@tangle-network/agent-eval 0.194.0 → 0.195.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/analyst/index.d.ts +5 -5
  4. package/dist/analyst/index.js +1 -1
  5. package/dist/{benchmark-command-Boqtfx4Z.js → benchmark-command-D6ZXuRFF.js} +2 -2
  6. package/dist/{benchmark-command-Boqtfx4Z.js.map → benchmark-command-D6ZXuRFF.js.map} +1 -1
  7. package/dist/benchmarks/index.d.ts +3 -3
  8. package/dist/benchmarks/index.js +2 -2
  9. package/dist/campaign/index.d.ts +5 -5
  10. package/dist/campaign/index.js +6 -6
  11. package/dist/{campaign-Cnx0Duzs.js → campaign-Df5qmzXW.js} +6 -6
  12. package/dist/{campaign-Cnx0Duzs.js.map → campaign-Df5qmzXW.js.map} +1 -1
  13. package/dist/cli.js +4 -4
  14. package/dist/{client-dMMKoQNS.d.ts → client-BzL9niqi.d.ts} +2 -2
  15. package/dist/{client-dMMKoQNS.d.ts.map → client-BzL9niqi.d.ts.map} +1 -1
  16. package/dist/contract/index.d.ts +7 -7
  17. package/dist/contract/index.js +3 -3
  18. package/dist/{default-registry-D44M_E4Q.d.ts → default-registry-D2csUBBQ.d.ts} +3 -3
  19. package/dist/{default-registry-D44M_E4Q.d.ts.map → default-registry-D2csUBBQ.d.ts.map} +1 -1
  20. package/dist/{define-agent-eval-CKTbbeyT.d.ts → define-agent-eval-C_oqRN9O.d.ts} +6 -6
  21. package/dist/{define-agent-eval-CKTbbeyT.d.ts.map → define-agent-eval-C_oqRN9O.d.ts.map} +1 -1
  22. package/dist/{define-agent-eval-Dq0MgLnw.js → define-agent-eval-h2qYe0xY.js} +3 -3
  23. package/dist/{define-agent-eval-Dq0MgLnw.js.map → define-agent-eval-h2qYe0xY.js.map} +1 -1
  24. package/dist/{engine-LkP-rfp1.d.ts → engine-De2hfuiQ.d.ts} +2 -2
  25. package/dist/{engine-LkP-rfp1.d.ts.map → engine-De2hfuiQ.d.ts.map} +1 -1
  26. package/dist/evaluation.d.ts +1 -1
  27. package/dist/experiment/index.d.ts +2 -2
  28. package/dist/hosted/index.d.ts +2 -2
  29. package/dist/hosted/index.js +1 -1
  30. package/dist/{index-DKUIMOst.d.ts → index-B7omS8L-.d.ts} +7 -7
  31. package/dist/{index-DKUIMOst.d.ts.map → index-B7omS8L-.d.ts.map} +1 -1
  32. package/dist/{index-Dvc9zaWD.d.ts → index-D7o8xhFO.d.ts} +85 -17
  33. package/dist/index-D7o8xhFO.d.ts.map +1 -0
  34. package/dist/{index-cWnRlQLW.d.ts → index-kKECJfib.d.ts} +3 -3
  35. package/dist/{index-cWnRlQLW.d.ts.map → index-kKECJfib.d.ts.map} +1 -1
  36. package/dist/index.d.ts +9 -9
  37. package/dist/index.js +3 -3
  38. package/dist/jev.d.ts +1 -1
  39. package/dist/{llm-judge-D6NIvyrM.js → llm-judge-CxfoYqdk.js} +373 -41
  40. package/dist/llm-judge-CxfoYqdk.js.map +1 -0
  41. package/dist/{matrix-CUzY_7Ml.d.ts → matrix-C1X-fIVX.d.ts} +2 -2
  42. package/dist/{matrix-CUzY_7Ml.d.ts.map → matrix-C1X-fIVX.d.ts.map} +1 -1
  43. package/dist/multishot/golden/index.d.ts +1 -1
  44. package/dist/multishot/index.d.ts +2 -2
  45. package/dist/openapi.json +1 -1
  46. package/dist/{produced-state-BjkNRWAe.js → produced-state-B0QMdiLh.js} +2 -2
  47. package/dist/{produced-state-BjkNRWAe.js.map → produced-state-B0QMdiLh.js.map} +1 -1
  48. package/dist/rl.d.ts +1 -1
  49. package/dist/{search-ledger-BbKtG_WF.js → search-ledger-9W0g5FP6.js} +8 -5
  50. package/dist/search-ledger-9W0g5FP6.js.map +1 -0
  51. package/dist/{search-ledger-types-B3El5xgS.d.ts → search-ledger-types-p2xKfiDJ.d.ts} +6 -2
  52. package/dist/{search-ledger-types-B3El5xgS.d.ts.map → search-ledger-types-p2xKfiDJ.d.ts.map} +1 -1
  53. package/dist/{search-shipper-Cm4Wt9SC.js → search-shipper-CYbmJQx8.js} +2 -2
  54. package/dist/{search-shipper-Cm4Wt9SC.js.map → search-shipper-CYbmJQx8.js.map} +1 -1
  55. package/dist/{search-summary-DnDGR702.js → search-summary-CcavvYLe.js} +2 -2
  56. package/dist/{search-summary-DnDGR702.js.map → search-summary-CcavvYLe.js.map} +1 -1
  57. package/dist/{skillopt-optimization-method-C4GiaBzp.js → skillopt-optimization-method-B3Oq4iim.js} +3 -3
  58. package/dist/{skillopt-optimization-method-C4GiaBzp.js.map → skillopt-optimization-method-B3Oq4iim.js.map} +1 -1
  59. package/dist/{statistical-heldout-CLfkCc_B.d.ts → statistical-heldout-BedVSZb2.d.ts} +2 -2
  60. package/dist/{statistical-heldout-CLfkCc_B.d.ts.map → statistical-heldout-BedVSZb2.d.ts.map} +1 -1
  61. package/dist/{store-tool-spans-E-mdWPAz.d.ts → store-tool-spans-CawP-WxM.d.ts} +4 -4
  62. package/dist/{store-tool-spans-E-mdWPAz.d.ts.map → store-tool-spans-CawP-WxM.d.ts.map} +1 -1
  63. package/dist/{tool-groups-CiFSpk6C.d.ts → tool-groups-B4JVrH3g.d.ts} +2 -2
  64. package/dist/tool-groups-B4JVrH3g.d.ts.map +1 -0
  65. package/dist/trace-repair/index.d.ts +1 -1
  66. package/dist/traces.d.ts +2 -2
  67. package/dist/{types-DLtLe21j.d.ts → types-B2673PBj.d.ts} +5 -3
  68. package/dist/types-B2673PBj.d.ts.map +1 -0
  69. package/dist/{types-M1Et9e6Z.d.ts → types-CjFDKMDe.d.ts} +2 -2
  70. package/dist/{types-M1Et9e6Z.d.ts.map → types-CjFDKMDe.d.ts.map} +1 -1
  71. package/docs/search-ledger.md +29 -7
  72. package/package.json +1 -1
  73. package/dist/index-Dvc9zaWD.d.ts.map +0 -1
  74. package/dist/llm-judge-D6NIvyrM.js.map +0 -1
  75. package/dist/search-ledger-BbKtG_WF.js.map +0 -1
  76. package/dist/tool-groups-CiFSpk6C.d.ts.map +0 -1
  77. package/dist/types-DLtLe21j.d.ts.map +0 -1
@@ -3,6 +3,7 @@ import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as L
3
3
  import { t as mulberry32 } from "./random-Dn5fPWkt.js";
4
4
  import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
5
5
  import { v as wilson } from "./paired-arms-JSDcs2zx.js";
6
+ import { t as minimumPairsForPairedDeltaTest } from "./paired-delta-test-BF-FEP8V.js";
6
7
  import { t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
7
8
  import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-DANVmyug.js";
8
9
  import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
@@ -17,8 +18,8 @@ import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./cla
17
18
  import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
18
19
  import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
19
20
  import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
20
- import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, t as FileSearchLedger, u as searchNodeId } from "./search-ledger-BbKtG_WF.js";
21
- import { c as seedFromDigest, i as estimateNode, n as searchProposerView, s as searchPosterior, t as renderSearchSummary } from "./search-summary-DnDGR702.js";
21
+ import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, t as FileSearchLedger, u as searchNodeId } from "./search-ledger-9W0g5FP6.js";
22
+ import { c as seedFromDigest, i as estimateNode, n as searchProposerView, s as searchPosterior, t as renderSearchSummary } from "./search-summary-CcavvYLe.js";
22
23
  import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-c_bKHTXz.js";
23
24
  import { createHash } from "node:crypto";
24
25
  import { writeFileSync } from "node:fs";
@@ -2537,14 +2538,25 @@ function paretoFrontierWithCrowding(candidates, objectives) {
2537
2538
  //#endregion
2538
2539
  //#region src/campaign/allocation.ts
2539
2540
  /**
2541
+ * Allocation: where a search spends rollouts.
2542
+ *
2543
+ * An allocator says which cells a node needs given the evidence so far; the
2544
+ * kernel allocates the ones the ledger does not hold yet, so a resumed search
2545
+ * plans exactly what the interrupted one planned. It also decides which nodes
2546
+ * earn more measurement (`advanced`) and, at close, which were left waiting
2547
+ * (`pruned`). Those are rank decisions: they spend budget and claim nothing.
2548
+ * Expansion (which parent, which operator) belongs to the `SearchPolicy`; the
2549
+ * claim belongs to the claim step.
2550
+ */
2551
+ /**
2540
2552
  * Every node gets every train and selection task at every repeat: the root as
2541
2553
  * `root` cells, every other node as `train` cells (the proposer's feedback) and
2542
2554
  * `screen` cells (the policy's private evidence). This is the fixed plan the
2543
- * generation loop ran and GEPA runs.
2555
+ * generation loop ran and GEPA runs. It has one rung, so it never advances or
2556
+ * prunes a node.
2544
2557
  */
2545
2558
  function uniform(options = {}) {
2546
- const reps = options.reps ?? 1;
2547
- if (!Number.isSafeInteger(reps) || reps < 1) throw new TypeError(`uniform: reps must be a positive integer, got ${String(reps)}`);
2559
+ const reps = positiveInteger("uniform", "reps", options.reps ?? 1);
2548
2560
  const cells = (state, root) => {
2549
2561
  const header = state.header;
2550
2562
  if (!header) return [];
@@ -2561,9 +2573,281 @@ function uniform(options = {}) {
2561
2573
  name: reps === 1 ? "uniform" : `uniform(reps=${reps})`,
2562
2574
  reps,
2563
2575
  plan: (state, nodeId) => cells(state, nodeId === state.rootNodeId),
2564
- screenSize: (state) => cells(state, false).length
2576
+ screenSize: (state) => cells(state, false).length,
2577
+ advance: () => [],
2578
+ prune: () => []
2579
+ };
2580
+ }
2581
+ /**
2582
+ * Asynchronous successive halving (Li et al., 2020) over one seeded
2583
+ * permutation of the ranked split's units, fixed by the search's seed.
2584
+ *
2585
+ * Rung k is the first `units × 2^k` units of the permutation; the top rung is
2586
+ * every unit. The root runs every unit before anything else, so every rung
2587
+ * pairs against it. A new node screens on rung 0 plus `trainUnits` train
2588
+ * units. Every node at a rung runs the same units as its parent, the root and
2589
+ * its siblings, so every contrast pairs.
2590
+ *
2591
+ * A node that finished rung k advances once it ranks in the top
2592
+ * floor(n / eta) of the n nodes that finished rung k, the root included, by
2593
+ * mean over the rung's units; a node with an unscored unit ranks last and
2594
+ * never advances. There is no barrier: a node outside the top waits, because
2595
+ * the set grows as more nodes finish. At close a waiting node is pruned with
2596
+ * its rank. The ranked split is selection, or train when a search declares no
2597
+ * selection split, as the policy's.
2598
+ */
2599
+ function asha(options = {}) {
2600
+ const firstRung = minimumPairsForPairedDeltaTest();
2601
+ const units = positiveInteger("asha", "units", options.units ?? firstRung);
2602
+ const eta = positiveInteger("asha", "eta", options.eta ?? 3);
2603
+ if (eta < 2) throw new TypeError(`asha: eta must be at least 2, got ${eta}`);
2604
+ const trainUnits = nonNegativeInteger("asha", "trainUnits", options.trainUnits ?? 2);
2605
+ const reps = positiveInteger("asha", "reps", options.reps ?? 1);
2606
+ const name = units === firstRung && eta === 3 && trainUnits === 2 && reps === 1 ? "asha" : `asha(units=${units},eta=${eta},train=${trainUnits},reps=${reps})`;
2607
+ const layouts = /* @__PURE__ */ new WeakMap();
2608
+ const layoutOf = (state) => {
2609
+ const header = state.header;
2610
+ if (!header) return null;
2611
+ let layout = layouts.get(header);
2612
+ if (!layout) {
2613
+ layout = ashaLayout(header, {
2614
+ units,
2615
+ trainUnits
2616
+ });
2617
+ layouts.set(header, layout);
2618
+ }
2619
+ return layout;
2620
+ };
2621
+ const cellsThrough = (layout, rung, root) => {
2622
+ const plan = [];
2623
+ const through = root ? layout.top : Math.min(rung, layout.top);
2624
+ layout.order.slice(0, layout.sizes[through]).forEach((unitId, position) => {
2625
+ const stage = root ? "root" : layout.split === "train" ? "train" : position < layout.sizes[0] ? "screen" : "rung";
2626
+ for (const task of layout.tasks.get(unitId)) for (let rep = 0; rep < reps; rep++) plan.push({
2627
+ taskId: task.taskId,
2628
+ split: layout.split,
2629
+ rep,
2630
+ stage
2631
+ });
2632
+ });
2633
+ for (const task of layout.feedback) for (let rep = 0; rep < reps; rep++) plan.push({
2634
+ taskId: task.taskId,
2635
+ split: "train",
2636
+ rep,
2637
+ stage: root ? "root" : "train"
2638
+ });
2639
+ return plan;
2640
+ };
2641
+ /**
2642
+ * Per node: the deepest rung it finished and its unit means on the ranked
2643
+ * split. A later decision (finalist, selected, pruned) does not unfinish a
2644
+ * rung, so every node but an invalid one keeps its place in the ranks; only
2645
+ * an undecided or advanced node is `open` to a new decision.
2646
+ */
2647
+ const standings = (view, layout) => {
2648
+ const { state } = view;
2649
+ const rootId = state.rootNodeId;
2650
+ const standing = [];
2651
+ for (const node of state.nodes()) {
2652
+ if (node.status === "invalid") continue;
2653
+ const root = node.nodeId === rootId;
2654
+ let advancedTo = 0;
2655
+ for (const { decision } of node.decisions) if (decision.status === "advanced") advancedTo = Math.max(advancedTo, decision.rung);
2656
+ const rung = root ? layout.top : Math.min(advancedTo, layout.top);
2657
+ const allocated = /* @__PURE__ */ new Set();
2658
+ let cancelled = false;
2659
+ for (const cell of state.cells({ nodeId: node.nodeId })) {
2660
+ if (cell.split !== layout.split) continue;
2661
+ const position = layout.position.get(cell.unitId);
2662
+ if (position !== void 0 && position < layout.sizes[rung]) {
2663
+ allocated.add(`${cell.taskId}\u0000${cell.rep}`);
2664
+ if (cell.cancelled !== null) cancelled = true;
2665
+ }
2666
+ }
2667
+ const covered = !cancelled && allocated.size === layout.cellsThrough[rung] * reps;
2668
+ const finished = view.idle(node.nodeId) && covered ? rung : root ? -1 : rung - 1;
2669
+ if (finished < 0) continue;
2670
+ const means = /* @__PURE__ */ new Map();
2671
+ for (const unit of state.unitScores(node.nodeId, layout.split)) means.set(unit.unitId, unit.mean);
2672
+ const open = node.status === null || node.status === "advanced";
2673
+ standing.push({
2674
+ nodeId: node.nodeId,
2675
+ ordinal: node.ordinal,
2676
+ root,
2677
+ open,
2678
+ rung,
2679
+ finished,
2680
+ means
2681
+ });
2682
+ }
2683
+ return standing;
2684
+ };
2685
+ /** The nodes that finished rung k, best first; an unscored unit ranks last. */
2686
+ const ranking = (state, layout, standing, k) => {
2687
+ const rungUnits = [...layout.order.slice(0, layout.sizes[k])].sort(compareCodeUnits);
2688
+ const sign = state.header.objective.direction === "maximize" ? 1 : -1;
2689
+ const ranked = [];
2690
+ for (const entry of standing) {
2691
+ if (entry.finished < k) continue;
2692
+ let sum = 0;
2693
+ let complete = true;
2694
+ for (const unitId of rungUnits) {
2695
+ const mean = entry.means.get(unitId);
2696
+ if (mean === void 0) {
2697
+ complete = false;
2698
+ break;
2699
+ }
2700
+ sum += mean;
2701
+ }
2702
+ ranked.push({
2703
+ ...entry,
2704
+ mean: complete ? sum / rungUnits.length : null,
2705
+ sign
2706
+ });
2707
+ }
2708
+ return ranked.sort(compareRanked);
2709
+ };
2710
+ const reason = (ranked, index, k, layout) => {
2711
+ const entry = ranked[index];
2712
+ const top = Math.floor(ranked.length / eta);
2713
+ const mean = entry.mean === null ? "an unscored unit" : `mean ${round$1(entry.mean)}`;
2714
+ return `rank ${index + 1} of ${ranked.length} on rung ${k} (${layout.sizes[k]} units, ${mean}); the top ${top} advance (eta ${eta})`;
2715
+ };
2716
+ return {
2717
+ name,
2718
+ reps,
2719
+ plan(state, nodeId, rung) {
2720
+ const layout = layoutOf(state);
2721
+ return layout ? cellsThrough(layout, rung, nodeId === state.rootNodeId) : [];
2722
+ },
2723
+ screenSize(state) {
2724
+ const layout = layoutOf(state);
2725
+ return layout ? cellsThrough(layout, 0, false).length : 0;
2726
+ },
2727
+ advance(view) {
2728
+ const { state } = view;
2729
+ const layout = layoutOf(state);
2730
+ const rootId = state.rootNodeId;
2731
+ if (!layout || layout.top === 0 || rootId === null) return [];
2732
+ const standing = standings(view, layout);
2733
+ const decisions = [];
2734
+ for (let k = layout.top - 1; k >= 0; k--) {
2735
+ const ranked = ranking(state, layout, standing, k);
2736
+ const top = Math.floor(ranked.length / eta);
2737
+ for (let index = 0; index < top; index++) {
2738
+ const entry = ranked[index];
2739
+ if (!entry.open || entry.root || entry.mean === null) continue;
2740
+ if (entry.finished !== k || entry.rung !== k) continue;
2741
+ decisions.push({
2742
+ nodeId: entry.nodeId,
2743
+ decision: {
2744
+ status: "advanced",
2745
+ rung: k + 1
2746
+ },
2747
+ basis: estimateNode(state, entry.nodeId, {
2748
+ against: rootId,
2749
+ split: layout.split
2750
+ }),
2751
+ rule: name,
2752
+ reason: reason(ranked, index, k, layout)
2753
+ });
2754
+ }
2755
+ }
2756
+ return decisions;
2757
+ },
2758
+ prune(view, keep) {
2759
+ const { state } = view;
2760
+ const layout = layoutOf(state);
2761
+ const rootId = state.rootNodeId;
2762
+ if (!layout || layout.top === 0 || rootId === null) return [];
2763
+ const standing = standings(view, layout);
2764
+ const decisions = [];
2765
+ for (let k = layout.top - 1; k >= 0; k--) {
2766
+ const ranked = ranking(state, layout, standing, k);
2767
+ const top = Math.floor(ranked.length / eta);
2768
+ ranked.forEach((entry, index) => {
2769
+ if (!entry.open || entry.root || entry.nodeId === keep) return;
2770
+ if (entry.finished !== k || entry.rung !== k || entry.means.size === 0) return;
2771
+ const why = index < top && entry.mean !== null ? `it earned rung ${k + 1}, but the cap, the cell limit or the deadline did not admit its cells` : "it waited outside the top when the search closed";
2772
+ decisions.push({
2773
+ nodeId: entry.nodeId,
2774
+ decision: { status: "pruned" },
2775
+ basis: estimateNode(state, entry.nodeId, {
2776
+ against: rootId,
2777
+ split: layout.split
2778
+ }),
2779
+ rule: name,
2780
+ reason: `${reason(ranked, index, k, layout)}; ${why}`
2781
+ });
2782
+ });
2783
+ }
2784
+ return decisions;
2785
+ }
2565
2786
  };
2566
2787
  }
2788
+ function compareRanked(left, right) {
2789
+ if (left.mean === null || right.mean === null) {
2790
+ if (left.mean !== right.mean) return left.mean === null ? 1 : -1;
2791
+ } else if (left.mean !== right.mean) return left.sign * (right.mean - left.mean);
2792
+ return left.ordinal - right.ordinal;
2793
+ }
2794
+ function ashaLayout(header, options) {
2795
+ const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
2796
+ const tasks = unitsOf(header.splits[split].tasks);
2797
+ const order = permute([...tasks.keys()], header.policy.seed, split);
2798
+ const sizes = [];
2799
+ for (let size = options.units;; size *= 2) {
2800
+ sizes.push(Math.min(size, order.length));
2801
+ if (size >= order.length) break;
2802
+ }
2803
+ const cellsThrough = sizes.map((size) => order.slice(0, size).reduce((sum, unitId) => sum + tasks.get(unitId).length, 0));
2804
+ let feedback = [];
2805
+ if (split === "selection") {
2806
+ const train = unitsOf(header.splits.train.tasks);
2807
+ feedback = permute([...train.keys()], header.policy.seed, "train").slice(0, options.trainUnits).flatMap((unitId) => train.get(unitId));
2808
+ }
2809
+ return {
2810
+ split,
2811
+ order,
2812
+ position: new Map(order.map((unitId, index) => [unitId, index])),
2813
+ tasks,
2814
+ sizes,
2815
+ cellsThrough,
2816
+ top: sizes.length - 1,
2817
+ feedback
2818
+ };
2819
+ }
2820
+ /** Tasks grouped by unit, units and tasks in code-unit order. */
2821
+ function unitsOf(tasks) {
2822
+ const byUnit = /* @__PURE__ */ new Map();
2823
+ for (const task of [...tasks].sort((left, right) => compareCodeUnits(left.taskId, right.taskId))) {
2824
+ const unit = byUnit.get(task.unitId);
2825
+ if (unit) unit.push(task);
2826
+ else byUnit.set(task.unitId, [task]);
2827
+ }
2828
+ return new Map([...byUnit.entries()].sort(([left], [right]) => compareCodeUnits(left, right)));
2829
+ }
2830
+ /** Fisher-Yates over units in code-unit order, from the search seed and the split. */
2831
+ function permute(unitIds, seed, split) {
2832
+ const rng = mulberry32(seed ^ (split === "train" ? 2050776593 : 731729763) | 0);
2833
+ const order = [...unitIds];
2834
+ for (let index = order.length - 1; index > 0; index--) {
2835
+ const other = Math.floor(rng() * (index + 1));
2836
+ [order[index], order[other]] = [order[other], order[index]];
2837
+ }
2838
+ return order;
2839
+ }
2840
+ function positiveInteger(owner, name, value) {
2841
+ if (!Number.isSafeInteger(value) || value < 1) throw new TypeError(`${owner}: ${name} must be a positive integer, got ${String(value)}`);
2842
+ return value;
2843
+ }
2844
+ function nonNegativeInteger(owner, name, value) {
2845
+ if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${owner}: ${name} must be a non-negative integer, got ${String(value)}`);
2846
+ return value;
2847
+ }
2848
+ function round$1(value) {
2849
+ return Math.round(value * 1e4) / 1e4;
2850
+ }
2567
2851
  //#endregion
2568
2852
  //#region src/paired-promotion-power.ts
2569
2853
  /**
@@ -3397,11 +3681,13 @@ const SEARCH_KERNEL_SOURCE = {
3397
3681
  estimate: "1.5 times the p99 of the lane settled cells once 20 settled, else the lane prior"
3398
3682
  },
3399
3683
  retries: "a retryable errored attempt runs again, up to maxAttempts",
3684
+ rungs: "when a node finishes its cells, the allocator names the nodes it advances, deepest rung first; each is recorded advanced and its rung allocated when the cap admits the rung",
3400
3685
  resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
3401
3686
  writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
3402
3687
  divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
3403
3688
  claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
3404
- close: "without a test split, the policy leader is selected when it has a scored cell; every other undecided node is rejected"
3689
+ close: "the allocator prunes the nodes left waiting on a rung, except the policy leader; then, with a test split, the claim decides; without one, the policy leader is selected when it has a scored cell; every other undecided node is rejected",
3690
+ complete: "a node leads only when it has a scored cell on the ranked split and no cell there ended unscored: a final unscored outcome, or an error with no attempt left"
3405
3691
  })
3406
3692
  };
3407
3693
  const USD_TOLERANCE = 1e-9;
@@ -3597,6 +3883,7 @@ var SearchKernel = class {
3597
3883
  for (const expansion of recorded) await this.finishProposal(expansion);
3598
3884
  for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
3599
3885
  for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) await this.screenDone(nodeId);
3886
+ await this.advance();
3600
3887
  }
3601
3888
  /**
3602
3889
  * A search that will claim must be able to: finalists come from a selection
@@ -3625,7 +3912,7 @@ var SearchKernel = class {
3625
3912
  cellUsd
3626
3913
  });
3627
3914
  if (reservedClaimUsd + USD_TOLERANCE < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
3628
- const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId).length;
3915
+ const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId, 0).length;
3629
3916
  const screens = (this.options.proposer.childrenPerProposal ?? 1) * this.options.allocation.screenSize(this.state);
3630
3917
  const round = this.operationReservation().usd + (rootCells + screens) * cellUsd;
3631
3918
  if (maxUsd - reservedClaimUsd + USD_TOLERANCE < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
@@ -3764,11 +4051,20 @@ var SearchKernel = class {
3764
4051
  if (cell.stage === "claim") return;
3765
4052
  const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
3766
4053
  this.pending.set(cell.nodeId, left);
3767
- if (left === 0) await this.onScreenDone(cell.nodeId);
4054
+ if (left === 0) await this.onNodeIdle(cell.nodeId);
3768
4055
  }
3769
- async onScreenDone(nodeId) {
4056
+ /** Every cell allocated to the node is done: its screen, or a rung. */
4057
+ async onNodeIdle(nodeId) {
3770
4058
  if (!await this.screenDone(nodeId)) return;
3771
- if (this.stopReason === null && !this.failure) await this.allocateFor(nodeId);
4059
+ await this.advance();
4060
+ }
4061
+ /** Record the rank decisions the allocator now supports and allocate the
4062
+ * rungs they open. Past the deadline, once the claim began, or once the
4063
+ * search fails or is aborted, no rung opens; a search that stopped
4064
+ * expanding still measures the nodes that earned it. */
4065
+ async advance() {
4066
+ if (this.claimPlan || this.failure || this.options.signal?.aborted || this.pastDeadline()) return;
4067
+ for (const decision of this.options.allocation.advance(this.allocationView())) await this.allocateFor(decision.nodeId, decision);
3772
4068
  }
3773
4069
  /**
3774
4070
  * A node's allocated cells all finished. A node that diverges (its train
@@ -3777,7 +4073,7 @@ var SearchKernel = class {
3777
4073
  * becomes a parent; any other node is screened. Returns whether it was.
3778
4074
  */
3779
4075
  async screenDone(nodeId) {
3780
- if (!this.claimPlan && !this.screenedSet.has(nodeId)) {
4076
+ if (!this.claimPlan && !this.screenedSet.has(nodeId) && this.state.node(nodeId).status !== "advanced") {
3781
4077
  const divergence = searchDivergence(this.state, nodeId);
3782
4078
  if (divergence) {
3783
4079
  await this.recorder.decideNode({
@@ -3807,10 +4103,19 @@ var SearchKernel = class {
3807
4103
  ...this.screened.slice(index)
3808
4104
  ];
3809
4105
  }
3810
- /** Allocate the cells the allocator plans for a node and the ledger lacks. */
3811
- async allocateFor(nodeId) {
3812
- const plans = this.options.allocation.plan(this.state, nodeId).filter((plan) => !this.state.cell(searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep)));
3813
- if (plans.length === 0) return;
4106
+ /**
4107
+ * Allocate the cells the allocator plans for a node through its rung and the
4108
+ * ledger lacks. With a rank decision, the rung is the decision's, and the
4109
+ * decision is recorded first, only once the cap admits the rung's cells.
4110
+ * Returns false when the cap or `maxCells` refused them.
4111
+ */
4112
+ async allocateFor(nodeId, decision) {
4113
+ const rung = decision?.decision.status === "advanced" ? decision.decision.rung : this.state.node(nodeId).rung ?? 0;
4114
+ const plans = this.options.allocation.plan(this.state, nodeId, rung).filter((plan) => !this.state.cell(searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep)));
4115
+ if (plans.length === 0) {
4116
+ if (decision) await this.decide(decision);
4117
+ return true;
4118
+ }
3814
4119
  const placed = plans.map((plan) => {
3815
4120
  const laneName = this.options.executor.place({
3816
4121
  ...plan,
@@ -3826,9 +4131,10 @@ var SearchKernel = class {
3826
4131
  });
3827
4132
  const hold = placed.reduce((sum, { reservation }) => sum + reservation.usd, 0);
3828
4133
  const { headroomUsd } = this.state.budget;
3829
- if (headroomUsd !== null && hold > headroomUsd + USD_TOLERANCE) return;
4134
+ if (headroomUsd !== null && hold > headroomUsd + USD_TOLERANCE) return false;
3830
4135
  const { maxCells } = this.state.header.budget;
3831
- if (maxCells !== null && this.state.audit.cells.allocated + placed.length > maxCells) return;
4136
+ if (maxCells !== null && this.state.audit.cells.allocated + placed.length > maxCells) return false;
4137
+ if (decision) await this.decide(decision);
3832
4138
  for (const { plan, lane, reservation } of placed) await this.recorder.allocateCell({
3833
4139
  nodeId,
3834
4140
  taskId: plan.taskId,
@@ -3845,6 +4151,11 @@ var SearchKernel = class {
3845
4151
  }
3846
4152
  this.pending.set(nodeId, (this.pending.get(nodeId) ?? 0) + placed.length);
3847
4153
  this.dispatch();
4154
+ return true;
4155
+ }
4156
+ async decide(decision) {
4157
+ await this.recorder.decideNode(decision);
4158
+ await this.refresh();
3848
4159
  }
3849
4160
  enqueue(cell) {
3850
4161
  this.lanes.get(cell.lane).queues[STAGE_PRIORITY[cell.stage]].push(cell.cellId);
@@ -4163,6 +4474,8 @@ var SearchKernel = class {
4163
4474
  }
4164
4475
  async close() {
4165
4476
  await this.refresh();
4477
+ const keep = this.options.policy.leader(this.policyView());
4478
+ for (const decision of this.options.allocation.prune(this.allocationView(), keep)) await this.decide(decision);
4166
4479
  if (this.claimPlan) {
4167
4480
  await this.closeWithClaim(this.claimPlan);
4168
4481
  return;
@@ -4172,6 +4485,7 @@ var SearchKernel = class {
4172
4485
  const { split } = view;
4173
4486
  const root = this.state.rootNodeId;
4174
4487
  const rule = this.options.policy.name;
4488
+ const leaderUnits = view.unitScores(leader).map((unit) => unit.unitId);
4175
4489
  const decisions = this.state.nodes().filter((node) => !isTerminal(node)).map((node) => {
4176
4490
  const scored = this.state.scoredCells(node.nodeId, split).length > 0;
4177
4491
  const lead = node.nodeId === leader && scored;
@@ -4180,12 +4494,13 @@ var SearchKernel = class {
4180
4494
  against,
4181
4495
  split
4182
4496
  });
4497
+ const units = new Set(view.unitScores(node.nodeId).map((unit) => unit.unitId));
4183
4498
  return {
4184
4499
  nodeId: node.nodeId,
4185
4500
  decision: { status: lead ? "selected" : "rejected" },
4186
4501
  basis,
4187
4502
  rule,
4188
- reason: lead ? `the leader when the search stopped (${this.stopReason}): a budget decision on the ${split} split, which claims nothing` : !this.screenedSet.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit of its screen unscored, so it could not lead" : "it did not beat the leader on the units they share"
4503
+ reason: lead ? `the leader when the search stopped (${this.stopReason}): a budget decision on the ${split} split, which claims nothing` : !this.screenedSet.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit unscored, so it could not lead" : leaderUnits.some((unitId) => !units.has(unitId)) ? "the search stopped before it was measured on every unit the leader was" : "it did not beat the leader on the units they share"
4189
4504
  };
4190
4505
  });
4191
4506
  for (const decision of decisions) await this.recorder.decideNode(decision);
@@ -4240,9 +4555,16 @@ var SearchKernel = class {
4240
4555
  return searchPolicyView(this.state, {
4241
4556
  screened: this.screened,
4242
4557
  screening: this.admitted.size - this.screenedSet.size,
4243
- expansions: this.completed.length
4558
+ expansions: this.completed.length,
4559
+ maxAttempts: this.maxAttempts
4244
4560
  });
4245
4561
  }
4562
+ allocationView() {
4563
+ return {
4564
+ state: this.state,
4565
+ idle: (nodeId) => this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0
4566
+ };
4567
+ }
4246
4568
  artifact(nodeId) {
4247
4569
  const held = this.artifacts.get(nodeId);
4248
4570
  if (held !== void 0) return held;
@@ -4300,8 +4622,11 @@ var SearchKernel = class {
4300
4622
  * finished, in registration order; `screening` counts the admitted nodes still
4301
4623
  * being screened. Nodes are ranked on the selection split, or on train when
4302
4624
  * the search declares no selection split. The view reads no test cell.
4625
+ * `maxAttempts` (default 3, the kernel's) says when an errored cell has no
4626
+ * attempt left, which makes its unit a dodge.
4303
4627
  */
4304
4628
  function searchPolicyView(state, input) {
4629
+ const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
4305
4630
  const header = state.header;
4306
4631
  if (!header) throw new Error(`search ${state.searchId} has not been opened`);
4307
4632
  const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
@@ -4315,8 +4640,13 @@ function searchPolicyView(state, input) {
4315
4640
  screened: [...input.screened],
4316
4641
  screening: input.screening,
4317
4642
  complete: (nodeId) => {
4318
- const cells = state.cells({ nodeId }).filter((cell) => cell.split === split);
4319
- return cells.length > 0 && cells.every((cell) => cell.score !== null);
4643
+ let scored = false;
4644
+ for (const cell of state.cells({ nodeId })) {
4645
+ if (cell.split !== split) continue;
4646
+ if (cell.score !== null) scored = true;
4647
+ else if (cell.final || cell.outcome === "errored" && cell.attempts >= maxAttempts) return false;
4648
+ }
4649
+ return scored;
4320
4650
  },
4321
4651
  unitScores: (nodeId) => state.unitScores(nodeId, split),
4322
4652
  estimate: (nodeId, against) => estimateNode(state, nodeId, {
@@ -4964,12 +5294,14 @@ function canonicalHeader(header) {
4964
5294
  * claim nothing; a claim comes only from the sealed test split.
4965
5295
  */
4966
5296
  /**
4967
- * The hill climb. The leader starts at the root; a screened node that scored
4968
- * every unit of its screen takes the lead when its mean on the units it shares
4969
- * with the leader beats the leader's mean on those units. Nodes are taken in
4970
- * registration order, so the leader is a pure function of the ledger. The
4971
- * policy expands the leader, and only once every earlier child is screened,
4972
- * so each proposal sees every result before it.
5297
+ * The hill climb. The leader starts at the root; a screened node that dodged no
5298
+ * unit takes the lead when it scored every unit the leader scored and its mean
5299
+ * on them beats the leader's. A node measured on fewer units than the leader,
5300
+ * such as one an allocator has only screened, cannot take the lead on less
5301
+ * evidence than the leader holds. Nodes are taken in registration order, so
5302
+ * the leader is a pure function of the ledger. The policy expands the leader,
5303
+ * and only once every earlier child is screened, so each proposal sees every
5304
+ * result before it.
4973
5305
  */
4974
5306
  function incumbent(options = {}) {
4975
5307
  const { patience } = options;
@@ -5054,20 +5386,20 @@ function hillClimb(spec) {
5054
5386
  }
5055
5387
  };
5056
5388
  }
5057
- /** True when `challenger` improves on `holder` over the units both scored. */
5389
+ /** True when `challenger` scored every unit `holder` scored and improves on
5390
+ * `holder` over them. */
5058
5391
  function beats(view, challenger, holder) {
5059
- const held = new Map(view.unitScores(holder).map((unit) => [unit.unitId, unit.mean]));
5392
+ const held = view.unitScores(holder);
5393
+ if (held.length === 0) return false;
5394
+ const own = new Map(view.unitScores(challenger).map((unit) => [unit.unitId, unit.mean]));
5060
5395
  let challengerSum = 0;
5061
5396
  let holderSum = 0;
5062
- let shared = 0;
5063
- for (const unit of view.unitScores(challenger)) {
5064
- const other = held.get(unit.unitId);
5065
- if (other === void 0) continue;
5066
- challengerSum += unit.mean;
5067
- holderSum += other;
5068
- shared += 1;
5397
+ for (const unit of held) {
5398
+ const mean = own.get(unit.unitId);
5399
+ if (mean === void 0) return false;
5400
+ challengerSum += mean;
5401
+ holderSum += unit.mean;
5069
5402
  }
5070
- if (shared === 0) return false;
5071
5403
  return view.direction === "maximize" ? challengerSum > holderSum : challengerSum < holderSum;
5072
5404
  }
5073
5405
  /** Non-dominated complete screened nodes over the root's units, with crowding. */
@@ -5446,7 +5778,7 @@ var SurfaceNodes = class {
5446
5778
  const ids = [];
5447
5779
  for (const child of blob.children) {
5448
5780
  const nodeId = state.nodeIdForDigest(child.node.artifactDigest);
5449
- if (nodeId === void 0 || ids.includes(nodeId)) continue;
5781
+ if (nodeId === void 0 || nodeId === state.rootNodeId || ids.includes(nodeId)) continue;
5450
5782
  if (this.origin(state, nodeId).expansion === expansion) ids.push(nodeId);
5451
5783
  }
5452
5784
  return ids;
@@ -7334,6 +7666,6 @@ function firstString(value) {
7334
7666
  return typeof value === "string" && value.trim() ? value : void 0;
7335
7667
  }
7336
7668
  //#endregion
7337
- export { DEFAULT_RED_TEAM_CORPUS as $, createSearchHistoryReceipt as A, decideSearchClaim as B, searchEdgeId as C, assertCompleteSearchHistory as D, SearchHistoryRequiredError as E, runSearch as F, requiredPairsForPairedPromotion as G, searchClaimReserveUsd as H, searchDivergence as I, paretoFrontier as J, uniform as K, searchExpansionIndex as L, verifySearchHistoryArtifact as M, verifySearchHistoryReceipt as N, assertSearchHistoryAdmissionOptions as O, SEARCH_KERNEL_SOURCE as P, defaultProductionGate as Q, searchPolicyView as R, developmentClaim as S, surfaceNode as T, verifySearchClaim as U, planSearchClaim as V, pairedPromotionPower as W, openAutoPr as X, runFinalComparison as Y, captureJudge as Z, isProposedCandidate as _, transientDispatchFailure as a, captureFinalEvidencePolicy as at, incumbent as b, compareOptimizationMethods as c, runEval as ct, combineComparisonCosts as d, resolveRunDir as dt, redTeamDataset as et, costFromLedgerSummary as f, tangleTracesRoot as ft, runOptimization as g, computeManifestHash as gt, runImprovementLoop as h, cellCachePath as ht, quotaExhaustedUntil as i, assertIndependentEvaluationSplit as it, searchHistoryCoverageRow as j, assertSearchHistoryMatchesState as k, optimizationTokenUsageFromSummary as l, runCampaign as lt, readGepaCandidatePopulationArtifact as m, buildCellSchedule as mt, JudgeParseError as n, scoreRedTeamOutput as nt, aggregateRunScore as o, evaluationUnitMap as ot, assertGepaCandidatePopulationSummary as p, readCachedCell as pt, dominates as q, isTransientTransportFailure as r, runCanaries as rt, clamp01 as s, reserveFinalEvidence as st, llmJudge as t, redTeamReport as tt, executeOptimizationMethod as u, planCampaignRun as ut, labelTrustRank as v, surfaceDiff as w, SearchRecorder as x, crowdedFrontierParent as y, SEARCH_CLAIM_RULE as z };
7669
+ export { defaultProductionGate as $, createSearchHistoryReceipt as A, decideSearchClaim as B, searchEdgeId as C, assertCompleteSearchHistory as D, SearchHistoryRequiredError as E, runSearch as F, requiredPairsForPairedPromotion as G, searchClaimReserveUsd as H, searchDivergence as I, dominates as J, asha as K, searchExpansionIndex as L, verifySearchHistoryArtifact as M, verifySearchHistoryReceipt as N, assertSearchHistoryAdmissionOptions as O, SEARCH_KERNEL_SOURCE as P, captureJudge as Q, searchPolicyView as R, developmentClaim as S, surfaceNode as T, verifySearchClaim as U, planSearchClaim as V, pairedPromotionPower as W, runFinalComparison as X, paretoFrontier as Y, openAutoPr as Z, isProposedCandidate as _, computeManifestHash as _t, transientDispatchFailure as a, assertIndependentEvaluationSplit as at, incumbent as b, compareOptimizationMethods as c, reserveFinalEvidence as ct, combineComparisonCosts as d, planCampaignRun as dt, DEFAULT_RED_TEAM_CORPUS as et, costFromLedgerSummary as f, resolveRunDir as ft, runOptimization as g, cellCachePath as gt, runImprovementLoop as h, buildCellSchedule as ht, quotaExhaustedUntil as i, runCanaries as it, searchHistoryCoverageRow as j, assertSearchHistoryMatchesState as k, optimizationTokenUsageFromSummary as l, runEval as lt, readGepaCandidatePopulationArtifact as m, readCachedCell as mt, JudgeParseError as n, redTeamReport as nt, aggregateRunScore as o, captureFinalEvidencePolicy as ot, assertGepaCandidatePopulationSummary as p, tangleTracesRoot as pt, uniform as q, isTransientTransportFailure as r, scoreRedTeamOutput as rt, clamp01 as s, evaluationUnitMap as st, llmJudge as t, redTeamDataset as tt, executeOptimizationMethod as u, runCampaign as ut, labelTrustRank as v, surfaceDiff as w, SearchRecorder as x, crowdedFrontierParent as y, SEARCH_CLAIM_RULE as z };
7338
7670
 
7339
- //# sourceMappingURL=llm-judge-D6NIvyrM.js.map
7671
+ //# sourceMappingURL=llm-judge-CxfoYqdk.js.map