@tangle-network/agent-eval 0.196.0 → 0.198.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/dist/active-curriculum-B_Lc9kfO.js +61 -0
- package/dist/active-curriculum-B_Lc9kfO.js.map +1 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +3 -3
- package/dist/{analyze-runs-B6gv7vAT.js → analyze-runs-DFbp2gB8.js} +5 -5
- package/dist/{analyze-runs-B6gv7vAT.js.map → analyze-runs-DFbp2gB8.js.map} +1 -1
- package/dist/{benchmark-command-Bw1he2JM.js → benchmark-command-D8E5TaPC.js} +6 -5
- package/dist/{benchmark-command-Bw1he2JM.js.map → benchmark-command-D8E5TaPC.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -3
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -4
- package/dist/campaign/index.js +13 -11
- package/dist/{campaign-qe1WC3Sz.js → campaign-Dlg2iOzH.js} +13 -12
- package/dist/{campaign-qe1WC3Sz.js.map → campaign-Dlg2iOzH.js.map} +1 -1
- package/dist/{campaign-evidence-DJc60F9C.js → campaign-evidence-jL_U7Ikv.js} +2 -2
- package/dist/{campaign-evidence-DJc60F9C.js.map → campaign-evidence-jL_U7Ikv.js.map} +1 -1
- package/dist/cli.js +243 -13
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +7 -6
- package/dist/contract/index.js +9 -9
- package/dist/{default-registry-D2csUBBQ.d.ts → default-registry-oK9hHfX_.d.ts} +3 -3
- package/dist/{default-registry-D2csUBBQ.d.ts.map → default-registry-oK9hHfX_.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DwAaPvvH.d.ts → define-agent-eval-By9PImpa.d.ts} +5 -4
- package/dist/define-agent-eval-By9PImpa.d.ts.map +1 -0
- package/dist/{define-agent-eval-h2qYe0xY.js → define-agent-eval-DvBgSzAI.js} +6 -6
- package/dist/{define-agent-eval-h2qYe0xY.js.map → define-agent-eval-DvBgSzAI.js.map} +1 -1
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-k_QdEBZo.js → dspy-rlm-engine-B5T43BkZ.js} +2 -2
- package/dist/{dspy-rlm-engine-k_QdEBZo.js.map → dspy-rlm-engine-B5T43BkZ.js.map} +1 -1
- package/dist/{engine-De2hfuiQ.d.ts → engine-Dc996kAg.d.ts} +2 -2
- package/dist/{engine-De2hfuiQ.d.ts.map → engine-Dc996kAg.d.ts.map} +1 -1
- package/dist/{eval-campaign-CXT2WYUe.js → eval-campaign-GapmPHIA.js} +2 -2
- package/dist/{eval-campaign-CXT2WYUe.js.map → eval-campaign-GapmPHIA.js.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.js +8 -6
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-_YIECYJ7.js → external-optimizer-process-CEXcCKqd.js} +2 -2
- package/dist/{external-optimizer-process-_YIECYJ7.js.map → external-optimizer-process-CEXcCKqd.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-BrghTvEp.js → external-optimizer-subprocess-laM1-UDV.js} +2 -61
- package/dist/external-optimizer-subprocess-laM1-UDV.js.map +1 -0
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +2 -3
- package/dist/fuzz.js.map +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CigojbNH.d.ts → index-BhoOCduB.d.ts} +38 -945
- package/dist/index-BhoOCduB.d.ts.map +1 -0
- package/dist/{index-CpCIXObe.d.ts → index-D-tI1vgf.d.ts} +6 -6
- package/dist/{index-CpCIXObe.d.ts.map → index-D-tI1vgf.d.ts.map} +1 -1
- package/dist/{index-kKECJfib.d.ts → index-lbwvO9vJ.d.ts} +3 -3
- package/dist/{index-kKECJfib.d.ts.map → index-lbwvO9vJ.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +15 -13
- package/dist/index.js.map +1 -1
- package/dist/jev-protocol.d.ts +17 -9
- package/dist/jev-protocol.d.ts.map +1 -1
- package/dist/jev-protocol.js +22 -14
- package/dist/jev-protocol.js.map +1 -1
- package/dist/jev.d.ts +3 -3
- package/dist/jev.js +3 -7
- package/dist/jev.js.map +1 -1
- package/dist/{llm-judge-CxfoYqdk.js → llm-judge-BYWXsxhZ.js} +529 -2079
- package/dist/llm-judge-BYWXsxhZ.js.map +1 -0
- package/dist/{matrix-C1X-fIVX.d.ts → matrix-BnksRaDD.d.ts} +2 -2
- package/dist/{matrix-C1X-fIVX.d.ts.map → matrix-BnksRaDD.d.ts.map} +1 -1
- package/dist/multiplicity-D9cjmStq.js +101 -0
- package/dist/multiplicity-D9cjmStq.js.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.js +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-JSDcs2zx.js → paired-arms-DKZFVPt8.js} +4 -398
- package/dist/paired-arms-DKZFVPt8.js.map +1 -0
- package/dist/paired-binary-fzkwENI4.js +399 -0
- package/dist/paired-binary-fzkwENI4.js.map +1 -0
- package/dist/{paired-promotion-decision-DTKl6-G8.js → paired-promotion-decision-D5B1Cxc2.js} +78 -5
- package/dist/paired-promotion-decision-D5B1Cxc2.js.map +1 -0
- package/dist/{power-and-mde-BJOpt6pq.js → power-and-mde-K_CmO3dc.js} +3 -101
- package/dist/power-and-mde-K_CmO3dc.js.map +1 -0
- package/dist/{produced-state-B0QMdiLh.js → produced-state-DOfmCEwP.js} +3 -3
- package/dist/{produced-state-B0QMdiLh.js.map → produced-state-DOfmCEwP.js.map} +1 -1
- package/dist/{promotion-policy-Cy5OV8BV.js → promotion-policy-C6FAJvmZ.js} +3 -3
- package/dist/{promotion-policy-Cy5OV8BV.js.map → promotion-policy-C6FAJvmZ.js.map} +1 -1
- package/dist/reporting.js +3 -3
- package/dist/rl.d.ts +17 -39
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +5 -5
- package/dist/search/index.d.ts +1162 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +187 -0
- package/dist/search/index.js.map +1 -0
- package/dist/search-kernel-CuvbakC_.d.ts +1176 -0
- package/dist/search-kernel-CuvbakC_.d.ts.map +1 -0
- package/dist/search-kernel-D8_kxChy.js +2399 -0
- package/dist/search-kernel-D8_kxChy.js.map +1 -0
- package/dist/{search-ledger-9W0g5FP6.js → search-ledger-BnUytR8X.js} +3 -1
- package/dist/search-ledger-BnUytR8X.js.map +1 -0
- package/dist/{search-shipper-CYbmJQx8.js → search-shipper-Cn8cxcyp.js} +2 -2
- package/dist/{search-shipper-CYbmJQx8.js.map → search-shipper-Cn8cxcyp.js.map} +1 -1
- package/dist/search-summary-Dmppg8kV.js +133 -0
- package/dist/search-summary-Dmppg8kV.js.map +1 -0
- package/dist/{semantic-concept-judge-Bh5lvKGg.js → semantic-concept-judge-2SE_kXGF.js} +2 -2
- package/dist/{semantic-concept-judge-Bh5lvKGg.js.map → semantic-concept-judge-2SE_kXGF.js.map} +1 -1
- package/dist/{sequential-C21_PP7F.js → sequential-Yz3OSskl.js} +2 -2
- package/dist/{sequential-C21_PP7F.js.map → sequential-Yz3OSskl.js.map} +1 -1
- package/dist/single-run-lock-fVu3MZWH.js +62 -0
- package/dist/single-run-lock-fVu3MZWH.js.map +1 -0
- package/dist/{skillopt-optimization-method-IbEhlA1V.js → skillopt-optimization-method-BMP2qV_1.js} +46 -33
- package/dist/{skillopt-optimization-method-IbEhlA1V.js.map → skillopt-optimization-method-BMP2qV_1.js.map} +1 -1
- package/dist/{statistical-heldout-DANVmyug.js → statistical-heldout-BAFDCo07.js} +2 -2
- package/dist/{statistical-heldout-DANVmyug.js.map → statistical-heldout-BAFDCo07.js.map} +1 -1
- package/dist/{statistical-heldout-BedVSZb2.d.ts → statistical-heldout-CMpxMjlQ.d.ts} +2 -2
- package/dist/{statistical-heldout-BedVSZb2.d.ts.map → statistical-heldout-CMpxMjlQ.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CawP-WxM.d.ts → store-tool-spans-CUwWSXoj.d.ts} +3 -3
- package/dist/{store-tool-spans-CawP-WxM.d.ts.map → store-tool-spans-CUwWSXoj.d.ts.map} +1 -1
- package/dist/{summary-report-BMvou2oZ.js → summary-report-BeYnvddy.js} +4 -3
- package/dist/{summary-report-BMvou2oZ.js.map → summary-report-BeYnvddy.js.map} +1 -1
- package/dist/{tool-groups-B4JVrH3g.d.ts → tool-groups-DVJAR1GE.d.ts} +2 -2
- package/dist/tool-groups-DVJAR1GE.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/tree-DHHWsH-X.js +4045 -0
- package/dist/tree-DHHWsH-X.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-B3jzCp0p2.js} +1 -1
- package/dist/types-B3jzCp0p2.js.map +1 -0
- package/dist/{types-CjFDKMDe.d.ts → types-D94EanDL.d.ts} +2 -2
- package/dist/{types-CjFDKMDe.d.ts.map → types-D94EanDL.d.ts.map} +1 -1
- package/dist/{types-B2673PBj.d.ts → types-EBUav9Bu.d.ts} +8 -4
- package/dist/types-EBUav9Bu.d.ts.map +1 -0
- package/docs/campaign-proposers.md +2 -0
- package/docs/design/statistics-decisions.md +1 -1
- package/docs/jev.md +3 -2
- package/docs/search-ledger.md +145 -8
- package/package.json +6 -1
- package/dist/active-curriculum-OjIWgrUJ.js +0 -204
- package/dist/active-curriculum-OjIWgrUJ.js.map +0 -1
- package/dist/define-agent-eval-DwAaPvvH.d.ts.map +0 -1
- package/dist/external-optimizer-subprocess-BrghTvEp.js.map +0 -1
- package/dist/index-CigojbNH.d.ts.map +0 -1
- package/dist/llm-judge-CxfoYqdk.js.map +0 -1
- package/dist/paired-arms-JSDcs2zx.js.map +0 -1
- package/dist/paired-delta-test-BF-FEP8V.js +0 -78
- package/dist/paired-delta-test-BF-FEP8V.js.map +0 -1
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +0 -1
- package/dist/power-and-mde-BJOpt6pq.js.map +0 -1
- package/dist/search-ledger-9W0g5FP6.js.map +0 -1
- package/dist/search-summary-CcavvYLe.js +0 -378
- package/dist/search-summary-CcavvYLe.js.map +0 -1
- package/dist/tool-groups-B4JVrH3g.d.ts.map +0 -1
- package/dist/types-B2673PBj.d.ts.map +0 -1
- package/dist/types-B3jzCp0p.js.map +0 -1
|
@@ -2,24 +2,23 @@ import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
|
|
|
2
2
|
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
|
|
3
3
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
4
4
|
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
|
|
8
|
-
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-DANVmyug.js";
|
|
5
|
+
import { r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-D5B1Cxc2.js";
|
|
6
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-BAFDCo07.js";
|
|
9
7
|
import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
|
|
10
8
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
11
9
|
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-tNnlQIrD.js";
|
|
12
10
|
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
|
|
13
|
-
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-
|
|
11
|
+
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-jL_U7Ikv.js";
|
|
14
12
|
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
15
13
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
16
|
-
import {
|
|
14
|
+
import { S as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate, x as createRunCostLedger } from "./external-optimizer-subprocess-laM1-UDV.js";
|
|
17
15
|
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-V9Zqbq0J.js";
|
|
18
16
|
import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
|
|
19
17
|
import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
|
|
20
18
|
import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
19
|
+
import { D as paretoFrontierWithCrowding, E as paretoFrontier, a as searchPolicyView, b as estimateNode, i as searchExpansionIndex, n as runSearch, p as EXPANSION_OPERATORS, t as SEARCH_KERNEL_SOURCE } from "./search-kernel-D8_kxChy.js";
|
|
20
|
+
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-BnUytR8X.js";
|
|
21
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-Dmppg8kV.js";
|
|
23
22
|
import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-c_bKHTXz.js";
|
|
24
23
|
import { createHash } from "node:crypto";
|
|
25
24
|
import { writeFileSync } from "node:fs";
|
|
@@ -1165,8 +1164,8 @@ function detectCalibrationDrift(runs, opts) {
|
|
|
1165
1164
|
alpha,
|
|
1166
1165
|
recentN: recent.length,
|
|
1167
1166
|
historyN: historical.length,
|
|
1168
|
-
recentMean: mean$
|
|
1169
|
-
historyMean: mean$
|
|
1167
|
+
recentMean: mean$2(recent),
|
|
1168
|
+
historyMean: mean$2(historical)
|
|
1170
1169
|
}
|
|
1171
1170
|
}],
|
|
1172
1171
|
evaluation: {
|
|
@@ -1403,7 +1402,7 @@ function chiSquareCritical(df, alpha) {
|
|
|
1403
1402
|
}
|
|
1404
1403
|
return TABLE[10][idx];
|
|
1405
1404
|
}
|
|
1406
|
-
function mean$
|
|
1405
|
+
function mean$2(xs) {
|
|
1407
1406
|
if (xs.length === 0) return 0;
|
|
1408
1407
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
1409
1408
|
}
|
|
@@ -2457,85 +2456,6 @@ async function runFinalComparison(opts) {
|
|
|
2457
2456
|
};
|
|
2458
2457
|
}
|
|
2459
2458
|
//#endregion
|
|
2460
|
-
//#region src/pareto.ts
|
|
2461
|
-
/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
|
|
2462
|
-
function dominates(a, b, objectives) {
|
|
2463
|
-
let strictlyBetter = false;
|
|
2464
|
-
for (const obj of objectives) {
|
|
2465
|
-
const av = obj.value(a);
|
|
2466
|
-
const bv = obj.value(b);
|
|
2467
|
-
if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
|
|
2468
|
-
const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
|
|
2469
|
-
if (obj.direction === "maximize" ? av < bv : av > bv) return false;
|
|
2470
|
-
if (aIsBetter) strictlyBetter = true;
|
|
2471
|
-
}
|
|
2472
|
-
return strictlyBetter;
|
|
2473
|
-
}
|
|
2474
|
-
/**
|
|
2475
|
-
* Compute the non-dominated frontier. Candidates with NaN/Infinity on any
|
|
2476
|
-
* objective are excluded (can't rank them). A candidate enters the frontier
|
|
2477
|
-
* iff no other candidate dominates it.
|
|
2478
|
-
*/
|
|
2479
|
-
function paretoFrontier(candidates, objectives) {
|
|
2480
|
-
if (objectives.length === 0) throw new Error("paretoFrontier: at least 1 objective required");
|
|
2481
|
-
const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))));
|
|
2482
|
-
const frontier = [];
|
|
2483
|
-
const dominated = [];
|
|
2484
|
-
for (const c of valid) if (valid.some((other) => other !== c && dominates(other, c, objectives))) dominated.push(c);
|
|
2485
|
-
else frontier.push(c);
|
|
2486
|
-
return {
|
|
2487
|
-
frontier,
|
|
2488
|
-
dominated,
|
|
2489
|
-
dominanceMap: frontier.map((d) => ({
|
|
2490
|
-
dominator: d,
|
|
2491
|
-
dominated: dominated.filter((x) => dominates(d, x, objectives))
|
|
2492
|
-
}))
|
|
2493
|
-
};
|
|
2494
|
-
}
|
|
2495
|
-
/**
|
|
2496
|
-
* NSGA-II crowding distance — secondary sort for ties on the frontier.
|
|
2497
|
-
*
|
|
2498
|
-
* When the Pareto front collapses to a single point (or many candidates tie
|
|
2499
|
-
* on dominance), naive selection picks arbitrarily and the population
|
|
2500
|
-
* degenerates over generations. NSGA-II preserves diversity by preferring
|
|
2501
|
-
* candidates with more empty space around them on the frontier.
|
|
2502
|
-
*
|
|
2503
|
-
* Returns an array of `{ candidate, distance }` in the SAME order as the
|
|
2504
|
-
* input. Higher distance = more isolated = should be preferred when
|
|
2505
|
-
* preserving diversity.
|
|
2506
|
-
*/
|
|
2507
|
-
function crowdingDistance(candidates, objectives) {
|
|
2508
|
-
const distances = new Map(candidates.map((c) => [c, 0]));
|
|
2509
|
-
for (const obj of objectives) {
|
|
2510
|
-
const sorted = [...candidates].sort((a, b) => obj.value(a) - obj.value(b));
|
|
2511
|
-
const min = obj.value(sorted[0]);
|
|
2512
|
-
const range = obj.value(sorted[sorted.length - 1]) - min || 1;
|
|
2513
|
-
distances.set(sorted[0], Infinity);
|
|
2514
|
-
distances.set(sorted[sorted.length - 1], Infinity);
|
|
2515
|
-
for (let i = 1; i < sorted.length - 1; i++) {
|
|
2516
|
-
const prev = obj.value(sorted[i - 1]);
|
|
2517
|
-
const next = obj.value(sorted[i + 1]);
|
|
2518
|
-
const current = distances.get(sorted[i]);
|
|
2519
|
-
if (current === Infinity) continue;
|
|
2520
|
-
distances.set(sorted[i], current + (next - prev) / range);
|
|
2521
|
-
}
|
|
2522
|
-
}
|
|
2523
|
-
return candidates.map((c) => ({
|
|
2524
|
-
candidate: c,
|
|
2525
|
-
distance: distances.get(c) ?? 0
|
|
2526
|
-
}));
|
|
2527
|
-
}
|
|
2528
|
-
/**
|
|
2529
|
-
* Pareto frontier with tie-break by crowding distance — the canonical
|
|
2530
|
-
* NSGA-II selection step. Returns the frontier sorted by descending crowding
|
|
2531
|
-
* distance so callers can `.slice(0, k)` to pick K diverse winners.
|
|
2532
|
-
*/
|
|
2533
|
-
function paretoFrontierWithCrowding(candidates, objectives) {
|
|
2534
|
-
const { frontier } = paretoFrontier(candidates, objectives);
|
|
2535
|
-
if (frontier.length === 0) return [];
|
|
2536
|
-
return crowdingDistance(frontier, objectives).sort((a, b) => b.distance - a.distance);
|
|
2537
|
-
}
|
|
2538
|
-
//#endregion
|
|
2539
2459
|
//#region src/campaign/allocation.ts
|
|
2540
2460
|
/**
|
|
2541
2461
|
* Allocation: where a search spends rollouts.
|
|
@@ -2603,9 +2523,11 @@ function asha(options = {}) {
|
|
|
2603
2523
|
if (eta < 2) throw new TypeError(`asha: eta must be at least 2, got ${eta}`);
|
|
2604
2524
|
const trainUnits = nonNegativeInteger("asha", "trainUnits", options.trainUnits ?? 2);
|
|
2605
2525
|
const reps = positiveInteger("asha", "reps", options.reps ?? 1);
|
|
2606
|
-
const
|
|
2526
|
+
const extend = options.extend ?? null;
|
|
2527
|
+
const name = units === firstRung && eta === 3 && trainUnits === 2 && reps === 1 && extend === null ? "asha" : `asha(units=${units},eta=${eta},train=${trainUnits},reps=${reps}${extend ? `,extend=${extend.name}` : ""})`;
|
|
2607
2528
|
const layouts = /* @__PURE__ */ new WeakMap();
|
|
2608
|
-
const
|
|
2529
|
+
const extensions = /* @__PURE__ */ new WeakMap();
|
|
2530
|
+
const seededLayout = (state) => {
|
|
2609
2531
|
const header = state.header;
|
|
2610
2532
|
if (!header) return null;
|
|
2611
2533
|
let layout = layouts.get(header);
|
|
@@ -2618,6 +2540,74 @@ function asha(options = {}) {
|
|
|
2618
2540
|
}
|
|
2619
2541
|
return layout;
|
|
2620
2542
|
};
|
|
2543
|
+
/**
|
|
2544
|
+
* With `extend`, each opened rung's added units come from the chooser at
|
|
2545
|
+
* the sequence of the first decision that opened it; units of rungs not yet
|
|
2546
|
+
* opened follow in the seeded order, which only the root, measured on every
|
|
2547
|
+
* unit, ever reads.
|
|
2548
|
+
*/
|
|
2549
|
+
const layoutOf = (state) => {
|
|
2550
|
+
const seeded = seededLayout(state);
|
|
2551
|
+
if (!seeded || extend === null || seeded.top === 0) return seeded;
|
|
2552
|
+
const opened = /* @__PURE__ */ new Map();
|
|
2553
|
+
for (const node of state.nodes()) for (const record of node.decisions) {
|
|
2554
|
+
if (record.decision.status !== "advanced") continue;
|
|
2555
|
+
const known = opened.get(record.decision.rung);
|
|
2556
|
+
if (known === void 0 || record.sequence < known) opened.set(record.decision.rung, record.sequence);
|
|
2557
|
+
}
|
|
2558
|
+
if (opened.size === 0) return seeded;
|
|
2559
|
+
const header = state.header;
|
|
2560
|
+
let cache = extensions.get(header);
|
|
2561
|
+
if (!cache) {
|
|
2562
|
+
cache = /* @__PURE__ */ new Map();
|
|
2563
|
+
extensions.set(header, cache);
|
|
2564
|
+
}
|
|
2565
|
+
const order = seeded.order.slice(0, seeded.sizes[0]);
|
|
2566
|
+
const chosen = new Set(order);
|
|
2567
|
+
for (let rung = 1; rung <= seeded.top; rung++) {
|
|
2568
|
+
const sequence = opened.get(rung);
|
|
2569
|
+
if (sequence === void 0) break;
|
|
2570
|
+
let entry = cache.get(rung);
|
|
2571
|
+
if (!entry || entry.sequence !== sequence) {
|
|
2572
|
+
const remaining = seeded.order.filter((unitId) => !chosen.has(unitId));
|
|
2573
|
+
const count = seeded.sizes[rung] - seeded.sizes[rung - 1];
|
|
2574
|
+
const allowed = new Set(remaining);
|
|
2575
|
+
const picked = [];
|
|
2576
|
+
for (const unitId of extend.choose({
|
|
2577
|
+
state,
|
|
2578
|
+
asOfSequence: sequence,
|
|
2579
|
+
split: seeded.split,
|
|
2580
|
+
rung,
|
|
2581
|
+
measured: [...order],
|
|
2582
|
+
remaining,
|
|
2583
|
+
count
|
|
2584
|
+
})) {
|
|
2585
|
+
if (picked.length === count) break;
|
|
2586
|
+
if (allowed.delete(unitId)) picked.push(unitId);
|
|
2587
|
+
}
|
|
2588
|
+
for (const unitId of remaining) {
|
|
2589
|
+
if (picked.length === count) break;
|
|
2590
|
+
if (allowed.delete(unitId)) picked.push(unitId);
|
|
2591
|
+
}
|
|
2592
|
+
entry = {
|
|
2593
|
+
sequence,
|
|
2594
|
+
units: picked
|
|
2595
|
+
};
|
|
2596
|
+
cache.set(rung, entry);
|
|
2597
|
+
}
|
|
2598
|
+
for (const unitId of entry.units) {
|
|
2599
|
+
order.push(unitId);
|
|
2600
|
+
chosen.add(unitId);
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
for (const unitId of seeded.order) if (!chosen.has(unitId)) order.push(unitId);
|
|
2604
|
+
return {
|
|
2605
|
+
...seeded,
|
|
2606
|
+
order,
|
|
2607
|
+
position: new Map(order.map((unitId, index) => [unitId, index])),
|
|
2608
|
+
cellsThrough: seeded.sizes.map((size) => order.slice(0, size).reduce((sum, unitId) => sum + seeded.tasks.get(unitId).length, 0))
|
|
2609
|
+
};
|
|
2610
|
+
};
|
|
2621
2611
|
const cellsThrough = (layout, rung, root) => {
|
|
2622
2612
|
const plan = [];
|
|
2623
2613
|
const through = root ? layout.top : Math.min(rung, layout.top);
|
|
@@ -2710,7 +2700,7 @@ function asha(options = {}) {
|
|
|
2710
2700
|
const reason = (ranked, index, k, layout) => {
|
|
2711
2701
|
const entry = ranked[index];
|
|
2712
2702
|
const top = Math.floor(ranked.length / eta);
|
|
2713
|
-
const mean = entry.mean === null ? "an unscored unit" : `mean ${round
|
|
2703
|
+
const mean = entry.mean === null ? "an unscored unit" : `mean ${round(entry.mean)}`;
|
|
2714
2704
|
return `rank ${index + 1} of ${ranked.length} on rung ${k} (${layout.sizes[k]} units, ${mean}); the top ${top} advance (eta ${eta})`;
|
|
2715
2705
|
};
|
|
2716
2706
|
return {
|
|
@@ -2845,1879 +2835,138 @@ function nonNegativeInteger(owner, name, value) {
|
|
|
2845
2835
|
if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${owner}: ${name} must be a non-negative integer, got ${String(value)}`);
|
|
2846
2836
|
return value;
|
|
2847
2837
|
}
|
|
2848
|
-
function round
|
|
2838
|
+
function round(value) {
|
|
2849
2839
|
return Math.round(value * 1e4) / 1e4;
|
|
2850
2840
|
}
|
|
2851
2841
|
//#endregion
|
|
2852
|
-
//#region src/
|
|
2853
|
-
/**
|
|
2854
|
-
* @module
|
|
2855
|
-
* Joint power of a configured {@link decidePairedPromotion} call, computed by
|
|
2856
|
-
* running that call on data drawn from a registered alternative.
|
|
2857
|
-
*
|
|
2858
|
-
* Why a simulation and not a formula: `decidePairedPromotion` is not one test.
|
|
2859
|
-
* It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
|
|
2860
|
-
* paired bootstrap at or above it) and by shape (Tango's score interval on a
|
|
2861
|
-
* two-point outcome), and on top of the deciding interval it applies a
|
|
2862
|
-
* sufficiency floor, a zero-width refusal and McNemar's exact veto. A
|
|
2863
|
-
* preregistration usually seals two such calls, a continuous primary and a
|
|
2864
|
-
* pass/fail veto drawn from the same episodes, and promotes only when both
|
|
2865
|
-
* promote. The sample size at which THAT procedure reaches a target power is
|
|
2866
|
-
* not the power of any one channel: the sign-test power at n can be 0.8 while
|
|
2867
|
-
* the promotion probability is well below it because the veto binds
|
|
2868
|
-
* (agent-eval#785). Closed forms exist for single channels only, and each one
|
|
2869
|
-
* shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
|
|
2870
|
-
* Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
|
|
2871
|
-
* pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
|
|
2872
|
-
* So the only faithful power is the procedure's own promotion rate under the
|
|
2873
|
-
* alternative, and that is what this module computes: draw n pairs from the
|
|
2874
|
-
* registered law, run every configured call on them exactly as the caller
|
|
2875
|
-
* will, repeat, count.
|
|
2876
|
-
*
|
|
2877
|
-
* The alternative is DECLARATIVE and serializable, never a sampler function,
|
|
2878
|
-
* so a preregistration can seal it as bytes: a mixture of cells, each with a
|
|
2879
|
-
* probability, an optional pass/fail outcome per arm for the binary calls,
|
|
2880
|
-
* and a law for the candidate-minus-baseline delta on the continuous scale.
|
|
2881
|
-
* Drawing the cell first and the delta given the cell is what makes the two
|
|
2882
|
-
* channels one joint law rather than two marginals.
|
|
2883
|
-
*
|
|
2884
|
-
* Everything is seeded (`mulberry32`), so a sizing is reproducible from its
|
|
2885
|
-
* inputs, and every estimate carries its Monte Carlo standard error and a
|
|
2886
|
-
* Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
|
|
2887
|
-
*/
|
|
2842
|
+
//#region src/campaign/search-campaign-cell.ts
|
|
2888
2843
|
/**
|
|
2889
|
-
*
|
|
2890
|
-
*
|
|
2844
|
+
* What `cell-settled` records for one campaign cell: `passed` with the
|
|
2845
|
+
* composite (and each judge's) when every judge scored it, else `errored`,
|
|
2846
|
+
* never retryable, because `runCampaign` already applied its own retry policy;
|
|
2847
|
+
* its tokens and cost, with an uncaptured cost kept as a lower bound; the
|
|
2848
|
+
* model the cell resolved, else the declared one.
|
|
2891
2849
|
*/
|
|
2892
|
-
function
|
|
2893
|
-
const n = options.n;
|
|
2894
|
-
if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
|
|
2895
|
-
const simulations = options.simulations ?? 1e3;
|
|
2896
|
-
if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
|
|
2897
|
-
const seed = options.seed ?? 1;
|
|
2898
|
-
if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
|
|
2899
|
-
const calls = validateCalls(options.calls);
|
|
2900
|
-
const alternative = validateAlternative(options.alternative, calls);
|
|
2901
|
-
const rng = mulberry32(seed);
|
|
2902
|
-
let promotions = 0;
|
|
2903
|
-
let refusals = 0;
|
|
2904
|
-
const perCall = calls.map(() => ({
|
|
2905
|
-
promote: 0,
|
|
2906
|
-
insufficient: 0,
|
|
2907
|
-
indeterminate: 0,
|
|
2908
|
-
exactTestVetoes: 0,
|
|
2909
|
-
hold: 0,
|
|
2910
|
-
methods: /* @__PURE__ */ new Map()
|
|
2911
|
-
}));
|
|
2912
|
-
for (let s = 0; s < simulations; s++) {
|
|
2913
|
-
const pairs = drawPairs(alternative, n, rng);
|
|
2914
|
-
let all = true;
|
|
2915
|
-
let anyRefusal = false;
|
|
2916
|
-
for (let c = 0; c < calls.length; c++) {
|
|
2917
|
-
const call = calls[c];
|
|
2918
|
-
const decision = runCall(call, pairs, n);
|
|
2919
|
-
const tally = perCall[c];
|
|
2920
|
-
tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
|
|
2921
|
-
if (decision.promote) tally.promote++;
|
|
2922
|
-
else {
|
|
2923
|
-
if (!decision.sufficient) tally.insufficient++;
|
|
2924
|
-
if (decision.indeterminate) tally.indeterminate++;
|
|
2925
|
-
if (decision.exactTestVetoes) tally.exactTestVetoes++;
|
|
2926
|
-
if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
|
|
2927
|
-
}
|
|
2928
|
-
if (!decision.promote) all = false;
|
|
2929
|
-
if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
|
|
2930
|
-
}
|
|
2931
|
-
if (all) promotions++;
|
|
2932
|
-
else if (anyRefusal) refusals++;
|
|
2933
|
-
}
|
|
2934
|
-
const power = promotions / simulations;
|
|
2935
|
-
const interval = wilson(promotions, simulations, .95);
|
|
2850
|
+
function campaignCellSearchResult(cell, input) {
|
|
2936
2851
|
return {
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
insufficient: tally.insufficient / simulations,
|
|
2950
|
-
indeterminate: tally.indeterminate / simulations,
|
|
2951
|
-
exactTestVetoes: tally.exactTestVetoes / simulations,
|
|
2952
|
-
hold: tally.hold / simulations,
|
|
2953
|
-
methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
|
|
2954
|
-
}))
|
|
2955
|
-
};
|
|
2956
|
-
}
|
|
2957
|
-
/**
|
|
2958
|
-
* First n at which {@link pairedPromotionPower} reaches the target, scanning
|
|
2959
|
-
* every n from `minPairs` upward. Both the point-estimate answer and the
|
|
2960
|
-
* confirmed lower-bound answer are returned; the scan runs until the latter is
|
|
2961
|
-
* reached (so the curve past `n` is retained) or `maxPairs` is exhausted. A
|
|
2962
|
-
* candidate selected by the scan's bound is certified only by an independent
|
|
2963
|
-
* simulation at that n on a different seed; a candidate that fails it is
|
|
2964
|
-
* recorded and the scan continues.
|
|
2965
|
-
*/
|
|
2966
|
-
function requiredPairsForPairedPromotion(options) {
|
|
2967
|
-
const target = options.target ?? .8;
|
|
2968
|
-
if (!Number.isFinite(target) || target <= 0 || target >= 1) throw new Error(`requiredPairsForPairedPromotion: target must be in (0,1), got ${target}`);
|
|
2969
|
-
const minPairs = options.minPairs ?? 1;
|
|
2970
|
-
const maxPairs = options.maxPairs ?? 200;
|
|
2971
|
-
if (!Number.isInteger(minPairs) || minPairs < 1) throw new Error(`requiredPairsForPairedPromotion: minPairs must be a positive integer, got ${minPairs}`);
|
|
2972
|
-
if (!Number.isInteger(maxPairs) || maxPairs < minPairs) throw new Error(`requiredPairsForPairedPromotion: maxPairs must be an integer >= minPairs, got ${maxPairs}`);
|
|
2973
|
-
const seed = options.seed ?? 1;
|
|
2974
|
-
const curve = [];
|
|
2975
|
-
const rejected = [];
|
|
2976
|
-
let n = null;
|
|
2977
|
-
let nAtLowerBound = null;
|
|
2978
|
-
let confirmation = null;
|
|
2979
|
-
for (let candidate = minPairs; candidate <= maxPairs; candidate++) {
|
|
2980
|
-
const result = pairedPromotionPower({
|
|
2981
|
-
n: candidate,
|
|
2982
|
-
alternative: options.alternative,
|
|
2983
|
-
calls: options.calls,
|
|
2984
|
-
simulations: options.simulations,
|
|
2985
|
-
seed
|
|
2986
|
-
});
|
|
2987
|
-
curve.push(result);
|
|
2988
|
-
if (n === null && result.power >= target) n = candidate;
|
|
2989
|
-
if (result.low >= target) {
|
|
2990
|
-
const check = pairedPromotionPower({
|
|
2991
|
-
n: candidate,
|
|
2992
|
-
alternative: options.alternative,
|
|
2993
|
-
calls: options.calls,
|
|
2994
|
-
simulations: options.simulations,
|
|
2995
|
-
seed: confirmationSeed(seed, candidate)
|
|
2996
|
-
});
|
|
2997
|
-
if (check.low >= target) {
|
|
2998
|
-
nAtLowerBound = candidate;
|
|
2999
|
-
confirmation = check;
|
|
3000
|
-
break;
|
|
3001
|
-
}
|
|
3002
|
-
rejected.push(check);
|
|
3003
|
-
}
|
|
3004
|
-
}
|
|
3005
|
-
return {
|
|
3006
|
-
target,
|
|
3007
|
-
n,
|
|
3008
|
-
nAtLowerBound,
|
|
3009
|
-
confirmation,
|
|
3010
|
-
curve,
|
|
3011
|
-
rejected
|
|
3012
|
-
};
|
|
3013
|
-
}
|
|
3014
|
-
/** A seed for the confirmation at `n` that never equals the scan seed. */
|
|
3015
|
-
function confirmationSeed(seed, n) {
|
|
3016
|
-
return (Math.imul((seed | 0) ^ 2654435769, 2246822507) + Math.imul(n, 3266489909) | 1) >>> 0;
|
|
3017
|
-
}
|
|
3018
|
-
function drawPairs(alternative, n, rng) {
|
|
3019
|
-
const baseline = alternative.baseline ?? {
|
|
3020
|
-
kind: "point",
|
|
3021
|
-
value: 0
|
|
3022
|
-
};
|
|
3023
|
-
const control = new Array(n);
|
|
3024
|
-
const candidate = new Array(n);
|
|
3025
|
-
const controlPass = new Array(n);
|
|
3026
|
-
const candidatePass = new Array(n);
|
|
3027
|
-
for (let i = 0; i < n; i++) {
|
|
3028
|
-
const cell = drawCell(alternative.cells, rng);
|
|
3029
|
-
const base = drawValue(baseline, rng);
|
|
3030
|
-
control[i] = base;
|
|
3031
|
-
candidate[i] = base + drawValue(cell.delta, rng);
|
|
3032
|
-
controlPass[i] = cell.pass?.control ?? false;
|
|
3033
|
-
candidatePass[i] = cell.pass?.treatment ?? false;
|
|
3034
|
-
}
|
|
3035
|
-
return {
|
|
3036
|
-
control,
|
|
3037
|
-
candidate,
|
|
3038
|
-
controlPass,
|
|
3039
|
-
candidatePass
|
|
3040
|
-
};
|
|
3041
|
-
}
|
|
3042
|
-
function drawCell(cells, rng) {
|
|
3043
|
-
const u = rng();
|
|
3044
|
-
let cumulative = 0;
|
|
3045
|
-
for (const cell of cells) {
|
|
3046
|
-
cumulative += cell.probability;
|
|
3047
|
-
if (u < cumulative) return cell;
|
|
3048
|
-
}
|
|
3049
|
-
return cells[cells.length - 1];
|
|
3050
|
-
}
|
|
3051
|
-
function drawValue(law, rng) {
|
|
3052
|
-
switch (law.kind) {
|
|
3053
|
-
case "point": return law.value;
|
|
3054
|
-
case "atoms": {
|
|
3055
|
-
const u = rng();
|
|
3056
|
-
let cumulative = 0;
|
|
3057
|
-
for (const atom of law.atoms) {
|
|
3058
|
-
cumulative += atom.probability;
|
|
3059
|
-
if (u < cumulative) return atom.value;
|
|
3060
|
-
}
|
|
3061
|
-
return law.atoms[law.atoms.length - 1].value;
|
|
3062
|
-
}
|
|
3063
|
-
case "normal": {
|
|
3064
|
-
const u1 = 1 - rng();
|
|
3065
|
-
const u2 = rng();
|
|
3066
|
-
return law.mean + law.sd * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
|
3067
|
-
}
|
|
3068
|
-
}
|
|
3069
|
-
}
|
|
3070
|
-
function runCall(call, pairs, n) {
|
|
3071
|
-
const options = {
|
|
3072
|
-
minPairs: n,
|
|
3073
|
-
...call.options
|
|
3074
|
-
};
|
|
3075
|
-
if (call.outcome === "pass") {
|
|
3076
|
-
const scale = call.options.binaryScale;
|
|
3077
|
-
return decidePairedPromotion(pairs.controlPass.map((p) => p ? scale : 0), pairs.candidatePass.map((p) => p ? scale : 0), options);
|
|
3078
|
-
}
|
|
3079
|
-
return decidePairedPromotion(pairs.control, pairs.candidate, options);
|
|
3080
|
-
}
|
|
3081
|
-
function validateCalls(calls) {
|
|
3082
|
-
if (!Array.isArray(calls) || calls.length === 0) throw new Error("pairedPromotionPower: at least one call is required");
|
|
3083
|
-
calls.forEach((call, i) => {
|
|
3084
|
-
if (call.outcome !== "delta" && call.outcome !== "pass") throw new Error(`pairedPromotionPower: calls[${i}].outcome must be 'delta' or 'pass'`);
|
|
3085
|
-
if (call.options === null || typeof call.options !== "object") throw new Error(`pairedPromotionPower: calls[${i}].options must be an object`);
|
|
3086
|
-
if (call.options.minPairs !== void 0 && (!Number.isInteger(call.options.minPairs) || call.options.minPairs < 1)) throw new Error(`pairedPromotionPower: calls[${i}].options.minPairs must be a positive integer`);
|
|
3087
|
-
if (call.outcome === "pass") {
|
|
3088
|
-
const scale = call.options.binaryScale;
|
|
3089
|
-
if (scale === void 0 || !Number.isFinite(scale) || scale <= 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call and must declare a finite positive options.binaryScale`);
|
|
3090
|
-
if (call.options.continuous) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; continuous cannot accompany it`);
|
|
3091
|
-
if (call.options.statistic === "median") throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; binaryScale requires the mean statistic, not median`);
|
|
3092
|
-
} else if (call.options.binaryScale !== void 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'delta' call and cannot declare binaryScale; use a 'pass' call for a two-point outcome`);
|
|
3093
|
-
});
|
|
3094
|
-
return calls;
|
|
3095
|
-
}
|
|
3096
|
-
function validateAlternative(alternative, calls) {
|
|
3097
|
-
if (!alternative || !Array.isArray(alternative.cells) || alternative.cells.length === 0) throw new Error("pairedPromotionPower: alternative.cells must be a non-empty array");
|
|
3098
|
-
const needsPass = calls.some((call) => call.outcome === "pass");
|
|
3099
|
-
let total = 0;
|
|
3100
|
-
alternative.cells.forEach((cell, i) => {
|
|
3101
|
-
if (!Number.isFinite(cell.probability) || cell.probability < 0) throw new Error(`pairedPromotionPower: cells[${i}].probability must be a finite number >= 0`);
|
|
3102
|
-
total += cell.probability;
|
|
3103
|
-
if (needsPass) {
|
|
3104
|
-
if (!cell.pass || typeof cell.pass.control !== "boolean" || typeof cell.pass.treatment !== "boolean") throw new Error(`pairedPromotionPower: cells[${i}].pass must give control and treatment booleans when a 'pass' call is configured`);
|
|
3105
|
-
}
|
|
3106
|
-
validateLaw(cell.delta, `cells[${i}].delta`);
|
|
3107
|
-
});
|
|
3108
|
-
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: cell probabilities must sum to 1, got ${total}`);
|
|
3109
|
-
if (alternative.baseline !== void 0) validateLaw(alternative.baseline, "baseline");
|
|
3110
|
-
return alternative;
|
|
3111
|
-
}
|
|
3112
|
-
function validateLaw(law, where) {
|
|
3113
|
-
if (!law || typeof law !== "object") throw new Error(`pairedPromotionPower: ${where} must be a law object`);
|
|
3114
|
-
switch (law.kind) {
|
|
3115
|
-
case "point":
|
|
3116
|
-
if (!Number.isFinite(law.value)) throw new Error(`pairedPromotionPower: ${where}.value must be finite`);
|
|
3117
|
-
return;
|
|
3118
|
-
case "atoms": {
|
|
3119
|
-
if (!Array.isArray(law.atoms) || law.atoms.length === 0) throw new Error(`pairedPromotionPower: ${where}.atoms must be a non-empty array`);
|
|
3120
|
-
let total = 0;
|
|
3121
|
-
law.atoms.forEach((atom, i) => {
|
|
3122
|
-
if (!Number.isFinite(atom.value)) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].value must be finite`);
|
|
3123
|
-
if (!Number.isFinite(atom.probability) || atom.probability < 0) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].probability must be a finite number >= 0`);
|
|
3124
|
-
total += atom.probability;
|
|
3125
|
-
});
|
|
3126
|
-
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: ${where}.atoms probabilities must sum to 1, got ${total}`);
|
|
3127
|
-
return;
|
|
3128
|
-
}
|
|
3129
|
-
case "normal":
|
|
3130
|
-
if (!Number.isFinite(law.mean) || !Number.isFinite(law.sd) || law.sd < 0) throw new Error(`pairedPromotionPower: ${where} needs a finite mean and a finite sd >= 0`);
|
|
3131
|
-
return;
|
|
3132
|
-
default: throw new Error(`pairedPromotionPower: ${where}.kind must be 'point', 'atoms' or 'normal'`);
|
|
3133
|
-
}
|
|
3134
|
-
}
|
|
3135
|
-
//#endregion
|
|
3136
|
-
//#region src/campaign/search-claim.ts
|
|
3137
|
-
/**
|
|
3138
|
-
* The claim: the one statement a search makes about its nodes, made once, on
|
|
3139
|
-
* its sealed test split, after the search stops expanding.
|
|
3140
|
-
*
|
|
3141
|
-
* `planSearchClaim` fixes the claim's design from the ledger before any test
|
|
3142
|
-
* cell runs: up to 3 finalists ranked by selection mean, the estimator the
|
|
3143
|
-
* paired decision will use, the number of finalists the family-wise confidence
|
|
3144
|
-
* is divided among, and a power check on selection data. The kernel stores the
|
|
3145
|
-
* plan on the ledger's `claim` operation, so the ledger proves the design was
|
|
3146
|
-
* fixed before the test data existed. `decideSearchClaim` then reads the test
|
|
3147
|
-
* cells of the root and the finalists, which ran together, and decides each
|
|
3148
|
-
* finalist against the root with `decidePairedPromotion` at the Bonferroni
|
|
3149
|
-
* confidence `1 - 0.05 / k`.
|
|
3150
|
-
*
|
|
3151
|
-
* Every number is a pure function of the ledger: the power simulation is
|
|
3152
|
-
* seeded from the search's seed and the decisions from the digest of the test
|
|
3153
|
-
* cells they read, so a verifier with the same ledger gets the same claim.
|
|
3154
|
-
*/
|
|
3155
|
-
const FAMILY_CONFIDENCE = .95;
|
|
3156
|
-
const TARGET_POWER = .8;
|
|
3157
|
-
const POWER_SIMULATIONS = 500;
|
|
3158
|
-
const RESAMPLES = 2e3;
|
|
3159
|
-
/**
|
|
3160
|
-
* Every rule and parameter the claim depends on. Its digest is the claim's
|
|
3161
|
-
* revision; change a value here whenever the procedure changes, so the
|
|
3162
|
-
* revision moves with it.
|
|
3163
|
-
*/
|
|
3164
|
-
const CLAIM_DEFINITION = {
|
|
3165
|
-
name: "tangle.search-claim.2026-09",
|
|
3166
|
-
finalists: `up to 3 non-root nodes not decided invalid that scored every selection unit and beat the root's mean on the selection units they share, ranked by selection mean in the objective's direction, then registration order`,
|
|
3167
|
-
familyConfidence: FAMILY_CONFIDENCE,
|
|
3168
|
-
correction: "Bonferroni: each of k finalists against the root at 1 - (1 - familyConfidence) / k",
|
|
3169
|
-
finalistCount: "the largest k whose test cells the unspent claim reserve and headroom cover and whose power reaches the target",
|
|
3170
|
-
estimator: "binary with scale s when every train, selection and test unit has one task at one repeat and every pre-test score is 0 or s; otherwise continuous",
|
|
3171
|
-
power: {
|
|
3172
|
-
method: "pairedPromotionPower of the claim decision, simulated at the claim minimumEffect",
|
|
3173
|
-
continuous: "per-unit improvement normal with the pooled selection variance",
|
|
3174
|
-
binary: "discordant units at the pooled selection variance over 2 s^2 per side, shifted by minimumEffect / s; concordant units pass at the root selection pass rate",
|
|
3175
|
-
targetPower: TARGET_POWER,
|
|
3176
|
-
simulations: POWER_SIMULATIONS,
|
|
3177
|
-
seed: "the first 32 bits of hashCanonical([search seed, k])"
|
|
3178
|
-
},
|
|
3179
|
-
decision: {
|
|
3180
|
-
call: "decidePairedPromotion(root, finalist) on per-unit test means in the objective direction",
|
|
3181
|
-
threshold: 0,
|
|
3182
|
-
minPairs: "every test unit",
|
|
3183
|
-
resamples: RESAMPLES,
|
|
3184
|
-
seed: "the first 32 bits of the finalist's test cellSetDigest"
|
|
3185
|
-
},
|
|
3186
|
-
select: "the promoted finalist with the largest improvement; ties to the better selection rank",
|
|
3187
|
-
ship: "a promoted finalist that scored every test unit, with the root, on held-out units, under a pinned judge"
|
|
3188
|
-
};
|
|
3189
|
-
/** The claim procedure every `SearchClaim` and claim plan names. */
|
|
3190
|
-
const SEARCH_CLAIM_RULE = {
|
|
3191
|
-
uri: "npm:@tangle-network/agent-eval#searchClaim",
|
|
3192
|
-
revision: hashCanonical(CLAIM_DEFINITION)
|
|
3193
|
-
};
|
|
3194
|
-
/** The rule name decisions made by the claim record. */
|
|
3195
|
-
const SEARCH_CLAIM_RULE_NAME = CLAIM_DEFINITION.name;
|
|
3196
|
-
/**
|
|
3197
|
-
* The claim reserve a search needs: the root and 3 finalists on every test
|
|
3198
|
-
* task at `reps` repeats, each held at `cellUsd`. Declare at least this as
|
|
3199
|
-
* `budget.reservedClaimUsd`; `runSearch` refuses to start a capped search
|
|
3200
|
-
* with less.
|
|
3201
|
-
*/
|
|
3202
|
-
function searchClaimReserveUsd(input) {
|
|
3203
|
-
return 4 * input.testTasks * input.reps * input.cellUsd;
|
|
3204
|
-
}
|
|
3205
|
-
/** Fix the claim's design from a search that stopped expanding. */
|
|
3206
|
-
function planSearchClaim(state, input) {
|
|
3207
|
-
const header = requireHeader(state);
|
|
3208
|
-
const root = state.rootNodeId;
|
|
3209
|
-
const plan = (finalists, power, test, reason) => ({
|
|
3210
|
-
kind: "search-claim-plan",
|
|
3211
|
-
rule: SEARCH_CLAIM_RULE,
|
|
3212
|
-
stopReason: input.stopReason,
|
|
3213
|
-
finalists,
|
|
3214
|
-
power,
|
|
3215
|
-
test,
|
|
3216
|
-
reason
|
|
3217
|
-
});
|
|
3218
|
-
const selectionUnits = unitIds(header.splits.selection.tasks);
|
|
3219
|
-
const testUnits = unitIds(header.splits.test.tasks);
|
|
3220
|
-
if (testUnits.length === 0) return plan([], { unknown: "the search has no test split" }, "hold", "the search has no test split, so it makes no claim");
|
|
3221
|
-
if (selectionUnits.length === 0) return plan([], { unknown: "the search has no selection split to choose finalists on" }, "hold", "the search has no selection split, so no finalist can be chosen without the test data");
|
|
3222
|
-
const candidates = rankFinalists(state, selectionUnits);
|
|
3223
|
-
if (candidates.length === 0) return plan([], { unknown: "no node went to test" }, "hold", "no node scored every selection unit and beat the root's selection mean, so the search keeps the root");
|
|
3224
|
-
const minimumEffect = header.objective.claim.minimumEffect;
|
|
3225
|
-
if (minimumEffect === void 0) return plan(candidates.slice(0, 1), { unknown: "the claim declares no minimumEffect" }, "test-cannot-resolve", "the claim declares no minimumEffect, so no test split can be shown to resolve it");
|
|
3226
|
-
const { pooledVariance } = searchPosterior(state, { split: "selection" });
|
|
3227
|
-
if (pooledVariance === null) return plan(candidates.slice(0, 1), { unknown: "no node shares 2 selection units with the root, so the variance is unknown" }, "test-cannot-resolve", "the selection split gives no variance for the power check, so the test cannot be shown to resolve the minimum effect");
|
|
3228
|
-
const estimator = claimEstimator(state, input.reps);
|
|
3229
|
-
const rootPassRate = passRate(state.unitScores(root, "selection"), estimator);
|
|
3230
|
-
let best = null;
|
|
3231
|
-
let budgetBound = false;
|
|
3232
|
-
for (let k = candidates.length; k >= 1; k--) {
|
|
3233
|
-
const power = claimPower({
|
|
3234
|
-
units: testUnits.length,
|
|
3235
|
-
finalists: k,
|
|
3236
|
-
minimumEffect,
|
|
3237
|
-
pooledVariance,
|
|
3238
|
-
estimator,
|
|
3239
|
-
rootPassRate,
|
|
3240
|
-
seed: header.policy.seed
|
|
3241
|
-
});
|
|
3242
|
-
best = power;
|
|
3243
|
-
if (!power.adequate) continue;
|
|
3244
|
-
if (!input.affordable(k)) {
|
|
3245
|
-
budgetBound = true;
|
|
3246
|
-
continue;
|
|
3247
|
-
}
|
|
3248
|
-
return plan(candidates.slice(0, k), power, "run", `${k} finalist${k === 1 ? "" : "s"} go to test: power ${round(power.powerAtMinimumEffect)} at an improvement of ${minimumEffect} on ${testUnits.length} test units reaches ${TARGET_POWER} at confidence ${round(bonferroni(k))} each`);
|
|
3249
|
-
}
|
|
3250
|
-
const power = best;
|
|
3251
|
-
return plan(candidates.slice(0, 1), power, "test-cannot-resolve", budgetBound ? "the unspent claim reserve and headroom do not cover the test cells of the root and one finalist" : `even one finalist reaches power ${round("adequate" in power ? power.powerAtMinimumEffect : 0)} at an improvement of ${minimumEffect} on ${testUnits.length} test units, below ${TARGET_POWER}`);
|
|
3252
|
-
}
|
|
3253
|
-
/**
|
|
3254
|
-
* Decide the claim from the test cells of the root and the finalists. Call it
|
|
3255
|
-
* once every claim cell is settled or cancelled. It returns the claim and the
|
|
3256
|
-
* terminal decisions of the root and every finalist.
|
|
3257
|
-
*/
|
|
3258
|
-
function decideSearchClaim(state, plan) {
|
|
3259
|
-
const header = requireHeader(state);
|
|
3260
|
-
const root = state.rootNodeId;
|
|
3261
|
-
const rootScored = [
|
|
3262
|
-
"train",
|
|
3263
|
-
"selection",
|
|
3264
|
-
"test"
|
|
3265
|
-
].some((split) => state.scoredCells(root, split).length > 0);
|
|
3266
|
-
const keepRoot = rootScored ? root : null;
|
|
3267
|
-
const rule = SEARCH_CLAIM_RULE_NAME;
|
|
3268
|
-
const decisions = [];
|
|
3269
|
-
const claimOf = (finalists, selected, decision, reason) => ({
|
|
3270
|
-
rule: SEARCH_CLAIM_RULE,
|
|
3271
|
-
confidence: FAMILY_CONFIDENCE,
|
|
3272
|
-
power: plan.power,
|
|
3273
|
-
finalists,
|
|
3274
|
-
selected,
|
|
3275
|
-
decision,
|
|
3276
|
-
reason
|
|
3277
|
-
});
|
|
3278
|
-
if (plan.test !== "run") {
|
|
3279
|
-
for (const finalist of plan.finalists) decisions.push({
|
|
3280
|
-
nodeId: finalist.nodeId,
|
|
3281
|
-
decision: { status: "rejected" },
|
|
3282
|
-
basis: null,
|
|
3283
|
-
rule,
|
|
3284
|
-
reason: `no test cell ran: ${plan.reason}`
|
|
3285
|
-
});
|
|
3286
|
-
decisions.push(rootDecision(root, rootScored ? "kept" : "unscored", rule, plan.reason));
|
|
3287
|
-
return {
|
|
3288
|
-
claim: claimOf(plan.finalists.map((finalist) => ({
|
|
3289
|
-
nodeId: finalist.nodeId,
|
|
3290
|
-
estimate: null,
|
|
3291
|
-
test: null,
|
|
3292
|
-
promote: false
|
|
3293
|
-
})), keepRoot, plan.test, plan.reason),
|
|
3294
|
-
decisions
|
|
3295
|
-
};
|
|
3296
|
-
}
|
|
3297
|
-
const power = plan.power;
|
|
3298
|
-
if (!("adequate" in power)) throw new Error("a claim that runs its test cells has a known power");
|
|
3299
|
-
const k = plan.finalists.length;
|
|
3300
|
-
const confidence = bonferroni(k);
|
|
3301
|
-
const testUnits = unitIds(header.splits.test.tasks);
|
|
3302
|
-
const maximize = header.objective.direction === "maximize";
|
|
3303
|
-
const rootUnits = state.unitScores(root, "test");
|
|
3304
|
-
const tested = plan.finalists.map(({ nodeId }) => {
|
|
3305
|
-
const estimate = estimateNode(state, nodeId, {
|
|
3306
|
-
against: root,
|
|
3307
|
-
split: "test"
|
|
3308
|
-
});
|
|
3309
|
-
return {
|
|
3310
|
-
nodeId,
|
|
3311
|
-
estimate,
|
|
3312
|
-
...pairedTest({
|
|
3313
|
-
root: rootUnits,
|
|
3314
|
-
node: state.unitScores(nodeId, "test"),
|
|
3315
|
-
testUnits,
|
|
3316
|
-
confidence,
|
|
3317
|
-
estimator: power.estimator,
|
|
3318
|
-
maximize,
|
|
3319
|
-
seed: seedFromDigest(estimate.cellSetDigest)
|
|
3320
|
-
})
|
|
3321
|
-
};
|
|
3322
|
-
});
|
|
3323
|
-
const winner = tested.map((entry, rank) => ({
|
|
3324
|
-
...entry,
|
|
3325
|
-
rank
|
|
3326
|
-
})).filter((entry) => entry.promote).sort((a, b) => b.improvement - a.improvement || a.rank - b.rank)[0] ?? null;
|
|
3327
|
-
const blockers = [];
|
|
3328
|
-
if (!header.splits.heldOutUnits) blockers.push("the test units also appear in train or selection");
|
|
3329
|
-
if ("unknown" in header.objective.judge) blockers.push("the judge is not pinned");
|
|
3330
|
-
const ship = winner !== null && blockers.length === 0;
|
|
3331
|
-
const selected = ship ? winner.nodeId : keepRoot;
|
|
3332
|
-
const reason = ship ? `finalist ${winner.nodeId} beat the root on the ${testUnits.length} test units at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})` : winner !== null ? `finalist ${winner.nodeId} beat the root, but the claim cannot ship: ${blockers.join("; ")}` : `no finalist beat the root on the test split at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})`;
|
|
3333
|
-
for (const entry of tested) decisions.push({
|
|
3334
|
-
nodeId: entry.nodeId,
|
|
3335
|
-
decision: { status: ship && entry.nodeId === winner.nodeId ? "selected" : "rejected" },
|
|
3336
|
-
basis: entry.estimate,
|
|
3337
|
-
rule,
|
|
3338
|
-
reason: ship && entry.nodeId === winner.nodeId ? reason : entry.promote ? ship ? `promoted, but finalist ${winner.nodeId} improved more on the test split` : reason : entry.why
|
|
3339
|
-
});
|
|
3340
|
-
decisions.push(rootDecision(root, ship ? "replaced" : rootScored ? "kept" : "unscored", rule, reason));
|
|
3341
|
-
return {
|
|
3342
|
-
claim: claimOf(tested.map((entry) => ({
|
|
3343
|
-
nodeId: entry.nodeId,
|
|
3344
|
-
estimate: entry.estimate,
|
|
3345
|
-
test: entry.test,
|
|
3346
|
-
promote: entry.promote
|
|
3347
|
-
})), selected, ship ? "ship" : "hold", reason),
|
|
3348
|
-
decisions
|
|
3349
|
-
};
|
|
3350
|
-
}
|
|
3351
|
-
/**
|
|
3352
|
-
* Make a closed search's claim again from its ledger alone and compare it with
|
|
3353
|
-
* the recorded one, byte for byte: the power check, the finalists, each
|
|
3354
|
-
* finalist's test estimate and deciding interval, the selection, the decision
|
|
3355
|
-
* and the reason. A producer's claim is not evidence until this holds; a store
|
|
3356
|
-
* keeps what this function derives, not what the producer wrote.
|
|
3357
|
-
*
|
|
3358
|
-
* It needs no blob, so it works on a store that keeps digests only. The
|
|
3359
|
-
* finalists are the nodes the ledger decided `finalist`, in decision order.
|
|
3360
|
-
* The one input the ledger cannot replay is whether the budget covered k
|
|
3361
|
-
* finalists' test cells; the claim cells that ran answer it (none ran: the
|
|
3362
|
-
* budget covered none). Test repeats come from the claim cells too; without
|
|
3363
|
-
* test cells, the recorded estimator stands for them (binary needs one repeat,
|
|
3364
|
-
* so any other count reproduces continuous).
|
|
3365
|
-
*
|
|
3366
|
-
* Returns null for a search that made no claim.
|
|
3367
|
-
*/
|
|
3368
|
-
function verifySearchClaim(state) {
|
|
3369
|
-
const closed = state.closed;
|
|
3370
|
-
if (!closed) throw new Error(`search ${state.searchId} is open; its claim is made when it closes`);
|
|
3371
|
-
const recorded = closed.claim;
|
|
3372
|
-
if (recorded === null) return null;
|
|
3373
|
-
if (recorded.rule.revision !== SEARCH_CLAIM_RULE.revision) return {
|
|
3374
|
-
status: "unknown",
|
|
3375
|
-
reason: `the claim names rule revision ${recorded.rule.revision}; this code implements ${SEARCH_CLAIM_RULE.revision}`
|
|
3376
|
-
};
|
|
3377
|
-
const claimCells = state.cells().filter((cell) => cell.stage === "claim");
|
|
3378
|
-
const decided = state.nodes().flatMap((node) => {
|
|
3379
|
-
const first = node.decisions.find((record) => record.decision.status === "finalist");
|
|
3380
|
-
return first ? [{
|
|
3381
|
-
nodeId: node.nodeId,
|
|
3382
|
-
sequence: first.sequence
|
|
3383
|
-
}] : [];
|
|
3384
|
-
}).sort((a, b) => a.sequence - b.sequence).map((entry) => entry.nodeId);
|
|
3385
|
-
const reps = claimCells.length > 0 ? 1 + Math.max(...claimCells.map((cell) => cell.rep)) : "adequate" in recorded.power && recorded.power.estimator.kind === "binary" ? 1 : 2;
|
|
3386
|
-
const plan = planSearchClaim(state, {
|
|
3387
|
-
stopReason: closed.reason,
|
|
3388
|
-
reps,
|
|
3389
|
-
affordable: (finalists) => claimCells.length > 0 && finalists <= decided.length
|
|
3390
|
-
});
|
|
3391
|
-
const differences = [];
|
|
3392
|
-
const planned = plan.finalists.map((finalist) => finalist.nodeId);
|
|
3393
|
-
if (canonicalString(planned) !== canonicalString(decided)) differences.push(`finalists: the rule picks [${planned.join(", ")}], the ledger decided [${decided.join(", ")}]`);
|
|
3394
|
-
if (plan.test === "run" !== claimCells.length > 0) differences.push(`test: the plan ${plan.test === "run" ? "runs" : "skips"} the test cells, and ${claimCells.length} ran`);
|
|
3395
|
-
const expected = decideSearchClaim(state, plan).claim;
|
|
3396
|
-
for (const key of [
|
|
3397
|
-
"decision",
|
|
3398
|
-
"selected",
|
|
3399
|
-
"confidence",
|
|
3400
|
-
"power",
|
|
3401
|
-
"reason"
|
|
3402
|
-
]) if (canonicalString(expected[key]) !== canonicalString(recorded[key])) differences.push(`${key}: the ledger supports ${canonicalString(expected[key])}, the claim records ${canonicalString(recorded[key])}`);
|
|
3403
|
-
const count = Math.max(expected.finalists.length, recorded.finalists.length);
|
|
3404
|
-
for (let index = 0; index < count; index++) {
|
|
3405
|
-
const want = expected.finalists[index];
|
|
3406
|
-
const have = recorded.finalists[index];
|
|
3407
|
-
if (canonicalString(want ?? null) === canonicalString(have ?? null)) continue;
|
|
3408
|
-
const fields = [
|
|
3409
|
-
"nodeId",
|
|
3410
|
-
"promote",
|
|
3411
|
-
"test",
|
|
3412
|
-
"estimate"
|
|
3413
|
-
].filter((field) => canonicalString(want?.[field] ?? null) !== canonicalString(have?.[field] ?? null));
|
|
3414
|
-
differences.push(`finalist ${index + 1} (${have?.nodeId ?? want?.nodeId}): ${fields.join(", ")} differ from what the ledger supports`);
|
|
3415
|
-
}
|
|
3416
|
-
return differences.length === 0 ? { status: "verified" } : {
|
|
3417
|
-
status: "mismatch",
|
|
3418
|
-
differences
|
|
3419
|
-
};
|
|
3420
|
-
}
|
|
3421
|
-
/** Per-finalist confidence for `k` finalists. */
|
|
3422
|
-
function bonferroni(k) {
|
|
3423
|
-
return 1 - (1 - FAMILY_CONFIDENCE) / k;
|
|
3424
|
-
}
|
|
3425
|
-
function rootDecision(root, fate, rule, reason) {
|
|
3426
|
-
return {
|
|
3427
|
-
nodeId: root,
|
|
3428
|
-
decision: { status: fate === "kept" ? "selected" : "rejected" },
|
|
3429
|
-
basis: null,
|
|
3430
|
-
rule,
|
|
3431
|
-
reason: fate === "kept" ? `the search keeps the root: ${reason}` : fate === "replaced" ? `a finalist replaces the root: ${reason}` : "the root has no scored cell, so the search keeps nothing"
|
|
3432
|
-
};
|
|
3433
|
-
}
|
|
3434
|
-
/** Nodes eligible to go to test, best first, at most `SEARCH_CLAIM_MAX_FINALISTS`. */
|
|
3435
|
-
function rankFinalists(state, selectionUnits) {
|
|
3436
|
-
const sign = requireHeader(state).objective.direction === "maximize" ? 1 : -1;
|
|
3437
|
-
const root = state.rootNodeId;
|
|
3438
|
-
const rootMeans = new Map(state.unitScores(root, "selection").map((unit) => [unit.unitId, unit.mean]));
|
|
3439
|
-
const ranked = [];
|
|
3440
|
-
for (const node of state.nodes()) {
|
|
3441
|
-
if (node.nodeId === root || node.status === "invalid") continue;
|
|
3442
|
-
const units = state.unitScores(node.nodeId, "selection");
|
|
3443
|
-
if (units.length !== selectionUnits.length) continue;
|
|
3444
|
-
let gain = 0;
|
|
3445
|
-
let shared = 0;
|
|
3446
|
-
for (const unit of units) {
|
|
3447
|
-
const rootMean = rootMeans.get(unit.unitId);
|
|
3448
|
-
if (rootMean === void 0) continue;
|
|
3449
|
-
gain += sign * (unit.mean - rootMean);
|
|
3450
|
-
shared += 1;
|
|
3451
|
-
}
|
|
3452
|
-
if (shared === 0 || gain <= 0) continue;
|
|
3453
|
-
ranked.push({
|
|
3454
|
-
nodeId: node.nodeId,
|
|
3455
|
-
selectionMean: mean$2(units.map((unit) => unit.mean)),
|
|
3456
|
-
ordinal: node.ordinal
|
|
3457
|
-
});
|
|
3458
|
-
}
|
|
3459
|
-
ranked.sort((a, b) => sign * (b.selectionMean - a.selectionMean) || a.ordinal - b.ordinal);
|
|
3460
|
-
return ranked.slice(0, 3).map(({ nodeId, selectionMean }) => ({
|
|
3461
|
-
nodeId,
|
|
3462
|
-
selectionMean
|
|
3463
|
-
}));
|
|
3464
|
-
}
|
|
3465
|
-
/**
|
|
3466
|
-
* The paired decision's estimator, fixed from pre-test data. A per-unit mean
|
|
3467
|
-
* is binary only when each unit holds one task at one repeat and every score
|
|
3468
|
-
* seen before the test is 0 or one positive value.
|
|
3469
|
-
*/
|
|
3470
|
-
function claimEstimator(state, reps) {
|
|
3471
|
-
const header = requireHeader(state);
|
|
3472
|
-
if (![
|
|
3473
|
-
"train",
|
|
3474
|
-
"selection",
|
|
3475
|
-
"test"
|
|
3476
|
-
].every((split) => {
|
|
3477
|
-
const tasks = header.splits[split].tasks;
|
|
3478
|
-
return unitIds(tasks).length === tasks.length;
|
|
3479
|
-
}) || reps !== 1) return { kind: "continuous" };
|
|
3480
|
-
let scale = null;
|
|
3481
|
-
for (const nodeId of state.nodeIds()) for (const split of ["train", "selection"]) for (const cell of state.scoredCells(nodeId, split)) {
|
|
3482
|
-
if (cell.score === 0) continue;
|
|
3483
|
-
if (cell.score < 0 || scale !== null && cell.score !== scale) return { kind: "continuous" };
|
|
3484
|
-
scale = cell.score;
|
|
3485
|
-
}
|
|
3486
|
-
return scale === null ? { kind: "continuous" } : {
|
|
3487
|
-
kind: "binary",
|
|
3488
|
-
scale
|
|
3489
|
-
};
|
|
3490
|
-
}
|
|
3491
|
-
function passRate(units, estimator) {
|
|
3492
|
-
if (estimator.kind !== "binary" || units.length === 0) return 0;
|
|
3493
|
-
return units.filter((unit) => unit.mean === estimator.scale).length / units.length;
|
|
3494
|
-
}
|
|
3495
|
-
function claimPower(input) {
|
|
3496
|
-
const { units, finalists, minimumEffect, pooledVariance, estimator } = input;
|
|
3497
|
-
const confidence = bonferroni(finalists);
|
|
3498
|
-
let alternative;
|
|
3499
|
-
let call;
|
|
3500
|
-
if (estimator.kind === "binary") {
|
|
3501
|
-
const s = estimator.scale;
|
|
3502
|
-
const shift = Math.min(1, minimumEffect / s);
|
|
3503
|
-
const discordance = Math.min(.5, pooledVariance / (2 * s * s));
|
|
3504
|
-
const loss = Math.max(0, discordance - shift / 2);
|
|
3505
|
-
const win = Math.min(1 - loss, loss + shift);
|
|
3506
|
-
const concordant = Math.max(0, 1 - win - loss);
|
|
3507
|
-
alternative = { cells: [
|
|
3508
|
-
{
|
|
3509
|
-
probability: win,
|
|
3510
|
-
pass: {
|
|
3511
|
-
control: false,
|
|
3512
|
-
treatment: true
|
|
3513
|
-
},
|
|
3514
|
-
delta: {
|
|
3515
|
-
kind: "point",
|
|
3516
|
-
value: s
|
|
3517
|
-
}
|
|
3518
|
-
},
|
|
3519
|
-
{
|
|
3520
|
-
probability: loss,
|
|
3521
|
-
pass: {
|
|
3522
|
-
control: true,
|
|
3523
|
-
treatment: false
|
|
3524
|
-
},
|
|
3525
|
-
delta: {
|
|
3526
|
-
kind: "point",
|
|
3527
|
-
value: -s
|
|
3528
|
-
}
|
|
3529
|
-
},
|
|
3530
|
-
{
|
|
3531
|
-
probability: concordant * input.rootPassRate,
|
|
3532
|
-
pass: {
|
|
3533
|
-
control: true,
|
|
3534
|
-
treatment: true
|
|
3535
|
-
},
|
|
3536
|
-
delta: {
|
|
3537
|
-
kind: "point",
|
|
3538
|
-
value: 0
|
|
3539
|
-
}
|
|
3540
|
-
},
|
|
3541
|
-
{
|
|
3542
|
-
probability: concordant * (1 - input.rootPassRate),
|
|
3543
|
-
pass: {
|
|
3544
|
-
control: false,
|
|
3545
|
-
treatment: false
|
|
3546
|
-
},
|
|
3547
|
-
delta: {
|
|
3548
|
-
kind: "point",
|
|
3549
|
-
value: 0
|
|
3550
|
-
}
|
|
3551
|
-
}
|
|
3552
|
-
] };
|
|
3553
|
-
call = {
|
|
3554
|
-
outcome: "pass",
|
|
3555
|
-
options: {
|
|
3556
|
-
binaryScale: s,
|
|
3557
|
-
confidence,
|
|
3558
|
-
threshold: 0
|
|
3559
|
-
}
|
|
3560
|
-
};
|
|
3561
|
-
} else {
|
|
3562
|
-
alternative = { cells: [{
|
|
3563
|
-
probability: 1,
|
|
3564
|
-
delta: {
|
|
3565
|
-
kind: "normal",
|
|
3566
|
-
mean: minimumEffect,
|
|
3567
|
-
sd: Math.sqrt(pooledVariance)
|
|
3568
|
-
}
|
|
3569
|
-
}] };
|
|
3570
|
-
call = {
|
|
3571
|
-
outcome: "delta",
|
|
3572
|
-
options: {
|
|
3573
|
-
continuous: true,
|
|
3574
|
-
confidence,
|
|
3575
|
-
threshold: 0,
|
|
3576
|
-
resamples: RESAMPLES
|
|
3577
|
-
}
|
|
3578
|
-
};
|
|
3579
|
-
}
|
|
3580
|
-
const simulated = pairedPromotionPower({
|
|
3581
|
-
n: units,
|
|
3582
|
-
alternative,
|
|
3583
|
-
calls: [call],
|
|
3584
|
-
simulations: POWER_SIMULATIONS,
|
|
3585
|
-
seed: seedFromDigest(hashCanonical([input.seed, finalists]))
|
|
3586
|
-
});
|
|
3587
|
-
return {
|
|
3588
|
-
adequate: simulated.power >= TARGET_POWER,
|
|
3589
|
-
minimumEffect,
|
|
3590
|
-
powerAtMinimumEffect: simulated.power,
|
|
3591
|
-
targetPower: TARGET_POWER,
|
|
3592
|
-
units,
|
|
3593
|
-
finalists,
|
|
3594
|
-
pooledVariance,
|
|
3595
|
-
estimator
|
|
2852
|
+
outcome: campaignCellOutcome(cell),
|
|
2853
|
+
accounting: campaignCellAccounting(cell),
|
|
2854
|
+
identity: {
|
|
2855
|
+
...input.execution,
|
|
2856
|
+
model: cell.resolvedModel === void 0 ? input.execution.model : searchModelIdentity(cell.resolvedModel, input.execution.model.provider)
|
|
2857
|
+
},
|
|
2858
|
+
wallMs: cell.durationMs,
|
|
2859
|
+
placement: {
|
|
2860
|
+
lane: input.lane,
|
|
2861
|
+
boxId: null
|
|
2862
|
+
},
|
|
2863
|
+
traceRef: { unknown: "runCampaign reports no trace id per cell" }
|
|
3596
2864
|
};
|
|
3597
2865
|
}
|
|
3598
|
-
/**
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
const rootMean = rootMeans.get(unit.unitId);
|
|
3608
|
-
if (rootMean === void 0) continue;
|
|
3609
|
-
before.push(input.maximize ? rootMean : unit.mean);
|
|
3610
|
-
after.push(input.maximize ? unit.mean : rootMean);
|
|
3611
|
-
}
|
|
3612
|
-
const pairs = before.length;
|
|
3613
|
-
if (pairs === 0) return {
|
|
3614
|
-
test: null,
|
|
3615
|
-
promote: false,
|
|
3616
|
-
improvement: null,
|
|
3617
|
-
why: "no test unit was scored by both it and the root"
|
|
3618
|
-
};
|
|
3619
|
-
const { estimator } = input;
|
|
3620
|
-
if (estimator.kind === "binary" && [...before, ...after].some((value) => value !== 0 && value !== estimator.scale)) return {
|
|
3621
|
-
test: null,
|
|
3622
|
-
promote: false,
|
|
3623
|
-
improvement: null,
|
|
3624
|
-
why: `a test score left the two-point scale {0, ${estimator.scale}} the claim fixed before the test`
|
|
3625
|
-
};
|
|
3626
|
-
const decision = decidePairedPromotion(before, after, {
|
|
3627
|
-
confidence: input.confidence,
|
|
3628
|
-
threshold: 0,
|
|
3629
|
-
minPairs: input.testUnits.length,
|
|
3630
|
-
resamples: RESAMPLES,
|
|
3631
|
-
seed: input.seed,
|
|
3632
|
-
...estimator.kind === "binary" ? { binaryScale: estimator.scale } : { continuous: true }
|
|
3633
|
-
});
|
|
3634
|
-
const test = {
|
|
3635
|
-
pairs,
|
|
3636
|
-
confidence: input.confidence,
|
|
3637
|
-
method: decision.method,
|
|
3638
|
-
delta: plain(input.maximize ? decision.delta : -decision.delta),
|
|
3639
|
-
interval: input.maximize ? [plain(decision.low), plain(decision.high)] : [plain(-decision.high), plain(-decision.low)]
|
|
3640
|
-
};
|
|
3641
|
-
const complete = pairs === input.testUnits.length;
|
|
3642
|
-
const promote = decision.promote && complete;
|
|
3643
|
-
const why = !complete ? `it and the root scored ${pairs} of ${input.testUnits.length} test units, and a claim needs every one` : decision.promote ? "promoted" : !decision.sufficient ? `${pairs} test units are below the ${decision.minimumPairs} its paired test needs` : decision.indeterminate ? `its test interval has no width: ${decision.indeterminateCause}` : decision.exactTestVetoes ? "McNemar's exact test vetoes the improvement" : `its improvement interval [${round(decision.low)}, ${round(decision.high)}] at confidence ${round(input.confidence)} does not exclude 0`;
|
|
3644
|
-
return {
|
|
3645
|
-
test,
|
|
3646
|
-
promote,
|
|
3647
|
-
improvement: decision.delta,
|
|
3648
|
-
why
|
|
2866
|
+
/** A provider-reported model: a snapshot when its name pins one, else a moving alias. */
|
|
2867
|
+
function searchModelIdentity(model, provider) {
|
|
2868
|
+
return modelHasSnapshot(model) ? {
|
|
2869
|
+
provider,
|
|
2870
|
+
snapshot: model
|
|
2871
|
+
} : {
|
|
2872
|
+
provider,
|
|
2873
|
+
alias: model,
|
|
2874
|
+
unknown: "the provider reported a moving alias, not a snapshot"
|
|
3649
2875
|
};
|
|
3650
2876
|
}
|
|
3651
|
-
function
|
|
3652
|
-
const
|
|
3653
|
-
if (
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
}
|
|
3665
|
-
function round(value) {
|
|
3666
|
-
return Math.round(value * 1e4) / 1e4;
|
|
3667
|
-
}
|
|
3668
|
-
/** Canonical JSON has one zero; never hand it a negative one. */
|
|
3669
|
-
function plain(value) {
|
|
3670
|
-
return value === 0 ? 0 : value;
|
|
3671
|
-
}
|
|
3672
|
-
/** The kernel every `runSearch` ledger names as its search implementation. */
|
|
3673
|
-
const SEARCH_KERNEL_SOURCE = {
|
|
3674
|
-
uri: "npm:@tangle-network/agent-eval#runSearch",
|
|
3675
|
-
revision: hashCanonical({
|
|
3676
|
-
name: "tangle.search-kernel.2026-09",
|
|
3677
|
-
dispatch: "claim, then rung and root, then screen, then train; first allocated first within a stage",
|
|
3678
|
-
expansion: "when no cell waits, fewer than twice the lanes capacity run, the cap admits one proposal and its expected screens, and overspend has not taken the search past its cap",
|
|
3679
|
-
reservation: {
|
|
3680
|
-
hard: "the lane per-cell maximum",
|
|
3681
|
-
estimate: "1.5 times the p99 of the lane settled cells once 20 settled, else the lane prior"
|
|
3682
|
-
},
|
|
3683
|
-
retries: "a retryable errored attempt runs again, up to maxAttempts",
|
|
3684
|
-
rungs: "when a node finishes its cells, the allocator names the nodes it advances, deepest rung first; each is recorded advanced and its rung allocated when the cap admits the rung",
|
|
3685
|
-
resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
|
|
3686
|
-
writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
|
|
3687
|
-
divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
|
|
3688
|
-
claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
|
|
3689
|
-
close: "the allocator prunes the nodes left waiting on a rung, except the policy leader; then, with a test split, the claim decides; without one, the policy leader is selected when it has a scored cell; every other undecided node is rejected",
|
|
3690
|
-
complete: "a node leads only when it has a scored cell on the ranked split and no cell there ended unscored: a final unscored outcome, or an error with no attempt left"
|
|
3691
|
-
})
|
|
3692
|
-
};
|
|
3693
|
-
const USD_TOLERANCE = 1e-9;
|
|
3694
|
-
/** The operation that holds the claim's plan. */
|
|
3695
|
-
const CLAIM_OPERATION = "claim";
|
|
3696
|
-
/** Settled cells a lane needs before its own cost distribution sets its estimate. */
|
|
3697
|
-
const ESTIMATE_FROM_CELLS = 20;
|
|
3698
|
-
const ESTIMATE_MARGIN = 1.5;
|
|
3699
|
-
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
3700
|
-
/** Dispatch order: claim cells first, then rungs and the root, then screens, then train. */
|
|
3701
|
-
const STAGE_PRIORITY = {
|
|
3702
|
-
claim: 0,
|
|
3703
|
-
rung: 1,
|
|
3704
|
-
root: 1,
|
|
3705
|
-
screen: 2,
|
|
3706
|
-
train: 3,
|
|
3707
|
-
external: 3
|
|
3708
|
-
};
|
|
3709
|
-
/** Run a search to its close, or continue one from its ledger. */
|
|
3710
|
-
async function runSearch(options) {
|
|
3711
|
-
return new SearchKernel(options).run();
|
|
3712
|
-
}
|
|
3713
|
-
/** The expansion index a kernel operation id encodes, or null. */
|
|
3714
|
-
function searchExpansionIndex(operationId) {
|
|
3715
|
-
const match = /^expand-(\d+)$/.exec(operationId);
|
|
3716
|
-
return match ? Number(match[1]) : null;
|
|
3717
|
-
}
|
|
3718
|
-
var SearchKernel = class {
|
|
3719
|
-
options;
|
|
3720
|
-
recorder;
|
|
3721
|
-
maxAttempts;
|
|
3722
|
-
now;
|
|
3723
|
-
state;
|
|
3724
|
-
lanes = /* @__PURE__ */ new Map();
|
|
3725
|
-
inFlight = /* @__PURE__ */ new Map();
|
|
3726
|
-
proposal = null;
|
|
3727
|
-
proposalController = null;
|
|
3728
|
-
completions = [];
|
|
3729
|
-
wake = null;
|
|
3730
|
-
/** Unfinished cells per node. */
|
|
3731
|
-
pending = /* @__PURE__ */ new Map();
|
|
3732
|
-
/** Admitted nodes whose screen finished, in registration order. */
|
|
3733
|
-
screened = [];
|
|
3734
|
-
screenedSet = /* @__PURE__ */ new Set();
|
|
3735
|
-
admitted = /* @__PURE__ */ new Set();
|
|
3736
|
-
expansionOf = /* @__PURE__ */ new Map();
|
|
3737
|
-
artifacts = /* @__PURE__ */ new Map();
|
|
3738
|
-
/** Children a proposal returned, by artifact digest, until they are registered. */
|
|
3739
|
-
proposed = /* @__PURE__ */ new Map();
|
|
3740
|
-
/** Cells the ledger showed unsettled at start: an earlier process may hold their attempt. */
|
|
3741
|
-
adoptable = /* @__PURE__ */ new Set();
|
|
3742
|
-
operationCosts = [];
|
|
3743
|
-
/** Operations started: the next one's id. */
|
|
3744
|
-
expansions = 0;
|
|
3745
|
-
/** Proposals that completed, by operation index. A proposal an interrupted
|
|
3746
|
-
* process lost counts toward none of the stop rules, so a resumed search
|
|
3747
|
-
* makes the proposals the uninterrupted one would have made. */
|
|
3748
|
-
completed = [];
|
|
3749
|
-
stopReason = null;
|
|
3750
|
-
failure = null;
|
|
3751
|
-
/** The claim's design once expansion stopped; the search then only claims. */
|
|
3752
|
-
claimPlan = null;
|
|
3753
|
-
constructor(options) {
|
|
3754
|
-
this.options = options;
|
|
3755
|
-
this.recorder = options.recorder;
|
|
3756
|
-
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
3757
|
-
this.now = options.now ?? Date.now;
|
|
3758
|
-
if (!Number.isSafeInteger(this.maxAttempts) || this.maxAttempts < 1) throw new TypeError("runSearch: maxAttempts must be a positive integer");
|
|
3759
|
-
const { maxExpansions } = options;
|
|
3760
|
-
if (maxExpansions !== void 0 && (!Number.isSafeInteger(maxExpansions) || maxExpansions < 0)) throw new TypeError("runSearch: maxExpansions must be a non-negative integer");
|
|
3761
|
-
for (const lane of options.executor.lanes()) {
|
|
3762
|
-
if (!Number.isSafeInteger(lane.capacity) || lane.capacity < 1) throw new TypeError(`runSearch: lane ${lane.name} needs a positive integer capacity`);
|
|
3763
|
-
if (!Number.isFinite(lane.cellUsd) || lane.cellUsd < 0) throw new TypeError(`runSearch: lane ${lane.name} needs a finite non-negative cellUsd`);
|
|
3764
|
-
if (this.lanes.has(lane.name)) throw new TypeError(`runSearch: lane ${lane.name} is declared twice`);
|
|
3765
|
-
this.lanes.set(lane.name, {
|
|
3766
|
-
...lane,
|
|
3767
|
-
inFlight: 0,
|
|
3768
|
-
queues: [
|
|
3769
|
-
[],
|
|
3770
|
-
[],
|
|
3771
|
-
[],
|
|
3772
|
-
[]
|
|
3773
|
-
],
|
|
3774
|
-
settledCosts: []
|
|
3775
|
-
});
|
|
3776
|
-
}
|
|
3777
|
-
if (this.lanes.size === 0) throw new TypeError("runSearch: the executor declares no lane");
|
|
3778
|
-
}
|
|
3779
|
-
async run() {
|
|
3780
|
-
const { ledger } = this.recorder;
|
|
3781
|
-
const lock = ledger instanceof FileSearchLedger ? acquireSingleRunLock({ lockPath: `${ledger.path}.run.lock` }) : null;
|
|
3782
|
-
try {
|
|
3783
|
-
return await this.runLocked();
|
|
3784
|
-
} finally {
|
|
3785
|
-
lock?.release();
|
|
3786
|
-
}
|
|
3787
|
-
}
|
|
3788
|
-
async runLocked() {
|
|
3789
|
-
await this.refresh();
|
|
3790
|
-
const header = this.state.header;
|
|
3791
|
-
if (!header) throw new Error(`runSearch: search ${this.recorder.searchId} has not been opened`);
|
|
3792
|
-
if (header.policy.expansion !== this.options.policy.name || header.policy.allocation !== this.options.allocation.name) throw new Error(`runSearch: search ${this.recorder.searchId} was opened with policy ${header.policy.expansion} and allocation ${header.policy.allocation}, not ${this.options.policy.name} and ${this.options.allocation.name}`);
|
|
3793
|
-
if (this.state.closed) return this.closedResult();
|
|
3794
|
-
await this.restore();
|
|
3795
|
-
const { signal } = this.options;
|
|
3796
|
-
const onAbort = () => {
|
|
3797
|
-
for (const controller of this.inFlight.values()) controller.abort(signal?.reason);
|
|
3798
|
-
this.proposalController?.abort(signal?.reason);
|
|
3799
|
-
this.wakeUp();
|
|
3800
|
-
};
|
|
3801
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
3802
|
-
try {
|
|
3803
|
-
if (this.claimPlan) await this.applyClaimPlan(this.claimPlan);
|
|
3804
|
-
await this.loop();
|
|
3805
|
-
if (!this.failure && !this.claimPlan && this.state.header.splits.test.tasks.length > 0) {
|
|
3806
|
-
await this.beginClaim();
|
|
3807
|
-
await this.loop();
|
|
3808
|
-
}
|
|
3809
|
-
} finally {
|
|
3810
|
-
signal?.removeEventListener("abort", onAbort);
|
|
3811
|
-
}
|
|
3812
|
-
if (this.failure) throw this.failure.error;
|
|
3813
|
-
await this.close();
|
|
3814
|
-
return this.closedResult();
|
|
3815
|
-
}
|
|
3816
|
-
async restore() {
|
|
3817
|
-
const rootInput = this.options.codec.node(this.recorder, this.options.root);
|
|
3818
|
-
const rootId = this.state.rootNodeId;
|
|
3819
|
-
if (rootId === null) {
|
|
3820
|
-
const { nodeId } = await this.recorder.registerNode(rootInput);
|
|
3821
|
-
await this.recorder.recordEdge({
|
|
3822
|
-
childNodeId: nodeId,
|
|
3823
|
-
parents: [],
|
|
3824
|
-
operator: "seed",
|
|
3825
|
-
attribution: "explicit",
|
|
3826
|
-
proposer: null,
|
|
3827
|
-
proposalKey: "root",
|
|
3828
|
-
rationale: { unknown: "the root is the caller-supplied starting artifact" },
|
|
3829
|
-
diffs: [],
|
|
3830
|
-
label: "root"
|
|
3831
|
-
});
|
|
3832
|
-
await this.refresh();
|
|
3833
|
-
} else if (this.state.node(rootId).artifactDigest !== rootInput.artifactDigest) throw new Error(`runSearch: search ${this.recorder.searchId} started from another root artifact; a changed start is a derived search`);
|
|
3834
|
-
const root = this.state.rootNodeId;
|
|
3835
|
-
this.artifacts.set(root, this.options.root);
|
|
3836
|
-
this.expansionOf.set(root, -1);
|
|
3837
|
-
if (this.state.audit.cells.allocated === 0) this.assertClaimReady();
|
|
3838
|
-
await this.restoreClaimPlan();
|
|
3839
|
-
const recorded = [];
|
|
3840
|
-
for (let expansion = 0;; expansion++) {
|
|
3841
|
-
const operationId = `expand-${expansion}`;
|
|
3842
|
-
const operation = this.state.operation(operationId);
|
|
3843
|
-
if (!operation) break;
|
|
3844
|
-
this.expansions = expansion + 1;
|
|
3845
|
-
if (operation.recorded) {
|
|
3846
|
-
this.operationCosts.push(operation.spentUsd);
|
|
3847
|
-
recorded.push(expansion);
|
|
3848
|
-
if (operation.outcome === "completed") this.completed.push(expansion);
|
|
3849
|
-
continue;
|
|
3850
|
-
}
|
|
3851
|
-
await this.recorder.recordOperation({
|
|
3852
|
-
operationId,
|
|
3853
|
-
operationKind: "candidate-generation",
|
|
3854
|
-
execution: this.options.proposer.execution,
|
|
3855
|
-
outcome: {
|
|
3856
|
-
status: "failed",
|
|
3857
|
-
failure: {
|
|
3858
|
-
code: "interrupted",
|
|
3859
|
-
message: "the search stopped while this proposal ran; its output was lost"
|
|
3860
|
-
}
|
|
3861
|
-
},
|
|
3862
|
-
accounting: interruptedAccounting()
|
|
3863
|
-
});
|
|
3864
|
-
await this.refresh();
|
|
3865
|
-
}
|
|
3866
|
-
for (const node of this.state.nodes()) {
|
|
3867
|
-
if (node.edgeIds.length === 0) continue;
|
|
3868
|
-
const edge = this.state.edge(node.edgeIds[0]);
|
|
3869
|
-
const expansion = edge.proposer?.operationId ? searchExpansionIndex(edge.proposer.operationId) : null;
|
|
3870
|
-
this.expansionOf.set(node.nodeId, expansion ?? -1);
|
|
3871
|
-
if (node.status !== "invalid") this.admitted.add(node.nodeId);
|
|
3872
|
-
}
|
|
3873
|
-
for (const cell of this.state.cells()) {
|
|
3874
|
-
const lane = cell.lane === null ? void 0 : this.lanes.get(cell.lane);
|
|
3875
|
-
if (cell.final && lane) lane.settledCosts.push(cell.spentUsd);
|
|
3876
|
-
if (this.cellDone(cell)) continue;
|
|
3877
|
-
if (!lane) throw new Error(`runSearch: cell ${cell.cellId} was placed on lane ${String(cell.lane)}, which the executor does not declare`);
|
|
3878
|
-
this.adoptable.add(cell.cellId);
|
|
3879
|
-
this.enqueue(cell);
|
|
3880
|
-
if (cell.stage !== "claim") this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
|
|
3881
|
-
}
|
|
3882
|
-
if (this.claimPlan) return;
|
|
3883
|
-
for (const expansion of recorded) await this.finishProposal(expansion);
|
|
3884
|
-
for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
|
|
3885
|
-
for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) await this.screenDone(nodeId);
|
|
3886
|
-
await this.advance();
|
|
3887
|
-
}
|
|
3888
|
-
/**
|
|
3889
|
-
* A search that will claim must be able to: finalists come from a selection
|
|
3890
|
-
* split, the power check needs the claim's minimum effect, a ship needs a
|
|
3891
|
-
* pinned judge, and under a cap the claim reserve must cover the root and 3
|
|
3892
|
-
* finalists on every test task while the rest of the cap covers one
|
|
3893
|
-
* screening round. Checked before the first cell, so a search that cannot
|
|
3894
|
-
* claim spends nothing.
|
|
3895
|
-
*/
|
|
3896
|
-
assertClaimReady() {
|
|
3897
|
-
const header = this.state.header;
|
|
3898
|
-
const testTasks = header.splits.test.tasks.length;
|
|
3899
|
-
if (testTasks === 0) return;
|
|
3900
|
-
const refuse = (why) => {
|
|
3901
|
-
throw new Error(`runSearch: search ${this.recorder.searchId} has a test split, but ${why}`);
|
|
3902
|
-
};
|
|
3903
|
-
if (header.splits.selection.tasks.length === 0) refuse("no selection split to choose its finalists on");
|
|
3904
|
-
if (header.objective.claim.minimumEffect === void 0) refuse("its claim declares no minimumEffect for the power check");
|
|
3905
|
-
if ("unknown" in header.objective.judge) refuse("its judge is not pinned; a claim with an unknown judge cannot ship, and a resumed search could mix two judges");
|
|
3906
|
-
const { maxUsd, reservedClaimUsd } = header.budget;
|
|
3907
|
-
if (maxUsd === null) return;
|
|
3908
|
-
const cellUsd = this.maxCellReservation();
|
|
3909
|
-
const need = searchClaimReserveUsd({
|
|
3910
|
-
testTasks,
|
|
3911
|
-
reps: this.options.allocation.reps,
|
|
3912
|
-
cellUsd
|
|
3913
|
-
});
|
|
3914
|
-
if (reservedClaimUsd + USD_TOLERANCE < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
|
|
3915
|
-
const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId, 0).length;
|
|
3916
|
-
const screens = (this.options.proposer.childrenPerProposal ?? 1) * this.options.allocation.screenSize(this.state);
|
|
3917
|
-
const round = this.operationReservation().usd + (rootCells + screens) * cellUsd;
|
|
3918
|
-
if (maxUsd - reservedClaimUsd + USD_TOLERANCE < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
|
|
3919
|
-
}
|
|
3920
|
-
/** A claim an earlier process began: continue it from its stored plan. */
|
|
3921
|
-
async restoreClaimPlan() {
|
|
3922
|
-
const operation = this.state.operation(CLAIM_OPERATION);
|
|
3923
|
-
if (!operation) return;
|
|
3924
|
-
const ref = operation.artifacts.find((artifact) => artifact.role === "claim-plan");
|
|
3925
|
-
if (!ref) throw new Error(`runSearch: the ${CLAIM_OPERATION} operation has no stored plan`);
|
|
3926
|
-
const plan = this.recorder.readBlob(ref);
|
|
3927
|
-
if (!operation.recorded) await this.recordClaimOperation();
|
|
3928
|
-
this.claimPlan = plan;
|
|
3929
|
-
this.stopReason = plan.stopReason;
|
|
3930
|
-
}
|
|
3931
|
-
async loop() {
|
|
3932
|
-
for (;;) {
|
|
3933
|
-
if (this.failure || this.options.signal?.aborted) {
|
|
3934
|
-
await this.drain(this.failure?.error ?? this.options.signal?.reason);
|
|
3935
|
-
this.failure ??= { error: this.options.signal?.reason ?? /* @__PURE__ */ new Error("aborted") };
|
|
3936
|
-
return;
|
|
3937
|
-
}
|
|
3938
|
-
if (this.pastDeadline()) {
|
|
3939
|
-
this.stopReason ??= "deadline";
|
|
3940
|
-
if (this.queuedCount() > 0) await this.cancelQueued("deadline");
|
|
3941
|
-
}
|
|
3942
|
-
this.dispatch();
|
|
3943
|
-
const blocked = this.stopReason === null && this.proposal === null ? await this.tryExpand() : null;
|
|
3944
|
-
if (!(this.inFlight.size > 0 || this.proposal !== null)) {
|
|
3945
|
-
if (this.queuedCount() > 0) continue;
|
|
3946
|
-
this.stopReason ??= blocked ?? "converged";
|
|
3947
|
-
return;
|
|
3948
|
-
}
|
|
3949
|
-
await this.handle(await this.next());
|
|
3950
|
-
}
|
|
3951
|
-
}
|
|
3952
|
-
/** The next completion, or null when an abort woke the loop without one. */
|
|
3953
|
-
async next() {
|
|
3954
|
-
if (this.completions.length === 0) await new Promise((resolve) => {
|
|
3955
|
-
this.wake = resolve;
|
|
3956
|
-
});
|
|
3957
|
-
return this.completions.shift() ?? null;
|
|
3958
|
-
}
|
|
3959
|
-
push(completion) {
|
|
3960
|
-
this.completions.push(completion);
|
|
3961
|
-
this.wakeUp();
|
|
3962
|
-
}
|
|
3963
|
-
wakeUp() {
|
|
3964
|
-
const wake = this.wake;
|
|
3965
|
-
this.wake = null;
|
|
3966
|
-
wake?.();
|
|
3967
|
-
}
|
|
3968
|
-
async handle(completion) {
|
|
3969
|
-
if (completion === null) return;
|
|
3970
|
-
if (completion.kind === "cell") await this.settle(completion);
|
|
3971
|
-
else await this.recordProposal(completion);
|
|
3972
|
-
}
|
|
3973
|
-
/** Abort everything in flight and record what still completes. */
|
|
3974
|
-
async drain(reason) {
|
|
3975
|
-
for (const controller of this.inFlight.values()) controller.abort(reason);
|
|
3976
|
-
this.proposalController?.abort(reason);
|
|
3977
|
-
while (this.inFlight.size > 0 || this.proposal !== null) await this.handle(await this.next());
|
|
3978
|
-
}
|
|
3979
|
-
/** Fill free lane slots with queued cells. A queued cell was admitted when
|
|
3980
|
-
* it was allocated, so starting it needs no further decision. */
|
|
3981
|
-
dispatch() {
|
|
3982
|
-
if (this.failure || this.options.signal?.aborted) return;
|
|
3983
|
-
const claimOnly = this.pastDeadline();
|
|
3984
|
-
const cap = this.state.header.budget.maxConcurrency;
|
|
3985
|
-
for (const lane of this.lanes.values()) while (lane.inFlight < lane.capacity && (cap === null || this.inFlight.size < cap)) {
|
|
3986
|
-
const cellId = (claimOnly ? lane.queues[STAGE_PRIORITY.claim] : lane.queues.find((waiting) => waiting.length > 0))?.shift();
|
|
3987
|
-
if (cellId === void 0) break;
|
|
3988
|
-
this.start(cellId, lane);
|
|
3989
|
-
}
|
|
3990
|
-
}
|
|
3991
|
-
start(cellId, lane) {
|
|
3992
|
-
const cell = this.state.cell(cellId);
|
|
3993
|
-
const attempt = cell.attempts + 1;
|
|
3994
|
-
const controller = new AbortController();
|
|
3995
|
-
const work = {
|
|
3996
|
-
searchId: this.recorder.searchId,
|
|
3997
|
-
cellId,
|
|
3998
|
-
nodeId: cell.nodeId,
|
|
3999
|
-
taskId: cell.taskId,
|
|
4000
|
-
unitId: cell.unitId,
|
|
4001
|
-
split: cell.split,
|
|
4002
|
-
rep: cell.rep,
|
|
4003
|
-
stage: cell.stage,
|
|
4004
|
-
lane: lane.name,
|
|
4005
|
-
attempt,
|
|
4006
|
-
runId: `${cellId}:${attempt}`,
|
|
4007
|
-
artifact: this.artifact(cell.nodeId),
|
|
4008
|
-
signal: controller.signal
|
|
4009
|
-
};
|
|
4010
|
-
const adopt = this.adoptable.delete(cellId);
|
|
4011
|
-
this.inFlight.set(cellId, controller);
|
|
4012
|
-
lane.inFlight += 1;
|
|
4013
|
-
const { executor } = this.options;
|
|
4014
|
-
(async () => (adopt ? await executor.adopt(work) : null) ?? await executor.run(work))().then((result) => this.push({
|
|
4015
|
-
kind: "cell",
|
|
4016
|
-
cellId,
|
|
4017
|
-
attempt,
|
|
4018
|
-
result
|
|
4019
|
-
}), (error) => this.push({
|
|
4020
|
-
kind: "cell",
|
|
4021
|
-
cellId,
|
|
4022
|
-
attempt,
|
|
4023
|
-
error
|
|
4024
|
-
}));
|
|
4025
|
-
}
|
|
4026
|
-
async settle(completion) {
|
|
4027
|
-
const { cellId, attempt } = completion;
|
|
4028
|
-
this.inFlight.delete(cellId);
|
|
4029
|
-
const before = this.state.cell(cellId);
|
|
4030
|
-
const lane = this.lanes.get(before.lane);
|
|
4031
|
-
lane.inFlight -= 1;
|
|
4032
|
-
if (!completion.result) {
|
|
4033
|
-
this.failure ??= { error: completion.error };
|
|
4034
|
-
return;
|
|
4035
|
-
}
|
|
4036
|
-
this.dispatch();
|
|
4037
|
-
if ((this.failure || this.options.signal?.aborted) && completion.result.outcome.status === "errored") return;
|
|
4038
|
-
await this.recorder.settleCell({
|
|
4039
|
-
...completion.result,
|
|
4040
|
-
cellId,
|
|
4041
|
-
attempt
|
|
4042
|
-
});
|
|
4043
|
-
await this.refresh();
|
|
4044
|
-
const cell = this.state.cell(cellId);
|
|
4045
|
-
if (cell.final) lane.settledCosts.push(cell.spentUsd);
|
|
4046
|
-
if (!this.cellDone(cell)) {
|
|
4047
|
-
if (this.failure || this.options.signal?.aborted) return;
|
|
4048
|
-
this.enqueue(cell);
|
|
4049
|
-
return;
|
|
4050
|
-
}
|
|
4051
|
-
if (cell.stage === "claim") return;
|
|
4052
|
-
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
4053
|
-
this.pending.set(cell.nodeId, left);
|
|
4054
|
-
if (left === 0) await this.onNodeIdle(cell.nodeId);
|
|
4055
|
-
}
|
|
4056
|
-
/** Every cell allocated to the node is done: its screen, or a rung. */
|
|
4057
|
-
async onNodeIdle(nodeId) {
|
|
4058
|
-
if (!await this.screenDone(nodeId)) return;
|
|
4059
|
-
await this.advance();
|
|
4060
|
-
}
|
|
4061
|
-
/** Record the rank decisions the allocator now supports and allocate the
|
|
4062
|
-
* rungs they open. Past the deadline, once the claim began, or once the
|
|
4063
|
-
* search fails or is aborted, no rung opens; a search that stopped
|
|
4064
|
-
* expanding still measures the nodes that earned it. */
|
|
4065
|
-
async advance() {
|
|
4066
|
-
if (this.claimPlan || this.failure || this.options.signal?.aborted || this.pastDeadline()) return;
|
|
4067
|
-
for (const decision of this.options.allocation.advance(this.allocationView())) await this.allocateFor(decision.nodeId, decision);
|
|
4068
|
-
}
|
|
4069
|
-
/**
|
|
4070
|
-
* A node's allocated cells all finished. A node that diverges (its train
|
|
4071
|
-
* mean rose over its parent's while its selection interval against the
|
|
4072
|
-
* parent lies wholly on the worse side) is decided `invalid` and never
|
|
4073
|
-
* becomes a parent; any other node is screened. Returns whether it was.
|
|
4074
|
-
*/
|
|
4075
|
-
async screenDone(nodeId) {
|
|
4076
|
-
if (!this.claimPlan && !this.screenedSet.has(nodeId) && this.state.node(nodeId).status !== "advanced") {
|
|
4077
|
-
const divergence = searchDivergence(this.state, nodeId);
|
|
4078
|
-
if (divergence) {
|
|
4079
|
-
await this.recorder.decideNode({
|
|
4080
|
-
nodeId,
|
|
4081
|
-
decision: { status: "invalid" },
|
|
4082
|
-
basis: divergence.basis,
|
|
4083
|
-
rule: "divergence",
|
|
4084
|
-
reason: divergence.reason
|
|
4085
|
-
});
|
|
4086
|
-
await this.refresh();
|
|
4087
|
-
this.admitted.delete(nodeId);
|
|
4088
|
-
return false;
|
|
4089
|
-
}
|
|
4090
|
-
}
|
|
4091
|
-
this.markScreened(nodeId);
|
|
4092
|
-
return true;
|
|
4093
|
-
}
|
|
4094
|
-
markScreened(nodeId) {
|
|
4095
|
-
if (this.screenedSet.has(nodeId)) return;
|
|
4096
|
-
this.screenedSet.add(nodeId);
|
|
4097
|
-
const ordinal = this.state.node(nodeId).ordinal;
|
|
4098
|
-
let index = this.screened.length;
|
|
4099
|
-
while (index > 0 && this.state.node(this.screened[index - 1]).ordinal > ordinal) index--;
|
|
4100
|
-
this.screened = [
|
|
4101
|
-
...this.screened.slice(0, index),
|
|
4102
|
-
nodeId,
|
|
4103
|
-
...this.screened.slice(index)
|
|
4104
|
-
];
|
|
4105
|
-
}
|
|
4106
|
-
/**
|
|
4107
|
-
* Allocate the cells the allocator plans for a node through its rung and the
|
|
4108
|
-
* ledger lacks. With a rank decision, the rung is the decision's, and the
|
|
4109
|
-
* decision is recorded first, only once the cap admits the rung's cells.
|
|
4110
|
-
* Returns false when the cap or `maxCells` refused them.
|
|
4111
|
-
*/
|
|
4112
|
-
async allocateFor(nodeId, decision) {
|
|
4113
|
-
const rung = decision?.decision.status === "advanced" ? decision.decision.rung : this.state.node(nodeId).rung ?? 0;
|
|
4114
|
-
const plans = this.options.allocation.plan(this.state, nodeId, rung).filter((plan) => !this.state.cell(searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep)));
|
|
4115
|
-
if (plans.length === 0) {
|
|
4116
|
-
if (decision) await this.decide(decision);
|
|
4117
|
-
return true;
|
|
4118
|
-
}
|
|
4119
|
-
const placed = plans.map((plan) => {
|
|
4120
|
-
const laneName = this.options.executor.place({
|
|
4121
|
-
...plan,
|
|
4122
|
-
nodeId
|
|
4123
|
-
});
|
|
4124
|
-
const lane = this.lanes.get(laneName);
|
|
4125
|
-
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
4126
|
-
return {
|
|
4127
|
-
plan,
|
|
4128
|
-
lane,
|
|
4129
|
-
reservation: this.cellReservation(lane)
|
|
4130
|
-
};
|
|
4131
|
-
});
|
|
4132
|
-
const hold = placed.reduce((sum, { reservation }) => sum + reservation.usd, 0);
|
|
4133
|
-
const { headroomUsd } = this.state.budget;
|
|
4134
|
-
if (headroomUsd !== null && hold > headroomUsd + USD_TOLERANCE) return false;
|
|
4135
|
-
const { maxCells } = this.state.header.budget;
|
|
4136
|
-
if (maxCells !== null && this.state.audit.cells.allocated + placed.length > maxCells) return false;
|
|
4137
|
-
if (decision) await this.decide(decision);
|
|
4138
|
-
for (const { plan, lane, reservation } of placed) await this.recorder.allocateCell({
|
|
4139
|
-
nodeId,
|
|
4140
|
-
taskId: plan.taskId,
|
|
4141
|
-
split: plan.split,
|
|
4142
|
-
rep: plan.rep,
|
|
4143
|
-
stage: plan.stage,
|
|
4144
|
-
lane: lane.name,
|
|
4145
|
-
reservation
|
|
4146
|
-
});
|
|
4147
|
-
await this.refresh();
|
|
4148
|
-
for (const { plan } of placed) {
|
|
4149
|
-
const cellId = searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep);
|
|
4150
|
-
this.enqueue(this.state.cell(cellId));
|
|
4151
|
-
}
|
|
4152
|
-
this.pending.set(nodeId, (this.pending.get(nodeId) ?? 0) + placed.length);
|
|
4153
|
-
this.dispatch();
|
|
4154
|
-
return true;
|
|
4155
|
-
}
|
|
4156
|
-
async decide(decision) {
|
|
4157
|
-
await this.recorder.decideNode(decision);
|
|
4158
|
-
await this.refresh();
|
|
4159
|
-
}
|
|
4160
|
-
enqueue(cell) {
|
|
4161
|
-
this.lanes.get(cell.lane).queues[STAGE_PRIORITY[cell.stage]].push(cell.cellId);
|
|
4162
|
-
}
|
|
4163
|
-
queuedCount() {
|
|
4164
|
-
let count = 0;
|
|
4165
|
-
for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
|
|
4166
|
-
return count;
|
|
4167
|
-
}
|
|
4168
|
-
/** Fix the claim's design, store it, and start its test cells. */
|
|
4169
|
-
async beginClaim() {
|
|
4170
|
-
await this.refresh();
|
|
4171
|
-
const stopReason = this.stopReason ?? "converged";
|
|
4172
|
-
const header = this.state.header;
|
|
4173
|
-
const reps = this.options.allocation.reps;
|
|
4174
|
-
const plan = planSearchClaim(this.state, {
|
|
4175
|
-
stopReason,
|
|
4176
|
-
reps,
|
|
4177
|
-
affordable: (finalists) => {
|
|
4178
|
-
const placed = this.claimCells(finalists, this.state.rootNodeId);
|
|
4179
|
-
const hold = placed.reduce((sum, cell) => sum + cell.reservation.usd, 0);
|
|
4180
|
-
const { headroomUsd, claimReserveUsd } = this.state.budget;
|
|
4181
|
-
const { maxCells } = header.budget;
|
|
4182
|
-
return (headroomUsd === null || hold <= headroomUsd + claimReserveUsd + USD_TOLERANCE) && (maxCells === null || this.state.audit.cells.allocated + placed.length <= maxCells);
|
|
4183
|
-
}
|
|
4184
|
-
});
|
|
4185
|
-
await this.recorder.startOperation({
|
|
4186
|
-
operationId: CLAIM_OPERATION,
|
|
4187
|
-
operationKind: "selection",
|
|
4188
|
-
reservation: null,
|
|
4189
|
-
artifacts: [this.recorder.blob("claim-plan", plan)]
|
|
4190
|
-
});
|
|
4191
|
-
await this.recordClaimOperation();
|
|
4192
|
-
this.claimPlan = plan;
|
|
4193
|
-
this.stopReason = stopReason;
|
|
4194
|
-
await this.applyClaimPlan(plan);
|
|
4195
|
-
}
|
|
4196
|
-
async recordClaimOperation() {
|
|
4197
|
-
await this.recorder.recordOperation({
|
|
4198
|
-
operationId: CLAIM_OPERATION,
|
|
4199
|
-
operationKind: "selection",
|
|
4200
|
-
execution: {
|
|
4201
|
-
kind: "deterministic",
|
|
4202
|
-
source: SEARCH_CLAIM_RULE
|
|
4203
|
-
},
|
|
4204
|
-
outcome: { status: "completed" },
|
|
4205
|
-
accounting: {
|
|
4206
|
-
tokens: {
|
|
4207
|
-
status: "known",
|
|
4208
|
-
inputTokens: 0,
|
|
4209
|
-
outputTokens: 0,
|
|
4210
|
-
cachedTokens: 0
|
|
4211
|
-
},
|
|
4212
|
-
cost: {
|
|
4213
|
-
status: "known",
|
|
4214
|
-
usd: 0,
|
|
4215
|
-
source: "free"
|
|
4216
|
-
}
|
|
4217
|
-
}
|
|
4218
|
-
});
|
|
4219
|
-
await this.refresh();
|
|
4220
|
-
}
|
|
4221
|
-
/**
|
|
4222
|
-
* Decide the plan's finalists and allocate the test cells of the root and
|
|
4223
|
-
* the finalists, interleaved by task so the arms of one task run side by
|
|
4224
|
-
* side. Allocation finishes before any claim cell starts. Idempotent.
|
|
4225
|
-
*/
|
|
4226
|
-
async applyClaimPlan(plan) {
|
|
4227
|
-
const root = this.state.rootNodeId;
|
|
4228
|
-
const bases = plan.finalists.map((finalist) => estimateNode(this.state, finalist.nodeId, {
|
|
4229
|
-
against: root,
|
|
4230
|
-
split: "selection"
|
|
4231
|
-
}));
|
|
4232
|
-
for (const [rank, finalist] of plan.finalists.entries()) await this.recorder.decideNode({
|
|
4233
|
-
nodeId: finalist.nodeId,
|
|
4234
|
-
decision: { status: "finalist" },
|
|
4235
|
-
basis: bases[rank],
|
|
4236
|
-
rule: SEARCH_CLAIM_RULE_NAME,
|
|
4237
|
-
reason: `rank ${rank + 1} of ${plan.finalists.length} by selection mean (${finalist.selectionMean}); ${plan.reason}`
|
|
4238
|
-
});
|
|
4239
|
-
await this.refresh();
|
|
4240
|
-
if (plan.test !== "run") return;
|
|
4241
|
-
const missing = this.claimCells(plan.finalists.length, root, plan).map((cell) => ({
|
|
4242
|
-
...cell,
|
|
4243
|
-
cellId: searchCellId(this.recorder.searchId, cell.nodeId, cell.plan.taskId, "test", cell.plan.rep)
|
|
4244
|
-
})).filter((cell) => !this.state.cell(cell.cellId));
|
|
4245
|
-
for (const cell of missing) await this.recorder.allocateCell({
|
|
4246
|
-
nodeId: cell.nodeId,
|
|
4247
|
-
taskId: cell.plan.taskId,
|
|
4248
|
-
split: "test",
|
|
4249
|
-
rep: cell.plan.rep,
|
|
4250
|
-
stage: "claim",
|
|
4251
|
-
lane: cell.lane.name,
|
|
4252
|
-
reservation: cell.reservation
|
|
4253
|
-
});
|
|
4254
|
-
await this.refresh();
|
|
4255
|
-
for (const cell of missing) this.enqueue(this.state.cell(cell.cellId));
|
|
4256
|
-
}
|
|
4257
|
-
/** The test cells of the root and the first `finalists` of the plan (or of
|
|
4258
|
-
* the finalist count being priced), task by task, with their lanes and holds. */
|
|
4259
|
-
claimCells(finalists, root, plan) {
|
|
4260
|
-
const header = this.state.header;
|
|
4261
|
-
const arms = [root, ...plan ? plan.finalists.map((entry) => entry.nodeId) : []];
|
|
4262
|
-
while (arms.length < finalists + 1) arms.push(root);
|
|
4263
|
-
const cells = [];
|
|
4264
|
-
for (const task of header.splits.test.tasks) for (let rep = 0; rep < this.options.allocation.reps; rep++) for (const nodeId of arms) {
|
|
4265
|
-
const cellPlan = {
|
|
4266
|
-
taskId: task.taskId,
|
|
4267
|
-
split: "test",
|
|
4268
|
-
rep,
|
|
4269
|
-
stage: "claim"
|
|
4270
|
-
};
|
|
4271
|
-
const laneName = this.options.executor.place({
|
|
4272
|
-
...cellPlan,
|
|
4273
|
-
nodeId
|
|
4274
|
-
});
|
|
4275
|
-
const lane = this.lanes.get(laneName);
|
|
4276
|
-
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
4277
|
-
cells.push({
|
|
4278
|
-
nodeId,
|
|
4279
|
-
plan: cellPlan,
|
|
4280
|
-
lane,
|
|
4281
|
-
reservation: this.cellReservation(lane)
|
|
4282
|
-
});
|
|
4283
|
-
}
|
|
4284
|
-
return cells;
|
|
4285
|
-
}
|
|
4286
|
-
async cancelQueued(reason) {
|
|
4287
|
-
for (const lane of this.lanes.values()) for (const [priority, queue] of lane.queues.entries()) {
|
|
4288
|
-
if (priority === STAGE_PRIORITY.claim) continue;
|
|
4289
|
-
for (const cellId of queue.splice(0)) {
|
|
4290
|
-
const cell = this.state.cell(cellId);
|
|
4291
|
-
if (cell.attempts === 0) {
|
|
4292
|
-
await this.recorder.cancelCell({
|
|
4293
|
-
cellId,
|
|
4294
|
-
reason
|
|
4295
|
-
});
|
|
4296
|
-
await this.refresh();
|
|
4297
|
-
}
|
|
4298
|
-
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
4299
|
-
this.pending.set(cell.nodeId, left);
|
|
4300
|
-
if (left === 0) this.markScreened(cell.nodeId);
|
|
4301
|
-
}
|
|
4302
|
-
}
|
|
4303
|
-
}
|
|
4304
|
-
/** No further attempt will run: a final outcome, a cancellation, or a
|
|
4305
|
-
* retryable error that used up its attempts. */
|
|
4306
|
-
cellDone(cell) {
|
|
4307
|
-
return cell.final || cell.cancelled !== null || cell.outcome === "errored" && cell.attempts >= this.maxAttempts;
|
|
4308
|
-
}
|
|
4309
|
-
/** Start a proposal when the rules admit one; otherwise say why not. */
|
|
4310
|
-
async tryExpand() {
|
|
4311
|
-
const { maxNodes, maxCells } = this.state.header.budget;
|
|
4312
|
-
if (maxNodes !== null && this.state.audit.nodes >= maxNodes) return "max-nodes";
|
|
4313
|
-
const { maxExpansions } = this.options;
|
|
4314
|
-
if (maxExpansions !== void 0 && this.completed.length >= maxExpansions) return "max-nodes";
|
|
4315
|
-
if (this.queuedCount() > 0) return "converged";
|
|
4316
|
-
if (this.inFlight.size >= 2 * this.capacity()) return "converged";
|
|
4317
|
-
const view = this.policyView();
|
|
4318
|
-
const { policy } = this.options;
|
|
4319
|
-
const leader = policy.leader(view);
|
|
4320
|
-
if (policy.patience !== void 0 && this.completed.filter((index) => index > (this.expansionOf.get(leader) ?? -1)).length >= policy.patience) return "patience";
|
|
4321
|
-
const screen = this.options.allocation.screenSize(this.state);
|
|
4322
|
-
const operation = this.operationReservation();
|
|
4323
|
-
const children = this.options.proposer.childrenPerProposal ?? 1;
|
|
4324
|
-
const need = operation.usd + children * screen * this.maxCellReservation();
|
|
4325
|
-
const { headroomUsd } = this.state.budget;
|
|
4326
|
-
if (headroomUsd !== null && (need > headroomUsd + USD_TOLERANCE || this.overCap())) return "budget";
|
|
4327
|
-
if (maxCells !== null && this.state.audit.cells.allocated + children * screen > maxCells) return "budget";
|
|
4328
|
-
const expansion = policy.expand(view);
|
|
4329
|
-
if (expansion === null) return "converged";
|
|
4330
|
-
const index = this.expansions++;
|
|
4331
|
-
const operationId = `expand-${index}`;
|
|
4332
|
-
await this.recorder.startOperation({
|
|
4333
|
-
operationId,
|
|
4334
|
-
operationKind: "candidate-generation",
|
|
4335
|
-
reservation: operation
|
|
4336
|
-
});
|
|
4337
|
-
await this.refresh();
|
|
4338
|
-
const controller = new AbortController();
|
|
4339
|
-
const request = {
|
|
4340
|
-
expansion: index,
|
|
4341
|
-
operationId,
|
|
4342
|
-
operator: expansion.operator,
|
|
4343
|
-
parents: expansion.parents.map((nodeId) => ({
|
|
4344
|
-
nodeId,
|
|
4345
|
-
artifact: this.artifact(nodeId)
|
|
4346
|
-
})),
|
|
4347
|
-
leader,
|
|
4348
|
-
signal: controller.signal
|
|
4349
|
-
};
|
|
4350
|
-
this.proposalController = controller;
|
|
4351
|
-
this.proposal = this.options.proposer.propose(request).then((result) => this.push({
|
|
4352
|
-
kind: "proposal",
|
|
4353
|
-
expansion: index,
|
|
4354
|
-
operationId,
|
|
4355
|
-
request,
|
|
4356
|
-
selection: expansion.selection,
|
|
4357
|
-
result
|
|
4358
|
-
}), (error) => this.push({
|
|
4359
|
-
kind: "proposal",
|
|
4360
|
-
expansion: index,
|
|
4361
|
-
operationId,
|
|
4362
|
-
request,
|
|
4363
|
-
selection: expansion.selection,
|
|
4364
|
-
error
|
|
4365
|
-
}));
|
|
4366
|
-
return null;
|
|
4367
|
-
}
|
|
4368
|
-
async recordProposal(completion) {
|
|
4369
|
-
this.proposal = null;
|
|
4370
|
-
this.proposalController = null;
|
|
4371
|
-
const { operationId, expansion } = completion;
|
|
4372
|
-
const { proposer, codec, admit } = this.options;
|
|
4373
|
-
const result = completion.result;
|
|
4374
|
-
if (!result) {
|
|
4375
|
-
await this.recorder.recordOperation({
|
|
4376
|
-
operationId,
|
|
4377
|
-
operationKind: "candidate-generation",
|
|
4378
|
-
execution: proposer.execution,
|
|
4379
|
-
outcome: {
|
|
4380
|
-
status: "failed",
|
|
4381
|
-
failure: {
|
|
4382
|
-
code: "proposer-error",
|
|
4383
|
-
message: errorMessage(completion.error)
|
|
4384
|
-
}
|
|
4385
|
-
},
|
|
4386
|
-
accounting: interruptedAccounting()
|
|
4387
|
-
});
|
|
4388
|
-
await this.refresh();
|
|
4389
|
-
this.failure ??= { error: completion.error };
|
|
4390
|
-
return;
|
|
4391
|
-
}
|
|
4392
|
-
const { maxNodes } = this.state.header.budget;
|
|
4393
|
-
const room = maxNodes === null ? Number.POSITIVE_INFINITY : maxNodes - this.state.audit.nodes;
|
|
4394
|
-
const parents = completion.request.parents;
|
|
4395
|
-
const blob = {
|
|
4396
|
-
kind: "search-proposal",
|
|
4397
|
-
operator: completion.request.operator,
|
|
4398
|
-
parents: parents.map((parent) => parent.nodeId),
|
|
4399
|
-
selection: completion.selection,
|
|
4400
|
-
children: result.children.slice(0, Math.max(0, room)).map((child) => ({
|
|
4401
|
-
node: codec.node(this.recorder, child.artifact),
|
|
4402
|
-
diffs: parents.map((parent) => codec.diff(this.recorder, parent.artifact, child.artifact)),
|
|
4403
|
-
label: redactText(child.label, { profile: "share" }),
|
|
4404
|
-
rationale: redactText(child.rationale, { profile: "share" }),
|
|
4405
|
-
attribution: child.attribution ?? null,
|
|
4406
|
-
invalid: admit?.(child.artifact) ?? null
|
|
4407
|
-
}))
|
|
4408
|
-
};
|
|
4409
|
-
await this.recorder.recordOperation({
|
|
4410
|
-
operationId,
|
|
4411
|
-
operationKind: "candidate-generation",
|
|
4412
|
-
execution: result.execution ?? proposer.execution,
|
|
4413
|
-
outcome: { status: "completed" },
|
|
4414
|
-
accounting: result.accounting,
|
|
4415
|
-
artifacts: [this.recorder.blob("proposal", blob)]
|
|
4416
|
-
});
|
|
4417
|
-
await this.refresh();
|
|
4418
|
-
this.operationCosts.push(this.state.operation(operationId).spentUsd);
|
|
4419
|
-
this.completed.push(expansion);
|
|
4420
|
-
for (const [index, child] of blob.children.entries()) this.proposed.set(child.node.artifactDigest, result.children[index].artifact);
|
|
4421
|
-
if (result.stop !== void 0 || blob.children.length === 0) this.stopReason ??= "converged";
|
|
4422
|
-
await this.finishProposal(expansion);
|
|
4423
|
-
}
|
|
4424
|
-
/** Register every child a recorded proposal produced, each once. */
|
|
4425
|
-
async finishProposal(expansion) {
|
|
4426
|
-
const operationId = `expand-${expansion}`;
|
|
4427
|
-
const operation = this.state.operation(operationId);
|
|
4428
|
-
const ref = operation?.artifacts.find((artifact) => artifact.role === "proposal");
|
|
4429
|
-
if (!operation?.recorded || operation.outcome !== "completed" || !ref) return;
|
|
4430
|
-
const blob = this.recorder.readBlob(ref);
|
|
4431
|
-
const { proposer } = this.options;
|
|
4432
|
-
for (const [index, child] of blob.children.entries()) {
|
|
4433
|
-
const { nodeId } = await this.recorder.registerNode(child.node);
|
|
4434
|
-
const held = this.proposed.get(child.node.artifactDigest);
|
|
4435
|
-
if (held !== void 0) {
|
|
4436
|
-
this.proposed.delete(child.node.artifactDigest);
|
|
4437
|
-
if (!this.artifacts.has(nodeId)) this.artifacts.set(nodeId, held);
|
|
4438
|
-
}
|
|
4439
|
-
const edgeId = await this.recorder.recordEdge({
|
|
4440
|
-
childNodeId: nodeId,
|
|
4441
|
-
parents: blob.parents,
|
|
4442
|
-
operator: blob.operator,
|
|
4443
|
-
attribution: "explicit",
|
|
4444
|
-
proposer: {
|
|
4445
|
-
kind: proposer.kind,
|
|
4446
|
-
name: proposer.name,
|
|
4447
|
-
operationId,
|
|
4448
|
-
source: proposer.source
|
|
4449
|
-
},
|
|
4450
|
-
proposalKey: `${operationId}:${index}`,
|
|
4451
|
-
selection: blob.selection,
|
|
4452
|
-
rationale: child.rationale,
|
|
4453
|
-
diffs: child.diffs,
|
|
4454
|
-
label: child.label
|
|
4455
|
-
});
|
|
4456
|
-
await this.refresh();
|
|
4457
|
-
if (this.state.node(nodeId).edgeIds[0] !== edgeId) continue;
|
|
4458
|
-
this.expansionOf.set(nodeId, expansion);
|
|
4459
|
-
if (child.invalid !== null) {
|
|
4460
|
-
await this.recorder.decideNode({
|
|
4461
|
-
nodeId,
|
|
4462
|
-
decision: { status: "invalid" },
|
|
4463
|
-
rule: "admission",
|
|
4464
|
-
reason: child.invalid
|
|
4465
|
-
});
|
|
4466
|
-
await this.refresh();
|
|
4467
|
-
continue;
|
|
4468
|
-
}
|
|
4469
|
-
if (this.admitted.has(nodeId)) continue;
|
|
4470
|
-
this.admitted.add(nodeId);
|
|
4471
|
-
await this.allocateFor(nodeId);
|
|
4472
|
-
if ((this.pending.get(nodeId) ?? 0) === 0) this.markScreened(nodeId);
|
|
4473
|
-
}
|
|
4474
|
-
}
|
|
4475
|
-
async close() {
|
|
4476
|
-
await this.refresh();
|
|
4477
|
-
const keep = this.options.policy.leader(this.policyView());
|
|
4478
|
-
for (const decision of this.options.allocation.prune(this.allocationView(), keep)) await this.decide(decision);
|
|
4479
|
-
if (this.claimPlan) {
|
|
4480
|
-
await this.closeWithClaim(this.claimPlan);
|
|
4481
|
-
return;
|
|
4482
|
-
}
|
|
4483
|
-
const view = this.policyView();
|
|
4484
|
-
const leader = this.options.policy.leader(view);
|
|
4485
|
-
const { split } = view;
|
|
4486
|
-
const root = this.state.rootNodeId;
|
|
4487
|
-
const rule = this.options.policy.name;
|
|
4488
|
-
const leaderUnits = view.unitScores(leader).map((unit) => unit.unitId);
|
|
4489
|
-
const decisions = this.state.nodes().filter((node) => !isTerminal(node)).map((node) => {
|
|
4490
|
-
const scored = this.state.scoredCells(node.nodeId, split).length > 0;
|
|
4491
|
-
const lead = node.nodeId === leader && scored;
|
|
4492
|
-
const against = node.nodeId === leader ? root : leader;
|
|
4493
|
-
const basis = node.nodeId === against || !scored ? null : estimateNode(this.state, node.nodeId, {
|
|
4494
|
-
against,
|
|
4495
|
-
split
|
|
4496
|
-
});
|
|
4497
|
-
const units = new Set(view.unitScores(node.nodeId).map((unit) => unit.unitId));
|
|
4498
|
-
return {
|
|
4499
|
-
nodeId: node.nodeId,
|
|
4500
|
-
decision: { status: lead ? "selected" : "rejected" },
|
|
4501
|
-
basis,
|
|
4502
|
-
rule,
|
|
4503
|
-
reason: lead ? `the leader when the search stopped (${this.stopReason}): a budget decision on the ${split} split, which claims nothing` : !this.screenedSet.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit unscored, so it could not lead" : leaderUnits.some((unitId) => !units.has(unitId)) ? "the search stopped before it was measured on every unit the leader was" : "it did not beat the leader on the units they share"
|
|
4504
|
-
};
|
|
4505
|
-
});
|
|
4506
|
-
for (const decision of decisions) await this.recorder.decideNode(decision);
|
|
4507
|
-
await this.recorder.close({
|
|
4508
|
-
reason: this.stopReason ?? "converged",
|
|
4509
|
-
claim: null
|
|
4510
|
-
});
|
|
4511
|
-
await this.refresh();
|
|
4512
|
-
}
|
|
4513
|
-
/** Close with the claim: the claim's decisions, then every other open node
|
|
4514
|
-
* rejected with its selection estimate against the root. */
|
|
4515
|
-
async closeWithClaim(plan) {
|
|
4516
|
-
const { claim, decisions } = decideSearchClaim(this.state, plan);
|
|
4517
|
-
const decided = new Set(decisions.map((decision) => decision.nodeId));
|
|
4518
|
-
const root = this.state.rootNodeId;
|
|
4519
|
-
const rest = this.state.nodes().filter((node) => !decided.has(node.nodeId) && !isTerminal(node)).map((node) => {
|
|
4520
|
-
const scored = this.state.scoredCells(node.nodeId, "selection").length > 0;
|
|
4521
|
-
return {
|
|
4522
|
-
nodeId: node.nodeId,
|
|
4523
|
-
decision: { status: "rejected" },
|
|
4524
|
-
basis: scored ? estimateNode(this.state, node.nodeId, {
|
|
4525
|
-
against: root,
|
|
4526
|
-
split: "selection"
|
|
4527
|
-
}) : null,
|
|
4528
|
-
rule: SEARCH_CLAIM_RULE_NAME,
|
|
4529
|
-
reason: scored ? "not a finalist: the claim tests at most 3 nodes that scored every selection unit and beat the root, best selection mean first" : "the search stopped before this node was measured on the selection split"
|
|
4530
|
-
};
|
|
4531
|
-
});
|
|
4532
|
-
for (const decision of [...decisions, ...rest]) await this.recorder.decideNode(decision);
|
|
4533
|
-
await this.recorder.close({
|
|
4534
|
-
reason: plan.stopReason,
|
|
4535
|
-
claim
|
|
4536
|
-
});
|
|
4537
|
-
await this.refresh();
|
|
4538
|
-
}
|
|
4539
|
-
/** The closed search, with its claim made again from the ledger alone. A
|
|
4540
|
-
* claim the ledger does not support throws: a tampered ledger, or a
|
|
4541
|
-
* producer and verifier that disagree. */
|
|
4542
|
-
closedResult() {
|
|
4543
|
-
const state = this.state;
|
|
4544
|
-
const claimVerification = verifySearchClaim(state);
|
|
4545
|
-
if (claimVerification?.status === "mismatch") throw new Error(`runSearch: search ${state.searchId} closed with a claim its ledger does not support: ${claimVerification.differences.join("; ")}`);
|
|
4546
|
-
return {
|
|
4547
|
-
state,
|
|
4548
|
-
leader: state.audit.selectedNodeId ?? state.rootNodeId,
|
|
4549
|
-
reason: state.closed.reason,
|
|
4550
|
-
claim: state.closed.claim,
|
|
4551
|
-
claimVerification
|
|
4552
|
-
};
|
|
4553
|
-
}
|
|
4554
|
-
policyView() {
|
|
4555
|
-
return searchPolicyView(this.state, {
|
|
4556
|
-
screened: this.screened,
|
|
4557
|
-
screening: this.admitted.size - this.screenedSet.size,
|
|
4558
|
-
expansions: this.completed.length,
|
|
4559
|
-
maxAttempts: this.maxAttempts
|
|
4560
|
-
});
|
|
4561
|
-
}
|
|
4562
|
-
allocationView() {
|
|
4563
|
-
return {
|
|
4564
|
-
state: this.state,
|
|
4565
|
-
idle: (nodeId) => this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0
|
|
4566
|
-
};
|
|
4567
|
-
}
|
|
4568
|
-
artifact(nodeId) {
|
|
4569
|
-
const held = this.artifacts.get(nodeId);
|
|
4570
|
-
if (held !== void 0) return held;
|
|
4571
|
-
const artifact = this.options.codec.load(this.recorder, this.state.node(nodeId));
|
|
4572
|
-
this.artifacts.set(nodeId, artifact);
|
|
4573
|
-
return artifact;
|
|
4574
|
-
}
|
|
4575
|
-
capacity() {
|
|
4576
|
-
let total = 0;
|
|
4577
|
-
for (const lane of this.lanes.values()) total += lane.capacity;
|
|
4578
|
-
const cap = this.state.header.budget.maxConcurrency;
|
|
4579
|
-
return cap === null ? total : Math.min(cap, total);
|
|
4580
|
-
}
|
|
4581
|
-
/** A hard lane holds its enforced maximum. An estimate lane holds 1.5 times
|
|
4582
|
-
* the 99th percentile of its settled cells once 20 settled, else its prior. */
|
|
4583
|
-
cellReservation(lane) {
|
|
4584
|
-
if (lane.costCap === "hard") return {
|
|
4585
|
-
kind: "hard",
|
|
4586
|
-
usd: lane.cellUsd
|
|
4587
|
-
};
|
|
4588
|
-
return {
|
|
4589
|
-
kind: "estimate",
|
|
4590
|
-
usd: estimate(lane.settledCosts, lane.cellUsd)
|
|
4591
|
-
};
|
|
4592
|
-
}
|
|
4593
|
-
maxCellReservation() {
|
|
4594
|
-
let most = 0;
|
|
4595
|
-
for (const lane of this.lanes.values()) most = Math.max(most, this.cellReservation(lane).usd);
|
|
4596
|
-
return most;
|
|
4597
|
-
}
|
|
4598
|
-
operationReservation() {
|
|
4599
|
-
return {
|
|
4600
|
-
kind: "estimate",
|
|
4601
|
-
usd: estimate(this.operationCosts, this.options.proposer.reservationUsd ?? 0)
|
|
4602
|
-
};
|
|
4603
|
-
}
|
|
4604
|
-
/** Overspend took committed spend, open holds and the unspent claim reserve
|
|
4605
|
-
* past the cap. Nothing new is proposed then, even at a zero-dollar prior. */
|
|
4606
|
-
overCap() {
|
|
4607
|
-
const { maxUsd } = this.state.header.budget;
|
|
4608
|
-
if (maxUsd === null) return false;
|
|
4609
|
-
const { committedUsd, openReservationUsd } = this.state.audit.spend;
|
|
4610
|
-
return committedUsd + openReservationUsd + this.state.budget.claimReserveUsd > maxUsd + USD_TOLERANCE;
|
|
4611
|
-
}
|
|
4612
|
-
pastDeadline() {
|
|
4613
|
-
const deadline = this.state.header.budget.deadline;
|
|
4614
|
-
return deadline !== null && this.now() >= Date.parse(deadline);
|
|
4615
|
-
}
|
|
4616
|
-
async refresh() {
|
|
4617
|
-
this.state = await this.recorder.state();
|
|
4618
|
-
}
|
|
4619
|
-
};
|
|
4620
|
-
/**
|
|
4621
|
-
* A policy's read of a search: `screened` lists the admitted nodes whose screen
|
|
4622
|
-
* finished, in registration order; `screening` counts the admitted nodes still
|
|
4623
|
-
* being screened. Nodes are ranked on the selection split, or on train when
|
|
4624
|
-
* the search declares no selection split. The view reads no test cell.
|
|
4625
|
-
* `maxAttempts` (default 3, the kernel's) says when an errored cell has no
|
|
4626
|
-
* attempt left, which makes its unit a dodge.
|
|
4627
|
-
*/
|
|
4628
|
-
function searchPolicyView(state, input) {
|
|
4629
|
-
const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
4630
|
-
const header = state.header;
|
|
4631
|
-
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
4632
|
-
const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
2877
|
+
function campaignCellOutcome(cell) {
|
|
2878
|
+
const quality = projectCampaignCellQuality(cell);
|
|
2879
|
+
if (quality.score === void 0) return {
|
|
2880
|
+
status: "errored",
|
|
2881
|
+
metrics: {},
|
|
2882
|
+
error: {
|
|
2883
|
+
code: cell.errorStage ?? "unscored",
|
|
2884
|
+
message: cell.error ?? "the cell produced no complete judge score",
|
|
2885
|
+
retryable: false
|
|
2886
|
+
}
|
|
2887
|
+
};
|
|
2888
|
+
const metrics = { composite: quality.score };
|
|
2889
|
+
for (const [judge, score] of Object.entries(quality.successfulJudgeScores)) metrics[`judge.${judge}`] = score.composite;
|
|
4633
2890
|
return {
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
split,
|
|
4638
|
-
rootNodeId: state.rootNodeId,
|
|
4639
|
-
expansions: input.expansions,
|
|
4640
|
-
screened: [...input.screened],
|
|
4641
|
-
screening: input.screening,
|
|
4642
|
-
complete: (nodeId) => {
|
|
4643
|
-
let scored = false;
|
|
4644
|
-
for (const cell of state.cells({ nodeId })) {
|
|
4645
|
-
if (cell.split !== split) continue;
|
|
4646
|
-
if (cell.score !== null) scored = true;
|
|
4647
|
-
else if (cell.final || cell.outcome === "errored" && cell.attempts >= maxAttempts) return false;
|
|
4648
|
-
}
|
|
4649
|
-
return scored;
|
|
4650
|
-
},
|
|
4651
|
-
unitScores: (nodeId) => state.unitScores(nodeId, split),
|
|
4652
|
-
estimate: (nodeId, against) => estimateNode(state, nodeId, {
|
|
4653
|
-
against,
|
|
4654
|
-
split
|
|
4655
|
-
})
|
|
2891
|
+
status: "passed",
|
|
2892
|
+
score: quality.score,
|
|
2893
|
+
metrics
|
|
4656
2894
|
};
|
|
4657
2895
|
}
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
* train units they share while its selection interval against the parent (6
|
|
4661
|
-
* or more units, not indeterminate) lies wholly on the worse side. Train is
|
|
4662
|
-
* what the proposer reads; a gain there that selection contradicts is the
|
|
4663
|
-
* signature of fitting the feedback instead of the task.
|
|
4664
|
-
*/
|
|
4665
|
-
function searchDivergence(state, nodeId) {
|
|
4666
|
-
const header = state.header;
|
|
4667
|
-
const parent = state.node(nodeId)?.primaryParentId ?? null;
|
|
4668
|
-
if (!header || parent === null || header.splits.selection.tasks.length === 0) return null;
|
|
4669
|
-
const selection = estimateNode(state, nodeId, {
|
|
4670
|
-
against: parent,
|
|
4671
|
-
split: "selection"
|
|
4672
|
-
});
|
|
4673
|
-
if (selection.interval === null) return null;
|
|
4674
|
-
const maximize = header.objective.direction === "maximize";
|
|
4675
|
-
const [low, high] = selection.interval;
|
|
4676
|
-
if (maximize ? high >= 0 : low <= 0) return null;
|
|
4677
|
-
const parentTrain = new Map(state.unitScores(parent, "train").map((unit) => [unit.unitId, unit.mean]));
|
|
4678
|
-
let gain = 0;
|
|
4679
|
-
let shared = 0;
|
|
4680
|
-
for (const unit of state.unitScores(nodeId, "train")) {
|
|
4681
|
-
const before = parentTrain.get(unit.unitId);
|
|
4682
|
-
if (before === void 0) continue;
|
|
4683
|
-
gain += maximize ? unit.mean - before : before - unit.mean;
|
|
4684
|
-
shared += 1;
|
|
4685
|
-
}
|
|
4686
|
-
if (shared === 0 || gain <= 0) return null;
|
|
2896
|
+
function campaignCellAccounting(cell) {
|
|
2897
|
+
const usage = cell.tokenUsage;
|
|
4687
2898
|
return {
|
|
4688
|
-
|
|
4689
|
-
|
|
2899
|
+
tokens: usage.tokensKnown === false ? {
|
|
2900
|
+
status: "unknown",
|
|
2901
|
+
reason: "a paid call in this cell reported no token usage"
|
|
2902
|
+
} : {
|
|
2903
|
+
status: "known",
|
|
2904
|
+
inputTokens: usage.input,
|
|
2905
|
+
outputTokens: usage.output,
|
|
2906
|
+
cachedTokens: 0
|
|
2907
|
+
},
|
|
2908
|
+
cost: cell.costProvenance.kind === "uncaptured" ? {
|
|
2909
|
+
status: "unknown",
|
|
2910
|
+
knownLowerBoundUsd: cell.costUsd,
|
|
2911
|
+
reason: "the cell recorded spend without a provider receipt"
|
|
2912
|
+
} : {
|
|
2913
|
+
status: "known",
|
|
2914
|
+
usd: cell.costUsd,
|
|
2915
|
+
source: cell.costProvenance.kind === "observed" ? "provider" : "pricing-table"
|
|
2916
|
+
}
|
|
4690
2917
|
};
|
|
4691
2918
|
}
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
2919
|
+
/** The accounting of a proposal's cost receipts: token and dollar sums, each
|
|
2920
|
+
* unknown (with the known dollars as a lower bound) when a receipt lacks it. */
|
|
2921
|
+
function searchReceiptAccounting(receipts) {
|
|
2922
|
+
let inputTokens = 0;
|
|
2923
|
+
let outputTokens = 0;
|
|
2924
|
+
let cachedTokens = 0;
|
|
2925
|
+
let usd = 0;
|
|
2926
|
+
let tokensKnown = true;
|
|
2927
|
+
let costKnown = true;
|
|
2928
|
+
for (const receipt of receipts) {
|
|
2929
|
+
if (receipt.usageUnknown === true) tokensKnown = false;
|
|
2930
|
+
inputTokens += receipt.inputTokens;
|
|
2931
|
+
outputTokens += receipt.outputTokens;
|
|
2932
|
+
cachedTokens += receipt.cachedTokens ?? 0;
|
|
2933
|
+
if (receipt.costUnknown) costKnown = false;
|
|
2934
|
+
else usd += receipt.costUsd;
|
|
2935
|
+
}
|
|
4706
2936
|
return {
|
|
4707
|
-
tokens: {
|
|
2937
|
+
tokens: tokensKnown ? {
|
|
2938
|
+
status: "known",
|
|
2939
|
+
inputTokens,
|
|
2940
|
+
outputTokens,
|
|
2941
|
+
cachedTokens
|
|
2942
|
+
} : {
|
|
4708
2943
|
status: "unknown",
|
|
4709
|
-
reason: "
|
|
2944
|
+
reason: "a candidate-generation call reported no token usage"
|
|
4710
2945
|
},
|
|
4711
|
-
cost: {
|
|
2946
|
+
cost: costKnown ? {
|
|
2947
|
+
status: "known",
|
|
2948
|
+
usd,
|
|
2949
|
+
source: usd === 0 ? "free" : "provider"
|
|
2950
|
+
} : {
|
|
4712
2951
|
status: "unknown",
|
|
4713
|
-
knownLowerBoundUsd:
|
|
4714
|
-
reason: "
|
|
2952
|
+
knownLowerBoundUsd: usd,
|
|
2953
|
+
reason: "a candidate-generation call recorded no provider cost"
|
|
4715
2954
|
}
|
|
4716
2955
|
};
|
|
4717
2956
|
}
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
2957
|
+
/** What ran a proposal: a model when it made a paid call other than a judge's,
|
|
2958
|
+
* else deterministic code. */
|
|
2959
|
+
function searchProposalExecution(receipts, provider, source) {
|
|
2960
|
+
const model = receipts.find((receipt) => receipt.channel !== "judge")?.model;
|
|
2961
|
+
if (model === void 0) return {
|
|
2962
|
+
kind: "deterministic",
|
|
2963
|
+
source
|
|
2964
|
+
};
|
|
2965
|
+
return {
|
|
2966
|
+
kind: "model",
|
|
2967
|
+
model: searchModelIdentity(model, provider),
|
|
2968
|
+
source
|
|
2969
|
+
};
|
|
4721
2970
|
}
|
|
4722
2971
|
//#endregion
|
|
4723
2972
|
//#region src/campaign/search-history-receipt.ts
|
|
@@ -5288,10 +3537,16 @@ function canonicalHeader(header) {
|
|
|
5288
3537
|
* Expansion policies: which node a search expands next, with which operator,
|
|
5289
3538
|
* and which node it currently keeps.
|
|
5290
3539
|
*
|
|
5291
|
-
* A policy reads a `SearchPolicyView`: nodes
|
|
5292
|
-
* per-unit scores on the policy's split
|
|
5293
|
-
*
|
|
5294
|
-
*
|
|
3540
|
+
* A policy reads a `SearchPolicyView`: the nodes, their lineage, their
|
|
3541
|
+
* per-unit scores and posterior on the policy's split, and how many of their
|
|
3542
|
+
* cells failed as defects. The view holds no test cells, so a policy cannot
|
|
3543
|
+
* see the sealed claim data. Its decisions spend budget and claim nothing; a
|
|
3544
|
+
* claim comes only from the sealed test split.
|
|
3545
|
+
*
|
|
3546
|
+
* `incumbent`, `incumbentWithOperatorBandit` and `crowdedFrontierParent` are
|
|
3547
|
+
* hill climbs and pair with the `uniform` allocator. `aide` and `beam` choose parents from each node's
|
|
3548
|
+
* posterior, which is honest about a node measured on few units, so they pair
|
|
3549
|
+
* with `asha`, which measures most nodes on only the first rung.
|
|
5295
3550
|
*/
|
|
5296
3551
|
/**
|
|
5297
3552
|
* The hill climb. The leader starts at the root; a screened node that dodged no
|
|
@@ -5316,6 +3571,65 @@ function incumbent(options = {}) {
|
|
|
5316
3571
|
});
|
|
5317
3572
|
}
|
|
5318
3573
|
/**
|
|
3574
|
+
* The hill climb (as `incumbent`), with its operator drawn from the
|
|
3575
|
+
* `operatorYield` lens instead of fixed at `improve` (search-tree-design
|
|
3576
|
+
* §12): a weighted draw, seeded from `seed` and the expansion index like
|
|
3577
|
+
* `crowdedFrontierParent`, over the lens's per-operator yield. The fallback
|
|
3578
|
+
* is per operator, not all-or-nothing: an operator the lens has not yet
|
|
3579
|
+
* measured `MIN_OUTCOMES_FOR_WEIGHT` outcomes for draws on `fixedWeights`
|
|
3580
|
+
* (uniform by default) instead, so it is never starved while unmeasured, and
|
|
3581
|
+
* an operator that does have enough outcomes draws on its own measured
|
|
3582
|
+
* yield right away rather than waiting for every other operator to catch
|
|
3583
|
+
* up — proved against real search-sim ledgers where a hill climb had only
|
|
3584
|
+
* ever proposed `improve`: an all-or-nothing gate would keep every draw
|
|
3585
|
+
* uniform forever, because `draft`/`debug`/`merge` are never measured by a
|
|
3586
|
+
* policy that never chooses them. A measured operator's weight is its own
|
|
3587
|
+
* `fixedWeights` prior plus its yield, not the yield alone: the two are on
|
|
3588
|
+
* different scales (yield is improvement per known dollar; the prior is an
|
|
3589
|
+
* arbitrary share), so using the yield by itself would let a small positive
|
|
3590
|
+
* yield draw LESS than an untested operator sitting on the default prior of
|
|
3591
|
+
* 1 — the same real ledgers caught exactly that before this was fixed. The
|
|
3592
|
+
* sum is floored just above zero, so a so-far-bad operator can still be
|
|
3593
|
+
* re-measured rather than excluded outright.
|
|
3594
|
+
*/
|
|
3595
|
+
function incumbentWithOperatorBandit(options = { seed: 0 }) {
|
|
3596
|
+
const { patience, seed } = options;
|
|
3597
|
+
if (!Number.isInteger(seed)) throw new TypeError(`incumbentWithOperatorBandit: seed must be an integer, got ${String(seed)}`);
|
|
3598
|
+
const fixedWeights = {
|
|
3599
|
+
draft: options.fixedWeights?.draft ?? 1,
|
|
3600
|
+
improve: options.fixedWeights?.improve ?? 1,
|
|
3601
|
+
debug: options.fixedWeights?.debug ?? 1,
|
|
3602
|
+
merge: options.fixedWeights?.merge ?? 1
|
|
3603
|
+
};
|
|
3604
|
+
for (const operator of EXPANSION_OPERATORS) {
|
|
3605
|
+
const weight = fixedWeights[operator];
|
|
3606
|
+
if (!Number.isFinite(weight) || weight < 0) throw new TypeError(`incumbentWithOperatorBandit: fixedWeights.${operator} must be >= 0, got ${weight}`);
|
|
3607
|
+
}
|
|
3608
|
+
return hillClimb({
|
|
3609
|
+
name: patience === void 0 ? `incumbent-operator-bandit(seed=${seed})` : `incumbent-operator-bandit(seed=${seed},patience=${patience})`,
|
|
3610
|
+
patience,
|
|
3611
|
+
parent: (_view, leader) => ({
|
|
3612
|
+
nodeId: leader,
|
|
3613
|
+
evidence: {}
|
|
3614
|
+
}),
|
|
3615
|
+
operator: (view) => chooseOperator(view.operatorWeights, fixedWeights, seed, view.expansions)
|
|
3616
|
+
});
|
|
3617
|
+
}
|
|
3618
|
+
function chooseOperator(measured, fixedWeights, seed, expansionIndex) {
|
|
3619
|
+
const weights = EXPANSION_OPERATORS.map((operator) => {
|
|
3620
|
+
const yieldMean = measured[operator];
|
|
3621
|
+
const prior = fixedWeights[operator];
|
|
3622
|
+
return yieldMean === null ? prior : Math.max(prior + yieldMean, 1e-6);
|
|
3623
|
+
});
|
|
3624
|
+
const total = weights.reduce((a, b) => a + b, 0);
|
|
3625
|
+
let draw = mulberry32(seed ^ Math.imul(expansionIndex + 1, 2654435761) | 0)() * total;
|
|
3626
|
+
for (let index = 0; index < EXPANSION_OPERATORS.length; index++) {
|
|
3627
|
+
draw -= weights[index];
|
|
3628
|
+
if (draw <= 0) return EXPANSION_OPERATORS[index];
|
|
3629
|
+
}
|
|
3630
|
+
return EXPANSION_OPERATORS[EXPANSION_OPERATORS.length - 1];
|
|
3631
|
+
}
|
|
3632
|
+
/**
|
|
5319
3633
|
* The hill climb with a different parent: each expansion draws two distinct
|
|
5320
3634
|
* members of the Pareto frontier (per-unit means on the policy split) with a
|
|
5321
3635
|
* PRNG seeded from `seed` and the expansion index, and keeps the one with the
|
|
@@ -5352,27 +3666,261 @@ function crowdedFrontierParent(options) {
|
|
|
5352
3666
|
}
|
|
5353
3667
|
});
|
|
5354
3668
|
}
|
|
5355
|
-
|
|
5356
|
-
|
|
5357
|
-
|
|
5358
|
-
|
|
5359
|
-
|
|
5360
|
-
|
|
3669
|
+
/**
|
|
3670
|
+
* AIDE's search policy (`aide/agent.py`, `search_policy`) with three changes
|
|
3671
|
+
* for noisy scores.
|
|
3672
|
+
*
|
|
3673
|
+
* 1. **Draft** while fewer than `drafts` nodes were drafted: the root is the
|
|
3674
|
+
* parent, and the proposer writes a whole alternative, not an edit.
|
|
3675
|
+
* 2. **Debug**, with probability `debugProbability`: a buggy leaf, drawn
|
|
3676
|
+
* uniformly, whose chain holds fewer than `maxDebugDepth` debug edges. A
|
|
3677
|
+
* node is buggy when at least half of its final cells outside the test
|
|
3678
|
+
* split failed: defects, not a low score. An `errored` cell is the
|
|
3679
|
+
* environment's fault; it is retried and never makes a node buggy.
|
|
3680
|
+
* 3. **Improve** otherwise. The parent is drawn by Thompson sampling, not
|
|
3681
|
+
* AIDE's argmax, from every screened node that is not buggy and has a
|
|
3682
|
+
* posterior (`searchPosterior`): the root sits at 0, and every other node
|
|
3683
|
+
* at its mean improvement over the root with the search's pooled
|
|
3684
|
+
* between-unit variance divided by its shared units, so a node measured on
|
|
3685
|
+
* few units draws from a wide posterior. Until some node shares 2 units
|
|
3686
|
+
* with the root the pooled variance is unknown, every posterior is as wide
|
|
3687
|
+
* as it can be, and the draw is uniform. With no such node, the policy
|
|
3688
|
+
* drafts.
|
|
3689
|
+
* 4. **Stall:** when the drawn parent's lineage (the nodes under its nearest
|
|
3690
|
+
* draft, or under the root) has had `stallAfter` improve children in a row
|
|
3691
|
+
* that did not raise the lineage's best posterior mean, the improve forks
|
|
3692
|
+
* from the node with the best posterior mean instead. This is AIDE²'s
|
|
3693
|
+
* inner policy. A fork's children restart the run of the lineage they
|
|
3694
|
+
* join, so a stalled lineage that holds the global best forks once, not
|
|
3695
|
+
* on every later expansion.
|
|
3696
|
+
*
|
|
3697
|
+
* A parent is always a node whose screen finished; a node refused at
|
|
3698
|
+
* admission or decided invalid never is. Draws come from a PRNG seeded by the
|
|
3699
|
+
* search seed and the count of completed expansions, so the policy is a pure
|
|
3700
|
+
* function of the ledger. The leader is the incumbent's.
|
|
3701
|
+
*/
|
|
3702
|
+
function aide(options = {}) {
|
|
3703
|
+
const drafts = integerOption("aide", "drafts", options.drafts ?? 5, 0);
|
|
3704
|
+
const debugProbability = options.debugProbability ?? .5;
|
|
3705
|
+
if (!(debugProbability >= 0 && debugProbability <= 1)) throw new TypeError(`aide: debugProbability must be in [0, 1], got ${String(options.debugProbability)}`);
|
|
3706
|
+
const maxDebugDepth = integerOption("aide", "maxDebugDepth", options.maxDebugDepth ?? 3, 0);
|
|
3707
|
+
const stallAfter = integerOption("aide", "stallAfter", options.stallAfter ?? 4, 1);
|
|
3708
|
+
const name = drafts === 5 && debugProbability === .5 && maxDebugDepth === 3 && stallAfter === 4 ? "aide" : `aide(drafts=${drafts},debug=${debugProbability},maxDebugDepth=${maxDebugDepth},stallAfter=${stallAfter})`;
|
|
3709
|
+
return {
|
|
3710
|
+
name,
|
|
3711
|
+
leader: leaderOf,
|
|
3712
|
+
expand(view) {
|
|
3713
|
+
const root = view.rootNodeId;
|
|
3714
|
+
const screened = new Set(view.screened);
|
|
3715
|
+
if (!screened.has(root)) return null;
|
|
3716
|
+
const nodes = view.nodes();
|
|
3717
|
+
const byId = new Map(nodes.map((node) => [node.nodeId, node]));
|
|
3718
|
+
const rng = mulberry32(view.seed ^ Math.imul(view.expansions + 1, 2654435761) ^ AIDE_SALT | 0);
|
|
3719
|
+
const drafted = nodes.filter((node) => node.operator === "draft").length;
|
|
3720
|
+
const draft = (evidence) => ({
|
|
3721
|
+
parents: [root],
|
|
3722
|
+
operator: "draft",
|
|
3723
|
+
selection: {
|
|
3724
|
+
rule: `${name}:draft`,
|
|
3725
|
+
evidence: {
|
|
3726
|
+
drafted,
|
|
3727
|
+
drafts,
|
|
3728
|
+
...evidence
|
|
3729
|
+
}
|
|
3730
|
+
}
|
|
3731
|
+
});
|
|
3732
|
+
if (drafted < drafts) return draft({});
|
|
3733
|
+
if (rng() < debugProbability) {
|
|
3734
|
+
const debuggable = nodes.filter((node) => screened.has(node.nodeId) && buggy(node) && node.children.length === 0 && debugDepth(node, byId) < maxDebugDepth);
|
|
3735
|
+
if (debuggable.length > 0) {
|
|
3736
|
+
const pick = debuggable[Math.floor(rng() * debuggable.length)];
|
|
3737
|
+
return {
|
|
3738
|
+
parents: [pick.nodeId],
|
|
3739
|
+
operator: "debug",
|
|
3740
|
+
selection: {
|
|
3741
|
+
rule: `${name}:debug`,
|
|
3742
|
+
evidence: {
|
|
3743
|
+
debuggable: debuggable.length,
|
|
3744
|
+
defects: pick.defects,
|
|
3745
|
+
outcomes: pick.outcomes,
|
|
3746
|
+
debugDepth: debugDepth(pick, byId)
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
};
|
|
3750
|
+
}
|
|
3751
|
+
}
|
|
3752
|
+
const posterior = new Map(view.posterior.nodes.map((entry) => [entry.nodeId, entry]));
|
|
3753
|
+
const good = nodes.filter((node) => screened.has(node.nodeId) && !buggy(node) && (posterior.get(node.nodeId)?.mean ?? null) !== null);
|
|
3754
|
+
if (good.length === 0) return draft({ good: 0 });
|
|
3755
|
+
const meanOf = (node) => posterior.get(node.nodeId).mean;
|
|
3756
|
+
const known = view.posterior.pooledVariance !== null;
|
|
3757
|
+
let drawn = good[0];
|
|
3758
|
+
let draw = Number.NEGATIVE_INFINITY;
|
|
3759
|
+
if (!known) drawn = good[Math.floor(rng() * good.length)];
|
|
3760
|
+
else for (const node of good) {
|
|
3761
|
+
const value = meanOf(node) + Math.sqrt(posterior.get(node.nodeId).variance) * normal(rng);
|
|
3762
|
+
if (value > draw) {
|
|
3763
|
+
draw = value;
|
|
3764
|
+
drawn = node;
|
|
3765
|
+
}
|
|
3766
|
+
}
|
|
3767
|
+
const stalled = stalledRun(drawn, nodes, screened, posterior, `${name}:stall-fork`);
|
|
3768
|
+
if (stalled >= stallAfter) {
|
|
3769
|
+
let best = good[0];
|
|
3770
|
+
for (const node of good) if (meanOf(node) > meanOf(best)) best = node;
|
|
3771
|
+
return {
|
|
3772
|
+
parents: [best.nodeId],
|
|
3773
|
+
operator: "improve",
|
|
3774
|
+
selection: {
|
|
3775
|
+
rule: `${name}:stall-fork`,
|
|
3776
|
+
evidence: {
|
|
3777
|
+
candidates: good.length,
|
|
3778
|
+
mean: plain(meanOf(best)),
|
|
3779
|
+
units: posterior.get(best.nodeId).pairs,
|
|
3780
|
+
drawnOrdinal: drawn.ordinal,
|
|
3781
|
+
stalledChildren: stalled
|
|
3782
|
+
}
|
|
3783
|
+
}
|
|
3784
|
+
};
|
|
3785
|
+
}
|
|
3786
|
+
const entry = posterior.get(drawn.nodeId);
|
|
3787
|
+
return {
|
|
3788
|
+
parents: [drawn.nodeId],
|
|
3789
|
+
operator: "improve",
|
|
3790
|
+
selection: {
|
|
3791
|
+
rule: `${name}:${known ? "thompson" : "uniform-draw"}`,
|
|
3792
|
+
evidence: {
|
|
3793
|
+
candidates: good.length,
|
|
3794
|
+
mean: plain(entry.mean),
|
|
3795
|
+
units: entry.pairs,
|
|
3796
|
+
...known ? {
|
|
3797
|
+
sd: plain(Math.sqrt(entry.variance)),
|
|
3798
|
+
draw: plain(draw)
|
|
3799
|
+
} : {}
|
|
3800
|
+
}
|
|
3801
|
+
}
|
|
3802
|
+
};
|
|
3803
|
+
}
|
|
3804
|
+
};
|
|
3805
|
+
}
|
|
3806
|
+
/**
|
|
3807
|
+
* Beam search: the beam is the top `width` nodes by posterior mean
|
|
3808
|
+
* (`searchPosterior`), and the member with the fewest children is expanded,
|
|
3809
|
+
* so each member is expanded in turn, as in a generation of `width`. A member
|
|
3810
|
+
* is the root, a node an allocator advanced, or a node that scored every unit
|
|
3811
|
+
* the root scored: a node an allocator has only screened waits for its rank
|
|
3812
|
+
* before it enters. Ties go to the higher mean, then to the earlier node. The
|
|
3813
|
+
* leader is the incumbent's. Strategy evolution is `beam` with its population
|
|
3814
|
+
* as the width.
|
|
3815
|
+
*/
|
|
3816
|
+
function beam(options) {
|
|
3817
|
+
const width = integerOption("beam", "width", options.width, 1);
|
|
3818
|
+
const name = `beam(width=${width})`;
|
|
3819
|
+
return {
|
|
3820
|
+
name,
|
|
3821
|
+
leader: leaderOf,
|
|
3822
|
+
expand(view) {
|
|
3823
|
+
const root = view.rootNodeId;
|
|
3824
|
+
const screened = new Set(view.screened);
|
|
3825
|
+
if (!screened.has(root)) return null;
|
|
3826
|
+
const posterior = new Map(view.posterior.nodes.map((entry) => [entry.nodeId, entry.mean]));
|
|
3827
|
+
const rootUnits = view.unitScores(root).map((unit) => unit.unitId);
|
|
3828
|
+
const members = view.nodes().filter((node) => screened.has(node.nodeId) && (posterior.get(node.nodeId) ?? null) !== null && (node.nodeId === root || node.status === "advanced" || scoredEvery(view, node.nodeId, rootUnits))).sort((left, right) => posterior.get(right.nodeId) - posterior.get(left.nodeId) || left.ordinal - right.ordinal).slice(0, width);
|
|
3829
|
+
let rank = 0;
|
|
3830
|
+
members.forEach((member, index) => {
|
|
3831
|
+
if (member.children.length < members[rank].children.length) rank = index;
|
|
3832
|
+
});
|
|
3833
|
+
const pick = members[rank];
|
|
3834
|
+
return {
|
|
3835
|
+
parents: [pick.nodeId],
|
|
3836
|
+
operator: "improve",
|
|
3837
|
+
selection: {
|
|
3838
|
+
rule: name,
|
|
3839
|
+
evidence: {
|
|
3840
|
+
rank: rank + 1,
|
|
3841
|
+
members: members.length,
|
|
3842
|
+
mean: plain(posterior.get(pick.nodeId)),
|
|
3843
|
+
children: pick.children.length
|
|
3844
|
+
}
|
|
3845
|
+
}
|
|
3846
|
+
};
|
|
5361
3847
|
}
|
|
5362
|
-
return current;
|
|
5363
3848
|
};
|
|
3849
|
+
}
|
|
3850
|
+
/** Salts the aide PRNG, so its draws differ from `crowdedFrontierParent`'s at one seed. */
|
|
3851
|
+
const AIDE_SALT = 1095320645;
|
|
3852
|
+
/** At least half of the node's final cells failed as defects. */
|
|
3853
|
+
function buggy(node) {
|
|
3854
|
+
return node.outcomes > 0 && 2 * node.defects >= node.outcomes;
|
|
3855
|
+
}
|
|
3856
|
+
/** Debug edges in a row ending at `node`. */
|
|
3857
|
+
function debugDepth(node, byId) {
|
|
3858
|
+
let depth = 0;
|
|
3859
|
+
let current = node;
|
|
3860
|
+
while (current?.operator === "debug") {
|
|
3861
|
+
depth += 1;
|
|
3862
|
+
current = current.parent === null ? void 0 : byId.get(current.parent);
|
|
3863
|
+
}
|
|
3864
|
+
return depth;
|
|
3865
|
+
}
|
|
3866
|
+
/**
|
|
3867
|
+
* Improve children in a row, in registration order, that did not raise the
|
|
3868
|
+
* best posterior mean of `node`'s lineage: the nodes under its nearest draft,
|
|
3869
|
+
* or under the root. Only screened nodes with a posterior count, and a child
|
|
3870
|
+
* placed by `forkRule` restarts the run.
|
|
3871
|
+
*/
|
|
3872
|
+
function stalledRun(node, nodes, screened, posterior, forkRule) {
|
|
3873
|
+
const heads = /* @__PURE__ */ new Map();
|
|
3874
|
+
for (const entry of nodes) {
|
|
3875
|
+
const parentHead = entry.parent === null ? void 0 : heads.get(entry.parent);
|
|
3876
|
+
const starts = entry.operator === "draft" || entry.operator === "seed" || entry.operator === "derive" || parentHead === void 0;
|
|
3877
|
+
heads.set(entry.nodeId, starts ? entry.nodeId : parentHead);
|
|
3878
|
+
}
|
|
3879
|
+
const headOf = (entry) => heads.get(entry.nodeId);
|
|
3880
|
+
const lineage = headOf(node);
|
|
3881
|
+
let best = Number.NEGATIVE_INFINITY;
|
|
3882
|
+
let run = 0;
|
|
3883
|
+
for (const entry of nodes) {
|
|
3884
|
+
const mean = posterior.get(entry.nodeId)?.mean ?? null;
|
|
3885
|
+
if (mean === null || !screened.has(entry.nodeId) || headOf(entry) !== lineage) continue;
|
|
3886
|
+
if (entry.rule === forkRule) run = 0;
|
|
3887
|
+
else if (entry.operator === "improve") run = mean > best ? 0 : run + 1;
|
|
3888
|
+
best = Math.max(best, mean);
|
|
3889
|
+
}
|
|
3890
|
+
return run;
|
|
3891
|
+
}
|
|
3892
|
+
/** True when the node scored every one of `units`. */
|
|
3893
|
+
function scoredEvery(view, nodeId, units) {
|
|
3894
|
+
if (units.length === 0 || !view.complete(nodeId)) return false;
|
|
3895
|
+
const own = new Set(view.unitScores(nodeId).map((unit) => unit.unitId));
|
|
3896
|
+
return units.every((unit) => own.has(unit));
|
|
3897
|
+
}
|
|
3898
|
+
/** A standard normal draw (Box-Muller). */
|
|
3899
|
+
function normal(rng) {
|
|
3900
|
+
return Math.sqrt(-2 * Math.log(1 - rng())) * Math.cos(2 * Math.PI * rng());
|
|
3901
|
+
}
|
|
3902
|
+
function integerOption(owner, name, value, minimum) {
|
|
3903
|
+
if (!Number.isSafeInteger(value) || value < minimum) throw new TypeError(`${owner}: ${name} must be an integer of at least ${minimum}, got ${String(value)}`);
|
|
3904
|
+
return value;
|
|
3905
|
+
}
|
|
3906
|
+
/** Evidence is canonical JSON, which has one zero. */
|
|
3907
|
+
function plain(value) {
|
|
3908
|
+
const rounded = Math.round(value * 1e9) / 1e9;
|
|
3909
|
+
return rounded === 0 ? 0 : rounded;
|
|
3910
|
+
}
|
|
3911
|
+
function hillClimb(spec) {
|
|
5364
3912
|
return {
|
|
5365
3913
|
name: spec.name,
|
|
5366
3914
|
...spec.patience === void 0 ? {} : { patience: spec.patience },
|
|
5367
|
-
leader,
|
|
3915
|
+
leader: leaderOf,
|
|
5368
3916
|
expand(view) {
|
|
5369
3917
|
if (view.screening > 0) return null;
|
|
5370
|
-
const kept =
|
|
3918
|
+
const kept = leaderOf(view);
|
|
5371
3919
|
const chosen = spec.parent(view, kept);
|
|
5372
3920
|
const units = view.unitScores(chosen.nodeId);
|
|
5373
3921
|
return {
|
|
5374
3922
|
parents: [chosen.nodeId],
|
|
5375
|
-
operator: "improve",
|
|
3923
|
+
operator: spec.operator ? spec.operator(view) : "improve",
|
|
5376
3924
|
selection: {
|
|
5377
3925
|
rule: spec.name,
|
|
5378
3926
|
evidence: {
|
|
@@ -5386,6 +3934,22 @@ function hillClimb(spec) {
|
|
|
5386
3934
|
}
|
|
5387
3935
|
};
|
|
5388
3936
|
}
|
|
3937
|
+
/**
|
|
3938
|
+
* The leader every built-in policy keeps: it starts at the root, and a
|
|
3939
|
+
* screened node that dodged no unit takes the lead when it scored every unit
|
|
3940
|
+
* the leader scored and its mean on them beats the leader's. Nodes are taken
|
|
3941
|
+
* in registration order, so the leader is a pure function of the ledger. A
|
|
3942
|
+
* node an allocator has only screened cannot take the lead from one measured
|
|
3943
|
+
* on more units.
|
|
3944
|
+
*/
|
|
3945
|
+
function leaderOf(view) {
|
|
3946
|
+
let current = view.rootNodeId;
|
|
3947
|
+
for (const nodeId of view.screened) {
|
|
3948
|
+
if (nodeId === current || !view.complete(nodeId)) continue;
|
|
3949
|
+
if (beats(view, nodeId, current)) current = nodeId;
|
|
3950
|
+
}
|
|
3951
|
+
return current;
|
|
3952
|
+
}
|
|
5389
3953
|
/** True when `challenger` scored every unit `holder` scored and improves on
|
|
5390
3954
|
* `holder` over them. */
|
|
5391
3955
|
function beats(view, challenger, holder) {
|
|
@@ -5603,8 +4167,8 @@ async function runOptimization(opts) {
|
|
|
5603
4167
|
const accounting = () => {
|
|
5604
4168
|
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receiptKey(receipt)));
|
|
5605
4169
|
return {
|
|
5606
|
-
accounting:
|
|
5607
|
-
execution: opts.searchLedger ? identity.proposer :
|
|
4170
|
+
accounting: searchReceiptAccounting(fresh),
|
|
4171
|
+
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
|
|
5608
4172
|
};
|
|
5609
4173
|
};
|
|
5610
4174
|
const earlier = await recorder.state();
|
|
@@ -6010,20 +4574,10 @@ var SurfaceNodes = class {
|
|
|
6010
4574
|
this.cells.set(nodeId, cells);
|
|
6011
4575
|
}
|
|
6012
4576
|
cellResult(cell, work) {
|
|
6013
|
-
return {
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
...this.execution,
|
|
6018
|
-
model: cellModel(cell, this.execution.model)
|
|
6019
|
-
},
|
|
6020
|
-
wallMs: cell.durationMs,
|
|
6021
|
-
placement: {
|
|
6022
|
-
lane: work.lane,
|
|
6023
|
-
boxId: null
|
|
6024
|
-
},
|
|
6025
|
-
traceRef: { unknown: "runCampaign reports no trace id per cell" }
|
|
6026
|
-
};
|
|
4577
|
+
return campaignCellSearchResult(cell, {
|
|
4578
|
+
execution: this.execution,
|
|
4579
|
+
lane: work.lane
|
|
4580
|
+
});
|
|
6027
4581
|
}
|
|
6028
4582
|
};
|
|
6029
4583
|
/** The default ledger and identities when the caller binds none. Revisions are
|
|
@@ -6074,110 +4628,6 @@ function artifactKindOf(surface) {
|
|
|
6074
4628
|
function receiptKey(receipt) {
|
|
6075
4629
|
return receipt.callId;
|
|
6076
4630
|
}
|
|
6077
|
-
/** A proposal that made a paid call ran a model; one that made none ran code. */
|
|
6078
|
-
function proposalExecution(receipts, provider, source) {
|
|
6079
|
-
const model = receipts.find((receipt) => receipt.channel !== "judge")?.model;
|
|
6080
|
-
if (model === void 0) return {
|
|
6081
|
-
kind: "deterministic",
|
|
6082
|
-
source
|
|
6083
|
-
};
|
|
6084
|
-
return {
|
|
6085
|
-
kind: "model",
|
|
6086
|
-
model: modelIdentity(model, provider),
|
|
6087
|
-
source
|
|
6088
|
-
};
|
|
6089
|
-
}
|
|
6090
|
-
function modelIdentity(model, provider) {
|
|
6091
|
-
return modelHasSnapshot(model) ? {
|
|
6092
|
-
provider,
|
|
6093
|
-
snapshot: model
|
|
6094
|
-
} : {
|
|
6095
|
-
provider,
|
|
6096
|
-
alias: model,
|
|
6097
|
-
unknown: "the provider reported a moving alias, not a snapshot"
|
|
6098
|
-
};
|
|
6099
|
-
}
|
|
6100
|
-
function cellModel(cell, fallback) {
|
|
6101
|
-
return cell.resolvedModel === void 0 ? fallback : modelIdentity(cell.resolvedModel, fallback.provider);
|
|
6102
|
-
}
|
|
6103
|
-
function cellOutcome(cell) {
|
|
6104
|
-
const quality = projectCampaignCellQuality(cell);
|
|
6105
|
-
if (quality.score === void 0) return {
|
|
6106
|
-
status: "errored",
|
|
6107
|
-
metrics: {},
|
|
6108
|
-
error: {
|
|
6109
|
-
code: cell.errorStage ?? "unscored",
|
|
6110
|
-
message: cell.error ?? "the cell produced no complete judge score",
|
|
6111
|
-
retryable: false
|
|
6112
|
-
}
|
|
6113
|
-
};
|
|
6114
|
-
const metrics = { composite: quality.score };
|
|
6115
|
-
for (const [judge, score] of Object.entries(quality.successfulJudgeScores)) metrics[`judge.${judge}`] = score.composite;
|
|
6116
|
-
return {
|
|
6117
|
-
status: "passed",
|
|
6118
|
-
score: quality.score,
|
|
6119
|
-
metrics
|
|
6120
|
-
};
|
|
6121
|
-
}
|
|
6122
|
-
function cellAccounting(cell) {
|
|
6123
|
-
const usage = cell.tokenUsage;
|
|
6124
|
-
return {
|
|
6125
|
-
tokens: usage.tokensKnown === false ? {
|
|
6126
|
-
status: "unknown",
|
|
6127
|
-
reason: "a paid call in this cell reported no token usage"
|
|
6128
|
-
} : {
|
|
6129
|
-
status: "known",
|
|
6130
|
-
inputTokens: usage.input,
|
|
6131
|
-
outputTokens: usage.output,
|
|
6132
|
-
cachedTokens: 0
|
|
6133
|
-
},
|
|
6134
|
-
cost: cell.costProvenance.kind === "uncaptured" ? {
|
|
6135
|
-
status: "unknown",
|
|
6136
|
-
knownLowerBoundUsd: cell.costUsd,
|
|
6137
|
-
reason: "the cell recorded spend without a provider receipt"
|
|
6138
|
-
} : {
|
|
6139
|
-
status: "known",
|
|
6140
|
-
usd: cell.costUsd,
|
|
6141
|
-
source: cell.costProvenance.kind === "observed" ? "provider" : "pricing-table"
|
|
6142
|
-
}
|
|
6143
|
-
};
|
|
6144
|
-
}
|
|
6145
|
-
function receiptAccounting(receipts) {
|
|
6146
|
-
let inputTokens = 0;
|
|
6147
|
-
let outputTokens = 0;
|
|
6148
|
-
let cachedTokens = 0;
|
|
6149
|
-
let usd = 0;
|
|
6150
|
-
let tokensKnown = true;
|
|
6151
|
-
let costKnown = true;
|
|
6152
|
-
for (const receipt of receipts) {
|
|
6153
|
-
if (receipt.usageUnknown === true) tokensKnown = false;
|
|
6154
|
-
inputTokens += receipt.inputTokens;
|
|
6155
|
-
outputTokens += receipt.outputTokens;
|
|
6156
|
-
cachedTokens += receipt.cachedTokens ?? 0;
|
|
6157
|
-
if (receipt.costUnknown) costKnown = false;
|
|
6158
|
-
else usd += receipt.costUsd;
|
|
6159
|
-
}
|
|
6160
|
-
return {
|
|
6161
|
-
tokens: tokensKnown ? {
|
|
6162
|
-
status: "known",
|
|
6163
|
-
inputTokens,
|
|
6164
|
-
outputTokens,
|
|
6165
|
-
cachedTokens
|
|
6166
|
-
} : {
|
|
6167
|
-
status: "unknown",
|
|
6168
|
-
reason: "a candidate-generation call reported no token usage"
|
|
6169
|
-
},
|
|
6170
|
-
cost: costKnown ? {
|
|
6171
|
-
status: "known",
|
|
6172
|
-
usd,
|
|
6173
|
-
source: usd === 0 ? "free" : "provider"
|
|
6174
|
-
} : {
|
|
6175
|
-
status: "unknown",
|
|
6176
|
-
knownLowerBoundUsd: usd,
|
|
6177
|
-
reason: "a candidate-generation call recorded no provider cost"
|
|
6178
|
-
}
|
|
6179
|
-
};
|
|
6180
|
-
}
|
|
6181
4631
|
function immutableProposalSnapshot(value, label) {
|
|
6182
4632
|
try {
|
|
6183
4633
|
return deepFreeze(structuredClone(value));
|
|
@@ -7666,6 +6116,6 @@ function firstString(value) {
|
|
|
7666
6116
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
7667
6117
|
}
|
|
7668
6118
|
//#endregion
|
|
7669
|
-
export {
|
|
6119
|
+
export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
|
|
7670
6120
|
|
|
7671
|
-
//# sourceMappingURL=llm-judge-
|
|
6121
|
+
//# sourceMappingURL=llm-judge-BYWXsxhZ.js.map
|