@tangle-network/agent-eval 0.197.0 → 0.198.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +3 -1
- package/dist/active-curriculum-B_Lc9kfO.js +61 -0
- package/dist/active-curriculum-B_Lc9kfO.js.map +1 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +3 -3
- package/dist/{analyze-runs-B6gv7vAT.js → analyze-runs-DFbp2gB8.js} +5 -5
- package/dist/{analyze-runs-B6gv7vAT.js.map → analyze-runs-DFbp2gB8.js.map} +1 -1
- package/dist/{benchmark-command-DB7pxlks.js → benchmark-command-DKQuLrrD.js} +6 -5
- package/dist/{benchmark-command-DB7pxlks.js.map → benchmark-command-DKQuLrrD.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -3
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -4
- package/dist/campaign/index.js +13 -11
- package/dist/{campaign-BUFSOI36.js → campaign-DRdKwxm-.js} +13 -12
- package/dist/{campaign-BUFSOI36.js.map → campaign-DRdKwxm-.js.map} +1 -1
- package/dist/{campaign-evidence-DJc60F9C.js → campaign-evidence-jL_U7Ikv.js} +2 -2
- package/dist/{campaign-evidence-DJc60F9C.js.map → campaign-evidence-jL_U7Ikv.js.map} +1 -1
- package/dist/cli.js +247 -13
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +7 -6
- package/dist/contract/index.js +9 -9
- package/dist/{default-registry-D2csUBBQ.d.ts → default-registry-oK9hHfX_.d.ts} +3 -3
- package/dist/{default-registry-D2csUBBQ.d.ts.map → default-registry-oK9hHfX_.d.ts.map} +1 -1
- package/dist/{define-agent-eval-ulRlLvse.d.ts → define-agent-eval-9JYoCkDI.d.ts} +5 -4
- package/dist/define-agent-eval-9JYoCkDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-4J02MdF7.js → define-agent-eval-DkfcqaCF.js} +6 -6
- package/dist/{define-agent-eval-4J02MdF7.js.map → define-agent-eval-DkfcqaCF.js.map} +1 -1
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-k_QdEBZo.js → dspy-rlm-engine-B5T43BkZ.js} +2 -2
- package/dist/{dspy-rlm-engine-k_QdEBZo.js.map → dspy-rlm-engine-B5T43BkZ.js.map} +1 -1
- package/dist/{engine-De2hfuiQ.d.ts → engine-Dc996kAg.d.ts} +2 -2
- package/dist/{engine-De2hfuiQ.d.ts.map → engine-Dc996kAg.d.ts.map} +1 -1
- package/dist/{eval-campaign-CXT2WYUe.js → eval-campaign-GapmPHIA.js} +2 -2
- package/dist/{eval-campaign-CXT2WYUe.js.map → eval-campaign-GapmPHIA.js.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.js +8 -6
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-_YIECYJ7.js → external-optimizer-process-CEXcCKqd.js} +2 -2
- package/dist/{external-optimizer-process-_YIECYJ7.js.map → external-optimizer-process-CEXcCKqd.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-BrghTvEp.js → external-optimizer-subprocess-laM1-UDV.js} +2 -61
- package/dist/external-optimizer-subprocess-laM1-UDV.js.map +1 -0
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +2 -3
- package/dist/fuzz.js.map +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-DJKZEldg.d.ts → index-BXJHJ_jE.d.ts} +12 -947
- package/dist/index-BXJHJ_jE.d.ts.map +1 -0
- package/dist/{index-CRsmcAGl.d.ts → index-DzQ6T34s.d.ts} +6 -6
- package/dist/{index-CRsmcAGl.d.ts.map → index-DzQ6T34s.d.ts.map} +1 -1
- package/dist/{index-kKECJfib.d.ts → index-lbwvO9vJ.d.ts} +3 -3
- package/dist/{index-kKECJfib.d.ts.map → index-lbwvO9vJ.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +15 -13
- package/dist/index.js.map +1 -1
- package/dist/jev.d.ts +1 -1
- package/dist/{llm-judge-fhkFQL1N.js → llm-judge-3ncNcWNT.js} +415 -1981
- package/dist/llm-judge-3ncNcWNT.js.map +1 -0
- package/dist/{matrix-C1X-fIVX.d.ts → matrix-BnksRaDD.d.ts} +2 -2
- package/dist/{matrix-C1X-fIVX.d.ts.map → matrix-BnksRaDD.d.ts.map} +1 -1
- package/dist/multiplicity-D9cjmStq.js +101 -0
- package/dist/multiplicity-D9cjmStq.js.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.js +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-JSDcs2zx.js → paired-arms-DKZFVPt8.js} +4 -398
- package/dist/paired-arms-DKZFVPt8.js.map +1 -0
- package/dist/paired-binary-fzkwENI4.js +399 -0
- package/dist/paired-binary-fzkwENI4.js.map +1 -0
- package/dist/{paired-promotion-decision-DTKl6-G8.js → paired-promotion-decision-D5B1Cxc2.js} +78 -5
- package/dist/paired-promotion-decision-D5B1Cxc2.js.map +1 -0
- package/dist/{power-and-mde-BJOpt6pq.js → power-and-mde-K_CmO3dc.js} +3 -101
- package/dist/power-and-mde-K_CmO3dc.js.map +1 -0
- package/dist/{produced-state-1RnbLrSa.js → produced-state-qIhqN3V5.js} +3 -3
- package/dist/{produced-state-1RnbLrSa.js.map → produced-state-qIhqN3V5.js.map} +1 -1
- package/dist/{promotion-policy-Cy5OV8BV.js → promotion-policy-C6FAJvmZ.js} +3 -3
- package/dist/{promotion-policy-Cy5OV8BV.js.map → promotion-policy-C6FAJvmZ.js.map} +1 -1
- package/dist/reporting.js +3 -3
- package/dist/rl.d.ts +17 -39
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +5 -5
- package/dist/search/index.d.ts +1187 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +187 -0
- package/dist/search/index.js.map +1 -0
- package/dist/search-kernel-CumYAjA3.js +2427 -0
- package/dist/search-kernel-CumYAjA3.js.map +1 -0
- package/dist/search-kernel-TLoiMMcq.d.ts +1185 -0
- package/dist/search-kernel-TLoiMMcq.d.ts.map +1 -0
- package/dist/{search-ledger-9W0g5FP6.js → search-ledger-BnUytR8X.js} +3 -1
- package/dist/search-ledger-BnUytR8X.js.map +1 -0
- package/dist/{search-shipper-CYbmJQx8.js → search-shipper-Cn8cxcyp.js} +2 -2
- package/dist/{search-shipper-CYbmJQx8.js.map → search-shipper-Cn8cxcyp.js.map} +1 -1
- package/dist/search-summary-dJOaZ3Dw.js +133 -0
- package/dist/search-summary-dJOaZ3Dw.js.map +1 -0
- package/dist/{semantic-concept-judge-Bh5lvKGg.js → semantic-concept-judge-2SE_kXGF.js} +2 -2
- package/dist/{semantic-concept-judge-Bh5lvKGg.js.map → semantic-concept-judge-2SE_kXGF.js.map} +1 -1
- package/dist/{sequential-C21_PP7F.js → sequential-Yz3OSskl.js} +2 -2
- package/dist/{sequential-C21_PP7F.js.map → sequential-Yz3OSskl.js.map} +1 -1
- package/dist/single-run-lock-fVu3MZWH.js +62 -0
- package/dist/single-run-lock-fVu3MZWH.js.map +1 -0
- package/dist/{skillopt-optimization-method-CHUcZ8w6.js → skillopt-optimization-method-BL_j6mX8.js} +8 -7
- package/dist/{skillopt-optimization-method-CHUcZ8w6.js.map → skillopt-optimization-method-BL_j6mX8.js.map} +1 -1
- package/dist/{statistical-heldout-DANVmyug.js → statistical-heldout-BAFDCo07.js} +2 -2
- package/dist/{statistical-heldout-DANVmyug.js.map → statistical-heldout-BAFDCo07.js.map} +1 -1
- package/dist/{statistical-heldout-BedVSZb2.d.ts → statistical-heldout-CMpxMjlQ.d.ts} +2 -2
- package/dist/{statistical-heldout-BedVSZb2.d.ts.map → statistical-heldout-CMpxMjlQ.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CawP-WxM.d.ts → store-tool-spans-CUwWSXoj.d.ts} +3 -3
- package/dist/{store-tool-spans-CawP-WxM.d.ts.map → store-tool-spans-CUwWSXoj.d.ts.map} +1 -1
- package/dist/{summary-report-BMvou2oZ.js → summary-report-BeYnvddy.js} +4 -3
- package/dist/{summary-report-BMvou2oZ.js.map → summary-report-BeYnvddy.js.map} +1 -1
- package/dist/{tool-groups-B4JVrH3g.d.ts → tool-groups-DVJAR1GE.d.ts} +2 -2
- package/dist/tool-groups-DVJAR1GE.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/tree-DxGfhISA.js +4100 -0
- package/dist/tree-DxGfhISA.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-B3jzCp0p2.js} +1 -1
- package/dist/types-B3jzCp0p2.js.map +1 -0
- package/dist/{types-CjFDKMDe.d.ts → types-D94EanDL.d.ts} +2 -2
- package/dist/{types-CjFDKMDe.d.ts.map → types-D94EanDL.d.ts.map} +1 -1
- package/dist/{types-B2673PBj.d.ts → types-EBUav9Bu.d.ts} +8 -4
- package/dist/types-EBUav9Bu.d.ts.map +1 -0
- package/docs/campaign-proposers.md +2 -0
- package/docs/design/statistics-decisions.md +1 -1
- package/docs/search-ledger.md +145 -8
- package/docs/trace-contracts.md +1 -3
- package/package.json +6 -1
- package/dist/active-curriculum-OjIWgrUJ.js +0 -204
- package/dist/active-curriculum-OjIWgrUJ.js.map +0 -1
- package/dist/define-agent-eval-ulRlLvse.d.ts.map +0 -1
- package/dist/external-optimizer-subprocess-BrghTvEp.js.map +0 -1
- package/dist/index-DJKZEldg.d.ts.map +0 -1
- package/dist/llm-judge-fhkFQL1N.js.map +0 -1
- package/dist/paired-arms-JSDcs2zx.js.map +0 -1
- package/dist/paired-delta-test-BF-FEP8V.js +0 -78
- package/dist/paired-delta-test-BF-FEP8V.js.map +0 -1
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +0 -1
- package/dist/power-and-mde-BJOpt6pq.js.map +0 -1
- package/dist/search-ledger-9W0g5FP6.js.map +0 -1
- package/dist/search-summary-CcavvYLe.js +0 -378
- package/dist/search-summary-CcavvYLe.js.map +0 -1
- package/dist/tool-groups-B4JVrH3g.d.ts.map +0 -1
- package/dist/types-B2673PBj.d.ts.map +0 -1
- package/dist/types-B3jzCp0p.js.map +0 -1
|
@@ -2,24 +2,23 @@ import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
|
|
|
2
2
|
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
|
|
3
3
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
4
4
|
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
|
|
8
|
-
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-DANVmyug.js";
|
|
5
|
+
import { r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-D5B1Cxc2.js";
|
|
6
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-BAFDCo07.js";
|
|
9
7
|
import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
|
|
10
8
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
11
9
|
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-tNnlQIrD.js";
|
|
12
10
|
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
|
|
13
|
-
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-
|
|
11
|
+
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-jL_U7Ikv.js";
|
|
14
12
|
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
15
13
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
16
|
-
import {
|
|
14
|
+
import { S as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate, x as createRunCostLedger } from "./external-optimizer-subprocess-laM1-UDV.js";
|
|
17
15
|
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-V9Zqbq0J.js";
|
|
18
16
|
import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
|
|
19
17
|
import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
|
|
20
18
|
import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
19
|
+
import { D as paretoFrontier, O as paretoFrontierWithCrowding, a as searchPolicyView, b as estimateNode, i as searchExpansionIndex, n as runSearch, p as EXPANSION_OPERATORS, t as SEARCH_KERNEL_SOURCE } from "./search-kernel-CumYAjA3.js";
|
|
20
|
+
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-BnUytR8X.js";
|
|
21
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-dJOaZ3Dw.js";
|
|
23
22
|
import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-c_bKHTXz.js";
|
|
24
23
|
import { createHash } from "node:crypto";
|
|
25
24
|
import { writeFileSync } from "node:fs";
|
|
@@ -1165,8 +1164,8 @@ function detectCalibrationDrift(runs, opts) {
|
|
|
1165
1164
|
alpha,
|
|
1166
1165
|
recentN: recent.length,
|
|
1167
1166
|
historyN: historical.length,
|
|
1168
|
-
recentMean: mean$
|
|
1169
|
-
historyMean: mean$
|
|
1167
|
+
recentMean: mean$2(recent),
|
|
1168
|
+
historyMean: mean$2(historical)
|
|
1170
1169
|
}
|
|
1171
1170
|
}],
|
|
1172
1171
|
evaluation: {
|
|
@@ -1403,7 +1402,7 @@ function chiSquareCritical(df, alpha) {
|
|
|
1403
1402
|
}
|
|
1404
1403
|
return TABLE[10][idx];
|
|
1405
1404
|
}
|
|
1406
|
-
function mean$
|
|
1405
|
+
function mean$2(xs) {
|
|
1407
1406
|
if (xs.length === 0) return 0;
|
|
1408
1407
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
1409
1408
|
}
|
|
@@ -2457,85 +2456,6 @@ async function runFinalComparison(opts) {
|
|
|
2457
2456
|
};
|
|
2458
2457
|
}
|
|
2459
2458
|
//#endregion
|
|
2460
|
-
//#region src/pareto.ts
|
|
2461
|
-
/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
|
|
2462
|
-
function dominates(a, b, objectives) {
|
|
2463
|
-
let strictlyBetter = false;
|
|
2464
|
-
for (const obj of objectives) {
|
|
2465
|
-
const av = obj.value(a);
|
|
2466
|
-
const bv = obj.value(b);
|
|
2467
|
-
if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
|
|
2468
|
-
const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
|
|
2469
|
-
if (obj.direction === "maximize" ? av < bv : av > bv) return false;
|
|
2470
|
-
if (aIsBetter) strictlyBetter = true;
|
|
2471
|
-
}
|
|
2472
|
-
return strictlyBetter;
|
|
2473
|
-
}
|
|
2474
|
-
/**
|
|
2475
|
-
* Compute the non-dominated frontier. Candidates with NaN/Infinity on any
|
|
2476
|
-
* objective are excluded (can't rank them). A candidate enters the frontier
|
|
2477
|
-
* iff no other candidate dominates it.
|
|
2478
|
-
*/
|
|
2479
|
-
function paretoFrontier(candidates, objectives) {
|
|
2480
|
-
if (objectives.length === 0) throw new Error("paretoFrontier: at least 1 objective required");
|
|
2481
|
-
const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))));
|
|
2482
|
-
const frontier = [];
|
|
2483
|
-
const dominated = [];
|
|
2484
|
-
for (const c of valid) if (valid.some((other) => other !== c && dominates(other, c, objectives))) dominated.push(c);
|
|
2485
|
-
else frontier.push(c);
|
|
2486
|
-
return {
|
|
2487
|
-
frontier,
|
|
2488
|
-
dominated,
|
|
2489
|
-
dominanceMap: frontier.map((d) => ({
|
|
2490
|
-
dominator: d,
|
|
2491
|
-
dominated: dominated.filter((x) => dominates(d, x, objectives))
|
|
2492
|
-
}))
|
|
2493
|
-
};
|
|
2494
|
-
}
|
|
2495
|
-
/**
|
|
2496
|
-
* NSGA-II crowding distance — secondary sort for ties on the frontier.
|
|
2497
|
-
*
|
|
2498
|
-
* When the Pareto front collapses to a single point (or many candidates tie
|
|
2499
|
-
* on dominance), naive selection picks arbitrarily and the population
|
|
2500
|
-
* degenerates over generations. NSGA-II preserves diversity by preferring
|
|
2501
|
-
* candidates with more empty space around them on the frontier.
|
|
2502
|
-
*
|
|
2503
|
-
* Returns an array of `{ candidate, distance }` in the SAME order as the
|
|
2504
|
-
* input. Higher distance = more isolated = should be preferred when
|
|
2505
|
-
* preserving diversity.
|
|
2506
|
-
*/
|
|
2507
|
-
function crowdingDistance(candidates, objectives) {
|
|
2508
|
-
const distances = new Map(candidates.map((c) => [c, 0]));
|
|
2509
|
-
for (const obj of objectives) {
|
|
2510
|
-
const sorted = [...candidates].sort((a, b) => obj.value(a) - obj.value(b));
|
|
2511
|
-
const min = obj.value(sorted[0]);
|
|
2512
|
-
const range = obj.value(sorted[sorted.length - 1]) - min || 1;
|
|
2513
|
-
distances.set(sorted[0], Infinity);
|
|
2514
|
-
distances.set(sorted[sorted.length - 1], Infinity);
|
|
2515
|
-
for (let i = 1; i < sorted.length - 1; i++) {
|
|
2516
|
-
const prev = obj.value(sorted[i - 1]);
|
|
2517
|
-
const next = obj.value(sorted[i + 1]);
|
|
2518
|
-
const current = distances.get(sorted[i]);
|
|
2519
|
-
if (current === Infinity) continue;
|
|
2520
|
-
distances.set(sorted[i], current + (next - prev) / range);
|
|
2521
|
-
}
|
|
2522
|
-
}
|
|
2523
|
-
return candidates.map((c) => ({
|
|
2524
|
-
candidate: c,
|
|
2525
|
-
distance: distances.get(c) ?? 0
|
|
2526
|
-
}));
|
|
2527
|
-
}
|
|
2528
|
-
/**
|
|
2529
|
-
* Pareto frontier with tie-break by crowding distance — the canonical
|
|
2530
|
-
* NSGA-II selection step. Returns the frontier sorted by descending crowding
|
|
2531
|
-
* distance so callers can `.slice(0, k)` to pick K diverse winners.
|
|
2532
|
-
*/
|
|
2533
|
-
function paretoFrontierWithCrowding(candidates, objectives) {
|
|
2534
|
-
const { frontier } = paretoFrontier(candidates, objectives);
|
|
2535
|
-
if (frontier.length === 0) return [];
|
|
2536
|
-
return crowdingDistance(frontier, objectives).sort((a, b) => b.distance - a.distance);
|
|
2537
|
-
}
|
|
2538
|
-
//#endregion
|
|
2539
2459
|
//#region src/campaign/allocation.ts
|
|
2540
2460
|
/**
|
|
2541
2461
|
* Allocation: where a search spends rollouts.
|
|
@@ -2603,9 +2523,11 @@ function asha(options = {}) {
|
|
|
2603
2523
|
if (eta < 2) throw new TypeError(`asha: eta must be at least 2, got ${eta}`);
|
|
2604
2524
|
const trainUnits = nonNegativeInteger("asha", "trainUnits", options.trainUnits ?? 2);
|
|
2605
2525
|
const reps = positiveInteger("asha", "reps", options.reps ?? 1);
|
|
2606
|
-
const
|
|
2526
|
+
const extend = options.extend ?? null;
|
|
2527
|
+
const name = units === firstRung && eta === 3 && trainUnits === 2 && reps === 1 && extend === null ? "asha" : `asha(units=${units},eta=${eta},train=${trainUnits},reps=${reps}${extend ? `,extend=${extend.name}` : ""})`;
|
|
2607
2528
|
const layouts = /* @__PURE__ */ new WeakMap();
|
|
2608
|
-
const
|
|
2529
|
+
const extensions = /* @__PURE__ */ new WeakMap();
|
|
2530
|
+
const seededLayout = (state) => {
|
|
2609
2531
|
const header = state.header;
|
|
2610
2532
|
if (!header) return null;
|
|
2611
2533
|
let layout = layouts.get(header);
|
|
@@ -2618,6 +2540,74 @@ function asha(options = {}) {
|
|
|
2618
2540
|
}
|
|
2619
2541
|
return layout;
|
|
2620
2542
|
};
|
|
2543
|
+
/**
|
|
2544
|
+
* With `extend`, each opened rung's added units come from the chooser at
|
|
2545
|
+
* the sequence of the first decision that opened it; units of rungs not yet
|
|
2546
|
+
* opened follow in the seeded order, which only the root, measured on every
|
|
2547
|
+
* unit, ever reads.
|
|
2548
|
+
*/
|
|
2549
|
+
const layoutOf = (state) => {
|
|
2550
|
+
const seeded = seededLayout(state);
|
|
2551
|
+
if (!seeded || extend === null || seeded.top === 0) return seeded;
|
|
2552
|
+
const opened = /* @__PURE__ */ new Map();
|
|
2553
|
+
for (const node of state.nodes()) for (const record of node.decisions) {
|
|
2554
|
+
if (record.decision.status !== "advanced") continue;
|
|
2555
|
+
const known = opened.get(record.decision.rung);
|
|
2556
|
+
if (known === void 0 || record.sequence < known) opened.set(record.decision.rung, record.sequence);
|
|
2557
|
+
}
|
|
2558
|
+
if (opened.size === 0) return seeded;
|
|
2559
|
+
const header = state.header;
|
|
2560
|
+
let cache = extensions.get(header);
|
|
2561
|
+
if (!cache) {
|
|
2562
|
+
cache = /* @__PURE__ */ new Map();
|
|
2563
|
+
extensions.set(header, cache);
|
|
2564
|
+
}
|
|
2565
|
+
const order = seeded.order.slice(0, seeded.sizes[0]);
|
|
2566
|
+
const chosen = new Set(order);
|
|
2567
|
+
for (let rung = 1; rung <= seeded.top; rung++) {
|
|
2568
|
+
const sequence = opened.get(rung);
|
|
2569
|
+
if (sequence === void 0) break;
|
|
2570
|
+
let entry = cache.get(rung);
|
|
2571
|
+
if (!entry || entry.sequence !== sequence) {
|
|
2572
|
+
const remaining = seeded.order.filter((unitId) => !chosen.has(unitId));
|
|
2573
|
+
const count = seeded.sizes[rung] - seeded.sizes[rung - 1];
|
|
2574
|
+
const allowed = new Set(remaining);
|
|
2575
|
+
const picked = [];
|
|
2576
|
+
for (const unitId of extend.choose({
|
|
2577
|
+
state,
|
|
2578
|
+
asOfSequence: sequence,
|
|
2579
|
+
split: seeded.split,
|
|
2580
|
+
rung,
|
|
2581
|
+
measured: [...order],
|
|
2582
|
+
remaining,
|
|
2583
|
+
count
|
|
2584
|
+
})) {
|
|
2585
|
+
if (picked.length === count) break;
|
|
2586
|
+
if (allowed.delete(unitId)) picked.push(unitId);
|
|
2587
|
+
}
|
|
2588
|
+
for (const unitId of remaining) {
|
|
2589
|
+
if (picked.length === count) break;
|
|
2590
|
+
if (allowed.delete(unitId)) picked.push(unitId);
|
|
2591
|
+
}
|
|
2592
|
+
entry = {
|
|
2593
|
+
sequence,
|
|
2594
|
+
units: picked
|
|
2595
|
+
};
|
|
2596
|
+
cache.set(rung, entry);
|
|
2597
|
+
}
|
|
2598
|
+
for (const unitId of entry.units) {
|
|
2599
|
+
order.push(unitId);
|
|
2600
|
+
chosen.add(unitId);
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
for (const unitId of seeded.order) if (!chosen.has(unitId)) order.push(unitId);
|
|
2604
|
+
return {
|
|
2605
|
+
...seeded,
|
|
2606
|
+
order,
|
|
2607
|
+
position: new Map(order.map((unitId, index) => [unitId, index])),
|
|
2608
|
+
cellsThrough: seeded.sizes.map((size) => order.slice(0, size).reduce((sum, unitId) => sum + seeded.tasks.get(unitId).length, 0))
|
|
2609
|
+
};
|
|
2610
|
+
};
|
|
2621
2611
|
const cellsThrough = (layout, rung, root) => {
|
|
2622
2612
|
const plan = [];
|
|
2623
2613
|
const through = root ? layout.top : Math.min(rung, layout.top);
|
|
@@ -2710,7 +2700,7 @@ function asha(options = {}) {
|
|
|
2710
2700
|
const reason = (ranked, index, k, layout) => {
|
|
2711
2701
|
const entry = ranked[index];
|
|
2712
2702
|
const top = Math.floor(ranked.length / eta);
|
|
2713
|
-
const mean = entry.mean === null ? "an unscored unit" : `mean ${round
|
|
2703
|
+
const mean = entry.mean === null ? "an unscored unit" : `mean ${round(entry.mean)}`;
|
|
2714
2704
|
return `rank ${index + 1} of ${ranked.length} on rung ${k} (${layout.sizes[k]} units, ${mean}); the top ${top} advance (eta ${eta})`;
|
|
2715
2705
|
};
|
|
2716
2706
|
return {
|
|
@@ -2845,7 +2835,7 @@ function nonNegativeInteger(owner, name, value) {
|
|
|
2845
2835
|
if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${owner}: ${name} must be a non-negative integer, got ${String(value)}`);
|
|
2846
2836
|
return value;
|
|
2847
2837
|
}
|
|
2848
|
-
function round
|
|
2838
|
+
function round(value) {
|
|
2849
2839
|
return Math.round(value * 1e4) / 1e4;
|
|
2850
2840
|
}
|
|
2851
2841
|
//#endregion
|
|
@@ -2979,1877 +2969,6 @@ function searchProposalExecution(receipts, provider, source) {
|
|
|
2979
2969
|
};
|
|
2980
2970
|
}
|
|
2981
2971
|
//#endregion
|
|
2982
|
-
//#region src/paired-promotion-power.ts
|
|
2983
|
-
/**
|
|
2984
|
-
* @module
|
|
2985
|
-
* Joint power of a configured {@link decidePairedPromotion} call, computed by
|
|
2986
|
-
* running that call on data drawn from a registered alternative.
|
|
2987
|
-
*
|
|
2988
|
-
* Why a simulation and not a formula: `decidePairedPromotion` is not one test.
|
|
2989
|
-
* It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
|
|
2990
|
-
* paired bootstrap at or above it) and by shape (Tango's score interval on a
|
|
2991
|
-
* two-point outcome), and on top of the deciding interval it applies a
|
|
2992
|
-
* sufficiency floor, a zero-width refusal and McNemar's exact veto. A
|
|
2993
|
-
* preregistration usually seals two such calls, a continuous primary and a
|
|
2994
|
-
* pass/fail veto drawn from the same episodes, and promotes only when both
|
|
2995
|
-
* promote. The sample size at which THAT procedure reaches a target power is
|
|
2996
|
-
* not the power of any one channel: the sign-test power at n can be 0.8 while
|
|
2997
|
-
* the promotion probability is well below it because the veto binds
|
|
2998
|
-
* (agent-eval#785). Closed forms exist for single channels only, and each one
|
|
2999
|
-
* shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
|
|
3000
|
-
* Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
|
|
3001
|
-
* pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
|
|
3002
|
-
* So the only faithful power is the procedure's own promotion rate under the
|
|
3003
|
-
* alternative, and that is what this module computes: draw n pairs from the
|
|
3004
|
-
* registered law, run every configured call on them exactly as the caller
|
|
3005
|
-
* will, repeat, count.
|
|
3006
|
-
*
|
|
3007
|
-
* The alternative is DECLARATIVE and serializable, never a sampler function,
|
|
3008
|
-
* so a preregistration can seal it as bytes: a mixture of cells, each with a
|
|
3009
|
-
* probability, an optional pass/fail outcome per arm for the binary calls,
|
|
3010
|
-
* and a law for the candidate-minus-baseline delta on the continuous scale.
|
|
3011
|
-
* Drawing the cell first and the delta given the cell is what makes the two
|
|
3012
|
-
* channels one joint law rather than two marginals.
|
|
3013
|
-
*
|
|
3014
|
-
* Everything is seeded (`mulberry32`), so a sizing is reproducible from its
|
|
3015
|
-
* inputs, and every estimate carries its Monte Carlo standard error and a
|
|
3016
|
-
* Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
|
|
3017
|
-
*/
|
|
3018
|
-
/**
|
|
3019
|
-
* Joint power of the configured calls at `n` under the registered alternative:
|
|
3020
|
-
* the fraction of simulated data sets on which every call promotes.
|
|
3021
|
-
*/
|
|
3022
|
-
function pairedPromotionPower(options) {
|
|
3023
|
-
const n = options.n;
|
|
3024
|
-
if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
|
|
3025
|
-
const simulations = options.simulations ?? 1e3;
|
|
3026
|
-
if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
|
|
3027
|
-
const seed = options.seed ?? 1;
|
|
3028
|
-
if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
|
|
3029
|
-
const calls = validateCalls(options.calls);
|
|
3030
|
-
const alternative = validateAlternative(options.alternative, calls);
|
|
3031
|
-
const rng = mulberry32(seed);
|
|
3032
|
-
let promotions = 0;
|
|
3033
|
-
let refusals = 0;
|
|
3034
|
-
const perCall = calls.map(() => ({
|
|
3035
|
-
promote: 0,
|
|
3036
|
-
insufficient: 0,
|
|
3037
|
-
indeterminate: 0,
|
|
3038
|
-
exactTestVetoes: 0,
|
|
3039
|
-
hold: 0,
|
|
3040
|
-
methods: /* @__PURE__ */ new Map()
|
|
3041
|
-
}));
|
|
3042
|
-
for (let s = 0; s < simulations; s++) {
|
|
3043
|
-
const pairs = drawPairs(alternative, n, rng);
|
|
3044
|
-
let all = true;
|
|
3045
|
-
let anyRefusal = false;
|
|
3046
|
-
for (let c = 0; c < calls.length; c++) {
|
|
3047
|
-
const call = calls[c];
|
|
3048
|
-
const decision = runCall(call, pairs, n);
|
|
3049
|
-
const tally = perCall[c];
|
|
3050
|
-
tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
|
|
3051
|
-
if (decision.promote) tally.promote++;
|
|
3052
|
-
else {
|
|
3053
|
-
if (!decision.sufficient) tally.insufficient++;
|
|
3054
|
-
if (decision.indeterminate) tally.indeterminate++;
|
|
3055
|
-
if (decision.exactTestVetoes) tally.exactTestVetoes++;
|
|
3056
|
-
if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
|
|
3057
|
-
}
|
|
3058
|
-
if (!decision.promote) all = false;
|
|
3059
|
-
if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
|
|
3060
|
-
}
|
|
3061
|
-
if (all) promotions++;
|
|
3062
|
-
else if (anyRefusal) refusals++;
|
|
3063
|
-
}
|
|
3064
|
-
const power = promotions / simulations;
|
|
3065
|
-
const interval = wilson(promotions, simulations, .95);
|
|
3066
|
-
return {
|
|
3067
|
-
n,
|
|
3068
|
-
simulations,
|
|
3069
|
-
seed,
|
|
3070
|
-
power,
|
|
3071
|
-
standardError: Math.sqrt(power * (1 - power) / simulations),
|
|
3072
|
-
low: interval.lower,
|
|
3073
|
-
high: interval.upper,
|
|
3074
|
-
refuse: refusals / simulations,
|
|
3075
|
-
hold: (simulations - promotions - refusals) / simulations,
|
|
3076
|
-
calls: perCall.map((tally, c) => ({
|
|
3077
|
-
outcome: calls[c].outcome,
|
|
3078
|
-
promote: tally.promote / simulations,
|
|
3079
|
-
insufficient: tally.insufficient / simulations,
|
|
3080
|
-
indeterminate: tally.indeterminate / simulations,
|
|
3081
|
-
exactTestVetoes: tally.exactTestVetoes / simulations,
|
|
3082
|
-
hold: tally.hold / simulations,
|
|
3083
|
-
methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
|
|
3084
|
-
}))
|
|
3085
|
-
};
|
|
3086
|
-
}
|
|
3087
|
-
/**
|
|
3088
|
-
* First n at which {@link pairedPromotionPower} reaches the target, scanning
|
|
3089
|
-
* every n from `minPairs` upward. Both the point-estimate answer and the
|
|
3090
|
-
* confirmed lower-bound answer are returned; the scan runs until the latter is
|
|
3091
|
-
* reached (so the curve past `n` is retained) or `maxPairs` is exhausted. A
|
|
3092
|
-
* candidate selected by the scan's bound is certified only by an independent
|
|
3093
|
-
* simulation at that n on a different seed; a candidate that fails it is
|
|
3094
|
-
* recorded and the scan continues.
|
|
3095
|
-
*/
|
|
3096
|
-
function requiredPairsForPairedPromotion(options) {
|
|
3097
|
-
const target = options.target ?? .8;
|
|
3098
|
-
if (!Number.isFinite(target) || target <= 0 || target >= 1) throw new Error(`requiredPairsForPairedPromotion: target must be in (0,1), got ${target}`);
|
|
3099
|
-
const minPairs = options.minPairs ?? 1;
|
|
3100
|
-
const maxPairs = options.maxPairs ?? 200;
|
|
3101
|
-
if (!Number.isInteger(minPairs) || minPairs < 1) throw new Error(`requiredPairsForPairedPromotion: minPairs must be a positive integer, got ${minPairs}`);
|
|
3102
|
-
if (!Number.isInteger(maxPairs) || maxPairs < minPairs) throw new Error(`requiredPairsForPairedPromotion: maxPairs must be an integer >= minPairs, got ${maxPairs}`);
|
|
3103
|
-
const seed = options.seed ?? 1;
|
|
3104
|
-
const curve = [];
|
|
3105
|
-
const rejected = [];
|
|
3106
|
-
let n = null;
|
|
3107
|
-
let nAtLowerBound = null;
|
|
3108
|
-
let confirmation = null;
|
|
3109
|
-
for (let candidate = minPairs; candidate <= maxPairs; candidate++) {
|
|
3110
|
-
const result = pairedPromotionPower({
|
|
3111
|
-
n: candidate,
|
|
3112
|
-
alternative: options.alternative,
|
|
3113
|
-
calls: options.calls,
|
|
3114
|
-
simulations: options.simulations,
|
|
3115
|
-
seed
|
|
3116
|
-
});
|
|
3117
|
-
curve.push(result);
|
|
3118
|
-
if (n === null && result.power >= target) n = candidate;
|
|
3119
|
-
if (result.low >= target) {
|
|
3120
|
-
const check = pairedPromotionPower({
|
|
3121
|
-
n: candidate,
|
|
3122
|
-
alternative: options.alternative,
|
|
3123
|
-
calls: options.calls,
|
|
3124
|
-
simulations: options.simulations,
|
|
3125
|
-
seed: confirmationSeed(seed, candidate)
|
|
3126
|
-
});
|
|
3127
|
-
if (check.low >= target) {
|
|
3128
|
-
nAtLowerBound = candidate;
|
|
3129
|
-
confirmation = check;
|
|
3130
|
-
break;
|
|
3131
|
-
}
|
|
3132
|
-
rejected.push(check);
|
|
3133
|
-
}
|
|
3134
|
-
}
|
|
3135
|
-
return {
|
|
3136
|
-
target,
|
|
3137
|
-
n,
|
|
3138
|
-
nAtLowerBound,
|
|
3139
|
-
confirmation,
|
|
3140
|
-
curve,
|
|
3141
|
-
rejected
|
|
3142
|
-
};
|
|
3143
|
-
}
|
|
3144
|
-
/** A seed for the confirmation at `n` that never equals the scan seed. */
|
|
3145
|
-
function confirmationSeed(seed, n) {
|
|
3146
|
-
return (Math.imul((seed | 0) ^ 2654435769, 2246822507) + Math.imul(n, 3266489909) | 1) >>> 0;
|
|
3147
|
-
}
|
|
3148
|
-
function drawPairs(alternative, n, rng) {
|
|
3149
|
-
const baseline = alternative.baseline ?? {
|
|
3150
|
-
kind: "point",
|
|
3151
|
-
value: 0
|
|
3152
|
-
};
|
|
3153
|
-
const control = new Array(n);
|
|
3154
|
-
const candidate = new Array(n);
|
|
3155
|
-
const controlPass = new Array(n);
|
|
3156
|
-
const candidatePass = new Array(n);
|
|
3157
|
-
for (let i = 0; i < n; i++) {
|
|
3158
|
-
const cell = drawCell(alternative.cells, rng);
|
|
3159
|
-
const base = drawValue(baseline, rng);
|
|
3160
|
-
control[i] = base;
|
|
3161
|
-
candidate[i] = base + drawValue(cell.delta, rng);
|
|
3162
|
-
controlPass[i] = cell.pass?.control ?? false;
|
|
3163
|
-
candidatePass[i] = cell.pass?.treatment ?? false;
|
|
3164
|
-
}
|
|
3165
|
-
return {
|
|
3166
|
-
control,
|
|
3167
|
-
candidate,
|
|
3168
|
-
controlPass,
|
|
3169
|
-
candidatePass
|
|
3170
|
-
};
|
|
3171
|
-
}
|
|
3172
|
-
function drawCell(cells, rng) {
|
|
3173
|
-
const u = rng();
|
|
3174
|
-
let cumulative = 0;
|
|
3175
|
-
for (const cell of cells) {
|
|
3176
|
-
cumulative += cell.probability;
|
|
3177
|
-
if (u < cumulative) return cell;
|
|
3178
|
-
}
|
|
3179
|
-
return cells[cells.length - 1];
|
|
3180
|
-
}
|
|
3181
|
-
function drawValue(law, rng) {
|
|
3182
|
-
switch (law.kind) {
|
|
3183
|
-
case "point": return law.value;
|
|
3184
|
-
case "atoms": {
|
|
3185
|
-
const u = rng();
|
|
3186
|
-
let cumulative = 0;
|
|
3187
|
-
for (const atom of law.atoms) {
|
|
3188
|
-
cumulative += atom.probability;
|
|
3189
|
-
if (u < cumulative) return atom.value;
|
|
3190
|
-
}
|
|
3191
|
-
return law.atoms[law.atoms.length - 1].value;
|
|
3192
|
-
}
|
|
3193
|
-
case "normal": {
|
|
3194
|
-
const u1 = 1 - rng();
|
|
3195
|
-
const u2 = rng();
|
|
3196
|
-
return law.mean + law.sd * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
|
3197
|
-
}
|
|
3198
|
-
}
|
|
3199
|
-
}
|
|
3200
|
-
function runCall(call, pairs, n) {
|
|
3201
|
-
const options = {
|
|
3202
|
-
minPairs: n,
|
|
3203
|
-
...call.options
|
|
3204
|
-
};
|
|
3205
|
-
if (call.outcome === "pass") {
|
|
3206
|
-
const scale = call.options.binaryScale;
|
|
3207
|
-
return decidePairedPromotion(pairs.controlPass.map((p) => p ? scale : 0), pairs.candidatePass.map((p) => p ? scale : 0), options);
|
|
3208
|
-
}
|
|
3209
|
-
return decidePairedPromotion(pairs.control, pairs.candidate, options);
|
|
3210
|
-
}
|
|
3211
|
-
function validateCalls(calls) {
|
|
3212
|
-
if (!Array.isArray(calls) || calls.length === 0) throw new Error("pairedPromotionPower: at least one call is required");
|
|
3213
|
-
calls.forEach((call, i) => {
|
|
3214
|
-
if (call.outcome !== "delta" && call.outcome !== "pass") throw new Error(`pairedPromotionPower: calls[${i}].outcome must be 'delta' or 'pass'`);
|
|
3215
|
-
if (call.options === null || typeof call.options !== "object") throw new Error(`pairedPromotionPower: calls[${i}].options must be an object`);
|
|
3216
|
-
if (call.options.minPairs !== void 0 && (!Number.isInteger(call.options.minPairs) || call.options.minPairs < 1)) throw new Error(`pairedPromotionPower: calls[${i}].options.minPairs must be a positive integer`);
|
|
3217
|
-
if (call.outcome === "pass") {
|
|
3218
|
-
const scale = call.options.binaryScale;
|
|
3219
|
-
if (scale === void 0 || !Number.isFinite(scale) || scale <= 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call and must declare a finite positive options.binaryScale`);
|
|
3220
|
-
if (call.options.continuous) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; continuous cannot accompany it`);
|
|
3221
|
-
if (call.options.statistic === "median") throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; binaryScale requires the mean statistic, not median`);
|
|
3222
|
-
} else if (call.options.binaryScale !== void 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'delta' call and cannot declare binaryScale; use a 'pass' call for a two-point outcome`);
|
|
3223
|
-
});
|
|
3224
|
-
return calls;
|
|
3225
|
-
}
|
|
3226
|
-
function validateAlternative(alternative, calls) {
|
|
3227
|
-
if (!alternative || !Array.isArray(alternative.cells) || alternative.cells.length === 0) throw new Error("pairedPromotionPower: alternative.cells must be a non-empty array");
|
|
3228
|
-
const needsPass = calls.some((call) => call.outcome === "pass");
|
|
3229
|
-
let total = 0;
|
|
3230
|
-
alternative.cells.forEach((cell, i) => {
|
|
3231
|
-
if (!Number.isFinite(cell.probability) || cell.probability < 0) throw new Error(`pairedPromotionPower: cells[${i}].probability must be a finite number >= 0`);
|
|
3232
|
-
total += cell.probability;
|
|
3233
|
-
if (needsPass) {
|
|
3234
|
-
if (!cell.pass || typeof cell.pass.control !== "boolean" || typeof cell.pass.treatment !== "boolean") throw new Error(`pairedPromotionPower: cells[${i}].pass must give control and treatment booleans when a 'pass' call is configured`);
|
|
3235
|
-
}
|
|
3236
|
-
validateLaw(cell.delta, `cells[${i}].delta`);
|
|
3237
|
-
});
|
|
3238
|
-
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: cell probabilities must sum to 1, got ${total}`);
|
|
3239
|
-
if (alternative.baseline !== void 0) validateLaw(alternative.baseline, "baseline");
|
|
3240
|
-
return alternative;
|
|
3241
|
-
}
|
|
3242
|
-
function validateLaw(law, where) {
|
|
3243
|
-
if (!law || typeof law !== "object") throw new Error(`pairedPromotionPower: ${where} must be a law object`);
|
|
3244
|
-
switch (law.kind) {
|
|
3245
|
-
case "point":
|
|
3246
|
-
if (!Number.isFinite(law.value)) throw new Error(`pairedPromotionPower: ${where}.value must be finite`);
|
|
3247
|
-
return;
|
|
3248
|
-
case "atoms": {
|
|
3249
|
-
if (!Array.isArray(law.atoms) || law.atoms.length === 0) throw new Error(`pairedPromotionPower: ${where}.atoms must be a non-empty array`);
|
|
3250
|
-
let total = 0;
|
|
3251
|
-
law.atoms.forEach((atom, i) => {
|
|
3252
|
-
if (!Number.isFinite(atom.value)) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].value must be finite`);
|
|
3253
|
-
if (!Number.isFinite(atom.probability) || atom.probability < 0) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].probability must be a finite number >= 0`);
|
|
3254
|
-
total += atom.probability;
|
|
3255
|
-
});
|
|
3256
|
-
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: ${where}.atoms probabilities must sum to 1, got ${total}`);
|
|
3257
|
-
return;
|
|
3258
|
-
}
|
|
3259
|
-
case "normal":
|
|
3260
|
-
if (!Number.isFinite(law.mean) || !Number.isFinite(law.sd) || law.sd < 0) throw new Error(`pairedPromotionPower: ${where} needs a finite mean and a finite sd >= 0`);
|
|
3261
|
-
return;
|
|
3262
|
-
default: throw new Error(`pairedPromotionPower: ${where}.kind must be 'point', 'atoms' or 'normal'`);
|
|
3263
|
-
}
|
|
3264
|
-
}
|
|
3265
|
-
//#endregion
|
|
3266
|
-
//#region src/campaign/search-claim.ts
|
|
3267
|
-
/**
|
|
3268
|
-
* The claim: the one statement a search makes about its nodes, made once, on
|
|
3269
|
-
* its sealed test split, after the search stops expanding.
|
|
3270
|
-
*
|
|
3271
|
-
* `planSearchClaim` fixes the claim's design from the ledger before any test
|
|
3272
|
-
* cell runs: up to 3 finalists ranked by selection mean, the estimator the
|
|
3273
|
-
* paired decision will use, the number of finalists the family-wise confidence
|
|
3274
|
-
* is divided among, and a power check on selection data. The kernel stores the
|
|
3275
|
-
* plan on the ledger's `claim` operation, so the ledger proves the design was
|
|
3276
|
-
* fixed before the test data existed. `decideSearchClaim` then reads the test
|
|
3277
|
-
* cells of the root and the finalists, which ran together, and decides each
|
|
3278
|
-
* finalist against the root with `decidePairedPromotion` at the Bonferroni
|
|
3279
|
-
* confidence `1 - 0.05 / k`.
|
|
3280
|
-
*
|
|
3281
|
-
* Every number is a pure function of the ledger: the power simulation is
|
|
3282
|
-
* seeded from the search's seed and the decisions from the digest of the test
|
|
3283
|
-
* cells they read, so a verifier with the same ledger gets the same claim.
|
|
3284
|
-
*/
|
|
3285
|
-
const FAMILY_CONFIDENCE = .95;
|
|
3286
|
-
const TARGET_POWER = .8;
|
|
3287
|
-
const POWER_SIMULATIONS = 500;
|
|
3288
|
-
const RESAMPLES = 2e3;
|
|
3289
|
-
/**
|
|
3290
|
-
* Every rule and parameter the claim depends on. Its digest is the claim's
|
|
3291
|
-
* revision; change a value here whenever the procedure changes, so the
|
|
3292
|
-
* revision moves with it.
|
|
3293
|
-
*/
|
|
3294
|
-
const CLAIM_DEFINITION = {
|
|
3295
|
-
name: "tangle.search-claim.2026-09",
|
|
3296
|
-
finalists: `up to 3 non-root nodes not decided invalid that scored every selection unit and beat the root's mean on the selection units they share, ranked by selection mean in the objective's direction, then registration order`,
|
|
3297
|
-
familyConfidence: FAMILY_CONFIDENCE,
|
|
3298
|
-
correction: "Bonferroni: each of k finalists against the root at 1 - (1 - familyConfidence) / k",
|
|
3299
|
-
finalistCount: "the largest k whose test cells the unspent claim reserve and headroom cover and whose power reaches the target",
|
|
3300
|
-
estimator: "binary with scale s when every train, selection and test unit has one task at one repeat and every pre-test score is 0 or s; otherwise continuous",
|
|
3301
|
-
power: {
|
|
3302
|
-
method: "pairedPromotionPower of the claim decision, simulated at the claim minimumEffect",
|
|
3303
|
-
continuous: "per-unit improvement normal with the pooled selection variance",
|
|
3304
|
-
binary: "discordant units at the pooled selection variance over 2 s^2 per side, shifted by minimumEffect / s; concordant units pass at the root selection pass rate",
|
|
3305
|
-
targetPower: TARGET_POWER,
|
|
3306
|
-
simulations: POWER_SIMULATIONS,
|
|
3307
|
-
seed: "the first 32 bits of hashCanonical([search seed, k])"
|
|
3308
|
-
},
|
|
3309
|
-
decision: {
|
|
3310
|
-
call: "decidePairedPromotion(root, finalist) on per-unit test means in the objective direction",
|
|
3311
|
-
threshold: 0,
|
|
3312
|
-
minPairs: "every test unit",
|
|
3313
|
-
resamples: RESAMPLES,
|
|
3314
|
-
seed: "the first 32 bits of the finalist's test cellSetDigest"
|
|
3315
|
-
},
|
|
3316
|
-
select: "the promoted finalist with the largest improvement; ties to the better selection rank",
|
|
3317
|
-
ship: "a promoted finalist that scored every test unit, with the root, on held-out units, under a pinned judge"
|
|
3318
|
-
};
|
|
3319
|
-
/** The claim procedure every `SearchClaim` and claim plan names. */
|
|
3320
|
-
const SEARCH_CLAIM_RULE = {
|
|
3321
|
-
uri: "npm:@tangle-network/agent-eval#searchClaim",
|
|
3322
|
-
revision: hashCanonical(CLAIM_DEFINITION)
|
|
3323
|
-
};
|
|
3324
|
-
/** The rule name decisions made by the claim record. */
|
|
3325
|
-
const SEARCH_CLAIM_RULE_NAME = CLAIM_DEFINITION.name;
|
|
3326
|
-
/**
|
|
3327
|
-
* The claim reserve a search needs: the root and 3 finalists on every test
|
|
3328
|
-
* task at `reps` repeats, each held at `cellUsd`. Declare at least this as
|
|
3329
|
-
* `budget.reservedClaimUsd`; `runSearch` refuses to start a capped search
|
|
3330
|
-
* with less.
|
|
3331
|
-
*/
|
|
3332
|
-
function searchClaimReserveUsd(input) {
|
|
3333
|
-
return 4 * input.testTasks * input.reps * input.cellUsd;
|
|
3334
|
-
}
|
|
3335
|
-
/** Fix the claim's design from a search that stopped expanding. */
|
|
3336
|
-
function planSearchClaim(state, input) {
|
|
3337
|
-
const header = requireHeader(state);
|
|
3338
|
-
const root = state.rootNodeId;
|
|
3339
|
-
const plan = (finalists, power, test, reason) => ({
|
|
3340
|
-
kind: "search-claim-plan",
|
|
3341
|
-
rule: SEARCH_CLAIM_RULE,
|
|
3342
|
-
stopReason: input.stopReason,
|
|
3343
|
-
finalists,
|
|
3344
|
-
power,
|
|
3345
|
-
test,
|
|
3346
|
-
reason
|
|
3347
|
-
});
|
|
3348
|
-
const selectionUnits = unitIds(header.splits.selection.tasks);
|
|
3349
|
-
const testUnits = unitIds(header.splits.test.tasks);
|
|
3350
|
-
if (testUnits.length === 0) return plan([], { unknown: "the search has no test split" }, "hold", "the search has no test split, so it makes no claim");
|
|
3351
|
-
if (selectionUnits.length === 0) return plan([], { unknown: "the search has no selection split to choose finalists on" }, "hold", "the search has no selection split, so no finalist can be chosen without the test data");
|
|
3352
|
-
const candidates = rankFinalists(state, selectionUnits);
|
|
3353
|
-
if (candidates.length === 0) return plan([], { unknown: "no node went to test" }, "hold", "no node scored every selection unit and beat the root's selection mean, so the search keeps the root");
|
|
3354
|
-
const minimumEffect = header.objective.claim.minimumEffect;
|
|
3355
|
-
if (minimumEffect === void 0) return plan(candidates.slice(0, 1), { unknown: "the claim declares no minimumEffect" }, "test-cannot-resolve", "the claim declares no minimumEffect, so no test split can be shown to resolve it");
|
|
3356
|
-
const { pooledVariance } = searchPosterior(state, { split: "selection" });
|
|
3357
|
-
if (pooledVariance === null) return plan(candidates.slice(0, 1), { unknown: "no node shares 2 selection units with the root, so the variance is unknown" }, "test-cannot-resolve", "the selection split gives no variance for the power check, so the test cannot be shown to resolve the minimum effect");
|
|
3358
|
-
const estimator = claimEstimator(state, input.reps);
|
|
3359
|
-
const rootPassRate = passRate(state.unitScores(root, "selection"), estimator);
|
|
3360
|
-
let best = null;
|
|
3361
|
-
let budgetBound = false;
|
|
3362
|
-
for (let k = candidates.length; k >= 1; k--) {
|
|
3363
|
-
const power = claimPower({
|
|
3364
|
-
units: testUnits.length,
|
|
3365
|
-
finalists: k,
|
|
3366
|
-
minimumEffect,
|
|
3367
|
-
pooledVariance,
|
|
3368
|
-
estimator,
|
|
3369
|
-
rootPassRate,
|
|
3370
|
-
seed: header.policy.seed
|
|
3371
|
-
});
|
|
3372
|
-
best = power;
|
|
3373
|
-
if (!power.adequate) continue;
|
|
3374
|
-
if (!input.affordable(k)) {
|
|
3375
|
-
budgetBound = true;
|
|
3376
|
-
continue;
|
|
3377
|
-
}
|
|
3378
|
-
return plan(candidates.slice(0, k), power, "run", `${k} finalist${k === 1 ? "" : "s"} go to test: power ${round(power.powerAtMinimumEffect)} at an improvement of ${minimumEffect} on ${testUnits.length} test units reaches ${TARGET_POWER} at confidence ${round(bonferroni(k))} each`);
|
|
3379
|
-
}
|
|
3380
|
-
const power = best;
|
|
3381
|
-
return plan(candidates.slice(0, 1), power, "test-cannot-resolve", budgetBound ? "the unspent claim reserve and headroom do not cover the test cells of the root and one finalist" : `even one finalist reaches power ${round("adequate" in power ? power.powerAtMinimumEffect : 0)} at an improvement of ${minimumEffect} on ${testUnits.length} test units, below ${TARGET_POWER}`);
|
|
3382
|
-
}
|
|
3383
|
-
/**
|
|
3384
|
-
* Decide the claim from the test cells of the root and the finalists. Call it
|
|
3385
|
-
* once every claim cell is settled or cancelled. It returns the claim and the
|
|
3386
|
-
* terminal decisions of the root and every finalist.
|
|
3387
|
-
*/
|
|
3388
|
-
function decideSearchClaim(state, plan) {
|
|
3389
|
-
const header = requireHeader(state);
|
|
3390
|
-
const root = state.rootNodeId;
|
|
3391
|
-
const rootScored = [
|
|
3392
|
-
"train",
|
|
3393
|
-
"selection",
|
|
3394
|
-
"test"
|
|
3395
|
-
].some((split) => state.scoredCells(root, split).length > 0);
|
|
3396
|
-
const keepRoot = rootScored ? root : null;
|
|
3397
|
-
const rule = SEARCH_CLAIM_RULE_NAME;
|
|
3398
|
-
const decisions = [];
|
|
3399
|
-
const claimOf = (finalists, selected, decision, reason) => ({
|
|
3400
|
-
rule: SEARCH_CLAIM_RULE,
|
|
3401
|
-
confidence: FAMILY_CONFIDENCE,
|
|
3402
|
-
power: plan.power,
|
|
3403
|
-
finalists,
|
|
3404
|
-
selected,
|
|
3405
|
-
decision,
|
|
3406
|
-
reason
|
|
3407
|
-
});
|
|
3408
|
-
if (plan.test !== "run") {
|
|
3409
|
-
for (const finalist of plan.finalists) decisions.push({
|
|
3410
|
-
nodeId: finalist.nodeId,
|
|
3411
|
-
decision: { status: "rejected" },
|
|
3412
|
-
basis: null,
|
|
3413
|
-
rule,
|
|
3414
|
-
reason: `no test cell ran: ${plan.reason}`
|
|
3415
|
-
});
|
|
3416
|
-
decisions.push(rootDecision(root, rootScored ? "kept" : "unscored", rule, plan.reason));
|
|
3417
|
-
return {
|
|
3418
|
-
claim: claimOf(plan.finalists.map((finalist) => ({
|
|
3419
|
-
nodeId: finalist.nodeId,
|
|
3420
|
-
estimate: null,
|
|
3421
|
-
test: null,
|
|
3422
|
-
promote: false
|
|
3423
|
-
})), keepRoot, plan.test, plan.reason),
|
|
3424
|
-
decisions
|
|
3425
|
-
};
|
|
3426
|
-
}
|
|
3427
|
-
const power = plan.power;
|
|
3428
|
-
if (!("adequate" in power)) throw new Error("a claim that runs its test cells has a known power");
|
|
3429
|
-
const k = plan.finalists.length;
|
|
3430
|
-
const confidence = bonferroni(k);
|
|
3431
|
-
const testUnits = unitIds(header.splits.test.tasks);
|
|
3432
|
-
const maximize = header.objective.direction === "maximize";
|
|
3433
|
-
const rootUnits = state.unitScores(root, "test");
|
|
3434
|
-
const tested = plan.finalists.map(({ nodeId }) => {
|
|
3435
|
-
const estimate = estimateNode(state, nodeId, {
|
|
3436
|
-
against: root,
|
|
3437
|
-
split: "test"
|
|
3438
|
-
});
|
|
3439
|
-
return {
|
|
3440
|
-
nodeId,
|
|
3441
|
-
estimate,
|
|
3442
|
-
...pairedTest({
|
|
3443
|
-
root: rootUnits,
|
|
3444
|
-
node: state.unitScores(nodeId, "test"),
|
|
3445
|
-
testUnits,
|
|
3446
|
-
confidence,
|
|
3447
|
-
estimator: power.estimator,
|
|
3448
|
-
maximize,
|
|
3449
|
-
seed: seedFromDigest(estimate.cellSetDigest)
|
|
3450
|
-
})
|
|
3451
|
-
};
|
|
3452
|
-
});
|
|
3453
|
-
const winner = tested.map((entry, rank) => ({
|
|
3454
|
-
...entry,
|
|
3455
|
-
rank
|
|
3456
|
-
})).filter((entry) => entry.promote).sort((a, b) => b.improvement - a.improvement || a.rank - b.rank)[0] ?? null;
|
|
3457
|
-
const blockers = [];
|
|
3458
|
-
if (!header.splits.heldOutUnits) blockers.push("the test units also appear in train or selection");
|
|
3459
|
-
if ("unknown" in header.objective.judge) blockers.push("the judge is not pinned");
|
|
3460
|
-
const ship = winner !== null && blockers.length === 0;
|
|
3461
|
-
const selected = ship ? winner.nodeId : keepRoot;
|
|
3462
|
-
const reason = ship ? `finalist ${winner.nodeId} beat the root on the ${testUnits.length} test units at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})` : winner !== null ? `finalist ${winner.nodeId} beat the root, but the claim cannot ship: ${blockers.join("; ")}` : `no finalist beat the root on the test split at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})`;
|
|
3463
|
-
for (const entry of tested) decisions.push({
|
|
3464
|
-
nodeId: entry.nodeId,
|
|
3465
|
-
decision: { status: ship && entry.nodeId === winner.nodeId ? "selected" : "rejected" },
|
|
3466
|
-
basis: entry.estimate,
|
|
3467
|
-
rule,
|
|
3468
|
-
reason: ship && entry.nodeId === winner.nodeId ? reason : entry.promote ? ship ? `promoted, but finalist ${winner.nodeId} improved more on the test split` : reason : entry.why
|
|
3469
|
-
});
|
|
3470
|
-
decisions.push(rootDecision(root, ship ? "replaced" : rootScored ? "kept" : "unscored", rule, reason));
|
|
3471
|
-
return {
|
|
3472
|
-
claim: claimOf(tested.map((entry) => ({
|
|
3473
|
-
nodeId: entry.nodeId,
|
|
3474
|
-
estimate: entry.estimate,
|
|
3475
|
-
test: entry.test,
|
|
3476
|
-
promote: entry.promote
|
|
3477
|
-
})), selected, ship ? "ship" : "hold", reason),
|
|
3478
|
-
decisions
|
|
3479
|
-
};
|
|
3480
|
-
}
|
|
3481
|
-
/**
|
|
3482
|
-
* Make a closed search's claim again from its ledger alone and compare it with
|
|
3483
|
-
* the recorded one, byte for byte: the power check, the finalists, each
|
|
3484
|
-
* finalist's test estimate and deciding interval, the selection, the decision
|
|
3485
|
-
* and the reason. A producer's claim is not evidence until this holds; a store
|
|
3486
|
-
* keeps what this function derives, not what the producer wrote.
|
|
3487
|
-
*
|
|
3488
|
-
* It needs no blob, so it works on a store that keeps digests only. The
|
|
3489
|
-
* finalists are the nodes the ledger decided `finalist`, in decision order.
|
|
3490
|
-
* The one input the ledger cannot replay is whether the budget covered k
|
|
3491
|
-
* finalists' test cells; the claim cells that ran answer it (none ran: the
|
|
3492
|
-
* budget covered none). Test repeats come from the claim cells too; without
|
|
3493
|
-
* test cells, the recorded estimator stands for them (binary needs one repeat,
|
|
3494
|
-
* so any other count reproduces continuous).
|
|
3495
|
-
*
|
|
3496
|
-
* Returns null for a search that made no claim.
|
|
3497
|
-
*/
|
|
3498
|
-
function verifySearchClaim(state) {
|
|
3499
|
-
const closed = state.closed;
|
|
3500
|
-
if (!closed) throw new Error(`search ${state.searchId} is open; its claim is made when it closes`);
|
|
3501
|
-
const recorded = closed.claim;
|
|
3502
|
-
if (recorded === null) return null;
|
|
3503
|
-
if (recorded.rule.revision !== SEARCH_CLAIM_RULE.revision) return {
|
|
3504
|
-
status: "unknown",
|
|
3505
|
-
reason: `the claim names rule revision ${recorded.rule.revision}; this code implements ${SEARCH_CLAIM_RULE.revision}`
|
|
3506
|
-
};
|
|
3507
|
-
const claimCells = state.cells().filter((cell) => cell.stage === "claim");
|
|
3508
|
-
const decided = state.nodes().flatMap((node) => {
|
|
3509
|
-
const first = node.decisions.find((record) => record.decision.status === "finalist");
|
|
3510
|
-
return first ? [{
|
|
3511
|
-
nodeId: node.nodeId,
|
|
3512
|
-
sequence: first.sequence
|
|
3513
|
-
}] : [];
|
|
3514
|
-
}).sort((a, b) => a.sequence - b.sequence).map((entry) => entry.nodeId);
|
|
3515
|
-
const reps = claimCells.length > 0 ? 1 + Math.max(...claimCells.map((cell) => cell.rep)) : "adequate" in recorded.power && recorded.power.estimator.kind === "binary" ? 1 : 2;
|
|
3516
|
-
const plan = planSearchClaim(state, {
|
|
3517
|
-
stopReason: closed.reason,
|
|
3518
|
-
reps,
|
|
3519
|
-
affordable: (finalists) => claimCells.length > 0 && finalists <= decided.length
|
|
3520
|
-
});
|
|
3521
|
-
const differences = [];
|
|
3522
|
-
const planned = plan.finalists.map((finalist) => finalist.nodeId);
|
|
3523
|
-
if (canonicalString(planned) !== canonicalString(decided)) differences.push(`finalists: the rule picks [${planned.join(", ")}], the ledger decided [${decided.join(", ")}]`);
|
|
3524
|
-
if (plan.test === "run" !== claimCells.length > 0) differences.push(`test: the plan ${plan.test === "run" ? "runs" : "skips"} the test cells, and ${claimCells.length} ran`);
|
|
3525
|
-
const expected = decideSearchClaim(state, plan).claim;
|
|
3526
|
-
for (const key of [
|
|
3527
|
-
"decision",
|
|
3528
|
-
"selected",
|
|
3529
|
-
"confidence",
|
|
3530
|
-
"power",
|
|
3531
|
-
"reason"
|
|
3532
|
-
]) if (canonicalString(expected[key]) !== canonicalString(recorded[key])) differences.push(`${key}: the ledger supports ${canonicalString(expected[key])}, the claim records ${canonicalString(recorded[key])}`);
|
|
3533
|
-
const count = Math.max(expected.finalists.length, recorded.finalists.length);
|
|
3534
|
-
for (let index = 0; index < count; index++) {
|
|
3535
|
-
const want = expected.finalists[index];
|
|
3536
|
-
const have = recorded.finalists[index];
|
|
3537
|
-
if (canonicalString(want ?? null) === canonicalString(have ?? null)) continue;
|
|
3538
|
-
const fields = [
|
|
3539
|
-
"nodeId",
|
|
3540
|
-
"promote",
|
|
3541
|
-
"test",
|
|
3542
|
-
"estimate"
|
|
3543
|
-
].filter((field) => canonicalString(want?.[field] ?? null) !== canonicalString(have?.[field] ?? null));
|
|
3544
|
-
differences.push(`finalist ${index + 1} (${have?.nodeId ?? want?.nodeId}): ${fields.join(", ")} differ from what the ledger supports`);
|
|
3545
|
-
}
|
|
3546
|
-
return differences.length === 0 ? { status: "verified" } : {
|
|
3547
|
-
status: "mismatch",
|
|
3548
|
-
differences
|
|
3549
|
-
};
|
|
3550
|
-
}
|
|
3551
|
-
/** Per-finalist confidence for `k` finalists. */
|
|
3552
|
-
function bonferroni(k) {
|
|
3553
|
-
return 1 - (1 - FAMILY_CONFIDENCE) / k;
|
|
3554
|
-
}
|
|
3555
|
-
function rootDecision(root, fate, rule, reason) {
|
|
3556
|
-
return {
|
|
3557
|
-
nodeId: root,
|
|
3558
|
-
decision: { status: fate === "kept" ? "selected" : "rejected" },
|
|
3559
|
-
basis: null,
|
|
3560
|
-
rule,
|
|
3561
|
-
reason: fate === "kept" ? `the search keeps the root: ${reason}` : fate === "replaced" ? `a finalist replaces the root: ${reason}` : "the root has no scored cell, so the search keeps nothing"
|
|
3562
|
-
};
|
|
3563
|
-
}
|
|
3564
|
-
/** Nodes eligible to go to test, best first, at most `SEARCH_CLAIM_MAX_FINALISTS`. */
|
|
3565
|
-
function rankFinalists(state, selectionUnits) {
|
|
3566
|
-
const sign = requireHeader(state).objective.direction === "maximize" ? 1 : -1;
|
|
3567
|
-
const root = state.rootNodeId;
|
|
3568
|
-
const rootMeans = new Map(state.unitScores(root, "selection").map((unit) => [unit.unitId, unit.mean]));
|
|
3569
|
-
const ranked = [];
|
|
3570
|
-
for (const node of state.nodes()) {
|
|
3571
|
-
if (node.nodeId === root || node.status === "invalid") continue;
|
|
3572
|
-
const units = state.unitScores(node.nodeId, "selection");
|
|
3573
|
-
if (units.length !== selectionUnits.length) continue;
|
|
3574
|
-
let gain = 0;
|
|
3575
|
-
let shared = 0;
|
|
3576
|
-
for (const unit of units) {
|
|
3577
|
-
const rootMean = rootMeans.get(unit.unitId);
|
|
3578
|
-
if (rootMean === void 0) continue;
|
|
3579
|
-
gain += sign * (unit.mean - rootMean);
|
|
3580
|
-
shared += 1;
|
|
3581
|
-
}
|
|
3582
|
-
if (shared === 0 || gain <= 0) continue;
|
|
3583
|
-
ranked.push({
|
|
3584
|
-
nodeId: node.nodeId,
|
|
3585
|
-
selectionMean: mean$2(units.map((unit) => unit.mean)),
|
|
3586
|
-
ordinal: node.ordinal
|
|
3587
|
-
});
|
|
3588
|
-
}
|
|
3589
|
-
ranked.sort((a, b) => sign * (b.selectionMean - a.selectionMean) || a.ordinal - b.ordinal);
|
|
3590
|
-
return ranked.slice(0, 3).map(({ nodeId, selectionMean }) => ({
|
|
3591
|
-
nodeId,
|
|
3592
|
-
selectionMean
|
|
3593
|
-
}));
|
|
3594
|
-
}
|
|
3595
|
-
/**
|
|
3596
|
-
* The paired decision's estimator, fixed from pre-test data. A per-unit mean
|
|
3597
|
-
* is binary only when each unit holds one task at one repeat and every score
|
|
3598
|
-
* seen before the test is 0 or one positive value.
|
|
3599
|
-
*/
|
|
3600
|
-
function claimEstimator(state, reps) {
|
|
3601
|
-
const header = requireHeader(state);
|
|
3602
|
-
if (![
|
|
3603
|
-
"train",
|
|
3604
|
-
"selection",
|
|
3605
|
-
"test"
|
|
3606
|
-
].every((split) => {
|
|
3607
|
-
const tasks = header.splits[split].tasks;
|
|
3608
|
-
return unitIds(tasks).length === tasks.length;
|
|
3609
|
-
}) || reps !== 1) return { kind: "continuous" };
|
|
3610
|
-
let scale = null;
|
|
3611
|
-
for (const nodeId of state.nodeIds()) for (const split of ["train", "selection"]) for (const cell of state.scoredCells(nodeId, split)) {
|
|
3612
|
-
if (cell.score === 0) continue;
|
|
3613
|
-
if (cell.score < 0 || scale !== null && cell.score !== scale) return { kind: "continuous" };
|
|
3614
|
-
scale = cell.score;
|
|
3615
|
-
}
|
|
3616
|
-
return scale === null ? { kind: "continuous" } : {
|
|
3617
|
-
kind: "binary",
|
|
3618
|
-
scale
|
|
3619
|
-
};
|
|
3620
|
-
}
|
|
3621
|
-
function passRate(units, estimator) {
|
|
3622
|
-
if (estimator.kind !== "binary" || units.length === 0) return 0;
|
|
3623
|
-
return units.filter((unit) => unit.mean === estimator.scale).length / units.length;
|
|
3624
|
-
}
|
|
3625
|
-
function claimPower(input) {
|
|
3626
|
-
const { units, finalists, minimumEffect, pooledVariance, estimator } = input;
|
|
3627
|
-
const confidence = bonferroni(finalists);
|
|
3628
|
-
let alternative;
|
|
3629
|
-
let call;
|
|
3630
|
-
if (estimator.kind === "binary") {
|
|
3631
|
-
const s = estimator.scale;
|
|
3632
|
-
const shift = Math.min(1, minimumEffect / s);
|
|
3633
|
-
const discordance = Math.min(.5, pooledVariance / (2 * s * s));
|
|
3634
|
-
const loss = Math.max(0, discordance - shift / 2);
|
|
3635
|
-
const win = Math.min(1 - loss, loss + shift);
|
|
3636
|
-
const concordant = Math.max(0, 1 - win - loss);
|
|
3637
|
-
alternative = { cells: [
|
|
3638
|
-
{
|
|
3639
|
-
probability: win,
|
|
3640
|
-
pass: {
|
|
3641
|
-
control: false,
|
|
3642
|
-
treatment: true
|
|
3643
|
-
},
|
|
3644
|
-
delta: {
|
|
3645
|
-
kind: "point",
|
|
3646
|
-
value: s
|
|
3647
|
-
}
|
|
3648
|
-
},
|
|
3649
|
-
{
|
|
3650
|
-
probability: loss,
|
|
3651
|
-
pass: {
|
|
3652
|
-
control: true,
|
|
3653
|
-
treatment: false
|
|
3654
|
-
},
|
|
3655
|
-
delta: {
|
|
3656
|
-
kind: "point",
|
|
3657
|
-
value: -s
|
|
3658
|
-
}
|
|
3659
|
-
},
|
|
3660
|
-
{
|
|
3661
|
-
probability: concordant * input.rootPassRate,
|
|
3662
|
-
pass: {
|
|
3663
|
-
control: true,
|
|
3664
|
-
treatment: true
|
|
3665
|
-
},
|
|
3666
|
-
delta: {
|
|
3667
|
-
kind: "point",
|
|
3668
|
-
value: 0
|
|
3669
|
-
}
|
|
3670
|
-
},
|
|
3671
|
-
{
|
|
3672
|
-
probability: concordant * (1 - input.rootPassRate),
|
|
3673
|
-
pass: {
|
|
3674
|
-
control: false,
|
|
3675
|
-
treatment: false
|
|
3676
|
-
},
|
|
3677
|
-
delta: {
|
|
3678
|
-
kind: "point",
|
|
3679
|
-
value: 0
|
|
3680
|
-
}
|
|
3681
|
-
}
|
|
3682
|
-
] };
|
|
3683
|
-
call = {
|
|
3684
|
-
outcome: "pass",
|
|
3685
|
-
options: {
|
|
3686
|
-
binaryScale: s,
|
|
3687
|
-
confidence,
|
|
3688
|
-
threshold: 0
|
|
3689
|
-
}
|
|
3690
|
-
};
|
|
3691
|
-
} else {
|
|
3692
|
-
alternative = { cells: [{
|
|
3693
|
-
probability: 1,
|
|
3694
|
-
delta: {
|
|
3695
|
-
kind: "normal",
|
|
3696
|
-
mean: minimumEffect,
|
|
3697
|
-
sd: Math.sqrt(pooledVariance)
|
|
3698
|
-
}
|
|
3699
|
-
}] };
|
|
3700
|
-
call = {
|
|
3701
|
-
outcome: "delta",
|
|
3702
|
-
options: {
|
|
3703
|
-
continuous: true,
|
|
3704
|
-
confidence,
|
|
3705
|
-
threshold: 0,
|
|
3706
|
-
resamples: RESAMPLES
|
|
3707
|
-
}
|
|
3708
|
-
};
|
|
3709
|
-
}
|
|
3710
|
-
const simulated = pairedPromotionPower({
|
|
3711
|
-
n: units,
|
|
3712
|
-
alternative,
|
|
3713
|
-
calls: [call],
|
|
3714
|
-
simulations: POWER_SIMULATIONS,
|
|
3715
|
-
seed: seedFromDigest(hashCanonical([input.seed, finalists]))
|
|
3716
|
-
});
|
|
3717
|
-
return {
|
|
3718
|
-
adequate: simulated.power >= TARGET_POWER,
|
|
3719
|
-
minimumEffect,
|
|
3720
|
-
powerAtMinimumEffect: simulated.power,
|
|
3721
|
-
targetPower: TARGET_POWER,
|
|
3722
|
-
units,
|
|
3723
|
-
finalists,
|
|
3724
|
-
pooledVariance,
|
|
3725
|
-
estimator
|
|
3726
|
-
};
|
|
3727
|
-
}
|
|
3728
|
-
/**
|
|
3729
|
-
* One finalist against the root on the test split, in the objective's
|
|
3730
|
-
* direction, requiring every test unit scored by both.
|
|
3731
|
-
*/
|
|
3732
|
-
function pairedTest(input) {
|
|
3733
|
-
const rootMeans = new Map(input.root.map((unit) => [unit.unitId, unit.mean]));
|
|
3734
|
-
const before = [];
|
|
3735
|
-
const after = [];
|
|
3736
|
-
for (const unit of input.node) {
|
|
3737
|
-
const rootMean = rootMeans.get(unit.unitId);
|
|
3738
|
-
if (rootMean === void 0) continue;
|
|
3739
|
-
before.push(input.maximize ? rootMean : unit.mean);
|
|
3740
|
-
after.push(input.maximize ? unit.mean : rootMean);
|
|
3741
|
-
}
|
|
3742
|
-
const pairs = before.length;
|
|
3743
|
-
if (pairs === 0) return {
|
|
3744
|
-
test: null,
|
|
3745
|
-
promote: false,
|
|
3746
|
-
improvement: null,
|
|
3747
|
-
why: "no test unit was scored by both it and the root"
|
|
3748
|
-
};
|
|
3749
|
-
const { estimator } = input;
|
|
3750
|
-
if (estimator.kind === "binary" && [...before, ...after].some((value) => value !== 0 && value !== estimator.scale)) return {
|
|
3751
|
-
test: null,
|
|
3752
|
-
promote: false,
|
|
3753
|
-
improvement: null,
|
|
3754
|
-
why: `a test score left the two-point scale {0, ${estimator.scale}} the claim fixed before the test`
|
|
3755
|
-
};
|
|
3756
|
-
const decision = decidePairedPromotion(before, after, {
|
|
3757
|
-
confidence: input.confidence,
|
|
3758
|
-
threshold: 0,
|
|
3759
|
-
minPairs: input.testUnits.length,
|
|
3760
|
-
resamples: RESAMPLES,
|
|
3761
|
-
seed: input.seed,
|
|
3762
|
-
...estimator.kind === "binary" ? { binaryScale: estimator.scale } : { continuous: true }
|
|
3763
|
-
});
|
|
3764
|
-
const test = {
|
|
3765
|
-
pairs,
|
|
3766
|
-
confidence: input.confidence,
|
|
3767
|
-
method: decision.method,
|
|
3768
|
-
delta: plain(input.maximize ? decision.delta : -decision.delta),
|
|
3769
|
-
interval: input.maximize ? [plain(decision.low), plain(decision.high)] : [plain(-decision.high), plain(-decision.low)]
|
|
3770
|
-
};
|
|
3771
|
-
const complete = pairs === input.testUnits.length;
|
|
3772
|
-
const promote = decision.promote && complete;
|
|
3773
|
-
const why = !complete ? `it and the root scored ${pairs} of ${input.testUnits.length} test units, and a claim needs every one` : decision.promote ? "promoted" : !decision.sufficient ? `${pairs} test units are below the ${decision.minimumPairs} its paired test needs` : decision.indeterminate ? `its test interval has no width: ${decision.indeterminateCause}` : decision.exactTestVetoes ? "McNemar's exact test vetoes the improvement" : `its improvement interval [${round(decision.low)}, ${round(decision.high)}] at confidence ${round(input.confidence)} does not exclude 0`;
|
|
3774
|
-
return {
|
|
3775
|
-
test,
|
|
3776
|
-
promote,
|
|
3777
|
-
improvement: decision.delta,
|
|
3778
|
-
why
|
|
3779
|
-
};
|
|
3780
|
-
}
|
|
3781
|
-
function requireHeader(state) {
|
|
3782
|
-
const header = state.header;
|
|
3783
|
-
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
3784
|
-
if (state.rootNodeId === null) throw new Error(`search ${state.searchId} has no root node`);
|
|
3785
|
-
return header;
|
|
3786
|
-
}
|
|
3787
|
-
function unitIds(tasks) {
|
|
3788
|
-
return [...new Set(tasks.map((task) => task.unitId))];
|
|
3789
|
-
}
|
|
3790
|
-
function mean$2(values) {
|
|
3791
|
-
let total = 0;
|
|
3792
|
-
for (const value of values) total += value;
|
|
3793
|
-
return total / values.length;
|
|
3794
|
-
}
|
|
3795
|
-
function round(value) {
|
|
3796
|
-
return Math.round(value * 1e4) / 1e4;
|
|
3797
|
-
}
|
|
3798
|
-
/** Canonical JSON has one zero; never hand it a negative one. */
|
|
3799
|
-
function plain(value) {
|
|
3800
|
-
return value === 0 ? 0 : value;
|
|
3801
|
-
}
|
|
3802
|
-
/** The kernel every `runSearch` ledger names as its search implementation. */
|
|
3803
|
-
const SEARCH_KERNEL_SOURCE = {
|
|
3804
|
-
uri: "npm:@tangle-network/agent-eval#runSearch",
|
|
3805
|
-
revision: hashCanonical({
|
|
3806
|
-
name: "tangle.search-kernel.2026-09",
|
|
3807
|
-
dispatch: "claim, then rung and root, then screen, then train; first allocated first within a stage",
|
|
3808
|
-
expansion: "when no cell waits, fewer than twice the lanes capacity run, the cap admits one proposal and its expected screens, and overspend has not taken the search past its cap",
|
|
3809
|
-
reservation: {
|
|
3810
|
-
hard: "the lane per-cell maximum",
|
|
3811
|
-
estimate: "1.5 times the p99 of the lane settled cells once 20 settled, else the lane prior"
|
|
3812
|
-
},
|
|
3813
|
-
retries: "a retryable errored attempt runs again, up to maxAttempts",
|
|
3814
|
-
rungs: "when a node finishes its cells, the allocator names the nodes it advances, deepest rung first; each is recorded advanced and its rung allocated when the cap admits the rung",
|
|
3815
|
-
resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
|
|
3816
|
-
writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
|
|
3817
|
-
divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
|
|
3818
|
-
claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
|
|
3819
|
-
close: "the allocator prunes the nodes left waiting on a rung, except the policy leader; then, with a test split, the claim decides; without one, the policy leader is selected when it has a scored cell; every other undecided node is rejected",
|
|
3820
|
-
complete: "a node leads only when it has a scored cell on the ranked split and no cell there ended unscored: a final unscored outcome, or an error with no attempt left"
|
|
3821
|
-
})
|
|
3822
|
-
};
|
|
3823
|
-
const USD_TOLERANCE = 1e-9;
|
|
3824
|
-
/** The operation that holds the claim's plan. */
|
|
3825
|
-
const CLAIM_OPERATION = "claim";
|
|
3826
|
-
/** Settled cells a lane needs before its own cost distribution sets its estimate. */
|
|
3827
|
-
const ESTIMATE_FROM_CELLS = 20;
|
|
3828
|
-
const ESTIMATE_MARGIN = 1.5;
|
|
3829
|
-
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
3830
|
-
/** Dispatch order: claim cells first, then rungs and the root, then screens, then train. */
|
|
3831
|
-
const STAGE_PRIORITY = {
|
|
3832
|
-
claim: 0,
|
|
3833
|
-
rung: 1,
|
|
3834
|
-
root: 1,
|
|
3835
|
-
screen: 2,
|
|
3836
|
-
train: 3,
|
|
3837
|
-
external: 3
|
|
3838
|
-
};
|
|
3839
|
-
/** Run a search to its close, or continue one from its ledger. */
|
|
3840
|
-
async function runSearch(options) {
|
|
3841
|
-
return new SearchKernel(options).run();
|
|
3842
|
-
}
|
|
3843
|
-
/** The expansion index a kernel operation id encodes, or null. */
|
|
3844
|
-
function searchExpansionIndex(operationId) {
|
|
3845
|
-
const match = /^expand-(\d+)$/.exec(operationId);
|
|
3846
|
-
return match ? Number(match[1]) : null;
|
|
3847
|
-
}
|
|
3848
|
-
var SearchKernel = class {
|
|
3849
|
-
options;
|
|
3850
|
-
recorder;
|
|
3851
|
-
maxAttempts;
|
|
3852
|
-
now;
|
|
3853
|
-
state;
|
|
3854
|
-
lanes = /* @__PURE__ */ new Map();
|
|
3855
|
-
inFlight = /* @__PURE__ */ new Map();
|
|
3856
|
-
proposal = null;
|
|
3857
|
-
proposalController = null;
|
|
3858
|
-
completions = [];
|
|
3859
|
-
wake = null;
|
|
3860
|
-
/** Unfinished cells per node. */
|
|
3861
|
-
pending = /* @__PURE__ */ new Map();
|
|
3862
|
-
/** Admitted nodes whose screen finished, in registration order. */
|
|
3863
|
-
screened = [];
|
|
3864
|
-
screenedSet = /* @__PURE__ */ new Set();
|
|
3865
|
-
admitted = /* @__PURE__ */ new Set();
|
|
3866
|
-
expansionOf = /* @__PURE__ */ new Map();
|
|
3867
|
-
artifacts = /* @__PURE__ */ new Map();
|
|
3868
|
-
/** Children a proposal returned, by artifact digest, until they are registered. */
|
|
3869
|
-
proposed = /* @__PURE__ */ new Map();
|
|
3870
|
-
/** Cells the ledger showed unsettled at start: an earlier process may hold their attempt. */
|
|
3871
|
-
adoptable = /* @__PURE__ */ new Set();
|
|
3872
|
-
operationCosts = [];
|
|
3873
|
-
/** Operations started: the next one's id. */
|
|
3874
|
-
expansions = 0;
|
|
3875
|
-
/** Proposals that completed, by operation index. A proposal an interrupted
|
|
3876
|
-
* process lost counts toward none of the stop rules, so a resumed search
|
|
3877
|
-
* makes the proposals the uninterrupted one would have made. */
|
|
3878
|
-
completed = [];
|
|
3879
|
-
stopReason = null;
|
|
3880
|
-
failure = null;
|
|
3881
|
-
/** The claim's design once expansion stopped; the search then only claims. */
|
|
3882
|
-
claimPlan = null;
|
|
3883
|
-
constructor(options) {
|
|
3884
|
-
this.options = options;
|
|
3885
|
-
this.recorder = options.recorder;
|
|
3886
|
-
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
3887
|
-
this.now = options.now ?? Date.now;
|
|
3888
|
-
if (!Number.isSafeInteger(this.maxAttempts) || this.maxAttempts < 1) throw new TypeError("runSearch: maxAttempts must be a positive integer");
|
|
3889
|
-
const { maxExpansions } = options;
|
|
3890
|
-
if (maxExpansions !== void 0 && (!Number.isSafeInteger(maxExpansions) || maxExpansions < 0)) throw new TypeError("runSearch: maxExpansions must be a non-negative integer");
|
|
3891
|
-
for (const lane of options.executor.lanes()) {
|
|
3892
|
-
if (!Number.isSafeInteger(lane.capacity) || lane.capacity < 1) throw new TypeError(`runSearch: lane ${lane.name} needs a positive integer capacity`);
|
|
3893
|
-
if (!Number.isFinite(lane.cellUsd) || lane.cellUsd < 0) throw new TypeError(`runSearch: lane ${lane.name} needs a finite non-negative cellUsd`);
|
|
3894
|
-
if (this.lanes.has(lane.name)) throw new TypeError(`runSearch: lane ${lane.name} is declared twice`);
|
|
3895
|
-
this.lanes.set(lane.name, {
|
|
3896
|
-
...lane,
|
|
3897
|
-
inFlight: 0,
|
|
3898
|
-
queues: [
|
|
3899
|
-
[],
|
|
3900
|
-
[],
|
|
3901
|
-
[],
|
|
3902
|
-
[]
|
|
3903
|
-
],
|
|
3904
|
-
settledCosts: []
|
|
3905
|
-
});
|
|
3906
|
-
}
|
|
3907
|
-
if (this.lanes.size === 0) throw new TypeError("runSearch: the executor declares no lane");
|
|
3908
|
-
}
|
|
3909
|
-
async run() {
|
|
3910
|
-
const { ledger } = this.recorder;
|
|
3911
|
-
const lock = ledger instanceof FileSearchLedger ? acquireSingleRunLock({ lockPath: `${ledger.path}.run.lock` }) : null;
|
|
3912
|
-
try {
|
|
3913
|
-
return await this.runLocked();
|
|
3914
|
-
} finally {
|
|
3915
|
-
lock?.release();
|
|
3916
|
-
}
|
|
3917
|
-
}
|
|
3918
|
-
async runLocked() {
|
|
3919
|
-
await this.refresh();
|
|
3920
|
-
const header = this.state.header;
|
|
3921
|
-
if (!header) throw new Error(`runSearch: search ${this.recorder.searchId} has not been opened`);
|
|
3922
|
-
if (header.policy.expansion !== this.options.policy.name || header.policy.allocation !== this.options.allocation.name) throw new Error(`runSearch: search ${this.recorder.searchId} was opened with policy ${header.policy.expansion} and allocation ${header.policy.allocation}, not ${this.options.policy.name} and ${this.options.allocation.name}`);
|
|
3923
|
-
if (this.state.closed) return this.closedResult();
|
|
3924
|
-
await this.restore();
|
|
3925
|
-
const { signal } = this.options;
|
|
3926
|
-
const onAbort = () => {
|
|
3927
|
-
for (const controller of this.inFlight.values()) controller.abort(signal?.reason);
|
|
3928
|
-
this.proposalController?.abort(signal?.reason);
|
|
3929
|
-
this.wakeUp();
|
|
3930
|
-
};
|
|
3931
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
3932
|
-
try {
|
|
3933
|
-
if (this.claimPlan) await this.applyClaimPlan(this.claimPlan);
|
|
3934
|
-
await this.loop();
|
|
3935
|
-
if (!this.failure && !this.claimPlan && this.state.header.splits.test.tasks.length > 0) {
|
|
3936
|
-
await this.beginClaim();
|
|
3937
|
-
await this.loop();
|
|
3938
|
-
}
|
|
3939
|
-
} finally {
|
|
3940
|
-
signal?.removeEventListener("abort", onAbort);
|
|
3941
|
-
}
|
|
3942
|
-
if (this.failure) throw this.failure.error;
|
|
3943
|
-
await this.close();
|
|
3944
|
-
return this.closedResult();
|
|
3945
|
-
}
|
|
3946
|
-
async restore() {
|
|
3947
|
-
const rootInput = this.options.codec.node(this.recorder, this.options.root);
|
|
3948
|
-
const rootId = this.state.rootNodeId;
|
|
3949
|
-
if (rootId === null) {
|
|
3950
|
-
const { nodeId } = await this.recorder.registerNode(rootInput);
|
|
3951
|
-
await this.recorder.recordEdge({
|
|
3952
|
-
childNodeId: nodeId,
|
|
3953
|
-
parents: [],
|
|
3954
|
-
operator: "seed",
|
|
3955
|
-
attribution: "explicit",
|
|
3956
|
-
proposer: null,
|
|
3957
|
-
proposalKey: "root",
|
|
3958
|
-
rationale: { unknown: "the root is the caller-supplied starting artifact" },
|
|
3959
|
-
diffs: [],
|
|
3960
|
-
label: "root"
|
|
3961
|
-
});
|
|
3962
|
-
await this.refresh();
|
|
3963
|
-
} else if (this.state.node(rootId).artifactDigest !== rootInput.artifactDigest) throw new Error(`runSearch: search ${this.recorder.searchId} started from another root artifact; a changed start is a derived search`);
|
|
3964
|
-
const root = this.state.rootNodeId;
|
|
3965
|
-
this.artifacts.set(root, this.options.root);
|
|
3966
|
-
this.expansionOf.set(root, -1);
|
|
3967
|
-
if (this.state.audit.cells.allocated === 0) this.assertClaimReady();
|
|
3968
|
-
await this.restoreClaimPlan();
|
|
3969
|
-
const recorded = [];
|
|
3970
|
-
for (let expansion = 0;; expansion++) {
|
|
3971
|
-
const operationId = `expand-${expansion}`;
|
|
3972
|
-
const operation = this.state.operation(operationId);
|
|
3973
|
-
if (!operation) break;
|
|
3974
|
-
this.expansions = expansion + 1;
|
|
3975
|
-
if (operation.recorded) {
|
|
3976
|
-
this.operationCosts.push(operation.spentUsd);
|
|
3977
|
-
recorded.push(expansion);
|
|
3978
|
-
if (operation.outcome === "completed") this.completed.push(expansion);
|
|
3979
|
-
continue;
|
|
3980
|
-
}
|
|
3981
|
-
await this.recorder.recordOperation({
|
|
3982
|
-
operationId,
|
|
3983
|
-
operationKind: "candidate-generation",
|
|
3984
|
-
execution: this.options.proposer.execution,
|
|
3985
|
-
outcome: {
|
|
3986
|
-
status: "failed",
|
|
3987
|
-
failure: {
|
|
3988
|
-
code: "interrupted",
|
|
3989
|
-
message: "the search stopped while this proposal ran; its output was lost"
|
|
3990
|
-
}
|
|
3991
|
-
},
|
|
3992
|
-
accounting: interruptedAccounting()
|
|
3993
|
-
});
|
|
3994
|
-
await this.refresh();
|
|
3995
|
-
}
|
|
3996
|
-
for (const node of this.state.nodes()) {
|
|
3997
|
-
if (node.edgeIds.length === 0) continue;
|
|
3998
|
-
const edge = this.state.edge(node.edgeIds[0]);
|
|
3999
|
-
const expansion = edge.proposer?.operationId ? searchExpansionIndex(edge.proposer.operationId) : null;
|
|
4000
|
-
this.expansionOf.set(node.nodeId, expansion ?? -1);
|
|
4001
|
-
if (node.status !== "invalid") this.admitted.add(node.nodeId);
|
|
4002
|
-
}
|
|
4003
|
-
for (const cell of this.state.cells()) {
|
|
4004
|
-
const lane = cell.lane === null ? void 0 : this.lanes.get(cell.lane);
|
|
4005
|
-
if (cell.final && lane) lane.settledCosts.push(cell.spentUsd);
|
|
4006
|
-
if (this.cellDone(cell)) continue;
|
|
4007
|
-
if (!lane) throw new Error(`runSearch: cell ${cell.cellId} was placed on lane ${String(cell.lane)}, which the executor does not declare`);
|
|
4008
|
-
this.adoptable.add(cell.cellId);
|
|
4009
|
-
this.enqueue(cell);
|
|
4010
|
-
if (cell.stage !== "claim") this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
|
|
4011
|
-
}
|
|
4012
|
-
if (this.claimPlan) return;
|
|
4013
|
-
for (const expansion of recorded) await this.finishProposal(expansion);
|
|
4014
|
-
for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
|
|
4015
|
-
for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) await this.screenDone(nodeId);
|
|
4016
|
-
await this.advance();
|
|
4017
|
-
}
|
|
4018
|
-
/**
|
|
4019
|
-
* A search that will claim must be able to: finalists come from a selection
|
|
4020
|
-
* split, the power check needs the claim's minimum effect, a ship needs a
|
|
4021
|
-
* pinned judge, and under a cap the claim reserve must cover the root and 3
|
|
4022
|
-
* finalists on every test task while the rest of the cap covers one
|
|
4023
|
-
* screening round. Checked before the first cell, so a search that cannot
|
|
4024
|
-
* claim spends nothing.
|
|
4025
|
-
*/
|
|
4026
|
-
assertClaimReady() {
|
|
4027
|
-
const header = this.state.header;
|
|
4028
|
-
const testTasks = header.splits.test.tasks.length;
|
|
4029
|
-
if (testTasks === 0) return;
|
|
4030
|
-
const refuse = (why) => {
|
|
4031
|
-
throw new Error(`runSearch: search ${this.recorder.searchId} has a test split, but ${why}`);
|
|
4032
|
-
};
|
|
4033
|
-
if (header.splits.selection.tasks.length === 0) refuse("no selection split to choose its finalists on");
|
|
4034
|
-
if (header.objective.claim.minimumEffect === void 0) refuse("its claim declares no minimumEffect for the power check");
|
|
4035
|
-
if ("unknown" in header.objective.judge) refuse("its judge is not pinned; a claim with an unknown judge cannot ship, and a resumed search could mix two judges");
|
|
4036
|
-
const { maxUsd, reservedClaimUsd } = header.budget;
|
|
4037
|
-
if (maxUsd === null) return;
|
|
4038
|
-
const cellUsd = this.maxCellReservation();
|
|
4039
|
-
const need = searchClaimReserveUsd({
|
|
4040
|
-
testTasks,
|
|
4041
|
-
reps: this.options.allocation.reps,
|
|
4042
|
-
cellUsd
|
|
4043
|
-
});
|
|
4044
|
-
if (reservedClaimUsd + USD_TOLERANCE < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
|
|
4045
|
-
const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId, 0).length;
|
|
4046
|
-
const screens = (this.options.proposer.childrenPerProposal ?? 1) * this.options.allocation.screenSize(this.state);
|
|
4047
|
-
const round = this.operationReservation().usd + (rootCells + screens) * cellUsd;
|
|
4048
|
-
if (maxUsd - reservedClaimUsd + USD_TOLERANCE < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
|
|
4049
|
-
}
|
|
4050
|
-
/** A claim an earlier process began: continue it from its stored plan. */
|
|
4051
|
-
async restoreClaimPlan() {
|
|
4052
|
-
const operation = this.state.operation(CLAIM_OPERATION);
|
|
4053
|
-
if (!operation) return;
|
|
4054
|
-
const ref = operation.artifacts.find((artifact) => artifact.role === "claim-plan");
|
|
4055
|
-
if (!ref) throw new Error(`runSearch: the ${CLAIM_OPERATION} operation has no stored plan`);
|
|
4056
|
-
const plan = this.recorder.readBlob(ref);
|
|
4057
|
-
if (!operation.recorded) await this.recordClaimOperation();
|
|
4058
|
-
this.claimPlan = plan;
|
|
4059
|
-
this.stopReason = plan.stopReason;
|
|
4060
|
-
}
|
|
4061
|
-
async loop() {
|
|
4062
|
-
for (;;) {
|
|
4063
|
-
if (this.failure || this.options.signal?.aborted) {
|
|
4064
|
-
await this.drain(this.failure?.error ?? this.options.signal?.reason);
|
|
4065
|
-
this.failure ??= { error: this.options.signal?.reason ?? /* @__PURE__ */ new Error("aborted") };
|
|
4066
|
-
return;
|
|
4067
|
-
}
|
|
4068
|
-
if (this.pastDeadline()) {
|
|
4069
|
-
this.stopReason ??= "deadline";
|
|
4070
|
-
if (this.queuedCount() > 0) await this.cancelQueued("deadline");
|
|
4071
|
-
}
|
|
4072
|
-
this.dispatch();
|
|
4073
|
-
const blocked = this.stopReason === null && this.proposal === null ? await this.tryExpand() : null;
|
|
4074
|
-
if (!(this.inFlight.size > 0 || this.proposal !== null)) {
|
|
4075
|
-
if (this.queuedCount() > 0) continue;
|
|
4076
|
-
this.stopReason ??= blocked ?? "converged";
|
|
4077
|
-
return;
|
|
4078
|
-
}
|
|
4079
|
-
await this.handle(await this.next());
|
|
4080
|
-
}
|
|
4081
|
-
}
|
|
4082
|
-
/** The next completion, or null when an abort woke the loop without one. */
|
|
4083
|
-
async next() {
|
|
4084
|
-
if (this.completions.length === 0) await new Promise((resolve) => {
|
|
4085
|
-
this.wake = resolve;
|
|
4086
|
-
});
|
|
4087
|
-
return this.completions.shift() ?? null;
|
|
4088
|
-
}
|
|
4089
|
-
push(completion) {
|
|
4090
|
-
this.completions.push(completion);
|
|
4091
|
-
this.wakeUp();
|
|
4092
|
-
}
|
|
4093
|
-
wakeUp() {
|
|
4094
|
-
const wake = this.wake;
|
|
4095
|
-
this.wake = null;
|
|
4096
|
-
wake?.();
|
|
4097
|
-
}
|
|
4098
|
-
async handle(completion) {
|
|
4099
|
-
if (completion === null) return;
|
|
4100
|
-
if (completion.kind === "cell") await this.settle(completion);
|
|
4101
|
-
else await this.recordProposal(completion);
|
|
4102
|
-
}
|
|
4103
|
-
/** Abort everything in flight and record what still completes. */
|
|
4104
|
-
async drain(reason) {
|
|
4105
|
-
for (const controller of this.inFlight.values()) controller.abort(reason);
|
|
4106
|
-
this.proposalController?.abort(reason);
|
|
4107
|
-
while (this.inFlight.size > 0 || this.proposal !== null) await this.handle(await this.next());
|
|
4108
|
-
}
|
|
4109
|
-
/** Fill free lane slots with queued cells. A queued cell was admitted when
|
|
4110
|
-
* it was allocated, so starting it needs no further decision. */
|
|
4111
|
-
dispatch() {
|
|
4112
|
-
if (this.failure || this.options.signal?.aborted) return;
|
|
4113
|
-
const claimOnly = this.pastDeadline();
|
|
4114
|
-
const cap = this.state.header.budget.maxConcurrency;
|
|
4115
|
-
for (const lane of this.lanes.values()) while (lane.inFlight < lane.capacity && (cap === null || this.inFlight.size < cap)) {
|
|
4116
|
-
const cellId = (claimOnly ? lane.queues[STAGE_PRIORITY.claim] : lane.queues.find((waiting) => waiting.length > 0))?.shift();
|
|
4117
|
-
if (cellId === void 0) break;
|
|
4118
|
-
this.start(cellId, lane);
|
|
4119
|
-
}
|
|
4120
|
-
}
|
|
4121
|
-
start(cellId, lane) {
|
|
4122
|
-
const cell = this.state.cell(cellId);
|
|
4123
|
-
const attempt = cell.attempts + 1;
|
|
4124
|
-
const controller = new AbortController();
|
|
4125
|
-
const work = {
|
|
4126
|
-
searchId: this.recorder.searchId,
|
|
4127
|
-
cellId,
|
|
4128
|
-
nodeId: cell.nodeId,
|
|
4129
|
-
taskId: cell.taskId,
|
|
4130
|
-
unitId: cell.unitId,
|
|
4131
|
-
split: cell.split,
|
|
4132
|
-
rep: cell.rep,
|
|
4133
|
-
stage: cell.stage,
|
|
4134
|
-
lane: lane.name,
|
|
4135
|
-
attempt,
|
|
4136
|
-
runId: `${cellId}:${attempt}`,
|
|
4137
|
-
artifact: this.artifact(cell.nodeId),
|
|
4138
|
-
signal: controller.signal
|
|
4139
|
-
};
|
|
4140
|
-
const adopt = this.adoptable.delete(cellId);
|
|
4141
|
-
this.inFlight.set(cellId, controller);
|
|
4142
|
-
lane.inFlight += 1;
|
|
4143
|
-
const { executor } = this.options;
|
|
4144
|
-
(async () => (adopt ? await executor.adopt(work) : null) ?? await executor.run(work))().then((result) => this.push({
|
|
4145
|
-
kind: "cell",
|
|
4146
|
-
cellId,
|
|
4147
|
-
attempt,
|
|
4148
|
-
result
|
|
4149
|
-
}), (error) => this.push({
|
|
4150
|
-
kind: "cell",
|
|
4151
|
-
cellId,
|
|
4152
|
-
attempt,
|
|
4153
|
-
error
|
|
4154
|
-
}));
|
|
4155
|
-
}
|
|
4156
|
-
async settle(completion) {
|
|
4157
|
-
const { cellId, attempt } = completion;
|
|
4158
|
-
this.inFlight.delete(cellId);
|
|
4159
|
-
const before = this.state.cell(cellId);
|
|
4160
|
-
const lane = this.lanes.get(before.lane);
|
|
4161
|
-
lane.inFlight -= 1;
|
|
4162
|
-
if (!completion.result) {
|
|
4163
|
-
this.failure ??= { error: completion.error };
|
|
4164
|
-
return;
|
|
4165
|
-
}
|
|
4166
|
-
this.dispatch();
|
|
4167
|
-
if ((this.failure || this.options.signal?.aborted) && completion.result.outcome.status === "errored") return;
|
|
4168
|
-
await this.recorder.settleCell({
|
|
4169
|
-
...completion.result,
|
|
4170
|
-
cellId,
|
|
4171
|
-
attempt
|
|
4172
|
-
});
|
|
4173
|
-
await this.refresh();
|
|
4174
|
-
const cell = this.state.cell(cellId);
|
|
4175
|
-
if (cell.final) lane.settledCosts.push(cell.spentUsd);
|
|
4176
|
-
if (!this.cellDone(cell)) {
|
|
4177
|
-
if (this.failure || this.options.signal?.aborted) return;
|
|
4178
|
-
this.enqueue(cell);
|
|
4179
|
-
return;
|
|
4180
|
-
}
|
|
4181
|
-
if (cell.stage === "claim") return;
|
|
4182
|
-
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
4183
|
-
this.pending.set(cell.nodeId, left);
|
|
4184
|
-
if (left === 0) await this.onNodeIdle(cell.nodeId);
|
|
4185
|
-
}
|
|
4186
|
-
/** Every cell allocated to the node is done: its screen, or a rung. */
|
|
4187
|
-
async onNodeIdle(nodeId) {
|
|
4188
|
-
if (!await this.screenDone(nodeId)) return;
|
|
4189
|
-
await this.advance();
|
|
4190
|
-
}
|
|
4191
|
-
/** Record the rank decisions the allocator now supports and allocate the
|
|
4192
|
-
* rungs they open. Past the deadline, once the claim began, or once the
|
|
4193
|
-
* search fails or is aborted, no rung opens; a search that stopped
|
|
4194
|
-
* expanding still measures the nodes that earned it. */
|
|
4195
|
-
async advance() {
|
|
4196
|
-
if (this.claimPlan || this.failure || this.options.signal?.aborted || this.pastDeadline()) return;
|
|
4197
|
-
for (const decision of this.options.allocation.advance(this.allocationView())) await this.allocateFor(decision.nodeId, decision);
|
|
4198
|
-
}
|
|
4199
|
-
/**
|
|
4200
|
-
* A node's allocated cells all finished. A node that diverges (its train
|
|
4201
|
-
* mean rose over its parent's while its selection interval against the
|
|
4202
|
-
* parent lies wholly on the worse side) is decided `invalid` and never
|
|
4203
|
-
* becomes a parent; any other node is screened. Returns whether it was.
|
|
4204
|
-
*/
|
|
4205
|
-
async screenDone(nodeId) {
|
|
4206
|
-
if (!this.claimPlan && !this.screenedSet.has(nodeId) && this.state.node(nodeId).status !== "advanced") {
|
|
4207
|
-
const divergence = searchDivergence(this.state, nodeId);
|
|
4208
|
-
if (divergence) {
|
|
4209
|
-
await this.recorder.decideNode({
|
|
4210
|
-
nodeId,
|
|
4211
|
-
decision: { status: "invalid" },
|
|
4212
|
-
basis: divergence.basis,
|
|
4213
|
-
rule: "divergence",
|
|
4214
|
-
reason: divergence.reason
|
|
4215
|
-
});
|
|
4216
|
-
await this.refresh();
|
|
4217
|
-
this.admitted.delete(nodeId);
|
|
4218
|
-
return false;
|
|
4219
|
-
}
|
|
4220
|
-
}
|
|
4221
|
-
this.markScreened(nodeId);
|
|
4222
|
-
return true;
|
|
4223
|
-
}
|
|
4224
|
-
markScreened(nodeId) {
|
|
4225
|
-
if (this.screenedSet.has(nodeId)) return;
|
|
4226
|
-
this.screenedSet.add(nodeId);
|
|
4227
|
-
const ordinal = this.state.node(nodeId).ordinal;
|
|
4228
|
-
let index = this.screened.length;
|
|
4229
|
-
while (index > 0 && this.state.node(this.screened[index - 1]).ordinal > ordinal) index--;
|
|
4230
|
-
this.screened = [
|
|
4231
|
-
...this.screened.slice(0, index),
|
|
4232
|
-
nodeId,
|
|
4233
|
-
...this.screened.slice(index)
|
|
4234
|
-
];
|
|
4235
|
-
}
|
|
4236
|
-
/**
|
|
4237
|
-
* Allocate the cells the allocator plans for a node through its rung and the
|
|
4238
|
-
* ledger lacks. With a rank decision, the rung is the decision's, and the
|
|
4239
|
-
* decision is recorded first, only once the cap admits the rung's cells.
|
|
4240
|
-
* Returns false when the cap or `maxCells` refused them.
|
|
4241
|
-
*/
|
|
4242
|
-
async allocateFor(nodeId, decision) {
|
|
4243
|
-
const rung = decision?.decision.status === "advanced" ? decision.decision.rung : this.state.node(nodeId).rung ?? 0;
|
|
4244
|
-
const plans = this.options.allocation.plan(this.state, nodeId, rung).filter((plan) => !this.state.cell(searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep)));
|
|
4245
|
-
if (plans.length === 0) {
|
|
4246
|
-
if (decision) await this.decide(decision);
|
|
4247
|
-
return true;
|
|
4248
|
-
}
|
|
4249
|
-
const placed = plans.map((plan) => {
|
|
4250
|
-
const laneName = this.options.executor.place({
|
|
4251
|
-
...plan,
|
|
4252
|
-
nodeId
|
|
4253
|
-
});
|
|
4254
|
-
const lane = this.lanes.get(laneName);
|
|
4255
|
-
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
4256
|
-
return {
|
|
4257
|
-
plan,
|
|
4258
|
-
lane,
|
|
4259
|
-
reservation: this.cellReservation(lane)
|
|
4260
|
-
};
|
|
4261
|
-
});
|
|
4262
|
-
const hold = placed.reduce((sum, { reservation }) => sum + reservation.usd, 0);
|
|
4263
|
-
const { headroomUsd } = this.state.budget;
|
|
4264
|
-
if (headroomUsd !== null && hold > headroomUsd + USD_TOLERANCE) return false;
|
|
4265
|
-
const { maxCells } = this.state.header.budget;
|
|
4266
|
-
if (maxCells !== null && this.state.audit.cells.allocated + placed.length > maxCells) return false;
|
|
4267
|
-
if (decision) await this.decide(decision);
|
|
4268
|
-
for (const { plan, lane, reservation } of placed) await this.recorder.allocateCell({
|
|
4269
|
-
nodeId,
|
|
4270
|
-
taskId: plan.taskId,
|
|
4271
|
-
split: plan.split,
|
|
4272
|
-
rep: plan.rep,
|
|
4273
|
-
stage: plan.stage,
|
|
4274
|
-
lane: lane.name,
|
|
4275
|
-
reservation
|
|
4276
|
-
});
|
|
4277
|
-
await this.refresh();
|
|
4278
|
-
for (const { plan } of placed) {
|
|
4279
|
-
const cellId = searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep);
|
|
4280
|
-
this.enqueue(this.state.cell(cellId));
|
|
4281
|
-
}
|
|
4282
|
-
this.pending.set(nodeId, (this.pending.get(nodeId) ?? 0) + placed.length);
|
|
4283
|
-
this.dispatch();
|
|
4284
|
-
return true;
|
|
4285
|
-
}
|
|
4286
|
-
async decide(decision) {
|
|
4287
|
-
await this.recorder.decideNode(decision);
|
|
4288
|
-
await this.refresh();
|
|
4289
|
-
}
|
|
4290
|
-
enqueue(cell) {
|
|
4291
|
-
this.lanes.get(cell.lane).queues[STAGE_PRIORITY[cell.stage]].push(cell.cellId);
|
|
4292
|
-
}
|
|
4293
|
-
queuedCount() {
|
|
4294
|
-
let count = 0;
|
|
4295
|
-
for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
|
|
4296
|
-
return count;
|
|
4297
|
-
}
|
|
4298
|
-
/** Fix the claim's design, store it, and start its test cells. */
|
|
4299
|
-
async beginClaim() {
|
|
4300
|
-
await this.refresh();
|
|
4301
|
-
const stopReason = this.stopReason ?? "converged";
|
|
4302
|
-
const header = this.state.header;
|
|
4303
|
-
const reps = this.options.allocation.reps;
|
|
4304
|
-
const plan = planSearchClaim(this.state, {
|
|
4305
|
-
stopReason,
|
|
4306
|
-
reps,
|
|
4307
|
-
affordable: (finalists) => {
|
|
4308
|
-
const placed = this.claimCells(finalists, this.state.rootNodeId);
|
|
4309
|
-
const hold = placed.reduce((sum, cell) => sum + cell.reservation.usd, 0);
|
|
4310
|
-
const { headroomUsd, claimReserveUsd } = this.state.budget;
|
|
4311
|
-
const { maxCells } = header.budget;
|
|
4312
|
-
return (headroomUsd === null || hold <= headroomUsd + claimReserveUsd + USD_TOLERANCE) && (maxCells === null || this.state.audit.cells.allocated + placed.length <= maxCells);
|
|
4313
|
-
}
|
|
4314
|
-
});
|
|
4315
|
-
await this.recorder.startOperation({
|
|
4316
|
-
operationId: CLAIM_OPERATION,
|
|
4317
|
-
operationKind: "selection",
|
|
4318
|
-
reservation: null,
|
|
4319
|
-
artifacts: [this.recorder.blob("claim-plan", plan)]
|
|
4320
|
-
});
|
|
4321
|
-
await this.recordClaimOperation();
|
|
4322
|
-
this.claimPlan = plan;
|
|
4323
|
-
this.stopReason = stopReason;
|
|
4324
|
-
await this.applyClaimPlan(plan);
|
|
4325
|
-
}
|
|
4326
|
-
async recordClaimOperation() {
|
|
4327
|
-
await this.recorder.recordOperation({
|
|
4328
|
-
operationId: CLAIM_OPERATION,
|
|
4329
|
-
operationKind: "selection",
|
|
4330
|
-
execution: {
|
|
4331
|
-
kind: "deterministic",
|
|
4332
|
-
source: SEARCH_CLAIM_RULE
|
|
4333
|
-
},
|
|
4334
|
-
outcome: { status: "completed" },
|
|
4335
|
-
accounting: {
|
|
4336
|
-
tokens: {
|
|
4337
|
-
status: "known",
|
|
4338
|
-
inputTokens: 0,
|
|
4339
|
-
outputTokens: 0,
|
|
4340
|
-
cachedTokens: 0
|
|
4341
|
-
},
|
|
4342
|
-
cost: {
|
|
4343
|
-
status: "known",
|
|
4344
|
-
usd: 0,
|
|
4345
|
-
source: "free"
|
|
4346
|
-
}
|
|
4347
|
-
}
|
|
4348
|
-
});
|
|
4349
|
-
await this.refresh();
|
|
4350
|
-
}
|
|
4351
|
-
/**
|
|
4352
|
-
* Decide the plan's finalists and allocate the test cells of the root and
|
|
4353
|
-
* the finalists, interleaved by task so the arms of one task run side by
|
|
4354
|
-
* side. Allocation finishes before any claim cell starts. Idempotent.
|
|
4355
|
-
*/
|
|
4356
|
-
async applyClaimPlan(plan) {
|
|
4357
|
-
const root = this.state.rootNodeId;
|
|
4358
|
-
const bases = plan.finalists.map((finalist) => estimateNode(this.state, finalist.nodeId, {
|
|
4359
|
-
against: root,
|
|
4360
|
-
split: "selection"
|
|
4361
|
-
}));
|
|
4362
|
-
for (const [rank, finalist] of plan.finalists.entries()) await this.recorder.decideNode({
|
|
4363
|
-
nodeId: finalist.nodeId,
|
|
4364
|
-
decision: { status: "finalist" },
|
|
4365
|
-
basis: bases[rank],
|
|
4366
|
-
rule: SEARCH_CLAIM_RULE_NAME,
|
|
4367
|
-
reason: `rank ${rank + 1} of ${plan.finalists.length} by selection mean (${finalist.selectionMean}); ${plan.reason}`
|
|
4368
|
-
});
|
|
4369
|
-
await this.refresh();
|
|
4370
|
-
if (plan.test !== "run") return;
|
|
4371
|
-
const missing = this.claimCells(plan.finalists.length, root, plan).map((cell) => ({
|
|
4372
|
-
...cell,
|
|
4373
|
-
cellId: searchCellId(this.recorder.searchId, cell.nodeId, cell.plan.taskId, "test", cell.plan.rep)
|
|
4374
|
-
})).filter((cell) => !this.state.cell(cell.cellId));
|
|
4375
|
-
for (const cell of missing) await this.recorder.allocateCell({
|
|
4376
|
-
nodeId: cell.nodeId,
|
|
4377
|
-
taskId: cell.plan.taskId,
|
|
4378
|
-
split: "test",
|
|
4379
|
-
rep: cell.plan.rep,
|
|
4380
|
-
stage: "claim",
|
|
4381
|
-
lane: cell.lane.name,
|
|
4382
|
-
reservation: cell.reservation
|
|
4383
|
-
});
|
|
4384
|
-
await this.refresh();
|
|
4385
|
-
for (const cell of missing) this.enqueue(this.state.cell(cell.cellId));
|
|
4386
|
-
}
|
|
4387
|
-
/** The test cells of the root and the first `finalists` of the plan (or of
|
|
4388
|
-
* the finalist count being priced), task by task, with their lanes and holds. */
|
|
4389
|
-
claimCells(finalists, root, plan) {
|
|
4390
|
-
const header = this.state.header;
|
|
4391
|
-
const arms = [root, ...plan ? plan.finalists.map((entry) => entry.nodeId) : []];
|
|
4392
|
-
while (arms.length < finalists + 1) arms.push(root);
|
|
4393
|
-
const cells = [];
|
|
4394
|
-
for (const task of header.splits.test.tasks) for (let rep = 0; rep < this.options.allocation.reps; rep++) for (const nodeId of arms) {
|
|
4395
|
-
const cellPlan = {
|
|
4396
|
-
taskId: task.taskId,
|
|
4397
|
-
split: "test",
|
|
4398
|
-
rep,
|
|
4399
|
-
stage: "claim"
|
|
4400
|
-
};
|
|
4401
|
-
const laneName = this.options.executor.place({
|
|
4402
|
-
...cellPlan,
|
|
4403
|
-
nodeId
|
|
4404
|
-
});
|
|
4405
|
-
const lane = this.lanes.get(laneName);
|
|
4406
|
-
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
4407
|
-
cells.push({
|
|
4408
|
-
nodeId,
|
|
4409
|
-
plan: cellPlan,
|
|
4410
|
-
lane,
|
|
4411
|
-
reservation: this.cellReservation(lane)
|
|
4412
|
-
});
|
|
4413
|
-
}
|
|
4414
|
-
return cells;
|
|
4415
|
-
}
|
|
4416
|
-
async cancelQueued(reason) {
|
|
4417
|
-
for (const lane of this.lanes.values()) for (const [priority, queue] of lane.queues.entries()) {
|
|
4418
|
-
if (priority === STAGE_PRIORITY.claim) continue;
|
|
4419
|
-
for (const cellId of queue.splice(0)) {
|
|
4420
|
-
const cell = this.state.cell(cellId);
|
|
4421
|
-
if (cell.attempts === 0) {
|
|
4422
|
-
await this.recorder.cancelCell({
|
|
4423
|
-
cellId,
|
|
4424
|
-
reason
|
|
4425
|
-
});
|
|
4426
|
-
await this.refresh();
|
|
4427
|
-
}
|
|
4428
|
-
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
4429
|
-
this.pending.set(cell.nodeId, left);
|
|
4430
|
-
if (left === 0) this.markScreened(cell.nodeId);
|
|
4431
|
-
}
|
|
4432
|
-
}
|
|
4433
|
-
}
|
|
4434
|
-
/** No further attempt will run: a final outcome, a cancellation, or a
|
|
4435
|
-
* retryable error that used up its attempts. */
|
|
4436
|
-
cellDone(cell) {
|
|
4437
|
-
return cell.final || cell.cancelled !== null || cell.outcome === "errored" && cell.attempts >= this.maxAttempts;
|
|
4438
|
-
}
|
|
4439
|
-
/** Start a proposal when the rules admit one; otherwise say why not. */
|
|
4440
|
-
async tryExpand() {
|
|
4441
|
-
const { maxNodes, maxCells } = this.state.header.budget;
|
|
4442
|
-
if (maxNodes !== null && this.state.audit.nodes >= maxNodes) return "max-nodes";
|
|
4443
|
-
const { maxExpansions } = this.options;
|
|
4444
|
-
if (maxExpansions !== void 0 && this.completed.length >= maxExpansions) return "max-nodes";
|
|
4445
|
-
if (this.queuedCount() > 0) return "converged";
|
|
4446
|
-
if (this.inFlight.size >= 2 * this.capacity()) return "converged";
|
|
4447
|
-
const view = this.policyView();
|
|
4448
|
-
const { policy } = this.options;
|
|
4449
|
-
const leader = policy.leader(view);
|
|
4450
|
-
if (policy.patience !== void 0 && this.completed.filter((index) => index > (this.expansionOf.get(leader) ?? -1)).length >= policy.patience) return "patience";
|
|
4451
|
-
const screen = this.options.allocation.screenSize(this.state);
|
|
4452
|
-
const operation = this.operationReservation();
|
|
4453
|
-
const children = this.options.proposer.childrenPerProposal ?? 1;
|
|
4454
|
-
const need = operation.usd + children * screen * this.maxCellReservation();
|
|
4455
|
-
const { headroomUsd } = this.state.budget;
|
|
4456
|
-
if (headroomUsd !== null && (need > headroomUsd + USD_TOLERANCE || this.overCap())) return "budget";
|
|
4457
|
-
if (maxCells !== null && this.state.audit.cells.allocated + children * screen > maxCells) return "budget";
|
|
4458
|
-
const expansion = policy.expand(view);
|
|
4459
|
-
if (expansion === null) return "converged";
|
|
4460
|
-
const index = this.expansions++;
|
|
4461
|
-
const operationId = `expand-${index}`;
|
|
4462
|
-
await this.recorder.startOperation({
|
|
4463
|
-
operationId,
|
|
4464
|
-
operationKind: "candidate-generation",
|
|
4465
|
-
reservation: operation
|
|
4466
|
-
});
|
|
4467
|
-
await this.refresh();
|
|
4468
|
-
const controller = new AbortController();
|
|
4469
|
-
const request = {
|
|
4470
|
-
expansion: index,
|
|
4471
|
-
operationId,
|
|
4472
|
-
operator: expansion.operator,
|
|
4473
|
-
parents: expansion.parents.map((nodeId) => ({
|
|
4474
|
-
nodeId,
|
|
4475
|
-
artifact: this.artifact(nodeId)
|
|
4476
|
-
})),
|
|
4477
|
-
leader,
|
|
4478
|
-
signal: controller.signal
|
|
4479
|
-
};
|
|
4480
|
-
this.proposalController = controller;
|
|
4481
|
-
this.proposal = this.options.proposer.propose(request).then((result) => this.push({
|
|
4482
|
-
kind: "proposal",
|
|
4483
|
-
expansion: index,
|
|
4484
|
-
operationId,
|
|
4485
|
-
request,
|
|
4486
|
-
selection: expansion.selection,
|
|
4487
|
-
result
|
|
4488
|
-
}), (error) => this.push({
|
|
4489
|
-
kind: "proposal",
|
|
4490
|
-
expansion: index,
|
|
4491
|
-
operationId,
|
|
4492
|
-
request,
|
|
4493
|
-
selection: expansion.selection,
|
|
4494
|
-
error
|
|
4495
|
-
}));
|
|
4496
|
-
return null;
|
|
4497
|
-
}
|
|
4498
|
-
async recordProposal(completion) {
|
|
4499
|
-
this.proposal = null;
|
|
4500
|
-
this.proposalController = null;
|
|
4501
|
-
const { operationId, expansion } = completion;
|
|
4502
|
-
const { proposer, codec, admit } = this.options;
|
|
4503
|
-
const result = completion.result;
|
|
4504
|
-
if (!result) {
|
|
4505
|
-
await this.recorder.recordOperation({
|
|
4506
|
-
operationId,
|
|
4507
|
-
operationKind: "candidate-generation",
|
|
4508
|
-
execution: proposer.execution,
|
|
4509
|
-
outcome: {
|
|
4510
|
-
status: "failed",
|
|
4511
|
-
failure: {
|
|
4512
|
-
code: "proposer-error",
|
|
4513
|
-
message: errorMessage(completion.error)
|
|
4514
|
-
}
|
|
4515
|
-
},
|
|
4516
|
-
accounting: interruptedAccounting()
|
|
4517
|
-
});
|
|
4518
|
-
await this.refresh();
|
|
4519
|
-
this.failure ??= { error: completion.error };
|
|
4520
|
-
return;
|
|
4521
|
-
}
|
|
4522
|
-
const { maxNodes } = this.state.header.budget;
|
|
4523
|
-
const room = maxNodes === null ? Number.POSITIVE_INFINITY : maxNodes - this.state.audit.nodes;
|
|
4524
|
-
const parents = completion.request.parents;
|
|
4525
|
-
const blob = {
|
|
4526
|
-
kind: "search-proposal",
|
|
4527
|
-
operator: completion.request.operator,
|
|
4528
|
-
parents: parents.map((parent) => parent.nodeId),
|
|
4529
|
-
selection: completion.selection,
|
|
4530
|
-
children: result.children.slice(0, Math.max(0, room)).map((child) => ({
|
|
4531
|
-
node: codec.node(this.recorder, child.artifact),
|
|
4532
|
-
diffs: parents.map((parent) => codec.diff(this.recorder, parent.artifact, child.artifact)),
|
|
4533
|
-
label: redactText(child.label, { profile: "share" }),
|
|
4534
|
-
rationale: redactText(child.rationale, { profile: "share" }),
|
|
4535
|
-
attribution: child.attribution ?? null,
|
|
4536
|
-
invalid: admit?.(child.artifact) ?? null
|
|
4537
|
-
}))
|
|
4538
|
-
};
|
|
4539
|
-
await this.recorder.recordOperation({
|
|
4540
|
-
operationId,
|
|
4541
|
-
operationKind: "candidate-generation",
|
|
4542
|
-
execution: result.execution ?? proposer.execution,
|
|
4543
|
-
outcome: { status: "completed" },
|
|
4544
|
-
accounting: result.accounting,
|
|
4545
|
-
artifacts: [this.recorder.blob("proposal", blob)]
|
|
4546
|
-
});
|
|
4547
|
-
await this.refresh();
|
|
4548
|
-
this.operationCosts.push(this.state.operation(operationId).spentUsd);
|
|
4549
|
-
this.completed.push(expansion);
|
|
4550
|
-
for (const [index, child] of blob.children.entries()) this.proposed.set(child.node.artifactDigest, result.children[index].artifact);
|
|
4551
|
-
if (result.stop !== void 0 || blob.children.length === 0) this.stopReason ??= "converged";
|
|
4552
|
-
await this.finishProposal(expansion);
|
|
4553
|
-
}
|
|
4554
|
-
/** Register every child a recorded proposal produced, each once. */
|
|
4555
|
-
async finishProposal(expansion) {
|
|
4556
|
-
const operationId = `expand-${expansion}`;
|
|
4557
|
-
const operation = this.state.operation(operationId);
|
|
4558
|
-
const ref = operation?.artifacts.find((artifact) => artifact.role === "proposal");
|
|
4559
|
-
if (!operation?.recorded || operation.outcome !== "completed" || !ref) return;
|
|
4560
|
-
const blob = this.recorder.readBlob(ref);
|
|
4561
|
-
const { proposer } = this.options;
|
|
4562
|
-
for (const [index, child] of blob.children.entries()) {
|
|
4563
|
-
const { nodeId } = await this.recorder.registerNode(child.node);
|
|
4564
|
-
const held = this.proposed.get(child.node.artifactDigest);
|
|
4565
|
-
if (held !== void 0) {
|
|
4566
|
-
this.proposed.delete(child.node.artifactDigest);
|
|
4567
|
-
if (!this.artifacts.has(nodeId)) this.artifacts.set(nodeId, held);
|
|
4568
|
-
}
|
|
4569
|
-
const edgeId = await this.recorder.recordEdge({
|
|
4570
|
-
childNodeId: nodeId,
|
|
4571
|
-
parents: blob.parents,
|
|
4572
|
-
operator: blob.operator,
|
|
4573
|
-
attribution: "explicit",
|
|
4574
|
-
proposer: {
|
|
4575
|
-
kind: proposer.kind,
|
|
4576
|
-
name: proposer.name,
|
|
4577
|
-
operationId,
|
|
4578
|
-
source: proposer.source
|
|
4579
|
-
},
|
|
4580
|
-
proposalKey: `${operationId}:${index}`,
|
|
4581
|
-
selection: blob.selection,
|
|
4582
|
-
rationale: child.rationale,
|
|
4583
|
-
diffs: child.diffs,
|
|
4584
|
-
label: child.label
|
|
4585
|
-
});
|
|
4586
|
-
await this.refresh();
|
|
4587
|
-
if (this.state.node(nodeId).edgeIds[0] !== edgeId) continue;
|
|
4588
|
-
this.expansionOf.set(nodeId, expansion);
|
|
4589
|
-
if (child.invalid !== null) {
|
|
4590
|
-
await this.recorder.decideNode({
|
|
4591
|
-
nodeId,
|
|
4592
|
-
decision: { status: "invalid" },
|
|
4593
|
-
rule: "admission",
|
|
4594
|
-
reason: child.invalid
|
|
4595
|
-
});
|
|
4596
|
-
await this.refresh();
|
|
4597
|
-
continue;
|
|
4598
|
-
}
|
|
4599
|
-
if (this.admitted.has(nodeId)) continue;
|
|
4600
|
-
this.admitted.add(nodeId);
|
|
4601
|
-
await this.allocateFor(nodeId);
|
|
4602
|
-
if ((this.pending.get(nodeId) ?? 0) === 0) this.markScreened(nodeId);
|
|
4603
|
-
}
|
|
4604
|
-
}
|
|
4605
|
-
async close() {
|
|
4606
|
-
await this.refresh();
|
|
4607
|
-
const keep = this.options.policy.leader(this.policyView());
|
|
4608
|
-
for (const decision of this.options.allocation.prune(this.allocationView(), keep)) await this.decide(decision);
|
|
4609
|
-
if (this.claimPlan) {
|
|
4610
|
-
await this.closeWithClaim(this.claimPlan);
|
|
4611
|
-
return;
|
|
4612
|
-
}
|
|
4613
|
-
const view = this.policyView();
|
|
4614
|
-
const leader = this.options.policy.leader(view);
|
|
4615
|
-
const { split } = view;
|
|
4616
|
-
const root = this.state.rootNodeId;
|
|
4617
|
-
const rule = this.options.policy.name;
|
|
4618
|
-
const leaderUnits = view.unitScores(leader).map((unit) => unit.unitId);
|
|
4619
|
-
const decisions = this.state.nodes().filter((node) => !isTerminal(node)).map((node) => {
|
|
4620
|
-
const scored = this.state.scoredCells(node.nodeId, split).length > 0;
|
|
4621
|
-
const lead = node.nodeId === leader && scored;
|
|
4622
|
-
const against = node.nodeId === leader ? root : leader;
|
|
4623
|
-
const basis = node.nodeId === against || !scored ? null : estimateNode(this.state, node.nodeId, {
|
|
4624
|
-
against,
|
|
4625
|
-
split
|
|
4626
|
-
});
|
|
4627
|
-
const units = new Set(view.unitScores(node.nodeId).map((unit) => unit.unitId));
|
|
4628
|
-
return {
|
|
4629
|
-
nodeId: node.nodeId,
|
|
4630
|
-
decision: { status: lead ? "selected" : "rejected" },
|
|
4631
|
-
basis,
|
|
4632
|
-
rule,
|
|
4633
|
-
reason: lead ? `the leader when the search stopped (${this.stopReason}): a budget decision on the ${split} split, which claims nothing` : !this.screenedSet.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit unscored, so it could not lead" : leaderUnits.some((unitId) => !units.has(unitId)) ? "the search stopped before it was measured on every unit the leader was" : "it did not beat the leader on the units they share"
|
|
4634
|
-
};
|
|
4635
|
-
});
|
|
4636
|
-
for (const decision of decisions) await this.recorder.decideNode(decision);
|
|
4637
|
-
await this.recorder.close({
|
|
4638
|
-
reason: this.stopReason ?? "converged",
|
|
4639
|
-
claim: null
|
|
4640
|
-
});
|
|
4641
|
-
await this.refresh();
|
|
4642
|
-
}
|
|
4643
|
-
/** Close with the claim: the claim's decisions, then every other open node
|
|
4644
|
-
* rejected with its selection estimate against the root. */
|
|
4645
|
-
async closeWithClaim(plan) {
|
|
4646
|
-
const { claim, decisions } = decideSearchClaim(this.state, plan);
|
|
4647
|
-
const decided = new Set(decisions.map((decision) => decision.nodeId));
|
|
4648
|
-
const root = this.state.rootNodeId;
|
|
4649
|
-
const rest = this.state.nodes().filter((node) => !decided.has(node.nodeId) && !isTerminal(node)).map((node) => {
|
|
4650
|
-
const scored = this.state.scoredCells(node.nodeId, "selection").length > 0;
|
|
4651
|
-
return {
|
|
4652
|
-
nodeId: node.nodeId,
|
|
4653
|
-
decision: { status: "rejected" },
|
|
4654
|
-
basis: scored ? estimateNode(this.state, node.nodeId, {
|
|
4655
|
-
against: root,
|
|
4656
|
-
split: "selection"
|
|
4657
|
-
}) : null,
|
|
4658
|
-
rule: SEARCH_CLAIM_RULE_NAME,
|
|
4659
|
-
reason: scored ? "not a finalist: the claim tests at most 3 nodes that scored every selection unit and beat the root, best selection mean first" : "the search stopped before this node was measured on the selection split"
|
|
4660
|
-
};
|
|
4661
|
-
});
|
|
4662
|
-
for (const decision of [...decisions, ...rest]) await this.recorder.decideNode(decision);
|
|
4663
|
-
await this.recorder.close({
|
|
4664
|
-
reason: plan.stopReason,
|
|
4665
|
-
claim
|
|
4666
|
-
});
|
|
4667
|
-
await this.refresh();
|
|
4668
|
-
}
|
|
4669
|
-
/** The closed search, with its claim made again from the ledger alone. A
|
|
4670
|
-
* claim the ledger does not support throws: a tampered ledger, or a
|
|
4671
|
-
* producer and verifier that disagree. */
|
|
4672
|
-
closedResult() {
|
|
4673
|
-
const state = this.state;
|
|
4674
|
-
const claimVerification = verifySearchClaim(state);
|
|
4675
|
-
if (claimVerification?.status === "mismatch") throw new Error(`runSearch: search ${state.searchId} closed with a claim its ledger does not support: ${claimVerification.differences.join("; ")}`);
|
|
4676
|
-
return {
|
|
4677
|
-
state,
|
|
4678
|
-
leader: state.audit.selectedNodeId ?? state.rootNodeId,
|
|
4679
|
-
reason: state.closed.reason,
|
|
4680
|
-
claim: state.closed.claim,
|
|
4681
|
-
claimVerification
|
|
4682
|
-
};
|
|
4683
|
-
}
|
|
4684
|
-
policyView() {
|
|
4685
|
-
return searchPolicyView(this.state, {
|
|
4686
|
-
screened: this.screened,
|
|
4687
|
-
screening: this.admitted.size - this.screenedSet.size,
|
|
4688
|
-
expansions: this.completed.length,
|
|
4689
|
-
maxAttempts: this.maxAttempts
|
|
4690
|
-
});
|
|
4691
|
-
}
|
|
4692
|
-
allocationView() {
|
|
4693
|
-
return {
|
|
4694
|
-
state: this.state,
|
|
4695
|
-
idle: (nodeId) => this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0
|
|
4696
|
-
};
|
|
4697
|
-
}
|
|
4698
|
-
artifact(nodeId) {
|
|
4699
|
-
const held = this.artifacts.get(nodeId);
|
|
4700
|
-
if (held !== void 0) return held;
|
|
4701
|
-
const artifact = this.options.codec.load(this.recorder, this.state.node(nodeId));
|
|
4702
|
-
this.artifacts.set(nodeId, artifact);
|
|
4703
|
-
return artifact;
|
|
4704
|
-
}
|
|
4705
|
-
capacity() {
|
|
4706
|
-
let total = 0;
|
|
4707
|
-
for (const lane of this.lanes.values()) total += lane.capacity;
|
|
4708
|
-
const cap = this.state.header.budget.maxConcurrency;
|
|
4709
|
-
return cap === null ? total : Math.min(cap, total);
|
|
4710
|
-
}
|
|
4711
|
-
/** A hard lane holds its enforced maximum. An estimate lane holds 1.5 times
|
|
4712
|
-
* the 99th percentile of its settled cells once 20 settled, else its prior. */
|
|
4713
|
-
cellReservation(lane) {
|
|
4714
|
-
if (lane.costCap === "hard") return {
|
|
4715
|
-
kind: "hard",
|
|
4716
|
-
usd: lane.cellUsd
|
|
4717
|
-
};
|
|
4718
|
-
return {
|
|
4719
|
-
kind: "estimate",
|
|
4720
|
-
usd: estimate(lane.settledCosts, lane.cellUsd)
|
|
4721
|
-
};
|
|
4722
|
-
}
|
|
4723
|
-
maxCellReservation() {
|
|
4724
|
-
let most = 0;
|
|
4725
|
-
for (const lane of this.lanes.values()) most = Math.max(most, this.cellReservation(lane).usd);
|
|
4726
|
-
return most;
|
|
4727
|
-
}
|
|
4728
|
-
operationReservation() {
|
|
4729
|
-
return {
|
|
4730
|
-
kind: "estimate",
|
|
4731
|
-
usd: estimate(this.operationCosts, this.options.proposer.reservationUsd ?? 0)
|
|
4732
|
-
};
|
|
4733
|
-
}
|
|
4734
|
-
/** Overspend took committed spend, open holds and the unspent claim reserve
|
|
4735
|
-
* past the cap. Nothing new is proposed then, even at a zero-dollar prior. */
|
|
4736
|
-
overCap() {
|
|
4737
|
-
const { maxUsd } = this.state.header.budget;
|
|
4738
|
-
if (maxUsd === null) return false;
|
|
4739
|
-
const { committedUsd, openReservationUsd } = this.state.audit.spend;
|
|
4740
|
-
return committedUsd + openReservationUsd + this.state.budget.claimReserveUsd > maxUsd + USD_TOLERANCE;
|
|
4741
|
-
}
|
|
4742
|
-
pastDeadline() {
|
|
4743
|
-
const deadline = this.state.header.budget.deadline;
|
|
4744
|
-
return deadline !== null && this.now() >= Date.parse(deadline);
|
|
4745
|
-
}
|
|
4746
|
-
async refresh() {
|
|
4747
|
-
this.state = await this.recorder.state();
|
|
4748
|
-
}
|
|
4749
|
-
};
|
|
4750
|
-
/**
|
|
4751
|
-
* A policy's read of a search: `screened` lists the admitted nodes whose screen
|
|
4752
|
-
* finished, in registration order; `screening` counts the admitted nodes still
|
|
4753
|
-
* being screened. Nodes are ranked on the selection split, or on train when
|
|
4754
|
-
* the search declares no selection split. The view reads no test cell.
|
|
4755
|
-
* `maxAttempts` (default 3, the kernel's) says when an errored cell has no
|
|
4756
|
-
* attempt left, which makes its unit a dodge.
|
|
4757
|
-
*/
|
|
4758
|
-
function searchPolicyView(state, input) {
|
|
4759
|
-
const maxAttempts = input.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
4760
|
-
const header = state.header;
|
|
4761
|
-
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
4762
|
-
const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
4763
|
-
return {
|
|
4764
|
-
searchId: state.searchId,
|
|
4765
|
-
seed: header.policy.seed,
|
|
4766
|
-
direction: header.objective.direction,
|
|
4767
|
-
split,
|
|
4768
|
-
rootNodeId: state.rootNodeId,
|
|
4769
|
-
expansions: input.expansions,
|
|
4770
|
-
screened: [...input.screened],
|
|
4771
|
-
screening: input.screening,
|
|
4772
|
-
complete: (nodeId) => {
|
|
4773
|
-
let scored = false;
|
|
4774
|
-
for (const cell of state.cells({ nodeId })) {
|
|
4775
|
-
if (cell.split !== split) continue;
|
|
4776
|
-
if (cell.score !== null) scored = true;
|
|
4777
|
-
else if (cell.final || cell.outcome === "errored" && cell.attempts >= maxAttempts) return false;
|
|
4778
|
-
}
|
|
4779
|
-
return scored;
|
|
4780
|
-
},
|
|
4781
|
-
unitScores: (nodeId) => state.unitScores(nodeId, split),
|
|
4782
|
-
estimate: (nodeId, against) => estimateNode(state, nodeId, {
|
|
4783
|
-
against,
|
|
4784
|
-
split
|
|
4785
|
-
})
|
|
4786
|
-
};
|
|
4787
|
-
}
|
|
4788
|
-
/**
|
|
4789
|
-
* The divergence rule: a node whose train mean rose over its parent's on the
|
|
4790
|
-
* train units they share while its selection interval against the parent (6
|
|
4791
|
-
* or more units, not indeterminate) lies wholly on the worse side. Train is
|
|
4792
|
-
* what the proposer reads; a gain there that selection contradicts is the
|
|
4793
|
-
* signature of fitting the feedback instead of the task.
|
|
4794
|
-
*/
|
|
4795
|
-
function searchDivergence(state, nodeId) {
|
|
4796
|
-
const header = state.header;
|
|
4797
|
-
const parent = state.node(nodeId)?.primaryParentId ?? null;
|
|
4798
|
-
if (!header || parent === null || header.splits.selection.tasks.length === 0) return null;
|
|
4799
|
-
const selection = estimateNode(state, nodeId, {
|
|
4800
|
-
against: parent,
|
|
4801
|
-
split: "selection"
|
|
4802
|
-
});
|
|
4803
|
-
if (selection.interval === null) return null;
|
|
4804
|
-
const maximize = header.objective.direction === "maximize";
|
|
4805
|
-
const [low, high] = selection.interval;
|
|
4806
|
-
if (maximize ? high >= 0 : low <= 0) return null;
|
|
4807
|
-
const parentTrain = new Map(state.unitScores(parent, "train").map((unit) => [unit.unitId, unit.mean]));
|
|
4808
|
-
let gain = 0;
|
|
4809
|
-
let shared = 0;
|
|
4810
|
-
for (const unit of state.unitScores(nodeId, "train")) {
|
|
4811
|
-
const before = parentTrain.get(unit.unitId);
|
|
4812
|
-
if (before === void 0) continue;
|
|
4813
|
-
gain += maximize ? unit.mean - before : before - unit.mean;
|
|
4814
|
-
shared += 1;
|
|
4815
|
-
}
|
|
4816
|
-
if (shared === 0 || gain <= 0) return null;
|
|
4817
|
-
return {
|
|
4818
|
-
basis: selection,
|
|
4819
|
-
reason: `its train mean rose ${Math.round(gain / shared * 1e4) / 1e4} over its parent's on ${shared} shared train unit${shared === 1 ? "" : "s"}, while its selection interval against the parent, [${round4(low)}, ${round4(high)}] on ${selection.pairs} units, lies wholly on the worse side`
|
|
4820
|
-
};
|
|
4821
|
-
}
|
|
4822
|
-
function round4(value) {
|
|
4823
|
-
return Math.round(value * 1e4) / 1e4;
|
|
4824
|
-
}
|
|
4825
|
-
function isTerminal(node) {
|
|
4826
|
-
return node.status !== null && node.status !== "advanced";
|
|
4827
|
-
}
|
|
4828
|
-
function estimate(costs, prior) {
|
|
4829
|
-
if (costs.length < ESTIMATE_FROM_CELLS) return prior;
|
|
4830
|
-
const sorted = [...costs].sort((a, b) => a - b);
|
|
4831
|
-
const p99 = sorted[Math.min(sorted.length - 1, Math.ceil(.99 * sorted.length) - 1)];
|
|
4832
|
-
return Math.round(ESTIMATE_MARGIN * p99 * 1e9) / 1e9;
|
|
4833
|
-
}
|
|
4834
|
-
/** Spend of work whose result was lost: unknown, with no proven floor. */
|
|
4835
|
-
function interruptedAccounting() {
|
|
4836
|
-
return {
|
|
4837
|
-
tokens: {
|
|
4838
|
-
status: "unknown",
|
|
4839
|
-
reason: "the result of this work was lost"
|
|
4840
|
-
},
|
|
4841
|
-
cost: {
|
|
4842
|
-
status: "unknown",
|
|
4843
|
-
knownLowerBoundUsd: 0,
|
|
4844
|
-
reason: "the result of this work was lost, so its spend is unknown"
|
|
4845
|
-
}
|
|
4846
|
-
};
|
|
4847
|
-
}
|
|
4848
|
-
function errorMessage(error) {
|
|
4849
|
-
const text = error instanceof Error ? error.message : String(error);
|
|
4850
|
-
return text.trim().length > 0 ? text : "the proposer failed without a message";
|
|
4851
|
-
}
|
|
4852
|
-
//#endregion
|
|
4853
2972
|
//#region src/campaign/search-history-receipt.ts
|
|
4854
2973
|
const SEARCH_HISTORY_RECEIPT_SCHEMA_VERSION = "2026-09";
|
|
4855
2974
|
const SEARCH_HISTORY_RECEIPT_DIGEST_ALGORITHM = "rfc8785-sha256";
|
|
@@ -5418,10 +3537,16 @@ function canonicalHeader(header) {
|
|
|
5418
3537
|
* Expansion policies: which node a search expands next, with which operator,
|
|
5419
3538
|
* and which node it currently keeps.
|
|
5420
3539
|
*
|
|
5421
|
-
* A policy reads a `SearchPolicyView`: nodes
|
|
5422
|
-
* per-unit scores on the policy's split
|
|
5423
|
-
*
|
|
5424
|
-
*
|
|
3540
|
+
* A policy reads a `SearchPolicyView`: the nodes, their lineage, their
|
|
3541
|
+
* per-unit scores and posterior on the policy's split, and how many of their
|
|
3542
|
+
* cells failed as defects. The view holds no test cells, so a policy cannot
|
|
3543
|
+
* see the sealed claim data. Its decisions spend budget and claim nothing; a
|
|
3544
|
+
* claim comes only from the sealed test split.
|
|
3545
|
+
*
|
|
3546
|
+
* `incumbent`, `incumbentWithOperatorBandit` and `crowdedFrontierParent` are
|
|
3547
|
+
* hill climbs and pair with the `uniform` allocator. `aide` and `beam` choose parents from each node's
|
|
3548
|
+
* posterior, which is honest about a node measured on few units, so they pair
|
|
3549
|
+
* with `asha`, which measures most nodes on only the first rung.
|
|
5425
3550
|
*/
|
|
5426
3551
|
/**
|
|
5427
3552
|
* The hill climb. The leader starts at the root; a screened node that dodged no
|
|
@@ -5446,6 +3571,65 @@ function incumbent(options = {}) {
|
|
|
5446
3571
|
});
|
|
5447
3572
|
}
|
|
5448
3573
|
/**
|
|
3574
|
+
* The hill climb (as `incumbent`), with its operator drawn from the
|
|
3575
|
+
* `operatorYield` lens instead of fixed at `improve` (search-tree-design
|
|
3576
|
+
* §12): a weighted draw, seeded from `seed` and the expansion index like
|
|
3577
|
+
* `crowdedFrontierParent`, over the lens's per-operator yield. The fallback
|
|
3578
|
+
* is per operator, not all-or-nothing: an operator the lens has not yet
|
|
3579
|
+
* measured `MIN_OUTCOMES_FOR_WEIGHT` outcomes for draws on `fixedWeights`
|
|
3580
|
+
* (uniform by default) instead, so it is never starved while unmeasured, and
|
|
3581
|
+
* an operator that does have enough outcomes draws on its own measured
|
|
3582
|
+
* yield right away rather than waiting for every other operator to catch
|
|
3583
|
+
* up — proved against real search-sim ledgers where a hill climb had only
|
|
3584
|
+
* ever proposed `improve`: an all-or-nothing gate would keep every draw
|
|
3585
|
+
* uniform forever, because `draft`/`debug`/`merge` are never measured by a
|
|
3586
|
+
* policy that never chooses them. A measured operator's weight is its own
|
|
3587
|
+
* `fixedWeights` prior plus its yield, not the yield alone: the two are on
|
|
3588
|
+
* different scales (yield is improvement per known dollar; the prior is an
|
|
3589
|
+
* arbitrary share), so using the yield by itself would let a small positive
|
|
3590
|
+
* yield draw LESS than an untested operator sitting on the default prior of
|
|
3591
|
+
* 1 — the same real ledgers caught exactly that before this was fixed. The
|
|
3592
|
+
* sum is floored just above zero, so a so-far-bad operator can still be
|
|
3593
|
+
* re-measured rather than excluded outright.
|
|
3594
|
+
*/
|
|
3595
|
+
function incumbentWithOperatorBandit(options = { seed: 0 }) {
|
|
3596
|
+
const { patience, seed } = options;
|
|
3597
|
+
if (!Number.isInteger(seed)) throw new TypeError(`incumbentWithOperatorBandit: seed must be an integer, got ${String(seed)}`);
|
|
3598
|
+
const fixedWeights = {
|
|
3599
|
+
draft: options.fixedWeights?.draft ?? 1,
|
|
3600
|
+
improve: options.fixedWeights?.improve ?? 1,
|
|
3601
|
+
debug: options.fixedWeights?.debug ?? 1,
|
|
3602
|
+
merge: options.fixedWeights?.merge ?? 1
|
|
3603
|
+
};
|
|
3604
|
+
for (const operator of EXPANSION_OPERATORS) {
|
|
3605
|
+
const weight = fixedWeights[operator];
|
|
3606
|
+
if (!Number.isFinite(weight) || weight < 0) throw new TypeError(`incumbentWithOperatorBandit: fixedWeights.${operator} must be >= 0, got ${weight}`);
|
|
3607
|
+
}
|
|
3608
|
+
return hillClimb({
|
|
3609
|
+
name: patience === void 0 ? `incumbent-operator-bandit(seed=${seed})` : `incumbent-operator-bandit(seed=${seed},patience=${patience})`,
|
|
3610
|
+
patience,
|
|
3611
|
+
parent: (_view, leader) => ({
|
|
3612
|
+
nodeId: leader,
|
|
3613
|
+
evidence: {}
|
|
3614
|
+
}),
|
|
3615
|
+
operator: (view) => chooseOperator(view.operatorWeights, fixedWeights, seed, view.expansions)
|
|
3616
|
+
});
|
|
3617
|
+
}
|
|
3618
|
+
function chooseOperator(measured, fixedWeights, seed, expansionIndex) {
|
|
3619
|
+
const weights = EXPANSION_OPERATORS.map((operator) => {
|
|
3620
|
+
const yieldMean = measured[operator];
|
|
3621
|
+
const prior = fixedWeights[operator];
|
|
3622
|
+
return yieldMean === null ? prior : Math.max(prior + yieldMean, 1e-6);
|
|
3623
|
+
});
|
|
3624
|
+
const total = weights.reduce((a, b) => a + b, 0);
|
|
3625
|
+
let draw = mulberry32(seed ^ Math.imul(expansionIndex + 1, 2654435761) | 0)() * total;
|
|
3626
|
+
for (let index = 0; index < EXPANSION_OPERATORS.length; index++) {
|
|
3627
|
+
draw -= weights[index];
|
|
3628
|
+
if (draw <= 0) return EXPANSION_OPERATORS[index];
|
|
3629
|
+
}
|
|
3630
|
+
return EXPANSION_OPERATORS[EXPANSION_OPERATORS.length - 1];
|
|
3631
|
+
}
|
|
3632
|
+
/**
|
|
5449
3633
|
* The hill climb with a different parent: each expansion draws two distinct
|
|
5450
3634
|
* members of the Pareto frontier (per-unit means on the policy split) with a
|
|
5451
3635
|
* PRNG seeded from `seed` and the expansion index, and keeps the one with the
|
|
@@ -5482,27 +3666,261 @@ function crowdedFrontierParent(options) {
|
|
|
5482
3666
|
}
|
|
5483
3667
|
});
|
|
5484
3668
|
}
|
|
5485
|
-
|
|
5486
|
-
|
|
5487
|
-
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
3669
|
+
/**
|
|
3670
|
+
* AIDE's search policy (`aide/agent.py`, `search_policy`) with three changes
|
|
3671
|
+
* for noisy scores.
|
|
3672
|
+
*
|
|
3673
|
+
* 1. **Draft** while fewer than `drafts` nodes were drafted: the root is the
|
|
3674
|
+
* parent, and the proposer writes a whole alternative, not an edit.
|
|
3675
|
+
* 2. **Debug**, with probability `debugProbability`: a buggy leaf, drawn
|
|
3676
|
+
* uniformly, whose chain holds fewer than `maxDebugDepth` debug edges. A
|
|
3677
|
+
* node is buggy when at least half of its final cells outside the test
|
|
3678
|
+
* split failed: defects, not a low score. An `errored` cell is the
|
|
3679
|
+
* environment's fault; it is retried and never makes a node buggy.
|
|
3680
|
+
* 3. **Improve** otherwise. The parent is drawn by Thompson sampling, not
|
|
3681
|
+
* AIDE's argmax, from every screened node that is not buggy and has a
|
|
3682
|
+
* posterior (`searchPosterior`): the root sits at 0, and every other node
|
|
3683
|
+
* at its mean improvement over the root with the search's pooled
|
|
3684
|
+
* between-unit variance divided by its shared units, so a node measured on
|
|
3685
|
+
* few units draws from a wide posterior. Until some node shares 2 units
|
|
3686
|
+
* with the root the pooled variance is unknown, every posterior is as wide
|
|
3687
|
+
* as it can be, and the draw is uniform. With no such node, the policy
|
|
3688
|
+
* drafts.
|
|
3689
|
+
* 4. **Stall:** when the drawn parent's lineage (the nodes under its nearest
|
|
3690
|
+
* draft, or under the root) has had `stallAfter` improve children in a row
|
|
3691
|
+
* that did not raise the lineage's best posterior mean, the improve forks
|
|
3692
|
+
* from the node with the best posterior mean instead. This is AIDE²'s
|
|
3693
|
+
* inner policy. A fork's children restart the run of the lineage they
|
|
3694
|
+
* join, so a stalled lineage that holds the global best forks once, not
|
|
3695
|
+
* on every later expansion.
|
|
3696
|
+
*
|
|
3697
|
+
* A parent is always a node whose screen finished; a node refused at
|
|
3698
|
+
* admission or decided invalid never is. Draws come from a PRNG seeded by the
|
|
3699
|
+
* search seed and the count of completed expansions, so the policy is a pure
|
|
3700
|
+
* function of the ledger. The leader is the incumbent's.
|
|
3701
|
+
*/
|
|
3702
|
+
function aide(options = {}) {
|
|
3703
|
+
const drafts = integerOption("aide", "drafts", options.drafts ?? 5, 0);
|
|
3704
|
+
const debugProbability = options.debugProbability ?? .5;
|
|
3705
|
+
if (!(debugProbability >= 0 && debugProbability <= 1)) throw new TypeError(`aide: debugProbability must be in [0, 1], got ${String(options.debugProbability)}`);
|
|
3706
|
+
const maxDebugDepth = integerOption("aide", "maxDebugDepth", options.maxDebugDepth ?? 3, 0);
|
|
3707
|
+
const stallAfter = integerOption("aide", "stallAfter", options.stallAfter ?? 4, 1);
|
|
3708
|
+
const name = drafts === 5 && debugProbability === .5 && maxDebugDepth === 3 && stallAfter === 4 ? "aide" : `aide(drafts=${drafts},debug=${debugProbability},maxDebugDepth=${maxDebugDepth},stallAfter=${stallAfter})`;
|
|
3709
|
+
return {
|
|
3710
|
+
name,
|
|
3711
|
+
leader: leaderOf,
|
|
3712
|
+
expand(view) {
|
|
3713
|
+
const root = view.rootNodeId;
|
|
3714
|
+
const screened = new Set(view.screened);
|
|
3715
|
+
if (!screened.has(root)) return null;
|
|
3716
|
+
const nodes = view.nodes();
|
|
3717
|
+
const byId = new Map(nodes.map((node) => [node.nodeId, node]));
|
|
3718
|
+
const rng = mulberry32(view.seed ^ Math.imul(view.expansions + 1, 2654435761) ^ AIDE_SALT | 0);
|
|
3719
|
+
const drafted = nodes.filter((node) => node.operator === "draft").length;
|
|
3720
|
+
const draft = (evidence) => ({
|
|
3721
|
+
parents: [root],
|
|
3722
|
+
operator: "draft",
|
|
3723
|
+
selection: {
|
|
3724
|
+
rule: `${name}:draft`,
|
|
3725
|
+
evidence: {
|
|
3726
|
+
drafted,
|
|
3727
|
+
drafts,
|
|
3728
|
+
...evidence
|
|
3729
|
+
}
|
|
3730
|
+
}
|
|
3731
|
+
});
|
|
3732
|
+
if (drafted < drafts) return draft({});
|
|
3733
|
+
if (rng() < debugProbability) {
|
|
3734
|
+
const debuggable = nodes.filter((node) => screened.has(node.nodeId) && buggy(node) && node.children.length === 0 && debugDepth(node, byId) < maxDebugDepth);
|
|
3735
|
+
if (debuggable.length > 0) {
|
|
3736
|
+
const pick = debuggable[Math.floor(rng() * debuggable.length)];
|
|
3737
|
+
return {
|
|
3738
|
+
parents: [pick.nodeId],
|
|
3739
|
+
operator: "debug",
|
|
3740
|
+
selection: {
|
|
3741
|
+
rule: `${name}:debug`,
|
|
3742
|
+
evidence: {
|
|
3743
|
+
debuggable: debuggable.length,
|
|
3744
|
+
defects: pick.defects,
|
|
3745
|
+
outcomes: pick.outcomes,
|
|
3746
|
+
debugDepth: debugDepth(pick, byId)
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
};
|
|
3750
|
+
}
|
|
3751
|
+
}
|
|
3752
|
+
const posterior = new Map(view.posterior.nodes.map((entry) => [entry.nodeId, entry]));
|
|
3753
|
+
const good = nodes.filter((node) => screened.has(node.nodeId) && !buggy(node) && (posterior.get(node.nodeId)?.mean ?? null) !== null);
|
|
3754
|
+
if (good.length === 0) return draft({ good: 0 });
|
|
3755
|
+
const meanOf = (node) => posterior.get(node.nodeId).mean;
|
|
3756
|
+
const known = view.posterior.pooledVariance !== null;
|
|
3757
|
+
let drawn = good[0];
|
|
3758
|
+
let draw = Number.NEGATIVE_INFINITY;
|
|
3759
|
+
if (!known) drawn = good[Math.floor(rng() * good.length)];
|
|
3760
|
+
else for (const node of good) {
|
|
3761
|
+
const value = meanOf(node) + Math.sqrt(posterior.get(node.nodeId).variance) * normal(rng);
|
|
3762
|
+
if (value > draw) {
|
|
3763
|
+
draw = value;
|
|
3764
|
+
drawn = node;
|
|
3765
|
+
}
|
|
3766
|
+
}
|
|
3767
|
+
const stalled = stalledRun(drawn, nodes, screened, posterior, `${name}:stall-fork`);
|
|
3768
|
+
if (stalled >= stallAfter) {
|
|
3769
|
+
let best = good[0];
|
|
3770
|
+
for (const node of good) if (meanOf(node) > meanOf(best)) best = node;
|
|
3771
|
+
return {
|
|
3772
|
+
parents: [best.nodeId],
|
|
3773
|
+
operator: "improve",
|
|
3774
|
+
selection: {
|
|
3775
|
+
rule: `${name}:stall-fork`,
|
|
3776
|
+
evidence: {
|
|
3777
|
+
candidates: good.length,
|
|
3778
|
+
mean: plain(meanOf(best)),
|
|
3779
|
+
units: posterior.get(best.nodeId).pairs,
|
|
3780
|
+
drawnOrdinal: drawn.ordinal,
|
|
3781
|
+
stalledChildren: stalled
|
|
3782
|
+
}
|
|
3783
|
+
}
|
|
3784
|
+
};
|
|
3785
|
+
}
|
|
3786
|
+
const entry = posterior.get(drawn.nodeId);
|
|
3787
|
+
return {
|
|
3788
|
+
parents: [drawn.nodeId],
|
|
3789
|
+
operator: "improve",
|
|
3790
|
+
selection: {
|
|
3791
|
+
rule: `${name}:${known ? "thompson" : "uniform-draw"}`,
|
|
3792
|
+
evidence: {
|
|
3793
|
+
candidates: good.length,
|
|
3794
|
+
mean: plain(entry.mean),
|
|
3795
|
+
units: entry.pairs,
|
|
3796
|
+
...known ? {
|
|
3797
|
+
sd: plain(Math.sqrt(entry.variance)),
|
|
3798
|
+
draw: plain(draw)
|
|
3799
|
+
} : {}
|
|
3800
|
+
}
|
|
3801
|
+
}
|
|
3802
|
+
};
|
|
5491
3803
|
}
|
|
5492
|
-
return current;
|
|
5493
3804
|
};
|
|
3805
|
+
}
|
|
3806
|
+
/**
|
|
3807
|
+
* Beam search: the beam is the top `width` nodes by posterior mean
|
|
3808
|
+
* (`searchPosterior`), and the member with the fewest children is expanded,
|
|
3809
|
+
* so each member is expanded in turn, as in a generation of `width`. A member
|
|
3810
|
+
* is the root, a node an allocator advanced, or a node that scored every unit
|
|
3811
|
+
* the root scored: a node an allocator has only screened waits for its rank
|
|
3812
|
+
* before it enters. Ties go to the higher mean, then to the earlier node. The
|
|
3813
|
+
* leader is the incumbent's. Strategy evolution is `beam` with its population
|
|
3814
|
+
* as the width.
|
|
3815
|
+
*/
|
|
3816
|
+
function beam(options) {
|
|
3817
|
+
const width = integerOption("beam", "width", options.width, 1);
|
|
3818
|
+
const name = `beam(width=${width})`;
|
|
3819
|
+
return {
|
|
3820
|
+
name,
|
|
3821
|
+
leader: leaderOf,
|
|
3822
|
+
expand(view) {
|
|
3823
|
+
const root = view.rootNodeId;
|
|
3824
|
+
const screened = new Set(view.screened);
|
|
3825
|
+
if (!screened.has(root)) return null;
|
|
3826
|
+
const posterior = new Map(view.posterior.nodes.map((entry) => [entry.nodeId, entry.mean]));
|
|
3827
|
+
const rootUnits = view.unitScores(root).map((unit) => unit.unitId);
|
|
3828
|
+
const members = view.nodes().filter((node) => screened.has(node.nodeId) && (posterior.get(node.nodeId) ?? null) !== null && (node.nodeId === root || node.status === "advanced" || scoredEvery(view, node.nodeId, rootUnits))).sort((left, right) => posterior.get(right.nodeId) - posterior.get(left.nodeId) || left.ordinal - right.ordinal).slice(0, width);
|
|
3829
|
+
let rank = 0;
|
|
3830
|
+
members.forEach((member, index) => {
|
|
3831
|
+
if (member.children.length < members[rank].children.length) rank = index;
|
|
3832
|
+
});
|
|
3833
|
+
const pick = members[rank];
|
|
3834
|
+
return {
|
|
3835
|
+
parents: [pick.nodeId],
|
|
3836
|
+
operator: "improve",
|
|
3837
|
+
selection: {
|
|
3838
|
+
rule: name,
|
|
3839
|
+
evidence: {
|
|
3840
|
+
rank: rank + 1,
|
|
3841
|
+
members: members.length,
|
|
3842
|
+
mean: plain(posterior.get(pick.nodeId)),
|
|
3843
|
+
children: pick.children.length
|
|
3844
|
+
}
|
|
3845
|
+
}
|
|
3846
|
+
};
|
|
3847
|
+
}
|
|
3848
|
+
};
|
|
3849
|
+
}
|
|
3850
|
+
/** Salts the aide PRNG, so its draws differ from `crowdedFrontierParent`'s at one seed. */
|
|
3851
|
+
const AIDE_SALT = 1095320645;
|
|
3852
|
+
/** At least half of the node's final cells failed as defects. */
|
|
3853
|
+
function buggy(node) {
|
|
3854
|
+
return node.outcomes > 0 && 2 * node.defects >= node.outcomes;
|
|
3855
|
+
}
|
|
3856
|
+
/** Debug edges in a row ending at `node`. */
|
|
3857
|
+
function debugDepth(node, byId) {
|
|
3858
|
+
let depth = 0;
|
|
3859
|
+
let current = node;
|
|
3860
|
+
while (current?.operator === "debug") {
|
|
3861
|
+
depth += 1;
|
|
3862
|
+
current = current.parent === null ? void 0 : byId.get(current.parent);
|
|
3863
|
+
}
|
|
3864
|
+
return depth;
|
|
3865
|
+
}
|
|
3866
|
+
/**
|
|
3867
|
+
* Improve children in a row, in registration order, that did not raise the
|
|
3868
|
+
* best posterior mean of `node`'s lineage: the nodes under its nearest draft,
|
|
3869
|
+
* or under the root. Only screened nodes with a posterior count, and a child
|
|
3870
|
+
* placed by `forkRule` restarts the run.
|
|
3871
|
+
*/
|
|
3872
|
+
function stalledRun(node, nodes, screened, posterior, forkRule) {
|
|
3873
|
+
const heads = /* @__PURE__ */ new Map();
|
|
3874
|
+
for (const entry of nodes) {
|
|
3875
|
+
const parentHead = entry.parent === null ? void 0 : heads.get(entry.parent);
|
|
3876
|
+
const starts = entry.operator === "draft" || entry.operator === "seed" || entry.operator === "derive" || parentHead === void 0;
|
|
3877
|
+
heads.set(entry.nodeId, starts ? entry.nodeId : parentHead);
|
|
3878
|
+
}
|
|
3879
|
+
const headOf = (entry) => heads.get(entry.nodeId);
|
|
3880
|
+
const lineage = headOf(node);
|
|
3881
|
+
let best = Number.NEGATIVE_INFINITY;
|
|
3882
|
+
let run = 0;
|
|
3883
|
+
for (const entry of nodes) {
|
|
3884
|
+
const mean = posterior.get(entry.nodeId)?.mean ?? null;
|
|
3885
|
+
if (mean === null || !screened.has(entry.nodeId) || headOf(entry) !== lineage) continue;
|
|
3886
|
+
if (entry.rule === forkRule) run = 0;
|
|
3887
|
+
else if (entry.operator === "improve") run = mean > best ? 0 : run + 1;
|
|
3888
|
+
best = Math.max(best, mean);
|
|
3889
|
+
}
|
|
3890
|
+
return run;
|
|
3891
|
+
}
|
|
3892
|
+
/** True when the node scored every one of `units`. */
|
|
3893
|
+
function scoredEvery(view, nodeId, units) {
|
|
3894
|
+
if (units.length === 0 || !view.complete(nodeId)) return false;
|
|
3895
|
+
const own = new Set(view.unitScores(nodeId).map((unit) => unit.unitId));
|
|
3896
|
+
return units.every((unit) => own.has(unit));
|
|
3897
|
+
}
|
|
3898
|
+
/** A standard normal draw (Box-Muller). */
|
|
3899
|
+
function normal(rng) {
|
|
3900
|
+
return Math.sqrt(-2 * Math.log(1 - rng())) * Math.cos(2 * Math.PI * rng());
|
|
3901
|
+
}
|
|
3902
|
+
function integerOption(owner, name, value, minimum) {
|
|
3903
|
+
if (!Number.isSafeInteger(value) || value < minimum) throw new TypeError(`${owner}: ${name} must be an integer of at least ${minimum}, got ${String(value)}`);
|
|
3904
|
+
return value;
|
|
3905
|
+
}
|
|
3906
|
+
/** Evidence is canonical JSON, which has one zero. */
|
|
3907
|
+
function plain(value) {
|
|
3908
|
+
const rounded = Math.round(value * 1e9) / 1e9;
|
|
3909
|
+
return rounded === 0 ? 0 : rounded;
|
|
3910
|
+
}
|
|
3911
|
+
function hillClimb(spec) {
|
|
5494
3912
|
return {
|
|
5495
3913
|
name: spec.name,
|
|
5496
3914
|
...spec.patience === void 0 ? {} : { patience: spec.patience },
|
|
5497
|
-
leader,
|
|
3915
|
+
leader: leaderOf,
|
|
5498
3916
|
expand(view) {
|
|
5499
3917
|
if (view.screening > 0) return null;
|
|
5500
|
-
const kept =
|
|
3918
|
+
const kept = leaderOf(view);
|
|
5501
3919
|
const chosen = spec.parent(view, kept);
|
|
5502
3920
|
const units = view.unitScores(chosen.nodeId);
|
|
5503
3921
|
return {
|
|
5504
3922
|
parents: [chosen.nodeId],
|
|
5505
|
-
operator: "improve",
|
|
3923
|
+
operator: spec.operator ? spec.operator(view) : "improve",
|
|
5506
3924
|
selection: {
|
|
5507
3925
|
rule: spec.name,
|
|
5508
3926
|
evidence: {
|
|
@@ -5516,6 +3934,22 @@ function hillClimb(spec) {
|
|
|
5516
3934
|
}
|
|
5517
3935
|
};
|
|
5518
3936
|
}
|
|
3937
|
+
/**
|
|
3938
|
+
* The leader every built-in policy keeps: it starts at the root, and a
|
|
3939
|
+
* screened node that dodged no unit takes the lead when it scored every unit
|
|
3940
|
+
* the leader scored and its mean on them beats the leader's. Nodes are taken
|
|
3941
|
+
* in registration order, so the leader is a pure function of the ledger. A
|
|
3942
|
+
* node an allocator has only screened cannot take the lead from one measured
|
|
3943
|
+
* on more units.
|
|
3944
|
+
*/
|
|
3945
|
+
function leaderOf(view) {
|
|
3946
|
+
let current = view.rootNodeId;
|
|
3947
|
+
for (const nodeId of view.screened) {
|
|
3948
|
+
if (nodeId === current || !view.complete(nodeId)) continue;
|
|
3949
|
+
if (beats(view, nodeId, current)) current = nodeId;
|
|
3950
|
+
}
|
|
3951
|
+
return current;
|
|
3952
|
+
}
|
|
5519
3953
|
/** True when `challenger` scored every unit `holder` scored and improves on
|
|
5520
3954
|
* `holder` over them. */
|
|
5521
3955
|
function beats(view, challenger, holder) {
|
|
@@ -7682,6 +6116,6 @@ function firstString(value) {
|
|
|
7682
6116
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
7683
6117
|
}
|
|
7684
6118
|
//#endregion
|
|
7685
|
-
export {
|
|
6119
|
+
export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
|
|
7686
6120
|
|
|
7687
|
-
//# sourceMappingURL=llm-judge-
|
|
6121
|
+
//# sourceMappingURL=llm-judge-3ncNcWNT.js.map
|