@tangle-network/agent-eval 0.192.0 → 0.193.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +1 -1
- package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
- package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
- package/dist/{benchmark-command-C458yEFK.js → benchmark-command-DVhUPU7H.js} +2 -2
- package/dist/{benchmark-command-C458yEFK.js.map → benchmark-command-DVhUPU7H.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +10 -9
- package/dist/{campaign-BVQVBEfH.js → campaign-DdE6S80e.js} +9 -8
- package/dist/{campaign-BVQVBEfH.js.map → campaign-DdE6S80e.js.map} +1 -1
- package/dist/{campaign-evidence-DcQiYhJI.js → campaign-evidence-DJc60F9C.js} +2 -2
- package/dist/{campaign-evidence-DcQiYhJI.js.map → campaign-evidence-DJc60F9C.js.map} +1 -1
- package/dist/cli.js +47 -10
- package/dist/cli.js.map +1 -1
- package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
- package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +8 -8
- package/dist/{default-registry-DdqA_-_o.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
- package/dist/{default-registry-DdqA_-_o.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DcludNJS.d.ts → define-agent-eval-DQJvVa1P.d.ts} +6 -6
- package/dist/{define-agent-eval-DcludNJS.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DQuos3Aw.js → define-agent-eval-DkPbnJEY.js} +5 -5
- package/dist/{define-agent-eval-DQuos3Aw.js.map → define-agent-eval-DkPbnJEY.js.map} +1 -1
- package/dist/diagnosis/index.js +1 -1
- package/dist/{engine-C1DIAEpn.d.ts → engine-DjRV55F9.d.ts} +2 -2
- package/dist/{engine-C1DIAEpn.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +4 -4
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +1 -1
- package/dist/{index-DBoXFaz0.d.ts → index-0jEtpSM7.d.ts} +3 -3
- package/dist/{index-DBoXFaz0.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
- package/dist/{index-DwQxdWve.d.ts → index-DMPLG226.d.ts} +89 -272
- package/dist/index-DMPLG226.d.ts.map +1 -0
- package/dist/{index-DNVzu9Vv.d.ts → index-DWQNDqPd.d.ts} +7 -7
- package/dist/{index-DNVzu9Vv.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +7 -290
- package/dist/index.js.map +1 -1
- package/dist/jev.d.ts +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/{llm-judge-Z6MkTkd0.js → llm-judge-ChbWfeWl.js} +1021 -234
- package/dist/llm-judge-ChbWfeWl.js.map +1 -0
- package/dist/{matrix-CQMMrEQ5.d.ts → matrix-CNu13gg8.d.ts} +2 -2
- package/dist/{matrix-CQMMrEQ5.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/paired-delta-test-BF-FEP8V.js +78 -0
- package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
- package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
- package/dist/{produced-state-CVmCPQDJ.js → produced-state-BqeIaIp2.js} +3 -3
- package/dist/{produced-state-CVmCPQDJ.js.map → produced-state-BqeIaIp2.js.map} +1 -1
- package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
- package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +1 -1
- package/dist/{search-ledger-Ck7tok0h.js → search-ledger-CJiAEU3j.js} +89 -27
- package/dist/search-ledger-CJiAEU3j.js.map +1 -0
- package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
- package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
- package/dist/{search-shipper-BG182p-U.js → search-shipper-DUlwlCsK.js} +2 -2
- package/dist/{search-shipper-BG182p-U.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
- package/dist/search-summary-BhY8p0Av.js +378 -0
- package/dist/search-summary-BhY8p0Av.js.map +1 -0
- package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
- package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DoayuM6D.js → skillopt-optimization-method-DXQdaBjx.js} +5 -5
- package/dist/{skillopt-optimization-method-DoayuM6D.js.map → skillopt-optimization-method-DXQdaBjx.js.map} +1 -1
- package/dist/{statistical-heldout-C1Ej5Ce8.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
- package/dist/{statistical-heldout-C1Ej5Ce8.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
- package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
- package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
- package/dist/{store-tool-spans-ByLdU26j.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
- package/dist/{store-tool-spans-ByLdU26j.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
- package/dist/{tool-groups-DiBi16ns.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
- package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-C1CAD70_.d.ts → types-BWwRXofp.d.ts} +328 -4
- package/dist/types-BWwRXofp.d.ts.map +1 -0
- package/dist/{types-ByZslIlr.d.ts → types-qoxUfKQg.d.ts} +2 -2
- package/dist/{types-ByZslIlr.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
- package/docs/search-ledger.md +71 -6
- package/package.json +1 -1
- package/dist/claim-BGOUqoMq.d.ts +0 -38
- package/dist/claim-BGOUqoMq.d.ts.map +0 -1
- package/dist/index-DwQxdWve.d.ts.map +0 -1
- package/dist/llm-judge-Z6MkTkd0.js.map +0 -1
- package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
- package/dist/search-ledger-Ck7tok0h.js.map +0 -1
- package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
- package/dist/tool-groups-DiBi16ns.d.ts.map +0 -1
- package/dist/types-C1CAD70_.d.ts.map +0 -1
|
@@ -2,23 +2,23 @@ import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
|
|
|
2
2
|
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
|
|
3
3
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
4
4
|
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-
|
|
5
|
+
import { v as wilson } from "./paired-arms-JSDcs2zx.js";
|
|
6
|
+
import { t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
|
|
7
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-DANVmyug.js";
|
|
8
8
|
import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
|
|
9
9
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
10
10
|
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-tNnlQIrD.js";
|
|
11
11
|
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
|
|
12
|
-
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-
|
|
12
|
+
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-DJc60F9C.js";
|
|
13
13
|
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
14
14
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
15
|
-
import { a as SearchLedgerError } from "./search-ledger-file-DY0AXX3W.js";
|
|
16
15
|
import { C as fsCampaignStorage, S as createRunCostLedger, g as isRecord, h as isExternalTextCandidate, x as acquireSingleRunLock } from "./external-optimizer-subprocess-kbqC-iWf.js";
|
|
17
16
|
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-V9Zqbq0J.js";
|
|
18
17
|
import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-B7rt9Zot.js";
|
|
19
18
|
import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
|
|
20
19
|
import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
|
|
21
|
-
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest,
|
|
20
|
+
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, t as FileSearchLedger, u as searchNodeId } from "./search-ledger-CJiAEU3j.js";
|
|
21
|
+
import { c as seedFromDigest, i as estimateNode, n as searchProposerView, s as searchPosterior, t as renderSearchSummary } from "./search-summary-BhY8p0Av.js";
|
|
22
22
|
import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-KYQBh9C8.js";
|
|
23
23
|
import { createHash } from "node:crypto";
|
|
24
24
|
import { writeFileSync } from "node:fs";
|
|
@@ -1164,8 +1164,8 @@ function detectCalibrationDrift(runs, opts) {
|
|
|
1164
1164
|
alpha,
|
|
1165
1165
|
recentN: recent.length,
|
|
1166
1166
|
historyN: historical.length,
|
|
1167
|
-
recentMean: mean$
|
|
1168
|
-
historyMean: mean$
|
|
1167
|
+
recentMean: mean$3(recent),
|
|
1168
|
+
historyMean: mean$3(historical)
|
|
1169
1169
|
}
|
|
1170
1170
|
}],
|
|
1171
1171
|
evaluation: {
|
|
@@ -1402,7 +1402,7 @@ function chiSquareCritical(df, alpha) {
|
|
|
1402
1402
|
}
|
|
1403
1403
|
return TABLE[10][idx];
|
|
1404
1404
|
}
|
|
1405
|
-
function mean$
|
|
1405
|
+
function mean$3(xs) {
|
|
1406
1406
|
if (xs.length === 0) return 0;
|
|
1407
1407
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
1408
1408
|
}
|
|
@@ -2559,249 +2559,757 @@ function uniform(options = {}) {
|
|
|
2559
2559
|
};
|
|
2560
2560
|
return {
|
|
2561
2561
|
name: reps === 1 ? "uniform" : `uniform(reps=${reps})`,
|
|
2562
|
+
reps,
|
|
2562
2563
|
plan: (state, nodeId) => cells(state, nodeId === state.rootNodeId),
|
|
2563
2564
|
screenSize: (state) => cells(state, false).length
|
|
2564
2565
|
};
|
|
2565
2566
|
}
|
|
2566
2567
|
//#endregion
|
|
2567
|
-
//#region src/
|
|
2568
|
+
//#region src/paired-promotion-power.ts
|
|
2568
2569
|
/**
|
|
2569
|
-
*
|
|
2570
|
+
* @module
|
|
2571
|
+
* Joint power of a configured {@link decidePairedPromotion} call, computed by
|
|
2572
|
+
* running that call on data drawn from a registered alternative.
|
|
2573
|
+
*
|
|
2574
|
+
* Why a simulation and not a formula: `decidePairedPromotion` is not one test.
|
|
2575
|
+
* It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
|
|
2576
|
+
* paired bootstrap at or above it) and by shape (Tango's score interval on a
|
|
2577
|
+
* two-point outcome), and on top of the deciding interval it applies a
|
|
2578
|
+
* sufficiency floor, a zero-width refusal and McNemar's exact veto. A
|
|
2579
|
+
* preregistration usually seals two such calls, a continuous primary and a
|
|
2580
|
+
* pass/fail veto drawn from the same episodes, and promotes only when both
|
|
2581
|
+
* promote. The sample size at which THAT procedure reaches a target power is
|
|
2582
|
+
* not the power of any one channel: the sign-test power at n can be 0.8 while
|
|
2583
|
+
* the promotion probability is well below it because the veto binds
|
|
2584
|
+
* (agent-eval#785). Closed forms exist for single channels only, and each one
|
|
2585
|
+
* shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
|
|
2586
|
+
* Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
|
|
2587
|
+
* pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
|
|
2588
|
+
* So the only faithful power is the procedure's own promotion rate under the
|
|
2589
|
+
* alternative, and that is what this module computes: draw n pairs from the
|
|
2590
|
+
* registered law, run every configured call on them exactly as the caller
|
|
2591
|
+
* will, repeat, count.
|
|
2570
2592
|
*
|
|
2571
|
-
*
|
|
2572
|
-
*
|
|
2573
|
-
*
|
|
2574
|
-
*
|
|
2575
|
-
*
|
|
2576
|
-
*
|
|
2577
|
-
* cells report the same bits.
|
|
2593
|
+
* The alternative is DECLARATIVE and serializable, never a sampler function,
|
|
2594
|
+
* so a preregistration can seal it as bytes: a mixture of cells, each with a
|
|
2595
|
+
* probability, an optional pass/fail outcome per arm for the binary calls,
|
|
2596
|
+
* and a law for the candidate-minus-baseline delta on the continuous scale.
|
|
2597
|
+
* Drawing the cell first and the delta given the cell is what makes the two
|
|
2598
|
+
* channels one joint law rather than two marginals.
|
|
2578
2599
|
*
|
|
2579
|
-
* `
|
|
2580
|
-
*
|
|
2581
|
-
*
|
|
2600
|
+
* Everything is seeded (`mulberry32`), so a sizing is reproducible from its
|
|
2601
|
+
* inputs, and every estimate carries its Monte Carlo standard error and a
|
|
2602
|
+
* Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
|
|
2582
2603
|
*/
|
|
2583
|
-
const CONFIDENCE = .95;
|
|
2584
|
-
const RESAMPLES = 2e3;
|
|
2585
|
-
/** 6 at 95 %: the smallest sample an exact one-sided sign test can resolve. */
|
|
2586
|
-
const DESCRIPTIVE_FROM = minimumPairsForPairedDeltaTest(CONFIDENCE);
|
|
2587
|
-
/** The estimator every `NodeEstimate` names. */
|
|
2588
|
-
const SEARCH_ESTIMATOR = {
|
|
2589
|
-
uri: "npm:@tangle-network/agent-eval#estimateNode",
|
|
2590
|
-
revision: hashCanonical({
|
|
2591
|
-
name: "tangle.estimate-node.2026-09",
|
|
2592
|
-
unit: "mean of the unit’s scored cells, summed in cellId order",
|
|
2593
|
-
pairing: "units both nodes scored, in unitId order",
|
|
2594
|
-
statistic: DECISION_PAIRED_DELTA_STATISTIC,
|
|
2595
|
-
interval: "percentile paired bootstrap",
|
|
2596
|
-
confidence: CONFIDENCE,
|
|
2597
|
-
resamples: RESAMPLES,
|
|
2598
|
-
rng: "mulberry32",
|
|
2599
|
-
seed: "the first 32 bits of cellSetDigest",
|
|
2600
|
-
signTest: "exact, one-sided toward improvement in the objective direction",
|
|
2601
|
-
methods: {
|
|
2602
|
-
none: "below 2 pairs",
|
|
2603
|
-
insufficient: `below ${DESCRIPTIVE_FROM} pairs`,
|
|
2604
|
-
descriptive: `below 20 pairs`,
|
|
2605
|
-
bootstrap: `from 20 pairs`
|
|
2606
|
-
},
|
|
2607
|
-
indeterminate: "every paired delta equal: no interval and no sign p"
|
|
2608
|
-
})
|
|
2609
|
-
};
|
|
2610
|
-
/** The method a paired sample of `pairs` units supports. */
|
|
2611
|
-
function searchEstimateMethod(pairs) {
|
|
2612
|
-
if (pairs < 2) return "none";
|
|
2613
|
-
if (pairs < DESCRIPTIVE_FROM) return "insufficient";
|
|
2614
|
-
if (pairs < 20) return "descriptive";
|
|
2615
|
-
return "bootstrap";
|
|
2616
|
-
}
|
|
2617
2604
|
/**
|
|
2618
|
-
*
|
|
2619
|
-
*
|
|
2620
|
-
* was read in.
|
|
2605
|
+
* Joint power of the configured calls at `n` under the registered alternative:
|
|
2606
|
+
* the fraction of simulated data sets on which every call promotes.
|
|
2621
2607
|
*/
|
|
2622
|
-
function
|
|
2623
|
-
const
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2608
|
+
function pairedPromotionPower(options) {
|
|
2609
|
+
const n = options.n;
|
|
2610
|
+
if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
|
|
2611
|
+
const simulations = options.simulations ?? 1e3;
|
|
2612
|
+
if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
|
|
2613
|
+
const seed = options.seed ?? 1;
|
|
2614
|
+
if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
|
|
2615
|
+
const calls = validateCalls(options.calls);
|
|
2616
|
+
const alternative = validateAlternative(options.alternative, calls);
|
|
2617
|
+
const rng = mulberry32(seed);
|
|
2618
|
+
let promotions = 0;
|
|
2619
|
+
let refusals = 0;
|
|
2620
|
+
const perCall = calls.map(() => ({
|
|
2621
|
+
promote: 0,
|
|
2622
|
+
insufficient: 0,
|
|
2623
|
+
indeterminate: 0,
|
|
2624
|
+
exactTestVetoes: 0,
|
|
2625
|
+
hold: 0,
|
|
2626
|
+
methods: /* @__PURE__ */ new Map()
|
|
2627
|
+
}));
|
|
2628
|
+
for (let s = 0; s < simulations; s++) {
|
|
2629
|
+
const pairs = drawPairs(alternative, n, rng);
|
|
2630
|
+
let all = true;
|
|
2631
|
+
let anyRefusal = false;
|
|
2632
|
+
for (let c = 0; c < calls.length; c++) {
|
|
2633
|
+
const call = calls[c];
|
|
2634
|
+
const decision = runCall(call, pairs, n);
|
|
2635
|
+
const tally = perCall[c];
|
|
2636
|
+
tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
|
|
2637
|
+
if (decision.promote) tally.promote++;
|
|
2638
|
+
else {
|
|
2639
|
+
if (!decision.sufficient) tally.insufficient++;
|
|
2640
|
+
if (decision.indeterminate) tally.indeterminate++;
|
|
2641
|
+
if (decision.exactTestVetoes) tally.exactTestVetoes++;
|
|
2642
|
+
if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
|
|
2643
|
+
}
|
|
2644
|
+
if (!decision.promote) all = false;
|
|
2645
|
+
if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
|
|
2646
|
+
}
|
|
2647
|
+
if (all) promotions++;
|
|
2648
|
+
else if (anyRefusal) refusals++;
|
|
2649
|
+
}
|
|
2650
|
+
const power = promotions / simulations;
|
|
2651
|
+
const interval = wilson(promotions, simulations, .95);
|
|
2652
|
+
return {
|
|
2653
|
+
n,
|
|
2654
|
+
simulations,
|
|
2655
|
+
seed,
|
|
2656
|
+
power,
|
|
2657
|
+
standardError: Math.sqrt(power * (1 - power) / simulations),
|
|
2658
|
+
low: interval.lower,
|
|
2659
|
+
high: interval.upper,
|
|
2660
|
+
refuse: refusals / simulations,
|
|
2661
|
+
hold: (simulations - promotions - refusals) / simulations,
|
|
2662
|
+
calls: perCall.map((tally, c) => ({
|
|
2663
|
+
outcome: calls[c].outcome,
|
|
2664
|
+
promote: tally.promote / simulations,
|
|
2665
|
+
insufficient: tally.insufficient / simulations,
|
|
2666
|
+
indeterminate: tally.indeterminate / simulations,
|
|
2667
|
+
exactTestVetoes: tally.exactTestVetoes / simulations,
|
|
2668
|
+
hold: tally.hold / simulations,
|
|
2669
|
+
methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
|
|
2670
|
+
}))
|
|
2671
|
+
};
|
|
2635
2672
|
}
|
|
2636
2673
|
/**
|
|
2637
|
-
*
|
|
2638
|
-
*
|
|
2639
|
-
*
|
|
2640
|
-
*
|
|
2641
|
-
*
|
|
2642
|
-
*
|
|
2643
|
-
* the
|
|
2644
|
-
*
|
|
2645
|
-
* - `none` (0 or 1 pair): no delta.
|
|
2646
|
-
* - `insufficient` (2 to 5): the delta only.
|
|
2647
|
-
* - `descriptive` (6 to 19): the bootstrap interval as spread, and the sign-test p.
|
|
2648
|
-
* - `bootstrap` (20 or more): the decision-grade bootstrap interval.
|
|
2649
|
-
*
|
|
2650
|
-
* When every paired delta is equal the interval would have zero width, which
|
|
2651
|
-
* is an absence of evidence; the estimate is then `indeterminate` and carries
|
|
2652
|
-
* neither an interval nor a p-value.
|
|
2674
|
+
* First n at which {@link pairedPromotionPower} reaches the target, scanning
|
|
2675
|
+
* every n from `minPairs` upward. Both the point-estimate answer and the
|
|
2676
|
+
* confirmed lower-bound answer are returned; the scan runs until the latter is
|
|
2677
|
+
* reached (so the curve past `n` is retained) or `maxPairs` is exhausted. A
|
|
2678
|
+
* candidate selected by the scan's bound is certified only by an independent
|
|
2679
|
+
* simulation at that n on a different seed; a candidate that fails it is
|
|
2680
|
+
* recorded and the scan continues.
|
|
2653
2681
|
*/
|
|
2654
|
-
function
|
|
2655
|
-
const
|
|
2656
|
-
if (
|
|
2657
|
-
|
|
2658
|
-
const
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
const
|
|
2662
|
-
const
|
|
2663
|
-
const
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2682
|
+
function requiredPairsForPairedPromotion(options) {
|
|
2683
|
+
const target = options.target ?? .8;
|
|
2684
|
+
if (!Number.isFinite(target) || target <= 0 || target >= 1) throw new Error(`requiredPairsForPairedPromotion: target must be in (0,1), got ${target}`);
|
|
2685
|
+
const minPairs = options.minPairs ?? 1;
|
|
2686
|
+
const maxPairs = options.maxPairs ?? 200;
|
|
2687
|
+
if (!Number.isInteger(minPairs) || minPairs < 1) throw new Error(`requiredPairsForPairedPromotion: minPairs must be a positive integer, got ${minPairs}`);
|
|
2688
|
+
if (!Number.isInteger(maxPairs) || maxPairs < minPairs) throw new Error(`requiredPairsForPairedPromotion: maxPairs must be an integer >= minPairs, got ${maxPairs}`);
|
|
2689
|
+
const seed = options.seed ?? 1;
|
|
2690
|
+
const curve = [];
|
|
2691
|
+
const rejected = [];
|
|
2692
|
+
let n = null;
|
|
2693
|
+
let nAtLowerBound = null;
|
|
2694
|
+
let confirmation = null;
|
|
2695
|
+
for (let candidate = minPairs; candidate <= maxPairs; candidate++) {
|
|
2696
|
+
const result = pairedPromotionPower({
|
|
2697
|
+
n: candidate,
|
|
2698
|
+
alternative: options.alternative,
|
|
2699
|
+
calls: options.calls,
|
|
2700
|
+
simulations: options.simulations,
|
|
2701
|
+
seed
|
|
2702
|
+
});
|
|
2703
|
+
curve.push(result);
|
|
2704
|
+
if (n === null && result.power >= target) n = candidate;
|
|
2705
|
+
if (result.low >= target) {
|
|
2706
|
+
const check = pairedPromotionPower({
|
|
2707
|
+
n: candidate,
|
|
2708
|
+
alternative: options.alternative,
|
|
2709
|
+
calls: options.calls,
|
|
2710
|
+
simulations: options.simulations,
|
|
2711
|
+
seed: confirmationSeed(seed, candidate)
|
|
2712
|
+
});
|
|
2713
|
+
if (check.low >= target) {
|
|
2714
|
+
nAtLowerBound = candidate;
|
|
2715
|
+
confirmation = check;
|
|
2716
|
+
break;
|
|
2717
|
+
}
|
|
2718
|
+
rejected.push(check);
|
|
2719
|
+
}
|
|
2720
|
+
}
|
|
2721
|
+
return {
|
|
2722
|
+
target,
|
|
2723
|
+
n,
|
|
2724
|
+
nAtLowerBound,
|
|
2725
|
+
confirmation,
|
|
2726
|
+
curve,
|
|
2727
|
+
rejected
|
|
2671
2728
|
};
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2729
|
+
}
|
|
2730
|
+
/** A seed for the confirmation at `n` that never equals the scan seed. */
|
|
2731
|
+
function confirmationSeed(seed, n) {
|
|
2732
|
+
return (Math.imul((seed | 0) ^ 2654435769, 2246822507) + Math.imul(n, 3266489909) | 1) >>> 0;
|
|
2733
|
+
}
|
|
2734
|
+
function drawPairs(alternative, n, rng) {
|
|
2735
|
+
const baseline = alternative.baseline ?? {
|
|
2736
|
+
kind: "point",
|
|
2737
|
+
value: 0
|
|
2678
2738
|
};
|
|
2679
|
-
const
|
|
2680
|
-
const
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2739
|
+
const control = new Array(n);
|
|
2740
|
+
const candidate = new Array(n);
|
|
2741
|
+
const controlPass = new Array(n);
|
|
2742
|
+
const candidatePass = new Array(n);
|
|
2743
|
+
for (let i = 0; i < n; i++) {
|
|
2744
|
+
const cell = drawCell(alternative.cells, rng);
|
|
2745
|
+
const base = drawValue(baseline, rng);
|
|
2746
|
+
control[i] = base;
|
|
2747
|
+
candidate[i] = base + drawValue(cell.delta, rng);
|
|
2748
|
+
controlPass[i] = cell.pass?.control ?? false;
|
|
2749
|
+
candidatePass[i] = cell.pass?.treatment ?? false;
|
|
2750
|
+
}
|
|
2689
2751
|
return {
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
indeterminate
|
|
2752
|
+
control,
|
|
2753
|
+
candidate,
|
|
2754
|
+
controlPass,
|
|
2755
|
+
candidatePass
|
|
2695
2756
|
};
|
|
2696
2757
|
}
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2758
|
+
function drawCell(cells, rng) {
|
|
2759
|
+
const u = rng();
|
|
2760
|
+
let cumulative = 0;
|
|
2761
|
+
for (const cell of cells) {
|
|
2762
|
+
cumulative += cell.probability;
|
|
2763
|
+
if (u < cumulative) return cell;
|
|
2764
|
+
}
|
|
2765
|
+
return cells[cells.length - 1];
|
|
2766
|
+
}
|
|
2767
|
+
function drawValue(law, rng) {
|
|
2768
|
+
switch (law.kind) {
|
|
2769
|
+
case "point": return law.value;
|
|
2770
|
+
case "atoms": {
|
|
2771
|
+
const u = rng();
|
|
2772
|
+
let cumulative = 0;
|
|
2773
|
+
for (const atom of law.atoms) {
|
|
2774
|
+
cumulative += atom.probability;
|
|
2775
|
+
if (u < cumulative) return atom.value;
|
|
2776
|
+
}
|
|
2777
|
+
return law.atoms[law.atoms.length - 1].value;
|
|
2778
|
+
}
|
|
2779
|
+
case "normal": {
|
|
2780
|
+
const u1 = 1 - rng();
|
|
2781
|
+
const u2 = rng();
|
|
2782
|
+
return law.mean + law.sd * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
}
|
|
2786
|
+
function runCall(call, pairs, n) {
|
|
2787
|
+
const options = {
|
|
2788
|
+
minPairs: n,
|
|
2789
|
+
...call.options
|
|
2790
|
+
};
|
|
2791
|
+
if (call.outcome === "pass") {
|
|
2792
|
+
const scale = call.options.binaryScale;
|
|
2793
|
+
return decidePairedPromotion(pairs.controlPass.map((p) => p ? scale : 0), pairs.candidatePass.map((p) => p ? scale : 0), options);
|
|
2794
|
+
}
|
|
2795
|
+
return decidePairedPromotion(pairs.control, pairs.candidate, options);
|
|
2796
|
+
}
|
|
2797
|
+
function validateCalls(calls) {
|
|
2798
|
+
if (!Array.isArray(calls) || calls.length === 0) throw new Error("pairedPromotionPower: at least one call is required");
|
|
2799
|
+
calls.forEach((call, i) => {
|
|
2800
|
+
if (call.outcome !== "delta" && call.outcome !== "pass") throw new Error(`pairedPromotionPower: calls[${i}].outcome must be 'delta' or 'pass'`);
|
|
2801
|
+
if (call.options === null || typeof call.options !== "object") throw new Error(`pairedPromotionPower: calls[${i}].options must be an object`);
|
|
2802
|
+
if (call.options.minPairs !== void 0 && (!Number.isInteger(call.options.minPairs) || call.options.minPairs < 1)) throw new Error(`pairedPromotionPower: calls[${i}].options.minPairs must be a positive integer`);
|
|
2803
|
+
if (call.outcome === "pass") {
|
|
2804
|
+
const scale = call.options.binaryScale;
|
|
2805
|
+
if (scale === void 0 || !Number.isFinite(scale) || scale <= 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call and must declare a finite positive options.binaryScale`);
|
|
2806
|
+
if (call.options.continuous) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; continuous cannot accompany it`);
|
|
2807
|
+
if (call.options.statistic === "median") throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; binaryScale requires the mean statistic, not median`);
|
|
2808
|
+
} else if (call.options.binaryScale !== void 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'delta' call and cannot declare binaryScale; use a 'pass' call for a two-point outcome`);
|
|
2713
2809
|
});
|
|
2810
|
+
return calls;
|
|
2811
|
+
}
|
|
2812
|
+
function validateAlternative(alternative, calls) {
|
|
2813
|
+
if (!alternative || !Array.isArray(alternative.cells) || alternative.cells.length === 0) throw new Error("pairedPromotionPower: alternative.cells must be a non-empty array");
|
|
2814
|
+
const needsPass = calls.some((call) => call.outcome === "pass");
|
|
2815
|
+
let total = 0;
|
|
2816
|
+
alternative.cells.forEach((cell, i) => {
|
|
2817
|
+
if (!Number.isFinite(cell.probability) || cell.probability < 0) throw new Error(`pairedPromotionPower: cells[${i}].probability must be a finite number >= 0`);
|
|
2818
|
+
total += cell.probability;
|
|
2819
|
+
if (needsPass) {
|
|
2820
|
+
if (!cell.pass || typeof cell.pass.control !== "boolean" || typeof cell.pass.treatment !== "boolean") throw new Error(`pairedPromotionPower: cells[${i}].pass must give control and treatment booleans when a 'pass' call is configured`);
|
|
2821
|
+
}
|
|
2822
|
+
validateLaw(cell.delta, `cells[${i}].delta`);
|
|
2823
|
+
});
|
|
2824
|
+
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: cell probabilities must sum to 1, got ${total}`);
|
|
2825
|
+
if (alternative.baseline !== void 0) validateLaw(alternative.baseline, "baseline");
|
|
2826
|
+
return alternative;
|
|
2827
|
+
}
|
|
2828
|
+
function validateLaw(law, where) {
|
|
2829
|
+
if (!law || typeof law !== "object") throw new Error(`pairedPromotionPower: ${where} must be a law object`);
|
|
2830
|
+
switch (law.kind) {
|
|
2831
|
+
case "point":
|
|
2832
|
+
if (!Number.isFinite(law.value)) throw new Error(`pairedPromotionPower: ${where}.value must be finite`);
|
|
2833
|
+
return;
|
|
2834
|
+
case "atoms": {
|
|
2835
|
+
if (!Array.isArray(law.atoms) || law.atoms.length === 0) throw new Error(`pairedPromotionPower: ${where}.atoms must be a non-empty array`);
|
|
2836
|
+
let total = 0;
|
|
2837
|
+
law.atoms.forEach((atom, i) => {
|
|
2838
|
+
if (!Number.isFinite(atom.value)) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].value must be finite`);
|
|
2839
|
+
if (!Number.isFinite(atom.probability) || atom.probability < 0) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].probability must be a finite number >= 0`);
|
|
2840
|
+
total += atom.probability;
|
|
2841
|
+
});
|
|
2842
|
+
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: ${where}.atoms probabilities must sum to 1, got ${total}`);
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
case "normal":
|
|
2846
|
+
if (!Number.isFinite(law.mean) || !Number.isFinite(law.sd) || law.sd < 0) throw new Error(`pairedPromotionPower: ${where} needs a finite mean and a finite sd >= 0`);
|
|
2847
|
+
return;
|
|
2848
|
+
default: throw new Error(`pairedPromotionPower: ${where}.kind must be 'point', 'atoms' or 'normal'`);
|
|
2849
|
+
}
|
|
2714
2850
|
}
|
|
2851
|
+
//#endregion
|
|
2852
|
+
//#region src/campaign/search-claim.ts
|
|
2715
2853
|
/**
|
|
2716
|
-
* The
|
|
2717
|
-
* its
|
|
2718
|
-
*
|
|
2719
|
-
*
|
|
2720
|
-
*
|
|
2854
|
+
* The claim: the one statement a search makes about its nodes, made once, on
|
|
2855
|
+
* its sealed test split, after the search stops expanding.
|
|
2856
|
+
*
|
|
2857
|
+
* `planSearchClaim` fixes the claim's design from the ledger before any test
|
|
2858
|
+
* cell runs: up to 3 finalists ranked by selection mean, the estimator the
|
|
2859
|
+
* paired decision will use, the number of finalists the family-wise confidence
|
|
2860
|
+
* is divided among, and a power check on selection data. The kernel stores the
|
|
2861
|
+
* plan on the ledger's `claim` operation, so the ledger proves the design was
|
|
2862
|
+
* fixed before the test data existed. `decideSearchClaim` then reads the test
|
|
2863
|
+
* cells of the root and the finalists, which ran together, and decides each
|
|
2864
|
+
* finalist against the root with `decidePairedPromotion` at the Bonferroni
|
|
2865
|
+
* confidence `1 - 0.05 / k`.
|
|
2866
|
+
*
|
|
2867
|
+
* Every number is a pure function of the ledger: the power simulation is
|
|
2868
|
+
* seeded from the search's seed and the decisions from the digest of the test
|
|
2869
|
+
* cells they read, so a verifier with the same ledger gets the same claim.
|
|
2721
2870
|
*/
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2871
|
+
const FAMILY_CONFIDENCE = .95;
|
|
2872
|
+
const TARGET_POWER = .8;
|
|
2873
|
+
const POWER_SIMULATIONS = 500;
|
|
2874
|
+
const RESAMPLES = 2e3;
|
|
2875
|
+
/**
|
|
2876
|
+
* Every rule and parameter the claim depends on. Its digest is the claim's
|
|
2877
|
+
* revision; change a value here whenever the procedure changes, so the
|
|
2878
|
+
* revision moves with it.
|
|
2879
|
+
*/
|
|
2880
|
+
const CLAIM_DEFINITION = {
|
|
2881
|
+
name: "tangle.search-claim.2026-09",
|
|
2882
|
+
finalists: `up to 3 non-root nodes not decided invalid that scored every selection unit and beat the root's mean on the selection units they share, ranked by selection mean in the objective's direction, then registration order`,
|
|
2883
|
+
familyConfidence: FAMILY_CONFIDENCE,
|
|
2884
|
+
correction: "Bonferroni: each of k finalists against the root at 1 - (1 - familyConfidence) / k",
|
|
2885
|
+
finalistCount: "the largest k whose test cells the unspent claim reserve and headroom cover and whose power reaches the target",
|
|
2886
|
+
estimator: "binary with scale s when every train, selection and test unit has one task at one repeat and every pre-test score is 0 or s; otherwise continuous",
|
|
2887
|
+
power: {
|
|
2888
|
+
method: "pairedPromotionPower of the claim decision, simulated at the claim minimumEffect",
|
|
2889
|
+
continuous: "per-unit improvement normal with the pooled selection variance",
|
|
2890
|
+
binary: "discordant units at the pooled selection variance over 2 s^2 per side, shifted by minimumEffect / s; concordant units pass at the root selection pass rate",
|
|
2891
|
+
targetPower: TARGET_POWER,
|
|
2892
|
+
simulations: POWER_SIMULATIONS,
|
|
2893
|
+
seed: "the first 32 bits of hashCanonical([search seed, k])"
|
|
2894
|
+
},
|
|
2895
|
+
decision: {
|
|
2896
|
+
call: "decidePairedPromotion(root, finalist) on per-unit test means in the objective direction",
|
|
2897
|
+
threshold: 0,
|
|
2898
|
+
minPairs: "every test unit",
|
|
2899
|
+
resamples: RESAMPLES,
|
|
2900
|
+
seed: "the first 32 bits of the finalist's test cellSetDigest"
|
|
2901
|
+
},
|
|
2902
|
+
select: "the promoted finalist with the largest improvement; ties to the better selection rank",
|
|
2903
|
+
ship: "a promoted finalist that scored every test unit, with the root, on held-out units, under a pinned judge"
|
|
2904
|
+
};
|
|
2905
|
+
/** The claim procedure every `SearchClaim` and claim plan names. */
|
|
2906
|
+
const SEARCH_CLAIM_RULE = {
|
|
2907
|
+
uri: "npm:@tangle-network/agent-eval#searchClaim",
|
|
2908
|
+
revision: hashCanonical(CLAIM_DEFINITION)
|
|
2909
|
+
};
|
|
2910
|
+
/** The rule name decisions made by the claim record. */
|
|
2911
|
+
const SEARCH_CLAIM_RULE_NAME = CLAIM_DEFINITION.name;
|
|
2912
|
+
/**
|
|
2913
|
+
* The claim reserve a search needs: the root and 3 finalists on every test
|
|
2914
|
+
* task at `reps` repeats, each held at `cellUsd`. Declare at least this as
|
|
2915
|
+
* `budget.reservedClaimUsd`; `runSearch` refuses to start a capped search
|
|
2916
|
+
* with less.
|
|
2917
|
+
*/
|
|
2918
|
+
function searchClaimReserveUsd(input) {
|
|
2919
|
+
return 4 * input.testTasks * input.reps * input.cellUsd;
|
|
2920
|
+
}
|
|
2921
|
+
/** Fix the claim's design from a search that stopped expanding. */
|
|
2922
|
+
function planSearchClaim(state, input) {
|
|
2923
|
+
const header = requireHeader(state);
|
|
2924
|
+
const root = state.rootNodeId;
|
|
2925
|
+
const plan = (finalists, power, test, reason) => ({
|
|
2926
|
+
kind: "search-claim-plan",
|
|
2927
|
+
rule: SEARCH_CLAIM_RULE,
|
|
2928
|
+
stopReason: input.stopReason,
|
|
2929
|
+
finalists,
|
|
2930
|
+
power,
|
|
2931
|
+
test,
|
|
2932
|
+
reason
|
|
2933
|
+
});
|
|
2934
|
+
const selectionUnits = unitIds(header.splits.selection.tasks);
|
|
2935
|
+
const testUnits = unitIds(header.splits.test.tasks);
|
|
2936
|
+
if (testUnits.length === 0) return plan([], { unknown: "the search has no test split" }, "hold", "the search has no test split, so it makes no claim");
|
|
2937
|
+
if (selectionUnits.length === 0) return plan([], { unknown: "the search has no selection split to choose finalists on" }, "hold", "the search has no selection split, so no finalist can be chosen without the test data");
|
|
2938
|
+
const candidates = rankFinalists(state, selectionUnits);
|
|
2939
|
+
if (candidates.length === 0) return plan([], { unknown: "no node went to test" }, "hold", "no node scored every selection unit and beat the root's selection mean, so the search keeps the root");
|
|
2940
|
+
const minimumEffect = header.objective.claim.minimumEffect;
|
|
2941
|
+
if (minimumEffect === void 0) return plan(candidates.slice(0, 1), { unknown: "the claim declares no minimumEffect" }, "test-cannot-resolve", "the claim declares no minimumEffect, so no test split can be shown to resolve it");
|
|
2942
|
+
const { pooledVariance } = searchPosterior(state, { split: "selection" });
|
|
2943
|
+
if (pooledVariance === null) return plan(candidates.slice(0, 1), { unknown: "no node shares 2 selection units with the root, so the variance is unknown" }, "test-cannot-resolve", "the selection split gives no variance for the power check, so the test cannot be shown to resolve the minimum effect");
|
|
2944
|
+
const estimator = claimEstimator(state, input.reps);
|
|
2945
|
+
const rootPassRate = passRate(state.unitScores(root, "selection"), estimator);
|
|
2946
|
+
let best = null;
|
|
2947
|
+
let budgetBound = false;
|
|
2948
|
+
for (let k = candidates.length; k >= 1; k--) {
|
|
2949
|
+
const power = claimPower({
|
|
2950
|
+
units: testUnits.length,
|
|
2951
|
+
finalists: k,
|
|
2952
|
+
minimumEffect,
|
|
2953
|
+
pooledVariance,
|
|
2954
|
+
estimator,
|
|
2955
|
+
rootPassRate,
|
|
2956
|
+
seed: header.policy.seed
|
|
2957
|
+
});
|
|
2958
|
+
best = power;
|
|
2959
|
+
if (!power.adequate) continue;
|
|
2960
|
+
if (!input.affordable(k)) {
|
|
2961
|
+
budgetBound = true;
|
|
2743
2962
|
continue;
|
|
2744
2963
|
}
|
|
2745
|
-
|
|
2746
|
-
const gains = node.map((value, index) => sign * (value - other[index]));
|
|
2747
|
-
const mean = gains.length > 0 ? sum(gains) / gains.length : null;
|
|
2748
|
-
if (mean !== null && gains.length >= 2) {
|
|
2749
|
-
squares += sum(gains.map((gain) => (gain - mean) ** 2));
|
|
2750
|
-
degreesOfFreedom += gains.length - 1;
|
|
2751
|
-
}
|
|
2752
|
-
nodes.push({
|
|
2753
|
-
nodeId,
|
|
2754
|
-
pairs: gains.length,
|
|
2755
|
-
mean: mean === null ? null : plain(mean),
|
|
2756
|
-
variance: null
|
|
2757
|
-
});
|
|
2964
|
+
return plan(candidates.slice(0, k), power, "run", `${k} finalist${k === 1 ? "" : "s"} go to test: power ${round(power.powerAtMinimumEffect)} at an improvement of ${minimumEffect} on ${testUnits.length} test units reaches ${TARGET_POWER} at confidence ${round(bonferroni(k))} each`);
|
|
2758
2965
|
}
|
|
2759
|
-
const
|
|
2760
|
-
|
|
2761
|
-
|
|
2966
|
+
const power = best;
|
|
2967
|
+
return plan(candidates.slice(0, 1), power, "test-cannot-resolve", budgetBound ? "the unspent claim reserve and headroom do not cover the test cells of the root and one finalist" : `even one finalist reaches power ${round("adequate" in power ? power.powerAtMinimumEffect : 0)} at an improvement of ${minimumEffect} on ${testUnits.length} test units, below ${TARGET_POWER}`);
|
|
2968
|
+
}
|
|
2969
|
+
/**
|
|
2970
|
+
* Decide the claim from the test cells of the root and the finalists. Call it
|
|
2971
|
+
* once every claim cell is settled or cancelled. It returns the claim and the
|
|
2972
|
+
* terminal decisions of the root and every finalist.
|
|
2973
|
+
*/
|
|
2974
|
+
function decideSearchClaim(state, plan) {
|
|
2975
|
+
const header = requireHeader(state);
|
|
2976
|
+
const root = state.rootNodeId;
|
|
2977
|
+
const rootScored = [
|
|
2978
|
+
"train",
|
|
2979
|
+
"selection",
|
|
2980
|
+
"test"
|
|
2981
|
+
].some((split) => state.scoredCells(root, split).length > 0);
|
|
2982
|
+
const keepRoot = rootScored ? root : null;
|
|
2983
|
+
const rule = SEARCH_CLAIM_RULE_NAME;
|
|
2984
|
+
const decisions = [];
|
|
2985
|
+
const claimOf = (finalists, selected, decision, reason) => ({
|
|
2986
|
+
rule: SEARCH_CLAIM_RULE,
|
|
2987
|
+
confidence: FAMILY_CONFIDENCE,
|
|
2988
|
+
power: plan.power,
|
|
2989
|
+
finalists,
|
|
2990
|
+
selected,
|
|
2991
|
+
decision,
|
|
2992
|
+
reason
|
|
2993
|
+
});
|
|
2994
|
+
if (plan.test !== "run") {
|
|
2995
|
+
for (const finalist of plan.finalists) decisions.push({
|
|
2996
|
+
nodeId: finalist.nodeId,
|
|
2997
|
+
decision: { status: "rejected" },
|
|
2998
|
+
basis: null,
|
|
2999
|
+
rule,
|
|
3000
|
+
reason: `no test cell ran: ${plan.reason}`
|
|
3001
|
+
});
|
|
3002
|
+
decisions.push(rootDecision(root, rootScored ? "kept" : "unscored", rule, plan.reason));
|
|
3003
|
+
return {
|
|
3004
|
+
claim: claimOf(plan.finalists.map((finalist) => ({
|
|
3005
|
+
nodeId: finalist.nodeId,
|
|
3006
|
+
estimate: null,
|
|
3007
|
+
test: null,
|
|
3008
|
+
promote: false
|
|
3009
|
+
})), keepRoot, plan.test, plan.reason),
|
|
3010
|
+
decisions
|
|
3011
|
+
};
|
|
2762
3012
|
}
|
|
3013
|
+
const power = plan.power;
|
|
3014
|
+
if (!("adequate" in power)) throw new Error("a claim that runs its test cells has a known power");
|
|
3015
|
+
const k = plan.finalists.length;
|
|
3016
|
+
const confidence = bonferroni(k);
|
|
3017
|
+
const testUnits = unitIds(header.splits.test.tasks);
|
|
3018
|
+
const maximize = header.objective.direction === "maximize";
|
|
3019
|
+
const rootUnits = state.unitScores(root, "test");
|
|
3020
|
+
const tested = plan.finalists.map(({ nodeId }) => {
|
|
3021
|
+
const estimate = estimateNode(state, nodeId, {
|
|
3022
|
+
against: root,
|
|
3023
|
+
split: "test"
|
|
3024
|
+
});
|
|
3025
|
+
return {
|
|
3026
|
+
nodeId,
|
|
3027
|
+
estimate,
|
|
3028
|
+
...pairedTest({
|
|
3029
|
+
root: rootUnits,
|
|
3030
|
+
node: state.unitScores(nodeId, "test"),
|
|
3031
|
+
testUnits,
|
|
3032
|
+
confidence,
|
|
3033
|
+
estimator: power.estimator,
|
|
3034
|
+
maximize,
|
|
3035
|
+
seed: seedFromDigest(estimate.cellSetDigest)
|
|
3036
|
+
})
|
|
3037
|
+
};
|
|
3038
|
+
});
|
|
3039
|
+
const winner = tested.map((entry, rank) => ({
|
|
3040
|
+
...entry,
|
|
3041
|
+
rank
|
|
3042
|
+
})).filter((entry) => entry.promote).sort((a, b) => b.improvement - a.improvement || a.rank - b.rank)[0] ?? null;
|
|
3043
|
+
const blockers = [];
|
|
3044
|
+
if (!header.splits.heldOutUnits) blockers.push("the test units also appear in train or selection");
|
|
3045
|
+
if ("unknown" in header.objective.judge) blockers.push("the judge is not pinned");
|
|
3046
|
+
const ship = winner !== null && blockers.length === 0;
|
|
3047
|
+
const selected = ship ? winner.nodeId : keepRoot;
|
|
3048
|
+
const reason = ship ? `finalist ${winner.nodeId} beat the root on the ${testUnits.length} test units at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})` : winner !== null ? `finalist ${winner.nodeId} beat the root, but the claim cannot ship: ${blockers.join("; ")}` : `no finalist beat the root on the test split at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})`;
|
|
3049
|
+
for (const entry of tested) decisions.push({
|
|
3050
|
+
nodeId: entry.nodeId,
|
|
3051
|
+
decision: { status: ship && entry.nodeId === winner.nodeId ? "selected" : "rejected" },
|
|
3052
|
+
basis: entry.estimate,
|
|
3053
|
+
rule,
|
|
3054
|
+
reason: ship && entry.nodeId === winner.nodeId ? reason : entry.promote ? ship ? `promoted, but finalist ${winner.nodeId} improved more on the test split` : reason : entry.why
|
|
3055
|
+
});
|
|
3056
|
+
decisions.push(rootDecision(root, ship ? "replaced" : rootScored ? "kept" : "unscored", rule, reason));
|
|
2763
3057
|
return {
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
3058
|
+
claim: claimOf(tested.map((entry) => ({
|
|
3059
|
+
nodeId: entry.nodeId,
|
|
3060
|
+
estimate: entry.estimate,
|
|
3061
|
+
test: entry.test,
|
|
3062
|
+
promote: entry.promote
|
|
3063
|
+
})), selected, ship ? "ship" : "hold", reason),
|
|
3064
|
+
decisions
|
|
2769
3065
|
};
|
|
2770
3066
|
}
|
|
2771
|
-
|
|
2772
|
-
|
|
3067
|
+
/** Per-finalist confidence for `k` finalists. */
|
|
3068
|
+
function bonferroni(k) {
|
|
3069
|
+
return 1 - (1 - FAMILY_CONFIDENCE) / k;
|
|
2773
3070
|
}
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
3071
|
+
function rootDecision(root, fate, rule, reason) {
|
|
3072
|
+
return {
|
|
3073
|
+
nodeId: root,
|
|
3074
|
+
decision: { status: fate === "kept" ? "selected" : "rejected" },
|
|
3075
|
+
basis: null,
|
|
3076
|
+
rule,
|
|
3077
|
+
reason: fate === "kept" ? `the search keeps the root: ${reason}` : fate === "replaced" ? `a finalist replaces the root: ${reason}` : "the root has no scored cell, so the search keeps nothing"
|
|
2780
3078
|
};
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
3079
|
+
}
|
|
3080
|
+
/** Nodes eligible to go to test, best first, at most `SEARCH_CLAIM_MAX_FINALISTS`. */
|
|
3081
|
+
function rankFinalists(state, selectionUnits) {
|
|
3082
|
+
const sign = requireHeader(state).objective.direction === "maximize" ? 1 : -1;
|
|
3083
|
+
const root = state.rootNodeId;
|
|
3084
|
+
const rootMeans = new Map(state.unitScores(root, "selection").map((unit) => [unit.unitId, unit.mean]));
|
|
3085
|
+
const ranked = [];
|
|
3086
|
+
for (const node of state.nodes()) {
|
|
3087
|
+
if (node.nodeId === root || node.status === "invalid") continue;
|
|
3088
|
+
const units = state.unitScores(node.nodeId, "selection");
|
|
3089
|
+
if (units.length !== selectionUnits.length) continue;
|
|
3090
|
+
let gain = 0;
|
|
3091
|
+
let shared = 0;
|
|
3092
|
+
for (const unit of units) {
|
|
3093
|
+
const rootMean = rootMeans.get(unit.unitId);
|
|
3094
|
+
if (rootMean === void 0) continue;
|
|
3095
|
+
gain += sign * (unit.mean - rootMean);
|
|
3096
|
+
shared += 1;
|
|
3097
|
+
}
|
|
3098
|
+
if (shared === 0 || gain <= 0) continue;
|
|
3099
|
+
ranked.push({
|
|
3100
|
+
nodeId: node.nodeId,
|
|
3101
|
+
selectionMean: mean$2(units.map((unit) => unit.mean)),
|
|
3102
|
+
ordinal: node.ordinal
|
|
3103
|
+
});
|
|
2786
3104
|
}
|
|
2787
|
-
|
|
3105
|
+
ranked.sort((a, b) => sign * (b.selectionMean - a.selectionMean) || a.ordinal - b.ordinal);
|
|
3106
|
+
return ranked.slice(0, 3).map(({ nodeId, selectionMean }) => ({
|
|
3107
|
+
nodeId,
|
|
3108
|
+
selectionMean
|
|
3109
|
+
}));
|
|
2788
3110
|
}
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
3111
|
+
/**
|
|
3112
|
+
* The paired decision's estimator, fixed from pre-test data. A per-unit mean
|
|
3113
|
+
* is binary only when each unit holds one task at one repeat and every score
|
|
3114
|
+
* seen before the test is 0 or one positive value.
|
|
3115
|
+
*/
|
|
3116
|
+
function claimEstimator(state, reps) {
|
|
3117
|
+
const header = requireHeader(state);
|
|
3118
|
+
if (![
|
|
3119
|
+
"train",
|
|
3120
|
+
"selection",
|
|
3121
|
+
"test"
|
|
3122
|
+
].every((split) => {
|
|
3123
|
+
const tasks = header.splits[split].tasks;
|
|
3124
|
+
return unitIds(tasks).length === tasks.length;
|
|
3125
|
+
}) || reps !== 1) return { kind: "continuous" };
|
|
3126
|
+
let scale = null;
|
|
3127
|
+
for (const nodeId of state.nodeIds()) for (const split of ["train", "selection"]) for (const cell of state.scoredCells(nodeId, split)) {
|
|
3128
|
+
if (cell.score === 0) continue;
|
|
3129
|
+
if (cell.score < 0 || scale !== null && cell.score !== scale) return { kind: "continuous" };
|
|
3130
|
+
scale = cell.score;
|
|
3131
|
+
}
|
|
3132
|
+
return scale === null ? { kind: "continuous" } : {
|
|
3133
|
+
kind: "binary",
|
|
3134
|
+
scale
|
|
3135
|
+
};
|
|
3136
|
+
}
|
|
3137
|
+
function passRate(units, estimator) {
|
|
3138
|
+
if (estimator.kind !== "binary" || units.length === 0) return 0;
|
|
3139
|
+
return units.filter((unit) => unit.mean === estimator.scale).length / units.length;
|
|
3140
|
+
}
|
|
3141
|
+
function claimPower(input) {
|
|
3142
|
+
const { units, finalists, minimumEffect, pooledVariance, estimator } = input;
|
|
3143
|
+
const confidence = bonferroni(finalists);
|
|
3144
|
+
let alternative;
|
|
3145
|
+
let call;
|
|
3146
|
+
if (estimator.kind === "binary") {
|
|
3147
|
+
const s = estimator.scale;
|
|
3148
|
+
const shift = Math.min(1, minimumEffect / s);
|
|
3149
|
+
const discordance = Math.min(.5, pooledVariance / (2 * s * s));
|
|
3150
|
+
const loss = Math.max(0, discordance - shift / 2);
|
|
3151
|
+
const win = Math.min(1 - loss, loss + shift);
|
|
3152
|
+
const concordant = Math.max(0, 1 - win - loss);
|
|
3153
|
+
alternative = { cells: [
|
|
3154
|
+
{
|
|
3155
|
+
probability: win,
|
|
3156
|
+
pass: {
|
|
3157
|
+
control: false,
|
|
3158
|
+
treatment: true
|
|
3159
|
+
},
|
|
3160
|
+
delta: {
|
|
3161
|
+
kind: "point",
|
|
3162
|
+
value: s
|
|
3163
|
+
}
|
|
3164
|
+
},
|
|
3165
|
+
{
|
|
3166
|
+
probability: loss,
|
|
3167
|
+
pass: {
|
|
3168
|
+
control: true,
|
|
3169
|
+
treatment: false
|
|
3170
|
+
},
|
|
3171
|
+
delta: {
|
|
3172
|
+
kind: "point",
|
|
3173
|
+
value: -s
|
|
3174
|
+
}
|
|
3175
|
+
},
|
|
3176
|
+
{
|
|
3177
|
+
probability: concordant * input.rootPassRate,
|
|
3178
|
+
pass: {
|
|
3179
|
+
control: true,
|
|
3180
|
+
treatment: true
|
|
3181
|
+
},
|
|
3182
|
+
delta: {
|
|
3183
|
+
kind: "point",
|
|
3184
|
+
value: 0
|
|
3185
|
+
}
|
|
3186
|
+
},
|
|
3187
|
+
{
|
|
3188
|
+
probability: concordant * (1 - input.rootPassRate),
|
|
3189
|
+
pass: {
|
|
3190
|
+
control: false,
|
|
3191
|
+
treatment: false
|
|
3192
|
+
},
|
|
3193
|
+
delta: {
|
|
3194
|
+
kind: "point",
|
|
3195
|
+
value: 0
|
|
3196
|
+
}
|
|
3197
|
+
}
|
|
3198
|
+
] };
|
|
3199
|
+
call = {
|
|
3200
|
+
outcome: "pass",
|
|
3201
|
+
options: {
|
|
3202
|
+
binaryScale: s,
|
|
3203
|
+
confidence,
|
|
3204
|
+
threshold: 0
|
|
3205
|
+
}
|
|
3206
|
+
};
|
|
3207
|
+
} else {
|
|
3208
|
+
alternative = { cells: [{
|
|
3209
|
+
probability: 1,
|
|
3210
|
+
delta: {
|
|
3211
|
+
kind: "normal",
|
|
3212
|
+
mean: minimumEffect,
|
|
3213
|
+
sd: Math.sqrt(pooledVariance)
|
|
3214
|
+
}
|
|
3215
|
+
}] };
|
|
3216
|
+
call = {
|
|
3217
|
+
outcome: "delta",
|
|
3218
|
+
options: {
|
|
3219
|
+
continuous: true,
|
|
3220
|
+
confidence,
|
|
3221
|
+
threshold: 0,
|
|
3222
|
+
resamples: RESAMPLES
|
|
3223
|
+
}
|
|
3224
|
+
};
|
|
2795
3225
|
}
|
|
3226
|
+
const simulated = pairedPromotionPower({
|
|
3227
|
+
n: units,
|
|
3228
|
+
alternative,
|
|
3229
|
+
calls: [call],
|
|
3230
|
+
simulations: POWER_SIMULATIONS,
|
|
3231
|
+
seed: seedFromDigest(hashCanonical([input.seed, finalists]))
|
|
3232
|
+
});
|
|
3233
|
+
return {
|
|
3234
|
+
adequate: simulated.power >= TARGET_POWER,
|
|
3235
|
+
minimumEffect,
|
|
3236
|
+
powerAtMinimumEffect: simulated.power,
|
|
3237
|
+
targetPower: TARGET_POWER,
|
|
3238
|
+
units,
|
|
3239
|
+
finalists,
|
|
3240
|
+
pooledVariance,
|
|
3241
|
+
estimator
|
|
3242
|
+
};
|
|
3243
|
+
}
|
|
3244
|
+
/**
|
|
3245
|
+
* One finalist against the root on the test split, in the objective's
|
|
3246
|
+
* direction, requiring every test unit scored by both.
|
|
3247
|
+
*/
|
|
3248
|
+
function pairedTest(input) {
|
|
3249
|
+
const rootMeans = new Map(input.root.map((unit) => [unit.unitId, unit.mean]));
|
|
3250
|
+
const before = [];
|
|
3251
|
+
const after = [];
|
|
3252
|
+
for (const unit of input.node) {
|
|
3253
|
+
const rootMean = rootMeans.get(unit.unitId);
|
|
3254
|
+
if (rootMean === void 0) continue;
|
|
3255
|
+
before.push(input.maximize ? rootMean : unit.mean);
|
|
3256
|
+
after.push(input.maximize ? unit.mean : rootMean);
|
|
3257
|
+
}
|
|
3258
|
+
const pairs = before.length;
|
|
3259
|
+
if (pairs === 0) return {
|
|
3260
|
+
test: null,
|
|
3261
|
+
promote: false,
|
|
3262
|
+
improvement: null,
|
|
3263
|
+
why: "no test unit was scored by both it and the root"
|
|
3264
|
+
};
|
|
3265
|
+
const { estimator } = input;
|
|
3266
|
+
if (estimator.kind === "binary" && [...before, ...after].some((value) => value !== 0 && value !== estimator.scale)) return {
|
|
3267
|
+
test: null,
|
|
3268
|
+
promote: false,
|
|
3269
|
+
improvement: null,
|
|
3270
|
+
why: `a test score left the two-point scale {0, ${estimator.scale}} the claim fixed before the test`
|
|
3271
|
+
};
|
|
3272
|
+
const decision = decidePairedPromotion(before, after, {
|
|
3273
|
+
confidence: input.confidence,
|
|
3274
|
+
threshold: 0,
|
|
3275
|
+
minPairs: input.testUnits.length,
|
|
3276
|
+
resamples: RESAMPLES,
|
|
3277
|
+
seed: input.seed,
|
|
3278
|
+
...estimator.kind === "binary" ? { binaryScale: estimator.scale } : { continuous: true }
|
|
3279
|
+
});
|
|
3280
|
+
const test = {
|
|
3281
|
+
pairs,
|
|
3282
|
+
confidence: input.confidence,
|
|
3283
|
+
method: decision.method,
|
|
3284
|
+
delta: plain(input.maximize ? decision.delta : -decision.delta),
|
|
3285
|
+
interval: input.maximize ? [plain(decision.low), plain(decision.high)] : [plain(-decision.high), plain(-decision.low)]
|
|
3286
|
+
};
|
|
3287
|
+
const complete = pairs === input.testUnits.length;
|
|
3288
|
+
const promote = decision.promote && complete;
|
|
3289
|
+
const why = !complete ? `it and the root scored ${pairs} of ${input.testUnits.length} test units, and a claim needs every one` : decision.promote ? "promoted" : !decision.sufficient ? `${pairs} test units are below the ${decision.minimumPairs} its paired test needs` : decision.indeterminate ? `its test interval has no width: ${decision.indeterminateCause}` : decision.exactTestVetoes ? "McNemar's exact test vetoes the improvement" : `its improvement interval [${round(decision.low)}, ${round(decision.high)}] at confidence ${round(input.confidence)} does not exclude 0`;
|
|
3290
|
+
return {
|
|
3291
|
+
test,
|
|
3292
|
+
promote,
|
|
3293
|
+
improvement: decision.delta,
|
|
3294
|
+
why
|
|
3295
|
+
};
|
|
2796
3296
|
}
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
3297
|
+
function requireHeader(state) {
|
|
3298
|
+
const header = state.header;
|
|
3299
|
+
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
3300
|
+
if (state.rootNodeId === null) throw new Error(`search ${state.searchId} has no root node`);
|
|
3301
|
+
return header;
|
|
3302
|
+
}
|
|
3303
|
+
function unitIds(tasks) {
|
|
3304
|
+
return [...new Set(tasks.map((task) => task.unitId))];
|
|
2800
3305
|
}
|
|
2801
|
-
function
|
|
3306
|
+
function mean$2(values) {
|
|
2802
3307
|
let total = 0;
|
|
2803
3308
|
for (const value of values) total += value;
|
|
2804
|
-
return total;
|
|
3309
|
+
return total / values.length;
|
|
3310
|
+
}
|
|
3311
|
+
function round(value) {
|
|
3312
|
+
return Math.round(value * 1e4) / 1e4;
|
|
2805
3313
|
}
|
|
2806
3314
|
/** Canonical JSON has one zero; never hand it a negative one. */
|
|
2807
3315
|
function plain(value) {
|
|
@@ -2821,10 +3329,14 @@ const SEARCH_KERNEL_SOURCE = {
|
|
|
2821
3329
|
retries: "a retryable errored attempt runs again, up to maxAttempts",
|
|
2822
3330
|
resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
|
|
2823
3331
|
writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
|
|
2824
|
-
|
|
3332
|
+
divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
|
|
3333
|
+
claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
|
|
3334
|
+
close: "without a test split, the policy leader is selected when it has a scored cell; every other undecided node is rejected"
|
|
2825
3335
|
})
|
|
2826
3336
|
};
|
|
2827
3337
|
const USD_TOLERANCE = 1e-9;
|
|
3338
|
+
/** The operation that holds the claim's plan. */
|
|
3339
|
+
const CLAIM_OPERATION = "claim";
|
|
2828
3340
|
/** Settled cells a lane needs before its own cost distribution sets its estimate. */
|
|
2829
3341
|
const ESTIMATE_FROM_CELLS = 20;
|
|
2830
3342
|
const ESTIMATE_MARGIN = 1.5;
|
|
@@ -2880,6 +3392,8 @@ var SearchKernel = class {
|
|
|
2880
3392
|
completed = [];
|
|
2881
3393
|
stopReason = null;
|
|
2882
3394
|
failure = null;
|
|
3395
|
+
/** The claim's design once expansion stopped; the search then only claims. */
|
|
3396
|
+
claimPlan = null;
|
|
2883
3397
|
constructor(options) {
|
|
2884
3398
|
this.options = options;
|
|
2885
3399
|
this.recorder = options.recorder;
|
|
@@ -2930,7 +3444,12 @@ var SearchKernel = class {
|
|
|
2930
3444
|
};
|
|
2931
3445
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2932
3446
|
try {
|
|
3447
|
+
if (this.claimPlan) await this.applyClaimPlan(this.claimPlan);
|
|
2933
3448
|
await this.loop();
|
|
3449
|
+
if (!this.failure && !this.claimPlan && this.state.header.splits.test.tasks.length > 0) {
|
|
3450
|
+
await this.beginClaim();
|
|
3451
|
+
await this.loop();
|
|
3452
|
+
}
|
|
2934
3453
|
} finally {
|
|
2935
3454
|
signal?.removeEventListener("abort", onAbort);
|
|
2936
3455
|
}
|
|
@@ -2959,6 +3478,8 @@ var SearchKernel = class {
|
|
|
2959
3478
|
const root = this.state.rootNodeId;
|
|
2960
3479
|
this.artifacts.set(root, this.options.root);
|
|
2961
3480
|
this.expansionOf.set(root, -1);
|
|
3481
|
+
if (this.state.audit.cells.allocated === 0) this.assertClaimReady();
|
|
3482
|
+
await this.restoreClaimPlan();
|
|
2962
3483
|
const recorded = [];
|
|
2963
3484
|
for (let expansion = 0;; expansion++) {
|
|
2964
3485
|
const operationId = `expand-${expansion}`;
|
|
@@ -3000,11 +3521,55 @@ var SearchKernel = class {
|
|
|
3000
3521
|
if (!lane) throw new Error(`runSearch: cell ${cell.cellId} was placed on lane ${String(cell.lane)}, which the executor does not declare`);
|
|
3001
3522
|
this.adoptable.add(cell.cellId);
|
|
3002
3523
|
this.enqueue(cell);
|
|
3003
|
-
this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
|
|
3524
|
+
if (cell.stage !== "claim") this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
|
|
3004
3525
|
}
|
|
3526
|
+
if (this.claimPlan) return;
|
|
3005
3527
|
for (const expansion of recorded) await this.finishProposal(expansion);
|
|
3006
3528
|
for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
|
|
3007
|
-
for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) this.
|
|
3529
|
+
for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) await this.screenDone(nodeId);
|
|
3530
|
+
}
|
|
3531
|
+
/**
|
|
3532
|
+
* A search that will claim must be able to: finalists come from a selection
|
|
3533
|
+
* split, the power check needs the claim's minimum effect, a ship needs a
|
|
3534
|
+
* pinned judge, and under a cap the claim reserve must cover the root and 3
|
|
3535
|
+
* finalists on every test task while the rest of the cap covers one
|
|
3536
|
+
* screening round. Checked before the first cell, so a search that cannot
|
|
3537
|
+
* claim spends nothing.
|
|
3538
|
+
*/
|
|
3539
|
+
assertClaimReady() {
|
|
3540
|
+
const header = this.state.header;
|
|
3541
|
+
const testTasks = header.splits.test.tasks.length;
|
|
3542
|
+
if (testTasks === 0) return;
|
|
3543
|
+
const refuse = (why) => {
|
|
3544
|
+
throw new Error(`runSearch: search ${this.recorder.searchId} has a test split, but ${why}`);
|
|
3545
|
+
};
|
|
3546
|
+
if (header.splits.selection.tasks.length === 0) refuse("no selection split to choose its finalists on");
|
|
3547
|
+
if (header.objective.claim.minimumEffect === void 0) refuse("its claim declares no minimumEffect for the power check");
|
|
3548
|
+
if ("unknown" in header.objective.judge) refuse("its judge is not pinned; a claim with an unknown judge cannot ship, and a resumed search could mix two judges");
|
|
3549
|
+
const { maxUsd, reservedClaimUsd } = header.budget;
|
|
3550
|
+
if (maxUsd === null) return;
|
|
3551
|
+
const cellUsd = this.maxCellReservation();
|
|
3552
|
+
const need = searchClaimReserveUsd({
|
|
3553
|
+
testTasks,
|
|
3554
|
+
reps: this.options.allocation.reps,
|
|
3555
|
+
cellUsd
|
|
3556
|
+
});
|
|
3557
|
+
if (reservedClaimUsd + USD_TOLERANCE < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
|
|
3558
|
+
const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId).length;
|
|
3559
|
+
const screens = (this.options.proposer.childrenPerProposal ?? 1) * this.options.allocation.screenSize(this.state);
|
|
3560
|
+
const round = this.operationReservation().usd + (rootCells + screens) * cellUsd;
|
|
3561
|
+
if (maxUsd - reservedClaimUsd + USD_TOLERANCE < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
|
|
3562
|
+
}
|
|
3563
|
+
/** A claim an earlier process began: continue it from its stored plan. */
|
|
3564
|
+
async restoreClaimPlan() {
|
|
3565
|
+
const operation = this.state.operation(CLAIM_OPERATION);
|
|
3566
|
+
if (!operation) return;
|
|
3567
|
+
const ref = operation.artifacts.find((artifact) => artifact.role === "claim-plan");
|
|
3568
|
+
if (!ref) throw new Error(`runSearch: the ${CLAIM_OPERATION} operation has no stored plan`);
|
|
3569
|
+
const plan = this.recorder.readBlob(ref);
|
|
3570
|
+
if (!operation.recorded) await this.recordClaimOperation();
|
|
3571
|
+
this.claimPlan = plan;
|
|
3572
|
+
this.stopReason = plan.stopReason;
|
|
3008
3573
|
}
|
|
3009
3574
|
async loop() {
|
|
3010
3575
|
for (;;) {
|
|
@@ -3057,10 +3622,11 @@ var SearchKernel = class {
|
|
|
3057
3622
|
/** Fill free lane slots with queued cells. A queued cell was admitted when
|
|
3058
3623
|
* it was allocated, so starting it needs no further decision. */
|
|
3059
3624
|
dispatch() {
|
|
3060
|
-
if (this.failure || this.options.signal?.aborted
|
|
3625
|
+
if (this.failure || this.options.signal?.aborted) return;
|
|
3626
|
+
const claimOnly = this.pastDeadline();
|
|
3061
3627
|
const cap = this.state.header.budget.maxConcurrency;
|
|
3062
3628
|
for (const lane of this.lanes.values()) while (lane.inFlight < lane.capacity && (cap === null || this.inFlight.size < cap)) {
|
|
3063
|
-
const cellId = lane.queues.find((
|
|
3629
|
+
const cellId = (claimOnly ? lane.queues[STAGE_PRIORITY.claim] : lane.queues.find((waiting) => waiting.length > 0))?.shift();
|
|
3064
3630
|
if (cellId === void 0) break;
|
|
3065
3631
|
this.start(cellId, lane);
|
|
3066
3632
|
}
|
|
@@ -3125,14 +3691,40 @@ var SearchKernel = class {
|
|
|
3125
3691
|
this.enqueue(cell);
|
|
3126
3692
|
return;
|
|
3127
3693
|
}
|
|
3694
|
+
if (cell.stage === "claim") return;
|
|
3128
3695
|
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
3129
3696
|
this.pending.set(cell.nodeId, left);
|
|
3130
3697
|
if (left === 0) await this.onScreenDone(cell.nodeId);
|
|
3131
3698
|
}
|
|
3132
3699
|
async onScreenDone(nodeId) {
|
|
3133
|
-
this.
|
|
3700
|
+
if (!await this.screenDone(nodeId)) return;
|
|
3134
3701
|
if (this.stopReason === null && !this.failure) await this.allocateFor(nodeId);
|
|
3135
3702
|
}
|
|
3703
|
+
/**
|
|
3704
|
+
* A node's allocated cells all finished. A node that diverges (its train
|
|
3705
|
+
* mean rose over its parent's while its selection interval against the
|
|
3706
|
+
* parent lies wholly on the worse side) is decided `invalid` and never
|
|
3707
|
+
* becomes a parent; any other node is screened. Returns whether it was.
|
|
3708
|
+
*/
|
|
3709
|
+
async screenDone(nodeId) {
|
|
3710
|
+
if (!this.claimPlan && !this.screenedSet.has(nodeId)) {
|
|
3711
|
+
const divergence = searchDivergence(this.state, nodeId);
|
|
3712
|
+
if (divergence) {
|
|
3713
|
+
await this.recorder.decideNode({
|
|
3714
|
+
nodeId,
|
|
3715
|
+
decision: { status: "invalid" },
|
|
3716
|
+
basis: divergence.basis,
|
|
3717
|
+
rule: "divergence",
|
|
3718
|
+
reason: divergence.reason
|
|
3719
|
+
});
|
|
3720
|
+
await this.refresh();
|
|
3721
|
+
this.admitted.delete(nodeId);
|
|
3722
|
+
return false;
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
this.markScreened(nodeId);
|
|
3726
|
+
return true;
|
|
3727
|
+
}
|
|
3136
3728
|
markScreened(nodeId) {
|
|
3137
3729
|
if (this.screenedSet.has(nodeId)) return;
|
|
3138
3730
|
this.screenedSet.add(nodeId);
|
|
@@ -3192,19 +3784,140 @@ var SearchKernel = class {
|
|
|
3192
3784
|
for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
|
|
3193
3785
|
return count;
|
|
3194
3786
|
}
|
|
3787
|
+
/** Fix the claim's design, store it, and start its test cells. */
|
|
3788
|
+
async beginClaim() {
|
|
3789
|
+
await this.refresh();
|
|
3790
|
+
const stopReason = this.stopReason ?? "converged";
|
|
3791
|
+
const header = this.state.header;
|
|
3792
|
+
const reps = this.options.allocation.reps;
|
|
3793
|
+
const plan = planSearchClaim(this.state, {
|
|
3794
|
+
stopReason,
|
|
3795
|
+
reps,
|
|
3796
|
+
affordable: (finalists) => {
|
|
3797
|
+
const placed = this.claimCells(finalists, this.state.rootNodeId);
|
|
3798
|
+
const hold = placed.reduce((sum, cell) => sum + cell.reservation.usd, 0);
|
|
3799
|
+
const { headroomUsd, claimReserveUsd } = this.state.budget;
|
|
3800
|
+
const { maxCells } = header.budget;
|
|
3801
|
+
return (headroomUsd === null || hold <= headroomUsd + claimReserveUsd + USD_TOLERANCE) && (maxCells === null || this.state.audit.cells.allocated + placed.length <= maxCells);
|
|
3802
|
+
}
|
|
3803
|
+
});
|
|
3804
|
+
await this.recorder.startOperation({
|
|
3805
|
+
operationId: CLAIM_OPERATION,
|
|
3806
|
+
operationKind: "selection",
|
|
3807
|
+
reservation: null,
|
|
3808
|
+
artifacts: [this.recorder.blob("claim-plan", plan)]
|
|
3809
|
+
});
|
|
3810
|
+
await this.recordClaimOperation();
|
|
3811
|
+
this.claimPlan = plan;
|
|
3812
|
+
this.stopReason = stopReason;
|
|
3813
|
+
await this.applyClaimPlan(plan);
|
|
3814
|
+
}
|
|
3815
|
+
async recordClaimOperation() {
|
|
3816
|
+
await this.recorder.recordOperation({
|
|
3817
|
+
operationId: CLAIM_OPERATION,
|
|
3818
|
+
operationKind: "selection",
|
|
3819
|
+
execution: {
|
|
3820
|
+
kind: "deterministic",
|
|
3821
|
+
source: SEARCH_CLAIM_RULE
|
|
3822
|
+
},
|
|
3823
|
+
outcome: { status: "completed" },
|
|
3824
|
+
accounting: {
|
|
3825
|
+
tokens: {
|
|
3826
|
+
status: "known",
|
|
3827
|
+
inputTokens: 0,
|
|
3828
|
+
outputTokens: 0,
|
|
3829
|
+
cachedTokens: 0
|
|
3830
|
+
},
|
|
3831
|
+
cost: {
|
|
3832
|
+
status: "known",
|
|
3833
|
+
usd: 0,
|
|
3834
|
+
source: "free"
|
|
3835
|
+
}
|
|
3836
|
+
}
|
|
3837
|
+
});
|
|
3838
|
+
await this.refresh();
|
|
3839
|
+
}
|
|
3840
|
+
/**
|
|
3841
|
+
* Decide the plan's finalists and allocate the test cells of the root and
|
|
3842
|
+
* the finalists, interleaved by task so the arms of one task run side by
|
|
3843
|
+
* side. Allocation finishes before any claim cell starts. Idempotent.
|
|
3844
|
+
*/
|
|
3845
|
+
async applyClaimPlan(plan) {
|
|
3846
|
+
const root = this.state.rootNodeId;
|
|
3847
|
+
const bases = plan.finalists.map((finalist) => estimateNode(this.state, finalist.nodeId, {
|
|
3848
|
+
against: root,
|
|
3849
|
+
split: "selection"
|
|
3850
|
+
}));
|
|
3851
|
+
for (const [rank, finalist] of plan.finalists.entries()) await this.recorder.decideNode({
|
|
3852
|
+
nodeId: finalist.nodeId,
|
|
3853
|
+
decision: { status: "finalist" },
|
|
3854
|
+
basis: bases[rank],
|
|
3855
|
+
rule: SEARCH_CLAIM_RULE_NAME,
|
|
3856
|
+
reason: `rank ${rank + 1} of ${plan.finalists.length} by selection mean (${finalist.selectionMean}); ${plan.reason}`
|
|
3857
|
+
});
|
|
3858
|
+
await this.refresh();
|
|
3859
|
+
if (plan.test !== "run") return;
|
|
3860
|
+
const missing = this.claimCells(plan.finalists.length, root, plan).map((cell) => ({
|
|
3861
|
+
...cell,
|
|
3862
|
+
cellId: searchCellId(this.recorder.searchId, cell.nodeId, cell.plan.taskId, "test", cell.plan.rep)
|
|
3863
|
+
})).filter((cell) => !this.state.cell(cell.cellId));
|
|
3864
|
+
for (const cell of missing) await this.recorder.allocateCell({
|
|
3865
|
+
nodeId: cell.nodeId,
|
|
3866
|
+
taskId: cell.plan.taskId,
|
|
3867
|
+
split: "test",
|
|
3868
|
+
rep: cell.plan.rep,
|
|
3869
|
+
stage: "claim",
|
|
3870
|
+
lane: cell.lane.name,
|
|
3871
|
+
reservation: cell.reservation
|
|
3872
|
+
});
|
|
3873
|
+
await this.refresh();
|
|
3874
|
+
for (const cell of missing) this.enqueue(this.state.cell(cell.cellId));
|
|
3875
|
+
}
|
|
3876
|
+
/** The test cells of the root and the first `finalists` of the plan (or of
|
|
3877
|
+
* the finalist count being priced), task by task, with their lanes and holds. */
|
|
3878
|
+
claimCells(finalists, root, plan) {
|
|
3879
|
+
const header = this.state.header;
|
|
3880
|
+
const arms = [root, ...plan ? plan.finalists.map((entry) => entry.nodeId) : []];
|
|
3881
|
+
while (arms.length < finalists + 1) arms.push(root);
|
|
3882
|
+
const cells = [];
|
|
3883
|
+
for (const task of header.splits.test.tasks) for (let rep = 0; rep < this.options.allocation.reps; rep++) for (const nodeId of arms) {
|
|
3884
|
+
const cellPlan = {
|
|
3885
|
+
taskId: task.taskId,
|
|
3886
|
+
split: "test",
|
|
3887
|
+
rep,
|
|
3888
|
+
stage: "claim"
|
|
3889
|
+
};
|
|
3890
|
+
const laneName = this.options.executor.place({
|
|
3891
|
+
...cellPlan,
|
|
3892
|
+
nodeId
|
|
3893
|
+
});
|
|
3894
|
+
const lane = this.lanes.get(laneName);
|
|
3895
|
+
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
3896
|
+
cells.push({
|
|
3897
|
+
nodeId,
|
|
3898
|
+
plan: cellPlan,
|
|
3899
|
+
lane,
|
|
3900
|
+
reservation: this.cellReservation(lane)
|
|
3901
|
+
});
|
|
3902
|
+
}
|
|
3903
|
+
return cells;
|
|
3904
|
+
}
|
|
3195
3905
|
async cancelQueued(reason) {
|
|
3196
|
-
for (const lane of this.lanes.values()) for (const queue of lane.queues
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3906
|
+
for (const lane of this.lanes.values()) for (const [priority, queue] of lane.queues.entries()) {
|
|
3907
|
+
if (priority === STAGE_PRIORITY.claim) continue;
|
|
3908
|
+
for (const cellId of queue.splice(0)) {
|
|
3909
|
+
const cell = this.state.cell(cellId);
|
|
3910
|
+
if (cell.attempts === 0) {
|
|
3911
|
+
await this.recorder.cancelCell({
|
|
3912
|
+
cellId,
|
|
3913
|
+
reason
|
|
3914
|
+
});
|
|
3915
|
+
await this.refresh();
|
|
3916
|
+
}
|
|
3917
|
+
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
3918
|
+
this.pending.set(cell.nodeId, left);
|
|
3919
|
+
if (left === 0) this.markScreened(cell.nodeId);
|
|
3204
3920
|
}
|
|
3205
|
-
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
3206
|
-
this.pending.set(cell.nodeId, left);
|
|
3207
|
-
if (left === 0) this.markScreened(cell.nodeId);
|
|
3208
3921
|
}
|
|
3209
3922
|
}
|
|
3210
3923
|
/** No further attempt will run: a final outcome, a cancellation, or a
|
|
@@ -3380,6 +4093,10 @@ var SearchKernel = class {
|
|
|
3380
4093
|
}
|
|
3381
4094
|
async close() {
|
|
3382
4095
|
await this.refresh();
|
|
4096
|
+
if (this.claimPlan) {
|
|
4097
|
+
await this.closeWithClaim(this.claimPlan);
|
|
4098
|
+
return;
|
|
4099
|
+
}
|
|
3383
4100
|
const view = this.policyView();
|
|
3384
4101
|
const leader = this.options.policy.leader(view);
|
|
3385
4102
|
const { split } = view;
|
|
@@ -3408,12 +4125,39 @@ var SearchKernel = class {
|
|
|
3408
4125
|
});
|
|
3409
4126
|
await this.refresh();
|
|
3410
4127
|
}
|
|
4128
|
+
/** Close with the claim: the claim's decisions, then every other open node
|
|
4129
|
+
* rejected with its selection estimate against the root. */
|
|
4130
|
+
async closeWithClaim(plan) {
|
|
4131
|
+
const { claim, decisions } = decideSearchClaim(this.state, plan);
|
|
4132
|
+
const decided = new Set(decisions.map((decision) => decision.nodeId));
|
|
4133
|
+
const root = this.state.rootNodeId;
|
|
4134
|
+
const rest = this.state.nodes().filter((node) => !decided.has(node.nodeId) && !isTerminal(node)).map((node) => {
|
|
4135
|
+
const scored = this.state.scoredCells(node.nodeId, "selection").length > 0;
|
|
4136
|
+
return {
|
|
4137
|
+
nodeId: node.nodeId,
|
|
4138
|
+
decision: { status: "rejected" },
|
|
4139
|
+
basis: scored ? estimateNode(this.state, node.nodeId, {
|
|
4140
|
+
against: root,
|
|
4141
|
+
split: "selection"
|
|
4142
|
+
}) : null,
|
|
4143
|
+
rule: SEARCH_CLAIM_RULE_NAME,
|
|
4144
|
+
reason: scored ? "not a finalist: the claim tests at most 3 nodes that scored every selection unit and beat the root, best selection mean first" : "the search stopped before this node was measured on the selection split"
|
|
4145
|
+
};
|
|
4146
|
+
});
|
|
4147
|
+
for (const decision of [...decisions, ...rest]) await this.recorder.decideNode(decision);
|
|
4148
|
+
await this.recorder.close({
|
|
4149
|
+
reason: plan.stopReason,
|
|
4150
|
+
claim
|
|
4151
|
+
});
|
|
4152
|
+
await this.refresh();
|
|
4153
|
+
}
|
|
3411
4154
|
closedResult() {
|
|
3412
4155
|
const state = this.state;
|
|
3413
4156
|
return {
|
|
3414
4157
|
state,
|
|
3415
4158
|
leader: state.audit.selectedNodeId ?? state.rootNodeId,
|
|
3416
|
-
reason: state.closed.reason
|
|
4159
|
+
reason: state.closed.reason,
|
|
4160
|
+
claim: state.closed.claim
|
|
3417
4161
|
};
|
|
3418
4162
|
}
|
|
3419
4163
|
policyView() {
|
|
@@ -3497,6 +4241,43 @@ function searchPolicyView(state, input) {
|
|
|
3497
4241
|
})
|
|
3498
4242
|
};
|
|
3499
4243
|
}
|
|
4244
|
+
/**
|
|
4245
|
+
* The divergence rule: a node whose train mean rose over its parent's on the
|
|
4246
|
+
* train units they share while its selection interval against the parent (6
|
|
4247
|
+
* or more units, not indeterminate) lies wholly on the worse side. Train is
|
|
4248
|
+
* what the proposer reads; a gain there that selection contradicts is the
|
|
4249
|
+
* signature of fitting the feedback instead of the task.
|
|
4250
|
+
*/
|
|
4251
|
+
function searchDivergence(state, nodeId) {
|
|
4252
|
+
const header = state.header;
|
|
4253
|
+
const parent = state.node(nodeId)?.primaryParentId ?? null;
|
|
4254
|
+
if (!header || parent === null || header.splits.selection.tasks.length === 0) return null;
|
|
4255
|
+
const selection = estimateNode(state, nodeId, {
|
|
4256
|
+
against: parent,
|
|
4257
|
+
split: "selection"
|
|
4258
|
+
});
|
|
4259
|
+
if (selection.interval === null) return null;
|
|
4260
|
+
const maximize = header.objective.direction === "maximize";
|
|
4261
|
+
const [low, high] = selection.interval;
|
|
4262
|
+
if (maximize ? high >= 0 : low <= 0) return null;
|
|
4263
|
+
const parentTrain = new Map(state.unitScores(parent, "train").map((unit) => [unit.unitId, unit.mean]));
|
|
4264
|
+
let gain = 0;
|
|
4265
|
+
let shared = 0;
|
|
4266
|
+
for (const unit of state.unitScores(nodeId, "train")) {
|
|
4267
|
+
const before = parentTrain.get(unit.unitId);
|
|
4268
|
+
if (before === void 0) continue;
|
|
4269
|
+
gain += maximize ? unit.mean - before : before - unit.mean;
|
|
4270
|
+
shared += 1;
|
|
4271
|
+
}
|
|
4272
|
+
if (shared === 0 || gain <= 0) return null;
|
|
4273
|
+
return {
|
|
4274
|
+
basis: selection,
|
|
4275
|
+
reason: `its train mean rose ${Math.round(gain / shared * 1e4) / 1e4} over its parent's on ${shared} shared train unit${shared === 1 ? "" : "s"}, while its selection interval against the parent, [${round4(low)}, ${round4(high)}] on ${selection.pairs} units, lies wholly on the worse side`
|
|
4276
|
+
};
|
|
4277
|
+
}
|
|
4278
|
+
function round4(value) {
|
|
4279
|
+
return Math.round(value * 1e4) / 1e4;
|
|
4280
|
+
}
|
|
3500
4281
|
function isTerminal(node) {
|
|
3501
4282
|
return node.status !== null && node.status !== "advanced";
|
|
3502
4283
|
}
|
|
@@ -3895,7 +4676,7 @@ var SearchRecorder = class SearchRecorder {
|
|
|
3895
4676
|
kind: "operation-started",
|
|
3896
4677
|
eventId: `operation-started:${input.operationId}`,
|
|
3897
4678
|
occurredAt: this.stamp(),
|
|
3898
|
-
artifacts: [],
|
|
4679
|
+
artifacts: input.artifacts ?? [],
|
|
3899
4680
|
operationId: input.operationId,
|
|
3900
4681
|
operationKind: input.operationKind,
|
|
3901
4682
|
reservation: input.reservation ?? null
|
|
@@ -4446,6 +5227,12 @@ async function runOptimization(opts) {
|
|
|
4446
5227
|
const context = Object.freeze({
|
|
4447
5228
|
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4448
5229
|
operator: request.operator,
|
|
5230
|
+
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
5231
|
+
nodeId,
|
|
5232
|
+
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
5233
|
+
})), "parents"),
|
|
5234
|
+
train: searchProposerView(state),
|
|
5235
|
+
summary: renderSearchSummary(state, { split: "train" }),
|
|
4449
5236
|
history: proposalHistory,
|
|
4450
5237
|
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4451
5238
|
populationSize: opts.populationSize,
|
|
@@ -6463,6 +7250,6 @@ function firstString(value) {
|
|
|
6463
7250
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
6464
7251
|
}
|
|
6465
7252
|
//#endregion
|
|
6466
|
-
export {
|
|
7253
|
+
export { redTeamDataset as $, createSearchHistoryReceipt as A, decideSearchClaim as B, searchEdgeId as C, assertCompleteSearchHistory as D, SearchHistoryRequiredError as E, runSearch as F, uniform as G, searchClaimReserveUsd as H, searchDivergence as I, runFinalComparison as J, dominates as K, searchExpansionIndex as L, verifySearchHistoryArtifact as M, verifySearchHistoryReceipt as N, assertSearchHistoryAdmissionOptions as O, SEARCH_KERNEL_SOURCE as P, DEFAULT_RED_TEAM_CORPUS as Q, searchPolicyView as R, developmentClaim as S, surfaceNode as T, pairedPromotionPower as U, planSearchClaim as V, requiredPairsForPairedPromotion as W, captureJudge as X, openAutoPr as Y, defaultProductionGate as Z, isProposedCandidate as _, transientDispatchFailure as a, evaluationUnitMap as at, incumbent as b, compareOptimizationMethods as c, runCampaign as ct, combineComparisonCosts as d, tangleTracesRoot as dt, redTeamReport as et, costFromLedgerSummary as f, readCachedCell as ft, runOptimization as g, runImprovementLoop as h, computeManifestHash as ht, quotaExhaustedUntil as i, captureFinalEvidencePolicy as it, searchHistoryCoverageRow as j, assertSearchHistoryMatchesState as k, optimizationTokenUsageFromSummary as l, planCampaignRun as lt, readGepaCandidatePopulationArtifact as m, cellCachePath as mt, JudgeParseError as n, runCanaries as nt, aggregateRunScore as o, reserveFinalEvidence as ot, assertGepaCandidatePopulationSummary as p, buildCellSchedule as pt, paretoFrontier as q, isTransientTransportFailure as r, assertIndependentEvaluationSplit as rt, clamp01 as s, runEval as st, llmJudge as t, scoreRedTeamOutput as tt, executeOptimizationMethod as u, resolveRunDir as ut, labelTrustRank as v, surfaceDiff as w, SearchRecorder as x, crowdedFrontierParent as y, SEARCH_CLAIM_RULE as z };
|
|
6467
7254
|
|
|
6468
|
-
//# sourceMappingURL=llm-judge-
|
|
7255
|
+
//# sourceMappingURL=llm-judge-ChbWfeWl.js.map
|