@tangle-network/agent-eval 0.190.1 → 0.192.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-DNV_U6QR.d.ts → agent-profile-DDfIjdyS.d.ts} +3 -3
- package/dist/{agent-profile-DNV_U6QR.d.ts.map → agent-profile-DDfIjdyS.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +11 -11
- package/dist/analyst/index.js +5 -5
- package/dist/{insight-report-CvPkEIcC.d.ts → analyze-runs-CPiKLJFs.d.ts} +73 -3
- package/dist/analyze-runs-CPiKLJFs.d.ts.map +1 -0
- package/dist/{attribute-vocabulary-DLJ6303h.d.ts → attribute-vocabulary-CcXqHODT.d.ts} +23 -12
- package/dist/attribute-vocabulary-CcXqHODT.d.ts.map +1 -0
- package/dist/{benchmark-command-BXIr_k8D.js → benchmark-command-C458yEFK.js} +9 -9
- package/dist/{benchmark-command-BXIr_k8D.js.map → benchmark-command-C458yEFK.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +8 -7
- package/dist/{campaign-B7q74uIh.js → campaign-BVQVBEfH.js} +13 -325
- package/dist/campaign-BVQVBEfH.js.map +1 -0
- package/dist/{campaign-evidence-D40-LHIL.js → campaign-evidence-DcQiYhJI.js} +18 -154
- package/dist/campaign-evidence-DcQiYhJI.js.map +1 -0
- package/dist/claim-V9Zqbq0J.js +54 -0
- package/dist/claim-V9Zqbq0J.js.map +1 -0
- package/dist/cli.js +99 -2
- package/dist/cli.js.map +1 -1
- package/dist/client-wFyDisxi.d.ts +236 -0
- package/dist/client-wFyDisxi.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -11
- package/dist/contract/index.js +10 -140
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-B5HrwrPM.d.ts → default-registry-DdqA_-_o.d.ts} +6 -6
- package/dist/{default-registry-B5HrwrPM.d.ts.map → default-registry-DdqA_-_o.d.ts.map} +1 -1
- package/dist/{default-registry-16I-Bxlf.js → default-registry-xMLX160E.js} +3 -3
- package/dist/{default-registry-16I-Bxlf.js.map → default-registry-xMLX160E.js.map} +1 -1
- package/dist/{define-agent-eval-DPizhHfy.js → define-agent-eval-DQuos3Aw.js} +30 -113
- package/dist/define-agent-eval-DQuos3Aw.js.map +1 -0
- package/dist/{define-agent-eval-DblGedQk.d.ts → define-agent-eval-DcludNJS.d.ts} +25 -38
- package/dist/define-agent-eval-DcludNJS.d.ts.map +1 -0
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-Cs5LzJ-f.js → dspy-rlm-engine-BX34vPVY.js} +3 -3
- package/dist/{dspy-rlm-engine-Cs5LzJ-f.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
- package/dist/{engine-BkLCnZQX.d.ts → engine-C1DIAEpn.d.ts} +4 -4
- package/dist/{engine-BkLCnZQX.d.ts.map → engine-C1DIAEpn.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-CfpP2A4Z.d.ts → exact-types-KGxBWH4b.d.ts} +2 -2
- package/dist/{exact-types-CfpP2A4Z.d.ts.map → exact-types-KGxBWH4b.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.js +2 -1
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-BnSmrkgL.js → external-optimizer-process-D-Sab4v3.js} +3 -62
- package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
- package/dist/{external-optimizer-subprocess-kbcPu5po.js → external-optimizer-subprocess-kbqC-iWf.js} +65 -27
- package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
- package/dist/{extract-usage-BrQ8mCLX.js → extract-usage-DFATEuWW.js} +2 -2
- package/dist/{extract-usage-BrQ8mCLX.js.map → extract-usage-DFATEuWW.js.map} +1 -1
- package/dist/{feedback-trajectory-BYjh7NEM.d.ts → feedback-trajectory-BXCGTRkz.d.ts} +3 -3
- package/dist/{feedback-trajectory-BYjh7NEM.d.ts.map → feedback-trajectory-BXCGTRkz.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +66 -6
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +2 -2
- package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
- package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
- package/dist/{index-2e_MTznj.d.ts → index-DBoXFaz0.d.ts} +6 -6
- package/dist/{index-2e_MTznj.d.ts.map → index-DBoXFaz0.d.ts.map} +1 -1
- package/dist/{index-CWoIJHmy.d.ts → index-DNVzu9Vv.d.ts} +10 -104
- package/dist/index-DNVzu9Vv.d.ts.map +1 -0
- package/dist/{index-CJv56rhX.d.ts → index-DwQxdWve.d.ts} +351 -135
- package/dist/index-DwQxdWve.d.ts.map +1 -0
- package/dist/index.d.ts +16 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -11
- package/dist/jev.d.ts +2 -2
- package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
- package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
- package/dist/{kind-factory-ByZeXDqF.js → kind-factory-CdPPHtGZ.js} +10 -26
- package/dist/kind-factory-CdPPHtGZ.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-BnxAv7ed.js → llm-judge-Z6MkTkd0.js} +1805 -2012
- package/dist/llm-judge-Z6MkTkd0.js.map +1 -0
- package/dist/{matrix-CNypzP-v.d.ts → matrix-CQMMrEQ5.d.ts} +2 -2
- package/dist/{matrix-CNypzP-v.d.ts.map → matrix-CQMMrEQ5.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-CQY1pNiN.js → otel-spans-DS7t1t1J.js} +8 -19
- package/dist/otel-spans-DS7t1t1J.js.map +1 -0
- package/dist/otlp-attributes-D9ptWaIJ.js +53 -0
- package/dist/otlp-attributes-D9ptWaIJ.js.map +1 -0
- package/dist/{prime-protocol-Fwvctmhm.d.ts → prime-protocol-T0i6xcAs.d.ts} +2 -2
- package/dist/{prime-protocol-Fwvctmhm.d.ts.map → prime-protocol-T0i6xcAs.d.ts.map} +1 -1
- package/dist/{produced-state-BXjXJHJT.js → produced-state-CVmCPQDJ.js} +3 -3
- package/dist/{produced-state-BXjXJHJT.js.map → produced-state-CVmCPQDJ.js.map} +1 -1
- package/dist/{registry-mAQVtuLx.d.ts → registry-Bu3lNole.d.ts} +3 -3
- package/dist/{registry-mAQVtuLx.d.ts.map → registry-Bu3lNole.d.ts.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +1 -1
- package/dist/{run-record-DnMFyg_g.js → run-record-tNnlQIrD.js} +2 -10
- package/dist/{run-record-DnMFyg_g.js.map → run-record-tNnlQIrD.js.map} +1 -1
- package/dist/search-ledger-Ck7tok0h.js +1517 -0
- package/dist/search-ledger-Ck7tok0h.js.map +1 -0
- package/dist/search-ledger-file-DY0AXX3W.js +27 -0
- package/dist/search-ledger-file-DY0AXX3W.js.map +1 -0
- package/dist/search-shipper-BG182p-U.js +781 -0
- package/dist/search-shipper-BG182p-U.js.map +1 -0
- package/dist/{semantic-concept-judge-ClB73cei.js → semantic-concept-judge-Bg-560H0.js} +4 -4
- package/dist/{semantic-concept-judge-ClB73cei.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
- package/dist/{skillopt-optimization-method-l2vDBtKp.js → skillopt-optimization-method-DoayuM6D.js} +6 -5
- package/dist/{skillopt-optimization-method-l2vDBtKp.js.map → skillopt-optimization-method-DoayuM6D.js.map} +1 -1
- package/dist/{statistical-heldout-CLpjYffJ.d.ts → statistical-heldout-C1Ej5Ce8.d.ts} +2 -2
- package/dist/{statistical-heldout-CLpjYffJ.d.ts.map → statistical-heldout-C1Ej5Ce8.d.ts.map} +1 -1
- package/dist/{store-otlp-84DH0OEH.js → store-otlp-BPowgPm6.js} +2 -2
- package/dist/{store-otlp-84DH0OEH.js.map → store-otlp-BPowgPm6.js.map} +1 -1
- package/dist/{store-tool-spans-DUiqyHvR.js → store-tool-spans-BcC_5ef-.js} +5 -5
- package/dist/{store-tool-spans-DUiqyHvR.js.map → store-tool-spans-BcC_5ef-.js.map} +1 -1
- package/dist/{store-tool-spans-BBXs8hDP.d.ts → store-tool-spans-ByLdU26j.d.ts} +5 -9
- package/dist/store-tool-spans-ByLdU26j.d.ts.map +1 -0
- package/dist/{task-failure-attributes-B34TiMBe.js → task-failure-attributes-CEqEfuIY.js} +17 -9
- package/dist/task-failure-attributes-CEqEfuIY.js.map +1 -0
- package/dist/{tool-groups-W7MJobX_.d.ts → tool-groups-DiBi16ns.d.ts} +3 -3
- package/dist/tool-groups-DiBi16ns.d.ts.map +1 -0
- package/dist/trace-attributes.d.ts +1 -1
- package/dist/trace-attributes.js +23 -75
- package/dist/trace-attributes.js.map +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +11 -18
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +16 -16
- package/dist/traces.js.map +1 -1
- package/dist/{types-Cvqy7asY.d.ts → types-ByZslIlr.d.ts} +2 -2
- package/dist/{types-Cvqy7asY.d.ts.map → types-ByZslIlr.d.ts.map} +1 -1
- package/dist/{types-BvmWNjjs.d.ts → types-C1CAD70_.d.ts} +15 -16
- package/dist/types-C1CAD70_.d.ts.map +1 -0
- package/dist/{types-Bay1dh2o.d.ts → types-CeDfCRXp.d.ts} +11 -16
- package/dist/types-CeDfCRXp.d.ts.map +1 -0
- package/dist/wire/index.d.ts +1 -1
- package/docs/campaign-proposers.md +14 -16
- package/docs/hosted-ingest-spec.md +92 -133
- package/docs/public-api.md +46 -35
- package/docs/search-ledger.md +79 -1
- package/package.json +2 -2
- package/dist/analyze-runs-BvK4-wSs.d.ts +0 -75
- package/dist/analyze-runs-BvK4-wSs.d.ts.map +0 -1
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +0 -1
- package/dist/campaign-B7q74uIh.js.map +0 -1
- package/dist/campaign-evidence-D40-LHIL.js.map +0 -1
- package/dist/client-BhEyAdfe.d.ts +0 -202
- package/dist/client-BhEyAdfe.d.ts.map +0 -1
- package/dist/client-QwJpsNYU.js +0 -655
- package/dist/client-QwJpsNYU.js.map +0 -1
- package/dist/define-agent-eval-DPizhHfy.js.map +0 -1
- package/dist/define-agent-eval-DblGedQk.d.ts.map +0 -1
- package/dist/external-optimizer-process-BnSmrkgL.js.map +0 -1
- package/dist/external-optimizer-subprocess-kbcPu5po.js.map +0 -1
- package/dist/index-CJv56rhX.d.ts.map +0 -1
- package/dist/index-CWoIJHmy.d.ts.map +0 -1
- package/dist/insight-report-CvPkEIcC.d.ts.map +0 -1
- package/dist/kind-factory-ByZeXDqF.js.map +0 -1
- package/dist/llm-judge-BnxAv7ed.js.map +0 -1
- package/dist/otel-spans-CQY1pNiN.js.map +0 -1
- package/dist/otlp-attributes-BpC_blLF.js +0 -83
- package/dist/otlp-attributes-BpC_blLF.js.map +0 -1
- package/dist/store-tool-spans-BBXs8hDP.d.ts.map +0 -1
- package/dist/task-failure-attributes-B34TiMBe.js.map +0 -1
- package/dist/tool-groups-W7MJobX_.d.ts.map +0 -1
- package/dist/types-Bay1dh2o.d.ts.map +0 -1
- package/dist/types-BvmWNjjs.d.ts.map +0 -1
|
@@ -1,20 +1,24 @@
|
|
|
1
1
|
import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
2
|
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
|
|
3
|
+
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
3
4
|
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
|
|
4
|
-
import {
|
|
5
|
+
import { n as DECISION_PAIRED_DELTA_STATISTIC } from "./paired-tests-C8iCsioC.js";
|
|
6
|
+
import { i as pairedDeltaTest, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-6cn3bLc3.js";
|
|
5
7
|
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-GcLIPxk4.js";
|
|
6
8
|
import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
|
|
7
9
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
8
|
-
import {
|
|
10
|
+
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-tNnlQIrD.js";
|
|
9
11
|
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
+
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-DcQiYhJI.js";
|
|
13
|
+
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
12
14
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
13
|
-
import "./ledger-
|
|
14
|
-
import {
|
|
15
|
+
import { a as SearchLedgerError } from "./search-ledger-file-DY0AXX3W.js";
|
|
16
|
+
import { C as fsCampaignStorage, S as createRunCostLedger, g as isRecord, h as isExternalTextCandidate, x as acquireSingleRunLock } from "./external-optimizer-subprocess-kbqC-iWf.js";
|
|
17
|
+
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-V9Zqbq0J.js";
|
|
15
18
|
import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-B7rt9Zot.js";
|
|
16
19
|
import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
|
|
17
20
|
import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
|
|
21
|
+
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, f as searchUnitScores, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, t as FileSearchLedger, u as searchNodeId } from "./search-ledger-Ck7tok0h.js";
|
|
18
22
|
import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-KYQBh9C8.js";
|
|
19
23
|
import { createHash } from "node:crypto";
|
|
20
24
|
import { writeFileSync } from "node:fs";
|
|
@@ -1160,8 +1164,8 @@ function detectCalibrationDrift(runs, opts) {
|
|
|
1160
1164
|
alpha,
|
|
1161
1165
|
recentN: recent.length,
|
|
1162
1166
|
historyN: historical.length,
|
|
1163
|
-
recentMean: mean$
|
|
1164
|
-
historyMean: mean$
|
|
1167
|
+
recentMean: mean$2(recent),
|
|
1168
|
+
historyMean: mean$2(historical)
|
|
1165
1169
|
}
|
|
1166
1170
|
}],
|
|
1167
1171
|
evaluation: {
|
|
@@ -1398,7 +1402,7 @@ function chiSquareCritical(df, alpha) {
|
|
|
1398
1402
|
}
|
|
1399
1403
|
return TABLE[10][idx];
|
|
1400
1404
|
}
|
|
1401
|
-
function mean$
|
|
1405
|
+
function mean$2(xs) {
|
|
1402
1406
|
if (xs.length === 0) return 0;
|
|
1403
1407
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
1404
1408
|
}
|
|
@@ -2531,1438 +2535,994 @@ function paretoFrontierWithCrowding(candidates, objectives) {
|
|
|
2531
2535
|
return crowdingDistance(frontier, objectives).sort((a, b) => b.distance - a.distance);
|
|
2532
2536
|
}
|
|
2533
2537
|
//#endregion
|
|
2534
|
-
//#region src/campaign/
|
|
2535
|
-
/**
|
|
2536
|
-
*
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2538
|
+
//#region src/campaign/allocation.ts
|
|
2539
|
+
/**
|
|
2540
|
+
* Every node gets every train and selection task at every repeat: the root as
|
|
2541
|
+
* `root` cells, every other node as `train` cells (the proposer's feedback) and
|
|
2542
|
+
* `screen` cells (the policy's private evidence). This is the fixed plan the
|
|
2543
|
+
* generation loop ran and GEPA runs.
|
|
2544
|
+
*/
|
|
2545
|
+
function uniform(options = {}) {
|
|
2546
|
+
const reps = options.reps ?? 1;
|
|
2547
|
+
if (!Number.isSafeInteger(reps) || reps < 1) throw new TypeError(`uniform: reps must be a positive integer, got ${String(reps)}`);
|
|
2548
|
+
const cells = (state, root) => {
|
|
2549
|
+
const header = state.header;
|
|
2550
|
+
if (!header) return [];
|
|
2551
|
+
const plan = [];
|
|
2552
|
+
for (const split of ["train", "selection"]) for (const task of header.splits[split].tasks) for (let rep = 0; rep < reps; rep++) plan.push({
|
|
2553
|
+
taskId: task.taskId,
|
|
2554
|
+
split,
|
|
2555
|
+
rep,
|
|
2556
|
+
stage: root ? "root" : split === "train" ? "train" : "screen"
|
|
2557
|
+
});
|
|
2558
|
+
return plan;
|
|
2559
|
+
};
|
|
2560
|
+
return {
|
|
2561
|
+
name: reps === 1 ? "uniform" : `uniform(reps=${reps})`,
|
|
2562
|
+
plan: (state, nodeId) => cells(state, nodeId === state.rootNodeId),
|
|
2563
|
+
screenSize: (state) => cells(state, false).length
|
|
2564
|
+
};
|
|
2548
2565
|
}
|
|
2549
2566
|
//#endregion
|
|
2550
|
-
//#region src/campaign/
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2567
|
+
//#region src/campaign/estimate-node.ts
|
|
2568
|
+
/**
|
|
2569
|
+
* Per-node statistics of a search.
|
|
2570
|
+
*
|
|
2571
|
+
* `estimateNode` is the one paired contrast of two nodes: it averages each
|
|
2572
|
+
* node's cells inside their units, pairs the units both nodes scored, and runs
|
|
2573
|
+
* `pairedDeltaTest` on the per-unit deltas. `estimateNodeFromCells` is the same
|
|
2574
|
+
* computation over cells a caller already holds, such as rows a server read
|
|
2575
|
+
* from its database. The bootstrap seed comes from `cellSetDigest`, the digest
|
|
2576
|
+
* of exactly the cells used, so a producer and a verifier that hold the same
|
|
2577
|
+
* cells report the same bits.
|
|
2578
|
+
*
|
|
2579
|
+
* `searchPosterior` gives every node a normal posterior on its improvement over
|
|
2580
|
+
* the root, which expansion policies sample parents from. Its numbers steer
|
|
2581
|
+
* spend and claim nothing.
|
|
2582
|
+
*/
|
|
2583
|
+
const CONFIDENCE = .95;
|
|
2584
|
+
const RESAMPLES = 2e3;
|
|
2585
|
+
/** 6 at 95 %: the smallest sample an exact one-sided sign test can resolve. */
|
|
2586
|
+
const DESCRIPTIVE_FROM = minimumPairsForPairedDeltaTest(CONFIDENCE);
|
|
2587
|
+
/** The estimator every `NodeEstimate` names. */
|
|
2588
|
+
const SEARCH_ESTIMATOR = {
|
|
2589
|
+
uri: "npm:@tangle-network/agent-eval#estimateNode",
|
|
2590
|
+
revision: hashCanonical({
|
|
2591
|
+
name: "tangle.estimate-node.2026-09",
|
|
2592
|
+
unit: "mean of the unit’s scored cells, summed in cellId order",
|
|
2593
|
+
pairing: "units both nodes scored, in unitId order",
|
|
2594
|
+
statistic: DECISION_PAIRED_DELTA_STATISTIC,
|
|
2595
|
+
interval: "percentile paired bootstrap",
|
|
2596
|
+
confidence: CONFIDENCE,
|
|
2597
|
+
resamples: RESAMPLES,
|
|
2598
|
+
rng: "mulberry32",
|
|
2599
|
+
seed: "the first 32 bits of cellSetDigest",
|
|
2600
|
+
signTest: "exact, one-sided toward improvement in the objective direction",
|
|
2601
|
+
methods: {
|
|
2602
|
+
none: "below 2 pairs",
|
|
2603
|
+
insufficient: `below ${DESCRIPTIVE_FROM} pairs`,
|
|
2604
|
+
descriptive: `below 20 pairs`,
|
|
2605
|
+
bootstrap: `from 20 pairs`
|
|
2606
|
+
},
|
|
2607
|
+
indeterminate: "every paired delta equal: no interval and no sign p"
|
|
2608
|
+
})
|
|
2609
|
+
};
|
|
2610
|
+
/** The method a paired sample of `pairs` units supports. */
|
|
2611
|
+
function searchEstimateMethod(pairs) {
|
|
2612
|
+
if (pairs < 2) return "none";
|
|
2613
|
+
if (pairs < DESCRIPTIVE_FROM) return "insufficient";
|
|
2614
|
+
if (pairs < 20) return "descriptive";
|
|
2615
|
+
return "bootstrap";
|
|
2556
2616
|
}
|
|
2557
|
-
//#endregion
|
|
2558
|
-
//#region src/campaign/search-ledger-types.ts
|
|
2559
2617
|
/**
|
|
2560
|
-
*
|
|
2561
|
-
*
|
|
2562
|
-
*
|
|
2618
|
+
* Digest of the cells an estimate reads, with the contrast they serve. Cells
|
|
2619
|
+
* are listed in cellId order, so the digest names the set, not the order it
|
|
2620
|
+
* was read in.
|
|
2563
2621
|
*/
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2622
|
+
function searchCellSetDigest(input) {
|
|
2623
|
+
const list = (cells) => [...cells].sort((left, right) => compareCodeUnits(left.cellId, right.cellId)).map((cell) => [
|
|
2624
|
+
cell.cellId,
|
|
2625
|
+
cell.unitId,
|
|
2626
|
+
cell.attempt,
|
|
2627
|
+
cell.score
|
|
2628
|
+
]);
|
|
2629
|
+
return hashCanonical({
|
|
2630
|
+
contrast: [input.nodeId, input.against],
|
|
2631
|
+
split: input.split,
|
|
2632
|
+
nodeCells: list(input.nodeCells),
|
|
2633
|
+
againstCells: list(input.againstCells)
|
|
2634
|
+
});
|
|
2635
|
+
}
|
|
2567
2636
|
/**
|
|
2568
|
-
* `
|
|
2637
|
+
* The paired contrast of `nodeId` against `against` on the units both scored.
|
|
2569
2638
|
*
|
|
2570
|
-
*
|
|
2571
|
-
*
|
|
2572
|
-
*
|
|
2573
|
-
*
|
|
2574
|
-
*
|
|
2639
|
+
* `delta` is the mean per-unit difference, node minus `against`, in the
|
|
2640
|
+
* metric's own units; for a `minimize` objective an improvement is negative.
|
|
2641
|
+
* The exact sign test is one-sided toward improvement in the objective's
|
|
2642
|
+
* direction. Unscored cells (errored, cancelled or in flight) are absent from
|
|
2643
|
+
* the inputs, never zero.
|
|
2575
2644
|
*
|
|
2576
|
-
*
|
|
2577
|
-
* `
|
|
2578
|
-
*
|
|
2579
|
-
*
|
|
2580
|
-
*
|
|
2581
|
-
*
|
|
2645
|
+
* - `none` (0 or 1 pair): no delta.
|
|
2646
|
+
* - `insufficient` (2 to 5): the delta only.
|
|
2647
|
+
* - `descriptive` (6 to 19): the bootstrap interval as spread, and the sign-test p.
|
|
2648
|
+
* - `bootstrap` (20 or more): the decision-grade bootstrap interval.
|
|
2649
|
+
*
|
|
2650
|
+
* When every paired delta is equal the interval would have zero width, which
|
|
2651
|
+
* is an absence of evidence; the estimate is then `indeterminate` and carries
|
|
2652
|
+
* neither an interval nor a p-value.
|
|
2582
2653
|
*/
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2654
|
+
function estimateNodeFromCells(input) {
|
|
2655
|
+
const { nodeId, against, split, direction } = input;
|
|
2656
|
+
if (nodeId === against) throw new SearchLedgerError(`node ${nodeId} cannot be estimated against itself`);
|
|
2657
|
+
assertDistinctCells([...input.nodeCells, ...input.againstCells], nodeId, against);
|
|
2658
|
+
const cellSetDigest = searchCellSetDigest(input);
|
|
2659
|
+
const nodeUnits = searchUnitScores(input.nodeCells);
|
|
2660
|
+
const { node, other } = pairUnits(nodeUnits, unitMeans(searchUnitScores(input.againstCells)));
|
|
2661
|
+
const pairs = node.length;
|
|
2662
|
+
const method = searchEstimateMethod(pairs);
|
|
2663
|
+
const base = {
|
|
2664
|
+
against,
|
|
2593
2665
|
split,
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2666
|
+
units: nodeUnits.length,
|
|
2667
|
+
pairs,
|
|
2668
|
+
method,
|
|
2669
|
+
cellSetDigest,
|
|
2670
|
+
estimator: SEARCH_ESTIMATOR
|
|
2671
|
+
};
|
|
2672
|
+
if (method === "none") return {
|
|
2673
|
+
...base,
|
|
2674
|
+
delta: null,
|
|
2675
|
+
interval: null,
|
|
2676
|
+
exactSignP: null,
|
|
2677
|
+
indeterminate: false
|
|
2678
|
+
};
|
|
2679
|
+
const maximize = direction === "maximize";
|
|
2680
|
+
const test = pairedDeltaTest(maximize ? other : node, maximize ? node : other, {
|
|
2681
|
+
statistic: DECISION_PAIRED_DELTA_STATISTIC,
|
|
2682
|
+
confidence: CONFIDENCE,
|
|
2683
|
+
resamples: RESAMPLES,
|
|
2684
|
+
seed: seedFromDigest(cellSetDigest)
|
|
2685
|
+
});
|
|
2686
|
+
const { mean, low, high } = test.bootstrap;
|
|
2687
|
+
const indeterminate = test.indeterminate;
|
|
2688
|
+
const spread = (method === "descriptive" || method === "bootstrap") && !indeterminate;
|
|
2689
|
+
return {
|
|
2690
|
+
...base,
|
|
2691
|
+
delta: plain(maximize ? mean : -mean),
|
|
2692
|
+
interval: spread ? maximize ? [plain(low), plain(high)] : [plain(-high), plain(-low)] : null,
|
|
2693
|
+
exactSignP: method === "descriptive" && !indeterminate ? test.pValue : null,
|
|
2694
|
+
indeterminate
|
|
2695
|
+
};
|
|
2605
2696
|
}
|
|
2606
|
-
const MAX_LABEL_CHARS$1 = 200;
|
|
2607
|
-
const USD_TOLERANCE = 1e-9;
|
|
2608
|
-
const SPLITS = [
|
|
2609
|
-
"train",
|
|
2610
|
-
"selection",
|
|
2611
|
-
"test"
|
|
2612
|
-
];
|
|
2613
2697
|
/**
|
|
2614
|
-
*
|
|
2615
|
-
*
|
|
2616
|
-
*
|
|
2617
|
-
* gets the same bits.
|
|
2698
|
+
* `estimateNodeFromCells` over the cells a search state holds: `nodeId`
|
|
2699
|
+
* against `against` on one split, in the direction the search's objective
|
|
2700
|
+
* declares.
|
|
2618
2701
|
*/
|
|
2619
|
-
function
|
|
2620
|
-
const
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
unitId,
|
|
2631
|
-
sum,
|
|
2632
|
-
count: unitCells.length,
|
|
2633
|
-
mean: sum / unitCells.length
|
|
2634
|
-
};
|
|
2702
|
+
function estimateNode(state, nodeId, options) {
|
|
2703
|
+
const header = state.header;
|
|
2704
|
+
if (!header) throw new SearchLedgerError(`search ${state.searchId} has not been opened`);
|
|
2705
|
+
for (const id of [nodeId, options.against]) if (!state.hasNode(id)) throw new SearchLedgerError(`node ${id} is not in search ${state.searchId}`);
|
|
2706
|
+
return estimateNodeFromCells({
|
|
2707
|
+
nodeId,
|
|
2708
|
+
against: options.against,
|
|
2709
|
+
split: options.split,
|
|
2710
|
+
direction: header.objective.direction,
|
|
2711
|
+
nodeCells: state.scoredCells(nodeId, options.split),
|
|
2712
|
+
againstCells: state.scoredCells(options.against, options.split)
|
|
2635
2713
|
});
|
|
2636
2714
|
}
|
|
2637
2715
|
/**
|
|
2638
|
-
* The
|
|
2639
|
-
*
|
|
2640
|
-
*
|
|
2716
|
+
* The posterior every node's parent-selection draw uses. Each node's mean is
|
|
2717
|
+
* its improvement over the root on shared units; its variance is the search's
|
|
2718
|
+
* pooled between-unit variance divided by its unit count, so a node measured
|
|
2719
|
+
* on one unit has a wide posterior rather than none. The root is the reference
|
|
2720
|
+
* and sits at exactly 0.
|
|
2641
2721
|
*/
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
header
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
nodes =
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
searchId,
|
|
2665
|
-
eventCount: 0,
|
|
2666
|
-
headHash: null,
|
|
2667
|
-
status: "open",
|
|
2668
|
-
closeReason: null,
|
|
2669
|
-
nodes: 0,
|
|
2670
|
-
nodesWithoutEdge: 0,
|
|
2671
|
-
undecidedNodes: 0,
|
|
2672
|
-
selectedNodeId: null,
|
|
2673
|
-
edges: {
|
|
2674
|
-
explicit: 0,
|
|
2675
|
-
correlated: 0,
|
|
2676
|
-
unknown: 0,
|
|
2677
|
-
reproposals: 0
|
|
2678
|
-
},
|
|
2679
|
-
cells: {
|
|
2680
|
-
allocated: 0,
|
|
2681
|
-
settled: 0,
|
|
2682
|
-
cancelled: 0,
|
|
2683
|
-
open: 0
|
|
2684
|
-
},
|
|
2685
|
-
attempts: 0,
|
|
2686
|
-
outcomes: {
|
|
2687
|
-
passed: 0,
|
|
2688
|
-
failed: 0,
|
|
2689
|
-
errored: 0
|
|
2690
|
-
},
|
|
2691
|
-
operations: {
|
|
2692
|
-
started: 0,
|
|
2693
|
-
recorded: 0,
|
|
2694
|
-
open: 0
|
|
2695
|
-
},
|
|
2696
|
-
spend: {
|
|
2697
|
-
knownUsd: 0,
|
|
2698
|
-
floorUsd: 0,
|
|
2699
|
-
unknownCostCells: 0,
|
|
2700
|
-
unknownCostOperations: 0,
|
|
2701
|
-
committedUsd: 0,
|
|
2702
|
-
openReservationUsd: 0,
|
|
2703
|
-
overspendUsd: 0,
|
|
2704
|
-
unreservedCells: 0,
|
|
2705
|
-
boxMinutes: 0
|
|
2706
|
-
},
|
|
2707
|
-
tokens: {
|
|
2708
|
-
inputTokens: 0,
|
|
2709
|
-
outputTokens: 0,
|
|
2710
|
-
cachedTokens: 0,
|
|
2711
|
-
unknownTokenAttempts: 0
|
|
2712
|
-
}
|
|
2713
|
-
};
|
|
2714
|
-
}
|
|
2715
|
-
apply(entry, index) {
|
|
2716
|
-
const event = entry.event;
|
|
2717
|
-
if (this.closed) throw integrity(`event ${event.eventId} follows search-closed ${this.closed.eventId}`);
|
|
2718
|
-
const occurredAt = Date.parse(event.occurredAt);
|
|
2719
|
-
if (occurredAt < this.lastOccurredAt) throw integrity(`event ${event.eventId} occurred before the preceding event`);
|
|
2720
|
-
assertUnique(event.artifacts.map(artifactKey), "artifact receipt", event.eventId);
|
|
2721
|
-
if (event.kind === "search-opened") {
|
|
2722
|
-
if (index !== 0 || this.header) throw integrity("search-opened must be the first and only opening event");
|
|
2723
|
-
this.open(event);
|
|
2724
|
-
} else {
|
|
2725
|
-
if (!this.header) throw integrity(`event ${event.eventId} appears before search-opened`);
|
|
2726
|
-
this.applyEvent(event, index);
|
|
2722
|
+
function searchPosterior(state, options = {}) {
|
|
2723
|
+
const split = options.split ?? "selection";
|
|
2724
|
+
const header = state.header;
|
|
2725
|
+
if (!header) throw new SearchLedgerError(`search ${state.searchId} has not been opened`);
|
|
2726
|
+
const rootNodeId = state.rootNodeId;
|
|
2727
|
+
if (rootNodeId === null) throw new SearchLedgerError(`search ${state.searchId} has no root node yet`);
|
|
2728
|
+
const sign = header.objective.direction === "maximize" ? 1 : -1;
|
|
2729
|
+
const root = searchUnitScores(state.scoredCells(rootNodeId, split));
|
|
2730
|
+
const rootMeans = unitMeans(root);
|
|
2731
|
+
const nodes = [];
|
|
2732
|
+
let squares = 0;
|
|
2733
|
+
let degreesOfFreedom = 0;
|
|
2734
|
+
for (const nodeId of state.nodeIds()) {
|
|
2735
|
+
if (nodeId === rootNodeId) {
|
|
2736
|
+
const measured = root.length > 0;
|
|
2737
|
+
nodes.push({
|
|
2738
|
+
nodeId,
|
|
2739
|
+
pairs: root.length,
|
|
2740
|
+
mean: measured ? 0 : null,
|
|
2741
|
+
variance: measured ? 0 : null
|
|
2742
|
+
});
|
|
2743
|
+
continue;
|
|
2727
2744
|
}
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2745
|
+
const { node, other } = pairUnits(searchUnitScores(state.scoredCells(nodeId, split)), rootMeans);
|
|
2746
|
+
const gains = node.map((value, index) => sign * (value - other[index]));
|
|
2747
|
+
const mean = gains.length > 0 ? sum(gains) / gains.length : null;
|
|
2748
|
+
if (mean !== null && gains.length >= 2) {
|
|
2749
|
+
squares += sum(gains.map((gain) => (gain - mean) ** 2));
|
|
2750
|
+
degreesOfFreedom += gains.length - 1;
|
|
2751
|
+
}
|
|
2752
|
+
nodes.push({
|
|
2753
|
+
nodeId,
|
|
2754
|
+
pairs: gains.length,
|
|
2755
|
+
mean: mean === null ? null : plain(mean),
|
|
2756
|
+
variance: null
|
|
2757
|
+
});
|
|
2737
2758
|
}
|
|
2738
|
-
|
|
2739
|
-
|
|
2759
|
+
const pooledVariance = degreesOfFreedom > 0 ? squares / degreesOfFreedom : null;
|
|
2760
|
+
if (pooledVariance !== null) {
|
|
2761
|
+
for (const posterior of nodes) if (posterior.nodeId !== rootNodeId && posterior.pairs > 0) posterior.variance = pooledVariance / posterior.pairs;
|
|
2740
2762
|
}
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2763
|
+
return {
|
|
2764
|
+
split,
|
|
2765
|
+
rootNodeId,
|
|
2766
|
+
pooledVariance,
|
|
2767
|
+
degreesOfFreedom,
|
|
2768
|
+
nodes
|
|
2769
|
+
};
|
|
2770
|
+
}
|
|
2771
|
+
function unitMeans(units) {
|
|
2772
|
+
return new Map(units.map((unit) => [unit.unitId, unit.mean]));
|
|
2773
|
+
}
|
|
2774
|
+
/** Unit means of two nodes on the units both scored, aligned in the node's
|
|
2775
|
+
* unitId order. */
|
|
2776
|
+
function pairUnits(node, otherMeans) {
|
|
2777
|
+
const paired = {
|
|
2778
|
+
node: [],
|
|
2779
|
+
other: []
|
|
2780
|
+
};
|
|
2781
|
+
for (const unit of node) {
|
|
2782
|
+
const mean = otherMeans.get(unit.unitId);
|
|
2783
|
+
if (mean === void 0) continue;
|
|
2784
|
+
paired.node.push(unit.mean);
|
|
2785
|
+
paired.other.push(mean);
|
|
2744
2786
|
}
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2787
|
+
return paired;
|
|
2788
|
+
}
|
|
2789
|
+
function assertDistinctCells(cells, nodeId, against) {
|
|
2790
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2791
|
+
for (const cell of cells) {
|
|
2792
|
+
if (seen.has(cell.cellId)) throw new SearchLedgerError(`cell ${cell.cellId} appears twice in the estimate of ${nodeId} against ${against}`);
|
|
2793
|
+
if (!Number.isFinite(cell.score)) throw new SearchLedgerError(`cell ${cell.cellId} has a non-finite score ${cell.score}`);
|
|
2794
|
+
seen.add(cell.cellId);
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
/** The first 32 bits of a `sha256:` digest, as the bootstrap's seed. */
|
|
2798
|
+
function seedFromDigest(digest) {
|
|
2799
|
+
return Number.parseInt(digest.slice(7, 15), 16) | 0;
|
|
2800
|
+
}
|
|
2801
|
+
function sum(values) {
|
|
2802
|
+
let total = 0;
|
|
2803
|
+
for (const value of values) total += value;
|
|
2804
|
+
return total;
|
|
2805
|
+
}
|
|
2806
|
+
/** Canonical JSON has one zero; never hand it a negative one. */
|
|
2807
|
+
function plain(value) {
|
|
2808
|
+
return value === 0 ? 0 : value;
|
|
2809
|
+
}
|
|
2810
|
+
/** The kernel every `runSearch` ledger names as its search implementation. */
|
|
2811
|
+
const SEARCH_KERNEL_SOURCE = {
|
|
2812
|
+
uri: "npm:@tangle-network/agent-eval#runSearch",
|
|
2813
|
+
revision: hashCanonical({
|
|
2814
|
+
name: "tangle.search-kernel.2026-09",
|
|
2815
|
+
dispatch: "claim, then rung and root, then screen, then train; first allocated first within a stage",
|
|
2816
|
+
expansion: "when no cell waits, fewer than twice the lanes capacity run, and the cap admits one proposal and its expected screens",
|
|
2817
|
+
reservation: {
|
|
2818
|
+
hard: "the lane per-cell maximum",
|
|
2819
|
+
estimate: "1.5 times the p99 of the lane settled cells once 20 settled, else the lane prior"
|
|
2820
|
+
},
|
|
2821
|
+
retries: "a retryable errored attempt runs again, up to maxAttempts",
|
|
2822
|
+
resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
|
|
2823
|
+
writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
|
|
2824
|
+
close: "the policy leader is selected when it has a scored cell; every other undecided node is rejected"
|
|
2825
|
+
})
|
|
2826
|
+
};
|
|
2827
|
+
const USD_TOLERANCE = 1e-9;
|
|
2828
|
+
/** Settled cells a lane needs before its own cost distribution sets its estimate. */
|
|
2829
|
+
const ESTIMATE_FROM_CELLS = 20;
|
|
2830
|
+
const ESTIMATE_MARGIN = 1.5;
|
|
2831
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
2832
|
+
/** Dispatch order: claim cells first, then rungs and the root, then screens, then train. */
|
|
2833
|
+
const STAGE_PRIORITY = {
|
|
2834
|
+
claim: 0,
|
|
2835
|
+
rung: 1,
|
|
2836
|
+
root: 1,
|
|
2837
|
+
screen: 2,
|
|
2838
|
+
train: 3,
|
|
2839
|
+
external: 3
|
|
2840
|
+
};
|
|
2841
|
+
/** Run a search to its close, or continue one from its ledger. */
|
|
2842
|
+
async function runSearch(options) {
|
|
2843
|
+
return new SearchKernel(options).run();
|
|
2844
|
+
}
|
|
2845
|
+
/** The expansion index a kernel operation id encodes, or null. */
|
|
2846
|
+
function searchExpansionIndex(operationId) {
|
|
2847
|
+
const match = /^expand-(\d+)$/.exec(operationId);
|
|
2848
|
+
return match ? Number(match[1]) : null;
|
|
2849
|
+
}
|
|
2850
|
+
var SearchKernel = class {
|
|
2851
|
+
options;
|
|
2852
|
+
recorder;
|
|
2853
|
+
maxAttempts;
|
|
2854
|
+
now;
|
|
2855
|
+
state;
|
|
2856
|
+
lanes = /* @__PURE__ */ new Map();
|
|
2857
|
+
inFlight = /* @__PURE__ */ new Map();
|
|
2858
|
+
proposal = null;
|
|
2859
|
+
proposalController = null;
|
|
2860
|
+
completions = [];
|
|
2861
|
+
wake = null;
|
|
2862
|
+
/** Unfinished cells per node. */
|
|
2863
|
+
pending = /* @__PURE__ */ new Map();
|
|
2864
|
+
/** Admitted nodes whose screen finished, in registration order. */
|
|
2865
|
+
screened = [];
|
|
2866
|
+
screenedSet = /* @__PURE__ */ new Set();
|
|
2867
|
+
admitted = /* @__PURE__ */ new Set();
|
|
2868
|
+
expansionOf = /* @__PURE__ */ new Map();
|
|
2869
|
+
artifacts = /* @__PURE__ */ new Map();
|
|
2870
|
+
/** Children a proposal returned, by artifact digest, until they are registered. */
|
|
2871
|
+
proposed = /* @__PURE__ */ new Map();
|
|
2872
|
+
/** Cells the ledger showed unsettled at start: an earlier process may hold their attempt. */
|
|
2873
|
+
adoptable = /* @__PURE__ */ new Set();
|
|
2874
|
+
operationCosts = [];
|
|
2875
|
+
/** Operations started: the next one's id. */
|
|
2876
|
+
expansions = 0;
|
|
2877
|
+
/** Proposals that completed, by operation index. A proposal an interrupted
|
|
2878
|
+
* process lost counts toward none of the stop rules, so a resumed search
|
|
2879
|
+
* makes the proposals the uninterrupted one would have made. */
|
|
2880
|
+
completed = [];
|
|
2881
|
+
stopReason = null;
|
|
2882
|
+
failure = null;
|
|
2883
|
+
constructor(options) {
|
|
2884
|
+
this.options = options;
|
|
2885
|
+
this.recorder = options.recorder;
|
|
2886
|
+
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
2887
|
+
this.now = options.now ?? Date.now;
|
|
2888
|
+
if (!Number.isSafeInteger(this.maxAttempts) || this.maxAttempts < 1) throw new TypeError("runSearch: maxAttempts must be a positive integer");
|
|
2889
|
+
const { maxExpansions } = options;
|
|
2890
|
+
if (maxExpansions !== void 0 && (!Number.isSafeInteger(maxExpansions) || maxExpansions < 0)) throw new TypeError("runSearch: maxExpansions must be a non-negative integer");
|
|
2891
|
+
for (const lane of options.executor.lanes()) {
|
|
2892
|
+
if (!Number.isSafeInteger(lane.capacity) || lane.capacity < 1) throw new TypeError(`runSearch: lane ${lane.name} needs a positive integer capacity`);
|
|
2893
|
+
if (!Number.isFinite(lane.cellUsd) || lane.cellUsd < 0) throw new TypeError(`runSearch: lane ${lane.name} needs a finite non-negative cellUsd`);
|
|
2894
|
+
if (this.lanes.has(lane.name)) throw new TypeError(`runSearch: lane ${lane.name} is declared twice`);
|
|
2895
|
+
this.lanes.set(lane.name, {
|
|
2896
|
+
...lane,
|
|
2897
|
+
inFlight: 0,
|
|
2898
|
+
queues: [
|
|
2899
|
+
[],
|
|
2900
|
+
[],
|
|
2901
|
+
[],
|
|
2902
|
+
[]
|
|
2903
|
+
],
|
|
2904
|
+
settledCosts: []
|
|
2905
|
+
});
|
|
2774
2906
|
}
|
|
2907
|
+
if (this.lanes.size === 0) throw new TypeError("runSearch: the executor declares no lane");
|
|
2775
2908
|
}
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
};
|
|
2784
|
-
for (const split of SPLITS) {
|
|
2785
|
-
const { tasks, taskSetDigest } = event.splits[split];
|
|
2786
|
-
if (searchTaskSetDigest(tasks) !== taskSetDigest) throw integrity(`the ${split} split's taskSetDigest does not match its tasks`);
|
|
2787
|
-
for (const task of tasks) {
|
|
2788
|
-
const owner = this.tasks.get(task.taskId);
|
|
2789
|
-
if (owner) throw integrity(`task ${task.taskId} is in both the ${owner.split} and ${split} splits`);
|
|
2790
|
-
this.tasks.set(task.taskId, {
|
|
2791
|
-
split,
|
|
2792
|
-
unitId: task.unitId
|
|
2793
|
-
});
|
|
2794
|
-
units[split].add(task.unitId);
|
|
2795
|
-
}
|
|
2909
|
+
async run() {
|
|
2910
|
+
const { ledger } = this.recorder;
|
|
2911
|
+
const lock = ledger instanceof FileSearchLedger ? acquireSingleRunLock({ lockPath: `${ledger.path}.run.lock` }) : null;
|
|
2912
|
+
try {
|
|
2913
|
+
return await this.runLocked();
|
|
2914
|
+
} finally {
|
|
2915
|
+
lock?.release();
|
|
2796
2916
|
}
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2917
|
+
}
|
|
2918
|
+
async runLocked() {
|
|
2919
|
+
await this.refresh();
|
|
2920
|
+
const header = this.state.header;
|
|
2921
|
+
if (!header) throw new Error(`runSearch: search ${this.recorder.searchId} has not been opened`);
|
|
2922
|
+
if (header.policy.expansion !== this.options.policy.name || header.policy.allocation !== this.options.allocation.name) throw new Error(`runSearch: search ${this.recorder.searchId} was opened with policy ${header.policy.expansion} and allocation ${header.policy.allocation}, not ${this.options.policy.name} and ${this.options.allocation.name}`);
|
|
2923
|
+
if (this.state.closed) return this.closedResult();
|
|
2924
|
+
await this.restore();
|
|
2925
|
+
const { signal } = this.options;
|
|
2926
|
+
const onAbort = () => {
|
|
2927
|
+
for (const controller of this.inFlight.values()) controller.abort(signal?.reason);
|
|
2928
|
+
this.proposalController?.abort(signal?.reason);
|
|
2929
|
+
this.wakeUp();
|
|
2930
|
+
};
|
|
2931
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2932
|
+
try {
|
|
2933
|
+
await this.loop();
|
|
2934
|
+
} finally {
|
|
2935
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2800
2936
|
}
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
this.
|
|
2804
|
-
}
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
if (
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
const
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
if (event.nodeId !== expected) throw integrity(`node ${event.nodeId} does not match its artifact digest (expected ${expected})`);
|
|
2837
|
-
if (this.nodes.has(event.nodeId)) throw integrity(`node ${event.nodeId} was registered twice; a re-proposal is a second edge into it`);
|
|
2838
|
-
assertUnique(event.surfaces.map((surface) => surface.surfaceId), "surfaceId", event.eventId);
|
|
2839
|
-
const maxNodes = this.header.budget.maxNodes;
|
|
2840
|
-
if (maxNodes !== null && this.nodes.size + 1 > maxNodes) throw integrity(`node ${event.nodeId} exceeds the search's maxNodes ${maxNodes}`);
|
|
2841
|
-
this.nodes.set(event.nodeId, {
|
|
2842
|
-
event,
|
|
2843
|
-
ordinal: this.nodeOrder.length,
|
|
2844
|
-
registeredSequence: index,
|
|
2845
|
-
updatedSequence: index,
|
|
2846
|
-
edgeIds: [],
|
|
2847
|
-
parents: [],
|
|
2848
|
-
parentKeys: /* @__PURE__ */ new Set(),
|
|
2849
|
-
primaryParentId: null,
|
|
2850
|
-
depth: null,
|
|
2851
|
-
children: [],
|
|
2852
|
-
decisions: [],
|
|
2853
|
-
finalist: false,
|
|
2854
|
-
cellIds: [],
|
|
2855
|
-
scoredCells: 0,
|
|
2856
|
-
spend: {
|
|
2857
|
-
knownUsd: 0,
|
|
2858
|
-
floorUsd: 0,
|
|
2859
|
-
unknownCostCells: 0,
|
|
2860
|
-
boxMinutes: 0
|
|
2861
|
-
}
|
|
2862
|
-
});
|
|
2863
|
-
this.nodeOrder.push(event.nodeId);
|
|
2864
|
-
this.nodeByDigest.set(event.artifactDigest, event.nodeId);
|
|
2865
|
-
this.audit.nodes += 1;
|
|
2866
|
-
this.audit.nodesWithoutEdge += 1;
|
|
2867
|
-
this.audit.undecidedNodes += 1;
|
|
2868
|
-
}
|
|
2869
|
-
recordEdge(event, index) {
|
|
2870
|
-
if (this.edges.has(event.edgeId)) throw integrity(`edge ${event.edgeId} was recorded twice`);
|
|
2871
|
-
const child = this.nodes.get(event.childNodeId);
|
|
2872
|
-
if (!child) throw integrity(`edge ${event.edgeId} names unregistered child ${event.childNodeId}`);
|
|
2873
|
-
if (event.label.length > MAX_LABEL_CHARS$1) throw integrity(`edge ${event.edgeId} label exceeds ${MAX_LABEL_CHARS$1} characters`);
|
|
2874
|
-
if (event.diffs.length !== event.parents.length) throw integrity(`edge ${event.edgeId} needs one diff per parent`);
|
|
2875
|
-
assertUnique(event.parents.map((parent) => canonicalString(parent)), "parent", event.eventId);
|
|
2876
|
-
const { operator, attribution, parents } = event;
|
|
2877
|
-
if (operator === "seed" !== (event.proposer === null)) throw integrity(`edge ${event.edgeId}: exactly the seed edge has no proposer`);
|
|
2878
|
-
if (operator === "seed" && (parents.length > 0 || attribution === "unknown")) throw integrity(`seed edge ${event.edgeId} must have no parents and a known attribution`);
|
|
2879
|
-
if (attribution === "unknown" && parents.length > 0) throw integrity(`edge ${event.edgeId} has unknown attribution but names parents`);
|
|
2880
|
-
if (operator !== "seed" && attribution !== "unknown" && parents.length === 0) throw integrity(`${operator} edge ${event.edgeId} names no parent`);
|
|
2881
|
-
if (operator === "merge" && attribution !== "unknown" && parents.length < 2) throw integrity(`merge edge ${event.edgeId} needs at least two parents`);
|
|
2882
|
-
const operationId = event.proposer?.operationId ?? null;
|
|
2883
|
-
if (operationId !== null && !this.operations.has(operationId)) throw integrity(`edge ${event.edgeId} names operation ${operationId}, which never started`);
|
|
2884
|
-
const inSearch = [];
|
|
2885
|
-
for (const parent of parents) {
|
|
2886
|
-
if (parent.searchId !== this.searchId) {
|
|
2887
|
-
const from = this.header.derivedFrom;
|
|
2888
|
-
if (operator !== "derive" || parents.length !== 1 || !from || from.searchId !== parent.searchId || from.nodeId !== parent.nodeId) throw integrity(`edge ${event.edgeId} names a parent in search ${parent.searchId}; only a derive edge matching derivedFrom may`);
|
|
2937
|
+
if (this.failure) throw this.failure.error;
|
|
2938
|
+
await this.close();
|
|
2939
|
+
return this.closedResult();
|
|
2940
|
+
}
|
|
2941
|
+
async restore() {
|
|
2942
|
+
const rootInput = this.options.codec.node(this.recorder, this.options.root);
|
|
2943
|
+
const rootId = this.state.rootNodeId;
|
|
2944
|
+
if (rootId === null) {
|
|
2945
|
+
const { nodeId } = await this.recorder.registerNode(rootInput);
|
|
2946
|
+
await this.recorder.recordEdge({
|
|
2947
|
+
childNodeId: nodeId,
|
|
2948
|
+
parents: [],
|
|
2949
|
+
operator: "seed",
|
|
2950
|
+
attribution: "explicit",
|
|
2951
|
+
proposer: null,
|
|
2952
|
+
proposalKey: "root",
|
|
2953
|
+
rationale: { unknown: "the root is the caller-supplied starting artifact" },
|
|
2954
|
+
diffs: [],
|
|
2955
|
+
label: "root"
|
|
2956
|
+
});
|
|
2957
|
+
await this.refresh();
|
|
2958
|
+
} else if (this.state.node(rootId).artifactDigest !== rootInput.artifactDigest) throw new Error(`runSearch: search ${this.recorder.searchId} started from another root artifact; a changed start is a derived search`);
|
|
2959
|
+
const root = this.state.rootNodeId;
|
|
2960
|
+
this.artifacts.set(root, this.options.root);
|
|
2961
|
+
this.expansionOf.set(root, -1);
|
|
2962
|
+
const recorded = [];
|
|
2963
|
+
for (let expansion = 0;; expansion++) {
|
|
2964
|
+
const operationId = `expand-${expansion}`;
|
|
2965
|
+
const operation = this.state.operation(operationId);
|
|
2966
|
+
if (!operation) break;
|
|
2967
|
+
this.expansions = expansion + 1;
|
|
2968
|
+
if (operation.recorded) {
|
|
2969
|
+
this.operationCosts.push(operation.spentUsd);
|
|
2970
|
+
recorded.push(expansion);
|
|
2971
|
+
if (operation.outcome === "completed") this.completed.push(expansion);
|
|
2889
2972
|
continue;
|
|
2890
2973
|
}
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2974
|
+
await this.recorder.recordOperation({
|
|
2975
|
+
operationId,
|
|
2976
|
+
operationKind: "candidate-generation",
|
|
2977
|
+
execution: this.options.proposer.execution,
|
|
2978
|
+
outcome: {
|
|
2979
|
+
status: "failed",
|
|
2980
|
+
failure: {
|
|
2981
|
+
code: "interrupted",
|
|
2982
|
+
message: "the search stopped while this proposal ran; its output was lost"
|
|
2983
|
+
}
|
|
2984
|
+
},
|
|
2985
|
+
accounting: interruptedAccounting()
|
|
2986
|
+
});
|
|
2987
|
+
await this.refresh();
|
|
2897
2988
|
}
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
this.
|
|
2903
|
-
|
|
2904
|
-
child.edgeIds.push(event.edgeId);
|
|
2905
|
-
for (const parent of parents) {
|
|
2906
|
-
const key = canonicalString(parent);
|
|
2907
|
-
if (child.parentKeys.has(key)) continue;
|
|
2908
|
-
child.parentKeys.add(key);
|
|
2909
|
-
child.parents.push(parent);
|
|
2989
|
+
for (const node of this.state.nodes()) {
|
|
2990
|
+
if (node.edgeIds.length === 0) continue;
|
|
2991
|
+
const edge = this.state.edge(node.edgeIds[0]);
|
|
2992
|
+
const expansion = edge.proposer?.operationId ? searchExpansionIndex(edge.proposer.operationId) : null;
|
|
2993
|
+
this.expansionOf.set(node.nodeId, expansion ?? -1);
|
|
2994
|
+
if (node.status !== "invalid") this.admitted.add(node.nodeId);
|
|
2910
2995
|
}
|
|
2911
|
-
for (const
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
if (event.cellId !== expected) throw integrity(`cell ${event.cellId} does not match its coordinates (expected ${expected})`);
|
|
2920
|
-
if (this.cells.has(event.cellId)) throw integrity(`cell ${event.cellId} was allocated twice`);
|
|
2921
|
-
const node = this.nodes.get(event.nodeId);
|
|
2922
|
-
if (!node) throw integrity(`cell ${event.cellId} names unregistered node ${event.nodeId}`);
|
|
2923
|
-
if (node.edgeIds.length === 0) throw integrity(`cell ${event.cellId}: node ${event.nodeId} has no edge yet`);
|
|
2924
|
-
const task = this.tasks.get(event.taskId);
|
|
2925
|
-
if (!task || task.split !== event.split) throw integrity(`cell ${event.cellId}: task ${event.taskId} is not in the ${event.split} split`);
|
|
2926
|
-
if (task.unitId !== event.unitId) throw integrity(`cell ${event.cellId}: task ${event.taskId} samples unit ${task.unitId}`);
|
|
2927
|
-
assertStageSplit(event.cellId, event.stage, event.split, node.ordinal === 0);
|
|
2928
|
-
if (event.split === "test" && node.ordinal !== 0 && !node.finalist) throw integrity(`cell ${event.cellId}: the test split is sealed; only the root and finalists run on it`);
|
|
2929
|
-
const maxCells = this.header.budget.maxCells;
|
|
2930
|
-
if (maxCells !== null && this.audit.cells.allocated + 1 > maxCells) throw integrity(`cell ${event.cellId} exceeds the search's maxCells ${maxCells}`);
|
|
2931
|
-
this.admit(event.reservation, event.stage === "claim", `cell ${event.cellId}`);
|
|
2932
|
-
this.cells.set(event.cellId, {
|
|
2933
|
-
cellId: event.cellId,
|
|
2934
|
-
nodeId: event.nodeId,
|
|
2935
|
-
taskId: event.taskId,
|
|
2936
|
-
unitId: event.unitId,
|
|
2937
|
-
split: event.split,
|
|
2938
|
-
rep: event.rep,
|
|
2939
|
-
stage: event.stage,
|
|
2940
|
-
lane: event.lane,
|
|
2941
|
-
reservation: event.reservation,
|
|
2942
|
-
attempts: 0,
|
|
2943
|
-
lastRunId: null,
|
|
2944
|
-
outcome: null,
|
|
2945
|
-
score: null,
|
|
2946
|
-
final: false,
|
|
2947
|
-
cancelled: null,
|
|
2948
|
-
spentUsd: 0,
|
|
2949
|
-
costKnown: true,
|
|
2950
|
-
traceId: null,
|
|
2951
|
-
allocatedSequence: index,
|
|
2952
|
-
updatedSequence: index
|
|
2953
|
-
});
|
|
2954
|
-
node.cellIds.push(event.cellId);
|
|
2955
|
-
node.updatedSequence = index;
|
|
2956
|
-
this.audit.cells.allocated += 1;
|
|
2957
|
-
this.audit.cells.open += 1;
|
|
2958
|
-
this.audit.spend.openReservationUsd += event.reservation?.usd ?? 0;
|
|
2959
|
-
if (event.stage === "claim") this.claimUsedUsd += event.reservation?.usd ?? 0;
|
|
2960
|
-
}
|
|
2961
|
-
settleCell(event, index) {
|
|
2962
|
-
const cell = this.cells.get(event.cellId);
|
|
2963
|
-
if (!cell) throw integrity(`attempt ${event.eventId} names unallocated cell ${event.cellId}`);
|
|
2964
|
-
if (cell.cancelled) throw integrity(`cell ${event.cellId} was settled after it was cancelled`);
|
|
2965
|
-
if (cell.final) throw integrity(`cell ${event.cellId} was attempted again after a final ${cell.outcome} outcome; only a retryable error admits another attempt`);
|
|
2966
|
-
if (event.attempt !== cell.attempts + 1) throw integrity(`cell ${event.cellId} attempt ${event.attempt} is not contiguous (expected ${cell.attempts + 1})`);
|
|
2967
|
-
const runId = searchCellRunId({
|
|
2968
|
-
cellId: event.cellId,
|
|
2969
|
-
attempt: event.attempt
|
|
2970
|
-
});
|
|
2971
|
-
if (event.runId !== runId) throw integrity(`cell ${event.cellId} attempt ${event.attempt} must have runId ${runId}`);
|
|
2972
|
-
const node = this.nodes.get(cell.nodeId);
|
|
2973
|
-
const declared = new Set(node.event.surfaces.map((surface) => surface.surfaceId));
|
|
2974
|
-
assertUnique(event.surfaceEvidence.map((evidence) => evidence.surfaceId), "surface evidence", event.eventId);
|
|
2975
|
-
for (const evidence of event.surfaceEvidence) if (!declared.has(evidence.surfaceId)) throw integrity(`attempt ${event.eventId} reports surface ${evidence.surfaceId}, which node ${cell.nodeId} does not declare`);
|
|
2976
|
-
if (event.artifacts.filter((artifact) => artifact.role === "run-record").length > 1) throw integrity(`attempt ${event.eventId} binds more than one RunRecord`);
|
|
2977
|
-
const before = this.reservationParts(cell);
|
|
2978
|
-
const cost = this.book(event.accounting);
|
|
2979
|
-
if (cell.attempts === 0) {
|
|
2980
|
-
this.audit.cells.settled += 1;
|
|
2981
|
-
this.audit.cells.open -= 1;
|
|
2982
|
-
if (cell.reservation === null) this.audit.spend.unreservedCells += 1;
|
|
2996
|
+
for (const cell of this.state.cells()) {
|
|
2997
|
+
const lane = cell.lane === null ? void 0 : this.lanes.get(cell.lane);
|
|
2998
|
+
if (cell.final && lane) lane.settledCosts.push(cell.spentUsd);
|
|
2999
|
+
if (this.cellDone(cell)) continue;
|
|
3000
|
+
if (!lane) throw new Error(`runSearch: cell ${cell.cellId} was placed on lane ${String(cell.lane)}, which the executor does not declare`);
|
|
3001
|
+
this.adoptable.add(cell.cellId);
|
|
3002
|
+
this.enqueue(cell);
|
|
3003
|
+
this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
|
|
2983
3004
|
}
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
3005
|
+
for (const expansion of recorded) await this.finishProposal(expansion);
|
|
3006
|
+
for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
|
|
3007
|
+
for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) this.markScreened(nodeId);
|
|
3008
|
+
}
|
|
3009
|
+
async loop() {
|
|
3010
|
+
for (;;) {
|
|
3011
|
+
if (this.failure || this.options.signal?.aborted) {
|
|
3012
|
+
await this.drain(this.failure?.error ?? this.options.signal?.reason);
|
|
3013
|
+
this.failure ??= { error: this.options.signal?.reason ?? /* @__PURE__ */ new Error("aborted") };
|
|
3014
|
+
return;
|
|
3015
|
+
}
|
|
3016
|
+
if (this.pastDeadline()) {
|
|
3017
|
+
this.stopReason ??= "deadline";
|
|
3018
|
+
if (this.queuedCount() > 0) await this.cancelQueued("deadline");
|
|
3019
|
+
}
|
|
3020
|
+
this.dispatch();
|
|
3021
|
+
const blocked = this.stopReason === null && this.proposal === null ? await this.tryExpand() : null;
|
|
3022
|
+
if (!(this.inFlight.size > 0 || this.proposal !== null)) {
|
|
3023
|
+
if (this.queuedCount() > 0) continue;
|
|
3024
|
+
this.stopReason ??= blocked ?? "converged";
|
|
3025
|
+
return;
|
|
3026
|
+
}
|
|
3027
|
+
await this.handle(await this.next());
|
|
2988
3028
|
}
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3029
|
+
}
|
|
3030
|
+
/** The next completion, or null when an abort woke the loop without one. */
|
|
3031
|
+
async next() {
|
|
3032
|
+
if (this.completions.length === 0) await new Promise((resolve) => {
|
|
3033
|
+
this.wake = resolve;
|
|
3034
|
+
});
|
|
3035
|
+
return this.completions.shift() ?? null;
|
|
3036
|
+
}
|
|
3037
|
+
push(completion) {
|
|
3038
|
+
this.completions.push(completion);
|
|
3039
|
+
this.wakeUp();
|
|
3040
|
+
}
|
|
3041
|
+
wakeUp() {
|
|
3042
|
+
const wake = this.wake;
|
|
3043
|
+
this.wake = null;
|
|
3044
|
+
wake?.();
|
|
3045
|
+
}
|
|
3046
|
+
async handle(completion) {
|
|
3047
|
+
if (completion === null) return;
|
|
3048
|
+
if (completion.kind === "cell") await this.settle(completion);
|
|
3049
|
+
else await this.recordProposal(completion);
|
|
3050
|
+
}
|
|
3051
|
+
/** Abort everything in flight and record what still completes. */
|
|
3052
|
+
async drain(reason) {
|
|
3053
|
+
for (const controller of this.inFlight.values()) controller.abort(reason);
|
|
3054
|
+
this.proposalController?.abort(reason);
|
|
3055
|
+
while (this.inFlight.size > 0 || this.proposal !== null) await this.handle(await this.next());
|
|
3056
|
+
}
|
|
3057
|
+
/** Fill free lane slots with queued cells. A queued cell was admitted when
|
|
3058
|
+
* it was allocated, so starting it needs no further decision. */
|
|
3059
|
+
dispatch() {
|
|
3060
|
+
if (this.failure || this.options.signal?.aborted || this.pastDeadline()) return;
|
|
3061
|
+
const cap = this.state.header.budget.maxConcurrency;
|
|
3062
|
+
for (const lane of this.lanes.values()) while (lane.inFlight < lane.capacity && (cap === null || this.inFlight.size < cap)) {
|
|
3063
|
+
const cellId = lane.queues.find((queue) => queue.length > 0)?.shift();
|
|
3064
|
+
if (cellId === void 0) break;
|
|
3065
|
+
this.start(cellId, lane);
|
|
3000
3066
|
}
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3067
|
+
}
|
|
3068
|
+
start(cellId, lane) {
|
|
3069
|
+
const cell = this.state.cell(cellId);
|
|
3070
|
+
const attempt = cell.attempts + 1;
|
|
3071
|
+
const controller = new AbortController();
|
|
3072
|
+
const work = {
|
|
3073
|
+
searchId: this.recorder.searchId,
|
|
3074
|
+
cellId,
|
|
3075
|
+
nodeId: cell.nodeId,
|
|
3076
|
+
taskId: cell.taskId,
|
|
3077
|
+
unitId: cell.unitId,
|
|
3078
|
+
split: cell.split,
|
|
3079
|
+
rep: cell.rep,
|
|
3080
|
+
stage: cell.stage,
|
|
3081
|
+
lane: lane.name,
|
|
3082
|
+
attempt,
|
|
3083
|
+
runId: `${cellId}:${attempt}`,
|
|
3084
|
+
artifact: this.artifact(cell.nodeId),
|
|
3085
|
+
signal: controller.signal
|
|
3086
|
+
};
|
|
3087
|
+
const adopt = this.adoptable.delete(cellId);
|
|
3088
|
+
this.inFlight.set(cellId, controller);
|
|
3089
|
+
lane.inFlight += 1;
|
|
3090
|
+
const { executor } = this.options;
|
|
3091
|
+
(async () => (adopt ? await executor.adopt(work) : null) ?? await executor.run(work))().then((result) => this.push({
|
|
3092
|
+
kind: "cell",
|
|
3093
|
+
cellId,
|
|
3094
|
+
attempt,
|
|
3095
|
+
result
|
|
3096
|
+
}), (error) => this.push({
|
|
3097
|
+
kind: "cell",
|
|
3098
|
+
cellId,
|
|
3099
|
+
attempt,
|
|
3100
|
+
error
|
|
3101
|
+
}));
|
|
3102
|
+
}
|
|
3103
|
+
async settle(completion) {
|
|
3104
|
+
const { cellId, attempt } = completion;
|
|
3105
|
+
this.inFlight.delete(cellId);
|
|
3106
|
+
const before = this.state.cell(cellId);
|
|
3107
|
+
const lane = this.lanes.get(before.lane);
|
|
3108
|
+
lane.inFlight -= 1;
|
|
3109
|
+
if (!completion.result) {
|
|
3110
|
+
this.failure ??= { error: completion.error };
|
|
3111
|
+
return;
|
|
3029
3112
|
}
|
|
3030
|
-
this.
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3113
|
+
this.dispatch();
|
|
3114
|
+
if ((this.failure || this.options.signal?.aborted) && completion.result.outcome.status === "errored") return;
|
|
3115
|
+
await this.recorder.settleCell({
|
|
3116
|
+
...completion.result,
|
|
3117
|
+
cellId,
|
|
3118
|
+
attempt
|
|
3119
|
+
});
|
|
3120
|
+
await this.refresh();
|
|
3121
|
+
const cell = this.state.cell(cellId);
|
|
3122
|
+
if (cell.final) lane.settledCosts.push(cell.spentUsd);
|
|
3123
|
+
if (!this.cellDone(cell)) {
|
|
3124
|
+
if (this.failure || this.options.signal?.aborted) return;
|
|
3125
|
+
this.enqueue(cell);
|
|
3126
|
+
return;
|
|
3040
3127
|
}
|
|
3041
|
-
const
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
if (
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3128
|
+
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
3129
|
+
this.pending.set(cell.nodeId, left);
|
|
3130
|
+
if (left === 0) await this.onScreenDone(cell.nodeId);
|
|
3131
|
+
}
|
|
3132
|
+
async onScreenDone(nodeId) {
|
|
3133
|
+
this.markScreened(nodeId);
|
|
3134
|
+
if (this.stopReason === null && !this.failure) await this.allocateFor(nodeId);
|
|
3135
|
+
}
|
|
3136
|
+
markScreened(nodeId) {
|
|
3137
|
+
if (this.screenedSet.has(nodeId)) return;
|
|
3138
|
+
this.screenedSet.add(nodeId);
|
|
3139
|
+
const ordinal = this.state.node(nodeId).ordinal;
|
|
3140
|
+
let index = this.screened.length;
|
|
3141
|
+
while (index > 0 && this.state.node(this.screened[index - 1]).ordinal > ordinal) index--;
|
|
3142
|
+
this.screened = [
|
|
3143
|
+
...this.screened.slice(0, index),
|
|
3144
|
+
nodeId,
|
|
3145
|
+
...this.screened.slice(index)
|
|
3146
|
+
];
|
|
3147
|
+
}
|
|
3148
|
+
/** Allocate the cells the allocator plans for a node and the ledger lacks. */
|
|
3149
|
+
async allocateFor(nodeId) {
|
|
3150
|
+
const plans = this.options.allocation.plan(this.state, nodeId).filter((plan) => !this.state.cell(searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep)));
|
|
3151
|
+
if (plans.length === 0) return;
|
|
3152
|
+
const placed = plans.map((plan) => {
|
|
3153
|
+
const laneName = this.options.executor.place({
|
|
3154
|
+
...plan,
|
|
3155
|
+
nodeId
|
|
3156
|
+
});
|
|
3157
|
+
const lane = this.lanes.get(laneName);
|
|
3158
|
+
if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
|
|
3159
|
+
return {
|
|
3160
|
+
plan,
|
|
3161
|
+
lane,
|
|
3162
|
+
reservation: this.cellReservation(lane)
|
|
3163
|
+
};
|
|
3054
3164
|
});
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
for (const
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3165
|
+
const hold = placed.reduce((sum, { reservation }) => sum + reservation.usd, 0);
|
|
3166
|
+
const { headroomUsd } = this.state.budget;
|
|
3167
|
+
if (headroomUsd !== null && hold > headroomUsd + USD_TOLERANCE) return;
|
|
3168
|
+
const { maxCells } = this.state.header.budget;
|
|
3169
|
+
if (maxCells !== null && this.state.audit.cells.allocated + placed.length > maxCells) return;
|
|
3170
|
+
for (const { plan, lane, reservation } of placed) await this.recorder.allocateCell({
|
|
3171
|
+
nodeId,
|
|
3172
|
+
taskId: plan.taskId,
|
|
3173
|
+
split: plan.split,
|
|
3174
|
+
rep: plan.rep,
|
|
3175
|
+
stage: plan.stage,
|
|
3176
|
+
lane: lane.name,
|
|
3177
|
+
reservation
|
|
3178
|
+
});
|
|
3179
|
+
await this.refresh();
|
|
3180
|
+
for (const { plan } of placed) {
|
|
3181
|
+
const cellId = searchCellId(this.recorder.searchId, nodeId, plan.taskId, plan.split, plan.rep);
|
|
3182
|
+
this.enqueue(this.state.cell(cellId));
|
|
3183
|
+
}
|
|
3184
|
+
this.pending.set(nodeId, (this.pending.get(nodeId) ?? 0) + placed.length);
|
|
3185
|
+
this.dispatch();
|
|
3186
|
+
}
|
|
3187
|
+
enqueue(cell) {
|
|
3188
|
+
this.lanes.get(cell.lane).queues[STAGE_PRIORITY[cell.stage]].push(cell.cellId);
|
|
3189
|
+
}
|
|
3190
|
+
queuedCount() {
|
|
3191
|
+
let count = 0;
|
|
3192
|
+
for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
|
|
3193
|
+
return count;
|
|
3194
|
+
}
|
|
3195
|
+
async cancelQueued(reason) {
|
|
3196
|
+
for (const lane of this.lanes.values()) for (const queue of lane.queues) for (const cellId of queue.splice(0)) {
|
|
3197
|
+
const cell = this.state.cell(cellId);
|
|
3198
|
+
if (cell.attempts === 0) {
|
|
3199
|
+
await this.recorder.cancelCell({
|
|
3200
|
+
cellId,
|
|
3201
|
+
reason
|
|
3202
|
+
});
|
|
3203
|
+
await this.refresh();
|
|
3204
|
+
}
|
|
3205
|
+
const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
|
|
3206
|
+
this.pending.set(cell.nodeId, left);
|
|
3207
|
+
if (left === 0) this.markScreened(cell.nodeId);
|
|
3080
3208
|
}
|
|
3081
3209
|
}
|
|
3082
|
-
/**
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
return
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
const
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
const
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
return
|
|
3106
|
-
|
|
3107
|
-
|
|
3210
|
+
/** No further attempt will run: a final outcome, a cancellation, or a
|
|
3211
|
+
* retryable error that used up its attempts. */
|
|
3212
|
+
cellDone(cell) {
|
|
3213
|
+
return cell.final || cell.cancelled !== null || cell.outcome === "errored" && cell.attempts >= this.maxAttempts;
|
|
3214
|
+
}
|
|
3215
|
+
/** Start a proposal when the rules admit one; otherwise say why not. */
|
|
3216
|
+
async tryExpand() {
|
|
3217
|
+
const { maxNodes, maxCells } = this.state.header.budget;
|
|
3218
|
+
if (maxNodes !== null && this.state.audit.nodes >= maxNodes) return "max-nodes";
|
|
3219
|
+
const { maxExpansions } = this.options;
|
|
3220
|
+
if (maxExpansions !== void 0 && this.completed.length >= maxExpansions) return "max-nodes";
|
|
3221
|
+
if (this.queuedCount() > 0) return "converged";
|
|
3222
|
+
if (this.inFlight.size >= 2 * this.capacity()) return "converged";
|
|
3223
|
+
const view = this.policyView();
|
|
3224
|
+
const { policy } = this.options;
|
|
3225
|
+
const leader = policy.leader(view);
|
|
3226
|
+
if (policy.patience !== void 0 && this.completed.filter((index) => index > (this.expansionOf.get(leader) ?? -1)).length >= policy.patience) return "patience";
|
|
3227
|
+
const screen = this.options.allocation.screenSize(this.state);
|
|
3228
|
+
const operation = this.operationReservation();
|
|
3229
|
+
const children = this.options.proposer.childrenPerProposal ?? 1;
|
|
3230
|
+
const need = operation.usd + children * screen * this.maxCellReservation();
|
|
3231
|
+
const { headroomUsd } = this.state.budget;
|
|
3232
|
+
if (headroomUsd !== null && need > headroomUsd + USD_TOLERANCE) return "budget";
|
|
3233
|
+
if (maxCells !== null && this.state.audit.cells.allocated + children * screen > maxCells) return "budget";
|
|
3234
|
+
const expansion = policy.expand(view);
|
|
3235
|
+
if (expansion === null) return "converged";
|
|
3236
|
+
const index = this.expansions++;
|
|
3237
|
+
const operationId = `expand-${index}`;
|
|
3238
|
+
await this.recorder.startOperation({
|
|
3239
|
+
operationId,
|
|
3240
|
+
operationKind: "candidate-generation",
|
|
3241
|
+
reservation: operation
|
|
3242
|
+
});
|
|
3243
|
+
await this.refresh();
|
|
3244
|
+
const controller = new AbortController();
|
|
3245
|
+
const request = {
|
|
3246
|
+
expansion: index,
|
|
3247
|
+
operationId,
|
|
3248
|
+
operator: expansion.operator,
|
|
3249
|
+
parents: expansion.parents.map((nodeId) => ({
|
|
3250
|
+
nodeId,
|
|
3251
|
+
artifact: this.artifact(nodeId)
|
|
3252
|
+
})),
|
|
3253
|
+
leader,
|
|
3254
|
+
signal: controller.signal
|
|
3108
3255
|
};
|
|
3256
|
+
this.proposalController = controller;
|
|
3257
|
+
this.proposal = this.options.proposer.propose(request).then((result) => this.push({
|
|
3258
|
+
kind: "proposal",
|
|
3259
|
+
expansion: index,
|
|
3260
|
+
operationId,
|
|
3261
|
+
request,
|
|
3262
|
+
selection: expansion.selection,
|
|
3263
|
+
result
|
|
3264
|
+
}), (error) => this.push({
|
|
3265
|
+
kind: "proposal",
|
|
3266
|
+
expansion: index,
|
|
3267
|
+
operationId,
|
|
3268
|
+
request,
|
|
3269
|
+
selection: expansion.selection,
|
|
3270
|
+
error
|
|
3271
|
+
}));
|
|
3272
|
+
return null;
|
|
3109
3273
|
}
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
const
|
|
3114
|
-
const
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3274
|
+
async recordProposal(completion) {
|
|
3275
|
+
this.proposal = null;
|
|
3276
|
+
this.proposalController = null;
|
|
3277
|
+
const { operationId, expansion } = completion;
|
|
3278
|
+
const { proposer, codec, admit } = this.options;
|
|
3279
|
+
const result = completion.result;
|
|
3280
|
+
if (!result) {
|
|
3281
|
+
await this.recorder.recordOperation({
|
|
3282
|
+
operationId,
|
|
3283
|
+
operationKind: "candidate-generation",
|
|
3284
|
+
execution: proposer.execution,
|
|
3285
|
+
outcome: {
|
|
3286
|
+
status: "failed",
|
|
3287
|
+
failure: {
|
|
3288
|
+
code: "proposer-error",
|
|
3289
|
+
message: errorMessage(completion.error)
|
|
3290
|
+
}
|
|
3291
|
+
},
|
|
3292
|
+
accounting: interruptedAccounting()
|
|
3293
|
+
});
|
|
3294
|
+
await this.refresh();
|
|
3295
|
+
this.failure ??= { error: completion.error };
|
|
3296
|
+
return;
|
|
3297
|
+
}
|
|
3298
|
+
const { maxNodes } = this.state.header.budget;
|
|
3299
|
+
const room = maxNodes === null ? Number.POSITIVE_INFINITY : maxNodes - this.state.audit.nodes;
|
|
3300
|
+
const parents = completion.request.parents;
|
|
3301
|
+
const blob = {
|
|
3302
|
+
kind: "search-proposal",
|
|
3303
|
+
operator: completion.request.operator,
|
|
3304
|
+
parents: parents.map((parent) => parent.nodeId),
|
|
3305
|
+
selection: completion.selection,
|
|
3306
|
+
children: result.children.slice(0, Math.max(0, room)).map((child) => ({
|
|
3307
|
+
node: codec.node(this.recorder, child.artifact),
|
|
3308
|
+
diffs: parents.map((parent) => codec.diff(this.recorder, parent.artifact, child.artifact)),
|
|
3309
|
+
label: redactText(child.label, { profile: "share" }),
|
|
3310
|
+
rationale: redactText(child.rationale, { profile: "share" }),
|
|
3311
|
+
attribution: child.attribution ?? null,
|
|
3312
|
+
invalid: admit?.(child.artifact) ?? null
|
|
3313
|
+
}))
|
|
3119
3314
|
};
|
|
3315
|
+
await this.recorder.recordOperation({
|
|
3316
|
+
operationId,
|
|
3317
|
+
operationKind: "candidate-generation",
|
|
3318
|
+
execution: result.execution ?? proposer.execution,
|
|
3319
|
+
outcome: { status: "completed" },
|
|
3320
|
+
accounting: result.accounting,
|
|
3321
|
+
artifacts: [this.recorder.blob("proposal", blob)]
|
|
3322
|
+
});
|
|
3323
|
+
await this.refresh();
|
|
3324
|
+
this.operationCosts.push(this.state.operation(operationId).spentUsd);
|
|
3325
|
+
this.completed.push(expansion);
|
|
3326
|
+
for (const [index, child] of blob.children.entries()) this.proposed.set(child.node.artifactDigest, result.children[index].artifact);
|
|
3327
|
+
if (result.stop !== void 0 || blob.children.length === 0) this.stopReason ??= "converged";
|
|
3328
|
+
await this.finishProposal(expansion);
|
|
3329
|
+
}
|
|
3330
|
+
/** Register every child a recorded proposal produced, each once. */
|
|
3331
|
+
async finishProposal(expansion) {
|
|
3332
|
+
const operationId = `expand-${expansion}`;
|
|
3333
|
+
const operation = this.state.operation(operationId);
|
|
3334
|
+
const ref = operation?.artifacts.find((artifact) => artifact.role === "proposal");
|
|
3335
|
+
if (!operation?.recorded || operation.outcome !== "completed" || !ref) return;
|
|
3336
|
+
const blob = this.recorder.readBlob(ref);
|
|
3337
|
+
const { proposer } = this.options;
|
|
3338
|
+
for (const [index, child] of blob.children.entries()) {
|
|
3339
|
+
const { nodeId } = await this.recorder.registerNode(child.node);
|
|
3340
|
+
const held = this.proposed.get(child.node.artifactDigest);
|
|
3341
|
+
if (held !== void 0) {
|
|
3342
|
+
this.proposed.delete(child.node.artifactDigest);
|
|
3343
|
+
if (!this.artifacts.has(nodeId)) this.artifacts.set(nodeId, held);
|
|
3344
|
+
}
|
|
3345
|
+
const edgeId = await this.recorder.recordEdge({
|
|
3346
|
+
childNodeId: nodeId,
|
|
3347
|
+
parents: blob.parents,
|
|
3348
|
+
operator: blob.operator,
|
|
3349
|
+
attribution: "explicit",
|
|
3350
|
+
proposer: {
|
|
3351
|
+
kind: proposer.kind,
|
|
3352
|
+
name: proposer.name,
|
|
3353
|
+
operationId,
|
|
3354
|
+
source: proposer.source
|
|
3355
|
+
},
|
|
3356
|
+
proposalKey: `${operationId}:${index}`,
|
|
3357
|
+
selection: blob.selection,
|
|
3358
|
+
rationale: child.rationale,
|
|
3359
|
+
diffs: child.diffs,
|
|
3360
|
+
label: child.label
|
|
3361
|
+
});
|
|
3362
|
+
await this.refresh();
|
|
3363
|
+
if (this.state.node(nodeId).edgeIds[0] !== edgeId) continue;
|
|
3364
|
+
this.expansionOf.set(nodeId, expansion);
|
|
3365
|
+
if (child.invalid !== null) {
|
|
3366
|
+
await this.recorder.decideNode({
|
|
3367
|
+
nodeId,
|
|
3368
|
+
decision: { status: "invalid" },
|
|
3369
|
+
rule: "admission",
|
|
3370
|
+
reason: child.invalid
|
|
3371
|
+
});
|
|
3372
|
+
await this.refresh();
|
|
3373
|
+
continue;
|
|
3374
|
+
}
|
|
3375
|
+
if (this.admitted.has(nodeId)) continue;
|
|
3376
|
+
this.admitted.add(nodeId);
|
|
3377
|
+
await this.allocateFor(nodeId);
|
|
3378
|
+
if ((this.pending.get(nodeId) ?? 0) === 0) this.markScreened(nodeId);
|
|
3379
|
+
}
|
|
3120
3380
|
}
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
const {
|
|
3126
|
-
|
|
3127
|
-
const
|
|
3128
|
-
const
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3381
|
+
async close() {
|
|
3382
|
+
await this.refresh();
|
|
3383
|
+
const view = this.policyView();
|
|
3384
|
+
const leader = this.options.policy.leader(view);
|
|
3385
|
+
const { split } = view;
|
|
3386
|
+
const root = this.state.rootNodeId;
|
|
3387
|
+
const rule = this.options.policy.name;
|
|
3388
|
+
const decisions = this.state.nodes().filter((node) => !isTerminal(node)).map((node) => {
|
|
3389
|
+
const scored = this.state.scoredCells(node.nodeId, split).length > 0;
|
|
3390
|
+
const lead = node.nodeId === leader && scored;
|
|
3391
|
+
const against = node.nodeId === leader ? root : leader;
|
|
3392
|
+
const basis = node.nodeId === against || !scored ? null : estimateNode(this.state, node.nodeId, {
|
|
3393
|
+
against,
|
|
3394
|
+
split
|
|
3395
|
+
});
|
|
3396
|
+
return {
|
|
3397
|
+
nodeId: node.nodeId,
|
|
3398
|
+
decision: { status: lead ? "selected" : "rejected" },
|
|
3399
|
+
basis,
|
|
3400
|
+
rule,
|
|
3401
|
+
reason: lead ? `the leader when the search stopped (${this.stopReason}): a budget decision on the ${split} split, which claims nothing` : !this.screenedSet.has(node.nodeId) || !scored ? "the search stopped before this node was measured" : !view.complete(node.nodeId) ? "it left a unit of its screen unscored, so it could not lead" : "it did not beat the leader on the units they share"
|
|
3402
|
+
};
|
|
3403
|
+
});
|
|
3404
|
+
for (const decision of decisions) await this.recorder.decideNode(decision);
|
|
3405
|
+
await this.recorder.close({
|
|
3406
|
+
reason: this.stopReason ?? "converged",
|
|
3407
|
+
claim: null
|
|
3408
|
+
});
|
|
3409
|
+
await this.refresh();
|
|
3410
|
+
}
|
|
3411
|
+
closedResult() {
|
|
3412
|
+
const state = this.state;
|
|
3150
3413
|
return {
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
outcomes: { ...audit.outcomes },
|
|
3155
|
-
operations: { ...audit.operations },
|
|
3156
|
-
spend: { ...audit.spend },
|
|
3157
|
-
tokens: { ...audit.tokens }
|
|
3414
|
+
state,
|
|
3415
|
+
leader: state.audit.selectedNodeId ?? state.rootNodeId,
|
|
3416
|
+
reason: state.closed.reason
|
|
3158
3417
|
};
|
|
3159
3418
|
}
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
readNodeIds() {
|
|
3167
|
-
return this.nodeOrder;
|
|
3168
|
-
}
|
|
3169
|
-
/** @internal */
|
|
3170
|
-
readHasNode(nodeId) {
|
|
3171
|
-
return this.nodes.has(nodeId);
|
|
3172
|
-
}
|
|
3173
|
-
/** @internal */
|
|
3174
|
-
readNodeIdForDigest(artifactDigest) {
|
|
3175
|
-
return this.nodeByDigest.get(artifactDigest);
|
|
3419
|
+
policyView() {
|
|
3420
|
+
return searchPolicyView(this.state, {
|
|
3421
|
+
screened: this.screened,
|
|
3422
|
+
screening: this.admitted.size - this.screenedSet.size,
|
|
3423
|
+
expansions: this.completed.length
|
|
3424
|
+
});
|
|
3176
3425
|
}
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3426
|
+
artifact(nodeId) {
|
|
3427
|
+
const held = this.artifacts.get(nodeId);
|
|
3428
|
+
if (held !== void 0) return held;
|
|
3429
|
+
const artifact = this.options.codec.load(this.recorder, this.state.node(nodeId));
|
|
3430
|
+
this.artifacts.set(nodeId, artifact);
|
|
3431
|
+
return artifact;
|
|
3180
3432
|
}
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3433
|
+
capacity() {
|
|
3434
|
+
let total = 0;
|
|
3435
|
+
for (const lane of this.lanes.values()) total += lane.capacity;
|
|
3436
|
+
const cap = this.state.header.budget.maxConcurrency;
|
|
3437
|
+
return cap === null ? total : Math.min(cap, total);
|
|
3438
|
+
}
|
|
3439
|
+
/** A hard lane holds its enforced maximum. An estimate lane holds 1.5 times
|
|
3440
|
+
* the 99th percentile of its settled cells once 20 settled, else its prior. */
|
|
3441
|
+
cellReservation(lane) {
|
|
3442
|
+
if (lane.costCap === "hard") return {
|
|
3443
|
+
kind: "hard",
|
|
3444
|
+
usd: lane.cellUsd
|
|
3445
|
+
};
|
|
3446
|
+
return {
|
|
3447
|
+
kind: "estimate",
|
|
3448
|
+
usd: estimate(lane.settledCosts, lane.cellUsd)
|
|
3449
|
+
};
|
|
3184
3450
|
}
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
const
|
|
3188
|
-
return
|
|
3451
|
+
maxCellReservation() {
|
|
3452
|
+
let most = 0;
|
|
3453
|
+
for (const lane of this.lanes.values()) most = Math.max(most, this.cellReservation(lane).usd);
|
|
3454
|
+
return most;
|
|
3189
3455
|
}
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3456
|
+
operationReservation() {
|
|
3457
|
+
return {
|
|
3458
|
+
kind: "estimate",
|
|
3459
|
+
usd: estimate(this.operationCosts, this.options.proposer.reservationUsd ?? 0)
|
|
3460
|
+
};
|
|
3194
3461
|
}
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
return operation ? { ...operation } : void 0;
|
|
3462
|
+
pastDeadline() {
|
|
3463
|
+
const deadline = this.state.header.budget.deadline;
|
|
3464
|
+
return deadline !== null && this.now() >= Date.parse(deadline);
|
|
3199
3465
|
}
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
const scored = [];
|
|
3203
|
-
for (const cellId of this.nodes.get(nodeId)?.cellIds ?? []) {
|
|
3204
|
-
const cell = this.cells.get(cellId);
|
|
3205
|
-
if (cell.split === split && cell.score !== null) scored.push({
|
|
3206
|
-
cellId,
|
|
3207
|
-
unitId: cell.unitId,
|
|
3208
|
-
attempt: cell.attempts,
|
|
3209
|
-
score: cell.score
|
|
3210
|
-
});
|
|
3211
|
-
}
|
|
3212
|
-
return scored;
|
|
3466
|
+
async refresh() {
|
|
3467
|
+
this.state = await this.recorder.state();
|
|
3213
3468
|
}
|
|
3214
3469
|
};
|
|
3215
3470
|
/**
|
|
3216
|
-
* A read of `
|
|
3217
|
-
*
|
|
3218
|
-
*
|
|
3471
|
+
* A policy's read of a search: `screened` lists the admitted nodes whose screen
|
|
3472
|
+
* finished, in registration order; `screening` counts the admitted nodes still
|
|
3473
|
+
* being screened. Nodes are ranked on the selection split, or on train when
|
|
3474
|
+
* the search declares no selection split. The view reads no test cell.
|
|
3219
3475
|
*/
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
header;
|
|
3223
|
-
|
|
3224
|
-
head;
|
|
3225
|
-
/** `occurredAt` of the head entry; a writer stamps later events at or after it. */
|
|
3226
|
-
lastOccurredAt;
|
|
3227
|
-
audit;
|
|
3228
|
-
completion;
|
|
3229
|
-
state;
|
|
3230
|
-
version;
|
|
3231
|
-
constructor(state, version) {
|
|
3232
|
-
this.state = state;
|
|
3233
|
-
this.version = version;
|
|
3234
|
-
this.searchId = state.searchId;
|
|
3235
|
-
this.header = state.readHeader();
|
|
3236
|
-
this.closed = state.readClosed();
|
|
3237
|
-
this.head = state.readHead();
|
|
3238
|
-
this.lastOccurredAt = state.readLastOccurredAt();
|
|
3239
|
-
this.audit = state.readAudit();
|
|
3240
|
-
const reasons = state.incompleteReasons();
|
|
3241
|
-
if (!this.closed) reasons.unshift("search is open");
|
|
3242
|
-
this.completion = {
|
|
3243
|
-
complete: this.closed !== null,
|
|
3244
|
-
reasons
|
|
3245
|
-
};
|
|
3246
|
-
}
|
|
3247
|
-
/** The node the search started from: the first registered. */
|
|
3248
|
-
get rootNodeId() {
|
|
3249
|
-
return this.live().readNodeIds()[0] ?? null;
|
|
3250
|
-
}
|
|
3251
|
-
node(nodeId) {
|
|
3252
|
-
return this.live().readNode(nodeId);
|
|
3253
|
-
}
|
|
3254
|
-
hasNode(nodeId) {
|
|
3255
|
-
return this.live().readHasNode(nodeId);
|
|
3256
|
-
}
|
|
3257
|
-
/** Node ids in registration order. */
|
|
3258
|
-
nodeIds() {
|
|
3259
|
-
return [...this.live().readNodeIds()];
|
|
3260
|
-
}
|
|
3261
|
-
/** Nodes in registration order. */
|
|
3262
|
-
nodes() {
|
|
3263
|
-
const state = this.live();
|
|
3264
|
-
return state.readNodeIds().map((nodeId) => state.readNode(nodeId));
|
|
3265
|
-
}
|
|
3266
|
-
nodeIdForDigest(artifactDigest) {
|
|
3267
|
-
return this.live().readNodeIdForDigest(artifactDigest);
|
|
3268
|
-
}
|
|
3269
|
-
edge(edgeId) {
|
|
3270
|
-
return this.live().readEdge(edgeId);
|
|
3271
|
-
}
|
|
3272
|
-
/** Edges in ledger order. */
|
|
3273
|
-
edges() {
|
|
3274
|
-
const state = this.live();
|
|
3275
|
-
return state.readEdgeIds().map((edgeId) => state.readEdge(edgeId));
|
|
3276
|
-
}
|
|
3277
|
-
cell(cellId) {
|
|
3278
|
-
return this.live().readCell(cellId);
|
|
3279
|
-
}
|
|
3280
|
-
/** Cells in allocation order, optionally of one node. */
|
|
3281
|
-
cells(filter = {}) {
|
|
3282
|
-
const state = this.live();
|
|
3283
|
-
return state.readCellIds(filter.nodeId).map((cellId) => state.readCell(cellId));
|
|
3284
|
-
}
|
|
3285
|
-
operation(operationId) {
|
|
3286
|
-
return this.live().readOperation(operationId);
|
|
3287
|
-
}
|
|
3288
|
-
/** A node's scored cells on one split, in allocation order. Unscored cells
|
|
3289
|
-
* (errored, cancelled or in flight) are absent, never zero. */
|
|
3290
|
-
scoredCells(nodeId, split) {
|
|
3291
|
-
return this.live().readScoredCells(nodeId, split);
|
|
3292
|
-
}
|
|
3293
|
-
/** Per-unit score means of a node on one split, in unitId order. */
|
|
3294
|
-
unitScores(nodeId, split) {
|
|
3295
|
-
return searchUnitScores(this.scoredCells(nodeId, split));
|
|
3296
|
-
}
|
|
3297
|
-
/** The lineage a RunRecord mint needs for one cell attempt. */
|
|
3298
|
-
lineage(coordinates) {
|
|
3299
|
-
const state = this.live();
|
|
3300
|
-
if (coordinates.searchId !== this.searchId) throw new SearchLedgerError(`coordinates name search ${coordinates.searchId}, not ${this.searchId}`);
|
|
3301
|
-
const cell = state.readCell(coordinates.cellId);
|
|
3302
|
-
if (!cell || cell.nodeId !== coordinates.nodeId) throw new SearchLedgerError(`cell ${coordinates.cellId} of node ${coordinates.nodeId} is not in search ${this.searchId}`);
|
|
3303
|
-
if (coordinates.attempt < 1 || coordinates.attempt > cell.attempts) throw new SearchLedgerError(`cell ${coordinates.cellId} has ${cell.attempts} settled attempt(s), not attempt ${coordinates.attempt}`);
|
|
3304
|
-
const node = state.readNode(cell.nodeId);
|
|
3305
|
-
if (node.depth === null) throw new SearchLedgerError(`node ${node.nodeId} has unknown lineage, so its depth is unknown`);
|
|
3306
|
-
const containment = this.header?.containment ?? null;
|
|
3307
|
-
return {
|
|
3308
|
-
depth: node.depth,
|
|
3309
|
-
ordinal: node.ordinal,
|
|
3310
|
-
rep: cell.rep,
|
|
3311
|
-
containingRunId: containment ? searchCellRunId({
|
|
3312
|
-
cellId: containment.cellId,
|
|
3313
|
-
attempt: containment.attempt
|
|
3314
|
-
}) : null
|
|
3315
|
-
};
|
|
3316
|
-
}
|
|
3317
|
-
live() {
|
|
3318
|
-
if (!this.state.isAt(this.version)) throw new SearchLedgerError(`this SearchStateView is from sequence ${this.head?.sequence ?? -1}, and the ledger has moved on; read the ledger's state again`);
|
|
3319
|
-
return this.state;
|
|
3320
|
-
}
|
|
3321
|
-
};
|
|
3322
|
-
function nodeView(record) {
|
|
3323
|
-
const latest = record.decisions.at(-1)?.decision ?? null;
|
|
3476
|
+
function searchPolicyView(state, input) {
|
|
3477
|
+
const header = state.header;
|
|
3478
|
+
if (!header) throw new Error(`search ${state.searchId} has not been opened`);
|
|
3479
|
+
const split = header.splits.selection.tasks.length > 0 ? "selection" : "train";
|
|
3324
3480
|
return {
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
updatedSequence: record.updatedSequence
|
|
3481
|
+
searchId: state.searchId,
|
|
3482
|
+
seed: header.policy.seed,
|
|
3483
|
+
direction: header.objective.direction,
|
|
3484
|
+
split,
|
|
3485
|
+
rootNodeId: state.rootNodeId,
|
|
3486
|
+
expansions: input.expansions,
|
|
3487
|
+
screened: [...input.screened],
|
|
3488
|
+
screening: input.screening,
|
|
3489
|
+
complete: (nodeId) => {
|
|
3490
|
+
const cells = state.cells({ nodeId }).filter((cell) => cell.split === split);
|
|
3491
|
+
return cells.length > 0 && cells.every((cell) => cell.score !== null);
|
|
3492
|
+
},
|
|
3493
|
+
unitScores: (nodeId) => state.unitScores(nodeId, split),
|
|
3494
|
+
estimate: (nodeId, against) => estimateNode(state, nodeId, {
|
|
3495
|
+
against,
|
|
3496
|
+
split
|
|
3497
|
+
})
|
|
3343
3498
|
};
|
|
3344
3499
|
}
|
|
3345
|
-
function
|
|
3346
|
-
|
|
3347
|
-
root: ["train", "selection"],
|
|
3348
|
-
train: ["train"],
|
|
3349
|
-
screen: ["selection"],
|
|
3350
|
-
rung: ["selection"],
|
|
3351
|
-
claim: ["test"],
|
|
3352
|
-
external: ["train", "selection"]
|
|
3353
|
-
}[stage].includes(split)) throw integrity(`cell ${cellId}: a ${stage} cell cannot run on the ${split} split`);
|
|
3354
|
-
if (stage === "root" && !isRoot) throw integrity(`cell ${cellId}: only the root node runs root cells`);
|
|
3355
|
-
}
|
|
3356
|
-
function assertUnique(values, label, eventId) {
|
|
3357
|
-
if (new Set(values).size !== values.length) throw integrity(`event ${eventId} contains duplicate ${label} values`);
|
|
3500
|
+
function isTerminal(node) {
|
|
3501
|
+
return node.status !== null && node.status !== "advanced";
|
|
3358
3502
|
}
|
|
3359
|
-
function
|
|
3360
|
-
|
|
3361
|
-
|
|
3503
|
+
function estimate(costs, prior) {
|
|
3504
|
+
if (costs.length < ESTIMATE_FROM_CELLS) return prior;
|
|
3505
|
+
const sorted = [...costs].sort((a, b) => a - b);
|
|
3506
|
+
const p99 = sorted[Math.min(sorted.length - 1, Math.ceil(.99 * sorted.length) - 1)];
|
|
3507
|
+
return Math.round(ESTIMATE_MARGIN * p99 * 1e9) / 1e9;
|
|
3362
3508
|
}
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
}
|
|
3366
|
-
function integrity(message) {
|
|
3367
|
-
return new SearchLedgerIntegrityError(message);
|
|
3368
|
-
}
|
|
3369
|
-
//#endregion
|
|
3370
|
-
//#region src/campaign/search-ledger.ts
|
|
3371
|
-
/**
|
|
3372
|
-
* Durable, append-only event log of one search: nodes (content-addressed
|
|
3373
|
-
* artifacts), edges (the proposals that derived them), cells (one run of one
|
|
3374
|
-
* node on one task), operations, decisions, and the claim.
|
|
3375
|
-
*
|
|
3376
|
-
* The ledger is the search's only lineage record and its only resume
|
|
3377
|
-
* checkpoint. A RunRecord owns one measured run and `CostLedger` owns per-call
|
|
3378
|
-
* accounting; this ledger binds them by digest instead of copying them.
|
|
3379
|
-
*
|
|
3380
|
-
* The file format is canonical JSONL with a SHA-256 hash chain. Every append is
|
|
3381
|
-
* serialized across processes, fsynced before acknowledgement, and idempotent
|
|
3382
|
-
* by `eventId`. A malformed, non-canonical, truncated, reordered, or conflicting
|
|
3383
|
-
* log fails loudly; a bad row is never skipped. A ledger written under another
|
|
3384
|
-
* schema tag is refused with that tag named, never translated.
|
|
3385
|
-
*
|
|
3386
|
-
* The journal machinery is the generic `ledger-core` journal. This module is
|
|
3387
|
-
* the search codec: event schemas and canonical ordering. `SearchState` is the
|
|
3388
|
-
* state machine and read model.
|
|
3389
|
-
*/
|
|
3390
|
-
const NON_EMPTY = z.string().min(1).refine((value) => value.trim() === value, "must not contain surrounding whitespace");
|
|
3391
|
-
const HASH = z.string().regex(/^sha256:[a-f0-9]{64}$/);
|
|
3392
|
-
const IMMUTABLE_REVISION = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64}|sha256:[a-f0-9]{64}|sha512:[A-Za-z0-9+/=]+)$/);
|
|
3393
|
-
const ISO_TIMESTAMP = z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/).refine((value) => Number.isFinite(Date.parse(value)), "invalid timestamp");
|
|
3394
|
-
const NON_NEGATIVE_INT = z.number().int().nonnegative().safe();
|
|
3395
|
-
const POSITIVE_INT = z.number().int().positive().safe();
|
|
3396
|
-
const FINITE_NUMBER = z.number().finite();
|
|
3397
|
-
const USD = z.number().finite().nonnegative();
|
|
3398
|
-
const NODE_ID = z.string().regex(/^node_[a-f0-9]{32}$/);
|
|
3399
|
-
const CELL_ID = z.string().regex(/^cell_[a-f0-9]{32}$/);
|
|
3400
|
-
const ArtifactRefSchema = z.object({
|
|
3401
|
-
role: NON_EMPTY,
|
|
3402
|
-
uri: NON_EMPTY,
|
|
3403
|
-
sha256: HASH,
|
|
3404
|
-
byteLength: NON_NEGATIVE_INT
|
|
3405
|
-
}).strict();
|
|
3406
|
-
const SourceRefSchema = z.object({
|
|
3407
|
-
uri: NON_EMPTY,
|
|
3408
|
-
revision: IMMUTABLE_REVISION
|
|
3409
|
-
}).strict();
|
|
3410
|
-
const UnknownRefSchema = z.object({ unknown: NON_EMPTY }).strict();
|
|
3411
|
-
const FailureReasonSchema = z.object({
|
|
3412
|
-
code: NON_EMPTY,
|
|
3413
|
-
message: NON_EMPTY
|
|
3414
|
-
}).strict();
|
|
3415
|
-
const ModelIdentitySchema = z.union([z.object({
|
|
3416
|
-
provider: NON_EMPTY,
|
|
3417
|
-
snapshot: NON_EMPTY.refine(modelHasSnapshot, "model must include an immutable snapshot")
|
|
3418
|
-
}).strict(), z.object({
|
|
3419
|
-
provider: NON_EMPTY,
|
|
3420
|
-
alias: NON_EMPTY,
|
|
3421
|
-
unknown: NON_EMPTY
|
|
3422
|
-
}).strict()]);
|
|
3423
|
-
const ExecutionIdentitySchema = z.object({
|
|
3424
|
-
model: ModelIdentitySchema,
|
|
3425
|
-
agent: SourceRefSchema,
|
|
3426
|
-
benchmark: SourceRefSchema
|
|
3427
|
-
}).strict();
|
|
3428
|
-
const SURFACE_KINDS = [
|
|
3429
|
-
"prompt",
|
|
3430
|
-
"tool-contract",
|
|
3431
|
-
"runtime-config",
|
|
3432
|
-
"memory",
|
|
3433
|
-
"knowledge",
|
|
3434
|
-
"agent-profile",
|
|
3435
|
-
"code",
|
|
3436
|
-
"deployment"
|
|
3437
|
-
];
|
|
3438
|
-
const SPLIT = z.enum([
|
|
3439
|
-
"train",
|
|
3440
|
-
"selection",
|
|
3441
|
-
"test"
|
|
3442
|
-
]);
|
|
3443
|
-
const ReservationSchema = z.object({
|
|
3444
|
-
kind: z.enum(["hard", "estimate"]),
|
|
3445
|
-
usd: USD
|
|
3446
|
-
}).strict();
|
|
3447
|
-
const EventBaseShape = {
|
|
3448
|
-
eventId: NON_EMPTY,
|
|
3449
|
-
occurredAt: ISO_TIMESTAMP,
|
|
3450
|
-
artifacts: z.array(ArtifactRefSchema)
|
|
3451
|
-
};
|
|
3452
|
-
const OperationKindSchema = z.enum([
|
|
3453
|
-
"candidate-generation",
|
|
3454
|
-
"analysis",
|
|
3455
|
-
"selection",
|
|
3456
|
-
"judge",
|
|
3457
|
-
"other"
|
|
3458
|
-
]);
|
|
3459
|
-
const KnownTokensSchema = z.object({
|
|
3460
|
-
status: z.literal("known"),
|
|
3461
|
-
inputTokens: NON_NEGATIVE_INT,
|
|
3462
|
-
outputTokens: NON_NEGATIVE_INT,
|
|
3463
|
-
cachedTokens: NON_NEGATIVE_INT
|
|
3464
|
-
}).strict();
|
|
3465
|
-
const UnknownTokensSchema = z.object({
|
|
3466
|
-
status: z.literal("unknown"),
|
|
3467
|
-
reason: NON_EMPTY
|
|
3468
|
-
}).strict();
|
|
3469
|
-
const KnownCostSchema = z.object({
|
|
3470
|
-
status: z.literal("known"),
|
|
3471
|
-
usd: USD,
|
|
3472
|
-
source: z.enum([
|
|
3473
|
-
"provider",
|
|
3474
|
-
"pricing-table",
|
|
3475
|
-
"free"
|
|
3476
|
-
])
|
|
3477
|
-
}).strict().superRefine((cost, ctx) => {
|
|
3478
|
-
if (cost.source === "free" && cost.usd !== 0) ctx.addIssue({
|
|
3479
|
-
code: "custom",
|
|
3480
|
-
message: "free cost source must have usd 0"
|
|
3481
|
-
});
|
|
3482
|
-
});
|
|
3483
|
-
const UnknownCostSchema = z.object({
|
|
3484
|
-
status: z.literal("unknown"),
|
|
3485
|
-
knownLowerBoundUsd: USD,
|
|
3486
|
-
reason: NON_EMPTY
|
|
3487
|
-
}).strict();
|
|
3488
|
-
const AccountingSchema = z.object({
|
|
3489
|
-
tokens: z.discriminatedUnion("status", [KnownTokensSchema, UnknownTokensSchema]),
|
|
3490
|
-
cost: z.discriminatedUnion("status", [KnownCostSchema, UnknownCostSchema])
|
|
3491
|
-
}).strict();
|
|
3492
|
-
const MetricsSchema = z.record(NON_EMPTY, FINITE_NUMBER).superRefine((metrics, ctx) => {
|
|
3493
|
-
for (const key of Object.keys(metrics)) if (key === "__proto__" || key === "constructor" || key === "prototype") ctx.addIssue({
|
|
3494
|
-
code: "custom",
|
|
3495
|
-
message: `unsafe metric key ${key}`
|
|
3496
|
-
});
|
|
3497
|
-
});
|
|
3498
|
-
const OutcomeSchema = z.discriminatedUnion("status", [
|
|
3499
|
-
z.object({
|
|
3500
|
-
status: z.literal("passed"),
|
|
3501
|
-
score: FINITE_NUMBER,
|
|
3502
|
-
metrics: MetricsSchema
|
|
3503
|
-
}).strict(),
|
|
3504
|
-
z.object({
|
|
3505
|
-
status: z.literal("failed"),
|
|
3506
|
-
score: FINITE_NUMBER,
|
|
3507
|
-
metrics: MetricsSchema,
|
|
3508
|
-
failure: FailureReasonSchema
|
|
3509
|
-
}).strict(),
|
|
3510
|
-
z.object({
|
|
3511
|
-
status: z.literal("errored"),
|
|
3512
|
-
metrics: MetricsSchema,
|
|
3513
|
-
error: FailureReasonSchema.extend({ retryable: z.boolean() }).strict()
|
|
3514
|
-
}).strict()
|
|
3515
|
-
]);
|
|
3516
|
-
const EffectSchema = z.discriminatedUnion("status", [z.object({
|
|
3517
|
-
status: z.literal("measured"),
|
|
3518
|
-
metric: NON_EMPTY,
|
|
3519
|
-
baselineValue: FINITE_NUMBER,
|
|
3520
|
-
candidateValue: FINITE_NUMBER,
|
|
3521
|
-
delta: FINITE_NUMBER
|
|
3522
|
-
}).strict().superRefine((effect, ctx) => {
|
|
3523
|
-
const expected = effect.candidateValue - effect.baselineValue;
|
|
3524
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(expected), Math.abs(effect.delta)) * 8;
|
|
3525
|
-
if (Math.abs(effect.delta - expected) > tolerance) ctx.addIssue({
|
|
3526
|
-
code: "custom",
|
|
3527
|
-
message: "delta must equal candidateValue - baselineValue"
|
|
3528
|
-
});
|
|
3529
|
-
}), z.object({
|
|
3530
|
-
status: z.literal("not-measured"),
|
|
3531
|
-
reason: NON_EMPTY
|
|
3532
|
-
}).strict()]);
|
|
3533
|
-
const SurfaceEvidenceSchema = z.object({
|
|
3534
|
-
surfaceId: NON_EMPTY,
|
|
3535
|
-
fired: z.boolean(),
|
|
3536
|
-
firingCount: NON_NEGATIVE_INT,
|
|
3537
|
-
effect: EffectSchema,
|
|
3538
|
-
evidence: z.array(ArtifactRefSchema).min(1)
|
|
3539
|
-
}).strict().superRefine((evidence, ctx) => {
|
|
3540
|
-
if (evidence.fired !== evidence.firingCount > 0) ctx.addIssue({
|
|
3541
|
-
code: "custom",
|
|
3542
|
-
message: "fired must equal firingCount > 0"
|
|
3543
|
-
});
|
|
3544
|
-
if (!evidence.fired && evidence.effect.status === "measured" && evidence.effect.delta !== 0) ctx.addIssue({
|
|
3545
|
-
code: "custom",
|
|
3546
|
-
message: "a surface that did not fire has no effect"
|
|
3547
|
-
});
|
|
3548
|
-
});
|
|
3549
|
-
const TaskSchema = z.object({
|
|
3550
|
-
taskId: NON_EMPTY,
|
|
3551
|
-
unitId: NON_EMPTY,
|
|
3552
|
-
source: SourceRefSchema
|
|
3553
|
-
}).strict();
|
|
3554
|
-
const SplitTasksSchema = z.object({
|
|
3555
|
-
taskSetDigest: HASH,
|
|
3556
|
-
tasks: z.array(TaskSchema)
|
|
3557
|
-
}).strict();
|
|
3558
|
-
const SearchOpenedSchema = z.object({
|
|
3559
|
-
...EventBaseShape,
|
|
3560
|
-
kind: z.literal("search-opened"),
|
|
3561
|
-
subject: NON_EMPTY,
|
|
3562
|
-
process: z.object({
|
|
3563
|
-
name: NON_EMPTY,
|
|
3564
|
-
executionRef: SourceRefSchema
|
|
3565
|
-
}).strict(),
|
|
3566
|
-
artifactKind: z.enum([...SURFACE_KINDS, "output"]),
|
|
3567
|
-
objective: z.object({
|
|
3568
|
-
metric: NON_EMPTY,
|
|
3569
|
-
direction: z.enum(["maximize", "minimize"]),
|
|
3570
|
-
judge: z.union([SourceRefSchema, UnknownRefSchema]),
|
|
3571
|
-
claim: evaluationClaimSchema
|
|
3572
|
-
}).strict(),
|
|
3573
|
-
splits: z.object({
|
|
3574
|
-
train: SplitTasksSchema,
|
|
3575
|
-
selection: SplitTasksSchema,
|
|
3576
|
-
test: SplitTasksSchema,
|
|
3577
|
-
heldOutUnits: z.boolean()
|
|
3578
|
-
}).strict(),
|
|
3579
|
-
policy: z.object({
|
|
3580
|
-
expansion: NON_EMPTY,
|
|
3581
|
-
allocation: NON_EMPTY,
|
|
3582
|
-
seed: z.number().int().safe()
|
|
3583
|
-
}).strict(),
|
|
3584
|
-
budget: z.object({
|
|
3585
|
-
maxUsd: USD.nullable(),
|
|
3586
|
-
maxCells: POSITIVE_INT.nullable(),
|
|
3587
|
-
maxNodes: POSITIVE_INT.nullable(),
|
|
3588
|
-
deadline: ISO_TIMESTAMP.nullable(),
|
|
3589
|
-
maxConcurrency: POSITIVE_INT.nullable(),
|
|
3590
|
-
reservedClaimUsd: USD
|
|
3591
|
-
}).strict(),
|
|
3592
|
-
containment: z.object({
|
|
3593
|
-
searchId: NON_EMPTY,
|
|
3594
|
-
cellId: CELL_ID,
|
|
3595
|
-
attempt: POSITIVE_INT
|
|
3596
|
-
}).strict().nullable(),
|
|
3597
|
-
derivedFrom: z.object({
|
|
3598
|
-
searchId: NON_EMPTY,
|
|
3599
|
-
nodeId: NODE_ID,
|
|
3600
|
-
headHash: HASH
|
|
3601
|
-
}).strict().nullable(),
|
|
3602
|
-
identity: ExecutionIdentitySchema
|
|
3603
|
-
}).strict();
|
|
3604
|
-
const OperationStartedSchema = z.object({
|
|
3605
|
-
...EventBaseShape,
|
|
3606
|
-
kind: z.literal("operation-started"),
|
|
3607
|
-
operationId: NON_EMPTY,
|
|
3608
|
-
operationKind: OperationKindSchema,
|
|
3609
|
-
reservation: ReservationSchema.nullable()
|
|
3610
|
-
}).strict();
|
|
3611
|
-
const OperationRecordedSchema = z.object({
|
|
3612
|
-
...EventBaseShape,
|
|
3613
|
-
kind: z.literal("operation-recorded"),
|
|
3614
|
-
operationId: NON_EMPTY,
|
|
3615
|
-
operationKind: OperationKindSchema,
|
|
3616
|
-
execution: z.discriminatedUnion("kind", [z.object({
|
|
3617
|
-
kind: z.literal("model"),
|
|
3618
|
-
model: ModelIdentitySchema,
|
|
3619
|
-
source: SourceRefSchema
|
|
3620
|
-
}).strict(), z.object({
|
|
3621
|
-
kind: z.literal("deterministic"),
|
|
3622
|
-
source: SourceRefSchema
|
|
3623
|
-
}).strict()]),
|
|
3624
|
-
outcome: z.discriminatedUnion("status", [
|
|
3625
|
-
z.object({ status: z.literal("completed") }).strict(),
|
|
3626
|
-
z.object({
|
|
3627
|
-
status: z.literal("partial"),
|
|
3628
|
-
failure: FailureReasonSchema
|
|
3629
|
-
}).strict(),
|
|
3630
|
-
z.object({
|
|
3631
|
-
status: z.literal("failed"),
|
|
3632
|
-
failure: FailureReasonSchema
|
|
3633
|
-
}).strict()
|
|
3634
|
-
]),
|
|
3635
|
-
accounting: AccountingSchema
|
|
3636
|
-
}).strict();
|
|
3637
|
-
const NodeRegisteredSchema = z.object({
|
|
3638
|
-
...EventBaseShape,
|
|
3639
|
-
kind: z.literal("node-registered"),
|
|
3640
|
-
nodeId: NODE_ID,
|
|
3641
|
-
artifactDigest: HASH,
|
|
3642
|
-
artifact: ArtifactRefSchema,
|
|
3643
|
-
surfaces: z.array(z.object({
|
|
3644
|
-
surfaceId: NON_EMPTY,
|
|
3645
|
-
kind: z.enum(SURFACE_KINDS),
|
|
3646
|
-
artifact: ArtifactRefSchema
|
|
3647
|
-
}).strict())
|
|
3648
|
-
}).strict();
|
|
3649
|
-
const NodeRefSchema = z.object({
|
|
3650
|
-
searchId: NON_EMPTY,
|
|
3651
|
-
nodeId: NODE_ID
|
|
3652
|
-
}).strict();
|
|
3653
|
-
const ArtifactOrUnknownSchema = z.union([ArtifactRefSchema, UnknownRefSchema]);
|
|
3654
|
-
const EdgeRecordedSchema = z.object({
|
|
3655
|
-
...EventBaseShape,
|
|
3656
|
-
kind: z.literal("edge-recorded"),
|
|
3657
|
-
edgeId: z.string().regex(/^edge_[a-f0-9]{32}$/),
|
|
3658
|
-
childNodeId: NODE_ID,
|
|
3659
|
-
parents: z.array(NodeRefSchema),
|
|
3660
|
-
operator: z.enum([
|
|
3661
|
-
"seed",
|
|
3662
|
-
"draft",
|
|
3663
|
-
"improve",
|
|
3664
|
-
"debug",
|
|
3665
|
-
"merge",
|
|
3666
|
-
"derive"
|
|
3667
|
-
]),
|
|
3668
|
-
attribution: z.enum([
|
|
3669
|
-
"explicit",
|
|
3670
|
-
"correlated",
|
|
3671
|
-
"unknown"
|
|
3672
|
-
]),
|
|
3673
|
-
proposer: z.object({
|
|
3674
|
-
kind: z.enum([
|
|
3675
|
-
"trace",
|
|
3676
|
-
"frontier-author",
|
|
3677
|
-
"human",
|
|
3678
|
-
"optimizer",
|
|
3679
|
-
"compound"
|
|
3680
|
-
]),
|
|
3681
|
-
name: NON_EMPTY,
|
|
3682
|
-
operationId: NON_EMPTY.nullable(),
|
|
3683
|
-
source: SourceRefSchema
|
|
3684
|
-
}).strict().nullable(),
|
|
3685
|
-
selection: z.object({
|
|
3686
|
-
rule: NON_EMPTY,
|
|
3687
|
-
evidence: z.record(NON_EMPTY, FINITE_NUMBER)
|
|
3688
|
-
}).strict().nullable(),
|
|
3689
|
-
rationale: ArtifactOrUnknownSchema,
|
|
3690
|
-
diffs: z.array(ArtifactOrUnknownSchema),
|
|
3691
|
-
label: z.string()
|
|
3692
|
-
}).strict();
|
|
3693
|
-
const CellAllocatedSchema = z.object({
|
|
3694
|
-
...EventBaseShape,
|
|
3695
|
-
kind: z.literal("cell-allocated"),
|
|
3696
|
-
cellId: CELL_ID,
|
|
3697
|
-
nodeId: NODE_ID,
|
|
3698
|
-
taskId: NON_EMPTY,
|
|
3699
|
-
unitId: NON_EMPTY,
|
|
3700
|
-
split: SPLIT,
|
|
3701
|
-
rep: NON_NEGATIVE_INT,
|
|
3702
|
-
stage: z.enum([
|
|
3703
|
-
"root",
|
|
3704
|
-
"train",
|
|
3705
|
-
"screen",
|
|
3706
|
-
"rung",
|
|
3707
|
-
"claim",
|
|
3708
|
-
"external"
|
|
3709
|
-
]),
|
|
3710
|
-
lane: NON_EMPTY.nullable(),
|
|
3711
|
-
reservation: ReservationSchema.nullable()
|
|
3712
|
-
}).strict();
|
|
3713
|
-
const TraceRefSchema = z.union([z.object({
|
|
3714
|
-
traceId: NON_EMPTY,
|
|
3715
|
-
execRunId: NON_EMPTY.nullable(),
|
|
3716
|
-
spansWritten: NON_NEGATIVE_INT.nullable(),
|
|
3717
|
-
spansDropped: NON_NEGATIVE_INT.nullable()
|
|
3718
|
-
}).strict(), UnknownRefSchema]);
|
|
3719
|
-
const NON_NEGATIVE = z.number().finite().nonnegative();
|
|
3720
|
-
const CellSettledSchema = z.object({
|
|
3721
|
-
...EventBaseShape,
|
|
3722
|
-
kind: z.literal("cell-settled"),
|
|
3723
|
-
cellId: CELL_ID,
|
|
3724
|
-
attempt: POSITIVE_INT,
|
|
3725
|
-
runId: NON_EMPTY,
|
|
3726
|
-
outcome: OutcomeSchema,
|
|
3727
|
-
accounting: AccountingSchema,
|
|
3728
|
-
boxMinutes: NON_NEGATIVE.nullable(),
|
|
3729
|
-
wallMs: NON_NEGATIVE.nullable(),
|
|
3730
|
-
queueMs: NON_NEGATIVE.nullable(),
|
|
3731
|
-
placement: z.object({
|
|
3732
|
-
lane: NON_EMPTY,
|
|
3733
|
-
boxId: NON_EMPTY.nullable()
|
|
3734
|
-
}).strict().nullable(),
|
|
3735
|
-
identity: ExecutionIdentitySchema,
|
|
3736
|
-
surfaceEvidence: z.array(SurfaceEvidenceSchema),
|
|
3737
|
-
traceRef: TraceRefSchema
|
|
3738
|
-
}).strict();
|
|
3739
|
-
const CellCancelledSchema = z.object({
|
|
3740
|
-
...EventBaseShape,
|
|
3741
|
-
kind: z.literal("cell-cancelled"),
|
|
3742
|
-
cellId: CELL_ID,
|
|
3743
|
-
reason: z.enum([
|
|
3744
|
-
"pruned",
|
|
3745
|
-
"budget",
|
|
3746
|
-
"deadline",
|
|
3747
|
-
"aborted"
|
|
3748
|
-
])
|
|
3749
|
-
}).strict();
|
|
3750
|
-
const NodeEstimateSchema = z.object({
|
|
3751
|
-
against: NODE_ID,
|
|
3752
|
-
split: SPLIT,
|
|
3753
|
-
units: NON_NEGATIVE_INT,
|
|
3754
|
-
pairs: NON_NEGATIVE_INT,
|
|
3755
|
-
delta: FINITE_NUMBER.nullable(),
|
|
3756
|
-
interval: z.tuple([FINITE_NUMBER, FINITE_NUMBER]).nullable(),
|
|
3757
|
-
method: z.enum([
|
|
3758
|
-
"none",
|
|
3759
|
-
"insufficient",
|
|
3760
|
-
"descriptive",
|
|
3761
|
-
"bootstrap"
|
|
3762
|
-
]),
|
|
3763
|
-
exactSignP: z.number().min(0).max(1).nullable(),
|
|
3764
|
-
indeterminate: z.boolean(),
|
|
3765
|
-
cellSetDigest: HASH,
|
|
3766
|
-
estimator: SourceRefSchema
|
|
3767
|
-
}).strict();
|
|
3768
|
-
const NodeDecidedSchema = z.object({
|
|
3769
|
-
...EventBaseShape,
|
|
3770
|
-
kind: z.literal("node-decided"),
|
|
3771
|
-
nodeId: NODE_ID,
|
|
3772
|
-
decision: z.discriminatedUnion("status", [
|
|
3773
|
-
z.object({
|
|
3774
|
-
status: z.literal("advanced"),
|
|
3775
|
-
rung: NON_NEGATIVE_INT
|
|
3776
|
-
}).strict(),
|
|
3777
|
-
z.object({ status: z.literal("pruned") }).strict(),
|
|
3778
|
-
z.object({ status: z.literal("invalid") }).strict(),
|
|
3779
|
-
z.object({ status: z.literal("finalist") }).strict(),
|
|
3780
|
-
z.object({ status: z.literal("selected") }).strict(),
|
|
3781
|
-
z.object({ status: z.literal("rejected") }).strict()
|
|
3782
|
-
]),
|
|
3783
|
-
basis: NodeEstimateSchema.nullable(),
|
|
3784
|
-
rule: NON_EMPTY,
|
|
3785
|
-
reason: NON_EMPTY
|
|
3786
|
-
}).strict();
|
|
3787
|
-
const SearchClosedSchema = z.object({
|
|
3788
|
-
...EventBaseShape,
|
|
3789
|
-
kind: z.literal("search-closed"),
|
|
3790
|
-
reason: z.enum([
|
|
3791
|
-
"budget",
|
|
3792
|
-
"deadline",
|
|
3793
|
-
"max-nodes",
|
|
3794
|
-
"patience",
|
|
3795
|
-
"converged",
|
|
3796
|
-
"aborted"
|
|
3797
|
-
]),
|
|
3798
|
-
claim: z.object({
|
|
3799
|
-
power: z.union([z.object({
|
|
3800
|
-
adequate: z.boolean(),
|
|
3801
|
-
minimumEffect: z.number().finite().positive(),
|
|
3802
|
-
powerAtMinimumEffect: z.number().min(0).max(1),
|
|
3803
|
-
units: NON_NEGATIVE_INT
|
|
3804
|
-
}).strict(), UnknownRefSchema]),
|
|
3805
|
-
finalists: z.array(z.object({
|
|
3806
|
-
nodeId: NODE_ID,
|
|
3807
|
-
estimate: NodeEstimateSchema.nullable(),
|
|
3808
|
-
promote: z.boolean()
|
|
3809
|
-
}).strict()),
|
|
3810
|
-
selected: NODE_ID.nullable(),
|
|
3811
|
-
decision: z.enum([
|
|
3812
|
-
"ship",
|
|
3813
|
-
"hold",
|
|
3814
|
-
"test-cannot-resolve"
|
|
3815
|
-
])
|
|
3816
|
-
}).strict().nullable()
|
|
3817
|
-
}).strict();
|
|
3818
|
-
const EventSchema = z.discriminatedUnion("kind", [
|
|
3819
|
-
SearchOpenedSchema,
|
|
3820
|
-
OperationStartedSchema,
|
|
3821
|
-
OperationRecordedSchema,
|
|
3822
|
-
NodeRegisteredSchema,
|
|
3823
|
-
EdgeRecordedSchema,
|
|
3824
|
-
CellAllocatedSchema,
|
|
3825
|
-
CellSettledSchema,
|
|
3826
|
-
CellCancelledSchema,
|
|
3827
|
-
NodeDecidedSchema,
|
|
3828
|
-
SearchClosedSchema
|
|
3829
|
-
]);
|
|
3830
|
-
const EntrySchema = z.object({
|
|
3831
|
-
schema: z.literal(SEARCH_LEDGER_SCHEMA),
|
|
3832
|
-
searchId: NON_EMPTY,
|
|
3833
|
-
sequence: NON_NEGATIVE_INT,
|
|
3834
|
-
previousHash: z.union([HASH, z.null()]),
|
|
3835
|
-
event: EventSchema,
|
|
3836
|
-
entryHash: HASH
|
|
3837
|
-
}).strict();
|
|
3838
|
-
/** Validate and return a canonical copy. Arrays whose order carries no meaning
|
|
3839
|
-
* are sorted, so retries from different processes produce identical bytes. */
|
|
3840
|
-
function validateSearchLedgerEvent(input) {
|
|
3841
|
-
const parsed = EventSchema.safeParse(input);
|
|
3842
|
-
if (!parsed.success) throw new SearchLedgerError(`invalid search ledger event: ${formatZodError(parsed.error)}`);
|
|
3843
|
-
return normalizeEvent(parsed.data);
|
|
3844
|
-
}
|
|
3845
|
-
/** Open a durable filesystem search ledger. Construction performs no I/O; the
|
|
3846
|
-
* first `append` or `state` verifies the whole existing file. */
|
|
3847
|
-
function openSearchLedger(options) {
|
|
3848
|
-
return new FileSearchLedger(options.path, options.searchId, options.trustedHead);
|
|
3849
|
-
}
|
|
3850
|
-
/** Verify and project immutable search-ledger JSONL through the same codec as
|
|
3851
|
-
* `FileSearchLedger`. */
|
|
3852
|
-
function replaySearchLedgerText(text, searchId, source) {
|
|
3853
|
-
return replayLedgerText(text, source, searchLedgerCodec(searchId));
|
|
3854
|
-
}
|
|
3855
|
-
function searchLedgerCodec(searchId) {
|
|
3509
|
+
/** Spend of work whose result was lost: unknown, with no proven floor. */
|
|
3510
|
+
function interruptedAccounting() {
|
|
3856
3511
|
return {
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
searchId
|
|
3861
|
-
},
|
|
3862
|
-
conflictError: (message) => new SearchLedgerConflictError(message),
|
|
3863
|
-
parseEntry: parseSearchLedgerEntry,
|
|
3864
|
-
checkEntryHeader: (entry, index) => {
|
|
3865
|
-
if (entry.searchId !== searchId) throw new SearchLedgerIntegrityError(`entry ${index} belongs to search ${entry.searchId}, expected ${searchId}`);
|
|
3512
|
+
tokens: {
|
|
3513
|
+
status: "unknown",
|
|
3514
|
+
reason: "the result of this work was lost"
|
|
3866
3515
|
},
|
|
3867
|
-
|
|
3516
|
+
cost: {
|
|
3517
|
+
status: "unknown",
|
|
3518
|
+
knownLowerBoundUsd: 0,
|
|
3519
|
+
reason: "the result of this work was lost, so its spend is unknown"
|
|
3520
|
+
}
|
|
3868
3521
|
};
|
|
3869
3522
|
}
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
searchId;
|
|
3874
|
-
trustedHeadPath;
|
|
3875
|
-
trustedHeadMode;
|
|
3876
|
-
journal;
|
|
3877
|
-
constructor(path, searchId, trustedHead = "pin") {
|
|
3878
|
-
if (path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
|
|
3879
|
-
if (searchId.length === 0 || searchId.trim() !== searchId) throw new SearchLedgerError("searchId must be non-empty without surrounding whitespace");
|
|
3880
|
-
this.searchId = searchId;
|
|
3881
|
-
this.trustedHeadMode = trustedHead;
|
|
3882
|
-
this.journal = new FileLedgerJournal(path, searchLedgerCodec(searchId), { requireTrustedHead: trustedHead === "require" });
|
|
3883
|
-
this.path = this.journal.path;
|
|
3884
|
-
this.trustedHeadPath = this.journal.trustedHeadPath;
|
|
3885
|
-
}
|
|
3886
|
-
async state() {
|
|
3887
|
-
return this.journal.replay();
|
|
3888
|
-
}
|
|
3889
|
-
async append(input) {
|
|
3890
|
-
const event = validateSearchLedgerEvent(input);
|
|
3891
|
-
const { entry, appended, projection } = await this.journal.append(event, { pinHead: this.trustedHeadMode !== "off" });
|
|
3892
|
-
return {
|
|
3893
|
-
entry,
|
|
3894
|
-
appended,
|
|
3895
|
-
state: projection
|
|
3896
|
-
};
|
|
3897
|
-
}
|
|
3898
|
-
async trustedHead() {
|
|
3899
|
-
return this.journal.trustedHead();
|
|
3900
|
-
}
|
|
3901
|
-
async pinTrustedHead() {
|
|
3902
|
-
return this.journal.pinTrustedHead();
|
|
3903
|
-
}
|
|
3904
|
-
async clearTrustedHead() {
|
|
3905
|
-
return this.journal.clearTrustedHead();
|
|
3906
|
-
}
|
|
3907
|
-
};
|
|
3908
|
-
function parseSearchLedgerEntry(raw, context) {
|
|
3909
|
-
const schema = raw?.schema;
|
|
3910
|
-
if (schema !== "tangle.search-ledger.2026-09") throw new SearchLedgerIntegrityError(`search ledger ${context.path} line ${context.line} was written under schema ${JSON.stringify(schema)}; this version reads only ${SEARCH_LEDGER_SCHEMA} and does not translate other formats`);
|
|
3911
|
-
const parsed = EntrySchema.safeParse(raw);
|
|
3912
|
-
if (!parsed.success) throw new SearchLedgerIntegrityError(`search ledger ${context.path} has a malformed entry at line ${context.line}: ${formatZodError(parsed.error)}`);
|
|
3913
|
-
const entry = parsed.data;
|
|
3914
|
-
if (canonicalString(validateSearchLedgerEvent(entry.event)) !== canonicalString(entry.event)) throw new SearchLedgerIntegrityError(`search ledger ${context.path} has non-canonical event ordering at line ${context.line}`);
|
|
3915
|
-
return entry;
|
|
3916
|
-
}
|
|
3917
|
-
function normalizeEvent(event) {
|
|
3918
|
-
const artifacts = sortArtifacts(event.artifacts);
|
|
3919
|
-
switch (event.kind) {
|
|
3920
|
-
case "search-opened": return {
|
|
3921
|
-
...event,
|
|
3922
|
-
artifacts,
|
|
3923
|
-
splits: {
|
|
3924
|
-
...event.splits,
|
|
3925
|
-
train: {
|
|
3926
|
-
...event.splits.train,
|
|
3927
|
-
tasks: sortTasks(event.splits.train.tasks)
|
|
3928
|
-
},
|
|
3929
|
-
selection: {
|
|
3930
|
-
...event.splits.selection,
|
|
3931
|
-
tasks: sortTasks(event.splits.selection.tasks)
|
|
3932
|
-
},
|
|
3933
|
-
test: {
|
|
3934
|
-
...event.splits.test,
|
|
3935
|
-
tasks: sortTasks(event.splits.test.tasks)
|
|
3936
|
-
}
|
|
3937
|
-
}
|
|
3938
|
-
};
|
|
3939
|
-
case "node-registered": return {
|
|
3940
|
-
...event,
|
|
3941
|
-
artifacts,
|
|
3942
|
-
surfaces: [...event.surfaces].sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
|
|
3943
|
-
};
|
|
3944
|
-
case "cell-settled": return {
|
|
3945
|
-
...event,
|
|
3946
|
-
artifacts,
|
|
3947
|
-
surfaceEvidence: [...event.surfaceEvidence].map((evidence) => ({
|
|
3948
|
-
...evidence,
|
|
3949
|
-
evidence: sortArtifacts(evidence.evidence)
|
|
3950
|
-
})).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
|
|
3951
|
-
};
|
|
3952
|
-
default: return {
|
|
3953
|
-
...event,
|
|
3954
|
-
artifacts
|
|
3955
|
-
};
|
|
3956
|
-
}
|
|
3957
|
-
}
|
|
3958
|
-
function sortTasks(tasks) {
|
|
3959
|
-
return [...tasks].sort((a, b) => compareStrings(a.taskId, b.taskId));
|
|
3960
|
-
}
|
|
3961
|
-
function sortArtifacts(artifacts) {
|
|
3962
|
-
return [...artifacts].sort((a, b) => compareStrings(artifactKey(a), artifactKey(b)));
|
|
3963
|
-
}
|
|
3964
|
-
function formatZodError(error) {
|
|
3965
|
-
return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.join(".") : "<root>"}: ${issue.message}`).join("; ");
|
|
3523
|
+
function errorMessage(error) {
|
|
3524
|
+
const text = error instanceof Error ? error.message : String(error);
|
|
3525
|
+
return text.trim().length > 0 ? text : "the proposer failed without a message";
|
|
3966
3526
|
}
|
|
3967
3527
|
//#endregion
|
|
3968
3528
|
//#region src/campaign/search-history-receipt.ts
|
|
@@ -4254,11 +3814,25 @@ var SearchRecorder = class SearchRecorder {
|
|
|
4254
3814
|
}
|
|
4255
3815
|
return {
|
|
4256
3816
|
role,
|
|
4257
|
-
uri:
|
|
3817
|
+
uri: pathToFileURL(path).href,
|
|
4258
3818
|
sha256: `sha256:${hex}`,
|
|
4259
3819
|
byteLength: bytes.byteLength
|
|
4260
3820
|
};
|
|
4261
3821
|
}
|
|
3822
|
+
/**
|
|
3823
|
+
* Read back a blob this recorder stored, by its content address. The bytes
|
|
3824
|
+
* must hash to the reference's digest and length, so a resumed search reads
|
|
3825
|
+
* exactly what the ledger bound.
|
|
3826
|
+
*/
|
|
3827
|
+
readBlob(ref) {
|
|
3828
|
+
const hex = ref.sha256.slice(7);
|
|
3829
|
+
const path = join(this.blobDir, `${hex}.json`);
|
|
3830
|
+
const text = this.storage.read(path);
|
|
3831
|
+
if (text === void 0) throw new Error(`search ${this.searchId}: blob ${ref.sha256} (${ref.role}) is missing at ${path}`);
|
|
3832
|
+
const bytes = Buffer.from(text, "utf8");
|
|
3833
|
+
if (createHash("sha256").update(bytes).digest("hex") !== hex || bytes.byteLength !== ref.byteLength) throw new Error(`search ${this.searchId}: blob at ${path} does not match ${ref.sha256}`);
|
|
3834
|
+
return JSON.parse(text);
|
|
3835
|
+
}
|
|
4262
3836
|
/** Register a node, or return the existing node with the same artifact digest. */
|
|
4263
3837
|
async registerNode(input) {
|
|
4264
3838
|
const existing = (await this.ledger.state()).nodeIdForDigest(input.artifactDigest);
|
|
@@ -4464,306 +4038,868 @@ function surfaceNode(recorder, surface) {
|
|
|
4464
4038
|
surface
|
|
4465
4039
|
});
|
|
4466
4040
|
return {
|
|
4467
|
-
artifactDigest: surfaceContentHash(surface),
|
|
4468
|
-
artifact,
|
|
4469
|
-
surfaces: declaredSurfaces(surface, artifact)
|
|
4041
|
+
artifactDigest: surfaceContentHash(surface),
|
|
4042
|
+
artifact,
|
|
4043
|
+
surfaces: declaredSurfaces(surface, artifact)
|
|
4044
|
+
};
|
|
4045
|
+
}
|
|
4046
|
+
/** A parent-to-child diff of two mutable surfaces, stored as a blob. */
|
|
4047
|
+
function surfaceDiff(recorder, parent, child) {
|
|
4048
|
+
return recorder.blob("diff", {
|
|
4049
|
+
kind: "surface-diff",
|
|
4050
|
+
from: surfaceContentHash(parent),
|
|
4051
|
+
to: surfaceContentHash(child),
|
|
4052
|
+
text: renderSurfaceDiff(child, parent)
|
|
4053
|
+
});
|
|
4054
|
+
}
|
|
4055
|
+
/** Declared surfaces of one node. A component surface declares one surface per
|
|
4056
|
+
* named component, so per-component evidence stays addressable. */
|
|
4057
|
+
function declaredSurfaces(surface, artifact) {
|
|
4058
|
+
if (typeof surface === "string") return [{
|
|
4059
|
+
surfaceId: "prompt",
|
|
4060
|
+
kind: "prompt",
|
|
4061
|
+
artifact
|
|
4062
|
+
}];
|
|
4063
|
+
if (surface.kind === "code") return [{
|
|
4064
|
+
surfaceId: "code",
|
|
4065
|
+
kind: "code",
|
|
4066
|
+
artifact
|
|
4067
|
+
}];
|
|
4068
|
+
return Object.keys(surface.components).sort().map((name) => ({
|
|
4069
|
+
surfaceId: `component:${name}`,
|
|
4070
|
+
kind: "prompt",
|
|
4071
|
+
artifact
|
|
4072
|
+
}));
|
|
4073
|
+
}
|
|
4074
|
+
function boundedLabel(label) {
|
|
4075
|
+
const redacted = redactText(label.trim(), { profile: "share" });
|
|
4076
|
+
return redacted.length <= MAX_LABEL_CHARS ? redacted : `${redacted.slice(0, MAX_LABEL_CHARS - 1)}…`;
|
|
4077
|
+
}
|
|
4078
|
+
/** The header without its stamp, for comparing a resumed opening to the stored one. */
|
|
4079
|
+
function headerMaterial(header) {
|
|
4080
|
+
const { occurredAt: _occurredAt, ...material } = header;
|
|
4081
|
+
return canonicalString(material);
|
|
4082
|
+
}
|
|
4083
|
+
/** The header as the ledger would store it, so the comparison ignores ordering. */
|
|
4084
|
+
function canonicalHeader(header) {
|
|
4085
|
+
return validateSearchLedgerEvent({
|
|
4086
|
+
...header,
|
|
4087
|
+
occurredAt: "2000-01-01T00:00:00.000Z"
|
|
4088
|
+
});
|
|
4089
|
+
}
|
|
4090
|
+
//#endregion
|
|
4091
|
+
//#region src/campaign/search-policy.ts
|
|
4092
|
+
/**
|
|
4093
|
+
* Expansion policies: which node a search expands next, with which operator,
|
|
4094
|
+
* and which node it currently keeps.
|
|
4095
|
+
*
|
|
4096
|
+
* A policy reads a `SearchPolicyView`: nodes whose screen finished and their
|
|
4097
|
+
* per-unit scores on the policy's split. The view holds no test cells, so a
|
|
4098
|
+
* policy cannot see the sealed claim data. Its decisions spend budget and
|
|
4099
|
+
* claim nothing; a claim comes only from the sealed test split.
|
|
4100
|
+
*/
|
|
4101
|
+
/**
|
|
4102
|
+
* The hill climb. The leader starts at the root; a screened node that scored
|
|
4103
|
+
* every unit of its screen takes the lead when its mean on the units it shares
|
|
4104
|
+
* with the leader beats the leader's mean on those units. Nodes are taken in
|
|
4105
|
+
* registration order, so the leader is a pure function of the ledger. The
|
|
4106
|
+
* policy expands the leader, and only once every earlier child is screened,
|
|
4107
|
+
* so each proposal sees every result before it.
|
|
4108
|
+
*/
|
|
4109
|
+
function incumbent(options = {}) {
|
|
4110
|
+
const { patience } = options;
|
|
4111
|
+
if (patience !== void 0 && (!Number.isSafeInteger(patience) || patience < 1)) throw new TypeError(`incumbent: patience must be a positive integer, got ${String(patience)}`);
|
|
4112
|
+
return hillClimb({
|
|
4113
|
+
name: patience === void 0 ? "incumbent" : `incumbent(patience=${patience})`,
|
|
4114
|
+
patience,
|
|
4115
|
+
parent: (_view, leader) => ({
|
|
4116
|
+
nodeId: leader,
|
|
4117
|
+
evidence: {}
|
|
4118
|
+
})
|
|
4119
|
+
});
|
|
4120
|
+
}
|
|
4121
|
+
/**
|
|
4122
|
+
* The hill climb with a different parent: each expansion draws two distinct
|
|
4123
|
+
* members of the Pareto frontier (per-unit means on the policy split) with a
|
|
4124
|
+
* PRNG seeded from `seed` and the expansion index, and keeps the one with the
|
|
4125
|
+
* larger crowding distance, the more isolated. Boundary members carry infinite
|
|
4126
|
+
* distance. Ties fall to the higher mean, then to the earlier node. The leader
|
|
4127
|
+
* rule is the incumbent's: a frontier parent explores, and only a node that
|
|
4128
|
+
* beats the leader leads.
|
|
4129
|
+
*/
|
|
4130
|
+
function crowdedFrontierParent(options) {
|
|
4131
|
+
const { seed } = options;
|
|
4132
|
+
if (!Number.isInteger(seed)) throw new TypeError(`crowdedFrontierParent: seed must be an integer, got ${String(seed)}`);
|
|
4133
|
+
return hillClimb({
|
|
4134
|
+
name: `crowded-frontier(seed=${seed})`,
|
|
4135
|
+
parent: (view, leader) => {
|
|
4136
|
+
const frontier = paretoMembers(view);
|
|
4137
|
+
if (frontier.length <= 1) return {
|
|
4138
|
+
nodeId: frontier[0]?.nodeId ?? leader,
|
|
4139
|
+
evidence: {}
|
|
4140
|
+
};
|
|
4141
|
+
const rng = mulberry32(seed ^ Math.imul(view.expansions + 1, 2654435761) | 0);
|
|
4142
|
+
const first = Math.floor(rng() * frontier.length);
|
|
4143
|
+
const offset = Math.floor(rng() * (frontier.length - 1));
|
|
4144
|
+
const second = offset >= first ? offset + 1 : offset;
|
|
4145
|
+
const a = frontier[first];
|
|
4146
|
+
const b = frontier[second];
|
|
4147
|
+
const winner = compareCrowded(a, b) <= 0 ? a : b;
|
|
4148
|
+
return {
|
|
4149
|
+
nodeId: winner.nodeId,
|
|
4150
|
+
evidence: {
|
|
4151
|
+
frontierSize: frontier.length,
|
|
4152
|
+
crowding: Number.isFinite(winner.distance) ? winner.distance : -1
|
|
4153
|
+
}
|
|
4154
|
+
};
|
|
4155
|
+
}
|
|
4156
|
+
});
|
|
4157
|
+
}
|
|
4158
|
+
function hillClimb(spec) {
|
|
4159
|
+
const leader = (view) => {
|
|
4160
|
+
let current = view.rootNodeId;
|
|
4161
|
+
for (const nodeId of view.screened) {
|
|
4162
|
+
if (nodeId === current || !view.complete(nodeId)) continue;
|
|
4163
|
+
if (beats(view, nodeId, current)) current = nodeId;
|
|
4164
|
+
}
|
|
4165
|
+
return current;
|
|
4166
|
+
};
|
|
4167
|
+
return {
|
|
4168
|
+
name: spec.name,
|
|
4169
|
+
...spec.patience === void 0 ? {} : { patience: spec.patience },
|
|
4170
|
+
leader,
|
|
4171
|
+
expand(view) {
|
|
4172
|
+
if (view.screening > 0) return null;
|
|
4173
|
+
const kept = leader(view);
|
|
4174
|
+
const chosen = spec.parent(view, kept);
|
|
4175
|
+
const units = view.unitScores(chosen.nodeId);
|
|
4176
|
+
return {
|
|
4177
|
+
parents: [chosen.nodeId],
|
|
4178
|
+
operator: "improve",
|
|
4179
|
+
selection: {
|
|
4180
|
+
rule: spec.name,
|
|
4181
|
+
evidence: {
|
|
4182
|
+
...chosen.evidence,
|
|
4183
|
+
units: units.length,
|
|
4184
|
+
...units.length > 0 ? { mean: mean$1(units.map((unit) => unit.mean)) } : {},
|
|
4185
|
+
isLeader: chosen.nodeId === kept ? 1 : 0
|
|
4186
|
+
}
|
|
4187
|
+
}
|
|
4188
|
+
};
|
|
4189
|
+
}
|
|
4190
|
+
};
|
|
4191
|
+
}
|
|
4192
|
+
/** True when `challenger` improves on `holder` over the units both scored. */
|
|
4193
|
+
function beats(view, challenger, holder) {
|
|
4194
|
+
const held = new Map(view.unitScores(holder).map((unit) => [unit.unitId, unit.mean]));
|
|
4195
|
+
let challengerSum = 0;
|
|
4196
|
+
let holderSum = 0;
|
|
4197
|
+
let shared = 0;
|
|
4198
|
+
for (const unit of view.unitScores(challenger)) {
|
|
4199
|
+
const other = held.get(unit.unitId);
|
|
4200
|
+
if (other === void 0) continue;
|
|
4201
|
+
challengerSum += unit.mean;
|
|
4202
|
+
holderSum += other;
|
|
4203
|
+
shared += 1;
|
|
4204
|
+
}
|
|
4205
|
+
if (shared === 0) return false;
|
|
4206
|
+
return view.direction === "maximize" ? challengerSum > holderSum : challengerSum < holderSum;
|
|
4207
|
+
}
|
|
4208
|
+
/** Non-dominated complete screened nodes over the root's units, with crowding. */
|
|
4209
|
+
function paretoMembers(view) {
|
|
4210
|
+
const units = view.unitScores(view.rootNodeId).map((unit) => unit.unitId);
|
|
4211
|
+
const sign = view.direction === "maximize" ? 1 : -1;
|
|
4212
|
+
const members = [];
|
|
4213
|
+
view.screened.forEach((nodeId, ordinal) => {
|
|
4214
|
+
if (!view.complete(nodeId)) return;
|
|
4215
|
+
const objectives = new Map(view.unitScores(nodeId).map((unit) => [unit.unitId, unit.mean]));
|
|
4216
|
+
if (units.some((unit) => !objectives.has(unit))) return;
|
|
4217
|
+
members.push({
|
|
4218
|
+
nodeId,
|
|
4219
|
+
ordinal,
|
|
4220
|
+
mean: sign * mean$1([...objectives.values()]),
|
|
4221
|
+
objectives,
|
|
4222
|
+
distance: 0
|
|
4223
|
+
});
|
|
4224
|
+
});
|
|
4225
|
+
if (members.length === 0 || units.length === 0) return members;
|
|
4226
|
+
return paretoFrontierWithCrowding(members, units.map((unit) => ({
|
|
4227
|
+
name: unit,
|
|
4228
|
+
direction: view.direction,
|
|
4229
|
+
value: (member) => member.objectives.get(unit)
|
|
4230
|
+
}))).map(({ candidate, distance }) => ({
|
|
4231
|
+
...candidate,
|
|
4232
|
+
distance
|
|
4233
|
+
}));
|
|
4234
|
+
}
|
|
4235
|
+
/** Negative when `a` wins the tournament. */
|
|
4236
|
+
function compareCrowded(a, b) {
|
|
4237
|
+
if (a.distance !== b.distance) return a.distance > b.distance ? -1 : 1;
|
|
4238
|
+
if (a.mean !== b.mean) return a.mean > b.mean ? -1 : 1;
|
|
4239
|
+
return a.ordinal - b.ordinal;
|
|
4240
|
+
}
|
|
4241
|
+
function mean$1(values) {
|
|
4242
|
+
let total = 0;
|
|
4243
|
+
for (const value of values) total += value;
|
|
4244
|
+
return total / values.length;
|
|
4245
|
+
}
|
|
4246
|
+
//#endregion
|
|
4247
|
+
//#region src/campaign/types.ts
|
|
4248
|
+
/** Type guard: a proposal carrying its rationale vs a bare
|
|
4249
|
+
* surface. The loop branches on this to populate `GenerationCandidate`. */
|
|
4250
|
+
function isProposedCandidate(value) {
|
|
4251
|
+
return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
|
|
4252
|
+
}
|
|
4253
|
+
const LABEL_TRUST_RANK = {
|
|
4254
|
+
unverified: 0,
|
|
4255
|
+
"verified-signal": 1,
|
|
4256
|
+
"human-rated": 2
|
|
4257
|
+
};
|
|
4258
|
+
/** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
|
|
4259
|
+
function labelTrustRank(trust) {
|
|
4260
|
+
return LABEL_TRUST_RANK[trust ?? "unverified"];
|
|
4261
|
+
}
|
|
4262
|
+
//#endregion
|
|
4263
|
+
//#region src/campaign/presets/run-optimization.ts
|
|
4264
|
+
/**
|
|
4265
|
+
* `runOptimization` improves a mutable surface with a caller-owned proposer,
|
|
4266
|
+
* as a search on the one kernel (`runSearch`).
|
|
4267
|
+
*
|
|
4268
|
+
* The baseline is the seeded root. The scenarios are the train split: the
|
|
4269
|
+
* proposer reads their scores, and the policy ranks on them because the loop
|
|
4270
|
+
* has no private selection split. A "generation" is one proposal of
|
|
4271
|
+
* `populationSize` candidates from the node the policy chose (by default the
|
|
4272
|
+
* incumbent, once every earlier candidate is measured). Every candidate is a
|
|
4273
|
+
* node measured on every scenario at every repeat, one cell at a time on the
|
|
4274
|
+
* run's lanes. The kept surface is the policy's leader: a budget decision on
|
|
4275
|
+
* the train split that claims nothing; `runImprovementLoop` compares it with
|
|
4276
|
+
* the baseline on separate cases.
|
|
4277
|
+
*
|
|
4278
|
+
* The search ledger is always written, beside the run (`<runDir>/search/`) or
|
|
4279
|
+
* where `searchLedger` puts it, and it is the run's only checkpoint: running
|
|
4280
|
+
* again on the same run directory and options continues an interrupted run
|
|
4281
|
+
* instead of starting over, and reruns nothing that settled.
|
|
4282
|
+
*/
|
|
4283
|
+
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
4284
|
+
async function runOptimization(opts) {
|
|
4285
|
+
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4286
|
+
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4287
|
+
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4288
|
+
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4289
|
+
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4290
|
+
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4291
|
+
const runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
4292
|
+
const storage = opts.storage ?? fsCampaignStorage();
|
|
4293
|
+
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
4294
|
+
storage,
|
|
4295
|
+
runDir,
|
|
4296
|
+
costCeilingUsd: opts.costCeiling
|
|
4297
|
+
});
|
|
4298
|
+
const reps = opts.reps ?? 1;
|
|
4299
|
+
const seed = opts.seed ?? 42;
|
|
4300
|
+
const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
|
|
4301
|
+
const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
|
|
4302
|
+
input: opts.premeasuredBaseline,
|
|
4303
|
+
baselineSurface,
|
|
4304
|
+
scenarios: opts.scenarios,
|
|
4305
|
+
reps,
|
|
4306
|
+
seed,
|
|
4307
|
+
judges: opts.judges ?? [],
|
|
4308
|
+
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
|
|
4309
|
+
requireJudgeScore
|
|
4310
|
+
}) : void 0;
|
|
4311
|
+
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
4312
|
+
const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
|
|
4313
|
+
const { identity } = binding;
|
|
4314
|
+
const execution = {
|
|
4315
|
+
model: identity.model,
|
|
4316
|
+
agent: identity.agent,
|
|
4317
|
+
benchmark: {
|
|
4318
|
+
uri: "campaign://scenarios",
|
|
4319
|
+
revision: splitDigest
|
|
4320
|
+
}
|
|
4321
|
+
};
|
|
4322
|
+
const policy = opts.policy ?? incumbent();
|
|
4323
|
+
const allocation = uniform({ reps });
|
|
4324
|
+
const recorder = await SearchRecorder.open({
|
|
4325
|
+
ledger: binding.ledger,
|
|
4326
|
+
storage
|
|
4327
|
+
}, {
|
|
4328
|
+
subject: identity.subject ?? opts.proposer.kind,
|
|
4329
|
+
process: {
|
|
4330
|
+
name: "runOptimization",
|
|
4331
|
+
executionRef: identity.search
|
|
4332
|
+
},
|
|
4333
|
+
artifactKind: artifactKindOf(baselineSurface),
|
|
4334
|
+
objective: {
|
|
4335
|
+
metric: "composite",
|
|
4336
|
+
direction: "maximize",
|
|
4337
|
+
judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
|
|
4338
|
+
claim: identity.claim ?? developmentClaim(splitDigest)
|
|
4339
|
+
},
|
|
4340
|
+
splits: {
|
|
4341
|
+
train: opts.scenarios.map((scenario) => ({
|
|
4342
|
+
taskId: scenario.id,
|
|
4343
|
+
unitId: scenario.id,
|
|
4344
|
+
source: {
|
|
4345
|
+
uri: `scenario://${scenario.id}`,
|
|
4346
|
+
revision: hashCanonical(scenario)
|
|
4347
|
+
}
|
|
4348
|
+
})),
|
|
4349
|
+
selection: [],
|
|
4350
|
+
test: [],
|
|
4351
|
+
heldOutUnits: true
|
|
4352
|
+
},
|
|
4353
|
+
policy: {
|
|
4354
|
+
expansion: policy.name,
|
|
4355
|
+
allocation: allocation.name,
|
|
4356
|
+
seed
|
|
4357
|
+
},
|
|
4358
|
+
budget: {
|
|
4359
|
+
maxUsd: opts.costCeiling ?? null,
|
|
4360
|
+
maxCells: null,
|
|
4361
|
+
maxNodes: 1 + opts.populationSize * opts.maxGenerations,
|
|
4362
|
+
deadline: null,
|
|
4363
|
+
maxConcurrency: null,
|
|
4364
|
+
reservedClaimUsd: 0
|
|
4365
|
+
},
|
|
4366
|
+
containment: null,
|
|
4367
|
+
derivedFrom: null,
|
|
4368
|
+
identity: execution
|
|
4369
|
+
});
|
|
4370
|
+
const nodes = new SurfaceNodes({
|
|
4371
|
+
opts,
|
|
4372
|
+
recorder,
|
|
4373
|
+
runDir,
|
|
4374
|
+
costLedger,
|
|
4375
|
+
execution,
|
|
4376
|
+
premeasured
|
|
4377
|
+
});
|
|
4378
|
+
const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
4379
|
+
let findings = initialFindings;
|
|
4380
|
+
const executor = {
|
|
4381
|
+
lanes: () => [{
|
|
4382
|
+
name: "campaign",
|
|
4383
|
+
capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
|
|
4384
|
+
costCap: "estimate",
|
|
4385
|
+
cellUsd: 0
|
|
4386
|
+
}],
|
|
4387
|
+
place: () => "campaign",
|
|
4388
|
+
run: (work) => nodes.runCell(work, scenarioById),
|
|
4389
|
+
adopt: (work) => nodes.adoptCell(work)
|
|
4390
|
+
};
|
|
4391
|
+
const proposer = opts.proposer;
|
|
4392
|
+
const proposerSource = identity.proposer.source;
|
|
4393
|
+
const port = {
|
|
4394
|
+
name: proposer.kind,
|
|
4395
|
+
kind: "optimizer",
|
|
4396
|
+
source: proposerSource,
|
|
4397
|
+
execution: identity.proposer,
|
|
4398
|
+
childrenPerProposal: opts.populationSize,
|
|
4399
|
+
async propose(request) {
|
|
4400
|
+
const phases = [
|
|
4401
|
+
"search.proposal",
|
|
4402
|
+
"analysis.baseline",
|
|
4403
|
+
"analysis.generation"
|
|
4404
|
+
];
|
|
4405
|
+
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map(receiptKey));
|
|
4406
|
+
const accounting = () => {
|
|
4407
|
+
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receiptKey(receipt)));
|
|
4408
|
+
return {
|
|
4409
|
+
accounting: receiptAccounting(fresh),
|
|
4410
|
+
execution: opts.searchLedger ? identity.proposer : proposalExecution(fresh, identity.model.provider, proposerSource)
|
|
4411
|
+
};
|
|
4412
|
+
};
|
|
4413
|
+
const earlier = await recorder.state();
|
|
4414
|
+
const root = earlier.rootNodeId;
|
|
4415
|
+
const rootCampaign = nodes.campaign(earlier, root);
|
|
4416
|
+
const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4417
|
+
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4418
|
+
const generation = nodes.generations(earlier).length;
|
|
4419
|
+
if (opts.analyzeGeneration) {
|
|
4420
|
+
const previous = generation - 1;
|
|
4421
|
+
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4422
|
+
const campaign = nodes.campaign(earlier, nodeId);
|
|
4423
|
+
return {
|
|
4424
|
+
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4425
|
+
campaign,
|
|
4426
|
+
composite: campaignMeanCompositeOrNull(campaign)
|
|
4427
|
+
};
|
|
4428
|
+
});
|
|
4429
|
+
const history = nodes.history(earlier, policy);
|
|
4430
|
+
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4431
|
+
const fresh = await opts.analyzeGeneration({
|
|
4432
|
+
generation: previous,
|
|
4433
|
+
runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
|
|
4434
|
+
candidates,
|
|
4435
|
+
history,
|
|
4436
|
+
costLedger,
|
|
4437
|
+
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4438
|
+
});
|
|
4439
|
+
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4440
|
+
findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4441
|
+
}
|
|
4442
|
+
}
|
|
4443
|
+
const state = await recorder.state();
|
|
4444
|
+
const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
|
|
4445
|
+
const parent = request.parents[0];
|
|
4446
|
+
const context = Object.freeze({
|
|
4447
|
+
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4448
|
+
operator: request.operator,
|
|
4449
|
+
history: proposalHistory,
|
|
4450
|
+
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4451
|
+
populationSize: opts.populationSize,
|
|
4452
|
+
generation,
|
|
4453
|
+
signal: request.signal,
|
|
4454
|
+
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4455
|
+
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4456
|
+
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4457
|
+
maxImprovementShots: opts.maxImprovementShots,
|
|
4458
|
+
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4459
|
+
costLedger,
|
|
4460
|
+
costPhase: "search.proposal"
|
|
4461
|
+
});
|
|
4462
|
+
if (proposer.decide?.({ history: proposalHistory }).stop) return {
|
|
4463
|
+
children: [],
|
|
4464
|
+
stop: "the proposer decided to stop",
|
|
4465
|
+
...accounting()
|
|
4466
|
+
};
|
|
4467
|
+
const proposed = await proposer.propose(context);
|
|
4468
|
+
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
4469
|
+
return {
|
|
4470
|
+
children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
|
|
4471
|
+
artifact: candidate.surface,
|
|
4472
|
+
label: candidate.label,
|
|
4473
|
+
rationale: candidate.rationale,
|
|
4474
|
+
...candidate.attribution ? { attribution: candidate.attribution } : {}
|
|
4475
|
+
} : {
|
|
4476
|
+
artifact: candidate,
|
|
4477
|
+
label: "",
|
|
4478
|
+
rationale: ""
|
|
4479
|
+
}),
|
|
4480
|
+
...accounting()
|
|
4481
|
+
};
|
|
4482
|
+
}
|
|
4483
|
+
};
|
|
4484
|
+
const result = await runSearch({
|
|
4485
|
+
recorder,
|
|
4486
|
+
root: baselineSurface,
|
|
4487
|
+
codec: surfaceCodec,
|
|
4488
|
+
policy,
|
|
4489
|
+
allocation,
|
|
4490
|
+
proposer: port,
|
|
4491
|
+
executor,
|
|
4492
|
+
maxExpansions: opts.maxGenerations,
|
|
4493
|
+
signal: opts.signal
|
|
4494
|
+
});
|
|
4495
|
+
const state = result.state;
|
|
4496
|
+
const winnerSurface = nodes.surface(state, result.leader);
|
|
4497
|
+
const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
|
|
4498
|
+
const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
|
|
4499
|
+
const expansions = nodes.generations(state);
|
|
4500
|
+
return {
|
|
4501
|
+
generations: nodes.history(state, policy).map((record) => ({
|
|
4502
|
+
record,
|
|
4503
|
+
surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
|
|
4504
|
+
const surface = nodes.surface(state, nodeId);
|
|
4505
|
+
return {
|
|
4506
|
+
surfaceHash: surfaceHash(surface),
|
|
4507
|
+
surface,
|
|
4508
|
+
campaign: nodes.campaign(state, nodeId)
|
|
4509
|
+
};
|
|
4510
|
+
})
|
|
4511
|
+
})),
|
|
4512
|
+
baselineSurface,
|
|
4513
|
+
winnerSurface,
|
|
4514
|
+
winnerSurfaceHash: surfaceHash(winnerSurface),
|
|
4515
|
+
...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
|
|
4516
|
+
...winnerRationale ? { winnerRationale } : {},
|
|
4517
|
+
baselineCampaign: nodes.campaign(state, state.rootNodeId),
|
|
4518
|
+
cost: costLedger.summary(),
|
|
4519
|
+
searchHistory: recorder.receipt({
|
|
4520
|
+
producerId: proposer.kind,
|
|
4521
|
+
runId: runDir
|
|
4522
|
+
}),
|
|
4523
|
+
paretoFrontier: nodes.frontier(state)
|
|
4470
4524
|
};
|
|
4471
4525
|
}
|
|
4472
|
-
/** A
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
}
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
if (surface.kind === "code") return [{
|
|
4490
|
-
surfaceId: "code",
|
|
4491
|
-
kind: "code",
|
|
4492
|
-
artifact
|
|
4493
|
-
}];
|
|
4494
|
-
return Object.keys(surface.components).sort().map((name) => ({
|
|
4495
|
-
surfaceId: `component:${name}`,
|
|
4496
|
-
kind: "prompt",
|
|
4497
|
-
artifact
|
|
4498
|
-
}));
|
|
4499
|
-
}
|
|
4500
|
-
function boundedLabel(label) {
|
|
4501
|
-
const redacted = redactText(label.trim(), { profile: "share" });
|
|
4502
|
-
return redacted.length <= MAX_LABEL_CHARS ? redacted : `${redacted.slice(0, MAX_LABEL_CHARS - 1)}…`;
|
|
4503
|
-
}
|
|
4504
|
-
/** The header without its stamp, for comparing a resumed opening to the stored one. */
|
|
4505
|
-
function headerMaterial(header) {
|
|
4506
|
-
const { occurredAt: _occurredAt, ...material } = header;
|
|
4507
|
-
return canonicalString(material);
|
|
4508
|
-
}
|
|
4509
|
-
/** The header as the ledger would store it, so the comparison ignores ordering. */
|
|
4510
|
-
function canonicalHeader(header) {
|
|
4511
|
-
return validateSearchLedgerEvent({
|
|
4512
|
-
...header,
|
|
4513
|
-
occurredAt: "2000-01-01T00:00:00.000Z"
|
|
4514
|
-
});
|
|
4515
|
-
}
|
|
4516
|
-
//#endregion
|
|
4517
|
-
//#region src/campaign/presets/run-optimization-recording.ts
|
|
4518
|
-
var OptimizationSearch = class OptimizationSearch {
|
|
4526
|
+
/** A mutable surface as a search artifact. */
|
|
4527
|
+
const surfaceCodec = {
|
|
4528
|
+
node: surfaceNode,
|
|
4529
|
+
diff: surfaceDiff,
|
|
4530
|
+
load: (recorder, node) => {
|
|
4531
|
+
const stored = recorder.readBlob(node.artifact);
|
|
4532
|
+
if (stored.kind !== "mutable-surface" || stored.surface === void 0) throw new Error(`runOptimization: node ${node.nodeId} does not hold a mutable surface`);
|
|
4533
|
+
return stored.surface;
|
|
4534
|
+
}
|
|
4535
|
+
};
|
|
4536
|
+
/**
|
|
4537
|
+
* The run's view of its surface nodes: where each one's cells run, its
|
|
4538
|
+
* campaign, and the proposer's reads over them. Everything here derives from
|
|
4539
|
+
* the ledger and the per-cell campaign cache, so a resumed run reads the same.
|
|
4540
|
+
*/
|
|
4541
|
+
var SurfaceNodes = class {
|
|
4542
|
+
opts;
|
|
4519
4543
|
recorder;
|
|
4520
|
-
|
|
4544
|
+
runDir;
|
|
4521
4545
|
costLedger;
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
this.
|
|
4531
|
-
this.
|
|
4532
|
-
this.
|
|
4533
|
-
this.
|
|
4534
|
-
this.
|
|
4535
|
-
|
|
4536
|
-
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4546
|
+
execution;
|
|
4547
|
+
premeasured;
|
|
4548
|
+
storage;
|
|
4549
|
+
campaigns = /* @__PURE__ */ new Map();
|
|
4550
|
+
proposals = /* @__PURE__ */ new Map();
|
|
4551
|
+
/** Cells this process ran or adopted, by node, keyed `scenarioId:rep`. */
|
|
4552
|
+
cells = /* @__PURE__ */ new Map();
|
|
4553
|
+
constructor(input) {
|
|
4554
|
+
this.opts = input.opts;
|
|
4555
|
+
this.recorder = input.recorder;
|
|
4556
|
+
this.runDir = input.runDir;
|
|
4557
|
+
this.costLedger = input.costLedger;
|
|
4558
|
+
this.execution = input.execution;
|
|
4559
|
+
this.premeasured = input.premeasured;
|
|
4560
|
+
this.storage = input.opts.storage ?? fsCampaignStorage();
|
|
4561
|
+
}
|
|
4562
|
+
/** `<runDir>/baseline` for the root, `<runDir>/candidates/<surfaceHash>`
|
|
4563
|
+
* for every other node: content-addressed, so a cell's directory needs no
|
|
4564
|
+
* ledger read and one surface keeps one cache across the run's searches. */
|
|
4565
|
+
nodeDir(surface, root) {
|
|
4566
|
+
return root ? `${this.runDir}/baseline` : `${this.runDir}/candidates/${surfaceHash(surface)}`;
|
|
4567
|
+
}
|
|
4568
|
+
surface(state, nodeId) {
|
|
4569
|
+
return surfaceCodec.load(this.recorder, state.node(nodeId));
|
|
4570
|
+
}
|
|
4571
|
+
/** Nodes proposal `expansion` registered, in the order it returned them. */
|
|
4572
|
+
expansionNodes(state, expansion) {
|
|
4573
|
+
const blob = this.proposal(state, expansion);
|
|
4574
|
+
if (!blob) return [];
|
|
4575
|
+
const ids = [];
|
|
4576
|
+
for (const child of blob.children) {
|
|
4577
|
+
const nodeId = state.nodeIdForDigest(child.node.artifactDigest);
|
|
4578
|
+
if (nodeId === void 0 || ids.includes(nodeId)) continue;
|
|
4579
|
+
if (this.origin(state, nodeId).expansion === expansion) ids.push(nodeId);
|
|
4580
|
+
}
|
|
4581
|
+
return ids;
|
|
4582
|
+
}
|
|
4583
|
+
/**
|
|
4584
|
+
* The scored attempt of a cell an earlier process ran, from its node's
|
|
4585
|
+
* cache; null otherwise, so the kernel runs the cell. A failure is not
|
|
4586
|
+
* adopted: the stop that interrupted the earlier process may have caused it.
|
|
4587
|
+
*/
|
|
4588
|
+
async adoptCell(work) {
|
|
4589
|
+
if (work.stage === "root" && this.premeasured) return this.premeasuredCell(work);
|
|
4590
|
+
const dir = this.nodeDir(work.artifact, work.stage === "root");
|
|
4591
|
+
const cached = this.storage.read(cellCachePath(dir, `${work.taskId}:${work.rep}`));
|
|
4592
|
+
const cell = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4593
|
+
if (cell?.manifestHash !== this.manifestFor(work.artifact)) return null;
|
|
4594
|
+
this.remember(work.nodeId, cell);
|
|
4595
|
+
return this.cellResult(cell, work);
|
|
4596
|
+
}
|
|
4597
|
+
/** Run one cell as a one-cell campaign in its node's directory; a cached
|
|
4598
|
+
* cell is read back, not dispatched again. */
|
|
4599
|
+
async runCell(work, scenarios) {
|
|
4600
|
+
if (work.stage === "root" && this.premeasured) return this.premeasuredCell(work);
|
|
4601
|
+
if (!scenarios.has(work.taskId)) throw new Error(`runOptimization: cell ${work.cellId} names unknown scenario ${work.taskId}`);
|
|
4602
|
+
const dir = this.nodeDir(work.artifact, work.stage === "root");
|
|
4603
|
+
const cell = (await this.campaignRun(dir, work.artifact, {
|
|
4604
|
+
signal: work.signal,
|
|
4605
|
+
costPhase: work.stage === "root" ? "search.baseline" : "search.candidate",
|
|
4606
|
+
cellFilter: ({ scenario, rep }) => scenario.id === work.taskId && rep === work.rep
|
|
4607
|
+
})).cells[0];
|
|
4608
|
+
if (!cell) throw new Error(`runOptimization: cell ${work.cellId} produced no campaign cell`);
|
|
4609
|
+
this.remember(work.nodeId, cell);
|
|
4610
|
+
return this.cellResult(cell, work);
|
|
4611
|
+
}
|
|
4612
|
+
/** The node's campaign over the cells the ledger settled: the cells this
|
|
4613
|
+
* process ran, else their cached results or failure receipts. */
|
|
4614
|
+
campaign(state, nodeId) {
|
|
4615
|
+
const root = nodeId === state.rootNodeId;
|
|
4616
|
+
if (root && this.premeasured) return this.premeasured;
|
|
4617
|
+
const surface = this.surface(state, nodeId);
|
|
4618
|
+
const dir = this.nodeDir(surface, root);
|
|
4619
|
+
const settled = state.cells({ nodeId }).filter((cell) => cell.attempts > 0).map((cell) => ({
|
|
4620
|
+
taskId: cell.taskId,
|
|
4621
|
+
rep: cell.rep
|
|
4554
4622
|
}));
|
|
4555
|
-
const
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
},
|
|
4564
|
-
artifactKind: "prompt",
|
|
4565
|
-
objective: {
|
|
4566
|
-
metric: "composite",
|
|
4567
|
-
direction: "maximize",
|
|
4568
|
-
judge: binding.identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
|
|
4569
|
-
claim: binding.identity.claim ?? developmentClaim(args.splitDigest)
|
|
4570
|
-
},
|
|
4571
|
-
splits: {
|
|
4572
|
-
train,
|
|
4573
|
-
selection: [],
|
|
4574
|
-
test: [],
|
|
4575
|
-
heldOutUnits: true
|
|
4576
|
-
},
|
|
4577
|
-
policy: {
|
|
4578
|
-
expansion: args.expansion,
|
|
4579
|
-
allocation: "uniform",
|
|
4580
|
-
seed: args.seed
|
|
4581
|
-
},
|
|
4582
|
-
budget: {
|
|
4583
|
-
maxUsd: args.maxUsd,
|
|
4584
|
-
maxCells: null,
|
|
4585
|
-
maxNodes: null,
|
|
4586
|
-
deadline: null,
|
|
4587
|
-
maxConcurrency: null,
|
|
4588
|
-
reservedClaimUsd: 0
|
|
4589
|
-
},
|
|
4590
|
-
containment: null,
|
|
4591
|
-
derivedFrom: null,
|
|
4592
|
-
identity
|
|
4593
|
-
});
|
|
4594
|
-
const search = new OptimizationSearch({
|
|
4595
|
-
recorder,
|
|
4596
|
-
binding,
|
|
4597
|
-
costLedger: args.costLedger,
|
|
4598
|
-
scenarioIds: args.scenarios.map((scenario) => scenario.id),
|
|
4599
|
-
reps: args.reps,
|
|
4600
|
-
identity,
|
|
4601
|
-
proposerName: args.proposerName
|
|
4623
|
+
const key = settled.map(({ taskId, rep }) => `${taskId}:${rep}`).sort().join("\n");
|
|
4624
|
+
const held = this.campaigns.get(nodeId);
|
|
4625
|
+
if (held?.cells === key) return held.campaign;
|
|
4626
|
+
const known = this.cells.get(nodeId);
|
|
4627
|
+
const cells = settled.map(({ taskId, rep }) => {
|
|
4628
|
+
const cell = known?.get(`${taskId}:${rep}`) ?? this.storedCell(dir, surface, taskId, rep);
|
|
4629
|
+
if (!cell) throw new Error(`runOptimization: cell ${taskId}:${rep} of node ${nodeId} settled, but ${dir} holds neither its cached result nor its failure receipt`);
|
|
4630
|
+
return cell;
|
|
4602
4631
|
});
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4632
|
+
cells.sort((a, b) => a.cellId < b.cellId ? -1 : a.cellId > b.cellId ? 1 : 0);
|
|
4633
|
+
const reps = this.opts.reps ?? 1;
|
|
4634
|
+
const seed = this.opts.seed ?? 42;
|
|
4635
|
+
const judges = this.opts.judges ?? [];
|
|
4636
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4637
|
+
const durationMs = cells.reduce((sum, cell) => sum + cell.durationMs, 0);
|
|
4638
|
+
const campaign = {
|
|
4639
|
+
manifestHash: this.manifestFor(surface),
|
|
4640
|
+
splitDigest: campaignSplitDigest(this.opts.scenarios, reps),
|
|
4641
|
+
seed,
|
|
4642
|
+
reps,
|
|
4643
|
+
startedAt: now,
|
|
4644
|
+
endedAt: now,
|
|
4645
|
+
durationMs,
|
|
4646
|
+
cells,
|
|
4647
|
+
aggregates: computeAggregates(cells, judges, seed, this.costLedger.summary({ tags: { runDir: dir } })),
|
|
4648
|
+
runDir: dir,
|
|
4649
|
+
artifactsByPath: {},
|
|
4650
|
+
scenarios: this.opts.scenarios.map(campaignScenarioIdentity)
|
|
4619
4651
|
};
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
await this.recorder.startOperation({
|
|
4624
|
-
operationId: generationOperationId(generation),
|
|
4625
|
-
operationKind: "candidate-generation"
|
|
4652
|
+
this.campaigns.set(nodeId, {
|
|
4653
|
+
cells: key,
|
|
4654
|
+
campaign
|
|
4626
4655
|
});
|
|
4656
|
+
return campaign;
|
|
4627
4657
|
}
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
const
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
|
|
4638
|
-
const
|
|
4639
|
-
|
|
4640
|
-
const
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4658
|
+
outcome(state, nodeId) {
|
|
4659
|
+
const campaign = this.campaign(state, nodeId);
|
|
4660
|
+
const reps = this.opts.reps ?? 1;
|
|
4661
|
+
const coverage = campaignCoverage(campaign.cells, this.opts.scenarios, reps, (this.opts.judges?.length ?? 0) > 0);
|
|
4662
|
+
const expansion = nodeId === state.rootNodeId ? -1 : this.generationOf(state, this.origin(state, nodeId).expansion);
|
|
4663
|
+
return toScoredSurfaceOutcome(surfaceHash(this.surface(state, nodeId)), campaign, coverage, expansion);
|
|
4664
|
+
}
|
|
4665
|
+
/** The Pareto frontier over every measured, complete node. */
|
|
4666
|
+
frontier(state) {
|
|
4667
|
+
const scored = [];
|
|
4668
|
+
for (const node of state.nodes()) {
|
|
4669
|
+
if (node.status === "invalid" || node.cellCount === 0) continue;
|
|
4670
|
+
const cells = state.cells({ nodeId: node.nodeId });
|
|
4671
|
+
if (cells.some((cell) => cell.attempts === 0 && cell.cancelled === null)) continue;
|
|
4672
|
+
if (cells.some((cell) => cell.score === null)) continue;
|
|
4673
|
+
const campaign = this.campaign(state, node.nodeId);
|
|
4674
|
+
const edge = state.edge(node.edgeIds[0]);
|
|
4675
|
+
scored.push(toParetoParent(this.surface(state, node.nodeId), campaign, node.nodeId === state.rootNodeId ? -1 : this.generationOf(state, this.origin(state, node.nodeId).expansion), edge.operator === "seed" ? void 0 : edge.label || void 0));
|
|
4676
|
+
}
|
|
4677
|
+
return computeParetoFrontier(scored);
|
|
4678
|
+
}
|
|
4679
|
+
/** Kernel expansions that registered a candidate, in order: generation g
|
|
4680
|
+
* is `generations(state)[g]`. */
|
|
4681
|
+
generations(state) {
|
|
4682
|
+
const expansions = [];
|
|
4683
|
+
for (let expansion = 0; state.operation(`expand-${expansion}`); expansion++) if (this.expansionNodes(state, expansion).length > 0) expansions.push(expansion);
|
|
4684
|
+
return expansions;
|
|
4685
|
+
}
|
|
4686
|
+
/** One record per generation, with its candidates and the one that took
|
|
4687
|
+
* the lead, if any. */
|
|
4688
|
+
history(state, policy) {
|
|
4689
|
+
const records = [];
|
|
4690
|
+
const measured = (nodeId) => state.node(nodeId).status !== "invalid" && state.cells({ nodeId }).every((cell) => cell.attempts > 0 || cell.cancelled !== null);
|
|
4691
|
+
const screened = [state.rootNodeId];
|
|
4692
|
+
let leader = policy.leader(searchPolicyView(state, {
|
|
4693
|
+
screened,
|
|
4694
|
+
screening: 0,
|
|
4695
|
+
expansions: 0
|
|
4696
|
+
}));
|
|
4697
|
+
for (const [generation, expansion] of this.generations(state).entries()) {
|
|
4698
|
+
const nodeIds = this.expansionNodes(state, expansion);
|
|
4699
|
+
screened.push(...nodeIds.filter(measured));
|
|
4700
|
+
screened.sort((a, b) => state.node(a).ordinal - state.node(b).ordinal);
|
|
4701
|
+
const next = policy.leader(searchPolicyView(state, {
|
|
4702
|
+
screened,
|
|
4703
|
+
screening: 0,
|
|
4704
|
+
expansions: expansion + 1
|
|
4705
|
+
}));
|
|
4706
|
+
const candidates = [];
|
|
4707
|
+
for (const nodeId of nodeIds) candidates.push(this.candidate(state, nodeId, expansion));
|
|
4708
|
+
records.push({
|
|
4709
|
+
generationIndex: generation,
|
|
4710
|
+
candidates,
|
|
4711
|
+
promoted: next !== leader && nodeIds.includes(next) ? [surfaceHash(this.surface(state, next))] : []
|
|
4660
4712
|
});
|
|
4661
|
-
|
|
4713
|
+
leader = next;
|
|
4662
4714
|
}
|
|
4663
|
-
return
|
|
4715
|
+
return records;
|
|
4664
4716
|
}
|
|
4665
|
-
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
|
|
4675
|
-
|
|
4676
|
-
|
|
4717
|
+
candidate(state, nodeId, expansion) {
|
|
4718
|
+
const campaign = this.campaign(state, nodeId);
|
|
4719
|
+
const coverage = campaignCoverage(campaign.cells, this.opts.scenarios, this.opts.reps ?? 1, (this.opts.judges?.length ?? 0) > 0);
|
|
4720
|
+
const breakdown = campaignBreakdown(campaign);
|
|
4721
|
+
const child = this.proposal(state, expansion).children.find((entry) => state.nodeIdForDigest(entry.node.artifactDigest) === nodeId);
|
|
4722
|
+
return {
|
|
4723
|
+
surfaceHash: surfaceHash(this.surface(state, nodeId)),
|
|
4724
|
+
composite: campaignMeanCompositeOrNull(campaign),
|
|
4725
|
+
eligibleForPromotion: coverage.complete,
|
|
4726
|
+
coverage: {
|
|
4727
|
+
expectedCells: coverage.expectedCellIds.length,
|
|
4728
|
+
scorableCells: coverage.scorableCellIds.length,
|
|
4729
|
+
unscorableCells: coverage.unscorableCells
|
|
4677
4730
|
},
|
|
4678
|
-
|
|
4679
|
-
|
|
4731
|
+
dimensions: breakdown.dimensions,
|
|
4732
|
+
scenarios: breakdown.scenarios,
|
|
4733
|
+
...child.label ? { label: child.label } : {},
|
|
4734
|
+
...child.rationale ? { rationale: child.rationale } : {},
|
|
4735
|
+
...child.attribution ? { attribution: child.attribution } : {}
|
|
4736
|
+
};
|
|
4680
4737
|
}
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
for (const taskId of this.scenarioIds) for (let rep = 0; rep < this.reps; rep++) await this.recorder.allocateCell({
|
|
4684
|
-
nodeId,
|
|
4685
|
-
taskId,
|
|
4686
|
-
split: "train",
|
|
4687
|
-
rep,
|
|
4688
|
-
stage: "train"
|
|
4689
|
-
});
|
|
4738
|
+
generationOf(state, expansion) {
|
|
4739
|
+
return this.generations(state).indexOf(expansion);
|
|
4690
4740
|
}
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
});
|
|
4701
|
-
await this.recorder.settleCell({
|
|
4702
|
-
cellId,
|
|
4703
|
-
attempt: 1,
|
|
4704
|
-
outcome: cellOutcome(cell),
|
|
4705
|
-
accounting: cellAccounting(cell),
|
|
4706
|
-
identity: {
|
|
4707
|
-
...this.identity,
|
|
4708
|
-
model: this.cellModel(cell)
|
|
4709
|
-
},
|
|
4710
|
-
wallMs: cell.durationMs,
|
|
4711
|
-
traceRef: { unknown: "runCampaign reports no trace id per cell" }
|
|
4712
|
-
});
|
|
4713
|
-
}
|
|
4741
|
+
origin(state, nodeId) {
|
|
4742
|
+
const node = state.node(nodeId);
|
|
4743
|
+
const operationId = state.edge(node.edgeIds[0])?.proposer?.operationId ?? null;
|
|
4744
|
+
const expansion = operationId === null ? null : searchExpansionIndex(operationId);
|
|
4745
|
+
if (expansion === null) throw new Error(`runOptimization: node ${nodeId} was not proposed by this run's search`);
|
|
4746
|
+
return {
|
|
4747
|
+
expansion,
|
|
4748
|
+
index: this.proposal(state, expansion).children.findIndex((child) => child.node.artifactDigest === node.artifactDigest)
|
|
4749
|
+
};
|
|
4714
4750
|
}
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
const
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4751
|
+
proposal(state, expansion) {
|
|
4752
|
+
const held = this.proposals.get(expansion);
|
|
4753
|
+
if (held) return held;
|
|
4754
|
+
const ref = state.operation(`expand-${expansion}`)?.artifacts.find((artifact) => artifact.role === "proposal");
|
|
4755
|
+
if (!ref) return void 0;
|
|
4756
|
+
const blob = this.recorder.readBlob(ref);
|
|
4757
|
+
this.proposals.set(expansion, blob);
|
|
4758
|
+
return blob;
|
|
4759
|
+
}
|
|
4760
|
+
async campaignRun(runDir, surface, input) {
|
|
4761
|
+
const { baselineSurface: _baselineSurface, premeasuredBaseline: _premeasuredBaseline, dispatchWithSurface, proposer: _proposer, populationSize: _populationSize, maxGenerations: _maxGenerations, candidateConcurrency: _candidateConcurrency, maxImprovementShots: _maxImprovementShots, findings: _findings, analyzeGeneration: _analyzeGeneration, policy: _policy, searchLedger: _searchLedger, ...campaignOptions } = this.opts;
|
|
4762
|
+
return runCampaign({
|
|
4763
|
+
...campaignOptions,
|
|
4764
|
+
signal: input.signal ?? this.opts.signal,
|
|
4765
|
+
costLedger: this.costLedger,
|
|
4766
|
+
costPhase: input.costPhase,
|
|
4767
|
+
dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
|
|
4768
|
+
dispatch: (scenario, ctx) => dispatchWithSurface(surface, scenario, ctx),
|
|
4769
|
+
runDir,
|
|
4770
|
+
cellFilter: input.cellFilter,
|
|
4771
|
+
resumable: true,
|
|
4772
|
+
maxConcurrency: 1
|
|
4730
4773
|
});
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
});
|
|
4735
|
-
return this.
|
|
4736
|
-
|
|
4737
|
-
|
|
4774
|
+
}
|
|
4775
|
+
premeasuredCell(work) {
|
|
4776
|
+
const cell = this.premeasured.cells.find((candidate) => candidate.scenarioId === work.taskId && candidate.rep === work.rep);
|
|
4777
|
+
if (!cell) throw new Error(`runOptimization: premeasured baseline lacks ${work.taskId}:${work.rep}`);
|
|
4778
|
+
return this.cellResult(cell, work);
|
|
4779
|
+
}
|
|
4780
|
+
/** A cell's result as its campaign stored it: the cache of a scored cell,
|
|
4781
|
+
* the final failure receipt of a failed one. */
|
|
4782
|
+
/** A cell's result as its campaign stored it for this surface: the cache of
|
|
4783
|
+
* a scored cell, the final failure receipt of a failed one. A result another
|
|
4784
|
+
* surface left in the same directory does not count. */
|
|
4785
|
+
storedCell(dir, surface, taskId, rep) {
|
|
4786
|
+
const cellId = `${taskId}:${rep}`;
|
|
4787
|
+
const manifestHash = this.manifestFor(surface);
|
|
4788
|
+
const cached = this.storage.read(cellCachePath(dir, cellId));
|
|
4789
|
+
const scored = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4790
|
+
if (scored?.manifestHash === manifestHash) return scored;
|
|
4791
|
+
const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
|
|
4792
|
+
const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
|
|
4793
|
+
return failed?.manifestHash === manifestHash ? failed : void 0;
|
|
4794
|
+
}
|
|
4795
|
+
manifestFor(surface) {
|
|
4796
|
+
return computeManifestHash({
|
|
4797
|
+
scenarios: this.opts.scenarios,
|
|
4798
|
+
judges: this.opts.judges ?? [],
|
|
4799
|
+
dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
|
|
4800
|
+
seed: this.opts.seed ?? 42,
|
|
4801
|
+
reps: this.opts.reps ?? 1
|
|
4738
4802
|
});
|
|
4739
4803
|
}
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4804
|
+
remember(nodeId, cell) {
|
|
4805
|
+
const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
|
|
4806
|
+
cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
|
|
4807
|
+
this.cells.set(nodeId, cells);
|
|
4808
|
+
}
|
|
4809
|
+
cellResult(cell, work) {
|
|
4810
|
+
return {
|
|
4811
|
+
outcome: cellOutcome(cell),
|
|
4812
|
+
accounting: cellAccounting(cell),
|
|
4813
|
+
identity: {
|
|
4814
|
+
...this.execution,
|
|
4815
|
+
model: cellModel(cell, this.execution.model)
|
|
4816
|
+
},
|
|
4817
|
+
wallMs: cell.durationMs,
|
|
4818
|
+
placement: {
|
|
4819
|
+
lane: work.lane,
|
|
4820
|
+
boxId: null
|
|
4821
|
+
},
|
|
4822
|
+
traceRef: { unknown: "runCampaign reports no trace id per cell" }
|
|
4758
4823
|
};
|
|
4759
4824
|
}
|
|
4760
4825
|
};
|
|
4761
|
-
|
|
4762
|
-
|
|
4826
|
+
/** The default ledger and identities when the caller binds none. Revisions are
|
|
4827
|
+
* content digests of what the caller declared, and each uri names what its
|
|
4828
|
+
* digest covers: the dispatch ref, the proposer kind, the kernel. */
|
|
4829
|
+
async function defaultBinding(opts, runDir, storage) {
|
|
4830
|
+
const base = `optimization-${createHash("sha256").update(runDir).digest("hex").slice(0, 16)}`;
|
|
4831
|
+
const dispatchRef = opts.dispatchRef ?? "anonymous";
|
|
4832
|
+
const identity = {
|
|
4833
|
+
agent: {
|
|
4834
|
+
uri: `dispatch-ref:${dispatchRef}`,
|
|
4835
|
+
revision: hashCanonical({ dispatchRef })
|
|
4836
|
+
},
|
|
4837
|
+
proposer: {
|
|
4838
|
+
kind: "deterministic",
|
|
4839
|
+
source: {
|
|
4840
|
+
uri: `proposer:${opts.proposer.kind}`,
|
|
4841
|
+
revision: hashCanonical({ proposer: opts.proposer.kind })
|
|
4842
|
+
}
|
|
4843
|
+
},
|
|
4844
|
+
search: SEARCH_KERNEL_SOURCE,
|
|
4845
|
+
model: {
|
|
4846
|
+
provider: "unspecified",
|
|
4847
|
+
alias: "unspecified",
|
|
4848
|
+
unknown: "runOptimization was not told which model the agent runs"
|
|
4849
|
+
}
|
|
4850
|
+
};
|
|
4851
|
+
for (let slot = 1;; slot++) {
|
|
4852
|
+
const path = `${slot === 1 ? `${runDir}/search` : `${runDir}/search-${slot}`}/ledger.jsonl`;
|
|
4853
|
+
const searchId = slot === 1 ? base : `${base}-${slot}`;
|
|
4854
|
+
const ledger = storage.kind === "filesystem" ? openSearchLedger({
|
|
4855
|
+
path,
|
|
4856
|
+
searchId
|
|
4857
|
+
}) : openSearchLedger({
|
|
4858
|
+
path,
|
|
4859
|
+
searchId,
|
|
4860
|
+
store: storage
|
|
4861
|
+
});
|
|
4862
|
+
if (!(await ledger.state()).closed) return {
|
|
4863
|
+
ledger,
|
|
4864
|
+
identity
|
|
4865
|
+
};
|
|
4866
|
+
}
|
|
4867
|
+
}
|
|
4868
|
+
function artifactKindOf(surface) {
|
|
4869
|
+
return typeof surface === "object" && surface.kind === "code" ? "code" : "prompt";
|
|
4870
|
+
}
|
|
4871
|
+
function receiptKey(receipt) {
|
|
4872
|
+
return receipt.callId;
|
|
4873
|
+
}
|
|
4874
|
+
/** A proposal that made a paid call ran a model; one that made none ran code. */
|
|
4875
|
+
function proposalExecution(receipts, provider, source) {
|
|
4876
|
+
const model = receipts.find((receipt) => receipt.channel !== "judge")?.model;
|
|
4877
|
+
if (model === void 0) return {
|
|
4878
|
+
kind: "deterministic",
|
|
4879
|
+
source
|
|
4880
|
+
};
|
|
4881
|
+
return {
|
|
4882
|
+
kind: "model",
|
|
4883
|
+
model: modelIdentity(model, provider),
|
|
4884
|
+
source
|
|
4885
|
+
};
|
|
4886
|
+
}
|
|
4887
|
+
function modelIdentity(model, provider) {
|
|
4888
|
+
return modelHasSnapshot(model) ? {
|
|
4889
|
+
provider,
|
|
4890
|
+
snapshot: model
|
|
4891
|
+
} : {
|
|
4892
|
+
provider,
|
|
4893
|
+
alias: model,
|
|
4894
|
+
unknown: "the provider reported a moving alias, not a snapshot"
|
|
4895
|
+
};
|
|
4896
|
+
}
|
|
4897
|
+
function cellModel(cell, fallback) {
|
|
4898
|
+
return cell.resolvedModel === void 0 ? fallback : modelIdentity(cell.resolvedModel, fallback.provider);
|
|
4763
4899
|
}
|
|
4764
4900
|
function cellOutcome(cell) {
|
|
4765
|
-
const
|
|
4766
|
-
if (
|
|
4901
|
+
const quality = projectCampaignCellQuality(cell);
|
|
4902
|
+
if (quality.score === void 0) return {
|
|
4767
4903
|
status: "errored",
|
|
4768
4904
|
metrics: {},
|
|
4769
4905
|
error: {
|
|
@@ -4772,12 +4908,11 @@ function cellOutcome(cell) {
|
|
|
4772
4908
|
retryable: false
|
|
4773
4909
|
}
|
|
4774
4910
|
};
|
|
4775
|
-
const
|
|
4776
|
-
const metrics =
|
|
4777
|
-
for (const [judge, score] of scores) metrics[`judge.${judge}`] = score.composite;
|
|
4911
|
+
const metrics = { composite: quality.score };
|
|
4912
|
+
for (const [judge, score] of Object.entries(quality.successfulJudgeScores)) metrics[`judge.${judge}`] = score.composite;
|
|
4778
4913
|
return {
|
|
4779
4914
|
status: "passed",
|
|
4780
|
-
score:
|
|
4915
|
+
score: quality.score,
|
|
4781
4916
|
metrics
|
|
4782
4917
|
};
|
|
4783
4918
|
}
|
|
@@ -4840,332 +4975,6 @@ function receiptAccounting(receipts) {
|
|
|
4840
4975
|
}
|
|
4841
4976
|
};
|
|
4842
4977
|
}
|
|
4843
|
-
//#endregion
|
|
4844
|
-
//#region src/campaign/presets/run-optimization.ts
|
|
4845
|
-
/**
|
|
4846
|
-
* `runOptimization` runs a caller-owned candidate generator for a bounded
|
|
4847
|
-
* number of rounds. Each candidate is measured on the same cases, and only a
|
|
4848
|
-
* candidate that beats the current best becomes the incumbent. By default the
|
|
4849
|
-
* incumbent is also the parent every generation mutates; a `selectParent`
|
|
4850
|
-
* policy draws the parent from the Pareto frontier instead.
|
|
4851
|
-
* The same loop accepts deterministic, model-backed, or agent-backed
|
|
4852
|
-
* proposers; they differ only in how `propose()` picks candidates.
|
|
4853
|
-
*
|
|
4854
|
-
* `runImprovementLoop` adds a separate final comparison, a release decision,
|
|
4855
|
-
* and optional pull request creation.
|
|
4856
|
-
*/
|
|
4857
|
-
/**
|
|
4858
|
-
* Improvement loop body: N generations of propose → campaign → rank, maintaining a Pareto frontier and one global incumbent across generations. The parent each generation mutates is the incumbent unless `selectParent` draws it from the frontier.
|
|
4859
|
-
*/
|
|
4860
|
-
async function runOptimization(opts) {
|
|
4861
|
-
const { proposer } = opts;
|
|
4862
|
-
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4863
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4864
|
-
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4865
|
-
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4866
|
-
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4867
|
-
opts.runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
4868
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
4869
|
-
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
4870
|
-
storage,
|
|
4871
|
-
runDir: opts.runDir,
|
|
4872
|
-
costCeilingUsd: opts.costCeiling
|
|
4873
|
-
});
|
|
4874
|
-
const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
|
|
4875
|
-
const reps = opts.reps ?? 1;
|
|
4876
|
-
const premeasuredBaseline = opts.premeasuredBaseline;
|
|
4877
|
-
const baselineCampaign = premeasuredBaseline ? validatedPremeasuredBaseline({
|
|
4878
|
-
input: premeasuredBaseline,
|
|
4879
|
-
baselineSurface,
|
|
4880
|
-
scenarios: opts.scenarios,
|
|
4881
|
-
reps,
|
|
4882
|
-
seed: opts.seed ?? 42,
|
|
4883
|
-
judges: opts.judges ?? [],
|
|
4884
|
-
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef)
|
|
4885
|
-
}) : await runCampaign({
|
|
4886
|
-
...opts,
|
|
4887
|
-
costLedger,
|
|
4888
|
-
costPhase: "search.baseline",
|
|
4889
|
-
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
|
|
4890
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(baselineSurface, scenario, ctx),
|
|
4891
|
-
runDir: `${opts.runDir}/baseline`
|
|
4892
|
-
});
|
|
4893
|
-
const baselineCoverage = campaignCoverage(baselineCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4894
|
-
if (!baselineCoverage.complete) {
|
|
4895
|
-
const label = opts.premeasuredBaseline ? "premeasured baseline" : "baseline";
|
|
4896
|
-
throw new Error(`runOptimization: ${label} is incomplete (${baselineCoverage.scorableCellIds.length}/${baselineCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(baselineCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4897
|
-
}
|
|
4898
|
-
const opened = opts.searchLedger ? await OptimizationSearch.open({
|
|
4899
|
-
binding: opts.searchLedger,
|
|
4900
|
-
storage,
|
|
4901
|
-
costLedger,
|
|
4902
|
-
scenarios: opts.scenarios,
|
|
4903
|
-
reps,
|
|
4904
|
-
seed: opts.seed ?? 42,
|
|
4905
|
-
splitDigest: baselineCampaign.splitDigest,
|
|
4906
|
-
proposerName: proposer.kind,
|
|
4907
|
-
expansion: opts.selectParent ? "select-parent" : "incumbent",
|
|
4908
|
-
maxUsd: opts.costCeiling ?? null,
|
|
4909
|
-
baselineSurface,
|
|
4910
|
-
baselineCells: baselineCampaign.cells
|
|
4911
|
-
}) : void 0;
|
|
4912
|
-
const search = opened?.search;
|
|
4913
|
-
const nodeIdByHash = /* @__PURE__ */ new Map();
|
|
4914
|
-
if (opened) nodeIdByHash.set(surfaceHash(baselineSurface), opened.rootNodeId);
|
|
4915
|
-
const incompleteNodeIds = /* @__PURE__ */ new Set();
|
|
4916
|
-
const promotedNodeIds = /* @__PURE__ */ new Set();
|
|
4917
|
-
let stopReason = "max-nodes";
|
|
4918
|
-
const generations = [];
|
|
4919
|
-
const history = [];
|
|
4920
|
-
let currentFindings = initialFindings;
|
|
4921
|
-
const selectionRankKey = opts.selectionRankKey ?? ((campaign) => [campaignMeanComposite(campaign)]);
|
|
4922
|
-
let winnerSurface = baselineSurface;
|
|
4923
|
-
let winnerSurfaceHash = surfaceHash(baselineSurface);
|
|
4924
|
-
const admittedCandidateHashes = /* @__PURE__ */ new Set([winnerSurfaceHash]);
|
|
4925
|
-
let winnerComposite = campaignMeanComposite(baselineCampaign);
|
|
4926
|
-
let winnerRankKey = selectionRankKey(baselineCampaign);
|
|
4927
|
-
assertFiniteRankKey(winnerRankKey, "selectionRankKey for baseline");
|
|
4928
|
-
const baselineOutcome = toScoredSurfaceOutcome(winnerSurfaceHash, baselineCampaign, baselineCoverage, -1);
|
|
4929
|
-
let winnerOutcome = baselineOutcome;
|
|
4930
|
-
let winnerLabel;
|
|
4931
|
-
let winnerRationale;
|
|
4932
|
-
const scored = [toParetoParent(baselineSurface, winnerSurfaceHash, baselineCampaign, -1)];
|
|
4933
|
-
const measuredByHash = /* @__PURE__ */ new Map([[winnerSurfaceHash, {
|
|
4934
|
-
parent: scored[0],
|
|
4935
|
-
outcome: baselineOutcome
|
|
4936
|
-
}]]);
|
|
4937
|
-
if (opts.analyzeGeneration && opts.maxGenerations > 0 && baselineCampaign.cells.length > 0) {
|
|
4938
|
-
const fresh = await opts.analyzeGeneration({
|
|
4939
|
-
generation: -1,
|
|
4940
|
-
runDir: baselineCampaign.runDir,
|
|
4941
|
-
candidates: [{
|
|
4942
|
-
surfaceHash: winnerSurfaceHash,
|
|
4943
|
-
campaign: baselineCampaign,
|
|
4944
|
-
composite: winnerComposite
|
|
4945
|
-
}],
|
|
4946
|
-
history,
|
|
4947
|
-
costLedger,
|
|
4948
|
-
costPhase: "analysis.baseline"
|
|
4949
|
-
});
|
|
4950
|
-
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4951
|
-
currentFindings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization baseline analysis findings"), "baseline analysis findings");
|
|
4952
|
-
}
|
|
4953
|
-
for (let gen = 0; gen < opts.maxGenerations; gen++) {
|
|
4954
|
-
const proposalHistory = immutableProposalSnapshot(history, "history");
|
|
4955
|
-
if (proposer.decide?.({ history: proposalHistory }).stop) {
|
|
4956
|
-
stopReason = "converged";
|
|
4957
|
-
break;
|
|
4958
|
-
}
|
|
4959
|
-
const paretoParents = immutableProposalSnapshot(computeParetoFrontier(scored), "Pareto parents");
|
|
4960
|
-
const incumbentOutcome = immutableProposalSnapshot(winnerOutcome, "incumbent outcome");
|
|
4961
|
-
const selected = opts.selectParent ? resolveSelectedParent(opts.selectParent(Object.freeze({
|
|
4962
|
-
frontier: paretoParents,
|
|
4963
|
-
incumbent: incumbentOutcome,
|
|
4964
|
-
history: proposalHistory,
|
|
4965
|
-
generation: gen
|
|
4966
|
-
})), measuredByHash, gen) : void 0;
|
|
4967
|
-
const parentSurface = selected ? selected.parent.surface : winnerSurface;
|
|
4968
|
-
const parentSurfaceHash = selected ? selected.parent.surfaceHash : winnerSurfaceHash;
|
|
4969
|
-
const parentComposite = selected ? selected.outcome.composite : winnerComposite;
|
|
4970
|
-
const parentOutcome = selected ? selected.outcome : winnerOutcome;
|
|
4971
|
-
const proposalContext = Object.freeze({
|
|
4972
|
-
currentSurface: immutableProposalSnapshot(parentSurface, "current surface"),
|
|
4973
|
-
history: proposalHistory,
|
|
4974
|
-
findings: immutableProposalSnapshot(assertProposalFindings(currentFindings, "runOptimization proposal findings"), "findings"),
|
|
4975
|
-
populationSize: opts.populationSize,
|
|
4976
|
-
generation: gen,
|
|
4977
|
-
signal: opts.signal ?? new AbortController().signal,
|
|
4978
|
-
baselineOutcome: immutableProposalSnapshot(baselineOutcome, "baseline outcome"),
|
|
4979
|
-
incumbentOutcome,
|
|
4980
|
-
parentOutcome: immutableProposalSnapshot(parentOutcome, "parent outcome"),
|
|
4981
|
-
maxImprovementShots: opts.maxImprovementShots,
|
|
4982
|
-
paretoParents,
|
|
4983
|
-
costLedger,
|
|
4984
|
-
costPhase: "search.proposal"
|
|
4985
|
-
});
|
|
4986
|
-
await search?.startGeneration(gen);
|
|
4987
|
-
const proposed = await proposer.propose(proposalContext);
|
|
4988
|
-
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
4989
|
-
const proposalSnapshot = immutableProposalSnapshot(proposed, "candidate outputs");
|
|
4990
|
-
if (proposalSnapshot.length === 0) {
|
|
4991
|
-
await search?.failGeneration(gen, "the proposer returned no candidates");
|
|
4992
|
-
stopReason = "converged";
|
|
4993
|
-
break;
|
|
4994
|
-
}
|
|
4995
|
-
const candidates = proposalSnapshot.map((p) => isProposedCandidate(p) ? p : {
|
|
4996
|
-
surface: p,
|
|
4997
|
-
label: "",
|
|
4998
|
-
rationale: ""
|
|
4999
|
-
});
|
|
5000
|
-
const generationHashes = /* @__PURE__ */ new Set();
|
|
5001
|
-
for (const { surface } of candidates) {
|
|
5002
|
-
const hash = surfaceHash(surface);
|
|
5003
|
-
if (admittedCandidateHashes.has(hash) || generationHashes.has(hash)) throw new Error(`runOptimization: duplicate candidate surface hash "${hash}" in generation ${gen}; candidate surfaces must be unique`);
|
|
5004
|
-
generationHashes.add(hash);
|
|
5005
|
-
}
|
|
5006
|
-
for (const hash of generationHashes) admittedCandidateHashes.add(hash);
|
|
5007
|
-
if (search) {
|
|
5008
|
-
const nodeIds = await search.recordProposal({
|
|
5009
|
-
generation: gen,
|
|
5010
|
-
parent: {
|
|
5011
|
-
nodeId: nodeIdByHash.get(parentSurfaceHash),
|
|
5012
|
-
surface: parentSurface,
|
|
5013
|
-
composite: parentComposite
|
|
5014
|
-
},
|
|
5015
|
-
selectionRule: opts.selectParent ? "select-parent" : "incumbent",
|
|
5016
|
-
candidates
|
|
5017
|
-
});
|
|
5018
|
-
for (const [index, nodeId] of nodeIds.entries()) {
|
|
5019
|
-
nodeIdByHash.set(surfaceHash(candidates[index].surface), nodeId);
|
|
5020
|
-
await search.allocate(nodeId);
|
|
5021
|
-
}
|
|
5022
|
-
}
|
|
5023
|
-
const surfaceResults = await mapConcurrent(candidates, candidateConcurrency, async ({ surface, label, rationale, attribution }, i, signal) => {
|
|
5024
|
-
const hash = surfaceHash(surface);
|
|
5025
|
-
const campaign = await runCampaign({
|
|
5026
|
-
...opts,
|
|
5027
|
-
signal,
|
|
5028
|
-
costLedger,
|
|
5029
|
-
costPhase: "search.candidate",
|
|
5030
|
-
dispatchRef: surfaceDispatchRef(surface, opts.dispatchRef),
|
|
5031
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
5032
|
-
runDir: `${opts.runDir}/gen-${gen}/candidate-${i}`
|
|
5033
|
-
});
|
|
5034
|
-
const coverage = campaignCoverage(campaign.cells, opts.scenarios, opts.reps ?? 1, requireJudgeScore);
|
|
5035
|
-
const composite = campaignMeanCompositeOrNull(campaign);
|
|
5036
|
-
const rankKey = coverage.complete ? selectionRankKey(campaign) : null;
|
|
5037
|
-
if (rankKey) assertFiniteRankKey(rankKey, `selectionRankKey for generation ${gen} candidate ${i}`, winnerRankKey.length);
|
|
5038
|
-
return {
|
|
5039
|
-
surfaceHash: hash,
|
|
5040
|
-
surface,
|
|
5041
|
-
label,
|
|
5042
|
-
rationale,
|
|
5043
|
-
...attribution ? { attribution } : {},
|
|
5044
|
-
campaign,
|
|
5045
|
-
composite,
|
|
5046
|
-
rankKey,
|
|
5047
|
-
coverage
|
|
5048
|
-
};
|
|
5049
|
-
}, opts.signal);
|
|
5050
|
-
for (const result of surfaceResults) {
|
|
5051
|
-
const { surface, surfaceHash: hash, campaign, coverage, label, rationale } = result;
|
|
5052
|
-
if (coverage.complete) {
|
|
5053
|
-
const parent = toParetoParent(surface, hash, campaign, gen, label || void 0, rationale || void 0);
|
|
5054
|
-
scored.push(parent);
|
|
5055
|
-
measuredByHash.set(hash, {
|
|
5056
|
-
parent,
|
|
5057
|
-
outcome: toScoredSurfaceOutcome(hash, campaign, coverage, gen)
|
|
5058
|
-
});
|
|
5059
|
-
}
|
|
5060
|
-
}
|
|
5061
|
-
if (search) for (const result of surfaceResults) {
|
|
5062
|
-
const nodeId = nodeIdByHash.get(result.surfaceHash);
|
|
5063
|
-
await search.recordCells(nodeId, "train", result.campaign.cells);
|
|
5064
|
-
if (!result.coverage.complete) incompleteNodeIds.add(nodeId);
|
|
5065
|
-
}
|
|
5066
|
-
surfaceResults.sort((a, b) => {
|
|
5067
|
-
if (a.coverage.complete !== b.coverage.complete) return a.coverage.complete ? -1 : 1;
|
|
5068
|
-
if (a.rankKey && b.rankKey) return compareRankKeys(b.rankKey, a.rankKey);
|
|
5069
|
-
return a.surfaceHash.localeCompare(b.surfaceHash);
|
|
5070
|
-
});
|
|
5071
|
-
const top = surfaceResults.filter((result) => result.coverage.complete && result.composite !== null && result.rankKey !== null)[0];
|
|
5072
|
-
const promoted = top && compareRankKeys(top.rankKey, winnerRankKey) > 0 ? [top] : [];
|
|
5073
|
-
if (promoted[0]) {
|
|
5074
|
-
const top = promoted[0];
|
|
5075
|
-
const promotedNodeId = nodeIdByHash.get(top.surfaceHash);
|
|
5076
|
-
if (promotedNodeId) promotedNodeIds.add(promotedNodeId);
|
|
5077
|
-
winnerSurface = top.surface;
|
|
5078
|
-
winnerSurfaceHash = top.surfaceHash;
|
|
5079
|
-
winnerComposite = top.composite;
|
|
5080
|
-
winnerRankKey = top.rankKey;
|
|
5081
|
-
winnerOutcome = toScoredSurfaceOutcome(top.surfaceHash, top.campaign, top.coverage, gen);
|
|
5082
|
-
winnerLabel = top.label || void 0;
|
|
5083
|
-
winnerRationale = top.rationale || void 0;
|
|
5084
|
-
}
|
|
5085
|
-
const record = {
|
|
5086
|
-
generationIndex: gen,
|
|
5087
|
-
candidates: surfaceResults.map((s) => {
|
|
5088
|
-
const breakdown = campaignBreakdown(s.campaign);
|
|
5089
|
-
const candidate = {
|
|
5090
|
-
surfaceHash: s.surfaceHash,
|
|
5091
|
-
composite: s.composite,
|
|
5092
|
-
ci95: null,
|
|
5093
|
-
parentSurfaceHash,
|
|
5094
|
-
parentComposite,
|
|
5095
|
-
...s.coverage.complete ? { observedDeltaFromParent: s.composite - parentComposite } : {},
|
|
5096
|
-
eligibleForPromotion: s.coverage.complete,
|
|
5097
|
-
coverage: {
|
|
5098
|
-
expectedCells: s.coverage.expectedCellIds.length,
|
|
5099
|
-
scorableCells: s.coverage.scorableCellIds.length,
|
|
5100
|
-
unscorableCells: s.coverage.unscorableCells
|
|
5101
|
-
},
|
|
5102
|
-
dimensions: breakdown.dimensions,
|
|
5103
|
-
scenarios: breakdown.scenarios
|
|
5104
|
-
};
|
|
5105
|
-
if (s.label) candidate.label = s.label;
|
|
5106
|
-
if (s.rationale) candidate.rationale = s.rationale;
|
|
5107
|
-
if (s.attribution) candidate.attribution = s.attribution;
|
|
5108
|
-
return candidate;
|
|
5109
|
-
}),
|
|
5110
|
-
promoted: promoted.map((p) => p.surfaceHash)
|
|
5111
|
-
};
|
|
5112
|
-
history.push(record);
|
|
5113
|
-
generations.push({
|
|
5114
|
-
record,
|
|
5115
|
-
surfaces: surfaceResults.map((s) => ({
|
|
5116
|
-
surfaceHash: s.surfaceHash,
|
|
5117
|
-
surface: s.surface,
|
|
5118
|
-
campaign: s.campaign
|
|
5119
|
-
}))
|
|
5120
|
-
});
|
|
5121
|
-
if (opts.analyzeGeneration && gen < opts.maxGenerations - 1) {
|
|
5122
|
-
const fresh = await opts.analyzeGeneration({
|
|
5123
|
-
generation: gen,
|
|
5124
|
-
runDir: `${opts.runDir}/gen-${gen}`,
|
|
5125
|
-
candidates: surfaceResults.map((s) => ({
|
|
5126
|
-
surfaceHash: s.surfaceHash,
|
|
5127
|
-
campaign: s.campaign,
|
|
5128
|
-
composite: s.composite
|
|
5129
|
-
})),
|
|
5130
|
-
history,
|
|
5131
|
-
costLedger,
|
|
5132
|
-
costPhase: "analysis.generation"
|
|
5133
|
-
});
|
|
5134
|
-
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
5135
|
-
currentFindings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization generation analysis findings"), "generation analysis findings");
|
|
5136
|
-
}
|
|
5137
|
-
}
|
|
5138
|
-
const searchHistory = await search?.finish({
|
|
5139
|
-
winnerNodeId: nodeIdByHash.get(winnerSurfaceHash),
|
|
5140
|
-
promotedNodeIds,
|
|
5141
|
-
incompleteNodeIds,
|
|
5142
|
-
reason: stopReason,
|
|
5143
|
-
runId: opts.runDir
|
|
5144
|
-
});
|
|
5145
|
-
return {
|
|
5146
|
-
generations,
|
|
5147
|
-
baselineSurface,
|
|
5148
|
-
winnerSurface,
|
|
5149
|
-
winnerSurfaceHash,
|
|
5150
|
-
winnerLabel,
|
|
5151
|
-
winnerRationale,
|
|
5152
|
-
baselineCampaign,
|
|
5153
|
-
...searchHistory ? { searchHistory } : {},
|
|
5154
|
-
paretoFrontier: computeParetoFrontier(scored),
|
|
5155
|
-
cost: costLedger.summary()
|
|
5156
|
-
};
|
|
5157
|
-
}
|
|
5158
|
-
/** Resolve the parent a `selectParent` policy returned to the measured record
|
|
5159
|
-
* this run holds for it. Refuses a surface the run never measured to
|
|
5160
|
-
* completion, and a parent whose surface does not hash to its `surfaceHash`. */
|
|
5161
|
-
function resolveSelectedParent(returned, measuredByHash, generation) {
|
|
5162
|
-
if (typeof returned !== "object" || returned === null || typeof returned.surfaceHash !== "string") throw new TypeError(`runOptimization: selectParent must return a ParetoParent (generation ${generation})`);
|
|
5163
|
-
const parent = returned;
|
|
5164
|
-
const measured = measuredByHash.get(parent.surfaceHash);
|
|
5165
|
-
if (!measured) throw new Error(`runOptimization: selectParent returned surface "${parent.surfaceHash}" in generation ${generation}, which this run has not measured to completion; a parent must be a scored surface from the frontier`);
|
|
5166
|
-
if (!surfaceHashMatches(parent.surface, parent.surfaceHash)) throw new Error(`runOptimization: selectParent returned a parent whose surface does not match its surfaceHash "${parent.surfaceHash}" (generation ${generation})`);
|
|
5167
|
-
return measured;
|
|
5168
|
-
}
|
|
5169
4978
|
function immutableProposalSnapshot(value, label) {
|
|
5170
4979
|
try {
|
|
5171
4980
|
return deepFreeze(structuredClone(value));
|
|
@@ -5199,58 +5008,42 @@ function validatedPremeasuredBaseline(args) {
|
|
|
5199
5008
|
reps: args.reps
|
|
5200
5009
|
});
|
|
5201
5010
|
if (campaign.manifestHash !== expectedManifest) throw new Error("runOptimization: premeasured baseline evaluator identity does not match the requested dispatch and judges");
|
|
5011
|
+
const coverage = campaignCoverage(campaign.cells, args.scenarios, args.reps, args.requireJudgeScore);
|
|
5012
|
+
if (!coverage.complete) throw new Error(`runOptimization: premeasured baseline is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
5202
5013
|
return campaign;
|
|
5203
5014
|
}
|
|
5204
|
-
/**
|
|
5205
|
-
|
|
5206
|
-
function toParetoParent(surface, hash, campaign, generation, label, rationale) {
|
|
5015
|
+
/** A scored campaign as a frontier member: per-scenario composite objectives. */
|
|
5016
|
+
function toParetoParent(surface, campaign, generation, label) {
|
|
5207
5017
|
const objectives = {};
|
|
5208
5018
|
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
|
|
5209
5019
|
const parent = {
|
|
5210
5020
|
surface,
|
|
5211
|
-
surfaceHash:
|
|
5021
|
+
surfaceHash: surfaceHash(surface),
|
|
5212
5022
|
objectives,
|
|
5213
5023
|
composite: campaignMeanComposite(campaign),
|
|
5214
5024
|
generation
|
|
5215
5025
|
};
|
|
5216
5026
|
if (label) parent.label = label;
|
|
5217
|
-
if (rationale) parent.rationale = rationale;
|
|
5218
5027
|
return parent;
|
|
5219
5028
|
}
|
|
5220
|
-
/** The non-dominated set over the per-scenario objective vectors.
|
|
5221
|
-
* scenario seen across the scored set becomes a `maximize` objective.
|
|
5222
|
-
* `runOptimization` admits only complete campaigns to this set; the finite
|
|
5223
|
-
* floor remains a defensive fallback for manually constructed/no-judge
|
|
5224
|
-
* vectors. Delegates dominance to the package-canonical `paretoFrontier`. */
|
|
5029
|
+
/** The non-dominated set over the per-scenario objective vectors. */
|
|
5225
5030
|
function computeParetoFrontier(scored) {
|
|
5226
5031
|
if (scored.length <= 1) return [...scored];
|
|
5227
5032
|
const ids = /* @__PURE__ */ new Set();
|
|
5228
5033
|
for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
|
|
5229
5034
|
if (ids.size === 0) return [...scored];
|
|
5230
|
-
const floor = {};
|
|
5231
|
-
for (const id of ids) {
|
|
5232
|
-
let min = Number.POSITIVE_INFINITY;
|
|
5233
|
-
for (const p of scored) {
|
|
5234
|
-
const v = p.objectives[id];
|
|
5235
|
-
if (typeof v === "number" && Number.isFinite(v) && v < min) min = v;
|
|
5236
|
-
}
|
|
5237
|
-
floor[id] = Number.isFinite(min) ? min : 0;
|
|
5238
|
-
}
|
|
5239
5035
|
return paretoFrontier(scored, [...ids].map((id) => ({
|
|
5240
5036
|
name: id,
|
|
5241
5037
|
direction: "maximize",
|
|
5242
|
-
value: (p) =>
|
|
5243
|
-
const v = p.objectives[id];
|
|
5244
|
-
return typeof v === "number" && Number.isFinite(v) ? v : floor[id] ?? 0;
|
|
5245
|
-
}
|
|
5038
|
+
value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
|
|
5246
5039
|
}))).frontier;
|
|
5247
5040
|
}
|
|
5248
|
-
function toScoredSurfaceOutcome(
|
|
5041
|
+
function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
|
|
5249
5042
|
const breakdown = campaignBreakdown(campaign);
|
|
5250
5043
|
return {
|
|
5251
5044
|
split: "search",
|
|
5252
5045
|
generation,
|
|
5253
|
-
surfaceHash,
|
|
5046
|
+
surfaceHash: hash,
|
|
5254
5047
|
composite: campaignMeanComposite(campaign),
|
|
5255
5048
|
dimensions: breakdown.dimensions,
|
|
5256
5049
|
scenarios: breakdown.scenarios,
|
|
@@ -6670,6 +6463,6 @@ function firstString(value) {
|
|
|
6670
6463
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
6671
6464
|
}
|
|
6672
6465
|
//#endregion
|
|
6673
|
-
export { scoreRedTeamOutput as $,
|
|
6466
|
+
export { scoreRedTeamOutput as $, createSearchHistoryReceipt as A, estimateNodeFromCells as B, searchEdgeId as C, assertCompleteSearchHistory as D, SearchHistoryRequiredError as E, runSearch as F, paretoFrontier as G, searchPosterior as H, searchExpansionIndex as I, captureJudge as J, runFinalComparison as K, searchPolicyView as L, verifySearchHistoryArtifact as M, verifySearchHistoryReceipt as N, assertSearchHistoryAdmissionOptions as O, SEARCH_KERNEL_SOURCE as P, redTeamReport as Q, SEARCH_ESTIMATOR as R, developmentClaim as S, surfaceNode as T, uniform as U, searchCellSetDigest as V, dominates as W, DEFAULT_RED_TEAM_CORPUS as X, defaultProductionGate as Y, redTeamDataset as Z, isProposedCandidate as _, transientDispatchFailure as a, runEval as at, incumbent as b, compareOptimizationMethods as c, resolveRunDir as ct, combineComparisonCosts as d, buildCellSchedule as dt, runCanaries as et, costFromLedgerSummary as f, cellCachePath as ft, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, reserveFinalEvidence as it, searchHistoryCoverageRow as j, assertSearchHistoryMatchesState as k, optimizationTokenUsageFromSummary as l, tangleTracesRoot as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, captureFinalEvidencePolicy as nt, aggregateRunScore as o, runCampaign as ot, assertGepaCandidatePopulationSummary as p, computeManifestHash as pt, openAutoPr as q, isTransientTransportFailure as r, evaluationUnitMap as rt, clamp01 as s, planCampaignRun as st, llmJudge as t, assertIndependentEvaluationSplit as tt, executeOptimizationMethod as u, readCachedCell as ut, labelTrustRank as v, surfaceDiff as w, SearchRecorder as x, crowdedFrontierParent as y, estimateNode as z };
|
|
6674
6467
|
|
|
6675
|
-
//# sourceMappingURL=llm-judge-
|
|
6468
|
+
//# sourceMappingURL=llm-judge-Z6MkTkd0.js.map
|