@tangle-network/agent-eval 0.191.0 → 0.193.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +6 -6
- package/dist/analyst/index.js +3 -3
- package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
- package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
- package/dist/{benchmark-command-3tZJnD82.js → benchmark-command-DVhUPU7H.js} +7 -7
- package/dist/{benchmark-command-3tZJnD82.js.map → benchmark-command-DVhUPU7H.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +12 -12
- package/dist/{campaign-C8iWEF0M.js → campaign-DdE6S80e.js} +15 -328
- package/dist/campaign-DdE6S80e.js.map +1 -0
- package/dist/{campaign-evidence-BqjlTM3P.js → campaign-evidence-DJc60F9C.js} +11 -46
- package/dist/campaign-evidence-DJc60F9C.js.map +1 -0
- package/dist/cli.js +47 -10
- package/dist/cli.js.map +1 -1
- package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
- package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +10 -10
- package/dist/{default-registry-DK_9aVJf.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
- package/dist/{default-registry-DK_9aVJf.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
- package/dist/{define-agent-eval-P28y7mI1.d.ts → define-agent-eval-DQJvVa1P.d.ts} +15 -24
- package/dist/{define-agent-eval-P28y7mI1.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BE43YTm4.js → define-agent-eval-DkPbnJEY.js} +9 -10
- package/dist/define-agent-eval-DkPbnJEY.js.map +1 -0
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-MfpwoEaj.js → dspy-rlm-engine-BX34vPVY.js} +2 -2
- package/dist/{dspy-rlm-engine-MfpwoEaj.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
- package/dist/{engine-vNiM02Ub.d.ts → engine-DjRV55F9.d.ts} +2 -2
- package/dist/{engine-vNiM02Ub.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +4 -4
- package/dist/{external-optimizer-process-xFTliuiK.js → external-optimizer-process-D-Sab4v3.js} +3 -62
- package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
- package/dist/{external-optimizer-subprocess-B234V9V0.js → external-optimizer-subprocess-kbqC-iWf.js} +64 -3
- package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +1 -1
- package/dist/{index-D-TN7ZRM.d.ts → index-0jEtpSM7.d.ts} +3 -3
- package/dist/{index-D-TN7ZRM.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
- package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
- package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
- package/dist/{index-CzfnASj_.d.ts → index-DMPLG226.d.ts} +400 -374
- package/dist/index-DMPLG226.d.ts.map +1 -0
- package/dist/{index-DH-MWH5f.d.ts → index-DWQNDqPd.d.ts} +7 -7
- package/dist/{index-DH-MWH5f.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +9 -292
- package/dist/index.js.map +1 -1
- package/dist/jev.d.ts +1 -1
- package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
- package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-NJOHM0iA.js → llm-judge-ChbWfeWl.js} +2653 -656
- package/dist/llm-judge-ChbWfeWl.js.map +1 -0
- package/dist/{matrix-CgqdFm5N.d.ts → matrix-CNu13gg8.d.ts} +2 -2
- package/dist/{matrix-CgqdFm5N.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/paired-delta-test-BF-FEP8V.js +78 -0
- package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
- package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
- package/dist/{produced-state-CY-L7CqS.js → produced-state-BqeIaIp2.js} +3 -3
- package/dist/{produced-state-CY-L7CqS.js.map → produced-state-BqeIaIp2.js.map} +1 -1
- package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
- package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +1 -1
- package/dist/{search-ledger-CQ8Uy4FK.js → search-ledger-CJiAEU3j.js} +164 -36
- package/dist/search-ledger-CJiAEU3j.js.map +1 -0
- package/dist/{search-ledger-file-EvdDZRhv.js → search-ledger-file-DY0AXX3W.js} +2 -2
- package/dist/{search-ledger-file-EvdDZRhv.js.map → search-ledger-file-DY0AXX3W.js.map} +1 -1
- package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
- package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
- package/dist/{search-shipper-C55V7HE3.js → search-shipper-DUlwlCsK.js} +3 -3
- package/dist/{search-shipper-C55V7HE3.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
- package/dist/search-summary-BhY8p0Av.js +378 -0
- package/dist/search-summary-BhY8p0Av.js.map +1 -0
- package/dist/{semantic-concept-judge-Bt7bXAAo.js → semantic-concept-judge-Bg-560H0.js} +3 -3
- package/dist/{semantic-concept-judge-Bt7bXAAo.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
- package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
- package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DwO3TOAV.js → skillopt-optimization-method-DXQdaBjx.js} +7 -7
- package/dist/{skillopt-optimization-method-DwO3TOAV.js.map → skillopt-optimization-method-DXQdaBjx.js.map} +1 -1
- package/dist/{statistical-heldout-BED3wBtX.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
- package/dist/{statistical-heldout-BED3wBtX.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
- package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
- package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
- package/dist/{tool-groups-D-mkeN0N.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
- package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-BmQRs661.d.ts → types-BWwRXofp.d.ts} +340 -17
- package/dist/types-BWwRXofp.d.ts.map +1 -0
- package/dist/{types-ClK-sHs8.d.ts → types-qoxUfKQg.d.ts} +2 -2
- package/dist/{types-ClK-sHs8.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
- package/docs/campaign-proposers.md +14 -16
- package/docs/search-ledger.md +129 -3
- package/package.json +1 -1
- package/dist/campaign-C8iWEF0M.js.map +0 -1
- package/dist/campaign-evidence-BqjlTM3P.js.map +0 -1
- package/dist/claim-BGOUqoMq.d.ts +0 -38
- package/dist/claim-BGOUqoMq.d.ts.map +0 -1
- package/dist/define-agent-eval-BE43YTm4.js.map +0 -1
- package/dist/external-optimizer-process-xFTliuiK.js.map +0 -1
- package/dist/external-optimizer-subprocess-B234V9V0.js.map +0 -1
- package/dist/index-CzfnASj_.d.ts.map +0 -1
- package/dist/llm-judge-NJOHM0iA.js.map +0 -1
- package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
- package/dist/search-ledger-CQ8Uy4FK.js.map +0 -1
- package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
- package/dist/tool-groups-D-mkeN0N.d.ts.map +0 -1
- package/dist/types-BmQRs661.d.ts.map +0 -1
|
@@ -1,8 +1,317 @@
|
|
|
1
1
|
import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { d as RunTokenUsage } from "./run-record-DF-fuu31.js";
|
|
2
|
+
import { d as RunTokenUsage, s as RunSearchCoordinates } from "./run-record-DF-fuu31.js";
|
|
3
3
|
import { _ as ProposalFinding } from "./types-CeDfCRXp.js";
|
|
4
4
|
import { M as LlmCallMetadata } from "./types-C0bWoSKC.js";
|
|
5
5
|
import { t as SeriesDistribution } from "./descriptive-B2iPaT9J.js";
|
|
6
|
+
import { A as SearchLedgerHash, G as SearchSplit, I as SearchNodeStatus, L as SearchOpenedEvent, N as SearchNodeDecision, O as SearchLedgerEntry, P as SearchNodeRef, Q as SearchTaskOutcome, R as SearchOperationKind, U as SearchReservation, Z as SearchTask, b as SearchClosedEvent, d as SearchCellCancelledEvent, i as SearchArtifactRef, l as SearchCandidateSurface, o as SearchAudit, p as SearchCellStage, t as NodeEstimate, w as SearchEdgeRecordedEvent, z as SearchOperationRecordedEvent } from "./search-ledger-types-B3El5xgS.js";
|
|
7
|
+
import { p as LedgerProjector } from "./index-Bj96W0jZ.js";
|
|
8
|
+
import { i as RolloutSearchLineage } from "./mint-C9Lojt1V.js";
|
|
9
|
+
//#region src/campaign/search-state.d.ts
|
|
10
|
+
/** Deterministic node id: identical content in one search is one node. */
|
|
11
|
+
declare function searchNodeId(searchId: string, artifactDigest: string): string;
|
|
12
|
+
/** Deterministic cell id: one node on one task in one split at one repeat. */
|
|
13
|
+
declare function searchCellId(searchId: string, nodeId: string, taskId: string, split: SearchSplit, rep: number): string;
|
|
14
|
+
/** Digest a split's task list in taskId order. */
|
|
15
|
+
declare function searchTaskSetDigest(tasks: readonly SearchTask[]): SearchLedgerHash;
|
|
16
|
+
interface SearchDecisionRecord {
|
|
17
|
+
sequence: number;
|
|
18
|
+
decision: SearchNodeDecision;
|
|
19
|
+
basis: NodeEstimate | null;
|
|
20
|
+
rule: string;
|
|
21
|
+
reason: string;
|
|
22
|
+
}
|
|
23
|
+
interface SearchSpend {
|
|
24
|
+
knownUsd: number;
|
|
25
|
+
floorUsd: number;
|
|
26
|
+
unknownCostCells: number;
|
|
27
|
+
boxMinutes: number;
|
|
28
|
+
}
|
|
29
|
+
/** Scores of one node on one unit of one split: the sum and count of its
|
|
30
|
+
* scored cells. Repeats and sibling tasks of the unit average inside it. */
|
|
31
|
+
interface SearchUnitScore {
|
|
32
|
+
unitId: string;
|
|
33
|
+
sum: number;
|
|
34
|
+
count: number;
|
|
35
|
+
mean: number;
|
|
36
|
+
}
|
|
37
|
+
/** A cell's scored attempt: the inputs of every per-unit statistic. */
|
|
38
|
+
interface SearchScoredCell {
|
|
39
|
+
cellId: string;
|
|
40
|
+
unitId: string;
|
|
41
|
+
/** The attempt that produced the score, which is the cell's last. */
|
|
42
|
+
attempt: number;
|
|
43
|
+
score: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Per-unit means of scored cells, in unitId order. Each unit sums its cells in
|
|
47
|
+
* cellId order, so the result depends on the set of cells alone, never on the
|
|
48
|
+
* order they settled in: a verifier that reads the same cells from a database
|
|
49
|
+
* gets the same bits.
|
|
50
|
+
*/
|
|
51
|
+
declare function searchUnitScores(cells: readonly SearchScoredCell[]): SearchUnitScore[];
|
|
52
|
+
interface SearchNode {
|
|
53
|
+
nodeId: string;
|
|
54
|
+
/** Registration order; the root is 0. */
|
|
55
|
+
ordinal: number;
|
|
56
|
+
artifactDigest: SearchLedgerHash;
|
|
57
|
+
artifact: SearchArtifactRef;
|
|
58
|
+
surfaces: readonly SearchCandidateSurface[];
|
|
59
|
+
/** First in-search parent of the node's first edge; null for the root, a
|
|
60
|
+
* derived root, or a node whose parents are unknown. */
|
|
61
|
+
primaryParentId: string | null;
|
|
62
|
+
/** Edges from the root along primary parents; null while unknown. */
|
|
63
|
+
depth: number | null;
|
|
64
|
+
/** Every parent named by any edge into the node, in edge order. */
|
|
65
|
+
parents: readonly SearchNodeRef[];
|
|
66
|
+
children: readonly string[];
|
|
67
|
+
edgeIds: readonly string[];
|
|
68
|
+
status: SearchNodeStatus | null;
|
|
69
|
+
rung: number | null;
|
|
70
|
+
/** The node was decided `finalist` at some point, so its test cells are legal. */
|
|
71
|
+
finalist: boolean;
|
|
72
|
+
decisions: readonly SearchDecisionRecord[];
|
|
73
|
+
cellCount: number;
|
|
74
|
+
spend: SearchSpend;
|
|
75
|
+
registeredSequence: number;
|
|
76
|
+
updatedSequence: number;
|
|
77
|
+
}
|
|
78
|
+
interface SearchCell {
|
|
79
|
+
cellId: string;
|
|
80
|
+
nodeId: string;
|
|
81
|
+
taskId: string;
|
|
82
|
+
unitId: string;
|
|
83
|
+
split: SearchSplit;
|
|
84
|
+
rep: number;
|
|
85
|
+
stage: SearchCellStage;
|
|
86
|
+
lane: string | null;
|
|
87
|
+
reservation: SearchReservation | null;
|
|
88
|
+
attempts: number;
|
|
89
|
+
lastRunId: string | null;
|
|
90
|
+
outcome: SearchTaskOutcome['status'] | null;
|
|
91
|
+
/** Score of the scored attempt; null when no attempt was scored. */
|
|
92
|
+
score: number | null;
|
|
93
|
+
/** No further attempt is legal: a scored or non-retryable outcome. */
|
|
94
|
+
final: boolean;
|
|
95
|
+
cancelled: SearchCellCancelledEvent['reason'] | null;
|
|
96
|
+
/** Known cost plus proven floors across attempts. */
|
|
97
|
+
spentUsd: number;
|
|
98
|
+
/** Every attempt's cost is known. */
|
|
99
|
+
costKnown: boolean;
|
|
100
|
+
traceId: string | null;
|
|
101
|
+
allocatedSequence: number;
|
|
102
|
+
updatedSequence: number;
|
|
103
|
+
}
|
|
104
|
+
interface SearchOperation {
|
|
105
|
+
operationId: string;
|
|
106
|
+
operationKind: SearchOperationKind;
|
|
107
|
+
reservation: SearchReservation | null;
|
|
108
|
+
recorded: boolean;
|
|
109
|
+
/** The recorded outcome; null until the operation is recorded. */
|
|
110
|
+
outcome: SearchOperationRecordedEvent['outcome']['status'] | null;
|
|
111
|
+
/** Artifacts the started event bound, then those the recorded event bound,
|
|
112
|
+
* for example a claim's plan or a proposal's output. */
|
|
113
|
+
artifacts: readonly SearchArtifactRef[];
|
|
114
|
+
spentUsd: number;
|
|
115
|
+
}
|
|
116
|
+
interface SearchCompletion {
|
|
117
|
+
complete: boolean;
|
|
118
|
+
reasons: string[];
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The pure, incremental projection of one search ledger. `apply` refuses any
|
|
122
|
+
* entry that breaks an invariant; the journal then discards this instance and
|
|
123
|
+
* replays into a new one, so a refused entry leaves no partial state behind.
|
|
124
|
+
*/
|
|
125
|
+
declare class SearchState implements LedgerProjector<SearchLedgerEntry, SearchStateView> {
|
|
126
|
+
readonly searchId: string;
|
|
127
|
+
private version;
|
|
128
|
+
private header;
|
|
129
|
+
private closed;
|
|
130
|
+
private head;
|
|
131
|
+
private lastOccurredAt;
|
|
132
|
+
private lastOccurredAtText;
|
|
133
|
+
private readonly tasks;
|
|
134
|
+
private readonly nodes;
|
|
135
|
+
private readonly nodeOrder;
|
|
136
|
+
private readonly nodeByDigest;
|
|
137
|
+
private readonly edges;
|
|
138
|
+
private readonly edgeOrder;
|
|
139
|
+
private readonly cells;
|
|
140
|
+
private readonly operations;
|
|
141
|
+
private selectedNodeId;
|
|
142
|
+
private claimUsedUsd;
|
|
143
|
+
/** Nodes ever decided `finalist`. */
|
|
144
|
+
private readonly finalists;
|
|
145
|
+
/** Claim cells allocated. Once one exists, the finalists are fixed. */
|
|
146
|
+
private claimCells;
|
|
147
|
+
private readonly audit;
|
|
148
|
+
constructor(searchId: string);
|
|
149
|
+
apply(entry: SearchLedgerEntry, index: number): void;
|
|
150
|
+
snapshot(): SearchStateView;
|
|
151
|
+
/** @internal Views check this before every read of the live indexes. */
|
|
152
|
+
isAt(version: number): boolean;
|
|
153
|
+
private applyEvent;
|
|
154
|
+
private open;
|
|
155
|
+
private startOperation;
|
|
156
|
+
private recordOperation;
|
|
157
|
+
private registerNode;
|
|
158
|
+
private recordEdge;
|
|
159
|
+
private allocateCell;
|
|
160
|
+
private settleCell;
|
|
161
|
+
private cancelCell;
|
|
162
|
+
private decideNode;
|
|
163
|
+
private close;
|
|
164
|
+
private checkClaim;
|
|
165
|
+
/** Once the first finalist is decided, the search only runs its claim:
|
|
166
|
+
* nothing new is proposed, registered or allocated outside the test. */
|
|
167
|
+
private assertSearching;
|
|
168
|
+
/** @internal */
|
|
169
|
+
incompleteReasons(): string[];
|
|
170
|
+
/** Book an attempt's or operation's accounting into the running totals. An
|
|
171
|
+
* unknown cost counts as its proven floor. */
|
|
172
|
+
private book;
|
|
173
|
+
/** A cell's share of the open reservations, its overspend, and what it draws
|
|
174
|
+
* from the claim reserve: the larger of its hold and its spend. */
|
|
175
|
+
private reservationParts;
|
|
176
|
+
/** The admission rule: committed spend, open holds, the unspent claim
|
|
177
|
+
* reserve and the new hold stay within the cap. A claim cell draws on the
|
|
178
|
+
* claim reserve instead of adding to it. */
|
|
179
|
+
private admit;
|
|
180
|
+
/** @internal The largest hold a new non-claim cell or operation may take,
|
|
181
|
+
* and the claim reserve still unspent. */
|
|
182
|
+
readBudget(): {
|
|
183
|
+
headroomUsd: number | null;
|
|
184
|
+
claimReserveUsd: number;
|
|
185
|
+
};
|
|
186
|
+
/** @internal */
|
|
187
|
+
readHeader(): SearchOpenedEvent | null;
|
|
188
|
+
/** @internal */
|
|
189
|
+
readClosed(): SearchClosedEvent | null;
|
|
190
|
+
/** @internal */
|
|
191
|
+
readHead(): {
|
|
192
|
+
sequence: number;
|
|
193
|
+
entryHash: SearchLedgerHash;
|
|
194
|
+
} | null;
|
|
195
|
+
/** @internal */
|
|
196
|
+
readLastOccurredAt(): string | null;
|
|
197
|
+
/** @internal */
|
|
198
|
+
readAudit(): SearchAudit;
|
|
199
|
+
/** @internal */
|
|
200
|
+
readNode(nodeId: string): SearchNode | undefined;
|
|
201
|
+
/** @internal */
|
|
202
|
+
readNodeIds(): readonly string[];
|
|
203
|
+
/** @internal */
|
|
204
|
+
readHasNode(nodeId: string): boolean;
|
|
205
|
+
/** @internal */
|
|
206
|
+
readNodeIdForDigest(artifactDigest: string): string | undefined;
|
|
207
|
+
/** @internal */
|
|
208
|
+
readEdge(edgeId: string): SearchEdgeRecordedEvent | undefined;
|
|
209
|
+
/** @internal */
|
|
210
|
+
readEdgeIds(): readonly string[];
|
|
211
|
+
/** @internal */
|
|
212
|
+
readCell(cellId: string): SearchCell | undefined;
|
|
213
|
+
/** @internal */
|
|
214
|
+
readCellIds(nodeId?: string): readonly string[];
|
|
215
|
+
/** @internal */
|
|
216
|
+
readOperation(operationId: string): SearchOperation | undefined;
|
|
217
|
+
/** @internal */
|
|
218
|
+
readScoredCells(nodeId: string, split: SearchSplit): SearchScoredCell[];
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* A read of `SearchState` at one ledger position. Header, head, audit and
|
|
222
|
+
* claim are values. Node, edge, cell and score reads go to the live indexes and
|
|
223
|
+
* throw once the state has applied a later entry.
|
|
224
|
+
*/
|
|
225
|
+
declare class SearchStateView {
|
|
226
|
+
readonly searchId: string;
|
|
227
|
+
readonly header: SearchOpenedEvent | null;
|
|
228
|
+
readonly closed: SearchClosedEvent | null;
|
|
229
|
+
readonly head: {
|
|
230
|
+
sequence: number;
|
|
231
|
+
entryHash: SearchLedgerHash;
|
|
232
|
+
} | null;
|
|
233
|
+
/** `occurredAt` of the head entry; a writer stamps later events at or after it. */
|
|
234
|
+
readonly lastOccurredAt: string | null;
|
|
235
|
+
readonly audit: SearchAudit;
|
|
236
|
+
readonly completion: SearchCompletion;
|
|
237
|
+
/**
|
|
238
|
+
* The admission rule's room: `headroomUsd` is the largest reservation a new
|
|
239
|
+
* non-claim cell or operation may hold (committed spend, open holds and the
|
|
240
|
+
* unspent claim reserve all count against the cap), or null when the search
|
|
241
|
+
* declares no cap. `claimReserveUsd` is what claim cells may still draw.
|
|
242
|
+
*/
|
|
243
|
+
readonly budget: {
|
|
244
|
+
headroomUsd: number | null;
|
|
245
|
+
claimReserveUsd: number;
|
|
246
|
+
};
|
|
247
|
+
private readonly state;
|
|
248
|
+
private readonly version;
|
|
249
|
+
constructor(state: SearchState, version: number);
|
|
250
|
+
/** The node the search started from: the first registered. */
|
|
251
|
+
get rootNodeId(): string | null;
|
|
252
|
+
node(nodeId: string): SearchNode | undefined;
|
|
253
|
+
hasNode(nodeId: string): boolean;
|
|
254
|
+
/** Node ids in registration order. */
|
|
255
|
+
nodeIds(): string[];
|
|
256
|
+
/** Nodes in registration order. */
|
|
257
|
+
nodes(): SearchNode[];
|
|
258
|
+
nodeIdForDigest(artifactDigest: string): string | undefined;
|
|
259
|
+
edge(edgeId: string): SearchEdgeRecordedEvent | undefined;
|
|
260
|
+
/** Edges in ledger order. */
|
|
261
|
+
edges(): SearchEdgeRecordedEvent[];
|
|
262
|
+
cell(cellId: string): SearchCell | undefined;
|
|
263
|
+
/** Cells in allocation order, optionally of one node. */
|
|
264
|
+
cells(filter?: {
|
|
265
|
+
nodeId?: string;
|
|
266
|
+
}): SearchCell[];
|
|
267
|
+
operation(operationId: string): SearchOperation | undefined;
|
|
268
|
+
/** A node's scored cells on one split, in allocation order. Unscored cells
|
|
269
|
+
* (errored, cancelled or in flight) are absent, never zero. */
|
|
270
|
+
scoredCells(nodeId: string, split: SearchSplit): SearchScoredCell[];
|
|
271
|
+
/** Per-unit score means of a node on one split, in unitId order. */
|
|
272
|
+
unitScores(nodeId: string, split: SearchSplit): SearchUnitScore[];
|
|
273
|
+
/** The lineage a RunRecord mint needs for one cell attempt. */
|
|
274
|
+
lineage(coordinates: RunSearchCoordinates): RolloutSearchLineage;
|
|
275
|
+
private live;
|
|
276
|
+
}
|
|
277
|
+
//#endregion
|
|
278
|
+
//#region src/campaign/search-summary.d.ts
|
|
279
|
+
/** The proposer's own read of a search: its node's cells on the train split,
|
|
280
|
+
* the only split a proposer may ever see. */
|
|
281
|
+
interface SearchProposerView {
|
|
282
|
+
readonly searchId: string;
|
|
283
|
+
readonly direction: 'maximize' | 'minimize';
|
|
284
|
+
/** `nodeId`'s scored train cells. Unscored cells (errored, cancelled or in
|
|
285
|
+
* flight) are absent, never zero. */
|
|
286
|
+
scoredCells(nodeId: string): readonly SearchScoredCell[];
|
|
287
|
+
/** Per-unit train-split means of `nodeId`, in unitId order. */
|
|
288
|
+
unitScores(nodeId: string): readonly SearchUnitScore[];
|
|
289
|
+
}
|
|
290
|
+
/** The train-only view `ProposeContext.train` carries. */
|
|
291
|
+
declare function searchProposerView(state: SearchStateView): SearchProposerView;
|
|
292
|
+
interface SearchSummaryOptions {
|
|
293
|
+
/**
|
|
294
|
+
* The split every estimate in the summary is computed on. A proposer's
|
|
295
|
+
* context must always pass `'train'`: the selection and test splits stay
|
|
296
|
+
* unread by anything a proposer's text can reach. The CLI passes the
|
|
297
|
+
* search's own ranking split, `'selection'` when the search declares one
|
|
298
|
+
* else `'train'`, matching `searchPolicyView`.
|
|
299
|
+
*/
|
|
300
|
+
split: SearchSplit;
|
|
301
|
+
/** Nodes shown under "Leading" and under "Recently discarded". Default 5. */
|
|
302
|
+
limit?: number;
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* `SearchState` as compact text: a header line with the search's shape and
|
|
306
|
+
* spend, the leading nodes against the root with their estimates, the most
|
|
307
|
+
* recently discarded nodes with the measurement that discarded them, and a
|
|
308
|
+
* log of recent proposals (operator and label — the ledger's inline,
|
|
309
|
+
* redacted summary of each rationale; the full rationale is a blob this
|
|
310
|
+
* function never reads, so it stays a pure, I/O-free projection of the
|
|
311
|
+
* state already in hand).
|
|
312
|
+
*/
|
|
313
|
+
declare function renderSearchSummary(state: SearchStateView, options: SearchSummaryOptions): string;
|
|
314
|
+
//#endregion
|
|
6
315
|
//#region src/campaign/types.d.ts
|
|
7
316
|
/** Stable identifier + kind tag for any scenario. Consumers
|
|
8
317
|
* extend with their per-domain payload (persona, task, requirement, ...). */
|
|
@@ -266,9 +575,30 @@ interface ScoredSurfaceOutcome {
|
|
|
266
575
|
/** Search state supplied to one candidate-generation call.
|
|
267
576
|
* Final evaluation data is not represented in this contract. */
|
|
268
577
|
interface ProposeContext<TFindings = ProposalFinding> {
|
|
269
|
-
/** The parent surface this proposal mutates: the
|
|
270
|
-
*
|
|
578
|
+
/** The parent surface this proposal mutates: the node the search policy
|
|
579
|
+
* chose (the incumbent by default). Equal to `parents[0].artifact` when
|
|
580
|
+
* `parents` is present. */
|
|
271
581
|
readonly currentSurface: MutableSurface;
|
|
582
|
+
/** How the child should derive from the parent: `improve` for a hill-climb
|
|
583
|
+
* step; other policies ask for `draft`, `debug` or `merge`. Absent outside
|
|
584
|
+
* a search. */
|
|
585
|
+
readonly operator?: 'draft' | 'improve' | 'debug' | 'merge';
|
|
586
|
+
/** Every parent the policy chose, primary first, with its artifact. A
|
|
587
|
+
* `merge` proposal needs every parent; other operators name exactly one.
|
|
588
|
+
* Absent outside a search. */
|
|
589
|
+
readonly parents?: ReadonlyArray<{
|
|
590
|
+
nodeId: string;
|
|
591
|
+
artifact: MutableSurface;
|
|
592
|
+
}>;
|
|
593
|
+
/** The proposer's own read of the search: its nodes' cells on the train
|
|
594
|
+
* split only. Absent outside a search. See {@link SearchProposerView}. */
|
|
595
|
+
readonly train?: SearchProposerView;
|
|
596
|
+
/** `renderSearchSummary` of the search so far, computed on the train
|
|
597
|
+
* split: the leading nodes, recently discarded ideas and recent proposals,
|
|
598
|
+
* as compact text. Absent outside a search. */
|
|
599
|
+
readonly summary?: string;
|
|
600
|
+
/** One record per earlier proposal: its candidates' measurements and the
|
|
601
|
+
* candidate that took the lead, if any. */
|
|
272
602
|
readonly history: ReadonlyArray<GenerationRecord>;
|
|
273
603
|
readonly findings: ReadonlyArray<TFindings>;
|
|
274
604
|
/** BREADTH: how many candidate surfaces to return this generation. */
|
|
@@ -284,8 +614,8 @@ interface ProposeContext<TFindings = ProposalFinding> {
|
|
|
284
614
|
* candidate must beat to promote. `runOptimization` always supplies it. */
|
|
285
615
|
readonly incumbentOutcome?: ScoredSurfaceOutcome;
|
|
286
616
|
/** Measured result for `currentSurface`, the parent this proposal mutates.
|
|
287
|
-
* Equal to `incumbentOutcome` unless the
|
|
288
|
-
* frontier
|
|
617
|
+
* Equal to `incumbentOutcome` unless the policy draws parents from the Pareto
|
|
618
|
+
* frontier. `runOptimization` always supplies it. */
|
|
289
619
|
readonly parentOutcome?: ScoredSurfaceOutcome;
|
|
290
620
|
/** DEPTH: max iterations the agentic generator may take per candidate.
|
|
291
621
|
* 1 = single-shot; >1 = it may iterate on its own change before handing it
|
|
@@ -569,17 +899,10 @@ interface GenerationRecord {
|
|
|
569
899
|
* handled — the evidence a blind `Mutator` cannot see. */
|
|
570
900
|
interface GenerationCandidate {
|
|
571
901
|
surfaceHash: string;
|
|
572
|
-
/** Mean over complete task-quality scores, or null when none were produced.
|
|
902
|
+
/** Mean over complete task-quality scores, or null when none were produced.
|
|
903
|
+
* Lineage and paired contrasts live in the search ledger: read the node's
|
|
904
|
+
* edges from its `SearchStateView` and its contrast from `estimateNode`. */
|
|
573
905
|
composite: number | null;
|
|
574
|
-
/** Estimated interval for `composite`, or null when uncertainty was not estimated. */
|
|
575
|
-
ci95: [number, number] | null;
|
|
576
|
-
/** Exact surface this candidate mutated. */
|
|
577
|
-
parentSurfaceHash?: string;
|
|
578
|
-
/** Measured search-split composite of the exact parent surface. */
|
|
579
|
-
parentComposite?: number;
|
|
580
|
-
/** Candidate composite minus its parent's composite. Present only when the
|
|
581
|
-
* candidate completed the designed denominator. */
|
|
582
|
-
observedDeltaFromParent?: number;
|
|
583
906
|
/** Whether this candidate had a scorable result for every designed campaign
|
|
584
907
|
* cell and was therefore eligible for ranking, promotion, and Pareto
|
|
585
908
|
* selection. */
|
|
@@ -669,5 +992,5 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
669
992
|
scenarios: Array<CampaignScenarioIdentity & Pick<TScenario, 'id' | 'kind'>>;
|
|
670
993
|
}
|
|
671
994
|
//#endregion
|
|
672
|
-
export { LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, GateDecision as _, CampaignResult as a, GenerationRecord as b, CampaignTraceWriter as c, DispatchContext as d, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, GateCheckStatus as m, CampaignArtifactWriter as n, CampaignScenarioIdentity as o, Gate as p, CampaignCellResult as r, CampaignTokenUsage as s, CampaignAggregates as t, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
673
|
-
//# sourceMappingURL=types-
|
|
995
|
+
export { SearchNode as $, LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, renderSearchSummary as J, SearchProposerView as K, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, SearchDecisionRecord as Q, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, SearchCell as X, searchProposerView as Y, SearchCompletion as Z, GateDecision as _, CampaignResult as a, SearchUnitScore as at, GenerationRecord as b, CampaignTraceWriter as c, searchTaskSetDigest as ct, DispatchContext as d, SearchOperation as et, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, SearchStateView as it, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, searchUnitScores as lt, GateCheckStatus as m, CampaignArtifactWriter as n, SearchSpend as nt, CampaignScenarioIdentity as o, searchCellId as ot, Gate as p, SearchSummaryOptions as q, CampaignCellResult as r, SearchState as rt, CampaignTokenUsage as s, searchNodeId as st, CampaignAggregates as t, SearchScoredCell as tt, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
996
|
+
//# sourceMappingURL=types-BWwRXofp.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-BWwRXofp.d.ts","names":[],"sources":["../src/campaign/search-state.ts","../src/campaign/search-summary.ts","../src/campaign/types.ts"],"mappings":";;;;;;;;;;iBAwDgB,aAAa,kBAAkB;;iBAK/B,aACd,kBACA,gBACA,gBACA,OAAO,aACP;;iBAMc,oBAAoB,gBAAgB,eAAe;UAkBlD;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;;EAEA;EACA;;;;;;;;iBASc,iBAAiB,gBAAgB,qBAAqB;UAoBrD;EACf;;EAEA;EACA,gBAAgB;EAChB,UAAU;EACV,mBAAmB;;;EAGnB;;EAEA;;EAEA,kBAAkB;EAClB;EACA;EACA,QAAQ;EACR;;EAEA;EACA,oBAAoB;EACpB;EACA,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,aAAa;EACb;EACA;EACA,SAAS;;EAET;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,aAAa;EACb;;EAEA,SAAS;;;EAGT,oBAAoB;EACpB;;UAGe;EACf;EACA;;;;;;;cA6BW,uBAAuB,gBAAgB,mBAAmB;WAC5D;UACD;UACA;UACA;UACA;UACA;UACA;mBACS;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;UACT;UACA;;mBAES;;UAET;mBACS;EAEjB,YAAY;EAgCZ,MAAM,OAAO,mBAAmB;EA6BhC,YAAY;;EAKZ,KAAK;UAIG;UAgCA;UA6CA;UAoBA;UA4BA;UA8CA;UA6GA;UAmEA;UA4EA;UAoBA;UAmDA;UAaA;;;UA2EA;;EASR;;;UAmBQ;;;UAkBA;;;;UAaA;;;EAqBR;IAAgB;IAA4B;;;EAa5C,cAAc;;EAKd,cAAc;;EAKd;IAAc;IAAkB,WAAW;;;EAK3C;;EAKA,aAAa;;EAcb,SAAS,iBAAiB;;EAM1B;;EAKA,YAAY;;EAKZ,oBAAoB;;EAKpB,SAAS,iBAAiB;;EAK1B;;EAKA,SAAS,iBAAiB;;EAM1B,YAAY;;EAMZ,cAAc,sBAAsB;;EAMpC,gBAAgB,gBAAgB,OAAO,cAAc;;;;;;;cAiB1C;WACF;WACA,QAAQ;WACR,QAAQ;WACR;IAAQ;IAAkB,WAAW;;;WAErC;WACA,OAAO;WACP,YAAY;;;;;;;WAOZ;IAAU;IAA4B;;mBAC9B;mBACA;EAEjB,YAAY,OAAO,aAAa;;MAgB5B;EAIJ,KAAK,iBAAiB;EAItB,QAAQ;;EAKR;;EAKA,SAAS;EAKT,gBAAgB;EAIhB,KAAK,iBAAiB;;EAKtB,SAAS;EAKT,KAAK,iBAAiB;;EAKtB,MAAM;IAAU;MAAyB;EAKzC,UAAU,sBAAsB;;;EAMhC,YAAY,gBAAgB,OAAO,cAAc;;EAKjD,WAAW,gBAAgB,OAAO,cAAc;;EAKhD,QAAQ,aAAa,uBAAuB;UAmCpC;;;;;;UCrsCO;WACN;WACA;;;EAGT,YAAY,0BAA0B;;EAEtC,WAAW,0BAA0B;;;iBAIvB,mBAAmB,OAAO,kBAAkB;UAW3C;;;;;;;;EAQf,OAAO;;EAEP;;;;;;;;;;;iBAYc,oBAAoB,OAAO,iBAAiB,SAAS;;;;;UCtCpD;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;;WAIjC,gBAAgB;;;;WAIhB;;;;WAIA,UAAU;IAAgB;IAAgB,UAAU;;;;WAGpD,QAAQ;;;;WAIR;;;WAGA,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;;;EAIA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { c as RunSplitTag } from "./run-record-DF-fuu31.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext } from "./types-
|
|
2
|
+
import { R as Scenario, d as DispatchContext } from "./types-BWwRXofp.js";
|
|
3
3
|
//#region src/benchmarks/types.d.ts
|
|
4
4
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
5
5
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -99,4 +99,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
|
99
99
|
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
100
100
|
//#endregion
|
|
101
101
|
export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
|
|
102
|
-
//# sourceMappingURL=types-
|
|
102
|
+
//# sourceMappingURL=types-qoxUfKQg.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-qoxUfKQg.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
|
|
@@ -100,7 +100,7 @@ Underreported spending and incomplete receipts remain explicit; `raw.method.cost
|
|
|
100
100
|
Inspect `cost.accountingComplete` and `cost.incompleteReasons` before treating the known subtotal as complete spending.
|
|
101
101
|
The shared dollar limit controls calls admitted through the cost ledger; arbitrary off-ledger callbacks must enforce their own spending limits.
|
|
102
102
|
|
|
103
|
-
Native generation records
|
|
103
|
+
Native generation records carry no interval; a candidate's paired contrast with its parent or the baseline is `estimateNode` over the run's search ledger.
|
|
104
104
|
Final comparisons retain their independently computed statistics.
|
|
105
105
|
Every final case and replica must have complete execution and judge results before comparison.
|
|
106
106
|
|
|
@@ -596,38 +596,36 @@ const proposer: SurfaceProposer = {
|
|
|
596
596
|
Return a label and rationale when they will help later analysis.
|
|
597
597
|
Candidate creation must not read final test results.
|
|
598
598
|
|
|
599
|
-
A proposer may attach `attribution`: an opaque JSON-safe record retained on `GenerationCandidate.attribution` and in loop provenance.
|
|
599
|
+
A proposer may attach `attribution`: an opaque JSON-safe record stored with the proposal in the search ledger and retained on `GenerationCandidate.attribution` and in loop provenance.
|
|
600
600
|
The loop never interprets it.
|
|
601
601
|
Tag it with a schema field and validate it on readback.
|
|
602
602
|
`makePolicyEditCandidateRecord` from `/analyst` records an edit forecast that can later be compared with the measured change.
|
|
603
603
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
The complete proposal is checked before candidate dispatch, so duplicates cannot consume candidate cells.
|
|
604
|
+
A candidate is a content-addressed node in the run's search ledger.
|
|
605
|
+
A candidate identical to a surface the search already holds, including the baseline, is a re-proposal: the ledger records a second edge into the existing node, and the surface is not measured again.
|
|
607
606
|
Use `reps` when one surface needs repeated measurements.
|
|
607
|
+
Lineage lives in the ledger: read a node's parents from its edges and its paired contrast from `estimateNode`, not from the generation record.
|
|
608
608
|
|
|
609
609
|
### Choose The Parent
|
|
610
610
|
|
|
611
|
-
`runOptimization()`
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
The loop hands that parent to the proposer as `ctx.currentSurface` and `ctx.parentOutcome`, and records it as every candidate's `parentSurfaceHash`.
|
|
617
|
-
`ctx.incumbentOutcome` stays the global promotion bar, and a candidate still has to beat the incumbent to promote.
|
|
618
|
-
`crowdedFrontierParent({ seed })` is the provided policy: a seeded NSGA-II crowded tournament that prefers isolated frontier parents.
|
|
619
|
-
The loop refuses a parent it has not measured to completion.
|
|
620
|
-
`selfImprove({ selectParent })` forwards the same policy in proposer mode.
|
|
611
|
+
`runOptimization()` runs on the search kernel with a `SearchPolicy` ([search ledger](./search-ledger.md#run-a-search-the-kernel)).
|
|
612
|
+
The default, `incumbent()`, is the hill climb: each proposal extends the leader once every earlier candidate is measured, and a candidate that scored every scenario leads when it beats the leader on the scenarios they share.
|
|
613
|
+
Pass `policy: crowdedFrontierParent({ seed })` to draw the parent from the Pareto frontier instead: a seeded NSGA-II crowded tournament that prefers isolated frontier members.
|
|
614
|
+
The proposer receives that parent as `ctx.currentSurface` and `ctx.parentOutcome`; `ctx.incumbentOutcome` stays the leader, and only a candidate that beats the leader leads.
|
|
615
|
+
`selfImprove({ policy })` forwards the same policy in proposer mode.
|
|
621
616
|
|
|
622
617
|
```ts
|
|
623
618
|
import { crowdedFrontierParent, runOptimization } from '@tangle-network/agent-eval/campaign'
|
|
624
619
|
|
|
625
620
|
const result = await runOptimization({
|
|
626
621
|
// ...scenarios, dispatchWithSurface, judges, proposer, populationSize, maxGenerations, runDir
|
|
627
|
-
|
|
622
|
+
policy: crowdedFrontierParent({ seed: 42 }),
|
|
628
623
|
})
|
|
629
624
|
```
|
|
630
625
|
|
|
626
|
+
`runOptimization` always writes its search ledger, at `<runDir>/search/ledger.jsonl` unless `searchLedger` puts it elsewhere, and returns the receipt on `searchHistory`.
|
|
627
|
+
Running it again on the same run directory continues an interrupted search and reruns nothing that settled; after a finished search it starts the next one beside it.
|
|
628
|
+
|
|
631
629
|
## Data And Cost Rules
|
|
632
630
|
|
|
633
631
|
- Train and selection cases are visible to complete optimization methods.
|