@tangle-network/agent-eval 0.106.1 → 0.106.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/campaign/index.d.ts +415 -2
- package/dist/campaign/index.js +653 -9
- package/dist/campaign/index.js.map +1 -1
- package/dist/{chunk-IA3X6CSF.js → chunk-J22CKVXN.js} +1 -1
- package/dist/chunk-J22CKVXN.js.map +1 -0
- package/dist/contract/index.d.ts +2 -2
- package/dist/contract/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{provenance-BGPBEFoV.d.ts → provenance-Bsyjc67Z.d.ts} +4 -0
- package/package.json +1 -1
- package/dist/chunk-IA3X6CSF.js.map +0 -1
package/dist/campaign/index.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeCon
|
|
|
5
5
|
export { i as CampaignAggregates, j as CampaignArtifactWriter, k as CampaignCellResult, l as CampaignCostMeter, z as CampaignTokenUsage, d as CampaignTraceWriter, D as DispatchFn, n as GateContext, h as GateDecision, G as GateResult, o as GenerationCandidate, A as JudgeAggregate, c as JudgeDimension, p as Mutator, O as OptimizationProposer, q as OptimizerConfig, P as ParetoParent, R as RedactionStatus, B as ScenarioAggregate, r as SessionScript, T as TraceSpan, E as isProposedCandidate, F as labelTrustRank } from '../types-Ctf7XIAL.js';
|
|
6
6
|
import { e as CampaignRunPlan, P as PlanCampaignRunOptions, C as CampaignStorage, b as RunCampaignOptions, c as RunImprovementLoopOptions } from '../gepa-DQ3ruj18.js';
|
|
7
7
|
export { h as CampaignRunPlanCell, j as GepaProposerConstraints, G as GepaProposerOptions, O as OpenAutoPrOptions, k as OpenAutoPrResult, a as RunImprovementLoopResult, R as RunOptimizationOptions, l as RunOptimizationResult, m as countSentenceEdits, n as defaultRenderDiff, o as extractH2Sections, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, p as openAutoPr, q as planCampaignRun, r as runCampaign, d as runImprovementLoop, s as runOptimization, t as surfaceHash } from '../gepa-DQ3ruj18.js';
|
|
8
|
-
export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, l as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, m as EmitLoopProvenanceArgs, n as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, o as LoopProvenanceBackend, q as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, c as ParetoSignificanceGateOptions, P as PowerPreflight, s as PowerPreflightOptions, d as PromotionObjective, e as PromotionPolicy, R as RunEvalOptions, f as buildEvidenceVector, t as buildLoopProvenanceRecord, g as composeGate, h as defaultProductionGate, u as emitLoopProvenance, i as evolutionaryProposer, j as heldOutGate, v as loopProvenanceSpans, p as paretoPolicy, k as paretoSignificanceGate, w as powerPreflight, x as provenanceRecordPath, y as provenanceSpansPath, r as runEval, z as surfaceContentHash } from '../provenance-
|
|
8
|
+
export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, l as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, m as EmitLoopProvenanceArgs, n as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, o as LoopProvenanceBackend, q as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, c as ParetoSignificanceGateOptions, P as PowerPreflight, s as PowerPreflightOptions, d as PromotionObjective, e as PromotionPolicy, R as RunEvalOptions, f as buildEvidenceVector, t as buildLoopProvenanceRecord, g as composeGate, h as defaultProductionGate, u as emitLoopProvenance, i as evolutionaryProposer, j as heldOutGate, v as loopProvenanceSpans, p as paretoPolicy, k as paretoSignificanceGate, w as powerPreflight, x as provenanceRecordPath, y as provenanceSpansPath, r as runEval, z as surfaceContentHash } from '../provenance-Bsyjc67Z.js';
|
|
9
9
|
import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-D88peojY.js';
|
|
10
10
|
import { L as LlmClientOptions } from '../llm-client-DyqEH4jH.js';
|
|
11
11
|
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
@@ -32,6 +32,242 @@ import '../judge-calibration-7C-IDmKr.js';
|
|
|
32
32
|
import '../types-C7DGg5ex.js';
|
|
33
33
|
import 'zod';
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Lineage DAG — a git-graph of improvement candidates.
|
|
37
|
+
*
|
|
38
|
+
* The improvement loop (`run-optimization.ts`) records a LINEAR `GenerationRecord[]`
|
|
39
|
+
* with no parent pointers, computes a Pareto frontier within a run, then collapses
|
|
40
|
+
* to one gated winner and discards the rest. This module adds the missing
|
|
41
|
+
* first-class structure: a directed acyclic graph of candidate versions where
|
|
42
|
+
*
|
|
43
|
+
* - a node with ONE parent is a mutation,
|
|
44
|
+
* - a node with ZERO parents is a root/seed,
|
|
45
|
+
* - a node with 2+ parents is a MERGE (a "collapse") — nothing special-cased,
|
|
46
|
+
* just a multi-parent node,
|
|
47
|
+
* - a `track` groups a lineage into an island, and each track runs a distinct
|
|
48
|
+
* `vision` (a proposer strategy — e.g. a "solve" vision, an "outside-the-box"
|
|
49
|
+
* vision, and an adversarial "contrarian" vision that attacks the leader).
|
|
50
|
+
*
|
|
51
|
+
* An agent-managed {@link Governor} decides the next operation (extend / branch /
|
|
52
|
+
* merge / prune / stop), so branching AND collapsing are choices a supervisor
|
|
53
|
+
* makes by reading the graph, not hard-coded control flow.
|
|
54
|
+
*
|
|
55
|
+
* DETERMINISM is a hard invariant: no `Date.now`, `Math.random`, or `new Date`.
|
|
56
|
+
* Ids are content+lineage hashes; order is a monotone insertion counter; every
|
|
57
|
+
* query returns nodes in `seq` order. Same inputs ⇒ identical graph.
|
|
58
|
+
*/
|
|
59
|
+
interface LineageNode {
|
|
60
|
+
/** Deterministic content+lineage hash (see {@link lineageNodeId}). */
|
|
61
|
+
id: string;
|
|
62
|
+
/** `[]` = root/seed; `[x]` = mutation; `[x, y, ...]` = MERGE (collapse). */
|
|
63
|
+
parentIds: string[];
|
|
64
|
+
/** Logical island/track this node belongs to. */
|
|
65
|
+
track: string;
|
|
66
|
+
/** Human label for the strategy driving this track (e.g. `solve`, `contrarian`). */
|
|
67
|
+
vision?: string;
|
|
68
|
+
/** Candidate content (prompt text, skill document, config JSON, ...). */
|
|
69
|
+
surface: string;
|
|
70
|
+
/** Scalar fitness (e.g. mean holdout composite); higher is better. */
|
|
71
|
+
score: number;
|
|
72
|
+
/** Optional per-scenario objective vector for Pareto dominance. */
|
|
73
|
+
scoreVector?: number[];
|
|
74
|
+
/** Which proposer produced it (`gepa`, `skill-opt`, `merge`, `seed`, ...). */
|
|
75
|
+
proposer: string;
|
|
76
|
+
rationale?: string;
|
|
77
|
+
/** Gate verdict, when this node was gated. */
|
|
78
|
+
gate?: 'ship' | 'hold';
|
|
79
|
+
/** Step index within its track (root = 0). */
|
|
80
|
+
generation: number;
|
|
81
|
+
/** Monotone global insertion order (assigned by {@link Lineage.addNode}). */
|
|
82
|
+
seq: number;
|
|
83
|
+
}
|
|
84
|
+
interface LineageEdge {
|
|
85
|
+
/** Parent node id. */
|
|
86
|
+
from: string;
|
|
87
|
+
/** Child node id. */
|
|
88
|
+
to: string;
|
|
89
|
+
}
|
|
90
|
+
interface LineageGraph {
|
|
91
|
+
nodes: LineageNode[];
|
|
92
|
+
edges: LineageEdge[];
|
|
93
|
+
}
|
|
94
|
+
/** Deterministic node id: a hash of the node's lineage + content + proposer.
|
|
95
|
+
* Pure — identical inputs always yield the same id (a re-derived node collapses
|
|
96
|
+
* onto its original rather than duplicating). */
|
|
97
|
+
declare function lineageNodeId(input: {
|
|
98
|
+
parentIds: string[];
|
|
99
|
+
track: string;
|
|
100
|
+
surface: string;
|
|
101
|
+
proposer: string;
|
|
102
|
+
}): string;
|
|
103
|
+
/** Input to {@link Lineage.addNode}: everything but the derived `id`/`seq` and the
|
|
104
|
+
* optional `generation` (derived from parents when omitted). */
|
|
105
|
+
type LineageNodeInput = Omit<LineageNode, 'id' | 'seq' | 'generation'> & {
|
|
106
|
+
generation?: number;
|
|
107
|
+
};
|
|
108
|
+
declare class Lineage {
|
|
109
|
+
private readonly byId;
|
|
110
|
+
private readonly childIds;
|
|
111
|
+
private nextSeq;
|
|
112
|
+
constructor(nodes?: readonly LineageNode[]);
|
|
113
|
+
/** Append a node. Derives `id` (via {@link lineageNodeId}), assigns the next
|
|
114
|
+
* `seq`, and derives `generation` as `max(parent.generation) + 1` (root = 0)
|
|
115
|
+
* when omitted. Throws on an unknown parent. Idempotent: re-adding an identical
|
|
116
|
+
* node returns the existing one.
|
|
117
|
+
*
|
|
118
|
+
* Acyclicity is guaranteed by construction, not by a runtime check: every
|
|
119
|
+
* parent must already exist (a child can never point at a not-yet-added node),
|
|
120
|
+
* ids are immutable content hashes (an existing node can never gain new
|
|
121
|
+
* parents), and the store is append-only — so no back-edge can form. (Traversal
|
|
122
|
+
* is still cycle-safe against hand-corrupted deserialized input via visited
|
|
123
|
+
* sets in {@link ancestors}/{@link descendants}.) */
|
|
124
|
+
addNode(input: LineageNodeInput): LineageNode;
|
|
125
|
+
/** Collapse 2+ parents into a single node (a merge/"collapse"). A merge is an
|
|
126
|
+
* ordinary multi-parent node; this is a guarded convenience. */
|
|
127
|
+
merge(input: {
|
|
128
|
+
parentIds: string[];
|
|
129
|
+
track: string;
|
|
130
|
+
surface: string;
|
|
131
|
+
score: number;
|
|
132
|
+
proposer?: string;
|
|
133
|
+
vision?: string;
|
|
134
|
+
scoreVector?: number[];
|
|
135
|
+
rationale?: string;
|
|
136
|
+
}): LineageNode;
|
|
137
|
+
get(id: string): LineageNode | undefined;
|
|
138
|
+
has(id: string): boolean;
|
|
139
|
+
/** All nodes, in insertion (`seq`) order. */
|
|
140
|
+
all(): LineageNode[];
|
|
141
|
+
roots(): LineageNode[];
|
|
142
|
+
parents(id: string): LineageNode[];
|
|
143
|
+
children(id: string): LineageNode[];
|
|
144
|
+
/** Transitive ancestors (excludes `id`). */
|
|
145
|
+
ancestors(id: string): Set<string>;
|
|
146
|
+
/** Transitive descendants (excludes `id`). */
|
|
147
|
+
descendants(id: string): Set<string>;
|
|
148
|
+
/** Nodes with no children (leaf/branch tips). */
|
|
149
|
+
tips(): LineageNode[];
|
|
150
|
+
/** Distinct track ids, in first-seen (`seq`) order. */
|
|
151
|
+
tracks(): string[];
|
|
152
|
+
trackNodes(track: string): LineageNode[];
|
|
153
|
+
/** The highest-`score` tip of a track (ties broken by lowest `seq`). */
|
|
154
|
+
trackTip(track: string): LineageNode | undefined;
|
|
155
|
+
/** The highest-`score` node overall (ties broken by lowest `seq`). */
|
|
156
|
+
best(): LineageNode | undefined;
|
|
157
|
+
/** The Pareto-non-dominated set among TIPS. Uses `scoreVector` when every
|
|
158
|
+
* compared tip carries one, else the scalar `score`. A dominates B iff A is
|
|
159
|
+
* >= B on every component and > B on at least one. */
|
|
160
|
+
frontier(): LineageNode[];
|
|
161
|
+
toGraph(): LineageGraph;
|
|
162
|
+
/** One JSON node per line, in `seq` order. */
|
|
163
|
+
toJSONL(): string;
|
|
164
|
+
static fromJSONL(text: string): Lineage;
|
|
165
|
+
private index;
|
|
166
|
+
}
|
|
167
|
+
interface LineageStore {
|
|
168
|
+
/** Load the persisted lineage (an empty `Lineage` when nothing is stored). */
|
|
169
|
+
load(): Promise<Lineage>;
|
|
170
|
+
/** Append one node durably. */
|
|
171
|
+
append(node: LineageNode): Promise<void>;
|
|
172
|
+
/** Overwrite with a full snapshot. */
|
|
173
|
+
save(lineage: Lineage): Promise<void>;
|
|
174
|
+
}
|
|
175
|
+
/** JSONL-file store: append-only durability, snapshot via rewrite, `load` parses
|
|
176
|
+
* through {@link Lineage.fromJSONL}. */
|
|
177
|
+
declare function fsLineageStore(path: string): LineageStore;
|
|
178
|
+
/** In-memory store (default; for tests and ephemeral runs). */
|
|
179
|
+
declare function memLineageStore(): LineageStore;
|
|
180
|
+
type GovernorOp = {
|
|
181
|
+
op: 'extend';
|
|
182
|
+
track: string;
|
|
183
|
+
} | {
|
|
184
|
+
op: 'branch';
|
|
185
|
+
fromNodeId: string;
|
|
186
|
+
track: string;
|
|
187
|
+
proposer: string;
|
|
188
|
+
vision?: string;
|
|
189
|
+
} | {
|
|
190
|
+
op: 'merge';
|
|
191
|
+
parentIds: string[];
|
|
192
|
+
track: string;
|
|
193
|
+
} | {
|
|
194
|
+
op: 'prune';
|
|
195
|
+
track: string;
|
|
196
|
+
} | {
|
|
197
|
+
op: 'stop';
|
|
198
|
+
};
|
|
199
|
+
interface GovernorContext {
|
|
200
|
+
lineage: Lineage;
|
|
201
|
+
/** Operations executed so far. */
|
|
202
|
+
step: number;
|
|
203
|
+
budgetRemaining: number;
|
|
204
|
+
/** Tracks already pruned — the governor must not target these for extend/branch. */
|
|
205
|
+
prunedTracks: string[];
|
|
206
|
+
}
|
|
207
|
+
interface Governor {
|
|
208
|
+
decide(ctx: GovernorContext): Promise<GovernorOp> | GovernorOp;
|
|
209
|
+
}
|
|
210
|
+
interface HeuristicGovernorOptions {
|
|
211
|
+
/** Cap on live tracks before branching a new one. Default 3. */
|
|
212
|
+
maxTracks?: number;
|
|
213
|
+
/** Non-improving steps a track may take before it is pruned. Default 2. */
|
|
214
|
+
plateauSteps?: number;
|
|
215
|
+
/** Frontier-tip count (across distinct tracks) that triggers a merge. Default 2. */
|
|
216
|
+
mergeFrontierAt?: number;
|
|
217
|
+
}
|
|
218
|
+
/** The reference deterministic policy an agent {@link Governor} can replace.
|
|
219
|
+
* Reads only the lineage + context — no LLM, no randomness. */
|
|
220
|
+
declare function heuristicGovernor(opts?: HeuristicGovernorOptions): Governor;
|
|
221
|
+
/** The LLM-supervisor slot: a governor whose `decide` defers to a caller-supplied
|
|
222
|
+
* async function (which may read `ctx.lineage.toGraph()`). */
|
|
223
|
+
declare function callbackGovernor(decide: (ctx: GovernorContext) => Promise<GovernorOp>): Governor;
|
|
224
|
+
interface RunLineageSeed {
|
|
225
|
+
surface: string;
|
|
226
|
+
track: string;
|
|
227
|
+
vision?: string;
|
|
228
|
+
proposer: string;
|
|
229
|
+
score: number;
|
|
230
|
+
scoreVector?: number[];
|
|
231
|
+
}
|
|
232
|
+
interface RunLineageStepResult {
|
|
233
|
+
surface: string;
|
|
234
|
+
score: number;
|
|
235
|
+
scoreVector?: number[];
|
|
236
|
+
rationale?: string;
|
|
237
|
+
gate?: 'ship' | 'hold';
|
|
238
|
+
}
|
|
239
|
+
interface RunLineageOptions {
|
|
240
|
+
seeds: RunLineageSeed[];
|
|
241
|
+
/** Produce one new candidate from a track's tip (propose + measure + gate in
|
|
242
|
+
* real use; a pure function in tests). */
|
|
243
|
+
step: (args: {
|
|
244
|
+
track: string;
|
|
245
|
+
proposer: string;
|
|
246
|
+
tip: LineageNode;
|
|
247
|
+
}) => Promise<RunLineageStepResult>;
|
|
248
|
+
/** Collapse 2+ parent surfaces into one (GEPA crossover / LLM merge in real use). */
|
|
249
|
+
merge: (args: {
|
|
250
|
+
parents: LineageNode[];
|
|
251
|
+
track: string;
|
|
252
|
+
}) => Promise<Omit<RunLineageStepResult, 'gate'>>;
|
|
253
|
+
governor: Governor;
|
|
254
|
+
budget: {
|
|
255
|
+
maxSteps: number;
|
|
256
|
+
};
|
|
257
|
+
store?: LineageStore;
|
|
258
|
+
log?: (msg: string, fields?: Record<string, unknown>) => void;
|
|
259
|
+
}
|
|
260
|
+
interface RunLineageResult {
|
|
261
|
+
lineage: Lineage;
|
|
262
|
+
best: LineageNode | undefined;
|
|
263
|
+
steps: number;
|
|
264
|
+
}
|
|
265
|
+
/** Drive a multi-track improvement DAG under an agent-managed governor. Seeds each
|
|
266
|
+
* entry as a root, then repeatedly asks the governor for the next operation
|
|
267
|
+
* (extend / branch / merge / prune / stop) up to `budget.maxSteps`, persisting
|
|
268
|
+
* every node. Honors `prune`: a pruned track is never extended or branched again. */
|
|
269
|
+
declare function runLineage(opts: RunLineageOptions): Promise<RunLineageResult>;
|
|
270
|
+
|
|
35
271
|
/**
|
|
36
272
|
* Make the trace-analyst's OWN prompt a GEPA-optimizable surface.
|
|
37
273
|
*
|
|
@@ -1051,6 +1287,129 @@ interface ScoreboardRenderOptions {
|
|
|
1051
1287
|
*/
|
|
1052
1288
|
declare function renderScoreboardMarkdown(rows: readonly ScoreboardRow[], opts?: ScoreboardRenderOptions): string;
|
|
1053
1289
|
|
|
1290
|
+
/**
|
|
1291
|
+
* `runLineageLoop` — the live adapter that wires the {@link runLineage} DAG's
|
|
1292
|
+
* two abstract seams (`step`, `merge`) to the REAL improvement machinery, so the
|
|
1293
|
+
* multi-track, multi-parent improvement DAG can run against a real proposer +
|
|
1294
|
+
* real measurement.
|
|
1295
|
+
*
|
|
1296
|
+
* INTEGRATION CHOICE (smallest correct integration):
|
|
1297
|
+
* - `step` = ONE `SurfaceProposer.propose` from the track tip (a single GEPA
|
|
1298
|
+
* reflective generation, small population) + one scoring campaign
|
|
1299
|
+
* per candidate. The best-scoring candidate (elitist: the tip is
|
|
1300
|
+
* kept in the pool so a step never regresses below its parent)
|
|
1301
|
+
* becomes the new DAG node.
|
|
1302
|
+
* - `merge` = the SAME proposer driven with `ctx.paretoParents` set to the
|
|
1303
|
+
* 2+ parent surfaces, which fires `gepaProposer`'s GEPA
|
|
1304
|
+
* combine-complementary-lessons CROSSOVER, then one scoring
|
|
1305
|
+
* campaign on the merged surface.
|
|
1306
|
+
*
|
|
1307
|
+
* We deliberately do NOT run a full {@link runImprovementLoop} (baseline
|
|
1308
|
+
* campaign + optimization + two holdout campaigns + gate + optional PR) per DAG
|
|
1309
|
+
* step — that is the OUTER gated-promotion shell for a single lineage, far too
|
|
1310
|
+
* heavy to fire once per node. The DAG {@link Governor} controls BREADTH across
|
|
1311
|
+
* steps (extend / branch / merge / prune); the inner step is intentionally one
|
|
1312
|
+
* small generation. This mirrors the task's guidance: "Keep the improvement
|
|
1313
|
+
* budget per step SMALL (1 generation, small population)."
|
|
1314
|
+
*
|
|
1315
|
+
* Both machinery halves are injectable seams so the loop is unit-testable
|
|
1316
|
+
* without a live model or a sandbox:
|
|
1317
|
+
* - `proposer` — defaults to {@link gepaProposer} (needs `llm` + `model`).
|
|
1318
|
+
* A test injects a pure stub `SurfaceProposer`.
|
|
1319
|
+
* - `scoreSurface` — defaults to a {@link runCampaign} pass over
|
|
1320
|
+
* `holdoutScenarios ?? scenarios` (the "agent" =
|
|
1321
|
+
* `dispatchWithSurface`, the judges = `judges`). A test
|
|
1322
|
+
* injects a deterministic function of the surface string,
|
|
1323
|
+
* or supplies a stub `dispatchWithSurface` + `judges` and
|
|
1324
|
+
* exercises the real `runCampaign` path.
|
|
1325
|
+
*
|
|
1326
|
+
* Additive: no changes to `lineage.ts`, `run-optimization.ts`, or `gepa.ts`.
|
|
1327
|
+
*/
|
|
1328
|
+
|
|
1329
|
+
/** A seed track: the initial surface + track identity. Unlike
|
|
1330
|
+
* {@link RunLineageSeed} there is NO `score` — `runLineageLoop` scores each
|
|
1331
|
+
* seed surface in an initial pass so seed fitness is measured, not asserted. */
|
|
1332
|
+
interface RunLineageLoopSeed {
|
|
1333
|
+
surface: MutableSurface;
|
|
1334
|
+
track: string;
|
|
1335
|
+
/** Human label for the strategy driving this track (e.g. `solve`,
|
|
1336
|
+
* `outside-the-box`, `contrarian`). */
|
|
1337
|
+
vision?: string;
|
|
1338
|
+
/** Proposer label recorded on the seed node (e.g. `gepa`, `seed`). */
|
|
1339
|
+
proposer: string;
|
|
1340
|
+
}
|
|
1341
|
+
/** The measured fitness of one surface — the value recorded on a DAG node. */
|
|
1342
|
+
interface SurfaceScore {
|
|
1343
|
+
score: number;
|
|
1344
|
+
/** Per-objective vector (per-scenario composite) for Pareto dominance across
|
|
1345
|
+
* track tips. Omitted ⇒ the DAG uses the scalar `score`. */
|
|
1346
|
+
scoreVector?: number[];
|
|
1347
|
+
}
|
|
1348
|
+
interface RunLineageLoopOptions<TScenario extends Scenario, TArtifact> {
|
|
1349
|
+
/** The visioned tracks to seed the DAG with. Each is scored once up front. */
|
|
1350
|
+
seeds: RunLineageLoopSeed[];
|
|
1351
|
+
/** Scenarios the candidate surfaces are proposed against + scored on. */
|
|
1352
|
+
scenarios: TScenario[];
|
|
1353
|
+
/** Held-out scenarios used to SCORE each DAG node's fitness. Defaults to
|
|
1354
|
+
* `scenarios` when omitted. Scored the same way for seeds, steps, and merges
|
|
1355
|
+
* so every node's `score` is comparable. */
|
|
1356
|
+
holdoutScenarios?: TScenario[];
|
|
1357
|
+
/** Judges for the scoring campaign. */
|
|
1358
|
+
judges?: JudgeConfig<TArtifact, TScenario>[];
|
|
1359
|
+
/** The "agent" seam: run the CURRENT surface on a scenario → artifact. Same
|
|
1360
|
+
* shape as `runOptimization`'s `dispatchWithSurface`. Required UNLESS a
|
|
1361
|
+
* custom `scoreSurface` is injected. */
|
|
1362
|
+
dispatchWithSurface?: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
|
|
1363
|
+
/** Where scoring campaigns write. Required UNLESS `scoreSurface` is injected. */
|
|
1364
|
+
runDir?: string;
|
|
1365
|
+
/** Router transport for the default `gepaProposer`. Required UNLESS a custom
|
|
1366
|
+
* `proposer` is injected. */
|
|
1367
|
+
llm?: LlmClientOptions;
|
|
1368
|
+
/** Model for the default `gepaProposer`. Required UNLESS a custom `proposer`
|
|
1369
|
+
* is injected. */
|
|
1370
|
+
model?: string;
|
|
1371
|
+
/** What is being optimized — appears in the GEPA reflection/combine prompts.
|
|
1372
|
+
* Default `'agent surface'`. */
|
|
1373
|
+
target?: string;
|
|
1374
|
+
/** Candidates proposed per extend/branch step (BREADTH within one step).
|
|
1375
|
+
* Default 4. The merge always proposes a single crossover. */
|
|
1376
|
+
populationSize?: number;
|
|
1377
|
+
/** Agent-managed decision layer. Default {@link heuristicGovernor}. */
|
|
1378
|
+
governor?: Governor;
|
|
1379
|
+
budget: {
|
|
1380
|
+
maxSteps: number;
|
|
1381
|
+
};
|
|
1382
|
+
store?: LineageStore;
|
|
1383
|
+
/** Override the per-step proposer. Default {@link gepaProposer}. Inject a
|
|
1384
|
+
* pure stub to unit-test without an LLM. */
|
|
1385
|
+
proposer?: SurfaceProposer;
|
|
1386
|
+
/** Override how a surface is scored into a DAG-node fitness. Default is a
|
|
1387
|
+
* {@link runCampaign} pass over `holdoutScenarios ?? scenarios`. Inject a
|
|
1388
|
+
* deterministic function to unit-test without a campaign. */
|
|
1389
|
+
scoreSurface?: (surface: MutableSurface) => Promise<SurfaceScore>;
|
|
1390
|
+
seed?: number;
|
|
1391
|
+
reps?: number;
|
|
1392
|
+
storage?: CampaignStorage;
|
|
1393
|
+
tracing?: 'on' | 'off';
|
|
1394
|
+
expectUsage?: 'assert' | 'warn' | 'off';
|
|
1395
|
+
maxConcurrency?: number;
|
|
1396
|
+
dispatchTimeoutMs?: number;
|
|
1397
|
+
/** Test seam — deterministic wall clock forwarded to `runCampaign`. */
|
|
1398
|
+
now?: () => Date;
|
|
1399
|
+
log?: (msg: string, fields?: Record<string, unknown>) => void;
|
|
1400
|
+
}
|
|
1401
|
+
interface RunLineageLoopResult {
|
|
1402
|
+
lineage: Lineage;
|
|
1403
|
+
best: LineageNode | undefined;
|
|
1404
|
+
steps: number;
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* Wire the {@link runLineage} DAG's `step`/`merge` seams to a real
|
|
1408
|
+
* `SurfaceProposer` + a real scoring campaign and run the multi-track improvement
|
|
1409
|
+
* DAG live under a {@link Governor}.
|
|
1410
|
+
*/
|
|
1411
|
+
declare function runLineageLoop<TScenario extends Scenario, TArtifact>(opts: RunLineageLoopOptions<TScenario, TArtifact>): Promise<RunLineageLoopResult>;
|
|
1412
|
+
|
|
1054
1413
|
/**
|
|
1055
1414
|
* SkillOpt patch primitives (Microsoft, arXiv:2605.23904 — "Executive
|
|
1056
1415
|
* Strategy for Self-Evolving Agent Skills"). Where GEPA regenerates a surface
|
|
@@ -1511,6 +1870,60 @@ declare function tangleTracesRoot(): string;
|
|
|
1511
1870
|
* compute so callers pass a *name*, not a path. */
|
|
1512
1871
|
declare function resolveRunDir(runDir: string, repo?: string): string;
|
|
1513
1872
|
|
|
1873
|
+
/**
|
|
1874
|
+
* Discriminative scenario selection (research claim E2).
|
|
1875
|
+
*
|
|
1876
|
+
* The OR benchmark is SATURATING: run 7 measured ~75% tied holdout cells — most
|
|
1877
|
+
* problems are solved optimally by the baseline AND every candidate, so those
|
|
1878
|
+
* paired cells carry zero signal. A random/balanced holdout split spends its
|
|
1879
|
+
* budget on scenarios that cannot separate candidates.
|
|
1880
|
+
*
|
|
1881
|
+
* This picks the holdout by DISCRIMINATION power instead: a scenario every
|
|
1882
|
+
* candidate scores identically (variance ~0) carries no signal; one where the
|
|
1883
|
+
* scores spread carries the most. We drop fully saturated ties so each paired
|
|
1884
|
+
* holdout cell is spent on a scenario that can actually move a verdict.
|
|
1885
|
+
*/
|
|
1886
|
+
/** Per-scenario observation: the composite scores each candidate earned on it. */
|
|
1887
|
+
interface ScenarioSignal {
|
|
1888
|
+
scenarioId: string;
|
|
1889
|
+
/** Per-candidate composite scores observed for this scenario (>=1 values). */
|
|
1890
|
+
scores: number[];
|
|
1891
|
+
}
|
|
1892
|
+
interface DiscriminationScore {
|
|
1893
|
+
scenarioId: string;
|
|
1894
|
+
/** Higher = separates candidates more (spread of their scores). */
|
|
1895
|
+
discrimination: number;
|
|
1896
|
+
/** Higher = easier / more-saturated (mean of candidate scores). */
|
|
1897
|
+
meanScore: number;
|
|
1898
|
+
variance: number;
|
|
1899
|
+
/** variance ~0 AND meanScore at/above the ceiling ⇒ a saturated tie, no signal. */
|
|
1900
|
+
tied: boolean;
|
|
1901
|
+
}
|
|
1902
|
+
/**
|
|
1903
|
+
* Rank scenarios by how well they DISCRIMINATE candidates.
|
|
1904
|
+
*
|
|
1905
|
+
* `discrimination = variance` (spread of the candidate scores) — kept simple on
|
|
1906
|
+
* purpose; the headroom term (`saturationCeiling - meanScore`) only breaks ties
|
|
1907
|
+
* so that, among equally spread scenarios, the one with more room to improve
|
|
1908
|
+
* ranks first. Returned sorted by the deterministic order above.
|
|
1909
|
+
*/
|
|
1910
|
+
declare function scoreDiscrimination(signals: ScenarioSignal[], opts?: {
|
|
1911
|
+
saturationCeiling?: number;
|
|
1912
|
+
}): DiscriminationScore[];
|
|
1913
|
+
/**
|
|
1914
|
+
* Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING
|
|
1915
|
+
* fully saturated ties when enough non-tied scenarios exist (a tie in the
|
|
1916
|
+
* holdout wastes a paired cell).
|
|
1917
|
+
*
|
|
1918
|
+
* Prefers non-tied scenarios; if fewer than `k` non-tied exist, fills with the
|
|
1919
|
+
* least-saturated tied ones (tied scenarios are already ordered least-saturated
|
|
1920
|
+
* first by `meanScore` asc). Deterministic. Throws if `k < 1`. If
|
|
1921
|
+
* `signals.length <= k`, returns all ids in discrimination order.
|
|
1922
|
+
*/
|
|
1923
|
+
declare function selectDiscriminative(signals: ScenarioSignal[], k: number, opts?: {
|
|
1924
|
+
saturationCeiling?: number;
|
|
1925
|
+
}): string[];
|
|
1926
|
+
|
|
1514
1927
|
/**
|
|
1515
1928
|
* Shared campaign-score reductions used by every optimizer preset
|
|
1516
1929
|
* (`runOptimization`, `runSkillOpt`, `compareProposers`). ONE definition of
|
|
@@ -1594,4 +2007,4 @@ declare function gitWorktreeAdapter(opts: GitWorktreeAdapterOptions): WorktreeAd
|
|
|
1594
2007
|
* as a ref under the adapter's worktree dir. */
|
|
1595
2008
|
declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
|
|
1596
2009
|
|
|
1597
|
-
export { type AcceptedEdit, type AceProposerOptions, type AnalystArtifact, type AnalystScenario, type ApplySkillPatchResult, type BuildAnalystSurfaceDispatchOptions, type CampaignBreakdown, CampaignResult, CampaignRunPlan, CampaignStorage, CodeSurface, type CompareProposersOptions, type DimensionRegression, DispatchContext, type EvalFixture, type EvalFixtureFile, type EvalFixtureLoadOptions, type EvalFixtureRunPlan, type EvalFixtureScenario, type EvalFixtureValidationMode, type FailureModeRecallJudgeOptions, type FapoAttributionSignals, type FapoEntryConfig, type FapoFailureCluster, type FapoOptimizationLevel, type FapoProposerOptions, type FapoReviewInput, type FapoReviewIssue, type FapoReviewResult, type FapoScopeContract, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, Gate, GenerationRecord, type GitWorktreeAdapterOptions, type HaloProposerOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, type JsonPrimitive, type JsonValue, JudgeConfig, JudgeScore, LabelTrust, LabeledScenarioRecord, LabeledScenarioSampleArgs, LabeledScenarioSource, LabeledScenarioStore, LabeledScenarioStoreError, LabeledScenarioWrite, type LoadEvalFixtureScenariosOptions, type MemoryCurationProposerOptions, MutableSurface, type OptimizerEntryConfig, type PairedHoldout, type ParameterCandidate, type ParameterChange, type ParameterSweepProposerOptions, PlanCampaignRunOptions, type PlanEvalFixtureRunOptions, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type PolicyEditProposerOptions, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, ProposeContext, type ProposePatchesArgs, ProposedCandidate, type ProposerComparison, type ProposerEntry, type ProposerPairwise, type ProposerScore, type RejectedEdit, RunCampaignOptions, RunImprovementLoopOptions, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, Scenario, type ScenarioRollup, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillOptProposer, type SkillOptProposerOptions, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, SurfaceProposer, type TraceAnalystProposerOptions, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceProposer, applySkillPatch, buildAnalystSurfaceDispatch, campaignBreakdown, campaignMeanComposite, compareProposers, detectScale, dimensionRegressions, discoverEvalFixtures, extractFapoAttributionSignals, failureModeRecallJudge, fapoEscalationEntry, fapoProposer, gepaParetoEntry, gepaReflectionEntry, gitWorktreeAdapter, haloProposer, heldoutSignificance, loadEvalFixture, loadEvalFixtureScenarios, makePlaybackDispatch, memoryCurationProposer, pairHoldout, parameterSweepProposer, parseSkillPatchResponse, patchEditCount, planEvalFixtureRun, policyEditProposer, renderScoreboardMarkdown, resolveRunDir, resolveWorktreePath, runProfileMatrix, runSkillOpt, scoreUserStory, scoreboardSummary, sequentialDecide, sequentialPairedGate, skillOptEntry, skillOptProposer, tangleTracesRoot, traceAnalystProposer, userStoryScoreboard };
|
|
2010
|
+
export { type AcceptedEdit, type AceProposerOptions, type AnalystArtifact, type AnalystScenario, type ApplySkillPatchResult, type BuildAnalystSurfaceDispatchOptions, type CampaignBreakdown, CampaignResult, CampaignRunPlan, CampaignStorage, CodeSurface, type CompareProposersOptions, type DimensionRegression, type DiscriminationScore, DispatchContext, type EvalFixture, type EvalFixtureFile, type EvalFixtureLoadOptions, type EvalFixtureRunPlan, type EvalFixtureScenario, type EvalFixtureValidationMode, type FailureModeRecallJudgeOptions, type FapoAttributionSignals, type FapoEntryConfig, type FapoFailureCluster, type FapoOptimizationLevel, type FapoProposerOptions, type FapoReviewInput, type FapoReviewIssue, type FapoReviewResult, type FapoScopeContract, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, Gate, GenerationRecord, type GitWorktreeAdapterOptions, type Governor, type GovernorContext, type GovernorOp, type HaloProposerOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, type HeuristicGovernorOptions, type JsonPrimitive, type JsonValue, JudgeConfig, JudgeScore, LabelTrust, LabeledScenarioRecord, LabeledScenarioSampleArgs, LabeledScenarioSource, LabeledScenarioStore, LabeledScenarioStoreError, LabeledScenarioWrite, Lineage, type LineageEdge, type LineageGraph, type LineageNode, type LineageNodeInput, type LineageStore, type LoadEvalFixtureScenariosOptions, type MemoryCurationProposerOptions, MutableSurface, type OptimizerEntryConfig, type PairedHoldout, type ParameterCandidate, type ParameterChange, type ParameterSweepProposerOptions, PlanCampaignRunOptions, type PlanEvalFixtureRunOptions, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type PolicyEditProposerOptions, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, ProposeContext, type ProposePatchesArgs, ProposedCandidate, type ProposerComparison, type ProposerEntry, type ProposerPairwise, type ProposerScore, type RejectedEdit, RunCampaignOptions, RunImprovementLoopOptions, type RunLineageLoopOptions, type RunLineageLoopResult, type RunLineageLoopSeed, type RunLineageOptions, type RunLineageResult, type RunLineageSeed, type RunLineageStepResult, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, Scenario, type ScenarioRollup, type ScenarioSignal, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillOptProposer, type SkillOptProposerOptions, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, SurfaceProposer, type SurfaceScore, type TraceAnalystProposerOptions, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceProposer, applySkillPatch, buildAnalystSurfaceDispatch, callbackGovernor, campaignBreakdown, campaignMeanComposite, compareProposers, detectScale, dimensionRegressions, discoverEvalFixtures, extractFapoAttributionSignals, failureModeRecallJudge, fapoEscalationEntry, fapoProposer, fsLineageStore, gepaParetoEntry, gepaReflectionEntry, gitWorktreeAdapter, haloProposer, heldoutSignificance, heuristicGovernor, lineageNodeId, loadEvalFixture, loadEvalFixtureScenarios, makePlaybackDispatch, memLineageStore, memoryCurationProposer, pairHoldout, parameterSweepProposer, parseSkillPatchResponse, patchEditCount, planEvalFixtureRun, policyEditProposer, renderScoreboardMarkdown, resolveRunDir, resolveWorktreePath, runLineage, runLineageLoop, runProfileMatrix, runSkillOpt, scoreDiscrimination, scoreUserStory, scoreboardSummary, selectDiscriminative, sequentialDecide, sequentialPairedGate, skillOptEntry, skillOptProposer, tangleTracesRoot, traceAnalystProposer, userStoryScoreboard };
|