@tangle-network/agent-eval 0.106.1 → 0.106.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/campaign/index.d.ts +415 -2
- package/dist/campaign/index.js +653 -9
- package/dist/campaign/index.js.map +1 -1
- package/dist/{chunk-IA3X6CSF.js → chunk-J22CKVXN.js} +1 -1
- package/dist/chunk-J22CKVXN.js.map +1 -0
- package/dist/contract/index.d.ts +2 -2
- package/dist/contract/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{provenance-BGPBEFoV.d.ts → provenance-Bsyjc67Z.d.ts} +4 -0
- package/package.json +1 -1
- package/dist/chunk-IA3X6CSF.js.map +0 -1
package/dist/campaign/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
paretoSignificanceGate,
|
|
8
8
|
powerPreflight,
|
|
9
9
|
runEval
|
|
10
|
-
} from "../chunk-
|
|
10
|
+
} from "../chunk-J22CKVXN.js";
|
|
11
11
|
import {
|
|
12
12
|
HARNESS_NATIVE_MODEL,
|
|
13
13
|
agentProfileHash,
|
|
@@ -103,6 +103,424 @@ import {
|
|
|
103
103
|
} from "../chunk-ONWEPEDO.js";
|
|
104
104
|
import "../chunk-PZ5AY32C.js";
|
|
105
105
|
|
|
106
|
+
// src/campaign/lineage.ts
|
|
107
|
+
import { createHash } from "crypto";
|
|
108
|
+
import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
|
|
109
|
+
import { dirname } from "path";
|
|
110
|
+
function lineageNodeId(input) {
|
|
111
|
+
const parents = [...input.parentIds].sort().join(",");
|
|
112
|
+
const payload = `${parents}|${input.track}|${input.surface}|${input.proposer}`;
|
|
113
|
+
return createHash("sha256").update(payload).digest("hex").slice(0, 16);
|
|
114
|
+
}
|
|
115
|
+
var Lineage = class _Lineage {
|
|
116
|
+
byId = /* @__PURE__ */ new Map();
|
|
117
|
+
childIds = /* @__PURE__ */ new Map();
|
|
118
|
+
nextSeq = 0;
|
|
119
|
+
constructor(nodes) {
|
|
120
|
+
if (!nodes) return;
|
|
121
|
+
for (const node of [...nodes].sort((a, b) => a.seq - b.seq)) {
|
|
122
|
+
this.index(node);
|
|
123
|
+
this.nextSeq = Math.max(this.nextSeq, node.seq + 1);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/** Append a node. Derives `id` (via {@link lineageNodeId}), assigns the next
|
|
127
|
+
* `seq`, and derives `generation` as `max(parent.generation) + 1` (root = 0)
|
|
128
|
+
* when omitted. Throws on an unknown parent. Idempotent: re-adding an identical
|
|
129
|
+
* node returns the existing one.
|
|
130
|
+
*
|
|
131
|
+
* Acyclicity is guaranteed by construction, not by a runtime check: every
|
|
132
|
+
* parent must already exist (a child can never point at a not-yet-added node),
|
|
133
|
+
* ids are immutable content hashes (an existing node can never gain new
|
|
134
|
+
* parents), and the store is append-only — so no back-edge can form. (Traversal
|
|
135
|
+
* is still cycle-safe against hand-corrupted deserialized input via visited
|
|
136
|
+
* sets in {@link ancestors}/{@link descendants}.) */
|
|
137
|
+
addNode(input) {
|
|
138
|
+
for (const parentId of input.parentIds) {
|
|
139
|
+
if (!this.byId.has(parentId)) {
|
|
140
|
+
throw new Error(`Lineage.addNode: unknown parent '${parentId}'`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const id = lineageNodeId({
|
|
144
|
+
parentIds: input.parentIds,
|
|
145
|
+
track: input.track,
|
|
146
|
+
surface: input.surface,
|
|
147
|
+
proposer: input.proposer
|
|
148
|
+
});
|
|
149
|
+
const existing = this.byId.get(id);
|
|
150
|
+
if (existing) return existing;
|
|
151
|
+
const generation = input.generation ?? (input.parentIds.length === 0 ? 0 : Math.max(...input.parentIds.map((p) => this.byId.get(p).generation)) + 1);
|
|
152
|
+
const node = {
|
|
153
|
+
id,
|
|
154
|
+
parentIds: [...input.parentIds],
|
|
155
|
+
track: input.track,
|
|
156
|
+
surface: input.surface,
|
|
157
|
+
score: input.score,
|
|
158
|
+
proposer: input.proposer,
|
|
159
|
+
generation,
|
|
160
|
+
seq: this.nextSeq++,
|
|
161
|
+
...input.vision !== void 0 ? { vision: input.vision } : {},
|
|
162
|
+
...input.scoreVector !== void 0 ? { scoreVector: [...input.scoreVector] } : {},
|
|
163
|
+
...input.rationale !== void 0 ? { rationale: input.rationale } : {},
|
|
164
|
+
...input.gate !== void 0 ? { gate: input.gate } : {}
|
|
165
|
+
};
|
|
166
|
+
this.index(node);
|
|
167
|
+
return node;
|
|
168
|
+
}
|
|
169
|
+
/** Collapse 2+ parents into a single node (a merge/"collapse"). A merge is an
|
|
170
|
+
* ordinary multi-parent node; this is a guarded convenience. */
|
|
171
|
+
merge(input) {
|
|
172
|
+
if (input.parentIds.length < 2) {
|
|
173
|
+
throw new Error(`Lineage.merge: a merge needs >= 2 parents, got ${input.parentIds.length}`);
|
|
174
|
+
}
|
|
175
|
+
return this.addNode({ ...input, proposer: input.proposer ?? "merge" });
|
|
176
|
+
}
|
|
177
|
+
get(id) {
|
|
178
|
+
return this.byId.get(id);
|
|
179
|
+
}
|
|
180
|
+
has(id) {
|
|
181
|
+
return this.byId.has(id);
|
|
182
|
+
}
|
|
183
|
+
/** All nodes, in insertion (`seq`) order. */
|
|
184
|
+
all() {
|
|
185
|
+
return [...this.byId.values()].sort(bySeq);
|
|
186
|
+
}
|
|
187
|
+
roots() {
|
|
188
|
+
return this.all().filter((n) => n.parentIds.length === 0);
|
|
189
|
+
}
|
|
190
|
+
parents(id) {
|
|
191
|
+
const node = this.byId.get(id);
|
|
192
|
+
if (!node) return [];
|
|
193
|
+
return node.parentIds.map((p) => this.byId.get(p)).filter((n) => n !== void 0).sort(bySeq);
|
|
194
|
+
}
|
|
195
|
+
children(id) {
|
|
196
|
+
return (this.childIds.get(id) ?? []).map((c) => this.byId.get(c)).sort(bySeq);
|
|
197
|
+
}
|
|
198
|
+
/** Transitive ancestors (excludes `id`). */
|
|
199
|
+
ancestors(id) {
|
|
200
|
+
const out = /* @__PURE__ */ new Set();
|
|
201
|
+
const stack = [...this.byId.get(id)?.parentIds ?? []];
|
|
202
|
+
while (stack.length > 0) {
|
|
203
|
+
const cur = stack.pop();
|
|
204
|
+
if (out.has(cur)) continue;
|
|
205
|
+
out.add(cur);
|
|
206
|
+
const node = this.byId.get(cur);
|
|
207
|
+
if (node) stack.push(...node.parentIds);
|
|
208
|
+
}
|
|
209
|
+
return out;
|
|
210
|
+
}
|
|
211
|
+
/** Transitive descendants (excludes `id`). */
|
|
212
|
+
descendants(id) {
|
|
213
|
+
const out = /* @__PURE__ */ new Set();
|
|
214
|
+
const stack = [...this.childIds.get(id) ?? []];
|
|
215
|
+
while (stack.length > 0) {
|
|
216
|
+
const cur = stack.pop();
|
|
217
|
+
if (out.has(cur)) continue;
|
|
218
|
+
out.add(cur);
|
|
219
|
+
stack.push(...this.childIds.get(cur) ?? []);
|
|
220
|
+
}
|
|
221
|
+
return out;
|
|
222
|
+
}
|
|
223
|
+
/** Nodes with no children (leaf/branch tips). */
|
|
224
|
+
tips() {
|
|
225
|
+
return this.all().filter((n) => (this.childIds.get(n.id) ?? []).length === 0);
|
|
226
|
+
}
|
|
227
|
+
/** Distinct track ids, in first-seen (`seq`) order. */
|
|
228
|
+
tracks() {
|
|
229
|
+
const seen = /* @__PURE__ */ new Set();
|
|
230
|
+
const out = [];
|
|
231
|
+
for (const node of this.all()) {
|
|
232
|
+
if (!seen.has(node.track)) {
|
|
233
|
+
seen.add(node.track);
|
|
234
|
+
out.push(node.track);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
trackNodes(track) {
|
|
240
|
+
return this.all().filter((n) => n.track === track);
|
|
241
|
+
}
|
|
242
|
+
/** The highest-`score` tip of a track (ties broken by lowest `seq`). */
|
|
243
|
+
trackTip(track) {
|
|
244
|
+
return pickBest(this.tips().filter((n) => n.track === track));
|
|
245
|
+
}
|
|
246
|
+
/** The highest-`score` node overall (ties broken by lowest `seq`). */
|
|
247
|
+
best() {
|
|
248
|
+
return pickBest(this.all());
|
|
249
|
+
}
|
|
250
|
+
/** The Pareto-non-dominated set among TIPS. Uses `scoreVector` when every
|
|
251
|
+
* compared tip carries one, else the scalar `score`. A dominates B iff A is
|
|
252
|
+
* >= B on every component and > B on at least one. */
|
|
253
|
+
frontier() {
|
|
254
|
+
const tips = this.tips();
|
|
255
|
+
const useVector = tips.length > 0 && tips.every((n) => n.scoreVector !== void 0);
|
|
256
|
+
const vecOf = (n) => useVector ? n.scoreVector : [n.score];
|
|
257
|
+
return tips.filter((a) => !tips.some((b) => b.id !== a.id && dominates(vecOf(b), vecOf(a)))).sort(bySeq);
|
|
258
|
+
}
|
|
259
|
+
toGraph() {
|
|
260
|
+
const nodes = this.all();
|
|
261
|
+
const edges = [];
|
|
262
|
+
for (const node of nodes) {
|
|
263
|
+
for (const parentId of node.parentIds) {
|
|
264
|
+
edges.push({ from: parentId, to: node.id });
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return { nodes, edges };
|
|
268
|
+
}
|
|
269
|
+
/** One JSON node per line, in `seq` order. */
|
|
270
|
+
toJSONL() {
|
|
271
|
+
return this.all().map((n) => JSON.stringify(n)).join("\n");
|
|
272
|
+
}
|
|
273
|
+
static fromJSONL(text) {
|
|
274
|
+
const nodes = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
|
|
275
|
+
return new _Lineage(nodes);
|
|
276
|
+
}
|
|
277
|
+
index(node) {
|
|
278
|
+
this.byId.set(node.id, node);
|
|
279
|
+
for (const parentId of node.parentIds) {
|
|
280
|
+
const list = this.childIds.get(parentId);
|
|
281
|
+
if (list) {
|
|
282
|
+
if (!list.includes(node.id)) list.push(node.id);
|
|
283
|
+
} else {
|
|
284
|
+
this.childIds.set(parentId, [node.id]);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
function bySeq(a, b) {
|
|
290
|
+
return a.seq - b.seq;
|
|
291
|
+
}
|
|
292
|
+
function pickBest(nodes) {
|
|
293
|
+
let best;
|
|
294
|
+
for (const node of nodes) {
|
|
295
|
+
if (!best || node.score > best.score || node.score === best.score && node.seq < best.seq) {
|
|
296
|
+
best = node;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return best;
|
|
300
|
+
}
|
|
301
|
+
function dominates(a, b) {
|
|
302
|
+
if (a.length !== b.length) return false;
|
|
303
|
+
let strictlyBetter = false;
|
|
304
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
305
|
+
if (a[i] < b[i]) return false;
|
|
306
|
+
if (a[i] > b[i]) strictlyBetter = true;
|
|
307
|
+
}
|
|
308
|
+
return strictlyBetter;
|
|
309
|
+
}
|
|
310
|
+
function fsLineageStore(path) {
|
|
311
|
+
const ensureDir = () => mkdir(dirname(path), { recursive: true });
|
|
312
|
+
return {
|
|
313
|
+
async load() {
|
|
314
|
+
try {
|
|
315
|
+
return Lineage.fromJSONL(await readFile(path, "utf8"));
|
|
316
|
+
} catch (err) {
|
|
317
|
+
if (err.code === "ENOENT") return new Lineage();
|
|
318
|
+
throw err;
|
|
319
|
+
}
|
|
320
|
+
},
|
|
321
|
+
async append(node) {
|
|
322
|
+
await ensureDir();
|
|
323
|
+
await appendFile(path, `${JSON.stringify(node)}
|
|
324
|
+
`, "utf8");
|
|
325
|
+
},
|
|
326
|
+
async save(lineage) {
|
|
327
|
+
await ensureDir();
|
|
328
|
+
const jsonl = lineage.toJSONL();
|
|
329
|
+
await writeFile(path, jsonl.length > 0 ? `${jsonl}
|
|
330
|
+
` : "", "utf8");
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
function memLineageStore() {
|
|
335
|
+
const nodes = [];
|
|
336
|
+
return {
|
|
337
|
+
async load() {
|
|
338
|
+
return new Lineage(nodes);
|
|
339
|
+
},
|
|
340
|
+
async append(node) {
|
|
341
|
+
nodes.push(node);
|
|
342
|
+
},
|
|
343
|
+
async save(lineage) {
|
|
344
|
+
nodes.length = 0;
|
|
345
|
+
nodes.push(...lineage.all());
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function heuristicGovernor(opts = {}) {
|
|
350
|
+
const maxTracks = opts.maxTracks ?? 3;
|
|
351
|
+
const plateauSteps = opts.plateauSteps ?? 2;
|
|
352
|
+
const mergeFrontierAt = opts.mergeFrontierAt ?? 2;
|
|
353
|
+
return {
|
|
354
|
+
decide(ctx) {
|
|
355
|
+
const { lineage } = ctx;
|
|
356
|
+
if (ctx.budgetRemaining <= 0) return { op: "stop" };
|
|
357
|
+
const pruned = new Set(ctx.prunedTracks);
|
|
358
|
+
const liveTracks = lineage.tracks().filter((t) => !pruned.has(t));
|
|
359
|
+
if (liveTracks.length === 0) return { op: "stop" };
|
|
360
|
+
const frontierByTrack = /* @__PURE__ */ new Map();
|
|
361
|
+
for (const tip of lineage.frontier()) {
|
|
362
|
+
if (pruned.has(tip.track)) continue;
|
|
363
|
+
if (!frontierByTrack.has(tip.track)) frontierByTrack.set(tip.track, tip);
|
|
364
|
+
}
|
|
365
|
+
if (frontierByTrack.size >= mergeFrontierAt) {
|
|
366
|
+
const tips = [...frontierByTrack.values()].sort(bySeq);
|
|
367
|
+
const target = pickBest(tips);
|
|
368
|
+
return { op: "merge", parentIds: tips.map((t) => t.id), track: target.track };
|
|
369
|
+
}
|
|
370
|
+
for (const track of liveTracks) {
|
|
371
|
+
if (isPlateaued(lineage.trackNodes(track), plateauSteps)) {
|
|
372
|
+
if (liveTracks.length > 1 || liveTracks.length < maxTracks) {
|
|
373
|
+
return { op: "prune", track };
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
const leader = pickBest(liveTracks.map((t) => lineage.trackTip(t)).filter(Boolean));
|
|
378
|
+
if (leader && liveTracks.length < maxTracks && isPlateaued(lineage.trackNodes(leader.track), plateauSteps)) {
|
|
379
|
+
return {
|
|
380
|
+
op: "branch",
|
|
381
|
+
fromNodeId: leader.id,
|
|
382
|
+
track: `${leader.track}+${lineage.tracks().length}`,
|
|
383
|
+
proposer: "gepa"
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
if (leader) return { op: "extend", track: leader.track };
|
|
387
|
+
return { op: "stop" };
|
|
388
|
+
}
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
function callbackGovernor(decide) {
|
|
392
|
+
return { decide };
|
|
393
|
+
}
|
|
394
|
+
function isPlateaued(trackNodes, window) {
|
|
395
|
+
if (trackNodes.length <= window) return false;
|
|
396
|
+
const ordered = [...trackNodes].sort(bySeq);
|
|
397
|
+
const head = ordered.slice(0, ordered.length - window);
|
|
398
|
+
const tail = ordered.slice(ordered.length - window);
|
|
399
|
+
const bestBefore = Math.max(...head.map((n) => n.score));
|
|
400
|
+
const bestRecent = Math.max(...tail.map((n) => n.score));
|
|
401
|
+
return bestRecent <= bestBefore;
|
|
402
|
+
}
|
|
403
|
+
async function runLineage(opts) {
|
|
404
|
+
const store = opts.store ?? memLineageStore();
|
|
405
|
+
const lineage = await store.load();
|
|
406
|
+
const log = opts.log ?? (() => {
|
|
407
|
+
});
|
|
408
|
+
const pruned = /* @__PURE__ */ new Set();
|
|
409
|
+
const trackProposer = /* @__PURE__ */ new Map();
|
|
410
|
+
const persist = async (node) => {
|
|
411
|
+
await store.append(node);
|
|
412
|
+
};
|
|
413
|
+
for (const seed of opts.seeds) {
|
|
414
|
+
const node = lineage.addNode({
|
|
415
|
+
parentIds: [],
|
|
416
|
+
track: seed.track,
|
|
417
|
+
surface: seed.surface,
|
|
418
|
+
score: seed.score,
|
|
419
|
+
proposer: seed.proposer,
|
|
420
|
+
...seed.vision !== void 0 ? { vision: seed.vision } : {},
|
|
421
|
+
...seed.scoreVector !== void 0 ? { scoreVector: seed.scoreVector } : {}
|
|
422
|
+
});
|
|
423
|
+
trackProposer.set(seed.track, seed.proposer);
|
|
424
|
+
await persist(node);
|
|
425
|
+
}
|
|
426
|
+
let steps = 0;
|
|
427
|
+
while (steps < opts.budget.maxSteps) {
|
|
428
|
+
const op = await opts.governor.decide({
|
|
429
|
+
lineage,
|
|
430
|
+
step: steps,
|
|
431
|
+
budgetRemaining: opts.budget.maxSteps - steps,
|
|
432
|
+
prunedTracks: [...pruned]
|
|
433
|
+
});
|
|
434
|
+
if (op.op === "stop") {
|
|
435
|
+
log("lineage: governor stop", { steps });
|
|
436
|
+
break;
|
|
437
|
+
}
|
|
438
|
+
if (op.op === "prune") {
|
|
439
|
+
pruned.add(op.track);
|
|
440
|
+
log("lineage: prune", { track: op.track });
|
|
441
|
+
steps += 1;
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
if (op.op === "merge") {
|
|
445
|
+
const parents = op.parentIds.map((id) => lineage.get(id)).filter((n) => n !== void 0);
|
|
446
|
+
if (parents.length < 2) {
|
|
447
|
+
log("lineage: merge skipped (fewer than 2 known parents)", { op });
|
|
448
|
+
steps += 1;
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
const result2 = await opts.merge({ parents, track: op.track });
|
|
452
|
+
const node2 = lineage.merge({
|
|
453
|
+
parentIds: parents.map((p) => p.id),
|
|
454
|
+
track: op.track,
|
|
455
|
+
surface: result2.surface,
|
|
456
|
+
score: result2.score,
|
|
457
|
+
...parents[0].vision !== void 0 ? { vision: parents[0].vision } : {},
|
|
458
|
+
...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
|
|
459
|
+
...result2.rationale !== void 0 ? { rationale: result2.rationale } : {}
|
|
460
|
+
});
|
|
461
|
+
await persist(node2);
|
|
462
|
+
steps += 1;
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
if (op.op === "branch") {
|
|
466
|
+
if (pruned.has(op.track)) {
|
|
467
|
+
log("lineage: branch skipped (target track pruned)", { track: op.track });
|
|
468
|
+
steps += 1;
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
const from = lineage.get(op.fromNodeId);
|
|
472
|
+
if (!from) {
|
|
473
|
+
log("lineage: branch skipped (unknown fromNodeId)", { op });
|
|
474
|
+
steps += 1;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
trackProposer.set(op.track, op.proposer);
|
|
478
|
+
const result2 = await opts.step({ track: op.track, proposer: op.proposer, tip: from });
|
|
479
|
+
const node2 = lineage.addNode({
|
|
480
|
+
parentIds: [from.id],
|
|
481
|
+
track: op.track,
|
|
482
|
+
surface: result2.surface,
|
|
483
|
+
score: result2.score,
|
|
484
|
+
proposer: op.proposer,
|
|
485
|
+
...op.vision !== void 0 ? { vision: op.vision } : {},
|
|
486
|
+
...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
|
|
487
|
+
...result2.rationale !== void 0 ? { rationale: result2.rationale } : {},
|
|
488
|
+
...result2.gate !== void 0 ? { gate: result2.gate } : {}
|
|
489
|
+
});
|
|
490
|
+
await persist(node2);
|
|
491
|
+
steps += 1;
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
if (pruned.has(op.track)) {
|
|
495
|
+
log("lineage: extend skipped (track pruned)", { track: op.track });
|
|
496
|
+
steps += 1;
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
const tip = lineage.trackTip(op.track);
|
|
500
|
+
if (!tip) {
|
|
501
|
+
log("lineage: extend skipped (no tip for track)", { track: op.track });
|
|
502
|
+
steps += 1;
|
|
503
|
+
continue;
|
|
504
|
+
}
|
|
505
|
+
const proposer = trackProposer.get(op.track) ?? tip.proposer;
|
|
506
|
+
const result = await opts.step({ track: op.track, proposer, tip });
|
|
507
|
+
const node = lineage.addNode({
|
|
508
|
+
parentIds: [tip.id],
|
|
509
|
+
track: op.track,
|
|
510
|
+
surface: result.surface,
|
|
511
|
+
score: result.score,
|
|
512
|
+
proposer,
|
|
513
|
+
...tip.vision !== void 0 ? { vision: tip.vision } : {},
|
|
514
|
+
...result.scoreVector !== void 0 ? { scoreVector: result.scoreVector } : {},
|
|
515
|
+
...result.rationale !== void 0 ? { rationale: result.rationale } : {},
|
|
516
|
+
...result.gate !== void 0 ? { gate: result.gate } : {}
|
|
517
|
+
});
|
|
518
|
+
await persist(node);
|
|
519
|
+
steps += 1;
|
|
520
|
+
}
|
|
521
|
+
return { lineage, best: lineage.best(), steps };
|
|
522
|
+
}
|
|
523
|
+
|
|
106
524
|
// src/campaign/analyst-surface.ts
|
|
107
525
|
function surfaceToText(surface) {
|
|
108
526
|
if (typeof surface === "string") return surface;
|
|
@@ -166,7 +584,7 @@ function failureModeRecallJudge(opts = {}) {
|
|
|
166
584
|
}
|
|
167
585
|
|
|
168
586
|
// src/campaign/fixtures.ts
|
|
169
|
-
import { createHash } from "crypto";
|
|
587
|
+
import { createHash as createHash2 } from "crypto";
|
|
170
588
|
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
|
|
171
589
|
import { isAbsolute, join, relative, resolve } from "path";
|
|
172
590
|
function discoverEvalFixtures(evalsDir) {
|
|
@@ -307,7 +725,7 @@ function collectFixtureFiles(fixturePath, base = "") {
|
|
|
307
725
|
const bytes = readFileSync(fullPath);
|
|
308
726
|
files.push({
|
|
309
727
|
path: relativePath,
|
|
310
|
-
sha256:
|
|
728
|
+
sha256: createHash2("sha256").update(bytes).digest("hex"),
|
|
311
729
|
bytes: bytes.byteLength
|
|
312
730
|
});
|
|
313
731
|
}
|
|
@@ -320,7 +738,7 @@ function assertInside(root, target, label) {
|
|
|
320
738
|
}
|
|
321
739
|
|
|
322
740
|
// src/campaign/gates/sequential.ts
|
|
323
|
-
import { createHash as
|
|
741
|
+
import { createHash as createHash3 } from "crypto";
|
|
324
742
|
function verifyManifestSync(m) {
|
|
325
743
|
if (m.algo !== void 0 && m.algo !== "sha256-content") {
|
|
326
744
|
throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
|
|
@@ -328,7 +746,7 @@ function verifyManifestSync(m) {
|
|
|
328
746
|
const { contentHash: contentHash2, algo: _algo, ...rest } = m;
|
|
329
747
|
void _algo;
|
|
330
748
|
const bytes = JSON.stringify(canonicalize(rest));
|
|
331
|
-
const hash =
|
|
749
|
+
const hash = createHash3("sha256").update(bytes, "utf8").digest("hex");
|
|
332
750
|
return hash === contentHash2;
|
|
333
751
|
}
|
|
334
752
|
function resolveConfig(opts) {
|
|
@@ -586,7 +1004,7 @@ function sequentialDecide(options = {}) {
|
|
|
586
1004
|
}
|
|
587
1005
|
|
|
588
1006
|
// src/campaign/labeled-store/fs-adapter.ts
|
|
589
|
-
import { createHash as
|
|
1007
|
+
import { createHash as createHash4 } from "crypto";
|
|
590
1008
|
import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
591
1009
|
import { join as join2 } from "path";
|
|
592
1010
|
var LabeledScenarioStoreError = class extends Error {
|
|
@@ -785,7 +1203,7 @@ function matchesFilter(record, args, source) {
|
|
|
785
1203
|
return true;
|
|
786
1204
|
}
|
|
787
1205
|
function sha256(input) {
|
|
788
|
-
return
|
|
1206
|
+
return createHash4("sha256").update(input).digest("hex").slice(0, 16);
|
|
789
1207
|
}
|
|
790
1208
|
function appendLine(path, line) {
|
|
791
1209
|
if (existsSync2(path)) {
|
|
@@ -2013,8 +2431,177 @@ function renderScoreboardMarkdown(rows, opts = {}) {
|
|
|
2013
2431
|
return out.join("\n");
|
|
2014
2432
|
}
|
|
2015
2433
|
|
|
2434
|
+
// src/campaign/presets/run-lineage-loop.ts
|
|
2435
|
+
function toCandidate(p) {
|
|
2436
|
+
return isProposedCandidate(p) ? { surface: p.surface, ...p.rationale ? { rationale: p.rationale } : {} } : { surface: p };
|
|
2437
|
+
}
|
|
2438
|
+
async function runLineageLoop(opts) {
|
|
2439
|
+
const governor = opts.governor ?? heuristicGovernor();
|
|
2440
|
+
const populationSize = opts.populationSize ?? 4;
|
|
2441
|
+
if (populationSize < 1) {
|
|
2442
|
+
throw new Error("runLineageLoop: populationSize must be >= 1");
|
|
2443
|
+
}
|
|
2444
|
+
let proposer = opts.proposer;
|
|
2445
|
+
if (!proposer) {
|
|
2446
|
+
if (!opts.llm || !opts.model) {
|
|
2447
|
+
throw new Error(
|
|
2448
|
+
"runLineageLoop: a proposer is required \u2014 either inject `proposer`, or provide `llm` + `model` for the default gepaProposer."
|
|
2449
|
+
);
|
|
2450
|
+
}
|
|
2451
|
+
proposer = gepaProposer({
|
|
2452
|
+
llm: opts.llm,
|
|
2453
|
+
model: opts.model,
|
|
2454
|
+
target: opts.target ?? "agent surface",
|
|
2455
|
+
// GEPA combine-complementary-lessons is what the `merge` seam relies on.
|
|
2456
|
+
combineParents: true
|
|
2457
|
+
});
|
|
2458
|
+
}
|
|
2459
|
+
const scoringScenarios = opts.holdoutScenarios ?? opts.scenarios;
|
|
2460
|
+
let scoreSurface = opts.scoreSurface;
|
|
2461
|
+
if (!scoreSurface) {
|
|
2462
|
+
const dispatchWithSurface = opts.dispatchWithSurface;
|
|
2463
|
+
const runDir = opts.runDir;
|
|
2464
|
+
if (!dispatchWithSurface || !runDir) {
|
|
2465
|
+
throw new Error(
|
|
2466
|
+
"runLineageLoop: scoring is required \u2014 either inject `scoreSurface`, or provide `dispatchWithSurface` (the agent) + `runDir` for the default runCampaign scorer."
|
|
2467
|
+
);
|
|
2468
|
+
}
|
|
2469
|
+
if (scoringScenarios.length === 0) {
|
|
2470
|
+
throw new Error(
|
|
2471
|
+
"runLineageLoop: the default scorer needs at least one scenario (holdoutScenarios ?? scenarios is empty)."
|
|
2472
|
+
);
|
|
2473
|
+
}
|
|
2474
|
+
scoreSurface = async (surface) => {
|
|
2475
|
+
const campaign = await runCampaign({
|
|
2476
|
+
scenarios: scoringScenarios,
|
|
2477
|
+
dispatch: (scenario, ctx) => dispatchWithSurface(surface, scenario, ctx),
|
|
2478
|
+
dispatchRef: `lineage-loop:${surfaceHash(surface)}`,
|
|
2479
|
+
runDir: `${runDir}/score/${surfaceHash(surface)}`,
|
|
2480
|
+
...opts.judges ? { judges: opts.judges } : {},
|
|
2481
|
+
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
2482
|
+
...opts.reps !== void 0 ? { reps: opts.reps } : {},
|
|
2483
|
+
...opts.storage ? { storage: opts.storage } : {},
|
|
2484
|
+
...opts.tracing ? { tracing: opts.tracing } : {},
|
|
2485
|
+
...opts.expectUsage ? { expectUsage: opts.expectUsage } : {},
|
|
2486
|
+
...opts.maxConcurrency !== void 0 ? { maxConcurrency: opts.maxConcurrency } : {},
|
|
2487
|
+
...opts.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: opts.dispatchTimeoutMs } : {},
|
|
2488
|
+
...opts.now ? { now: opts.now } : {}
|
|
2489
|
+
});
|
|
2490
|
+
const score = campaignMeanComposite(campaign);
|
|
2491
|
+
const byId = new Map(
|
|
2492
|
+
campaignBreakdown(campaign).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
2493
|
+
);
|
|
2494
|
+
const scoreVector = scoringScenarios.map((s) => byId.get(s.id) ?? 0);
|
|
2495
|
+
return { score, scoreVector };
|
|
2496
|
+
};
|
|
2497
|
+
}
|
|
2498
|
+
const objectivesOf = (node) => {
|
|
2499
|
+
const objectives = {};
|
|
2500
|
+
if (node.scoreVector && node.scoreVector.length > 0) {
|
|
2501
|
+
scoringScenarios.forEach((s, i) => {
|
|
2502
|
+
objectives[s.id] = node.scoreVector[i] ?? 0;
|
|
2503
|
+
});
|
|
2504
|
+
if (Object.keys(objectives).length > 0) return objectives;
|
|
2505
|
+
}
|
|
2506
|
+
objectives.composite = node.score;
|
|
2507
|
+
return objectives;
|
|
2508
|
+
};
|
|
2509
|
+
const scoredSeeds = [];
|
|
2510
|
+
for (const seed of opts.seeds) {
|
|
2511
|
+
const measured = await scoreSurface(seed.surface);
|
|
2512
|
+
scoredSeeds.push({
|
|
2513
|
+
surface: seed.surface,
|
|
2514
|
+
track: seed.track,
|
|
2515
|
+
proposer: seed.proposer,
|
|
2516
|
+
score: measured.score,
|
|
2517
|
+
...seed.vision !== void 0 ? { vision: seed.vision } : {},
|
|
2518
|
+
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {}
|
|
2519
|
+
});
|
|
2520
|
+
}
|
|
2521
|
+
const step = async (args) => {
|
|
2522
|
+
const proposed = await proposer.propose({
|
|
2523
|
+
currentSurface: args.tip.surface,
|
|
2524
|
+
history: [],
|
|
2525
|
+
findings: [],
|
|
2526
|
+
populationSize,
|
|
2527
|
+
generation: 0,
|
|
2528
|
+
signal: new AbortController().signal,
|
|
2529
|
+
paretoParents: []
|
|
2530
|
+
});
|
|
2531
|
+
const pool = [
|
|
2532
|
+
{
|
|
2533
|
+
surface: args.tip.surface,
|
|
2534
|
+
score: args.tip.score,
|
|
2535
|
+
...args.tip.scoreVector !== void 0 ? { scoreVector: args.tip.scoreVector } : {},
|
|
2536
|
+
...args.tip.rationale !== void 0 ? { rationale: args.tip.rationale } : {}
|
|
2537
|
+
}
|
|
2538
|
+
];
|
|
2539
|
+
for (const p of proposed) {
|
|
2540
|
+
const { surface, rationale } = toCandidate(p);
|
|
2541
|
+
const measured = await scoreSurface(surface);
|
|
2542
|
+
pool.push({
|
|
2543
|
+
surface,
|
|
2544
|
+
score: measured.score,
|
|
2545
|
+
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
|
|
2546
|
+
...rationale !== void 0 ? { rationale } : {}
|
|
2547
|
+
});
|
|
2548
|
+
}
|
|
2549
|
+
let best = pool[0];
|
|
2550
|
+
for (const entry of pool) {
|
|
2551
|
+
if (entry.score > best.score) best = entry;
|
|
2552
|
+
}
|
|
2553
|
+
return {
|
|
2554
|
+
surface: best.surface,
|
|
2555
|
+
score: best.score,
|
|
2556
|
+
...best.scoreVector !== void 0 ? { scoreVector: best.scoreVector } : {},
|
|
2557
|
+
...best.rationale !== void 0 ? { rationale: best.rationale } : {}
|
|
2558
|
+
};
|
|
2559
|
+
};
|
|
2560
|
+
const merge = async (args) => {
|
|
2561
|
+
const ordered = [...args.parents].sort((a, b) => b.score - a.score);
|
|
2562
|
+
const paretoParents = ordered.map((node) => ({
|
|
2563
|
+
surface: node.surface,
|
|
2564
|
+
surfaceHash: surfaceHash(node.surface),
|
|
2565
|
+
objectives: objectivesOf(node),
|
|
2566
|
+
composite: node.score,
|
|
2567
|
+
generation: node.generation
|
|
2568
|
+
}));
|
|
2569
|
+
const proposed = await proposer.propose({
|
|
2570
|
+
currentSurface: ordered[0].surface,
|
|
2571
|
+
history: [],
|
|
2572
|
+
findings: [],
|
|
2573
|
+
// A merge is a single crossover; combineParents fires because
|
|
2574
|
+
// paretoParents has > 1 string member.
|
|
2575
|
+
populationSize: 1,
|
|
2576
|
+
// generation >= 1 keeps the combine slot semantically a "merge" step.
|
|
2577
|
+
generation: 1,
|
|
2578
|
+
signal: new AbortController().signal,
|
|
2579
|
+
paretoParents
|
|
2580
|
+
});
|
|
2581
|
+
const first = proposed[0];
|
|
2582
|
+
const merged = first ? toCandidate(first) : { surface: ordered[0].surface };
|
|
2583
|
+
const measured = await scoreSurface(merged.surface);
|
|
2584
|
+
return {
|
|
2585
|
+
surface: merged.surface,
|
|
2586
|
+
score: measured.score,
|
|
2587
|
+
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
|
|
2588
|
+
...merged.rationale !== void 0 ? { rationale: merged.rationale } : {}
|
|
2589
|
+
};
|
|
2590
|
+
};
|
|
2591
|
+
const result = await runLineage({
|
|
2592
|
+
seeds: scoredSeeds,
|
|
2593
|
+
step,
|
|
2594
|
+
merge,
|
|
2595
|
+
governor,
|
|
2596
|
+
budget: opts.budget,
|
|
2597
|
+
...opts.store ? { store: opts.store } : {},
|
|
2598
|
+
...opts.log ? { log: opts.log } : {}
|
|
2599
|
+
});
|
|
2600
|
+
return { lineage: result.lineage, best: result.best, steps: result.steps };
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2016
2603
|
// src/campaign/presets/run-profile-matrix.ts
|
|
2017
|
-
import { createHash as
|
|
2604
|
+
import { createHash as createHash5 } from "crypto";
|
|
2018
2605
|
import { join as join3 } from "path";
|
|
2019
2606
|
var ProfileMatrixError = class extends AgentEvalError {
|
|
2020
2607
|
constructor(message) {
|
|
@@ -2025,7 +2612,7 @@ function sanitize(id) {
|
|
|
2025
2612
|
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2026
2613
|
}
|
|
2027
2614
|
function sha(input) {
|
|
2028
|
-
return
|
|
2615
|
+
return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
|
|
2029
2616
|
}
|
|
2030
2617
|
function mean2(xs) {
|
|
2031
2618
|
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
@@ -2665,6 +3252,53 @@ ${report.slice(0, 800)}`,
|
|
|
2665
3252
|
});
|
|
2666
3253
|
}
|
|
2667
3254
|
|
|
3255
|
+
// src/campaign/scenario-selection.ts
|
|
3256
|
+
var DEFAULT_SATURATION_CEILING = 0.999;
|
|
3257
|
+
var VARIANCE_EPSILON = 1e-9;
|
|
3258
|
+
function moments(scores) {
|
|
3259
|
+
const n = scores.length;
|
|
3260
|
+
if (n === 0) return { meanScore: 0, variance: 0 };
|
|
3261
|
+
let sum = 0;
|
|
3262
|
+
for (const s of scores) sum += s;
|
|
3263
|
+
const meanScore = sum / n;
|
|
3264
|
+
let sqDev = 0;
|
|
3265
|
+
for (const s of scores) {
|
|
3266
|
+
const d = s - meanScore;
|
|
3267
|
+
sqDev += d * d;
|
|
3268
|
+
}
|
|
3269
|
+
return { meanScore, variance: sqDev / n };
|
|
3270
|
+
}
|
|
3271
|
+
function compareDiscrimination(a, b) {
|
|
3272
|
+
if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
|
|
3273
|
+
if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
|
|
3274
|
+
return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
|
|
3275
|
+
}
|
|
3276
|
+
function scoreDiscrimination(signals, opts) {
|
|
3277
|
+
const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
|
|
3278
|
+
const scored = signals.map((signal) => {
|
|
3279
|
+
const { meanScore, variance } = moments(signal.scores);
|
|
3280
|
+
const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
|
|
3281
|
+
return {
|
|
3282
|
+
scenarioId: signal.scenarioId,
|
|
3283
|
+
discrimination: variance,
|
|
3284
|
+
meanScore,
|
|
3285
|
+
variance,
|
|
3286
|
+
tied
|
|
3287
|
+
};
|
|
3288
|
+
});
|
|
3289
|
+
return scored.sort(compareDiscrimination);
|
|
3290
|
+
}
|
|
3291
|
+
function selectDiscriminative(signals, k, opts) {
|
|
3292
|
+
if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
|
|
3293
|
+
const ranked = scoreDiscrimination(signals, opts);
|
|
3294
|
+
if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
|
|
3295
|
+
const nonTied = ranked.filter((s) => !s.tied);
|
|
3296
|
+
if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
|
|
3297
|
+
const tied = ranked.filter((s) => s.tied);
|
|
3298
|
+
const fill = tied.slice(0, k - nonTied.length);
|
|
3299
|
+
return [...nonTied, ...fill].map((s) => s.scenarioId);
|
|
3300
|
+
}
|
|
3301
|
+
|
|
2668
3302
|
// src/campaign/worktree/index.ts
|
|
2669
3303
|
import { execFileSync } from "child_process";
|
|
2670
3304
|
import { existsSync as existsSync3 } from "fs";
|
|
@@ -2727,6 +3361,7 @@ function resolveWorktreePath(surface, worktreeDir) {
|
|
|
2727
3361
|
export {
|
|
2728
3362
|
FsLabeledScenarioStore,
|
|
2729
3363
|
LabeledScenarioStoreError,
|
|
3364
|
+
Lineage,
|
|
2730
3365
|
ProfileMatrixError,
|
|
2731
3366
|
SkillPatchParseError,
|
|
2732
3367
|
WorktreeAdapterError,
|
|
@@ -2735,6 +3370,7 @@ export {
|
|
|
2735
3370
|
buildAnalystSurfaceDispatch,
|
|
2736
3371
|
buildEvidenceVector,
|
|
2737
3372
|
buildLoopProvenanceRecord,
|
|
3373
|
+
callbackGovernor,
|
|
2738
3374
|
campaignBreakdown,
|
|
2739
3375
|
campaignMeanComposite,
|
|
2740
3376
|
compareProposers,
|
|
@@ -2753,6 +3389,7 @@ export {
|
|
|
2753
3389
|
fapoEscalationEntry,
|
|
2754
3390
|
fapoProposer,
|
|
2755
3391
|
fsCampaignStorage,
|
|
3392
|
+
fsLineageStore,
|
|
2756
3393
|
gepaParetoEntry,
|
|
2757
3394
|
gepaProposer,
|
|
2758
3395
|
gepaReflectionEntry,
|
|
@@ -2760,14 +3397,17 @@ export {
|
|
|
2760
3397
|
haloProposer,
|
|
2761
3398
|
heldOutGate,
|
|
2762
3399
|
heldoutSignificance,
|
|
3400
|
+
heuristicGovernor,
|
|
2763
3401
|
inMemoryCampaignStorage,
|
|
2764
3402
|
isProposedCandidate,
|
|
2765
3403
|
labelTrustRank,
|
|
3404
|
+
lineageNodeId,
|
|
2766
3405
|
llmJudge,
|
|
2767
3406
|
loadEvalFixture,
|
|
2768
3407
|
loadEvalFixtureScenarios,
|
|
2769
3408
|
loopProvenanceSpans,
|
|
2770
3409
|
makePlaybackDispatch,
|
|
3410
|
+
memLineageStore,
|
|
2771
3411
|
memoryCurationProposer,
|
|
2772
3412
|
openAutoPr,
|
|
2773
3413
|
pairHoldout,
|
|
@@ -2788,11 +3428,15 @@ export {
|
|
|
2788
3428
|
runCampaign,
|
|
2789
3429
|
runEval,
|
|
2790
3430
|
runImprovementLoop,
|
|
3431
|
+
runLineage,
|
|
3432
|
+
runLineageLoop,
|
|
2791
3433
|
runOptimization,
|
|
2792
3434
|
runProfileMatrix,
|
|
2793
3435
|
runSkillOpt,
|
|
3436
|
+
scoreDiscrimination,
|
|
2794
3437
|
scoreUserStory,
|
|
2795
3438
|
scoreboardSummary,
|
|
3439
|
+
selectDiscriminative,
|
|
2796
3440
|
sequentialDecide,
|
|
2797
3441
|
sequentialPairedGate,
|
|
2798
3442
|
skillOptEntry,
|