@tangle-network/agent-eval 0.106.2 → 0.107.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/http.d.ts +1 -1
- package/dist/adapters/langchain.d.ts +1 -1
- package/dist/adapters/otel.d.ts +1 -1
- package/dist/campaign/index.d.ts +486 -8
- package/dist/campaign/index.js +751 -19
- package/dist/campaign/index.js.map +1 -1
- package/dist/{chunk-2HLM4SJJ.js → chunk-6PF6LXRY.js} +2 -2
- package/dist/{chunk-3E5KUXYZ.js → chunk-G4DLZAV5.js} +21 -1
- package/dist/chunk-G4DLZAV5.js.map +1 -0
- package/dist/{chunk-J22CKVXN.js → chunk-V75NN2ZR.js} +2 -2
- package/dist/contract/index.d.ts +6 -6
- package/dist/contract/index.js +2 -2
- package/dist/{gepa-DQ3ruj18.d.ts → gepa-BmqTrdtg.d.ts} +9 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{pre-registration-BkkTmtQG.d.ts → pre-registration-x3f0VpxT.d.ts} +1 -1
- package/dist/{provenance-Bsyjc67Z.d.ts → provenance-CIjRcI6m.d.ts} +2 -2
- package/dist/rl.d.ts +1 -1
- package/dist/{types-Ctf7XIAL.d.ts → types-CNZ0tHET.d.ts} +10 -0
- package/package.json +1 -1
- package/dist/chunk-3E5KUXYZ.js.map +0 -1
- /package/dist/{chunk-2HLM4SJJ.js.map → chunk-6PF6LXRY.js.map} +0 -0
- /package/dist/{chunk-J22CKVXN.js.map → chunk-V75NN2ZR.js.map} +0 -0
package/dist/campaign/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
paretoSignificanceGate,
|
|
8
8
|
powerPreflight,
|
|
9
9
|
runEval
|
|
10
|
-
} from "../chunk-
|
|
10
|
+
} from "../chunk-V75NN2ZR.js";
|
|
11
11
|
import {
|
|
12
12
|
HARNESS_NATIVE_MODEL,
|
|
13
13
|
agentProfileHash,
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
harnessAxisOf,
|
|
18
18
|
llmJudge,
|
|
19
19
|
verifyCompletion
|
|
20
|
-
} from "../chunk-
|
|
20
|
+
} from "../chunk-6PF6LXRY.js";
|
|
21
21
|
import {
|
|
22
22
|
buildLoopProvenanceRecord,
|
|
23
23
|
campaignBreakdown,
|
|
@@ -43,7 +43,7 @@ import {
|
|
|
43
43
|
runOptimization,
|
|
44
44
|
surfaceContentHash,
|
|
45
45
|
surfaceHash
|
|
46
|
-
} from "../chunk-
|
|
46
|
+
} from "../chunk-G4DLZAV5.js";
|
|
47
47
|
import {
|
|
48
48
|
assertRealBackend,
|
|
49
49
|
contentHash,
|
|
@@ -103,6 +103,424 @@ import {
|
|
|
103
103
|
} from "../chunk-ONWEPEDO.js";
|
|
104
104
|
import "../chunk-PZ5AY32C.js";
|
|
105
105
|
|
|
106
|
+
// src/campaign/lineage.ts
|
|
107
|
+
import { createHash } from "crypto";
|
|
108
|
+
import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
|
|
109
|
+
import { dirname } from "path";
|
|
110
|
+
function lineageNodeId(input) {
|
|
111
|
+
const parents = [...input.parentIds].sort().join(",");
|
|
112
|
+
const payload = `${parents}|${input.track}|${input.surface}|${input.proposer}`;
|
|
113
|
+
return createHash("sha256").update(payload).digest("hex").slice(0, 16);
|
|
114
|
+
}
|
|
115
|
+
var Lineage = class _Lineage {
|
|
116
|
+
byId = /* @__PURE__ */ new Map();
|
|
117
|
+
childIds = /* @__PURE__ */ new Map();
|
|
118
|
+
nextSeq = 0;
|
|
119
|
+
constructor(nodes) {
|
|
120
|
+
if (!nodes) return;
|
|
121
|
+
for (const node of [...nodes].sort((a, b) => a.seq - b.seq)) {
|
|
122
|
+
this.index(node);
|
|
123
|
+
this.nextSeq = Math.max(this.nextSeq, node.seq + 1);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/** Append a node. Derives `id` (via {@link lineageNodeId}), assigns the next
|
|
127
|
+
* `seq`, and derives `generation` as `max(parent.generation) + 1` (root = 0)
|
|
128
|
+
* when omitted. Throws on an unknown parent. Idempotent: re-adding an identical
|
|
129
|
+
* node returns the existing one.
|
|
130
|
+
*
|
|
131
|
+
* Acyclicity is guaranteed by construction, not by a runtime check: every
|
|
132
|
+
* parent must already exist (a child can never point at a not-yet-added node),
|
|
133
|
+
* ids are immutable content hashes (an existing node can never gain new
|
|
134
|
+
* parents), and the store is append-only — so no back-edge can form. (Traversal
|
|
135
|
+
* is still cycle-safe against hand-corrupted deserialized input via visited
|
|
136
|
+
* sets in {@link ancestors}/{@link descendants}.) */
|
|
137
|
+
addNode(input) {
|
|
138
|
+
for (const parentId of input.parentIds) {
|
|
139
|
+
if (!this.byId.has(parentId)) {
|
|
140
|
+
throw new Error(`Lineage.addNode: unknown parent '${parentId}'`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const id = lineageNodeId({
|
|
144
|
+
parentIds: input.parentIds,
|
|
145
|
+
track: input.track,
|
|
146
|
+
surface: input.surface,
|
|
147
|
+
proposer: input.proposer
|
|
148
|
+
});
|
|
149
|
+
const existing = this.byId.get(id);
|
|
150
|
+
if (existing) return existing;
|
|
151
|
+
const generation = input.generation ?? (input.parentIds.length === 0 ? 0 : Math.max(...input.parentIds.map((p) => this.byId.get(p).generation)) + 1);
|
|
152
|
+
const node = {
|
|
153
|
+
id,
|
|
154
|
+
parentIds: [...input.parentIds],
|
|
155
|
+
track: input.track,
|
|
156
|
+
surface: input.surface,
|
|
157
|
+
score: input.score,
|
|
158
|
+
proposer: input.proposer,
|
|
159
|
+
generation,
|
|
160
|
+
seq: this.nextSeq++,
|
|
161
|
+
...input.vision !== void 0 ? { vision: input.vision } : {},
|
|
162
|
+
...input.scoreVector !== void 0 ? { scoreVector: [...input.scoreVector] } : {},
|
|
163
|
+
...input.rationale !== void 0 ? { rationale: input.rationale } : {},
|
|
164
|
+
...input.gate !== void 0 ? { gate: input.gate } : {}
|
|
165
|
+
};
|
|
166
|
+
this.index(node);
|
|
167
|
+
return node;
|
|
168
|
+
}
|
|
169
|
+
/** Collapse 2+ parents into a single node (a merge/"collapse"). A merge is an
|
|
170
|
+
* ordinary multi-parent node; this is a guarded convenience. */
|
|
171
|
+
merge(input) {
|
|
172
|
+
if (input.parentIds.length < 2) {
|
|
173
|
+
throw new Error(`Lineage.merge: a merge needs >= 2 parents, got ${input.parentIds.length}`);
|
|
174
|
+
}
|
|
175
|
+
return this.addNode({ ...input, proposer: input.proposer ?? "merge" });
|
|
176
|
+
}
|
|
177
|
+
get(id) {
|
|
178
|
+
return this.byId.get(id);
|
|
179
|
+
}
|
|
180
|
+
has(id) {
|
|
181
|
+
return this.byId.has(id);
|
|
182
|
+
}
|
|
183
|
+
/** All nodes, in insertion (`seq`) order. */
|
|
184
|
+
all() {
|
|
185
|
+
return [...this.byId.values()].sort(bySeq);
|
|
186
|
+
}
|
|
187
|
+
roots() {
|
|
188
|
+
return this.all().filter((n) => n.parentIds.length === 0);
|
|
189
|
+
}
|
|
190
|
+
parents(id) {
|
|
191
|
+
const node = this.byId.get(id);
|
|
192
|
+
if (!node) return [];
|
|
193
|
+
return node.parentIds.map((p) => this.byId.get(p)).filter((n) => n !== void 0).sort(bySeq);
|
|
194
|
+
}
|
|
195
|
+
children(id) {
|
|
196
|
+
return (this.childIds.get(id) ?? []).map((c) => this.byId.get(c)).sort(bySeq);
|
|
197
|
+
}
|
|
198
|
+
/** Transitive ancestors (excludes `id`). */
|
|
199
|
+
ancestors(id) {
|
|
200
|
+
const out = /* @__PURE__ */ new Set();
|
|
201
|
+
const stack = [...this.byId.get(id)?.parentIds ?? []];
|
|
202
|
+
while (stack.length > 0) {
|
|
203
|
+
const cur = stack.pop();
|
|
204
|
+
if (out.has(cur)) continue;
|
|
205
|
+
out.add(cur);
|
|
206
|
+
const node = this.byId.get(cur);
|
|
207
|
+
if (node) stack.push(...node.parentIds);
|
|
208
|
+
}
|
|
209
|
+
return out;
|
|
210
|
+
}
|
|
211
|
+
/** Transitive descendants (excludes `id`). */
|
|
212
|
+
descendants(id) {
|
|
213
|
+
const out = /* @__PURE__ */ new Set();
|
|
214
|
+
const stack = [...this.childIds.get(id) ?? []];
|
|
215
|
+
while (stack.length > 0) {
|
|
216
|
+
const cur = stack.pop();
|
|
217
|
+
if (out.has(cur)) continue;
|
|
218
|
+
out.add(cur);
|
|
219
|
+
stack.push(...this.childIds.get(cur) ?? []);
|
|
220
|
+
}
|
|
221
|
+
return out;
|
|
222
|
+
}
|
|
223
|
+
/** Nodes with no children (leaf/branch tips). */
|
|
224
|
+
tips() {
|
|
225
|
+
return this.all().filter((n) => (this.childIds.get(n.id) ?? []).length === 0);
|
|
226
|
+
}
|
|
227
|
+
/** Distinct track ids, in first-seen (`seq`) order. */
|
|
228
|
+
tracks() {
|
|
229
|
+
const seen = /* @__PURE__ */ new Set();
|
|
230
|
+
const out = [];
|
|
231
|
+
for (const node of this.all()) {
|
|
232
|
+
if (!seen.has(node.track)) {
|
|
233
|
+
seen.add(node.track);
|
|
234
|
+
out.push(node.track);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
trackNodes(track) {
|
|
240
|
+
return this.all().filter((n) => n.track === track);
|
|
241
|
+
}
|
|
242
|
+
/** The highest-`score` tip of a track (ties broken by lowest `seq`). */
|
|
243
|
+
trackTip(track) {
|
|
244
|
+
return pickBest(this.tips().filter((n) => n.track === track));
|
|
245
|
+
}
|
|
246
|
+
/** The highest-`score` node overall (ties broken by lowest `seq`). */
|
|
247
|
+
best() {
|
|
248
|
+
return pickBest(this.all());
|
|
249
|
+
}
|
|
250
|
+
/** The Pareto-non-dominated set among TIPS. Uses `scoreVector` when every
|
|
251
|
+
* compared tip carries one, else the scalar `score`. A dominates B iff A is
|
|
252
|
+
* >= B on every component and > B on at least one. */
|
|
253
|
+
frontier() {
|
|
254
|
+
const tips = this.tips();
|
|
255
|
+
const useVector = tips.length > 0 && tips.every((n) => n.scoreVector !== void 0);
|
|
256
|
+
const vecOf = (n) => useVector ? n.scoreVector : [n.score];
|
|
257
|
+
return tips.filter((a) => !tips.some((b) => b.id !== a.id && dominates(vecOf(b), vecOf(a)))).sort(bySeq);
|
|
258
|
+
}
|
|
259
|
+
toGraph() {
|
|
260
|
+
const nodes = this.all();
|
|
261
|
+
const edges = [];
|
|
262
|
+
for (const node of nodes) {
|
|
263
|
+
for (const parentId of node.parentIds) {
|
|
264
|
+
edges.push({ from: parentId, to: node.id });
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return { nodes, edges };
|
|
268
|
+
}
|
|
269
|
+
/** One JSON node per line, in `seq` order. */
|
|
270
|
+
toJSONL() {
|
|
271
|
+
return this.all().map((n) => JSON.stringify(n)).join("\n");
|
|
272
|
+
}
|
|
273
|
+
static fromJSONL(text) {
|
|
274
|
+
const nodes = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
|
|
275
|
+
return new _Lineage(nodes);
|
|
276
|
+
}
|
|
277
|
+
index(node) {
|
|
278
|
+
this.byId.set(node.id, node);
|
|
279
|
+
for (const parentId of node.parentIds) {
|
|
280
|
+
const list = this.childIds.get(parentId);
|
|
281
|
+
if (list) {
|
|
282
|
+
if (!list.includes(node.id)) list.push(node.id);
|
|
283
|
+
} else {
|
|
284
|
+
this.childIds.set(parentId, [node.id]);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
function bySeq(a, b) {
|
|
290
|
+
return a.seq - b.seq;
|
|
291
|
+
}
|
|
292
|
+
function pickBest(nodes) {
|
|
293
|
+
let best;
|
|
294
|
+
for (const node of nodes) {
|
|
295
|
+
if (!best || node.score > best.score || node.score === best.score && node.seq < best.seq) {
|
|
296
|
+
best = node;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return best;
|
|
300
|
+
}
|
|
301
|
+
function dominates(a, b) {
|
|
302
|
+
if (a.length !== b.length) return false;
|
|
303
|
+
let strictlyBetter = false;
|
|
304
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
305
|
+
if (a[i] < b[i]) return false;
|
|
306
|
+
if (a[i] > b[i]) strictlyBetter = true;
|
|
307
|
+
}
|
|
308
|
+
return strictlyBetter;
|
|
309
|
+
}
|
|
310
|
+
function fsLineageStore(path) {
|
|
311
|
+
const ensureDir = () => mkdir(dirname(path), { recursive: true });
|
|
312
|
+
return {
|
|
313
|
+
async load() {
|
|
314
|
+
try {
|
|
315
|
+
return Lineage.fromJSONL(await readFile(path, "utf8"));
|
|
316
|
+
} catch (err) {
|
|
317
|
+
if (err.code === "ENOENT") return new Lineage();
|
|
318
|
+
throw err;
|
|
319
|
+
}
|
|
320
|
+
},
|
|
321
|
+
async append(node) {
|
|
322
|
+
await ensureDir();
|
|
323
|
+
await appendFile(path, `${JSON.stringify(node)}
|
|
324
|
+
`, "utf8");
|
|
325
|
+
},
|
|
326
|
+
async save(lineage) {
|
|
327
|
+
await ensureDir();
|
|
328
|
+
const jsonl = lineage.toJSONL();
|
|
329
|
+
await writeFile(path, jsonl.length > 0 ? `${jsonl}
|
|
330
|
+
` : "", "utf8");
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
function memLineageStore() {
|
|
335
|
+
const nodes = [];
|
|
336
|
+
return {
|
|
337
|
+
async load() {
|
|
338
|
+
return new Lineage(nodes);
|
|
339
|
+
},
|
|
340
|
+
async append(node) {
|
|
341
|
+
nodes.push(node);
|
|
342
|
+
},
|
|
343
|
+
async save(lineage) {
|
|
344
|
+
nodes.length = 0;
|
|
345
|
+
nodes.push(...lineage.all());
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function heuristicGovernor(opts = {}) {
|
|
350
|
+
const maxTracks = opts.maxTracks ?? 3;
|
|
351
|
+
const plateauSteps = opts.plateauSteps ?? 2;
|
|
352
|
+
const mergeFrontierAt = opts.mergeFrontierAt ?? 2;
|
|
353
|
+
return {
|
|
354
|
+
decide(ctx) {
|
|
355
|
+
const { lineage } = ctx;
|
|
356
|
+
if (ctx.budgetRemaining <= 0) return { op: "stop" };
|
|
357
|
+
const pruned = new Set(ctx.prunedTracks);
|
|
358
|
+
const liveTracks = lineage.tracks().filter((t) => !pruned.has(t));
|
|
359
|
+
if (liveTracks.length === 0) return { op: "stop" };
|
|
360
|
+
const frontierByTrack = /* @__PURE__ */ new Map();
|
|
361
|
+
for (const tip of lineage.frontier()) {
|
|
362
|
+
if (pruned.has(tip.track)) continue;
|
|
363
|
+
if (!frontierByTrack.has(tip.track)) frontierByTrack.set(tip.track, tip);
|
|
364
|
+
}
|
|
365
|
+
if (frontierByTrack.size >= mergeFrontierAt) {
|
|
366
|
+
const tips = [...frontierByTrack.values()].sort(bySeq);
|
|
367
|
+
const target = pickBest(tips);
|
|
368
|
+
return { op: "merge", parentIds: tips.map((t) => t.id), track: target.track };
|
|
369
|
+
}
|
|
370
|
+
for (const track of liveTracks) {
|
|
371
|
+
if (isPlateaued(lineage.trackNodes(track), plateauSteps)) {
|
|
372
|
+
if (liveTracks.length > 1 || liveTracks.length < maxTracks) {
|
|
373
|
+
return { op: "prune", track };
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
const leader = pickBest(liveTracks.map((t) => lineage.trackTip(t)).filter(Boolean));
|
|
378
|
+
if (leader && liveTracks.length < maxTracks && isPlateaued(lineage.trackNodes(leader.track), plateauSteps)) {
|
|
379
|
+
return {
|
|
380
|
+
op: "branch",
|
|
381
|
+
fromNodeId: leader.id,
|
|
382
|
+
track: `${leader.track}+${lineage.tracks().length}`,
|
|
383
|
+
proposer: "gepa"
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
if (leader) return { op: "extend", track: leader.track };
|
|
387
|
+
return { op: "stop" };
|
|
388
|
+
}
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
function callbackGovernor(decide) {
|
|
392
|
+
return { decide };
|
|
393
|
+
}
|
|
394
|
+
function isPlateaued(trackNodes, window) {
|
|
395
|
+
if (trackNodes.length <= window) return false;
|
|
396
|
+
const ordered = [...trackNodes].sort(bySeq);
|
|
397
|
+
const head = ordered.slice(0, ordered.length - window);
|
|
398
|
+
const tail = ordered.slice(ordered.length - window);
|
|
399
|
+
const bestBefore = Math.max(...head.map((n) => n.score));
|
|
400
|
+
const bestRecent = Math.max(...tail.map((n) => n.score));
|
|
401
|
+
return bestRecent <= bestBefore;
|
|
402
|
+
}
|
|
403
|
+
async function runLineage(opts) {
|
|
404
|
+
const store = opts.store ?? memLineageStore();
|
|
405
|
+
const lineage = await store.load();
|
|
406
|
+
const log = opts.log ?? (() => {
|
|
407
|
+
});
|
|
408
|
+
const pruned = /* @__PURE__ */ new Set();
|
|
409
|
+
const trackProposer = /* @__PURE__ */ new Map();
|
|
410
|
+
const persist = async (node) => {
|
|
411
|
+
await store.append(node);
|
|
412
|
+
};
|
|
413
|
+
for (const seed of opts.seeds) {
|
|
414
|
+
const node = lineage.addNode({
|
|
415
|
+
parentIds: [],
|
|
416
|
+
track: seed.track,
|
|
417
|
+
surface: seed.surface,
|
|
418
|
+
score: seed.score,
|
|
419
|
+
proposer: seed.proposer,
|
|
420
|
+
...seed.vision !== void 0 ? { vision: seed.vision } : {},
|
|
421
|
+
...seed.scoreVector !== void 0 ? { scoreVector: seed.scoreVector } : {}
|
|
422
|
+
});
|
|
423
|
+
trackProposer.set(seed.track, seed.proposer);
|
|
424
|
+
await persist(node);
|
|
425
|
+
}
|
|
426
|
+
let steps = 0;
|
|
427
|
+
while (steps < opts.budget.maxSteps) {
|
|
428
|
+
const op = await opts.governor.decide({
|
|
429
|
+
lineage,
|
|
430
|
+
step: steps,
|
|
431
|
+
budgetRemaining: opts.budget.maxSteps - steps,
|
|
432
|
+
prunedTracks: [...pruned]
|
|
433
|
+
});
|
|
434
|
+
if (op.op === "stop") {
|
|
435
|
+
log("lineage: governor stop", { steps });
|
|
436
|
+
break;
|
|
437
|
+
}
|
|
438
|
+
if (op.op === "prune") {
|
|
439
|
+
pruned.add(op.track);
|
|
440
|
+
log("lineage: prune", { track: op.track });
|
|
441
|
+
steps += 1;
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
if (op.op === "merge") {
|
|
445
|
+
const parents = op.parentIds.map((id) => lineage.get(id)).filter((n) => n !== void 0);
|
|
446
|
+
if (parents.length < 2) {
|
|
447
|
+
log("lineage: merge skipped (fewer than 2 known parents)", { op });
|
|
448
|
+
steps += 1;
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
const result2 = await opts.merge({ parents, track: op.track });
|
|
452
|
+
const node2 = lineage.merge({
|
|
453
|
+
parentIds: parents.map((p) => p.id),
|
|
454
|
+
track: op.track,
|
|
455
|
+
surface: result2.surface,
|
|
456
|
+
score: result2.score,
|
|
457
|
+
...parents[0].vision !== void 0 ? { vision: parents[0].vision } : {},
|
|
458
|
+
...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
|
|
459
|
+
...result2.rationale !== void 0 ? { rationale: result2.rationale } : {}
|
|
460
|
+
});
|
|
461
|
+
await persist(node2);
|
|
462
|
+
steps += 1;
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
if (op.op === "branch") {
|
|
466
|
+
if (pruned.has(op.track)) {
|
|
467
|
+
log("lineage: branch skipped (target track pruned)", { track: op.track });
|
|
468
|
+
steps += 1;
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
const from = lineage.get(op.fromNodeId);
|
|
472
|
+
if (!from) {
|
|
473
|
+
log("lineage: branch skipped (unknown fromNodeId)", { op });
|
|
474
|
+
steps += 1;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
trackProposer.set(op.track, op.proposer);
|
|
478
|
+
const result2 = await opts.step({ track: op.track, proposer: op.proposer, tip: from });
|
|
479
|
+
const node2 = lineage.addNode({
|
|
480
|
+
parentIds: [from.id],
|
|
481
|
+
track: op.track,
|
|
482
|
+
surface: result2.surface,
|
|
483
|
+
score: result2.score,
|
|
484
|
+
proposer: op.proposer,
|
|
485
|
+
...op.vision !== void 0 ? { vision: op.vision } : {},
|
|
486
|
+
...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
|
|
487
|
+
...result2.rationale !== void 0 ? { rationale: result2.rationale } : {},
|
|
488
|
+
...result2.gate !== void 0 ? { gate: result2.gate } : {}
|
|
489
|
+
});
|
|
490
|
+
await persist(node2);
|
|
491
|
+
steps += 1;
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
if (pruned.has(op.track)) {
|
|
495
|
+
log("lineage: extend skipped (track pruned)", { track: op.track });
|
|
496
|
+
steps += 1;
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
const tip = lineage.trackTip(op.track);
|
|
500
|
+
if (!tip) {
|
|
501
|
+
log("lineage: extend skipped (no tip for track)", { track: op.track });
|
|
502
|
+
steps += 1;
|
|
503
|
+
continue;
|
|
504
|
+
}
|
|
505
|
+
const proposer = trackProposer.get(op.track) ?? tip.proposer;
|
|
506
|
+
const result = await opts.step({ track: op.track, proposer, tip });
|
|
507
|
+
const node = lineage.addNode({
|
|
508
|
+
parentIds: [tip.id],
|
|
509
|
+
track: op.track,
|
|
510
|
+
surface: result.surface,
|
|
511
|
+
score: result.score,
|
|
512
|
+
proposer,
|
|
513
|
+
...tip.vision !== void 0 ? { vision: tip.vision } : {},
|
|
514
|
+
...result.scoreVector !== void 0 ? { scoreVector: result.scoreVector } : {},
|
|
515
|
+
...result.rationale !== void 0 ? { rationale: result.rationale } : {},
|
|
516
|
+
...result.gate !== void 0 ? { gate: result.gate } : {}
|
|
517
|
+
});
|
|
518
|
+
await persist(node);
|
|
519
|
+
steps += 1;
|
|
520
|
+
}
|
|
521
|
+
return { lineage, best: lineage.best(), steps };
|
|
522
|
+
}
|
|
523
|
+
|
|
106
524
|
// src/campaign/analyst-surface.ts
|
|
107
525
|
function surfaceToText(surface) {
|
|
108
526
|
if (typeof surface === "string") return surface;
|
|
@@ -166,7 +584,7 @@ function failureModeRecallJudge(opts = {}) {
|
|
|
166
584
|
}
|
|
167
585
|
|
|
168
586
|
// src/campaign/fixtures.ts
|
|
169
|
-
import { createHash } from "crypto";
|
|
587
|
+
import { createHash as createHash2 } from "crypto";
|
|
170
588
|
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
|
|
171
589
|
import { isAbsolute, join, relative, resolve } from "path";
|
|
172
590
|
function discoverEvalFixtures(evalsDir) {
|
|
@@ -307,7 +725,7 @@ function collectFixtureFiles(fixturePath, base = "") {
|
|
|
307
725
|
const bytes = readFileSync(fullPath);
|
|
308
726
|
files.push({
|
|
309
727
|
path: relativePath,
|
|
310
|
-
sha256:
|
|
728
|
+
sha256: createHash2("sha256").update(bytes).digest("hex"),
|
|
311
729
|
bytes: bytes.byteLength
|
|
312
730
|
});
|
|
313
731
|
}
|
|
@@ -319,8 +737,88 @@ function assertInside(root, target, label) {
|
|
|
319
737
|
throw new Error(`loadEvalFixture: fixture path escapes evalsDir: ${label}`);
|
|
320
738
|
}
|
|
321
739
|
|
|
740
|
+
// src/campaign/gates/neutralization-gate.ts
|
|
741
|
+
function mean(xs) {
|
|
742
|
+
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
743
|
+
}
|
|
744
|
+
function pairedLift(arm, baseline, scenarioIds) {
|
|
745
|
+
const paired = pairHoldout(arm, baseline, scenarioIds, (s) => s.composite);
|
|
746
|
+
const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0));
|
|
747
|
+
return { lift: mean(deltas), n: deltas.length };
|
|
748
|
+
}
|
|
749
|
+
function neutralizationGate(options) {
|
|
750
|
+
const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5;
|
|
751
|
+
return {
|
|
752
|
+
name: "neutralizationGate",
|
|
753
|
+
async decide(ctx) {
|
|
754
|
+
if (!ctx.baselineJudgeScores) {
|
|
755
|
+
throw new Error(
|
|
756
|
+
"neutralizationGate: ctx.baselineJudgeScores is required \u2014 the placebo control measures lift OVER baseline."
|
|
757
|
+
);
|
|
758
|
+
}
|
|
759
|
+
if (!ctx.neutralizedJudgeScores) {
|
|
760
|
+
throw new Error(
|
|
761
|
+
"neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied \u2014 composing this gate without that wiring would pass an unproven candidate."
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
const scenarioIds = new Set(options.scenarios.map((s) => s.id));
|
|
765
|
+
const cand = pairedLift(
|
|
766
|
+
ctx.judgeScores,
|
|
767
|
+
ctx.baselineJudgeScores,
|
|
768
|
+
scenarioIds
|
|
769
|
+
);
|
|
770
|
+
const neut = pairedLift(
|
|
771
|
+
ctx.neutralizedJudgeScores,
|
|
772
|
+
ctx.baselineJudgeScores,
|
|
773
|
+
scenarioIds
|
|
774
|
+
);
|
|
775
|
+
if (cand.lift <= 0) {
|
|
776
|
+
return {
|
|
777
|
+
decision: "hold",
|
|
778
|
+
reasons: [
|
|
779
|
+
`neutralization: candidate held-out lift ${cand.lift.toFixed(3)} \u2264 0 \u2014 no positive lift to attribute to content`
|
|
780
|
+
],
|
|
781
|
+
contributingGates: [
|
|
782
|
+
{
|
|
783
|
+
name: "neutralizationGate",
|
|
784
|
+
passed: false,
|
|
785
|
+
detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
|
|
786
|
+
}
|
|
787
|
+
],
|
|
788
|
+
delta: cand.lift
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
const decorativeFraction = neut.lift / cand.lift;
|
|
792
|
+
const passed = decorativeFraction < maxDecorativeFraction;
|
|
793
|
+
const pct = (decorativeFraction * 100).toFixed(0);
|
|
794
|
+
return {
|
|
795
|
+
decision: passed ? "ship" : "hold",
|
|
796
|
+
reasons: passed ? [
|
|
797
|
+
`neutralization: content is causal \u2014 blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
|
|
798
|
+
] : [
|
|
799
|
+
`neutralization: lift is DECORATIVE \u2014 blanking the content (footprint-matched) reproduces ${pct}% of the lift (\u2265 ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
|
|
800
|
+
],
|
|
801
|
+
contributingGates: [
|
|
802
|
+
{
|
|
803
|
+
name: "neutralizationGate",
|
|
804
|
+
passed,
|
|
805
|
+
detail: {
|
|
806
|
+
candidateLift: cand.lift,
|
|
807
|
+
neutralizedLift: neut.lift,
|
|
808
|
+
decorativeFraction,
|
|
809
|
+
maxDecorativeFraction,
|
|
810
|
+
n: cand.n
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
],
|
|
814
|
+
delta: cand.lift
|
|
815
|
+
};
|
|
816
|
+
}
|
|
817
|
+
};
|
|
818
|
+
}
|
|
819
|
+
|
|
322
820
|
// src/campaign/gates/sequential.ts
|
|
323
|
-
import { createHash as
|
|
821
|
+
import { createHash as createHash3 } from "crypto";
|
|
324
822
|
function verifyManifestSync(m) {
|
|
325
823
|
if (m.algo !== void 0 && m.algo !== "sha256-content") {
|
|
326
824
|
throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
|
|
@@ -328,7 +826,7 @@ function verifyManifestSync(m) {
|
|
|
328
826
|
const { contentHash: contentHash2, algo: _algo, ...rest } = m;
|
|
329
827
|
void _algo;
|
|
330
828
|
const bytes = JSON.stringify(canonicalize(rest));
|
|
331
|
-
const hash =
|
|
829
|
+
const hash = createHash3("sha256").update(bytes, "utf8").digest("hex");
|
|
332
830
|
return hash === contentHash2;
|
|
333
831
|
}
|
|
334
832
|
function resolveConfig(opts) {
|
|
@@ -586,7 +1084,7 @@ function sequentialDecide(options = {}) {
|
|
|
586
1084
|
}
|
|
587
1085
|
|
|
588
1086
|
// src/campaign/labeled-store/fs-adapter.ts
|
|
589
|
-
import { createHash as
|
|
1087
|
+
import { createHash as createHash4 } from "crypto";
|
|
590
1088
|
import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
591
1089
|
import { join as join2 } from "path";
|
|
592
1090
|
var LabeledScenarioStoreError = class extends Error {
|
|
@@ -785,7 +1283,7 @@ function matchesFilter(record, args, source) {
|
|
|
785
1283
|
return true;
|
|
786
1284
|
}
|
|
787
1285
|
function sha256(input) {
|
|
788
|
-
return
|
|
1286
|
+
return createHash4("sha256").update(input).digest("hex").slice(0, 16);
|
|
789
1287
|
}
|
|
790
1288
|
function appendLine(path, line) {
|
|
791
1289
|
if (existsSync2(path)) {
|
|
@@ -796,6 +1294,12 @@ function appendLine(path, line) {
|
|
|
796
1294
|
}
|
|
797
1295
|
}
|
|
798
1296
|
|
|
1297
|
+
// src/campaign/neutralize.ts
|
|
1298
|
+
var FILLER = "#";
|
|
1299
|
+
function neutralizeText(content) {
|
|
1300
|
+
return content.replace(/\S/g, FILLER);
|
|
1301
|
+
}
|
|
1302
|
+
|
|
799
1303
|
// src/campaign/proposers/fapo.ts
|
|
800
1304
|
var FAPO_LEVELS = ["prompt", "parameter", "structural"];
|
|
801
1305
|
var MAX_FINDING_DEPTH = 16;
|
|
@@ -1737,8 +2241,8 @@ async function compareProposerEntries(opts) {
|
|
|
1737
2241
|
});
|
|
1738
2242
|
const score = {
|
|
1739
2243
|
name: w.name,
|
|
1740
|
-
baselineComposite:
|
|
1741
|
-
winnerComposite:
|
|
2244
|
+
baselineComposite: mean2(baselineArr),
|
|
2245
|
+
winnerComposite: mean2(w.arr),
|
|
1742
2246
|
lift: boot.mean,
|
|
1743
2247
|
liftCi: { low: boot.low, high: boot.high },
|
|
1744
2248
|
costUsd: w.costUsd,
|
|
@@ -1775,7 +2279,7 @@ async function compareProposerEntries(opts) {
|
|
|
1775
2279
|
});
|
|
1776
2280
|
return { scores, best, pairwise, holdoutScenarioIds: scenarioIds };
|
|
1777
2281
|
}
|
|
1778
|
-
function
|
|
2282
|
+
function mean2(xs) {
|
|
1779
2283
|
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
1780
2284
|
}
|
|
1781
2285
|
function slug(name) {
|
|
@@ -2013,8 +2517,177 @@ function renderScoreboardMarkdown(rows, opts = {}) {
|
|
|
2013
2517
|
return out.join("\n");
|
|
2014
2518
|
}
|
|
2015
2519
|
|
|
2520
|
+
// src/campaign/presets/run-lineage-loop.ts
|
|
2521
|
+
function toCandidate(p) {
|
|
2522
|
+
return isProposedCandidate(p) ? { surface: p.surface, ...p.rationale ? { rationale: p.rationale } : {} } : { surface: p };
|
|
2523
|
+
}
|
|
2524
|
+
async function runLineageLoop(opts) {
|
|
2525
|
+
const governor = opts.governor ?? heuristicGovernor();
|
|
2526
|
+
const populationSize = opts.populationSize ?? 4;
|
|
2527
|
+
if (populationSize < 1) {
|
|
2528
|
+
throw new Error("runLineageLoop: populationSize must be >= 1");
|
|
2529
|
+
}
|
|
2530
|
+
let proposer = opts.proposer;
|
|
2531
|
+
if (!proposer) {
|
|
2532
|
+
if (!opts.llm || !opts.model) {
|
|
2533
|
+
throw new Error(
|
|
2534
|
+
"runLineageLoop: a proposer is required \u2014 either inject `proposer`, or provide `llm` + `model` for the default gepaProposer."
|
|
2535
|
+
);
|
|
2536
|
+
}
|
|
2537
|
+
proposer = gepaProposer({
|
|
2538
|
+
llm: opts.llm,
|
|
2539
|
+
model: opts.model,
|
|
2540
|
+
target: opts.target ?? "agent surface",
|
|
2541
|
+
// GEPA combine-complementary-lessons is what the `merge` seam relies on.
|
|
2542
|
+
combineParents: true
|
|
2543
|
+
});
|
|
2544
|
+
}
|
|
2545
|
+
const scoringScenarios = opts.holdoutScenarios ?? opts.scenarios;
|
|
2546
|
+
let scoreSurface = opts.scoreSurface;
|
|
2547
|
+
if (!scoreSurface) {
|
|
2548
|
+
const dispatchWithSurface = opts.dispatchWithSurface;
|
|
2549
|
+
const runDir = opts.runDir;
|
|
2550
|
+
if (!dispatchWithSurface || !runDir) {
|
|
2551
|
+
throw new Error(
|
|
2552
|
+
"runLineageLoop: scoring is required \u2014 either inject `scoreSurface`, or provide `dispatchWithSurface` (the agent) + `runDir` for the default runCampaign scorer."
|
|
2553
|
+
);
|
|
2554
|
+
}
|
|
2555
|
+
if (scoringScenarios.length === 0) {
|
|
2556
|
+
throw new Error(
|
|
2557
|
+
"runLineageLoop: the default scorer needs at least one scenario (holdoutScenarios ?? scenarios is empty)."
|
|
2558
|
+
);
|
|
2559
|
+
}
|
|
2560
|
+
scoreSurface = async (surface) => {
|
|
2561
|
+
const campaign = await runCampaign({
|
|
2562
|
+
scenarios: scoringScenarios,
|
|
2563
|
+
dispatch: (scenario, ctx) => dispatchWithSurface(surface, scenario, ctx),
|
|
2564
|
+
dispatchRef: `lineage-loop:${surfaceHash(surface)}`,
|
|
2565
|
+
runDir: `${runDir}/score/${surfaceHash(surface)}`,
|
|
2566
|
+
...opts.judges ? { judges: opts.judges } : {},
|
|
2567
|
+
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
2568
|
+
...opts.reps !== void 0 ? { reps: opts.reps } : {},
|
|
2569
|
+
...opts.storage ? { storage: opts.storage } : {},
|
|
2570
|
+
...opts.tracing ? { tracing: opts.tracing } : {},
|
|
2571
|
+
...opts.expectUsage ? { expectUsage: opts.expectUsage } : {},
|
|
2572
|
+
...opts.maxConcurrency !== void 0 ? { maxConcurrency: opts.maxConcurrency } : {},
|
|
2573
|
+
...opts.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: opts.dispatchTimeoutMs } : {},
|
|
2574
|
+
...opts.now ? { now: opts.now } : {}
|
|
2575
|
+
});
|
|
2576
|
+
const score = campaignMeanComposite(campaign);
|
|
2577
|
+
const byId = new Map(
|
|
2578
|
+
campaignBreakdown(campaign).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
2579
|
+
);
|
|
2580
|
+
const scoreVector = scoringScenarios.map((s) => byId.get(s.id) ?? 0);
|
|
2581
|
+
return { score, scoreVector };
|
|
2582
|
+
};
|
|
2583
|
+
}
|
|
2584
|
+
const objectivesOf = (node) => {
|
|
2585
|
+
const objectives = {};
|
|
2586
|
+
if (node.scoreVector && node.scoreVector.length > 0) {
|
|
2587
|
+
scoringScenarios.forEach((s, i) => {
|
|
2588
|
+
objectives[s.id] = node.scoreVector[i] ?? 0;
|
|
2589
|
+
});
|
|
2590
|
+
if (Object.keys(objectives).length > 0) return objectives;
|
|
2591
|
+
}
|
|
2592
|
+
objectives.composite = node.score;
|
|
2593
|
+
return objectives;
|
|
2594
|
+
};
|
|
2595
|
+
const scoredSeeds = [];
|
|
2596
|
+
for (const seed of opts.seeds) {
|
|
2597
|
+
const measured = await scoreSurface(seed.surface);
|
|
2598
|
+
scoredSeeds.push({
|
|
2599
|
+
surface: seed.surface,
|
|
2600
|
+
track: seed.track,
|
|
2601
|
+
proposer: seed.proposer,
|
|
2602
|
+
score: measured.score,
|
|
2603
|
+
...seed.vision !== void 0 ? { vision: seed.vision } : {},
|
|
2604
|
+
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {}
|
|
2605
|
+
});
|
|
2606
|
+
}
|
|
2607
|
+
const step = async (args) => {
|
|
2608
|
+
const proposed = await proposer.propose({
|
|
2609
|
+
currentSurface: args.tip.surface,
|
|
2610
|
+
history: [],
|
|
2611
|
+
findings: [],
|
|
2612
|
+
populationSize,
|
|
2613
|
+
generation: 0,
|
|
2614
|
+
signal: new AbortController().signal,
|
|
2615
|
+
paretoParents: []
|
|
2616
|
+
});
|
|
2617
|
+
const pool = [
|
|
2618
|
+
{
|
|
2619
|
+
surface: args.tip.surface,
|
|
2620
|
+
score: args.tip.score,
|
|
2621
|
+
...args.tip.scoreVector !== void 0 ? { scoreVector: args.tip.scoreVector } : {},
|
|
2622
|
+
...args.tip.rationale !== void 0 ? { rationale: args.tip.rationale } : {}
|
|
2623
|
+
}
|
|
2624
|
+
];
|
|
2625
|
+
for (const p of proposed) {
|
|
2626
|
+
const { surface, rationale } = toCandidate(p);
|
|
2627
|
+
const measured = await scoreSurface(surface);
|
|
2628
|
+
pool.push({
|
|
2629
|
+
surface,
|
|
2630
|
+
score: measured.score,
|
|
2631
|
+
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
|
|
2632
|
+
...rationale !== void 0 ? { rationale } : {}
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
2635
|
+
let best = pool[0];
|
|
2636
|
+
for (const entry of pool) {
|
|
2637
|
+
if (entry.score > best.score) best = entry;
|
|
2638
|
+
}
|
|
2639
|
+
return {
|
|
2640
|
+
surface: best.surface,
|
|
2641
|
+
score: best.score,
|
|
2642
|
+
...best.scoreVector !== void 0 ? { scoreVector: best.scoreVector } : {},
|
|
2643
|
+
...best.rationale !== void 0 ? { rationale: best.rationale } : {}
|
|
2644
|
+
};
|
|
2645
|
+
};
|
|
2646
|
+
const merge = async (args) => {
|
|
2647
|
+
const ordered = [...args.parents].sort((a, b) => b.score - a.score);
|
|
2648
|
+
const paretoParents = ordered.map((node) => ({
|
|
2649
|
+
surface: node.surface,
|
|
2650
|
+
surfaceHash: surfaceHash(node.surface),
|
|
2651
|
+
objectives: objectivesOf(node),
|
|
2652
|
+
composite: node.score,
|
|
2653
|
+
generation: node.generation
|
|
2654
|
+
}));
|
|
2655
|
+
const proposed = await proposer.propose({
|
|
2656
|
+
currentSurface: ordered[0].surface,
|
|
2657
|
+
history: [],
|
|
2658
|
+
findings: [],
|
|
2659
|
+
// A merge is a single crossover; combineParents fires because
|
|
2660
|
+
// paretoParents has > 1 string member.
|
|
2661
|
+
populationSize: 1,
|
|
2662
|
+
// generation >= 1 keeps the combine slot semantically a "merge" step.
|
|
2663
|
+
generation: 1,
|
|
2664
|
+
signal: new AbortController().signal,
|
|
2665
|
+
paretoParents
|
|
2666
|
+
});
|
|
2667
|
+
const first = proposed[0];
|
|
2668
|
+
const merged = first ? toCandidate(first) : { surface: ordered[0].surface };
|
|
2669
|
+
const measured = await scoreSurface(merged.surface);
|
|
2670
|
+
return {
|
|
2671
|
+
surface: merged.surface,
|
|
2672
|
+
score: measured.score,
|
|
2673
|
+
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
|
|
2674
|
+
...merged.rationale !== void 0 ? { rationale: merged.rationale } : {}
|
|
2675
|
+
};
|
|
2676
|
+
};
|
|
2677
|
+
const result = await runLineage({
|
|
2678
|
+
seeds: scoredSeeds,
|
|
2679
|
+
step,
|
|
2680
|
+
merge,
|
|
2681
|
+
governor,
|
|
2682
|
+
budget: opts.budget,
|
|
2683
|
+
...opts.store ? { store: opts.store } : {},
|
|
2684
|
+
...opts.log ? { log: opts.log } : {}
|
|
2685
|
+
});
|
|
2686
|
+
return { lineage: result.lineage, best: result.best, steps: result.steps };
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2016
2689
|
// src/campaign/presets/run-profile-matrix.ts
|
|
2017
|
-
import { createHash as
|
|
2690
|
+
import { createHash as createHash5 } from "crypto";
|
|
2018
2691
|
import { join as join3 } from "path";
|
|
2019
2692
|
var ProfileMatrixError = class extends AgentEvalError {
|
|
2020
2693
|
constructor(message) {
|
|
@@ -2025,14 +2698,14 @@ function sanitize(id) {
|
|
|
2025
2698
|
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2026
2699
|
}
|
|
2027
2700
|
function sha(input) {
|
|
2028
|
-
return
|
|
2701
|
+
return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
|
|
2029
2702
|
}
|
|
2030
|
-
function
|
|
2703
|
+
function mean3(xs) {
|
|
2031
2704
|
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
2032
2705
|
}
|
|
2033
2706
|
function cellComposite(cell) {
|
|
2034
2707
|
const composites = Object.values(cell.judgeScores).map((s) => s.composite);
|
|
2035
|
-
return composites.length === 0 ? 0 :
|
|
2708
|
+
return composites.length === 0 ? 0 : mean3(composites);
|
|
2036
2709
|
}
|
|
2037
2710
|
function requireResolvedModel(cell, profileId) {
|
|
2038
2711
|
const resolved = cell.resolvedModel?.trim();
|
|
@@ -2068,7 +2741,7 @@ function buildRunRecord(args) {
|
|
|
2068
2741
|
if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
|
|
2069
2742
|
}
|
|
2070
2743
|
const perDimMean = {};
|
|
2071
|
-
for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] =
|
|
2744
|
+
for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean3(values);
|
|
2072
2745
|
let costUsd = cell.costUsd;
|
|
2073
2746
|
let costEstimated = false;
|
|
2074
2747
|
if (costUsd === 0 && cell.tokenUsage.output > 0 && isModelPriced(model)) {
|
|
@@ -2232,7 +2905,7 @@ async function runProfileMatrix(opts) {
|
|
|
2232
2905
|
// one harness, so the first record's model is representative).
|
|
2233
2906
|
model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
|
|
2234
2907
|
records: profileRecords.length,
|
|
2235
|
-
meanComposite:
|
|
2908
|
+
meanComposite: mean3(profileRecords.map(compositeOf)),
|
|
2236
2909
|
totalCostUsd: pricedTotalCostUsd,
|
|
2237
2910
|
integrity: summarizeBackendIntegrity(profileRecords)
|
|
2238
2911
|
};
|
|
@@ -2262,7 +2935,7 @@ function rollup(records, keyOf) {
|
|
|
2262
2935
|
groups.set(key, arr);
|
|
2263
2936
|
}
|
|
2264
2937
|
const out = {};
|
|
2265
|
-
for (const [key, xs] of groups) out[key] = { meanComposite:
|
|
2938
|
+
for (const [key, xs] of groups) out[key] = { meanComposite: mean3(xs), n: xs.length };
|
|
2266
2939
|
return out;
|
|
2267
2940
|
}
|
|
2268
2941
|
function rollupByPersona(records, scenarios, personaOf) {
|
|
@@ -2665,6 +3338,53 @@ ${report.slice(0, 800)}`,
|
|
|
2665
3338
|
});
|
|
2666
3339
|
}
|
|
2667
3340
|
|
|
3341
|
+
// src/campaign/scenario-selection.ts
|
|
3342
|
+
var DEFAULT_SATURATION_CEILING = 0.999;
|
|
3343
|
+
var VARIANCE_EPSILON = 1e-9;
|
|
3344
|
+
function moments(scores) {
|
|
3345
|
+
const n = scores.length;
|
|
3346
|
+
if (n === 0) return { meanScore: 0, variance: 0 };
|
|
3347
|
+
let sum = 0;
|
|
3348
|
+
for (const s of scores) sum += s;
|
|
3349
|
+
const meanScore = sum / n;
|
|
3350
|
+
let sqDev = 0;
|
|
3351
|
+
for (const s of scores) {
|
|
3352
|
+
const d = s - meanScore;
|
|
3353
|
+
sqDev += d * d;
|
|
3354
|
+
}
|
|
3355
|
+
return { meanScore, variance: sqDev / n };
|
|
3356
|
+
}
|
|
3357
|
+
function compareDiscrimination(a, b) {
|
|
3358
|
+
if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
|
|
3359
|
+
if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
|
|
3360
|
+
return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
|
|
3361
|
+
}
|
|
3362
|
+
function scoreDiscrimination(signals, opts) {
|
|
3363
|
+
const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
|
|
3364
|
+
const scored = signals.map((signal) => {
|
|
3365
|
+
const { meanScore, variance } = moments(signal.scores);
|
|
3366
|
+
const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
|
|
3367
|
+
return {
|
|
3368
|
+
scenarioId: signal.scenarioId,
|
|
3369
|
+
discrimination: variance,
|
|
3370
|
+
meanScore,
|
|
3371
|
+
variance,
|
|
3372
|
+
tied
|
|
3373
|
+
};
|
|
3374
|
+
});
|
|
3375
|
+
return scored.sort(compareDiscrimination);
|
|
3376
|
+
}
|
|
3377
|
+
function selectDiscriminative(signals, k, opts) {
|
|
3378
|
+
if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
|
|
3379
|
+
const ranked = scoreDiscrimination(signals, opts);
|
|
3380
|
+
if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
|
|
3381
|
+
const nonTied = ranked.filter((s) => !s.tied);
|
|
3382
|
+
if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
|
|
3383
|
+
const tied = ranked.filter((s) => s.tied);
|
|
3384
|
+
const fill = tied.slice(0, k - nonTied.length);
|
|
3385
|
+
return [...nonTied, ...fill].map((s) => s.scenarioId);
|
|
3386
|
+
}
|
|
3387
|
+
|
|
2668
3388
|
// src/campaign/worktree/index.ts
|
|
2669
3389
|
import { execFileSync } from "child_process";
|
|
2670
3390
|
import { existsSync as existsSync3 } from "fs";
|
|
@@ -2727,6 +3447,7 @@ function resolveWorktreePath(surface, worktreeDir) {
|
|
|
2727
3447
|
export {
|
|
2728
3448
|
FsLabeledScenarioStore,
|
|
2729
3449
|
LabeledScenarioStoreError,
|
|
3450
|
+
Lineage,
|
|
2730
3451
|
ProfileMatrixError,
|
|
2731
3452
|
SkillPatchParseError,
|
|
2732
3453
|
WorktreeAdapterError,
|
|
@@ -2735,6 +3456,7 @@ export {
|
|
|
2735
3456
|
buildAnalystSurfaceDispatch,
|
|
2736
3457
|
buildEvidenceVector,
|
|
2737
3458
|
buildLoopProvenanceRecord,
|
|
3459
|
+
callbackGovernor,
|
|
2738
3460
|
campaignBreakdown,
|
|
2739
3461
|
campaignMeanComposite,
|
|
2740
3462
|
compareProposers,
|
|
@@ -2753,6 +3475,7 @@ export {
|
|
|
2753
3475
|
fapoEscalationEntry,
|
|
2754
3476
|
fapoProposer,
|
|
2755
3477
|
fsCampaignStorage,
|
|
3478
|
+
fsLineageStore,
|
|
2756
3479
|
gepaParetoEntry,
|
|
2757
3480
|
gepaProposer,
|
|
2758
3481
|
gepaReflectionEntry,
|
|
@@ -2760,15 +3483,20 @@ export {
|
|
|
2760
3483
|
haloProposer,
|
|
2761
3484
|
heldOutGate,
|
|
2762
3485
|
heldoutSignificance,
|
|
3486
|
+
heuristicGovernor,
|
|
2763
3487
|
inMemoryCampaignStorage,
|
|
2764
3488
|
isProposedCandidate,
|
|
2765
3489
|
labelTrustRank,
|
|
3490
|
+
lineageNodeId,
|
|
2766
3491
|
llmJudge,
|
|
2767
3492
|
loadEvalFixture,
|
|
2768
3493
|
loadEvalFixtureScenarios,
|
|
2769
3494
|
loopProvenanceSpans,
|
|
2770
3495
|
makePlaybackDispatch,
|
|
3496
|
+
memLineageStore,
|
|
2771
3497
|
memoryCurationProposer,
|
|
3498
|
+
neutralizationGate,
|
|
3499
|
+
neutralizeText,
|
|
2772
3500
|
openAutoPr,
|
|
2773
3501
|
pairHoldout,
|
|
2774
3502
|
parameterSweepProposer,
|
|
@@ -2788,11 +3516,15 @@ export {
|
|
|
2788
3516
|
runCampaign,
|
|
2789
3517
|
runEval,
|
|
2790
3518
|
runImprovementLoop,
|
|
3519
|
+
runLineage,
|
|
3520
|
+
runLineageLoop,
|
|
2791
3521
|
runOptimization,
|
|
2792
3522
|
runProfileMatrix,
|
|
2793
3523
|
runSkillOpt,
|
|
3524
|
+
scoreDiscrimination,
|
|
2794
3525
|
scoreUserStory,
|
|
2795
3526
|
scoreboardSummary,
|
|
3527
|
+
selectDiscriminative,
|
|
2796
3528
|
sequentialDecide,
|
|
2797
3529
|
sequentialPairedGate,
|
|
2798
3530
|
skillOptEntry,
|