@tangle-network/agent-eval 0.106.2 → 0.107.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@ import {
7
7
  paretoSignificanceGate,
8
8
  powerPreflight,
9
9
  runEval
10
- } from "../chunk-J22CKVXN.js";
10
+ } from "../chunk-V75NN2ZR.js";
11
11
  import {
12
12
  HARNESS_NATIVE_MODEL,
13
13
  agentProfileHash,
@@ -17,7 +17,7 @@ import {
17
17
  harnessAxisOf,
18
18
  llmJudge,
19
19
  verifyCompletion
20
- } from "../chunk-2HLM4SJJ.js";
20
+ } from "../chunk-6PF6LXRY.js";
21
21
  import {
22
22
  buildLoopProvenanceRecord,
23
23
  campaignBreakdown,
@@ -43,7 +43,7 @@ import {
43
43
  runOptimization,
44
44
  surfaceContentHash,
45
45
  surfaceHash
46
- } from "../chunk-3E5KUXYZ.js";
46
+ } from "../chunk-G4DLZAV5.js";
47
47
  import {
48
48
  assertRealBackend,
49
49
  contentHash,
@@ -103,6 +103,424 @@ import {
103
103
  } from "../chunk-ONWEPEDO.js";
104
104
  import "../chunk-PZ5AY32C.js";
105
105
 
106
+ // src/campaign/lineage.ts
107
+ import { createHash } from "crypto";
108
+ import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
109
+ import { dirname } from "path";
110
+ function lineageNodeId(input) {
111
+ const parents = [...input.parentIds].sort().join(",");
112
+ const payload = `${parents}|${input.track}|${input.surface}|${input.proposer}`;
113
+ return createHash("sha256").update(payload).digest("hex").slice(0, 16);
114
+ }
115
+ var Lineage = class _Lineage {
116
+ byId = /* @__PURE__ */ new Map();
117
+ childIds = /* @__PURE__ */ new Map();
118
+ nextSeq = 0;
119
+ constructor(nodes) {
120
+ if (!nodes) return;
121
+ for (const node of [...nodes].sort((a, b) => a.seq - b.seq)) {
122
+ this.index(node);
123
+ this.nextSeq = Math.max(this.nextSeq, node.seq + 1);
124
+ }
125
+ }
126
+ /** Append a node. Derives `id` (via {@link lineageNodeId}), assigns the next
127
+ * `seq`, and derives `generation` as `max(parent.generation) + 1` (root = 0)
128
+ * when omitted. Throws on an unknown parent. Idempotent: re-adding an identical
129
+ * node returns the existing one.
130
+ *
131
+ * Acyclicity is guaranteed by construction, not by a runtime check: every
132
+ * parent must already exist (a child can never point at a not-yet-added node),
133
+ * ids are immutable content hashes (an existing node can never gain new
134
+ * parents), and the store is append-only — so no back-edge can form. (Traversal
135
+ * is still cycle-safe against hand-corrupted deserialized input via visited
136
+ * sets in {@link ancestors}/{@link descendants}.) */
137
+ addNode(input) {
138
+ for (const parentId of input.parentIds) {
139
+ if (!this.byId.has(parentId)) {
140
+ throw new Error(`Lineage.addNode: unknown parent '${parentId}'`);
141
+ }
142
+ }
143
+ const id = lineageNodeId({
144
+ parentIds: input.parentIds,
145
+ track: input.track,
146
+ surface: input.surface,
147
+ proposer: input.proposer
148
+ });
149
+ const existing = this.byId.get(id);
150
+ if (existing) return existing;
151
+ const generation = input.generation ?? (input.parentIds.length === 0 ? 0 : Math.max(...input.parentIds.map((p) => this.byId.get(p).generation)) + 1);
152
+ const node = {
153
+ id,
154
+ parentIds: [...input.parentIds],
155
+ track: input.track,
156
+ surface: input.surface,
157
+ score: input.score,
158
+ proposer: input.proposer,
159
+ generation,
160
+ seq: this.nextSeq++,
161
+ ...input.vision !== void 0 ? { vision: input.vision } : {},
162
+ ...input.scoreVector !== void 0 ? { scoreVector: [...input.scoreVector] } : {},
163
+ ...input.rationale !== void 0 ? { rationale: input.rationale } : {},
164
+ ...input.gate !== void 0 ? { gate: input.gate } : {}
165
+ };
166
+ this.index(node);
167
+ return node;
168
+ }
169
+ /** Collapse 2+ parents into a single node (a merge/"collapse"). A merge is an
170
+ * ordinary multi-parent node; this is a guarded convenience. */
171
+ merge(input) {
172
+ if (input.parentIds.length < 2) {
173
+ throw new Error(`Lineage.merge: a merge needs >= 2 parents, got ${input.parentIds.length}`);
174
+ }
175
+ return this.addNode({ ...input, proposer: input.proposer ?? "merge" });
176
+ }
177
+ get(id) {
178
+ return this.byId.get(id);
179
+ }
180
+ has(id) {
181
+ return this.byId.has(id);
182
+ }
183
+ /** All nodes, in insertion (`seq`) order. */
184
+ all() {
185
+ return [...this.byId.values()].sort(bySeq);
186
+ }
187
+ roots() {
188
+ return this.all().filter((n) => n.parentIds.length === 0);
189
+ }
190
+ parents(id) {
191
+ const node = this.byId.get(id);
192
+ if (!node) return [];
193
+ return node.parentIds.map((p) => this.byId.get(p)).filter((n) => n !== void 0).sort(bySeq);
194
+ }
195
+ children(id) {
196
+ return (this.childIds.get(id) ?? []).map((c) => this.byId.get(c)).sort(bySeq);
197
+ }
198
+ /** Transitive ancestors (excludes `id`). */
199
+ ancestors(id) {
200
+ const out = /* @__PURE__ */ new Set();
201
+ const stack = [...this.byId.get(id)?.parentIds ?? []];
202
+ while (stack.length > 0) {
203
+ const cur = stack.pop();
204
+ if (out.has(cur)) continue;
205
+ out.add(cur);
206
+ const node = this.byId.get(cur);
207
+ if (node) stack.push(...node.parentIds);
208
+ }
209
+ return out;
210
+ }
211
+ /** Transitive descendants (excludes `id`). */
212
+ descendants(id) {
213
+ const out = /* @__PURE__ */ new Set();
214
+ const stack = [...this.childIds.get(id) ?? []];
215
+ while (stack.length > 0) {
216
+ const cur = stack.pop();
217
+ if (out.has(cur)) continue;
218
+ out.add(cur);
219
+ stack.push(...this.childIds.get(cur) ?? []);
220
+ }
221
+ return out;
222
+ }
223
+ /** Nodes with no children (leaf/branch tips). */
224
+ tips() {
225
+ return this.all().filter((n) => (this.childIds.get(n.id) ?? []).length === 0);
226
+ }
227
+ /** Distinct track ids, in first-seen (`seq`) order. */
228
+ tracks() {
229
+ const seen = /* @__PURE__ */ new Set();
230
+ const out = [];
231
+ for (const node of this.all()) {
232
+ if (!seen.has(node.track)) {
233
+ seen.add(node.track);
234
+ out.push(node.track);
235
+ }
236
+ }
237
+ return out;
238
+ }
239
+ trackNodes(track) {
240
+ return this.all().filter((n) => n.track === track);
241
+ }
242
+ /** The highest-`score` tip of a track (ties broken by lowest `seq`). */
243
+ trackTip(track) {
244
+ return pickBest(this.tips().filter((n) => n.track === track));
245
+ }
246
+ /** The highest-`score` node overall (ties broken by lowest `seq`). */
247
+ best() {
248
+ return pickBest(this.all());
249
+ }
250
+ /** The Pareto-non-dominated set among TIPS. Uses `scoreVector` when every
251
+ * compared tip carries one, else the scalar `score`. A dominates B iff A is
252
+ * >= B on every component and > B on at least one. */
253
+ frontier() {
254
+ const tips = this.tips();
255
+ const useVector = tips.length > 0 && tips.every((n) => n.scoreVector !== void 0);
256
+ const vecOf = (n) => useVector ? n.scoreVector : [n.score];
257
+ return tips.filter((a) => !tips.some((b) => b.id !== a.id && dominates(vecOf(b), vecOf(a)))).sort(bySeq);
258
+ }
259
+ toGraph() {
260
+ const nodes = this.all();
261
+ const edges = [];
262
+ for (const node of nodes) {
263
+ for (const parentId of node.parentIds) {
264
+ edges.push({ from: parentId, to: node.id });
265
+ }
266
+ }
267
+ return { nodes, edges };
268
+ }
269
+ /** One JSON node per line, in `seq` order. */
270
+ toJSONL() {
271
+ return this.all().map((n) => JSON.stringify(n)).join("\n");
272
+ }
273
+ static fromJSONL(text) {
274
+ const nodes = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
275
+ return new _Lineage(nodes);
276
+ }
277
+ index(node) {
278
+ this.byId.set(node.id, node);
279
+ for (const parentId of node.parentIds) {
280
+ const list = this.childIds.get(parentId);
281
+ if (list) {
282
+ if (!list.includes(node.id)) list.push(node.id);
283
+ } else {
284
+ this.childIds.set(parentId, [node.id]);
285
+ }
286
+ }
287
+ }
288
+ };
289
+ function bySeq(a, b) {
290
+ return a.seq - b.seq;
291
+ }
292
+ function pickBest(nodes) {
293
+ let best;
294
+ for (const node of nodes) {
295
+ if (!best || node.score > best.score || node.score === best.score && node.seq < best.seq) {
296
+ best = node;
297
+ }
298
+ }
299
+ return best;
300
+ }
301
+ function dominates(a, b) {
302
+ if (a.length !== b.length) return false;
303
+ let strictlyBetter = false;
304
+ for (let i = 0; i < a.length; i += 1) {
305
+ if (a[i] < b[i]) return false;
306
+ if (a[i] > b[i]) strictlyBetter = true;
307
+ }
308
+ return strictlyBetter;
309
+ }
310
+ function fsLineageStore(path) {
311
+ const ensureDir = () => mkdir(dirname(path), { recursive: true });
312
+ return {
313
+ async load() {
314
+ try {
315
+ return Lineage.fromJSONL(await readFile(path, "utf8"));
316
+ } catch (err) {
317
+ if (err.code === "ENOENT") return new Lineage();
318
+ throw err;
319
+ }
320
+ },
321
+ async append(node) {
322
+ await ensureDir();
323
+ await appendFile(path, `${JSON.stringify(node)}
324
+ `, "utf8");
325
+ },
326
+ async save(lineage) {
327
+ await ensureDir();
328
+ const jsonl = lineage.toJSONL();
329
+ await writeFile(path, jsonl.length > 0 ? `${jsonl}
330
+ ` : "", "utf8");
331
+ }
332
+ };
333
+ }
334
+ function memLineageStore() {
335
+ const nodes = [];
336
+ return {
337
+ async load() {
338
+ return new Lineage(nodes);
339
+ },
340
+ async append(node) {
341
+ nodes.push(node);
342
+ },
343
+ async save(lineage) {
344
+ nodes.length = 0;
345
+ nodes.push(...lineage.all());
346
+ }
347
+ };
348
+ }
349
+ function heuristicGovernor(opts = {}) {
350
+ const maxTracks = opts.maxTracks ?? 3;
351
+ const plateauSteps = opts.plateauSteps ?? 2;
352
+ const mergeFrontierAt = opts.mergeFrontierAt ?? 2;
353
+ return {
354
+ decide(ctx) {
355
+ const { lineage } = ctx;
356
+ if (ctx.budgetRemaining <= 0) return { op: "stop" };
357
+ const pruned = new Set(ctx.prunedTracks);
358
+ const liveTracks = lineage.tracks().filter((t) => !pruned.has(t));
359
+ if (liveTracks.length === 0) return { op: "stop" };
360
+ const frontierByTrack = /* @__PURE__ */ new Map();
361
+ for (const tip of lineage.frontier()) {
362
+ if (pruned.has(tip.track)) continue;
363
+ if (!frontierByTrack.has(tip.track)) frontierByTrack.set(tip.track, tip);
364
+ }
365
+ if (frontierByTrack.size >= mergeFrontierAt) {
366
+ const tips = [...frontierByTrack.values()].sort(bySeq);
367
+ const target = pickBest(tips);
368
+ return { op: "merge", parentIds: tips.map((t) => t.id), track: target.track };
369
+ }
370
+ for (const track of liveTracks) {
371
+ if (isPlateaued(lineage.trackNodes(track), plateauSteps)) {
372
+ if (liveTracks.length > 1 || liveTracks.length < maxTracks) {
373
+ return { op: "prune", track };
374
+ }
375
+ }
376
+ }
377
+ const leader = pickBest(liveTracks.map((t) => lineage.trackTip(t)).filter(Boolean));
378
+ if (leader && liveTracks.length < maxTracks && isPlateaued(lineage.trackNodes(leader.track), plateauSteps)) {
379
+ return {
380
+ op: "branch",
381
+ fromNodeId: leader.id,
382
+ track: `${leader.track}+${lineage.tracks().length}`,
383
+ proposer: "gepa"
384
+ };
385
+ }
386
+ if (leader) return { op: "extend", track: leader.track };
387
+ return { op: "stop" };
388
+ }
389
+ };
390
+ }
391
+ function callbackGovernor(decide) {
392
+ return { decide };
393
+ }
394
+ function isPlateaued(trackNodes, window) {
395
+ if (trackNodes.length <= window) return false;
396
+ const ordered = [...trackNodes].sort(bySeq);
397
+ const head = ordered.slice(0, ordered.length - window);
398
+ const tail = ordered.slice(ordered.length - window);
399
+ const bestBefore = Math.max(...head.map((n) => n.score));
400
+ const bestRecent = Math.max(...tail.map((n) => n.score));
401
+ return bestRecent <= bestBefore;
402
+ }
403
+ async function runLineage(opts) {
404
+ const store = opts.store ?? memLineageStore();
405
+ const lineage = await store.load();
406
+ const log = opts.log ?? (() => {
407
+ });
408
+ const pruned = /* @__PURE__ */ new Set();
409
+ const trackProposer = /* @__PURE__ */ new Map();
410
+ const persist = async (node) => {
411
+ await store.append(node);
412
+ };
413
+ for (const seed of opts.seeds) {
414
+ const node = lineage.addNode({
415
+ parentIds: [],
416
+ track: seed.track,
417
+ surface: seed.surface,
418
+ score: seed.score,
419
+ proposer: seed.proposer,
420
+ ...seed.vision !== void 0 ? { vision: seed.vision } : {},
421
+ ...seed.scoreVector !== void 0 ? { scoreVector: seed.scoreVector } : {}
422
+ });
423
+ trackProposer.set(seed.track, seed.proposer);
424
+ await persist(node);
425
+ }
426
+ let steps = 0;
427
+ while (steps < opts.budget.maxSteps) {
428
+ const op = await opts.governor.decide({
429
+ lineage,
430
+ step: steps,
431
+ budgetRemaining: opts.budget.maxSteps - steps,
432
+ prunedTracks: [...pruned]
433
+ });
434
+ if (op.op === "stop") {
435
+ log("lineage: governor stop", { steps });
436
+ break;
437
+ }
438
+ if (op.op === "prune") {
439
+ pruned.add(op.track);
440
+ log("lineage: prune", { track: op.track });
441
+ steps += 1;
442
+ continue;
443
+ }
444
+ if (op.op === "merge") {
445
+ const parents = op.parentIds.map((id) => lineage.get(id)).filter((n) => n !== void 0);
446
+ if (parents.length < 2) {
447
+ log("lineage: merge skipped (fewer than 2 known parents)", { op });
448
+ steps += 1;
449
+ continue;
450
+ }
451
+ const result2 = await opts.merge({ parents, track: op.track });
452
+ const node2 = lineage.merge({
453
+ parentIds: parents.map((p) => p.id),
454
+ track: op.track,
455
+ surface: result2.surface,
456
+ score: result2.score,
457
+ ...parents[0].vision !== void 0 ? { vision: parents[0].vision } : {},
458
+ ...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
459
+ ...result2.rationale !== void 0 ? { rationale: result2.rationale } : {}
460
+ });
461
+ await persist(node2);
462
+ steps += 1;
463
+ continue;
464
+ }
465
+ if (op.op === "branch") {
466
+ if (pruned.has(op.track)) {
467
+ log("lineage: branch skipped (target track pruned)", { track: op.track });
468
+ steps += 1;
469
+ continue;
470
+ }
471
+ const from = lineage.get(op.fromNodeId);
472
+ if (!from) {
473
+ log("lineage: branch skipped (unknown fromNodeId)", { op });
474
+ steps += 1;
475
+ continue;
476
+ }
477
+ trackProposer.set(op.track, op.proposer);
478
+ const result2 = await opts.step({ track: op.track, proposer: op.proposer, tip: from });
479
+ const node2 = lineage.addNode({
480
+ parentIds: [from.id],
481
+ track: op.track,
482
+ surface: result2.surface,
483
+ score: result2.score,
484
+ proposer: op.proposer,
485
+ ...op.vision !== void 0 ? { vision: op.vision } : {},
486
+ ...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
487
+ ...result2.rationale !== void 0 ? { rationale: result2.rationale } : {},
488
+ ...result2.gate !== void 0 ? { gate: result2.gate } : {}
489
+ });
490
+ await persist(node2);
491
+ steps += 1;
492
+ continue;
493
+ }
494
+ if (pruned.has(op.track)) {
495
+ log("lineage: extend skipped (track pruned)", { track: op.track });
496
+ steps += 1;
497
+ continue;
498
+ }
499
+ const tip = lineage.trackTip(op.track);
500
+ if (!tip) {
501
+ log("lineage: extend skipped (no tip for track)", { track: op.track });
502
+ steps += 1;
503
+ continue;
504
+ }
505
+ const proposer = trackProposer.get(op.track) ?? tip.proposer;
506
+ const result = await opts.step({ track: op.track, proposer, tip });
507
+ const node = lineage.addNode({
508
+ parentIds: [tip.id],
509
+ track: op.track,
510
+ surface: result.surface,
511
+ score: result.score,
512
+ proposer,
513
+ ...tip.vision !== void 0 ? { vision: tip.vision } : {},
514
+ ...result.scoreVector !== void 0 ? { scoreVector: result.scoreVector } : {},
515
+ ...result.rationale !== void 0 ? { rationale: result.rationale } : {},
516
+ ...result.gate !== void 0 ? { gate: result.gate } : {}
517
+ });
518
+ await persist(node);
519
+ steps += 1;
520
+ }
521
+ return { lineage, best: lineage.best(), steps };
522
+ }
523
+
106
524
  // src/campaign/analyst-surface.ts
107
525
  function surfaceToText(surface) {
108
526
  if (typeof surface === "string") return surface;
@@ -166,7 +584,7 @@ function failureModeRecallJudge(opts = {}) {
166
584
  }
167
585
 
168
586
  // src/campaign/fixtures.ts
169
- import { createHash } from "crypto";
587
+ import { createHash as createHash2 } from "crypto";
170
588
  import { existsSync, readdirSync, readFileSync, statSync } from "fs";
171
589
  import { isAbsolute, join, relative, resolve } from "path";
172
590
  function discoverEvalFixtures(evalsDir) {
@@ -307,7 +725,7 @@ function collectFixtureFiles(fixturePath, base = "") {
307
725
  const bytes = readFileSync(fullPath);
308
726
  files.push({
309
727
  path: relativePath,
310
- sha256: createHash("sha256").update(bytes).digest("hex"),
728
+ sha256: createHash2("sha256").update(bytes).digest("hex"),
311
729
  bytes: bytes.byteLength
312
730
  });
313
731
  }
@@ -319,8 +737,88 @@ function assertInside(root, target, label) {
319
737
  throw new Error(`loadEvalFixture: fixture path escapes evalsDir: ${label}`);
320
738
  }
321
739
 
740
+ // src/campaign/gates/neutralization-gate.ts
741
+ function mean(xs) {
742
+ return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
743
+ }
744
+ function pairedLift(arm, baseline, scenarioIds) {
745
+ const paired = pairHoldout(arm, baseline, scenarioIds, (s) => s.composite);
746
+ const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0));
747
+ return { lift: mean(deltas), n: deltas.length };
748
+ }
749
+ function neutralizationGate(options) {
750
+ const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5;
751
+ return {
752
+ name: "neutralizationGate",
753
+ async decide(ctx) {
754
+ if (!ctx.baselineJudgeScores) {
755
+ throw new Error(
756
+ "neutralizationGate: ctx.baselineJudgeScores is required \u2014 the placebo control measures lift OVER baseline."
757
+ );
758
+ }
759
+ if (!ctx.neutralizedJudgeScores) {
760
+ throw new Error(
761
+ "neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied \u2014 composing this gate without that wiring would pass an unproven candidate."
762
+ );
763
+ }
764
+ const scenarioIds = new Set(options.scenarios.map((s) => s.id));
765
+ const cand = pairedLift(
766
+ ctx.judgeScores,
767
+ ctx.baselineJudgeScores,
768
+ scenarioIds
769
+ );
770
+ const neut = pairedLift(
771
+ ctx.neutralizedJudgeScores,
772
+ ctx.baselineJudgeScores,
773
+ scenarioIds
774
+ );
775
+ if (cand.lift <= 0) {
776
+ return {
777
+ decision: "hold",
778
+ reasons: [
779
+ `neutralization: candidate held-out lift ${cand.lift.toFixed(3)} \u2264 0 \u2014 no positive lift to attribute to content`
780
+ ],
781
+ contributingGates: [
782
+ {
783
+ name: "neutralizationGate",
784
+ passed: false,
785
+ detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
786
+ }
787
+ ],
788
+ delta: cand.lift
789
+ };
790
+ }
791
+ const decorativeFraction = neut.lift / cand.lift;
792
+ const passed = decorativeFraction < maxDecorativeFraction;
793
+ const pct = (decorativeFraction * 100).toFixed(0);
794
+ return {
795
+ decision: passed ? "ship" : "hold",
796
+ reasons: passed ? [
797
+ `neutralization: content is causal \u2014 blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
798
+ ] : [
799
+ `neutralization: lift is DECORATIVE \u2014 blanking the content (footprint-matched) reproduces ${pct}% of the lift (\u2265 ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
800
+ ],
801
+ contributingGates: [
802
+ {
803
+ name: "neutralizationGate",
804
+ passed,
805
+ detail: {
806
+ candidateLift: cand.lift,
807
+ neutralizedLift: neut.lift,
808
+ decorativeFraction,
809
+ maxDecorativeFraction,
810
+ n: cand.n
811
+ }
812
+ }
813
+ ],
814
+ delta: cand.lift
815
+ };
816
+ }
817
+ };
818
+ }
819
+
322
820
  // src/campaign/gates/sequential.ts
323
- import { createHash as createHash2 } from "crypto";
821
+ import { createHash as createHash3 } from "crypto";
324
822
  function verifyManifestSync(m) {
325
823
  if (m.algo !== void 0 && m.algo !== "sha256-content") {
326
824
  throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
@@ -328,7 +826,7 @@ function verifyManifestSync(m) {
328
826
  const { contentHash: contentHash2, algo: _algo, ...rest } = m;
329
827
  void _algo;
330
828
  const bytes = JSON.stringify(canonicalize(rest));
331
- const hash = createHash2("sha256").update(bytes, "utf8").digest("hex");
829
+ const hash = createHash3("sha256").update(bytes, "utf8").digest("hex");
332
830
  return hash === contentHash2;
333
831
  }
334
832
  function resolveConfig(opts) {
@@ -586,7 +1084,7 @@ function sequentialDecide(options = {}) {
586
1084
  }
587
1085
 
588
1086
  // src/campaign/labeled-store/fs-adapter.ts
589
- import { createHash as createHash3 } from "crypto";
1087
+ import { createHash as createHash4 } from "crypto";
590
1088
  import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
591
1089
  import { join as join2 } from "path";
592
1090
  var LabeledScenarioStoreError = class extends Error {
@@ -785,7 +1283,7 @@ function matchesFilter(record, args, source) {
785
1283
  return true;
786
1284
  }
787
1285
  function sha256(input) {
788
- return createHash3("sha256").update(input).digest("hex").slice(0, 16);
1286
+ return createHash4("sha256").update(input).digest("hex").slice(0, 16);
789
1287
  }
790
1288
  function appendLine(path, line) {
791
1289
  if (existsSync2(path)) {
@@ -796,6 +1294,12 @@ function appendLine(path, line) {
796
1294
  }
797
1295
  }
798
1296
 
1297
+ // src/campaign/neutralize.ts
1298
+ var FILLER = "#";
1299
+ function neutralizeText(content) {
1300
+ return content.replace(/\S/g, FILLER);
1301
+ }
1302
+
799
1303
  // src/campaign/proposers/fapo.ts
800
1304
  var FAPO_LEVELS = ["prompt", "parameter", "structural"];
801
1305
  var MAX_FINDING_DEPTH = 16;
@@ -1737,8 +2241,8 @@ async function compareProposerEntries(opts) {
1737
2241
  });
1738
2242
  const score = {
1739
2243
  name: w.name,
1740
- baselineComposite: mean(baselineArr),
1741
- winnerComposite: mean(w.arr),
2244
+ baselineComposite: mean2(baselineArr),
2245
+ winnerComposite: mean2(w.arr),
1742
2246
  lift: boot.mean,
1743
2247
  liftCi: { low: boot.low, high: boot.high },
1744
2248
  costUsd: w.costUsd,
@@ -1775,7 +2279,7 @@ async function compareProposerEntries(opts) {
1775
2279
  });
1776
2280
  return { scores, best, pairwise, holdoutScenarioIds: scenarioIds };
1777
2281
  }
1778
- function mean(xs) {
2282
+ function mean2(xs) {
1779
2283
  return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
1780
2284
  }
1781
2285
  function slug(name) {
@@ -2013,8 +2517,177 @@ function renderScoreboardMarkdown(rows, opts = {}) {
2013
2517
  return out.join("\n");
2014
2518
  }
2015
2519
 
2520
+ // src/campaign/presets/run-lineage-loop.ts
2521
+ function toCandidate(p) {
2522
+ return isProposedCandidate(p) ? { surface: p.surface, ...p.rationale ? { rationale: p.rationale } : {} } : { surface: p };
2523
+ }
2524
+ async function runLineageLoop(opts) {
2525
+ const governor = opts.governor ?? heuristicGovernor();
2526
+ const populationSize = opts.populationSize ?? 4;
2527
+ if (populationSize < 1) {
2528
+ throw new Error("runLineageLoop: populationSize must be >= 1");
2529
+ }
2530
+ let proposer = opts.proposer;
2531
+ if (!proposer) {
2532
+ if (!opts.llm || !opts.model) {
2533
+ throw new Error(
2534
+ "runLineageLoop: a proposer is required \u2014 either inject `proposer`, or provide `llm` + `model` for the default gepaProposer."
2535
+ );
2536
+ }
2537
+ proposer = gepaProposer({
2538
+ llm: opts.llm,
2539
+ model: opts.model,
2540
+ target: opts.target ?? "agent surface",
2541
+ // GEPA combine-complementary-lessons is what the `merge` seam relies on.
2542
+ combineParents: true
2543
+ });
2544
+ }
2545
+ const scoringScenarios = opts.holdoutScenarios ?? opts.scenarios;
2546
+ let scoreSurface = opts.scoreSurface;
2547
+ if (!scoreSurface) {
2548
+ const dispatchWithSurface = opts.dispatchWithSurface;
2549
+ const runDir = opts.runDir;
2550
+ if (!dispatchWithSurface || !runDir) {
2551
+ throw new Error(
2552
+ "runLineageLoop: scoring is required \u2014 either inject `scoreSurface`, or provide `dispatchWithSurface` (the agent) + `runDir` for the default runCampaign scorer."
2553
+ );
2554
+ }
2555
+ if (scoringScenarios.length === 0) {
2556
+ throw new Error(
2557
+ "runLineageLoop: the default scorer needs at least one scenario (holdoutScenarios ?? scenarios is empty)."
2558
+ );
2559
+ }
2560
+ scoreSurface = async (surface) => {
2561
+ const campaign = await runCampaign({
2562
+ scenarios: scoringScenarios,
2563
+ dispatch: (scenario, ctx) => dispatchWithSurface(surface, scenario, ctx),
2564
+ dispatchRef: `lineage-loop:${surfaceHash(surface)}`,
2565
+ runDir: `${runDir}/score/${surfaceHash(surface)}`,
2566
+ ...opts.judges ? { judges: opts.judges } : {},
2567
+ ...opts.seed !== void 0 ? { seed: opts.seed } : {},
2568
+ ...opts.reps !== void 0 ? { reps: opts.reps } : {},
2569
+ ...opts.storage ? { storage: opts.storage } : {},
2570
+ ...opts.tracing ? { tracing: opts.tracing } : {},
2571
+ ...opts.expectUsage ? { expectUsage: opts.expectUsage } : {},
2572
+ ...opts.maxConcurrency !== void 0 ? { maxConcurrency: opts.maxConcurrency } : {},
2573
+ ...opts.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: opts.dispatchTimeoutMs } : {},
2574
+ ...opts.now ? { now: opts.now } : {}
2575
+ });
2576
+ const score = campaignMeanComposite(campaign);
2577
+ const byId = new Map(
2578
+ campaignBreakdown(campaign).scenarios.map((s) => [s.scenarioId, s.composite])
2579
+ );
2580
+ const scoreVector = scoringScenarios.map((s) => byId.get(s.id) ?? 0);
2581
+ return { score, scoreVector };
2582
+ };
2583
+ }
2584
+ const objectivesOf = (node) => {
2585
+ const objectives = {};
2586
+ if (node.scoreVector && node.scoreVector.length > 0) {
2587
+ scoringScenarios.forEach((s, i) => {
2588
+ objectives[s.id] = node.scoreVector[i] ?? 0;
2589
+ });
2590
+ if (Object.keys(objectives).length > 0) return objectives;
2591
+ }
2592
+ objectives.composite = node.score;
2593
+ return objectives;
2594
+ };
2595
+ const scoredSeeds = [];
2596
+ for (const seed of opts.seeds) {
2597
+ const measured = await scoreSurface(seed.surface);
2598
+ scoredSeeds.push({
2599
+ surface: seed.surface,
2600
+ track: seed.track,
2601
+ proposer: seed.proposer,
2602
+ score: measured.score,
2603
+ ...seed.vision !== void 0 ? { vision: seed.vision } : {},
2604
+ ...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {}
2605
+ });
2606
+ }
2607
+ const step = async (args) => {
2608
+ const proposed = await proposer.propose({
2609
+ currentSurface: args.tip.surface,
2610
+ history: [],
2611
+ findings: [],
2612
+ populationSize,
2613
+ generation: 0,
2614
+ signal: new AbortController().signal,
2615
+ paretoParents: []
2616
+ });
2617
+ const pool = [
2618
+ {
2619
+ surface: args.tip.surface,
2620
+ score: args.tip.score,
2621
+ ...args.tip.scoreVector !== void 0 ? { scoreVector: args.tip.scoreVector } : {},
2622
+ ...args.tip.rationale !== void 0 ? { rationale: args.tip.rationale } : {}
2623
+ }
2624
+ ];
2625
+ for (const p of proposed) {
2626
+ const { surface, rationale } = toCandidate(p);
2627
+ const measured = await scoreSurface(surface);
2628
+ pool.push({
2629
+ surface,
2630
+ score: measured.score,
2631
+ ...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
2632
+ ...rationale !== void 0 ? { rationale } : {}
2633
+ });
2634
+ }
2635
+ let best = pool[0];
2636
+ for (const entry of pool) {
2637
+ if (entry.score > best.score) best = entry;
2638
+ }
2639
+ return {
2640
+ surface: best.surface,
2641
+ score: best.score,
2642
+ ...best.scoreVector !== void 0 ? { scoreVector: best.scoreVector } : {},
2643
+ ...best.rationale !== void 0 ? { rationale: best.rationale } : {}
2644
+ };
2645
+ };
2646
+ const merge = async (args) => {
2647
+ const ordered = [...args.parents].sort((a, b) => b.score - a.score);
2648
+ const paretoParents = ordered.map((node) => ({
2649
+ surface: node.surface,
2650
+ surfaceHash: surfaceHash(node.surface),
2651
+ objectives: objectivesOf(node),
2652
+ composite: node.score,
2653
+ generation: node.generation
2654
+ }));
2655
+ const proposed = await proposer.propose({
2656
+ currentSurface: ordered[0].surface,
2657
+ history: [],
2658
+ findings: [],
2659
+ // A merge is a single crossover; combineParents fires because
2660
+ // paretoParents has > 1 string member.
2661
+ populationSize: 1,
2662
+ // generation >= 1 keeps the combine slot semantically a "merge" step.
2663
+ generation: 1,
2664
+ signal: new AbortController().signal,
2665
+ paretoParents
2666
+ });
2667
+ const first = proposed[0];
2668
+ const merged = first ? toCandidate(first) : { surface: ordered[0].surface };
2669
+ const measured = await scoreSurface(merged.surface);
2670
+ return {
2671
+ surface: merged.surface,
2672
+ score: measured.score,
2673
+ ...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
2674
+ ...merged.rationale !== void 0 ? { rationale: merged.rationale } : {}
2675
+ };
2676
+ };
2677
+ const result = await runLineage({
2678
+ seeds: scoredSeeds,
2679
+ step,
2680
+ merge,
2681
+ governor,
2682
+ budget: opts.budget,
2683
+ ...opts.store ? { store: opts.store } : {},
2684
+ ...opts.log ? { log: opts.log } : {}
2685
+ });
2686
+ return { lineage: result.lineage, best: result.best, steps: result.steps };
2687
+ }
2688
+
2016
2689
  // src/campaign/presets/run-profile-matrix.ts
2017
- import { createHash as createHash4 } from "crypto";
2690
+ import { createHash as createHash5 } from "crypto";
2018
2691
  import { join as join3 } from "path";
2019
2692
  var ProfileMatrixError = class extends AgentEvalError {
2020
2693
  constructor(message) {
@@ -2025,14 +2698,14 @@ function sanitize(id) {
2025
2698
  return id.replace(/[^a-zA-Z0-9_-]/g, "_");
2026
2699
  }
2027
2700
  function sha(input) {
2028
- return createHash4("sha256").update(JSON.stringify(input)).digest("hex");
2701
+ return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
2029
2702
  }
2030
- function mean2(xs) {
2703
+ function mean3(xs) {
2031
2704
  return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
2032
2705
  }
2033
2706
  function cellComposite(cell) {
2034
2707
  const composites = Object.values(cell.judgeScores).map((s) => s.composite);
2035
- return composites.length === 0 ? 0 : mean2(composites);
2708
+ return composites.length === 0 ? 0 : mean3(composites);
2036
2709
  }
2037
2710
  function requireResolvedModel(cell, profileId) {
2038
2711
  const resolved = cell.resolvedModel?.trim();
@@ -2068,7 +2741,7 @@ function buildRunRecord(args) {
2068
2741
  if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
2069
2742
  }
2070
2743
  const perDimMean = {};
2071
- for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean2(values);
2744
+ for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean3(values);
2072
2745
  let costUsd = cell.costUsd;
2073
2746
  let costEstimated = false;
2074
2747
  if (costUsd === 0 && cell.tokenUsage.output > 0 && isModelPriced(model)) {
@@ -2232,7 +2905,7 @@ async function runProfileMatrix(opts) {
2232
2905
  // one harness, so the first record's model is representative).
2233
2906
  model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
2234
2907
  records: profileRecords.length,
2235
- meanComposite: mean2(profileRecords.map(compositeOf)),
2908
+ meanComposite: mean3(profileRecords.map(compositeOf)),
2236
2909
  totalCostUsd: pricedTotalCostUsd,
2237
2910
  integrity: summarizeBackendIntegrity(profileRecords)
2238
2911
  };
@@ -2262,7 +2935,7 @@ function rollup(records, keyOf) {
2262
2935
  groups.set(key, arr);
2263
2936
  }
2264
2937
  const out = {};
2265
- for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
2938
+ for (const [key, xs] of groups) out[key] = { meanComposite: mean3(xs), n: xs.length };
2266
2939
  return out;
2267
2940
  }
2268
2941
  function rollupByPersona(records, scenarios, personaOf) {
@@ -2665,6 +3338,53 @@ ${report.slice(0, 800)}`,
2665
3338
  });
2666
3339
  }
2667
3340
 
3341
+ // src/campaign/scenario-selection.ts
3342
+ var DEFAULT_SATURATION_CEILING = 0.999;
3343
+ var VARIANCE_EPSILON = 1e-9;
3344
+ function moments(scores) {
3345
+ const n = scores.length;
3346
+ if (n === 0) return { meanScore: 0, variance: 0 };
3347
+ let sum = 0;
3348
+ for (const s of scores) sum += s;
3349
+ const meanScore = sum / n;
3350
+ let sqDev = 0;
3351
+ for (const s of scores) {
3352
+ const d = s - meanScore;
3353
+ sqDev += d * d;
3354
+ }
3355
+ return { meanScore, variance: sqDev / n };
3356
+ }
3357
+ function compareDiscrimination(a, b) {
3358
+ if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
3359
+ if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
3360
+ return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
3361
+ }
3362
+ function scoreDiscrimination(signals, opts) {
3363
+ const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
3364
+ const scored = signals.map((signal) => {
3365
+ const { meanScore, variance } = moments(signal.scores);
3366
+ const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
3367
+ return {
3368
+ scenarioId: signal.scenarioId,
3369
+ discrimination: variance,
3370
+ meanScore,
3371
+ variance,
3372
+ tied
3373
+ };
3374
+ });
3375
+ return scored.sort(compareDiscrimination);
3376
+ }
3377
+ function selectDiscriminative(signals, k, opts) {
3378
+ if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
3379
+ const ranked = scoreDiscrimination(signals, opts);
3380
+ if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
3381
+ const nonTied = ranked.filter((s) => !s.tied);
3382
+ if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
3383
+ const tied = ranked.filter((s) => s.tied);
3384
+ const fill = tied.slice(0, k - nonTied.length);
3385
+ return [...nonTied, ...fill].map((s) => s.scenarioId);
3386
+ }
3387
+
2668
3388
  // src/campaign/worktree/index.ts
2669
3389
  import { execFileSync } from "child_process";
2670
3390
  import { existsSync as existsSync3 } from "fs";
@@ -2727,6 +3447,7 @@ function resolveWorktreePath(surface, worktreeDir) {
2727
3447
  export {
2728
3448
  FsLabeledScenarioStore,
2729
3449
  LabeledScenarioStoreError,
3450
+ Lineage,
2730
3451
  ProfileMatrixError,
2731
3452
  SkillPatchParseError,
2732
3453
  WorktreeAdapterError,
@@ -2735,6 +3456,7 @@ export {
2735
3456
  buildAnalystSurfaceDispatch,
2736
3457
  buildEvidenceVector,
2737
3458
  buildLoopProvenanceRecord,
3459
+ callbackGovernor,
2738
3460
  campaignBreakdown,
2739
3461
  campaignMeanComposite,
2740
3462
  compareProposers,
@@ -2753,6 +3475,7 @@ export {
2753
3475
  fapoEscalationEntry,
2754
3476
  fapoProposer,
2755
3477
  fsCampaignStorage,
3478
+ fsLineageStore,
2756
3479
  gepaParetoEntry,
2757
3480
  gepaProposer,
2758
3481
  gepaReflectionEntry,
@@ -2760,15 +3483,20 @@ export {
2760
3483
  haloProposer,
2761
3484
  heldOutGate,
2762
3485
  heldoutSignificance,
3486
+ heuristicGovernor,
2763
3487
  inMemoryCampaignStorage,
2764
3488
  isProposedCandidate,
2765
3489
  labelTrustRank,
3490
+ lineageNodeId,
2766
3491
  llmJudge,
2767
3492
  loadEvalFixture,
2768
3493
  loadEvalFixtureScenarios,
2769
3494
  loopProvenanceSpans,
2770
3495
  makePlaybackDispatch,
3496
+ memLineageStore,
2771
3497
  memoryCurationProposer,
3498
+ neutralizationGate,
3499
+ neutralizeText,
2772
3500
  openAutoPr,
2773
3501
  pairHoldout,
2774
3502
  parameterSweepProposer,
@@ -2788,11 +3516,15 @@ export {
2788
3516
  runCampaign,
2789
3517
  runEval,
2790
3518
  runImprovementLoop,
3519
+ runLineage,
3520
+ runLineageLoop,
2791
3521
  runOptimization,
2792
3522
  runProfileMatrix,
2793
3523
  runSkillOpt,
3524
+ scoreDiscrimination,
2794
3525
  scoreUserStory,
2795
3526
  scoreboardSummary,
3527
+ selectDiscriminative,
2796
3528
  sequentialDecide,
2797
3529
  sequentialPairedGate,
2798
3530
  skillOptEntry,