@tangle-network/agent-eval 0.106.3 → 0.108.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +2 -2
  3. package/dist/adapters/http.d.ts +1 -1
  4. package/dist/adapters/langchain.d.ts +1 -1
  5. package/dist/adapters/otel.d.ts +1 -1
  6. package/dist/benchmarks/index.d.ts +3 -1
  7. package/dist/benchmarks/index.js +49 -3
  8. package/dist/campaign/index.d.ts +75 -8
  9. package/dist/campaign/index.js +74 -3321
  10. package/dist/campaign/index.js.map +1 -1
  11. package/dist/chunk-4VLZEPJ3.js +4306 -0
  12. package/dist/chunk-4VLZEPJ3.js.map +1 -0
  13. package/dist/{chunk-3E5KUXYZ.js → chunk-OVPVM4JC.js} +1064 -643
  14. package/dist/chunk-OVPVM4JC.js.map +1 -0
  15. package/dist/chunk-T6W5ADLG.js +766 -0
  16. package/dist/chunk-T6W5ADLG.js.map +1 -0
  17. package/dist/contract/index.d.ts +16 -6
  18. package/dist/contract/index.js +10 -11
  19. package/dist/contract/index.js.map +1 -1
  20. package/dist/{gepa-DQ3ruj18.d.ts → gepa-B3x5Ulcv.d.ts} +11 -40
  21. package/dist/hosted/index.d.ts +1 -1
  22. package/dist/index-pPtfoIJO.d.ts +423 -0
  23. package/dist/index.d.ts +6 -5
  24. package/dist/index.js +8 -7
  25. package/dist/index.js.map +1 -1
  26. package/dist/multishot/index.d.ts +1 -1
  27. package/dist/openapi.json +1 -1
  28. package/dist/{pre-registration-BkkTmtQG.d.ts → pre-registration-BUhVPzE7.d.ts} +1 -1
  29. package/dist/{provenance-Bsyjc67Z.d.ts → provenance-DdDhf6cg.d.ts} +3 -2
  30. package/dist/rl.d.ts +1 -1
  31. package/dist/rl.js +6 -6
  32. package/dist/storage-Dw_f7WMt.d.ts +39 -0
  33. package/dist/{types-Ctf7XIAL.d.ts → types-BdIv5dvA.d.ts} +11 -1
  34. package/docs/improvement-glossary.md +5 -1
  35. package/package.json +1 -1
  36. package/dist/chunk-2HLM4SJJ.js +0 -861
  37. package/dist/chunk-2HLM4SJJ.js.map +0 -1
  38. package/dist/chunk-3E5KUXYZ.js.map +0 -1
  39. package/dist/chunk-6QDKWHLS.js +0 -223
  40. package/dist/chunk-6QDKWHLS.js.map +0 -1
  41. package/dist/chunk-J22CKVXN.js +0 -421
  42. package/dist/chunk-J22CKVXN.js.map +0 -1
  43. package/dist/index-C2CC_dry.d.ts +0 -159
@@ -2,6 +2,9 @@ import {
2
2
  runCampaign,
3
3
  summarizeBackendIntegrity
4
4
  } from "./chunk-3PFZBGMR.js";
5
+ import {
6
+ detectRewardHacking
7
+ } from "./chunk-N22ZO7FV.js";
5
8
  import {
6
9
  callLlm
7
10
  } from "./chunk-FUCQVFMU.js";
@@ -549,6 +552,150 @@ function excerptAt(source, at, needleLength) {
549
552
  return (start > 0 ? "\u2026" : "") + source.slice(start, end) + (end < source.length ? "\u2026" : "");
550
553
  }
551
554
 
555
+ // src/campaign/auto-pr.ts
556
+ import { execSync } from "child_process";
557
+ import { writeFileSync } from "fs";
558
+ import { tmpdir } from "os";
559
+ import { join } from "path";
560
+ function openAutoPr(options) {
561
+ if (options.gate.decision !== "ship") {
562
+ return {
563
+ opened: false,
564
+ dryRun: false,
565
+ reason: `gate verdict was "${options.gate.decision}" \u2014 refusing to open PR`
566
+ };
567
+ }
568
+ const dryRun = options.dryRun ?? !process.env.GH_AUTO_PR_TOKEN;
569
+ const branch = options.branch ?? `auto/${options.result.manifestHash.slice(0, 12)}`;
570
+ const title = options.title ?? `auto: campaign ${options.result.manifestHash.slice(0, 8)} promoted by gate`;
571
+ const body = renderPrBody(options.result, options.gate, options.promotedDiff);
572
+ const bodyPath = join(tmpdir(), `auto-pr-body-${Date.now()}.md`);
573
+ writeFileSync(bodyPath, body);
574
+ if (dryRun) {
575
+ return {
576
+ opened: false,
577
+ dryRun: true,
578
+ reason: `dry-run (GH_AUTO_PR_TOKEN not set). Would create PR on ${options.ghOwner}/${options.ghRepo} branch ${branch}. Body at ${bodyPath}.`
579
+ };
580
+ }
581
+ const ghExec = options.ghExec ?? defaultGhExec;
582
+ const result = ghExec([
583
+ "pr",
584
+ "create",
585
+ "--repo",
586
+ `${options.ghOwner}/${options.ghRepo}`,
587
+ "--head",
588
+ branch,
589
+ "--title",
590
+ title,
591
+ "--body-file",
592
+ bodyPath
593
+ ]);
594
+ if (result.status !== 0) {
595
+ return {
596
+ opened: false,
597
+ dryRun: false,
598
+ reason: `gh pr create failed (exit ${result.status}): ${result.stderr.slice(0, 400)}`
599
+ };
600
+ }
601
+ const prUrl = result.stdout.trim();
602
+ return { opened: true, prUrl, dryRun: false, reason: "PR opened" };
603
+ }
604
+ function renderPrBody(result, gate, diff) {
605
+ const lines = [];
606
+ lines.push(`## Automated promotion by \`runImprovementLoop\``);
607
+ lines.push("");
608
+ lines.push(`**Manifest**: \`${result.manifestHash}\``);
609
+ lines.push(`**Seed**: ${result.seed}`);
610
+ lines.push(`**Duration**: ${Math.round(result.durationMs / 1e3)}s`);
611
+ lines.push(
612
+ `**Cells**: executed ${result.aggregates.cellsExecuted}, cached ${result.aggregates.cellsCached}, skipped ${result.aggregates.cellsSkipped}, failed ${result.aggregates.cellsFailed}`
613
+ );
614
+ lines.push(`**Total spend**: $${result.aggregates.totalCostUsd.toFixed(2)}`);
615
+ lines.push("");
616
+ lines.push(`### Gate verdict: \`${gate.decision}\``);
617
+ lines.push("");
618
+ for (const reason of gate.reasons) lines.push(`- ${reason}`);
619
+ if (gate.delta !== void 0) lines.push(`- delta: ${gate.delta.toFixed(3)}`);
620
+ lines.push("");
621
+ lines.push("### Contributing gates");
622
+ lines.push("");
623
+ lines.push("| gate | passed | detail |");
624
+ lines.push("|---|---|---|");
625
+ for (const c of gate.contributingGates) {
626
+ const detail = typeof c.detail === "object" ? JSON.stringify(c.detail).slice(0, 80) : String(c.detail).slice(0, 80);
627
+ lines.push(`| ${c.name} | ${c.passed ? "\u2713" : "\u2717"} | ${detail} |`);
628
+ }
629
+ lines.push("");
630
+ lines.push("### Promoted surface");
631
+ lines.push("");
632
+ lines.push("```diff");
633
+ lines.push(diff.slice(0, 8e3));
634
+ lines.push("```");
635
+ lines.push("");
636
+ lines.push("### By-judge aggregates");
637
+ lines.push("");
638
+ lines.push("| judge | mean | ci95 | n |");
639
+ lines.push("|---|---|---|---|");
640
+ for (const [name, agg] of Object.entries(result.aggregates.byJudge)) {
641
+ lines.push(
642
+ `| ${name} | ${agg.mean.toFixed(3)} | [${agg.ci95[0].toFixed(3)}, ${agg.ci95[1].toFixed(3)}] | ${agg.n} |`
643
+ );
644
+ }
645
+ return lines.join("\n");
646
+ }
647
+ function defaultGhExec(args) {
648
+ try {
649
+ const stdout = execSync(`gh ${args.map(quoteArg).join(" ")}`, {
650
+ env: { ...process.env, GH_TOKEN: process.env.GH_AUTO_PR_TOKEN ?? process.env.GH_TOKEN ?? "" },
651
+ stdio: ["ignore", "pipe", "pipe"]
652
+ }).toString("utf8");
653
+ return { stdout, stderr: "", status: 0 };
654
+ } catch (err) {
655
+ const e = err;
656
+ return {
657
+ stdout: e.stdout?.toString("utf8") ?? "",
658
+ stderr: e.stderr?.toString("utf8") ?? "",
659
+ status: e.status ?? 1
660
+ };
661
+ }
662
+ }
663
+ function quoteArg(arg) {
664
+ if (/^[a-zA-Z0-9_/\-:.@]+$/.test(arg)) return arg;
665
+ return `"${arg.replace(/"/g, '\\"')}"`;
666
+ }
667
+
668
+ // src/campaign/gates/compose.ts
669
+ function composeGate(...gates) {
670
+ if (gates.length === 0) {
671
+ throw new Error("composeGate requires at least one gate");
672
+ }
673
+ return {
674
+ name: `composed(${gates.map((g) => g.name).join(",")})`,
675
+ async decide(ctx) {
676
+ const results = [];
677
+ for (const gate of gates) {
678
+ const res = await gate.decide(ctx);
679
+ results.push({ gate, res });
680
+ }
681
+ const decisions = results.map((r) => r.res.decision);
682
+ const overall = decisions.every((d) => d === "ship") ? "ship" : decisions.includes("arch_ceiling") ? "arch_ceiling" : decisions.includes("model_ceiling") ? "model_ceiling" : decisions.includes("hold") ? "hold" : "need_more_work";
683
+ const contributing = results.flatMap(
684
+ (r) => r.res.contributingGates.length > 0 ? r.res.contributingGates : [{ name: r.gate.name, passed: r.res.decision === "ship", detail: r.res }]
685
+ );
686
+ const reasons = results.flatMap(
687
+ (r) => r.res.reasons.map((reason) => `[${r.gate.name}] ${reason}`)
688
+ );
689
+ return {
690
+ decision: overall,
691
+ reasons,
692
+ contributingGates: contributing,
693
+ delta: results[0]?.res.delta
694
+ };
695
+ }
696
+ };
697
+ }
698
+
552
699
  // src/canary.ts
553
700
  function runCanaries(runs, opts = {}) {
554
701
  const alerts = [
@@ -767,383 +914,279 @@ function mean(xs) {
767
914
  return xs.reduce((s, x) => s + x, 0) / xs.length;
768
915
  }
769
916
 
770
- // src/json-recovery.ts
771
- function autoCloseTruncatedJson(raw) {
772
- const stack = [];
773
- let inString = false;
774
- let escaped = false;
775
- for (const c of raw) {
776
- if (escaped) {
777
- escaped = false;
778
- continue;
779
- }
780
- if (inString) {
781
- if (c === "\\") {
782
- escaped = true;
783
- continue;
784
- }
785
- if (c === '"') {
786
- inString = false;
787
- continue;
788
- }
789
- continue;
790
- }
791
- if (c === '"') {
792
- inString = true;
793
- continue;
794
- }
795
- if (c === "{" || c === "[") stack.push(c);
796
- else if (c === "}") {
797
- if (stack.pop() !== "{") return null;
798
- } else if (c === "]") {
799
- if (stack.pop() !== "[") return null;
917
+ // src/campaign/gates/statistical-heldout.ts
918
+ var TIE_WARN_FRACTION = 0.4;
919
+ function pairHoldout(candidate, baseline, scenarioIds, select) {
920
+ const cellValue = (byCell, cellId) => {
921
+ const scores = byCell.get(cellId);
922
+ if (!scores) return void 0;
923
+ const vals = [];
924
+ for (const s of Object.values(scores)) {
925
+ const v = select(s);
926
+ if (typeof v === "number" && Number.isFinite(v)) vals.push(v);
800
927
  }
928
+ if (vals.length === 0) return void 0;
929
+ return vals.reduce((a, b) => a + b, 0) / vals.length;
930
+ };
931
+ const inScope = (cellId) => scenarioIds.has(cellId.split(":")[0] ?? "");
932
+ const candCells = [...candidate.keys()].filter(inScope).sort();
933
+ const baseCells = [...baseline.keys()].filter(inScope).sort();
934
+ if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {
935
+ throw new Error(
936
+ `pairHoldout: candidate/baseline holdout cells do not align \u2014 candidate=[${candCells.join(",")}] baseline=[${baseCells.join(",")}]. Both holdout campaigns must run the same scenarios with the same seed base.`
937
+ );
801
938
  }
802
- if (stack.length === 0 && !inString) return raw;
803
- let suffix = "";
804
- if (escaped) suffix += "\\";
805
- if (inString) suffix += '"';
806
- while (stack.length > 0) {
807
- const opener = stack.pop();
808
- suffix += opener === "{" ? "}" : "]";
939
+ const before = [];
940
+ const after = [];
941
+ const cellIds = [];
942
+ for (const cellId of candCells) {
943
+ const b = cellValue(baseline, cellId);
944
+ const a = cellValue(candidate, cellId);
945
+ if (b === void 0 || a === void 0) continue;
946
+ before.push(b);
947
+ after.push(a);
948
+ cellIds.push(cellId);
809
949
  }
810
- return raw + suffix;
950
+ return { before, after, cellIds };
811
951
  }
812
- var UNPARSEABLE = /* @__PURE__ */ Symbol("unparseable");
813
- function tryParse(candidate) {
814
- try {
815
- return JSON.parse(candidate);
816
- } catch {
817
- return UNPARSEABLE;
952
+ function heldoutSignificance(paired, opts = {}) {
953
+ const deltaThreshold = opts.deltaThreshold ?? 0;
954
+ const minProductiveRuns = opts.minProductiveRuns ?? 3;
955
+ const confidence = opts.confidence ?? 0.95;
956
+ const resamples = opts.resamples ?? 2e3;
957
+ const seed = opts.seed ?? 1337;
958
+ const statistic = opts.statistic ?? "mean";
959
+ const bootstrap = pairedBootstrap(paired.before, paired.after, {
960
+ confidence,
961
+ resamples,
962
+ statistic,
963
+ seed
964
+ });
965
+ const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
966
+ confidence,
967
+ resamples,
968
+ statistic: "median",
969
+ seed
970
+ });
971
+ const n = paired.before.length;
972
+ let ties = 0;
973
+ for (let i = 0; i < n; i += 1) {
974
+ const after = paired.after[i] ?? 0;
975
+ const before = paired.before[i] ?? 0;
976
+ if (Math.abs(after - before) < 1e-9) ties += 1;
818
977
  }
978
+ const tieFraction = n === 0 ? 0 : ties / n;
979
+ const fewRuns = n < minProductiveRuns;
980
+ const significant = !fewRuns && bootstrap.low > deltaThreshold;
981
+ return { paired, bootstrap, medianBootstrap, tieFraction, n, significant, fewRuns };
819
982
  }
820
- function lastCommaOutsideString(s) {
821
- let inString = false;
822
- let escaped = false;
823
- let last = -1;
824
- for (let i = 0; i < s.length; i++) {
825
- const c = s[i];
826
- if (escaped) {
827
- escaped = false;
828
- continue;
829
- }
830
- if (inString) {
831
- if (c === "\\") escaped = true;
832
- else if (c === '"') inString = false;
833
- continue;
834
- }
835
- if (c === '"') inString = true;
836
- else if (c === ",") last = i;
837
- }
838
- return last;
983
+ function detectScale(values) {
984
+ return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
839
985
  }
840
- function recoverTruncatedJson(text) {
841
- const objStart = text.indexOf("{");
842
- const arrStart = text.indexOf("[");
843
- const starts = [objStart, arrStart].filter((i) => i >= 0).sort((a, b) => a - b);
844
- for (const start of starts) {
845
- const recovered = recoverFrom(text.slice(start));
846
- if (recovered !== UNPARSEABLE) return recovered;
847
- }
848
- return null;
849
- }
850
- function recoverFrom(slice) {
851
- let candidate = slice;
852
- const lastClose = Math.max(candidate.lastIndexOf("}"), candidate.lastIndexOf("]"));
853
- if (lastClose > 0) {
854
- const balanced = tryParse(candidate.slice(0, lastClose + 1));
855
- if (balanced !== UNPARSEABLE) return balanced;
856
- }
857
- for (let i = 0; i < 64; i++) {
858
- const closed = autoCloseTruncatedJson(candidate);
859
- if (closed !== null) {
860
- const parsed = tryParse(closed);
861
- if (parsed !== UNPARSEABLE) return parsed;
862
- }
863
- const cut = lastCommaOutsideString(candidate);
864
- if (cut <= 0) return UNPARSEABLE;
865
- candidate = candidate.slice(0, cut);
986
+ function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
987
+ const out = [];
988
+ for (const dim of criticalDimensions) {
989
+ const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
990
+ if (paired.before.length === 0) continue;
991
+ const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
992
+ const bootstrap = pairedBootstrap(paired.before, paired.after, {
993
+ confidence: opts.confidence ?? 0.95,
994
+ resamples: opts.resamples ?? 2e3,
995
+ statistic: "median",
996
+ seed: opts.seed ?? 1337
997
+ });
998
+ out.push({
999
+ dimension: dim,
1000
+ bootstrap,
1001
+ regressed: bootstrap.low < -tolerance,
1002
+ tolerance,
1003
+ n: paired.before.length
1004
+ });
866
1005
  }
867
- return UNPARSEABLE;
1006
+ return out;
868
1007
  }
869
1008
 
870
- // src/reflective-mutation.ts
871
- var DEFAULT_MUTATION_PRIMITIVES = [
872
- 'Strengthen an imperative ("should" \u2192 "must")',
873
- "Add a concrete example pulled from a missed-golden phrase",
874
- "Remove a redundant rule that did not improve recall",
875
- 'Add a counterfactual ("if X is missing, the score is capped at Y")',
876
- "Reorder sections so the highest-impact rule is first",
877
- "Replace abstract language with a domain-specific noun the trial misses"
878
- ];
879
- function buildReflectionPrompt(ctx) {
880
- const primitives = ctx.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES;
881
- const sections = [];
882
- sections.push(`# Mutation target: ${ctx.target}`);
883
- sections.push("");
884
- sections.push(
885
- `You are tuning the prompt component named \`${ctx.target}\`. The current variant is shown below; you have ${ctx.topTrials.length} top trials and ${ctx.bottomTrials.length} bottom trials as evidence. Propose ${ctx.childCount} mutation${ctx.childCount === 1 ? "" : "s"} that fix specific weaknesses visible in the bottom trials. Avoid blank rephrasings.`
886
- );
887
- sections.push("");
888
- sections.push("## Current variant");
889
- sections.push("```json");
890
- sections.push(JSON.stringify(ctx.parentPayload, null, 2));
891
- sections.push("```");
892
- sections.push("");
893
- if (ctx.bottomTrials.length > 0) {
894
- sections.push("## Failures (bottom trials) \u2014 what went wrong");
895
- sections.push("");
896
- for (const trial of ctx.bottomTrials) {
897
- sections.push(
898
- `### Trial \`${trial.id}\` \u2014 score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
1009
+ // src/campaign/gates/default-production-gate.ts
1010
+ function defaultProductionGate(options) {
1011
+ const deltaThreshold = options.deltaThreshold ?? 0;
1012
+ const confidence = options.confidence ?? 0.95;
1013
+ const resamples = options.bootstrapResamples ?? 2e3;
1014
+ const seed = options.bootstrapSeed ?? 1337;
1015
+ const minProductiveRuns = options.minProductiveRuns ?? 3;
1016
+ const heldoutStatistic = options.heldoutStatistic ?? "mean";
1017
+ const blockOnGaming = options.blockOnRewardHackingGaming ?? true;
1018
+ return {
1019
+ name: "defaultProductionGate",
1020
+ async decide(ctx) {
1021
+ const reasons = [];
1022
+ const contributing = [];
1023
+ const scenarioIds = new Set(options.holdoutScenarios.map((s) => s.id));
1024
+ const sig = heldoutSignificance(
1025
+ pairHoldout(
1026
+ ctx.judgeScores,
1027
+ ctx.baselineJudgeScores ?? ctx.judgeScores,
1028
+ scenarioIds,
1029
+ (s) => s.composite
1030
+ ),
1031
+ {
1032
+ deltaThreshold,
1033
+ minProductiveRuns,
1034
+ confidence,
1035
+ resamples,
1036
+ seed,
1037
+ statistic: heldoutStatistic
1038
+ }
899
1039
  );
900
- if (trial.failureNote) {
901
- sections.push("");
902
- sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 600)}`);
1040
+ const delta = heldoutStatistic === "median" ? sig.bootstrap.median : sig.bootstrap.mean;
1041
+ const heldoutPass = sig.significant;
1042
+ contributing.push({
1043
+ name: "heldout-significance",
1044
+ passed: heldoutPass,
1045
+ detail: {
1046
+ n: sig.n,
1047
+ delta,
1048
+ deltaMean: sig.bootstrap.mean,
1049
+ deltaMedianDiagnostic: sig.medianBootstrap.median,
1050
+ // Back-compat: prior consumers read `deltaMedian`. It now always carries
1051
+ // the median diagnostic (the ship decision keys on `delta`/mean).
1052
+ deltaMedian: sig.medianBootstrap.median,
1053
+ tieFraction: sig.tieFraction,
1054
+ ciLow: sig.bootstrap.low,
1055
+ ciHigh: sig.bootstrap.high,
1056
+ confidence: sig.bootstrap.confidence,
1057
+ deltaThreshold,
1058
+ fewRuns: sig.fewRuns
1059
+ }
1060
+ });
1061
+ if (!heldoutPass) {
1062
+ const tieNote = sig.tieFraction >= TIE_WARN_FRACTION ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied scenarios` : "";
1063
+ reasons.push(
1064
+ sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${minProductiveRuns}) \u2014 too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 threshold ${deltaThreshold} (${heldoutStatistic} \u0394 ${delta.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]${tieNote})`
1065
+ );
903
1066
  }
904
- const missed = (trial.expectations ?? []).filter((e) => !e.matched);
905
- if (missed.length > 0) {
906
- sections.push("");
907
- sections.push("**Missed expectations:**");
908
- for (const m of missed) {
909
- sections.push(`- \`${m.id}\`: should match phrase \`${quote(m.phrase)}\``);
1067
+ const dimRegs = options.criticalDimensions?.length ? dimensionRegressions(
1068
+ ctx.judgeScores,
1069
+ ctx.baselineJudgeScores ?? ctx.judgeScores,
1070
+ scenarioIds,
1071
+ options.criticalDimensions,
1072
+ { tolerance: options.regressionTolerance, confidence, resamples, seed }
1073
+ ) : [];
1074
+ const regressed = dimRegs.filter((d) => d.regressed);
1075
+ const dimPass = regressed.length === 0;
1076
+ contributing.push({
1077
+ name: "dimension-regression",
1078
+ passed: dimPass,
1079
+ detail: {
1080
+ guarded: options.criticalDimensions ?? [],
1081
+ regressions: dimRegs.map((d) => ({
1082
+ dimension: d.dimension,
1083
+ ciLow: d.bootstrap.low,
1084
+ median: d.bootstrap.median,
1085
+ tolerance: d.tolerance,
1086
+ n: d.n,
1087
+ regressed: d.regressed
1088
+ }))
910
1089
  }
1090
+ });
1091
+ if (!dimPass) {
1092
+ reasons.push(
1093
+ `critical dimension(s) regressed: ${regressed.map((d) => `${d.dimension} CI.low ${d.bootstrap.low.toFixed(3)} < -${d.tolerance}`).join("; ")}`
1094
+ );
911
1095
  }
912
- if (trial.emitted) {
913
- sections.push("");
914
- sections.push("**What the agent emitted:**");
915
- sections.push("```");
916
- sections.push(truncate(trial.emitted, 600));
917
- sections.push("```");
1096
+ const budgetPass = options.budgetUsd === void 0 || ctx.cost.candidate + ctx.cost.baseline <= options.budgetUsd;
1097
+ contributing.push({
1098
+ name: "budget",
1099
+ passed: budgetPass,
1100
+ detail: {
1101
+ candidateUsd: ctx.cost.candidate,
1102
+ baselineUsd: ctx.cost.baseline,
1103
+ budgetUsd: options.budgetUsd
1104
+ }
1105
+ });
1106
+ if (!budgetPass) {
1107
+ reasons.push(
1108
+ `spend ${(ctx.cost.candidate + ctx.cost.baseline).toFixed(2)} > budget ${options.budgetUsd}`
1109
+ );
918
1110
  }
919
- sections.push("");
920
- }
921
- }
922
- if (ctx.topTrials.length > 0) {
923
- sections.push("## Successes (top trials) \u2014 what to preserve");
924
- sections.push("");
925
- for (const trial of ctx.topTrials) {
926
- sections.push(
927
- `- \`${trial.id}\`: score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
1111
+ const redTeamFindings = options.redTeamBattery ? probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery) : { passed: true, findings: [] };
1112
+ contributing.push({
1113
+ name: "red-team",
1114
+ passed: redTeamFindings.passed,
1115
+ detail: {
1116
+ failures: redTeamFindings.findings.length,
1117
+ sample: redTeamFindings.findings.slice(0, 3)
1118
+ }
1119
+ });
1120
+ if (!redTeamFindings.passed) {
1121
+ reasons.push(`red-team probe failed (${redTeamFindings.findings.length} findings)`);
1122
+ }
1123
+ let rewardHackingReport = null;
1124
+ if (options.recentRuns && options.recentRuns.length >= 10) {
1125
+ rewardHackingReport = detectRewardHacking({ runs: options.recentRuns });
1126
+ }
1127
+ const gamingThreshold = 0.6;
1128
+ const gamingFindings = (rewardHackingReport?.findings ?? []).filter(
1129
+ (f) => f.severity >= gamingThreshold
928
1130
  );
929
- }
930
- sections.push("");
931
- }
932
- sections.push("## Allowed mutation primitives");
933
- sections.push("");
934
- for (const p of primitives) sections.push(`- ${p}`);
935
- sections.push("");
936
- sections.push("## Output schema");
937
- sections.push("");
938
- sections.push("Respond with a JSON object \u2014 no prose, no markdown fences:");
939
- sections.push("```json");
940
- sections.push(
941
- JSON.stringify(
942
- {
943
- proposals: [
944
- {
945
- label: "<short label, \u2264 40 chars>",
946
- rationale: "<which failure this targets and which primitive you used>",
947
- payload: "<full payload of the new variant \u2014 same shape as the current variant>"
948
- }
949
- ]
950
- },
951
- null,
952
- 2
953
- )
954
- );
955
- sections.push("```");
956
- return sections.join("\n");
957
- }
958
- function renderAnalystEvidence(findings, report) {
959
- const sections = [];
960
- const rows = Array.isArray(findings) ? findings : [];
961
- if (rows.length > 0) {
962
- const order = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
963
- const ranked = rows.map((f) => f && typeof f === "object" ? f : {}).sort((a, b) => (order[String(a.severity)] ?? 9) - (order[String(b.severity)] ?? 9)).slice(0, 12);
964
- sections.push("## Diagnosed findings (act on these root causes)");
965
- sections.push("");
966
- for (const f of ranked) {
967
- const sev = typeof f.severity === "string" ? f.severity : "info";
968
- const area = typeof f.area === "string" ? ` [${f.area}]` : "";
969
- const claim = typeof f.claim === "string" ? f.claim : JSON.stringify(f);
970
- sections.push(`- **${sev}**${area} ${truncate(claim, 300)}`);
971
- if (typeof f.recommended_action === "string" && f.recommended_action.trim()) {
972
- sections.push(` - fix: ${truncate(f.recommended_action, 200)}`);
1131
+ const rewardHackingPass = !rewardHackingReport || !blockOnGaming || gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
1132
+ contributing.push({
1133
+ name: "reward-hacking",
1134
+ passed: rewardHackingPass,
1135
+ detail: { report: rewardHackingReport, gamingFindingCount: gamingFindings.length }
1136
+ });
1137
+ if (!rewardHackingPass) {
1138
+ reasons.push(
1139
+ `reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`
1140
+ );
1141
+ }
1142
+ let canaryReport = null;
1143
+ if (options.recentRuns && options.recentRuns.length >= 10) {
1144
+ canaryReport = runCanaries(options.recentRuns, {});
1145
+ }
1146
+ const errorAlerts = (canaryReport?.alerts ?? []).filter((a) => a.severity === "error");
1147
+ const canaryPass = errorAlerts.length === 0;
1148
+ contributing.push({
1149
+ name: "canary",
1150
+ passed: canaryPass,
1151
+ detail: { totalAlerts: canaryReport?.alerts.length ?? 0, errorAlerts: errorAlerts.length }
1152
+ });
1153
+ if (!canaryPass) {
1154
+ reasons.push(`canary error alerts: ${errorAlerts.length}`);
973
1155
  }
1156
+ const allPassed = contributing.every((c) => c.passed);
1157
+ const decision = allPassed ? "ship" : "hold";
1158
+ return {
1159
+ decision,
1160
+ reasons: reasons.length > 0 ? reasons : ["all gates passed"],
1161
+ contributingGates: contributing,
1162
+ delta
1163
+ };
974
1164
  }
975
- sections.push("");
976
- }
977
- const reportText = typeof report === "string" ? report : report && typeof report === "object" ? JSON.stringify(report) : "";
978
- if (reportText.trim()) {
979
- sections.push("## Research report");
980
- sections.push("");
981
- sections.push(truncate(reportText.trim(), 1500));
982
- sections.push("");
983
- }
984
- if (sections.length === 0) return null;
985
- return sections.join("\n");
986
- }
987
- function truncate(s, max) {
988
- if (s.length <= max) return s;
989
- return `${s.slice(0, max)}\u2026 [truncated]`;
990
- }
991
- function quote(s) {
992
- return s.replace(/`/g, "\\`");
1165
+ };
993
1166
  }
994
- function parseReflectionResponse(raw, maxProposals) {
995
- let text = raw.trim();
996
- if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
997
- let parsed = null;
998
- const objectStart = text.indexOf("{");
999
- const objectEnd = text.lastIndexOf("}");
1000
- const arrayStart = text.indexOf("[");
1001
- const arrayEnd = text.lastIndexOf("]");
1002
- const tryObjectFirst = objectStart >= 0 && (arrayStart < 0 || objectStart < arrayStart);
1003
- const candidates = [];
1004
- if (tryObjectFirst) {
1005
- if (objectStart >= 0 && objectEnd > objectStart)
1006
- candidates.push(text.slice(objectStart, objectEnd + 1));
1007
- if (arrayStart >= 0 && arrayEnd > arrayStart)
1008
- candidates.push(text.slice(arrayStart, arrayEnd + 1));
1009
- } else {
1010
- if (arrayStart >= 0 && arrayEnd > arrayStart)
1011
- candidates.push(text.slice(arrayStart, arrayEnd + 1));
1012
- if (objectStart >= 0 && objectEnd > objectStart)
1013
- candidates.push(text.slice(objectStart, objectEnd + 1));
1014
- }
1015
- for (const slice of candidates) {
1016
- try {
1017
- parsed = JSON.parse(slice);
1018
- break;
1019
- } catch {
1020
- }
1021
- }
1022
- if (parsed == null) {
1023
- for (const slice of candidates) {
1024
- const closed = autoCloseTruncatedJson(slice);
1025
- if (closed != null && closed !== slice) {
1026
- try {
1027
- parsed = JSON.parse(closed);
1028
- break;
1029
- } catch {
1030
- }
1167
+ function probeRedTeam(artifacts, battery) {
1168
+ const findings = [];
1169
+ for (const [_cellId, artifact] of artifacts) {
1170
+ const text = extractText(artifact);
1171
+ if (text === void 0) continue;
1172
+ for (const rtCase of battery) {
1173
+ const finding = scoreRedTeamOutput(text, [], rtCase);
1174
+ if (!finding.passed) {
1175
+ findings.push({ scenarioId: rtCase.id, reason: finding.reason ?? "red-team probe failed" });
1031
1176
  }
1032
1177
  }
1033
1178
  }
1034
- if (parsed == null) return [];
1035
- let proposalsRaw;
1036
- if (Array.isArray(parsed)) {
1037
- proposalsRaw = parsed;
1038
- } else if (parsed && typeof parsed === "object") {
1039
- proposalsRaw = parsed.proposals;
1040
- }
1041
- if (!Array.isArray(proposalsRaw)) return [];
1042
- const out = [];
1043
- for (const p of proposalsRaw) {
1044
- if (!p || typeof p !== "object") continue;
1045
- const obj = p;
1046
- if (!("payload" in obj)) continue;
1047
- out.push({
1048
- label: typeof obj.label === "string" ? obj.label : "mutation",
1049
- rationale: typeof obj.rationale === "string" ? obj.rationale : "",
1050
- payload: obj.payload
1051
- });
1052
- if (maxProposals !== void 0 && out.length >= maxProposals) break;
1179
+ return { passed: findings.length === 0, findings };
1180
+ }
1181
+ function extractText(artifact) {
1182
+ if (typeof artifact === "string") return artifact;
1183
+ if (artifact && typeof artifact === "object") {
1184
+ const rec = artifact;
1185
+ if (typeof rec.text === "string") return rec.text;
1186
+ if (typeof rec.output === "string") return rec.output;
1187
+ if (typeof rec.content === "string") return rec.content;
1053
1188
  }
1054
- return out;
1055
- }
1056
-
1057
- // src/campaign/gates/statistical-heldout.ts
1058
- var TIE_WARN_FRACTION = 0.4;
1059
- function pairHoldout(candidate, baseline, scenarioIds, select) {
1060
- const cellValue = (byCell, cellId) => {
1061
- const scores = byCell.get(cellId);
1062
- if (!scores) return void 0;
1063
- const vals = [];
1064
- for (const s of Object.values(scores)) {
1065
- const v = select(s);
1066
- if (typeof v === "number" && Number.isFinite(v)) vals.push(v);
1067
- }
1068
- if (vals.length === 0) return void 0;
1069
- return vals.reduce((a, b) => a + b, 0) / vals.length;
1070
- };
1071
- const inScope = (cellId) => scenarioIds.has(cellId.split(":")[0] ?? "");
1072
- const candCells = [...candidate.keys()].filter(inScope).sort();
1073
- const baseCells = [...baseline.keys()].filter(inScope).sort();
1074
- if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {
1075
- throw new Error(
1076
- `pairHoldout: candidate/baseline holdout cells do not align \u2014 candidate=[${candCells.join(",")}] baseline=[${baseCells.join(",")}]. Both holdout campaigns must run the same scenarios with the same seed base.`
1077
- );
1078
- }
1079
- const before = [];
1080
- const after = [];
1081
- const cellIds = [];
1082
- for (const cellId of candCells) {
1083
- const b = cellValue(baseline, cellId);
1084
- const a = cellValue(candidate, cellId);
1085
- if (b === void 0 || a === void 0) continue;
1086
- before.push(b);
1087
- after.push(a);
1088
- cellIds.push(cellId);
1089
- }
1090
- return { before, after, cellIds };
1091
- }
1092
- function heldoutSignificance(paired, opts = {}) {
1093
- const deltaThreshold = opts.deltaThreshold ?? 0;
1094
- const minProductiveRuns = opts.minProductiveRuns ?? 3;
1095
- const confidence = opts.confidence ?? 0.95;
1096
- const resamples = opts.resamples ?? 2e3;
1097
- const seed = opts.seed ?? 1337;
1098
- const statistic = opts.statistic ?? "mean";
1099
- const bootstrap = pairedBootstrap(paired.before, paired.after, {
1100
- confidence,
1101
- resamples,
1102
- statistic,
1103
- seed
1104
- });
1105
- const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
1106
- confidence,
1107
- resamples,
1108
- statistic: "median",
1109
- seed
1110
- });
1111
- const n = paired.before.length;
1112
- let ties = 0;
1113
- for (let i = 0; i < n; i += 1) {
1114
- const after = paired.after[i] ?? 0;
1115
- const before = paired.before[i] ?? 0;
1116
- if (Math.abs(after - before) < 1e-9) ties += 1;
1117
- }
1118
- const tieFraction = n === 0 ? 0 : ties / n;
1119
- const fewRuns = n < minProductiveRuns;
1120
- const significant = !fewRuns && bootstrap.low > deltaThreshold;
1121
- return { paired, bootstrap, medianBootstrap, tieFraction, n, significant, fewRuns };
1122
- }
1123
- function detectScale(values) {
1124
- return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
1125
- }
1126
- function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
1127
- const out = [];
1128
- for (const dim of criticalDimensions) {
1129
- const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
1130
- if (paired.before.length === 0) continue;
1131
- const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
1132
- const bootstrap = pairedBootstrap(paired.before, paired.after, {
1133
- confidence: opts.confidence ?? 0.95,
1134
- resamples: opts.resamples ?? 2e3,
1135
- statistic: "median",
1136
- seed: opts.seed ?? 1337
1137
- });
1138
- out.push({
1139
- dimension: dim,
1140
- bootstrap,
1141
- regressed: bootstrap.low < -tolerance,
1142
- tolerance,
1143
- n: paired.before.length
1144
- });
1145
- }
1146
- return out;
1189
+ return void 0;
1147
1190
  }
1148
1191
 
1149
1192
  // src/campaign/gates/heldout-gate.ts
@@ -1207,117 +1250,628 @@ function heldOutGate(options) {
1207
1250
  };
1208
1251
  }
1209
1252
 
1210
- // src/campaign/auto-pr.ts
1211
- import { execSync } from "child_process";
1212
- import { writeFileSync } from "fs";
1213
- import { tmpdir } from "os";
1214
- import { join } from "path";
1215
- function openAutoPr(options) {
1216
- if (options.gate.decision !== "ship") {
1217
- return {
1218
- opened: false,
1219
- dryRun: false,
1220
- reason: `gate verdict was "${options.gate.decision}" \u2014 refusing to open PR`
1221
- };
1253
+ // src/campaign/gates/power-preflight.ts
1254
+ function zFor(confidence) {
1255
+ if (confidence >= 0.99) return 2.576;
1256
+ if (confidence >= 0.95) return 1.96;
1257
+ if (confidence >= 0.9) return 1.645;
1258
+ return 1.282;
1259
+ }
1260
+ function powerPreflight(opts) {
1261
+ const composites = opts.baselineComposites.filter((v) => Number.isFinite(v));
1262
+ if (composites.length < 3) {
1263
+ throw new Error(
1264
+ `powerPreflight: need >= 3 finite baseline composites to estimate variance, got ${composites.length}`
1265
+ );
1222
1266
  }
1223
- const dryRun = options.dryRun ?? !process.env.GH_AUTO_PR_TOKEN;
1224
- const branch = options.branch ?? `auto/${options.result.manifestHash.slice(0, 12)}`;
1225
- const title = options.title ?? `auto: campaign ${options.result.manifestHash.slice(0, 8)} promoted by gate`;
1226
- const body = renderPrBody(options.result, options.gate, options.promotedDiff);
1227
- const bodyPath = join(tmpdir(), `auto-pr-body-${Date.now()}.md`);
1228
- writeFileSync(bodyPath, body);
1229
- if (dryRun) {
1230
- return {
1231
- opened: false,
1232
- dryRun: true,
1233
- reason: `dry-run (GH_AUTO_PR_TOKEN not set). Would create PR on ${options.ghOwner}/${options.ghRepo} branch ${branch}. Body at ${bodyPath}.`
1234
- };
1267
+ const deltaThreshold = opts.deltaThreshold ?? 0.05;
1268
+ const confidence = opts.confidence ?? 0.95;
1269
+ const n = opts.pairedN ?? composites.length;
1270
+ if (n < 2) throw new Error(`powerPreflight: pairedN must be >= 2, got ${n}`);
1271
+ const mean2 = composites.reduce((a, b) => a + b, 0) / composites.length;
1272
+ const variance = composites.reduce((a, b) => a + (b - mean2) * (b - mean2), 0) / (composites.length - 1);
1273
+ const sd = Math.sqrt(variance);
1274
+ const z = zFor(confidence);
1275
+ const mde = deltaThreshold + z * Math.SQRT2 * sd / Math.sqrt(n);
1276
+ const scaleAssumed = composites.every((v) => v >= -1e-3 && v <= 1.5);
1277
+ const headroom = Math.max(0, 1 - mean2);
1278
+ const underpowered = scaleAssumed && mde > headroom;
1279
+ const sharedChannelCaveat = opts.sharedScorerChannel ? "Holdout and gate share one scoring channel: raising n/reps reduces only idiosyncratic noise \u2014 systematic judge bias remains and this MDE is a lower bound. Full debiasing needs an independent second scoring channel (different judge/benchmark family)." : void 0;
1280
+ const recommendation = underpowered ? `UNDERPOWERED: minimum detectable lift ${mde.toFixed(3)} exceeds the ${headroom.toFixed(3)} headroom above the baseline (${mean2.toFixed(3)}) \u2014 no achievable effect can ship at this budget. Raise paired n (scenarios x reps) to ~${Math.ceil((z * Math.SQRT2 * sd / Math.max(headroom - deltaThreshold, 0.01)) ** 2)} or reduce worker variance before searching.` : `Minimum detectable lift at n=${n}: ${mde.toFixed(3)} (baseline sd ${sd.toFixed(3)}). Effects smaller than this cannot clear the gate; budget the search for effects you believe exceed it.`;
1281
+ return {
1282
+ n,
1283
+ sd,
1284
+ mde,
1285
+ baselineMean: mean2,
1286
+ headroom,
1287
+ underpowered,
1288
+ scaleAssumed,
1289
+ deltaThreshold,
1290
+ confidence,
1291
+ ...sharedChannelCaveat ? { sharedChannelCaveat } : {},
1292
+ recommendation: sharedChannelCaveat ? `${recommendation} ${sharedChannelCaveat}` : recommendation
1293
+ };
1294
+ }
1295
+
1296
+ // src/campaign/gates/promotion-policy.ts
1297
+ function buildEvidenceVector(ctx, objectives, opts = {}) {
1298
+ if (objectives.length === 0) {
1299
+ throw new Error("buildEvidenceVector: at least 1 objective required");
1235
1300
  }
1236
- const ghExec = options.ghExec ?? defaultGhExec;
1237
- const result = ghExec([
1238
- "pr",
1239
- "create",
1240
- "--repo",
1241
- `${options.ghOwner}/${options.ghRepo}`,
1242
- "--head",
1243
- branch,
1244
- "--title",
1245
- title,
1246
- "--body-file",
1247
- bodyPath
1248
- ]);
1249
- if (result.status !== 0) {
1250
- return {
1251
- opened: false,
1252
- dryRun: false,
1253
- reason: `gh pr create failed (exit ${result.status}): ${result.stderr.slice(0, 400)}`
1254
- };
1301
+ const minProductiveRuns = opts.minProductiveRuns ?? 3;
1302
+ const confidence = opts.confidence ?? 0.95;
1303
+ const resamples = opts.resamples ?? 2e3;
1304
+ const seed = opts.seed ?? 1337;
1305
+ const baseline = ctx.baselineJudgeScores ?? ctx.judgeScores;
1306
+ const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
1307
+ const axes = [];
1308
+ for (const obj of objectives) {
1309
+ let select;
1310
+ if (obj.source.kind === "composite") {
1311
+ select = (s) => s.composite;
1312
+ } else {
1313
+ const dim = obj.source.dimension;
1314
+ select = (s) => s.dimensions[dim];
1315
+ }
1316
+ const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select);
1317
+ const before = obj.direction === "maximize" ? paired.before : paired.after;
1318
+ const after = obj.direction === "maximize" ? paired.after : paired.before;
1319
+ const bootstrap = pairedBootstrap(before, after, {
1320
+ confidence,
1321
+ resamples,
1322
+ statistic: "median",
1323
+ seed
1324
+ });
1325
+ const n = paired.before.length;
1326
+ const floorTolerance = obj.floorTolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
1327
+ const gainThreshold = obj.gainThreshold ?? 0;
1328
+ const verdict = n < minProductiveRuns ? "few_runs" : bootstrap.low < -floorTolerance ? "regressed" : bootstrap.low > gainThreshold ? "improved" : "flat";
1329
+ axes.push({
1330
+ name: obj.name,
1331
+ source: obj.source,
1332
+ direction: obj.direction,
1333
+ bootstrap,
1334
+ n,
1335
+ gainThreshold,
1336
+ floorTolerance,
1337
+ verdict
1338
+ });
1255
1339
  }
1256
- const prUrl = result.stdout.trim();
1257
- return { opened: true, prUrl, dryRun: false, reason: "PR opened" };
1340
+ const ns = axes.map((a) => a.n).filter((n) => n > 0);
1341
+ const minN = ns.length > 0 ? Math.min(...ns) : 0;
1342
+ return { axes, minN, cost: { candidate: ctx.cost.candidate, baseline: ctx.cost.baseline } };
1343
+ }
1344
+ var paretoPolicy = (ev) => {
1345
+ const contributingGates = ev.axes.map((ax) => ({
1346
+ name: `objective:${ax.name}`,
1347
+ passed: ax.verdict === "improved",
1348
+ detail: {
1349
+ direction: ax.direction,
1350
+ source: ax.source,
1351
+ verdict: ax.verdict,
1352
+ n: ax.n,
1353
+ deltaMedian: ax.bootstrap.median,
1354
+ ciLow: ax.bootstrap.low,
1355
+ ciHigh: ax.bootstrap.high,
1356
+ confidence: ax.bootstrap.confidence,
1357
+ gainThreshold: ax.gainThreshold,
1358
+ floorTolerance: ax.floorTolerance
1359
+ }
1360
+ }));
1361
+ const regressed = ev.axes.filter((a) => a.verdict === "regressed");
1362
+ const fewRuns = ev.axes.filter((a) => a.verdict === "few_runs");
1363
+ const improved = ev.axes.filter((a) => a.verdict === "improved");
1364
+ let decision;
1365
+ const reasons = [];
1366
+ if (regressed.length > 0) {
1367
+ decision = "hold";
1368
+ for (const a of regressed) {
1369
+ reasons.push(
1370
+ `objective '${a.name}' regressed: good-direction CI.low ${a.bootstrap.low.toFixed(3)} < -${a.floorTolerance} (n=${a.n})`
1371
+ );
1372
+ }
1373
+ } else if (fewRuns.length > 0) {
1374
+ decision = "need_more_work";
1375
+ for (const a of fewRuns) {
1376
+ reasons.push(
1377
+ `objective '${a.name}' has only n=${a.n} paired runs \u2014 insufficient evidence to claim significance`
1378
+ );
1379
+ }
1380
+ } else if (improved.length > 0) {
1381
+ decision = "ship";
1382
+ reasons.push(
1383
+ `Pareto improvement at the confidence level: ${improved.map(
1384
+ (a) => `'${a.name}' +${a.bootstrap.median.toFixed(3)} (CI.low ${a.bootstrap.low.toFixed(3)})`
1385
+ ).join(", ")}; no objective regressed`
1386
+ );
1387
+ } else {
1388
+ decision = "hold";
1389
+ reasons.push(
1390
+ "no Pareto improvement: candidate statistically equivalent to baseline on every objective"
1391
+ );
1392
+ }
1393
+ const composite = ev.axes.find((a) => a.source.kind === "composite") ?? ev.axes[0];
1394
+ return { decision, reasons, contributingGates, delta: composite?.bootstrap.median };
1395
+ };
1396
+ function paretoSignificanceGate(options) {
1397
+ if (options.objectives.length === 0) {
1398
+ throw new Error("paretoSignificanceGate: at least 1 objective required");
1399
+ }
1400
+ const policy = options.policy ?? paretoPolicy;
1401
+ return {
1402
+ name: options.name ?? "paretoSignificanceGate",
1403
+ async decide(ctx) {
1404
+ const ev = buildEvidenceVector(ctx, options.objectives, options);
1405
+ return policy(ev);
1406
+ }
1407
+ };
1258
1408
  }
1259
- function renderPrBody(result, gate, diff) {
1260
- const lines = [];
1261
- lines.push(`## Automated promotion by \`runImprovementLoop\``);
1262
- lines.push("");
1263
- lines.push(`**Manifest**: \`${result.manifestHash}\``);
1264
- lines.push(`**Seed**: ${result.seed}`);
1265
- lines.push(`**Duration**: ${Math.round(result.durationMs / 1e3)}s`);
1266
- lines.push(
1267
- `**Cells**: executed ${result.aggregates.cellsExecuted}, cached ${result.aggregates.cellsCached}, skipped ${result.aggregates.cellsSkipped}, failed ${result.aggregates.cellsFailed}`
1268
- );
1269
- lines.push(`**Total spend**: $${result.aggregates.totalCostUsd.toFixed(2)}`);
1270
- lines.push("");
1271
- lines.push(`### Gate verdict: \`${gate.decision}\``);
1272
- lines.push("");
1273
- for (const reason of gate.reasons) lines.push(`- ${reason}`);
1274
- if (gate.delta !== void 0) lines.push(`- delta: ${gate.delta.toFixed(3)}`);
1275
- lines.push("");
1276
- lines.push("### Contributing gates");
1277
- lines.push("");
1278
- lines.push("| gate | passed | detail |");
1279
- lines.push("|---|---|---|");
1280
- for (const c of gate.contributingGates) {
1281
- const detail = typeof c.detail === "object" ? JSON.stringify(c.detail).slice(0, 80) : String(c.detail).slice(0, 80);
1282
- lines.push(`| ${c.name} | ${c.passed ? "\u2713" : "\u2717"} | ${detail} |`);
1409
+
1410
+ // src/campaign/types.ts
1411
+ function isProposedCandidate(value) {
1412
+ return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
1413
+ }
1414
+ var LABEL_TRUST_RANK = {
1415
+ unverified: 0,
1416
+ "verified-signal": 1,
1417
+ "human-rated": 2
1418
+ };
1419
+ function labelTrustRank(trust) {
1420
+ return LABEL_TRUST_RANK[trust ?? "unverified"];
1421
+ }
1422
+
1423
+ // src/json-recovery.ts
1424
+ function autoCloseTruncatedJson(raw) {
1425
+ const stack = [];
1426
+ let inString = false;
1427
+ let escaped = false;
1428
+ for (const c of raw) {
1429
+ if (escaped) {
1430
+ escaped = false;
1431
+ continue;
1432
+ }
1433
+ if (inString) {
1434
+ if (c === "\\") {
1435
+ escaped = true;
1436
+ continue;
1437
+ }
1438
+ if (c === '"') {
1439
+ inString = false;
1440
+ continue;
1441
+ }
1442
+ continue;
1443
+ }
1444
+ if (c === '"') {
1445
+ inString = true;
1446
+ continue;
1447
+ }
1448
+ if (c === "{" || c === "[") stack.push(c);
1449
+ else if (c === "}") {
1450
+ if (stack.pop() !== "{") return null;
1451
+ } else if (c === "]") {
1452
+ if (stack.pop() !== "[") return null;
1453
+ }
1283
1454
  }
1284
- lines.push("");
1285
- lines.push("### Promoted surface");
1286
- lines.push("");
1287
- lines.push("```diff");
1288
- lines.push(diff.slice(0, 8e3));
1289
- lines.push("```");
1290
- lines.push("");
1291
- lines.push("### By-judge aggregates");
1292
- lines.push("");
1293
- lines.push("| judge | mean | ci95 | n |");
1294
- lines.push("|---|---|---|---|");
1295
- for (const [name, agg] of Object.entries(result.aggregates.byJudge)) {
1296
- lines.push(
1297
- `| ${name} | ${agg.mean.toFixed(3)} | [${agg.ci95[0].toFixed(3)}, ${agg.ci95[1].toFixed(3)}] | ${agg.n} |`
1298
- );
1455
+ if (stack.length === 0 && !inString) return raw;
1456
+ let suffix = "";
1457
+ if (escaped) suffix += "\\";
1458
+ if (inString) suffix += '"';
1459
+ while (stack.length > 0) {
1460
+ const opener = stack.pop();
1461
+ suffix += opener === "{" ? "}" : "]";
1462
+ }
1463
+ return raw + suffix;
1464
+ }
1465
+ var UNPARSEABLE = /* @__PURE__ */ Symbol("unparseable");
1466
+ function tryParse(candidate) {
1467
+ try {
1468
+ return JSON.parse(candidate);
1469
+ } catch {
1470
+ return UNPARSEABLE;
1471
+ }
1472
+ }
1473
+ function lastCommaOutsideString(s) {
1474
+ let inString = false;
1475
+ let escaped = false;
1476
+ let last = -1;
1477
+ for (let i = 0; i < s.length; i++) {
1478
+ const c = s[i];
1479
+ if (escaped) {
1480
+ escaped = false;
1481
+ continue;
1482
+ }
1483
+ if (inString) {
1484
+ if (c === "\\") escaped = true;
1485
+ else if (c === '"') inString = false;
1486
+ continue;
1487
+ }
1488
+ if (c === '"') inString = true;
1489
+ else if (c === ",") last = i;
1490
+ }
1491
+ return last;
1492
+ }
1493
+ function recoverTruncatedJson(text) {
1494
+ const objStart = text.indexOf("{");
1495
+ const arrStart = text.indexOf("[");
1496
+ const starts = [objStart, arrStart].filter((i) => i >= 0).sort((a, b) => a - b);
1497
+ for (const start of starts) {
1498
+ const recovered = recoverFrom(text.slice(start));
1499
+ if (recovered !== UNPARSEABLE) return recovered;
1500
+ }
1501
+ return null;
1502
+ }
1503
+ function recoverFrom(slice) {
1504
+ let candidate = slice;
1505
+ const lastClose = Math.max(candidate.lastIndexOf("}"), candidate.lastIndexOf("]"));
1506
+ if (lastClose > 0) {
1507
+ const balanced = tryParse(candidate.slice(0, lastClose + 1));
1508
+ if (balanced !== UNPARSEABLE) return balanced;
1509
+ }
1510
+ for (let i = 0; i < 64; i++) {
1511
+ const closed = autoCloseTruncatedJson(candidate);
1512
+ if (closed !== null) {
1513
+ const parsed = tryParse(closed);
1514
+ if (parsed !== UNPARSEABLE) return parsed;
1515
+ }
1516
+ const cut = lastCommaOutsideString(candidate);
1517
+ if (cut <= 0) return UNPARSEABLE;
1518
+ candidate = candidate.slice(0, cut);
1519
+ }
1520
+ return UNPARSEABLE;
1521
+ }
1522
+
1523
+ // src/reflective-mutation.ts
1524
+ var DEFAULT_MUTATION_PRIMITIVES = [
1525
+ 'Strengthen an imperative ("should" \u2192 "must")',
1526
+ "Add a concrete example pulled from a missed-golden phrase",
1527
+ "Remove a redundant rule that did not improve recall",
1528
+ 'Add a counterfactual ("if X is missing, the score is capped at Y")',
1529
+ "Reorder sections so the highest-impact rule is first",
1530
+ "Replace abstract language with a domain-specific noun the trial misses"
1531
+ ];
1532
+ function buildReflectionPrompt(ctx) {
1533
+ const primitives = ctx.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES;
1534
+ const sections = [];
1535
+ sections.push(`# Mutation target: ${ctx.target}`);
1536
+ sections.push("");
1537
+ sections.push(
1538
+ `You are tuning the prompt component named \`${ctx.target}\`. The current variant is shown below; you have ${ctx.topTrials.length} top trials and ${ctx.bottomTrials.length} bottom trials as evidence. Propose ${ctx.childCount} mutation${ctx.childCount === 1 ? "" : "s"} that fix specific weaknesses visible in the bottom trials. Avoid blank rephrasings.`
1539
+ );
1540
+ sections.push("");
1541
+ sections.push("## Current variant");
1542
+ sections.push("```json");
1543
+ sections.push(JSON.stringify(ctx.parentPayload, null, 2));
1544
+ sections.push("```");
1545
+ sections.push("");
1546
+ if (ctx.bottomTrials.length > 0) {
1547
+ sections.push("## Failures (bottom trials) \u2014 what went wrong");
1548
+ sections.push("");
1549
+ for (const trial of ctx.bottomTrials) {
1550
+ sections.push(
1551
+ `### Trial \`${trial.id}\` \u2014 score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
1552
+ );
1553
+ if (trial.failureNote) {
1554
+ sections.push("");
1555
+ sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 600)}`);
1556
+ }
1557
+ const missed = (trial.expectations ?? []).filter((e) => !e.matched);
1558
+ if (missed.length > 0) {
1559
+ sections.push("");
1560
+ sections.push("**Missed expectations:**");
1561
+ for (const m of missed) {
1562
+ sections.push(`- \`${m.id}\`: should match phrase \`${quote(m.phrase)}\``);
1563
+ }
1564
+ }
1565
+ if (trial.emitted) {
1566
+ sections.push("");
1567
+ sections.push("**What the agent emitted:**");
1568
+ sections.push("```");
1569
+ sections.push(truncate(trial.emitted, 600));
1570
+ sections.push("```");
1571
+ }
1572
+ sections.push("");
1573
+ }
1574
+ }
1575
+ if (ctx.topTrials.length > 0) {
1576
+ sections.push("## Successes (top trials) \u2014 what to preserve");
1577
+ sections.push("");
1578
+ for (const trial of ctx.topTrials) {
1579
+ sections.push(
1580
+ `- \`${trial.id}\`: score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
1581
+ );
1582
+ }
1583
+ sections.push("");
1584
+ }
1585
+ sections.push("## Allowed mutation primitives");
1586
+ sections.push("");
1587
+ for (const p of primitives) sections.push(`- ${p}`);
1588
+ sections.push("");
1589
+ sections.push("## Output schema");
1590
+ sections.push("");
1591
+ sections.push("Respond with a JSON object \u2014 no prose, no markdown fences:");
1592
+ sections.push("```json");
1593
+ sections.push(
1594
+ JSON.stringify(
1595
+ {
1596
+ proposals: [
1597
+ {
1598
+ label: "<short label, \u2264 40 chars>",
1599
+ rationale: "<which failure this targets and which primitive you used>",
1600
+ payload: "<full payload of the new variant \u2014 same shape as the current variant>"
1601
+ }
1602
+ ]
1603
+ },
1604
+ null,
1605
+ 2
1606
+ )
1607
+ );
1608
+ sections.push("```");
1609
+ return sections.join("\n");
1610
+ }
1611
+ function renderAnalystEvidence(findings, report) {
1612
+ const sections = [];
1613
+ const rows = Array.isArray(findings) ? findings : [];
1614
+ if (rows.length > 0) {
1615
+ const order = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
1616
+ const ranked = rows.map((f) => f && typeof f === "object" ? f : {}).sort((a, b) => (order[String(a.severity)] ?? 9) - (order[String(b.severity)] ?? 9)).slice(0, 12);
1617
+ sections.push("## Diagnosed findings (act on these root causes)");
1618
+ sections.push("");
1619
+ for (const f of ranked) {
1620
+ const sev = typeof f.severity === "string" ? f.severity : "info";
1621
+ const area = typeof f.area === "string" ? ` [${f.area}]` : "";
1622
+ const claim = typeof f.claim === "string" ? f.claim : JSON.stringify(f);
1623
+ sections.push(`- **${sev}**${area} ${truncate(claim, 300)}`);
1624
+ if (typeof f.recommended_action === "string" && f.recommended_action.trim()) {
1625
+ sections.push(` - fix: ${truncate(f.recommended_action, 200)}`);
1626
+ }
1627
+ }
1628
+ sections.push("");
1629
+ }
1630
+ const reportText = typeof report === "string" ? report : report && typeof report === "object" ? JSON.stringify(report) : "";
1631
+ if (reportText.trim()) {
1632
+ sections.push("## Research report");
1633
+ sections.push("");
1634
+ sections.push(truncate(reportText.trim(), 1500));
1635
+ sections.push("");
1636
+ }
1637
+ if (sections.length === 0) return null;
1638
+ return sections.join("\n");
1639
+ }
1640
+ function truncate(s, max) {
1641
+ if (s.length <= max) return s;
1642
+ return `${s.slice(0, max)}\u2026 [truncated]`;
1643
+ }
1644
+ function quote(s) {
1645
+ return s.replace(/`/g, "\\`");
1646
+ }
1647
+ function parseReflectionResponse(raw, maxProposals) {
1648
+ let text = raw.trim();
1649
+ if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
1650
+ let parsed = null;
1651
+ const objectStart = text.indexOf("{");
1652
+ const objectEnd = text.lastIndexOf("}");
1653
+ const arrayStart = text.indexOf("[");
1654
+ const arrayEnd = text.lastIndexOf("]");
1655
+ const tryObjectFirst = objectStart >= 0 && (arrayStart < 0 || objectStart < arrayStart);
1656
+ const candidates = [];
1657
+ if (tryObjectFirst) {
1658
+ if (objectStart >= 0 && objectEnd > objectStart)
1659
+ candidates.push(text.slice(objectStart, objectEnd + 1));
1660
+ if (arrayStart >= 0 && arrayEnd > arrayStart)
1661
+ candidates.push(text.slice(arrayStart, arrayEnd + 1));
1662
+ } else {
1663
+ if (arrayStart >= 0 && arrayEnd > arrayStart)
1664
+ candidates.push(text.slice(arrayStart, arrayEnd + 1));
1665
+ if (objectStart >= 0 && objectEnd > objectStart)
1666
+ candidates.push(text.slice(objectStart, objectEnd + 1));
1667
+ }
1668
+ for (const slice of candidates) {
1669
+ try {
1670
+ parsed = JSON.parse(slice);
1671
+ break;
1672
+ } catch {
1673
+ }
1674
+ }
1675
+ if (parsed == null) {
1676
+ for (const slice of candidates) {
1677
+ const closed = autoCloseTruncatedJson(slice);
1678
+ if (closed != null && closed !== slice) {
1679
+ try {
1680
+ parsed = JSON.parse(closed);
1681
+ break;
1682
+ } catch {
1683
+ }
1684
+ }
1685
+ }
1686
+ }
1687
+ if (parsed == null) return [];
1688
+ let proposalsRaw;
1689
+ if (Array.isArray(parsed)) {
1690
+ proposalsRaw = parsed;
1691
+ } else if (parsed && typeof parsed === "object") {
1692
+ proposalsRaw = parsed.proposals;
1693
+ }
1694
+ if (!Array.isArray(proposalsRaw)) return [];
1695
+ const out = [];
1696
+ for (const p of proposalsRaw) {
1697
+ if (!p || typeof p !== "object") continue;
1698
+ const obj = p;
1699
+ if (!("payload" in obj)) continue;
1700
+ out.push({
1701
+ label: typeof obj.label === "string" ? obj.label : "mutation",
1702
+ rationale: typeof obj.rationale === "string" ? obj.rationale : "",
1703
+ payload: obj.payload
1704
+ });
1705
+ if (maxProposals !== void 0 && out.length >= maxProposals) break;
1706
+ }
1707
+ return out;
1708
+ }
1709
+
1710
+ // src/campaign/proposers/gepa.ts
1711
+ var REFLECTION_SYSTEM = 'You are an expert prompt engineer performing GEPA-style reflective mutation. You are given a prompt surface, its top trials (preserve what works) and its bottom trials (the evidence to fix). For each proposal, reason in this order before writing the payload: (1) LOCALIZE \u2014 point to the exact span of the current surface responsible for a bottom-trial failure; (2) DIAGNOSE the root cause (a missing rule, an ambiguous instruction, an over-broad directive), not just the symptom; (3) propose the MINIMAL, GENERALIZABLE edit that fixes the whole failure class \u2014 state it as a rule the agent should follow, never a patch memorized to the shown trials (that is overfitting and will not transfer to the held-out set); (4) PRESERVE every instruction the top trials depend on \u2014 do not delete or weaken working guidance. Put this localize\u2192diagnose\u2192fix reasoning in each proposal\'s `rationale`. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} where each `payload` is the FULL improved surface text. No prose outside the JSON.';
1712
+ var COMBINE_SYSTEM = 'You are an expert prompt engineer performing a GEPA "combine complementary lessons" merge. You are given several non-dominated versions of one surface; each is uniquely best on different scenarios. Produce ONE new version that keeps what makes each version strong on its winning scenarios and resolves conflicts in favor of the more general rule. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} with exactly one proposal whose `payload` is the FULL merged surface text. No prose outside the JSON.';
1713
+ function gepaProposer(opts) {
1714
+ const evidenceK = opts.evidenceK ?? 3;
1715
+ const combineParents = opts.combineParents ?? true;
1716
+ const combineMaxParents = opts.combineMaxParents ?? 4;
1717
+ if (combineParents && combineMaxParents < 1) {
1718
+ throw new Error("gepaProposer: combineMaxParents must be >= 1 when combineParents is enabled");
1719
+ }
1720
+ return {
1721
+ kind: "gepa",
1722
+ async propose(ctx) {
1723
+ const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
1724
+ const constraints = opts.constraints;
1725
+ const preserveSections = constraints?.preserveSections !== void 0 ? constraints.preserveSections.length === 0 ? extractH2Sections(parent) : constraints.preserveSections : null;
1726
+ const maxEdits = constraints?.maxSentenceEdits;
1727
+ const out = [];
1728
+ const seen = /* @__PURE__ */ new Set();
1729
+ const accept = (payload, label, rationale) => {
1730
+ const text = typeof payload === "string" ? payload.trim() : "";
1731
+ if (!text || text === parent || seen.has(text)) return;
1732
+ if (preserveSections && !validatePreservedSections(text, preserveSections)) return;
1733
+ if (maxEdits !== void 0 && countSentenceEdits(parent, text) > maxEdits * 2) return;
1734
+ seen.add(text);
1735
+ out.push({ surface: text, label, rationale });
1736
+ };
1737
+ const stringParents = (combineParents ? ctx.paretoParents ?? [] : []).filter((p) => typeof p.surface === "string").sort((a, b) => b.composite - a.composite).slice(0, combineMaxParents);
1738
+ if (stringParents.length > 1) {
1739
+ const combinePrompt = buildCombinePrompt({
1740
+ target: opts.target,
1741
+ parents: stringParents,
1742
+ evidenceK
1743
+ });
1744
+ const combineResult = await callLlm(
1745
+ {
1746
+ model: opts.model,
1747
+ messages: [
1748
+ { role: "system", content: COMBINE_SYSTEM },
1749
+ { role: "user", content: combinePrompt }
1750
+ ],
1751
+ jsonMode: true,
1752
+ temperature: opts.temperature ?? 0.7,
1753
+ maxTokens: opts.maxTokens ?? 6e3
1754
+ },
1755
+ opts.llm
1756
+ );
1757
+ const merged = parseReflectionResponse(combineResult.content, 1)[0];
1758
+ if (merged) {
1759
+ accept(
1760
+ merged.payload,
1761
+ merged.label || "pareto-combine",
1762
+ merged.rationale || `combined ${stringParents.length} non-dominated parents (gen ${stringParents.map((p) => p.generation).join(",")})`
1763
+ );
1764
+ }
1765
+ }
1766
+ const reflectCount = Math.max(0, ctx.populationSize - out.length);
1767
+ if (reflectCount > 0) {
1768
+ const { top, bottom, target } = buildEvidence(ctx, evidenceK, opts.target);
1769
+ const userPrompt = buildReflectionPrompt({
1770
+ target,
1771
+ parentPayload: parent,
1772
+ topTrials: top,
1773
+ bottomTrials: bottom,
1774
+ childCount: reflectCount,
1775
+ mutationPrimitives: opts.mutationPrimitives
1776
+ });
1777
+ const analyst = renderAnalystEvidence(ctx.findings, ctx.report);
1778
+ const finalPrompt = analyst ? `${userPrompt}
1779
+
1780
+ ${analyst}` : userPrompt;
1781
+ const result = await callLlm(
1782
+ {
1783
+ model: opts.model,
1784
+ messages: [
1785
+ { role: "system", content: REFLECTION_SYSTEM },
1786
+ { role: "user", content: finalPrompt }
1787
+ ],
1788
+ jsonMode: true,
1789
+ temperature: opts.temperature ?? 0.7,
1790
+ maxTokens: opts.maxTokens ?? 6e3
1791
+ },
1792
+ opts.llm
1793
+ );
1794
+ for (const proposal of parseReflectionResponse(result.content, reflectCount)) {
1795
+ accept(proposal.payload, proposal.label, proposal.rationale);
1796
+ }
1797
+ }
1798
+ return out.slice(0, ctx.populationSize);
1799
+ }
1800
+ };
1801
+ }
1802
+ function buildCombinePrompt(args) {
1803
+ const lines = [
1804
+ `You are merging ${args.parents.length} versions of: ${args.target}.`,
1805
+ "",
1806
+ "Each version is on the Pareto frontier \u2014 none dominates the others; each",
1807
+ "wins on different scenarios. Combine their complementary strengths into",
1808
+ "ONE version. Below, each version lists the scenarios it scores highest on.",
1809
+ ""
1810
+ ];
1811
+ args.parents.forEach((p, i) => {
1812
+ const tag = String.fromCharCode(65 + i);
1813
+ const best = Object.entries(p.objectives).sort((a, b) => b[1] - a[1]).slice(0, args.evidenceK).map(([id, score]) => `${id} (${score.toFixed(2)})`);
1814
+ lines.push(
1815
+ `### Version ${tag} (mean ${p.composite.toFixed(2)}; strongest on: ${best.join(", ") || "n/a"})`,
1816
+ "```",
1817
+ p.surface,
1818
+ "```",
1819
+ ""
1820
+ );
1821
+ });
1822
+ lines.push(
1823
+ "Return ONE merged version that would score well on the union of every",
1824
+ "version's winning scenarios. Keep each version's specific winning rule;",
1825
+ "where two rules conflict, prefer the more general one and note the choice",
1826
+ "in your rationale."
1827
+ );
1828
+ return lines.join("\n");
1829
+ }
1830
+ function extractH2Sections(text) {
1831
+ const out = [];
1832
+ for (const line of text.split("\n")) {
1833
+ const match = /^##\s+(.+?)\s*$/.exec(line);
1834
+ if (match) out.push(match[1]);
1299
1835
  }
1300
- return lines.join("\n");
1836
+ return out;
1301
1837
  }
1302
- function defaultGhExec(args) {
1303
- try {
1304
- const stdout = execSync(`gh ${args.map(quoteArg).join(" ")}`, {
1305
- env: { ...process.env, GH_TOKEN: process.env.GH_AUTO_PR_TOKEN ?? process.env.GH_TOKEN ?? "" },
1306
- stdio: ["ignore", "pipe", "pipe"]
1307
- }).toString("utf8");
1308
- return { stdout, stderr: "", status: 0 };
1309
- } catch (err) {
1310
- const e = err;
1311
- return {
1312
- stdout: e.stdout?.toString("utf8") ?? "",
1313
- stderr: e.stderr?.toString("utf8") ?? "",
1314
- status: e.status ?? 1
1315
- };
1838
+ function countSentenceEdits(baseline, candidate) {
1839
+ const norm = (s) => s.split(/(?<=[.!?])\s+|\n/g).map((p) => p.trim()).filter((p) => p.length > 0);
1840
+ const a = new Set(norm(baseline));
1841
+ const b = new Set(norm(candidate));
1842
+ let edits = 0;
1843
+ for (const s of a) if (!b.has(s)) edits++;
1844
+ for (const s of b) if (!a.has(s)) edits++;
1845
+ return edits;
1846
+ }
1847
+ function validatePreservedSections(candidate, required) {
1848
+ if (required.length === 0) return true;
1849
+ const have = new Set(extractH2Sections(candidate));
1850
+ for (const section of required) {
1851
+ if (!have.has(section)) return false;
1316
1852
  }
1853
+ return true;
1317
1854
  }
1318
- function quoteArg(arg) {
1319
- if (/^[a-zA-Z0-9_/\-:.@]+$/.test(arg)) return arg;
1320
- return `"${arg.replace(/"/g, '\\"')}"`;
1855
+ function buildEvidence(ctx, evidenceK, baseTarget) {
1856
+ const last = ctx.history.at(-1);
1857
+ if (!last || last.candidates.length === 0) {
1858
+ return { top: [], bottom: [], target: baseTarget };
1859
+ }
1860
+ const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
1861
+ if (!best) return { top: [], bottom: [], target: baseTarget };
1862
+ const byScore = [...best.scenarios].sort((a, b) => b.composite - a.composite);
1863
+ const toTrace = (s) => ({
1864
+ id: s.scenarioId,
1865
+ score: s.composite,
1866
+ // The judge's "why it scored low" — grounds the reflection on real failure
1867
+ // patterns instead of blind rephrasing. Generalizable by the judge contract.
1868
+ ...s.notes ? { failureNote: s.notes } : {}
1869
+ });
1870
+ const top = byScore.slice(0, evidenceK).map(toTrace);
1871
+ const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
1872
+ const weakest = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([dim, value]) => `${dim} (${value.toFixed(2)})`);
1873
+ const target = weakest.length > 0 ? `${baseTarget} \u2014 weakest dimensions: ${weakest.join(", ")}` : baseTarget;
1874
+ return { top, bottom, target };
1321
1875
  }
1322
1876
 
1323
1877
  // src/campaign/score-utils.ts
@@ -1374,19 +1928,6 @@ function campaignBreakdown(campaign) {
1374
1928
  return { dimensions, scenarios };
1375
1929
  }
1376
1930
 
1377
- // src/campaign/types.ts
1378
- function isProposedCandidate(value) {
1379
- return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
1380
- }
1381
- var LABEL_TRUST_RANK = {
1382
- unverified: 0,
1383
- "verified-signal": 1,
1384
- "human-rated": 2
1385
- };
1386
- function labelTrustRank(trust) {
1387
- return LABEL_TRUST_RANK[trust ?? "unverified"];
1388
- }
1389
-
1390
1931
  // src/campaign/presets/run-optimization.ts
1391
1932
  import { createHash } from "crypto";
1392
1933
  async function runOptimization(opts) {
@@ -1619,6 +2160,24 @@ async function runImprovementLoop(opts) {
1619
2160
  baselineArtifacts.set(cell.cellId, cell.artifact);
1620
2161
  baselineJudgeScores.set(cell.cellId, cell.judgeScores);
1621
2162
  }
2163
+ let neutralizedArtifacts;
2164
+ let neutralizedJudgeScores;
2165
+ if (opts.neutralize && !winnerIsBaseline) {
2166
+ const neutralizedSurface = opts.neutralize(optimization.winnerSurface, opts.baselineSurface);
2167
+ const neutralizedOnHoldout = await runCampaign2({
2168
+ ...opts,
2169
+ dispatchTimeoutMs,
2170
+ scenarios: opts.holdoutScenarios,
2171
+ dispatch: (scenario, ctx) => opts.dispatchWithSurface(neutralizedSurface, scenario, ctx),
2172
+ runDir: `${opts.runDir}/holdout-neutralized`
2173
+ });
2174
+ neutralizedArtifacts = /* @__PURE__ */ new Map();
2175
+ neutralizedJudgeScores = /* @__PURE__ */ new Map();
2176
+ for (const cell of neutralizedOnHoldout.cells) {
2177
+ neutralizedArtifacts.set(cell.cellId, cell.artifact);
2178
+ neutralizedJudgeScores.set(cell.cellId, cell.judgeScores);
2179
+ }
2180
+ }
1622
2181
  const gateResult = winnerIsBaseline ? {
1623
2182
  decision: "hold",
1624
2183
  reasons: [
@@ -1633,6 +2192,8 @@ async function runImprovementLoop(opts) {
1633
2192
  baselineArtifacts,
1634
2193
  judgeScores,
1635
2194
  baselineJudgeScores,
2195
+ neutralizedArtifacts,
2196
+ neutralizedJudgeScores,
1636
2197
  scenarios: opts.holdoutScenarios,
1637
2198
  cost: {
1638
2199
  candidate: winnerOnHoldout.aggregates.totalCostUsd,
@@ -1678,172 +2239,25 @@ ${fmt(winnerSurface)}`;
1678
2239
  return lines.join("\n");
1679
2240
  }
1680
2241
 
1681
- // src/campaign/proposers/gepa.ts
1682
- var REFLECTION_SYSTEM = 'You are an expert prompt engineer performing GEPA-style reflective mutation. You are given a prompt surface, its top trials (preserve what works) and its bottom trials (the evidence to fix). For each proposal, reason in this order before writing the payload: (1) LOCALIZE \u2014 point to the exact span of the current surface responsible for a bottom-trial failure; (2) DIAGNOSE the root cause (a missing rule, an ambiguous instruction, an over-broad directive), not just the symptom; (3) propose the MINIMAL, GENERALIZABLE edit that fixes the whole failure class \u2014 state it as a rule the agent should follow, never a patch memorized to the shown trials (that is overfitting and will not transfer to the held-out set); (4) PRESERVE every instruction the top trials depend on \u2014 do not delete or weaken working guidance. Put this localize\u2192diagnose\u2192fix reasoning in each proposal\'s `rationale`. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} where each `payload` is the FULL improved surface text. No prose outside the JSON.';
1683
- var COMBINE_SYSTEM = 'You are an expert prompt engineer performing a GEPA "combine complementary lessons" merge. You are given several non-dominated versions of one surface; each is uniquely best on different scenarios. Produce ONE new version that keeps what makes each version strong on its winning scenarios and resolves conflicts in favor of the more general rule. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} with exactly one proposal whose `payload` is the FULL merged surface text. No prose outside the JSON.';
1684
- function gepaProposer(opts) {
1685
- const evidenceK = opts.evidenceK ?? 3;
1686
- const combineParents = opts.combineParents ?? true;
1687
- const combineMaxParents = opts.combineMaxParents ?? 4;
1688
- if (combineParents && combineMaxParents < 1) {
1689
- throw new Error("gepaProposer: combineMaxParents must be >= 1 when combineParents is enabled");
1690
- }
1691
- return {
1692
- kind: "gepa",
1693
- async propose(ctx) {
1694
- const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
1695
- const constraints = opts.constraints;
1696
- const preserveSections = constraints?.preserveSections !== void 0 ? constraints.preserveSections.length === 0 ? extractH2Sections(parent) : constraints.preserveSections : null;
1697
- const maxEdits = constraints?.maxSentenceEdits;
1698
- const out = [];
1699
- const seen = /* @__PURE__ */ new Set();
1700
- const accept = (payload, label, rationale) => {
1701
- const text = typeof payload === "string" ? payload.trim() : "";
1702
- if (!text || text === parent || seen.has(text)) return;
1703
- if (preserveSections && !validatePreservedSections(text, preserveSections)) return;
1704
- if (maxEdits !== void 0 && countSentenceEdits(parent, text) > maxEdits * 2) return;
1705
- seen.add(text);
1706
- out.push({ surface: text, label, rationale });
1707
- };
1708
- const stringParents = (combineParents ? ctx.paretoParents ?? [] : []).filter((p) => typeof p.surface === "string").sort((a, b) => b.composite - a.composite).slice(0, combineMaxParents);
1709
- if (stringParents.length > 1) {
1710
- const combinePrompt = buildCombinePrompt({
1711
- target: opts.target,
1712
- parents: stringParents,
1713
- evidenceK
1714
- });
1715
- const combineResult = await callLlm(
1716
- {
1717
- model: opts.model,
1718
- messages: [
1719
- { role: "system", content: COMBINE_SYSTEM },
1720
- { role: "user", content: combinePrompt }
1721
- ],
1722
- jsonMode: true,
1723
- temperature: opts.temperature ?? 0.7,
1724
- maxTokens: opts.maxTokens ?? 6e3
1725
- },
1726
- opts.llm
1727
- );
1728
- const merged = parseReflectionResponse(combineResult.content, 1)[0];
1729
- if (merged) {
1730
- accept(
1731
- merged.payload,
1732
- merged.label || "pareto-combine",
1733
- merged.rationale || `combined ${stringParents.length} non-dominated parents (gen ${stringParents.map((p) => p.generation).join(",")})`
1734
- );
1735
- }
1736
- }
1737
- const reflectCount = Math.max(0, ctx.populationSize - out.length);
1738
- if (reflectCount > 0) {
1739
- const { top, bottom, target } = buildEvidence(ctx, evidenceK, opts.target);
1740
- const userPrompt = buildReflectionPrompt({
1741
- target,
1742
- parentPayload: parent,
1743
- topTrials: top,
1744
- bottomTrials: bottom,
1745
- childCount: reflectCount,
1746
- mutationPrimitives: opts.mutationPrimitives
1747
- });
1748
- const analyst = renderAnalystEvidence(ctx.findings, ctx.report);
1749
- const finalPrompt = analyst ? `${userPrompt}
2242
+ // src/campaign/presets/run-eval.ts
2243
+ async function runEval(opts) {
2244
+ return runCampaign(opts);
2245
+ }
1750
2246
 
1751
- ${analyst}` : userPrompt;
1752
- const result = await callLlm(
1753
- {
1754
- model: opts.model,
1755
- messages: [
1756
- { role: "system", content: REFLECTION_SYSTEM },
1757
- { role: "user", content: finalPrompt }
1758
- ],
1759
- jsonMode: true,
1760
- temperature: opts.temperature ?? 0.7,
1761
- maxTokens: opts.maxTokens ?? 6e3
1762
- },
1763
- opts.llm
1764
- );
1765
- for (const proposal of parseReflectionResponse(result.content, reflectCount)) {
1766
- accept(proposal.payload, proposal.label, proposal.rationale);
1767
- }
1768
- }
1769
- return out.slice(0, ctx.populationSize);
2247
+ // src/campaign/proposers/evolutionary.ts
2248
+ function evolutionaryProposer(opts) {
2249
+ return {
2250
+ kind: `evolutionary:${opts.mutator.kind}`,
2251
+ async propose({ currentSurface, findings, populationSize, signal }) {
2252
+ return opts.mutator.mutate({
2253
+ findings: findings.length > 0 ? findings : opts.findings ?? [],
2254
+ currentSurface,
2255
+ populationSize,
2256
+ signal
2257
+ });
1770
2258
  }
1771
2259
  };
1772
2260
  }
1773
- function buildCombinePrompt(args) {
1774
- const lines = [
1775
- `You are merging ${args.parents.length} versions of: ${args.target}.`,
1776
- "",
1777
- "Each version is on the Pareto frontier \u2014 none dominates the others; each",
1778
- "wins on different scenarios. Combine their complementary strengths into",
1779
- "ONE version. Below, each version lists the scenarios it scores highest on.",
1780
- ""
1781
- ];
1782
- args.parents.forEach((p, i) => {
1783
- const tag = String.fromCharCode(65 + i);
1784
- const best = Object.entries(p.objectives).sort((a, b) => b[1] - a[1]).slice(0, args.evidenceK).map(([id, score]) => `${id} (${score.toFixed(2)})`);
1785
- lines.push(
1786
- `### Version ${tag} (mean ${p.composite.toFixed(2)}; strongest on: ${best.join(", ") || "n/a"})`,
1787
- "```",
1788
- p.surface,
1789
- "```",
1790
- ""
1791
- );
1792
- });
1793
- lines.push(
1794
- "Return ONE merged version that would score well on the union of every",
1795
- "version's winning scenarios. Keep each version's specific winning rule;",
1796
- "where two rules conflict, prefer the more general one and note the choice",
1797
- "in your rationale."
1798
- );
1799
- return lines.join("\n");
1800
- }
1801
- function extractH2Sections(text) {
1802
- const out = [];
1803
- for (const line of text.split("\n")) {
1804
- const match = /^##\s+(.+?)\s*$/.exec(line);
1805
- if (match) out.push(match[1]);
1806
- }
1807
- return out;
1808
- }
1809
- function countSentenceEdits(baseline, candidate) {
1810
- const norm = (s) => s.split(/(?<=[.!?])\s+|\n/g).map((p) => p.trim()).filter((p) => p.length > 0);
1811
- const a = new Set(norm(baseline));
1812
- const b = new Set(norm(candidate));
1813
- let edits = 0;
1814
- for (const s of a) if (!b.has(s)) edits++;
1815
- for (const s of b) if (!a.has(s)) edits++;
1816
- return edits;
1817
- }
1818
- function validatePreservedSections(candidate, required) {
1819
- if (required.length === 0) return true;
1820
- const have = new Set(extractH2Sections(candidate));
1821
- for (const section of required) {
1822
- if (!have.has(section)) return false;
1823
- }
1824
- return true;
1825
- }
1826
- function buildEvidence(ctx, evidenceK, baseTarget) {
1827
- const last = ctx.history.at(-1);
1828
- if (!last || last.candidates.length === 0) {
1829
- return { top: [], bottom: [], target: baseTarget };
1830
- }
1831
- const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
1832
- if (!best) return { top: [], bottom: [], target: baseTarget };
1833
- const byScore = [...best.scenarios].sort((a, b) => b.composite - a.composite);
1834
- const toTrace = (s) => ({
1835
- id: s.scenarioId,
1836
- score: s.composite,
1837
- // The judge's "why it scored low" — grounds the reflection on real failure
1838
- // patterns instead of blind rephrasing. Generalizable by the judge contract.
1839
- ...s.notes ? { failureNote: s.notes } : {}
1840
- });
1841
- const top = byScore.slice(0, evidenceK).map(toTrace);
1842
- const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
1843
- const weakest = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([dim, value]) => `${dim} (${value.toFixed(2)})`);
1844
- const target = weakest.length > 0 ? `${baseTarget} \u2014 weakest dimensions: ${weakest.join(", ")}` : baseTarget;
1845
- return { top, bottom, target };
1846
- }
1847
2261
 
1848
2262
  // src/campaign/provenance.ts
1849
2263
  import { createHash as createHash2 } from "crypto";
@@ -2127,29 +2541,36 @@ export {
2127
2541
  scoreRedTeamOutput,
2128
2542
  redTeamReport,
2129
2543
  toolNamesForRun,
2544
+ openAutoPr,
2545
+ composeGate,
2130
2546
  runCanaries,
2131
- DEFAULT_MUTATION_PRIMITIVES,
2132
- buildReflectionPrompt,
2133
- renderAnalystEvidence,
2134
- parseReflectionResponse,
2135
- TIE_WARN_FRACTION,
2136
2547
  pairHoldout,
2137
2548
  heldoutSignificance,
2138
2549
  detectScale,
2139
2550
  dimensionRegressions,
2551
+ defaultProductionGate,
2140
2552
  heldOutGate,
2141
- openAutoPr,
2142
- campaignMeanComposite,
2143
- campaignBreakdown,
2553
+ powerPreflight,
2554
+ buildEvidenceVector,
2555
+ paretoPolicy,
2556
+ paretoSignificanceGate,
2144
2557
  isProposedCandidate,
2145
2558
  labelTrustRank,
2559
+ DEFAULT_MUTATION_PRIMITIVES,
2560
+ buildReflectionPrompt,
2561
+ renderAnalystEvidence,
2562
+ parseReflectionResponse,
2563
+ gepaProposer,
2564
+ extractH2Sections,
2565
+ countSentenceEdits,
2566
+ campaignMeanComposite,
2567
+ campaignBreakdown,
2146
2568
  runOptimization,
2147
2569
  surfaceHash,
2148
2570
  runImprovementLoop,
2149
2571
  defaultRenderDiff,
2150
- gepaProposer,
2151
- extractH2Sections,
2152
- countSentenceEdits,
2572
+ runEval,
2573
+ evolutionaryProposer,
2153
2574
  surfaceContentHash,
2154
2575
  buildLoopProvenanceRecord,
2155
2576
  loopProvenanceSpans,
@@ -2157,4 +2578,4 @@ export {
2157
2578
  provenanceSpansPath,
2158
2579
  emitLoopProvenance
2159
2580
  };
2160
- //# sourceMappingURL=chunk-3E5KUXYZ.js.map
2581
+ //# sourceMappingURL=chunk-OVPVM4JC.js.map