@tangle-network/agent-eval 0.107.0 → 0.108.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +1 -1
- package/dist/adapters/langchain.d.ts +1 -1
- package/dist/adapters/otel.d.ts +1 -1
- package/dist/benchmarks/index.d.ts +3 -1
- package/dist/benchmarks/index.js +49 -3
- package/dist/campaign/index.d.ts +10 -7
- package/dist/campaign/index.js +72 -3407
- package/dist/campaign/index.js.map +1 -1
- package/dist/chunk-6SKVFBTR.js +4306 -0
- package/dist/chunk-6SKVFBTR.js.map +1 -0
- package/dist/{chunk-G4DLZAV5.js → chunk-OVPVM4JC.js} +1044 -643
- package/dist/chunk-OVPVM4JC.js.map +1 -0
- package/dist/chunk-T6W5ADLG.js +766 -0
- package/dist/chunk-T6W5ADLG.js.map +1 -0
- package/dist/contract/index.d.ts +16 -6
- package/dist/contract/index.js +10 -11
- package/dist/contract/index.js.map +1 -1
- package/dist/{gepa-BmqTrdtg.d.ts → gepa-B3x5Ulcv.d.ts} +3 -40
- package/dist/hosted/index.d.ts +1 -1
- package/dist/index-pPtfoIJO.d.ts +423 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.js +8 -7
- package/dist/index.js.map +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{pre-registration-x3f0VpxT.d.ts → pre-registration-BUhVPzE7.d.ts} +1 -1
- package/dist/{provenance-CIjRcI6m.d.ts → provenance-DdDhf6cg.d.ts} +3 -2
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +6 -6
- package/dist/storage-Dw_f7WMt.d.ts +39 -0
- package/dist/{types-CNZ0tHET.d.ts → types-BdIv5dvA.d.ts} +1 -1
- package/docs/improvement-glossary.md +5 -1
- package/package.json +1 -1
- package/dist/chunk-6PF6LXRY.js +0 -861
- package/dist/chunk-6PF6LXRY.js.map +0 -1
- package/dist/chunk-6QDKWHLS.js +0 -223
- package/dist/chunk-6QDKWHLS.js.map +0 -1
- package/dist/chunk-G4DLZAV5.js.map +0 -1
- package/dist/chunk-V75NN2ZR.js +0 -421
- package/dist/chunk-V75NN2ZR.js.map +0 -1
- package/dist/index-C2CC_dry.d.ts +0 -159
|
@@ -2,6 +2,9 @@ import {
|
|
|
2
2
|
runCampaign,
|
|
3
3
|
summarizeBackendIntegrity
|
|
4
4
|
} from "./chunk-3PFZBGMR.js";
|
|
5
|
+
import {
|
|
6
|
+
detectRewardHacking
|
|
7
|
+
} from "./chunk-N22ZO7FV.js";
|
|
5
8
|
import {
|
|
6
9
|
callLlm
|
|
7
10
|
} from "./chunk-FUCQVFMU.js";
|
|
@@ -549,6 +552,150 @@ function excerptAt(source, at, needleLength) {
|
|
|
549
552
|
return (start > 0 ? "\u2026" : "") + source.slice(start, end) + (end < source.length ? "\u2026" : "");
|
|
550
553
|
}
|
|
551
554
|
|
|
555
|
+
// src/campaign/auto-pr.ts
|
|
556
|
+
import { execSync } from "child_process";
|
|
557
|
+
import { writeFileSync } from "fs";
|
|
558
|
+
import { tmpdir } from "os";
|
|
559
|
+
import { join } from "path";
|
|
560
|
+
function openAutoPr(options) {
|
|
561
|
+
if (options.gate.decision !== "ship") {
|
|
562
|
+
return {
|
|
563
|
+
opened: false,
|
|
564
|
+
dryRun: false,
|
|
565
|
+
reason: `gate verdict was "${options.gate.decision}" \u2014 refusing to open PR`
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
const dryRun = options.dryRun ?? !process.env.GH_AUTO_PR_TOKEN;
|
|
569
|
+
const branch = options.branch ?? `auto/${options.result.manifestHash.slice(0, 12)}`;
|
|
570
|
+
const title = options.title ?? `auto: campaign ${options.result.manifestHash.slice(0, 8)} promoted by gate`;
|
|
571
|
+
const body = renderPrBody(options.result, options.gate, options.promotedDiff);
|
|
572
|
+
const bodyPath = join(tmpdir(), `auto-pr-body-${Date.now()}.md`);
|
|
573
|
+
writeFileSync(bodyPath, body);
|
|
574
|
+
if (dryRun) {
|
|
575
|
+
return {
|
|
576
|
+
opened: false,
|
|
577
|
+
dryRun: true,
|
|
578
|
+
reason: `dry-run (GH_AUTO_PR_TOKEN not set). Would create PR on ${options.ghOwner}/${options.ghRepo} branch ${branch}. Body at ${bodyPath}.`
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
const ghExec = options.ghExec ?? defaultGhExec;
|
|
582
|
+
const result = ghExec([
|
|
583
|
+
"pr",
|
|
584
|
+
"create",
|
|
585
|
+
"--repo",
|
|
586
|
+
`${options.ghOwner}/${options.ghRepo}`,
|
|
587
|
+
"--head",
|
|
588
|
+
branch,
|
|
589
|
+
"--title",
|
|
590
|
+
title,
|
|
591
|
+
"--body-file",
|
|
592
|
+
bodyPath
|
|
593
|
+
]);
|
|
594
|
+
if (result.status !== 0) {
|
|
595
|
+
return {
|
|
596
|
+
opened: false,
|
|
597
|
+
dryRun: false,
|
|
598
|
+
reason: `gh pr create failed (exit ${result.status}): ${result.stderr.slice(0, 400)}`
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
const prUrl = result.stdout.trim();
|
|
602
|
+
return { opened: true, prUrl, dryRun: false, reason: "PR opened" };
|
|
603
|
+
}
|
|
604
|
+
function renderPrBody(result, gate, diff) {
|
|
605
|
+
const lines = [];
|
|
606
|
+
lines.push(`## Automated promotion by \`runImprovementLoop\``);
|
|
607
|
+
lines.push("");
|
|
608
|
+
lines.push(`**Manifest**: \`${result.manifestHash}\``);
|
|
609
|
+
lines.push(`**Seed**: ${result.seed}`);
|
|
610
|
+
lines.push(`**Duration**: ${Math.round(result.durationMs / 1e3)}s`);
|
|
611
|
+
lines.push(
|
|
612
|
+
`**Cells**: executed ${result.aggregates.cellsExecuted}, cached ${result.aggregates.cellsCached}, skipped ${result.aggregates.cellsSkipped}, failed ${result.aggregates.cellsFailed}`
|
|
613
|
+
);
|
|
614
|
+
lines.push(`**Total spend**: $${result.aggregates.totalCostUsd.toFixed(2)}`);
|
|
615
|
+
lines.push("");
|
|
616
|
+
lines.push(`### Gate verdict: \`${gate.decision}\``);
|
|
617
|
+
lines.push("");
|
|
618
|
+
for (const reason of gate.reasons) lines.push(`- ${reason}`);
|
|
619
|
+
if (gate.delta !== void 0) lines.push(`- delta: ${gate.delta.toFixed(3)}`);
|
|
620
|
+
lines.push("");
|
|
621
|
+
lines.push("### Contributing gates");
|
|
622
|
+
lines.push("");
|
|
623
|
+
lines.push("| gate | passed | detail |");
|
|
624
|
+
lines.push("|---|---|---|");
|
|
625
|
+
for (const c of gate.contributingGates) {
|
|
626
|
+
const detail = typeof c.detail === "object" ? JSON.stringify(c.detail).slice(0, 80) : String(c.detail).slice(0, 80);
|
|
627
|
+
lines.push(`| ${c.name} | ${c.passed ? "\u2713" : "\u2717"} | ${detail} |`);
|
|
628
|
+
}
|
|
629
|
+
lines.push("");
|
|
630
|
+
lines.push("### Promoted surface");
|
|
631
|
+
lines.push("");
|
|
632
|
+
lines.push("```diff");
|
|
633
|
+
lines.push(diff.slice(0, 8e3));
|
|
634
|
+
lines.push("```");
|
|
635
|
+
lines.push("");
|
|
636
|
+
lines.push("### By-judge aggregates");
|
|
637
|
+
lines.push("");
|
|
638
|
+
lines.push("| judge | mean | ci95 | n |");
|
|
639
|
+
lines.push("|---|---|---|---|");
|
|
640
|
+
for (const [name, agg] of Object.entries(result.aggregates.byJudge)) {
|
|
641
|
+
lines.push(
|
|
642
|
+
`| ${name} | ${agg.mean.toFixed(3)} | [${agg.ci95[0].toFixed(3)}, ${agg.ci95[1].toFixed(3)}] | ${agg.n} |`
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
return lines.join("\n");
|
|
646
|
+
}
|
|
647
|
+
function defaultGhExec(args) {
|
|
648
|
+
try {
|
|
649
|
+
const stdout = execSync(`gh ${args.map(quoteArg).join(" ")}`, {
|
|
650
|
+
env: { ...process.env, GH_TOKEN: process.env.GH_AUTO_PR_TOKEN ?? process.env.GH_TOKEN ?? "" },
|
|
651
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
652
|
+
}).toString("utf8");
|
|
653
|
+
return { stdout, stderr: "", status: 0 };
|
|
654
|
+
} catch (err) {
|
|
655
|
+
const e = err;
|
|
656
|
+
return {
|
|
657
|
+
stdout: e.stdout?.toString("utf8") ?? "",
|
|
658
|
+
stderr: e.stderr?.toString("utf8") ?? "",
|
|
659
|
+
status: e.status ?? 1
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
function quoteArg(arg) {
|
|
664
|
+
if (/^[a-zA-Z0-9_/\-:.@]+$/.test(arg)) return arg;
|
|
665
|
+
return `"${arg.replace(/"/g, '\\"')}"`;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// src/campaign/gates/compose.ts
|
|
669
|
+
function composeGate(...gates) {
|
|
670
|
+
if (gates.length === 0) {
|
|
671
|
+
throw new Error("composeGate requires at least one gate");
|
|
672
|
+
}
|
|
673
|
+
return {
|
|
674
|
+
name: `composed(${gates.map((g) => g.name).join(",")})`,
|
|
675
|
+
async decide(ctx) {
|
|
676
|
+
const results = [];
|
|
677
|
+
for (const gate of gates) {
|
|
678
|
+
const res = await gate.decide(ctx);
|
|
679
|
+
results.push({ gate, res });
|
|
680
|
+
}
|
|
681
|
+
const decisions = results.map((r) => r.res.decision);
|
|
682
|
+
const overall = decisions.every((d) => d === "ship") ? "ship" : decisions.includes("arch_ceiling") ? "arch_ceiling" : decisions.includes("model_ceiling") ? "model_ceiling" : decisions.includes("hold") ? "hold" : "need_more_work";
|
|
683
|
+
const contributing = results.flatMap(
|
|
684
|
+
(r) => r.res.contributingGates.length > 0 ? r.res.contributingGates : [{ name: r.gate.name, passed: r.res.decision === "ship", detail: r.res }]
|
|
685
|
+
);
|
|
686
|
+
const reasons = results.flatMap(
|
|
687
|
+
(r) => r.res.reasons.map((reason) => `[${r.gate.name}] ${reason}`)
|
|
688
|
+
);
|
|
689
|
+
return {
|
|
690
|
+
decision: overall,
|
|
691
|
+
reasons,
|
|
692
|
+
contributingGates: contributing,
|
|
693
|
+
delta: results[0]?.res.delta
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
|
|
552
699
|
// src/canary.ts
|
|
553
700
|
function runCanaries(runs, opts = {}) {
|
|
554
701
|
const alerts = [
|
|
@@ -767,383 +914,279 @@ function mean(xs) {
|
|
|
767
914
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
768
915
|
}
|
|
769
916
|
|
|
770
|
-
// src/
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
if (inString) {
|
|
781
|
-
if (c === "\\") {
|
|
782
|
-
escaped = true;
|
|
783
|
-
continue;
|
|
784
|
-
}
|
|
785
|
-
if (c === '"') {
|
|
786
|
-
inString = false;
|
|
787
|
-
continue;
|
|
788
|
-
}
|
|
789
|
-
continue;
|
|
790
|
-
}
|
|
791
|
-
if (c === '"') {
|
|
792
|
-
inString = true;
|
|
793
|
-
continue;
|
|
794
|
-
}
|
|
795
|
-
if (c === "{" || c === "[") stack.push(c);
|
|
796
|
-
else if (c === "}") {
|
|
797
|
-
if (stack.pop() !== "{") return null;
|
|
798
|
-
} else if (c === "]") {
|
|
799
|
-
if (stack.pop() !== "[") return null;
|
|
917
|
+
// src/campaign/gates/statistical-heldout.ts
|
|
918
|
+
var TIE_WARN_FRACTION = 0.4;
|
|
919
|
+
function pairHoldout(candidate, baseline, scenarioIds, select) {
|
|
920
|
+
const cellValue = (byCell, cellId) => {
|
|
921
|
+
const scores = byCell.get(cellId);
|
|
922
|
+
if (!scores) return void 0;
|
|
923
|
+
const vals = [];
|
|
924
|
+
for (const s of Object.values(scores)) {
|
|
925
|
+
const v = select(s);
|
|
926
|
+
if (typeof v === "number" && Number.isFinite(v)) vals.push(v);
|
|
800
927
|
}
|
|
928
|
+
if (vals.length === 0) return void 0;
|
|
929
|
+
return vals.reduce((a, b) => a + b, 0) / vals.length;
|
|
930
|
+
};
|
|
931
|
+
const inScope = (cellId) => scenarioIds.has(cellId.split(":")[0] ?? "");
|
|
932
|
+
const candCells = [...candidate.keys()].filter(inScope).sort();
|
|
933
|
+
const baseCells = [...baseline.keys()].filter(inScope).sort();
|
|
934
|
+
if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {
|
|
935
|
+
throw new Error(
|
|
936
|
+
`pairHoldout: candidate/baseline holdout cells do not align \u2014 candidate=[${candCells.join(",")}] baseline=[${baseCells.join(",")}]. Both holdout campaigns must run the same scenarios with the same seed base.`
|
|
937
|
+
);
|
|
801
938
|
}
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
const
|
|
808
|
-
|
|
939
|
+
const before = [];
|
|
940
|
+
const after = [];
|
|
941
|
+
const cellIds = [];
|
|
942
|
+
for (const cellId of candCells) {
|
|
943
|
+
const b = cellValue(baseline, cellId);
|
|
944
|
+
const a = cellValue(candidate, cellId);
|
|
945
|
+
if (b === void 0 || a === void 0) continue;
|
|
946
|
+
before.push(b);
|
|
947
|
+
after.push(a);
|
|
948
|
+
cellIds.push(cellId);
|
|
809
949
|
}
|
|
810
|
-
return
|
|
950
|
+
return { before, after, cellIds };
|
|
811
951
|
}
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
952
|
+
function heldoutSignificance(paired, opts = {}) {
|
|
953
|
+
const deltaThreshold = opts.deltaThreshold ?? 0;
|
|
954
|
+
const minProductiveRuns = opts.minProductiveRuns ?? 3;
|
|
955
|
+
const confidence = opts.confidence ?? 0.95;
|
|
956
|
+
const resamples = opts.resamples ?? 2e3;
|
|
957
|
+
const seed = opts.seed ?? 1337;
|
|
958
|
+
const statistic = opts.statistic ?? "mean";
|
|
959
|
+
const bootstrap = pairedBootstrap(paired.before, paired.after, {
|
|
960
|
+
confidence,
|
|
961
|
+
resamples,
|
|
962
|
+
statistic,
|
|
963
|
+
seed
|
|
964
|
+
});
|
|
965
|
+
const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
|
|
966
|
+
confidence,
|
|
967
|
+
resamples,
|
|
968
|
+
statistic: "median",
|
|
969
|
+
seed
|
|
970
|
+
});
|
|
971
|
+
const n = paired.before.length;
|
|
972
|
+
let ties = 0;
|
|
973
|
+
for (let i = 0; i < n; i += 1) {
|
|
974
|
+
const after = paired.after[i] ?? 0;
|
|
975
|
+
const before = paired.before[i] ?? 0;
|
|
976
|
+
if (Math.abs(after - before) < 1e-9) ties += 1;
|
|
818
977
|
}
|
|
978
|
+
const tieFraction = n === 0 ? 0 : ties / n;
|
|
979
|
+
const fewRuns = n < minProductiveRuns;
|
|
980
|
+
const significant = !fewRuns && bootstrap.low > deltaThreshold;
|
|
981
|
+
return { paired, bootstrap, medianBootstrap, tieFraction, n, significant, fewRuns };
|
|
819
982
|
}
|
|
820
|
-
function
|
|
821
|
-
|
|
822
|
-
let escaped = false;
|
|
823
|
-
let last = -1;
|
|
824
|
-
for (let i = 0; i < s.length; i++) {
|
|
825
|
-
const c = s[i];
|
|
826
|
-
if (escaped) {
|
|
827
|
-
escaped = false;
|
|
828
|
-
continue;
|
|
829
|
-
}
|
|
830
|
-
if (inString) {
|
|
831
|
-
if (c === "\\") escaped = true;
|
|
832
|
-
else if (c === '"') inString = false;
|
|
833
|
-
continue;
|
|
834
|
-
}
|
|
835
|
-
if (c === '"') inString = true;
|
|
836
|
-
else if (c === ",") last = i;
|
|
837
|
-
}
|
|
838
|
-
return last;
|
|
983
|
+
function detectScale(values) {
|
|
984
|
+
return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
|
|
839
985
|
}
|
|
840
|
-
function
|
|
841
|
-
const
|
|
842
|
-
const
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
const
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
if (closed !== null) {
|
|
860
|
-
const parsed = tryParse(closed);
|
|
861
|
-
if (parsed !== UNPARSEABLE) return parsed;
|
|
862
|
-
}
|
|
863
|
-
const cut = lastCommaOutsideString(candidate);
|
|
864
|
-
if (cut <= 0) return UNPARSEABLE;
|
|
865
|
-
candidate = candidate.slice(0, cut);
|
|
986
|
+
function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
|
|
987
|
+
const out = [];
|
|
988
|
+
for (const dim of criticalDimensions) {
|
|
989
|
+
const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
|
|
990
|
+
if (paired.before.length === 0) continue;
|
|
991
|
+
const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
|
|
992
|
+
const bootstrap = pairedBootstrap(paired.before, paired.after, {
|
|
993
|
+
confidence: opts.confidence ?? 0.95,
|
|
994
|
+
resamples: opts.resamples ?? 2e3,
|
|
995
|
+
statistic: "median",
|
|
996
|
+
seed: opts.seed ?? 1337
|
|
997
|
+
});
|
|
998
|
+
out.push({
|
|
999
|
+
dimension: dim,
|
|
1000
|
+
bootstrap,
|
|
1001
|
+
regressed: bootstrap.low < -tolerance,
|
|
1002
|
+
tolerance,
|
|
1003
|
+
n: paired.before.length
|
|
1004
|
+
});
|
|
866
1005
|
}
|
|
867
|
-
return
|
|
1006
|
+
return out;
|
|
868
1007
|
}
|
|
869
1008
|
|
|
870
|
-
// src/
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
1009
|
+
// src/campaign/gates/default-production-gate.ts
|
|
1010
|
+
function defaultProductionGate(options) {
|
|
1011
|
+
const deltaThreshold = options.deltaThreshold ?? 0;
|
|
1012
|
+
const confidence = options.confidence ?? 0.95;
|
|
1013
|
+
const resamples = options.bootstrapResamples ?? 2e3;
|
|
1014
|
+
const seed = options.bootstrapSeed ?? 1337;
|
|
1015
|
+
const minProductiveRuns = options.minProductiveRuns ?? 3;
|
|
1016
|
+
const heldoutStatistic = options.heldoutStatistic ?? "mean";
|
|
1017
|
+
const blockOnGaming = options.blockOnRewardHackingGaming ?? true;
|
|
1018
|
+
return {
|
|
1019
|
+
name: "defaultProductionGate",
|
|
1020
|
+
async decide(ctx) {
|
|
1021
|
+
const reasons = [];
|
|
1022
|
+
const contributing = [];
|
|
1023
|
+
const scenarioIds = new Set(options.holdoutScenarios.map((s) => s.id));
|
|
1024
|
+
const sig = heldoutSignificance(
|
|
1025
|
+
pairHoldout(
|
|
1026
|
+
ctx.judgeScores,
|
|
1027
|
+
ctx.baselineJudgeScores ?? ctx.judgeScores,
|
|
1028
|
+
scenarioIds,
|
|
1029
|
+
(s) => s.composite
|
|
1030
|
+
),
|
|
1031
|
+
{
|
|
1032
|
+
deltaThreshold,
|
|
1033
|
+
minProductiveRuns,
|
|
1034
|
+
confidence,
|
|
1035
|
+
resamples,
|
|
1036
|
+
seed,
|
|
1037
|
+
statistic: heldoutStatistic
|
|
1038
|
+
}
|
|
899
1039
|
);
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
1040
|
+
const delta = heldoutStatistic === "median" ? sig.bootstrap.median : sig.bootstrap.mean;
|
|
1041
|
+
const heldoutPass = sig.significant;
|
|
1042
|
+
contributing.push({
|
|
1043
|
+
name: "heldout-significance",
|
|
1044
|
+
passed: heldoutPass,
|
|
1045
|
+
detail: {
|
|
1046
|
+
n: sig.n,
|
|
1047
|
+
delta,
|
|
1048
|
+
deltaMean: sig.bootstrap.mean,
|
|
1049
|
+
deltaMedianDiagnostic: sig.medianBootstrap.median,
|
|
1050
|
+
// Back-compat: prior consumers read `deltaMedian`. It now always carries
|
|
1051
|
+
// the median diagnostic (the ship decision keys on `delta`/mean).
|
|
1052
|
+
deltaMedian: sig.medianBootstrap.median,
|
|
1053
|
+
tieFraction: sig.tieFraction,
|
|
1054
|
+
ciLow: sig.bootstrap.low,
|
|
1055
|
+
ciHigh: sig.bootstrap.high,
|
|
1056
|
+
confidence: sig.bootstrap.confidence,
|
|
1057
|
+
deltaThreshold,
|
|
1058
|
+
fewRuns: sig.fewRuns
|
|
1059
|
+
}
|
|
1060
|
+
});
|
|
1061
|
+
if (!heldoutPass) {
|
|
1062
|
+
const tieNote = sig.tieFraction >= TIE_WARN_FRACTION ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied scenarios` : "";
|
|
1063
|
+
reasons.push(
|
|
1064
|
+
sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${minProductiveRuns}) \u2014 too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 threshold ${deltaThreshold} (${heldoutStatistic} \u0394 ${delta.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]${tieNote})`
|
|
1065
|
+
);
|
|
903
1066
|
}
|
|
904
|
-
const
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
1067
|
+
const dimRegs = options.criticalDimensions?.length ? dimensionRegressions(
|
|
1068
|
+
ctx.judgeScores,
|
|
1069
|
+
ctx.baselineJudgeScores ?? ctx.judgeScores,
|
|
1070
|
+
scenarioIds,
|
|
1071
|
+
options.criticalDimensions,
|
|
1072
|
+
{ tolerance: options.regressionTolerance, confidence, resamples, seed }
|
|
1073
|
+
) : [];
|
|
1074
|
+
const regressed = dimRegs.filter((d) => d.regressed);
|
|
1075
|
+
const dimPass = regressed.length === 0;
|
|
1076
|
+
contributing.push({
|
|
1077
|
+
name: "dimension-regression",
|
|
1078
|
+
passed: dimPass,
|
|
1079
|
+
detail: {
|
|
1080
|
+
guarded: options.criticalDimensions ?? [],
|
|
1081
|
+
regressions: dimRegs.map((d) => ({
|
|
1082
|
+
dimension: d.dimension,
|
|
1083
|
+
ciLow: d.bootstrap.low,
|
|
1084
|
+
median: d.bootstrap.median,
|
|
1085
|
+
tolerance: d.tolerance,
|
|
1086
|
+
n: d.n,
|
|
1087
|
+
regressed: d.regressed
|
|
1088
|
+
}))
|
|
910
1089
|
}
|
|
1090
|
+
});
|
|
1091
|
+
if (!dimPass) {
|
|
1092
|
+
reasons.push(
|
|
1093
|
+
`critical dimension(s) regressed: ${regressed.map((d) => `${d.dimension} CI.low ${d.bootstrap.low.toFixed(3)} < -${d.tolerance}`).join("; ")}`
|
|
1094
|
+
);
|
|
911
1095
|
}
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
1096
|
+
const budgetPass = options.budgetUsd === void 0 || ctx.cost.candidate + ctx.cost.baseline <= options.budgetUsd;
|
|
1097
|
+
contributing.push({
|
|
1098
|
+
name: "budget",
|
|
1099
|
+
passed: budgetPass,
|
|
1100
|
+
detail: {
|
|
1101
|
+
candidateUsd: ctx.cost.candidate,
|
|
1102
|
+
baselineUsd: ctx.cost.baseline,
|
|
1103
|
+
budgetUsd: options.budgetUsd
|
|
1104
|
+
}
|
|
1105
|
+
});
|
|
1106
|
+
if (!budgetPass) {
|
|
1107
|
+
reasons.push(
|
|
1108
|
+
`spend ${(ctx.cost.candidate + ctx.cost.baseline).toFixed(2)} > budget ${options.budgetUsd}`
|
|
1109
|
+
);
|
|
918
1110
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1111
|
+
const redTeamFindings = options.redTeamBattery ? probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery) : { passed: true, findings: [] };
|
|
1112
|
+
contributing.push({
|
|
1113
|
+
name: "red-team",
|
|
1114
|
+
passed: redTeamFindings.passed,
|
|
1115
|
+
detail: {
|
|
1116
|
+
failures: redTeamFindings.findings.length,
|
|
1117
|
+
sample: redTeamFindings.findings.slice(0, 3)
|
|
1118
|
+
}
|
|
1119
|
+
});
|
|
1120
|
+
if (!redTeamFindings.passed) {
|
|
1121
|
+
reasons.push(`red-team probe failed (${redTeamFindings.findings.length} findings)`);
|
|
1122
|
+
}
|
|
1123
|
+
let rewardHackingReport = null;
|
|
1124
|
+
if (options.recentRuns && options.recentRuns.length >= 10) {
|
|
1125
|
+
rewardHackingReport = detectRewardHacking({ runs: options.recentRuns });
|
|
1126
|
+
}
|
|
1127
|
+
const gamingThreshold = 0.6;
|
|
1128
|
+
const gamingFindings = (rewardHackingReport?.findings ?? []).filter(
|
|
1129
|
+
(f) => f.severity >= gamingThreshold
|
|
928
1130
|
);
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
)
|
|
954
|
-
);
|
|
955
|
-
sections.push("```");
|
|
956
|
-
return sections.join("\n");
|
|
957
|
-
}
|
|
958
|
-
function renderAnalystEvidence(findings, report) {
|
|
959
|
-
const sections = [];
|
|
960
|
-
const rows = Array.isArray(findings) ? findings : [];
|
|
961
|
-
if (rows.length > 0) {
|
|
962
|
-
const order = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
|
|
963
|
-
const ranked = rows.map((f) => f && typeof f === "object" ? f : {}).sort((a, b) => (order[String(a.severity)] ?? 9) - (order[String(b.severity)] ?? 9)).slice(0, 12);
|
|
964
|
-
sections.push("## Diagnosed findings (act on these root causes)");
|
|
965
|
-
sections.push("");
|
|
966
|
-
for (const f of ranked) {
|
|
967
|
-
const sev = typeof f.severity === "string" ? f.severity : "info";
|
|
968
|
-
const area = typeof f.area === "string" ? ` [${f.area}]` : "";
|
|
969
|
-
const claim = typeof f.claim === "string" ? f.claim : JSON.stringify(f);
|
|
970
|
-
sections.push(`- **${sev}**${area} ${truncate(claim, 300)}`);
|
|
971
|
-
if (typeof f.recommended_action === "string" && f.recommended_action.trim()) {
|
|
972
|
-
sections.push(` - fix: ${truncate(f.recommended_action, 200)}`);
|
|
1131
|
+
const rewardHackingPass = !rewardHackingReport || !blockOnGaming || gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
|
|
1132
|
+
contributing.push({
|
|
1133
|
+
name: "reward-hacking",
|
|
1134
|
+
passed: rewardHackingPass,
|
|
1135
|
+
detail: { report: rewardHackingReport, gamingFindingCount: gamingFindings.length }
|
|
1136
|
+
});
|
|
1137
|
+
if (!rewardHackingPass) {
|
|
1138
|
+
reasons.push(
|
|
1139
|
+
`reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
let canaryReport = null;
|
|
1143
|
+
if (options.recentRuns && options.recentRuns.length >= 10) {
|
|
1144
|
+
canaryReport = runCanaries(options.recentRuns, {});
|
|
1145
|
+
}
|
|
1146
|
+
const errorAlerts = (canaryReport?.alerts ?? []).filter((a) => a.severity === "error");
|
|
1147
|
+
const canaryPass = errorAlerts.length === 0;
|
|
1148
|
+
contributing.push({
|
|
1149
|
+
name: "canary",
|
|
1150
|
+
passed: canaryPass,
|
|
1151
|
+
detail: { totalAlerts: canaryReport?.alerts.length ?? 0, errorAlerts: errorAlerts.length }
|
|
1152
|
+
});
|
|
1153
|
+
if (!canaryPass) {
|
|
1154
|
+
reasons.push(`canary error alerts: ${errorAlerts.length}`);
|
|
973
1155
|
}
|
|
1156
|
+
const allPassed = contributing.every((c) => c.passed);
|
|
1157
|
+
const decision = allPassed ? "ship" : "hold";
|
|
1158
|
+
return {
|
|
1159
|
+
decision,
|
|
1160
|
+
reasons: reasons.length > 0 ? reasons : ["all gates passed"],
|
|
1161
|
+
contributingGates: contributing,
|
|
1162
|
+
delta
|
|
1163
|
+
};
|
|
974
1164
|
}
|
|
975
|
-
|
|
976
|
-
}
|
|
977
|
-
const reportText = typeof report === "string" ? report : report && typeof report === "object" ? JSON.stringify(report) : "";
|
|
978
|
-
if (reportText.trim()) {
|
|
979
|
-
sections.push("## Research report");
|
|
980
|
-
sections.push("");
|
|
981
|
-
sections.push(truncate(reportText.trim(), 1500));
|
|
982
|
-
sections.push("");
|
|
983
|
-
}
|
|
984
|
-
if (sections.length === 0) return null;
|
|
985
|
-
return sections.join("\n");
|
|
986
|
-
}
|
|
987
|
-
function truncate(s, max) {
|
|
988
|
-
if (s.length <= max) return s;
|
|
989
|
-
return `${s.slice(0, max)}\u2026 [truncated]`;
|
|
990
|
-
}
|
|
991
|
-
function quote(s) {
|
|
992
|
-
return s.replace(/`/g, "\\`");
|
|
1165
|
+
};
|
|
993
1166
|
}
|
|
994
|
-
function
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
const candidates = [];
|
|
1004
|
-
if (tryObjectFirst) {
|
|
1005
|
-
if (objectStart >= 0 && objectEnd > objectStart)
|
|
1006
|
-
candidates.push(text.slice(objectStart, objectEnd + 1));
|
|
1007
|
-
if (arrayStart >= 0 && arrayEnd > arrayStart)
|
|
1008
|
-
candidates.push(text.slice(arrayStart, arrayEnd + 1));
|
|
1009
|
-
} else {
|
|
1010
|
-
if (arrayStart >= 0 && arrayEnd > arrayStart)
|
|
1011
|
-
candidates.push(text.slice(arrayStart, arrayEnd + 1));
|
|
1012
|
-
if (objectStart >= 0 && objectEnd > objectStart)
|
|
1013
|
-
candidates.push(text.slice(objectStart, objectEnd + 1));
|
|
1014
|
-
}
|
|
1015
|
-
for (const slice of candidates) {
|
|
1016
|
-
try {
|
|
1017
|
-
parsed = JSON.parse(slice);
|
|
1018
|
-
break;
|
|
1019
|
-
} catch {
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
|
-
if (parsed == null) {
|
|
1023
|
-
for (const slice of candidates) {
|
|
1024
|
-
const closed = autoCloseTruncatedJson(slice);
|
|
1025
|
-
if (closed != null && closed !== slice) {
|
|
1026
|
-
try {
|
|
1027
|
-
parsed = JSON.parse(closed);
|
|
1028
|
-
break;
|
|
1029
|
-
} catch {
|
|
1030
|
-
}
|
|
1167
|
+
function probeRedTeam(artifacts, battery) {
|
|
1168
|
+
const findings = [];
|
|
1169
|
+
for (const [_cellId, artifact] of artifacts) {
|
|
1170
|
+
const text = extractText(artifact);
|
|
1171
|
+
if (text === void 0) continue;
|
|
1172
|
+
for (const rtCase of battery) {
|
|
1173
|
+
const finding = scoreRedTeamOutput(text, [], rtCase);
|
|
1174
|
+
if (!finding.passed) {
|
|
1175
|
+
findings.push({ scenarioId: rtCase.id, reason: finding.reason ?? "red-team probe failed" });
|
|
1031
1176
|
}
|
|
1032
1177
|
}
|
|
1033
1178
|
}
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
for (const p of proposalsRaw) {
|
|
1044
|
-
if (!p || typeof p !== "object") continue;
|
|
1045
|
-
const obj = p;
|
|
1046
|
-
if (!("payload" in obj)) continue;
|
|
1047
|
-
out.push({
|
|
1048
|
-
label: typeof obj.label === "string" ? obj.label : "mutation",
|
|
1049
|
-
rationale: typeof obj.rationale === "string" ? obj.rationale : "",
|
|
1050
|
-
payload: obj.payload
|
|
1051
|
-
});
|
|
1052
|
-
if (maxProposals !== void 0 && out.length >= maxProposals) break;
|
|
1179
|
+
return { passed: findings.length === 0, findings };
|
|
1180
|
+
}
|
|
1181
|
+
function extractText(artifact) {
|
|
1182
|
+
if (typeof artifact === "string") return artifact;
|
|
1183
|
+
if (artifact && typeof artifact === "object") {
|
|
1184
|
+
const rec = artifact;
|
|
1185
|
+
if (typeof rec.text === "string") return rec.text;
|
|
1186
|
+
if (typeof rec.output === "string") return rec.output;
|
|
1187
|
+
if (typeof rec.content === "string") return rec.content;
|
|
1053
1188
|
}
|
|
1054
|
-
return
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
// src/campaign/gates/statistical-heldout.ts
|
|
1058
|
-
var TIE_WARN_FRACTION = 0.4;
|
|
1059
|
-
function pairHoldout(candidate, baseline, scenarioIds, select) {
|
|
1060
|
-
const cellValue = (byCell, cellId) => {
|
|
1061
|
-
const scores = byCell.get(cellId);
|
|
1062
|
-
if (!scores) return void 0;
|
|
1063
|
-
const vals = [];
|
|
1064
|
-
for (const s of Object.values(scores)) {
|
|
1065
|
-
const v = select(s);
|
|
1066
|
-
if (typeof v === "number" && Number.isFinite(v)) vals.push(v);
|
|
1067
|
-
}
|
|
1068
|
-
if (vals.length === 0) return void 0;
|
|
1069
|
-
return vals.reduce((a, b) => a + b, 0) / vals.length;
|
|
1070
|
-
};
|
|
1071
|
-
const inScope = (cellId) => scenarioIds.has(cellId.split(":")[0] ?? "");
|
|
1072
|
-
const candCells = [...candidate.keys()].filter(inScope).sort();
|
|
1073
|
-
const baseCells = [...baseline.keys()].filter(inScope).sort();
|
|
1074
|
-
if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {
|
|
1075
|
-
throw new Error(
|
|
1076
|
-
`pairHoldout: candidate/baseline holdout cells do not align \u2014 candidate=[${candCells.join(",")}] baseline=[${baseCells.join(",")}]. Both holdout campaigns must run the same scenarios with the same seed base.`
|
|
1077
|
-
);
|
|
1078
|
-
}
|
|
1079
|
-
const before = [];
|
|
1080
|
-
const after = [];
|
|
1081
|
-
const cellIds = [];
|
|
1082
|
-
for (const cellId of candCells) {
|
|
1083
|
-
const b = cellValue(baseline, cellId);
|
|
1084
|
-
const a = cellValue(candidate, cellId);
|
|
1085
|
-
if (b === void 0 || a === void 0) continue;
|
|
1086
|
-
before.push(b);
|
|
1087
|
-
after.push(a);
|
|
1088
|
-
cellIds.push(cellId);
|
|
1089
|
-
}
|
|
1090
|
-
return { before, after, cellIds };
|
|
1091
|
-
}
|
|
1092
|
-
function heldoutSignificance(paired, opts = {}) {
|
|
1093
|
-
const deltaThreshold = opts.deltaThreshold ?? 0;
|
|
1094
|
-
const minProductiveRuns = opts.minProductiveRuns ?? 3;
|
|
1095
|
-
const confidence = opts.confidence ?? 0.95;
|
|
1096
|
-
const resamples = opts.resamples ?? 2e3;
|
|
1097
|
-
const seed = opts.seed ?? 1337;
|
|
1098
|
-
const statistic = opts.statistic ?? "mean";
|
|
1099
|
-
const bootstrap = pairedBootstrap(paired.before, paired.after, {
|
|
1100
|
-
confidence,
|
|
1101
|
-
resamples,
|
|
1102
|
-
statistic,
|
|
1103
|
-
seed
|
|
1104
|
-
});
|
|
1105
|
-
const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
|
|
1106
|
-
confidence,
|
|
1107
|
-
resamples,
|
|
1108
|
-
statistic: "median",
|
|
1109
|
-
seed
|
|
1110
|
-
});
|
|
1111
|
-
const n = paired.before.length;
|
|
1112
|
-
let ties = 0;
|
|
1113
|
-
for (let i = 0; i < n; i += 1) {
|
|
1114
|
-
const after = paired.after[i] ?? 0;
|
|
1115
|
-
const before = paired.before[i] ?? 0;
|
|
1116
|
-
if (Math.abs(after - before) < 1e-9) ties += 1;
|
|
1117
|
-
}
|
|
1118
|
-
const tieFraction = n === 0 ? 0 : ties / n;
|
|
1119
|
-
const fewRuns = n < minProductiveRuns;
|
|
1120
|
-
const significant = !fewRuns && bootstrap.low > deltaThreshold;
|
|
1121
|
-
return { paired, bootstrap, medianBootstrap, tieFraction, n, significant, fewRuns };
|
|
1122
|
-
}
|
|
1123
|
-
function detectScale(values) {
|
|
1124
|
-
return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
|
|
1125
|
-
}
|
|
1126
|
-
function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
|
|
1127
|
-
const out = [];
|
|
1128
|
-
for (const dim of criticalDimensions) {
|
|
1129
|
-
const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
|
|
1130
|
-
if (paired.before.length === 0) continue;
|
|
1131
|
-
const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
|
|
1132
|
-
const bootstrap = pairedBootstrap(paired.before, paired.after, {
|
|
1133
|
-
confidence: opts.confidence ?? 0.95,
|
|
1134
|
-
resamples: opts.resamples ?? 2e3,
|
|
1135
|
-
statistic: "median",
|
|
1136
|
-
seed: opts.seed ?? 1337
|
|
1137
|
-
});
|
|
1138
|
-
out.push({
|
|
1139
|
-
dimension: dim,
|
|
1140
|
-
bootstrap,
|
|
1141
|
-
regressed: bootstrap.low < -tolerance,
|
|
1142
|
-
tolerance,
|
|
1143
|
-
n: paired.before.length
|
|
1144
|
-
});
|
|
1145
|
-
}
|
|
1146
|
-
return out;
|
|
1189
|
+
return void 0;
|
|
1147
1190
|
}
|
|
1148
1191
|
|
|
1149
1192
|
// src/campaign/gates/heldout-gate.ts
|
|
@@ -1207,117 +1250,628 @@ function heldOutGate(options) {
|
|
|
1207
1250
|
};
|
|
1208
1251
|
}
|
|
1209
1252
|
|
|
1210
|
-
// src/campaign/
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1253
|
+
// src/campaign/gates/power-preflight.ts
|
|
1254
|
+
function zFor(confidence) {
|
|
1255
|
+
if (confidence >= 0.99) return 2.576;
|
|
1256
|
+
if (confidence >= 0.95) return 1.96;
|
|
1257
|
+
if (confidence >= 0.9) return 1.645;
|
|
1258
|
+
return 1.282;
|
|
1259
|
+
}
|
|
1260
|
+
function powerPreflight(opts) {
|
|
1261
|
+
const composites = opts.baselineComposites.filter((v) => Number.isFinite(v));
|
|
1262
|
+
if (composites.length < 3) {
|
|
1263
|
+
throw new Error(
|
|
1264
|
+
`powerPreflight: need >= 3 finite baseline composites to estimate variance, got ${composites.length}`
|
|
1265
|
+
);
|
|
1222
1266
|
}
|
|
1223
|
-
const
|
|
1224
|
-
const
|
|
1225
|
-
const
|
|
1226
|
-
|
|
1227
|
-
const
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1267
|
+
const deltaThreshold = opts.deltaThreshold ?? 0.05;
|
|
1268
|
+
const confidence = opts.confidence ?? 0.95;
|
|
1269
|
+
const n = opts.pairedN ?? composites.length;
|
|
1270
|
+
if (n < 2) throw new Error(`powerPreflight: pairedN must be >= 2, got ${n}`);
|
|
1271
|
+
const mean2 = composites.reduce((a, b) => a + b, 0) / composites.length;
|
|
1272
|
+
const variance = composites.reduce((a, b) => a + (b - mean2) * (b - mean2), 0) / (composites.length - 1);
|
|
1273
|
+
const sd = Math.sqrt(variance);
|
|
1274
|
+
const z = zFor(confidence);
|
|
1275
|
+
const mde = deltaThreshold + z * Math.SQRT2 * sd / Math.sqrt(n);
|
|
1276
|
+
const scaleAssumed = composites.every((v) => v >= -1e-3 && v <= 1.5);
|
|
1277
|
+
const headroom = Math.max(0, 1 - mean2);
|
|
1278
|
+
const underpowered = scaleAssumed && mde > headroom;
|
|
1279
|
+
const sharedChannelCaveat = opts.sharedScorerChannel ? "Holdout and gate share one scoring channel: raising n/reps reduces only idiosyncratic noise \u2014 systematic judge bias remains and this MDE is a lower bound. Full debiasing needs an independent second scoring channel (different judge/benchmark family)." : void 0;
|
|
1280
|
+
const recommendation = underpowered ? `UNDERPOWERED: minimum detectable lift ${mde.toFixed(3)} exceeds the ${headroom.toFixed(3)} headroom above the baseline (${mean2.toFixed(3)}) \u2014 no achievable effect can ship at this budget. Raise paired n (scenarios x reps) to ~${Math.ceil((z * Math.SQRT2 * sd / Math.max(headroom - deltaThreshold, 0.01)) ** 2)} or reduce worker variance before searching.` : `Minimum detectable lift at n=${n}: ${mde.toFixed(3)} (baseline sd ${sd.toFixed(3)}). Effects smaller than this cannot clear the gate; budget the search for effects you believe exceed it.`;
|
|
1281
|
+
return {
|
|
1282
|
+
n,
|
|
1283
|
+
sd,
|
|
1284
|
+
mde,
|
|
1285
|
+
baselineMean: mean2,
|
|
1286
|
+
headroom,
|
|
1287
|
+
underpowered,
|
|
1288
|
+
scaleAssumed,
|
|
1289
|
+
deltaThreshold,
|
|
1290
|
+
confidence,
|
|
1291
|
+
...sharedChannelCaveat ? { sharedChannelCaveat } : {},
|
|
1292
|
+
recommendation: sharedChannelCaveat ? `${recommendation} ${sharedChannelCaveat}` : recommendation
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
// src/campaign/gates/promotion-policy.ts
|
|
1297
|
+
function buildEvidenceVector(ctx, objectives, opts = {}) {
|
|
1298
|
+
if (objectives.length === 0) {
|
|
1299
|
+
throw new Error("buildEvidenceVector: at least 1 objective required");
|
|
1235
1300
|
}
|
|
1236
|
-
const
|
|
1237
|
-
const
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1301
|
+
const minProductiveRuns = opts.minProductiveRuns ?? 3;
|
|
1302
|
+
const confidence = opts.confidence ?? 0.95;
|
|
1303
|
+
const resamples = opts.resamples ?? 2e3;
|
|
1304
|
+
const seed = opts.seed ?? 1337;
|
|
1305
|
+
const baseline = ctx.baselineJudgeScores ?? ctx.judgeScores;
|
|
1306
|
+
const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
|
|
1307
|
+
const axes = [];
|
|
1308
|
+
for (const obj of objectives) {
|
|
1309
|
+
let select;
|
|
1310
|
+
if (obj.source.kind === "composite") {
|
|
1311
|
+
select = (s) => s.composite;
|
|
1312
|
+
} else {
|
|
1313
|
+
const dim = obj.source.dimension;
|
|
1314
|
+
select = (s) => s.dimensions[dim];
|
|
1315
|
+
}
|
|
1316
|
+
const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select);
|
|
1317
|
+
const before = obj.direction === "maximize" ? paired.before : paired.after;
|
|
1318
|
+
const after = obj.direction === "maximize" ? paired.after : paired.before;
|
|
1319
|
+
const bootstrap = pairedBootstrap(before, after, {
|
|
1320
|
+
confidence,
|
|
1321
|
+
resamples,
|
|
1322
|
+
statistic: "median",
|
|
1323
|
+
seed
|
|
1324
|
+
});
|
|
1325
|
+
const n = paired.before.length;
|
|
1326
|
+
const floorTolerance = obj.floorTolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
|
|
1327
|
+
const gainThreshold = obj.gainThreshold ?? 0;
|
|
1328
|
+
const verdict = n < minProductiveRuns ? "few_runs" : bootstrap.low < -floorTolerance ? "regressed" : bootstrap.low > gainThreshold ? "improved" : "flat";
|
|
1329
|
+
axes.push({
|
|
1330
|
+
name: obj.name,
|
|
1331
|
+
source: obj.source,
|
|
1332
|
+
direction: obj.direction,
|
|
1333
|
+
bootstrap,
|
|
1334
|
+
n,
|
|
1335
|
+
gainThreshold,
|
|
1336
|
+
floorTolerance,
|
|
1337
|
+
verdict
|
|
1338
|
+
});
|
|
1255
1339
|
}
|
|
1256
|
-
const
|
|
1257
|
-
|
|
1340
|
+
const ns = axes.map((a) => a.n).filter((n) => n > 0);
|
|
1341
|
+
const minN = ns.length > 0 ? Math.min(...ns) : 0;
|
|
1342
|
+
return { axes, minN, cost: { candidate: ctx.cost.candidate, baseline: ctx.cost.baseline } };
|
|
1343
|
+
}
|
|
1344
|
+
var paretoPolicy = (ev) => {
|
|
1345
|
+
const contributingGates = ev.axes.map((ax) => ({
|
|
1346
|
+
name: `objective:${ax.name}`,
|
|
1347
|
+
passed: ax.verdict === "improved",
|
|
1348
|
+
detail: {
|
|
1349
|
+
direction: ax.direction,
|
|
1350
|
+
source: ax.source,
|
|
1351
|
+
verdict: ax.verdict,
|
|
1352
|
+
n: ax.n,
|
|
1353
|
+
deltaMedian: ax.bootstrap.median,
|
|
1354
|
+
ciLow: ax.bootstrap.low,
|
|
1355
|
+
ciHigh: ax.bootstrap.high,
|
|
1356
|
+
confidence: ax.bootstrap.confidence,
|
|
1357
|
+
gainThreshold: ax.gainThreshold,
|
|
1358
|
+
floorTolerance: ax.floorTolerance
|
|
1359
|
+
}
|
|
1360
|
+
}));
|
|
1361
|
+
const regressed = ev.axes.filter((a) => a.verdict === "regressed");
|
|
1362
|
+
const fewRuns = ev.axes.filter((a) => a.verdict === "few_runs");
|
|
1363
|
+
const improved = ev.axes.filter((a) => a.verdict === "improved");
|
|
1364
|
+
let decision;
|
|
1365
|
+
const reasons = [];
|
|
1366
|
+
if (regressed.length > 0) {
|
|
1367
|
+
decision = "hold";
|
|
1368
|
+
for (const a of regressed) {
|
|
1369
|
+
reasons.push(
|
|
1370
|
+
`objective '${a.name}' regressed: good-direction CI.low ${a.bootstrap.low.toFixed(3)} < -${a.floorTolerance} (n=${a.n})`
|
|
1371
|
+
);
|
|
1372
|
+
}
|
|
1373
|
+
} else if (fewRuns.length > 0) {
|
|
1374
|
+
decision = "need_more_work";
|
|
1375
|
+
for (const a of fewRuns) {
|
|
1376
|
+
reasons.push(
|
|
1377
|
+
`objective '${a.name}' has only n=${a.n} paired runs \u2014 insufficient evidence to claim significance`
|
|
1378
|
+
);
|
|
1379
|
+
}
|
|
1380
|
+
} else if (improved.length > 0) {
|
|
1381
|
+
decision = "ship";
|
|
1382
|
+
reasons.push(
|
|
1383
|
+
`Pareto improvement at the confidence level: ${improved.map(
|
|
1384
|
+
(a) => `'${a.name}' +${a.bootstrap.median.toFixed(3)} (CI.low ${a.bootstrap.low.toFixed(3)})`
|
|
1385
|
+
).join(", ")}; no objective regressed`
|
|
1386
|
+
);
|
|
1387
|
+
} else {
|
|
1388
|
+
decision = "hold";
|
|
1389
|
+
reasons.push(
|
|
1390
|
+
"no Pareto improvement: candidate statistically equivalent to baseline on every objective"
|
|
1391
|
+
);
|
|
1392
|
+
}
|
|
1393
|
+
const composite = ev.axes.find((a) => a.source.kind === "composite") ?? ev.axes[0];
|
|
1394
|
+
return { decision, reasons, contributingGates, delta: composite?.bootstrap.median };
|
|
1395
|
+
};
|
|
1396
|
+
function paretoSignificanceGate(options) {
|
|
1397
|
+
if (options.objectives.length === 0) {
|
|
1398
|
+
throw new Error("paretoSignificanceGate: at least 1 objective required");
|
|
1399
|
+
}
|
|
1400
|
+
const policy = options.policy ?? paretoPolicy;
|
|
1401
|
+
return {
|
|
1402
|
+
name: options.name ?? "paretoSignificanceGate",
|
|
1403
|
+
async decide(ctx) {
|
|
1404
|
+
const ev = buildEvidenceVector(ctx, options.objectives, options);
|
|
1405
|
+
return policy(ev);
|
|
1406
|
+
}
|
|
1407
|
+
};
|
|
1258
1408
|
}
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1409
|
+
|
|
1410
|
+
// src/campaign/types.ts
|
|
1411
|
+
function isProposedCandidate(value) {
|
|
1412
|
+
return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
|
|
1413
|
+
}
|
|
1414
|
+
var LABEL_TRUST_RANK = {
|
|
1415
|
+
unverified: 0,
|
|
1416
|
+
"verified-signal": 1,
|
|
1417
|
+
"human-rated": 2
|
|
1418
|
+
};
|
|
1419
|
+
function labelTrustRank(trust) {
|
|
1420
|
+
return LABEL_TRUST_RANK[trust ?? "unverified"];
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
// src/json-recovery.ts
|
|
1424
|
+
function autoCloseTruncatedJson(raw) {
|
|
1425
|
+
const stack = [];
|
|
1426
|
+
let inString = false;
|
|
1427
|
+
let escaped = false;
|
|
1428
|
+
for (const c of raw) {
|
|
1429
|
+
if (escaped) {
|
|
1430
|
+
escaped = false;
|
|
1431
|
+
continue;
|
|
1432
|
+
}
|
|
1433
|
+
if (inString) {
|
|
1434
|
+
if (c === "\\") {
|
|
1435
|
+
escaped = true;
|
|
1436
|
+
continue;
|
|
1437
|
+
}
|
|
1438
|
+
if (c === '"') {
|
|
1439
|
+
inString = false;
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1442
|
+
continue;
|
|
1443
|
+
}
|
|
1444
|
+
if (c === '"') {
|
|
1445
|
+
inString = true;
|
|
1446
|
+
continue;
|
|
1447
|
+
}
|
|
1448
|
+
if (c === "{" || c === "[") stack.push(c);
|
|
1449
|
+
else if (c === "}") {
|
|
1450
|
+
if (stack.pop() !== "{") return null;
|
|
1451
|
+
} else if (c === "]") {
|
|
1452
|
+
if (stack.pop() !== "[") return null;
|
|
1453
|
+
}
|
|
1283
1454
|
}
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1455
|
+
if (stack.length === 0 && !inString) return raw;
|
|
1456
|
+
let suffix = "";
|
|
1457
|
+
if (escaped) suffix += "\\";
|
|
1458
|
+
if (inString) suffix += '"';
|
|
1459
|
+
while (stack.length > 0) {
|
|
1460
|
+
const opener = stack.pop();
|
|
1461
|
+
suffix += opener === "{" ? "}" : "]";
|
|
1462
|
+
}
|
|
1463
|
+
return raw + suffix;
|
|
1464
|
+
}
|
|
1465
|
+
var UNPARSEABLE = /* @__PURE__ */ Symbol("unparseable");
|
|
1466
|
+
function tryParse(candidate) {
|
|
1467
|
+
try {
|
|
1468
|
+
return JSON.parse(candidate);
|
|
1469
|
+
} catch {
|
|
1470
|
+
return UNPARSEABLE;
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
function lastCommaOutsideString(s) {
|
|
1474
|
+
let inString = false;
|
|
1475
|
+
let escaped = false;
|
|
1476
|
+
let last = -1;
|
|
1477
|
+
for (let i = 0; i < s.length; i++) {
|
|
1478
|
+
const c = s[i];
|
|
1479
|
+
if (escaped) {
|
|
1480
|
+
escaped = false;
|
|
1481
|
+
continue;
|
|
1482
|
+
}
|
|
1483
|
+
if (inString) {
|
|
1484
|
+
if (c === "\\") escaped = true;
|
|
1485
|
+
else if (c === '"') inString = false;
|
|
1486
|
+
continue;
|
|
1487
|
+
}
|
|
1488
|
+
if (c === '"') inString = true;
|
|
1489
|
+
else if (c === ",") last = i;
|
|
1490
|
+
}
|
|
1491
|
+
return last;
|
|
1492
|
+
}
|
|
1493
|
+
function recoverTruncatedJson(text) {
|
|
1494
|
+
const objStart = text.indexOf("{");
|
|
1495
|
+
const arrStart = text.indexOf("[");
|
|
1496
|
+
const starts = [objStart, arrStart].filter((i) => i >= 0).sort((a, b) => a - b);
|
|
1497
|
+
for (const start of starts) {
|
|
1498
|
+
const recovered = recoverFrom(text.slice(start));
|
|
1499
|
+
if (recovered !== UNPARSEABLE) return recovered;
|
|
1500
|
+
}
|
|
1501
|
+
return null;
|
|
1502
|
+
}
|
|
1503
|
+
function recoverFrom(slice) {
|
|
1504
|
+
let candidate = slice;
|
|
1505
|
+
const lastClose = Math.max(candidate.lastIndexOf("}"), candidate.lastIndexOf("]"));
|
|
1506
|
+
if (lastClose > 0) {
|
|
1507
|
+
const balanced = tryParse(candidate.slice(0, lastClose + 1));
|
|
1508
|
+
if (balanced !== UNPARSEABLE) return balanced;
|
|
1509
|
+
}
|
|
1510
|
+
for (let i = 0; i < 64; i++) {
|
|
1511
|
+
const closed = autoCloseTruncatedJson(candidate);
|
|
1512
|
+
if (closed !== null) {
|
|
1513
|
+
const parsed = tryParse(closed);
|
|
1514
|
+
if (parsed !== UNPARSEABLE) return parsed;
|
|
1515
|
+
}
|
|
1516
|
+
const cut = lastCommaOutsideString(candidate);
|
|
1517
|
+
if (cut <= 0) return UNPARSEABLE;
|
|
1518
|
+
candidate = candidate.slice(0, cut);
|
|
1519
|
+
}
|
|
1520
|
+
return UNPARSEABLE;
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
// src/reflective-mutation.ts
|
|
1524
|
+
var DEFAULT_MUTATION_PRIMITIVES = [
|
|
1525
|
+
'Strengthen an imperative ("should" \u2192 "must")',
|
|
1526
|
+
"Add a concrete example pulled from a missed-golden phrase",
|
|
1527
|
+
"Remove a redundant rule that did not improve recall",
|
|
1528
|
+
'Add a counterfactual ("if X is missing, the score is capped at Y")',
|
|
1529
|
+
"Reorder sections so the highest-impact rule is first",
|
|
1530
|
+
"Replace abstract language with a domain-specific noun the trial misses"
|
|
1531
|
+
];
|
|
1532
|
+
function buildReflectionPrompt(ctx) {
|
|
1533
|
+
const primitives = ctx.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES;
|
|
1534
|
+
const sections = [];
|
|
1535
|
+
sections.push(`# Mutation target: ${ctx.target}`);
|
|
1536
|
+
sections.push("");
|
|
1537
|
+
sections.push(
|
|
1538
|
+
`You are tuning the prompt component named \`${ctx.target}\`. The current variant is shown below; you have ${ctx.topTrials.length} top trials and ${ctx.bottomTrials.length} bottom trials as evidence. Propose ${ctx.childCount} mutation${ctx.childCount === 1 ? "" : "s"} that fix specific weaknesses visible in the bottom trials. Avoid blank rephrasings.`
|
|
1539
|
+
);
|
|
1540
|
+
sections.push("");
|
|
1541
|
+
sections.push("## Current variant");
|
|
1542
|
+
sections.push("```json");
|
|
1543
|
+
sections.push(JSON.stringify(ctx.parentPayload, null, 2));
|
|
1544
|
+
sections.push("```");
|
|
1545
|
+
sections.push("");
|
|
1546
|
+
if (ctx.bottomTrials.length > 0) {
|
|
1547
|
+
sections.push("## Failures (bottom trials) \u2014 what went wrong");
|
|
1548
|
+
sections.push("");
|
|
1549
|
+
for (const trial of ctx.bottomTrials) {
|
|
1550
|
+
sections.push(
|
|
1551
|
+
`### Trial \`${trial.id}\` \u2014 score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
|
|
1552
|
+
);
|
|
1553
|
+
if (trial.failureNote) {
|
|
1554
|
+
sections.push("");
|
|
1555
|
+
sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 600)}`);
|
|
1556
|
+
}
|
|
1557
|
+
const missed = (trial.expectations ?? []).filter((e) => !e.matched);
|
|
1558
|
+
if (missed.length > 0) {
|
|
1559
|
+
sections.push("");
|
|
1560
|
+
sections.push("**Missed expectations:**");
|
|
1561
|
+
for (const m of missed) {
|
|
1562
|
+
sections.push(`- \`${m.id}\`: should match phrase \`${quote(m.phrase)}\``);
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
if (trial.emitted) {
|
|
1566
|
+
sections.push("");
|
|
1567
|
+
sections.push("**What the agent emitted:**");
|
|
1568
|
+
sections.push("```");
|
|
1569
|
+
sections.push(truncate(trial.emitted, 600));
|
|
1570
|
+
sections.push("```");
|
|
1571
|
+
}
|
|
1572
|
+
sections.push("");
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
if (ctx.topTrials.length > 0) {
|
|
1576
|
+
sections.push("## Successes (top trials) \u2014 what to preserve");
|
|
1577
|
+
sections.push("");
|
|
1578
|
+
for (const trial of ctx.topTrials) {
|
|
1579
|
+
sections.push(
|
|
1580
|
+
`- \`${trial.id}\`: score ${trial.score.toFixed(2)}${trial.inputName ? ` (${trial.inputName})` : ""}`
|
|
1581
|
+
);
|
|
1582
|
+
}
|
|
1583
|
+
sections.push("");
|
|
1584
|
+
}
|
|
1585
|
+
sections.push("## Allowed mutation primitives");
|
|
1586
|
+
sections.push("");
|
|
1587
|
+
for (const p of primitives) sections.push(`- ${p}`);
|
|
1588
|
+
sections.push("");
|
|
1589
|
+
sections.push("## Output schema");
|
|
1590
|
+
sections.push("");
|
|
1591
|
+
sections.push("Respond with a JSON object \u2014 no prose, no markdown fences:");
|
|
1592
|
+
sections.push("```json");
|
|
1593
|
+
sections.push(
|
|
1594
|
+
JSON.stringify(
|
|
1595
|
+
{
|
|
1596
|
+
proposals: [
|
|
1597
|
+
{
|
|
1598
|
+
label: "<short label, \u2264 40 chars>",
|
|
1599
|
+
rationale: "<which failure this targets and which primitive you used>",
|
|
1600
|
+
payload: "<full payload of the new variant \u2014 same shape as the current variant>"
|
|
1601
|
+
}
|
|
1602
|
+
]
|
|
1603
|
+
},
|
|
1604
|
+
null,
|
|
1605
|
+
2
|
|
1606
|
+
)
|
|
1607
|
+
);
|
|
1608
|
+
sections.push("```");
|
|
1609
|
+
return sections.join("\n");
|
|
1610
|
+
}
|
|
1611
|
+
function renderAnalystEvidence(findings, report) {
|
|
1612
|
+
const sections = [];
|
|
1613
|
+
const rows = Array.isArray(findings) ? findings : [];
|
|
1614
|
+
if (rows.length > 0) {
|
|
1615
|
+
const order = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
|
|
1616
|
+
const ranked = rows.map((f) => f && typeof f === "object" ? f : {}).sort((a, b) => (order[String(a.severity)] ?? 9) - (order[String(b.severity)] ?? 9)).slice(0, 12);
|
|
1617
|
+
sections.push("## Diagnosed findings (act on these root causes)");
|
|
1618
|
+
sections.push("");
|
|
1619
|
+
for (const f of ranked) {
|
|
1620
|
+
const sev = typeof f.severity === "string" ? f.severity : "info";
|
|
1621
|
+
const area = typeof f.area === "string" ? ` [${f.area}]` : "";
|
|
1622
|
+
const claim = typeof f.claim === "string" ? f.claim : JSON.stringify(f);
|
|
1623
|
+
sections.push(`- **${sev}**${area} ${truncate(claim, 300)}`);
|
|
1624
|
+
if (typeof f.recommended_action === "string" && f.recommended_action.trim()) {
|
|
1625
|
+
sections.push(` - fix: ${truncate(f.recommended_action, 200)}`);
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
sections.push("");
|
|
1629
|
+
}
|
|
1630
|
+
const reportText = typeof report === "string" ? report : report && typeof report === "object" ? JSON.stringify(report) : "";
|
|
1631
|
+
if (reportText.trim()) {
|
|
1632
|
+
sections.push("## Research report");
|
|
1633
|
+
sections.push("");
|
|
1634
|
+
sections.push(truncate(reportText.trim(), 1500));
|
|
1635
|
+
sections.push("");
|
|
1636
|
+
}
|
|
1637
|
+
if (sections.length === 0) return null;
|
|
1638
|
+
return sections.join("\n");
|
|
1639
|
+
}
|
|
1640
|
+
function truncate(s, max) {
|
|
1641
|
+
if (s.length <= max) return s;
|
|
1642
|
+
return `${s.slice(0, max)}\u2026 [truncated]`;
|
|
1643
|
+
}
|
|
1644
|
+
function quote(s) {
|
|
1645
|
+
return s.replace(/`/g, "\\`");
|
|
1646
|
+
}
|
|
1647
|
+
function parseReflectionResponse(raw, maxProposals) {
|
|
1648
|
+
let text = raw.trim();
|
|
1649
|
+
if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
|
|
1650
|
+
let parsed = null;
|
|
1651
|
+
const objectStart = text.indexOf("{");
|
|
1652
|
+
const objectEnd = text.lastIndexOf("}");
|
|
1653
|
+
const arrayStart = text.indexOf("[");
|
|
1654
|
+
const arrayEnd = text.lastIndexOf("]");
|
|
1655
|
+
const tryObjectFirst = objectStart >= 0 && (arrayStart < 0 || objectStart < arrayStart);
|
|
1656
|
+
const candidates = [];
|
|
1657
|
+
if (tryObjectFirst) {
|
|
1658
|
+
if (objectStart >= 0 && objectEnd > objectStart)
|
|
1659
|
+
candidates.push(text.slice(objectStart, objectEnd + 1));
|
|
1660
|
+
if (arrayStart >= 0 && arrayEnd > arrayStart)
|
|
1661
|
+
candidates.push(text.slice(arrayStart, arrayEnd + 1));
|
|
1662
|
+
} else {
|
|
1663
|
+
if (arrayStart >= 0 && arrayEnd > arrayStart)
|
|
1664
|
+
candidates.push(text.slice(arrayStart, arrayEnd + 1));
|
|
1665
|
+
if (objectStart >= 0 && objectEnd > objectStart)
|
|
1666
|
+
candidates.push(text.slice(objectStart, objectEnd + 1));
|
|
1667
|
+
}
|
|
1668
|
+
for (const slice of candidates) {
|
|
1669
|
+
try {
|
|
1670
|
+
parsed = JSON.parse(slice);
|
|
1671
|
+
break;
|
|
1672
|
+
} catch {
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
if (parsed == null) {
|
|
1676
|
+
for (const slice of candidates) {
|
|
1677
|
+
const closed = autoCloseTruncatedJson(slice);
|
|
1678
|
+
if (closed != null && closed !== slice) {
|
|
1679
|
+
try {
|
|
1680
|
+
parsed = JSON.parse(closed);
|
|
1681
|
+
break;
|
|
1682
|
+
} catch {
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if (parsed == null) return [];
|
|
1688
|
+
let proposalsRaw;
|
|
1689
|
+
if (Array.isArray(parsed)) {
|
|
1690
|
+
proposalsRaw = parsed;
|
|
1691
|
+
} else if (parsed && typeof parsed === "object") {
|
|
1692
|
+
proposalsRaw = parsed.proposals;
|
|
1693
|
+
}
|
|
1694
|
+
if (!Array.isArray(proposalsRaw)) return [];
|
|
1695
|
+
const out = [];
|
|
1696
|
+
for (const p of proposalsRaw) {
|
|
1697
|
+
if (!p || typeof p !== "object") continue;
|
|
1698
|
+
const obj = p;
|
|
1699
|
+
if (!("payload" in obj)) continue;
|
|
1700
|
+
out.push({
|
|
1701
|
+
label: typeof obj.label === "string" ? obj.label : "mutation",
|
|
1702
|
+
rationale: typeof obj.rationale === "string" ? obj.rationale : "",
|
|
1703
|
+
payload: obj.payload
|
|
1704
|
+
});
|
|
1705
|
+
if (maxProposals !== void 0 && out.length >= maxProposals) break;
|
|
1706
|
+
}
|
|
1707
|
+
return out;
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
// src/campaign/proposers/gepa.ts
|
|
1711
|
+
var REFLECTION_SYSTEM = 'You are an expert prompt engineer performing GEPA-style reflective mutation. You are given a prompt surface, its top trials (preserve what works) and its bottom trials (the evidence to fix). For each proposal, reason in this order before writing the payload: (1) LOCALIZE \u2014 point to the exact span of the current surface responsible for a bottom-trial failure; (2) DIAGNOSE the root cause (a missing rule, an ambiguous instruction, an over-broad directive), not just the symptom; (3) propose the MINIMAL, GENERALIZABLE edit that fixes the whole failure class \u2014 state it as a rule the agent should follow, never a patch memorized to the shown trials (that is overfitting and will not transfer to the held-out set); (4) PRESERVE every instruction the top trials depend on \u2014 do not delete or weaken working guidance. Put this localize\u2192diagnose\u2192fix reasoning in each proposal\'s `rationale`. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} where each `payload` is the FULL improved surface text. No prose outside the JSON.';
|
|
1712
|
+
var COMBINE_SYSTEM = 'You are an expert prompt engineer performing a GEPA "combine complementary lessons" merge. You are given several non-dominated versions of one surface; each is uniquely best on different scenarios. Produce ONE new version that keeps what makes each version strong on its winning scenarios and resolves conflicts in favor of the more general rule. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} with exactly one proposal whose `payload` is the FULL merged surface text. No prose outside the JSON.';
|
|
1713
|
+
function gepaProposer(opts) {
|
|
1714
|
+
const evidenceK = opts.evidenceK ?? 3;
|
|
1715
|
+
const combineParents = opts.combineParents ?? true;
|
|
1716
|
+
const combineMaxParents = opts.combineMaxParents ?? 4;
|
|
1717
|
+
if (combineParents && combineMaxParents < 1) {
|
|
1718
|
+
throw new Error("gepaProposer: combineMaxParents must be >= 1 when combineParents is enabled");
|
|
1719
|
+
}
|
|
1720
|
+
return {
|
|
1721
|
+
kind: "gepa",
|
|
1722
|
+
async propose(ctx) {
|
|
1723
|
+
const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
|
|
1724
|
+
const constraints = opts.constraints;
|
|
1725
|
+
const preserveSections = constraints?.preserveSections !== void 0 ? constraints.preserveSections.length === 0 ? extractH2Sections(parent) : constraints.preserveSections : null;
|
|
1726
|
+
const maxEdits = constraints?.maxSentenceEdits;
|
|
1727
|
+
const out = [];
|
|
1728
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1729
|
+
const accept = (payload, label, rationale) => {
|
|
1730
|
+
const text = typeof payload === "string" ? payload.trim() : "";
|
|
1731
|
+
if (!text || text === parent || seen.has(text)) return;
|
|
1732
|
+
if (preserveSections && !validatePreservedSections(text, preserveSections)) return;
|
|
1733
|
+
if (maxEdits !== void 0 && countSentenceEdits(parent, text) > maxEdits * 2) return;
|
|
1734
|
+
seen.add(text);
|
|
1735
|
+
out.push({ surface: text, label, rationale });
|
|
1736
|
+
};
|
|
1737
|
+
const stringParents = (combineParents ? ctx.paretoParents ?? [] : []).filter((p) => typeof p.surface === "string").sort((a, b) => b.composite - a.composite).slice(0, combineMaxParents);
|
|
1738
|
+
if (stringParents.length > 1) {
|
|
1739
|
+
const combinePrompt = buildCombinePrompt({
|
|
1740
|
+
target: opts.target,
|
|
1741
|
+
parents: stringParents,
|
|
1742
|
+
evidenceK
|
|
1743
|
+
});
|
|
1744
|
+
const combineResult = await callLlm(
|
|
1745
|
+
{
|
|
1746
|
+
model: opts.model,
|
|
1747
|
+
messages: [
|
|
1748
|
+
{ role: "system", content: COMBINE_SYSTEM },
|
|
1749
|
+
{ role: "user", content: combinePrompt }
|
|
1750
|
+
],
|
|
1751
|
+
jsonMode: true,
|
|
1752
|
+
temperature: opts.temperature ?? 0.7,
|
|
1753
|
+
maxTokens: opts.maxTokens ?? 6e3
|
|
1754
|
+
},
|
|
1755
|
+
opts.llm
|
|
1756
|
+
);
|
|
1757
|
+
const merged = parseReflectionResponse(combineResult.content, 1)[0];
|
|
1758
|
+
if (merged) {
|
|
1759
|
+
accept(
|
|
1760
|
+
merged.payload,
|
|
1761
|
+
merged.label || "pareto-combine",
|
|
1762
|
+
merged.rationale || `combined ${stringParents.length} non-dominated parents (gen ${stringParents.map((p) => p.generation).join(",")})`
|
|
1763
|
+
);
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
const reflectCount = Math.max(0, ctx.populationSize - out.length);
|
|
1767
|
+
if (reflectCount > 0) {
|
|
1768
|
+
const { top, bottom, target } = buildEvidence(ctx, evidenceK, opts.target);
|
|
1769
|
+
const userPrompt = buildReflectionPrompt({
|
|
1770
|
+
target,
|
|
1771
|
+
parentPayload: parent,
|
|
1772
|
+
topTrials: top,
|
|
1773
|
+
bottomTrials: bottom,
|
|
1774
|
+
childCount: reflectCount,
|
|
1775
|
+
mutationPrimitives: opts.mutationPrimitives
|
|
1776
|
+
});
|
|
1777
|
+
const analyst = renderAnalystEvidence(ctx.findings, ctx.report);
|
|
1778
|
+
const finalPrompt = analyst ? `${userPrompt}
|
|
1779
|
+
|
|
1780
|
+
${analyst}` : userPrompt;
|
|
1781
|
+
const result = await callLlm(
|
|
1782
|
+
{
|
|
1783
|
+
model: opts.model,
|
|
1784
|
+
messages: [
|
|
1785
|
+
{ role: "system", content: REFLECTION_SYSTEM },
|
|
1786
|
+
{ role: "user", content: finalPrompt }
|
|
1787
|
+
],
|
|
1788
|
+
jsonMode: true,
|
|
1789
|
+
temperature: opts.temperature ?? 0.7,
|
|
1790
|
+
maxTokens: opts.maxTokens ?? 6e3
|
|
1791
|
+
},
|
|
1792
|
+
opts.llm
|
|
1793
|
+
);
|
|
1794
|
+
for (const proposal of parseReflectionResponse(result.content, reflectCount)) {
|
|
1795
|
+
accept(proposal.payload, proposal.label, proposal.rationale);
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
return out.slice(0, ctx.populationSize);
|
|
1799
|
+
}
|
|
1800
|
+
};
|
|
1801
|
+
}
|
|
1802
|
+
function buildCombinePrompt(args) {
|
|
1803
|
+
const lines = [
|
|
1804
|
+
`You are merging ${args.parents.length} versions of: ${args.target}.`,
|
|
1805
|
+
"",
|
|
1806
|
+
"Each version is on the Pareto frontier \u2014 none dominates the others; each",
|
|
1807
|
+
"wins on different scenarios. Combine their complementary strengths into",
|
|
1808
|
+
"ONE version. Below, each version lists the scenarios it scores highest on.",
|
|
1809
|
+
""
|
|
1810
|
+
];
|
|
1811
|
+
args.parents.forEach((p, i) => {
|
|
1812
|
+
const tag = String.fromCharCode(65 + i);
|
|
1813
|
+
const best = Object.entries(p.objectives).sort((a, b) => b[1] - a[1]).slice(0, args.evidenceK).map(([id, score]) => `${id} (${score.toFixed(2)})`);
|
|
1814
|
+
lines.push(
|
|
1815
|
+
`### Version ${tag} (mean ${p.composite.toFixed(2)}; strongest on: ${best.join(", ") || "n/a"})`,
|
|
1816
|
+
"```",
|
|
1817
|
+
p.surface,
|
|
1818
|
+
"```",
|
|
1819
|
+
""
|
|
1820
|
+
);
|
|
1821
|
+
});
|
|
1822
|
+
lines.push(
|
|
1823
|
+
"Return ONE merged version that would score well on the union of every",
|
|
1824
|
+
"version's winning scenarios. Keep each version's specific winning rule;",
|
|
1825
|
+
"where two rules conflict, prefer the more general one and note the choice",
|
|
1826
|
+
"in your rationale."
|
|
1827
|
+
);
|
|
1828
|
+
return lines.join("\n");
|
|
1829
|
+
}
|
|
1830
|
+
function extractH2Sections(text) {
|
|
1831
|
+
const out = [];
|
|
1832
|
+
for (const line of text.split("\n")) {
|
|
1833
|
+
const match = /^##\s+(.+?)\s*$/.exec(line);
|
|
1834
|
+
if (match) out.push(match[1]);
|
|
1299
1835
|
}
|
|
1300
|
-
return
|
|
1836
|
+
return out;
|
|
1301
1837
|
}
|
|
1302
|
-
function
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1838
|
+
function countSentenceEdits(baseline, candidate) {
|
|
1839
|
+
const norm = (s) => s.split(/(?<=[.!?])\s+|\n/g).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
1840
|
+
const a = new Set(norm(baseline));
|
|
1841
|
+
const b = new Set(norm(candidate));
|
|
1842
|
+
let edits = 0;
|
|
1843
|
+
for (const s of a) if (!b.has(s)) edits++;
|
|
1844
|
+
for (const s of b) if (!a.has(s)) edits++;
|
|
1845
|
+
return edits;
|
|
1846
|
+
}
|
|
1847
|
+
function validatePreservedSections(candidate, required) {
|
|
1848
|
+
if (required.length === 0) return true;
|
|
1849
|
+
const have = new Set(extractH2Sections(candidate));
|
|
1850
|
+
for (const section of required) {
|
|
1851
|
+
if (!have.has(section)) return false;
|
|
1316
1852
|
}
|
|
1853
|
+
return true;
|
|
1317
1854
|
}
|
|
1318
|
-
function
|
|
1319
|
-
|
|
1320
|
-
|
|
1855
|
+
function buildEvidence(ctx, evidenceK, baseTarget) {
|
|
1856
|
+
const last = ctx.history.at(-1);
|
|
1857
|
+
if (!last || last.candidates.length === 0) {
|
|
1858
|
+
return { top: [], bottom: [], target: baseTarget };
|
|
1859
|
+
}
|
|
1860
|
+
const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
|
|
1861
|
+
if (!best) return { top: [], bottom: [], target: baseTarget };
|
|
1862
|
+
const byScore = [...best.scenarios].sort((a, b) => b.composite - a.composite);
|
|
1863
|
+
const toTrace = (s) => ({
|
|
1864
|
+
id: s.scenarioId,
|
|
1865
|
+
score: s.composite,
|
|
1866
|
+
// The judge's "why it scored low" — grounds the reflection on real failure
|
|
1867
|
+
// patterns instead of blind rephrasing. Generalizable by the judge contract.
|
|
1868
|
+
...s.notes ? { failureNote: s.notes } : {}
|
|
1869
|
+
});
|
|
1870
|
+
const top = byScore.slice(0, evidenceK).map(toTrace);
|
|
1871
|
+
const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
|
|
1872
|
+
const weakest = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([dim, value]) => `${dim} (${value.toFixed(2)})`);
|
|
1873
|
+
const target = weakest.length > 0 ? `${baseTarget} \u2014 weakest dimensions: ${weakest.join(", ")}` : baseTarget;
|
|
1874
|
+
return { top, bottom, target };
|
|
1321
1875
|
}
|
|
1322
1876
|
|
|
1323
1877
|
// src/campaign/score-utils.ts
|
|
@@ -1374,19 +1928,6 @@ function campaignBreakdown(campaign) {
|
|
|
1374
1928
|
return { dimensions, scenarios };
|
|
1375
1929
|
}
|
|
1376
1930
|
|
|
1377
|
-
// src/campaign/types.ts
|
|
1378
|
-
function isProposedCandidate(value) {
|
|
1379
|
-
return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
|
|
1380
|
-
}
|
|
1381
|
-
var LABEL_TRUST_RANK = {
|
|
1382
|
-
unverified: 0,
|
|
1383
|
-
"verified-signal": 1,
|
|
1384
|
-
"human-rated": 2
|
|
1385
|
-
};
|
|
1386
|
-
function labelTrustRank(trust) {
|
|
1387
|
-
return LABEL_TRUST_RANK[trust ?? "unverified"];
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
1931
|
// src/campaign/presets/run-optimization.ts
|
|
1391
1932
|
import { createHash } from "crypto";
|
|
1392
1933
|
async function runOptimization(opts) {
|
|
@@ -1698,172 +2239,25 @@ ${fmt(winnerSurface)}`;
|
|
|
1698
2239
|
return lines.join("\n");
|
|
1699
2240
|
}
|
|
1700
2241
|
|
|
1701
|
-
// src/campaign/
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
const evidenceK = opts.evidenceK ?? 3;
|
|
1706
|
-
const combineParents = opts.combineParents ?? true;
|
|
1707
|
-
const combineMaxParents = opts.combineMaxParents ?? 4;
|
|
1708
|
-
if (combineParents && combineMaxParents < 1) {
|
|
1709
|
-
throw new Error("gepaProposer: combineMaxParents must be >= 1 when combineParents is enabled");
|
|
1710
|
-
}
|
|
1711
|
-
return {
|
|
1712
|
-
kind: "gepa",
|
|
1713
|
-
async propose(ctx) {
|
|
1714
|
-
const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
|
|
1715
|
-
const constraints = opts.constraints;
|
|
1716
|
-
const preserveSections = constraints?.preserveSections !== void 0 ? constraints.preserveSections.length === 0 ? extractH2Sections(parent) : constraints.preserveSections : null;
|
|
1717
|
-
const maxEdits = constraints?.maxSentenceEdits;
|
|
1718
|
-
const out = [];
|
|
1719
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1720
|
-
const accept = (payload, label, rationale) => {
|
|
1721
|
-
const text = typeof payload === "string" ? payload.trim() : "";
|
|
1722
|
-
if (!text || text === parent || seen.has(text)) return;
|
|
1723
|
-
if (preserveSections && !validatePreservedSections(text, preserveSections)) return;
|
|
1724
|
-
if (maxEdits !== void 0 && countSentenceEdits(parent, text) > maxEdits * 2) return;
|
|
1725
|
-
seen.add(text);
|
|
1726
|
-
out.push({ surface: text, label, rationale });
|
|
1727
|
-
};
|
|
1728
|
-
const stringParents = (combineParents ? ctx.paretoParents ?? [] : []).filter((p) => typeof p.surface === "string").sort((a, b) => b.composite - a.composite).slice(0, combineMaxParents);
|
|
1729
|
-
if (stringParents.length > 1) {
|
|
1730
|
-
const combinePrompt = buildCombinePrompt({
|
|
1731
|
-
target: opts.target,
|
|
1732
|
-
parents: stringParents,
|
|
1733
|
-
evidenceK
|
|
1734
|
-
});
|
|
1735
|
-
const combineResult = await callLlm(
|
|
1736
|
-
{
|
|
1737
|
-
model: opts.model,
|
|
1738
|
-
messages: [
|
|
1739
|
-
{ role: "system", content: COMBINE_SYSTEM },
|
|
1740
|
-
{ role: "user", content: combinePrompt }
|
|
1741
|
-
],
|
|
1742
|
-
jsonMode: true,
|
|
1743
|
-
temperature: opts.temperature ?? 0.7,
|
|
1744
|
-
maxTokens: opts.maxTokens ?? 6e3
|
|
1745
|
-
},
|
|
1746
|
-
opts.llm
|
|
1747
|
-
);
|
|
1748
|
-
const merged = parseReflectionResponse(combineResult.content, 1)[0];
|
|
1749
|
-
if (merged) {
|
|
1750
|
-
accept(
|
|
1751
|
-
merged.payload,
|
|
1752
|
-
merged.label || "pareto-combine",
|
|
1753
|
-
merged.rationale || `combined ${stringParents.length} non-dominated parents (gen ${stringParents.map((p) => p.generation).join(",")})`
|
|
1754
|
-
);
|
|
1755
|
-
}
|
|
1756
|
-
}
|
|
1757
|
-
const reflectCount = Math.max(0, ctx.populationSize - out.length);
|
|
1758
|
-
if (reflectCount > 0) {
|
|
1759
|
-
const { top, bottom, target } = buildEvidence(ctx, evidenceK, opts.target);
|
|
1760
|
-
const userPrompt = buildReflectionPrompt({
|
|
1761
|
-
target,
|
|
1762
|
-
parentPayload: parent,
|
|
1763
|
-
topTrials: top,
|
|
1764
|
-
bottomTrials: bottom,
|
|
1765
|
-
childCount: reflectCount,
|
|
1766
|
-
mutationPrimitives: opts.mutationPrimitives
|
|
1767
|
-
});
|
|
1768
|
-
const analyst = renderAnalystEvidence(ctx.findings, ctx.report);
|
|
1769
|
-
const finalPrompt = analyst ? `${userPrompt}
|
|
2242
|
+
// src/campaign/presets/run-eval.ts
|
|
2243
|
+
async function runEval(opts) {
|
|
2244
|
+
return runCampaign(opts);
|
|
2245
|
+
}
|
|
1770
2246
|
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
},
|
|
1783
|
-
opts.llm
|
|
1784
|
-
);
|
|
1785
|
-
for (const proposal of parseReflectionResponse(result.content, reflectCount)) {
|
|
1786
|
-
accept(proposal.payload, proposal.label, proposal.rationale);
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
return out.slice(0, ctx.populationSize);
|
|
2247
|
+
// src/campaign/proposers/evolutionary.ts
|
|
2248
|
+
function evolutionaryProposer(opts) {
|
|
2249
|
+
return {
|
|
2250
|
+
kind: `evolutionary:${opts.mutator.kind}`,
|
|
2251
|
+
async propose({ currentSurface, findings, populationSize, signal }) {
|
|
2252
|
+
return opts.mutator.mutate({
|
|
2253
|
+
findings: findings.length > 0 ? findings : opts.findings ?? [],
|
|
2254
|
+
currentSurface,
|
|
2255
|
+
populationSize,
|
|
2256
|
+
signal
|
|
2257
|
+
});
|
|
1790
2258
|
}
|
|
1791
2259
|
};
|
|
1792
2260
|
}
|
|
1793
|
-
function buildCombinePrompt(args) {
|
|
1794
|
-
const lines = [
|
|
1795
|
-
`You are merging ${args.parents.length} versions of: ${args.target}.`,
|
|
1796
|
-
"",
|
|
1797
|
-
"Each version is on the Pareto frontier \u2014 none dominates the others; each",
|
|
1798
|
-
"wins on different scenarios. Combine their complementary strengths into",
|
|
1799
|
-
"ONE version. Below, each version lists the scenarios it scores highest on.",
|
|
1800
|
-
""
|
|
1801
|
-
];
|
|
1802
|
-
args.parents.forEach((p, i) => {
|
|
1803
|
-
const tag = String.fromCharCode(65 + i);
|
|
1804
|
-
const best = Object.entries(p.objectives).sort((a, b) => b[1] - a[1]).slice(0, args.evidenceK).map(([id, score]) => `${id} (${score.toFixed(2)})`);
|
|
1805
|
-
lines.push(
|
|
1806
|
-
`### Version ${tag} (mean ${p.composite.toFixed(2)}; strongest on: ${best.join(", ") || "n/a"})`,
|
|
1807
|
-
"```",
|
|
1808
|
-
p.surface,
|
|
1809
|
-
"```",
|
|
1810
|
-
""
|
|
1811
|
-
);
|
|
1812
|
-
});
|
|
1813
|
-
lines.push(
|
|
1814
|
-
"Return ONE merged version that would score well on the union of every",
|
|
1815
|
-
"version's winning scenarios. Keep each version's specific winning rule;",
|
|
1816
|
-
"where two rules conflict, prefer the more general one and note the choice",
|
|
1817
|
-
"in your rationale."
|
|
1818
|
-
);
|
|
1819
|
-
return lines.join("\n");
|
|
1820
|
-
}
|
|
1821
|
-
function extractH2Sections(text) {
|
|
1822
|
-
const out = [];
|
|
1823
|
-
for (const line of text.split("\n")) {
|
|
1824
|
-
const match = /^##\s+(.+?)\s*$/.exec(line);
|
|
1825
|
-
if (match) out.push(match[1]);
|
|
1826
|
-
}
|
|
1827
|
-
return out;
|
|
1828
|
-
}
|
|
1829
|
-
function countSentenceEdits(baseline, candidate) {
|
|
1830
|
-
const norm = (s) => s.split(/(?<=[.!?])\s+|\n/g).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
1831
|
-
const a = new Set(norm(baseline));
|
|
1832
|
-
const b = new Set(norm(candidate));
|
|
1833
|
-
let edits = 0;
|
|
1834
|
-
for (const s of a) if (!b.has(s)) edits++;
|
|
1835
|
-
for (const s of b) if (!a.has(s)) edits++;
|
|
1836
|
-
return edits;
|
|
1837
|
-
}
|
|
1838
|
-
function validatePreservedSections(candidate, required) {
|
|
1839
|
-
if (required.length === 0) return true;
|
|
1840
|
-
const have = new Set(extractH2Sections(candidate));
|
|
1841
|
-
for (const section of required) {
|
|
1842
|
-
if (!have.has(section)) return false;
|
|
1843
|
-
}
|
|
1844
|
-
return true;
|
|
1845
|
-
}
|
|
1846
|
-
function buildEvidence(ctx, evidenceK, baseTarget) {
|
|
1847
|
-
const last = ctx.history.at(-1);
|
|
1848
|
-
if (!last || last.candidates.length === 0) {
|
|
1849
|
-
return { top: [], bottom: [], target: baseTarget };
|
|
1850
|
-
}
|
|
1851
|
-
const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
|
|
1852
|
-
if (!best) return { top: [], bottom: [], target: baseTarget };
|
|
1853
|
-
const byScore = [...best.scenarios].sort((a, b) => b.composite - a.composite);
|
|
1854
|
-
const toTrace = (s) => ({
|
|
1855
|
-
id: s.scenarioId,
|
|
1856
|
-
score: s.composite,
|
|
1857
|
-
// The judge's "why it scored low" — grounds the reflection on real failure
|
|
1858
|
-
// patterns instead of blind rephrasing. Generalizable by the judge contract.
|
|
1859
|
-
...s.notes ? { failureNote: s.notes } : {}
|
|
1860
|
-
});
|
|
1861
|
-
const top = byScore.slice(0, evidenceK).map(toTrace);
|
|
1862
|
-
const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
|
|
1863
|
-
const weakest = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([dim, value]) => `${dim} (${value.toFixed(2)})`);
|
|
1864
|
-
const target = weakest.length > 0 ? `${baseTarget} \u2014 weakest dimensions: ${weakest.join(", ")}` : baseTarget;
|
|
1865
|
-
return { top, bottom, target };
|
|
1866
|
-
}
|
|
1867
2261
|
|
|
1868
2262
|
// src/campaign/provenance.ts
|
|
1869
2263
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -2147,29 +2541,36 @@ export {
|
|
|
2147
2541
|
scoreRedTeamOutput,
|
|
2148
2542
|
redTeamReport,
|
|
2149
2543
|
toolNamesForRun,
|
|
2544
|
+
openAutoPr,
|
|
2545
|
+
composeGate,
|
|
2150
2546
|
runCanaries,
|
|
2151
|
-
DEFAULT_MUTATION_PRIMITIVES,
|
|
2152
|
-
buildReflectionPrompt,
|
|
2153
|
-
renderAnalystEvidence,
|
|
2154
|
-
parseReflectionResponse,
|
|
2155
|
-
TIE_WARN_FRACTION,
|
|
2156
2547
|
pairHoldout,
|
|
2157
2548
|
heldoutSignificance,
|
|
2158
2549
|
detectScale,
|
|
2159
2550
|
dimensionRegressions,
|
|
2551
|
+
defaultProductionGate,
|
|
2160
2552
|
heldOutGate,
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2553
|
+
powerPreflight,
|
|
2554
|
+
buildEvidenceVector,
|
|
2555
|
+
paretoPolicy,
|
|
2556
|
+
paretoSignificanceGate,
|
|
2164
2557
|
isProposedCandidate,
|
|
2165
2558
|
labelTrustRank,
|
|
2559
|
+
DEFAULT_MUTATION_PRIMITIVES,
|
|
2560
|
+
buildReflectionPrompt,
|
|
2561
|
+
renderAnalystEvidence,
|
|
2562
|
+
parseReflectionResponse,
|
|
2563
|
+
gepaProposer,
|
|
2564
|
+
extractH2Sections,
|
|
2565
|
+
countSentenceEdits,
|
|
2566
|
+
campaignMeanComposite,
|
|
2567
|
+
campaignBreakdown,
|
|
2166
2568
|
runOptimization,
|
|
2167
2569
|
surfaceHash,
|
|
2168
2570
|
runImprovementLoop,
|
|
2169
2571
|
defaultRenderDiff,
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
countSentenceEdits,
|
|
2572
|
+
runEval,
|
|
2573
|
+
evolutionaryProposer,
|
|
2173
2574
|
surfaceContentHash,
|
|
2174
2575
|
buildLoopProvenanceRecord,
|
|
2175
2576
|
loopProvenanceSpans,
|
|
@@ -2177,4 +2578,4 @@ export {
|
|
|
2177
2578
|
provenanceSpansPath,
|
|
2178
2579
|
emitLoopProvenance
|
|
2179
2580
|
};
|
|
2180
|
-
//# sourceMappingURL=chunk-
|
|
2581
|
+
//# sourceMappingURL=chunk-OVPVM4JC.js.map
|