@tangle-network/agent-eval 0.217.1 → 0.218.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/CHANGELOG.md +36 -1
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/analyst/index.d.ts +1 -1
  4. package/dist/analyst/index.d.ts.map +1 -1
  5. package/dist/analyst/index.js +9 -9
  6. package/dist/analyst/index.js.map +1 -1
  7. package/dist/{analyze-runs-DyeNcenW.d.ts → analyze-runs-DjlZOVXx.d.ts} +2 -2
  8. package/dist/{analyze-runs-DyeNcenW.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
  9. package/dist/{benchmark-command-xQK8nOI6.js → benchmark-command-BupdHRyf.js} +661 -562
  10. package/dist/benchmark-command-BupdHRyf.js.map +1 -0
  11. package/dist/benchmarks/index.d.ts +4 -4
  12. package/dist/benchmarks/index.js +3 -3
  13. package/dist/campaign/index.d.ts +4 -4
  14. package/dist/campaign/index.js +10 -10
  15. package/dist/{campaign-CJGkxAQ6.js → campaign-CGvUTnmc.js} +312 -281
  16. package/dist/{campaign-CJGkxAQ6.js.map → campaign-CGvUTnmc.js.map} +1 -1
  17. package/dist/{campaign-evidence-CQYw8_0H.js → campaign-evidence-DunzcJGv.js} +2 -2
  18. package/dist/{campaign-evidence-CQYw8_0H.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
  19. package/dist/{chat-client-ldqxYYTN.js → chat-client-B2pXlPdx.js} +2 -2
  20. package/dist/{chat-client-ldqxYYTN.js.map → chat-client-B2pXlPdx.js.map} +1 -1
  21. package/dist/{chat-json-call-DavzexSp.js → chat-json-call-Dm9c-b41.js} +2 -2
  22. package/dist/{chat-json-call-DavzexSp.js.map → chat-json-call-Dm9c-b41.js.map} +1 -1
  23. package/dist/cli.js +10 -10
  24. package/dist/{completion-verifier-B_jw6C9p.js → completion-verifier-DbLVf3i8.js} +4 -4
  25. package/dist/{completion-verifier-B_jw6C9p.js.map → completion-verifier-DbLVf3i8.js.map} +1 -1
  26. package/dist/contract/index.d.ts +7 -7
  27. package/dist/contract/index.js +90 -10
  28. package/dist/contract/index.js.map +1 -1
  29. package/dist/{default-registry-ClYjq3QE.js → default-registry-BJ-Pq6yL.js} +309 -277
  30. package/dist/{default-registry-ClYjq3QE.js.map → default-registry-BJ-Pq6yL.js.map} +1 -1
  31. package/dist/{define-agent-eval-BDxNlP3D.js → define-agent-eval-BGdFKlrc.js} +165 -105
  32. package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
  33. package/dist/{define-agent-eval-BIkH_gAR.d.ts → define-agent-eval-BlH10zEw.d.ts} +6 -6
  34. package/dist/{define-agent-eval-BIkH_gAR.d.ts.map → define-agent-eval-BlH10zEw.d.ts.map} +1 -1
  35. package/dist/diagnosis/index.d.ts +1 -1
  36. package/dist/{dspy-rlm-engine-DhdCr1Dc.js → dspy-rlm-engine-CSMwudLO.js} +2 -2
  37. package/dist/{dspy-rlm-engine-DhdCr1Dc.js.map → dspy-rlm-engine-CSMwudLO.js.map} +1 -1
  38. package/dist/eval-campaign-BUMIxKTE.js +393 -0
  39. package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
  40. package/dist/{evaluation-jTjC_QQD.d.ts → evaluation-BOZ38vOH.d.ts} +2 -2
  41. package/dist/{evaluation-jTjC_QQD.d.ts.map → evaluation-BOZ38vOH.d.ts.map} +1 -1
  42. package/dist/evaluation.d.ts +1 -1
  43. package/dist/experiment/index.d.ts +1 -1
  44. package/dist/experiment/index.d.ts.map +1 -1
  45. package/dist/experiment/index.js +110 -71
  46. package/dist/experiment/index.js.map +1 -1
  47. package/dist/{exporters-Cl6IeNAu.js → exporters-DWvM16LE.js} +2 -2
  48. package/dist/{exporters-Cl6IeNAu.js.map → exporters-DWvM16LE.js.map} +1 -1
  49. package/dist/{external-optimizer-process-KVFGvYX1.js → external-optimizer-process-C3Wm7Vhn.js} +2 -2
  50. package/dist/{external-optimizer-process-KVFGvYX1.js.map → external-optimizer-process-C3Wm7Vhn.js.map} +1 -1
  51. package/dist/{external-optimizer-subprocess-XIHBOSKt.js → external-optimizer-subprocess-B5gdLRZv.js} +3 -3
  52. package/dist/{external-optimizer-subprocess-XIHBOSKt.js.map → external-optimizer-subprocess-B5gdLRZv.js.map} +1 -1
  53. package/dist/{findings-store-BMVQtEsm.js → findings-store-BtTzPL0d.js} +5 -5
  54. package/dist/{findings-store-BMVQtEsm.js.map → findings-store-BtTzPL0d.js.map} +1 -1
  55. package/dist/{hf-dataset-D9f7-oX7.js → hf-dataset-CKxIJQba.js} +3 -3
  56. package/dist/{hf-dataset-D9f7-oX7.js.map → hf-dataset-CKxIJQba.js.map} +1 -1
  57. package/dist/hosted/index.js +1 -1
  58. package/dist/{index-ZuWx52WG.d.ts → index-B16FNdW5.d.ts} +34 -17
  59. package/dist/index-B16FNdW5.d.ts.map +1 -0
  60. package/dist/{index-9jJ2fKSX.d.ts → index-CalAsXMR.d.ts} +7 -5
  61. package/dist/index-CalAsXMR.d.ts.map +1 -0
  62. package/dist/{index-BiT6RtNH.d.ts → index-DbMOXYnP.d.ts} +6 -6
  63. package/dist/{index-BiT6RtNH.d.ts.map → index-DbMOXYnP.d.ts.map} +1 -1
  64. package/dist/index.d.ts +9 -9
  65. package/dist/index.d.ts.map +1 -1
  66. package/dist/index.js +611 -688
  67. package/dist/index.js.map +1 -1
  68. package/dist/{integrity-DOd9Q86O.js → integrity-C8oUeB0g.js} +322 -197
  69. package/dist/integrity-C8oUeB0g.js.map +1 -0
  70. package/dist/{journal-CR4QikYp.js → journal-C7om8Gum.js} +48 -38
  71. package/dist/journal-C7om8Gum.js.map +1 -0
  72. package/dist/ledger-core/index.d.ts +1 -1
  73. package/dist/ledger-core/index.js +2 -2
  74. package/dist/ledger-core/lock-holder.d.ts +2 -0
  75. package/dist/ledger-core/lock-holder.js +181 -0
  76. package/dist/ledger-core/lock-holder.js.map +1 -0
  77. package/dist/{ledger-core-DrJhbms9.js → ledger-core-CAxUdDk0.js} +2 -2
  78. package/dist/{ledger-core-DrJhbms9.js.map → ledger-core-CAxUdDk0.js.map} +1 -1
  79. package/dist/{llm-client-BshCREn6.js → llm-client-_Y6ic-tf.js} +238 -248
  80. package/dist/llm-client-_Y6ic-tf.js.map +1 -0
  81. package/dist/{llm-judge-C8jMTNNG.js → llm-judge-Djxh-Uhs.js} +646 -458
  82. package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
  83. package/dist/lock-holder-DaJbfpci.d.ts +86 -0
  84. package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
  85. package/dist/{matrix-DHe-dwVJ.d.ts → matrix-CCSBna4q.d.ts} +2 -2
  86. package/dist/{matrix-DHe-dwVJ.d.ts.map → matrix-CCSBna4q.d.ts.map} +1 -1
  87. package/dist/{mint-zszyYlpA.js → mint-CbdwjTPW.js} +2 -2
  88. package/dist/{mint-zszyYlpA.js.map → mint-CbdwjTPW.js.map} +1 -1
  89. package/dist/multishot/golden/index.d.ts +1 -1
  90. package/dist/multishot/index.d.ts +2 -2
  91. package/dist/openapi.json +1 -1
  92. package/dist/{pareto-BzsxidFD.js → pareto-CTv8vozZ.js} +4 -4
  93. package/dist/{pareto-BzsxidFD.js.map → pareto-CTv8vozZ.js.map} +1 -1
  94. package/dist/{promotion-gate-DaY95yU6.d.ts → promotion-gate-CxEHRsZW.d.ts} +2 -2
  95. package/dist/{promotion-gate-DaY95yU6.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
  96. package/dist/{proposal-findings-BbWidvUk.js → proposal-findings-n-5WBbEx.js} +2 -2
  97. package/dist/{proposal-findings-BbWidvUk.js.map → proposal-findings-n-5WBbEx.js.map} +1 -1
  98. package/dist/{render-C-w2EZ6p.js → render-Cs34y0iX.js} +404 -208
  99. package/dist/render-Cs34y0iX.js.map +1 -0
  100. package/dist/reporting.d.ts +2 -2
  101. package/dist/{researcher-0ABVIdGa.d.ts → researcher-GU1tV4dn.d.ts} +2 -2
  102. package/dist/{researcher-0ABVIdGa.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
  103. package/dist/rl.d.ts +2 -2
  104. package/dist/rl.js +4 -4
  105. package/dist/rollout/index.js +5 -5
  106. package/dist/{rollout-D6C50mfA.js → rollout-Dd1ucT_M.js} +5 -5
  107. package/dist/{rollout-D6C50mfA.js.map → rollout-Dd1ucT_M.js.map} +1 -1
  108. package/dist/schema-C-ziyJD7.d.ts.map +1 -1
  109. package/dist/{schema-DxcKTzbi.js → schema-CvhfLjtZ.js} +77 -47
  110. package/dist/schema-CvhfLjtZ.js.map +1 -0
  111. package/dist/search/index.d.ts +2 -2
  112. package/dist/search/index.d.ts.map +1 -1
  113. package/dist/search/index.js +2 -2
  114. package/dist/{search-kernel-DxqjBxA-.d.ts → search-kernel-p9JB3SmL.d.ts} +3 -3
  115. package/dist/{search-kernel-DxqjBxA-.d.ts.map → search-kernel-p9JB3SmL.d.ts.map} +1 -1
  116. package/dist/{search-ledger-TUvB5gWH.js → search-ledger-ClpN4I3e.js} +4 -4
  117. package/dist/{search-ledger-TUvB5gWH.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
  118. package/dist/{search-ledger-file-UKCO2r8a.js → search-ledger-file-Bd5Tioq6.js} +3 -3
  119. package/dist/{search-ledger-file-UKCO2r8a.js.map → search-ledger-file-Bd5Tioq6.js.map} +1 -1
  120. package/dist/{search-shipper-Cw2z3-td.js → search-shipper-Bg2HW3Uv.js} +3 -3
  121. package/dist/{search-shipper-Cw2z3-td.js.map → search-shipper-Bg2HW3Uv.js.map} +1 -1
  122. package/dist/{search-summary-DQIvJsSB.js → search-summary-CrBuvVnf.js} +3 -3
  123. package/dist/{search-summary-DQIvJsSB.js.map → search-summary-CrBuvVnf.js.map} +1 -1
  124. package/dist/{server-DoVbqqtJ.js → server-C6M0qR8h.js} +2 -2
  125. package/dist/{server-DoVbqqtJ.js.map → server-C6M0qR8h.js.map} +1 -1
  126. package/dist/{single-run-lock-U58yf8N6.js → single-run-lock-Dekd8_Wv.js} +7 -7
  127. package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
  128. package/dist/{skillopt-optimization-method-kyL94-ri.js → skillopt-optimization-method-5dlex7MJ.js} +1339 -1059
  129. package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
  130. package/dist/{statistical-heldout-C9AbgREN.d.ts → statistical-heldout-BHfLZHrm.d.ts} +2 -2
  131. package/dist/{statistical-heldout-C9AbgREN.d.ts.map → statistical-heldout-BHfLZHrm.d.ts.map} +1 -1
  132. package/dist/{summary-report-DM7tlml1.d.ts → summary-report-CkbRcSuu.d.ts} +16 -1
  133. package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
  134. package/dist/supervisor-run/index.d.ts.map +1 -1
  135. package/dist/supervisor-run/index.js +3 -3
  136. package/dist/{systemone-Bp_Ac0p1.d.ts → systemone-BMzDJb1i.d.ts} +3 -3
  137. package/dist/{systemone-Bp_Ac0p1.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
  138. package/dist/systemone.d.ts +2 -2
  139. package/dist/{terminal-record-78g6E4vh.js → terminal-record-D4Rv-j1W.js} +241 -193
  140. package/dist/{terminal-record-78g6E4vh.js.map → terminal-record-D4Rv-j1W.js.map} +1 -1
  141. package/dist/trajectory-replay/index.d.ts +4 -1
  142. package/dist/trajectory-replay/index.d.ts.map +1 -1
  143. package/dist/trajectory-replay/index.js +834 -824
  144. package/dist/trajectory-replay/index.js.map +1 -1
  145. package/dist/{tree-BGRglkKd.js → tree-B5e1uAeL.js} +122 -14
  146. package/dist/tree-B5e1uAeL.js.map +1 -0
  147. package/dist/{types-DtMbFw6U.d.ts → types-BKCzzmit.d.ts} +2 -2
  148. package/dist/{types-DtMbFw6U.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
  149. package/dist/{types-D8GhJoHk.d.ts → types-DtkO1G9L.d.ts} +2 -2
  150. package/dist/{types-D8GhJoHk.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
  151. package/dist/wire/index.js +1 -1
  152. package/docs/design/statistics-decisions.md +1 -1
  153. package/docs/eval-surface-map.md +1 -0
  154. package/docs/profile-json-identity.md +1 -2
  155. package/docs/public-api.md +27 -16
  156. package/package.json +7 -2
  157. package/dist/benchmark-command-xQK8nOI6.js.map +0 -1
  158. package/dist/define-agent-eval-BDxNlP3D.js.map +0 -1
  159. package/dist/eval-campaign-BLEfBGbA.js +0 -343
  160. package/dist/eval-campaign-BLEfBGbA.js.map +0 -1
  161. package/dist/index-9jJ2fKSX.d.ts.map +0 -1
  162. package/dist/index-ZuWx52WG.d.ts.map +0 -1
  163. package/dist/integrity-DOd9Q86O.js.map +0 -1
  164. package/dist/journal-CR4QikYp.js.map +0 -1
  165. package/dist/llm-client-BshCREn6.js.map +0 -1
  166. package/dist/llm-judge-C8jMTNNG.js.map +0 -1
  167. package/dist/render-C-w2EZ6p.js.map +0 -1
  168. package/dist/schema-DxcKTzbi.js.map +0 -1
  169. package/dist/single-run-lock-U58yf8N6.js.map +0 -1
  170. package/dist/skillopt-optimization-method-kyL94-ri.js.map +0 -1
  171. package/dist/summary-report-DM7tlml1.d.ts.map +0 -1
  172. package/dist/tree-BGRglkKd.js.map +0 -1
@@ -3,17 +3,17 @@ import { t as mean$1 } from "./mean-BbA_JCzW.js";
3
3
  import { t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
4
4
  import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
5
5
  import { n as contentHash, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
6
- import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-C8jMTNNG.js";
7
- import { D as surfaceContentHash, I as assertCompleteCampaign, P as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-CQYw8_0H.js";
8
- import { a as mapConcurrent } from "./journal-CR4QikYp.js";
6
+ import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-Djxh-Uhs.js";
7
+ import { D as surfaceContentHash, I as assertCompleteCampaign, P as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-DunzcJGv.js";
8
+ import { a as mapConcurrent } from "./journal-C7om8Gum.js";
9
9
  import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
10
- import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-XIHBOSKt.js";
10
+ import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-B5gdLRZv.js";
11
11
  import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
12
- import { n as openSearchLedger } from "./search-ledger-TUvB5gWH.js";
13
- import { t as acquireSingleRunLock } from "./single-run-lock-U58yf8N6.js";
12
+ import { n as openSearchLedger } from "./search-ledger-ClpN4I3e.js";
13
+ import { t as acquireSingleRunLock } from "./single-run-lock-Dekd8_Wv.js";
14
14
  import { n as isRecord } from "./json-BhdsJaOQ.js";
15
- import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-BshCREn6.js";
16
- import { t as startExternalOptimizerCallback } from "./external-optimizer-process-KVFGvYX1.js";
15
+ import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-_Y6ic-tf.js";
16
+ import { t as startExternalOptimizerCallback } from "./external-optimizer-process-C3Wm7Vhn.js";
17
17
  import { createHash, randomBytes } from "node:crypto";
18
18
  import { mkdir } from "node:fs/promises";
19
19
  import { isDeepStrictEqual } from "node:util";
@@ -567,270 +567,341 @@ function externalTextOptimizationMethod(config) {
567
567
  const snapshot = snapshotExternalTextOptimizationConfig(config);
568
568
  return {
569
569
  name: snapshot.name,
570
- async optimize(input) {
571
- const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
572
- const expectsComponents = typeof seedCandidate !== "string";
573
- const maxCandidateChars = snapshot.maxCandidateChars ?? DEFAULT_MAX_CANDIDATE_CHARS;
574
- const maxEvidenceChars = snapshot.maxEvidenceChars ?? DEFAULT_MAX_EVIDENCE_CHARS;
575
- const attemptId = randomBytes(16).toString("hex");
576
- const storage = input.runOptions.storage ?? fsCampaignStorage();
577
- const resume = snapshot.resume ?? "never";
578
- const trainSet = Object.freeze(input.trainScenarios.map((scenario) => describeExternalScenario(scenario, snapshot.name, maxEvidenceChars, snapshot.describeScenario)));
579
- const selectionSet = Object.freeze(input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, snapshot.name, maxEvidenceChars, snapshot.describeScenario)));
580
- const methodDir = `${input.runDir}/external/${safePathComponent(snapshot.name)}`;
581
- const runId = externalOptimizerRunKey({
582
- material: {
583
- optimizer: {
584
- kind: snapshot.source.kind,
585
- package: snapshot.source.package,
586
- version: snapshot.source.version,
587
- ...snapshot.source.sourceUrl ? { sourceUrl: snapshot.source.sourceUrl } : {},
588
- ...snapshot.source.revision ? { revision: snapshot.source.revision } : {}
589
- },
590
- method: snapshot.name,
591
- evaluationId: snapshot.evaluationId,
592
- dispatchRef: input.runOptions.dispatchRef ?? null,
593
- objective: snapshot.objective,
594
- background: snapshot.background ?? "",
595
- seed: input.seed,
596
- seedCandidate,
597
- trainSet,
598
- selectionSet,
599
- maxEvaluations: snapshot.maxEvaluations,
600
- maxOptimizerCostUsd: snapshot.maxOptimizerCostUsd,
601
- maxCandidateChars,
602
- maxEvidenceChars
603
- },
604
- attemptId,
605
- resumeEnabled: resume !== "never"
606
- });
607
- const stateDir = `${methodDir}/state/${runId}`;
608
- const artifactDir = `${methodDir}/attempts/${attemptId}`;
609
- const manifestPath = `${stateDir}/run-manifest.jsonl`;
610
- const evaluationCostPhase = "external.evaluation";
611
- const optimizerCostPhase = "external.optimizer";
612
- storage.ensureDir(stateDir);
613
- storage.ensureDir(artifactDir);
614
- if (resume !== "never") await mkdir(stateDir, { recursive: true });
615
- const lock = resume === "never" ? void 0 : acquireSingleRunLock({
616
- lockPath: `${stateDir}.lock`,
617
- releaseOnExit: true
618
- });
619
- let releaseLock = true;
620
- try {
621
- let manifest = resume === "never" ? void 0 : readExternalOptimizerRunManifest(storage.read(manifestPath), storage.exists(manifestPath), manifestPath, runId);
622
- const restoreRequested = manifest !== void 0;
623
- if (resume === "required" && !restoreRequested) throw new Error(`${snapshot.name}: no compatible run state is available to resume`);
624
- if (resume === "never" && restoreRequested) throw new Error(`${snapshot.name}: fresh optimizer run unexpectedly found prior state`);
625
- if (resume !== "never") manifest = appendExternalOptimizerRunManifestEvent({
626
- storage,
627
- path: manifestPath,
628
- revision: manifest?.revision ?? 0,
629
- event: {
630
- runId,
631
- attemptId,
632
- status: "partial"
633
- }
634
- });
635
- const runBudget = openExternalOptimizerRunBudget({
636
- storage,
637
- runDir: methodDir,
638
- runKey: runId,
639
- attemptId,
640
- maxEvaluations: snapshot.maxEvaluations
641
- });
642
- const optimizerLedger = createRunCostLedger({
643
- storage,
644
- runDir: `${stateDir}/cost`,
645
- costCeilingUsd: snapshot.maxOptimizerCostUsd
646
- });
647
- const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, `${snapshot.name} optimizer`);
648
- const score = createExternalTextEvaluator({
649
- input,
650
- label: `${snapshot.name} optimizer`,
651
- runDir: artifactDir,
652
- compatibleRunId: runId,
653
- costPhase: evaluationCostPhase,
654
- costTags: runBudget.attemptTags,
655
- costLedger: input.costLedger,
656
- scenarioById,
657
- maxCandidateChars,
658
- maxEvidenceChars,
659
- describeArtifact: snapshot.describeArtifact
660
- });
661
- let acceptingEvaluations = true;
662
- const activeEvaluations = /* @__PURE__ */ new Set();
663
- const controller = new AbortController();
664
- const optimizerCostMeter = { async runPaidCall(request) {
665
- const call = {
666
- methodResult: void 0,
667
- providerStarted: false
668
- };
669
- const subLimitResult = await optimizerLedger.runPaidCall({
670
- ...request,
671
- channel: request.channel ?? "optimizer",
672
- phase: optimizerCostPhase,
673
- tags: runBudget.attemptTags,
674
- signal: request.signal ? AbortSignal.any([controller.signal, request.signal]) : controller.signal,
675
- execute: async (signal, callId) => {
676
- call.methodResult = await input.costLedger.runPaidCall({
677
- ...request,
678
- callId,
679
- channel: request.channel ?? "optimizer",
680
- phase: optimizerCostPhase,
681
- tags: runBudget.attemptTags,
682
- signal,
683
- execute: (methodSignal, methodCallId) => {
684
- call.providerStarted = true;
685
- return request.execute(methodSignal, methodCallId);
686
- }
687
- });
688
- if (!call.methodResult.succeeded) throw new Error(`${snapshot.name}: method cost account rejected optimizer call`, { cause: call.methodResult.error });
689
- return call.methodResult.value;
690
- },
691
- receipt: () => {
692
- if (!call.methodResult?.succeeded) throw new Error(`${snapshot.name}: optimizer call completed without a receipt`);
693
- return mirroredCostReceipt(call.methodResult.receipt);
694
- },
695
- receiptFromError: () => {
696
- if (call.methodResult?.receipt) return mirroredCostReceipt(call.methodResult.receipt);
697
- if (!call.providerStarted) return noChargeReceipt(modelBeforeExecution(request));
698
- }
699
- });
700
- if (call.methodResult === void 0) return subLimitResult;
701
- if (!call.methodResult.succeeded) return call.methodResult;
702
- return subLimitResult.succeeded ? call.methodResult : subLimitResult;
703
- } };
704
- Object.freeze(optimizerCostMeter);
705
- const evaluate = (request, signal = controller.signal) => {
706
- if (!acceptingEvaluations) throw new Error(`${snapshot.name}: evaluate cannot be called after run() completes`);
707
- if (runBudget.acceptEvaluation() === void 0) throw new Error(`${snapshot.name}: maxEvaluations limit reached`);
708
- const combinedSignal = signal === controller.signal ? controller.signal : AbortSignal.any([controller.signal, signal]);
709
- const pending = score(cloneExternalEvaluationRequest(request), combinedSignal);
710
- activeEvaluations.add(pending);
711
- const remove = () => activeEvaluations.delete(pending);
712
- pending.then(remove, remove);
713
- return pending;
714
- };
715
- const started = Date.now();
716
- const timeoutMs = snapshot.timeoutMs ?? DEFAULT_TIMEOUT_MS;
717
- const timeout = setTimeout(() => controller.abort(/* @__PURE__ */ new Error(`${snapshot.name}: optimizer exceeded ${timeoutMs}ms`)), timeoutMs);
718
- let result;
719
- let runError;
720
- let outstandingAtCompletion = 0;
721
- let paidCallsAtCompletion = 0;
722
- let runSettled = false;
723
- const runPromise = Promise.resolve().then(() => snapshot.run(Object.freeze({
724
- runId,
725
- name: snapshot.name,
726
- objective: snapshot.objective,
727
- evaluationId: snapshot.evaluationId,
728
- ...snapshot.background ? { background: snapshot.background } : {},
729
- seedCandidate: cloneExternalTextCandidate(seedCandidate),
730
- trainSet,
731
- selectionSet,
732
- maxEvaluations: snapshot.maxEvaluations,
733
- seed: input.seed,
734
- stateDir,
735
- restoreRequested,
736
- artifactDir,
737
- signal: controller.signal,
738
- cost: optimizerCostMeter,
739
- evaluate
740
- })));
741
- runPromise.then(() => {
742
- runSettled = true;
743
- }, () => {
744
- runSettled = true;
745
- });
746
- try {
747
- result = await raceWithAbort(runPromise, controller.signal);
748
- outstandingAtCompletion = activeEvaluations.size;
749
- paidCallsAtCompletion = optimizerLedger.summary().pendingCalls;
750
- } catch (error) {
751
- runError = error;
752
- } finally {
753
- acceptingEvaluations = false;
754
- clearTimeout(timeout);
755
- }
756
- await Promise.allSettled([...activeEvaluations]);
757
- const paidCallsSettled = await optimizerLedger.waitForIdle({ timeoutMs: 5e3 });
758
- if (runError !== void 0) {
759
- if ((!runSettled || !paidCallsSettled) && lock) {
760
- releaseLock = false;
761
- Promise.allSettled([runPromise, optimizerLedger.waitForIdle({ timeoutMs: MAX_TIMER_DELAY_MS$2 })]).then(() => lock.release());
762
- }
763
- throw runError;
764
- }
765
- if (!paidCallsSettled) {
766
- if (lock) {
767
- releaseLock = false;
768
- Promise.allSettled([runPromise, optimizerLedger.waitForIdle({ timeoutMs: MAX_TIMER_DELAY_MS$2 })]).then(() => lock.release());
769
- }
770
- throw new Error(`${snapshot.name}: optimizer paid calls did not settle`);
771
- }
772
- if (paidCallsAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${paidCallsAtCompletion} outstanding paid call(s); await every context.cost.runPaidCall()`);
773
- if (outstandingAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${outstandingAtCompletion} outstanding evaluation(s); await every evaluate() call`);
774
- assertExternalTextOptimizerResult(result, snapshot.name, maxCandidateChars, expectsComponents);
775
- if (result.resumed !== restoreRequested) throw new Error(`${snapshot.name}: optimizer reported resumed=${String(result.resumed)} but restoreRequested=${String(restoreRequested)}`);
776
- const evaluationCost = costFromLedgerSummary(input.costLedger.summary({
777
- phase: evaluationCostPhase,
778
- tags: runBudget.runTags
779
- }));
780
- const optimizerFilter = {
781
- phase: optimizerCostPhase,
782
- tags: runBudget.runTags
783
- };
784
- const optimizerReceipts = input.costLedger.list(optimizerFilter);
785
- const optimizerSummary = input.costLedger.summary(optimizerFilter);
786
- const optimizerCost = costFromLedgerSummary(optimizerSummary);
787
- assertExternalCostAccounting(result.costAccounting, optimizerSummary.totalCalls, snapshot.name);
788
- const externalCostReason = result.costAccounting.kind === "external" ? result.costAccounting.reason : void 0;
789
- const tokenUsage = optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts);
790
- if (resume !== "never") appendExternalOptimizerRunManifestEvent({
791
- storage,
792
- path: manifestPath,
793
- revision: manifest.revision,
794
- event: {
795
- runId,
796
- attemptId,
797
- status: "completed"
570
+ optimize: (input) => optimizeExternalText(snapshot, input)
571
+ };
572
+ }
573
+ const EVALUATION_COST_PHASE = "external.evaluation";
574
+ const OPTIMIZER_COST_PHASE = "external.optimizer";
575
+ async function optimizeExternalText(snapshot, input) {
576
+ const run = await prepareExternalRun(snapshot, input);
577
+ try {
578
+ const { manifest, restoreRequested } = restoreRunManifest(run);
579
+ const accounts = openRunAccounts(run);
580
+ const session = startOptimizerSession(run, accounts, restoreRequested);
581
+ return finishRun(run, accounts, session, manifest, await awaitOptimizerResult(run, accounts, session, restoreRequested));
582
+ } finally {
583
+ if (run.releaseLock) run.lock?.release();
584
+ }
585
+ }
586
+ function describeExternalScenarios(scenarios, snapshot, maxEvidenceChars) {
587
+ return Object.freeze(scenarios.map((scenario) => describeExternalScenario(scenario, snapshot.name, maxEvidenceChars, snapshot.describeScenario)));
588
+ }
589
+ /** Run identity, scenario descriptions and directories, then the resume lock. */
590
+ async function prepareExternalRun(snapshot, input) {
591
+ const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
592
+ const maxCandidateChars = snapshot.maxCandidateChars ?? DEFAULT_MAX_CANDIDATE_CHARS;
593
+ const maxEvidenceChars = snapshot.maxEvidenceChars ?? DEFAULT_MAX_EVIDENCE_CHARS;
594
+ const attemptId = randomBytes(16).toString("hex");
595
+ const storage = input.runOptions.storage ?? fsCampaignStorage();
596
+ const resume = snapshot.resume ?? "never";
597
+ const trainSet = describeExternalScenarios(input.trainScenarios, snapshot, maxEvidenceChars);
598
+ const selectionSet = describeExternalScenarios(input.selectionScenarios, snapshot, maxEvidenceChars);
599
+ const methodDir = `${input.runDir}/external/${safePathComponent(snapshot.name)}`;
600
+ const runId = externalOptimizerRunKey({
601
+ material: {
602
+ optimizer: {
603
+ kind: snapshot.source.kind,
604
+ package: snapshot.source.package,
605
+ version: snapshot.source.version,
606
+ ...snapshot.source.sourceUrl ? { sourceUrl: snapshot.source.sourceUrl } : {},
607
+ ...snapshot.source.revision ? { revision: snapshot.source.revision } : {}
608
+ },
609
+ method: snapshot.name,
610
+ evaluationId: snapshot.evaluationId,
611
+ dispatchRef: input.runOptions.dispatchRef ?? null,
612
+ objective: snapshot.objective,
613
+ background: snapshot.background ?? "",
614
+ seed: input.seed,
615
+ seedCandidate,
616
+ trainSet,
617
+ selectionSet,
618
+ maxEvaluations: snapshot.maxEvaluations,
619
+ maxOptimizerCostUsd: snapshot.maxOptimizerCostUsd,
620
+ maxCandidateChars,
621
+ maxEvidenceChars
622
+ },
623
+ attemptId,
624
+ resumeEnabled: resume !== "never"
625
+ });
626
+ const stateDir = `${methodDir}/state/${runId}`;
627
+ const artifactDir = `${methodDir}/attempts/${attemptId}`;
628
+ storage.ensureDir(stateDir);
629
+ storage.ensureDir(artifactDir);
630
+ if (resume !== "never") await mkdir(stateDir, { recursive: true });
631
+ const lock = resume === "never" ? void 0 : acquireSingleRunLock({
632
+ lockPath: `${stateDir}.lock`,
633
+ releaseOnExit: true
634
+ });
635
+ return {
636
+ snapshot,
637
+ input,
638
+ seedCandidate,
639
+ expectsComponents: typeof seedCandidate !== "string",
640
+ maxCandidateChars,
641
+ maxEvidenceChars,
642
+ attemptId,
643
+ storage,
644
+ resume,
645
+ trainSet,
646
+ selectionSet,
647
+ methodDir,
648
+ runId,
649
+ stateDir,
650
+ artifactDir,
651
+ manifestPath: `${stateDir}/run-manifest.jsonl`,
652
+ lock,
653
+ releaseLock: true
654
+ };
655
+ }
656
+ /** Reads prior state when resuming, and records this attempt as partial. */
657
+ function restoreRunManifest(run) {
658
+ const { snapshot, storage, manifestPath, runId, attemptId, resume } = run;
659
+ let manifest = resume === "never" ? void 0 : readExternalOptimizerRunManifest(storage.read(manifestPath), storage.exists(manifestPath), manifestPath, runId);
660
+ const restoreRequested = manifest !== void 0;
661
+ if (resume === "required" && !restoreRequested) throw new Error(`${snapshot.name}: no compatible run state is available to resume`);
662
+ if (resume === "never" && restoreRequested) throw new Error(`${snapshot.name}: fresh optimizer run unexpectedly found prior state`);
663
+ if (resume !== "never") manifest = appendExternalOptimizerRunManifestEvent({
664
+ storage,
665
+ path: manifestPath,
666
+ revision: manifest?.revision ?? 0,
667
+ event: {
668
+ runId,
669
+ attemptId,
670
+ status: "partial"
671
+ }
672
+ });
673
+ return {
674
+ manifest,
675
+ restoreRequested
676
+ };
677
+ }
678
+ function openRunAccounts(run) {
679
+ const { snapshot, storage, methodDir, runId, attemptId, stateDir } = run;
680
+ return {
681
+ runBudget: openExternalOptimizerRunBudget({
682
+ storage,
683
+ runDir: methodDir,
684
+ runKey: runId,
685
+ attemptId,
686
+ maxEvaluations: snapshot.maxEvaluations
687
+ }),
688
+ optimizerLedger: createRunCostLedger({
689
+ storage,
690
+ runDir: `${stateDir}/cost`,
691
+ costCeilingUsd: snapshot.maxOptimizerCostUsd
692
+ })
693
+ };
694
+ }
695
+ /** Optimizer paid calls, charged to the optimizer's own limit and mirrored into the method's account. */
696
+ function createOptimizerCostMeter(run, accounts, controller) {
697
+ const { snapshot, input } = run;
698
+ const { runBudget, optimizerLedger } = accounts;
699
+ return Object.freeze({ async runPaidCall(request) {
700
+ const call = {
701
+ methodResult: void 0,
702
+ providerStarted: false
703
+ };
704
+ const subLimitResult = await optimizerLedger.runPaidCall({
705
+ ...request,
706
+ channel: request.channel ?? "optimizer",
707
+ phase: OPTIMIZER_COST_PHASE,
708
+ tags: runBudget.attemptTags,
709
+ signal: request.signal ? AbortSignal.any([controller.signal, request.signal]) : controller.signal,
710
+ execute: async (signal, callId) => {
711
+ call.methodResult = await input.costLedger.runPaidCall({
712
+ ...request,
713
+ callId,
714
+ channel: request.channel ?? "optimizer",
715
+ phase: OPTIMIZER_COST_PHASE,
716
+ tags: runBudget.attemptTags,
717
+ signal,
718
+ execute: (methodSignal, methodCallId) => {
719
+ call.providerStarted = true;
720
+ return request.execute(methodSignal, methodCallId);
798
721
  }
799
722
  });
800
- const measuredCost = combineComparisonCosts([{
801
- label: "evaluation",
802
- cost: evaluationCost
803
- }, {
804
- label: "optimizer",
805
- cost: optimizerCost
806
- }]);
807
- return {
808
- winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
809
- cost: {
810
- ...measuredCost,
811
- ...externalCostReason ? { costProvenance: {
812
- kind: "uncaptured",
813
- usd: null
814
- } } : {},
815
- accountingComplete: measuredCost.accountingComplete && externalCostReason === void 0,
816
- incompleteReasons: [...measuredCost.incompleteReasons, ...externalCostReason ? [`optimizer: external spend is not observed: ${externalCostReason}`] : []]
817
- },
818
- durationMs: Date.now() - started,
819
- provenance: {
820
- source: {
821
- ...snapshot.source,
822
- evidence: "declared"
823
- },
824
- runId,
825
- resumed: result.resumed,
826
- evaluationCount: runBudget.acceptedEvaluations(),
827
- artifactDir,
828
- ...tokenUsage ? { tokenUsage } : {}
829
- }
830
- };
831
- } finally {
832
- if (releaseLock) lock?.release();
723
+ if (!call.methodResult.succeeded) throw new Error(`${snapshot.name}: method cost account rejected optimizer call`, { cause: call.methodResult.error });
724
+ return call.methodResult.value;
725
+ },
726
+ receipt: () => {
727
+ if (!call.methodResult?.succeeded) throw new Error(`${snapshot.name}: optimizer call completed without a receipt`);
728
+ return mirroredCostReceipt(call.methodResult.receipt);
729
+ },
730
+ receiptFromError: () => {
731
+ if (call.methodResult?.receipt) return mirroredCostReceipt(call.methodResult.receipt);
732
+ if (!call.providerStarted) return noChargeReceipt(modelBeforeExecution(request));
833
733
  }
734
+ });
735
+ if (call.methodResult === void 0) return subLimitResult;
736
+ if (!call.methodResult.succeeded) return call.methodResult;
737
+ return subLimitResult.succeeded ? call.methodResult : subLimitResult;
738
+ } });
739
+ }
740
+ /** Starts the optimizer with its evaluator, cost meter, timeout and frozen context. */
741
+ function startOptimizerSession(run, accounts, restoreRequested) {
742
+ const { snapshot, input, runId, stateDir, artifactDir } = run;
743
+ const { runBudget } = accounts;
744
+ const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, `${snapshot.name} optimizer`);
745
+ const score = createExternalTextEvaluator({
746
+ input,
747
+ label: `${snapshot.name} optimizer`,
748
+ runDir: artifactDir,
749
+ compatibleRunId: runId,
750
+ costPhase: EVALUATION_COST_PHASE,
751
+ costTags: runBudget.attemptTags,
752
+ costLedger: input.costLedger,
753
+ scenarioById,
754
+ maxCandidateChars: run.maxCandidateChars,
755
+ maxEvidenceChars: run.maxEvidenceChars,
756
+ describeArtifact: snapshot.describeArtifact
757
+ });
758
+ const controller = new AbortController();
759
+ const activeEvaluations = /* @__PURE__ */ new Set();
760
+ const state = {
761
+ acceptingEvaluations: true,
762
+ runSettled: false
763
+ };
764
+ const optimizerCostMeter = createOptimizerCostMeter(run, accounts, controller);
765
+ const evaluate = (request, signal = controller.signal) => {
766
+ if (!state.acceptingEvaluations) throw new Error(`${snapshot.name}: evaluate cannot be called after run() completes`);
767
+ if (runBudget.acceptEvaluation() === void 0) throw new Error(`${snapshot.name}: maxEvaluations limit reached`);
768
+ const combinedSignal = signal === controller.signal ? controller.signal : AbortSignal.any([controller.signal, signal]);
769
+ const pending = score(cloneExternalEvaluationRequest(request), combinedSignal);
770
+ activeEvaluations.add(pending);
771
+ const remove = () => activeEvaluations.delete(pending);
772
+ pending.then(remove, remove);
773
+ return pending;
774
+ };
775
+ const started = Date.now();
776
+ const timeoutMs = snapshot.timeoutMs ?? DEFAULT_TIMEOUT_MS;
777
+ const timeout = setTimeout(() => controller.abort(/* @__PURE__ */ new Error(`${snapshot.name}: optimizer exceeded ${timeoutMs}ms`)), timeoutMs);
778
+ const runPromise = Promise.resolve().then(() => snapshot.run(Object.freeze({
779
+ runId,
780
+ name: snapshot.name,
781
+ objective: snapshot.objective,
782
+ evaluationId: snapshot.evaluationId,
783
+ ...snapshot.background ? { background: snapshot.background } : {},
784
+ seedCandidate: cloneExternalTextCandidate(run.seedCandidate),
785
+ trainSet: run.trainSet,
786
+ selectionSet: run.selectionSet,
787
+ maxEvaluations: snapshot.maxEvaluations,
788
+ seed: input.seed,
789
+ stateDir,
790
+ restoreRequested,
791
+ artifactDir,
792
+ signal: controller.signal,
793
+ cost: optimizerCostMeter,
794
+ evaluate
795
+ })));
796
+ const markSettled = () => {
797
+ state.runSettled = true;
798
+ };
799
+ runPromise.then(markSettled, markSettled);
800
+ return {
801
+ controller,
802
+ activeEvaluations,
803
+ runPromise,
804
+ timeout,
805
+ started,
806
+ state
807
+ };
808
+ }
809
+ /** Waits for the optimizer, then refuses a run that left evaluations or paid calls outstanding. */
810
+ async function awaitOptimizerResult(run, accounts, session, restoreRequested) {
811
+ const { snapshot, lock } = run;
812
+ const { optimizerLedger } = accounts;
813
+ let result;
814
+ let runError;
815
+ let outstandingAtCompletion = 0;
816
+ let paidCallsAtCompletion = 0;
817
+ try {
818
+ result = await raceWithAbort(session.runPromise, session.controller.signal);
819
+ outstandingAtCompletion = session.activeEvaluations.size;
820
+ paidCallsAtCompletion = optimizerLedger.summary().pendingCalls;
821
+ } catch (error) {
822
+ runError = error;
823
+ } finally {
824
+ session.state.acceptingEvaluations = false;
825
+ clearTimeout(session.timeout);
826
+ }
827
+ await Promise.allSettled([...session.activeEvaluations]);
828
+ const paidCallsSettled = await optimizerLedger.waitForIdle({ timeoutMs: 5e3 });
829
+ const handOffLock = () => {
830
+ if (!lock) return;
831
+ run.releaseLock = false;
832
+ Promise.allSettled([session.runPromise, optimizerLedger.waitForIdle({ timeoutMs: MAX_TIMER_DELAY_MS$2 })]).then(() => lock.release());
833
+ };
834
+ if (runError !== void 0) {
835
+ if (!session.state.runSettled || !paidCallsSettled) handOffLock();
836
+ throw runError;
837
+ }
838
+ if (!paidCallsSettled) {
839
+ handOffLock();
840
+ throw new Error(`${snapshot.name}: optimizer paid calls did not settle`);
841
+ }
842
+ if (paidCallsAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${paidCallsAtCompletion} outstanding paid call(s); await every context.cost.runPaidCall()`);
843
+ if (outstandingAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${outstandingAtCompletion} outstanding evaluation(s); await every evaluate() call`);
844
+ assertExternalTextOptimizerResult(result, snapshot.name, run.maxCandidateChars, run.expectsComponents);
845
+ if (result.resumed !== restoreRequested) throw new Error(`${snapshot.name}: optimizer reported resumed=${String(result.resumed)} but restoreRequested=${String(restoreRequested)}`);
846
+ return result;
847
+ }
848
+ /** Measures evaluation and optimizer spend, records completion, and builds the method result. */
849
+ function finishRun(run, accounts, session, manifest, result) {
850
+ const { snapshot, input, storage, runId, attemptId } = run;
851
+ const { runBudget } = accounts;
852
+ const evaluationCost = costFromLedgerSummary(input.costLedger.summary({
853
+ phase: EVALUATION_COST_PHASE,
854
+ tags: runBudget.runTags
855
+ }));
856
+ const optimizerFilter = {
857
+ phase: OPTIMIZER_COST_PHASE,
858
+ tags: runBudget.runTags
859
+ };
860
+ const optimizerReceipts = input.costLedger.list(optimizerFilter);
861
+ const optimizerSummary = input.costLedger.summary(optimizerFilter);
862
+ const optimizerCost = costFromLedgerSummary(optimizerSummary);
863
+ assertExternalCostAccounting(result.costAccounting, optimizerSummary.totalCalls, snapshot.name);
864
+ const externalCostReason = result.costAccounting.kind === "external" ? result.costAccounting.reason : void 0;
865
+ const tokenUsage = optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts);
866
+ if (run.resume !== "never") appendExternalOptimizerRunManifestEvent({
867
+ storage,
868
+ path: run.manifestPath,
869
+ revision: manifest.revision,
870
+ event: {
871
+ runId,
872
+ attemptId,
873
+ status: "completed"
874
+ }
875
+ });
876
+ const measuredCost = combineComparisonCosts([{
877
+ label: "evaluation",
878
+ cost: evaluationCost
879
+ }, {
880
+ label: "optimizer",
881
+ cost: optimizerCost
882
+ }]);
883
+ return {
884
+ winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
885
+ cost: {
886
+ ...measuredCost,
887
+ ...externalCostReason ? { costProvenance: {
888
+ kind: "uncaptured",
889
+ usd: null
890
+ } } : {},
891
+ accountingComplete: measuredCost.accountingComplete && externalCostReason === void 0,
892
+ incompleteReasons: [...measuredCost.incompleteReasons, ...externalCostReason ? [`optimizer: external spend is not observed: ${externalCostReason}`] : []]
893
+ },
894
+ durationMs: Date.now() - session.started,
895
+ provenance: {
896
+ source: {
897
+ ...snapshot.source,
898
+ evidence: "declared"
899
+ },
900
+ runId,
901
+ resumed: result.resumed,
902
+ evaluationCount: runBudget.acceptedEvaluations(),
903
+ artifactDir: run.artifactDir,
904
+ ...tokenUsage ? { tokenUsage } : {}
834
905
  }
835
906
  };
836
907
  }
@@ -1000,30 +1071,6 @@ function heldOutGate(options) {
1000
1071
  };
1001
1072
  }
1002
1073
  //#endregion
1003
- //#region src/campaign/external-optimizer-accounting.ts
1004
- const TOKEN_USAGE_FIELDS = [
1005
- "inputTokens",
1006
- "outputTokens",
1007
- "totalTokens",
1008
- "calls"
1009
- ];
1010
- function assertPriorExternalOptimizerUsage(summary, budget, name) {
1011
- if (!summary.usageComplete || budget.maxCostUsd !== void 0 && !summary.accountingComplete) throw new Error(`${name}: cannot resume optimizer-model work with incomplete bounded usage`);
1012
- if (summary.totalCalls > budget.maxRequests || budget.maxCostUsd !== void 0 && summary.totalCostUsd > budget.maxCostUsd + Number.EPSILON) throw new Error(`${name}: prior optimizer-model usage exceeds the configured budget`);
1013
- }
1014
- function assertExternalOptimizerTokenUsage(usage, name, optimizer) {
1015
- if (usage === void 0) return;
1016
- for (const field of TOKEN_USAGE_FIELDS) if (!Number.isSafeInteger(usage[field]) || usage[field] < 0) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.${field}`);
1017
- if (usage.totalTokens !== usage.inputTokens + usage.outputTokens) throw new Error(`${name}: ${optimizer} bridge returned inconsistent tokenUsage.totalTokens`);
1018
- if (usage.requestAttempts !== void 0 && (!Number.isSafeInteger(usage.requestAttempts) || usage.requestAttempts < usage.calls)) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.requestAttempts`);
1019
- }
1020
- function assertExternalOptimizerCompletionCount(upstream, requestAttempts, successfulCompletions, name, optimizer) {
1021
- if (!upstream) throw new Error(`${name}: ${optimizer} did not report optimizer token usage`);
1022
- if (!Number.isSafeInteger(requestAttempts) || !Number.isSafeInteger(successfulCompletions) || requestAttempts < 0 || successfulCompletions < 0 || successfulCompletions > requestAttempts) throw new Error(`${name}: optimizer model proxy returned invalid request counts`);
1023
- if (upstream.calls !== successfulCompletions) throw new Error(`${name}: ${optimizer} reported ${upstream.calls} successful model calls but the proxy completed ${successfulCompletions} across ${requestAttempts} attempts`);
1024
- if (upstream.requestAttempts !== void 0 && upstream.requestAttempts !== requestAttempts) throw new Error(`${name}: ${optimizer} reported ${upstream.requestAttempts} model attempts but the proxy received ${requestAttempts}`);
1025
- }
1026
- //#endregion
1027
1074
  //#region src/campaign/external-optimizer-run-config.ts
1028
1075
  function externalOptimizerRunnerIdentity(runner, module) {
1029
1076
  return {
@@ -1048,72 +1095,6 @@ function snapshotJson(value, label) {
1048
1095
  return JSON.parse(serialized);
1049
1096
  }
1050
1097
  //#endregion
1051
- //#region src/campaign/external-optimizer-source.ts
1052
- function assertExternalOptimizerPackageSource(value, expectedPackage, name, optimizer) {
1053
- assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer);
1054
- assertExternalOptimizerSourceDetails(value, name, optimizer);
1055
- }
1056
- function assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer) {
1057
- if (!isRecord(value) || value.package !== expectedPackage || typeof value.version !== "string" || value.version.length === 0 || value.version !== value.version.trim()) throw new Error(`${name}: ${optimizer} bridge returned invalid upstream package provenance`);
1058
- }
1059
- function assertExternalOptimizerSourceDetails(value, name, optimizer) {
1060
- if (value.sourceUrl !== void 0 && (typeof value.sourceUrl !== "string" || value.sourceUrl.length === 0 || value.sourceUrl !== value.sourceUrl.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceUrl`);
1061
- if (value.revision !== void 0 && (typeof value.revision !== "string" || value.revision.length === 0 || value.revision !== value.revision.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream revision`);
1062
- if (value.sourceSha256 !== void 0 && (typeof value.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(value.sourceSha256))) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceSha256`);
1063
- }
1064
- function observedExternalOptimizerPackageSource(value) {
1065
- return {
1066
- kind: "package",
1067
- evidence: "observed",
1068
- package: value.package,
1069
- version: value.version,
1070
- ...value.sourceUrl ? { sourceUrl: value.sourceUrl } : {},
1071
- ...value.revision ? { revision: value.revision } : {},
1072
- ...value.sourceSha256 ? { sourceSha256: value.sourceSha256 } : {}
1073
- };
1074
- }
1075
- //#endregion
1076
- //#region src/campaign/external-optimizer-runtime.ts
1077
- async function inspectExternalOptimizerRuntime(args) {
1078
- const result = await runExternalOptimizerProcess({
1079
- label: `${args.label} source inspection`,
1080
- tempPrefix: `agent-eval-${args.package}-inspect-`,
1081
- module: args.module,
1082
- input: {
1083
- operation: "inspect",
1084
- engineModules: [...args.engineModules ?? []]
1085
- },
1086
- ...args.runner ? { runner: args.runner } : {},
1087
- timeoutMs: args.timeoutMs,
1088
- ...args.signal ? { signal: args.signal } : {}
1089
- });
1090
- assertExternalOptimizerRuntimeIdentity(result.runtime, args.package, args.label);
1091
- return result.runtime;
1092
- }
1093
- function observedExternalOptimizerRuntime(runtime) {
1094
- return {
1095
- source: observedExternalOptimizerPackageSource(runtime.optimizer),
1096
- bridge: observedExternalOptimizerPackageSource(runtime.bridge),
1097
- modules: runtime.engineModules.map((module) => ({ ...module })),
1098
- python: { ...runtime.python }
1099
- };
1100
- }
1101
- function assertExternalOptimizerRunBinding(args) {
1102
- if (!isDeepStrictEqual(args.returnedSource, args.runtime.optimizer)) throw new Error(`${args.label}: optimizer package changed after source inspection`);
1103
- if (args.returnedRunId !== args.runId) throw new Error(`${args.label}: bridge returned a different run ID`);
1104
- if (args.resume === "never" && args.resumed) throw new Error(`${args.label}: fresh run reported restored state`);
1105
- if (args.resume === "required" && !args.resumed) throw new Error(`${args.label}: required resume did not restore state`);
1106
- if (!args.runId.startsWith(args.compatibleRunId)) throw new Error(`${args.label}: run ID is not bound to its compatible identity`);
1107
- }
1108
- function assertExternalOptimizerRuntimeIdentity(value, expectedPackage, label) {
1109
- if (!isRecord(value)) throw new Error(`${label}: source inspection returned no runtime identity`);
1110
- assertExternalOptimizerPackageSource(value.bridge, "agent-eval-rpc", label, "bridge");
1111
- assertExternalOptimizerPackageSource(value.optimizer, expectedPackage, label, expectedPackage);
1112
- if (!value.bridge.sourceSha256 || !value.optimizer.sourceSha256) throw new Error(`${label}: source inspection omitted package source hashes`);
1113
- if (!isRecord(value.python) || typeof value.python.implementation !== "string" || value.python.implementation.length === 0 || value.python.implementation !== value.python.implementation.trim() || typeof value.python.version !== "string" || value.python.version.length === 0 || value.python.version !== value.python.version.trim()) throw new Error(`${label}: source inspection returned an invalid Python runtime`);
1114
- if (!Array.isArray(value.engineModules) || value.engineModules.some((module) => !isRecord(module) || typeof module.module !== "string" || module.module.length === 0 || module.module !== module.module.trim() || typeof module.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(module.sourceSha256))) throw new Error(`${label}: source inspection returned invalid engine modules`);
1115
- }
1116
- //#endregion
1117
1098
  //#region src/campaign/optimizer-model.ts
1118
1099
  function assertOptimizerModel(value, label) {
1119
1100
  if (!value || typeof value !== "object") throw new Error(`${label} is required`);
@@ -1311,6 +1292,96 @@ function addEvaluationLimit(total, increment) {
1311
1292
  return next;
1312
1293
  }
1313
1294
  //#endregion
1295
+ //#region src/campaign/external-optimizer-accounting.ts
1296
+ const TOKEN_USAGE_FIELDS = [
1297
+ "inputTokens",
1298
+ "outputTokens",
1299
+ "totalTokens",
1300
+ "calls"
1301
+ ];
1302
+ function assertPriorExternalOptimizerUsage(summary, budget, name) {
1303
+ if (!summary.usageComplete || budget.maxCostUsd !== void 0 && !summary.accountingComplete) throw new Error(`${name}: cannot resume optimizer-model work with incomplete bounded usage`);
1304
+ if (summary.totalCalls > budget.maxRequests || budget.maxCostUsd !== void 0 && summary.totalCostUsd > budget.maxCostUsd + Number.EPSILON) throw new Error(`${name}: prior optimizer-model usage exceeds the configured budget`);
1305
+ }
1306
+ function assertExternalOptimizerTokenUsage(usage, name, optimizer) {
1307
+ if (usage === void 0) return;
1308
+ for (const field of TOKEN_USAGE_FIELDS) if (!Number.isSafeInteger(usage[field]) || usage[field] < 0) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.${field}`);
1309
+ if (usage.totalTokens !== usage.inputTokens + usage.outputTokens) throw new Error(`${name}: ${optimizer} bridge returned inconsistent tokenUsage.totalTokens`);
1310
+ if (usage.requestAttempts !== void 0 && (!Number.isSafeInteger(usage.requestAttempts) || usage.requestAttempts < usage.calls)) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.requestAttempts`);
1311
+ }
1312
+ function assertExternalOptimizerCompletionCount(upstream, requestAttempts, successfulCompletions, name, optimizer) {
1313
+ if (!upstream) throw new Error(`${name}: ${optimizer} did not report optimizer token usage`);
1314
+ if (!Number.isSafeInteger(requestAttempts) || !Number.isSafeInteger(successfulCompletions) || requestAttempts < 0 || successfulCompletions < 0 || successfulCompletions > requestAttempts) throw new Error(`${name}: optimizer model proxy returned invalid request counts`);
1315
+ if (upstream.calls !== successfulCompletions) throw new Error(`${name}: ${optimizer} reported ${upstream.calls} successful model calls but the proxy completed ${successfulCompletions} across ${requestAttempts} attempts`);
1316
+ if (upstream.requestAttempts !== void 0 && upstream.requestAttempts !== requestAttempts) throw new Error(`${name}: ${optimizer} reported ${upstream.requestAttempts} model attempts but the proxy received ${requestAttempts}`);
1317
+ }
1318
+ //#endregion
1319
+ //#region src/campaign/external-optimizer-source.ts
1320
+ function assertExternalOptimizerPackageSource(value, expectedPackage, name, optimizer) {
1321
+ assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer);
1322
+ assertExternalOptimizerSourceDetails(value, name, optimizer);
1323
+ }
1324
+ function assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer) {
1325
+ if (!isRecord(value) || value.package !== expectedPackage || typeof value.version !== "string" || value.version.length === 0 || value.version !== value.version.trim()) throw new Error(`${name}: ${optimizer} bridge returned invalid upstream package provenance`);
1326
+ }
1327
+ function assertExternalOptimizerSourceDetails(value, name, optimizer) {
1328
+ if (value.sourceUrl !== void 0 && (typeof value.sourceUrl !== "string" || value.sourceUrl.length === 0 || value.sourceUrl !== value.sourceUrl.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceUrl`);
1329
+ if (value.revision !== void 0 && (typeof value.revision !== "string" || value.revision.length === 0 || value.revision !== value.revision.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream revision`);
1330
+ if (value.sourceSha256 !== void 0 && (typeof value.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(value.sourceSha256))) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceSha256`);
1331
+ }
1332
+ function observedExternalOptimizerPackageSource(value) {
1333
+ return {
1334
+ kind: "package",
1335
+ evidence: "observed",
1336
+ package: value.package,
1337
+ version: value.version,
1338
+ ...value.sourceUrl ? { sourceUrl: value.sourceUrl } : {},
1339
+ ...value.revision ? { revision: value.revision } : {},
1340
+ ...value.sourceSha256 ? { sourceSha256: value.sourceSha256 } : {}
1341
+ };
1342
+ }
1343
+ //#endregion
1344
+ //#region src/campaign/external-optimizer-runtime.ts
1345
+ async function inspectExternalOptimizerRuntime(args) {
1346
+ const result = await runExternalOptimizerProcess({
1347
+ label: `${args.label} source inspection`,
1348
+ tempPrefix: `agent-eval-${args.package}-inspect-`,
1349
+ module: args.module,
1350
+ input: {
1351
+ operation: "inspect",
1352
+ engineModules: [...args.engineModules ?? []]
1353
+ },
1354
+ ...args.runner ? { runner: args.runner } : {},
1355
+ timeoutMs: args.timeoutMs,
1356
+ ...args.signal ? { signal: args.signal } : {}
1357
+ });
1358
+ assertExternalOptimizerRuntimeIdentity(result.runtime, args.package, args.label);
1359
+ return result.runtime;
1360
+ }
1361
+ function observedExternalOptimizerRuntime(runtime) {
1362
+ return {
1363
+ source: observedExternalOptimizerPackageSource(runtime.optimizer),
1364
+ bridge: observedExternalOptimizerPackageSource(runtime.bridge),
1365
+ modules: runtime.engineModules.map((module) => ({ ...module })),
1366
+ python: { ...runtime.python }
1367
+ };
1368
+ }
1369
+ function assertExternalOptimizerRunBinding(args) {
1370
+ if (!isDeepStrictEqual(args.returnedSource, args.runtime.optimizer)) throw new Error(`${args.label}: optimizer package changed after source inspection`);
1371
+ if (args.returnedRunId !== args.runId) throw new Error(`${args.label}: bridge returned a different run ID`);
1372
+ if (args.resume === "never" && args.resumed) throw new Error(`${args.label}: fresh run reported restored state`);
1373
+ if (args.resume === "required" && !args.resumed) throw new Error(`${args.label}: required resume did not restore state`);
1374
+ if (!args.runId.startsWith(args.compatibleRunId)) throw new Error(`${args.label}: run ID is not bound to its compatible identity`);
1375
+ }
1376
+ function assertExternalOptimizerRuntimeIdentity(value, expectedPackage, label) {
1377
+ if (!isRecord(value)) throw new Error(`${label}: source inspection returned no runtime identity`);
1378
+ assertExternalOptimizerPackageSource(value.bridge, "agent-eval-rpc", label, "bridge");
1379
+ assertExternalOptimizerPackageSource(value.optimizer, expectedPackage, label, expectedPackage);
1380
+ if (!value.bridge.sourceSha256 || !value.optimizer.sourceSha256) throw new Error(`${label}: source inspection omitted package source hashes`);
1381
+ if (!isRecord(value.python) || typeof value.python.implementation !== "string" || value.python.implementation.length === 0 || value.python.implementation !== value.python.implementation.trim() || typeof value.python.version !== "string" || value.python.version.length === 0 || value.python.version !== value.python.version.trim()) throw new Error(`${label}: source inspection returned an invalid Python runtime`);
1382
+ if (!Array.isArray(value.engineModules) || value.engineModules.some((module) => !isRecord(module) || typeof module.module !== "string" || module.module.length === 0 || module.module !== module.module.trim() || typeof module.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(module.sourceSha256))) throw new Error(`${label}: source inspection returned invalid engine modules`);
1383
+ }
1384
+ //#endregion
1314
1385
  //#region src/campaign/gepa-optimization-result.ts
1315
1386
  function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, maxEvaluations, maxPopulationCandidates, scenarioIds, expectsComponents, requiresCandidatePopulation) {
1316
1387
  if (result.recipeKind !== recipeKind) throw new Error(`${name}: GEPA bridge reported recipe '${String(result.recipeKind)}'`);
@@ -1733,24 +1804,7 @@ async function recordExternalSearch(input) {
1733
1804
  });
1734
1805
  }
1735
1806
  //#endregion
1736
- //#region src/campaign/gepa-optimization-method.ts
1737
- /**
1738
- * Turn an optional GEPA installation into an `OptimizationMethod`.
1739
- *
1740
- * GEPA receives only serialized train and selection cases. The caller's final
1741
- * test partition stays inside `compareOptimizationMethods`, which invokes this
1742
- * method without a test-set field. The local callback routes every candidate
1743
- * evaluation through the same dispatch and judges used by other methods.
1744
- */
1745
- /**
1746
- * Environment for a bridge child that spawns `claude` CLI subprocesses.
1747
- *
1748
- * Both agent CLI engines start the CLI with the bridge's own environment, so
1749
- * these variables travel bridge → engine → CLI. They merge AFTER
1750
- * `removeCredentialEnvironment`: the ephemeral loopback token is the only
1751
- * credential the child may see, and a caller-supplied ANTHROPIC_AUTH_TOKEN is
1752
- * stripped like any other credential.
1753
- */
1807
+ //#region src/campaign/gepa-optimization-run.ts
1754
1808
  function anthropicShimEnvironment(proxy, optimizer) {
1755
1809
  return {
1756
1810
  ANTHROPIC_BASE_URL: new URL(proxy.baseUrl).origin,
@@ -1763,370 +1817,501 @@ function anthropicShimEnvironment(proxy, optimizer) {
1763
1817
  API_TIMEOUT_MS: String((optimizer.budget.requestTimeoutMs ?? 3e5) + 6e4)
1764
1818
  };
1765
1819
  }
1766
- function gepaOptimizationMethod(config) {
1767
- assertGepaOptimizationConfig(config);
1768
- config = snapshotGepaOptimizationConfig(config);
1769
- const name = config.name ?? defaultGepaMethodName(config.recipe);
1820
+ /** Spend the optimizer booked to its own candidate generation. Unknown stays
1821
+ * unknown: the bridge reports proposer cost only when the engine measured it. */
1822
+ function optimizerAccounting(tokenUsage, proposerCostUsd) {
1823
+ return {
1824
+ tokens: tokenUsage?.inputTokens === void 0 || tokenUsage.outputTokens === void 0 ? {
1825
+ status: "unknown",
1826
+ reason: "the optimizer bridge reported no token usage"
1827
+ } : {
1828
+ status: "known",
1829
+ inputTokens: tokenUsage.inputTokens,
1830
+ outputTokens: tokenUsage.outputTokens,
1831
+ cachedTokens: 0
1832
+ },
1833
+ cost: proposerCostUsd === void 0 ? {
1834
+ status: "unknown",
1835
+ knownLowerBoundUsd: 0,
1836
+ reason: "the optimizer bridge reported no proposer cost"
1837
+ } : {
1838
+ status: "known",
1839
+ usd: proposerCostUsd,
1840
+ source: "provider"
1841
+ }
1842
+ };
1843
+ }
1844
+ /** The settled inputs of one GEPA run: its identity, budgets, evaluation callback and logs. */
1845
+ async function prepareGepaRun(config, input, name) {
1846
+ const signal = input.runOptions.signal;
1847
+ signal?.throwIfAborted();
1848
+ if (typeof input.baselineSurface !== "string" && input.baselineSurface.kind !== "components") throw new Error(`${name}: GEPA bridge supports text and component surfaces`);
1849
+ const expectsComponents = typeof input.baselineSurface !== "string";
1850
+ if (expectsComponents) assertGepaComponentRecipe(config.recipe, name);
1851
+ const started = Date.now();
1852
+ const maxCandidateChars = config.maxCandidateChars ?? 2e5;
1853
+ const maxEvidenceChars = config.maxEvidenceChars ?? 1e5;
1854
+ const timeoutMs = config.timeoutMs ?? 18e5;
1855
+ const storage = input.runOptions.storage ?? fsCampaignStorage();
1856
+ const runDir = `${input.runDir}/gepa`;
1857
+ storage.ensureDir(runDir);
1858
+ const costLedger = input.costLedger;
1859
+ const attemptId = randomBytes(16).toString("hex");
1860
+ const resume = config.resume ?? "never";
1861
+ const bridgeRunner = config.optimizer && config.runner ? {
1862
+ ...config.runner,
1863
+ env: removeCredentialEnvironment(config.runner.env ?? {})
1864
+ } : config.runner;
1865
+ const runtimeIdentity = await inspectExternalOptimizerRuntime({
1866
+ label: name,
1867
+ package: "gepa",
1868
+ module: "agent_eval_rpc.gepa_bridge",
1869
+ engineModules: config.engineModules,
1870
+ ...bridgeRunner ? { runner: bridgeRunner } : {},
1871
+ timeoutMs,
1872
+ ...signal ? { signal } : {}
1873
+ });
1874
+ const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
1875
+ const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
1876
+ const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
1877
+ const evaluationLimit = gepaRecipeEvaluationLimit(config.recipe, input.selectionScenarios.length);
1878
+ const maxPopulationCandidates = Math.min(Number.MAX_SAFE_INTEGER, evaluationLimit + 1);
1879
+ const populationScenarioIds = (selectionSet.length > 0 ? selectionSet : trainSet).map((scenario) => scenario.id);
1880
+ const runMaterial = {
1881
+ optimizer: "gepa",
1882
+ runtime: runtimeIdentity,
1883
+ method: name,
1884
+ evaluationId: config.evaluationId,
1885
+ dispatchRef: input.runOptions.dispatchRef ?? null,
1886
+ seed: input.seed,
1887
+ recipe: snapshotJson(config.recipe, "GEPA run settings"),
1888
+ engineModules: config.engineModules ?? [],
1889
+ objective: config.objective,
1890
+ background: config.background ?? "",
1891
+ seedCandidate,
1892
+ trainSet,
1893
+ selectionSet,
1894
+ maxCandidateChars,
1895
+ maxPopulationCandidates,
1896
+ maxEvidenceChars,
1897
+ evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
1898
+ optimizerModel: config.optimizer ? {
1899
+ model: config.optimizer.model,
1900
+ callRef: config.optimizer.callRef,
1901
+ budget: config.optimizer.budget,
1902
+ ...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
1903
+ } : null,
1904
+ runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.gepa_bridge"),
1905
+ trustResumeState: config.trustResumeState === true
1906
+ };
1907
+ const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
1908
+ const runId = externalOptimizerRunKey({
1909
+ material: runMaterial,
1910
+ attemptId,
1911
+ resumeEnabled: resume !== "never" && gepaRecipeSupportsResume(config.recipe)
1912
+ });
1913
+ const runBudget = openExternalOptimizerRunBudget({
1914
+ storage,
1915
+ runDir,
1916
+ runKey: runId,
1917
+ attemptId,
1918
+ maxEvaluations: evaluationLimit
1919
+ });
1920
+ const observationLog = openExternalOptimizerObservationLog({
1921
+ storage,
1922
+ path: `${runDir}/observations-${attemptId}.jsonl`
1923
+ });
1924
+ const executionLog = config.optimizer ? openExternalOptimizerExecutionLog({
1925
+ storage,
1926
+ path: `${runDir}/model-executions-${attemptId}.jsonl`
1927
+ }) : void 0;
1928
+ const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "GEPA bridge");
1929
+ const evaluate = createExternalTextEvaluator({
1930
+ input,
1931
+ label: "GEPA bridge",
1932
+ runDir,
1933
+ compatibleRunId: runId,
1934
+ costPhase: "gepa.external-evaluation",
1935
+ costTags: runBudget.attemptTags,
1936
+ costLedger,
1937
+ scenarioById,
1938
+ maxCandidateChars,
1939
+ maxEvidenceChars,
1940
+ describeArtifact: config.describeArtifact
1941
+ });
1942
+ let attemptEvaluationCount = 0;
1770
1943
  return {
1944
+ signal,
1945
+ expectsComponents,
1946
+ started,
1947
+ maxCandidateChars,
1948
+ maxEvidenceChars,
1949
+ timeoutMs,
1950
+ storage,
1951
+ runDir,
1952
+ costLedger,
1953
+ attemptId,
1954
+ resume,
1955
+ bridgeRunner,
1956
+ runtimeIdentity,
1957
+ seedCandidate,
1958
+ trainSet,
1959
+ selectionSet,
1960
+ evaluationLimit,
1961
+ maxPopulationCandidates,
1962
+ populationScenarioIds,
1963
+ compatibleRunId,
1964
+ runId,
1965
+ runBudget,
1966
+ observationLog,
1967
+ executionLog,
1968
+ callback: await startExternalOptimizerCallback({
1969
+ token: randomBytes(32).toString("hex"),
1970
+ maxEvaluations: evaluationLimit,
1971
+ acceptEvaluation: () => {
1972
+ if (runBudget.acceptEvaluation() === void 0) return void 0;
1973
+ attemptEvaluationCount += 1;
1974
+ return attemptEvaluationCount;
1975
+ },
1976
+ evaluate,
1977
+ observe: observationLog.observe,
1978
+ ...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
1979
+ ...signal ? { signal } : {}
1980
+ }),
1771
1981
  name,
1772
- async optimize(input) {
1773
- const signal = input.runOptions.signal;
1774
- signal?.throwIfAborted();
1775
- if (typeof input.baselineSurface !== "string" && input.baselineSurface.kind !== "components") throw new Error(`${name}: GEPA bridge supports text and component surfaces`);
1776
- const expectsComponents = typeof input.baselineSurface !== "string";
1777
- if (expectsComponents) assertGepaComponentRecipe(config.recipe, name);
1778
- const started = Date.now();
1779
- const maxCandidateChars = config.maxCandidateChars ?? 2e5;
1780
- const maxEvidenceChars = config.maxEvidenceChars ?? 1e5;
1781
- const timeoutMs = config.timeoutMs ?? 18e5;
1782
- const storage = input.runOptions.storage ?? fsCampaignStorage();
1783
- const runDir = `${input.runDir}/gepa`;
1784
- storage.ensureDir(runDir);
1785
- const costLedger = input.costLedger;
1786
- const attemptId = randomBytes(16).toString("hex");
1787
- const resume = config.resume ?? "never";
1788
- const bridgeRunner = config.optimizer && config.runner ? {
1789
- ...config.runner,
1790
- env: removeCredentialEnvironment(config.runner.env ?? {})
1791
- } : config.runner;
1792
- const runtimeIdentity = await inspectExternalOptimizerRuntime({
1793
- label: name,
1794
- package: "gepa",
1795
- module: "agent_eval_rpc.gepa_bridge",
1796
- engineModules: config.engineModules,
1797
- ...bridgeRunner ? { runner: bridgeRunner } : {},
1798
- timeoutMs,
1799
- ...signal ? { signal } : {}
1800
- });
1801
- const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
1802
- const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
1803
- const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
1804
- const evaluationLimit = gepaRecipeEvaluationLimit(config.recipe, input.selectionScenarios.length);
1805
- const maxPopulationCandidates = Math.min(Number.MAX_SAFE_INTEGER, evaluationLimit + 1);
1806
- const populationScenarioIds = (selectionSet.length > 0 ? selectionSet : trainSet).map((scenario) => scenario.id);
1807
- const runMaterial = {
1808
- optimizer: "gepa",
1809
- runtime: runtimeIdentity,
1810
- method: name,
1811
- evaluationId: config.evaluationId,
1812
- dispatchRef: input.runOptions.dispatchRef ?? null,
1813
- seed: input.seed,
1814
- recipe: snapshotJson(config.recipe, "GEPA run settings"),
1815
- engineModules: config.engineModules ?? [],
1816
- objective: config.objective,
1817
- background: config.background ?? "",
1818
- seedCandidate,
1819
- trainSet,
1820
- selectionSet,
1821
- maxCandidateChars,
1822
- maxPopulationCandidates,
1823
- maxEvidenceChars,
1824
- evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
1825
- optimizerModel: config.optimizer ? {
1826
- model: config.optimizer.model,
1982
+ config,
1983
+ input
1984
+ };
1985
+ }
1986
+ /** Start the optimizer model proxy when one is configured, run the GEPA bridge, and release the resources either way. */
1987
+ async function runBridge(ctx) {
1988
+ const { signal, maxCandidateChars, maxEvidenceChars, timeoutMs, runDir, costLedger, attemptId, resume, bridgeRunner, runtimeIdentity, seedCandidate, trainSet, selectionSet, maxPopulationCandidates, compatibleRunId, runId, runBudget, executionLog, callback, name, config, input } = ctx;
1989
+ const runnerEnv = bridgeRunner?.env ?? {};
1990
+ let modelProxy;
1991
+ const bridgeRunnerForSpawn = () => {
1992
+ if (!modelProxy) return bridgeRunner;
1993
+ const anthropicChildEnv = config.optimizer?.anthropicEndpoint === true && gepaRecipeHasAgentCliEngine(config.recipe) ? anthropicShimEnvironment(modelProxy, config.optimizer) : void 0;
1994
+ if (!bridgeRunner && !anthropicChildEnv) return bridgeRunner;
1995
+ return {
1996
+ ...bridgeRunner ?? {},
1997
+ env: {
1998
+ ...removeCredentialEnvironment(runnerEnv),
1999
+ ...anthropicChildEnv ?? {}
2000
+ }
2001
+ };
2002
+ };
2003
+ const closeResources = () => closeExternalOptimizerResources({
2004
+ label: name,
2005
+ callback,
2006
+ ...modelProxy ? { modelProxy } : {}
2007
+ });
2008
+ const { result, outputDir } = await runWithCleanup({
2009
+ label: `${name} optimizer resources`,
2010
+ run: async () => {
2011
+ if (config.optimizer) {
2012
+ const priorOptimizerUsage = costLedger.summary({
2013
+ phase: "gepa.optimizer-model",
2014
+ tags: runBudget.runTags
2015
+ });
2016
+ assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
2017
+ modelProxy = await startExternalOptimizerModelProxy({
2018
+ call: config.optimizer.call,
1827
2019
  callRef: config.optimizer.callRef,
2020
+ recordExecution: executionLog.observe,
2021
+ model: config.optimizer.model,
1828
2022
  budget: config.optimizer.budget,
1829
- ...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
1830
- } : null,
1831
- runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.gepa_bridge"),
1832
- trustResumeState: config.trustResumeState === true
1833
- };
1834
- const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
1835
- const runId = externalOptimizerRunKey({
1836
- material: runMaterial,
1837
- attemptId,
1838
- resumeEnabled: resume !== "never" && gepaRecipeSupportsResume(config.recipe)
1839
- });
1840
- const runBudget = openExternalOptimizerRunBudget({
1841
- storage,
1842
- runDir,
1843
- runKey: runId,
1844
- attemptId,
1845
- maxEvaluations: evaluationLimit
1846
- });
1847
- const observationLog = openExternalOptimizerObservationLog({
1848
- storage,
1849
- path: `${runDir}/observations-${attemptId}.jsonl`
1850
- });
1851
- const executionLog = config.optimizer ? openExternalOptimizerExecutionLog({
1852
- storage,
1853
- path: `${runDir}/model-executions-${attemptId}.jsonl`
1854
- }) : void 0;
1855
- const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "GEPA bridge");
1856
- const evaluate = createExternalTextEvaluator({
1857
- input,
1858
- label: "GEPA bridge",
1859
- runDir,
1860
- compatibleRunId: runId,
1861
- costPhase: "gepa.external-evaluation",
1862
- costTags: runBudget.attemptTags,
1863
- costLedger,
1864
- scenarioById,
1865
- maxCandidateChars,
1866
- maxEvidenceChars,
1867
- describeArtifact: config.describeArtifact
1868
- });
1869
- let attemptEvaluationCount = 0;
1870
- const callback = await startExternalOptimizerCallback({
1871
- token: randomBytes(32).toString("hex"),
1872
- maxEvaluations: evaluationLimit,
1873
- acceptEvaluation: () => {
1874
- if (runBudget.acceptEvaluation() === void 0) return void 0;
1875
- attemptEvaluationCount += 1;
1876
- return attemptEvaluationCount;
1877
- },
1878
- evaluate,
1879
- observe: observationLog.observe,
1880
- ...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
1881
- ...signal ? { signal } : {}
1882
- });
1883
- const runnerEnv = bridgeRunner?.env ?? {};
1884
- let modelProxy;
1885
- const bridgeRunnerForSpawn = () => {
1886
- if (!modelProxy) return bridgeRunner;
1887
- const anthropicChildEnv = config.optimizer?.anthropicEndpoint === true && gepaRecipeHasAgentCliEngine(config.recipe) ? anthropicShimEnvironment(modelProxy, config.optimizer) : void 0;
1888
- if (!bridgeRunner && !anthropicChildEnv) return bridgeRunner;
1889
- return {
1890
- ...bridgeRunner ?? {},
1891
- env: {
1892
- ...removeCredentialEnvironment(runnerEnv),
1893
- ...anthropicChildEnv ?? {}
1894
- }
1895
- };
1896
- };
1897
- const closeResources = () => closeExternalOptimizerResources({
1898
- label: name,
1899
- callback,
1900
- ...modelProxy ? { modelProxy } : {}
1901
- });
1902
- const { result, outputDir } = await runWithCleanup({
1903
- label: `${name} optimizer resources`,
1904
- run: async () => {
1905
- if (config.optimizer) {
1906
- const priorOptimizerUsage = costLedger.summary({
1907
- phase: "gepa.optimizer-model",
1908
- tags: runBudget.runTags
1909
- });
1910
- assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
1911
- modelProxy = await startExternalOptimizerModelProxy({
1912
- call: config.optimizer.call,
1913
- callRef: config.optimizer.callRef,
1914
- recordExecution: executionLog.observe,
1915
- model: config.optimizer.model,
1916
- budget: config.optimizer.budget,
1917
- ...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
1918
- ...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {},
1919
- costLedger,
1920
- phase: "gepa.optimizer-model",
1921
- actor: name,
1922
- tags: { ...runBudget.attemptTags },
1923
- initialUsage: {
1924
- requests: priorOptimizerUsage.totalCalls,
1925
- ...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
1926
- },
1927
- ...signal ? { signal } : {}
1928
- });
1929
- }
1930
- const outputDir = `${runDir}/external`;
1931
- await mkdir(outputDir, { recursive: true });
1932
- return {
1933
- result: await runExternalOptimizerProcess({
1934
- label: "GEPA bridge",
1935
- tempPrefix: "agent-eval-gepa-",
1936
- module: "agent_eval_rpc.gepa_bridge",
1937
- input: {
1938
- attemptId,
1939
- compatibleRunId,
1940
- runId,
1941
- runtimeIdentity,
1942
- resume,
1943
- trustedResumeState: config.trustResumeState === true,
1944
- evaluationId: config.evaluationId,
1945
- seed: input.seed,
1946
- callbackUrl: callback.url,
1947
- callbackToken: callback.token,
1948
- timeoutMs,
1949
- engineModules: config.engineModules ?? [],
1950
- recipe: config.recipe,
1951
- objective: config.objective,
1952
- ...config.background ? { background: config.background } : {},
1953
- seedCandidate,
1954
- trainSet,
1955
- selectionSet,
1956
- maxCandidateChars,
1957
- maxPopulationCandidates,
1958
- maxEvidenceChars,
1959
- outputDir,
1960
- ...modelProxy && config.optimizer ? { modelProxy: {
1961
- baseUrl: modelProxy.baseUrl,
1962
- apiKey: modelProxy.apiKey,
1963
- model: config.optimizer.model,
1964
- budget: config.optimizer.budget,
1965
- ...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
1966
- } } : {}
1967
- },
1968
- runner: bridgeRunnerForSpawn(),
1969
- timeoutMs,
1970
- ...signal ? { signal } : {}
1971
- }),
1972
- outputDir
1973
- };
1974
- },
1975
- cleanup: closeResources
1976
- });
1977
- signal?.throwIfAborted();
1978
- assertGepaBridgeOutput(result, name, maxCandidateChars, config.recipe.kind, evaluationLimit, maxPopulationCandidates, populationScenarioIds, expectsComponents, config.recipe.kind === "engine" && config.recipe.run.engine === "gepa");
1979
- assertExternalOptimizerRunBinding({
1980
- label: name,
1981
- runtime: runtimeIdentity,
1982
- returnedSource: result.upstream,
1983
- compatibleRunId,
1984
- runId,
1985
- returnedRunId: result.runId,
1986
- resume,
1987
- resumed: result.resumed
1988
- });
1989
- if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
1990
- const population = result.candidatePopulation ? readGepaCandidatePopulationArtifact({
1991
- summary: result.candidatePopulation,
1992
- storage
1993
- }) : null;
1994
- if (population) {
1995
- const selected = population.candidates[population.bestIndex];
1996
- const selectedHash = contentHash({
1997
- kind: "external-text-candidate",
1998
- candidate: result.bestCandidate
2023
+ ...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
2024
+ ...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {},
2025
+ costLedger,
2026
+ phase: "gepa.optimizer-model",
2027
+ actor: name,
2028
+ tags: { ...runBudget.attemptTags },
2029
+ initialUsage: {
2030
+ requests: priorOptimizerUsage.totalCalls,
2031
+ ...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
2032
+ },
2033
+ ...signal ? { signal } : {}
1999
2034
  });
2000
- if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
2001
2035
  }
2002
- const searchHistory = config.searchLedger ? await recordGepaSearch({
2003
- name,
2004
- path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
2005
- searchId: runId,
2006
- identity: config.searchLedger.identity,
2007
- storage,
2008
- seed: input.seed,
2009
- baselineSurface: input.baselineSurface,
2010
- trainScenarios: input.trainScenarios,
2011
- selectionScenarios: input.selectionScenarios,
2012
- evaluationLimit,
2013
- ...population ? { population } : {
2014
- population: null,
2015
- selected: decodeExternalTextCandidate(result.bestCandidate)
2016
- },
2017
- observations: readExternalOptimizerObservationArtifact({
2018
- summary: observationLog.summary(),
2019
- storage
2020
- }),
2021
- generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd)
2022
- }) : void 0;
2023
- const evaluationCost = costFromLedgerSummary(costLedger.summary({
2024
- phase: "gepa.external-evaluation",
2025
- tags: runBudget.runTags
2026
- }));
2027
- const optimizerSummary = costLedger.summary({
2028
- phase: "gepa.optimizer-model",
2029
- tags: runBudget.runTags
2030
- });
2031
- const optimizerReceipts = costLedger.list({
2032
- phase: "gepa.optimizer-model",
2033
- tags: runBudget.runTags
2034
- });
2035
- const optimizerCost = costFromLedgerSummary(optimizerSummary);
2036
- const reportedProposerCost = result.proposerCostUsd ?? 0;
2037
- if (modelProxy) {
2038
- modelProxy.assertExecutionComplete();
2039
- const wires = modelProxy.wireUsage();
2040
- assertExternalOptimizerCompletionCount(result.tokenUsage, wires.openai.requestAttempts, wires.openai.successfulCompletions, name, "GEPA");
2041
- if (config.optimizer?.anthropicEndpoint === true) {
2042
- const anthropicReceipts = costLedger.list({
2043
- phase: "gepa.optimizer-model",
2044
- tags: {
2045
- ...runBudget.attemptTags,
2046
- wire: "anthropic"
2047
- }
2048
- });
2049
- if (anthropicReceipts.length !== wires.anthropic.requestAttempts) throw new Error(`${name}: the Anthropic endpoint admitted ${wires.anthropic.requestAttempts} model calls but the ledger holds ${anthropicReceipts.length} receipts`);
2050
- }
2051
- }
2052
- const tokenUsage = modelProxy ? optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts) : void 0;
2053
- const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
2054
- const meteredCost = modelProxy ? combineComparisonCosts([{
2055
- label: "evaluation",
2056
- cost: evaluationCost
2057
- }, {
2058
- label: "optimizer model",
2059
- cost: optimizerCost
2060
- }]) : void 0;
2061
- const externalTotalCostUsd = evaluationCost.totalCostUsd + reportedProposerCost;
2036
+ const outputDir = `${runDir}/external`;
2037
+ await mkdir(outputDir, { recursive: true });
2062
2038
  return {
2063
- winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
2064
- ...searchHistory ? { searchHistory } : {},
2065
- cost: modelProxy ? meteredCost : {
2066
- totalCostUsd: externalTotalCostUsd,
2067
- costProvenance: result.proposerCostAccounting === "reported" && evaluationCost.costProvenance.kind !== "uncaptured" ? {
2068
- kind: "estimated",
2069
- usd: externalTotalCostUsd
2070
- } : {
2071
- kind: "uncaptured",
2072
- usd: null
2039
+ result: await runExternalOptimizerProcess({
2040
+ label: "GEPA bridge",
2041
+ tempPrefix: "agent-eval-gepa-",
2042
+ module: "agent_eval_rpc.gepa_bridge",
2043
+ input: {
2044
+ attemptId,
2045
+ compatibleRunId,
2046
+ runId,
2047
+ runtimeIdentity,
2048
+ resume,
2049
+ trustedResumeState: config.trustResumeState === true,
2050
+ evaluationId: config.evaluationId,
2051
+ seed: input.seed,
2052
+ callbackUrl: callback.url,
2053
+ callbackToken: callback.token,
2054
+ timeoutMs,
2055
+ engineModules: config.engineModules ?? [],
2056
+ recipe: config.recipe,
2057
+ objective: config.objective,
2058
+ ...config.background ? { background: config.background } : {},
2059
+ seedCandidate,
2060
+ trainSet,
2061
+ selectionSet,
2062
+ maxCandidateChars,
2063
+ maxPopulationCandidates,
2064
+ maxEvidenceChars,
2065
+ outputDir,
2066
+ ...modelProxy && config.optimizer ? { modelProxy: {
2067
+ baseUrl: modelProxy.baseUrl,
2068
+ apiKey: modelProxy.apiKey,
2069
+ model: config.optimizer.model,
2070
+ budget: config.optimizer.budget,
2071
+ ...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
2072
+ } } : {}
2073
2073
  },
2074
- accountingComplete: false,
2075
- incompleteReasons: [
2076
- ...evaluationCost.incompleteReasons,
2077
- result.proposerCostAccounting === "reported" ? "GEPA proposer cost is externally reported and has no agent-eval receipt" : "GEPA proposer cost is unavailable",
2078
- ...result.resumed ? ["GEPA proposer cost before this resumed attempt is unavailable"] : []
2079
- ]
2080
- },
2081
- durationMs: Date.now() - started,
2082
- provenance: {
2083
- ...runtime,
2084
- ...config.optimizer ? {
2085
- optimizerModel: config.optimizer.model,
2086
- optimizerCallRef: config.optimizer.callRef
2087
- } : {},
2088
- ...modelProxy && config.optimizer?.anthropicEndpoint === true ? { anthropicEndpoint: modelProxy.wireUsage().anthropic } : {},
2089
- compatibleRunId,
2090
- runId,
2091
- resumed: result.resumed,
2092
- seedApplied: result.seedApplied,
2093
- evaluationCount: runBudget.acceptedEvaluations(),
2094
- ...result.upstreamReportedEvaluations !== void 0 ? { upstreamReportedEvaluations: result.upstreamReportedEvaluations } : {},
2095
- artifactDir: outputDir,
2096
- ...tokenUsage ? { tokenUsage } : {},
2097
- observations: observationLog.summary(),
2098
- ...result.candidatePopulation ? { gepaCandidatePopulation: result.candidatePopulation } : {},
2099
- ...executionLog ? { modelExecutions: executionLog.summary() } : {}
2074
+ runner: bridgeRunnerForSpawn(),
2075
+ timeoutMs,
2076
+ ...signal ? { signal } : {}
2077
+ }),
2078
+ outputDir
2079
+ };
2080
+ },
2081
+ cleanup: closeResources
2082
+ });
2083
+ return {
2084
+ result,
2085
+ outputDir,
2086
+ modelProxy
2087
+ };
2088
+ }
2089
+ /** Check the bridge output against the run it was started for, and return the candidate population it reports. */
2090
+ function verifyBridgeResult(ctx, result) {
2091
+ const { signal, expectsComponents, maxCandidateChars, storage, resume, runtimeIdentity, evaluationLimit, maxPopulationCandidates, populationScenarioIds, compatibleRunId, runId, callback, name, config } = ctx;
2092
+ signal?.throwIfAborted();
2093
+ assertGepaBridgeOutput(result, name, maxCandidateChars, config.recipe.kind, evaluationLimit, maxPopulationCandidates, populationScenarioIds, expectsComponents, config.recipe.kind === "engine" && config.recipe.run.engine === "gepa");
2094
+ assertExternalOptimizerRunBinding({
2095
+ label: name,
2096
+ runtime: runtimeIdentity,
2097
+ returnedSource: result.upstream,
2098
+ compatibleRunId,
2099
+ runId,
2100
+ returnedRunId: result.runId,
2101
+ resume,
2102
+ resumed: result.resumed
2103
+ });
2104
+ if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
2105
+ const population = result.candidatePopulation ? readGepaCandidatePopulationArtifact({
2106
+ summary: result.candidatePopulation,
2107
+ storage
2108
+ }) : null;
2109
+ if (population) {
2110
+ const selected = population.candidates[population.bestIndex];
2111
+ const selectedHash = contentHash({
2112
+ kind: "external-text-candidate",
2113
+ candidate: result.bestCandidate
2114
+ });
2115
+ if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
2116
+ }
2117
+ return population;
2118
+ }
2119
+ /** Every recipe records its search in the search ledger when one is configured. */
2120
+ async function recordGepaSearchHistory(ctx, result, population) {
2121
+ const { storage, runDir, evaluationLimit, runId, observationLog, name, config, input } = ctx;
2122
+ return config.searchLedger ? await recordGepaSearch({
2123
+ name,
2124
+ path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
2125
+ searchId: runId,
2126
+ identity: config.searchLedger.identity,
2127
+ storage,
2128
+ seed: input.seed,
2129
+ baselineSurface: input.baselineSurface,
2130
+ trainScenarios: input.trainScenarios,
2131
+ selectionScenarios: input.selectionScenarios,
2132
+ evaluationLimit,
2133
+ ...population ? { population } : {
2134
+ population: null,
2135
+ selected: decodeExternalTextCandidate(result.bestCandidate)
2136
+ },
2137
+ observations: readExternalOptimizerObservationArtifact({
2138
+ summary: observationLog.summary(),
2139
+ storage
2140
+ }),
2141
+ generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd)
2142
+ }) : void 0;
2143
+ }
2144
+ /** Evaluation and optimizer-model cost, with the proxy's completeness checks. */
2145
+ function gepaCosts(ctx, result, modelProxy) {
2146
+ const { costLedger, runtimeIdentity, runBudget, name, config } = ctx;
2147
+ const evaluationCost = costFromLedgerSummary(costLedger.summary({
2148
+ phase: "gepa.external-evaluation",
2149
+ tags: runBudget.runTags
2150
+ }));
2151
+ const optimizerSummary = costLedger.summary({
2152
+ phase: "gepa.optimizer-model",
2153
+ tags: runBudget.runTags
2154
+ });
2155
+ const optimizerReceipts = costLedger.list({
2156
+ phase: "gepa.optimizer-model",
2157
+ tags: runBudget.runTags
2158
+ });
2159
+ const optimizerCost = costFromLedgerSummary(optimizerSummary);
2160
+ const reportedProposerCost = result.proposerCostUsd ?? 0;
2161
+ if (modelProxy) {
2162
+ modelProxy.assertExecutionComplete();
2163
+ const wires = modelProxy.wireUsage();
2164
+ assertExternalOptimizerCompletionCount(result.tokenUsage, wires.openai.requestAttempts, wires.openai.successfulCompletions, name, "GEPA");
2165
+ if (config.optimizer?.anthropicEndpoint === true) {
2166
+ const anthropicReceipts = costLedger.list({
2167
+ phase: "gepa.optimizer-model",
2168
+ tags: {
2169
+ ...runBudget.attemptTags,
2170
+ wire: "anthropic"
2100
2171
  }
2101
- };
2172
+ });
2173
+ if (anthropicReceipts.length !== wires.anthropic.requestAttempts) throw new Error(`${name}: the Anthropic endpoint admitted ${wires.anthropic.requestAttempts} model calls but the ledger holds ${anthropicReceipts.length} receipts`);
2102
2174
  }
2175
+ }
2176
+ return {
2177
+ evaluationCost,
2178
+ tokenUsage: modelProxy ? optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts) : void 0,
2179
+ runtime: observedExternalOptimizerRuntime(runtimeIdentity),
2180
+ meteredCost: modelProxy ? combineComparisonCosts([{
2181
+ label: "evaluation",
2182
+ cost: evaluationCost
2183
+ }, {
2184
+ label: "optimizer model",
2185
+ cost: optimizerCost
2186
+ }]) : void 0,
2187
+ externalTotalCostUsd: evaluationCost.totalCostUsd + reportedProposerCost
2103
2188
  };
2104
2189
  }
2105
- /** Spend the optimizer booked to its own candidate generation. Unknown stays
2106
- * unknown: the bridge reports proposer cost only when the engine measured it. */
2107
- function optimizerAccounting(tokenUsage, proposerCostUsd) {
2190
+ /** The method result: the winning surface, its cost and the provenance of the run. */
2191
+ function buildGepaOutcome(ctx, result, outputDir, modelProxy, searchHistory, costs) {
2192
+ const { started, compatibleRunId, runId, runBudget, observationLog, executionLog, config } = ctx;
2193
+ const { evaluationCost, tokenUsage, runtime, meteredCost, externalTotalCostUsd } = costs;
2108
2194
  return {
2109
- tokens: tokenUsage?.inputTokens === void 0 || tokenUsage.outputTokens === void 0 ? {
2110
- status: "unknown",
2111
- reason: "the optimizer bridge reported no token usage"
2112
- } : {
2113
- status: "known",
2114
- inputTokens: tokenUsage.inputTokens,
2115
- outputTokens: tokenUsage.outputTokens,
2116
- cachedTokens: 0
2195
+ winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
2196
+ ...searchHistory ? { searchHistory } : {},
2197
+ cost: modelProxy ? meteredCost : {
2198
+ totalCostUsd: externalTotalCostUsd,
2199
+ costProvenance: result.proposerCostAccounting === "reported" && evaluationCost.costProvenance.kind !== "uncaptured" ? {
2200
+ kind: "estimated",
2201
+ usd: externalTotalCostUsd
2202
+ } : {
2203
+ kind: "uncaptured",
2204
+ usd: null
2205
+ },
2206
+ accountingComplete: false,
2207
+ incompleteReasons: [
2208
+ ...evaluationCost.incompleteReasons,
2209
+ result.proposerCostAccounting === "reported" ? "GEPA proposer cost is externally reported and has no agent-eval receipt" : "GEPA proposer cost is unavailable",
2210
+ ...result.resumed ? ["GEPA proposer cost before this resumed attempt is unavailable"] : []
2211
+ ]
2117
2212
  },
2118
- cost: proposerCostUsd === void 0 ? {
2119
- status: "unknown",
2120
- knownLowerBoundUsd: 0,
2121
- reason: "the optimizer bridge reported no proposer cost"
2122
- } : {
2123
- status: "known",
2124
- usd: proposerCostUsd,
2125
- source: "provider"
2213
+ durationMs: Date.now() - started,
2214
+ provenance: {
2215
+ ...runtime,
2216
+ ...config.optimizer ? {
2217
+ optimizerModel: config.optimizer.model,
2218
+ optimizerCallRef: config.optimizer.callRef
2219
+ } : {},
2220
+ ...modelProxy && config.optimizer?.anthropicEndpoint === true ? { anthropicEndpoint: modelProxy.wireUsage().anthropic } : {},
2221
+ compatibleRunId,
2222
+ runId,
2223
+ resumed: result.resumed,
2224
+ seedApplied: result.seedApplied,
2225
+ evaluationCount: runBudget.acceptedEvaluations(),
2226
+ ...result.upstreamReportedEvaluations !== void 0 ? { upstreamReportedEvaluations: result.upstreamReportedEvaluations } : {},
2227
+ artifactDir: outputDir,
2228
+ ...tokenUsage ? { tokenUsage } : {},
2229
+ observations: observationLog.summary(),
2230
+ ...result.candidatePopulation ? { gepaCandidatePopulation: result.candidatePopulation } : {},
2231
+ ...executionLog ? { modelExecutions: executionLog.summary() } : {}
2232
+ }
2233
+ };
2234
+ }
2235
+ //#endregion
2236
+ //#region src/campaign/gepa-optimization-method.ts
2237
+ /**
2238
+ * Turn an optional GEPA installation into an `OptimizationMethod`.
2239
+ *
2240
+ * GEPA receives only serialized train and selection cases. The caller's final
2241
+ * test partition stays inside `compareOptimizationMethods`, which invokes this
2242
+ * method without a test-set field. The local callback routes every candidate
2243
+ * evaluation through the same dispatch and judges used by other methods.
2244
+ */
2245
+ /**
2246
+ * Environment for a bridge child that spawns `claude` CLI subprocesses.
2247
+ *
2248
+ * Both agent CLI engines start the CLI with the bridge's own environment, so
2249
+ * these variables travel bridge → engine → CLI. They merge AFTER
2250
+ * `removeCredentialEnvironment`: the ephemeral loopback token is the only
2251
+ * credential the child may see, and a caller-supplied ANTHROPIC_AUTH_TOKEN is
2252
+ * stripped like any other credential.
2253
+ */
2254
+ function gepaOptimizationMethod(config) {
2255
+ assertGepaOptimizationConfig(config);
2256
+ config = snapshotGepaOptimizationConfig(config);
2257
+ const name = config.name ?? defaultGepaMethodName(config.recipe);
2258
+ return {
2259
+ name,
2260
+ async optimize(input) {
2261
+ const ctx = await prepareGepaRun(config, input, name);
2262
+ const { result, outputDir, modelProxy } = await runBridge(ctx);
2263
+ return buildGepaOutcome(ctx, result, outputDir, modelProxy, await recordGepaSearchHistory(ctx, result, verifyBridgeResult(ctx, result)), gepaCosts(ctx, result, modelProxy));
2126
2264
  }
2127
2265
  };
2128
2266
  }
2129
2267
  //#endregion
2268
+ //#region src/campaign/presets/comparison-controls.ts
2269
+ /** Admission checks for the shared controls of a method comparison. */
2270
+ function assertComparisonControls(opts, seed, resamples, confidence) {
2271
+ if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
2272
+ if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
2273
+ if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
2274
+ if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
2275
+ const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
2276
+ if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
2277
+ if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
2278
+ try {
2279
+ surfaceContentHash(opts.baselineSurface);
2280
+ } catch (cause) {
2281
+ throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
2282
+ }
2283
+ const judgeNames = /* @__PURE__ */ new Set();
2284
+ for (const judge of opts.judges) {
2285
+ if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
2286
+ if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
2287
+ judgeNames.add(judge.name);
2288
+ const dimensionKeys = /* @__PURE__ */ new Set();
2289
+ for (const dimension of judge.dimensions) {
2290
+ if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
2291
+ if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
2292
+ dimensionKeys.add(dimension.key);
2293
+ }
2294
+ }
2295
+ if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
2296
+ if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
2297
+ if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
2298
+ assertConfidence(confidence);
2299
+ const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
2300
+ if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
2301
+ if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
2302
+ if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
2303
+ if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
2304
+ if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
2305
+ if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
2306
+ }
2307
+ function assertConfidence(confidence) {
2308
+ if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
2309
+ }
2310
+ function minimumBootstrapResamples(confidence, comparisonCount) {
2311
+ const exact = 2 * comparisonCount / (1 - confidence);
2312
+ return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
2313
+ }
2314
+ //#endregion
2130
2315
  //#region src/campaign/presets/compare-optimization-methods.ts
2131
2316
  /**
2132
2317
  * Compare optimization methods on shared train, selection, and test data.
@@ -2140,16 +2325,21 @@ function optimizerAccounting(tokenUsage, proposerCostUsd) {
2140
2325
  /**
2141
2326
  * Compare complete optimization methods on disjoint train, selection, and final test data.
2142
2327
  */
2143
- async function compareOptimizationMethods(opts) {
2144
- opts = {
2145
- ...opts,
2146
- methods: opts.methods.map((method) => ({ ...method })),
2147
- trainScenarios: structuredClone(opts.trainScenarios),
2148
- selectionScenarios: structuredClone(opts.selectionScenarios),
2149
- testScenarios: structuredClone(opts.testScenarios),
2150
- judges: opts.judges.map(captureJudge),
2151
- claim: opts.claim && defineEvaluationClaim(opts.claim),
2152
- finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence)
2328
+ async function compareOptimizationMethods(rawOpts) {
2329
+ return comparisonResult(pairwiseAgainstBest(scoreMethods(await measureWinners(await reserveTestMeasurement(await optimizeAllMethods(await finalTestScoring(normalizeComparisonOptions({ rawOpts }))))))));
2330
+ }
2331
+ /** The captured options, the derived statistics settings, the run directory and the cost ledger. */
2332
+ function normalizeComparisonOptions(prev) {
2333
+ const { rawOpts } = prev;
2334
+ const opts = {
2335
+ ...rawOpts,
2336
+ methods: rawOpts.methods.map((method) => ({ ...method })),
2337
+ trainScenarios: structuredClone(rawOpts.trainScenarios),
2338
+ selectionScenarios: structuredClone(rawOpts.selectionScenarios),
2339
+ testScenarios: structuredClone(rawOpts.testScenarios),
2340
+ judges: rawOpts.judges.map(captureJudge),
2341
+ claim: rawOpts.claim && defineEvaluationClaim(rawOpts.claim),
2342
+ finalEvidence: rawOpts.finalEvidence && captureFinalEvidencePolicy(rawOpts.finalEvidence)
2153
2343
  };
2154
2344
  assertOptimizationMethods(opts.methods);
2155
2345
  assertComparisonPartitions(opts);
@@ -2174,6 +2364,28 @@ async function compareOptimizationMethods(opts) {
2174
2364
  runDir: `${resolvedRunDir}/cost`,
2175
2365
  costCeilingUsd: opts.costCeiling
2176
2366
  });
2367
+ return {
2368
+ ...prev,
2369
+ opts,
2370
+ searchHistoryPolicy,
2371
+ seed,
2372
+ confidence,
2373
+ optimizationConcurrency,
2374
+ comparisonCount,
2375
+ intervalConfidence,
2376
+ minimumResamples,
2377
+ resamples,
2378
+ evidence,
2379
+ evidenceBySurface,
2380
+ storage,
2381
+ resolvedRunDir,
2382
+ baselineSurface,
2383
+ costLedger
2384
+ };
2385
+ }
2386
+ /** Scoring a surface on the final test set, and aligning the scores to the designed scenarios. */
2387
+ async function finalTestScoring(prev) {
2388
+ const { opts, seed, intervalConfidence, resamples, evidence, evidenceBySurface, storage, resolvedRunDir, costLedger } = prev;
2177
2389
  const scoreOnTest = async (surface, tag, costPhase) => {
2178
2390
  const measuredSurface = structuredClone(surface);
2179
2391
  const campaign = await runCampaign({
@@ -2227,6 +2439,20 @@ async function compareOptimizationMethods(opts) {
2227
2439
  if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
2228
2440
  return scenarioIds.map((id) => byScenario[id]);
2229
2441
  };
2442
+ return {
2443
+ ...prev,
2444
+ scoreOnTest,
2445
+ scenarioIds,
2446
+ unitByScenario,
2447
+ units,
2448
+ aggregateUnits,
2449
+ decisionOptions,
2450
+ align
2451
+ };
2452
+ }
2453
+ /** Reserve final evidence, then finish every method before the first final-test call. */
2454
+ async function optimizeAllMethods(prev) {
2455
+ const { opts, seed, optimizationConcurrency, storage, resolvedRunDir, baselineSurface, costLedger } = prev;
2230
2456
  if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
2231
2457
  if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
2232
2458
  const optimizationOwner = new AbortController();
@@ -2258,7 +2484,16 @@ async function compareOptimizationMethods(opts) {
2258
2484
  label: `method '${result.name}'`,
2259
2485
  cost: result.cost
2260
2486
  }))).totalCostUsd, opts.costCeiling, "optimization");
2261
- const testCostPhase = `compareOptimizationMethods.test:${contentHash({
2487
+ return {
2488
+ ...prev,
2489
+ optimizationOwner,
2490
+ optimized
2491
+ };
2492
+ }
2493
+ /** The identity of the final test and the exposure of its evidence. */
2494
+ async function reserveTestMeasurement(prev) {
2495
+ const { opts, seed, baselineSurface, optimized } = prev;
2496
+ const testCostIdentity = contentHash({
2262
2497
  baseline: surfaceContentHash(baselineSurface),
2263
2498
  winners: optimized.map((winner) => ({
2264
2499
  name: winner.name,
@@ -2276,8 +2511,19 @@ async function compareOptimizationMethods(opts) {
2276
2511
  dispatch: callerDispatchRef(opts),
2277
2512
  seed,
2278
2513
  reps: opts.reps ?? 1
2279
- })}`;
2514
+ });
2515
+ const testCostPhase = `compareOptimizationMethods.test:${testCostIdentity}`;
2280
2516
  const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
2517
+ return {
2518
+ ...prev,
2519
+ testCostIdentity,
2520
+ testCostPhase,
2521
+ finalEvidence
2522
+ };
2523
+ }
2524
+ /** One final-test measurement per distinct surface. */
2525
+ async function measureWinners(prev) {
2526
+ const { baselineSurface, scoreOnTest, align, optimized, testCostPhase } = prev;
2281
2527
  const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
2282
2528
  const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
2283
2529
  const winners = [];
@@ -2293,6 +2539,16 @@ async function compareOptimizationMethods(opts) {
2293
2539
  arr
2294
2540
  });
2295
2541
  }
2542
+ return {
2543
+ ...prev,
2544
+ baselineArr,
2545
+ testScoresBySurface,
2546
+ winners
2547
+ };
2548
+ }
2549
+ /** Per-method scores, ranked by lift with cost breaking complete ties. */
2550
+ function scoreMethods(prev) {
2551
+ const { evidence, evidenceBySurface, baselineSurface, scenarioIds, units, aggregateUnits, decisionOptions, baselineArr, winners } = prev;
2296
2552
  const scores = winners.map((w) => {
2297
2553
  const baselineUnits = aggregateUnits(baselineArr);
2298
2554
  const winnerUnits = aggregateUnits(w.arr);
@@ -2353,6 +2609,15 @@ async function compareOptimizationMethods(opts) {
2353
2609
  s.rank = i + 1;
2354
2610
  });
2355
2611
  const best = scores[0];
2612
+ return {
2613
+ ...prev,
2614
+ scores,
2615
+ best
2616
+ };
2617
+ }
2618
+ /** The best method against every other, in both directions. */
2619
+ function pairwiseAgainstBest(prev) {
2620
+ const { aggregateUnits, decisionOptions, winners, scores, best } = prev;
2356
2621
  const byName = new Map(winners.map((w) => [w.name, w]));
2357
2622
  const bestArr = aggregateUnits(byName.get(best.name).arr);
2358
2623
  const pairwise = scores.slice(1).map((other) => {
@@ -2370,6 +2635,16 @@ async function compareOptimizationMethods(opts) {
2370
2635
  decision
2371
2636
  };
2372
2637
  });
2638
+ return {
2639
+ ...prev,
2640
+ byName,
2641
+ bestArr,
2642
+ pairwise
2643
+ };
2644
+ }
2645
+ /** Total cost against the ceiling, then the comparison record. */
2646
+ function comparisonResult(prev) {
2647
+ const { opts, searchHistoryPolicy, seed, confidence, comparisonCount, intervalConfidence, resamples, costLedger, scenarioIds, units, optimized, testCostPhase, finalEvidence, scores, best, pairwise } = prev;
2373
2648
  const optimizationCost = combineComparisonCosts(scores.map((score) => ({
2374
2649
  label: `method '${score.name}'`,
2375
2650
  cost: score.optimizationCost
@@ -2431,43 +2706,6 @@ function assertOptimizationMethods(methods) {
2431
2706
  pathOwners.set(pathKey, method.name);
2432
2707
  }
2433
2708
  }
2434
- function assertComparisonControls(opts, seed, resamples, confidence) {
2435
- if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
2436
- if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
2437
- if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
2438
- if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
2439
- const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
2440
- if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
2441
- if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
2442
- try {
2443
- surfaceContentHash(opts.baselineSurface);
2444
- } catch (cause) {
2445
- throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
2446
- }
2447
- const judgeNames = /* @__PURE__ */ new Set();
2448
- for (const judge of opts.judges) {
2449
- if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
2450
- if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
2451
- judgeNames.add(judge.name);
2452
- const dimensionKeys = /* @__PURE__ */ new Set();
2453
- for (const dimension of judge.dimensions) {
2454
- if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
2455
- if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
2456
- dimensionKeys.add(dimension.key);
2457
- }
2458
- }
2459
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
2460
- if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
2461
- if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
2462
- assertConfidence(confidence);
2463
- const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
2464
- if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
2465
- if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
2466
- if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
2467
- if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
2468
- if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
2469
- if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
2470
- }
2471
2709
  function assertComparisonPartitions(opts) {
2472
2710
  const partitions = [
2473
2711
  {
@@ -2508,13 +2746,6 @@ function assertComparisonPartitions(opts) {
2508
2746
  function slug(name) {
2509
2747
  return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
2510
2748
  }
2511
- function assertConfidence(confidence) {
2512
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
2513
- }
2514
- function minimumBootstrapResamples(confidence, comparisonCount) {
2515
- const exact = 2 * comparisonCount / (1 - confidence);
2516
- return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
2517
- }
2518
2749
  function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
2519
2750
  const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
2520
2751
  const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
@@ -2626,298 +2857,347 @@ function assertSkillOptBridgeOutput(result, name, maxCandidateChars, maxEvaluati
2626
2857
  function skillOptOptimizationMethod(config) {
2627
2858
  assertSkillOptOptimizationConfig(config);
2628
2859
  config = snapshotSkillOptOptimizationConfig(config);
2629
- const name = config.name ?? "skillopt";
2860
+ const snapshot = config;
2861
+ const name = snapshot.name ?? "skillopt";
2630
2862
  return {
2631
2863
  name,
2632
- async optimize(input) {
2633
- const signal = input.runOptions.signal;
2634
- signal?.throwIfAborted();
2635
- if (typeof input.baselineSurface !== "string") throw new Error(`${name}: SkillOpt requires a string baselineSurface`);
2636
- const started = Date.now();
2637
- const maxCandidateChars = config.maxCandidateChars ?? 2e5;
2638
- const maxEvidenceChars = config.maxEvidenceChars ?? 1e5;
2639
- const storage = input.runOptions.storage ?? fsCampaignStorage();
2640
- const runDir = `${input.runDir}/skillopt`;
2641
- storage.ensureDir(runDir);
2642
- const costLedger = input.costLedger;
2643
- const attemptId = randomBytes(16).toString("hex");
2644
- const resume = config.resume ?? "never";
2645
- const bridgeRunner = config.runner ? {
2646
- ...config.runner,
2647
- env: removeCredentialEnvironment(config.runner.env ?? {})
2648
- } : void 0;
2649
- const runtimeIdentity = await inspectExternalOptimizerRuntime({
2650
- label: name,
2651
- package: "skillopt",
2652
- module: "agent_eval_rpc.skillopt_bridge",
2653
- ...bridgeRunner ? { runner: bridgeRunner } : {},
2654
- timeoutMs: config.timeoutMs ?? 36e5,
2655
- ...signal ? { signal } : {}
2656
- });
2657
- const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario));
2658
- const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario));
2659
- const runMaterial = {
2660
- optimizer: "skillopt",
2661
- runtime: runtimeIdentity,
2662
- method: name,
2663
- evaluationId: config.evaluationId,
2664
- dispatchRef: input.runOptions.dispatchRef ?? null,
2665
- seed: input.seed,
2666
- trainer: snapshotJson(config.trainer, "SkillOpt run settings"),
2667
- objective: config.objective,
2668
- background: config.background ?? "",
2669
- optimizerModel: {
2670
- model: config.optimizer.model,
2671
- callRef: config.optimizer.callRef,
2672
- budget: config.optimizer.budget
2673
- },
2674
- seedCandidate: input.baselineSurface,
2675
- trainSet,
2676
- selectionSet,
2677
- maxEvaluations: config.maxEvaluations,
2678
- hardScoreThreshold: config.hardScoreThreshold ?? 1,
2679
- maxCandidateChars,
2680
- maxEvidenceChars,
2681
- evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
2682
- runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.skillopt_bridge")
2683
- };
2684
- const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
2685
- const runId = externalOptimizerRunKey({
2686
- material: runMaterial,
2687
- attemptId,
2688
- resumeEnabled: resume !== "never"
2689
- });
2690
- const runBudget = openExternalOptimizerRunBudget({
2691
- storage,
2692
- runDir,
2693
- runKey: runId,
2694
- attemptId,
2695
- maxEvaluations: config.maxEvaluations
2696
- });
2697
- const observationLog = openExternalOptimizerObservationLog({
2698
- storage,
2699
- path: `${runDir}/observations-${attemptId}.jsonl`
2700
- });
2701
- const executionLog = openExternalOptimizerExecutionLog({
2702
- storage,
2703
- path: `${runDir}/model-executions-${attemptId}.jsonl`
2704
- });
2705
- const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "SkillOpt bridge");
2706
- const evaluate = createExternalTextEvaluator({
2707
- input,
2708
- label: "SkillOpt bridge",
2709
- runDir,
2710
- compatibleRunId: runId,
2711
- costPhase: "skillopt.external-evaluation",
2712
- costTags: runBudget.attemptTags,
2713
- costLedger,
2714
- scenarioById,
2715
- maxCandidateChars,
2716
- maxEvidenceChars,
2717
- describeArtifact: config.describeArtifact
2718
- });
2719
- let attemptEvaluationCount = 0;
2720
- const callback = await startExternalOptimizerCallback({
2721
- token: randomBytes(32).toString("hex"),
2722
- maxEvaluations: config.maxEvaluations,
2723
- acceptEvaluation: () => {
2724
- if (runBudget.acceptEvaluation() === void 0) return void 0;
2725
- attemptEvaluationCount += 1;
2726
- return attemptEvaluationCount;
2727
- },
2728
- evaluate,
2729
- observe: observationLog.observe,
2730
- ...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
2731
- ...signal ? { signal } : {}
2732
- });
2733
- const runnerEnv = bridgeRunner?.env ?? {};
2734
- let activeModelProxy;
2735
- const closeResources = () => closeExternalOptimizerResources({
2736
- label: name,
2737
- callback,
2738
- ...activeModelProxy ? { modelProxy: activeModelProxy } : {}
2739
- });
2740
- const { result, outputDir, modelProxy } = await runWithCleanup({
2741
- label: `${name} optimizer resources`,
2742
- run: async () => {
2743
- const priorOptimizerUsage = costLedger.summary({
2744
- phase: "skillopt.optimizer-model",
2745
- tags: runBudget.runTags
2746
- });
2747
- assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
2748
- const modelProxy = await startExternalOptimizerModelProxy({
2749
- call: config.optimizer.call,
2750
- callRef: config.optimizer.callRef,
2751
- recordExecution: executionLog.observe,
2752
- model: config.optimizer.model,
2753
- budget: config.optimizer.budget,
2754
- ...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
2755
- costLedger,
2756
- phase: "skillopt.optimizer-model",
2757
- actor: name,
2758
- tags: { ...runBudget.attemptTags },
2759
- initialUsage: {
2760
- requests: priorOptimizerUsage.totalCalls,
2761
- ...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
2762
- },
2763
- ...signal ? { signal } : {}
2764
- });
2765
- activeModelProxy = modelProxy;
2766
- const outputDir = `${runDir}/external`;
2767
- await mkdir(outputDir, { recursive: true });
2768
- return {
2769
- result: await runExternalOptimizerProcess({
2770
- label: "SkillOpt bridge",
2771
- tempPrefix: "agent-eval-skillopt-",
2772
- module: "agent_eval_rpc.skillopt_bridge",
2773
- input: {
2774
- attemptId,
2775
- compatibleRunId,
2776
- runId,
2777
- runtimeIdentity,
2778
- resume,
2779
- evaluationId: config.evaluationId,
2780
- seed: input.seed,
2781
- callbackUrl: callback.url,
2782
- callbackToken: callback.token,
2783
- objective: config.objective,
2784
- ...config.background ? { background: config.background } : {},
2785
- trainer: config.trainer,
2786
- optimizerModel: config.optimizer.model,
2787
- modelBudget: config.optimizer.budget,
2788
- seedCandidate: input.baselineSurface,
2789
- trainSet,
2790
- selectionSet,
2791
- maxEvaluations: config.maxEvaluations,
2792
- hardScoreThreshold: config.hardScoreThreshold ?? 1,
2793
- maxCandidateChars,
2794
- maxEvidenceChars,
2795
- outputDir
2796
- },
2797
- runner: {
2798
- ...bridgeRunner,
2799
- env: {
2800
- ...removeCredentialEnvironment(runnerEnv),
2801
- OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
2802
- OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
2803
- OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
2804
- OPTIMIZER_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
2805
- TARGET_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
2806
- TARGET_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
2807
- OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
2808
- OPENAI_COMPATIBLE_MAX_TOKENS: String(config.optimizer.budget.maxOutputTokensPerRequest),
2809
- OPTIMIZER_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
2810
- OPTIMIZER_OPENAI_COMPATIBLE_MAX_TOKENS: String(config.optimizer.budget.maxOutputTokensPerRequest),
2811
- TARGET_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
2812
- TARGET_OPENAI_COMPATIBLE_MAX_TOKENS: String(config.optimizer.budget.maxOutputTokensPerRequest)
2813
- }
2814
- },
2815
- timeoutMs: config.timeoutMs ?? 36e5,
2816
- ...signal ? { signal } : {}
2817
- }),
2818
- outputDir,
2819
- modelProxy
2820
- };
2821
- },
2822
- cleanup: closeResources
2823
- });
2824
- signal?.throwIfAborted();
2825
- assertSkillOptBridgeOutput(result, name, maxCandidateChars, config.maxEvaluations);
2826
- assertExternalOptimizerRunBinding({
2827
- label: name,
2828
- runtime: runtimeIdentity,
2829
- returnedSource: result.upstream,
2830
- compatibleRunId,
2831
- runId,
2832
- returnedRunId: result.runId,
2833
- resume,
2834
- resumed: result.resumed
2835
- });
2836
- if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: SkillOpt reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
2837
- const evaluationCost = costFromLedgerSummary(costLedger.summary({
2838
- phase: "skillopt.external-evaluation",
2839
- tags: runBudget.runTags
2840
- }));
2841
- const optimizerUsage = costLedger.summary({
2864
+ optimize: (input) => optimizeWithSkillOpt(snapshot, name, input)
2865
+ };
2866
+ }
2867
+ async function optimizeWithSkillOpt(config, name, input) {
2868
+ const signal = input.runOptions.signal;
2869
+ signal?.throwIfAborted();
2870
+ if (typeof input.baselineSurface !== "string") throw new Error(`${name}: SkillOpt requires a string baselineSurface`);
2871
+ const context = await prepareSkillOptRun(config, name, input);
2872
+ const { result, outputDir, modelProxy } = await runSkillOptBridge(context);
2873
+ signal?.throwIfAborted();
2874
+ return finishSkillOptRun(context, result, outputDir, modelProxy);
2875
+ }
2876
+ /** Resolve identities, open the run's budget and logs, and start the evaluation callback. */
2877
+ async function prepareSkillOptRun(config, name, input) {
2878
+ const signal = input.runOptions.signal;
2879
+ const started = Date.now();
2880
+ const maxCandidateChars = config.maxCandidateChars ?? 2e5;
2881
+ const maxEvidenceChars = config.maxEvidenceChars ?? 1e5;
2882
+ const storage = input.runOptions.storage ?? fsCampaignStorage();
2883
+ const runDir = `${input.runDir}/skillopt`;
2884
+ storage.ensureDir(runDir);
2885
+ const costLedger = input.costLedger;
2886
+ const attemptId = randomBytes(16).toString("hex");
2887
+ const resume = config.resume ?? "never";
2888
+ const bridgeRunner = config.runner ? {
2889
+ ...config.runner,
2890
+ env: removeCredentialEnvironment(config.runner.env ?? {})
2891
+ } : void 0;
2892
+ const runtimeIdentity = await inspectExternalOptimizerRuntime({
2893
+ label: name,
2894
+ package: "skillopt",
2895
+ module: "agent_eval_rpc.skillopt_bridge",
2896
+ ...bridgeRunner ? { runner: bridgeRunner } : {},
2897
+ timeoutMs: config.timeoutMs ?? 36e5,
2898
+ ...signal ? { signal } : {}
2899
+ });
2900
+ const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario));
2901
+ const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario));
2902
+ const runMaterial = {
2903
+ optimizer: "skillopt",
2904
+ runtime: runtimeIdentity,
2905
+ method: name,
2906
+ evaluationId: config.evaluationId,
2907
+ dispatchRef: input.runOptions.dispatchRef ?? null,
2908
+ seed: input.seed,
2909
+ trainer: snapshotJson(config.trainer, "SkillOpt run settings"),
2910
+ objective: config.objective,
2911
+ background: config.background ?? "",
2912
+ optimizerModel: {
2913
+ model: config.optimizer.model,
2914
+ callRef: config.optimizer.callRef,
2915
+ budget: config.optimizer.budget
2916
+ },
2917
+ seedCandidate: input.baselineSurface,
2918
+ trainSet,
2919
+ selectionSet,
2920
+ maxEvaluations: config.maxEvaluations,
2921
+ hardScoreThreshold: config.hardScoreThreshold ?? 1,
2922
+ maxCandidateChars,
2923
+ maxEvidenceChars,
2924
+ evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
2925
+ runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.skillopt_bridge")
2926
+ };
2927
+ const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
2928
+ const runId = externalOptimizerRunKey({
2929
+ material: runMaterial,
2930
+ attemptId,
2931
+ resumeEnabled: resume !== "never"
2932
+ });
2933
+ const runBudget = openExternalOptimizerRunBudget({
2934
+ storage,
2935
+ runDir,
2936
+ runKey: runId,
2937
+ attemptId,
2938
+ maxEvaluations: config.maxEvaluations
2939
+ });
2940
+ const observationLog = openExternalOptimizerObservationLog({
2941
+ storage,
2942
+ path: `${runDir}/observations-${attemptId}.jsonl`
2943
+ });
2944
+ const executionLog = openExternalOptimizerExecutionLog({
2945
+ storage,
2946
+ path: `${runDir}/model-executions-${attemptId}.jsonl`
2947
+ });
2948
+ const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "SkillOpt bridge");
2949
+ const evaluate = createExternalTextEvaluator({
2950
+ input,
2951
+ label: "SkillOpt bridge",
2952
+ runDir,
2953
+ compatibleRunId: runId,
2954
+ costPhase: "skillopt.external-evaluation",
2955
+ costTags: runBudget.attemptTags,
2956
+ costLedger,
2957
+ scenarioById,
2958
+ maxCandidateChars,
2959
+ maxEvidenceChars,
2960
+ describeArtifact: config.describeArtifact
2961
+ });
2962
+ let attemptEvaluationCount = 0;
2963
+ return {
2964
+ config,
2965
+ input,
2966
+ name,
2967
+ signal,
2968
+ started,
2969
+ maxCandidateChars,
2970
+ maxEvidenceChars,
2971
+ storage,
2972
+ runDir,
2973
+ attemptId,
2974
+ resume,
2975
+ bridgeRunner,
2976
+ runtimeIdentity,
2977
+ trainSet,
2978
+ selectionSet,
2979
+ compatibleRunId,
2980
+ runId,
2981
+ runBudget,
2982
+ observationLog,
2983
+ executionLog,
2984
+ callback: await startExternalOptimizerCallback({
2985
+ token: randomBytes(32).toString("hex"),
2986
+ maxEvaluations: config.maxEvaluations,
2987
+ acceptEvaluation: () => {
2988
+ if (runBudget.acceptEvaluation() === void 0) return void 0;
2989
+ attemptEvaluationCount += 1;
2990
+ return attemptEvaluationCount;
2991
+ },
2992
+ evaluate,
2993
+ observe: observationLog.observe,
2994
+ ...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
2995
+ ...signal ? { signal } : {}
2996
+ })
2997
+ };
2998
+ }
2999
+ /** Start the metered model proxy and run the Python bridge, always closing the callback and proxy. */
3000
+ async function runSkillOptBridge(context) {
3001
+ const { config, input, name, signal, runDir, runBudget, callback } = context;
3002
+ const costLedger = input.costLedger;
3003
+ const runnerEnv = context.bridgeRunner?.env ?? {};
3004
+ let activeModelProxy;
3005
+ const closeResources = () => closeExternalOptimizerResources({
3006
+ label: name,
3007
+ callback,
3008
+ ...activeModelProxy ? { modelProxy: activeModelProxy } : {}
3009
+ });
3010
+ return runWithCleanup({
3011
+ label: `${name} optimizer resources`,
3012
+ run: async () => {
3013
+ const priorOptimizerUsage = costLedger.summary({
2842
3014
  phase: "skillopt.optimizer-model",
2843
3015
  tags: runBudget.runTags
2844
3016
  });
2845
- const optimizerReceipts = costLedger.list({
3017
+ assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
3018
+ const modelProxy = await startExternalOptimizerModelProxy({
3019
+ call: config.optimizer.call,
3020
+ callRef: config.optimizer.callRef,
3021
+ recordExecution: context.executionLog.observe,
3022
+ model: config.optimizer.model,
3023
+ budget: config.optimizer.budget,
3024
+ ...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
3025
+ costLedger,
2846
3026
  phase: "skillopt.optimizer-model",
2847
- tags: runBudget.runTags
3027
+ actor: name,
3028
+ tags: { ...runBudget.attemptTags },
3029
+ initialUsage: {
3030
+ requests: priorOptimizerUsage.totalCalls,
3031
+ ...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
3032
+ },
3033
+ ...signal ? { signal } : {}
2848
3034
  });
2849
- const optimizerCost = costFromLedgerSummary(optimizerUsage);
2850
- modelProxy.assertExecutionComplete();
2851
- assertExternalOptimizerCompletionCount(result.tokenUsage, modelProxy.requestAttempts(), modelProxy.successfulCompletions(), name, "SkillOpt");
2852
- const tokenUsage = optimizationTokenUsageFromSummary(optimizerUsage, optimizerReceipts);
2853
- const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
2854
- const combinedCost = combineComparisonCosts([{
2855
- label: "evaluation",
2856
- cost: evaluationCost
2857
- }, {
2858
- label: "optimizer model",
2859
- cost: optimizerCost
2860
- }]);
2861
- const searchHistory = config.searchLedger ? await recordSkillOptSearch({
2862
- name,
2863
- path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
2864
- searchId: runId,
2865
- identity: config.searchLedger.identity,
2866
- storage,
2867
- seed: input.seed,
2868
- baselineSurface: input.baselineSurface,
2869
- trainScenarios: input.trainScenarios,
2870
- selectionScenarios: input.selectionScenarios,
2871
- evaluationLimit: config.maxEvaluations,
2872
- observations: readExternalOptimizerObservationArtifact({
2873
- summary: observationLog.summary(),
2874
- storage
2875
- }),
2876
- selected: result.bestCandidate,
2877
- generationAccounting: {
2878
- tokens: optimizerUsage.usageComplete ? {
2879
- status: "known",
2880
- inputTokens: optimizerUsage.inputTokens,
2881
- outputTokens: optimizerUsage.outputTokens,
2882
- cachedTokens: optimizerUsage.cachedTokens
2883
- } : {
2884
- status: "unknown",
2885
- reason: "an optimizer call reported no token usage"
2886
- },
2887
- cost: optimizerCost.accountingComplete ? {
2888
- status: "known",
2889
- usd: optimizerCost.totalCostUsd,
2890
- source: "provider"
2891
- } : {
2892
- status: "unknown",
2893
- knownLowerBoundUsd: optimizerCost.totalCostUsd,
2894
- reason: optimizerCost.incompleteReasons.join("; ") || "optimizer spend incomplete"
2895
- }
2896
- }
2897
- }) : void 0;
3035
+ activeModelProxy = modelProxy;
3036
+ const outputDir = `${runDir}/external`;
3037
+ await mkdir(outputDir, { recursive: true });
2898
3038
  return {
2899
- winnerSurface: result.bestCandidate,
2900
- ...searchHistory ? { searchHistory } : {},
2901
- cost: combinedCost,
2902
- durationMs: Date.now() - started,
2903
- provenance: {
2904
- ...runtime,
2905
- optimizerModel: config.optimizer.model,
2906
- optimizerCallRef: config.optimizer.callRef,
2907
- compatibleRunId,
2908
- runId,
2909
- resumed: result.resumed,
2910
- evaluationCount: runBudget.acceptedEvaluations(),
2911
- artifactDir: outputDir,
2912
- ...tokenUsage ? { tokenUsage } : {},
2913
- observations: observationLog.summary(),
2914
- modelExecutions: executionLog.summary()
2915
- }
3039
+ result: await runExternalOptimizerProcess({
3040
+ label: "SkillOpt bridge",
3041
+ tempPrefix: "agent-eval-skillopt-",
3042
+ module: "agent_eval_rpc.skillopt_bridge",
3043
+ input: {
3044
+ attemptId: context.attemptId,
3045
+ compatibleRunId: context.compatibleRunId,
3046
+ runId: context.runId,
3047
+ runtimeIdentity: context.runtimeIdentity,
3048
+ resume: context.resume,
3049
+ evaluationId: config.evaluationId,
3050
+ seed: input.seed,
3051
+ callbackUrl: callback.url,
3052
+ callbackToken: callback.token,
3053
+ objective: config.objective,
3054
+ ...config.background ? { background: config.background } : {},
3055
+ trainer: config.trainer,
3056
+ optimizerModel: config.optimizer.model,
3057
+ modelBudget: config.optimizer.budget,
3058
+ seedCandidate: input.baselineSurface,
3059
+ trainSet: context.trainSet,
3060
+ selectionSet: context.selectionSet,
3061
+ maxEvaluations: config.maxEvaluations,
3062
+ hardScoreThreshold: config.hardScoreThreshold ?? 1,
3063
+ maxCandidateChars: context.maxCandidateChars,
3064
+ maxEvidenceChars: context.maxEvidenceChars,
3065
+ outputDir
3066
+ },
3067
+ runner: {
3068
+ ...context.bridgeRunner,
3069
+ env: {
3070
+ ...removeCredentialEnvironment(runnerEnv),
3071
+ ...skillOptModelEnvironment(config, modelProxy)
3072
+ }
3073
+ },
3074
+ timeoutMs: config.timeoutMs ?? 36e5,
3075
+ ...signal ? { signal } : {}
3076
+ }),
3077
+ outputDir,
3078
+ modelProxy
2916
3079
  };
3080
+ },
3081
+ cleanup: closeResources
3082
+ });
3083
+ }
3084
+ /** Point every OpenAI-compatible client the bridge uses at the metered model proxy. */
3085
+ function skillOptModelEnvironment(config, modelProxy) {
3086
+ const maxTokens = String(config.optimizer.budget.maxOutputTokensPerRequest);
3087
+ return {
3088
+ OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
3089
+ OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
3090
+ OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
3091
+ OPTIMIZER_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
3092
+ TARGET_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
3093
+ TARGET_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
3094
+ OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
3095
+ OPENAI_COMPATIBLE_MAX_TOKENS: maxTokens,
3096
+ OPTIMIZER_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
3097
+ OPTIMIZER_OPENAI_COMPATIBLE_MAX_TOKENS: maxTokens,
3098
+ TARGET_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
3099
+ TARGET_OPENAI_COMPATIBLE_MAX_TOKENS: maxTokens
3100
+ };
3101
+ }
3102
+ /** Validate the bridge output against the run, total its spend, and build the method result. */
3103
+ async function finishSkillOptRun(context, result, outputDir, modelProxy) {
3104
+ const { config, input, name, runBudget, callback, observationLog, executionLog } = context;
3105
+ const { runtimeIdentity, compatibleRunId, runId } = context;
3106
+ const costLedger = input.costLedger;
3107
+ assertSkillOptBridgeOutput(result, name, context.maxCandidateChars, config.maxEvaluations);
3108
+ assertExternalOptimizerRunBinding({
3109
+ label: name,
3110
+ runtime: runtimeIdentity,
3111
+ returnedSource: result.upstream,
3112
+ compatibleRunId,
3113
+ runId,
3114
+ returnedRunId: result.runId,
3115
+ resume: context.resume,
3116
+ resumed: result.resumed
3117
+ });
3118
+ if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: SkillOpt reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
3119
+ const evaluationCost = costFromLedgerSummary(costLedger.summary({
3120
+ phase: "skillopt.external-evaluation",
3121
+ tags: runBudget.runTags
3122
+ }));
3123
+ const optimizerUsage = costLedger.summary({
3124
+ phase: "skillopt.optimizer-model",
3125
+ tags: runBudget.runTags
3126
+ });
3127
+ const optimizerReceipts = costLedger.list({
3128
+ phase: "skillopt.optimizer-model",
3129
+ tags: runBudget.runTags
3130
+ });
3131
+ const optimizerCost = costFromLedgerSummary(optimizerUsage);
3132
+ modelProxy.assertExecutionComplete();
3133
+ assertExternalOptimizerCompletionCount(result.tokenUsage, modelProxy.requestAttempts(), modelProxy.successfulCompletions(), name, "SkillOpt");
3134
+ const tokenUsage = optimizationTokenUsageFromSummary(optimizerUsage, optimizerReceipts);
3135
+ const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
3136
+ const combinedCost = combineComparisonCosts([{
3137
+ label: "evaluation",
3138
+ cost: evaluationCost
3139
+ }, {
3140
+ label: "optimizer model",
3141
+ cost: optimizerCost
3142
+ }]);
3143
+ const searchHistory = config.searchLedger ? await recordSkillOptSearch({
3144
+ name,
3145
+ path: config.searchLedger.path ?? `${context.runDir}/search-ledger.jsonl`,
3146
+ searchId: runId,
3147
+ identity: config.searchLedger.identity,
3148
+ storage: context.storage,
3149
+ seed: input.seed,
3150
+ baselineSurface: input.baselineSurface,
3151
+ trainScenarios: input.trainScenarios,
3152
+ selectionScenarios: input.selectionScenarios,
3153
+ evaluationLimit: config.maxEvaluations,
3154
+ observations: readExternalOptimizerObservationArtifact({
3155
+ summary: observationLog.summary(),
3156
+ storage: context.storage
3157
+ }),
3158
+ selected: result.bestCandidate,
3159
+ generationAccounting: {
3160
+ tokens: optimizerUsage.usageComplete ? {
3161
+ status: "known",
3162
+ inputTokens: optimizerUsage.inputTokens,
3163
+ outputTokens: optimizerUsage.outputTokens,
3164
+ cachedTokens: optimizerUsage.cachedTokens
3165
+ } : {
3166
+ status: "unknown",
3167
+ reason: "an optimizer call reported no token usage"
3168
+ },
3169
+ cost: optimizerCost.accountingComplete ? {
3170
+ status: "known",
3171
+ usd: optimizerCost.totalCostUsd,
3172
+ source: "provider"
3173
+ } : {
3174
+ status: "unknown",
3175
+ knownLowerBoundUsd: optimizerCost.totalCostUsd,
3176
+ reason: optimizerCost.incompleteReasons.join("; ") || "optimizer spend incomplete"
3177
+ }
3178
+ }
3179
+ }) : void 0;
3180
+ return {
3181
+ winnerSurface: result.bestCandidate,
3182
+ ...searchHistory ? { searchHistory } : {},
3183
+ cost: combinedCost,
3184
+ durationMs: Date.now() - context.started,
3185
+ provenance: {
3186
+ ...runtime,
3187
+ optimizerModel: config.optimizer.model,
3188
+ optimizerCallRef: config.optimizer.callRef,
3189
+ compatibleRunId,
3190
+ runId,
3191
+ resumed: result.resumed,
3192
+ evaluationCount: runBudget.acceptedEvaluations(),
3193
+ artifactDir: outputDir,
3194
+ ...tokenUsage ? { tokenUsage } : {},
3195
+ observations: observationLog.summary(),
3196
+ modelExecutions: executionLog.summary()
2917
3197
  }
2918
3198
  };
2919
3199
  }
2920
3200
  //#endregion
2921
3201
  export { importExternalEvaluations as a, composeGate as c, readExternalOptimizerObservationArtifact as d, externalSurface as i, externalTextOptimizationMethod as l, compareOptimizationMethods as n, importGepaPopulation as o, gepaOptimizationMethod as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
2922
3202
 
2923
- //# sourceMappingURL=skillopt-optimization-method-kyL94-ri.js.map
3203
+ //# sourceMappingURL=skillopt-optimization-method-5dlex7MJ.js.map