@tangle-network/agent-eval 0.217.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -1
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +9 -9
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DyeNcenW.d.ts → analyze-runs-DjlZOVXx.d.ts} +2 -2
- package/dist/{analyze-runs-DyeNcenW.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{benchmark-command-xQK8nOI6.js → benchmark-command-BupdHRyf.js} +661 -562
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +10 -10
- package/dist/{campaign-CJGkxAQ6.js → campaign-CGvUTnmc.js} +312 -281
- package/dist/{campaign-CJGkxAQ6.js.map → campaign-CGvUTnmc.js.map} +1 -1
- package/dist/{campaign-evidence-CQYw8_0H.js → campaign-evidence-DunzcJGv.js} +2 -2
- package/dist/{campaign-evidence-CQYw8_0H.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{chat-client-ldqxYYTN.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-ldqxYYTN.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-DavzexSp.js → chat-json-call-Dm9c-b41.js} +2 -2
- package/dist/{chat-json-call-DavzexSp.js.map → chat-json-call-Dm9c-b41.js.map} +1 -1
- package/dist/cli.js +10 -10
- package/dist/{completion-verifier-B_jw6C9p.js → completion-verifier-DbLVf3i8.js} +4 -4
- package/dist/{completion-verifier-B_jw6C9p.js.map → completion-verifier-DbLVf3i8.js.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +90 -10
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-ClYjq3QE.js → default-registry-BJ-Pq6yL.js} +309 -277
- package/dist/{default-registry-ClYjq3QE.js.map → default-registry-BJ-Pq6yL.js.map} +1 -1
- package/dist/{define-agent-eval-BDxNlP3D.js → define-agent-eval-BGdFKlrc.js} +165 -105
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-BIkH_gAR.d.ts → define-agent-eval-BlH10zEw.d.ts} +6 -6
- package/dist/{define-agent-eval-BIkH_gAR.d.ts.map → define-agent-eval-BlH10zEw.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +1 -1
- package/dist/{dspy-rlm-engine-DhdCr1Dc.js → dspy-rlm-engine-CSMwudLO.js} +2 -2
- package/dist/{dspy-rlm-engine-DhdCr1Dc.js.map → dspy-rlm-engine-CSMwudLO.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-jTjC_QQD.d.ts → evaluation-BOZ38vOH.d.ts} +2 -2
- package/dist/{evaluation-jTjC_QQD.d.ts.map → evaluation-BOZ38vOH.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +110 -71
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-Cl6IeNAu.js → exporters-DWvM16LE.js} +2 -2
- package/dist/{exporters-Cl6IeNAu.js.map → exporters-DWvM16LE.js.map} +1 -1
- package/dist/{external-optimizer-process-KVFGvYX1.js → external-optimizer-process-C3Wm7Vhn.js} +2 -2
- package/dist/{external-optimizer-process-KVFGvYX1.js.map → external-optimizer-process-C3Wm7Vhn.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-XIHBOSKt.js → external-optimizer-subprocess-B5gdLRZv.js} +3 -3
- package/dist/{external-optimizer-subprocess-XIHBOSKt.js.map → external-optimizer-subprocess-B5gdLRZv.js.map} +1 -1
- package/dist/{findings-store-BMVQtEsm.js → findings-store-BtTzPL0d.js} +5 -5
- package/dist/{findings-store-BMVQtEsm.js.map → findings-store-BtTzPL0d.js.map} +1 -1
- package/dist/{hf-dataset-D9f7-oX7.js → hf-dataset-CKxIJQba.js} +3 -3
- package/dist/{hf-dataset-D9f7-oX7.js.map → hf-dataset-CKxIJQba.js.map} +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-ZuWx52WG.d.ts → index-B16FNdW5.d.ts} +34 -17
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-9jJ2fKSX.d.ts → index-CalAsXMR.d.ts} +7 -5
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BiT6RtNH.d.ts → index-DbMOXYnP.d.ts} +6 -6
- package/dist/{index-BiT6RtNH.d.ts.map → index-DbMOXYnP.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +611 -688
- package/dist/index.js.map +1 -1
- package/dist/{integrity-DOd9Q86O.js → integrity-C8oUeB0g.js} +322 -197
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{journal-CR4QikYp.js → journal-C7om8Gum.js} +48 -38
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-DrJhbms9.js → ledger-core-CAxUdDk0.js} +2 -2
- package/dist/{ledger-core-DrJhbms9.js.map → ledger-core-CAxUdDk0.js.map} +1 -1
- package/dist/{llm-client-BshCREn6.js → llm-client-_Y6ic-tf.js} +238 -248
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-C8jMTNNG.js → llm-judge-Djxh-Uhs.js} +646 -458
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/{matrix-DHe-dwVJ.d.ts → matrix-CCSBna4q.d.ts} +2 -2
- package/dist/{matrix-DHe-dwVJ.d.ts.map → matrix-CCSBna4q.d.ts.map} +1 -1
- package/dist/{mint-zszyYlpA.js → mint-CbdwjTPW.js} +2 -2
- package/dist/{mint-zszyYlpA.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{pareto-BzsxidFD.js → pareto-CTv8vozZ.js} +4 -4
- package/dist/{pareto-BzsxidFD.js.map → pareto-CTv8vozZ.js.map} +1 -1
- package/dist/{promotion-gate-DaY95yU6.d.ts → promotion-gate-CxEHRsZW.d.ts} +2 -2
- package/dist/{promotion-gate-DaY95yU6.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{proposal-findings-BbWidvUk.js → proposal-findings-n-5WBbEx.js} +2 -2
- package/dist/{proposal-findings-BbWidvUk.js.map → proposal-findings-n-5WBbEx.js.map} +1 -1
- package/dist/{render-C-w2EZ6p.js → render-Cs34y0iX.js} +404 -208
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +2 -2
- package/dist/{researcher-0ABVIdGa.d.ts → researcher-GU1tV4dn.d.ts} +2 -2
- package/dist/{researcher-0ABVIdGa.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/rl.d.ts +2 -2
- package/dist/rl.js +4 -4
- package/dist/rollout/index.js +5 -5
- package/dist/{rollout-D6C50mfA.js → rollout-Dd1ucT_M.js} +5 -5
- package/dist/{rollout-D6C50mfA.js.map → rollout-Dd1ucT_M.js.map} +1 -1
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-DxcKTzbi.js → schema-CvhfLjtZ.js} +77 -47
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-DxqjBxA-.d.ts → search-kernel-p9JB3SmL.d.ts} +3 -3
- package/dist/{search-kernel-DxqjBxA-.d.ts.map → search-kernel-p9JB3SmL.d.ts.map} +1 -1
- package/dist/{search-ledger-TUvB5gWH.js → search-ledger-ClpN4I3e.js} +4 -4
- package/dist/{search-ledger-TUvB5gWH.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-UKCO2r8a.js → search-ledger-file-Bd5Tioq6.js} +3 -3
- package/dist/{search-ledger-file-UKCO2r8a.js.map → search-ledger-file-Bd5Tioq6.js.map} +1 -1
- package/dist/{search-shipper-Cw2z3-td.js → search-shipper-Bg2HW3Uv.js} +3 -3
- package/dist/{search-shipper-Cw2z3-td.js.map → search-shipper-Bg2HW3Uv.js.map} +1 -1
- package/dist/{search-summary-DQIvJsSB.js → search-summary-CrBuvVnf.js} +3 -3
- package/dist/{search-summary-DQIvJsSB.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{server-DoVbqqtJ.js → server-C6M0qR8h.js} +2 -2
- package/dist/{server-DoVbqqtJ.js.map → server-C6M0qR8h.js.map} +1 -1
- package/dist/{single-run-lock-U58yf8N6.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-kyL94-ri.js → skillopt-optimization-method-5dlex7MJ.js} +1339 -1059
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{statistical-heldout-C9AbgREN.d.ts → statistical-heldout-BHfLZHrm.d.ts} +2 -2
- package/dist/{statistical-heldout-C9AbgREN.d.ts.map → statistical-heldout-BHfLZHrm.d.ts.map} +1 -1
- package/dist/{summary-report-DM7tlml1.d.ts → summary-report-CkbRcSuu.d.ts} +16 -1
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +3 -3
- package/dist/{systemone-Bp_Ac0p1.d.ts → systemone-BMzDJb1i.d.ts} +3 -3
- package/dist/{systemone-Bp_Ac0p1.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone.d.ts +2 -2
- package/dist/{terminal-record-78g6E4vh.js → terminal-record-D4Rv-j1W.js} +241 -193
- package/dist/{terminal-record-78g6E4vh.js.map → terminal-record-D4Rv-j1W.js.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +4 -1
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +834 -824
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-BGRglkKd.js → tree-B5e1uAeL.js} +122 -14
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-DtMbFw6U.d.ts → types-BKCzzmit.d.ts} +2 -2
- package/dist/{types-DtMbFw6U.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-D8GhJoHk.d.ts → types-DtkO1G9L.d.ts} +2 -2
- package/dist/{types-D8GhJoHk.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/design/statistics-decisions.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/profile-json-identity.md +1 -2
- package/docs/public-api.md +27 -16
- package/package.json +7 -2
- package/dist/benchmark-command-xQK8nOI6.js.map +0 -1
- package/dist/define-agent-eval-BDxNlP3D.js.map +0 -1
- package/dist/eval-campaign-BLEfBGbA.js +0 -343
- package/dist/eval-campaign-BLEfBGbA.js.map +0 -1
- package/dist/index-9jJ2fKSX.d.ts.map +0 -1
- package/dist/index-ZuWx52WG.d.ts.map +0 -1
- package/dist/integrity-DOd9Q86O.js.map +0 -1
- package/dist/journal-CR4QikYp.js.map +0 -1
- package/dist/llm-client-BshCREn6.js.map +0 -1
- package/dist/llm-judge-C8jMTNNG.js.map +0 -1
- package/dist/render-C-w2EZ6p.js.map +0 -1
- package/dist/schema-DxcKTzbi.js.map +0 -1
- package/dist/single-run-lock-U58yf8N6.js.map +0 -1
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +0 -1
- package/dist/summary-report-DM7tlml1.d.ts.map +0 -1
- package/dist/tree-BGRglkKd.js.map +0 -1
package/dist/{skillopt-optimization-method-kyL94-ri.js → skillopt-optimization-method-5dlex7MJ.js}
RENAMED
|
@@ -3,17 +3,17 @@ import { t as mean$1 } from "./mean-BbA_JCzW.js";
|
|
|
3
3
|
import { t as decidePairedPromotion } from "./paired-promotion-decision-D0SHb-ZL.js";
|
|
4
4
|
import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
|
|
5
5
|
import { n as contentHash, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
|
|
6
|
-
import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-
|
|
7
|
-
import { D as surfaceContentHash, I as assertCompleteCampaign, P as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-
|
|
8
|
-
import { a as mapConcurrent } from "./journal-
|
|
6
|
+
import { E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, Q as runCampaign, S as developmentClaim, T as surfaceNode, X as reserveFinalEvidence, Y as exposeFinalEvidence, c as assertGepaCandidatePopulationSummary, d as costFromLedgerSummary, et as resolveRunDir, f as optimizationTokenUsageFromSummary, l as readGepaCandidatePopulationArtifact, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, w as surfaceDiff, x as SearchRecorder, z as captureJudge } from "./llm-judge-Djxh-Uhs.js";
|
|
7
|
+
import { D as surfaceContentHash, I as assertCompleteCampaign, P as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-DunzcJGv.js";
|
|
8
|
+
import { a as mapConcurrent } from "./journal-C7om8Gum.js";
|
|
9
9
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
10
|
-
import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-
|
|
10
|
+
import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-B5gdLRZv.js";
|
|
11
11
|
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
|
|
12
|
-
import { n as openSearchLedger } from "./search-ledger-
|
|
13
|
-
import { t as acquireSingleRunLock } from "./single-run-lock-
|
|
12
|
+
import { n as openSearchLedger } from "./search-ledger-ClpN4I3e.js";
|
|
13
|
+
import { t as acquireSingleRunLock } from "./single-run-lock-Dekd8_Wv.js";
|
|
14
14
|
import { n as isRecord } from "./json-BhdsJaOQ.js";
|
|
15
|
-
import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-
|
|
16
|
-
import { t as startExternalOptimizerCallback } from "./external-optimizer-process-
|
|
15
|
+
import { _ as combineAbortSignals, d as assertServedModelPolicy } from "./llm-client-_Y6ic-tf.js";
|
|
16
|
+
import { t as startExternalOptimizerCallback } from "./external-optimizer-process-C3Wm7Vhn.js";
|
|
17
17
|
import { createHash, randomBytes } from "node:crypto";
|
|
18
18
|
import { mkdir } from "node:fs/promises";
|
|
19
19
|
import { isDeepStrictEqual } from "node:util";
|
|
@@ -567,270 +567,341 @@ function externalTextOptimizationMethod(config) {
|
|
|
567
567
|
const snapshot = snapshotExternalTextOptimizationConfig(config);
|
|
568
568
|
return {
|
|
569
569
|
name: snapshot.name,
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
let paidCallsAtCompletion = 0;
|
|
722
|
-
let runSettled = false;
|
|
723
|
-
const runPromise = Promise.resolve().then(() => snapshot.run(Object.freeze({
|
|
724
|
-
runId,
|
|
725
|
-
name: snapshot.name,
|
|
726
|
-
objective: snapshot.objective,
|
|
727
|
-
evaluationId: snapshot.evaluationId,
|
|
728
|
-
...snapshot.background ? { background: snapshot.background } : {},
|
|
729
|
-
seedCandidate: cloneExternalTextCandidate(seedCandidate),
|
|
730
|
-
trainSet,
|
|
731
|
-
selectionSet,
|
|
732
|
-
maxEvaluations: snapshot.maxEvaluations,
|
|
733
|
-
seed: input.seed,
|
|
734
|
-
stateDir,
|
|
735
|
-
restoreRequested,
|
|
736
|
-
artifactDir,
|
|
737
|
-
signal: controller.signal,
|
|
738
|
-
cost: optimizerCostMeter,
|
|
739
|
-
evaluate
|
|
740
|
-
})));
|
|
741
|
-
runPromise.then(() => {
|
|
742
|
-
runSettled = true;
|
|
743
|
-
}, () => {
|
|
744
|
-
runSettled = true;
|
|
745
|
-
});
|
|
746
|
-
try {
|
|
747
|
-
result = await raceWithAbort(runPromise, controller.signal);
|
|
748
|
-
outstandingAtCompletion = activeEvaluations.size;
|
|
749
|
-
paidCallsAtCompletion = optimizerLedger.summary().pendingCalls;
|
|
750
|
-
} catch (error) {
|
|
751
|
-
runError = error;
|
|
752
|
-
} finally {
|
|
753
|
-
acceptingEvaluations = false;
|
|
754
|
-
clearTimeout(timeout);
|
|
755
|
-
}
|
|
756
|
-
await Promise.allSettled([...activeEvaluations]);
|
|
757
|
-
const paidCallsSettled = await optimizerLedger.waitForIdle({ timeoutMs: 5e3 });
|
|
758
|
-
if (runError !== void 0) {
|
|
759
|
-
if ((!runSettled || !paidCallsSettled) && lock) {
|
|
760
|
-
releaseLock = false;
|
|
761
|
-
Promise.allSettled([runPromise, optimizerLedger.waitForIdle({ timeoutMs: MAX_TIMER_DELAY_MS$2 })]).then(() => lock.release());
|
|
762
|
-
}
|
|
763
|
-
throw runError;
|
|
764
|
-
}
|
|
765
|
-
if (!paidCallsSettled) {
|
|
766
|
-
if (lock) {
|
|
767
|
-
releaseLock = false;
|
|
768
|
-
Promise.allSettled([runPromise, optimizerLedger.waitForIdle({ timeoutMs: MAX_TIMER_DELAY_MS$2 })]).then(() => lock.release());
|
|
769
|
-
}
|
|
770
|
-
throw new Error(`${snapshot.name}: optimizer paid calls did not settle`);
|
|
771
|
-
}
|
|
772
|
-
if (paidCallsAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${paidCallsAtCompletion} outstanding paid call(s); await every context.cost.runPaidCall()`);
|
|
773
|
-
if (outstandingAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${outstandingAtCompletion} outstanding evaluation(s); await every evaluate() call`);
|
|
774
|
-
assertExternalTextOptimizerResult(result, snapshot.name, maxCandidateChars, expectsComponents);
|
|
775
|
-
if (result.resumed !== restoreRequested) throw new Error(`${snapshot.name}: optimizer reported resumed=${String(result.resumed)} but restoreRequested=${String(restoreRequested)}`);
|
|
776
|
-
const evaluationCost = costFromLedgerSummary(input.costLedger.summary({
|
|
777
|
-
phase: evaluationCostPhase,
|
|
778
|
-
tags: runBudget.runTags
|
|
779
|
-
}));
|
|
780
|
-
const optimizerFilter = {
|
|
781
|
-
phase: optimizerCostPhase,
|
|
782
|
-
tags: runBudget.runTags
|
|
783
|
-
};
|
|
784
|
-
const optimizerReceipts = input.costLedger.list(optimizerFilter);
|
|
785
|
-
const optimizerSummary = input.costLedger.summary(optimizerFilter);
|
|
786
|
-
const optimizerCost = costFromLedgerSummary(optimizerSummary);
|
|
787
|
-
assertExternalCostAccounting(result.costAccounting, optimizerSummary.totalCalls, snapshot.name);
|
|
788
|
-
const externalCostReason = result.costAccounting.kind === "external" ? result.costAccounting.reason : void 0;
|
|
789
|
-
const tokenUsage = optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts);
|
|
790
|
-
if (resume !== "never") appendExternalOptimizerRunManifestEvent({
|
|
791
|
-
storage,
|
|
792
|
-
path: manifestPath,
|
|
793
|
-
revision: manifest.revision,
|
|
794
|
-
event: {
|
|
795
|
-
runId,
|
|
796
|
-
attemptId,
|
|
797
|
-
status: "completed"
|
|
570
|
+
optimize: (input) => optimizeExternalText(snapshot, input)
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
const EVALUATION_COST_PHASE = "external.evaluation";
|
|
574
|
+
const OPTIMIZER_COST_PHASE = "external.optimizer";
|
|
575
|
+
async function optimizeExternalText(snapshot, input) {
|
|
576
|
+
const run = await prepareExternalRun(snapshot, input);
|
|
577
|
+
try {
|
|
578
|
+
const { manifest, restoreRequested } = restoreRunManifest(run);
|
|
579
|
+
const accounts = openRunAccounts(run);
|
|
580
|
+
const session = startOptimizerSession(run, accounts, restoreRequested);
|
|
581
|
+
return finishRun(run, accounts, session, manifest, await awaitOptimizerResult(run, accounts, session, restoreRequested));
|
|
582
|
+
} finally {
|
|
583
|
+
if (run.releaseLock) run.lock?.release();
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
function describeExternalScenarios(scenarios, snapshot, maxEvidenceChars) {
|
|
587
|
+
return Object.freeze(scenarios.map((scenario) => describeExternalScenario(scenario, snapshot.name, maxEvidenceChars, snapshot.describeScenario)));
|
|
588
|
+
}
|
|
589
|
+
/** Run identity, scenario descriptions and directories, then the resume lock. */
|
|
590
|
+
async function prepareExternalRun(snapshot, input) {
|
|
591
|
+
const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
|
|
592
|
+
const maxCandidateChars = snapshot.maxCandidateChars ?? DEFAULT_MAX_CANDIDATE_CHARS;
|
|
593
|
+
const maxEvidenceChars = snapshot.maxEvidenceChars ?? DEFAULT_MAX_EVIDENCE_CHARS;
|
|
594
|
+
const attemptId = randomBytes(16).toString("hex");
|
|
595
|
+
const storage = input.runOptions.storage ?? fsCampaignStorage();
|
|
596
|
+
const resume = snapshot.resume ?? "never";
|
|
597
|
+
const trainSet = describeExternalScenarios(input.trainScenarios, snapshot, maxEvidenceChars);
|
|
598
|
+
const selectionSet = describeExternalScenarios(input.selectionScenarios, snapshot, maxEvidenceChars);
|
|
599
|
+
const methodDir = `${input.runDir}/external/${safePathComponent(snapshot.name)}`;
|
|
600
|
+
const runId = externalOptimizerRunKey({
|
|
601
|
+
material: {
|
|
602
|
+
optimizer: {
|
|
603
|
+
kind: snapshot.source.kind,
|
|
604
|
+
package: snapshot.source.package,
|
|
605
|
+
version: snapshot.source.version,
|
|
606
|
+
...snapshot.source.sourceUrl ? { sourceUrl: snapshot.source.sourceUrl } : {},
|
|
607
|
+
...snapshot.source.revision ? { revision: snapshot.source.revision } : {}
|
|
608
|
+
},
|
|
609
|
+
method: snapshot.name,
|
|
610
|
+
evaluationId: snapshot.evaluationId,
|
|
611
|
+
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
612
|
+
objective: snapshot.objective,
|
|
613
|
+
background: snapshot.background ?? "",
|
|
614
|
+
seed: input.seed,
|
|
615
|
+
seedCandidate,
|
|
616
|
+
trainSet,
|
|
617
|
+
selectionSet,
|
|
618
|
+
maxEvaluations: snapshot.maxEvaluations,
|
|
619
|
+
maxOptimizerCostUsd: snapshot.maxOptimizerCostUsd,
|
|
620
|
+
maxCandidateChars,
|
|
621
|
+
maxEvidenceChars
|
|
622
|
+
},
|
|
623
|
+
attemptId,
|
|
624
|
+
resumeEnabled: resume !== "never"
|
|
625
|
+
});
|
|
626
|
+
const stateDir = `${methodDir}/state/${runId}`;
|
|
627
|
+
const artifactDir = `${methodDir}/attempts/${attemptId}`;
|
|
628
|
+
storage.ensureDir(stateDir);
|
|
629
|
+
storage.ensureDir(artifactDir);
|
|
630
|
+
if (resume !== "never") await mkdir(stateDir, { recursive: true });
|
|
631
|
+
const lock = resume === "never" ? void 0 : acquireSingleRunLock({
|
|
632
|
+
lockPath: `${stateDir}.lock`,
|
|
633
|
+
releaseOnExit: true
|
|
634
|
+
});
|
|
635
|
+
return {
|
|
636
|
+
snapshot,
|
|
637
|
+
input,
|
|
638
|
+
seedCandidate,
|
|
639
|
+
expectsComponents: typeof seedCandidate !== "string",
|
|
640
|
+
maxCandidateChars,
|
|
641
|
+
maxEvidenceChars,
|
|
642
|
+
attemptId,
|
|
643
|
+
storage,
|
|
644
|
+
resume,
|
|
645
|
+
trainSet,
|
|
646
|
+
selectionSet,
|
|
647
|
+
methodDir,
|
|
648
|
+
runId,
|
|
649
|
+
stateDir,
|
|
650
|
+
artifactDir,
|
|
651
|
+
manifestPath: `${stateDir}/run-manifest.jsonl`,
|
|
652
|
+
lock,
|
|
653
|
+
releaseLock: true
|
|
654
|
+
};
|
|
655
|
+
}
|
|
656
|
+
/** Reads prior state when resuming, and records this attempt as partial. */
|
|
657
|
+
function restoreRunManifest(run) {
|
|
658
|
+
const { snapshot, storage, manifestPath, runId, attemptId, resume } = run;
|
|
659
|
+
let manifest = resume === "never" ? void 0 : readExternalOptimizerRunManifest(storage.read(manifestPath), storage.exists(manifestPath), manifestPath, runId);
|
|
660
|
+
const restoreRequested = manifest !== void 0;
|
|
661
|
+
if (resume === "required" && !restoreRequested) throw new Error(`${snapshot.name}: no compatible run state is available to resume`);
|
|
662
|
+
if (resume === "never" && restoreRequested) throw new Error(`${snapshot.name}: fresh optimizer run unexpectedly found prior state`);
|
|
663
|
+
if (resume !== "never") manifest = appendExternalOptimizerRunManifestEvent({
|
|
664
|
+
storage,
|
|
665
|
+
path: manifestPath,
|
|
666
|
+
revision: manifest?.revision ?? 0,
|
|
667
|
+
event: {
|
|
668
|
+
runId,
|
|
669
|
+
attemptId,
|
|
670
|
+
status: "partial"
|
|
671
|
+
}
|
|
672
|
+
});
|
|
673
|
+
return {
|
|
674
|
+
manifest,
|
|
675
|
+
restoreRequested
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
function openRunAccounts(run) {
|
|
679
|
+
const { snapshot, storage, methodDir, runId, attemptId, stateDir } = run;
|
|
680
|
+
return {
|
|
681
|
+
runBudget: openExternalOptimizerRunBudget({
|
|
682
|
+
storage,
|
|
683
|
+
runDir: methodDir,
|
|
684
|
+
runKey: runId,
|
|
685
|
+
attemptId,
|
|
686
|
+
maxEvaluations: snapshot.maxEvaluations
|
|
687
|
+
}),
|
|
688
|
+
optimizerLedger: createRunCostLedger({
|
|
689
|
+
storage,
|
|
690
|
+
runDir: `${stateDir}/cost`,
|
|
691
|
+
costCeilingUsd: snapshot.maxOptimizerCostUsd
|
|
692
|
+
})
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
/** Optimizer paid calls, charged to the optimizer's own limit and mirrored into the method's account. */
|
|
696
|
+
function createOptimizerCostMeter(run, accounts, controller) {
|
|
697
|
+
const { snapshot, input } = run;
|
|
698
|
+
const { runBudget, optimizerLedger } = accounts;
|
|
699
|
+
return Object.freeze({ async runPaidCall(request) {
|
|
700
|
+
const call = {
|
|
701
|
+
methodResult: void 0,
|
|
702
|
+
providerStarted: false
|
|
703
|
+
};
|
|
704
|
+
const subLimitResult = await optimizerLedger.runPaidCall({
|
|
705
|
+
...request,
|
|
706
|
+
channel: request.channel ?? "optimizer",
|
|
707
|
+
phase: OPTIMIZER_COST_PHASE,
|
|
708
|
+
tags: runBudget.attemptTags,
|
|
709
|
+
signal: request.signal ? AbortSignal.any([controller.signal, request.signal]) : controller.signal,
|
|
710
|
+
execute: async (signal, callId) => {
|
|
711
|
+
call.methodResult = await input.costLedger.runPaidCall({
|
|
712
|
+
...request,
|
|
713
|
+
callId,
|
|
714
|
+
channel: request.channel ?? "optimizer",
|
|
715
|
+
phase: OPTIMIZER_COST_PHASE,
|
|
716
|
+
tags: runBudget.attemptTags,
|
|
717
|
+
signal,
|
|
718
|
+
execute: (methodSignal, methodCallId) => {
|
|
719
|
+
call.providerStarted = true;
|
|
720
|
+
return request.execute(methodSignal, methodCallId);
|
|
798
721
|
}
|
|
799
722
|
});
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
...measuredCost,
|
|
811
|
-
...externalCostReason ? { costProvenance: {
|
|
812
|
-
kind: "uncaptured",
|
|
813
|
-
usd: null
|
|
814
|
-
} } : {},
|
|
815
|
-
accountingComplete: measuredCost.accountingComplete && externalCostReason === void 0,
|
|
816
|
-
incompleteReasons: [...measuredCost.incompleteReasons, ...externalCostReason ? [`optimizer: external spend is not observed: ${externalCostReason}`] : []]
|
|
817
|
-
},
|
|
818
|
-
durationMs: Date.now() - started,
|
|
819
|
-
provenance: {
|
|
820
|
-
source: {
|
|
821
|
-
...snapshot.source,
|
|
822
|
-
evidence: "declared"
|
|
823
|
-
},
|
|
824
|
-
runId,
|
|
825
|
-
resumed: result.resumed,
|
|
826
|
-
evaluationCount: runBudget.acceptedEvaluations(),
|
|
827
|
-
artifactDir,
|
|
828
|
-
...tokenUsage ? { tokenUsage } : {}
|
|
829
|
-
}
|
|
830
|
-
};
|
|
831
|
-
} finally {
|
|
832
|
-
if (releaseLock) lock?.release();
|
|
723
|
+
if (!call.methodResult.succeeded) throw new Error(`${snapshot.name}: method cost account rejected optimizer call`, { cause: call.methodResult.error });
|
|
724
|
+
return call.methodResult.value;
|
|
725
|
+
},
|
|
726
|
+
receipt: () => {
|
|
727
|
+
if (!call.methodResult?.succeeded) throw new Error(`${snapshot.name}: optimizer call completed without a receipt`);
|
|
728
|
+
return mirroredCostReceipt(call.methodResult.receipt);
|
|
729
|
+
},
|
|
730
|
+
receiptFromError: () => {
|
|
731
|
+
if (call.methodResult?.receipt) return mirroredCostReceipt(call.methodResult.receipt);
|
|
732
|
+
if (!call.providerStarted) return noChargeReceipt(modelBeforeExecution(request));
|
|
833
733
|
}
|
|
734
|
+
});
|
|
735
|
+
if (call.methodResult === void 0) return subLimitResult;
|
|
736
|
+
if (!call.methodResult.succeeded) return call.methodResult;
|
|
737
|
+
return subLimitResult.succeeded ? call.methodResult : subLimitResult;
|
|
738
|
+
} });
|
|
739
|
+
}
|
|
740
|
+
/** Starts the optimizer with its evaluator, cost meter, timeout and frozen context. */
|
|
741
|
+
function startOptimizerSession(run, accounts, restoreRequested) {
|
|
742
|
+
const { snapshot, input, runId, stateDir, artifactDir } = run;
|
|
743
|
+
const { runBudget } = accounts;
|
|
744
|
+
const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, `${snapshot.name} optimizer`);
|
|
745
|
+
const score = createExternalTextEvaluator({
|
|
746
|
+
input,
|
|
747
|
+
label: `${snapshot.name} optimizer`,
|
|
748
|
+
runDir: artifactDir,
|
|
749
|
+
compatibleRunId: runId,
|
|
750
|
+
costPhase: EVALUATION_COST_PHASE,
|
|
751
|
+
costTags: runBudget.attemptTags,
|
|
752
|
+
costLedger: input.costLedger,
|
|
753
|
+
scenarioById,
|
|
754
|
+
maxCandidateChars: run.maxCandidateChars,
|
|
755
|
+
maxEvidenceChars: run.maxEvidenceChars,
|
|
756
|
+
describeArtifact: snapshot.describeArtifact
|
|
757
|
+
});
|
|
758
|
+
const controller = new AbortController();
|
|
759
|
+
const activeEvaluations = /* @__PURE__ */ new Set();
|
|
760
|
+
const state = {
|
|
761
|
+
acceptingEvaluations: true,
|
|
762
|
+
runSettled: false
|
|
763
|
+
};
|
|
764
|
+
const optimizerCostMeter = createOptimizerCostMeter(run, accounts, controller);
|
|
765
|
+
const evaluate = (request, signal = controller.signal) => {
|
|
766
|
+
if (!state.acceptingEvaluations) throw new Error(`${snapshot.name}: evaluate cannot be called after run() completes`);
|
|
767
|
+
if (runBudget.acceptEvaluation() === void 0) throw new Error(`${snapshot.name}: maxEvaluations limit reached`);
|
|
768
|
+
const combinedSignal = signal === controller.signal ? controller.signal : AbortSignal.any([controller.signal, signal]);
|
|
769
|
+
const pending = score(cloneExternalEvaluationRequest(request), combinedSignal);
|
|
770
|
+
activeEvaluations.add(pending);
|
|
771
|
+
const remove = () => activeEvaluations.delete(pending);
|
|
772
|
+
pending.then(remove, remove);
|
|
773
|
+
return pending;
|
|
774
|
+
};
|
|
775
|
+
const started = Date.now();
|
|
776
|
+
const timeoutMs = snapshot.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
777
|
+
const timeout = setTimeout(() => controller.abort(/* @__PURE__ */ new Error(`${snapshot.name}: optimizer exceeded ${timeoutMs}ms`)), timeoutMs);
|
|
778
|
+
const runPromise = Promise.resolve().then(() => snapshot.run(Object.freeze({
|
|
779
|
+
runId,
|
|
780
|
+
name: snapshot.name,
|
|
781
|
+
objective: snapshot.objective,
|
|
782
|
+
evaluationId: snapshot.evaluationId,
|
|
783
|
+
...snapshot.background ? { background: snapshot.background } : {},
|
|
784
|
+
seedCandidate: cloneExternalTextCandidate(run.seedCandidate),
|
|
785
|
+
trainSet: run.trainSet,
|
|
786
|
+
selectionSet: run.selectionSet,
|
|
787
|
+
maxEvaluations: snapshot.maxEvaluations,
|
|
788
|
+
seed: input.seed,
|
|
789
|
+
stateDir,
|
|
790
|
+
restoreRequested,
|
|
791
|
+
artifactDir,
|
|
792
|
+
signal: controller.signal,
|
|
793
|
+
cost: optimizerCostMeter,
|
|
794
|
+
evaluate
|
|
795
|
+
})));
|
|
796
|
+
const markSettled = () => {
|
|
797
|
+
state.runSettled = true;
|
|
798
|
+
};
|
|
799
|
+
runPromise.then(markSettled, markSettled);
|
|
800
|
+
return {
|
|
801
|
+
controller,
|
|
802
|
+
activeEvaluations,
|
|
803
|
+
runPromise,
|
|
804
|
+
timeout,
|
|
805
|
+
started,
|
|
806
|
+
state
|
|
807
|
+
};
|
|
808
|
+
}
|
|
809
|
+
/** Waits for the optimizer, then refuses a run that left evaluations or paid calls outstanding. */
|
|
810
|
+
async function awaitOptimizerResult(run, accounts, session, restoreRequested) {
|
|
811
|
+
const { snapshot, lock } = run;
|
|
812
|
+
const { optimizerLedger } = accounts;
|
|
813
|
+
let result;
|
|
814
|
+
let runError;
|
|
815
|
+
let outstandingAtCompletion = 0;
|
|
816
|
+
let paidCallsAtCompletion = 0;
|
|
817
|
+
try {
|
|
818
|
+
result = await raceWithAbort(session.runPromise, session.controller.signal);
|
|
819
|
+
outstandingAtCompletion = session.activeEvaluations.size;
|
|
820
|
+
paidCallsAtCompletion = optimizerLedger.summary().pendingCalls;
|
|
821
|
+
} catch (error) {
|
|
822
|
+
runError = error;
|
|
823
|
+
} finally {
|
|
824
|
+
session.state.acceptingEvaluations = false;
|
|
825
|
+
clearTimeout(session.timeout);
|
|
826
|
+
}
|
|
827
|
+
await Promise.allSettled([...session.activeEvaluations]);
|
|
828
|
+
const paidCallsSettled = await optimizerLedger.waitForIdle({ timeoutMs: 5e3 });
|
|
829
|
+
const handOffLock = () => {
|
|
830
|
+
if (!lock) return;
|
|
831
|
+
run.releaseLock = false;
|
|
832
|
+
Promise.allSettled([session.runPromise, optimizerLedger.waitForIdle({ timeoutMs: MAX_TIMER_DELAY_MS$2 })]).then(() => lock.release());
|
|
833
|
+
};
|
|
834
|
+
if (runError !== void 0) {
|
|
835
|
+
if (!session.state.runSettled || !paidCallsSettled) handOffLock();
|
|
836
|
+
throw runError;
|
|
837
|
+
}
|
|
838
|
+
if (!paidCallsSettled) {
|
|
839
|
+
handOffLock();
|
|
840
|
+
throw new Error(`${snapshot.name}: optimizer paid calls did not settle`);
|
|
841
|
+
}
|
|
842
|
+
if (paidCallsAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${paidCallsAtCompletion} outstanding paid call(s); await every context.cost.runPaidCall()`);
|
|
843
|
+
if (outstandingAtCompletion > 0) throw new Error(`${snapshot.name}: run() completed with ${outstandingAtCompletion} outstanding evaluation(s); await every evaluate() call`);
|
|
844
|
+
assertExternalTextOptimizerResult(result, snapshot.name, run.maxCandidateChars, run.expectsComponents);
|
|
845
|
+
if (result.resumed !== restoreRequested) throw new Error(`${snapshot.name}: optimizer reported resumed=${String(result.resumed)} but restoreRequested=${String(restoreRequested)}`);
|
|
846
|
+
return result;
|
|
847
|
+
}
|
|
848
|
+
/** Measures evaluation and optimizer spend, records completion, and builds the method result. */
|
|
849
|
+
function finishRun(run, accounts, session, manifest, result) {
|
|
850
|
+
const { snapshot, input, storage, runId, attemptId } = run;
|
|
851
|
+
const { runBudget } = accounts;
|
|
852
|
+
const evaluationCost = costFromLedgerSummary(input.costLedger.summary({
|
|
853
|
+
phase: EVALUATION_COST_PHASE,
|
|
854
|
+
tags: runBudget.runTags
|
|
855
|
+
}));
|
|
856
|
+
const optimizerFilter = {
|
|
857
|
+
phase: OPTIMIZER_COST_PHASE,
|
|
858
|
+
tags: runBudget.runTags
|
|
859
|
+
};
|
|
860
|
+
const optimizerReceipts = input.costLedger.list(optimizerFilter);
|
|
861
|
+
const optimizerSummary = input.costLedger.summary(optimizerFilter);
|
|
862
|
+
const optimizerCost = costFromLedgerSummary(optimizerSummary);
|
|
863
|
+
assertExternalCostAccounting(result.costAccounting, optimizerSummary.totalCalls, snapshot.name);
|
|
864
|
+
const externalCostReason = result.costAccounting.kind === "external" ? result.costAccounting.reason : void 0;
|
|
865
|
+
const tokenUsage = optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts);
|
|
866
|
+
if (run.resume !== "never") appendExternalOptimizerRunManifestEvent({
|
|
867
|
+
storage,
|
|
868
|
+
path: run.manifestPath,
|
|
869
|
+
revision: manifest.revision,
|
|
870
|
+
event: {
|
|
871
|
+
runId,
|
|
872
|
+
attemptId,
|
|
873
|
+
status: "completed"
|
|
874
|
+
}
|
|
875
|
+
});
|
|
876
|
+
const measuredCost = combineComparisonCosts([{
|
|
877
|
+
label: "evaluation",
|
|
878
|
+
cost: evaluationCost
|
|
879
|
+
}, {
|
|
880
|
+
label: "optimizer",
|
|
881
|
+
cost: optimizerCost
|
|
882
|
+
}]);
|
|
883
|
+
return {
|
|
884
|
+
winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
|
|
885
|
+
cost: {
|
|
886
|
+
...measuredCost,
|
|
887
|
+
...externalCostReason ? { costProvenance: {
|
|
888
|
+
kind: "uncaptured",
|
|
889
|
+
usd: null
|
|
890
|
+
} } : {},
|
|
891
|
+
accountingComplete: measuredCost.accountingComplete && externalCostReason === void 0,
|
|
892
|
+
incompleteReasons: [...measuredCost.incompleteReasons, ...externalCostReason ? [`optimizer: external spend is not observed: ${externalCostReason}`] : []]
|
|
893
|
+
},
|
|
894
|
+
durationMs: Date.now() - session.started,
|
|
895
|
+
provenance: {
|
|
896
|
+
source: {
|
|
897
|
+
...snapshot.source,
|
|
898
|
+
evidence: "declared"
|
|
899
|
+
},
|
|
900
|
+
runId,
|
|
901
|
+
resumed: result.resumed,
|
|
902
|
+
evaluationCount: runBudget.acceptedEvaluations(),
|
|
903
|
+
artifactDir: run.artifactDir,
|
|
904
|
+
...tokenUsage ? { tokenUsage } : {}
|
|
834
905
|
}
|
|
835
906
|
};
|
|
836
907
|
}
|
|
@@ -1000,30 +1071,6 @@ function heldOutGate(options) {
|
|
|
1000
1071
|
};
|
|
1001
1072
|
}
|
|
1002
1073
|
//#endregion
|
|
1003
|
-
//#region src/campaign/external-optimizer-accounting.ts
|
|
1004
|
-
const TOKEN_USAGE_FIELDS = [
|
|
1005
|
-
"inputTokens",
|
|
1006
|
-
"outputTokens",
|
|
1007
|
-
"totalTokens",
|
|
1008
|
-
"calls"
|
|
1009
|
-
];
|
|
1010
|
-
function assertPriorExternalOptimizerUsage(summary, budget, name) {
|
|
1011
|
-
if (!summary.usageComplete || budget.maxCostUsd !== void 0 && !summary.accountingComplete) throw new Error(`${name}: cannot resume optimizer-model work with incomplete bounded usage`);
|
|
1012
|
-
if (summary.totalCalls > budget.maxRequests || budget.maxCostUsd !== void 0 && summary.totalCostUsd > budget.maxCostUsd + Number.EPSILON) throw new Error(`${name}: prior optimizer-model usage exceeds the configured budget`);
|
|
1013
|
-
}
|
|
1014
|
-
function assertExternalOptimizerTokenUsage(usage, name, optimizer) {
|
|
1015
|
-
if (usage === void 0) return;
|
|
1016
|
-
for (const field of TOKEN_USAGE_FIELDS) if (!Number.isSafeInteger(usage[field]) || usage[field] < 0) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.${field}`);
|
|
1017
|
-
if (usage.totalTokens !== usage.inputTokens + usage.outputTokens) throw new Error(`${name}: ${optimizer} bridge returned inconsistent tokenUsage.totalTokens`);
|
|
1018
|
-
if (usage.requestAttempts !== void 0 && (!Number.isSafeInteger(usage.requestAttempts) || usage.requestAttempts < usage.calls)) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.requestAttempts`);
|
|
1019
|
-
}
|
|
1020
|
-
function assertExternalOptimizerCompletionCount(upstream, requestAttempts, successfulCompletions, name, optimizer) {
|
|
1021
|
-
if (!upstream) throw new Error(`${name}: ${optimizer} did not report optimizer token usage`);
|
|
1022
|
-
if (!Number.isSafeInteger(requestAttempts) || !Number.isSafeInteger(successfulCompletions) || requestAttempts < 0 || successfulCompletions < 0 || successfulCompletions > requestAttempts) throw new Error(`${name}: optimizer model proxy returned invalid request counts`);
|
|
1023
|
-
if (upstream.calls !== successfulCompletions) throw new Error(`${name}: ${optimizer} reported ${upstream.calls} successful model calls but the proxy completed ${successfulCompletions} across ${requestAttempts} attempts`);
|
|
1024
|
-
if (upstream.requestAttempts !== void 0 && upstream.requestAttempts !== requestAttempts) throw new Error(`${name}: ${optimizer} reported ${upstream.requestAttempts} model attempts but the proxy received ${requestAttempts}`);
|
|
1025
|
-
}
|
|
1026
|
-
//#endregion
|
|
1027
1074
|
//#region src/campaign/external-optimizer-run-config.ts
|
|
1028
1075
|
function externalOptimizerRunnerIdentity(runner, module) {
|
|
1029
1076
|
return {
|
|
@@ -1048,72 +1095,6 @@ function snapshotJson(value, label) {
|
|
|
1048
1095
|
return JSON.parse(serialized);
|
|
1049
1096
|
}
|
|
1050
1097
|
//#endregion
|
|
1051
|
-
//#region src/campaign/external-optimizer-source.ts
|
|
1052
|
-
function assertExternalOptimizerPackageSource(value, expectedPackage, name, optimizer) {
|
|
1053
|
-
assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer);
|
|
1054
|
-
assertExternalOptimizerSourceDetails(value, name, optimizer);
|
|
1055
|
-
}
|
|
1056
|
-
function assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer) {
|
|
1057
|
-
if (!isRecord(value) || value.package !== expectedPackage || typeof value.version !== "string" || value.version.length === 0 || value.version !== value.version.trim()) throw new Error(`${name}: ${optimizer} bridge returned invalid upstream package provenance`);
|
|
1058
|
-
}
|
|
1059
|
-
function assertExternalOptimizerSourceDetails(value, name, optimizer) {
|
|
1060
|
-
if (value.sourceUrl !== void 0 && (typeof value.sourceUrl !== "string" || value.sourceUrl.length === 0 || value.sourceUrl !== value.sourceUrl.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceUrl`);
|
|
1061
|
-
if (value.revision !== void 0 && (typeof value.revision !== "string" || value.revision.length === 0 || value.revision !== value.revision.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream revision`);
|
|
1062
|
-
if (value.sourceSha256 !== void 0 && (typeof value.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(value.sourceSha256))) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceSha256`);
|
|
1063
|
-
}
|
|
1064
|
-
function observedExternalOptimizerPackageSource(value) {
|
|
1065
|
-
return {
|
|
1066
|
-
kind: "package",
|
|
1067
|
-
evidence: "observed",
|
|
1068
|
-
package: value.package,
|
|
1069
|
-
version: value.version,
|
|
1070
|
-
...value.sourceUrl ? { sourceUrl: value.sourceUrl } : {},
|
|
1071
|
-
...value.revision ? { revision: value.revision } : {},
|
|
1072
|
-
...value.sourceSha256 ? { sourceSha256: value.sourceSha256 } : {}
|
|
1073
|
-
};
|
|
1074
|
-
}
|
|
1075
|
-
//#endregion
|
|
1076
|
-
//#region src/campaign/external-optimizer-runtime.ts
|
|
1077
|
-
async function inspectExternalOptimizerRuntime(args) {
|
|
1078
|
-
const result = await runExternalOptimizerProcess({
|
|
1079
|
-
label: `${args.label} source inspection`,
|
|
1080
|
-
tempPrefix: `agent-eval-${args.package}-inspect-`,
|
|
1081
|
-
module: args.module,
|
|
1082
|
-
input: {
|
|
1083
|
-
operation: "inspect",
|
|
1084
|
-
engineModules: [...args.engineModules ?? []]
|
|
1085
|
-
},
|
|
1086
|
-
...args.runner ? { runner: args.runner } : {},
|
|
1087
|
-
timeoutMs: args.timeoutMs,
|
|
1088
|
-
...args.signal ? { signal: args.signal } : {}
|
|
1089
|
-
});
|
|
1090
|
-
assertExternalOptimizerRuntimeIdentity(result.runtime, args.package, args.label);
|
|
1091
|
-
return result.runtime;
|
|
1092
|
-
}
|
|
1093
|
-
function observedExternalOptimizerRuntime(runtime) {
|
|
1094
|
-
return {
|
|
1095
|
-
source: observedExternalOptimizerPackageSource(runtime.optimizer),
|
|
1096
|
-
bridge: observedExternalOptimizerPackageSource(runtime.bridge),
|
|
1097
|
-
modules: runtime.engineModules.map((module) => ({ ...module })),
|
|
1098
|
-
python: { ...runtime.python }
|
|
1099
|
-
};
|
|
1100
|
-
}
|
|
1101
|
-
function assertExternalOptimizerRunBinding(args) {
|
|
1102
|
-
if (!isDeepStrictEqual(args.returnedSource, args.runtime.optimizer)) throw new Error(`${args.label}: optimizer package changed after source inspection`);
|
|
1103
|
-
if (args.returnedRunId !== args.runId) throw new Error(`${args.label}: bridge returned a different run ID`);
|
|
1104
|
-
if (args.resume === "never" && args.resumed) throw new Error(`${args.label}: fresh run reported restored state`);
|
|
1105
|
-
if (args.resume === "required" && !args.resumed) throw new Error(`${args.label}: required resume did not restore state`);
|
|
1106
|
-
if (!args.runId.startsWith(args.compatibleRunId)) throw new Error(`${args.label}: run ID is not bound to its compatible identity`);
|
|
1107
|
-
}
|
|
1108
|
-
function assertExternalOptimizerRuntimeIdentity(value, expectedPackage, label) {
|
|
1109
|
-
if (!isRecord(value)) throw new Error(`${label}: source inspection returned no runtime identity`);
|
|
1110
|
-
assertExternalOptimizerPackageSource(value.bridge, "agent-eval-rpc", label, "bridge");
|
|
1111
|
-
assertExternalOptimizerPackageSource(value.optimizer, expectedPackage, label, expectedPackage);
|
|
1112
|
-
if (!value.bridge.sourceSha256 || !value.optimizer.sourceSha256) throw new Error(`${label}: source inspection omitted package source hashes`);
|
|
1113
|
-
if (!isRecord(value.python) || typeof value.python.implementation !== "string" || value.python.implementation.length === 0 || value.python.implementation !== value.python.implementation.trim() || typeof value.python.version !== "string" || value.python.version.length === 0 || value.python.version !== value.python.version.trim()) throw new Error(`${label}: source inspection returned an invalid Python runtime`);
|
|
1114
|
-
if (!Array.isArray(value.engineModules) || value.engineModules.some((module) => !isRecord(module) || typeof module.module !== "string" || module.module.length === 0 || module.module !== module.module.trim() || typeof module.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(module.sourceSha256))) throw new Error(`${label}: source inspection returned invalid engine modules`);
|
|
1115
|
-
}
|
|
1116
|
-
//#endregion
|
|
1117
1098
|
//#region src/campaign/optimizer-model.ts
|
|
1118
1099
|
function assertOptimizerModel(value, label) {
|
|
1119
1100
|
if (!value || typeof value !== "object") throw new Error(`${label} is required`);
|
|
@@ -1311,6 +1292,96 @@ function addEvaluationLimit(total, increment) {
|
|
|
1311
1292
|
return next;
|
|
1312
1293
|
}
|
|
1313
1294
|
//#endregion
|
|
1295
|
+
//#region src/campaign/external-optimizer-accounting.ts
|
|
1296
|
+
const TOKEN_USAGE_FIELDS = [
|
|
1297
|
+
"inputTokens",
|
|
1298
|
+
"outputTokens",
|
|
1299
|
+
"totalTokens",
|
|
1300
|
+
"calls"
|
|
1301
|
+
];
|
|
1302
|
+
function assertPriorExternalOptimizerUsage(summary, budget, name) {
|
|
1303
|
+
if (!summary.usageComplete || budget.maxCostUsd !== void 0 && !summary.accountingComplete) throw new Error(`${name}: cannot resume optimizer-model work with incomplete bounded usage`);
|
|
1304
|
+
if (summary.totalCalls > budget.maxRequests || budget.maxCostUsd !== void 0 && summary.totalCostUsd > budget.maxCostUsd + Number.EPSILON) throw new Error(`${name}: prior optimizer-model usage exceeds the configured budget`);
|
|
1305
|
+
}
|
|
1306
|
+
function assertExternalOptimizerTokenUsage(usage, name, optimizer) {
|
|
1307
|
+
if (usage === void 0) return;
|
|
1308
|
+
for (const field of TOKEN_USAGE_FIELDS) if (!Number.isSafeInteger(usage[field]) || usage[field] < 0) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.${field}`);
|
|
1309
|
+
if (usage.totalTokens !== usage.inputTokens + usage.outputTokens) throw new Error(`${name}: ${optimizer} bridge returned inconsistent tokenUsage.totalTokens`);
|
|
1310
|
+
if (usage.requestAttempts !== void 0 && (!Number.isSafeInteger(usage.requestAttempts) || usage.requestAttempts < usage.calls)) throw new Error(`${name}: ${optimizer} bridge returned invalid tokenUsage.requestAttempts`);
|
|
1311
|
+
}
|
|
1312
|
+
function assertExternalOptimizerCompletionCount(upstream, requestAttempts, successfulCompletions, name, optimizer) {
|
|
1313
|
+
if (!upstream) throw new Error(`${name}: ${optimizer} did not report optimizer token usage`);
|
|
1314
|
+
if (!Number.isSafeInteger(requestAttempts) || !Number.isSafeInteger(successfulCompletions) || requestAttempts < 0 || successfulCompletions < 0 || successfulCompletions > requestAttempts) throw new Error(`${name}: optimizer model proxy returned invalid request counts`);
|
|
1315
|
+
if (upstream.calls !== successfulCompletions) throw new Error(`${name}: ${optimizer} reported ${upstream.calls} successful model calls but the proxy completed ${successfulCompletions} across ${requestAttempts} attempts`);
|
|
1316
|
+
if (upstream.requestAttempts !== void 0 && upstream.requestAttempts !== requestAttempts) throw new Error(`${name}: ${optimizer} reported ${upstream.requestAttempts} model attempts but the proxy received ${requestAttempts}`);
|
|
1317
|
+
}
|
|
1318
|
+
//#endregion
|
|
1319
|
+
//#region src/campaign/external-optimizer-source.ts
|
|
1320
|
+
function assertExternalOptimizerPackageSource(value, expectedPackage, name, optimizer) {
|
|
1321
|
+
assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer);
|
|
1322
|
+
assertExternalOptimizerSourceDetails(value, name, optimizer);
|
|
1323
|
+
}
|
|
1324
|
+
function assertExternalOptimizerPackageIdentity(value, expectedPackage, name, optimizer) {
|
|
1325
|
+
if (!isRecord(value) || value.package !== expectedPackage || typeof value.version !== "string" || value.version.length === 0 || value.version !== value.version.trim()) throw new Error(`${name}: ${optimizer} bridge returned invalid upstream package provenance`);
|
|
1326
|
+
}
|
|
1327
|
+
function assertExternalOptimizerSourceDetails(value, name, optimizer) {
|
|
1328
|
+
if (value.sourceUrl !== void 0 && (typeof value.sourceUrl !== "string" || value.sourceUrl.length === 0 || value.sourceUrl !== value.sourceUrl.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceUrl`);
|
|
1329
|
+
if (value.revision !== void 0 && (typeof value.revision !== "string" || value.revision.length === 0 || value.revision !== value.revision.trim())) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream revision`);
|
|
1330
|
+
if (value.sourceSha256 !== void 0 && (typeof value.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(value.sourceSha256))) throw new Error(`${name}: ${optimizer} bridge returned an invalid upstream sourceSha256`);
|
|
1331
|
+
}
|
|
1332
|
+
function observedExternalOptimizerPackageSource(value) {
|
|
1333
|
+
return {
|
|
1334
|
+
kind: "package",
|
|
1335
|
+
evidence: "observed",
|
|
1336
|
+
package: value.package,
|
|
1337
|
+
version: value.version,
|
|
1338
|
+
...value.sourceUrl ? { sourceUrl: value.sourceUrl } : {},
|
|
1339
|
+
...value.revision ? { revision: value.revision } : {},
|
|
1340
|
+
...value.sourceSha256 ? { sourceSha256: value.sourceSha256 } : {}
|
|
1341
|
+
};
|
|
1342
|
+
}
|
|
1343
|
+
//#endregion
|
|
1344
|
+
//#region src/campaign/external-optimizer-runtime.ts
|
|
1345
|
+
async function inspectExternalOptimizerRuntime(args) {
|
|
1346
|
+
const result = await runExternalOptimizerProcess({
|
|
1347
|
+
label: `${args.label} source inspection`,
|
|
1348
|
+
tempPrefix: `agent-eval-${args.package}-inspect-`,
|
|
1349
|
+
module: args.module,
|
|
1350
|
+
input: {
|
|
1351
|
+
operation: "inspect",
|
|
1352
|
+
engineModules: [...args.engineModules ?? []]
|
|
1353
|
+
},
|
|
1354
|
+
...args.runner ? { runner: args.runner } : {},
|
|
1355
|
+
timeoutMs: args.timeoutMs,
|
|
1356
|
+
...args.signal ? { signal: args.signal } : {}
|
|
1357
|
+
});
|
|
1358
|
+
assertExternalOptimizerRuntimeIdentity(result.runtime, args.package, args.label);
|
|
1359
|
+
return result.runtime;
|
|
1360
|
+
}
|
|
1361
|
+
function observedExternalOptimizerRuntime(runtime) {
|
|
1362
|
+
return {
|
|
1363
|
+
source: observedExternalOptimizerPackageSource(runtime.optimizer),
|
|
1364
|
+
bridge: observedExternalOptimizerPackageSource(runtime.bridge),
|
|
1365
|
+
modules: runtime.engineModules.map((module) => ({ ...module })),
|
|
1366
|
+
python: { ...runtime.python }
|
|
1367
|
+
};
|
|
1368
|
+
}
|
|
1369
|
+
function assertExternalOptimizerRunBinding(args) {
|
|
1370
|
+
if (!isDeepStrictEqual(args.returnedSource, args.runtime.optimizer)) throw new Error(`${args.label}: optimizer package changed after source inspection`);
|
|
1371
|
+
if (args.returnedRunId !== args.runId) throw new Error(`${args.label}: bridge returned a different run ID`);
|
|
1372
|
+
if (args.resume === "never" && args.resumed) throw new Error(`${args.label}: fresh run reported restored state`);
|
|
1373
|
+
if (args.resume === "required" && !args.resumed) throw new Error(`${args.label}: required resume did not restore state`);
|
|
1374
|
+
if (!args.runId.startsWith(args.compatibleRunId)) throw new Error(`${args.label}: run ID is not bound to its compatible identity`);
|
|
1375
|
+
}
|
|
1376
|
+
function assertExternalOptimizerRuntimeIdentity(value, expectedPackage, label) {
|
|
1377
|
+
if (!isRecord(value)) throw new Error(`${label}: source inspection returned no runtime identity`);
|
|
1378
|
+
assertExternalOptimizerPackageSource(value.bridge, "agent-eval-rpc", label, "bridge");
|
|
1379
|
+
assertExternalOptimizerPackageSource(value.optimizer, expectedPackage, label, expectedPackage);
|
|
1380
|
+
if (!value.bridge.sourceSha256 || !value.optimizer.sourceSha256) throw new Error(`${label}: source inspection omitted package source hashes`);
|
|
1381
|
+
if (!isRecord(value.python) || typeof value.python.implementation !== "string" || value.python.implementation.length === 0 || value.python.implementation !== value.python.implementation.trim() || typeof value.python.version !== "string" || value.python.version.length === 0 || value.python.version !== value.python.version.trim()) throw new Error(`${label}: source inspection returned an invalid Python runtime`);
|
|
1382
|
+
if (!Array.isArray(value.engineModules) || value.engineModules.some((module) => !isRecord(module) || typeof module.module !== "string" || module.module.length === 0 || module.module !== module.module.trim() || typeof module.sourceSha256 !== "string" || !/^[0-9a-f]{64}$/.test(module.sourceSha256))) throw new Error(`${label}: source inspection returned invalid engine modules`);
|
|
1383
|
+
}
|
|
1384
|
+
//#endregion
|
|
1314
1385
|
//#region src/campaign/gepa-optimization-result.ts
|
|
1315
1386
|
function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, maxEvaluations, maxPopulationCandidates, scenarioIds, expectsComponents, requiresCandidatePopulation) {
|
|
1316
1387
|
if (result.recipeKind !== recipeKind) throw new Error(`${name}: GEPA bridge reported recipe '${String(result.recipeKind)}'`);
|
|
@@ -1733,24 +1804,7 @@ async function recordExternalSearch(input) {
|
|
|
1733
1804
|
});
|
|
1734
1805
|
}
|
|
1735
1806
|
//#endregion
|
|
1736
|
-
//#region src/campaign/gepa-optimization-
|
|
1737
|
-
/**
|
|
1738
|
-
* Turn an optional GEPA installation into an `OptimizationMethod`.
|
|
1739
|
-
*
|
|
1740
|
-
* GEPA receives only serialized train and selection cases. The caller's final
|
|
1741
|
-
* test partition stays inside `compareOptimizationMethods`, which invokes this
|
|
1742
|
-
* method without a test-set field. The local callback routes every candidate
|
|
1743
|
-
* evaluation through the same dispatch and judges used by other methods.
|
|
1744
|
-
*/
|
|
1745
|
-
/**
|
|
1746
|
-
* Environment for a bridge child that spawns `claude` CLI subprocesses.
|
|
1747
|
-
*
|
|
1748
|
-
* Both agent CLI engines start the CLI with the bridge's own environment, so
|
|
1749
|
-
* these variables travel bridge → engine → CLI. They merge AFTER
|
|
1750
|
-
* `removeCredentialEnvironment`: the ephemeral loopback token is the only
|
|
1751
|
-
* credential the child may see, and a caller-supplied ANTHROPIC_AUTH_TOKEN is
|
|
1752
|
-
* stripped like any other credential.
|
|
1753
|
-
*/
|
|
1807
|
+
//#region src/campaign/gepa-optimization-run.ts
|
|
1754
1808
|
function anthropicShimEnvironment(proxy, optimizer) {
|
|
1755
1809
|
return {
|
|
1756
1810
|
ANTHROPIC_BASE_URL: new URL(proxy.baseUrl).origin,
|
|
@@ -1763,370 +1817,501 @@ function anthropicShimEnvironment(proxy, optimizer) {
|
|
|
1763
1817
|
API_TIMEOUT_MS: String((optimizer.budget.requestTimeoutMs ?? 3e5) + 6e4)
|
|
1764
1818
|
};
|
|
1765
1819
|
}
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1820
|
+
/** Spend the optimizer booked to its own candidate generation. Unknown stays
|
|
1821
|
+
* unknown: the bridge reports proposer cost only when the engine measured it. */
|
|
1822
|
+
function optimizerAccounting(tokenUsage, proposerCostUsd) {
|
|
1823
|
+
return {
|
|
1824
|
+
tokens: tokenUsage?.inputTokens === void 0 || tokenUsage.outputTokens === void 0 ? {
|
|
1825
|
+
status: "unknown",
|
|
1826
|
+
reason: "the optimizer bridge reported no token usage"
|
|
1827
|
+
} : {
|
|
1828
|
+
status: "known",
|
|
1829
|
+
inputTokens: tokenUsage.inputTokens,
|
|
1830
|
+
outputTokens: tokenUsage.outputTokens,
|
|
1831
|
+
cachedTokens: 0
|
|
1832
|
+
},
|
|
1833
|
+
cost: proposerCostUsd === void 0 ? {
|
|
1834
|
+
status: "unknown",
|
|
1835
|
+
knownLowerBoundUsd: 0,
|
|
1836
|
+
reason: "the optimizer bridge reported no proposer cost"
|
|
1837
|
+
} : {
|
|
1838
|
+
status: "known",
|
|
1839
|
+
usd: proposerCostUsd,
|
|
1840
|
+
source: "provider"
|
|
1841
|
+
}
|
|
1842
|
+
};
|
|
1843
|
+
}
|
|
1844
|
+
/** The settled inputs of one GEPA run: its identity, budgets, evaluation callback and logs. */
|
|
1845
|
+
async function prepareGepaRun(config, input, name) {
|
|
1846
|
+
const signal = input.runOptions.signal;
|
|
1847
|
+
signal?.throwIfAborted();
|
|
1848
|
+
if (typeof input.baselineSurface !== "string" && input.baselineSurface.kind !== "components") throw new Error(`${name}: GEPA bridge supports text and component surfaces`);
|
|
1849
|
+
const expectsComponents = typeof input.baselineSurface !== "string";
|
|
1850
|
+
if (expectsComponents) assertGepaComponentRecipe(config.recipe, name);
|
|
1851
|
+
const started = Date.now();
|
|
1852
|
+
const maxCandidateChars = config.maxCandidateChars ?? 2e5;
|
|
1853
|
+
const maxEvidenceChars = config.maxEvidenceChars ?? 1e5;
|
|
1854
|
+
const timeoutMs = config.timeoutMs ?? 18e5;
|
|
1855
|
+
const storage = input.runOptions.storage ?? fsCampaignStorage();
|
|
1856
|
+
const runDir = `${input.runDir}/gepa`;
|
|
1857
|
+
storage.ensureDir(runDir);
|
|
1858
|
+
const costLedger = input.costLedger;
|
|
1859
|
+
const attemptId = randomBytes(16).toString("hex");
|
|
1860
|
+
const resume = config.resume ?? "never";
|
|
1861
|
+
const bridgeRunner = config.optimizer && config.runner ? {
|
|
1862
|
+
...config.runner,
|
|
1863
|
+
env: removeCredentialEnvironment(config.runner.env ?? {})
|
|
1864
|
+
} : config.runner;
|
|
1865
|
+
const runtimeIdentity = await inspectExternalOptimizerRuntime({
|
|
1866
|
+
label: name,
|
|
1867
|
+
package: "gepa",
|
|
1868
|
+
module: "agent_eval_rpc.gepa_bridge",
|
|
1869
|
+
engineModules: config.engineModules,
|
|
1870
|
+
...bridgeRunner ? { runner: bridgeRunner } : {},
|
|
1871
|
+
timeoutMs,
|
|
1872
|
+
...signal ? { signal } : {}
|
|
1873
|
+
});
|
|
1874
|
+
const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
|
|
1875
|
+
const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
|
|
1876
|
+
const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
|
|
1877
|
+
const evaluationLimit = gepaRecipeEvaluationLimit(config.recipe, input.selectionScenarios.length);
|
|
1878
|
+
const maxPopulationCandidates = Math.min(Number.MAX_SAFE_INTEGER, evaluationLimit + 1);
|
|
1879
|
+
const populationScenarioIds = (selectionSet.length > 0 ? selectionSet : trainSet).map((scenario) => scenario.id);
|
|
1880
|
+
const runMaterial = {
|
|
1881
|
+
optimizer: "gepa",
|
|
1882
|
+
runtime: runtimeIdentity,
|
|
1883
|
+
method: name,
|
|
1884
|
+
evaluationId: config.evaluationId,
|
|
1885
|
+
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
1886
|
+
seed: input.seed,
|
|
1887
|
+
recipe: snapshotJson(config.recipe, "GEPA run settings"),
|
|
1888
|
+
engineModules: config.engineModules ?? [],
|
|
1889
|
+
objective: config.objective,
|
|
1890
|
+
background: config.background ?? "",
|
|
1891
|
+
seedCandidate,
|
|
1892
|
+
trainSet,
|
|
1893
|
+
selectionSet,
|
|
1894
|
+
maxCandidateChars,
|
|
1895
|
+
maxPopulationCandidates,
|
|
1896
|
+
maxEvidenceChars,
|
|
1897
|
+
evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
|
|
1898
|
+
optimizerModel: config.optimizer ? {
|
|
1899
|
+
model: config.optimizer.model,
|
|
1900
|
+
callRef: config.optimizer.callRef,
|
|
1901
|
+
budget: config.optimizer.budget,
|
|
1902
|
+
...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
|
|
1903
|
+
} : null,
|
|
1904
|
+
runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.gepa_bridge"),
|
|
1905
|
+
trustResumeState: config.trustResumeState === true
|
|
1906
|
+
};
|
|
1907
|
+
const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
|
|
1908
|
+
const runId = externalOptimizerRunKey({
|
|
1909
|
+
material: runMaterial,
|
|
1910
|
+
attemptId,
|
|
1911
|
+
resumeEnabled: resume !== "never" && gepaRecipeSupportsResume(config.recipe)
|
|
1912
|
+
});
|
|
1913
|
+
const runBudget = openExternalOptimizerRunBudget({
|
|
1914
|
+
storage,
|
|
1915
|
+
runDir,
|
|
1916
|
+
runKey: runId,
|
|
1917
|
+
attemptId,
|
|
1918
|
+
maxEvaluations: evaluationLimit
|
|
1919
|
+
});
|
|
1920
|
+
const observationLog = openExternalOptimizerObservationLog({
|
|
1921
|
+
storage,
|
|
1922
|
+
path: `${runDir}/observations-${attemptId}.jsonl`
|
|
1923
|
+
});
|
|
1924
|
+
const executionLog = config.optimizer ? openExternalOptimizerExecutionLog({
|
|
1925
|
+
storage,
|
|
1926
|
+
path: `${runDir}/model-executions-${attemptId}.jsonl`
|
|
1927
|
+
}) : void 0;
|
|
1928
|
+
const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "GEPA bridge");
|
|
1929
|
+
const evaluate = createExternalTextEvaluator({
|
|
1930
|
+
input,
|
|
1931
|
+
label: "GEPA bridge",
|
|
1932
|
+
runDir,
|
|
1933
|
+
compatibleRunId: runId,
|
|
1934
|
+
costPhase: "gepa.external-evaluation",
|
|
1935
|
+
costTags: runBudget.attemptTags,
|
|
1936
|
+
costLedger,
|
|
1937
|
+
scenarioById,
|
|
1938
|
+
maxCandidateChars,
|
|
1939
|
+
maxEvidenceChars,
|
|
1940
|
+
describeArtifact: config.describeArtifact
|
|
1941
|
+
});
|
|
1942
|
+
let attemptEvaluationCount = 0;
|
|
1770
1943
|
return {
|
|
1944
|
+
signal,
|
|
1945
|
+
expectsComponents,
|
|
1946
|
+
started,
|
|
1947
|
+
maxCandidateChars,
|
|
1948
|
+
maxEvidenceChars,
|
|
1949
|
+
timeoutMs,
|
|
1950
|
+
storage,
|
|
1951
|
+
runDir,
|
|
1952
|
+
costLedger,
|
|
1953
|
+
attemptId,
|
|
1954
|
+
resume,
|
|
1955
|
+
bridgeRunner,
|
|
1956
|
+
runtimeIdentity,
|
|
1957
|
+
seedCandidate,
|
|
1958
|
+
trainSet,
|
|
1959
|
+
selectionSet,
|
|
1960
|
+
evaluationLimit,
|
|
1961
|
+
maxPopulationCandidates,
|
|
1962
|
+
populationScenarioIds,
|
|
1963
|
+
compatibleRunId,
|
|
1964
|
+
runId,
|
|
1965
|
+
runBudget,
|
|
1966
|
+
observationLog,
|
|
1967
|
+
executionLog,
|
|
1968
|
+
callback: await startExternalOptimizerCallback({
|
|
1969
|
+
token: randomBytes(32).toString("hex"),
|
|
1970
|
+
maxEvaluations: evaluationLimit,
|
|
1971
|
+
acceptEvaluation: () => {
|
|
1972
|
+
if (runBudget.acceptEvaluation() === void 0) return void 0;
|
|
1973
|
+
attemptEvaluationCount += 1;
|
|
1974
|
+
return attemptEvaluationCount;
|
|
1975
|
+
},
|
|
1976
|
+
evaluate,
|
|
1977
|
+
observe: observationLog.observe,
|
|
1978
|
+
...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
|
|
1979
|
+
...signal ? { signal } : {}
|
|
1980
|
+
}),
|
|
1771
1981
|
name,
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
...
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
runtime: runtimeIdentity,
|
|
1810
|
-
method: name,
|
|
1811
|
-
evaluationId: config.evaluationId,
|
|
1812
|
-
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
1813
|
-
seed: input.seed,
|
|
1814
|
-
recipe: snapshotJson(config.recipe, "GEPA run settings"),
|
|
1815
|
-
engineModules: config.engineModules ?? [],
|
|
1816
|
-
objective: config.objective,
|
|
1817
|
-
background: config.background ?? "",
|
|
1818
|
-
seedCandidate,
|
|
1819
|
-
trainSet,
|
|
1820
|
-
selectionSet,
|
|
1821
|
-
maxCandidateChars,
|
|
1822
|
-
maxPopulationCandidates,
|
|
1823
|
-
maxEvidenceChars,
|
|
1824
|
-
evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
|
|
1825
|
-
optimizerModel: config.optimizer ? {
|
|
1826
|
-
model: config.optimizer.model,
|
|
1982
|
+
config,
|
|
1983
|
+
input
|
|
1984
|
+
};
|
|
1985
|
+
}
|
|
1986
|
+
/** Start the optimizer model proxy when one is configured, run the GEPA bridge, and release the resources either way. */
|
|
1987
|
+
async function runBridge(ctx) {
|
|
1988
|
+
const { signal, maxCandidateChars, maxEvidenceChars, timeoutMs, runDir, costLedger, attemptId, resume, bridgeRunner, runtimeIdentity, seedCandidate, trainSet, selectionSet, maxPopulationCandidates, compatibleRunId, runId, runBudget, executionLog, callback, name, config, input } = ctx;
|
|
1989
|
+
const runnerEnv = bridgeRunner?.env ?? {};
|
|
1990
|
+
let modelProxy;
|
|
1991
|
+
const bridgeRunnerForSpawn = () => {
|
|
1992
|
+
if (!modelProxy) return bridgeRunner;
|
|
1993
|
+
const anthropicChildEnv = config.optimizer?.anthropicEndpoint === true && gepaRecipeHasAgentCliEngine(config.recipe) ? anthropicShimEnvironment(modelProxy, config.optimizer) : void 0;
|
|
1994
|
+
if (!bridgeRunner && !anthropicChildEnv) return bridgeRunner;
|
|
1995
|
+
return {
|
|
1996
|
+
...bridgeRunner ?? {},
|
|
1997
|
+
env: {
|
|
1998
|
+
...removeCredentialEnvironment(runnerEnv),
|
|
1999
|
+
...anthropicChildEnv ?? {}
|
|
2000
|
+
}
|
|
2001
|
+
};
|
|
2002
|
+
};
|
|
2003
|
+
const closeResources = () => closeExternalOptimizerResources({
|
|
2004
|
+
label: name,
|
|
2005
|
+
callback,
|
|
2006
|
+
...modelProxy ? { modelProxy } : {}
|
|
2007
|
+
});
|
|
2008
|
+
const { result, outputDir } = await runWithCleanup({
|
|
2009
|
+
label: `${name} optimizer resources`,
|
|
2010
|
+
run: async () => {
|
|
2011
|
+
if (config.optimizer) {
|
|
2012
|
+
const priorOptimizerUsage = costLedger.summary({
|
|
2013
|
+
phase: "gepa.optimizer-model",
|
|
2014
|
+
tags: runBudget.runTags
|
|
2015
|
+
});
|
|
2016
|
+
assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
|
|
2017
|
+
modelProxy = await startExternalOptimizerModelProxy({
|
|
2018
|
+
call: config.optimizer.call,
|
|
1827
2019
|
callRef: config.optimizer.callRef,
|
|
2020
|
+
recordExecution: executionLog.observe,
|
|
2021
|
+
model: config.optimizer.model,
|
|
1828
2022
|
budget: config.optimizer.budget,
|
|
1829
|
-
...config.optimizer.
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
const runBudget = openExternalOptimizerRunBudget({
|
|
1841
|
-
storage,
|
|
1842
|
-
runDir,
|
|
1843
|
-
runKey: runId,
|
|
1844
|
-
attemptId,
|
|
1845
|
-
maxEvaluations: evaluationLimit
|
|
1846
|
-
});
|
|
1847
|
-
const observationLog = openExternalOptimizerObservationLog({
|
|
1848
|
-
storage,
|
|
1849
|
-
path: `${runDir}/observations-${attemptId}.jsonl`
|
|
1850
|
-
});
|
|
1851
|
-
const executionLog = config.optimizer ? openExternalOptimizerExecutionLog({
|
|
1852
|
-
storage,
|
|
1853
|
-
path: `${runDir}/model-executions-${attemptId}.jsonl`
|
|
1854
|
-
}) : void 0;
|
|
1855
|
-
const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "GEPA bridge");
|
|
1856
|
-
const evaluate = createExternalTextEvaluator({
|
|
1857
|
-
input,
|
|
1858
|
-
label: "GEPA bridge",
|
|
1859
|
-
runDir,
|
|
1860
|
-
compatibleRunId: runId,
|
|
1861
|
-
costPhase: "gepa.external-evaluation",
|
|
1862
|
-
costTags: runBudget.attemptTags,
|
|
1863
|
-
costLedger,
|
|
1864
|
-
scenarioById,
|
|
1865
|
-
maxCandidateChars,
|
|
1866
|
-
maxEvidenceChars,
|
|
1867
|
-
describeArtifact: config.describeArtifact
|
|
1868
|
-
});
|
|
1869
|
-
let attemptEvaluationCount = 0;
|
|
1870
|
-
const callback = await startExternalOptimizerCallback({
|
|
1871
|
-
token: randomBytes(32).toString("hex"),
|
|
1872
|
-
maxEvaluations: evaluationLimit,
|
|
1873
|
-
acceptEvaluation: () => {
|
|
1874
|
-
if (runBudget.acceptEvaluation() === void 0) return void 0;
|
|
1875
|
-
attemptEvaluationCount += 1;
|
|
1876
|
-
return attemptEvaluationCount;
|
|
1877
|
-
},
|
|
1878
|
-
evaluate,
|
|
1879
|
-
observe: observationLog.observe,
|
|
1880
|
-
...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
|
|
1881
|
-
...signal ? { signal } : {}
|
|
1882
|
-
});
|
|
1883
|
-
const runnerEnv = bridgeRunner?.env ?? {};
|
|
1884
|
-
let modelProxy;
|
|
1885
|
-
const bridgeRunnerForSpawn = () => {
|
|
1886
|
-
if (!modelProxy) return bridgeRunner;
|
|
1887
|
-
const anthropicChildEnv = config.optimizer?.anthropicEndpoint === true && gepaRecipeHasAgentCliEngine(config.recipe) ? anthropicShimEnvironment(modelProxy, config.optimizer) : void 0;
|
|
1888
|
-
if (!bridgeRunner && !anthropicChildEnv) return bridgeRunner;
|
|
1889
|
-
return {
|
|
1890
|
-
...bridgeRunner ?? {},
|
|
1891
|
-
env: {
|
|
1892
|
-
...removeCredentialEnvironment(runnerEnv),
|
|
1893
|
-
...anthropicChildEnv ?? {}
|
|
1894
|
-
}
|
|
1895
|
-
};
|
|
1896
|
-
};
|
|
1897
|
-
const closeResources = () => closeExternalOptimizerResources({
|
|
1898
|
-
label: name,
|
|
1899
|
-
callback,
|
|
1900
|
-
...modelProxy ? { modelProxy } : {}
|
|
1901
|
-
});
|
|
1902
|
-
const { result, outputDir } = await runWithCleanup({
|
|
1903
|
-
label: `${name} optimizer resources`,
|
|
1904
|
-
run: async () => {
|
|
1905
|
-
if (config.optimizer) {
|
|
1906
|
-
const priorOptimizerUsage = costLedger.summary({
|
|
1907
|
-
phase: "gepa.optimizer-model",
|
|
1908
|
-
tags: runBudget.runTags
|
|
1909
|
-
});
|
|
1910
|
-
assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
|
|
1911
|
-
modelProxy = await startExternalOptimizerModelProxy({
|
|
1912
|
-
call: config.optimizer.call,
|
|
1913
|
-
callRef: config.optimizer.callRef,
|
|
1914
|
-
recordExecution: executionLog.observe,
|
|
1915
|
-
model: config.optimizer.model,
|
|
1916
|
-
budget: config.optimizer.budget,
|
|
1917
|
-
...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
|
|
1918
|
-
...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {},
|
|
1919
|
-
costLedger,
|
|
1920
|
-
phase: "gepa.optimizer-model",
|
|
1921
|
-
actor: name,
|
|
1922
|
-
tags: { ...runBudget.attemptTags },
|
|
1923
|
-
initialUsage: {
|
|
1924
|
-
requests: priorOptimizerUsage.totalCalls,
|
|
1925
|
-
...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
|
|
1926
|
-
},
|
|
1927
|
-
...signal ? { signal } : {}
|
|
1928
|
-
});
|
|
1929
|
-
}
|
|
1930
|
-
const outputDir = `${runDir}/external`;
|
|
1931
|
-
await mkdir(outputDir, { recursive: true });
|
|
1932
|
-
return {
|
|
1933
|
-
result: await runExternalOptimizerProcess({
|
|
1934
|
-
label: "GEPA bridge",
|
|
1935
|
-
tempPrefix: "agent-eval-gepa-",
|
|
1936
|
-
module: "agent_eval_rpc.gepa_bridge",
|
|
1937
|
-
input: {
|
|
1938
|
-
attemptId,
|
|
1939
|
-
compatibleRunId,
|
|
1940
|
-
runId,
|
|
1941
|
-
runtimeIdentity,
|
|
1942
|
-
resume,
|
|
1943
|
-
trustedResumeState: config.trustResumeState === true,
|
|
1944
|
-
evaluationId: config.evaluationId,
|
|
1945
|
-
seed: input.seed,
|
|
1946
|
-
callbackUrl: callback.url,
|
|
1947
|
-
callbackToken: callback.token,
|
|
1948
|
-
timeoutMs,
|
|
1949
|
-
engineModules: config.engineModules ?? [],
|
|
1950
|
-
recipe: config.recipe,
|
|
1951
|
-
objective: config.objective,
|
|
1952
|
-
...config.background ? { background: config.background } : {},
|
|
1953
|
-
seedCandidate,
|
|
1954
|
-
trainSet,
|
|
1955
|
-
selectionSet,
|
|
1956
|
-
maxCandidateChars,
|
|
1957
|
-
maxPopulationCandidates,
|
|
1958
|
-
maxEvidenceChars,
|
|
1959
|
-
outputDir,
|
|
1960
|
-
...modelProxy && config.optimizer ? { modelProxy: {
|
|
1961
|
-
baseUrl: modelProxy.baseUrl,
|
|
1962
|
-
apiKey: modelProxy.apiKey,
|
|
1963
|
-
model: config.optimizer.model,
|
|
1964
|
-
budget: config.optimizer.budget,
|
|
1965
|
-
...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
|
|
1966
|
-
} } : {}
|
|
1967
|
-
},
|
|
1968
|
-
runner: bridgeRunnerForSpawn(),
|
|
1969
|
-
timeoutMs,
|
|
1970
|
-
...signal ? { signal } : {}
|
|
1971
|
-
}),
|
|
1972
|
-
outputDir
|
|
1973
|
-
};
|
|
1974
|
-
},
|
|
1975
|
-
cleanup: closeResources
|
|
1976
|
-
});
|
|
1977
|
-
signal?.throwIfAborted();
|
|
1978
|
-
assertGepaBridgeOutput(result, name, maxCandidateChars, config.recipe.kind, evaluationLimit, maxPopulationCandidates, populationScenarioIds, expectsComponents, config.recipe.kind === "engine" && config.recipe.run.engine === "gepa");
|
|
1979
|
-
assertExternalOptimizerRunBinding({
|
|
1980
|
-
label: name,
|
|
1981
|
-
runtime: runtimeIdentity,
|
|
1982
|
-
returnedSource: result.upstream,
|
|
1983
|
-
compatibleRunId,
|
|
1984
|
-
runId,
|
|
1985
|
-
returnedRunId: result.runId,
|
|
1986
|
-
resume,
|
|
1987
|
-
resumed: result.resumed
|
|
1988
|
-
});
|
|
1989
|
-
if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
|
|
1990
|
-
const population = result.candidatePopulation ? readGepaCandidatePopulationArtifact({
|
|
1991
|
-
summary: result.candidatePopulation,
|
|
1992
|
-
storage
|
|
1993
|
-
}) : null;
|
|
1994
|
-
if (population) {
|
|
1995
|
-
const selected = population.candidates[population.bestIndex];
|
|
1996
|
-
const selectedHash = contentHash({
|
|
1997
|
-
kind: "external-text-candidate",
|
|
1998
|
-
candidate: result.bestCandidate
|
|
2023
|
+
...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
|
|
2024
|
+
...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {},
|
|
2025
|
+
costLedger,
|
|
2026
|
+
phase: "gepa.optimizer-model",
|
|
2027
|
+
actor: name,
|
|
2028
|
+
tags: { ...runBudget.attemptTags },
|
|
2029
|
+
initialUsage: {
|
|
2030
|
+
requests: priorOptimizerUsage.totalCalls,
|
|
2031
|
+
...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
|
|
2032
|
+
},
|
|
2033
|
+
...signal ? { signal } : {}
|
|
1999
2034
|
});
|
|
2000
|
-
if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
|
|
2001
2035
|
}
|
|
2002
|
-
const
|
|
2003
|
-
|
|
2004
|
-
path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
|
|
2005
|
-
searchId: runId,
|
|
2006
|
-
identity: config.searchLedger.identity,
|
|
2007
|
-
storage,
|
|
2008
|
-
seed: input.seed,
|
|
2009
|
-
baselineSurface: input.baselineSurface,
|
|
2010
|
-
trainScenarios: input.trainScenarios,
|
|
2011
|
-
selectionScenarios: input.selectionScenarios,
|
|
2012
|
-
evaluationLimit,
|
|
2013
|
-
...population ? { population } : {
|
|
2014
|
-
population: null,
|
|
2015
|
-
selected: decodeExternalTextCandidate(result.bestCandidate)
|
|
2016
|
-
},
|
|
2017
|
-
observations: readExternalOptimizerObservationArtifact({
|
|
2018
|
-
summary: observationLog.summary(),
|
|
2019
|
-
storage
|
|
2020
|
-
}),
|
|
2021
|
-
generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd)
|
|
2022
|
-
}) : void 0;
|
|
2023
|
-
const evaluationCost = costFromLedgerSummary(costLedger.summary({
|
|
2024
|
-
phase: "gepa.external-evaluation",
|
|
2025
|
-
tags: runBudget.runTags
|
|
2026
|
-
}));
|
|
2027
|
-
const optimizerSummary = costLedger.summary({
|
|
2028
|
-
phase: "gepa.optimizer-model",
|
|
2029
|
-
tags: runBudget.runTags
|
|
2030
|
-
});
|
|
2031
|
-
const optimizerReceipts = costLedger.list({
|
|
2032
|
-
phase: "gepa.optimizer-model",
|
|
2033
|
-
tags: runBudget.runTags
|
|
2034
|
-
});
|
|
2035
|
-
const optimizerCost = costFromLedgerSummary(optimizerSummary);
|
|
2036
|
-
const reportedProposerCost = result.proposerCostUsd ?? 0;
|
|
2037
|
-
if (modelProxy) {
|
|
2038
|
-
modelProxy.assertExecutionComplete();
|
|
2039
|
-
const wires = modelProxy.wireUsage();
|
|
2040
|
-
assertExternalOptimizerCompletionCount(result.tokenUsage, wires.openai.requestAttempts, wires.openai.successfulCompletions, name, "GEPA");
|
|
2041
|
-
if (config.optimizer?.anthropicEndpoint === true) {
|
|
2042
|
-
const anthropicReceipts = costLedger.list({
|
|
2043
|
-
phase: "gepa.optimizer-model",
|
|
2044
|
-
tags: {
|
|
2045
|
-
...runBudget.attemptTags,
|
|
2046
|
-
wire: "anthropic"
|
|
2047
|
-
}
|
|
2048
|
-
});
|
|
2049
|
-
if (anthropicReceipts.length !== wires.anthropic.requestAttempts) throw new Error(`${name}: the Anthropic endpoint admitted ${wires.anthropic.requestAttempts} model calls but the ledger holds ${anthropicReceipts.length} receipts`);
|
|
2050
|
-
}
|
|
2051
|
-
}
|
|
2052
|
-
const tokenUsage = modelProxy ? optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts) : void 0;
|
|
2053
|
-
const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
|
|
2054
|
-
const meteredCost = modelProxy ? combineComparisonCosts([{
|
|
2055
|
-
label: "evaluation",
|
|
2056
|
-
cost: evaluationCost
|
|
2057
|
-
}, {
|
|
2058
|
-
label: "optimizer model",
|
|
2059
|
-
cost: optimizerCost
|
|
2060
|
-
}]) : void 0;
|
|
2061
|
-
const externalTotalCostUsd = evaluationCost.totalCostUsd + reportedProposerCost;
|
|
2036
|
+
const outputDir = `${runDir}/external`;
|
|
2037
|
+
await mkdir(outputDir, { recursive: true });
|
|
2062
2038
|
return {
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2039
|
+
result: await runExternalOptimizerProcess({
|
|
2040
|
+
label: "GEPA bridge",
|
|
2041
|
+
tempPrefix: "agent-eval-gepa-",
|
|
2042
|
+
module: "agent_eval_rpc.gepa_bridge",
|
|
2043
|
+
input: {
|
|
2044
|
+
attemptId,
|
|
2045
|
+
compatibleRunId,
|
|
2046
|
+
runId,
|
|
2047
|
+
runtimeIdentity,
|
|
2048
|
+
resume,
|
|
2049
|
+
trustedResumeState: config.trustResumeState === true,
|
|
2050
|
+
evaluationId: config.evaluationId,
|
|
2051
|
+
seed: input.seed,
|
|
2052
|
+
callbackUrl: callback.url,
|
|
2053
|
+
callbackToken: callback.token,
|
|
2054
|
+
timeoutMs,
|
|
2055
|
+
engineModules: config.engineModules ?? [],
|
|
2056
|
+
recipe: config.recipe,
|
|
2057
|
+
objective: config.objective,
|
|
2058
|
+
...config.background ? { background: config.background } : {},
|
|
2059
|
+
seedCandidate,
|
|
2060
|
+
trainSet,
|
|
2061
|
+
selectionSet,
|
|
2062
|
+
maxCandidateChars,
|
|
2063
|
+
maxPopulationCandidates,
|
|
2064
|
+
maxEvidenceChars,
|
|
2065
|
+
outputDir,
|
|
2066
|
+
...modelProxy && config.optimizer ? { modelProxy: {
|
|
2067
|
+
baseUrl: modelProxy.baseUrl,
|
|
2068
|
+
apiKey: modelProxy.apiKey,
|
|
2069
|
+
model: config.optimizer.model,
|
|
2070
|
+
budget: config.optimizer.budget,
|
|
2071
|
+
...config.optimizer.anthropicEndpoint === true ? { anthropicEndpoint: true } : {}
|
|
2072
|
+
} } : {}
|
|
2073
2073
|
},
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2074
|
+
runner: bridgeRunnerForSpawn(),
|
|
2075
|
+
timeoutMs,
|
|
2076
|
+
...signal ? { signal } : {}
|
|
2077
|
+
}),
|
|
2078
|
+
outputDir
|
|
2079
|
+
};
|
|
2080
|
+
},
|
|
2081
|
+
cleanup: closeResources
|
|
2082
|
+
});
|
|
2083
|
+
return {
|
|
2084
|
+
result,
|
|
2085
|
+
outputDir,
|
|
2086
|
+
modelProxy
|
|
2087
|
+
};
|
|
2088
|
+
}
|
|
2089
|
+
/** Check the bridge output against the run it was started for, and return the candidate population it reports. */
|
|
2090
|
+
function verifyBridgeResult(ctx, result) {
|
|
2091
|
+
const { signal, expectsComponents, maxCandidateChars, storage, resume, runtimeIdentity, evaluationLimit, maxPopulationCandidates, populationScenarioIds, compatibleRunId, runId, callback, name, config } = ctx;
|
|
2092
|
+
signal?.throwIfAborted();
|
|
2093
|
+
assertGepaBridgeOutput(result, name, maxCandidateChars, config.recipe.kind, evaluationLimit, maxPopulationCandidates, populationScenarioIds, expectsComponents, config.recipe.kind === "engine" && config.recipe.run.engine === "gepa");
|
|
2094
|
+
assertExternalOptimizerRunBinding({
|
|
2095
|
+
label: name,
|
|
2096
|
+
runtime: runtimeIdentity,
|
|
2097
|
+
returnedSource: result.upstream,
|
|
2098
|
+
compatibleRunId,
|
|
2099
|
+
runId,
|
|
2100
|
+
returnedRunId: result.runId,
|
|
2101
|
+
resume,
|
|
2102
|
+
resumed: result.resumed
|
|
2103
|
+
});
|
|
2104
|
+
if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
|
|
2105
|
+
const population = result.candidatePopulation ? readGepaCandidatePopulationArtifact({
|
|
2106
|
+
summary: result.candidatePopulation,
|
|
2107
|
+
storage
|
|
2108
|
+
}) : null;
|
|
2109
|
+
if (population) {
|
|
2110
|
+
const selected = population.candidates[population.bestIndex];
|
|
2111
|
+
const selectedHash = contentHash({
|
|
2112
|
+
kind: "external-text-candidate",
|
|
2113
|
+
candidate: result.bestCandidate
|
|
2114
|
+
});
|
|
2115
|
+
if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
|
|
2116
|
+
}
|
|
2117
|
+
return population;
|
|
2118
|
+
}
|
|
2119
|
+
/** Every recipe records its search in the search ledger when one is configured. */
|
|
2120
|
+
async function recordGepaSearchHistory(ctx, result, population) {
|
|
2121
|
+
const { storage, runDir, evaluationLimit, runId, observationLog, name, config, input } = ctx;
|
|
2122
|
+
return config.searchLedger ? await recordGepaSearch({
|
|
2123
|
+
name,
|
|
2124
|
+
path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
|
|
2125
|
+
searchId: runId,
|
|
2126
|
+
identity: config.searchLedger.identity,
|
|
2127
|
+
storage,
|
|
2128
|
+
seed: input.seed,
|
|
2129
|
+
baselineSurface: input.baselineSurface,
|
|
2130
|
+
trainScenarios: input.trainScenarios,
|
|
2131
|
+
selectionScenarios: input.selectionScenarios,
|
|
2132
|
+
evaluationLimit,
|
|
2133
|
+
...population ? { population } : {
|
|
2134
|
+
population: null,
|
|
2135
|
+
selected: decodeExternalTextCandidate(result.bestCandidate)
|
|
2136
|
+
},
|
|
2137
|
+
observations: readExternalOptimizerObservationArtifact({
|
|
2138
|
+
summary: observationLog.summary(),
|
|
2139
|
+
storage
|
|
2140
|
+
}),
|
|
2141
|
+
generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd)
|
|
2142
|
+
}) : void 0;
|
|
2143
|
+
}
|
|
2144
|
+
/** Evaluation and optimizer-model cost, with the proxy's completeness checks. */
|
|
2145
|
+
function gepaCosts(ctx, result, modelProxy) {
|
|
2146
|
+
const { costLedger, runtimeIdentity, runBudget, name, config } = ctx;
|
|
2147
|
+
const evaluationCost = costFromLedgerSummary(costLedger.summary({
|
|
2148
|
+
phase: "gepa.external-evaluation",
|
|
2149
|
+
tags: runBudget.runTags
|
|
2150
|
+
}));
|
|
2151
|
+
const optimizerSummary = costLedger.summary({
|
|
2152
|
+
phase: "gepa.optimizer-model",
|
|
2153
|
+
tags: runBudget.runTags
|
|
2154
|
+
});
|
|
2155
|
+
const optimizerReceipts = costLedger.list({
|
|
2156
|
+
phase: "gepa.optimizer-model",
|
|
2157
|
+
tags: runBudget.runTags
|
|
2158
|
+
});
|
|
2159
|
+
const optimizerCost = costFromLedgerSummary(optimizerSummary);
|
|
2160
|
+
const reportedProposerCost = result.proposerCostUsd ?? 0;
|
|
2161
|
+
if (modelProxy) {
|
|
2162
|
+
modelProxy.assertExecutionComplete();
|
|
2163
|
+
const wires = modelProxy.wireUsage();
|
|
2164
|
+
assertExternalOptimizerCompletionCount(result.tokenUsage, wires.openai.requestAttempts, wires.openai.successfulCompletions, name, "GEPA");
|
|
2165
|
+
if (config.optimizer?.anthropicEndpoint === true) {
|
|
2166
|
+
const anthropicReceipts = costLedger.list({
|
|
2167
|
+
phase: "gepa.optimizer-model",
|
|
2168
|
+
tags: {
|
|
2169
|
+
...runBudget.attemptTags,
|
|
2170
|
+
wire: "anthropic"
|
|
2100
2171
|
}
|
|
2101
|
-
};
|
|
2172
|
+
});
|
|
2173
|
+
if (anthropicReceipts.length !== wires.anthropic.requestAttempts) throw new Error(`${name}: the Anthropic endpoint admitted ${wires.anthropic.requestAttempts} model calls but the ledger holds ${anthropicReceipts.length} receipts`);
|
|
2102
2174
|
}
|
|
2175
|
+
}
|
|
2176
|
+
return {
|
|
2177
|
+
evaluationCost,
|
|
2178
|
+
tokenUsage: modelProxy ? optimizationTokenUsageFromSummary(optimizerSummary, optimizerReceipts) : void 0,
|
|
2179
|
+
runtime: observedExternalOptimizerRuntime(runtimeIdentity),
|
|
2180
|
+
meteredCost: modelProxy ? combineComparisonCosts([{
|
|
2181
|
+
label: "evaluation",
|
|
2182
|
+
cost: evaluationCost
|
|
2183
|
+
}, {
|
|
2184
|
+
label: "optimizer model",
|
|
2185
|
+
cost: optimizerCost
|
|
2186
|
+
}]) : void 0,
|
|
2187
|
+
externalTotalCostUsd: evaluationCost.totalCostUsd + reportedProposerCost
|
|
2103
2188
|
};
|
|
2104
2189
|
}
|
|
2105
|
-
/**
|
|
2106
|
-
|
|
2107
|
-
|
|
2190
|
+
/** The method result: the winning surface, its cost and the provenance of the run. */
|
|
2191
|
+
function buildGepaOutcome(ctx, result, outputDir, modelProxy, searchHistory, costs) {
|
|
2192
|
+
const { started, compatibleRunId, runId, runBudget, observationLog, executionLog, config } = ctx;
|
|
2193
|
+
const { evaluationCost, tokenUsage, runtime, meteredCost, externalTotalCostUsd } = costs;
|
|
2108
2194
|
return {
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2195
|
+
winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
|
|
2196
|
+
...searchHistory ? { searchHistory } : {},
|
|
2197
|
+
cost: modelProxy ? meteredCost : {
|
|
2198
|
+
totalCostUsd: externalTotalCostUsd,
|
|
2199
|
+
costProvenance: result.proposerCostAccounting === "reported" && evaluationCost.costProvenance.kind !== "uncaptured" ? {
|
|
2200
|
+
kind: "estimated",
|
|
2201
|
+
usd: externalTotalCostUsd
|
|
2202
|
+
} : {
|
|
2203
|
+
kind: "uncaptured",
|
|
2204
|
+
usd: null
|
|
2205
|
+
},
|
|
2206
|
+
accountingComplete: false,
|
|
2207
|
+
incompleteReasons: [
|
|
2208
|
+
...evaluationCost.incompleteReasons,
|
|
2209
|
+
result.proposerCostAccounting === "reported" ? "GEPA proposer cost is externally reported and has no agent-eval receipt" : "GEPA proposer cost is unavailable",
|
|
2210
|
+
...result.resumed ? ["GEPA proposer cost before this resumed attempt is unavailable"] : []
|
|
2211
|
+
]
|
|
2117
2212
|
},
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2213
|
+
durationMs: Date.now() - started,
|
|
2214
|
+
provenance: {
|
|
2215
|
+
...runtime,
|
|
2216
|
+
...config.optimizer ? {
|
|
2217
|
+
optimizerModel: config.optimizer.model,
|
|
2218
|
+
optimizerCallRef: config.optimizer.callRef
|
|
2219
|
+
} : {},
|
|
2220
|
+
...modelProxy && config.optimizer?.anthropicEndpoint === true ? { anthropicEndpoint: modelProxy.wireUsage().anthropic } : {},
|
|
2221
|
+
compatibleRunId,
|
|
2222
|
+
runId,
|
|
2223
|
+
resumed: result.resumed,
|
|
2224
|
+
seedApplied: result.seedApplied,
|
|
2225
|
+
evaluationCount: runBudget.acceptedEvaluations(),
|
|
2226
|
+
...result.upstreamReportedEvaluations !== void 0 ? { upstreamReportedEvaluations: result.upstreamReportedEvaluations } : {},
|
|
2227
|
+
artifactDir: outputDir,
|
|
2228
|
+
...tokenUsage ? { tokenUsage } : {},
|
|
2229
|
+
observations: observationLog.summary(),
|
|
2230
|
+
...result.candidatePopulation ? { gepaCandidatePopulation: result.candidatePopulation } : {},
|
|
2231
|
+
...executionLog ? { modelExecutions: executionLog.summary() } : {}
|
|
2232
|
+
}
|
|
2233
|
+
};
|
|
2234
|
+
}
|
|
2235
|
+
//#endregion
|
|
2236
|
+
//#region src/campaign/gepa-optimization-method.ts
|
|
2237
|
+
/**
|
|
2238
|
+
* Turn an optional GEPA installation into an `OptimizationMethod`.
|
|
2239
|
+
*
|
|
2240
|
+
* GEPA receives only serialized train and selection cases. The caller's final
|
|
2241
|
+
* test partition stays inside `compareOptimizationMethods`, which invokes this
|
|
2242
|
+
* method without a test-set field. The local callback routes every candidate
|
|
2243
|
+
* evaluation through the same dispatch and judges used by other methods.
|
|
2244
|
+
*/
|
|
2245
|
+
/**
|
|
2246
|
+
* Environment for a bridge child that spawns `claude` CLI subprocesses.
|
|
2247
|
+
*
|
|
2248
|
+
* Both agent CLI engines start the CLI with the bridge's own environment, so
|
|
2249
|
+
* these variables travel bridge → engine → CLI. They merge AFTER
|
|
2250
|
+
* `removeCredentialEnvironment`: the ephemeral loopback token is the only
|
|
2251
|
+
* credential the child may see, and a caller-supplied ANTHROPIC_AUTH_TOKEN is
|
|
2252
|
+
* stripped like any other credential.
|
|
2253
|
+
*/
|
|
2254
|
+
function gepaOptimizationMethod(config) {
|
|
2255
|
+
assertGepaOptimizationConfig(config);
|
|
2256
|
+
config = snapshotGepaOptimizationConfig(config);
|
|
2257
|
+
const name = config.name ?? defaultGepaMethodName(config.recipe);
|
|
2258
|
+
return {
|
|
2259
|
+
name,
|
|
2260
|
+
async optimize(input) {
|
|
2261
|
+
const ctx = await prepareGepaRun(config, input, name);
|
|
2262
|
+
const { result, outputDir, modelProxy } = await runBridge(ctx);
|
|
2263
|
+
return buildGepaOutcome(ctx, result, outputDir, modelProxy, await recordGepaSearchHistory(ctx, result, verifyBridgeResult(ctx, result)), gepaCosts(ctx, result, modelProxy));
|
|
2126
2264
|
}
|
|
2127
2265
|
};
|
|
2128
2266
|
}
|
|
2129
2267
|
//#endregion
|
|
2268
|
+
//#region src/campaign/presets/comparison-controls.ts
|
|
2269
|
+
/** Admission checks for the shared controls of a method comparison. */
|
|
2270
|
+
function assertComparisonControls(opts, seed, resamples, confidence) {
|
|
2271
|
+
if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
|
|
2272
|
+
if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
|
|
2273
|
+
if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
|
|
2274
|
+
if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
|
|
2275
|
+
const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
|
|
2276
|
+
if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
|
|
2277
|
+
if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
|
|
2278
|
+
try {
|
|
2279
|
+
surfaceContentHash(opts.baselineSurface);
|
|
2280
|
+
} catch (cause) {
|
|
2281
|
+
throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
|
|
2282
|
+
}
|
|
2283
|
+
const judgeNames = /* @__PURE__ */ new Set();
|
|
2284
|
+
for (const judge of opts.judges) {
|
|
2285
|
+
if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
|
|
2286
|
+
if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
|
|
2287
|
+
judgeNames.add(judge.name);
|
|
2288
|
+
const dimensionKeys = /* @__PURE__ */ new Set();
|
|
2289
|
+
for (const dimension of judge.dimensions) {
|
|
2290
|
+
if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
|
|
2291
|
+
if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
|
|
2292
|
+
dimensionKeys.add(dimension.key);
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
|
|
2296
|
+
if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
|
|
2297
|
+
if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
|
|
2298
|
+
assertConfidence(confidence);
|
|
2299
|
+
const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
|
|
2300
|
+
if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
|
|
2301
|
+
if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
|
|
2302
|
+
if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
|
|
2303
|
+
if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
|
|
2304
|
+
if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
|
|
2305
|
+
if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
|
|
2306
|
+
}
|
|
2307
|
+
function assertConfidence(confidence) {
|
|
2308
|
+
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
|
|
2309
|
+
}
|
|
2310
|
+
function minimumBootstrapResamples(confidence, comparisonCount) {
|
|
2311
|
+
const exact = 2 * comparisonCount / (1 - confidence);
|
|
2312
|
+
return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
|
|
2313
|
+
}
|
|
2314
|
+
//#endregion
|
|
2130
2315
|
//#region src/campaign/presets/compare-optimization-methods.ts
|
|
2131
2316
|
/**
|
|
2132
2317
|
* Compare optimization methods on shared train, selection, and test data.
|
|
@@ -2140,16 +2325,21 @@ function optimizerAccounting(tokenUsage, proposerCostUsd) {
|
|
|
2140
2325
|
/**
|
|
2141
2326
|
* Compare complete optimization methods on disjoint train, selection, and final test data.
|
|
2142
2327
|
*/
|
|
2143
|
-
async function compareOptimizationMethods(
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2328
|
+
async function compareOptimizationMethods(rawOpts) {
|
|
2329
|
+
return comparisonResult(pairwiseAgainstBest(scoreMethods(await measureWinners(await reserveTestMeasurement(await optimizeAllMethods(await finalTestScoring(normalizeComparisonOptions({ rawOpts }))))))));
|
|
2330
|
+
}
|
|
2331
|
+
/** The captured options, the derived statistics settings, the run directory and the cost ledger. */
|
|
2332
|
+
function normalizeComparisonOptions(prev) {
|
|
2333
|
+
const { rawOpts } = prev;
|
|
2334
|
+
const opts = {
|
|
2335
|
+
...rawOpts,
|
|
2336
|
+
methods: rawOpts.methods.map((method) => ({ ...method })),
|
|
2337
|
+
trainScenarios: structuredClone(rawOpts.trainScenarios),
|
|
2338
|
+
selectionScenarios: structuredClone(rawOpts.selectionScenarios),
|
|
2339
|
+
testScenarios: structuredClone(rawOpts.testScenarios),
|
|
2340
|
+
judges: rawOpts.judges.map(captureJudge),
|
|
2341
|
+
claim: rawOpts.claim && defineEvaluationClaim(rawOpts.claim),
|
|
2342
|
+
finalEvidence: rawOpts.finalEvidence && captureFinalEvidencePolicy(rawOpts.finalEvidence)
|
|
2153
2343
|
};
|
|
2154
2344
|
assertOptimizationMethods(opts.methods);
|
|
2155
2345
|
assertComparisonPartitions(opts);
|
|
@@ -2174,6 +2364,28 @@ async function compareOptimizationMethods(opts) {
|
|
|
2174
2364
|
runDir: `${resolvedRunDir}/cost`,
|
|
2175
2365
|
costCeilingUsd: opts.costCeiling
|
|
2176
2366
|
});
|
|
2367
|
+
return {
|
|
2368
|
+
...prev,
|
|
2369
|
+
opts,
|
|
2370
|
+
searchHistoryPolicy,
|
|
2371
|
+
seed,
|
|
2372
|
+
confidence,
|
|
2373
|
+
optimizationConcurrency,
|
|
2374
|
+
comparisonCount,
|
|
2375
|
+
intervalConfidence,
|
|
2376
|
+
minimumResamples,
|
|
2377
|
+
resamples,
|
|
2378
|
+
evidence,
|
|
2379
|
+
evidenceBySurface,
|
|
2380
|
+
storage,
|
|
2381
|
+
resolvedRunDir,
|
|
2382
|
+
baselineSurface,
|
|
2383
|
+
costLedger
|
|
2384
|
+
};
|
|
2385
|
+
}
|
|
2386
|
+
/** Scoring a surface on the final test set, and aligning the scores to the designed scenarios. */
|
|
2387
|
+
async function finalTestScoring(prev) {
|
|
2388
|
+
const { opts, seed, intervalConfidence, resamples, evidence, evidenceBySurface, storage, resolvedRunDir, costLedger } = prev;
|
|
2177
2389
|
const scoreOnTest = async (surface, tag, costPhase) => {
|
|
2178
2390
|
const measuredSurface = structuredClone(surface);
|
|
2179
2391
|
const campaign = await runCampaign({
|
|
@@ -2227,6 +2439,20 @@ async function compareOptimizationMethods(opts) {
|
|
|
2227
2439
|
if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
|
|
2228
2440
|
return scenarioIds.map((id) => byScenario[id]);
|
|
2229
2441
|
};
|
|
2442
|
+
return {
|
|
2443
|
+
...prev,
|
|
2444
|
+
scoreOnTest,
|
|
2445
|
+
scenarioIds,
|
|
2446
|
+
unitByScenario,
|
|
2447
|
+
units,
|
|
2448
|
+
aggregateUnits,
|
|
2449
|
+
decisionOptions,
|
|
2450
|
+
align
|
|
2451
|
+
};
|
|
2452
|
+
}
|
|
2453
|
+
/** Reserve final evidence, then finish every method before the first final-test call. */
|
|
2454
|
+
async function optimizeAllMethods(prev) {
|
|
2455
|
+
const { opts, seed, optimizationConcurrency, storage, resolvedRunDir, baselineSurface, costLedger } = prev;
|
|
2230
2456
|
if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
|
|
2231
2457
|
if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
|
|
2232
2458
|
const optimizationOwner = new AbortController();
|
|
@@ -2258,7 +2484,16 @@ async function compareOptimizationMethods(opts) {
|
|
|
2258
2484
|
label: `method '${result.name}'`,
|
|
2259
2485
|
cost: result.cost
|
|
2260
2486
|
}))).totalCostUsd, opts.costCeiling, "optimization");
|
|
2261
|
-
|
|
2487
|
+
return {
|
|
2488
|
+
...prev,
|
|
2489
|
+
optimizationOwner,
|
|
2490
|
+
optimized
|
|
2491
|
+
};
|
|
2492
|
+
}
|
|
2493
|
+
/** The identity of the final test and the exposure of its evidence. */
|
|
2494
|
+
async function reserveTestMeasurement(prev) {
|
|
2495
|
+
const { opts, seed, baselineSurface, optimized } = prev;
|
|
2496
|
+
const testCostIdentity = contentHash({
|
|
2262
2497
|
baseline: surfaceContentHash(baselineSurface),
|
|
2263
2498
|
winners: optimized.map((winner) => ({
|
|
2264
2499
|
name: winner.name,
|
|
@@ -2276,8 +2511,19 @@ async function compareOptimizationMethods(opts) {
|
|
|
2276
2511
|
dispatch: callerDispatchRef(opts),
|
|
2277
2512
|
seed,
|
|
2278
2513
|
reps: opts.reps ?? 1
|
|
2279
|
-
})
|
|
2514
|
+
});
|
|
2515
|
+
const testCostPhase = `compareOptimizationMethods.test:${testCostIdentity}`;
|
|
2280
2516
|
const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
|
|
2517
|
+
return {
|
|
2518
|
+
...prev,
|
|
2519
|
+
testCostIdentity,
|
|
2520
|
+
testCostPhase,
|
|
2521
|
+
finalEvidence
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
/** One final-test measurement per distinct surface. */
|
|
2525
|
+
async function measureWinners(prev) {
|
|
2526
|
+
const { baselineSurface, scoreOnTest, align, optimized, testCostPhase } = prev;
|
|
2281
2527
|
const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
|
|
2282
2528
|
const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
|
|
2283
2529
|
const winners = [];
|
|
@@ -2293,6 +2539,16 @@ async function compareOptimizationMethods(opts) {
|
|
|
2293
2539
|
arr
|
|
2294
2540
|
});
|
|
2295
2541
|
}
|
|
2542
|
+
return {
|
|
2543
|
+
...prev,
|
|
2544
|
+
baselineArr,
|
|
2545
|
+
testScoresBySurface,
|
|
2546
|
+
winners
|
|
2547
|
+
};
|
|
2548
|
+
}
|
|
2549
|
+
/** Per-method scores, ranked by lift with cost breaking complete ties. */
|
|
2550
|
+
function scoreMethods(prev) {
|
|
2551
|
+
const { evidence, evidenceBySurface, baselineSurface, scenarioIds, units, aggregateUnits, decisionOptions, baselineArr, winners } = prev;
|
|
2296
2552
|
const scores = winners.map((w) => {
|
|
2297
2553
|
const baselineUnits = aggregateUnits(baselineArr);
|
|
2298
2554
|
const winnerUnits = aggregateUnits(w.arr);
|
|
@@ -2353,6 +2609,15 @@ async function compareOptimizationMethods(opts) {
|
|
|
2353
2609
|
s.rank = i + 1;
|
|
2354
2610
|
});
|
|
2355
2611
|
const best = scores[0];
|
|
2612
|
+
return {
|
|
2613
|
+
...prev,
|
|
2614
|
+
scores,
|
|
2615
|
+
best
|
|
2616
|
+
};
|
|
2617
|
+
}
|
|
2618
|
+
/** The best method against every other, in both directions. */
|
|
2619
|
+
function pairwiseAgainstBest(prev) {
|
|
2620
|
+
const { aggregateUnits, decisionOptions, winners, scores, best } = prev;
|
|
2356
2621
|
const byName = new Map(winners.map((w) => [w.name, w]));
|
|
2357
2622
|
const bestArr = aggregateUnits(byName.get(best.name).arr);
|
|
2358
2623
|
const pairwise = scores.slice(1).map((other) => {
|
|
@@ -2370,6 +2635,16 @@ async function compareOptimizationMethods(opts) {
|
|
|
2370
2635
|
decision
|
|
2371
2636
|
};
|
|
2372
2637
|
});
|
|
2638
|
+
return {
|
|
2639
|
+
...prev,
|
|
2640
|
+
byName,
|
|
2641
|
+
bestArr,
|
|
2642
|
+
pairwise
|
|
2643
|
+
};
|
|
2644
|
+
}
|
|
2645
|
+
/** Total cost against the ceiling, then the comparison record. */
|
|
2646
|
+
function comparisonResult(prev) {
|
|
2647
|
+
const { opts, searchHistoryPolicy, seed, confidence, comparisonCount, intervalConfidence, resamples, costLedger, scenarioIds, units, optimized, testCostPhase, finalEvidence, scores, best, pairwise } = prev;
|
|
2373
2648
|
const optimizationCost = combineComparisonCosts(scores.map((score) => ({
|
|
2374
2649
|
label: `method '${score.name}'`,
|
|
2375
2650
|
cost: score.optimizationCost
|
|
@@ -2431,43 +2706,6 @@ function assertOptimizationMethods(methods) {
|
|
|
2431
2706
|
pathOwners.set(pathKey, method.name);
|
|
2432
2707
|
}
|
|
2433
2708
|
}
|
|
2434
|
-
function assertComparisonControls(opts, seed, resamples, confidence) {
|
|
2435
|
-
if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
|
|
2436
|
-
if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
|
|
2437
|
-
if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
|
|
2438
|
-
if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
|
|
2439
|
-
const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
|
|
2440
|
-
if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
|
|
2441
|
-
if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
|
|
2442
|
-
try {
|
|
2443
|
-
surfaceContentHash(opts.baselineSurface);
|
|
2444
|
-
} catch (cause) {
|
|
2445
|
-
throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
|
|
2446
|
-
}
|
|
2447
|
-
const judgeNames = /* @__PURE__ */ new Set();
|
|
2448
|
-
for (const judge of opts.judges) {
|
|
2449
|
-
if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
|
|
2450
|
-
if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
|
|
2451
|
-
judgeNames.add(judge.name);
|
|
2452
|
-
const dimensionKeys = /* @__PURE__ */ new Set();
|
|
2453
|
-
for (const dimension of judge.dimensions) {
|
|
2454
|
-
if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
|
|
2455
|
-
if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
|
|
2456
|
-
dimensionKeys.add(dimension.key);
|
|
2457
|
-
}
|
|
2458
|
-
}
|
|
2459
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
|
|
2460
|
-
if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
|
|
2461
|
-
if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
|
|
2462
|
-
assertConfidence(confidence);
|
|
2463
|
-
const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
|
|
2464
|
-
if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
|
|
2465
|
-
if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
|
|
2466
|
-
if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
|
|
2467
|
-
if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
|
|
2468
|
-
if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
|
|
2469
|
-
if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
|
|
2470
|
-
}
|
|
2471
2709
|
function assertComparisonPartitions(opts) {
|
|
2472
2710
|
const partitions = [
|
|
2473
2711
|
{
|
|
@@ -2508,13 +2746,6 @@ function assertComparisonPartitions(opts) {
|
|
|
2508
2746
|
function slug(name) {
|
|
2509
2747
|
return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
|
|
2510
2748
|
}
|
|
2511
|
-
function assertConfidence(confidence) {
|
|
2512
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
|
|
2513
|
-
}
|
|
2514
|
-
function minimumBootstrapResamples(confidence, comparisonCount) {
|
|
2515
|
-
const exact = 2 * comparisonCount / (1 - confidence);
|
|
2516
|
-
return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
|
|
2517
|
-
}
|
|
2518
2749
|
function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
|
|
2519
2750
|
const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
|
|
2520
2751
|
const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
|
|
@@ -2626,298 +2857,347 @@ function assertSkillOptBridgeOutput(result, name, maxCandidateChars, maxEvaluati
|
|
|
2626
2857
|
function skillOptOptimizationMethod(config) {
|
|
2627
2858
|
assertSkillOptOptimizationConfig(config);
|
|
2628
2859
|
config = snapshotSkillOptOptimizationConfig(config);
|
|
2629
|
-
const
|
|
2860
|
+
const snapshot = config;
|
|
2861
|
+
const name = snapshot.name ?? "skillopt";
|
|
2630
2862
|
return {
|
|
2631
2863
|
name,
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
callbackToken: callback.token,
|
|
2783
|
-
objective: config.objective,
|
|
2784
|
-
...config.background ? { background: config.background } : {},
|
|
2785
|
-
trainer: config.trainer,
|
|
2786
|
-
optimizerModel: config.optimizer.model,
|
|
2787
|
-
modelBudget: config.optimizer.budget,
|
|
2788
|
-
seedCandidate: input.baselineSurface,
|
|
2789
|
-
trainSet,
|
|
2790
|
-
selectionSet,
|
|
2791
|
-
maxEvaluations: config.maxEvaluations,
|
|
2792
|
-
hardScoreThreshold: config.hardScoreThreshold ?? 1,
|
|
2793
|
-
maxCandidateChars,
|
|
2794
|
-
maxEvidenceChars,
|
|
2795
|
-
outputDir
|
|
2796
|
-
},
|
|
2797
|
-
runner: {
|
|
2798
|
-
...bridgeRunner,
|
|
2799
|
-
env: {
|
|
2800
|
-
...removeCredentialEnvironment(runnerEnv),
|
|
2801
|
-
OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
|
|
2802
|
-
OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
|
|
2803
|
-
OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
|
|
2804
|
-
OPTIMIZER_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
|
|
2805
|
-
TARGET_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
|
|
2806
|
-
TARGET_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
|
|
2807
|
-
OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
|
|
2808
|
-
OPENAI_COMPATIBLE_MAX_TOKENS: String(config.optimizer.budget.maxOutputTokensPerRequest),
|
|
2809
|
-
OPTIMIZER_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
|
|
2810
|
-
OPTIMIZER_OPENAI_COMPATIBLE_MAX_TOKENS: String(config.optimizer.budget.maxOutputTokensPerRequest),
|
|
2811
|
-
TARGET_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
|
|
2812
|
-
TARGET_OPENAI_COMPATIBLE_MAX_TOKENS: String(config.optimizer.budget.maxOutputTokensPerRequest)
|
|
2813
|
-
}
|
|
2814
|
-
},
|
|
2815
|
-
timeoutMs: config.timeoutMs ?? 36e5,
|
|
2816
|
-
...signal ? { signal } : {}
|
|
2817
|
-
}),
|
|
2818
|
-
outputDir,
|
|
2819
|
-
modelProxy
|
|
2820
|
-
};
|
|
2821
|
-
},
|
|
2822
|
-
cleanup: closeResources
|
|
2823
|
-
});
|
|
2824
|
-
signal?.throwIfAborted();
|
|
2825
|
-
assertSkillOptBridgeOutput(result, name, maxCandidateChars, config.maxEvaluations);
|
|
2826
|
-
assertExternalOptimizerRunBinding({
|
|
2827
|
-
label: name,
|
|
2828
|
-
runtime: runtimeIdentity,
|
|
2829
|
-
returnedSource: result.upstream,
|
|
2830
|
-
compatibleRunId,
|
|
2831
|
-
runId,
|
|
2832
|
-
returnedRunId: result.runId,
|
|
2833
|
-
resume,
|
|
2834
|
-
resumed: result.resumed
|
|
2835
|
-
});
|
|
2836
|
-
if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: SkillOpt reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
|
|
2837
|
-
const evaluationCost = costFromLedgerSummary(costLedger.summary({
|
|
2838
|
-
phase: "skillopt.external-evaluation",
|
|
2839
|
-
tags: runBudget.runTags
|
|
2840
|
-
}));
|
|
2841
|
-
const optimizerUsage = costLedger.summary({
|
|
2864
|
+
optimize: (input) => optimizeWithSkillOpt(snapshot, name, input)
|
|
2865
|
+
};
|
|
2866
|
+
}
|
|
2867
|
+
async function optimizeWithSkillOpt(config, name, input) {
|
|
2868
|
+
const signal = input.runOptions.signal;
|
|
2869
|
+
signal?.throwIfAborted();
|
|
2870
|
+
if (typeof input.baselineSurface !== "string") throw new Error(`${name}: SkillOpt requires a string baselineSurface`);
|
|
2871
|
+
const context = await prepareSkillOptRun(config, name, input);
|
|
2872
|
+
const { result, outputDir, modelProxy } = await runSkillOptBridge(context);
|
|
2873
|
+
signal?.throwIfAborted();
|
|
2874
|
+
return finishSkillOptRun(context, result, outputDir, modelProxy);
|
|
2875
|
+
}
|
|
2876
|
+
/** Resolve identities, open the run's budget and logs, and start the evaluation callback. */
|
|
2877
|
+
async function prepareSkillOptRun(config, name, input) {
|
|
2878
|
+
const signal = input.runOptions.signal;
|
|
2879
|
+
const started = Date.now();
|
|
2880
|
+
const maxCandidateChars = config.maxCandidateChars ?? 2e5;
|
|
2881
|
+
const maxEvidenceChars = config.maxEvidenceChars ?? 1e5;
|
|
2882
|
+
const storage = input.runOptions.storage ?? fsCampaignStorage();
|
|
2883
|
+
const runDir = `${input.runDir}/skillopt`;
|
|
2884
|
+
storage.ensureDir(runDir);
|
|
2885
|
+
const costLedger = input.costLedger;
|
|
2886
|
+
const attemptId = randomBytes(16).toString("hex");
|
|
2887
|
+
const resume = config.resume ?? "never";
|
|
2888
|
+
const bridgeRunner = config.runner ? {
|
|
2889
|
+
...config.runner,
|
|
2890
|
+
env: removeCredentialEnvironment(config.runner.env ?? {})
|
|
2891
|
+
} : void 0;
|
|
2892
|
+
const runtimeIdentity = await inspectExternalOptimizerRuntime({
|
|
2893
|
+
label: name,
|
|
2894
|
+
package: "skillopt",
|
|
2895
|
+
module: "agent_eval_rpc.skillopt_bridge",
|
|
2896
|
+
...bridgeRunner ? { runner: bridgeRunner } : {},
|
|
2897
|
+
timeoutMs: config.timeoutMs ?? 36e5,
|
|
2898
|
+
...signal ? { signal } : {}
|
|
2899
|
+
});
|
|
2900
|
+
const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario));
|
|
2901
|
+
const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario));
|
|
2902
|
+
const runMaterial = {
|
|
2903
|
+
optimizer: "skillopt",
|
|
2904
|
+
runtime: runtimeIdentity,
|
|
2905
|
+
method: name,
|
|
2906
|
+
evaluationId: config.evaluationId,
|
|
2907
|
+
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
2908
|
+
seed: input.seed,
|
|
2909
|
+
trainer: snapshotJson(config.trainer, "SkillOpt run settings"),
|
|
2910
|
+
objective: config.objective,
|
|
2911
|
+
background: config.background ?? "",
|
|
2912
|
+
optimizerModel: {
|
|
2913
|
+
model: config.optimizer.model,
|
|
2914
|
+
callRef: config.optimizer.callRef,
|
|
2915
|
+
budget: config.optimizer.budget
|
|
2916
|
+
},
|
|
2917
|
+
seedCandidate: input.baselineSurface,
|
|
2918
|
+
trainSet,
|
|
2919
|
+
selectionSet,
|
|
2920
|
+
maxEvaluations: config.maxEvaluations,
|
|
2921
|
+
hardScoreThreshold: config.hardScoreThreshold ?? 1,
|
|
2922
|
+
maxCandidateChars,
|
|
2923
|
+
maxEvidenceChars,
|
|
2924
|
+
evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
|
|
2925
|
+
runner: externalOptimizerRunnerIdentity(bridgeRunner, "agent_eval_rpc.skillopt_bridge")
|
|
2926
|
+
};
|
|
2927
|
+
const compatibleRunId = externalOptimizerCompatibleRunKey(runMaterial);
|
|
2928
|
+
const runId = externalOptimizerRunKey({
|
|
2929
|
+
material: runMaterial,
|
|
2930
|
+
attemptId,
|
|
2931
|
+
resumeEnabled: resume !== "never"
|
|
2932
|
+
});
|
|
2933
|
+
const runBudget = openExternalOptimizerRunBudget({
|
|
2934
|
+
storage,
|
|
2935
|
+
runDir,
|
|
2936
|
+
runKey: runId,
|
|
2937
|
+
attemptId,
|
|
2938
|
+
maxEvaluations: config.maxEvaluations
|
|
2939
|
+
});
|
|
2940
|
+
const observationLog = openExternalOptimizerObservationLog({
|
|
2941
|
+
storage,
|
|
2942
|
+
path: `${runDir}/observations-${attemptId}.jsonl`
|
|
2943
|
+
});
|
|
2944
|
+
const executionLog = openExternalOptimizerExecutionLog({
|
|
2945
|
+
storage,
|
|
2946
|
+
path: `${runDir}/model-executions-${attemptId}.jsonl`
|
|
2947
|
+
});
|
|
2948
|
+
const scenarioById = mapExternalScenarios(input.trainScenarios, input.selectionScenarios, "SkillOpt bridge");
|
|
2949
|
+
const evaluate = createExternalTextEvaluator({
|
|
2950
|
+
input,
|
|
2951
|
+
label: "SkillOpt bridge",
|
|
2952
|
+
runDir,
|
|
2953
|
+
compatibleRunId: runId,
|
|
2954
|
+
costPhase: "skillopt.external-evaluation",
|
|
2955
|
+
costTags: runBudget.attemptTags,
|
|
2956
|
+
costLedger,
|
|
2957
|
+
scenarioById,
|
|
2958
|
+
maxCandidateChars,
|
|
2959
|
+
maxEvidenceChars,
|
|
2960
|
+
describeArtifact: config.describeArtifact
|
|
2961
|
+
});
|
|
2962
|
+
let attemptEvaluationCount = 0;
|
|
2963
|
+
return {
|
|
2964
|
+
config,
|
|
2965
|
+
input,
|
|
2966
|
+
name,
|
|
2967
|
+
signal,
|
|
2968
|
+
started,
|
|
2969
|
+
maxCandidateChars,
|
|
2970
|
+
maxEvidenceChars,
|
|
2971
|
+
storage,
|
|
2972
|
+
runDir,
|
|
2973
|
+
attemptId,
|
|
2974
|
+
resume,
|
|
2975
|
+
bridgeRunner,
|
|
2976
|
+
runtimeIdentity,
|
|
2977
|
+
trainSet,
|
|
2978
|
+
selectionSet,
|
|
2979
|
+
compatibleRunId,
|
|
2980
|
+
runId,
|
|
2981
|
+
runBudget,
|
|
2982
|
+
observationLog,
|
|
2983
|
+
executionLog,
|
|
2984
|
+
callback: await startExternalOptimizerCallback({
|
|
2985
|
+
token: randomBytes(32).toString("hex"),
|
|
2986
|
+
maxEvaluations: config.maxEvaluations,
|
|
2987
|
+
acceptEvaluation: () => {
|
|
2988
|
+
if (runBudget.acceptEvaluation() === void 0) return void 0;
|
|
2989
|
+
attemptEvaluationCount += 1;
|
|
2990
|
+
return attemptEvaluationCount;
|
|
2991
|
+
},
|
|
2992
|
+
evaluate,
|
|
2993
|
+
observe: observationLog.observe,
|
|
2994
|
+
...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
|
|
2995
|
+
...signal ? { signal } : {}
|
|
2996
|
+
})
|
|
2997
|
+
};
|
|
2998
|
+
}
|
|
2999
|
+
/** Start the metered model proxy and run the Python bridge, always closing the callback and proxy. */
|
|
3000
|
+
async function runSkillOptBridge(context) {
|
|
3001
|
+
const { config, input, name, signal, runDir, runBudget, callback } = context;
|
|
3002
|
+
const costLedger = input.costLedger;
|
|
3003
|
+
const runnerEnv = context.bridgeRunner?.env ?? {};
|
|
3004
|
+
let activeModelProxy;
|
|
3005
|
+
const closeResources = () => closeExternalOptimizerResources({
|
|
3006
|
+
label: name,
|
|
3007
|
+
callback,
|
|
3008
|
+
...activeModelProxy ? { modelProxy: activeModelProxy } : {}
|
|
3009
|
+
});
|
|
3010
|
+
return runWithCleanup({
|
|
3011
|
+
label: `${name} optimizer resources`,
|
|
3012
|
+
run: async () => {
|
|
3013
|
+
const priorOptimizerUsage = costLedger.summary({
|
|
2842
3014
|
phase: "skillopt.optimizer-model",
|
|
2843
3015
|
tags: runBudget.runTags
|
|
2844
3016
|
});
|
|
2845
|
-
|
|
3017
|
+
assertPriorExternalOptimizerUsage(priorOptimizerUsage, config.optimizer.budget, name);
|
|
3018
|
+
const modelProxy = await startExternalOptimizerModelProxy({
|
|
3019
|
+
call: config.optimizer.call,
|
|
3020
|
+
callRef: config.optimizer.callRef,
|
|
3021
|
+
recordExecution: context.executionLog.observe,
|
|
3022
|
+
model: config.optimizer.model,
|
|
3023
|
+
budget: config.optimizer.budget,
|
|
3024
|
+
...config.optimizer.servedModelPolicy ? { servedModelPolicy: config.optimizer.servedModelPolicy } : {},
|
|
3025
|
+
costLedger,
|
|
2846
3026
|
phase: "skillopt.optimizer-model",
|
|
2847
|
-
|
|
3027
|
+
actor: name,
|
|
3028
|
+
tags: { ...runBudget.attemptTags },
|
|
3029
|
+
initialUsage: {
|
|
3030
|
+
requests: priorOptimizerUsage.totalCalls,
|
|
3031
|
+
...priorOptimizerUsage.costProvenance.kind === "uncaptured" ? {} : { costUsd: priorOptimizerUsage.totalCostUsd }
|
|
3032
|
+
},
|
|
3033
|
+
...signal ? { signal } : {}
|
|
2848
3034
|
});
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
const tokenUsage = optimizationTokenUsageFromSummary(optimizerUsage, optimizerReceipts);
|
|
2853
|
-
const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
|
|
2854
|
-
const combinedCost = combineComparisonCosts([{
|
|
2855
|
-
label: "evaluation",
|
|
2856
|
-
cost: evaluationCost
|
|
2857
|
-
}, {
|
|
2858
|
-
label: "optimizer model",
|
|
2859
|
-
cost: optimizerCost
|
|
2860
|
-
}]);
|
|
2861
|
-
const searchHistory = config.searchLedger ? await recordSkillOptSearch({
|
|
2862
|
-
name,
|
|
2863
|
-
path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
|
|
2864
|
-
searchId: runId,
|
|
2865
|
-
identity: config.searchLedger.identity,
|
|
2866
|
-
storage,
|
|
2867
|
-
seed: input.seed,
|
|
2868
|
-
baselineSurface: input.baselineSurface,
|
|
2869
|
-
trainScenarios: input.trainScenarios,
|
|
2870
|
-
selectionScenarios: input.selectionScenarios,
|
|
2871
|
-
evaluationLimit: config.maxEvaluations,
|
|
2872
|
-
observations: readExternalOptimizerObservationArtifact({
|
|
2873
|
-
summary: observationLog.summary(),
|
|
2874
|
-
storage
|
|
2875
|
-
}),
|
|
2876
|
-
selected: result.bestCandidate,
|
|
2877
|
-
generationAccounting: {
|
|
2878
|
-
tokens: optimizerUsage.usageComplete ? {
|
|
2879
|
-
status: "known",
|
|
2880
|
-
inputTokens: optimizerUsage.inputTokens,
|
|
2881
|
-
outputTokens: optimizerUsage.outputTokens,
|
|
2882
|
-
cachedTokens: optimizerUsage.cachedTokens
|
|
2883
|
-
} : {
|
|
2884
|
-
status: "unknown",
|
|
2885
|
-
reason: "an optimizer call reported no token usage"
|
|
2886
|
-
},
|
|
2887
|
-
cost: optimizerCost.accountingComplete ? {
|
|
2888
|
-
status: "known",
|
|
2889
|
-
usd: optimizerCost.totalCostUsd,
|
|
2890
|
-
source: "provider"
|
|
2891
|
-
} : {
|
|
2892
|
-
status: "unknown",
|
|
2893
|
-
knownLowerBoundUsd: optimizerCost.totalCostUsd,
|
|
2894
|
-
reason: optimizerCost.incompleteReasons.join("; ") || "optimizer spend incomplete"
|
|
2895
|
-
}
|
|
2896
|
-
}
|
|
2897
|
-
}) : void 0;
|
|
3035
|
+
activeModelProxy = modelProxy;
|
|
3036
|
+
const outputDir = `${runDir}/external`;
|
|
3037
|
+
await mkdir(outputDir, { recursive: true });
|
|
2898
3038
|
return {
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
3039
|
+
result: await runExternalOptimizerProcess({
|
|
3040
|
+
label: "SkillOpt bridge",
|
|
3041
|
+
tempPrefix: "agent-eval-skillopt-",
|
|
3042
|
+
module: "agent_eval_rpc.skillopt_bridge",
|
|
3043
|
+
input: {
|
|
3044
|
+
attemptId: context.attemptId,
|
|
3045
|
+
compatibleRunId: context.compatibleRunId,
|
|
3046
|
+
runId: context.runId,
|
|
3047
|
+
runtimeIdentity: context.runtimeIdentity,
|
|
3048
|
+
resume: context.resume,
|
|
3049
|
+
evaluationId: config.evaluationId,
|
|
3050
|
+
seed: input.seed,
|
|
3051
|
+
callbackUrl: callback.url,
|
|
3052
|
+
callbackToken: callback.token,
|
|
3053
|
+
objective: config.objective,
|
|
3054
|
+
...config.background ? { background: config.background } : {},
|
|
3055
|
+
trainer: config.trainer,
|
|
3056
|
+
optimizerModel: config.optimizer.model,
|
|
3057
|
+
modelBudget: config.optimizer.budget,
|
|
3058
|
+
seedCandidate: input.baselineSurface,
|
|
3059
|
+
trainSet: context.trainSet,
|
|
3060
|
+
selectionSet: context.selectionSet,
|
|
3061
|
+
maxEvaluations: config.maxEvaluations,
|
|
3062
|
+
hardScoreThreshold: config.hardScoreThreshold ?? 1,
|
|
3063
|
+
maxCandidateChars: context.maxCandidateChars,
|
|
3064
|
+
maxEvidenceChars: context.maxEvidenceChars,
|
|
3065
|
+
outputDir
|
|
3066
|
+
},
|
|
3067
|
+
runner: {
|
|
3068
|
+
...context.bridgeRunner,
|
|
3069
|
+
env: {
|
|
3070
|
+
...removeCredentialEnvironment(runnerEnv),
|
|
3071
|
+
...skillOptModelEnvironment(config, modelProxy)
|
|
3072
|
+
}
|
|
3073
|
+
},
|
|
3074
|
+
timeoutMs: config.timeoutMs ?? 36e5,
|
|
3075
|
+
...signal ? { signal } : {}
|
|
3076
|
+
}),
|
|
3077
|
+
outputDir,
|
|
3078
|
+
modelProxy
|
|
2916
3079
|
};
|
|
3080
|
+
},
|
|
3081
|
+
cleanup: closeResources
|
|
3082
|
+
});
|
|
3083
|
+
}
|
|
3084
|
+
/** Point every OpenAI-compatible client the bridge uses at the metered model proxy. */
|
|
3085
|
+
function skillOptModelEnvironment(config, modelProxy) {
|
|
3086
|
+
const maxTokens = String(config.optimizer.budget.maxOutputTokensPerRequest);
|
|
3087
|
+
return {
|
|
3088
|
+
OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
|
|
3089
|
+
OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
|
|
3090
|
+
OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
|
|
3091
|
+
OPTIMIZER_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
|
|
3092
|
+
TARGET_OPENAI_COMPATIBLE_BASE_URL: modelProxy.baseUrl,
|
|
3093
|
+
TARGET_OPENAI_COMPATIBLE_API_KEY: modelProxy.apiKey,
|
|
3094
|
+
OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
|
|
3095
|
+
OPENAI_COMPATIBLE_MAX_TOKENS: maxTokens,
|
|
3096
|
+
OPTIMIZER_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
|
|
3097
|
+
OPTIMIZER_OPENAI_COMPATIBLE_MAX_TOKENS: maxTokens,
|
|
3098
|
+
TARGET_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
|
|
3099
|
+
TARGET_OPENAI_COMPATIBLE_MAX_TOKENS: maxTokens
|
|
3100
|
+
};
|
|
3101
|
+
}
|
|
3102
|
+
/** Validate the bridge output against the run, total its spend, and build the method result. */
|
|
3103
|
+
async function finishSkillOptRun(context, result, outputDir, modelProxy) {
|
|
3104
|
+
const { config, input, name, runBudget, callback, observationLog, executionLog } = context;
|
|
3105
|
+
const { runtimeIdentity, compatibleRunId, runId } = context;
|
|
3106
|
+
const costLedger = input.costLedger;
|
|
3107
|
+
assertSkillOptBridgeOutput(result, name, context.maxCandidateChars, config.maxEvaluations);
|
|
3108
|
+
assertExternalOptimizerRunBinding({
|
|
3109
|
+
label: name,
|
|
3110
|
+
runtime: runtimeIdentity,
|
|
3111
|
+
returnedSource: result.upstream,
|
|
3112
|
+
compatibleRunId,
|
|
3113
|
+
runId,
|
|
3114
|
+
returnedRunId: result.runId,
|
|
3115
|
+
resume: context.resume,
|
|
3116
|
+
resumed: result.resumed
|
|
3117
|
+
});
|
|
3118
|
+
if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: SkillOpt reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
|
|
3119
|
+
const evaluationCost = costFromLedgerSummary(costLedger.summary({
|
|
3120
|
+
phase: "skillopt.external-evaluation",
|
|
3121
|
+
tags: runBudget.runTags
|
|
3122
|
+
}));
|
|
3123
|
+
const optimizerUsage = costLedger.summary({
|
|
3124
|
+
phase: "skillopt.optimizer-model",
|
|
3125
|
+
tags: runBudget.runTags
|
|
3126
|
+
});
|
|
3127
|
+
const optimizerReceipts = costLedger.list({
|
|
3128
|
+
phase: "skillopt.optimizer-model",
|
|
3129
|
+
tags: runBudget.runTags
|
|
3130
|
+
});
|
|
3131
|
+
const optimizerCost = costFromLedgerSummary(optimizerUsage);
|
|
3132
|
+
modelProxy.assertExecutionComplete();
|
|
3133
|
+
assertExternalOptimizerCompletionCount(result.tokenUsage, modelProxy.requestAttempts(), modelProxy.successfulCompletions(), name, "SkillOpt");
|
|
3134
|
+
const tokenUsage = optimizationTokenUsageFromSummary(optimizerUsage, optimizerReceipts);
|
|
3135
|
+
const runtime = observedExternalOptimizerRuntime(runtimeIdentity);
|
|
3136
|
+
const combinedCost = combineComparisonCosts([{
|
|
3137
|
+
label: "evaluation",
|
|
3138
|
+
cost: evaluationCost
|
|
3139
|
+
}, {
|
|
3140
|
+
label: "optimizer model",
|
|
3141
|
+
cost: optimizerCost
|
|
3142
|
+
}]);
|
|
3143
|
+
const searchHistory = config.searchLedger ? await recordSkillOptSearch({
|
|
3144
|
+
name,
|
|
3145
|
+
path: config.searchLedger.path ?? `${context.runDir}/search-ledger.jsonl`,
|
|
3146
|
+
searchId: runId,
|
|
3147
|
+
identity: config.searchLedger.identity,
|
|
3148
|
+
storage: context.storage,
|
|
3149
|
+
seed: input.seed,
|
|
3150
|
+
baselineSurface: input.baselineSurface,
|
|
3151
|
+
trainScenarios: input.trainScenarios,
|
|
3152
|
+
selectionScenarios: input.selectionScenarios,
|
|
3153
|
+
evaluationLimit: config.maxEvaluations,
|
|
3154
|
+
observations: readExternalOptimizerObservationArtifact({
|
|
3155
|
+
summary: observationLog.summary(),
|
|
3156
|
+
storage: context.storage
|
|
3157
|
+
}),
|
|
3158
|
+
selected: result.bestCandidate,
|
|
3159
|
+
generationAccounting: {
|
|
3160
|
+
tokens: optimizerUsage.usageComplete ? {
|
|
3161
|
+
status: "known",
|
|
3162
|
+
inputTokens: optimizerUsage.inputTokens,
|
|
3163
|
+
outputTokens: optimizerUsage.outputTokens,
|
|
3164
|
+
cachedTokens: optimizerUsage.cachedTokens
|
|
3165
|
+
} : {
|
|
3166
|
+
status: "unknown",
|
|
3167
|
+
reason: "an optimizer call reported no token usage"
|
|
3168
|
+
},
|
|
3169
|
+
cost: optimizerCost.accountingComplete ? {
|
|
3170
|
+
status: "known",
|
|
3171
|
+
usd: optimizerCost.totalCostUsd,
|
|
3172
|
+
source: "provider"
|
|
3173
|
+
} : {
|
|
3174
|
+
status: "unknown",
|
|
3175
|
+
knownLowerBoundUsd: optimizerCost.totalCostUsd,
|
|
3176
|
+
reason: optimizerCost.incompleteReasons.join("; ") || "optimizer spend incomplete"
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
}) : void 0;
|
|
3180
|
+
return {
|
|
3181
|
+
winnerSurface: result.bestCandidate,
|
|
3182
|
+
...searchHistory ? { searchHistory } : {},
|
|
3183
|
+
cost: combinedCost,
|
|
3184
|
+
durationMs: Date.now() - context.started,
|
|
3185
|
+
provenance: {
|
|
3186
|
+
...runtime,
|
|
3187
|
+
optimizerModel: config.optimizer.model,
|
|
3188
|
+
optimizerCallRef: config.optimizer.callRef,
|
|
3189
|
+
compatibleRunId,
|
|
3190
|
+
runId,
|
|
3191
|
+
resumed: result.resumed,
|
|
3192
|
+
evaluationCount: runBudget.acceptedEvaluations(),
|
|
3193
|
+
artifactDir: outputDir,
|
|
3194
|
+
...tokenUsage ? { tokenUsage } : {},
|
|
3195
|
+
observations: observationLog.summary(),
|
|
3196
|
+
modelExecutions: executionLog.summary()
|
|
2917
3197
|
}
|
|
2918
3198
|
};
|
|
2919
3199
|
}
|
|
2920
3200
|
//#endregion
|
|
2921
3201
|
export { importExternalEvaluations as a, composeGate as c, readExternalOptimizerObservationArtifact as d, externalSurface as i, externalTextOptimizationMethod as l, compareOptimizationMethods as n, importGepaPopulation as o, gepaOptimizationMethod as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
|
|
2922
3202
|
|
|
2923
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
3203
|
+
//# sourceMappingURL=skillopt-optimization-method-5dlex7MJ.js.map
|