@tangle-network/agent-eval 0.173.2 → 0.174.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/dist/{proposal-findings-bko3GGy-.js → abort-signal-CtzAM_sJ.js} +11 -11
- package/dist/abort-signal-CtzAM_sJ.js.map +1 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/agent-profile-_xPxqVJt.d.ts +488 -0
- package/dist/agent-profile-_xPxqVJt.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +7 -9
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +8 -8
- package/dist/{benchmark-C4wk_Sjr.js → benchmark-DQKzykkO.js} +2 -2
- package/dist/{benchmark-C4wk_Sjr.js.map → benchmark-DQKzykkO.js.map} +1 -1
- package/dist/{benchmark-command-CS6gVHVq.js → benchmark-command-mZIlR-ra.js} +13 -13
- package/dist/{benchmark-command-CS6gVHVq.js.map → benchmark-command-mZIlR-ra.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -4
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -9
- package/dist/campaign/index.js +7 -7
- package/dist/{campaign-B3kPMU8S.js → campaign-BzMSCejE.js} +8 -8
- package/dist/{campaign-B3kPMU8S.js.map → campaign-BzMSCejE.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-DlqdbM7n.d.ts → client-vyYQg3bm.d.ts} +2 -2
- package/dist/{client-DlqdbM7n.d.ts.map → client-vyYQg3bm.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -10
- package/dist/contract/index.js +8 -8
- package/dist/{default-registry-B0bKikCb.js → default-registry-CrAp0pYq.js} +4 -4
- package/dist/{default-registry-B0bKikCb.js.map → default-registry-CrAp0pYq.js.map} +1 -1
- package/dist/{default-registry-BKwc8bN5.d.ts → default-registry-FfNzaUHV.d.ts} +3 -3
- package/dist/{default-registry-BKwc8bN5.d.ts.map → default-registry-FfNzaUHV.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CY6qdlGV.d.ts → define-agent-eval-V1jQyCDR.d.ts} +102 -11
- package/dist/define-agent-eval-V1jQyCDR.d.ts.map +1 -0
- package/dist/{define-agent-eval-8h3lXXee.js → define-agent-eval-ox5McL6e.js} +331 -144
- package/dist/define-agent-eval-ox5McL6e.js.map +1 -0
- package/dist/{dspy-rlm-engine-CF0t2ITD.js → dspy-rlm-engine-Caz2pl4L.js} +3 -3
- package/dist/{dspy-rlm-engine-CF0t2ITD.js.map → dspy-rlm-engine-Caz2pl4L.js.map} +1 -1
- package/dist/{engine-DhFir3Ys.d.ts → engine-CvW_I72-.d.ts} +2 -2
- package/dist/{engine-DhFir3Ys.d.ts.map → engine-CvW_I72-.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +1 -4
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/{external-optimizer-process-BwITA9Jp.js → external-optimizer-process-CxnFL1hd.js} +2 -2
- package/dist/{external-optimizer-process-BwITA9Jp.js.map → external-optimizer-process-CxnFL1hd.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-wBWeoG6A.js → external-optimizer-subprocess-CQi27uEI.js} +2 -2
- package/dist/{external-optimizer-subprocess-wBWeoG6A.js.map → external-optimizer-subprocess-CQi27uEI.js.map} +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/fuzz.js.map +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/{index-BQqOjerE.d.ts → index-BTrx5s8m.d.ts} +8 -9
- package/dist/index-BTrx5s8m.d.ts.map +1 -0
- package/dist/{index-D0Db5X-4.d.ts → index-Bn-nlnSV.d.ts} +4 -4
- package/dist/{index-D0Db5X-4.d.ts.map → index-Bn-nlnSV.d.ts.map} +1 -1
- package/dist/index-DKXuBPXf.d.ts +3840 -0
- package/dist/index-DKXuBPXf.d.ts.map +1 -0
- package/dist/index.d.ts +11 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -10
- package/dist/{kind-factory-gP6lDySe.js → kind-factory-BLvL-E44.js} +2 -2
- package/dist/{kind-factory-gP6lDySe.js.map → kind-factory-BLvL-E44.js.map} +1 -1
- package/dist/{llm-judge-BfqMFo4h.js → llm-judge-DmNaBrXB.js} +2541 -2435
- package/dist/llm-judge-DmNaBrXB.js.map +1 -0
- package/dist/{matrix-DGu8KhSs.d.ts → matrix-CJtXz1ky.d.ts} +2 -2
- package/dist/{matrix-DGu8KhSs.d.ts.map → matrix-CJtXz1ky.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-D91uDvQw.js → produced-state-B8mw6zj9.js} +2 -2
- package/dist/{produced-state-D91uDvQw.js.map → produced-state-B8mw6zj9.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js.map +1 -1
- package/dist/{semantic-concept-judge-Dok7_35a.js → semantic-concept-judge-E3s_fEjB.js} +3 -3
- package/dist/{semantic-concept-judge-Dok7_35a.js.map → semantic-concept-judge-E3s_fEjB.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DDw3v3gA.js → skillopt-optimization-method-f7399oGb.js} +5 -5
- package/dist/{skillopt-optimization-method-DDw3v3gA.js.map → skillopt-optimization-method-f7399oGb.js.map} +1 -1
- package/dist/statistical-heldout-Cqb73yE9.d.ts +1127 -0
- package/dist/statistical-heldout-Cqb73yE9.d.ts.map +1 -0
- package/dist/{store-otlp-Dow0pk_5.js → store-otlp-DV_H2HDu.js} +2 -2
- package/dist/{store-otlp-Dow0pk_5.js.map → store-otlp-DV_H2HDu.js.map} +1 -1
- package/dist/{store-tool-spans-CCZNsihA.d.ts → store-tool-spans-4o55ABER.d.ts} +3 -3
- package/dist/{store-tool-spans-CCZNsihA.d.ts.map → store-tool-spans-4o55ABER.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CeNj_m2L.js → store-tool-spans-B9tjys_h.js} +3 -3
- package/dist/{store-tool-spans-CeNj_m2L.js.map → store-tool-spans-B9tjys_h.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +25 -7
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{task-failure-attributes-CZjZeBsY.js → task-failure-attributes-CUy9mkIY.js} +2 -2
- package/dist/{task-failure-attributes-CZjZeBsY.js.map → task-failure-attributes-CUy9mkIY.js.map} +1 -1
- package/dist/{tool-groups-Cp4Xdzrp.d.ts → tool-groups-DAe1t6zb.d.ts} +2 -2
- package/dist/tool-groups-DAe1t6zb.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/traces.js +4 -4
- package/dist/{types-Ba5UQyVD.d.ts → types-BJz2CPTM.d.ts} +2 -2
- package/dist/{types-Ba5UQyVD.d.ts.map → types-BJz2CPTM.d.ts.map} +1 -1
- package/dist/{types-CiWITkGo.js → types-DQ0e2E7y.js} +2 -2
- package/dist/types-DQ0e2E7y.js.map +1 -0
- package/dist/{types-BDV4PiMR.d.ts → types-Dd1ejaeI.d.ts} +2 -2
- package/dist/{types-BDV4PiMR.d.ts.map → types-Dd1ejaeI.d.ts.map} +1 -1
- package/docs/campaign-proposers.md +42 -0
- package/package.json +1 -1
- package/dist/agent-profile-B9_GGsG8.d.ts +0 -84
- package/dist/agent-profile-B9_GGsG8.d.ts.map +0 -1
- package/dist/backend-integrity-CeuTgqsd.d.ts +0 -280
- package/dist/backend-integrity-CeuTgqsd.d.ts.map +0 -1
- package/dist/benchmark-BjLGkfnN.d.ts +0 -236
- package/dist/benchmark-BjLGkfnN.d.ts.map +0 -1
- package/dist/define-agent-eval-8h3lXXee.js.map +0 -1
- package/dist/define-agent-eval-CY6qdlGV.d.ts.map +0 -1
- package/dist/external-optimizer-contracts-CQCpyrIL.d.ts +0 -172
- package/dist/external-optimizer-contracts-CQCpyrIL.d.ts.map +0 -1
- package/dist/heldout-gate-Df5hsqmm.d.ts +0 -453
- package/dist/heldout-gate-Df5hsqmm.d.ts.map +0 -1
- package/dist/index-BQqOjerE.d.ts.map +0 -1
- package/dist/index-CFDffsKz.d.ts +0 -1135
- package/dist/index-CFDffsKz.d.ts.map +0 -1
- package/dist/llm-judge-BfqMFo4h.js.map +0 -1
- package/dist/power-preflight-Ptse_Kq7.d.ts +0 -117
- package/dist/power-preflight-Ptse_Kq7.d.ts.map +0 -1
- package/dist/pre-registration-BoI4ucR3.d.ts +0 -592
- package/dist/pre-registration-BoI4ucR3.d.ts.map +0 -1
- package/dist/promotion-policy-CvMda3kU.d.ts +0 -134
- package/dist/promotion-policy-CvMda3kU.d.ts.map +0 -1
- package/dist/proposal-findings-bko3GGy-.js.map +0 -1
- package/dist/provenance-CRY67X50.d.ts +0 -1995
- package/dist/provenance-CRY67X50.d.ts.map +0 -1
- package/dist/statistical-heldout-DTyB_6-1.d.ts +0 -295
- package/dist/statistical-heldout-DTyB_6-1.d.ts.map +0 -1
- package/dist/tool-groups-Cp4Xdzrp.d.ts.map +0 -1
- package/dist/types-CiWITkGo.js.map +0 -1
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"provenance-CRY67X50.d.ts","names":[],"sources":["../src/campaign/storage.ts","../src/campaign/cell-schedule.ts","../src/campaign/cell-cache.ts","../src/campaign/plan-campaign-run.ts","../src/campaign/run-campaign.ts","../src/campaign/presets/run-eval.ts","../src/campaign/auto-pr.ts","../src/campaign/parent-selection.ts","../src/campaign/search-ledger.ts","../src/campaign/search-history-receipt.ts","../src/campaign/gepa-candidate-population.ts","../src/campaign/search-ledger-recording.ts","../src/campaign/presets/run-optimization.ts","../src/campaign/presets/run-improvement-loop.ts","../src/campaign/transient-failure.ts","../src/llm-judge.ts","../src/campaign/external-optimizer-observations.ts","../src/campaign/presets/compare-optimization-methods.ts","../src/canary.ts","../src/red-team.ts","../src/campaign/provenance.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;UAoBiB;;EAEf,UAAU;;EAEV,OAAO;;EAEP,KAAK;;EAEL,MAAM,cAAc,kBAAkB;;;EAGtC,OAAO,cAAc,iBAAiB;;;;;;;;;iBAUxB,qBAAqB;;;;iBA0CrB,2BAA2B;;iBAmC3B,oBAAoB;EAClC,SAAS;EACT;EACA;;EAEA;IACE;;;iBClHY,kBAAkB,kBAAkB,UAClD,WAAW,aACX,cACA,eACC;EAAQ,UAAU;EAAW;EAAa;EAAgB;;KA4BjD,iBAAiB,kBAAkB,YAAY,kBAClD,kBAAkB;iBAOX,cAAc,gBAAgB;;;KCkElC;KASA,UAAU;EAChB;EAAe,MAAM,mBAAmB;;EACxC;EAAgB,QAAQ;;iBAEd,eAAe,WAAW;EACxC,SAAS;EACT;EACA;EACA;IACE,UAAU;;;UCxHG;EACf;EACA;EACA;EACA;EACA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;UAGQ,uBAAuB,kBAAkB,UAAU;EAClE,WAAW;EACX,WAAW,WAAW,WAAW;EACjC;EACA,SAAS,YAAY,WAAW;EAChC;EACA;EACA;;EAEA;EACA;;EAEA;EACA,UAAU;;EAEV,aAAa;;EAEb,WAAW,SAAS;;;;;;iBAON,gBAAgB,kBAAkB,UAAU,WAC1D,MAAM,uBAAuB,WAAW,aACvC;;;UCjBc,mBAAmB,kBAAkB,UAAU;EAC9D,WAAW;EACX,UAAU,WAAW,WAAW;;EAEhC,SAAS;;;;;;EAMT;EACA,SAAS,YAAY,WAAW;;EAEhC;;;EAGA;;;EAGA;;;;;;EAMA,cAAc;IAAS,UAAU;IAAW;;;;;;EAK5C;;;;;;EAMA;;;;EAIA,eAAe;EACf;EACA;;EAEA;;;EAGA,aAAa;;EAEb;;EAEA,WAAW,SAAS;;EAEpB;;;;;;;;;;EAUA;;;;;;;;EAQA,YAAY;;;;;;;;;;EAUZ;;;;;EAKA;;;;EAIA;;;EAGA;;;;EAIA;;;;;;;;;;;EAWA;;EAEA,YAAY;;EAEZ,oBAAoB,gBAAgB,gBAAgB;;;;;;EAMpD,UAAU;;;;;;;;;;;;EAYV,iBAAiB;IACf,UAAU;IACV;IACA;;;;;;;UAQa,2BAA2B;EAC1C;EACA;EACA;EACA;IACE;IACA;IACA;MACE;MACA;MACA;;;EAGJ,MAAM,mBAAmB;EACzB,MAAM;;;;;;;;;;;;UAaS;;EAEf;;;;;EAKA,YAAY,SAAS;;;;;iBAMD,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,eAAe,WAAW;;;UCvNpB,eAAe,kBAAkB,UAAU,mBAClD,KAAK,mBAAmB,WAAW;EAC3C;;;;;iBAMoB,QAAQ,kBAAkB,UAAU,WACxD,MAAM,eAAe,WAAW,aAC/B,QAAQ,eAAe,WAAW;;;UCHpB,kBAAkB,WAAW,kBAAkB;;EAE9D,QAAQ,eAAe,WAAW;;;EAGlC,MAAM;;;EAGN;;EAEA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA,UAAU;IAAqB;IAAgB;IAAgB;;;UAGhD;EACf;EACA;EACA;EACA;;;;;iBAMc,WAAW,WAAW,kBAAkB,UACtD,SAAS,kBAAkB,WAAW,aACrC;;;;UCtCc;;;WAGN,UAAU,cAAc;;;WAGxB,WAAW;;WAEX,SAAS,cAAc;;WAEvB;;;;;KAMC,kBAAkB,KAAK,2BAA2B;UAE7C;;;EAGf;;;;;;;;;;;iBAYc,sBAAsB,SAAS,+BAA+B;;;cCPxE;KAEM,mBAAmB;KAEnB;;;UAYK;EACf;EACA;EACA,QAAQ;EACR;;;;UAKe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,MAAM;EACN,UAAU;;UAGK;;EAEf;EACA;EACA;EACA;EACA,gBAAgB;;KAGN;UAOK;EACf;EACA,QAAQ;EACR,WAAW;;;EAGX;;UAGe;EACf;EACA,MAAM;;UAGS;EACf;;;EAGA;;UAGe;;EAEf,gBAAgB;;EAEhB,OAAO;;EAEP,YAAY;;KAGF;EAEN;EACA;EACA;EACA;;EAGA;EACA;;KAGM;EAEN;EACA;EACA;;EAGA;;EAEA;EACA;;UAGW;EACf,QAAQ;EACR,MAAM;;UAGS;EACf;EACA;;KAGU;EAEN;EACA;EACA,SAAS;;EAGT;EACA;EACA,SAAS;EACT,SAAS;;EAGT;EACA,SAAS;EACT,OAAO;IAAwB;;;KAGzB;EAEN;EACA;EACA;EACA;EACA;;EAGA;EACA;;;;UAKW;EACf;EACA;EACA;EACA,QAAQ;EACR,UAAU;;UAGF;EACR;EACA;EACA,WAAW;;UAGI,2BAA2B;EAC1C;EACA,MAAM;;;;;;UAOS,gCAAgC;EAC/C;EACA;IACE,gBAAgB;IAChB,YAAY;;;UAIC,uCAAuC;EACtD;EACA;EACA;EACA;EACA,SAAS;EACT,UAAU;;UAGK,uCAAuC;EACtD;EACA;EACA;EACA,QAAQ;;UAGO,iCAAiC;EAChD;EACA;EACA;EACA;EACA;IACE;IACA,QAAQ;;EAEV;IACE,OAAO;IACP,OAAO;IACP,WAAW;;EAEb,SAAS;EACT,YAAY;EACZ,iBAAiB;;UAGF,qCAAqC;EACpD;EACA;EACA,eAAe;EACf;IAEM;IACA,OAAO;IACP,QAAQ;;IAGR;IACA,QAAQ;;EAEd;IACM;;IACA;IAAmB,SAAS;;IAC5B;IAAkB,SAAS;;EACjC,YAAY;;UAGG,oCAAoC;EACnD;EACA;EACA;IACM;;IAEA;IACA,QAAQ;;;UAIC,6BAA6B;EAC5C;EACA;IAEM;IACA;;IAGA;IACA,QAAQ;;;KAIJ,oBACR,qBACA,0BACA,iCACA,iCACA,2BACA,+BACA,8BACA;UAEa;EACf,eAAe;EACf;EACA;EACA,cAAc;EACd,OAAO;EACP,WAAW;;KAGD;EAEN;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IAAY;IAAgB;IAAgB;;EAC5C;IAAqB;IAAmB;IAAiB;;EACzD;IAAa;IAAkB;IAAkB;;EACjD;IACE;IACA;IACA;IACA;IACA;IACA;;EAEF;EACA;EACA,YAAY;EACZ,UAAU;;UAGK;EACf,SAAS;EACT,MAAM;;;EAGN,gBAAgB;EAChB,YAAY;EACZ,sBAAsB;EACtB,UAAU;EACV,YAAY;EACZ,WAAW;EACX,YAAY;EACZ,OAAO;;UAGQ;EACf,OAAO;;EAEP;EACA,QAAQ;;;;iBA8aM,0BAA0B,iBAAiB;;;;;;;;;;;;;;;KAsB/C;UAEK;EACf;EACA;EACA,cAAc;;UAGC;WACN;WACA;;WAEA;EACT,OAAO,OAAO,oBAAoB,QAAQ;EAC1C,UAAU,QAAQ;;EAElB,eAAe,QAAQ;;;EAGvB,kBAAkB,QAAQ;;;;;;;EAO1B,oBAAoB,QAAQ;;;;iBAKd,iBAAiB,SAAS,0BAA0B;;cA8BvD,4BAA4B;WAC9B;WACA;WACA;mBACQ;mBACA;EAMjB,YAAY,cAAc,oBAAoB,cAAa;EAerD,UAAU,QAAQ;EAIlB,OAAO,OAAO,oBAAoB,QAAQ;EAU1C,eAAe,QAAQ;EAIvB,kBAAkB,QAAQ;EAI1B,oBAAoB,QAAQ;;;;cC56B9B;cACA;;UAGW;WACN;WACA,UAAU;WACV,QAAQ;WACR;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;;;;;;;;;UAWM;WACN,sBAAsB;WACtB;WACA,wBAAwB;WACxB,eAAe;;WAEf;;WAEA;WACA,QAAQ;;WAER,aAAa;WACb,SAAS;WACT;WACA;;UAGM;WACN;WACA;;WAEA,QAAQ;;WAER,QAAQ;;KAGP;UAEK;WACN;WACA;WACA;WACA,UAAU;;UAGJ;WACN,QAAQ;WACR;WACA,oBAAoB;;cAGlB,mCAAmC;WACrC;WACA;EAET,YAAY,oBAAoB;;;iBAUlB,2BACd,OAAO,kCACN;;iBA2Ba,2BAA2B,SAAS,uBAAuB;;iBA6C3D,iCACd,SAAS,sBACT,QAAQ;;iBAeM,4BACd,oBACA,SAAS,2CACA,WAAW;;iBAcN,yBACd,oBACA,SAAS,mCACR;;;UCrMc;WACN;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;;UAGM;;WAEN;WACA,WAAW;WACX;WACA;;WAEA;;WAEA;WACA,0BAA0B;WAC1B;;UAGM;WACN,SAAS;WACT;WACA;WACA,qBAAqB;;;;;;;;;;iBAWhB,oCAAoC;EAClD,SAAS;EACT,UAAU;IACR;;;;KCpBQ,0BAA0B;;UAGrB;;EAEf,OAAO;;EAEP,UAAU;;EAEV,QAAQ;;EAER,OAAO;;UAGQ;EACf,QAAQ;EACR,UAAU;;;UAIK;EACf,SAAS;EACT;EACA;;;UAIe,wBAAwB;EACvC,SAAS;EACT;EACA,OAAO,cAAc,mBAAmB;EACxC;;EAEA;;UAGe,sBAAsB,kBAAkB;EACvD,SAAS;EACT,SAAS;EACT;EACA,WAAW,cAAc;EACzB;EACA;EACA;;EAEA;;EAEA;EACA,YAAY;;;;;;;;;;cAmBD,eAAe,kBAAkB,UAAU;mBACrC;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;UACT;UACA;UAED;;;SAOM,KAAK,kBAAkB,UAAU,WAC5C,MAAM,sBAAsB,aAC3B,QAAQ,eAAe,WAAW;;;;;;EAY/B,iBAAiB;IACrB;IACA;IACA,YAAY,cAAc;MACxB;;EAgEE,cACJ,YAAY,cAAc,wBAAwB,cACjD;;;;;;;;;;EAmEG,OAAO;IACX;IACA;IACA;MACE,QAAQ;;EA0FN,QAAQ,gBAAgB,QAAQ;;UAkBxB;UAsBA;UAkCN;UAIA;UAMM;UA8BA;;UAmBN;UAOA;UAWA;;UAcA;UAcM;;UAON;;;;;;;;;;;;;;;;;iBAgKY,gCAAgC,kBAAkB,UAAU;EAChF,QAAQ;EACR,SAAS;EACT;EACA,UAAU;EACV,YAAY;;EAEZ,WAAW,cAAc;;EAEzB,sBAAsB;EACtB;EACA;IACE,QAAQ;;;UCrqBK,gCAAgC,WAAW,kBAAkB;;EAE5E;;EAEA,UAAU,eAAe,WAAW;;UAGrB,2BAA2B,kBAAkB,UAAU,mBAC9D,KAAK,mBAAmB,WAAW;;EAE3C,iBAAiB;;;;;;;;;EASjB,sBAAsB,gCAAgC,WAAW;;EAEjE,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;;EAEb,UAAU,gBAAgB;EAC1B;EACA;;;EAGA;;;EAGA;;EAEA,WAAW,cAAc;;;;;;;;;;;EAWzB,qBAAqB;IACnB;IACA;IACA,YAAY;MACV;MACA,UAAU,eAAe,WAAW;MACpC;;IAEF,SAAS;;IAET,aAAa;IACb;QACI,QAAQ,cAAc;;;;;;;;;;;;;;;;;;EAkB5B,oBAAoB,UAAU,eAAe,WAAW;;;;;;;;;;;;;EAaxD,eAAe;;;;;;;;;;;;EAYf,eAAe;;KAGL,uBACV,kBAAkB,UAClB,aACE,2BAA2B,WAAW;UAEzB,sBAAsB,WAAW,kBAAkB;EAClE,aAAa;IACX,QAAQ;IACR,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;;EAIxC,iBAAiB;EACjB,eAAe;EACf;;;;EAIA;;;;EAIA;EACA,kBAAkB,eAAe,WAAW;;EAE5C,MAAM;;;;EAIN,gBAAgB;;;;;;EAMhB,gBAAgB;;;;;iBAMI,gBAAgB,kBAAkB,UAAU,WAChE,MAAM,uBAAuB,WAAW,aACvC,QAAQ,sBAAsB,WAAW;;;KCtLhC,0BACV,kBAAkB,UAClB,aACE,uBAAuB,WAAW;;;EAGpC,kBAAkB;;;;;;;;;EASlB;;;;EAIA,MAAM,KAAK,WAAW;;;;;EAKtB;;EAEA;EACA;;;;;;;;EAQA,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;UAGlE,yBAAyB,WAAW,kBAAkB,kBAC7D,sBAAsB,WAAW;EACzC,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,uBAAuB,eAAe,WAAW;EACjD,qBAAqB;EACrB,YAAY,QAAQ,WAAW,KAAK,WAAW;;;;EAI/C;;;;;EAKA;EACA,WAAW,kBAAkB;;;;;iBAMT,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,0BAA0B,WAAW,aAC1C,QAAQ,yBAAyB,WAAW;;;UC1D9B;;;;;;;WAON;;WAEA,yBAAyB;;;;;WAKzB;;;;;;;;;;;;;;;;;;;;;;;;iBAiFK,oBAAoB,qCAAqC;;;;;iBAiCzD,4BACd,oCACA,OAAM;;;;;;;;;;;;iBAyBQ,yBACd,OAAM,2BACJ,SAAS;;;;;KCzID,6BAA6B;UAExB,gBAAgB,WAAW,kBAAkB,WAAW;;;;EAIvE,MAAM;;;EAGN,aAAa;;EAEb;;EAEA;EACA;EACA;;;EAGA,UAAU;;;;;;;;;;;;;;;;EAgBV;IAAY;;IAAwB;IAAuB;;;;;;EAK3D;;EAEA,aAAa,UAAU;;;EAGvB,cAAc;IAAS,UAAU;IAAW,UAAU;;;EAEtD,aAAa;EACb;IAAmB;IAAc,QAAQ,EAAE;;;;;;;;;;;;;iBAoB7B,SAAS,qBAAqB,kBAAkB,WAAW,UACzE,cACA,gBACA,MAAM,gBAAgB,WAAW,aAChC,YAAY,WAAW;;;UCxGT;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;;WAEN,WAAW;;WAEX;WACA;WACA;;WAEA;aACE;aACA;;;UAII;WACN,SAAS;WACT,uBAAuB;;WAEvB,qBAAqB;;;;;;;;;;iBAWhB,yCAAyC;EACvD,SAAS;EACT,UAAU;IACR;;;;KCfQ,6BAA6B,kBAAkB,UAAU,aAAa,KAChF,mBAAmB,WAAW;;UAKf;;EAEf;;EAEA,gBAAgB;EAChB;EACA;;UAGe;EACf;;EAEA;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV,SAAS;;EAET;;EAEA;EACA;;EAEA;EACA;;EAEA;;EAEA;;;;;;EAMA;EACA;EACA,aAAa;;EAEb,eAAe;;EAEf,0BAA0B;;EAE1B,kBAAkB;;EAElB,oBAAoB;;;UAIL,wBAAwB,kBAAkB,UAAU;;WAE1D,iBAAiB;;WAEjB,yBAAyB;;WAEzB,6BAA6B;;WAE7B,sBACP,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;;WAEJ,iBAAiB,YAAY,WAAW;;WAExC;WACA;;WAEA,YAAY,SAAS,6BAA6B,WAAW;;WAE7D,YAAY;;UAGN;;EAEf,eAAe;;EAEf,MAAM;;EAEN;;EAEA,aAAa;;EAEb,gBAAgB;;;UAID,mBAAmB,kBAAkB,WAAW,UAAU;;EAEzE;EACA,WACE,OAAO,wBAAwB,WAAW,eACvC,QAAQ;;UAGE;EACf;;EAEA;;EAEA;;EAEA;;;EAGA;IAAU;IAAa;;;EAEvB,kBAAkB;;EAElB;;EAEA,aAAa;;EAEb,gBAAgB;IACd;IACA;IACA;IACA;;EAEF,eAAe;;EAEf;;UAGe;;EAEf;EACA;;EAEA;EACA;EACA;;EAEA;;UAGe;;EAEf,QAAQ;EACR,MAAM;;EAEN,UAAU;EACV;;EAEA,kBAAkB;;EAElB,UAAU;;EAEV,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,eAAe;;UAGA,kCAAkC,kBAAkB,UAAU,mBACrE,KAAK,mBAAmB,WAAW;EAC3C,SAAS,mBAAmB,WAAW;EACvC,iBAAiB;;EAEjB,gBAAgB;;EAEhB,oBAAoB;;EAEpB,eAAe;;EAEf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;EACb,QAAQ,YAAY,WAAW;;;EAG/B;;EAEA,yBAAyB,6BAA6B,WAAW;;EAEjE;;;EAGA;;EAEA;;;;;EAKA,sBAAsB;;;;;iBAMF,2BAA2B,kBAAkB,UAAU,WAC3E,MAAM,kCAAkC,WAAW,aAClD,QAAQ;;iBAqxBK,sBAAsB,SAAS,oBAAoB;;iBAYnD,kCACd,SAAS,mBACT,mBAAmB,gBAClB;;iBA0Ba,uBACd,SAAS;EAAgB;EAAe,MAAM;KAC7C;;;KCxkCS;KAEA;UAEK;EACf,MAAM;EACN,UAAU;EACV;;;EAGA,UAAU;;UAGK;EACf,QAAQ;;EAER,QAAQ,OAAO;;EAEf,aAAa;;UAGE;EACf,MAAM;EACN;EACA;EACA;;UAGe;;;;;;;;;;EAUf;IACE;IACA;;IAEA;;;;;;;;;;;;;EAcF;IACE;IACA;IACA;IACA;;;;;;;;;;EAWF;IACE,WAAW,KAAK;IAChB;IACA;IACA;IACA;;;;;;;;iBASY,YAAY,MAAM,aAAa,OAAM,gBAAqB;;;KClG9D;UAUK;EACf,UAAU;;EAEV;;;;;EAKA;;EAEA;;EAEA;;UAGe,oBAAoB;EACnC,SAAS;;UAGM;EACf;EACA,UAAU;EACV;EACA;EACA;;UAGe;EACf,UAAU;EACV,oBAAoB,OAAO;EAC3B;;;cA0DW,yBAAyB;iBA6FtB,eAAe,aAAY,gBAAqB;;;;;iBAkBhD,mBACd,gBACA,qBACA,QAAQ,cACP;;iBA4Fa,cAAc,UAAU,mBAAmB;;;UCrQ1C;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;EAIA,cAAc,SAAS;;EAEvB;;EAEA;;EAEA;;EAEA;;EAEA,UAAU,YAAY;;EAEtB;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;EACA;EACA;;UAGe;EACf;IACE;IACA;;EAEF;IACE;IACA;IACA;IACA;MACE;MACA;MACA;MACA;;;EAGJ;;UAGe;EACf;EACA,MAAM;EACN;EACA,aAAa;;;;;;;UAQE;EACf;;EAEA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;EACA;;EAEA;;EAEA,YAAY;;EAEZ,qBAAqB;;EAErB,UAAU;;EAEV;;EAEA;IACE,UAAU;IACV;IACA;IACA,mBAAmB;;;;;;EAMrB;;EAEA;;EAEA;;;EAGA;;EAEA,SAAS;EACT;EACA;;UAGe,wBAAwB,WAAW,kBAAkB;EACpE;EACA;EACA;EACA,iBAAiB;EACjB,eAAe;EACf;EACA;;EAEA,wBAAwB,eAAe,WAAW;;EAElD,aAAa;IACX;IACA,YAAY;IACZ;;;IAGA,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;EAGxC,MAAM;;;;EAIN;EACA,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,qBAAqB;EACrB,uBAAuB,eAAe,WAAW;;EAEjD,cAAc,cAAc;EAC5B;EACA;EACA,qBAAqB;;UAGN,6BAA6B,WAAW,kBAAkB;EACzE;EACA;EACA;EACA,iBAAiB;EACjB,QAAQ,yBAAyB,WAAW;EAC5C,cAAc,cAAc;EAC5B;EACA;;;iBAIc,6BAA6B,WAAW,kBAAkB,UACxE,OAAO,6BAA6B,WAAW,aAC9C,wBAAwB,WAAW;;iBA4CtB,0BAA0B,WAAW,kBAAkB,UACrE,MAAM,wBAAwB,WAAW,aACxC;;iBAqRa,0BAA0B,WAAW,kBAAkB,UACrE,UAAU,eAAe,WAAW;;iBAqCtB,2BAA2B,QAAQ,uBAAuB;;;iBAc1D,gBAAgB;;;;;;;;;;;;iBA2JhB,oBACd,QAAQ,sBACR;EAAQ;IACP;;iBAiJa,qBAAqB;;;;iBAMrB,oBAAoB;UAInB;EACf,QAAQ;EACR,OAAO;;EAEP;EACA;;UAGe,uBAAuB,WAAW,kBAAkB,kBAC3D,wBAAwB,WAAW;;EAE3C,SAAS;;;EAGT,eAAe;;;;;;;;;;;;iBA+EK,mBAAmB,WAAW,kBAAkB,UACpE,MAAM,uBAAuB,WAAW,aACvC,QAAQ"}
|
|
@@ -1,295 +0,0 @@
|
|
|
1
|
-
import { T as EProcessState, r as SignedManifest } from "./pre-registration-BoI4ucR3.js";
|
|
2
|
-
import { a as PairedPromotionDecision, d as PairedBootstrapResult, i as PairedMcNemarEvidence, r as PairedDecisionStatistic, t as PairedDecisionMethod } from "./paired-promotion-decision-CGzg0cI_.js";
|
|
3
|
-
import { R as Scenario, b as GenerationRecord, p as Gate, w as JudgeScore } from "./types-Ba5UQyVD.js";
|
|
4
|
-
//#region src/campaign/gates/sequential.d.ts
|
|
5
|
-
type SequentialDecision = 'promote' | 'continue' | 'undecided-at-maxN';
|
|
6
|
-
interface SequentialObservation {
|
|
7
|
-
decision: SequentialDecision;
|
|
8
|
-
/** Current e-value (the betting wealth) against H0. */
|
|
9
|
-
eValue: number;
|
|
10
|
-
/** Paired deltas consumed so far. */
|
|
11
|
-
n: number;
|
|
12
|
-
/** Names the decision basis. For 'undecided-at-maxN' it states explicitly
|
|
13
|
-
* that exhausting the budget is NOT evidence of no effect. */
|
|
14
|
-
reason: string;
|
|
15
|
-
}
|
|
16
|
-
interface SequentialPairedGateOptions {
|
|
17
|
-
/** Type-I budget. With `preRegistration` bound this MUST match
|
|
18
|
-
* `manifest.alpha` (conflict throws). Default 0.05. */
|
|
19
|
-
alpha?: number;
|
|
20
|
-
/** Minimum paired deltas before a promote may fire. The stopping rule is
|
|
21
|
-
* "first n ≥ minN with e-value ≥ 1/alpha" — still a valid stopping time.
|
|
22
|
-
* Default 5. */
|
|
23
|
-
minN?: number;
|
|
24
|
-
/** Pre-registered observation budget. Required unless `preRegistration`
|
|
25
|
-
* supplies it via `preRegisteredN` (conflict throws). */
|
|
26
|
-
maxN?: number;
|
|
27
|
-
/** Bet truncation forwarded to `eProcess`. Default 0.5. */
|
|
28
|
-
maxBet?: number;
|
|
29
|
-
/** Bound on |delta| in the judge's native scale; deltas are mapped to
|
|
30
|
-
* x = (d/scale + 1)/2 ∈ [0,1]. A delta outside ±scale throws (use
|
|
31
|
-
* `detectScale` to pick 1 vs 100 BEFORE streaming). Default 1. */
|
|
32
|
-
scale?: number;
|
|
33
|
-
/** Seed for the data-independent shuffle of paired deltas in `decide(ctx)`
|
|
34
|
-
* (exchangeability guard). Default 1337. */
|
|
35
|
-
shuffleSeed?: number;
|
|
36
|
-
/** Bind the pre-registered hypothesis. Verified (content hash) at
|
|
37
|
-
* construction; alpha/maxN/direction/minEffect come FROM the manifest. */
|
|
38
|
-
preRegistration?: SignedManifest;
|
|
39
|
-
/** Override the gate name in reports. */
|
|
40
|
-
name?: string;
|
|
41
|
-
/** Continue the observe-stream from a `state()` snapshot taken before a
|
|
42
|
-
* restart. The gate's parameters still resolve from this options object
|
|
43
|
-
* (and the manifest); the snapshot must have been recorded under the same
|
|
44
|
-
* alpha, maxBet, and null boundary, and its decision must be one this
|
|
45
|
-
* configuration could have reached at its n, or construction throws.
|
|
46
|
-
* `decide(ctx)` is unaffected — it always runs on a fresh stream. */
|
|
47
|
-
resume?: SequentialStreamState;
|
|
48
|
-
}
|
|
49
|
-
/** Snapshot of one observe-stream: the e-process state plus the gate's own
|
|
50
|
-
* decision, which applies `minN` and `maxN` on top of the core latch. A
|
|
51
|
-
* gate rebuilt with `resume` from this value continues exactly where the
|
|
52
|
-
* snapshot was taken. */
|
|
53
|
-
type SequentialStreamState = EProcessState & {
|
|
54
|
-
decision: SequentialDecision;
|
|
55
|
-
};
|
|
56
|
-
interface SequentialPairedGate<TArtifact = unknown, TScenario extends Scenario = Scenario> extends Gate<TArtifact, TScenario> {
|
|
57
|
-
/** Streaming entry point: feed one paired per-scenario delta
|
|
58
|
-
* (candidate − baseline, native scale). Each gate instance carries ONE
|
|
59
|
-
* observe-stream; `decide(ctx)` runs on its own fresh stream and never
|
|
60
|
-
* consumes or advances this one. 'promote' is sticky; observing past the
|
|
61
|
-
* pre-registered maxN throws (extending a finished stream after seeing
|
|
62
|
-
* the result reopens optional stopping — start a NEW pre-registered
|
|
63
|
-
* test). */
|
|
64
|
-
observe(delta: number): SequentialObservation;
|
|
65
|
-
/** Read-only snapshot of the observe-stream. Pass it back as
|
|
66
|
-
* `resume` to continue the stream after a restart. */
|
|
67
|
-
state(): SequentialStreamState;
|
|
68
|
-
}
|
|
69
|
-
/**
|
|
70
|
-
* Anytime-valid sequential paired gate. Conforms to the existing `Gate`
|
|
71
|
-
* contract (`decide(ctx)` consumes candidate vs baseline judge scores via
|
|
72
|
-
* `pairHoldout` — same pairing granularity as the fixed-n gates: full cellId,
|
|
73
|
-
* never scenarioId) and adds a streaming `observe(delta)` entry for campaigns
|
|
74
|
-
* that score cells incrementally and want to stop mid-stream.
|
|
75
|
-
*
|
|
76
|
-
* Decision mapping onto the substrate's five-valued `GateDecision`:
|
|
77
|
-
* - 'promote' → 'ship'
|
|
78
|
-
* - 'continue' → 'need_more_work' (stream ended before maxN with
|
|
79
|
-
* the e-value undecided — more reps could decide)
|
|
80
|
-
* - 'undecided-at-maxN' → 'hold', with the reason stating it is NOT
|
|
81
|
-
* evidence of no effect (never a silent default)
|
|
82
|
-
*/
|
|
83
|
-
declare function sequentialPairedGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(options: SequentialPairedGateOptions): SequentialPairedGate<TArtifact, TScenario>;
|
|
84
|
-
interface SequentialDecideOptions {
|
|
85
|
-
/** Type-I budget for the early-stop evidence. Default 0.05. */
|
|
86
|
-
alpha?: number;
|
|
87
|
-
/** Minimum paired deltas before a stop may fire. Default 5. */
|
|
88
|
-
minN?: number;
|
|
89
|
-
/** Bet truncation forwarded to `eProcess`. Default 0.5. */
|
|
90
|
-
maxBet?: number;
|
|
91
|
-
/** Bound on |per-scenario composite delta|. Default 1. */
|
|
92
|
-
scale?: number;
|
|
93
|
-
}
|
|
94
|
-
interface SequentialDecideFn {
|
|
95
|
-
(args: {
|
|
96
|
-
history: GenerationRecord[];
|
|
97
|
-
}): {
|
|
98
|
-
stop: boolean;
|
|
99
|
-
reason?: string;
|
|
100
|
-
};
|
|
101
|
-
/** Read-only snapshot of the accumulated e-process (observability + tests). */
|
|
102
|
-
state(): EProcessState;
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* `SurfaceProposer.decide` adapter — stops the optimization loop the moment
|
|
106
|
-
* the e-process decides the loop has produced a real improvement, instead of
|
|
107
|
-
* always running `maxGenerations`.
|
|
108
|
-
*
|
|
109
|
-
* Stream: for each generation g ≥ 1, the per-scenario composite deltas of
|
|
110
|
-
* generation g's top candidate vs the generation-0 top candidate (the
|
|
111
|
-
* incumbent the loop set out to beat), paired by scenarioId. H0: no proposed
|
|
112
|
-
* surface improves any scenario's expected composite over the incumbent —
|
|
113
|
-
* under it every delta has conditional mean ≤ 0 and the e-process is valid.
|
|
114
|
-
* Once wealth ≥ 1/alpha the loop stops and hands the winner to the promotion
|
|
115
|
-
* gate (which re-scores on HELD-OUT data — this adapter only spends the
|
|
116
|
-
* exploration budget, it never promotes).
|
|
117
|
-
*
|
|
118
|
-
* Honesty caveats: (1) the incumbent's scores are measured once and shared
|
|
119
|
-
* across all generations' deltas, so type-I control is exact only insofar as
|
|
120
|
-
* those scores approximate the incumbent's true per-scenario means (more reps
|
|
121
|
-
* → tighter); (2) an UNDECIDED process never stops the loop — absence of a
|
|
122
|
-
* crossing is NOT evidence of no effect, so the loop simply runs its normal
|
|
123
|
-
* course. Calling the adapter repeatedly with a growing history consumes each
|
|
124
|
-
* generation exactly once (re-feeding an already-seen record would double-count
|
|
125
|
-
* evidence).
|
|
126
|
-
*/
|
|
127
|
-
declare function sequentialDecide(options?: SequentialDecideOptions): SequentialDecideFn;
|
|
128
|
-
//#endregion
|
|
129
|
-
//#region src/campaign/gates/statistical-heldout.d.ts
|
|
130
|
-
interface PairedHoldout {
|
|
131
|
-
/** Baseline scalar per paired cell (same order as `after`/`cellIds`). */
|
|
132
|
-
before: number[];
|
|
133
|
-
/** Candidate scalar per paired cell. */
|
|
134
|
-
after: number[];
|
|
135
|
-
/** The full cellIds (`scenario:rep`) that paired, in order. */
|
|
136
|
-
cellIds: string[];
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Pair candidate vs baseline holdout observations by FULL cellId. `select`
|
|
140
|
-
* pulls the scalar from a cell's judge reports (composite, or a named
|
|
141
|
-
* dimension); a cell contributes the mean of `select` across its judges. Cells
|
|
142
|
-
* whose scenario is not in `scenarioIds`, or where `select` is undefined for
|
|
143
|
-
* every judge on either side, are skipped on BOTH sides so the arrays stay
|
|
144
|
-
* paired. Throws when the two maps disagree on which holdout cells exist — a
|
|
145
|
-
* load-bearing invariant: the baseline + winner holdout campaigns run the same
|
|
146
|
-
* scenarios with the same seed base, so their cellIds MUST align; a mismatch
|
|
147
|
-
* means a silent pairing bug, not a soft fallback.
|
|
148
|
-
*/
|
|
149
|
-
declare function pairHoldout(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, select: (s: JudgeScore) => number | undefined): PairedHoldout;
|
|
150
|
-
interface HeldoutSignificance {
|
|
151
|
-
paired: PairedHoldout;
|
|
152
|
-
/**
|
|
153
|
-
* The paired bootstrap on the requested statistic (MEAN by default — see the
|
|
154
|
-
* tie note on `heldoutSignificance`).
|
|
155
|
-
*
|
|
156
|
-
* DIAGNOSTIC, not necessarily the interval the verdict keyed on. On a
|
|
157
|
-
* two-point (pass/fail) outcome the decision routes to Tango's score interval
|
|
158
|
-
* instead, because a percentile bootstrap of the mean over a three-atom
|
|
159
|
-
* lattice is not a valid interval at a nonzero margin. Read
|
|
160
|
-
* `decision.low`/`decision.high` for the interval that actually decided, and
|
|
161
|
-
* `decisionStatistic` for which one it is.
|
|
162
|
-
*/
|
|
163
|
-
bootstrap: PairedBootstrapResult;
|
|
164
|
-
/** The MEDIAN paired-delta bootstrap, reported as a diagnostic. When many
|
|
165
|
-
* scenarios are tied (both sides solve them), the median is pinned near 0
|
|
166
|
-
* regardless of the mean lift — comparing the two exposes tie-domination. */
|
|
167
|
-
medianBootstrap: PairedBootstrapResult;
|
|
168
|
-
/**
|
|
169
|
-
* The full promotion decision: which estimator the outcome's shape admits,
|
|
170
|
-
* the interval it produced, McNemar's exact veto on the two-point path, and
|
|
171
|
-
* whether the interval was zero-width (no evidence in either direction). The
|
|
172
|
-
* single source of `significant`.
|
|
173
|
-
*/
|
|
174
|
-
decision: PairedPromotionDecision;
|
|
175
|
-
/** Which paired estimator the verdict was decided on. */
|
|
176
|
-
decisionStatistic: PairedDecisionStatistic;
|
|
177
|
-
/** McNemar's exact evidence on the two-point path; null otherwise. */
|
|
178
|
-
mcnemar: PairedMcNemarEvidence | null;
|
|
179
|
-
/** Fraction of paired observations that are exact ties (|delta| < 1e-9). A
|
|
180
|
-
* high tie fraction is WHY a median-based gate would have missed a real lift;
|
|
181
|
-
* it is the observability the tie fix adds. */
|
|
182
|
-
tieFraction: number;
|
|
183
|
-
/** n paired observations. */
|
|
184
|
-
n: number;
|
|
185
|
-
/** Effective minimum after applying the bootstrap's hard statistical floor. */
|
|
186
|
-
minimumRequired: number;
|
|
187
|
-
/** Statistical method that carried the decision. */
|
|
188
|
-
decisionMethod: PairedDecisionMethod;
|
|
189
|
-
/** Exact one-sided p-value on the small-sample path; otherwise null. */
|
|
190
|
-
pValue: number | null;
|
|
191
|
-
/** True iff n >= minimumRequired, the DECIDING interval has nonzero width,
|
|
192
|
-
* its lower bound clears the threshold, and McNemar's exact test does not
|
|
193
|
-
* veto at a non-negative threshold. */
|
|
194
|
-
significant: boolean;
|
|
195
|
-
/** Set when n < minimumRequired — too little evidence to claim significance. */
|
|
196
|
-
fewRuns: boolean;
|
|
197
|
-
}
|
|
198
|
-
interface HeldoutSignificanceOptions {
|
|
199
|
-
deltaThreshold?: number;
|
|
200
|
-
minProductiveRuns?: number;
|
|
201
|
-
confidence?: number;
|
|
202
|
-
resamples?: number;
|
|
203
|
-
/** Fixed by default for a deterministic, reproducible gate verdict. */
|
|
204
|
-
seed?: number;
|
|
205
|
-
statistic?: 'mean' | 'median';
|
|
206
|
-
}
|
|
207
|
-
/**
|
|
208
|
-
* Significance of the held-out composite lift: ship only when the lower bound
|
|
209
|
-
* of the interval the outcome's shape ADMITS exceeds `deltaThreshold` (default
|
|
210
|
-
* 0 ⇒ "confidently positive"). Interpret `deltaThreshold` in the judge's native
|
|
211
|
-
* scale.
|
|
212
|
-
*
|
|
213
|
-
* The decision is delegated whole to {@link decidePairedPromotion}, the one
|
|
214
|
-
* copy of the rule (`src/paired-promotion-decision.ts`), which `HeldOutGate`
|
|
215
|
-
* also calls. That module's header carries the measurements; the short version
|
|
216
|
-
* is three guards a bare `bootstrap.low > threshold` does not have:
|
|
217
|
-
*
|
|
218
|
-
* - a two-point (pass/fail) outcome decides on Tango's SCORE interval, the
|
|
219
|
-
* only paired-binary construction that stays valid at a nonzero margin;
|
|
220
|
-
* - McNemar's exact test VETOES at any non-negative threshold;
|
|
221
|
-
* - a ZERO-WIDTH interval is refused rather than promoted, in either
|
|
222
|
-
* direction — [0,0] clears every negative threshold and [g,g] clears every
|
|
223
|
-
* threshold below g, and both are an absence of evidence, not a result.
|
|
224
|
-
*
|
|
225
|
-
* Measured on this function before those guards landed, at a nominal 5 %:
|
|
226
|
-
* 14.60 % false promotion at n = 40 on a paired-binary noninferiority boundary,
|
|
227
|
-
* and 88.50 % at n = 6 under a bounded asymmetric null whose true mean paired
|
|
228
|
-
* delta is exactly 0.
|
|
229
|
-
*
|
|
230
|
-
* At small n, where the percentile bootstrap is descriptive only, a
|
|
231
|
-
* pre-registered exact sign test still carries the bootstrap path.
|
|
232
|
-
*/
|
|
233
|
-
declare function heldoutSignificance(paired: PairedHoldout, opts?: HeldoutSignificanceOptions): HeldoutSignificance;
|
|
234
|
-
interface DimensionRegression {
|
|
235
|
-
dimension: string;
|
|
236
|
-
/** Paired bootstrap on (candidate − baseline). DIAGNOSTIC on a pass/fail
|
|
237
|
-
* dimension, where `ci` carries the interval that decided instead. */
|
|
238
|
-
bootstrap: PairedBootstrapResult;
|
|
239
|
-
/** Which paired statistic `bootstrap.low` is the lower bound of. `'mean'`
|
|
240
|
-
* unless the caller asked for the median. `bootstrap.median` still carries
|
|
241
|
-
* the median point estimate either way. */
|
|
242
|
-
bootstrapStatistic: 'median' | 'mean';
|
|
243
|
-
/** The interval `regressed` was decided on, in the dimension's native units. */
|
|
244
|
-
ci: {
|
|
245
|
-
low: number;
|
|
246
|
-
high: number;
|
|
247
|
-
};
|
|
248
|
-
/** Which estimator produced `ci`. */
|
|
249
|
-
decisionStatistic: PairedDecisionStatistic;
|
|
250
|
-
/** McNemar's exact evidence on a pass/fail dimension; null otherwise. */
|
|
251
|
-
mcnemar: PairedMcNemarEvidence | null;
|
|
252
|
-
/** `ci` has zero width — no evidence in either direction. */
|
|
253
|
-
indeterminate: boolean;
|
|
254
|
-
/** True iff the candidate may have regressed this dimension by more than
|
|
255
|
-
* tolerance: the lower bound of the DECIDING interval on (candidate −
|
|
256
|
-
* baseline) is below −tolerance, OR the exact small-sample test proves a drop
|
|
257
|
-
* past tolerance. */
|
|
258
|
-
regressed: boolean;
|
|
259
|
-
tolerance: number;
|
|
260
|
-
n: number;
|
|
261
|
-
}
|
|
262
|
-
/** Detect the native scale of a set of scores: 0-100 when any magnitude clears
|
|
263
|
-
* 1.5, else [0,1]. Used to auto-scale the regression tolerance so a default
|
|
264
|
-
* expressed for [0,1] is not silently a no-op on a 0-100 dimension. */
|
|
265
|
-
declare function detectScale(values: number[]): 1 | 100;
|
|
266
|
-
/** Per-critical-dimension regression guard. For each dimension, pair the
|
|
267
|
-
* candidate vs baseline values by full cellId and bootstrap the paired delta;
|
|
268
|
-
* a dimension is "regressed" when the CI lower bound < −tolerance (conservative
|
|
269
|
-
* — blocks if the credible worst case exceeds tolerance, which is the right
|
|
270
|
-
* posture for safety dimensions like `hallucination_free`). When `tolerance`
|
|
271
|
-
* is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100.
|
|
272
|
-
*
|
|
273
|
-
* The interval comes from {@link decidePairedPromotion}, so a pass/fail
|
|
274
|
-
* dimension is judged on Tango's score interval rather than a percentile
|
|
275
|
-
* bootstrap of the mean — `tolerance` is a NONZERO margin, and the bootstrap
|
|
276
|
-
* is not a valid interval at one. That matters most here because this guard
|
|
277
|
-
* fails OPEN by construction: `tolerance` is positive, so an interval pinned at
|
|
278
|
-
* [0,0] never satisfies `low < −tolerance` and a real regression on a safety
|
|
279
|
-
* dimension would be reported as `regressed: false`. On the median it fails the
|
|
280
|
-
* same way for the same reason — when most pairs tie, which is automatic for a
|
|
281
|
-
* pass/fail dimension on {0,1} and on the 0-100 encoding `detectScale` exists
|
|
282
|
-
* to support, the median CI collapses to [0,0]. Pass `statistic: 'median'` to
|
|
283
|
-
* restore the pre-0.134 behaviour. */
|
|
284
|
-
declare function dimensionRegressions(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, criticalDimensions: string[], opts?: {
|
|
285
|
-
tolerance?: number;
|
|
286
|
-
confidence?: number;
|
|
287
|
-
resamples?: number;
|
|
288
|
-
seed?: number;
|
|
289
|
-
/** Paired statistic the CI is computed on. Default `'mean'` — see
|
|
290
|
-
* {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */
|
|
291
|
-
statistic?: 'mean' | 'median';
|
|
292
|
-
}): DimensionRegression[];
|
|
293
|
-
//#endregion
|
|
294
|
-
export { sequentialPairedGate as _, detectScale as a, pairHoldout as c, SequentialDecision as d, SequentialObservation as f, sequentialDecide as g, SequentialStreamState as h, PairedHoldout as i, SequentialDecideFn as l, SequentialPairedGateOptions as m, HeldoutSignificance as n, dimensionRegressions as o, SequentialPairedGate as p, HeldoutSignificanceOptions as r, heldoutSignificance as s, DimensionRegression as t, SequentialDecideOptions as u };
|
|
295
|
-
//# sourceMappingURL=statistical-heldout-DTyB_6-1.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"statistical-heldout-DTyB_6-1.d.ts","names":[],"sources":["../src/campaign/gates/sequential.ts","../src/campaign/gates/statistical-heldout.ts"],"mappings":";;;;KAsCY;UAEK;EACf,UAAU;;EAEV;;EAEA;;;EAGA;;UAGe;;;EAGf;;;;EAIA;;;EAGA;;EAEA;;;;EAIA;;;EAGA;;;EAGA,kBAAkB;;EAElB;;;;;;;EAOA,SAAS;;;;;;KAOC,wBAAwB;EAAkB,UAAU;;UAE/C,qBAAqB,qBAAqB,kBAAkB,WAAW,kBAC9E,KAAK,WAAW;;;;;;;;EAQxB,QAAQ,gBAAgB;;;EAGxB,SAAS;;;;;;;;;;;;;;;;iBAsPK,qBAAqB,qBAAqB,kBAAkB,WAAW,UACrF,SAAS,8BACR,qBAAqB,WAAW;UAsFlB;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;GACd;IAAQ,SAAS;;IAAyB;IAAe;;;EAE1D,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;iBA0BK,iBAAiB,UAAS,0BAA+B;;;UC/axD;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;;iBAcc,YACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,SAAS,GAAG,oCACX;UAsDc;EACf,QAAQ;;;;;;;;;;;;EAYR,WAAW;;;;EAIX,iBAAiB;;;;;;;EAOjB,UAAU;;EAEV,mBAAmB;;EAEnB,SAAS;;;;EAIT;;EAEA;;EAEA;;EAEA,gBAAgB;;EAEhB;;;;EAIA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;EACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6Bc,oBACd,QAAQ,eACR,OAAM,6BACL;UAkEc;EACf;;;EAGA,WAAW;;;;EAIX;;EAEA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;EAET;;;;;EAKA;EACA;EACA;;;;;iBAMc,YAAY;;;;;;;;;;;;;;;;;;;iBAsBZ,qBACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,8BACA;EACE;EACA;EACA;EACA;;;EAGA;IAED"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tool-groups-Cp4Xdzrp.d.ts","names":[],"sources":["../src/analyst/tool-groups.ts"],"mappings":";;;;KAmBY;;;;;;;;;;;;;;;;;;;;iBAgDI,wBACd,OAAO,oBACP,OAAO,qBACN"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types-CiWITkGo.js","names":[],"sources":["../src/ledger-core/deep-freeze.ts","../src/analyst/usage-receipt.ts","../src/analyst/types.ts"],"sourcesContent":["/** Freeze a detached canonical-JSON graph. Canonicalization has already ruled out cycles.\n *\n * Lives outside canonical.ts so the analyst-benchmark implementation digest,\n * which covers canonical.ts, stays bound to the published benchmark evidence. */\nexport function deepFreezeCanonicalJson<T>(value: T): T {\n if (value && typeof value === 'object' && !Object.isFrozen(value)) {\n Object.freeze(value)\n for (const nested of Object.values(value)) deepFreezeCanonicalJson(nested)\n }\n return value\n}\n","import type { CostChannel, CostLedgerFilter, CostLedgerHandle } from '../cost-ledger'\nimport type { AnalystUsageReceipt } from './types'\n\nexport const DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS = 5_000\n\n/** Convert one ledger channel's complete call set into one analyst receipt. */\nexport function usageReceiptFromCostLedger(\n ledger: CostLedgerHandle,\n filter: CostChannel | CostLedgerFilter = 'analyst',\n): AnalystUsageReceipt {\n const resolvedFilter = typeof filter === 'string' ? { channel: filter } : filter\n const summary = ledger.summary(resolvedFilter)\n const receipts = ledger.list(resolvedFilter)\n const hasReasoningUsage = receipts.some((receipt) => receipt.reasoningTokens !== undefined)\n const hasCacheWriteUsage = receipts.some((receipt) => receipt.cacheWriteTokens !== undefined)\n const cost = summary.costProvenance\n return {\n calls: summary.totalCalls + summary.pendingCalls,\n tokens: summary.usageComplete\n ? {\n input: summary.inputTokens,\n output: summary.outputTokens,\n ...(hasReasoningUsage ? { reasoning: summary.reasoningTokens ?? 0 } : {}),\n ...(summary.cachedTokens > 0 ? { cached: summary.cachedTokens } : {}),\n ...(hasCacheWriteUsage ? { cacheWrite: summary.cacheWriteTokens ?? 0 } : {}),\n }\n : null,\n cost,\n ...(cost.kind === 'uncaptured' ? { knownCostUsd: summary.totalCostUsd } : {}),\n }\n}\n\nexport interface SettledUsageReceipt {\n settled: boolean\n pendingCalls: number\n receipt: AnalystUsageReceipt\n}\n\n/** Wait a bounded time for late provider receipts, then take one immutable snapshot. */\nexport async function settleUsageReceiptFromCostLedger(\n ledger: CostLedgerHandle,\n options: CostLedgerFilter & { timeoutMs?: number } = {},\n): Promise<SettledUsageReceipt> {\n const { timeoutMs: requestedTimeoutMs, ...requestedFilter } = options\n const filter: CostLedgerFilter = {\n channel: requestedFilter.channel ?? 'analyst',\n ...(requestedFilter.phase === undefined ? {} : { phase: requestedFilter.phase }),\n ...(requestedFilter.tags === undefined ? {} : { tags: requestedFilter.tags }),\n }\n const timeoutMs = validateUsageSettlementTimeout(requestedTimeoutMs)\n const initial = ledger.summary(filter)\n const waitResult =\n initial.pendingCalls === 0\n ? true\n : ledger.waitForIdle\n ? await ledger.waitForIdle({ timeoutMs })\n : false\n const pendingCalls = ledger.summary(filter).pendingCalls\n return {\n settled: waitResult && pendingCalls === 0,\n pendingCalls,\n receipt: usageReceiptFromCostLedger(ledger, filter),\n }\n}\n\nexport function validateUsageSettlementTimeout(timeoutMs?: number): number {\n const resolved = timeoutMs ?? DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS\n if (!Number.isSafeInteger(resolved) || resolved < 0 || resolved > 2_147_483_647) {\n throw new TypeError(\n 'settlementTimeoutMs must be a non-negative safe integer no greater than 2147483647',\n )\n }\n return resolved\n}\n\nexport function assertValidAnalystUsageReceipt(\n receipt: AnalystUsageReceipt,\n context = 'AnalystContext.recordUsage',\n): void {\n if (receipt.calls !== null && (!Number.isSafeInteger(receipt.calls) || receipt.calls < 0)) {\n throw new Error(`${context}: calls must be a non-negative safe integer or null`)\n }\n if (receipt.tokens) {\n assertNonNegativeSafeInteger(receipt.tokens.input, 'tokens.input', context)\n assertNonNegativeSafeInteger(receipt.tokens.output, 'tokens.output', context)\n if (receipt.tokens.reasoning !== undefined) {\n assertNonNegativeSafeInteger(receipt.tokens.reasoning, 'tokens.reasoning', context)\n if (receipt.tokens.reasoning > receipt.tokens.output) {\n throw new Error(`${context}: tokens.reasoning must not exceed tokens.output`)\n }\n }\n if (receipt.tokens.cached !== undefined) {\n assertNonNegativeSafeInteger(receipt.tokens.cached, 'tokens.cached', context)\n }\n if (receipt.tokens.cacheWrite !== undefined) {\n assertNonNegativeSafeInteger(receipt.tokens.cacheWrite, 'tokens.cacheWrite', context)\n }\n }\n if (receipt.cost.kind !== 'uncaptured') {\n assertNonNegativeFinite(receipt.cost.usd, 'cost.usd', context)\n } else if (receipt.cost.usd !== null) {\n throw new Error(`${context}: uncaptured cost.usd must be null`)\n }\n if (receipt.knownCostUsd !== undefined) {\n assertNonNegativeFinite(receipt.knownCostUsd, 'knownCostUsd', context)\n }\n if (receipt.partialTokens) {\n const { input, output } = receipt.partialTokens\n if (receipt.tokens) {\n throw new Error(`${context}: partialTokens must be absent when tokens is complete`)\n }\n if (input === null && output === null) {\n throw new Error(`${context}: partialTokens must carry at least one reported side`)\n }\n if (input !== null) assertNonNegativeSafeInteger(input, 'partialTokens.input', context)\n if (output !== null) assertNonNegativeSafeInteger(output, 'partialTokens.output', context)\n }\n}\n\nfunction assertNonNegativeSafeInteger(value: number, field: string, context: string): void {\n if (!Number.isSafeInteger(value) || value < 0) {\n throw new Error(`${context}: ${field} must be a non-negative safe integer`)\n }\n}\n\nfunction assertNonNegativeFinite(value: number, field: string, context: string): void {\n if (!Number.isFinite(value) || value < 0) {\n throw new Error(`${context}: ${field} must be a non-negative finite number`)\n }\n}\n","/**\n * Analyst contract — the missing orchestration layer over agent-eval's\n * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,\n * SemanticConceptJudge, JudgeFn, ...).\n *\n * Each existing primitive returns its own output shape. The Analyst\n * contract is the single envelope every primitive lifts into, so a\n * registry can run N analysts against a run and a single renderer can\n * compose findings without knowing which analyzer produced them.\n *\n * The contract is intentionally domain-agnostic: nothing here knows\n * about code, voice, RAG, or any particular agent stack. Analysts\n * declare what INPUT KIND they need (a trace store, an artifact dir,\n * a RunRecord, a JudgeInput, or `custom`), and the registry routes\n * the matching input from `AnalystRunInputs`.\n */\n\nimport { createHash } from 'node:crypto'\nimport type { CostLedgerHandle } from '../cost-ledger'\nimport type { RunCostProvenance, RunRecord, RunTokenUsage } from '../run-record'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { JudgeInput } from '../types'\nimport type { ChatClient } from './chat-client'\n\n/**\n * Unified envelope every analyst emits. Schema-versioned so renderers\n * and time-series diffs survive future field additions.\n */\nexport interface AnalystFinding {\n schema_version: '1.0.0'\n /**\n * Stable hash over identity-defining fields (analyst_id + canonical\n * claim + area + optional subject). Two findings from two runs that\n * \"are the same finding\" share this id — that's what `diffFindings`\n * uses to compute appeared/disappeared sets across runs.\n */\n finding_id: string\n analyst_id: string\n produced_at: string\n severity: AnalystSeverity\n /**\n * Coarse classification. Renderers group by this. Free-form so\n * domain-specific analysts can introduce categories without a\n * schema change ('agent-reasoning', 'verification', 'cost',\n * 'tool-use', 'safety', 'latency', 'data-quality', ...).\n */\n area: string\n claim: string\n rationale?: string\n evidence_refs: EvidenceRef[]\n recommended_action?: string\n validation_plan?: string\n /** 0..1 — the analyst's own confidence. Not calibrated across analysts. */\n confidence: number\n /**\n * Optional subject the finding is about — leaf id, agent id, request\n * id. Included in finding_id when present so per-subject findings\n * diff cleanly across runs.\n */\n subject?: string\n /** True when this finding was lifted from a judge result rather than observed\n * directly in a trace or artifact. Descriptive only: proposal access is\n * controlled by `ProposalFinding.proposal_origin`. */\n derived_from_judge?: boolean\n /** Analyst-private extras; renderers ignore unless they know the analyst. */\n metadata?: Record<string, unknown>\n}\n\n/** Finding severity. `AnalystSeverity` derives from this array, so the type\n * and the schema that validates a finding cannot name different levels. */\nexport const ANALYST_SEVERITIES = ['critical', 'high', 'medium', 'low', 'info'] as const\n\nexport type AnalystSeverity = (typeof ANALYST_SEVERITIES)[number]\n\n/** Data sources that candidate generation may intentionally learn from. */\nexport type ProposalFindingOrigin = 'search' | 'production'\n\n/** A finding explicitly admitted as candidate-generation input. */\nexport type ProposalFinding = AnalystFinding & {\n readonly proposal_origin: ProposalFindingOrigin\n}\n\nexport interface EvidenceRef {\n /**\n * Where the evidence lives. `span` and `event` refer to OTLP trace\n * elements; `artifact` to a file inside the run's artifact tree;\n * `finding` to another AnalystFinding (cross-analyst chaining);\n * `metric` to a named scalar reading the renderer knows how to read.\n */\n kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric'\n uri: string\n excerpt?: string\n}\n\n// ── Analyst contract ─────────────────────────────────────────────────\n\n/**\n * The discriminator the registry uses to pass the right input.\n * `custom` is the escape hatch — analysts that need something else\n * (e.g. an embedding cache, a partner SDK handle) read it from\n * `AnalystRunInputs.custom[<analyst id>]`.\n */\nexport type AnalystInputKind =\n | 'trace-store'\n | 'artifact-dir'\n | 'run-record'\n | 'judge-input'\n | 'custom'\n\nexport interface AnalystCost {\n /** `deterministic` analysts MUST NOT call the LLM. */\n kind: 'deterministic' | 'llm'\n /** Optional declared upper bound; the registry can enforce a budget. */\n est_usd_per_run?: number\n /** Models the analyst expects to use (informational). */\n models?: string[]\n /** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */\n settlement_timeout_ms?: number\n}\n\nexport interface AnalystRequirements {\n /** Min number of shots / samples the analyst needs to produce signal. */\n min_shots?: number\n /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */\n capabilities?: string[]\n}\n\n/**\n * What's passed to every analyst call. The registry resolves which\n * field the analyst's `inputKind` selects and asserts it's present.\n */\nexport interface AnalystRunInputs {\n traceStore?: TraceAnalysisStore\n artifactDir?: string\n runRecord?: RunRecord\n judgeInput?: JudgeInput\n /** Keyed by analyst id; populated by callers that registered custom analysts. */\n custom?: Record<string, unknown>\n}\n\nexport interface AnalystContext {\n runId: string\n /** Stable correlation id so logs from a single registry.run() share a tag. */\n correlationId: string\n /** Enforced wall-clock deadline (epoch ms). */\n deadlineMs?: number\n /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */\n budgetUsd?: number\n /** Shared paid-call account when the analyst runs inside a larger campaign. */\n costLedger?: CostLedgerHandle\n /** Attribution phase used when writing to the shared paid-call account. */\n costPhase?: string\n /**\n * Shared chat client. Analysts that call an LLM go through this so\n * the operator picks transport (sandbox-sdk | router | cli-bridge |\n * direct-provider | mock) at the registry boundary without touching\n * analyst code.\n */\n chat?: ChatClient\n /**\n * Findings from a prior run the operator wants the analyst to see as\n * retrieval context. Kinds that take advantage of cross-run memory\n * (failure-mode \"I saw this cluster last run\", knowledge-gap \"the wiki\n * page I asked for is still missing\") render these into the actor's\n * working set. Filtering is the operator's job: pass the slice that\n * matches the analyst's id, or pass everything and let the kind\n * filter. Empty / absent means no cross-run context.\n */\n priorFindings?: ReadonlyArray<AnalystFinding>\n /**\n * Findings emitted by analysts that completed earlier in this registry run.\n * This is separate from `priorFindings`: upstream findings are dependency\n * context for the current pass, while prior findings are cross-run memory.\n * The registry populates this only when `RegistryRunOpts.chainFindings` is on.\n */\n upstreamFindings?: ReadonlyArray<AnalystFinding>\n /**\n * Report metered work independently of findings. This keeps an empty finding\n * set from erasing token/cost telemetry. Multiple receipts are accumulated.\n */\n recordUsage?: (receipt: AnalystUsageReceipt) => void\n /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */\n tags?: Record<string, string>\n /** Logger callback — analysts SHOULD prefer this over console.* for testability. */\n log?: (msg: string, fields?: Record<string, unknown>) => void\n /** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */\n signal?: AbortSignal\n /**\n * Optional live-execution port. A runtime that owns a sandbox or checkout\n * fills it so an analyst can execute a bounded probe against the run's\n * produced state instead of reasoning about it from the trace alone. This\n * package defines only the port: no field here reaches for an agent loop,\n * and an absent probe means the analyst works from recorded evidence.\n */\n probe?: ExecutionProbe\n}\n\n// ── Live-execution port ─────────────────────────────────────────────\n\n/** One bounded command an analyst asks the probe to run. */\nexport interface ExecutionProbeRequest {\n command: string\n /** Working directory inside the probed environment. */\n cwd?: string\n /** Hard wall-clock deadline for this one execution. */\n timeoutMs: number\n /** Bytes of combined output retained; the prober truncates beyond it. */\n maxOutputBytes?: number\n signal?: AbortSignal\n}\n\n/**\n * Typed outcome of one probe execution. `succeeded: false` is a PROBE failure\n * (the environment could not run the command); a command that ran and exited\n * non-zero is a successful observation with a non-zero `exitCode`.\n */\nexport type ExecutionProbeOutcome =\n | {\n succeeded: true\n exitCode: number\n stdout: string\n stderr: string\n durationMs: number\n /** True when output was cut at `maxOutputBytes`. */\n truncated: boolean\n }\n | { succeeded: false; error: { class: string; message: string } }\n\n/**\n * The seam a runtime fills to let analysts observe produced state live.\n * Implementations own sandboxing, credentials, and cleanup; analysts only\n * submit bounded requests and read typed outcomes.\n */\nexport interface ExecutionProbe {\n /** One plain sentence naming what is being probed (e.g. a sandbox id). */\n readonly description: string\n execute(request: ExecutionProbeRequest): Promise<ExecutionProbeOutcome>\n}\n\n/**\n * The minimal contract. Concrete analysts can refine `TInput` so\n * implementations stay type-safe (e.g. a trace analyst's `TInput` is\n * `TraceAnalysisStore`); the registry passes the right field from\n * `AnalystRunInputs` based on `inputKind`.\n */\nexport interface Analyst<TInput = unknown> {\n /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */\n readonly id: string\n /** Human-readable. One sentence. */\n readonly description: string\n readonly inputKind: AnalystInputKind\n readonly cost: AnalystCost\n readonly requires?: AnalystRequirements\n /** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */\n readonly version: string\n analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>\n}\n\n/** Metered work performed by one analyst call. */\nexport interface AnalystUsageReceipt {\n /** Number of model-usage records observed at the provider boundary. */\n calls: number | null\n /** Null when the provider did not return token accounting. */\n tokens: RunTokenUsage | null\n /** Observed, estimated, or explicitly uncaptured dollar cost. */\n cost: RunCostProvenance\n /** Known lower bound when one or more calls have uncaptured cost. */\n knownCostUsd?: number\n /**\n * Token counts the provider reported on only one side. Present exactly when\n * `tokens` is null and at least one side WAS reported: `RunTokenUsage` has no\n * nullable side, so a one-sided count cannot live in `tokens` without writing\n * a zero nobody measured. Read it as a lower bound, never as a total — the\n * field exists so a null `tokens` cannot hide a real count.\n */\n partialTokens?: { input: number | null; output: number | null }\n /**\n * True when the token counts were DERIVED by the transport (from character\n * lengths, say) rather than measured by the model provider. `cost.kind` is\n * `estimated` both for a rate estimate over exact tokens and for one over\n * derived tokens; this is the field that separates them.\n */\n tokensEstimated?: boolean\n}\n\n// ── finding_id stability ─────────────────────────────────────────────\n\n/**\n * Compute the stable finding_id from the identity-defining fields.\n * Default implementation hashes {analyst_id, area, subject, normalized claim}.\n * Analysts that emit findings whose claim text varies per run (timestamps,\n * counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,\n * or (b) move the variable part into `rationale`/`metadata` and keep the\n * `claim` static.\n */\nexport function computeFindingId(input: {\n analyst_id: string\n area: string\n subject?: string\n claim: string\n /** Override the claim for hashing — use when the displayed claim has run-specific bits. */\n id_basis?: string\n}): string {\n const basis = JSON.stringify({\n a: input.analyst_id,\n r: input.area,\n s: input.subject ?? '',\n c: normalizeClaim(input.id_basis ?? input.claim),\n })\n return `f_${createHash('sha256').update(basis).digest('hex').slice(0, 20)}`\n}\n\nfunction normalizeClaim(c: string): string {\n // Lowercase, collapse whitespace, strip trailing punctuation. Goal:\n // \"Leaf X failed install\" and \"Leaf X failed install.\" hash the same.\n return c\n .toLowerCase()\n .replace(/\\s+/g, ' ')\n .replace(/[.!?;:,]+$/g, '')\n .trim()\n}\n\n/**\n * Convenience factory: produce a fully-formed AnalystFinding with the\n * id computed automatically. Analyst code stays terse.\n */\nexport function makeFinding(\n init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n id_basis?: string\n produced_at?: string\n },\n): AnalystFinding {\n const { id_basis, produced_at, ...rest } = init\n return {\n schema_version: '1.0.0',\n finding_id: computeFindingId({\n analyst_id: rest.analyst_id,\n area: rest.area,\n subject: rest.subject,\n claim: rest.claim,\n id_basis,\n }),\n produced_at: produced_at ?? new Date().toISOString(),\n ...rest,\n }\n}\n\n/** Build a finding whose source is explicitly allowed during candidate generation. */\nexport function makeProposalFinding(\n init: Omit<ProposalFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n id_basis?: string\n produced_at?: string\n },\n): ProposalFinding {\n const { proposal_origin, ...finding } = init\n return { ...makeFinding(finding), proposal_origin }\n}\n\n// ── Registry result envelope ────────────────────────────────────────\n\nexport interface AnalystRunSummary {\n analyst_id: string\n status: 'ok' | 'skipped' | 'failed'\n /** Why skipped — missing input, budget exceeded, capability unmet. */\n reason?: string\n findings_count: number\n latency_ms: number\n /** Additive model usage and cost provenance for this analyst. */\n usage: AnalystUsageReceipt\n /** When `status='failed'`: the error class + message, never the full stack. */\n error?: { class: string; message: string }\n}\n\nexport interface AnalystRunResult {\n run_id: string\n correlation_id: string\n started_at: string\n ended_at: string\n findings: AnalystFinding[]\n per_analyst: AnalystRunSummary[]\n /** Total LLM cost in USD across all analysts in this registry.run(). */\n total_cost_usd: number\n /**\n * Provenance for `total_cost_usd`. When uncaptured, the numeric field is only\n * the known subtotal and must not be treated as the run's total spend.\n */\n total_cost_provenance?: RunCostProvenance\n}\n\n// ── Streaming event envelope ────────────────────────────────────────\n\n/**\n * Events emitted by `AnalystRegistry.runStream(...)` in real time as\n * the registry executes. UIs subscribe via `for await (const ev of\n * registry.runStream(...))`; `registry.run(...)` is a thin collector\n * over the same stream, so the two surfaces share their invariants.\n *\n * Per-finding events are intentionally omitted — analyzers are batch\n * operations (a recursive engine returns the full `findings:json[]` at the\n * end of the responder), so streaming inside one analyst would only\n * emit partial JSON consumers can't render. The kind-completion event\n * is the right granularity; subscribers wanting per-finding rendering\n * iterate `event.findings` themselves.\n */\nexport type AnalystRunEvent =\n | {\n type: 'run-started'\n run_id: string\n correlation_id: string\n started_at: string\n /** The ordered list of analyst ids the registry will run. */\n analyst_ids: ReadonlyArray<string>\n }\n | {\n type: 'analyst-skipped'\n summary: AnalystRunSummary\n }\n | {\n type: 'analyst-started'\n analyst_id: string\n started_at: string\n }\n | {\n type: 'analyst-completed'\n /** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */\n summary: AnalystRunSummary\n findings: ReadonlyArray<AnalystFinding>\n }\n | {\n type: 'run-completed'\n result: AnalystRunResult\n }\n"],"mappings":";;;;;;AAIA,SAAgB,wBAA2B,OAAa;CACtD,IAAI,SAAS,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;EACjE,OAAO,OAAO,KAAK;EACnB,KAAK,MAAM,UAAU,OAAO,OAAO,KAAK,GAAG,wBAAwB,MAAM;CAC3E;CACA,OAAO;AACT;;ACJA,SAAgB,2BACd,QACA,SAAyC,WACpB;CACrB,MAAM,iBAAiB,OAAO,WAAW,WAAW,EAAE,SAAS,OAAO,IAAI;CAC1E,MAAM,UAAU,OAAO,QAAQ,cAAc;CAC7C,MAAM,WAAW,OAAO,KAAK,cAAc;CAC3C,MAAM,oBAAoB,SAAS,MAAM,YAAY,QAAQ,oBAAoB,KAAA,CAAS;CAC1F,MAAM,qBAAqB,SAAS,MAAM,YAAY,QAAQ,qBAAqB,KAAA,CAAS;CAC5F,MAAM,OAAO,QAAQ;CACrB,OAAO;EACL,OAAO,QAAQ,aAAa,QAAQ;EACpC,QAAQ,QAAQ,gBACZ;GACE,OAAO,QAAQ;GACf,QAAQ,QAAQ;GAChB,GAAI,oBAAoB,EAAE,WAAW,QAAQ,mBAAmB,EAAE,IAAI,CAAC;GACvE,GAAI,QAAQ,eAAe,IAAI,EAAE,QAAQ,QAAQ,aAAa,IAAI,CAAC;GACnE,GAAI,qBAAqB,EAAE,YAAY,QAAQ,oBAAoB,EAAE,IAAI,CAAC;EAC5E,IACA;EACJ;EACA,GAAI,KAAK,SAAS,eAAe,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;CAC7E;AACF;;AASA,eAAsB,iCACpB,QACA,UAAqD,CAAC,GACxB;CAC9B,MAAM,EAAE,WAAW,oBAAoB,GAAG,oBAAoB;CAC9D,MAAM,SAA2B;EAC/B,SAAS,gBAAgB,WAAW;EACpC,GAAI,gBAAgB,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,gBAAgB,MAAM;EAC9E,GAAI,gBAAgB,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,gBAAgB,KAAK;CAC7E;CACA,MAAM,YAAY,+BAA+B,kBAAkB;CAEnE,MAAM,aADU,OAAO,QAAQ,MAEvB,CAAC,CAAC,iBAAiB,IACrB,OACA,OAAO,cACL,MAAM,OAAO,YAAY,EAAE,UAAU,CAAC,IACtC;CACR,MAAM,eAAe,OAAO,QAAQ,MAAM,CAAC,CAAC;CAC5C,OAAO;EACL,SAAS,cAAc,iBAAiB;EACxC;EACA,SAAS,2BAA2B,QAAQ,MAAM;CACpD;AACF;AAEA,SAAgB,+BAA+B,WAA4B;CACzE,MAAM,WAAW,aAAA;CACjB,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,KAAK,WAAW,YAChE,MAAM,IAAI,UACR,oFACF;CAEF,OAAO;AACT;AAEA,SAAgB,+BACd,SACA,UAAU,8BACJ;CACN,IAAI,QAAQ,UAAU,SAAS,CAAC,OAAO,cAAc,QAAQ,KAAK,KAAK,QAAQ,QAAQ,IACrF,MAAM,IAAI,MAAM,GAAG,QAAQ,oDAAoD;CAEjF,IAAI,QAAQ,QAAQ;EAClB,6BAA6B,QAAQ,OAAO,OAAO,gBAAgB,OAAO;EAC1E,6BAA6B,QAAQ,OAAO,QAAQ,iBAAiB,OAAO;EAC5E,IAAI,QAAQ,OAAO,cAAc,KAAA,GAAW;GAC1C,6BAA6B,QAAQ,OAAO,WAAW,oBAAoB,OAAO;GAClF,IAAI,QAAQ,OAAO,YAAY,QAAQ,OAAO,QAC5C,MAAM,IAAI,MAAM,GAAG,QAAQ,iDAAiD;EAEhF;EACA,IAAI,QAAQ,OAAO,WAAW,KAAA,GAC5B,6BAA6B,QAAQ,OAAO,QAAQ,iBAAiB,OAAO;EAE9E,IAAI,QAAQ,OAAO,eAAe,KAAA,GAChC,6BAA6B,QAAQ,OAAO,YAAY,qBAAqB,OAAO;CAExF;CACA,IAAI,QAAQ,KAAK,SAAS,cACxB,wBAAwB,QAAQ,KAAK,KAAK,YAAY,OAAO;MACxD,IAAI,QAAQ,KAAK,QAAQ,MAC9B,MAAM,IAAI,MAAM,GAAG,QAAQ,mCAAmC;CAEhE,IAAI,QAAQ,iBAAiB,KAAA,GAC3B,wBAAwB,QAAQ,cAAc,gBAAgB,OAAO;CAEvE,IAAI,QAAQ,eAAe;EACzB,MAAM,EAAE,OAAO,WAAW,QAAQ;EAClC,IAAI,QAAQ,QACV,MAAM,IAAI,MAAM,GAAG,QAAQ,uDAAuD;EAEpF,IAAI,UAAU,QAAQ,WAAW,MAC/B,MAAM,IAAI,MAAM,GAAG,QAAQ,sDAAsD;EAEnF,IAAI,UAAU,MAAM,6BAA6B,OAAO,uBAAuB,OAAO;EACtF,IAAI,WAAW,MAAM,6BAA6B,QAAQ,wBAAwB,OAAO;CAC3F;AACF;AAEA,SAAS,6BAA6B,OAAe,OAAe,SAAuB;CACzF,IAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAC1C,MAAM,IAAI,MAAM,GAAG,QAAQ,IAAI,MAAM,qCAAqC;AAE9E;AAEA,SAAS,wBAAwB,OAAe,OAAe,SAAuB;CACpF,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,MAAM,GAAG,QAAQ,IAAI,MAAM,sCAAsC;AAE/E;;;;;;;;;;;;;;;;;;;;;AC3DA,MAAa,qBAAqB;CAAC;CAAY;CAAQ;CAAU;CAAO;AAAM;;;;;;;;;AAiO9E,SAAgB,iBAAiB,OAOtB;CACT,MAAM,QAAQ,KAAK,UAAU;EAC3B,GAAG,MAAM;EACT,GAAG,MAAM;EACT,GAAG,MAAM,WAAW;EACpB,GAAG,eAAe,MAAM,YAAY,MAAM,KAAK;CACjD,CAAC;CACD,OAAO,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;AAC1E;AAEA,SAAS,eAAe,GAAmB;CAGzC,OAAO,EACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,eAAe,EAAE,CAAC,CAC1B,KAAK;AACV;;;;;AAMA,SAAgB,YACd,MAIgB;CAChB,MAAM,EAAE,UAAU,aAAa,GAAG,SAAS;CAC3C,OAAO;EACL,gBAAgB;EAChB,YAAY,iBAAiB;GAC3B,YAAY,KAAK;GACjB,MAAM,KAAK;GACX,SAAS,KAAK;GACd,OAAO,KAAK;GACZ;EACF,CAAC;EACD,aAAa,gCAAe,IAAI,KAAK,EAAA,CAAE,YAAY;EACnD,GAAG;CACL;AACF;;AAGA,SAAgB,oBACd,MAIiB;CACjB,MAAM,EAAE,iBAAiB,GAAG,YAAY;CACxC,OAAO;EAAE,GAAG,YAAY,OAAO;EAAG;CAAgB;AACpD"}
|