@tangle-network/agent-eval 0.192.0 → 0.193.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +6 -6
- package/dist/analyst/index.js +4 -4
- package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
- package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
- package/dist/{benchmark-command-C458yEFK.js → benchmark-command-N5r3GXMP.js} +8 -8
- package/dist/{benchmark-command-C458yEFK.js.map → benchmark-command-N5r3GXMP.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +11 -10
- package/dist/{campaign-BVQVBEfH.js → campaign-DnL0S4XL.js} +11 -10
- package/dist/{campaign-BVQVBEfH.js.map → campaign-DnL0S4XL.js.map} +1 -1
- package/dist/{campaign-evidence-DcQiYhJI.js → campaign-evidence-DJc60F9C.js} +2 -2
- package/dist/{campaign-evidence-DcQiYhJI.js.map → campaign-evidence-DJc60F9C.js.map} +1 -1
- package/dist/{chat-client-B_ItsSBw.js → chat-client-CJORSG_J.js} +2 -2
- package/dist/{chat-client-B_ItsSBw.js.map → chat-client-CJORSG_J.js.map} +1 -1
- package/dist/{chat-json-call-CXHBvUz8.js → chat-json-call-DnlDjpco.js} +2 -2
- package/dist/{chat-json-call-CXHBvUz8.js.map → chat-json-call-DnlDjpco.js.map} +1 -1
- package/dist/cli.js +49 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
- package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +11 -11
- package/dist/{default-registry-DdqA_-_o.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
- package/dist/{default-registry-DdqA_-_o.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DcludNJS.d.ts → define-agent-eval-DQJvVa1P.d.ts} +6 -6
- package/dist/{define-agent-eval-DcludNJS.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
- package/dist/{define-agent-eval-DQuos3Aw.js → define-agent-eval-qxAPjMSL.js} +6 -6
- package/dist/{define-agent-eval-DQuos3Aw.js.map → define-agent-eval-qxAPjMSL.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +1 -1
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +12 -6
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-BX34vPVY.js → dspy-rlm-engine-k_QdEBZo.js} +2 -2
- package/dist/{dspy-rlm-engine-BX34vPVY.js.map → dspy-rlm-engine-k_QdEBZo.js.map} +1 -1
- package/dist/{engine-C1DIAEpn.d.ts → engine-DjRV55F9.d.ts} +2 -2
- package/dist/{engine-C1DIAEpn.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
- package/dist/{eval-campaign-CwsBn0Gm.js → eval-campaign-CXT2WYUe.js} +2 -2
- package/dist/{eval-campaign-CwsBn0Gm.js.map → eval-campaign-CXT2WYUe.js.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +4 -4
- package/dist/{external-optimizer-process-D-Sab4v3.js → external-optimizer-process-_YIECYJ7.js} +2 -2
- package/dist/{external-optimizer-process-D-Sab4v3.js.map → external-optimizer-process-_YIECYJ7.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-kbqC-iWf.js → external-optimizer-subprocess-BrghTvEp.js} +2 -2
- package/dist/{external-optimizer-subprocess-kbqC-iWf.js.map → external-optimizer-subprocess-BrghTvEp.js.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +1 -1
- package/dist/{index-DBoXFaz0.d.ts → index-0jEtpSM7.d.ts} +3 -3
- package/dist/{index-DBoXFaz0.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
- package/dist/{index-DwQxdWve.d.ts → index-DMPLG226.d.ts} +89 -272
- package/dist/index-DMPLG226.d.ts.map +1 -0
- package/dist/{index-DNVzu9Vv.d.ts → index-DWQNDqPd.d.ts} +7 -7
- package/dist/{index-DNVzu9Vv.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.js +60 -308
- package/dist/index.js.map +1 -1
- package/dist/jev.d.ts +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/{llm-client-KYQBh9C8.js → llm-client-c_bKHTXz.js} +2 -2
- package/dist/{llm-client-KYQBh9C8.js.map → llm-client-c_bKHTXz.js.map} +1 -1
- package/dist/{llm-judge-Z6MkTkd0.js → llm-judge-Cififl_V.js} +1024 -237
- package/dist/llm-judge-Cififl_V.js.map +1 -0
- package/dist/{matrix-CQMMrEQ5.d.ts → matrix-CNu13gg8.d.ts} +2 -2
- package/dist/{matrix-CQMMrEQ5.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-DS7t1t1J.js → otel-spans-CQmSxtcS.js} +2 -2
- package/dist/{otel-spans-DS7t1t1J.js.map → otel-spans-CQmSxtcS.js.map} +1 -1
- package/dist/paired-delta-test-BF-FEP8V.js +78 -0
- package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
- package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
- package/dist/pipelines/index.d.ts +1 -1
- package/dist/{produced-state-CVmCPQDJ.js → produced-state-B2AhDRnq.js} +4 -4
- package/dist/{produced-state-CVmCPQDJ.js.map → produced-state-B2AhDRnq.js.map} +1 -1
- package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
- package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
- package/dist/{raw-provider-sink-DYh4Bszw.js → raw-provider-sink-CXuOhLvT.js} +2 -2
- package/dist/{raw-provider-sink-DYh4Bszw.js.map → raw-provider-sink-CXuOhLvT.js.map} +1 -1
- package/dist/{redact-CopuMxll.d.ts → redact-Cveoq6fJ.d.ts} +2 -2
- package/dist/{redact-CopuMxll.d.ts.map → redact-Cveoq6fJ.d.ts.map} +1 -1
- package/dist/{redact-B7rt9Zot.js → redact-Dk1ZdJEm.js} +59 -26
- package/dist/redact-Dk1ZdJEm.js.map +1 -0
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +2 -2
- package/dist/{search-ledger-Ck7tok0h.js → search-ledger-CJiAEU3j.js} +89 -27
- package/dist/search-ledger-CJiAEU3j.js.map +1 -0
- package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
- package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
- package/dist/{search-shipper-BG182p-U.js → search-shipper-DUlwlCsK.js} +2 -2
- package/dist/{search-shipper-BG182p-U.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
- package/dist/search-summary-BhY8p0Av.js +378 -0
- package/dist/search-summary-BhY8p0Av.js.map +1 -0
- package/dist/{semantic-concept-judge-Bg-560H0.js → semantic-concept-judge-Bh5lvKGg.js} +4 -4
- package/dist/{semantic-concept-judge-Bg-560H0.js.map → semantic-concept-judge-Bh5lvKGg.js.map} +1 -1
- package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
- package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
- package/dist/{server-DtetGGoA.js → server-D56r1lhp.js} +2 -2
- package/dist/{server-DtetGGoA.js.map → server-D56r1lhp.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DoayuM6D.js → skillopt-optimization-method-C6DbFKnw.js} +8 -8
- package/dist/{skillopt-optimization-method-DoayuM6D.js.map → skillopt-optimization-method-C6DbFKnw.js.map} +1 -1
- package/dist/{spans-BWtjk_9c.d.ts → spans-D9_ehFuk.d.ts} +2 -2
- package/dist/{spans-BWtjk_9c.d.ts.map → spans-D9_ehFuk.d.ts.map} +1 -1
- package/dist/{statistical-heldout-C1Ej5Ce8.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
- package/dist/{statistical-heldout-C1Ej5Ce8.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
- package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
- package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
- package/dist/{store-tool-spans-BcC_5ef-.js → store-tool-spans-BaGakFc2.js} +2 -2
- package/dist/{store-tool-spans-BcC_5ef-.js.map → store-tool-spans-BaGakFc2.js.map} +1 -1
- package/dist/{store-tool-spans-ByLdU26j.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
- package/dist/{store-tool-spans-ByLdU26j.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
- package/dist/{task-failure-attributes-CEqEfuIY.js → task-failure-attributes-DngaLdE4.js} +4 -3
- package/dist/task-failure-attributes-DngaLdE4.js.map +1 -0
- package/dist/{tool-groups-DiBi16ns.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
- package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +4 -4
- package/dist/{types-C1CAD70_.d.ts → types-BWwRXofp.d.ts} +328 -4
- package/dist/types-BWwRXofp.d.ts.map +1 -0
- package/dist/{types-ByZslIlr.d.ts → types-qoxUfKQg.d.ts} +2 -2
- package/dist/{types-ByZslIlr.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/search-ledger.md +71 -6
- package/package.json +2 -2
- package/dist/claim-BGOUqoMq.d.ts +0 -38
- package/dist/claim-BGOUqoMq.d.ts.map +0 -1
- package/dist/index-DwQxdWve.d.ts.map +0 -1
- package/dist/llm-judge-Z6MkTkd0.js.map +0 -1
- package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
- package/dist/redact-B7rt9Zot.js.map +0 -1
- package/dist/search-ledger-Ck7tok0h.js.map +0 -1
- package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
- package/dist/task-failure-attributes-CEqEfuIY.js.map +0 -1
- package/dist/tool-groups-DiBi16ns.d.ts.map +0 -1
- package/dist/types-C1CAD70_.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-
|
|
1
|
+
{"version":3,"file":"index-DWQNDqPd.d.ts","names":[],"sources":["../src/contract/measured-comparison.ts","../src/contract/profile-measured-comparison.ts","../src/contract/intake/run-record-dir.ts","../src/contract/eval-reporting-suite.ts","../src/contract/intake/agent-trace.ts","../src/contract/intake/code-agent-observation.ts","../src/contract/intake/code-agent-session.ts","../src/contract/intake/code-agent-store.ts","../src/contract/intake/feedback-table.ts","../src/contract/intake/otel-spans.ts"],"mappings":";;;;;;;;;;;;UAiCiB;EACf,QAAQ,gCAAgC;EACxC;EACA;;UAGe;EACf,YAAY;EACZ;EACA,QAAQ;EACR,MAAM;EACN,eAAe;EACf;EACA,SAAS;;UAGM;EACf,YAAY;EACZ,QAAQ,OAAO,oCAAoC,QAAQ;;EAE3D;;EAEA,aAAa;EACb,SAAS;;UAGM;EACf,cAAc;EACd;IACE;IACA,MAAM;;;UAIO;EACf,YAAY;EACZ,cAAc;EACd;IACE;IACA,MAAM;;EAER,aAAa;EACb;EACA,YAAY;EACZ;EACA,WAAW;;;UAII,kBAAkB;EACjC;EACA,UAAU;EACV,WAAW;;;UAII,yBAAyB;EACxC,MAAM,KAAK;EACX,WAAW,KAAK;IAAkB;IAAc;;EAChD,QAAQ,KAAK;EACb,eAAe,KAAK,OAAO;EAC3B,UAAU,KAAK;EACf,UAAU,KAAK;EACf,OAAO,KAAK;;UAGG,kCAAkC;EACjD,uBAAuB,kBAAkB;EACzC,QAAQ;EACR,SAAS,yBAAyB;;EAElC;;EAEA,kBAAkB;;EAElB,kBAAkB;;;KAIR,8BAA8B,KACxC;EAGA,iBAAiB;EACjB,WAAW;EACX;;;iBAIc,2BACd,UAAU,sCACT;;iBAQa,4BACd,SAAS,qCACR;;iBAiBa,wBACd,UAAU,mCACT;iBAQa,0BAA0B,iBAAiB;;iBAarC,uBACpB,SAAS,gCACR,QAAQ;;;;;;;;iBAkEK,2BAA2B,MACzC,SAAS,kCAAkC,QAC1C;;iBA8Ya,0CACd,SAAS,oCACR;;iBAuEa,oCACd,iBACC;;;UCjrBc;EACf,aAAa;EACb,QAAQ,gCAAgC;EACxC;EACA;;UAGe;EACf,YAAY;EACZ;EACA,aAAa;EACb,MAAM;EACN,SAAS;EACT;EACA,SAAS;;UAGM;EACf,YAAY;EACZ,QACE,OAAO,kDACN,QAAQ;;EAEX;;EAEA,aAAa;EACb,SAAS;;UAGM;EACf,cAAc;EACd;IACE;IACA,MAAM;;;UAIO;EACf,YAAY;EACZ,cAAc;EACd;IACE;IACA,MAAM;;EAER,aAAa;EACb;EACA,YAAY;EACZ;EACA,WAAW;;;iBAIG,gCACd,UAAU,sCACT;;iBAQa,iCACd,SAAS,0CACR;;iBAqBa,sCACd,UAAU,4CACT;;;;;;iBAsBmB,qCACpB,SAAS,8CACR,QAAQ;;iBA0CK,wDACd,SAAS,kDACR;;iBAuEa,kDACd,iBACC;;;;UC7Oc;;EAEf;;EAEA;;EAEA;;UAGe;;;;;;EAMf;;;;;;;EAOA,WAAW;;;;;;EAMX;;UAGe;;EAEf,MAAM;;EAEN,UAAU;;EAEV;;;;;;;;;;iBAkBoB,iBACpB,cACA,UAAS,0BACR,QAAQ;;;;;KC7CC,0BAA0B;UAErB;;;;;EAKf,UAAU,KAAK;;EAEf,OAAO;;;;;;;;;;EAUP;;;;UAKe;;;EAGf,QAAQ;;EAER;;IAEE;;IAEA;;;IAGA;;IAEA;;;IAGA,UAAU;;;EAGZ;;;;;;;iBAUoB,mBACpB,OAAO,yBACP,UAAS,4BACR,QAAQ;;;KChEC;UAEK;EACf,MAAM;;EAEN;;UAGe;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,cAAc;EACd,QAAQ;;UAGO;EACf;EACA,eAAe;;UAGA;EACf;EACA;EACA;EACA;IAAQ;IAAc;;EACtB;IAAS;IAAe;;EACxB,OAAO;;;;UAOQ;EACf;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;KAGU,kBAAkB,YAAY;;;;;;iBAW1B,gBAAgB,SAAS,qBAAqB;UAmE7C;;;;EAIf,SAAS,YAAY;;;EAGrB,cAAc;;;;;;;;iBASA,8BACd,MAAM,aACN,OAAO,kBACN;;;KCjLS;KAEA;KAEA;KAEA;KASA;UAEK;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA,MAAM;EACN,SAAS;EACT;EACA,QAAQ;EACR;EACA;EACA,UAAU;;UAGK;EACf,QAAQ;EACR;EACA;EACA;IACE,QAAQ;IACR;;EAEF,SAAS;;UAGM;EACf,QAAQ;EACR;EACA;EACA,YAAY;;;;;;;iBAeE,wBACd,SAAS,iCACR;;;UCjCc;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;;UAGe;EACf,MAAM;EACN,aAAa;EACb,SAAS;EACT,cAAc;;UAGC;EACf;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;;;EAGA,iBAAiB;;;EAGjB,YAAY;;;;KAKF;EACN;EAAe;EAAoB;;EACnC;EAAmB;;iBAcT,oBAAoB,gBAAgB;;;;;;;;;;iBAuB7B,yBAAyB,eAAe,eAAe;;;;;iBA6BxD,wBAAwB,eAAe,QAAQ;iBAUrD,iBACd,SAAS,gCACR;iBAIa,sBACd,SAAS,gCACR;iBAIa,oBACd,SAAS,gCACR;iBAIa,oBACd,SAAS,gCACR;iBAIa,cACd,SAAS,gCACR;;;;UC/Lc;;;;;WAKN;;WAEA,QAAQ;;;;;WAKR;;;;;WAKA;aACE;;aAEA;;;;KAKD;;UAUK;WACN;WACA;WACA;WACA;WACA,aAAa;;;UAIP;WACN;WACA,UAAU,SAAS,QAAQ,OAAO;;WAElC,QAAQ;aACN;aACA,QAAQ;;;;;;;;;KAUT;WAEG;WACA,OAAO;WACP;WACA,mBAAmB;;WAEnB;;WAEA;;WAEA;WACA,WAAW,SAAS,QAAQ,OAAO;WACnC,YAAY;;WAGZ;WACA,OAAO;WACP;;WAEA;WACA,YAAY;;;;;;;;;iBAaL,sBACpB,OAAO,sBACN,QAAQ;;;UCtHM;;;EAGf;;EAEA;;;EAGA;;;EAGA,WAAW;;UAGI;EACf;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,WAAW;;;EAGX,SAAS;;UAGM;;EAEf,SAAS;;;EAGT,OAAO;;;;EAIP;IAAU;IAAa;;;;;EAIvB;;UAGe;EACf,MAAM;;;EAGN,aAAa;IAAQ;IAAe;IAAe;;;iBAGrC,kBAAkB,MAAM,2BAA2B;;;UC3BlD;EACf,OAAO;;EAEP,eAAe;;EAEf;;;;;;EAMA,eAAe,eAAe,gBAAgB;;iBAGhC,cAAc,MAAM,uBAAuB"}
|
package/dist/index.d.ts
CHANGED
|
@@ -9,15 +9,15 @@ import { a as ContinuousCalibrationResult, d as calibrateJudgeContinuous, f as c
|
|
|
9
9
|
import { a as CorpusAgreementPerDimension, c as corpusInterRaterAgreement, i as CorpusAgreementOptions, l as corpusInterRaterAgreementFromJudgeScores, n as SeriesConvergenceResult, o as CorpusAgreementReport, r as analyzeSeries, s as CorpusScoreRecord, t as SeriesConvergenceOptions, u as interRaterReliability } from "./series-convergence-DY7ha0kx.js";
|
|
10
10
|
import { a as partialCredit, c as spearmanR, d as weightedMean, i as confidenceInterval, l as summarizeNumberSeries, n as WeightedCompositeInput, o as pearsonR, r as WeightedCompositeResult, s as ranks, t as SeriesDistribution, u as weightedComposite } from "./descriptive-B2iPaT9J.js";
|
|
11
11
|
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-DIWHvysC.js";
|
|
12
|
-
import { A as PairArmsOptions, Bt as RiskDifferenceResult, C as hashJson, Ct as verifyAttestation, D as ComparePairedArmsOptions, F as PairedCorrectness, Ft as EProcessStep, Gt as pairedRiskDifference, Ht as isBinaryOutcomeVector, I as PairedMetricDelta, It as eProcess, Jt as passAtK, Kt as pairedRiskDifferenceExact, L as comparePairedArms, Lt as ExactRiskDifferenceResult, M as PairRunRecordsResult, Mt as EProcess, N as PairedArmRow, Nt as EProcessOptions, O as MatchedPair, P as PairedArmsComparison, Pt as EProcessState, R as pairArms, Rt as McNemarResult, St as attest, Ut as mcnemar, Vt as ScoreRiskDifferenceResult, Wt as pairedBinaryScale, Yt as wilson, bt as AttestationVerification, et as Objective, j as PairArmsResult, k as MatchedRunRecordPair, nt as dominates, qt as pairedRiskDifferenceScore, rt as paretoFrontier, tt as ParetoResult, vt as ATTESTATION_ALGORITHM, w as manifestContentDigest, xt as AttestedReport, yt as AttestationProvenance, z as pairRunRecords, zt as ProportionInterval } from "./statistical-heldout-
|
|
12
|
+
import { A as PairArmsOptions, Bt as RiskDifferenceResult, C as hashJson, Ct as verifyAttestation, D as ComparePairedArmsOptions, F as PairedCorrectness, Ft as EProcessStep, Gt as pairedRiskDifference, Ht as isBinaryOutcomeVector, I as PairedMetricDelta, It as eProcess, Jt as passAtK, Kt as pairedRiskDifferenceExact, L as comparePairedArms, Lt as ExactRiskDifferenceResult, M as PairRunRecordsResult, Mt as EProcess, N as PairedArmRow, Nt as EProcessOptions, O as MatchedPair, P as PairedArmsComparison, Pt as EProcessState, R as pairArms, Rt as McNemarResult, St as attest, Ut as mcnemar, Vt as ScoreRiskDifferenceResult, Wt as pairedBinaryScale, Yt as wilson, bt as AttestationVerification, et as Objective, j as PairArmsResult, k as MatchedRunRecordPair, nt as dominates, qt as pairedRiskDifferenceScore, rt as paretoFrontier, tt as ParetoResult, vt as ATTESTATION_ALGORITHM, w as manifestContentDigest, xt as AttestedReport, yt as AttestationProvenance, z as pairRunRecords, zt as ProportionInterval } from "./statistical-heldout-D-xBpbrd.js";
|
|
13
13
|
import { _ as pairedSignTest, a as PairedPromotionDecision, c as BOOTSTRAP_GATE_MIN_N, d as PairedBootstrapResult, f as PairedSignTestResult, g as pairedDeltaTieFraction, h as pairedBootstrap, i as PairedMcNemarEvidence, l as DECISION_PAIRED_DELTA_STATISTIC, m as SignTestAlternative, n as PairedDecisionShape, o as PairedPromotionDecisionOptions, p as PairedTTestResult, r as PairedDecisionStatistic, s as decidePairedPromotion, t as PairedDecisionMethod, u as PairedBootstrapOptions, v as pairedTTest } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
14
14
|
import { a as requiredPairedSampleSize, i as pairedMde, n as mcnemarPower, o as requiredSampleSize, r as mcnemarRequiredN, t as mulberry32 } from "./random-DGEyr3zG.js";
|
|
15
15
|
import { C as RankTestMethod, D as WilcoxonSignedRankResult, E as WILCOXON_EXACT_MAX_N, O as mannWhitneyU, S as MannWhitneyResult, T as RankTestOptions, _ as bootstrapCi, a as ReleaseConfidenceIssue, b as MANN_WHITNEY_EXACT_MAX_STATES, f as evaluateReleaseConfidence, g as Verdict, i as ReleaseConfidenceInput, k as wilcoxonSignedRank, m as BootstrapResult, o as ReleaseConfidenceMetrics, p as BootstrapOptions, s as ReleaseConfidenceScorecard, t as ActionableSideInfo, u as ReleaseTraceEvidence, w as RankTestMethodRequest, x as MANN_WHITNEY_EXACT_MAX_WORK, y as DEFAULT_PERMUTATIONS } from "./release-confidence-CkoXFx9I.js";
|
|
16
|
-
import { R as Scenario$1, S as JudgeConfig, _ as GateDecision, a as CampaignResult, w as JudgeScore$1 } from "./types-C1CAD70_.js";
|
|
17
|
-
import { $r as redTeamDataset, At as RuntimeEventLike, Bt as SatisfiedBy, Dt as summarizeBackendIntegrity, Et as assertRealBackend, Fo as RunCampaignOptions, Ft as CorrectnessChecker, Ht as completionVerdict, Io as runCampaign, It as LlmCorrectnessCheckerOpts, Jr as DEFAULT_RED_TEAM_CORPUS, Lt as ProducedProposal, Mt as extractProducedState, No as CampaignCellFailureReceipt, Nt as CompletionRequirement, Ot as ArtifactEventLike, Po as CampaignCellRetryPolicy, Pt as CompletionVerdict, Qr as RedTeamReport, Rt as ProducedState, Tt as BackendIntegrityReport, Ut as createLlmCorrectnessChecker, Vt as TaskGold, Wt as verifyCompletion, Xr as RedTeamCategory, Yr as RedTeamCase, Zr as RedTeamFinding, ai as CanaryOptions, da as LlmJudgeOptions, ei as redTeamReport, fa as llmJudge, ga as transientDispatchFailure, ha as quotaExhaustedUntil, ii as CanaryKind, jt as ToolCallEventLike, kt as ProposalEventLike, ni as CanaryAlert, oi as CanaryReport, ri as CanaryEvaluation, si as runCanaries, ti as scoreRedTeamOutput, wt as BackendIntegrityError, zt as RequirementCheck } from "./index-DwQxdWve.js";
|
|
18
16
|
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, s as TraceStore, t as EventFilter } from "./store-C4avFjIi.js";
|
|
19
17
|
import { a as MintedRolloutLine } from "./schema-B5LtYvn9.js";
|
|
20
18
|
import { a as mintRolloutRows, c as ScorePreference, i as RolloutSearchLineage, n as MintRolloutResult, t as MintRolloutOptions } from "./mint-C9Lojt1V.js";
|
|
19
|
+
import { R as Scenario$1, S as JudgeConfig, _ as GateDecision, a as CampaignResult, w as JudgeScore$1 } from "./types-BWwRXofp.js";
|
|
20
|
+
import { Ao as runCampaign, Bt as extractProducedState, Ca as quotaExhaustedUntil, Do as CampaignCellFailureReceipt, Ft as summarizeBackendIntegrity, Gt as ProducedProposal, Ht as CompletionVerdict, It as ArtifactEventLike, Jt as SatisfiedBy, Kt as ProducedState, Lt as ProposalEventLike, Mt as BackendIntegrityError, Nt as BackendIntegrityReport, Oo as CampaignCellRetryPolicy, Pt as assertRealBackend, Qt as verifyCompletion, Rt as RuntimeEventLike, Ut as CorrectnessChecker, Vt as CompletionRequirement, Wt as LlmCorrectnessCheckerOpts, Xt as completionVerdict, Yt as TaskGold, Zt as createLlmCorrectnessChecker, ai as RedTeamFinding, ba as llmJudge, ci as redTeamReport, di as CanaryEvaluation, fi as CanaryKind, hi as runCanaries, ii as RedTeamCategory, ko as RunCampaignOptions, li as scoreRedTeamOutput, mi as CanaryReport, ni as DEFAULT_RED_TEAM_CORPUS, oi as RedTeamReport, pi as CanaryOptions, qt as RequirementCheck, ri as RedTeamCase, si as redTeamDataset, ui as CanaryAlert, wa as transientDispatchFailure, ya as LlmJudgeOptions, zt as ToolCallEventLike } from "./index-DMPLG226.js";
|
|
21
21
|
import { a as ProfileAxisSpec, c as expandProfileAxes, i as HarnessType, l as harnessAxisOf, n as CODING_HARNESSES, o as agentProfileHash, r as HARNESS_NATIVE_MODEL, s as agentProfileId, t as AgentProfile } from "./agent-profile-DDfIjdyS.js";
|
|
22
22
|
import { i as DatasetSplit, n as DatasetManifest, r as DatasetScenario, t as Dataset } from "./dataset-DQqhOCPt.js";
|
|
23
23
|
import { C as defineEquivalenceCheck, S as buildEquivalenceRecord, _ as StrategyChecker, a as CheckerIdentity, b as VerificationStrategyProfile, c as EquivalenceCheckDefinition, d as EquivalenceCheckerInput, f as EquivalenceCheckerResult, g as EquivalenceRecord, h as EquivalenceProtocolError, i as equivalenceVerdict, l as EquivalenceCheckSpec, m as EquivalenceObligationStatus, n as VerdictCertification, o as CheckerOutcome, p as EquivalenceObligation, r as certificationEvidenceDigest, s as EquivalenceArm, t as DefaultVerdict, u as EquivalenceChecker, v as VERIFICATION_STRATEGIES, w as runEquivalenceCheck, x as VerificationStrategySource, y as VERIFICATION_STRATEGY_SOURCES } from "./verdict-E4eRNf7-.js";
|
|
@@ -27,20 +27,20 @@ import { a as FAILURE_BLAME, c as FailureContext, d as INFRA_FAILURE_BLAMES, f a
|
|
|
27
27
|
import { d as InsightReport, i as analyzeRuns, t as AnalyzeRunsOptions } from "./analyze-runs-CPiKLJFs.js";
|
|
28
28
|
import { i as ExactAnalystRunEvent, l as ExactCapableAnalyst, o as ExactAnalystRunResult } from "./exact-types-KGxBWH4b.js";
|
|
29
29
|
import { c as RegistryRunOpts, i as BudgetPolicy, n as AnalystRegistry, r as AnalystRegistryOptions, s as ExactRegistryRunOpts } from "./registry-Bu3lNole.js";
|
|
30
|
-
import { _ as SelfImproveMethodResult, a as DefinedAgentEval, c as SelfImproveMethodOptions, d as SelfImproveProposerOptions, f as SelfImproveProposerResult, g as SelfImproveMethodProvenance, h as selfImprove, i as DefineAgentEvalOptions, l as SelfImproveOptions, n as AgentEvalEvaluateOptions, o as defineAgentEval, p as SelfImproveResult, r as AgentEvalImproveOptions, s as SelfImproveBudget, t as AgentEvalAgent, u as SelfImproveProgressEvent } from "./define-agent-eval-
|
|
31
|
-
import { a as createTraceAnalyst, i as TraceAnalystDefinition, n as buildDefaultAnalystRegistry, r as CreateTraceAnalystOptions, t as DefaultAnalystRegistryOptions } from "./default-registry-
|
|
32
|
-
import { i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS } from "./engine-
|
|
33
|
-
import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as ConceptFinding, E as createChatTraceEngine, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine } from "./index-
|
|
34
|
-
import { i as BenchmarkEvaluation } from "./types-
|
|
30
|
+
import { _ as SelfImproveMethodResult, a as DefinedAgentEval, c as SelfImproveMethodOptions, d as SelfImproveProposerOptions, f as SelfImproveProposerResult, g as SelfImproveMethodProvenance, h as selfImprove, i as DefineAgentEvalOptions, l as SelfImproveOptions, n as AgentEvalEvaluateOptions, o as defineAgentEval, p as SelfImproveResult, r as AgentEvalImproveOptions, s as SelfImproveBudget, t as AgentEvalAgent, u as SelfImproveProgressEvent } from "./define-agent-eval-DQJvVa1P.js";
|
|
31
|
+
import { a as createTraceAnalyst, i as TraceAnalystDefinition, n as buildDefaultAnalystRegistry, r as CreateTraceAnalystOptions, t as DefaultAnalystRegistryOptions } from "./default-registry-C3HTE08O.js";
|
|
32
|
+
import { i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS } from "./engine-DjRV55F9.js";
|
|
33
|
+
import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as ConceptFinding, E as createChatTraceEngine, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine } from "./index-0jEtpSM7.js";
|
|
34
|
+
import { i as BenchmarkEvaluation } from "./types-qoxUfKQg.js";
|
|
35
35
|
import { a as SpanHandle, n as CaptureWrite, o as TraceEmitter, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-HcCmFQl_.js";
|
|
36
36
|
import { n as SandboxDriver } from "./sandbox-harness-BlSOu4LX.js";
|
|
37
|
-
import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-
|
|
37
|
+
import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-Cveoq6fJ.js";
|
|
38
38
|
import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-BhsrMupG.js";
|
|
39
39
|
import { n as TrajectoryStep, r as buildTrajectory, t as Trajectory } from "./trajectory-Y1fmdlw7.js";
|
|
40
40
|
import { a as runCounterfactual, i as CounterfactualRunner, n as CounterfactualMutation, r as CounterfactualResult, t as CounterfactualContext } from "./counterfactual-CPaWN99-.js";
|
|
41
41
|
import { A as AnalystFindingDigest, B as ControlRunResult, C as createFeedbackTrajectory, D as renderPreferenceMemoryMarkdown, E as feedbackTrajectoryToOptimizerRow, F as ControlActionOutcome, G as StopDecision, H as ControlRuntimeError, I as ControlBudget, J as subjectiveEval, K as objectiveEval, L as ControlContext, M as AnalystRunDigest, N as analystFindingDigest, O as summarizePreferenceMemory, P as analystRunDigest, R as ControlDecision, S as controlRunToFeedbackTrajectory, T as feedbackTrajectoriesToOptimizerRows, U as ControlSeverity, V as ControlRuntimeConfig, W as ControlStep, _ as PreferenceMemoryEntry, a as FeedbackLabel, b as analystRunToReviewRequests, c as FeedbackOptimizerRow, d as FeedbackTask, f as FeedbackTrajectory, g as InMemoryFeedbackTrajectoryStore, h as FileSystemFeedbackTrajectoryStore, i as FeedbackAttempt, j as AnalystReviewDecision, k as withAssignedFeedbackSplit, l as FeedbackOutcome, m as FeedbackTrajectoryStore, n as AnalystReviewRequest, o as FeedbackLabelKind, p as FeedbackTrajectoryFilter, q as runAgentControlLoop, r as FeedbackArtifactType, s as FeedbackLabelSource, t as AnalystFeedbackTrajectoryOptions, u as FeedbackSplitPolicy, v as ProposedSideEffect, w as feedbackTrajectoriesToDatasetScenarios, x as assignFeedbackSplit, y as analystRunToFeedbackTrajectory, z as ControlEvalResult } from "./feedback-trajectory-BXCGTRkz.js";
|
|
42
42
|
import { y as OUTPUT_VALUE } from "./attribute-vocabulary-CcXqHODT.js";
|
|
43
|
-
import { C as inferDomainKeywords, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, W as ExtractedUsage, g as TraceInsightSuite, h as TraceInsightReadiness, k as tokenizeDomainWords, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore, v as buildTraceInsightContext, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-
|
|
43
|
+
import { C as inferDomainKeywords, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, W as ExtractedUsage, g as TraceInsightSuite, h as TraceInsightReadiness, k as tokenizeDomainWords, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore, v as buildTraceInsightContext, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-BxrfuU2t.js";
|
|
44
44
|
import { a as RunIntegrityReport, o as assertRunCaptured, t as RunIntegrityError } from "./integrity-CaeNeusS.js";
|
|
45
45
|
import { c as judgeSpans, f as runsForScenario, i as argHash } from "./query-DBUQhehH.js";
|
|
46
46
|
import { a as SteeringChange, c as CampaignRunContext, d as CampaignVariant, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, l as CampaignRunOutcome, m as FailedRun, n as ExperimentResult, o as CampaignFactoryParams, p as EvalCampaignResult, r as FailureMode, s as CampaignIntegrityPolicy, t as ExperimentPlan, u as CampaignScenario } from "./researcher-DojEMb7E.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
|
|
2
|
-
import { i as JudgeError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
|
|
2
|
+
import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
|
|
3
3
|
import { a as hashCanonical, r as canonicalString } from "./canonical-BsZVgBhW.js";
|
|
4
|
-
import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-
|
|
4
|
+
import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-B2AhDRnq.js";
|
|
5
5
|
import { c as groupRunsByAgentProfileCell, d as validateAgentProfileCell, f as verifyAgentProfileCell, h as manifestContentDigest, i as agentProfileCellKey, m as hashJson, r as agentProfileCellHashMaterial, s as buildAgentProfileCell, t as AGENT_PROFILE_KINDS, u as toAgentProfileJson } from "./agent-profile-cell-DcFW4vOc.js";
|
|
6
6
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
7
7
|
import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
|
|
@@ -13,20 +13,21 @@ import { _ as passAtK, a as MANN_WHITNEY_EXACT_MAX_STATES, c as mannWhitneyU, d
|
|
|
13
13
|
import { a as pairedSignTest, i as pairedDeltaTieFraction, n as DECISION_PAIRED_DELTA_STATISTIC, o as pairedTTest, r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "./paired-tests-C8iCsioC.js";
|
|
14
14
|
import { t as eProcess } from "./sequential-eprocess-D1jKoihe.js";
|
|
15
15
|
import { n as iqr, r as welchsTTest } from "./baseline-BC-eBZ7U.js";
|
|
16
|
-
import {
|
|
16
|
+
import { n as pairedDeltaTest, t as minimumPairsForPairedDeltaTest } from "./paired-delta-test-BF-FEP8V.js";
|
|
17
|
+
import { n as pairedDecisionShape, t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
|
|
17
18
|
import { a as isToolSpan, i as isLlmSpan, r as isJudgeSpan, t as FAILURE_CLASSES } from "./schema-CSf6qWgZ.js";
|
|
18
19
|
import { d as runsForScenario, r as argHash, s as judgeSpans } from "./query-DlV0pdx4.js";
|
|
19
|
-
import { r as checkCanaries, t as analyzeRuns } from "./analyze-runs-
|
|
20
|
+
import { r as checkCanaries, t as analyzeRuns } from "./analyze-runs-B6gv7vAT.js";
|
|
20
21
|
import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
21
22
|
import { a as parseRunRecordSafe, c as runTaskScore, i as modelHasSnapshot, l as searchCellRunId, n as UNKNOWN_MODEL, o as roundTripRunRecord, r as isRunRecord, s as runCostFloorUsd, t as RunRecordValidationError, u as validateRunRecord } from "./run-record-BRlM99rX.js";
|
|
22
23
|
import { a as summaryTable, n as gainHistogram, r as paretoChart } from "./summary-report-BMvou2oZ.js";
|
|
23
24
|
import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
|
|
24
|
-
import { $ as
|
|
25
|
+
import { $ as redTeamDataset, K as dominates, Q as DEFAULT_RED_TEAM_CORPUS, U as pairedPromotionPower, W as requiredPairsForPairedPromotion, a as transientDispatchFailure, ct as runCampaign, et as redTeamReport, i as quotaExhaustedUntil, nt as runCanaries, o as aggregateRunScore, q as paretoFrontier, s as clamp01, t as llmJudge, tt as scoreRedTeamOutput } from "./llm-judge-Cififl_V.js";
|
|
25
26
|
import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-DOMpkoeb.js";
|
|
26
27
|
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-_IbkejCF.js";
|
|
27
|
-
import { B as assertRealBackend, F as surfaceContentHash, T as parseReflectionResponse, V as summarizeBackendIntegrity, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation, w as buildReflectionPrompt, z as BackendIntegrityError } from "./campaign-evidence-
|
|
28
|
-
import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-
|
|
29
|
-
import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-
|
|
28
|
+
import { B as assertRealBackend, F as surfaceContentHash, T as parseReflectionResponse, V as summarizeBackendIntegrity, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation, w as buildReflectionPrompt, z as BackendIntegrityError } from "./campaign-evidence-DJc60F9C.js";
|
|
29
|
+
import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-Dk1ZdJEm.js";
|
|
30
|
+
import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-qxAPjMSL.js";
|
|
30
31
|
import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-DhPoXApk.js";
|
|
31
32
|
import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
|
|
32
33
|
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
@@ -35,26 +36,26 @@ import { G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, U as parseFindingSubject, h a
|
|
|
35
36
|
import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-xMLX160E.js";
|
|
36
37
|
import { OUTPUT_VALUE } from "./trace-attributes.js";
|
|
37
38
|
import { r as extractUsageFromSse, t as extractUsage } from "./extract-usage-DFATEuWW.js";
|
|
38
|
-
import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-
|
|
39
|
-
import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-
|
|
39
|
+
import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-CXuOhLvT.js";
|
|
40
|
+
import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-BaGakFc2.js";
|
|
40
41
|
import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-BFP4kjGj.js";
|
|
41
42
|
import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-DNe_Uv1Q.js";
|
|
42
43
|
import { t as packageVersion$1 } from "./package-version-D7lQHt_-.js";
|
|
43
44
|
import { a as classifyFailure, i as INFRA_FAILURE_BLAMES, o as classifyFailureReason, r as FAILURE_BLAME, s as failureBlame, t as DEFAULT_FAILURE_REASON_RULES } from "./failure-taxonomy-CjabTATj.js";
|
|
44
45
|
import { n as runBoundedProcess, t as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS } from "./bounded-process-BBZob7vl.js";
|
|
45
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
46
|
+
import { t as runEvalCampaign } from "./eval-campaign-CXT2WYUe.js";
|
|
46
47
|
import "./rollout-B-XBReTC.js";
|
|
47
48
|
import { t as mintRolloutRows } from "./mint-iCD-W43s.js";
|
|
48
49
|
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
|
|
49
|
-
import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-
|
|
50
|
-
import { n as paidJsonChat } from "./chat-json-call-
|
|
50
|
+
import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-c_bKHTXz.js";
|
|
51
|
+
import { n as paidJsonChat } from "./chat-json-call-DnlDjpco.js";
|
|
51
52
|
import { i as makeProposalFinding, n as computeFindingId, r as makeFinding } from "./types-CW5b90JH.js";
|
|
52
|
-
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-
|
|
53
|
-
import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-
|
|
53
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-k_QdEBZo.js";
|
|
54
|
+
import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-Bh5lvKGg.js";
|
|
54
55
|
import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
|
|
55
56
|
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-BPowgPm6.js";
|
|
56
57
|
import { n as evaluateReleaseConfidence, r as bootstrapCi } from "./release-confidence-XQm_xjhR.js";
|
|
57
|
-
import { t as createChatClient } from "./chat-client-
|
|
58
|
+
import { t as createChatClient } from "./chat-client-CJORSG_J.js";
|
|
58
59
|
import { createHash } from "node:crypto";
|
|
59
60
|
import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
60
61
|
import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
@@ -1357,7 +1358,7 @@ function assertPredicate(value, where) {
|
|
|
1357
1358
|
]) if (p[field] !== void 0) assertMatcher(p[field], `${where} "${field}"`);
|
|
1358
1359
|
if (p.kind !== void 0 && !SPAN_KINDS.includes(p.kind)) throw new ValidationError(`${where}: "kind" must be one of ${SPAN_KINDS.join(", ")}`);
|
|
1359
1360
|
if (p.attr !== void 0 && (p.attr === null || typeof p.attr !== "object" || Array.isArray(p.attr))) throw new ValidationError(`${where}: "attr" must be a plain object`);
|
|
1360
|
-
for (const [key, expected] of Object.entries(p.attr ?? {})) if (
|
|
1361
|
+
for (const [key, expected] of Object.entries(p.attr ?? {})) if (expected !== null && typeof expected === "object") assertMatcher(expected, `${where} attr "${key}"`);
|
|
1361
1362
|
if (p.not !== void 0) assertPredicate(p.not, `${where} "not"`);
|
|
1362
1363
|
if (p.requiresCustom && typeof p.custom !== "function") throw new ValidationError(`${where}: rule was built with a custom predicate function, which does not survive JSON serialization — re-attach \`custom\` after deserializing or drop the rule`);
|
|
1363
1364
|
const hasAttr = p.attr !== void 0 && Object.keys(p.attr).length > 0;
|
|
@@ -1370,7 +1371,7 @@ function predicateMatches(span, p) {
|
|
|
1370
1371
|
if (p.model !== void 0 && !matchText(spanModel(span), p.model)) return false;
|
|
1371
1372
|
if (p.attr !== void 0) for (const [key, expected] of Object.entries(p.attr)) {
|
|
1372
1373
|
const actual = span.attributes?.[key];
|
|
1373
|
-
if (isRegexValue(expected)) {
|
|
1374
|
+
if (isRegexValue(expected) || isOneOfMatcher(expected)) {
|
|
1374
1375
|
if (!matchText(typeof actual === "string" ? actual : void 0, expected)) return false;
|
|
1375
1376
|
} else if (actual !== expected) return false;
|
|
1376
1377
|
}
|
|
@@ -1812,6 +1813,33 @@ function traceContract(name) {
|
|
|
1812
1813
|
function spanRef(span, index) {
|
|
1813
1814
|
return span.spanId ?? `#${index}`;
|
|
1814
1815
|
}
|
|
1816
|
+
/**
|
|
1817
|
+
* Whether this trace shows any sign that spans of `kind` are actually
|
|
1818
|
+
* captured, independent of whether any span happens to match a given rule's
|
|
1819
|
+
* predicate. A `never`, `atMost`, `tokensAtMost` or `retrySafe` rule that
|
|
1820
|
+
* targets TOOL or LLM spans and finds zero matches cannot otherwise tell "no
|
|
1821
|
+
* calls happened" from "calls were not recorded": an exporter that silently
|
|
1822
|
+
* drops a span kind makes `forbidden`, `allowed`, `maxCalls` and `retries`
|
|
1823
|
+
* checks on that kind pass vacuously. Evidence is: a span of that kind
|
|
1824
|
+
* anywhere in the trace, or — for TOOL — a recorded offered-tool set
|
|
1825
|
+
* (`gen_ai.tool.definitions`), which proves the capture pipeline ran even on
|
|
1826
|
+
* a turn that called no tool.
|
|
1827
|
+
*/
|
|
1828
|
+
function hasCaptureEvidence(spans, kind) {
|
|
1829
|
+
if (spans.some((span) => contractSpanKind(span) === kind)) return true;
|
|
1830
|
+
if (kind === "TOOL") return spans.some((span) => spanOfferedTools(span).recorded);
|
|
1831
|
+
return false;
|
|
1832
|
+
}
|
|
1833
|
+
/** Only `TOOL`/`LLM`-scoped predicates have capture evidence to check. */
|
|
1834
|
+
function capturedKindOf(p) {
|
|
1835
|
+
return p.kind === "TOOL" || p.kind === "LLM" ? p.kind : void 0;
|
|
1836
|
+
}
|
|
1837
|
+
function requireCaptureEvidence(spans, p, rule, situation) {
|
|
1838
|
+
const kind = capturedKindOf(p);
|
|
1839
|
+
if (kind === void 0) return;
|
|
1840
|
+
if (hasCaptureEvidence(spans, kind)) return;
|
|
1841
|
+
throw new CaptureIntegrityError(`rule '${rule}': ${situation} and the trace has no ${kind} capture evidence (no ${kind} span` + (kind === "TOOL" ? `, no ${TOOL_DEFINITIONS_ATTR}` : "") + `) — cannot tell whether ${describePredicate(p)} never happened or was never recorded`);
|
|
1842
|
+
}
|
|
1815
1843
|
function indexed(spans, p) {
|
|
1816
1844
|
const out = [];
|
|
1817
1845
|
spans.forEach((span, i) => {
|
|
@@ -2065,7 +2093,9 @@ function checkRetrySafe(rule, spans) {
|
|
|
2065
2093
|
const out = [];
|
|
2066
2094
|
const groups = /* @__PURE__ */ new Map();
|
|
2067
2095
|
const unseen = /* @__PURE__ */ new Map();
|
|
2068
|
-
|
|
2096
|
+
const calls = toolSpans(spans);
|
|
2097
|
+
if (calls.length === 0 && !hasCaptureEvidence(spans, "TOOL")) throw new CaptureIntegrityError(`rule '${rule.label}': retrySafe found 0 TOOL spans and the trace has no TOOL capture evidence (no TOOL span, no ${TOOL_DEFINITIONS_ATTR}) — cannot tell whether no reads or writes happened or the trace lost its TOOL spans`);
|
|
2098
|
+
for (const call of calls) {
|
|
2069
2099
|
const name = contractSpanToolName(call.span);
|
|
2070
2100
|
if (name === void 0) {
|
|
2071
2101
|
out.push({
|
|
@@ -2181,11 +2211,15 @@ function checkRule(rule, ctx) {
|
|
|
2181
2211
|
spanId: span.spanId,
|
|
2182
2212
|
detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") fails always(${describePredicate(rule.p)})`
|
|
2183
2213
|
}]);
|
|
2184
|
-
case "never":
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2214
|
+
case "never": {
|
|
2215
|
+
const matches = indexed(spans, rule.p);
|
|
2216
|
+
if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `never(${describePredicate(rule.p)}) matched 0 spans`);
|
|
2217
|
+
return matches.map(({ span, ref }) => ({
|
|
2218
|
+
rule: rule.label,
|
|
2219
|
+
spanId: span.spanId,
|
|
2220
|
+
detail: `span ${ref} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
|
|
2221
|
+
}));
|
|
2222
|
+
}
|
|
2189
2223
|
case "eventually": return spans.some((span) => predicateMatches(span, rule.p)) ? [] : [{
|
|
2190
2224
|
rule: rule.label,
|
|
2191
2225
|
detail: `no span matches eventually(${describePredicate(rule.p)}) over ${spans.length} span(s)`
|
|
@@ -2208,6 +2242,7 @@ function checkRule(rule, ctx) {
|
|
|
2208
2242
|
case "neverUnless": return orderViolations(rule.label, indexed(spans, rule.prior), indexed(spans, rule.p), rule.order ?? "start-order", `${describePredicate(rule.p)} without an earlier ${describePredicate(rule.prior)}`);
|
|
2209
2243
|
case "atMost": {
|
|
2210
2244
|
const matches = indexed(spans, rule.p);
|
|
2245
|
+
if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `atMost(${describePredicate(rule.p)}) matched 0 spans`);
|
|
2211
2246
|
return matches.length <= rule.max ? [] : [{
|
|
2212
2247
|
rule: rule.label,
|
|
2213
2248
|
spanId: matches[rule.max]?.span.spanId,
|
|
@@ -2216,6 +2251,7 @@ function checkRule(rule, ctx) {
|
|
|
2216
2251
|
}
|
|
2217
2252
|
case "tokensAtMost": {
|
|
2218
2253
|
const matches = indexed(spans, rule.p);
|
|
2254
|
+
if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `tokensAtMost(${describePredicate(rule.p)}) matched 0 spans`);
|
|
2219
2255
|
const unknown = matches.find((m) => spanTotalTokens(m.span) === void 0);
|
|
2220
2256
|
if (unknown) return [{
|
|
2221
2257
|
rule: rule.label,
|
|
@@ -9077,290 +9113,6 @@ function isObject(v) {
|
|
|
9077
9113
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
9078
9114
|
}
|
|
9079
9115
|
//#endregion
|
|
9080
|
-
//#region src/paired-promotion-power.ts
|
|
9081
|
-
/**
|
|
9082
|
-
* @module
|
|
9083
|
-
* Joint power of a configured {@link decidePairedPromotion} call, computed by
|
|
9084
|
-
* running that call on data drawn from a registered alternative.
|
|
9085
|
-
*
|
|
9086
|
-
* Why a simulation and not a formula: `decidePairedPromotion` is not one test.
|
|
9087
|
-
* It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
|
|
9088
|
-
* paired bootstrap at or above it) and by shape (Tango's score interval on a
|
|
9089
|
-
* two-point outcome), and on top of the deciding interval it applies a
|
|
9090
|
-
* sufficiency floor, a zero-width refusal and McNemar's exact veto. A
|
|
9091
|
-
* preregistration usually seals two such calls, a continuous primary and a
|
|
9092
|
-
* pass/fail veto drawn from the same episodes, and promotes only when both
|
|
9093
|
-
* promote. The sample size at which THAT procedure reaches a target power is
|
|
9094
|
-
* not the power of any one channel: the sign-test power at n can be 0.8 while
|
|
9095
|
-
* the promotion probability is well below it because the veto binds
|
|
9096
|
-
* (agent-eval#785). Closed forms exist for single channels only, and each one
|
|
9097
|
-
* shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
|
|
9098
|
-
* Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
|
|
9099
|
-
* pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
|
|
9100
|
-
* So the only faithful power is the procedure's own promotion rate under the
|
|
9101
|
-
* alternative, and that is what this module computes: draw n pairs from the
|
|
9102
|
-
* registered law, run every configured call on them exactly as the caller
|
|
9103
|
-
* will, repeat, count.
|
|
9104
|
-
*
|
|
9105
|
-
* The alternative is DECLARATIVE and serializable, never a sampler function,
|
|
9106
|
-
* so a preregistration can seal it as bytes: a mixture of cells, each with a
|
|
9107
|
-
* probability, an optional pass/fail outcome per arm for the binary calls,
|
|
9108
|
-
* and a law for the candidate-minus-baseline delta on the continuous scale.
|
|
9109
|
-
* Drawing the cell first and the delta given the cell is what makes the two
|
|
9110
|
-
* channels one joint law rather than two marginals.
|
|
9111
|
-
*
|
|
9112
|
-
* Everything is seeded (`mulberry32`), so a sizing is reproducible from its
|
|
9113
|
-
* inputs, and every estimate carries its Monte Carlo standard error and a
|
|
9114
|
-
* Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
|
|
9115
|
-
*/
|
|
9116
|
-
/**
|
|
9117
|
-
* Joint power of the configured calls at `n` under the registered alternative:
|
|
9118
|
-
* the fraction of simulated data sets on which every call promotes.
|
|
9119
|
-
*/
|
|
9120
|
-
function pairedPromotionPower(options) {
|
|
9121
|
-
const n = options.n;
|
|
9122
|
-
if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
|
|
9123
|
-
const simulations = options.simulations ?? 1e3;
|
|
9124
|
-
if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
|
|
9125
|
-
const seed = options.seed ?? 1;
|
|
9126
|
-
if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
|
|
9127
|
-
const calls = validateCalls(options.calls);
|
|
9128
|
-
const alternative = validateAlternative(options.alternative, calls);
|
|
9129
|
-
const rng = mulberry32(seed);
|
|
9130
|
-
let promotions = 0;
|
|
9131
|
-
let refusals = 0;
|
|
9132
|
-
const perCall = calls.map(() => ({
|
|
9133
|
-
promote: 0,
|
|
9134
|
-
insufficient: 0,
|
|
9135
|
-
indeterminate: 0,
|
|
9136
|
-
exactTestVetoes: 0,
|
|
9137
|
-
hold: 0,
|
|
9138
|
-
methods: /* @__PURE__ */ new Map()
|
|
9139
|
-
}));
|
|
9140
|
-
for (let s = 0; s < simulations; s++) {
|
|
9141
|
-
const pairs = drawPairs(alternative, n, rng);
|
|
9142
|
-
let all = true;
|
|
9143
|
-
let anyRefusal = false;
|
|
9144
|
-
for (let c = 0; c < calls.length; c++) {
|
|
9145
|
-
const call = calls[c];
|
|
9146
|
-
const decision = runCall(call, pairs, n);
|
|
9147
|
-
const tally = perCall[c];
|
|
9148
|
-
tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
|
|
9149
|
-
if (decision.promote) tally.promote++;
|
|
9150
|
-
else {
|
|
9151
|
-
if (!decision.sufficient) tally.insufficient++;
|
|
9152
|
-
if (decision.indeterminate) tally.indeterminate++;
|
|
9153
|
-
if (decision.exactTestVetoes) tally.exactTestVetoes++;
|
|
9154
|
-
if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
|
|
9155
|
-
}
|
|
9156
|
-
if (!decision.promote) all = false;
|
|
9157
|
-
if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
|
|
9158
|
-
}
|
|
9159
|
-
if (all) promotions++;
|
|
9160
|
-
else if (anyRefusal) refusals++;
|
|
9161
|
-
}
|
|
9162
|
-
const power = promotions / simulations;
|
|
9163
|
-
const interval = wilson(promotions, simulations, .95);
|
|
9164
|
-
return {
|
|
9165
|
-
n,
|
|
9166
|
-
simulations,
|
|
9167
|
-
seed,
|
|
9168
|
-
power,
|
|
9169
|
-
standardError: Math.sqrt(power * (1 - power) / simulations),
|
|
9170
|
-
low: interval.lower,
|
|
9171
|
-
high: interval.upper,
|
|
9172
|
-
refuse: refusals / simulations,
|
|
9173
|
-
hold: (simulations - promotions - refusals) / simulations,
|
|
9174
|
-
calls: perCall.map((tally, c) => ({
|
|
9175
|
-
outcome: calls[c].outcome,
|
|
9176
|
-
promote: tally.promote / simulations,
|
|
9177
|
-
insufficient: tally.insufficient / simulations,
|
|
9178
|
-
indeterminate: tally.indeterminate / simulations,
|
|
9179
|
-
exactTestVetoes: tally.exactTestVetoes / simulations,
|
|
9180
|
-
hold: tally.hold / simulations,
|
|
9181
|
-
methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
|
|
9182
|
-
}))
|
|
9183
|
-
};
|
|
9184
|
-
}
|
|
9185
|
-
/**
|
|
9186
|
-
* First n at which {@link pairedPromotionPower} reaches the target, scanning
|
|
9187
|
-
* every n from `minPairs` upward. Both the point-estimate answer and the
|
|
9188
|
-
* confirmed lower-bound answer are returned; the scan runs until the latter is
|
|
9189
|
-
* reached (so the curve past `n` is retained) or `maxPairs` is exhausted. A
|
|
9190
|
-
* candidate selected by the scan's bound is certified only by an independent
|
|
9191
|
-
* simulation at that n on a different seed; a candidate that fails it is
|
|
9192
|
-
* recorded and the scan continues.
|
|
9193
|
-
*/
|
|
9194
|
-
function requiredPairsForPairedPromotion(options) {
|
|
9195
|
-
const target = options.target ?? .8;
|
|
9196
|
-
if (!Number.isFinite(target) || target <= 0 || target >= 1) throw new Error(`requiredPairsForPairedPromotion: target must be in (0,1), got ${target}`);
|
|
9197
|
-
const minPairs = options.minPairs ?? 1;
|
|
9198
|
-
const maxPairs = options.maxPairs ?? 200;
|
|
9199
|
-
if (!Number.isInteger(minPairs) || minPairs < 1) throw new Error(`requiredPairsForPairedPromotion: minPairs must be a positive integer, got ${minPairs}`);
|
|
9200
|
-
if (!Number.isInteger(maxPairs) || maxPairs < minPairs) throw new Error(`requiredPairsForPairedPromotion: maxPairs must be an integer >= minPairs, got ${maxPairs}`);
|
|
9201
|
-
const seed = options.seed ?? 1;
|
|
9202
|
-
const curve = [];
|
|
9203
|
-
const rejected = [];
|
|
9204
|
-
let n = null;
|
|
9205
|
-
let nAtLowerBound = null;
|
|
9206
|
-
let confirmation = null;
|
|
9207
|
-
for (let candidate = minPairs; candidate <= maxPairs; candidate++) {
|
|
9208
|
-
const result = pairedPromotionPower({
|
|
9209
|
-
n: candidate,
|
|
9210
|
-
alternative: options.alternative,
|
|
9211
|
-
calls: options.calls,
|
|
9212
|
-
simulations: options.simulations,
|
|
9213
|
-
seed
|
|
9214
|
-
});
|
|
9215
|
-
curve.push(result);
|
|
9216
|
-
if (n === null && result.power >= target) n = candidate;
|
|
9217
|
-
if (result.low >= target) {
|
|
9218
|
-
const check = pairedPromotionPower({
|
|
9219
|
-
n: candidate,
|
|
9220
|
-
alternative: options.alternative,
|
|
9221
|
-
calls: options.calls,
|
|
9222
|
-
simulations: options.simulations,
|
|
9223
|
-
seed: confirmationSeed(seed, candidate)
|
|
9224
|
-
});
|
|
9225
|
-
if (check.low >= target) {
|
|
9226
|
-
nAtLowerBound = candidate;
|
|
9227
|
-
confirmation = check;
|
|
9228
|
-
break;
|
|
9229
|
-
}
|
|
9230
|
-
rejected.push(check);
|
|
9231
|
-
}
|
|
9232
|
-
}
|
|
9233
|
-
return {
|
|
9234
|
-
target,
|
|
9235
|
-
n,
|
|
9236
|
-
nAtLowerBound,
|
|
9237
|
-
confirmation,
|
|
9238
|
-
curve,
|
|
9239
|
-
rejected
|
|
9240
|
-
};
|
|
9241
|
-
}
|
|
9242
|
-
/** A seed for the confirmation at `n` that never equals the scan seed. */
|
|
9243
|
-
function confirmationSeed(seed, n) {
|
|
9244
|
-
return (Math.imul((seed | 0) ^ 2654435769, 2246822507) + Math.imul(n, 3266489909) | 1) >>> 0;
|
|
9245
|
-
}
|
|
9246
|
-
function drawPairs(alternative, n, rng) {
|
|
9247
|
-
const baseline = alternative.baseline ?? {
|
|
9248
|
-
kind: "point",
|
|
9249
|
-
value: 0
|
|
9250
|
-
};
|
|
9251
|
-
const control = new Array(n);
|
|
9252
|
-
const candidate = new Array(n);
|
|
9253
|
-
const controlPass = new Array(n);
|
|
9254
|
-
const candidatePass = new Array(n);
|
|
9255
|
-
for (let i = 0; i < n; i++) {
|
|
9256
|
-
const cell = drawCell(alternative.cells, rng);
|
|
9257
|
-
const base = drawValue(baseline, rng);
|
|
9258
|
-
control[i] = base;
|
|
9259
|
-
candidate[i] = base + drawValue(cell.delta, rng);
|
|
9260
|
-
controlPass[i] = cell.pass?.control ?? false;
|
|
9261
|
-
candidatePass[i] = cell.pass?.treatment ?? false;
|
|
9262
|
-
}
|
|
9263
|
-
return {
|
|
9264
|
-
control,
|
|
9265
|
-
candidate,
|
|
9266
|
-
controlPass,
|
|
9267
|
-
candidatePass
|
|
9268
|
-
};
|
|
9269
|
-
}
|
|
9270
|
-
function drawCell(cells, rng) {
|
|
9271
|
-
const u = rng();
|
|
9272
|
-
let cumulative = 0;
|
|
9273
|
-
for (const cell of cells) {
|
|
9274
|
-
cumulative += cell.probability;
|
|
9275
|
-
if (u < cumulative) return cell;
|
|
9276
|
-
}
|
|
9277
|
-
return cells[cells.length - 1];
|
|
9278
|
-
}
|
|
9279
|
-
function drawValue(law, rng) {
|
|
9280
|
-
switch (law.kind) {
|
|
9281
|
-
case "point": return law.value;
|
|
9282
|
-
case "atoms": {
|
|
9283
|
-
const u = rng();
|
|
9284
|
-
let cumulative = 0;
|
|
9285
|
-
for (const atom of law.atoms) {
|
|
9286
|
-
cumulative += atom.probability;
|
|
9287
|
-
if (u < cumulative) return atom.value;
|
|
9288
|
-
}
|
|
9289
|
-
return law.atoms[law.atoms.length - 1].value;
|
|
9290
|
-
}
|
|
9291
|
-
case "normal": {
|
|
9292
|
-
const u1 = 1 - rng();
|
|
9293
|
-
const u2 = rng();
|
|
9294
|
-
return law.mean + law.sd * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
|
9295
|
-
}
|
|
9296
|
-
}
|
|
9297
|
-
}
|
|
9298
|
-
function runCall(call, pairs, n) {
|
|
9299
|
-
const options = {
|
|
9300
|
-
minPairs: n,
|
|
9301
|
-
...call.options
|
|
9302
|
-
};
|
|
9303
|
-
if (call.outcome === "pass") {
|
|
9304
|
-
const scale = call.options.binaryScale;
|
|
9305
|
-
return decidePairedPromotion(pairs.controlPass.map((p) => p ? scale : 0), pairs.candidatePass.map((p) => p ? scale : 0), options);
|
|
9306
|
-
}
|
|
9307
|
-
return decidePairedPromotion(pairs.control, pairs.candidate, options);
|
|
9308
|
-
}
|
|
9309
|
-
function validateCalls(calls) {
|
|
9310
|
-
if (!Array.isArray(calls) || calls.length === 0) throw new Error("pairedPromotionPower: at least one call is required");
|
|
9311
|
-
calls.forEach((call, i) => {
|
|
9312
|
-
if (call.outcome !== "delta" && call.outcome !== "pass") throw new Error(`pairedPromotionPower: calls[${i}].outcome must be 'delta' or 'pass'`);
|
|
9313
|
-
if (call.options === null || typeof call.options !== "object") throw new Error(`pairedPromotionPower: calls[${i}].options must be an object`);
|
|
9314
|
-
if (call.options.minPairs !== void 0 && (!Number.isInteger(call.options.minPairs) || call.options.minPairs < 1)) throw new Error(`pairedPromotionPower: calls[${i}].options.minPairs must be a positive integer`);
|
|
9315
|
-
if (call.outcome === "pass") {
|
|
9316
|
-
const scale = call.options.binaryScale;
|
|
9317
|
-
if (scale === void 0 || !Number.isFinite(scale) || scale <= 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call and must declare a finite positive options.binaryScale`);
|
|
9318
|
-
if (call.options.continuous) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; continuous cannot accompany it`);
|
|
9319
|
-
if (call.options.statistic === "median") throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; binaryScale requires the mean statistic, not median`);
|
|
9320
|
-
} else if (call.options.binaryScale !== void 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'delta' call and cannot declare binaryScale; use a 'pass' call for a two-point outcome`);
|
|
9321
|
-
});
|
|
9322
|
-
return calls;
|
|
9323
|
-
}
|
|
9324
|
-
function validateAlternative(alternative, calls) {
|
|
9325
|
-
if (!alternative || !Array.isArray(alternative.cells) || alternative.cells.length === 0) throw new Error("pairedPromotionPower: alternative.cells must be a non-empty array");
|
|
9326
|
-
const needsPass = calls.some((call) => call.outcome === "pass");
|
|
9327
|
-
let total = 0;
|
|
9328
|
-
alternative.cells.forEach((cell, i) => {
|
|
9329
|
-
if (!Number.isFinite(cell.probability) || cell.probability < 0) throw new Error(`pairedPromotionPower: cells[${i}].probability must be a finite number >= 0`);
|
|
9330
|
-
total += cell.probability;
|
|
9331
|
-
if (needsPass) {
|
|
9332
|
-
if (!cell.pass || typeof cell.pass.control !== "boolean" || typeof cell.pass.treatment !== "boolean") throw new Error(`pairedPromotionPower: cells[${i}].pass must give control and treatment booleans when a 'pass' call is configured`);
|
|
9333
|
-
}
|
|
9334
|
-
validateLaw(cell.delta, `cells[${i}].delta`);
|
|
9335
|
-
});
|
|
9336
|
-
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: cell probabilities must sum to 1, got ${total}`);
|
|
9337
|
-
if (alternative.baseline !== void 0) validateLaw(alternative.baseline, "baseline");
|
|
9338
|
-
return alternative;
|
|
9339
|
-
}
|
|
9340
|
-
function validateLaw(law, where) {
|
|
9341
|
-
if (!law || typeof law !== "object") throw new Error(`pairedPromotionPower: ${where} must be a law object`);
|
|
9342
|
-
switch (law.kind) {
|
|
9343
|
-
case "point":
|
|
9344
|
-
if (!Number.isFinite(law.value)) throw new Error(`pairedPromotionPower: ${where}.value must be finite`);
|
|
9345
|
-
return;
|
|
9346
|
-
case "atoms": {
|
|
9347
|
-
if (!Array.isArray(law.atoms) || law.atoms.length === 0) throw new Error(`pairedPromotionPower: ${where}.atoms must be a non-empty array`);
|
|
9348
|
-
let total = 0;
|
|
9349
|
-
law.atoms.forEach((atom, i) => {
|
|
9350
|
-
if (!Number.isFinite(atom.value)) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].value must be finite`);
|
|
9351
|
-
if (!Number.isFinite(atom.probability) || atom.probability < 0) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].probability must be a finite number >= 0`);
|
|
9352
|
-
total += atom.probability;
|
|
9353
|
-
});
|
|
9354
|
-
if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: ${where}.atoms probabilities must sum to 1, got ${total}`);
|
|
9355
|
-
return;
|
|
9356
|
-
}
|
|
9357
|
-
case "normal":
|
|
9358
|
-
if (!Number.isFinite(law.mean) || !Number.isFinite(law.sd) || law.sd < 0) throw new Error(`pairedPromotionPower: ${where} needs a finite mean and a finite sd >= 0`);
|
|
9359
|
-
return;
|
|
9360
|
-
default: throw new Error(`pairedPromotionPower: ${where}.kind must be 'point', 'atoms' or 'normal'`);
|
|
9361
|
-
}
|
|
9362
|
-
}
|
|
9363
|
-
//#endregion
|
|
9364
9116
|
//#region src/treatment-gate.ts
|
|
9365
9117
|
/**
|
|
9366
9118
|
* Core predicate. Gated (treatment-not-applied) ONLY when telemetry was
|