@tangle-network/agent-eval 0.192.0 → 0.193.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/CHANGELOG.md +45 -2
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/analyst/index.d.ts +6 -6
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
  6. package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
  7. package/dist/{benchmark-command-C458yEFK.js → benchmark-command-N5r3GXMP.js} +8 -8
  8. package/dist/{benchmark-command-C458yEFK.js.map → benchmark-command-N5r3GXMP.js.map} +1 -1
  9. package/dist/benchmarks/index.d.ts +3 -3
  10. package/dist/benchmarks/index.js +3 -3
  11. package/dist/campaign/index.d.ts +5 -5
  12. package/dist/campaign/index.js +11 -10
  13. package/dist/{campaign-BVQVBEfH.js → campaign-DnL0S4XL.js} +11 -10
  14. package/dist/{campaign-BVQVBEfH.js.map → campaign-DnL0S4XL.js.map} +1 -1
  15. package/dist/{campaign-evidence-DcQiYhJI.js → campaign-evidence-DJc60F9C.js} +2 -2
  16. package/dist/{campaign-evidence-DcQiYhJI.js.map → campaign-evidence-DJc60F9C.js.map} +1 -1
  17. package/dist/{chat-client-B_ItsSBw.js → chat-client-CJORSG_J.js} +2 -2
  18. package/dist/{chat-client-B_ItsSBw.js.map → chat-client-CJORSG_J.js.map} +1 -1
  19. package/dist/{chat-json-call-CXHBvUz8.js → chat-json-call-DnlDjpco.js} +2 -2
  20. package/dist/{chat-json-call-CXHBvUz8.js.map → chat-json-call-DnlDjpco.js.map} +1 -1
  21. package/dist/cli.js +49 -12
  22. package/dist/cli.js.map +1 -1
  23. package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
  24. package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
  25. package/dist/contract/index.d.ts +7 -7
  26. package/dist/contract/index.js +11 -11
  27. package/dist/{default-registry-DdqA_-_o.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
  28. package/dist/{default-registry-DdqA_-_o.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
  29. package/dist/{define-agent-eval-DcludNJS.d.ts → define-agent-eval-DQJvVa1P.d.ts} +6 -6
  30. package/dist/{define-agent-eval-DcludNJS.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
  31. package/dist/{define-agent-eval-DQuos3Aw.js → define-agent-eval-qxAPjMSL.js} +6 -6
  32. package/dist/{define-agent-eval-DQuos3Aw.js.map → define-agent-eval-qxAPjMSL.js.map} +1 -1
  33. package/dist/diagnosis/index.d.ts +1 -1
  34. package/dist/diagnosis/index.d.ts.map +1 -1
  35. package/dist/diagnosis/index.js +12 -6
  36. package/dist/diagnosis/index.js.map +1 -1
  37. package/dist/{dspy-rlm-engine-BX34vPVY.js → dspy-rlm-engine-k_QdEBZo.js} +2 -2
  38. package/dist/{dspy-rlm-engine-BX34vPVY.js.map → dspy-rlm-engine-k_QdEBZo.js.map} +1 -1
  39. package/dist/{engine-C1DIAEpn.d.ts → engine-DjRV55F9.d.ts} +2 -2
  40. package/dist/{engine-C1DIAEpn.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
  41. package/dist/{eval-campaign-CwsBn0Gm.js → eval-campaign-CXT2WYUe.js} +2 -2
  42. package/dist/{eval-campaign-CwsBn0Gm.js.map → eval-campaign-CXT2WYUe.js.map} +1 -1
  43. package/dist/evaluation.d.ts +1 -1
  44. package/dist/experiment/index.d.ts +2 -2
  45. package/dist/experiment/index.js +4 -4
  46. package/dist/{external-optimizer-process-D-Sab4v3.js → external-optimizer-process-_YIECYJ7.js} +2 -2
  47. package/dist/{external-optimizer-process-D-Sab4v3.js.map → external-optimizer-process-_YIECYJ7.js.map} +1 -1
  48. package/dist/{external-optimizer-subprocess-kbqC-iWf.js → external-optimizer-subprocess-BrghTvEp.js} +2 -2
  49. package/dist/{external-optimizer-subprocess-kbqC-iWf.js.map → external-optimizer-subprocess-BrghTvEp.js.map} +1 -1
  50. package/dist/hosted/index.d.ts +2 -2
  51. package/dist/hosted/index.js +1 -1
  52. package/dist/{index-DBoXFaz0.d.ts → index-0jEtpSM7.d.ts} +3 -3
  53. package/dist/{index-DBoXFaz0.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
  54. package/dist/{index-DwQxdWve.d.ts → index-DMPLG226.d.ts} +89 -272
  55. package/dist/index-DMPLG226.d.ts.map +1 -0
  56. package/dist/{index-DNVzu9Vv.d.ts → index-DWQNDqPd.d.ts} +7 -7
  57. package/dist/{index-DNVzu9Vv.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
  58. package/dist/index.d.ts +10 -10
  59. package/dist/index.js +60 -308
  60. package/dist/index.js.map +1 -1
  61. package/dist/jev.d.ts +1 -1
  62. package/dist/ledger-core/index.d.ts +1 -1
  63. package/dist/{llm-client-KYQBh9C8.js → llm-client-c_bKHTXz.js} +2 -2
  64. package/dist/{llm-client-KYQBh9C8.js.map → llm-client-c_bKHTXz.js.map} +1 -1
  65. package/dist/{llm-judge-Z6MkTkd0.js → llm-judge-Cififl_V.js} +1024 -237
  66. package/dist/llm-judge-Cififl_V.js.map +1 -0
  67. package/dist/{matrix-CQMMrEQ5.d.ts → matrix-CNu13gg8.d.ts} +2 -2
  68. package/dist/{matrix-CQMMrEQ5.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
  69. package/dist/multishot/golden/index.d.ts +1 -1
  70. package/dist/multishot/index.d.ts +2 -2
  71. package/dist/openapi.json +1 -1
  72. package/dist/{otel-spans-DS7t1t1J.js → otel-spans-CQmSxtcS.js} +2 -2
  73. package/dist/{otel-spans-DS7t1t1J.js.map → otel-spans-CQmSxtcS.js.map} +1 -1
  74. package/dist/paired-delta-test-BF-FEP8V.js +78 -0
  75. package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
  76. package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
  77. package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
  78. package/dist/pipelines/index.d.ts +1 -1
  79. package/dist/{produced-state-CVmCPQDJ.js → produced-state-B2AhDRnq.js} +4 -4
  80. package/dist/{produced-state-CVmCPQDJ.js.map → produced-state-B2AhDRnq.js.map} +1 -1
  81. package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
  82. package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
  83. package/dist/{raw-provider-sink-DYh4Bszw.js → raw-provider-sink-CXuOhLvT.js} +2 -2
  84. package/dist/{raw-provider-sink-DYh4Bszw.js.map → raw-provider-sink-CXuOhLvT.js.map} +1 -1
  85. package/dist/{redact-CopuMxll.d.ts → redact-Cveoq6fJ.d.ts} +2 -2
  86. package/dist/{redact-CopuMxll.d.ts.map → redact-Cveoq6fJ.d.ts.map} +1 -1
  87. package/dist/{redact-B7rt9Zot.js → redact-Dk1ZdJEm.js} +59 -26
  88. package/dist/redact-Dk1ZdJEm.js.map +1 -0
  89. package/dist/rl.d.ts +1 -1
  90. package/dist/rl.js +2 -2
  91. package/dist/{search-ledger-Ck7tok0h.js → search-ledger-CJiAEU3j.js} +89 -27
  92. package/dist/search-ledger-CJiAEU3j.js.map +1 -0
  93. package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
  94. package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
  95. package/dist/{search-shipper-BG182p-U.js → search-shipper-DUlwlCsK.js} +2 -2
  96. package/dist/{search-shipper-BG182p-U.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
  97. package/dist/search-summary-BhY8p0Av.js +378 -0
  98. package/dist/search-summary-BhY8p0Av.js.map +1 -0
  99. package/dist/{semantic-concept-judge-Bg-560H0.js → semantic-concept-judge-Bh5lvKGg.js} +4 -4
  100. package/dist/{semantic-concept-judge-Bg-560H0.js.map → semantic-concept-judge-Bh5lvKGg.js.map} +1 -1
  101. package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
  102. package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
  103. package/dist/{server-DtetGGoA.js → server-D56r1lhp.js} +2 -2
  104. package/dist/{server-DtetGGoA.js.map → server-D56r1lhp.js.map} +1 -1
  105. package/dist/{skillopt-optimization-method-DoayuM6D.js → skillopt-optimization-method-C6DbFKnw.js} +8 -8
  106. package/dist/{skillopt-optimization-method-DoayuM6D.js.map → skillopt-optimization-method-C6DbFKnw.js.map} +1 -1
  107. package/dist/{spans-BWtjk_9c.d.ts → spans-D9_ehFuk.d.ts} +2 -2
  108. package/dist/{spans-BWtjk_9c.d.ts.map → spans-D9_ehFuk.d.ts.map} +1 -1
  109. package/dist/{statistical-heldout-C1Ej5Ce8.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
  110. package/dist/{statistical-heldout-C1Ej5Ce8.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
  111. package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
  112. package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
  113. package/dist/{store-tool-spans-BcC_5ef-.js → store-tool-spans-BaGakFc2.js} +2 -2
  114. package/dist/{store-tool-spans-BcC_5ef-.js.map → store-tool-spans-BaGakFc2.js.map} +1 -1
  115. package/dist/{store-tool-spans-ByLdU26j.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
  116. package/dist/{store-tool-spans-ByLdU26j.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
  117. package/dist/{task-failure-attributes-CEqEfuIY.js → task-failure-attributes-DngaLdE4.js} +4 -3
  118. package/dist/task-failure-attributes-DngaLdE4.js.map +1 -0
  119. package/dist/{tool-groups-DiBi16ns.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
  120. package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
  121. package/dist/trace-repair/index.d.ts +1 -1
  122. package/dist/traces.d.ts +3 -3
  123. package/dist/traces.js +4 -4
  124. package/dist/{types-C1CAD70_.d.ts → types-BWwRXofp.d.ts} +328 -4
  125. package/dist/types-BWwRXofp.d.ts.map +1 -0
  126. package/dist/{types-ByZslIlr.d.ts → types-qoxUfKQg.d.ts} +2 -2
  127. package/dist/{types-ByZslIlr.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
  128. package/dist/wire/index.js +1 -1
  129. package/docs/search-ledger.md +71 -6
  130. package/package.json +2 -2
  131. package/dist/claim-BGOUqoMq.d.ts +0 -38
  132. package/dist/claim-BGOUqoMq.d.ts.map +0 -1
  133. package/dist/index-DwQxdWve.d.ts.map +0 -1
  134. package/dist/llm-judge-Z6MkTkd0.js.map +0 -1
  135. package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
  136. package/dist/redact-B7rt9Zot.js.map +0 -1
  137. package/dist/search-ledger-Ck7tok0h.js.map +0 -1
  138. package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
  139. package/dist/task-failure-attributes-CEqEfuIY.js.map +0 -1
  140. package/dist/tool-groups-DiBi16ns.d.ts.map +0 -1
  141. package/dist/types-C1CAD70_.d.ts.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"index-DNVzu9Vv.d.ts","names":[],"sources":["../src/contract/measured-comparison.ts","../src/contract/profile-measured-comparison.ts","../src/contract/intake/run-record-dir.ts","../src/contract/eval-reporting-suite.ts","../src/contract/intake/agent-trace.ts","../src/contract/intake/code-agent-observation.ts","../src/contract/intake/code-agent-session.ts","../src/contract/intake/code-agent-store.ts","../src/contract/intake/feedback-table.ts","../src/contract/intake/otel-spans.ts"],"mappings":";;;;;;;;;;;;UAiCiB;EACf,QAAQ,gCAAgC;EACxC;EACA;;UAGe;EACf,YAAY;EACZ;EACA,QAAQ;EACR,MAAM;EACN,eAAe;EACf;EACA,SAAS;;UAGM;EACf,YAAY;EACZ,QAAQ,OAAO,oCAAoC,QAAQ;;EAE3D;;EAEA,aAAa;EACb,SAAS;;UAGM;EACf,cAAc;EACd;IACE;IACA,MAAM;;;UAIO;EACf,YAAY;EACZ,cAAc;EACd;IACE;IACA,MAAM;;EAER,aAAa;EACb;EACA,YAAY;EACZ;EACA,WAAW;;;UAII,kBAAkB;EACjC;EACA,UAAU;EACV,WAAW;;;UAII,yBAAyB;EACxC,MAAM,KAAK;EACX,WAAW,KAAK;IAAkB;IAAc;;EAChD,QAAQ,KAAK;EACb,eAAe,KAAK,OAAO;EAC3B,UAAU,KAAK;EACf,UAAU,KAAK;EACf,OAAO,KAAK;;UAGG,kCAAkC;EACjD,uBAAuB,kBAAkB;EACzC,QAAQ;EACR,SAAS,yBAAyB;;EAElC;;EAEA,kBAAkB;;EAElB,kBAAkB;;;KAIR,8BAA8B,KACxC;EAGA,iBAAiB;EACjB,WAAW;EACX;;;iBAIc,2BACd,UAAU,sCACT;;iBAQa,4BACd,SAAS,qCACR;;iBAiBa,wBACd,UAAU,mCACT;iBAQa,0BAA0B,iBAAiB;;iBAarC,uBACpB,SAAS,gCACR,QAAQ;;;;;;;;iBAkEK,2BAA2B,MACzC,SAAS,kCAAkC,QAC1C;;iBA8Ya,0CACd,SAAS,oCACR;;iBAuEa,oCACd,iBACC;;;UCjrBc;EACf,aAAa;EACb,QAAQ,gCAAgC;EACxC;EACA;;UAGe;EACf,YAAY;EACZ;EACA,aAAa;EACb,MAAM;EACN,SAAS;EACT;EACA,SAAS;;UAGM;EACf,YAAY;EACZ,QACE,OAAO,kDACN,QAAQ;;EAEX;;EAEA,aAAa;EACb,SAAS;;UAGM;EACf,cAAc;EACd;IACE;IACA,MAAM;;;UAIO;EACf,YAAY;EACZ,cAAc;EACd;IACE;IACA,MAAM;;EAER,aAAa;EACb;EACA,YAAY;EACZ;EACA,WAAW;;;iBAIG,gCACd,UAAU,sCACT;;iBAQa,iCACd,SAAS,0CACR;;iBAqBa,sCACd,UAAU,4CACT;;;;;;iBAsBmB,qCACpB,SAAS,8CACR,QAAQ;;iBA0CK,wDACd,SAAS,kDACR;;iBAuEa,kDACd,iBACC;;;;UC7Oc;;EAEf;;EAEA;;EAEA;;UAGe;;;;;;EAMf;;;;;;;EAOA,WAAW;;;;;;EAMX;;UAGe;;EAEf,MAAM;;EAEN,UAAU;;EAEV;;;;;;;;;;iBAkBoB,iBACpB,cACA,UAAS,0BACR,QAAQ;;;;;KC7CC,0BAA0B;UAErB;;;;;EAKf,UAAU,KAAK;;EAEf,OAAO;;;;;;;;;;EAUP;;;;UAKe;;;EAGf,QAAQ;;EAER;;IAEE;;IAEA;;;IAGA;;IAEA;;;IAGA,UAAU;;;EAGZ;;;;;;;iBAUoB,mBACpB,OAAO,yBACP,UAAS,4BACR,QAAQ;;;KChEC;UAEK;EACf,MAAM;;EAEN;;UAGe;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,cAAc;EACd,QAAQ;;UAGO;EACf;EACA,eAAe;;UAGA;EACf;EACA;EACA;EACA;IAAQ;IAAc;;EACtB;IAAS;IAAe;;EACxB,OAAO;;;;UAOQ;EACf;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;KAGU,kBAAkB,YAAY;;;;;;iBAW1B,gBAAgB,SAAS,qBAAqB;UAmE7C;;;;EAIf,SAAS,YAAY;;;EAGrB,cAAc;;;;;;;;iBASA,8BACd,MAAM,aACN,OAAO,kBACN;;;KCjLS;KAEA;KAEA;KAEA;KASA;UAEK;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA,MAAM;EACN,SAAS;EACT;EACA,QAAQ;EACR;EACA;EACA,UAAU;;UAGK;EACf,QAAQ;EACR;EACA;EACA;IACE,QAAQ;IACR;;EAEF,SAAS;;UAGM;EACf,QAAQ;EACR;EACA;EACA,YAAY;;;;;;;iBAeE,wBACd,SAAS,iCACR;;;UCjCc;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;;UAGe;EACf,MAAM;EACN,aAAa;EACb,SAAS;EACT,cAAc;;UAGC;EACf;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;;;EAGA,iBAAiB;;;EAGjB,YAAY;;;;KAKF;EACN;EAAe;EAAoB;;EACnC;EAAmB;;iBAcT,oBAAoB,gBAAgB;;;;;;;;;;iBAuB7B,yBAAyB,eAAe,eAAe;;;;;iBA6BxD,wBAAwB,eAAe,QAAQ;iBAUrD,iBACd,SAAS,gCACR;iBAIa,sBACd,SAAS,gCACR;iBAIa,oBACd,SAAS,gCACR;iBAIa,oBACd,SAAS,gCACR;iBAIa,cACd,SAAS,gCACR;;;;UC/Lc;;;;;WAKN;;WAEA,QAAQ;;;;;WAKR;;;;;WAKA;aACE;;aAEA;;;;KAKD;;UAUK;WACN;WACA;WACA;WACA;WACA,aAAa;;;UAIP;WACN;WACA,UAAU,SAAS,QAAQ,OAAO;;WAElC,QAAQ;aACN;aACA,QAAQ;;;;;;;;;KAUT;WAEG;WACA,OAAO;WACP;WACA,mBAAmB;;WAEnB;;WAEA;;WAEA;WACA,WAAW,SAAS,QAAQ,OAAO;WACnC,YAAY;;WAGZ;WACA,OAAO;WACP;;WAEA;WACA,YAAY;;;;;;;;;iBAaL,sBACpB,OAAO,sBACN,QAAQ;;;UCtHM;;;EAGf;;EAEA;;;EAGA;;;EAGA,WAAW;;UAGI;EACf;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,WAAW;;;EAGX,SAAS;;UAGM;;EAEf,SAAS;;;EAGT,OAAO;;;;EAIP;IAAU;IAAa;;;;;EAIvB;;UAGe;EACf,MAAM;;;EAGN,aAAa;IAAQ;IAAe;IAAe;;;iBAGrC,kBAAkB,MAAM,2BAA2B;;;UC3BlD;EACf,OAAO;;EAEP,eAAe;;EAEf;;;;;;EAMA,eAAe,eAAe,gBAAgB;;iBAGhC,cAAc,MAAM,uBAAuB"}
1
+ {"version":3,"file":"index-DWQNDqPd.d.ts","names":[],"sources":["../src/contract/measured-comparison.ts","../src/contract/profile-measured-comparison.ts","../src/contract/intake/run-record-dir.ts","../src/contract/eval-reporting-suite.ts","../src/contract/intake/agent-trace.ts","../src/contract/intake/code-agent-observation.ts","../src/contract/intake/code-agent-session.ts","../src/contract/intake/code-agent-store.ts","../src/contract/intake/feedback-table.ts","../src/contract/intake/otel-spans.ts"],"mappings":";;;;;;;;;;;;UAiCiB;EACf,QAAQ,gCAAgC;EACxC;EACA;;UAGe;EACf,YAAY;EACZ;EACA,QAAQ;EACR,MAAM;EACN,eAAe;EACf;EACA,SAAS;;UAGM;EACf,YAAY;EACZ,QAAQ,OAAO,oCAAoC,QAAQ;;EAE3D;;EAEA,aAAa;EACb,SAAS;;UAGM;EACf,cAAc;EACd;IACE;IACA,MAAM;;;UAIO;EACf,YAAY;EACZ,cAAc;EACd;IACE;IACA,MAAM;;EAER,aAAa;EACb;EACA,YAAY;EACZ;EACA,WAAW;;;UAII,kBAAkB;EACjC;EACA,UAAU;EACV,WAAW;;;UAII,yBAAyB;EACxC,MAAM,KAAK;EACX,WAAW,KAAK;IAAkB;IAAc;;EAChD,QAAQ,KAAK;EACb,eAAe,KAAK,OAAO;EAC3B,UAAU,KAAK;EACf,UAAU,KAAK;EACf,OAAO,KAAK;;UAGG,kCAAkC;EACjD,uBAAuB,kBAAkB;EACzC,QAAQ;EACR,SAAS,yBAAyB;;EAElC;;EAEA,kBAAkB;;EAElB,kBAAkB;;;KAIR,8BAA8B,KACxC;EAGA,iBAAiB;EACjB,WAAW;EACX;;;iBAIc,2BACd,UAAU,sCACT;;iBAQa,4BACd,SAAS,qCACR;;iBAiBa,wBACd,UAAU,mCACT;iBAQa,0BAA0B,iBAAiB;;iBAarC,uBACpB,SAAS,gCACR,QAAQ;;;;;;;;iBAkEK,2BAA2B,MACzC,SAAS,kCAAkC,QAC1C;;iBA8Ya,0CACd,SAAS,oCACR;;iBAuEa,oCACd,iBACC;;;UCjrBc;EACf,aAAa;EACb,QAAQ,gCAAgC;EACxC;EACA;;UAGe;EACf,YAAY;EACZ;EACA,aAAa;EACb,MAAM;EACN,SAAS;EACT;EACA,SAAS;;UAGM;EACf,YAAY;EACZ,QACE,OAAO,kDACN,QAAQ;;EAEX;;EAEA,aAAa;EACb,SAAS;;UAGM;EACf,cAAc;EACd;IACE;IACA,MAAM;;;UAIO;EACf,YAAY;EACZ,cAAc;EACd;IACE;IACA,MAAM;;EAER,aAAa;EACb;EACA,YAAY;EACZ;EACA,WAAW;;;iBAIG,gCACd,UAAU,sCACT;;iBAQa,iCACd,SAAS,0CACR;;iBAqBa,sCACd,UAAU,4CACT;;;;;;iBAsBmB,qCACpB,SAAS,8CACR,QAAQ;;iBA0CK,wDACd,SAAS,kDACR;;iBAuEa,kDACd,iBACC;;;;UC7Oc;;EAEf;;EAEA;;EAEA;;UAGe;;;;;;EAMf;;;;;;;EAOA,WAAW;;;;;;EAMX;;UAGe;;EAEf,MAAM;;EAEN,UAAU;;EAEV;;;;;;;;;;iBAkBoB,iBACpB,cACA,UAAS,0BACR,QAAQ;;;;;KC7CC,0BAA0B;UAErB;;;;;EAKf,UAAU,KAAK;;EAEf,OAAO;;;;;;;;;;EAUP;;;;UAKe;;;EAGf,QAAQ;;EAER;;IAEE;;IAEA;;;IAGA;;IAEA;;;IAGA,UAAU;;;EAGZ;;;;;;;iBAUoB,mBACpB,OAAO,yBACP,UAAS,4BACR,QAAQ;;;KChEC;UAEK;EACf,MAAM;;EAEN;;UAGe;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,cAAc;EACd,QAAQ;;UAGO;EACf;EACA,eAAe;;UAGA;EACf;EACA;EACA;EACA;IAAQ;IAAc;;EACtB;IAAS;IAAe;;EACxB,OAAO;;;;UAOQ;EACf;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;KAGU,kBAAkB,YAAY;;;;;;iBAW1B,gBAAgB,SAAS,qBAAqB;UAmE7C;;;;EAIf,SAAS,YAAY;;;EAGrB,cAAc;;;;;;;;iBASA,8BACd,MAAM,aACN,OAAO,kBACN;;;KCjLS;KAEA;KAEA;KAEA;KASA;UAEK;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA,MAAM;EACN,SAAS;EACT;EACA,QAAQ;EACR;EACA;EACA,UAAU;;UAGK;EACf,QAAQ;EACR;EACA;EACA;IACE,QAAQ;IACR;;EAEF,SAAS;;UAGM;EACf,QAAQ;EACR;EACA;EACA,YAAY;;;;;;;iBAeE,wBACd,SAAS,iCACR;;;UCjCc;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;;UAGe;EACf,MAAM;EACN,aAAa;EACb,SAAS;EACT,cAAc;;UAGC;EACf;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;;;EAGA,iBAAiB;;;EAGjB,YAAY;;;;KAKF;EACN;EAAe;EAAoB;;EACnC;EAAmB;;iBAcT,oBAAoB,gBAAgB;;;;;;;;;;iBAuB7B,yBAAyB,eAAe,eAAe;;;;;iBA6BxD,wBAAwB,eAAe,QAAQ;iBAUrD,iBACd,SAAS,gCACR;iBAIa,sBACd,SAAS,gCACR;iBAIa,oBACd,SAAS,gCACR;iBAIa,oBACd,SAAS,gCACR;iBAIa,cACd,SAAS,gCACR;;;;UC/Lc;;;;;WAKN;;WAEA,QAAQ;;;;;WAKR;;;;;WAKA;aACE;;aAEA;;;;KAKD;;UAUK;WACN;WACA;WACA;WACA;WACA,aAAa;;;UAIP;WACN;WACA,UAAU,SAAS,QAAQ,OAAO;;WAElC,QAAQ;aACN;aACA,QAAQ;;;;;;;;;KAUT;WAEG;WACA,OAAO;WACP;WACA,mBAAmB;;WAEnB;;WAEA;;WAEA;WACA,WAAW,SAAS,QAAQ,OAAO;WACnC,YAAY;;WAGZ;WACA,OAAO;WACP;;WAEA;WACA,YAAY;;;;;;;;;iBAaL,sBACpB,OAAO,sBACN,QAAQ;;;UCtHM;;;EAGf;;EAEA;;;EAGA;;;EAGA,WAAW;;UAGI;EACf;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,WAAW;;;EAGX,SAAS;;UAGM;;EAEf,SAAS;;;EAGT,OAAO;;;;EAIP;IAAU;IAAa;;;;;EAIvB;;UAGe;EACf,MAAM;;;EAGN,aAAa;IAAQ;IAAe;IAAe;;;iBAGrC,kBAAkB,MAAM,2BAA2B;;;UC3BlD;EACf,OAAO;;EAEP,eAAe;;EAEf;;;;;;EAMA,eAAe,eAAe,gBAAgB;;iBAGhC,cAAc,MAAM,uBAAuB"}
package/dist/index.d.ts CHANGED
@@ -9,15 +9,15 @@ import { a as ContinuousCalibrationResult, d as calibrateJudgeContinuous, f as c
9
9
  import { a as CorpusAgreementPerDimension, c as corpusInterRaterAgreement, i as CorpusAgreementOptions, l as corpusInterRaterAgreementFromJudgeScores, n as SeriesConvergenceResult, o as CorpusAgreementReport, r as analyzeSeries, s as CorpusScoreRecord, t as SeriesConvergenceOptions, u as interRaterReliability } from "./series-convergence-DY7ha0kx.js";
10
10
  import { a as partialCredit, c as spearmanR, d as weightedMean, i as confidenceInterval, l as summarizeNumberSeries, n as WeightedCompositeInput, o as pearsonR, r as WeightedCompositeResult, s as ranks, t as SeriesDistribution, u as weightedComposite } from "./descriptive-B2iPaT9J.js";
11
11
  import { n as bonferroni, r as holm, t as benjaminiHochberg } from "./multiplicity-DIWHvysC.js";
12
- import { A as PairArmsOptions, Bt as RiskDifferenceResult, C as hashJson, Ct as verifyAttestation, D as ComparePairedArmsOptions, F as PairedCorrectness, Ft as EProcessStep, Gt as pairedRiskDifference, Ht as isBinaryOutcomeVector, I as PairedMetricDelta, It as eProcess, Jt as passAtK, Kt as pairedRiskDifferenceExact, L as comparePairedArms, Lt as ExactRiskDifferenceResult, M as PairRunRecordsResult, Mt as EProcess, N as PairedArmRow, Nt as EProcessOptions, O as MatchedPair, P as PairedArmsComparison, Pt as EProcessState, R as pairArms, Rt as McNemarResult, St as attest, Ut as mcnemar, Vt as ScoreRiskDifferenceResult, Wt as pairedBinaryScale, Yt as wilson, bt as AttestationVerification, et as Objective, j as PairArmsResult, k as MatchedRunRecordPair, nt as dominates, qt as pairedRiskDifferenceScore, rt as paretoFrontier, tt as ParetoResult, vt as ATTESTATION_ALGORITHM, w as manifestContentDigest, xt as AttestedReport, yt as AttestationProvenance, z as pairRunRecords, zt as ProportionInterval } from "./statistical-heldout-C1Ej5Ce8.js";
12
+ import { A as PairArmsOptions, Bt as RiskDifferenceResult, C as hashJson, Ct as verifyAttestation, D as ComparePairedArmsOptions, F as PairedCorrectness, Ft as EProcessStep, Gt as pairedRiskDifference, Ht as isBinaryOutcomeVector, I as PairedMetricDelta, It as eProcess, Jt as passAtK, Kt as pairedRiskDifferenceExact, L as comparePairedArms, Lt as ExactRiskDifferenceResult, M as PairRunRecordsResult, Mt as EProcess, N as PairedArmRow, Nt as EProcessOptions, O as MatchedPair, P as PairedArmsComparison, Pt as EProcessState, R as pairArms, Rt as McNemarResult, St as attest, Ut as mcnemar, Vt as ScoreRiskDifferenceResult, Wt as pairedBinaryScale, Yt as wilson, bt as AttestationVerification, et as Objective, j as PairArmsResult, k as MatchedRunRecordPair, nt as dominates, qt as pairedRiskDifferenceScore, rt as paretoFrontier, tt as ParetoResult, vt as ATTESTATION_ALGORITHM, w as manifestContentDigest, xt as AttestedReport, yt as AttestationProvenance, z as pairRunRecords, zt as ProportionInterval } from "./statistical-heldout-D-xBpbrd.js";
13
13
  import { _ as pairedSignTest, a as PairedPromotionDecision, c as BOOTSTRAP_GATE_MIN_N, d as PairedBootstrapResult, f as PairedSignTestResult, g as pairedDeltaTieFraction, h as pairedBootstrap, i as PairedMcNemarEvidence, l as DECISION_PAIRED_DELTA_STATISTIC, m as SignTestAlternative, n as PairedDecisionShape, o as PairedPromotionDecisionOptions, p as PairedTTestResult, r as PairedDecisionStatistic, s as decidePairedPromotion, t as PairedDecisionMethod, u as PairedBootstrapOptions, v as pairedTTest } from "./paired-promotion-decision-DPKz8ZnF.js";
14
14
  import { a as requiredPairedSampleSize, i as pairedMde, n as mcnemarPower, o as requiredSampleSize, r as mcnemarRequiredN, t as mulberry32 } from "./random-DGEyr3zG.js";
15
15
  import { C as RankTestMethod, D as WilcoxonSignedRankResult, E as WILCOXON_EXACT_MAX_N, O as mannWhitneyU, S as MannWhitneyResult, T as RankTestOptions, _ as bootstrapCi, a as ReleaseConfidenceIssue, b as MANN_WHITNEY_EXACT_MAX_STATES, f as evaluateReleaseConfidence, g as Verdict, i as ReleaseConfidenceInput, k as wilcoxonSignedRank, m as BootstrapResult, o as ReleaseConfidenceMetrics, p as BootstrapOptions, s as ReleaseConfidenceScorecard, t as ActionableSideInfo, u as ReleaseTraceEvidence, w as RankTestMethodRequest, x as MANN_WHITNEY_EXACT_MAX_WORK, y as DEFAULT_PERMUTATIONS } from "./release-confidence-CkoXFx9I.js";
16
- import { R as Scenario$1, S as JudgeConfig, _ as GateDecision, a as CampaignResult, w as JudgeScore$1 } from "./types-C1CAD70_.js";
17
- import { $r as redTeamDataset, At as RuntimeEventLike, Bt as SatisfiedBy, Dt as summarizeBackendIntegrity, Et as assertRealBackend, Fo as RunCampaignOptions, Ft as CorrectnessChecker, Ht as completionVerdict, Io as runCampaign, It as LlmCorrectnessCheckerOpts, Jr as DEFAULT_RED_TEAM_CORPUS, Lt as ProducedProposal, Mt as extractProducedState, No as CampaignCellFailureReceipt, Nt as CompletionRequirement, Ot as ArtifactEventLike, Po as CampaignCellRetryPolicy, Pt as CompletionVerdict, Qr as RedTeamReport, Rt as ProducedState, Tt as BackendIntegrityReport, Ut as createLlmCorrectnessChecker, Vt as TaskGold, Wt as verifyCompletion, Xr as RedTeamCategory, Yr as RedTeamCase, Zr as RedTeamFinding, ai as CanaryOptions, da as LlmJudgeOptions, ei as redTeamReport, fa as llmJudge, ga as transientDispatchFailure, ha as quotaExhaustedUntil, ii as CanaryKind, jt as ToolCallEventLike, kt as ProposalEventLike, ni as CanaryAlert, oi as CanaryReport, ri as CanaryEvaluation, si as runCanaries, ti as scoreRedTeamOutput, wt as BackendIntegrityError, zt as RequirementCheck } from "./index-DwQxdWve.js";
18
16
  import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, s as TraceStore, t as EventFilter } from "./store-C4avFjIi.js";
19
17
  import { a as MintedRolloutLine } from "./schema-B5LtYvn9.js";
20
18
  import { a as mintRolloutRows, c as ScorePreference, i as RolloutSearchLineage, n as MintRolloutResult, t as MintRolloutOptions } from "./mint-C9Lojt1V.js";
19
+ import { R as Scenario$1, S as JudgeConfig, _ as GateDecision, a as CampaignResult, w as JudgeScore$1 } from "./types-BWwRXofp.js";
20
+ import { Ao as runCampaign, Bt as extractProducedState, Ca as quotaExhaustedUntil, Do as CampaignCellFailureReceipt, Ft as summarizeBackendIntegrity, Gt as ProducedProposal, Ht as CompletionVerdict, It as ArtifactEventLike, Jt as SatisfiedBy, Kt as ProducedState, Lt as ProposalEventLike, Mt as BackendIntegrityError, Nt as BackendIntegrityReport, Oo as CampaignCellRetryPolicy, Pt as assertRealBackend, Qt as verifyCompletion, Rt as RuntimeEventLike, Ut as CorrectnessChecker, Vt as CompletionRequirement, Wt as LlmCorrectnessCheckerOpts, Xt as completionVerdict, Yt as TaskGold, Zt as createLlmCorrectnessChecker, ai as RedTeamFinding, ba as llmJudge, ci as redTeamReport, di as CanaryEvaluation, fi as CanaryKind, hi as runCanaries, ii as RedTeamCategory, ko as RunCampaignOptions, li as scoreRedTeamOutput, mi as CanaryReport, ni as DEFAULT_RED_TEAM_CORPUS, oi as RedTeamReport, pi as CanaryOptions, qt as RequirementCheck, ri as RedTeamCase, si as redTeamDataset, ui as CanaryAlert, wa as transientDispatchFailure, ya as LlmJudgeOptions, zt as ToolCallEventLike } from "./index-DMPLG226.js";
21
21
  import { a as ProfileAxisSpec, c as expandProfileAxes, i as HarnessType, l as harnessAxisOf, n as CODING_HARNESSES, o as agentProfileHash, r as HARNESS_NATIVE_MODEL, s as agentProfileId, t as AgentProfile } from "./agent-profile-DDfIjdyS.js";
22
22
  import { i as DatasetSplit, n as DatasetManifest, r as DatasetScenario, t as Dataset } from "./dataset-DQqhOCPt.js";
23
23
  import { C as defineEquivalenceCheck, S as buildEquivalenceRecord, _ as StrategyChecker, a as CheckerIdentity, b as VerificationStrategyProfile, c as EquivalenceCheckDefinition, d as EquivalenceCheckerInput, f as EquivalenceCheckerResult, g as EquivalenceRecord, h as EquivalenceProtocolError, i as equivalenceVerdict, l as EquivalenceCheckSpec, m as EquivalenceObligationStatus, n as VerdictCertification, o as CheckerOutcome, p as EquivalenceObligation, r as certificationEvidenceDigest, s as EquivalenceArm, t as DefaultVerdict, u as EquivalenceChecker, v as VERIFICATION_STRATEGIES, w as runEquivalenceCheck, x as VerificationStrategySource, y as VERIFICATION_STRATEGY_SOURCES } from "./verdict-E4eRNf7-.js";
@@ -27,20 +27,20 @@ import { a as FAILURE_BLAME, c as FailureContext, d as INFRA_FAILURE_BLAMES, f a
27
27
  import { d as InsightReport, i as analyzeRuns, t as AnalyzeRunsOptions } from "./analyze-runs-CPiKLJFs.js";
28
28
  import { i as ExactAnalystRunEvent, l as ExactCapableAnalyst, o as ExactAnalystRunResult } from "./exact-types-KGxBWH4b.js";
29
29
  import { c as RegistryRunOpts, i as BudgetPolicy, n as AnalystRegistry, r as AnalystRegistryOptions, s as ExactRegistryRunOpts } from "./registry-Bu3lNole.js";
30
- import { _ as SelfImproveMethodResult, a as DefinedAgentEval, c as SelfImproveMethodOptions, d as SelfImproveProposerOptions, f as SelfImproveProposerResult, g as SelfImproveMethodProvenance, h as selfImprove, i as DefineAgentEvalOptions, l as SelfImproveOptions, n as AgentEvalEvaluateOptions, o as defineAgentEval, p as SelfImproveResult, r as AgentEvalImproveOptions, s as SelfImproveBudget, t as AgentEvalAgent, u as SelfImproveProgressEvent } from "./define-agent-eval-DcludNJS.js";
31
- import { a as createTraceAnalyst, i as TraceAnalystDefinition, n as buildDefaultAnalystRegistry, r as CreateTraceAnalystOptions, t as DefaultAnalystRegistryOptions } from "./default-registry-DdqA_-_o.js";
32
- import { i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS } from "./engine-C1DIAEpn.js";
33
- import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as ConceptFinding, E as createChatTraceEngine, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine } from "./index-DBoXFaz0.js";
34
- import { i as BenchmarkEvaluation } from "./types-ByZslIlr.js";
30
+ import { _ as SelfImproveMethodResult, a as DefinedAgentEval, c as SelfImproveMethodOptions, d as SelfImproveProposerOptions, f as SelfImproveProposerResult, g as SelfImproveMethodProvenance, h as selfImprove, i as DefineAgentEvalOptions, l as SelfImproveOptions, n as AgentEvalEvaluateOptions, o as defineAgentEval, p as SelfImproveResult, r as AgentEvalImproveOptions, s as SelfImproveBudget, t as AgentEvalAgent, u as SelfImproveProgressEvent } from "./define-agent-eval-DQJvVa1P.js";
31
+ import { a as createTraceAnalyst, i as TraceAnalystDefinition, n as buildDefaultAnalystRegistry, r as CreateTraceAnalystOptions, t as DefaultAnalystRegistryOptions } from "./default-registry-C3HTE08O.js";
32
+ import { i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine, t as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS } from "./engine-DjRV55F9.js";
33
+ import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as ConceptFinding, E as createChatTraceEngine, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, T as ChatTraceEngineOptions, _ as FindingSubject, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubjectKind, w as createDspyRlmTraceEngine } from "./index-0jEtpSM7.js";
34
+ import { i as BenchmarkEvaluation } from "./types-qoxUfKQg.js";
35
35
  import { a as SpanHandle, n as CaptureWrite, o as TraceEmitter, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-HcCmFQl_.js";
36
36
  import { n as SandboxDriver } from "./sandbox-harness-BlSOu4LX.js";
37
- import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-CopuMxll.js";
37
+ import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-Cveoq6fJ.js";
38
38
  import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-BhsrMupG.js";
39
39
  import { n as TrajectoryStep, r as buildTrajectory, t as Trajectory } from "./trajectory-Y1fmdlw7.js";
40
40
  import { a as runCounterfactual, i as CounterfactualRunner, n as CounterfactualMutation, r as CounterfactualResult, t as CounterfactualContext } from "./counterfactual-CPaWN99-.js";
41
41
  import { A as AnalystFindingDigest, B as ControlRunResult, C as createFeedbackTrajectory, D as renderPreferenceMemoryMarkdown, E as feedbackTrajectoryToOptimizerRow, F as ControlActionOutcome, G as StopDecision, H as ControlRuntimeError, I as ControlBudget, J as subjectiveEval, K as objectiveEval, L as ControlContext, M as AnalystRunDigest, N as analystFindingDigest, O as summarizePreferenceMemory, P as analystRunDigest, R as ControlDecision, S as controlRunToFeedbackTrajectory, T as feedbackTrajectoriesToOptimizerRows, U as ControlSeverity, V as ControlRuntimeConfig, W as ControlStep, _ as PreferenceMemoryEntry, a as FeedbackLabel, b as analystRunToReviewRequests, c as FeedbackOptimizerRow, d as FeedbackTask, f as FeedbackTrajectory, g as InMemoryFeedbackTrajectoryStore, h as FileSystemFeedbackTrajectoryStore, i as FeedbackAttempt, j as AnalystReviewDecision, k as withAssignedFeedbackSplit, l as FeedbackOutcome, m as FeedbackTrajectoryStore, n as AnalystReviewRequest, o as FeedbackLabelKind, p as FeedbackTrajectoryFilter, q as runAgentControlLoop, r as FeedbackArtifactType, s as FeedbackLabelSource, t as AnalystFeedbackTrajectoryOptions, u as FeedbackSplitPolicy, v as ProposedSideEffect, w as feedbackTrajectoriesToDatasetScenarios, x as assignFeedbackSplit, y as analystRunToFeedbackTrajectory, z as ControlEvalResult } from "./feedback-trajectory-BXCGTRkz.js";
42
42
  import { y as OUTPUT_VALUE } from "./attribute-vocabulary-CcXqHODT.js";
43
- import { C as inferDomainKeywords, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, W as ExtractedUsage, g as TraceInsightSuite, h as TraceInsightReadiness, k as tokenizeDomainWords, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore, v as buildTraceInsightContext, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-ByLdU26j.js";
43
+ import { C as inferDomainKeywords, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, W as ExtractedUsage, g as TraceInsightSuite, h as TraceInsightReadiness, k as tokenizeDomainWords, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, r as OtlpFileTraceStore, v as buildTraceInsightContext, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-BxrfuU2t.js";
44
44
  import { a as RunIntegrityReport, o as assertRunCaptured, t as RunIntegrityError } from "./integrity-CaeNeusS.js";
45
45
  import { c as judgeSpans, f as runsForScenario, i as argHash } from "./query-DBUQhehH.js";
46
46
  import { a as SteeringChange, c as CampaignRunContext, d as CampaignVariant, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, l as CampaignRunOutcome, m as FailedRun, n as ExperimentResult, o as CampaignFactoryParams, p as EvalCampaignResult, r as FailureMode, s as CampaignIntegrityPolicy, t as ExperimentPlan, u as CampaignScenario } from "./researcher-DojEMb7E.js";
package/dist/index.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
2
- import { i as JudgeError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
2
+ import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
3
3
  import { a as hashCanonical, r as canonicalString } from "./canonical-BsZVgBhW.js";
4
- import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-CVmCPQDJ.js";
4
+ import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-B2AhDRnq.js";
5
5
  import { c as groupRunsByAgentProfileCell, d as validateAgentProfileCell, f as verifyAgentProfileCell, h as manifestContentDigest, i as agentProfileCellKey, m as hashJson, r as agentProfileCellHashMaterial, s as buildAgentProfileCell, t as AGENT_PROFILE_KINDS, u as toAgentProfileJson } from "./agent-profile-cell-DcFW4vOc.js";
6
6
  import { t as mulberry32 } from "./random-Dn5fPWkt.js";
7
7
  import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
@@ -13,20 +13,21 @@ import { _ as passAtK, a as MANN_WHITNEY_EXACT_MAX_STATES, c as mannWhitneyU, d
13
13
  import { a as pairedSignTest, i as pairedDeltaTieFraction, n as DECISION_PAIRED_DELTA_STATISTIC, o as pairedTTest, r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "./paired-tests-C8iCsioC.js";
14
14
  import { t as eProcess } from "./sequential-eprocess-D1jKoihe.js";
15
15
  import { n as iqr, r as welchsTTest } from "./baseline-BC-eBZ7U.js";
16
- import { i as pairedDeltaTest, n as pairedDecisionShape, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-6cn3bLc3.js";
16
+ import { n as pairedDeltaTest, t as minimumPairsForPairedDeltaTest } from "./paired-delta-test-BF-FEP8V.js";
17
+ import { n as pairedDecisionShape, t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
17
18
  import { a as isToolSpan, i as isLlmSpan, r as isJudgeSpan, t as FAILURE_CLASSES } from "./schema-CSf6qWgZ.js";
18
19
  import { d as runsForScenario, r as argHash, s as judgeSpans } from "./query-DlV0pdx4.js";
19
- import { r as checkCanaries, t as analyzeRuns } from "./analyze-runs-SCSWYpy0.js";
20
+ import { r as checkCanaries, t as analyzeRuns } from "./analyze-runs-B6gv7vAT.js";
20
21
  import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
21
22
  import { a as parseRunRecordSafe, c as runTaskScore, i as modelHasSnapshot, l as searchCellRunId, n as UNKNOWN_MODEL, o as roundTripRunRecord, r as isRunRecord, s as runCostFloorUsd, t as RunRecordValidationError, u as validateRunRecord } from "./run-record-BRlM99rX.js";
22
23
  import { a as summaryTable, n as gainHistogram, r as paretoChart } from "./summary-report-BMvou2oZ.js";
23
24
  import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
24
- import { $ as scoreRedTeamOutput, G as paretoFrontier, Q as redTeamReport, W as dominates, X as DEFAULT_RED_TEAM_CORPUS, Z as redTeamDataset, a as transientDispatchFailure, et as runCanaries, i as quotaExhaustedUntil, o as aggregateRunScore, ot as runCampaign, s as clamp01, t as llmJudge } from "./llm-judge-Z6MkTkd0.js";
25
+ import { $ as redTeamDataset, K as dominates, Q as DEFAULT_RED_TEAM_CORPUS, U as pairedPromotionPower, W as requiredPairsForPairedPromotion, a as transientDispatchFailure, ct as runCampaign, et as redTeamReport, i as quotaExhaustedUntil, nt as runCanaries, o as aggregateRunScore, q as paretoFrontier, s as clamp01, t as llmJudge, tt as scoreRedTeamOutput } from "./llm-judge-Cififl_V.js";
25
26
  import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-DOMpkoeb.js";
26
27
  import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-_IbkejCF.js";
27
- import { B as assertRealBackend, F as surfaceContentHash, T as parseReflectionResponse, V as summarizeBackendIntegrity, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation, w as buildReflectionPrompt, z as BackendIntegrityError } from "./campaign-evidence-DcQiYhJI.js";
28
- import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-B7rt9Zot.js";
29
- import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-DQuos3Aw.js";
28
+ import { B as assertRealBackend, F as surfaceContentHash, T as parseReflectionResponse, V as summarizeBackendIntegrity, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation, w as buildReflectionPrompt, z as BackendIntegrityError } from "./campaign-evidence-DJc60F9C.js";
29
+ import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-Dk1ZdJEm.js";
30
+ import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-qxAPjMSL.js";
30
31
  import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-DhPoXApk.js";
31
32
  import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
32
33
  import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
@@ -35,26 +36,26 @@ import { G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, U as parseFindingSubject, h a
35
36
  import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-xMLX160E.js";
36
37
  import { OUTPUT_VALUE } from "./trace-attributes.js";
37
38
  import { r as extractUsageFromSse, t as extractUsage } from "./extract-usage-DFATEuWW.js";
38
- import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-DYh4Bszw.js";
39
- import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-BcC_5ef-.js";
39
+ import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-CXuOhLvT.js";
40
+ import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-BaGakFc2.js";
40
41
  import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-BFP4kjGj.js";
41
42
  import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-DNe_Uv1Q.js";
42
43
  import { t as packageVersion$1 } from "./package-version-D7lQHt_-.js";
43
44
  import { a as classifyFailure, i as INFRA_FAILURE_BLAMES, o as classifyFailureReason, r as FAILURE_BLAME, s as failureBlame, t as DEFAULT_FAILURE_REASON_RULES } from "./failure-taxonomy-CjabTATj.js";
44
45
  import { n as runBoundedProcess, t as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS } from "./bounded-process-BBZob7vl.js";
45
- import { t as runEvalCampaign } from "./eval-campaign-CwsBn0Gm.js";
46
+ import { t as runEvalCampaign } from "./eval-campaign-CXT2WYUe.js";
46
47
  import "./rollout-B-XBReTC.js";
47
48
  import { t as mintRolloutRows } from "./mint-iCD-W43s.js";
48
49
  import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
49
- import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-KYQBh9C8.js";
50
- import { n as paidJsonChat } from "./chat-json-call-CXHBvUz8.js";
50
+ import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-c_bKHTXz.js";
51
+ import { n as paidJsonChat } from "./chat-json-call-DnlDjpco.js";
51
52
  import { i as makeProposalFinding, n as computeFindingId, r as makeFinding } from "./types-CW5b90JH.js";
52
- import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-BX34vPVY.js";
53
- import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-Bg-560H0.js";
53
+ import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-k_QdEBZo.js";
54
+ import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-Bh5lvKGg.js";
54
55
  import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
55
56
  import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-BPowgPm6.js";
56
57
  import { n as evaluateReleaseConfidence, r as bootstrapCi } from "./release-confidence-XQm_xjhR.js";
57
- import { t as createChatClient } from "./chat-client-B_ItsSBw.js";
58
+ import { t as createChatClient } from "./chat-client-CJORSG_J.js";
58
59
  import { createHash } from "node:crypto";
59
60
  import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
60
61
  import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
@@ -1357,7 +1358,7 @@ function assertPredicate(value, where) {
1357
1358
  ]) if (p[field] !== void 0) assertMatcher(p[field], `${where} "${field}"`);
1358
1359
  if (p.kind !== void 0 && !SPAN_KINDS.includes(p.kind)) throw new ValidationError(`${where}: "kind" must be one of ${SPAN_KINDS.join(", ")}`);
1359
1360
  if (p.attr !== void 0 && (p.attr === null || typeof p.attr !== "object" || Array.isArray(p.attr))) throw new ValidationError(`${where}: "attr" must be a plain object`);
1360
- for (const [key, expected] of Object.entries(p.attr ?? {})) if (isSerializedRegex(expected)) assertMatcher(expected, `${where} attr "${key}"`);
1361
+ for (const [key, expected] of Object.entries(p.attr ?? {})) if (expected !== null && typeof expected === "object") assertMatcher(expected, `${where} attr "${key}"`);
1361
1362
  if (p.not !== void 0) assertPredicate(p.not, `${where} "not"`);
1362
1363
  if (p.requiresCustom && typeof p.custom !== "function") throw new ValidationError(`${where}: rule was built with a custom predicate function, which does not survive JSON serialization — re-attach \`custom\` after deserializing or drop the rule`);
1363
1364
  const hasAttr = p.attr !== void 0 && Object.keys(p.attr).length > 0;
@@ -1370,7 +1371,7 @@ function predicateMatches(span, p) {
1370
1371
  if (p.model !== void 0 && !matchText(spanModel(span), p.model)) return false;
1371
1372
  if (p.attr !== void 0) for (const [key, expected] of Object.entries(p.attr)) {
1372
1373
  const actual = span.attributes?.[key];
1373
- if (isRegexValue(expected)) {
1374
+ if (isRegexValue(expected) || isOneOfMatcher(expected)) {
1374
1375
  if (!matchText(typeof actual === "string" ? actual : void 0, expected)) return false;
1375
1376
  } else if (actual !== expected) return false;
1376
1377
  }
@@ -1812,6 +1813,33 @@ function traceContract(name) {
1812
1813
  function spanRef(span, index) {
1813
1814
  return span.spanId ?? `#${index}`;
1814
1815
  }
1816
+ /**
1817
+ * Whether this trace shows any sign that spans of `kind` are actually
1818
+ * captured, independent of whether any span happens to match a given rule's
1819
+ * predicate. A `never`, `atMost`, `tokensAtMost` or `retrySafe` rule that
1820
+ * targets TOOL or LLM spans and finds zero matches cannot otherwise tell "no
1821
+ * calls happened" from "calls were not recorded": an exporter that silently
1822
+ * drops a span kind makes `forbidden`, `allowed`, `maxCalls` and `retries`
1823
+ * checks on that kind pass vacuously. Evidence is: a span of that kind
1824
+ * anywhere in the trace, or — for TOOL — a recorded offered-tool set
1825
+ * (`gen_ai.tool.definitions`), which proves the capture pipeline ran even on
1826
+ * a turn that called no tool.
1827
+ */
1828
+ function hasCaptureEvidence(spans, kind) {
1829
+ if (spans.some((span) => contractSpanKind(span) === kind)) return true;
1830
+ if (kind === "TOOL") return spans.some((span) => spanOfferedTools(span).recorded);
1831
+ return false;
1832
+ }
1833
+ /** Only `TOOL`/`LLM`-scoped predicates have capture evidence to check. */
1834
+ function capturedKindOf(p) {
1835
+ return p.kind === "TOOL" || p.kind === "LLM" ? p.kind : void 0;
1836
+ }
1837
+ function requireCaptureEvidence(spans, p, rule, situation) {
1838
+ const kind = capturedKindOf(p);
1839
+ if (kind === void 0) return;
1840
+ if (hasCaptureEvidence(spans, kind)) return;
1841
+ throw new CaptureIntegrityError(`rule '${rule}': ${situation} and the trace has no ${kind} capture evidence (no ${kind} span` + (kind === "TOOL" ? `, no ${TOOL_DEFINITIONS_ATTR}` : "") + `) — cannot tell whether ${describePredicate(p)} never happened or was never recorded`);
1842
+ }
1815
1843
  function indexed(spans, p) {
1816
1844
  const out = [];
1817
1845
  spans.forEach((span, i) => {
@@ -2065,7 +2093,9 @@ function checkRetrySafe(rule, spans) {
2065
2093
  const out = [];
2066
2094
  const groups = /* @__PURE__ */ new Map();
2067
2095
  const unseen = /* @__PURE__ */ new Map();
2068
- for (const call of toolSpans(spans)) {
2096
+ const calls = toolSpans(spans);
2097
+ if (calls.length === 0 && !hasCaptureEvidence(spans, "TOOL")) throw new CaptureIntegrityError(`rule '${rule.label}': retrySafe found 0 TOOL spans and the trace has no TOOL capture evidence (no TOOL span, no ${TOOL_DEFINITIONS_ATTR}) — cannot tell whether no reads or writes happened or the trace lost its TOOL spans`);
2098
+ for (const call of calls) {
2069
2099
  const name = contractSpanToolName(call.span);
2070
2100
  if (name === void 0) {
2071
2101
  out.push({
@@ -2181,11 +2211,15 @@ function checkRule(rule, ctx) {
2181
2211
  spanId: span.spanId,
2182
2212
  detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") fails always(${describePredicate(rule.p)})`
2183
2213
  }]);
2184
- case "never": return indexed(spans, rule.p).map(({ span, ref }) => ({
2185
- rule: rule.label,
2186
- spanId: span.spanId,
2187
- detail: `span ${ref} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
2188
- }));
2214
+ case "never": {
2215
+ const matches = indexed(spans, rule.p);
2216
+ if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `never(${describePredicate(rule.p)}) matched 0 spans`);
2217
+ return matches.map(({ span, ref }) => ({
2218
+ rule: rule.label,
2219
+ spanId: span.spanId,
2220
+ detail: `span ${ref} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
2221
+ }));
2222
+ }
2189
2223
  case "eventually": return spans.some((span) => predicateMatches(span, rule.p)) ? [] : [{
2190
2224
  rule: rule.label,
2191
2225
  detail: `no span matches eventually(${describePredicate(rule.p)}) over ${spans.length} span(s)`
@@ -2208,6 +2242,7 @@ function checkRule(rule, ctx) {
2208
2242
  case "neverUnless": return orderViolations(rule.label, indexed(spans, rule.prior), indexed(spans, rule.p), rule.order ?? "start-order", `${describePredicate(rule.p)} without an earlier ${describePredicate(rule.prior)}`);
2209
2243
  case "atMost": {
2210
2244
  const matches = indexed(spans, rule.p);
2245
+ if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `atMost(${describePredicate(rule.p)}) matched 0 spans`);
2211
2246
  return matches.length <= rule.max ? [] : [{
2212
2247
  rule: rule.label,
2213
2248
  spanId: matches[rule.max]?.span.spanId,
@@ -2216,6 +2251,7 @@ function checkRule(rule, ctx) {
2216
2251
  }
2217
2252
  case "tokensAtMost": {
2218
2253
  const matches = indexed(spans, rule.p);
2254
+ if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `tokensAtMost(${describePredicate(rule.p)}) matched 0 spans`);
2219
2255
  const unknown = matches.find((m) => spanTotalTokens(m.span) === void 0);
2220
2256
  if (unknown) return [{
2221
2257
  rule: rule.label,
@@ -9077,290 +9113,6 @@ function isObject(v) {
9077
9113
  return typeof v === "object" && v !== null && !Array.isArray(v);
9078
9114
  }
9079
9115
  //#endregion
9080
- //#region src/paired-promotion-power.ts
9081
- /**
9082
- * @module
9083
- * Joint power of a configured {@link decidePairedPromotion} call, computed by
9084
- * running that call on data drawn from a registered alternative.
9085
- *
9086
- * Why a simulation and not a formula: `decidePairedPromotion` is not one test.
9087
- * It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
9088
- * paired bootstrap at or above it) and by shape (Tango's score interval on a
9089
- * two-point outcome), and on top of the deciding interval it applies a
9090
- * sufficiency floor, a zero-width refusal and McNemar's exact veto. A
9091
- * preregistration usually seals two such calls, a continuous primary and a
9092
- * pass/fail veto drawn from the same episodes, and promotes only when both
9093
- * promote. The sample size at which THAT procedure reaches a target power is
9094
- * not the power of any one channel: the sign-test power at n can be 0.8 while
9095
- * the promotion probability is well below it because the veto binds
9096
- * (agent-eval#785). Closed forms exist for single channels only, and each one
9097
- * shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
9098
- * Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
9099
- * pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
9100
- * So the only faithful power is the procedure's own promotion rate under the
9101
- * alternative, and that is what this module computes: draw n pairs from the
9102
- * registered law, run every configured call on them exactly as the caller
9103
- * will, repeat, count.
9104
- *
9105
- * The alternative is DECLARATIVE and serializable, never a sampler function,
9106
- * so a preregistration can seal it as bytes: a mixture of cells, each with a
9107
- * probability, an optional pass/fail outcome per arm for the binary calls,
9108
- * and a law for the candidate-minus-baseline delta on the continuous scale.
9109
- * Drawing the cell first and the delta given the cell is what makes the two
9110
- * channels one joint law rather than two marginals.
9111
- *
9112
- * Everything is seeded (`mulberry32`), so a sizing is reproducible from its
9113
- * inputs, and every estimate carries its Monte Carlo standard error and a
9114
- * Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
9115
- */
9116
- /**
9117
- * Joint power of the configured calls at `n` under the registered alternative:
9118
- * the fraction of simulated data sets on which every call promotes.
9119
- */
9120
- function pairedPromotionPower(options) {
9121
- const n = options.n;
9122
- if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
9123
- const simulations = options.simulations ?? 1e3;
9124
- if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
9125
- const seed = options.seed ?? 1;
9126
- if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
9127
- const calls = validateCalls(options.calls);
9128
- const alternative = validateAlternative(options.alternative, calls);
9129
- const rng = mulberry32(seed);
9130
- let promotions = 0;
9131
- let refusals = 0;
9132
- const perCall = calls.map(() => ({
9133
- promote: 0,
9134
- insufficient: 0,
9135
- indeterminate: 0,
9136
- exactTestVetoes: 0,
9137
- hold: 0,
9138
- methods: /* @__PURE__ */ new Map()
9139
- }));
9140
- for (let s = 0; s < simulations; s++) {
9141
- const pairs = drawPairs(alternative, n, rng);
9142
- let all = true;
9143
- let anyRefusal = false;
9144
- for (let c = 0; c < calls.length; c++) {
9145
- const call = calls[c];
9146
- const decision = runCall(call, pairs, n);
9147
- const tally = perCall[c];
9148
- tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
9149
- if (decision.promote) tally.promote++;
9150
- else {
9151
- if (!decision.sufficient) tally.insufficient++;
9152
- if (decision.indeterminate) tally.indeterminate++;
9153
- if (decision.exactTestVetoes) tally.exactTestVetoes++;
9154
- if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
9155
- }
9156
- if (!decision.promote) all = false;
9157
- if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
9158
- }
9159
- if (all) promotions++;
9160
- else if (anyRefusal) refusals++;
9161
- }
9162
- const power = promotions / simulations;
9163
- const interval = wilson(promotions, simulations, .95);
9164
- return {
9165
- n,
9166
- simulations,
9167
- seed,
9168
- power,
9169
- standardError: Math.sqrt(power * (1 - power) / simulations),
9170
- low: interval.lower,
9171
- high: interval.upper,
9172
- refuse: refusals / simulations,
9173
- hold: (simulations - promotions - refusals) / simulations,
9174
- calls: perCall.map((tally, c) => ({
9175
- outcome: calls[c].outcome,
9176
- promote: tally.promote / simulations,
9177
- insufficient: tally.insufficient / simulations,
9178
- indeterminate: tally.indeterminate / simulations,
9179
- exactTestVetoes: tally.exactTestVetoes / simulations,
9180
- hold: tally.hold / simulations,
9181
- methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
9182
- }))
9183
- };
9184
- }
9185
- /**
9186
- * First n at which {@link pairedPromotionPower} reaches the target, scanning
9187
- * every n from `minPairs` upward. Both the point-estimate answer and the
9188
- * confirmed lower-bound answer are returned; the scan runs until the latter is
9189
- * reached (so the curve past `n` is retained) or `maxPairs` is exhausted. A
9190
- * candidate selected by the scan's bound is certified only by an independent
9191
- * simulation at that n on a different seed; a candidate that fails it is
9192
- * recorded and the scan continues.
9193
- */
9194
- function requiredPairsForPairedPromotion(options) {
9195
- const target = options.target ?? .8;
9196
- if (!Number.isFinite(target) || target <= 0 || target >= 1) throw new Error(`requiredPairsForPairedPromotion: target must be in (0,1), got ${target}`);
9197
- const minPairs = options.minPairs ?? 1;
9198
- const maxPairs = options.maxPairs ?? 200;
9199
- if (!Number.isInteger(minPairs) || minPairs < 1) throw new Error(`requiredPairsForPairedPromotion: minPairs must be a positive integer, got ${minPairs}`);
9200
- if (!Number.isInteger(maxPairs) || maxPairs < minPairs) throw new Error(`requiredPairsForPairedPromotion: maxPairs must be an integer >= minPairs, got ${maxPairs}`);
9201
- const seed = options.seed ?? 1;
9202
- const curve = [];
9203
- const rejected = [];
9204
- let n = null;
9205
- let nAtLowerBound = null;
9206
- let confirmation = null;
9207
- for (let candidate = minPairs; candidate <= maxPairs; candidate++) {
9208
- const result = pairedPromotionPower({
9209
- n: candidate,
9210
- alternative: options.alternative,
9211
- calls: options.calls,
9212
- simulations: options.simulations,
9213
- seed
9214
- });
9215
- curve.push(result);
9216
- if (n === null && result.power >= target) n = candidate;
9217
- if (result.low >= target) {
9218
- const check = pairedPromotionPower({
9219
- n: candidate,
9220
- alternative: options.alternative,
9221
- calls: options.calls,
9222
- simulations: options.simulations,
9223
- seed: confirmationSeed(seed, candidate)
9224
- });
9225
- if (check.low >= target) {
9226
- nAtLowerBound = candidate;
9227
- confirmation = check;
9228
- break;
9229
- }
9230
- rejected.push(check);
9231
- }
9232
- }
9233
- return {
9234
- target,
9235
- n,
9236
- nAtLowerBound,
9237
- confirmation,
9238
- curve,
9239
- rejected
9240
- };
9241
- }
9242
- /** A seed for the confirmation at `n` that never equals the scan seed. */
9243
- function confirmationSeed(seed, n) {
9244
- return (Math.imul((seed | 0) ^ 2654435769, 2246822507) + Math.imul(n, 3266489909) | 1) >>> 0;
9245
- }
9246
- function drawPairs(alternative, n, rng) {
9247
- const baseline = alternative.baseline ?? {
9248
- kind: "point",
9249
- value: 0
9250
- };
9251
- const control = new Array(n);
9252
- const candidate = new Array(n);
9253
- const controlPass = new Array(n);
9254
- const candidatePass = new Array(n);
9255
- for (let i = 0; i < n; i++) {
9256
- const cell = drawCell(alternative.cells, rng);
9257
- const base = drawValue(baseline, rng);
9258
- control[i] = base;
9259
- candidate[i] = base + drawValue(cell.delta, rng);
9260
- controlPass[i] = cell.pass?.control ?? false;
9261
- candidatePass[i] = cell.pass?.treatment ?? false;
9262
- }
9263
- return {
9264
- control,
9265
- candidate,
9266
- controlPass,
9267
- candidatePass
9268
- };
9269
- }
9270
- function drawCell(cells, rng) {
9271
- const u = rng();
9272
- let cumulative = 0;
9273
- for (const cell of cells) {
9274
- cumulative += cell.probability;
9275
- if (u < cumulative) return cell;
9276
- }
9277
- return cells[cells.length - 1];
9278
- }
9279
- function drawValue(law, rng) {
9280
- switch (law.kind) {
9281
- case "point": return law.value;
9282
- case "atoms": {
9283
- const u = rng();
9284
- let cumulative = 0;
9285
- for (const atom of law.atoms) {
9286
- cumulative += atom.probability;
9287
- if (u < cumulative) return atom.value;
9288
- }
9289
- return law.atoms[law.atoms.length - 1].value;
9290
- }
9291
- case "normal": {
9292
- const u1 = 1 - rng();
9293
- const u2 = rng();
9294
- return law.mean + law.sd * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
9295
- }
9296
- }
9297
- }
9298
- function runCall(call, pairs, n) {
9299
- const options = {
9300
- minPairs: n,
9301
- ...call.options
9302
- };
9303
- if (call.outcome === "pass") {
9304
- const scale = call.options.binaryScale;
9305
- return decidePairedPromotion(pairs.controlPass.map((p) => p ? scale : 0), pairs.candidatePass.map((p) => p ? scale : 0), options);
9306
- }
9307
- return decidePairedPromotion(pairs.control, pairs.candidate, options);
9308
- }
9309
- function validateCalls(calls) {
9310
- if (!Array.isArray(calls) || calls.length === 0) throw new Error("pairedPromotionPower: at least one call is required");
9311
- calls.forEach((call, i) => {
9312
- if (call.outcome !== "delta" && call.outcome !== "pass") throw new Error(`pairedPromotionPower: calls[${i}].outcome must be 'delta' or 'pass'`);
9313
- if (call.options === null || typeof call.options !== "object") throw new Error(`pairedPromotionPower: calls[${i}].options must be an object`);
9314
- if (call.options.minPairs !== void 0 && (!Number.isInteger(call.options.minPairs) || call.options.minPairs < 1)) throw new Error(`pairedPromotionPower: calls[${i}].options.minPairs must be a positive integer`);
9315
- if (call.outcome === "pass") {
9316
- const scale = call.options.binaryScale;
9317
- if (scale === void 0 || !Number.isFinite(scale) || scale <= 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call and must declare a finite positive options.binaryScale`);
9318
- if (call.options.continuous) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; continuous cannot accompany it`);
9319
- if (call.options.statistic === "median") throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; binaryScale requires the mean statistic, not median`);
9320
- } else if (call.options.binaryScale !== void 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'delta' call and cannot declare binaryScale; use a 'pass' call for a two-point outcome`);
9321
- });
9322
- return calls;
9323
- }
9324
- function validateAlternative(alternative, calls) {
9325
- if (!alternative || !Array.isArray(alternative.cells) || alternative.cells.length === 0) throw new Error("pairedPromotionPower: alternative.cells must be a non-empty array");
9326
- const needsPass = calls.some((call) => call.outcome === "pass");
9327
- let total = 0;
9328
- alternative.cells.forEach((cell, i) => {
9329
- if (!Number.isFinite(cell.probability) || cell.probability < 0) throw new Error(`pairedPromotionPower: cells[${i}].probability must be a finite number >= 0`);
9330
- total += cell.probability;
9331
- if (needsPass) {
9332
- if (!cell.pass || typeof cell.pass.control !== "boolean" || typeof cell.pass.treatment !== "boolean") throw new Error(`pairedPromotionPower: cells[${i}].pass must give control and treatment booleans when a 'pass' call is configured`);
9333
- }
9334
- validateLaw(cell.delta, `cells[${i}].delta`);
9335
- });
9336
- if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: cell probabilities must sum to 1, got ${total}`);
9337
- if (alternative.baseline !== void 0) validateLaw(alternative.baseline, "baseline");
9338
- return alternative;
9339
- }
9340
- function validateLaw(law, where) {
9341
- if (!law || typeof law !== "object") throw new Error(`pairedPromotionPower: ${where} must be a law object`);
9342
- switch (law.kind) {
9343
- case "point":
9344
- if (!Number.isFinite(law.value)) throw new Error(`pairedPromotionPower: ${where}.value must be finite`);
9345
- return;
9346
- case "atoms": {
9347
- if (!Array.isArray(law.atoms) || law.atoms.length === 0) throw new Error(`pairedPromotionPower: ${where}.atoms must be a non-empty array`);
9348
- let total = 0;
9349
- law.atoms.forEach((atom, i) => {
9350
- if (!Number.isFinite(atom.value)) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].value must be finite`);
9351
- if (!Number.isFinite(atom.probability) || atom.probability < 0) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].probability must be a finite number >= 0`);
9352
- total += atom.probability;
9353
- });
9354
- if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: ${where}.atoms probabilities must sum to 1, got ${total}`);
9355
- return;
9356
- }
9357
- case "normal":
9358
- if (!Number.isFinite(law.mean) || !Number.isFinite(law.sd) || law.sd < 0) throw new Error(`pairedPromotionPower: ${where} needs a finite mean and a finite sd >= 0`);
9359
- return;
9360
- default: throw new Error(`pairedPromotionPower: ${where}.kind must be 'point', 'atoms' or 'normal'`);
9361
- }
9362
- }
9363
- //#endregion
9364
9116
  //#region src/treatment-gate.ts
9365
9117
  /**
9366
9118
  * Core predicate. Gated (treatment-not-applied) ONLY when telemetry was