@tangle-network/agent-eval 0.128.2 → 0.130.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (424) hide show
  1. package/CHANGELOG.md +279 -0
  2. package/README.md +19 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +83 -2932
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -364
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1205
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1710
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -894
  34. package/dist/benchmarks/index.js +2 -59
  35. package/dist/benchmarks-DviOvUNr.js +754 -0
  36. package/dist/benchmarks-DviOvUNr.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6390
  44. package/dist/campaign/index.js +3 -212
  45. package/dist/campaign-CBKZvQ1H.js +3885 -0
  46. package/dist/campaign-CBKZvQ1H.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -174
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5605
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1937
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -32
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -617
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CAPUUKaM.d.ts +335 -0
  116. package/dist/index-CAPUUKaM.d.ts.map +1 -0
  117. package/dist/index-DE5fb3EC.d.ts +2244 -0
  118. package/dist/index-DE5fb3EC.d.ts.map +1 -0
  119. package/dist/index-DSC51roc.d.ts +102 -0
  120. package/dist/index-DSC51roc.d.ts.map +1 -0
  121. package/dist/index.d.ts +3776 -15120
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11185 -11191
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -481
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1298
  196. package/dist/reporting.js +6 -50
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +916 -3596
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2362 -1751
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -1048
  211. package/dist/rollout/index.js +8 -110
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/run-record-BuoE80Dq.js.map +1 -0
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
  253. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
  254. package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
  255. package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -849
  273. package/dist/supervisor-run/index.js +2 -64
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -251
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1174
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/feature-guide.md +1 -1
  301. package/docs/rollout.md +116 -2
  302. package/package.json +18 -10
  303. package/dist/benchmarks/index.js.map +0 -1
  304. package/dist/campaign/index.js.map +0 -1
  305. package/dist/chunk-2JX3CFMB.js +0 -695
  306. package/dist/chunk-2JX3CFMB.js.map +0 -1
  307. package/dist/chunk-2MKQIFS4.js +0 -183
  308. package/dist/chunk-2MKQIFS4.js.map +0 -1
  309. package/dist/chunk-3RF76KTD.js +0 -84
  310. package/dist/chunk-3RF76KTD.js.map +0 -1
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7ZZMD7UK.js +0 -386
  314. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  315. package/dist/chunk-BOD4O7OF.js +0 -40
  316. package/dist/chunk-BOD4O7OF.js.map +0 -1
  317. package/dist/chunk-BYT7ELPS.js +0 -1553
  318. package/dist/chunk-BYT7ELPS.js.map +0 -1
  319. package/dist/chunk-DJKY2TSY.js +0 -2428
  320. package/dist/chunk-DJKY2TSY.js.map +0 -1
  321. package/dist/chunk-DPUHNQLN.js +0 -232
  322. package/dist/chunk-DPUHNQLN.js.map +0 -1
  323. package/dist/chunk-DRYIUNWY.js +0 -622
  324. package/dist/chunk-DRYIUNWY.js.map +0 -1
  325. package/dist/chunk-EJGRPCO3.js +0 -617
  326. package/dist/chunk-EJGRPCO3.js.map +0 -1
  327. package/dist/chunk-EOSZT7PL.js +0 -2001
  328. package/dist/chunk-EOSZT7PL.js.map +0 -1
  329. package/dist/chunk-EZJEIH2R.js +0 -1559
  330. package/dist/chunk-EZJEIH2R.js.map +0 -1
  331. package/dist/chunk-GGE4NNQT.js +0 -65
  332. package/dist/chunk-GGE4NNQT.js.map +0 -1
  333. package/dist/chunk-HHWE3POT.js +0 -94
  334. package/dist/chunk-HHWE3POT.js.map +0 -1
  335. package/dist/chunk-IHQDPH7D.js +0 -171
  336. package/dist/chunk-IHQDPH7D.js.map +0 -1
  337. package/dist/chunk-JHCHEVET.js +0 -274
  338. package/dist/chunk-JHCHEVET.js.map +0 -1
  339. package/dist/chunk-K4DBDHLK.js +0 -158
  340. package/dist/chunk-K4DBDHLK.js.map +0 -1
  341. package/dist/chunk-K6N6XJJX.js +0 -306
  342. package/dist/chunk-K6N6XJJX.js.map +0 -1
  343. package/dist/chunk-MA6HLL3S.js +0 -65
  344. package/dist/chunk-MA6HLL3S.js.map +0 -1
  345. package/dist/chunk-MAZ26DC7.js +0 -99
  346. package/dist/chunk-MAZ26DC7.js.map +0 -1
  347. package/dist/chunk-MHELPNRP.js +0 -1212
  348. package/dist/chunk-MHELPNRP.js.map +0 -1
  349. package/dist/chunk-NACAGYSY.js +0 -1040
  350. package/dist/chunk-NACAGYSY.js.map +0 -1
  351. package/dist/chunk-NKAGIDE2.js +0 -7633
  352. package/dist/chunk-NKAGIDE2.js.map +0 -1
  353. package/dist/chunk-NPCTHQIO.js +0 -91
  354. package/dist/chunk-NPCTHQIO.js.map +0 -1
  355. package/dist/chunk-NYLOYM6N.js +0 -332
  356. package/dist/chunk-NYLOYM6N.js.map +0 -1
  357. package/dist/chunk-ONWEPEDO.js +0 -57
  358. package/dist/chunk-ONWEPEDO.js.map +0 -1
  359. package/dist/chunk-P5W7RQKK.js +0 -576
  360. package/dist/chunk-P5W7RQKK.js.map +0 -1
  361. package/dist/chunk-P6FYH6K4.js +0 -1161
  362. package/dist/chunk-P6FYH6K4.js.map +0 -1
  363. package/dist/chunk-PBE2LOSS.js +0 -669
  364. package/dist/chunk-PBE2LOSS.js.map +0 -1
  365. package/dist/chunk-PC4UYEBM.js +0 -166
  366. package/dist/chunk-PC4UYEBM.js.map +0 -1
  367. package/dist/chunk-PXE2VKMX.js +0 -140
  368. package/dist/chunk-PXE2VKMX.js.map +0 -1
  369. package/dist/chunk-PZ5AY32C.js +0 -10
  370. package/dist/chunk-PZ5AY32C.js.map +0 -1
  371. package/dist/chunk-RZTMDUO7.js +0 -49
  372. package/dist/chunk-RZTMDUO7.js.map +0 -1
  373. package/dist/chunk-S5YLIBFX.js +0 -136
  374. package/dist/chunk-S5YLIBFX.js.map +0 -1
  375. package/dist/chunk-SZLVEKMJ.js +0 -1446
  376. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  377. package/dist/chunk-T4SQEITX.js +0 -95
  378. package/dist/chunk-T4SQEITX.js.map +0 -1
  379. package/dist/chunk-TBL77AUT.js +0 -355
  380. package/dist/chunk-TBL77AUT.js.map +0 -1
  381. package/dist/chunk-TSN7JT6D.js +0 -1646
  382. package/dist/chunk-TSN7JT6D.js.map +0 -1
  383. package/dist/chunk-TT4KNT67.js +0 -124
  384. package/dist/chunk-TT4KNT67.js.map +0 -1
  385. package/dist/chunk-UB2LOJ6Q.js +0 -4461
  386. package/dist/chunk-UB2LOJ6Q.js.map +0 -1
  387. package/dist/chunk-UWZZKKU7.js +0 -237
  388. package/dist/chunk-UWZZKKU7.js.map +0 -1
  389. package/dist/chunk-VBQ3CRKH.js +0 -291
  390. package/dist/chunk-VBQ3CRKH.js.map +0 -1
  391. package/dist/chunk-VGRCHJON.js +0 -163
  392. package/dist/chunk-VGRCHJON.js.map +0 -1
  393. package/dist/chunk-VI2UW6B6.js +0 -162
  394. package/dist/chunk-VI2UW6B6.js.map +0 -1
  395. package/dist/chunk-VLOATJQ2.js +0 -908
  396. package/dist/chunk-VLOATJQ2.js.map +0 -1
  397. package/dist/chunk-VQMK5FMP.js +0 -247
  398. package/dist/chunk-VQMK5FMP.js.map +0 -1
  399. package/dist/chunk-VZSRQ272.js +0 -149
  400. package/dist/chunk-VZSRQ272.js.map +0 -1
  401. package/dist/chunk-WGXIEX7P.js +0 -116
  402. package/dist/chunk-WGXIEX7P.js.map +0 -1
  403. package/dist/chunk-WS3NZZQQ.js +0 -929
  404. package/dist/chunk-WS3NZZQQ.js.map +0 -1
  405. package/dist/chunk-XDWDC2MP.js +0 -695
  406. package/dist/chunk-XDWDC2MP.js.map +0 -1
  407. package/dist/chunk-XPRT64IE.js +0 -766
  408. package/dist/chunk-XPRT64IE.js.map +0 -1
  409. package/dist/chunk-YJBNWCAA.js +0 -1056
  410. package/dist/chunk-YJBNWCAA.js.map +0 -1
  411. package/dist/chunk-ZET2UAYW.js +0 -89
  412. package/dist/chunk-ZET2UAYW.js.map +0 -1
  413. package/dist/chunk-ZUUWPZCV.js +0 -752
  414. package/dist/chunk-ZUUWPZCV.js.map +0 -1
  415. package/dist/control.js.map +0 -1
  416. package/dist/hosted/index.js.map +0 -1
  417. package/dist/matrix/index.js.map +0 -1
  418. package/dist/reporting.js.map +0 -1
  419. package/dist/rollout/index.js.map +0 -1
  420. package/dist/run-campaign-ISHFZ7FJ.js +0 -17
  421. package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
  422. package/dist/supervisor-run/index.js.map +0 -1
  423. package/dist/traces.js.map +0 -1
  424. package/dist/wire/index.js.map +0 -1
@@ -1,1212 +0,0 @@
1
- import {
2
- ValidationError
3
- } from "./chunk-ONWEPEDO.js";
4
-
5
- // src/statistics.ts
6
- var normalizeScores = (scores) => scores;
7
- function weightedMean(scores) {
8
- if (scores.length === 0) return 0;
9
- let totalWeight = 0;
10
- let weightedSum = 0;
11
- for (const { score, weight } of scores) {
12
- const w = weight ?? 1;
13
- weightedSum += score * w;
14
- totalWeight += w;
15
- }
16
- return totalWeight > 0 ? weightedSum / totalWeight : 0;
17
- }
18
- function confidenceInterval(scores, confidence = 0.95, opts = {}) {
19
- if (scores.length === 0) return { mean: 0, lower: 0, upper: 0 };
20
- if (scores.length === 1) return { mean: scores[0], lower: scores[0], upper: scores[0] };
21
- const n = scores.length;
22
- const mean = scores.reduce((a, b) => a + b, 0) / n;
23
- const B = opts.resamples ?? 1e3;
24
- const rng = makeRng(opts.seed);
25
- const bootstrapMeans = [];
26
- for (let i = 0; i < B; i++) {
27
- let sum = 0;
28
- for (let j = 0; j < n; j++) {
29
- sum += scores[Math.floor(rng() * n)];
30
- }
31
- bootstrapMeans.push(sum / n);
32
- }
33
- bootstrapMeans.sort((a, b) => a - b);
34
- const alpha = 1 - confidence;
35
- const lowerIdx = Math.floor(alpha / 2 * B);
36
- const upperIdx = Math.floor((1 - alpha / 2) * B) - 1;
37
- return {
38
- mean,
39
- lower: bootstrapMeans[lowerIdx],
40
- upper: bootstrapMeans[Math.min(upperIdx, B - 1)]
41
- };
42
- }
43
- function interRaterReliability(judgeScores) {
44
- if (judgeScores.length < 2) return 1;
45
- const dimensionMap = /* @__PURE__ */ new Map();
46
- for (const judgeSet of judgeScores) {
47
- for (const s of judgeSet) {
48
- if (!dimensionMap.has(s.dimension)) dimensionMap.set(s.dimension, []);
49
- const arr = dimensionMap.get(s.dimension);
50
- if (arr.length === 0 || arr[arr.length - 1].length >= judgeScores.length) {
51
- arr.push([s.score]);
52
- } else {
53
- arr[arr.length - 1].push(s.score);
54
- }
55
- }
56
- }
57
- const allValues = [];
58
- const pairDiffs = [];
59
- for (const items of dimensionMap.values()) {
60
- for (const ratings of items) {
61
- if (ratings.length < 2) continue;
62
- for (const v of ratings) allValues.push(v);
63
- for (let i = 0; i < ratings.length; i++) {
64
- for (let j = i + 1; j < ratings.length; j++) {
65
- pairDiffs.push((ratings[i] - ratings[j]) ** 2);
66
- }
67
- }
68
- }
69
- }
70
- if (pairDiffs.length === 0 || allValues.length < 2) return 1;
71
- const observedDisagreement = pairDiffs.reduce((a, b) => a + b, 0) / pairDiffs.length;
72
- let expectedDisagreement = 0;
73
- let expectedCount = 0;
74
- for (let i = 0; i < allValues.length; i++) {
75
- for (let j = i + 1; j < allValues.length; j++) {
76
- expectedDisagreement += (allValues[i] - allValues[j]) ** 2;
77
- expectedCount++;
78
- }
79
- }
80
- expectedDisagreement = expectedCount > 0 ? expectedDisagreement / expectedCount : 0;
81
- if (expectedDisagreement === 0) return 1;
82
- return 1 - observedDisagreement / expectedDisagreement;
83
- }
84
- function mannWhitneyU(a, b) {
85
- if (a.length === 0 || b.length === 0) return { u: 0, p: 1 };
86
- const n1 = a.length;
87
- const n2 = b.length;
88
- const combined = [
89
- ...a.map((v) => ({ v, group: "a" })),
90
- ...b.map((v) => ({ v, group: "b" }))
91
- ].sort((x, y) => x.v - y.v);
92
- const ranks2 = new Array(combined.length);
93
- let i = 0;
94
- while (i < combined.length) {
95
- let j = i;
96
- while (j < combined.length && combined[j].v === combined[i].v) j++;
97
- const avgRank = (i + 1 + j) / 2;
98
- for (let k = i; k < j; k++) ranks2[k] = avgRank;
99
- i = j;
100
- }
101
- let r1 = 0;
102
- for (let k = 0; k < combined.length; k++) {
103
- if (combined[k].group === "a") r1 += ranks2[k];
104
- }
105
- const u1 = r1 - n1 * (n1 + 1) / 2;
106
- const u2 = n1 * n2 - u1;
107
- const u = Math.min(u1, u2);
108
- const mu = n1 * n2 / 2;
109
- const sigma = Math.sqrt(n1 * n2 * (n1 + n2 + 1) / 12);
110
- if (sigma === 0) return { u, p: 1 };
111
- const z = Math.abs(u - mu) / sigma;
112
- const p = 2 * (1 - normalCdf(z));
113
- return { u, p };
114
- }
115
- function partialCredit(current, target) {
116
- if (target <= 0) return 1;
117
- return Math.min(1, Math.max(0, current / target));
118
- }
119
- function pairedTTest(before, after) {
120
- if (before.length !== after.length) {
121
- throw new ValidationError(
122
- `pairedTTest: unequal sample sizes (${before.length} vs ${after.length})`
123
- );
124
- }
125
- const n = before.length;
126
- if (n < 2) return { t: 0, df: 0, p: 1 };
127
- const diffs = before.map((b, i) => after[i] - b);
128
- const mean = diffs.reduce((a, b) => a + b, 0) / n;
129
- const variance = diffs.reduce((acc, d) => acc + (d - mean) ** 2, 0) / (n - 1);
130
- const se = Math.sqrt(variance / n);
131
- if (se === 0) return { t: mean === 0 ? 0 : Infinity, df: n - 1, p: mean === 0 ? 1 : 0 };
132
- const t = mean / se;
133
- const df = n - 1;
134
- const p = 2 * (1 - studentTCdf(Math.abs(t), df));
135
- return { t, df, p };
136
- }
137
- function wilcoxonSignedRank(before, after) {
138
- if (before.length !== after.length) {
139
- throw new ValidationError(
140
- `wilcoxonSignedRank: unequal sample sizes (${before.length} vs ${after.length})`
141
- );
142
- }
143
- const diffs = before.map((b, i2) => after[i2] - b).filter((d) => d !== 0);
144
- const n = diffs.length;
145
- if (n < 6) return { w: 0, p: 1 };
146
- const absRanks = diffs.map((d, i2) => ({ abs: Math.abs(d), sign: Math.sign(d), i: i2 })).sort((a, b) => a.abs - b.abs);
147
- const ranks2 = new Array(n);
148
- let i = 0;
149
- while (i < n) {
150
- let j = i;
151
- while (j < n && absRanks[j].abs === absRanks[i].abs) j++;
152
- const avg = (i + 1 + j) / 2;
153
- for (let k = i; k < j; k++) ranks2[absRanks[k].i] = avg;
154
- i = j;
155
- }
156
- let wPlus = 0;
157
- for (let k = 0; k < n; k++) if (diffs[k] > 0) wPlus += ranks2[k];
158
- const mean = n * (n + 1) / 4;
159
- const variance = n * (n + 1) * (2 * n + 1) / 24;
160
- const z = (wPlus - mean) / Math.sqrt(variance);
161
- const p = 2 * (1 - normalCdf(Math.abs(z)));
162
- return { w: wPlus, p };
163
- }
164
- function cohensD(a, b) {
165
- if (a.length < 2 || b.length < 2) return 0;
166
- const meanA = a.reduce((x, y) => x + y, 0) / a.length;
167
- const meanB = b.reduce((x, y) => x + y, 0) / b.length;
168
- const varA = a.reduce((acc, x) => acc + (x - meanA) ** 2, 0) / (a.length - 1);
169
- const varB = b.reduce((acc, x) => acc + (x - meanB) ** 2, 0) / (b.length - 1);
170
- const pooled = Math.sqrt(
171
- ((a.length - 1) * varA + (b.length - 1) * varB) / (a.length + b.length - 2)
172
- );
173
- if (pooled === 0) return 0;
174
- return (meanB - meanA) / pooled;
175
- }
176
- function pairedCohensDz(before, after) {
177
- if (before.length !== after.length) {
178
- throw new ValidationError(
179
- `pairedCohensDz: unequal sample sizes (${before.length} vs ${after.length})`
180
- );
181
- }
182
- if (before.length < 2) return null;
183
- const deltas = before.map((value, index) => after[index] - value);
184
- if (deltas.some((value) => !Number.isFinite(value))) {
185
- throw new ValidationError("pairedCohensDz: all paired values must be finite");
186
- }
187
- const meanDelta = deltas.reduce((sum, value) => sum + value, 0) / deltas.length;
188
- const variance = deltas.reduce((sum, value) => sum + (value - meanDelta) ** 2, 0) / (deltas.length - 1);
189
- const standardDeviation = Math.sqrt(variance);
190
- const scale = Math.max(1, Math.abs(meanDelta), ...deltas.map(Math.abs));
191
- if (standardDeviation <= Number.EPSILON * scale) return meanDelta === 0 ? 0 : null;
192
- return meanDelta / standardDeviation;
193
- }
194
- function cliffsDelta(before, after) {
195
- const n = before.length * after.length;
196
- if (n === 0) return 0;
197
- let dominance = 0;
198
- for (const a of after) {
199
- for (const b of before) {
200
- if (a > b) dominance += 1;
201
- else if (a < b) dominance -= 1;
202
- }
203
- }
204
- return dominance / n;
205
- }
206
- function interpretCliffs(delta) {
207
- const d = Math.abs(delta);
208
- if (d < 0.147) return "negligible";
209
- if (d < 0.33) return "small";
210
- if (d < 0.474) return "medium";
211
- return "large";
212
- }
213
- function ranks(xs) {
214
- const indexed = xs.map((v, i2) => ({ v, i: i2 })).sort((a, b) => a.v - b.v);
215
- const r = new Array(xs.length);
216
- let i = 0;
217
- while (i < indexed.length) {
218
- let j = i;
219
- while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
220
- const avg = (i + j) / 2 + 1;
221
- for (let k = i; k <= j; k++) r[indexed[k].i] = avg;
222
- i = j + 1;
223
- }
224
- return r;
225
- }
226
- function pearsonR(a, b) {
227
- if (a.length !== b.length || a.length < 2) return Number.NaN;
228
- const n = a.length;
229
- const meanA = a.reduce((s, v) => s + v, 0) / n;
230
- const meanB = b.reduce((s, v) => s + v, 0) / n;
231
- let num = 0;
232
- let varA = 0;
233
- let varB = 0;
234
- for (let i = 0; i < n; i++) {
235
- const da = a[i] - meanA;
236
- const db = b[i] - meanB;
237
- num += da * db;
238
- varA += da * da;
239
- varB += db * db;
240
- }
241
- if (varA === 0 || varB === 0) return varA === 0 && varB === 0 ? 1 : 0;
242
- return num / Math.sqrt(varA * varB);
243
- }
244
- function spearmanR(a, b) {
245
- if (a.length !== b.length || a.length < 2) return Number.NaN;
246
- return pearsonR(ranks(a), ranks(b));
247
- }
248
- function weightedComposite(input) {
249
- const entries = Object.entries(input.weights);
250
- if (entries.length === 0) {
251
- throw new Error("weightedComposite: `weights` is empty \u2014 nothing to combine");
252
- }
253
- let weightedSum = 0;
254
- let weightTotal = 0;
255
- for (const [dim, weight] of entries) {
256
- if (weight < 0) {
257
- throw new Error(`weightedComposite: weight for '${dim}' is negative (${weight})`);
258
- }
259
- if (!(dim in input.dims)) {
260
- throw new Error(
261
- `weightedComposite: weighted dimension '${dim}' is absent from \`dims\` \u2014 refusing to renormalise onto a different denominator`
262
- );
263
- }
264
- weightedSum += input.dims[dim] * weight;
265
- weightTotal += weight;
266
- }
267
- if (weightTotal === 0) {
268
- throw new Error("weightedComposite: weights sum to 0 \u2014 composite is undefined");
269
- }
270
- const composite = weightedSum / weightTotal;
271
- return input.threshold === void 0 ? { composite } : { composite, pass: composite >= input.threshold };
272
- }
273
- function corpusInterRaterAgreement(records, opts = {}) {
274
- if (records.length === 0) {
275
- throw new ValidationError("corpusInterRaterAgreement: no score records supplied");
276
- }
277
- const judgesSeen = /* @__PURE__ */ new Set();
278
- const dimsSeen = /* @__PURE__ */ new Set();
279
- const grid = /* @__PURE__ */ new Map();
280
- for (const r of records) {
281
- if (!Number.isFinite(r.score)) {
282
- throw new ValidationError(
283
- `corpusInterRaterAgreement: non-finite score for (item=${r.itemId}, judge=${r.judgeName}, dim=${r.dimension})`
284
- );
285
- }
286
- judgesSeen.add(r.judgeName);
287
- dimsSeen.add(r.dimension);
288
- const byJudge = grid.get(r.dimension) ?? /* @__PURE__ */ new Map();
289
- const byItem = byJudge.get(r.judgeName) ?? /* @__PURE__ */ new Map();
290
- if (byItem.has(r.itemId)) {
291
- throw new ValidationError(
292
- `corpusInterRaterAgreement: duplicate record for (item=${r.itemId}, judge=${r.judgeName}, dim=${r.dimension})`
293
- );
294
- }
295
- byItem.set(r.itemId, r.score);
296
- byJudge.set(r.judgeName, byItem);
297
- grid.set(r.dimension, byJudge);
298
- }
299
- const targetDims = opts.dimensions ?? [...dimsSeen].sort();
300
- for (const d of targetDims) {
301
- if (!dimsSeen.has(d)) {
302
- throw new ValidationError(
303
- `corpusInterRaterAgreement: dimension '${d}' was requested but no records carry it`
304
- );
305
- }
306
- }
307
- const targetJudges = opts.judges ? [...opts.judges] : [...judgesSeen].sort();
308
- for (const j of targetJudges) {
309
- if (!judgesSeen.has(j)) {
310
- throw new ValidationError(
311
- `corpusInterRaterAgreement: judge '${j}' was requested but produced no records`
312
- );
313
- }
314
- }
315
- if (targetJudges.length < 2) {
316
- throw new ValidationError(
317
- `corpusInterRaterAgreement: need \u22652 judges, got ${targetJudges.length}`
318
- );
319
- }
320
- const perDimension = [];
321
- const iccs = [];
322
- const kappas = [];
323
- for (const dim of targetDims) {
324
- const byJudge = grid.get(dim);
325
- const judgeItemCounts = {};
326
- for (const j of targetJudges) {
327
- const m = byJudge.get(j);
328
- judgeItemCounts[j] = m?.size ?? 0;
329
- }
330
- const emptyJudges = targetJudges.filter((j) => judgeItemCounts[j] === 0);
331
- if (emptyJudges.length > 0) {
332
- throw new ValidationError(
333
- `corpusInterRaterAgreement: dimension '${dim}' has no scores from judge(s) ${emptyJudges.join(", ")} (counts: ${JSON.stringify(judgeItemCounts)})`
334
- );
335
- }
336
- let commonItems = null;
337
- for (const j of targetJudges) {
338
- const ids = new Set(byJudge.get(j).keys());
339
- if (commonItems === null) {
340
- commonItems = ids;
341
- } else {
342
- const prev = commonItems;
343
- commonItems = new Set([...prev].filter((x) => ids.has(x)));
344
- }
345
- }
346
- const sortedItems = [...commonItems ?? /* @__PURE__ */ new Set()].sort();
347
- if (sortedItems.length < 2) {
348
- throw new ValidationError(
349
- `corpusInterRaterAgreement: dimension '${dim}' has ${sortedItems.length} item(s) rated by all ${targetJudges.length} judges (need \u22652)`
350
- );
351
- }
352
- const matrix = sortedItems.map(
353
- (itemId) => targetJudges.map((j) => byJudge.get(j).get(itemId))
354
- );
355
- const agreement = continuousAgreement(matrix, opts);
356
- perDimension.push({
357
- ...agreement,
358
- dimension: dim,
359
- itemIds: sortedItems,
360
- judgeIds: [...targetJudges]
361
- });
362
- if (Number.isFinite(agreement.icc)) iccs.push(agreement.icc);
363
- if (Number.isFinite(agreement.weightedKappa)) kappas.push(agreement.weightedKappa);
364
- }
365
- const mean = (xs) => xs.length === 0 ? Number.NaN : xs.reduce((a, b) => a + b, 0) / xs.length;
366
- return {
367
- perDimension,
368
- overallIcc: mean(iccs),
369
- overallWeightedKappa: mean(kappas),
370
- dimensions: targetDims,
371
- judgeIds: targetJudges
372
- };
373
- }
374
- function corpusInterRaterAgreementFromJudgeScores(itemsScores, opts = {}) {
375
- const records = [];
376
- const seen = /* @__PURE__ */ new Set();
377
- for (const { itemId, scores } of itemsScores) {
378
- if (seen.has(itemId)) {
379
- throw new ValidationError(
380
- `corpusInterRaterAgreementFromJudgeScores: duplicate itemId '${itemId}'`
381
- );
382
- }
383
- seen.add(itemId);
384
- for (const s of scores) {
385
- records.push({
386
- itemId,
387
- judgeName: s.judgeName,
388
- dimension: s.dimension,
389
- score: s.score
390
- });
391
- }
392
- }
393
- return corpusInterRaterAgreement(records, opts);
394
- }
395
- function studentTCdf(t, df) {
396
- if (df <= 0) return 0.5;
397
- if (df > 100) return normalCdf(t);
398
- const x = df / (df + t * t);
399
- const a = df / 2;
400
- const b = 0.5;
401
- const ib = incompleteBeta(x, a, b);
402
- return t >= 0 ? 1 - 0.5 * ib : 0.5 * ib;
403
- }
404
- function incompleteBeta(x, a, b) {
405
- if (x <= 0) return 0;
406
- if (x >= 1) return 1;
407
- const lnBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b);
408
- const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - lnBeta) / a;
409
- const maxIter = 200;
410
- const eps = 3e-7;
411
- let c = 1;
412
- let d = 1 - (a + b) * x / (a + 1);
413
- if (Math.abs(d) < 1e-30) d = 1e-30;
414
- d = 1 / d;
415
- let f = d;
416
- for (let m = 1; m <= maxIter; m++) {
417
- const m2 = 2 * m;
418
- let num = m * (b - m) * x / ((a + m2 - 1) * (a + m2));
419
- d = 1 + num * d;
420
- if (Math.abs(d) < 1e-30) d = 1e-30;
421
- c = 1 + num / c;
422
- if (Math.abs(c) < 1e-30) c = 1e-30;
423
- d = 1 / d;
424
- f *= d * c;
425
- num = -((a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1));
426
- d = 1 + num * d;
427
- if (Math.abs(d) < 1e-30) d = 1e-30;
428
- c = 1 + num / c;
429
- if (Math.abs(c) < 1e-30) c = 1e-30;
430
- d = 1 / d;
431
- const delta = d * c;
432
- f *= delta;
433
- if (Math.abs(delta - 1) < eps) break;
434
- }
435
- return front * f;
436
- }
437
- function lnGamma(z) {
438
- const g = 7;
439
- const coefs = [
440
- 0.9999999999998099,
441
- 676.5203681218851,
442
- -1259.1392167224028,
443
- 771.3234287776531,
444
- -176.6150291621406,
445
- 12.507343278686905,
446
- -0.13857109526572012,
447
- 9984369578019572e-21,
448
- 15056327351493116e-23
449
- ];
450
- if (z < 0.5) {
451
- return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z);
452
- }
453
- z -= 1;
454
- let x = coefs[0];
455
- for (let i = 1; i < g + 2; i++) x += coefs[i] / (z + i);
456
- const t = z + g + 0.5;
457
- return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
458
- }
459
- function normalCdf(x) {
460
- const a1 = 0.254829592;
461
- const a2 = -0.284496736;
462
- const a3 = 1.421413741;
463
- const a4 = -1.453152027;
464
- const a5 = 1.061405429;
465
- const p = 0.3275911;
466
- const sign = x < 0 ? -1 : 1;
467
- const absX = Math.abs(x);
468
- const t = 1 / (1 + p * absX);
469
- const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX / 2);
470
- return 0.5 * (1 + sign * y);
471
- }
472
- function requiredSampleSize(opts) {
473
- const effect = opts.effect;
474
- if (!Number.isFinite(effect) || effect <= 0) return Infinity;
475
- const alpha = opts.alpha ?? 0.05;
476
- const power = opts.power ?? 0.8;
477
- const twoSided = opts.twoSided ?? true;
478
- const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
479
- const zBeta = zQuantile(power);
480
- const n = 2 * ((zAlpha + zBeta) / effect) ** 2;
481
- return Math.ceil(n);
482
- }
483
- function requiredPairedSampleSize(opts) {
484
- const effect = opts.effect;
485
- if (!Number.isFinite(effect) || effect <= 0) return Infinity;
486
- const alpha = opts.alpha ?? 0.05;
487
- const power = opts.power ?? 0.8;
488
- const twoSided = opts.twoSided ?? true;
489
- const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
490
- const zBeta = zQuantile(power);
491
- return Math.ceil(((zAlpha + zBeta) / effect) ** 2);
492
- }
493
- function pairedMde(opts) {
494
- if (!Number.isFinite(opts.nPaired) || opts.nPaired <= 0) return Infinity;
495
- const alpha = opts.alpha ?? 0.05;
496
- const power = opts.power ?? 0.8;
497
- const twoSided = opts.twoSided ?? true;
498
- const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
499
- const zBeta = zQuantile(power);
500
- return (zAlpha + zBeta) / Math.sqrt(opts.nPaired);
501
- }
502
- function mcnemarRequiredN(opts) {
503
- const { p10, p01 } = opts;
504
- if (p10 < 0 || p01 < 0 || p10 + p01 > 1) {
505
- throw new Error(`mcnemarRequiredN: require p10,p01 \u2265 0 and p10+p01 \u2264 1 (got ${p10}, ${p01})`);
506
- }
507
- const delta = p10 - p01;
508
- if (delta === 0) return Infinity;
509
- const alpha = opts.alpha ?? 0.05;
510
- const power = opts.power ?? 0.8;
511
- const twoSided = opts.twoSided ?? true;
512
- const pDisc = p10 + p01;
513
- const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
514
- const zBeta = zQuantile(power);
515
- const n = (zAlpha * Math.sqrt(pDisc) + zBeta * Math.sqrt(Math.max(0, pDisc - delta * delta))) ** 2 / (delta * delta);
516
- return Math.ceil(n);
517
- }
518
- function mcnemarPower(opts) {
519
- const { p10, p01, nPairs } = opts;
520
- if (p10 < 0 || p01 < 0 || p10 + p01 > 1) {
521
- throw new Error(`mcnemarPower: require p10,p01 \u2265 0 and p10+p01 \u2264 1 (got ${p10}, ${p01})`);
522
- }
523
- const alpha = opts.alpha ?? 0.05;
524
- const twoSided = opts.twoSided ?? true;
525
- const delta = p10 - p01;
526
- if (delta === 0 || nPairs <= 0) return alpha;
527
- const pDisc = p10 + p01;
528
- const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
529
- const denom = Math.sqrt(Math.max(1e-12, pDisc - delta * delta));
530
- const zBeta = (Math.sqrt(nPairs) * Math.abs(delta) - zAlpha * Math.sqrt(pDisc)) / denom;
531
- return Math.min(1, Math.max(0, normalCdf(zBeta)));
532
- }
533
- function bonferroni(pValues, alpha = 0.05) {
534
- const k = pValues.length;
535
- const adjusted = pValues.map((p) => Math.min(1, p * k));
536
- const significant = adjusted.map((p) => p < alpha);
537
- return { adjusted, significant };
538
- }
539
- function holm(pValues, alpha = 0.05) {
540
- if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) {
541
- throw new ValidationError(`holm: alpha must be in (0,1), got ${alpha}`);
542
- }
543
- for (const [index, pValue] of pValues.entries()) {
544
- if (!Number.isFinite(pValue) || pValue < 0 || pValue > 1) {
545
- throw new ValidationError(`holm: pValues[${index}] must be in [0,1], got ${pValue}`);
546
- }
547
- }
548
- const count = pValues.length;
549
- if (count === 0) return { adjusted: [], significant: [] };
550
- const ordered = pValues.map((pValue, index) => ({ pValue, index })).sort((a, b) => a.pValue - b.pValue || a.index - b.index);
551
- const adjusted = new Array(count);
552
- let previous = 0;
553
- for (let rank = 0; rank < count; rank++) {
554
- const entry = ordered[rank];
555
- const stepAdjusted = Math.min(1, entry.pValue * (count - rank));
556
- previous = Math.max(previous, stepAdjusted);
557
- adjusted[entry.index] = previous;
558
- }
559
- return { adjusted, significant: adjusted.map((pValue) => pValue <= alpha) };
560
- }
561
- function benjaminiHochberg(pValues, fdr = 0.05) {
562
- const n = pValues.length;
563
- if (n === 0) return { qValues: [], significant: [] };
564
- const indexed = pValues.map((p, i) => ({ p, i })).sort((a, b) => a.p - b.p);
565
- const q = new Array(n);
566
- let minRight = 1;
567
- for (let k = n - 1; k >= 0; k--) {
568
- const rank = k + 1;
569
- const entry = indexed[k];
570
- const raw = entry.p * n / rank;
571
- const bounded = Math.min(minRight, raw);
572
- minRight = bounded;
573
- q[entry.i] = Math.min(1, bounded);
574
- }
575
- const significant = q.map((v) => v < fdr);
576
- return { qValues: q, significant };
577
- }
578
- function pairedBootstrap(before, after, opts = {}) {
579
- if (before.length !== after.length) {
580
- throw new Error(`pairedBootstrap: unequal sample sizes (${before.length} vs ${after.length})`);
581
- }
582
- const confidence = opts.confidence ?? 0.95;
583
- const resamples = opts.resamples ?? 2e3;
584
- const statistic = opts.statistic ?? "median";
585
- if (confidence <= 0 || confidence >= 1) {
586
- throw new Error(`pairedBootstrap: confidence must be in (0,1), got ${confidence}`);
587
- }
588
- const n = before.length;
589
- const deltas = before.map((b, i) => after[i] - b);
590
- if (n === 0) {
591
- return { n: 0, median: 0, mean: 0, low: 0, high: 0, confidence, resamples };
592
- }
593
- if (n === 1) {
594
- const d = deltas[0];
595
- return { n: 1, median: d, mean: d, low: d, high: d, confidence, resamples };
596
- }
597
- const rng = makeRng(opts.seed);
598
- const samples = new Array(resamples);
599
- for (let b = 0; b < resamples; b++) {
600
- if (statistic === "mean") {
601
- let sum = 0;
602
- for (let k = 0; k < n; k++) {
603
- sum += deltas[Math.floor(rng() * n)];
604
- }
605
- samples[b] = sum / n;
606
- } else {
607
- const acc = new Array(n);
608
- for (let k = 0; k < n; k++) {
609
- acc[k] = deltas[Math.floor(rng() * n)];
610
- }
611
- samples[b] = medianInPlace(acc);
612
- }
613
- }
614
- samples.sort((a, b) => a - b);
615
- const alpha = 1 - confidence;
616
- const lowIdx = Math.floor(alpha / 2 * resamples);
617
- const highIdx = Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1);
618
- return {
619
- n,
620
- median: medianInPlace([...deltas]),
621
- mean: deltas.reduce((s, x) => s + x, 0) / n,
622
- low: samples[lowIdx],
623
- high: samples[Math.max(highIdx, lowIdx)],
624
- confidence,
625
- resamples
626
- };
627
- }
628
- function pairedSignTest(differences, alternative) {
629
- if (alternative !== "greater" && alternative !== "less") {
630
- throw new ValidationError(
631
- `pairedSignTest: alternative must be 'greater' or 'less', got ${alternative}`
632
- );
633
- }
634
- let positive = 0;
635
- let negative = 0;
636
- let ties = 0;
637
- for (let i = 0; i < differences.length; i++) {
638
- const difference = differences[i];
639
- if (!Number.isFinite(difference)) {
640
- throw new ValidationError(
641
- `pairedSignTest: difference at index ${i} must be finite, got ${difference}`
642
- );
643
- }
644
- if (difference > 0) positive++;
645
- else if (difference < 0) negative++;
646
- else ties++;
647
- }
648
- const nNonTies = positive + negative;
649
- const successes = alternative === "greater" ? positive : negative;
650
- return {
651
- n: differences.length,
652
- positive,
653
- negative,
654
- ties,
655
- nNonTies,
656
- alternative,
657
- pValue: binomialHalfUpperTail(successes, nNonTies)
658
- };
659
- }
660
- function wilson(successes, n, confidence = 0.95) {
661
- if (n <= 0) return { estimate: 0, lower: 0, upper: 0 };
662
- if (successes < 0 || successes > n) {
663
- throw new Error(`wilson: successes (${successes}) must be in [0, ${n}]`);
664
- }
665
- const z = zQuantile(1 - (1 - confidence) / 2);
666
- const p = successes / n;
667
- const z2 = z * z;
668
- const denom = 1 + z2 / n;
669
- const center = (p + z2 / (2 * n)) / denom;
670
- const half = z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n) / denom;
671
- return {
672
- estimate: p,
673
- lower: Math.max(0, center - half),
674
- upper: Math.min(1, center + half)
675
- };
676
- }
677
- function mcnemar(control, treatment) {
678
- if (control.length !== treatment.length) {
679
- throw new Error(`mcnemar: unequal sample sizes (${control.length} vs ${treatment.length})`);
680
- }
681
- const n = control.length;
682
- let b = 0;
683
- let c = 0;
684
- for (let i = 0; i < n; i++) {
685
- const ctrl = control[i] ? 1 : 0;
686
- const treat = treatment[i] ? 1 : 0;
687
- if (treat === 1 && ctrl === 0) b++;
688
- else if (treat === 0 && ctrl === 1) c++;
689
- }
690
- const nDiscordant = b + c;
691
- const statistic = nDiscordant === 0 ? 0 : (Math.abs(b - c) - 1) ** 2 / nDiscordant;
692
- return { n, nDiscordant, b, c, statistic, pValue: binomialSignTwoSided(b, c) };
693
- }
694
- function pairedRiskDifference(control, treatment, confidence = 0.95) {
695
- if (control.length !== treatment.length) {
696
- throw new Error(
697
- `pairedRiskDifference: unequal sample sizes (${control.length} vs ${treatment.length})`
698
- );
699
- }
700
- const n = control.length;
701
- if (n === 0) return { n: 0, b: 0, c: 0, riskDifference: 0, lower: 0, upper: 0, confidence };
702
- let b = 0;
703
- let c = 0;
704
- for (let i = 0; i < n; i++) {
705
- const ctrl = control[i] ? 1 : 0;
706
- const treat = treatment[i] ? 1 : 0;
707
- if (treat === 1 && ctrl === 0) b++;
708
- else if (treat === 0 && ctrl === 1) c++;
709
- }
710
- const rd = (b - c) / n;
711
- const variance = (b + c - (b - c) ** 2 / n) / (n * n);
712
- const z = zQuantile(1 - (1 - confidence) / 2);
713
- const half = z * Math.sqrt(Math.max(0, variance));
714
- return {
715
- n,
716
- b,
717
- c,
718
- riskDifference: rd,
719
- lower: Math.max(-1, rd - half),
720
- upper: Math.min(1, rd + half),
721
- confidence
722
- };
723
- }
724
- function passAtK(n, c, k) {
725
- if (!Number.isInteger(n) || !Number.isInteger(c) || !Number.isInteger(k)) {
726
- throw new Error(`passAtK: n, c, k must be integers (got n=${n}, c=${c}, k=${k})`);
727
- }
728
- if (k < 1 || k > n || c < 0 || c > n) {
729
- throw new Error(`passAtK: require 1 \u2264 k \u2264 n and 0 \u2264 c \u2264 n (got n=${n}, c=${c}, k=${k})`);
730
- }
731
- if (n - c < k) return 1;
732
- let prob = 1;
733
- for (let i = n - c + 1; i <= n; i++) prob *= 1 - k / i;
734
- return 1 - prob;
735
- }
736
- function binomialSignTwoSided(b, c) {
737
- const nd = b + c;
738
- if (nd === 0) return 1;
739
- return Math.min(1, 2 * binomialHalfLowerTail(Math.min(b, c), nd));
740
- }
741
- function binomialHalfUpperTail(successes, n) {
742
- if (successes <= 0) return 1;
743
- if (successes > n) return 0;
744
- if (successes <= n / 2) {
745
- return Math.max(0, 1 - binomialHalfLowerTail(successes - 1, n));
746
- }
747
- return binomialHalfLowerTail(n - successes, n);
748
- }
749
- function binomialHalfLowerTail(maxSuccesses, n) {
750
- if (maxSuccesses < 0) return 0;
751
- if (maxSuccesses >= n) return 1;
752
- if (maxSuccesses === 0) return 2 ** -n;
753
- const logHalfN = n * Math.log(0.5);
754
- let logTail = Number.NEGATIVE_INFINITY;
755
- for (let i = 0; i <= maxSuccesses; i++) {
756
- const logChoose = lnGamma(n + 1) - lnGamma(i + 1) - lnGamma(n - i + 1);
757
- logTail = logAddExp(logTail, logChoose + logHalfN);
758
- }
759
- return Math.min(1, Math.exp(logTail));
760
- }
761
- function logAddExp(a, b) {
762
- if (a === Number.NEGATIVE_INFINITY) return b;
763
- if (b === Number.NEGATIVE_INFINITY) return a;
764
- const max = Math.max(a, b);
765
- const min = Math.min(a, b);
766
- return max + Math.log1p(Math.exp(min - max));
767
- }
768
- function eProcess(opts = {}) {
769
- const alpha = opts.alpha ?? 0.05;
770
- const maxBet = opts.maxBet ?? 0.5;
771
- const nullMean = opts.nullMean ?? 0.5;
772
- if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) {
773
- throw new ValidationError(`eProcess: alpha must be in (0,1), got ${alpha}`);
774
- }
775
- if (!Number.isFinite(nullMean) || nullMean <= 0 || nullMean >= 1) {
776
- throw new ValidationError(`eProcess: nullMean must be in (0,1), got ${nullMean}`);
777
- }
778
- if (!Number.isFinite(maxBet) || maxBet <= 0 || maxBet >= 1 / nullMean) {
779
- throw new ValidationError(
780
- `eProcess: maxBet must be in (0, 1/nullMean=${(1 / nullMean).toFixed(4)}) so wealth factors stay positive, got ${maxBet}`
781
- );
782
- }
783
- const threshold = 1 / alpha;
784
- let wealth = 1;
785
- let n = 0;
786
- let decided = false;
787
- let decidedAtN;
788
- let sumX = 0;
789
- let varSum = 0;
790
- return {
791
- update(x) {
792
- if (typeof x !== "number" || !Number.isFinite(x) || x < 0 || x > 1) {
793
- throw new ValidationError(
794
- `eProcess: observation must be a finite number in [0,1], got ${x}`
795
- );
796
- }
797
- const muPrev = (0.5 + sumX) / (n + 1);
798
- const varPrev = (0.25 + varSum) / (n + 1);
799
- const edge = muPrev - nullMean;
800
- const lambda = Math.min(maxBet, Math.max(0, edge / (varPrev + edge * edge)));
801
- wealth *= 1 + lambda * (x - nullMean);
802
- n += 1;
803
- sumX += x;
804
- const muNow = (0.5 + sumX) / (n + 1);
805
- varSum += (x - muNow) ** 2;
806
- if (!decided && wealth >= threshold) {
807
- decided = true;
808
- decidedAtN = n;
809
- }
810
- return { wealth, n, decided };
811
- },
812
- state() {
813
- return { wealth, n, decided, alpha, maxBet, nullMean, threshold, decidedAtN };
814
- }
815
- };
816
- }
817
- function zQuantile(p) {
818
- if (p <= 0 || p >= 1) {
819
- if (p === 0) return -Infinity;
820
- if (p === 1) return Infinity;
821
- return NaN;
822
- }
823
- const a = [
824
- -39.69683028665376,
825
- 220.9460984245205,
826
- -275.9285104469687,
827
- 138.357751867269,
828
- -30.66479806614716,
829
- 2.506628277459239
830
- ];
831
- const b = [
832
- -54.47609879822406,
833
- 161.5858368580409,
834
- -155.6989798598866,
835
- 66.80131188771972,
836
- -13.28068155288572
837
- ];
838
- const c = [
839
- -0.007784894002430293,
840
- -0.3223964580411365,
841
- -2.400758277161838,
842
- -2.549732539343734,
843
- 4.374664141464968,
844
- 2.938163982698783
845
- ];
846
- const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
847
- const pLow = 0.02425;
848
- const pHigh = 1 - pLow;
849
- let q;
850
- let r;
851
- if (p < pLow) {
852
- q = Math.sqrt(-2 * Math.log(p));
853
- return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
854
- }
855
- if (p <= pHigh) {
856
- q = p - 0.5;
857
- r = q * q;
858
- return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
859
- }
860
- q = Math.sqrt(-2 * Math.log(1 - p));
861
- return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
862
- }
863
- function medianInPlace(xs) {
864
- if (xs.length === 0) return 0;
865
- xs.sort((a, b) => a - b);
866
- const mid = Math.floor(xs.length / 2);
867
- return xs.length % 2 === 0 ? (xs[mid - 1] + xs[mid]) / 2 : xs[mid];
868
- }
869
- function makeRng(seed) {
870
- if (seed === void 0) return Math.random;
871
- return mulberry32(seed);
872
- }
873
- function mulberry32(seed) {
874
- let s = seed | 0 || 2654435769;
875
- return () => {
876
- s = s + 1831565813 | 0;
877
- let t = s;
878
- t = Math.imul(t ^ t >>> 15, t | 1);
879
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
880
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
881
- };
882
- }
883
-
884
- // src/judge-calibration.ts
885
- function calibrateJudge(golden, candidate) {
886
- const map = /* @__PURE__ */ new Map();
887
- for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN });
888
- for (const c of candidate) {
889
- const entry = map.get(c.itemId);
890
- if (entry) entry.j = c.score;
891
- }
892
- const common = [...map.values()].filter((v) => Number.isFinite(v.j));
893
- const n = common.length;
894
- if (n < 2) {
895
- return { n, pearson: NaN, kappa: NaN, mae: NaN, worstItems: [] };
896
- }
897
- const humans = common.map((c) => c.h);
898
- const judges = common.map((c) => c.j);
899
- const pearson = pearsonR(humans, judges);
900
- const kappa = weightedKappa(humans.map(Math.round), judges.map(Math.round));
901
- const absDiffs = common.map((c) => Math.abs(c.j - c.h));
902
- const mae = absDiffs.reduce((a, b) => a + b, 0) / n;
903
- const worst = [...map.entries()].filter(([, v]) => Number.isFinite(v.j)).map(([itemId, v]) => ({ itemId, judge: v.j, human: v.h, delta: Math.abs(v.j - v.h) })).sort((a, b) => b.delta - a.delta).slice(0, 5);
904
- return { n, pearson, kappa, mae, worstItems: worst };
905
- }
906
- function positionalBias(scores) {
907
- const pairs = /* @__PURE__ */ new Map();
908
- for (const s of scores) {
909
- const slot = pairs.get(s.itemId) ?? {};
910
- if (s.positionOfAInput === "first") slot.first = s.score;
911
- else if (s.positionOfAInput === "second") slot.second = s.score;
912
- pairs.set(s.itemId, slot);
913
- }
914
- const deltas = [];
915
- for (const { first, second } of pairs.values()) {
916
- if (first !== void 0 && second !== void 0) deltas.push(first - second);
917
- }
918
- if (deltas.length === 0) return { avgDelta: 0, n: 0 };
919
- return { avgDelta: deltas.reduce((a, b) => a + b, 0) / deltas.length, n: deltas.length };
920
- }
921
- function verbosityBias(samples) {
922
- const n = samples.length;
923
- if (n < 3) return { pearson: NaN, n };
924
- return {
925
- pearson: pearsonR(
926
- samples.map((s) => s.outputLen),
927
- samples.map((s) => s.score)
928
- ),
929
- n
930
- };
931
- }
932
- function selfPreference(samples) {
933
- const inF = samples.filter((s) => s.inFamily).map((s) => s.score);
934
- const outF = samples.filter((s) => !s.inFamily).map((s) => s.score);
935
- if (inF.length === 0 || outF.length === 0)
936
- return { inFamilyMean: 0, outOfFamilyMean: 0, deltaMean: 0, n: 0 };
937
- const inMean = inF.reduce((a, b) => a + b, 0) / inF.length;
938
- const outMean = outF.reduce((a, b) => a + b, 0) / outF.length;
939
- return {
940
- inFamilyMean: inMean,
941
- outOfFamilyMean: outMean,
942
- deltaMean: inMean - outMean,
943
- n: samples.length
944
- };
945
- }
946
- function weightedKappa(a, b) {
947
- if (a.length !== b.length || a.length === 0) return NaN;
948
- const min = Math.min(...a, ...b);
949
- const max = Math.max(...a, ...b);
950
- const K = max - min + 1;
951
- if (K < 2) return 1;
952
- const observed = Array.from({ length: K }, () => new Array(K).fill(0));
953
- const rowMarg = new Array(K).fill(0);
954
- const colMarg = new Array(K).fill(0);
955
- for (let i = 0; i < a.length; i++) {
956
- const ai = a[i] - min;
957
- const bi = b[i] - min;
958
- const row = observed[ai];
959
- row[bi] = (row[bi] ?? 0) + 1;
960
- rowMarg[ai]++;
961
- colMarg[bi]++;
962
- }
963
- let num = 0;
964
- let den = 0;
965
- for (let i = 0; i < K; i++) {
966
- for (let j = 0; j < K; j++) {
967
- const w = (i - j) ** 2 / (K - 1) ** 2;
968
- const expected = rowMarg[i] * colMarg[j] / a.length;
969
- num += w * observed[i][j];
970
- den += w * expected;
971
- }
972
- }
973
- if (den === 0) return 1;
974
- return 1 - num / den;
975
- }
976
- function continuousAgreement(scores, opts = {}) {
977
- const bootstrap = opts.bootstrap ?? 1e3;
978
- const weights = opts.weights ?? "quadratic";
979
- const seed = opts.seed ?? 12648430;
980
- const ciLevel = opts.ciLevel ?? 0.95;
981
- const matrix = scores.filter((row) => row.length >= 2 && row.every((v) => Number.isFinite(v)));
982
- const raters = matrix[0]?.length ?? 0;
983
- const clean = matrix.filter((row) => row.length === raters);
984
- const nClean = clean.length;
985
- if (nClean < 2 || raters < 2) {
986
- return {
987
- weightedKappa: NaN,
988
- icc: NaN,
989
- pearson: NaN,
990
- spearman: NaN,
991
- ci: { icc: [NaN, NaN], weightedKappa: [NaN, NaN] },
992
- n: nClean,
993
- raters
994
- };
995
- }
996
- const kappa = continuousWeightedKappa(clean, weights);
997
- const icc = icc21(clean);
998
- const pearson = avgPairwise(clean, pearsonR);
999
- const spearman = avgPairwise(clean, spearmanR);
1000
- const ciIcc = [NaN, NaN];
1001
- const ciKappa = [NaN, NaN];
1002
- if (bootstrap > 0) {
1003
- const rng = mulberry322(seed);
1004
- const iccs = [];
1005
- const kappas = [];
1006
- for (let b = 0; b < bootstrap; b++) {
1007
- const sample = new Array(nClean);
1008
- for (let i = 0; i < nClean; i++) {
1009
- sample[i] = clean[Math.floor(rng() * nClean)];
1010
- }
1011
- const iccB = icc21(sample);
1012
- const kB = continuousWeightedKappa(sample, weights);
1013
- if (Number.isFinite(iccB)) iccs.push(iccB);
1014
- if (Number.isFinite(kB)) kappas.push(kB);
1015
- }
1016
- const [lo, hi] = percentileBounds(ciLevel);
1017
- if (iccs.length > 0) {
1018
- iccs.sort((a, b) => a - b);
1019
- ciIcc[0] = quantile(iccs, lo);
1020
- ciIcc[1] = quantile(iccs, hi);
1021
- }
1022
- if (kappas.length > 0) {
1023
- kappas.sort((a, b) => a - b);
1024
- ciKappa[0] = quantile(kappas, lo);
1025
- ciKappa[1] = quantile(kappas, hi);
1026
- }
1027
- }
1028
- return {
1029
- weightedKappa: kappa,
1030
- icc,
1031
- pearson,
1032
- spearman,
1033
- ci: { icc: ciIcc, weightedKappa: ciKappa },
1034
- n: nClean,
1035
- raters
1036
- };
1037
- }
1038
- function calibrateJudgeContinuous(golden, candidate, opts = {}) {
1039
- const base = calibrateJudge(golden, candidate);
1040
- const map = /* @__PURE__ */ new Map();
1041
- for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN });
1042
- for (const c of candidate) {
1043
- const entry = map.get(c.itemId);
1044
- if (entry) entry.j = c.score;
1045
- }
1046
- const rows = [];
1047
- for (const v of map.values()) {
1048
- if (Number.isFinite(v.j)) rows.push([v.h, v.j]);
1049
- }
1050
- const agreement = continuousAgreement(rows, opts);
1051
- return {
1052
- ...base,
1053
- weightedKappaContinuous: agreement.weightedKappa,
1054
- icc: agreement.icc,
1055
- spearman: agreement.spearman,
1056
- ci: agreement.ci
1057
- };
1058
- }
1059
- function continuousWeightedKappa(rows, scheme) {
1060
- if (rows.length === 0) return NaN;
1061
- const raters = rows[0].length;
1062
- if (raters < 2) return NaN;
1063
- const wFn = scheme === "linear" ? (x, y) => Math.abs(x - y) : (x, y) => (x - y) ** 2;
1064
- let sum = 0;
1065
- let pairs = 0;
1066
- for (let r1 = 0; r1 < raters; r1++) {
1067
- for (let r2 = r1 + 1; r2 < raters; r2++) {
1068
- const a = rows.map((row) => row[r1]);
1069
- const b = rows.map((row) => row[r2]);
1070
- const n = a.length;
1071
- let obs = 0;
1072
- for (let i = 0; i < n; i++) obs += wFn(a[i], b[i]);
1073
- obs /= n;
1074
- let exp = 0;
1075
- for (let i = 0; i < n; i++) {
1076
- for (let j = 0; j < n; j++) exp += wFn(a[i], b[j]);
1077
- }
1078
- exp /= n * n;
1079
- if (exp === 0) {
1080
- sum += obs === 0 ? 1 : 0;
1081
- } else {
1082
- sum += 1 - obs / exp;
1083
- }
1084
- pairs++;
1085
- }
1086
- }
1087
- return pairs === 0 ? NaN : sum / pairs;
1088
- }
1089
- function icc21(rows) {
1090
- const n = rows.length;
1091
- if (n < 2) return NaN;
1092
- const k = rows[0].length;
1093
- if (k < 2) return NaN;
1094
- const rowMeans = rows.map((row) => row.reduce((s, v) => s + v, 0) / k);
1095
- const colMeans = new Array(k).fill(0);
1096
- for (let j = 0; j < k; j++) {
1097
- let s = 0;
1098
- for (let i = 0; i < n; i++) s += rows[i][j];
1099
- colMeans[j] = s / n;
1100
- }
1101
- let grand = 0;
1102
- for (let i = 0; i < n; i++) grand += rowMeans[i];
1103
- grand /= n;
1104
- let ssR = 0;
1105
- for (let i = 0; i < n; i++) ssR += (rowMeans[i] - grand) ** 2;
1106
- ssR *= k;
1107
- let ssC = 0;
1108
- for (let j = 0; j < k; j++) ssC += (colMeans[j] - grand) ** 2;
1109
- ssC *= n;
1110
- let ssT = 0;
1111
- for (let i = 0; i < n; i++) {
1112
- for (let j = 0; j < k; j++) ssT += (rows[i][j] - grand) ** 2;
1113
- }
1114
- const ssE = ssT - ssR - ssC;
1115
- const dfR = n - 1;
1116
- const dfC = k - 1;
1117
- const dfE = (n - 1) * (k - 1);
1118
- const msR = ssR / dfR;
1119
- const msC = ssC / dfC;
1120
- const msE = dfE > 0 ? ssE / dfE : 0;
1121
- const denom = msR + (k - 1) * msE + k * (msC - msE) / n;
1122
- if (denom === 0) {
1123
- return msR === 0 && msE === 0 ? 1 : 0;
1124
- }
1125
- return (msR - msE) / denom;
1126
- }
1127
- function avgPairwise(rows, fn) {
1128
- const k = rows[0]?.length ?? 0;
1129
- if (k < 2) return NaN;
1130
- let sum = 0;
1131
- let pairs = 0;
1132
- for (let i = 0; i < k; i++) {
1133
- for (let j = i + 1; j < k; j++) {
1134
- const a = rows.map((row) => row[i]);
1135
- const b = rows.map((row) => row[j]);
1136
- const r = fn(a, b);
1137
- if (Number.isFinite(r)) {
1138
- sum += r;
1139
- pairs++;
1140
- }
1141
- }
1142
- }
1143
- return pairs === 0 ? NaN : sum / pairs;
1144
- }
1145
- function mulberry322(seed) {
1146
- let a = seed >>> 0;
1147
- return () => {
1148
- a = a + 1831565813 >>> 0;
1149
- let t = a;
1150
- t = Math.imul(t ^ t >>> 15, t | 1);
1151
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
1152
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
1153
- };
1154
- }
1155
- function percentileBounds(ciLevel) {
1156
- const tail = (1 - ciLevel) / 2;
1157
- return [tail, 1 - tail];
1158
- }
1159
- function quantile(sorted, q) {
1160
- if (sorted.length === 0) return NaN;
1161
- if (sorted.length === 1) return sorted[0];
1162
- const pos = q * (sorted.length - 1);
1163
- const lo = Math.floor(pos);
1164
- const hi = Math.ceil(pos);
1165
- if (lo === hi) return sorted[lo];
1166
- const frac = pos - lo;
1167
- return sorted[lo] * (1 - frac) + sorted[hi] * frac;
1168
- }
1169
-
1170
- export {
1171
- calibrateJudge,
1172
- positionalBias,
1173
- verbosityBias,
1174
- selfPreference,
1175
- continuousAgreement,
1176
- calibrateJudgeContinuous,
1177
- normalizeScores,
1178
- weightedMean,
1179
- confidenceInterval,
1180
- interRaterReliability,
1181
- mannWhitneyU,
1182
- partialCredit,
1183
- pairedTTest,
1184
- wilcoxonSignedRank,
1185
- cohensD,
1186
- pairedCohensDz,
1187
- cliffsDelta,
1188
- interpretCliffs,
1189
- ranks,
1190
- pearsonR,
1191
- spearmanR,
1192
- weightedComposite,
1193
- corpusInterRaterAgreement,
1194
- corpusInterRaterAgreementFromJudgeScores,
1195
- requiredSampleSize,
1196
- requiredPairedSampleSize,
1197
- pairedMde,
1198
- mcnemarRequiredN,
1199
- mcnemarPower,
1200
- bonferroni,
1201
- holm,
1202
- benjaminiHochberg,
1203
- pairedBootstrap,
1204
- pairedSignTest,
1205
- wilson,
1206
- mcnemar,
1207
- pairedRiskDifference,
1208
- passAtK,
1209
- eProcess,
1210
- mulberry32
1211
- };
1212
- //# sourceMappingURL=chunk-MHELPNRP.js.map