@tangle-network/agent-eval 0.216.0 → 0.217.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (553) hide show
  1. package/CHANGELOG.md +141 -57
  2. package/README.md +2 -2
  3. package/dist/adapters/http.d.ts +3 -3
  4. package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-DTt8uGP9.d.ts} +5 -18
  5. package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
  6. package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-B1V1gUZs.js} +4 -53
  7. package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
  8. package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
  9. package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
  10. package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BhAqjluA.js} +5 -5
  11. package/dist/agreement-irr-BhAqjluA.js.map +1 -0
  12. package/dist/analyst/index.d.ts +37 -170
  13. package/dist/analyst/index.d.ts.map +1 -1
  14. package/dist/analyst/index.js +17 -192
  15. package/dist/analyst/index.js.map +1 -1
  16. package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-DqPJaJ4X.js} +763 -555
  17. package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
  18. package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-DyeNcenW.d.ts} +18 -16
  19. package/dist/analyze-runs-DyeNcenW.d.ts.map +1 -0
  20. package/dist/authenticity/index.d.ts +0 -23
  21. package/dist/authenticity/index.d.ts.map +1 -1
  22. package/dist/authenticity/index.js +1 -34
  23. package/dist/authenticity/index.js.map +1 -1
  24. package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-xQK8nOI6.js} +591 -109
  25. package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
  26. package/dist/benchmarks/index.d.ts +5 -6
  27. package/dist/benchmarks/index.d.ts.map +1 -1
  28. package/dist/benchmarks/index.js +4 -4
  29. package/dist/benchmarks/index.js.map +1 -1
  30. package/dist/builder-eval/index.d.ts +3 -3
  31. package/dist/builder-eval/index.js +2 -2
  32. package/dist/builder-eval/index.js.map +1 -1
  33. package/dist/campaign/index.d.ts +8 -8
  34. package/dist/campaign/index.js +15 -15
  35. package/dist/{campaign-BQ-Jujuj.js → campaign-CJGkxAQ6.js} +256 -765
  36. package/dist/campaign-CJGkxAQ6.js.map +1 -0
  37. package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-CQYw8_0H.js} +10 -18
  38. package/dist/{campaign-evidence-OcUciD-N.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
  39. package/dist/{canonical-BsZVgBhW.js → canonical-Dq2hOQ8L.js} +2 -2
  40. package/dist/{canonical-BsZVgBhW.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
  41. package/dist/cell-spend-DEWR3nj-.js +90 -0
  42. package/dist/cell-spend-DEWR3nj-.js.map +1 -0
  43. package/dist/{chat-client-DthPOYGd.js → chat-client-ldqxYYTN.js} +2 -2
  44. package/dist/{chat-client-DthPOYGd.js.map → chat-client-ldqxYYTN.js.map} +1 -1
  45. package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-DavzexSp.js} +3 -2
  46. package/dist/chat-json-call-DavzexSp.js.map +1 -0
  47. package/dist/{claim-BXcZw-jo.js → claim-BPVuyHuy.js} +4 -4
  48. package/dist/claim-BPVuyHuy.js.map +1 -0
  49. package/dist/cli.js +113 -12
  50. package/dist/cli.js.map +1 -1
  51. package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
  52. package/dist/client-hwo3cJjR.d.ts.map +1 -0
  53. package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
  54. package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
  55. package/dist/{produced-state-ZCtDBxAE.js → completion-verifier-B_jw6C9p.js} +9 -64
  56. package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
  57. package/dist/contract/index.d.ts +13 -13
  58. package/dist/contract/index.js +1041 -1473
  59. package/dist/contract/index.js.map +1 -1
  60. package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-D0352Z4d.js} +832 -817
  61. package/dist/cost-ledger-D0352Z4d.js.map +1 -0
  62. package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
  63. package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
  64. package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
  65. package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-ClYjq3QE.js} +8 -11
  66. package/dist/default-registry-ClYjq3QE.js.map +1 -0
  67. package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
  68. package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
  69. package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-BDxNlP3D.js} +10 -10
  70. package/dist/define-agent-eval-BDxNlP3D.js.map +1 -0
  71. package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +9 -9
  72. package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
  73. package/dist/{descriptive-CnU8Yq29.js → descriptive-CVNy48rR.js} +2 -2
  74. package/dist/{descriptive-CnU8Yq29.js.map → descriptive-CVNy48rR.js.map} +1 -1
  75. package/dist/diagnosis/index.d.ts +4 -14
  76. package/dist/diagnosis/index.d.ts.map +1 -1
  77. package/dist/diagnosis/index.js +67 -28
  78. package/dist/diagnosis/index.js.map +1 -1
  79. package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
  80. package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
  81. package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
  82. package/dist/emitter-DABYYTcH.d.ts.map +1 -0
  83. package/dist/emitter-DbMXUIVP.js.map +1 -1
  84. package/dist/{engine-C_zNWfM8.d.ts → engine-BvZ0R-36.d.ts} +18 -22
  85. package/dist/engine-BvZ0R-36.d.ts.map +1 -0
  86. package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-DbPGcVtI.js} +2 -2
  87. package/dist/equal-terms-DbPGcVtI.js.map +1 -0
  88. package/dist/{errors-Dngq5h35.js → errors-BcOs5MKW.js} +6 -8
  89. package/dist/{errors-Dngq5h35.js.map → errors-BcOs5MKW.js.map} +1 -1
  90. package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-BLEfBGbA.js} +6 -6
  91. package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
  92. package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-jTjC_QQD.d.ts} +8 -8
  93. package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
  94. package/dist/evaluation.d.ts +2 -2
  95. package/dist/evaluation.js +1 -1
  96. package/dist/evaluation.js.map +1 -1
  97. package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
  98. package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
  99. package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
  100. package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
  101. package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
  102. package/dist/experiment/index.d.ts +331 -33
  103. package/dist/experiment/index.d.ts.map +1 -1
  104. package/dist/experiment/index.js +303 -101
  105. package/dist/experiment/index.js.map +1 -1
  106. package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
  107. package/dist/exporters-Cl6IeNAu.js.map +1 -0
  108. package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
  109. package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
  110. package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-KVFGvYX1.js} +4 -3
  111. package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
  112. package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-XIHBOSKt.js} +551 -553
  113. package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
  114. package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
  115. package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
  116. package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
  117. package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
  118. package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
  119. package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
  120. package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +8 -13
  121. package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
  122. package/dist/{semantic-concept-judge-zQBG9hHU.js → findings-store-BMVQtEsm.js} +13 -255
  123. package/dist/findings-store-BMVQtEsm.js.map +1 -0
  124. package/dist/fuzz.d.ts +20 -63
  125. package/dist/fuzz.d.ts.map +1 -1
  126. package/dist/fuzz.js +62 -108
  127. package/dist/fuzz.js.map +1 -1
  128. package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
  129. package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
  130. package/dist/hosted/index.d.ts +2 -2
  131. package/dist/hosted/index.js +2 -2
  132. package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
  133. package/dist/index-9jJ2fKSX.d.ts.map +1 -0
  134. package/dist/{index-oQ4aHziK.d.ts → index-BLGqZ81N.d.ts} +11 -16
  135. package/dist/{index-oQ4aHziK.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
  136. package/dist/{index-DlH0Y9gI.d.ts → index-BiT6RtNH.d.ts} +31 -43
  137. package/dist/index-BiT6RtNH.d.ts.map +1 -0
  138. package/dist/{index-Ch9VxCpK.d.ts → index-CX-GAUpw.d.ts} +4 -8
  139. package/dist/index-CX-GAUpw.d.ts.map +1 -0
  140. package/dist/{index-CPJJr98o.d.ts → index-CiRwLv2R.d.ts} +5 -68
  141. package/dist/index-CiRwLv2R.d.ts.map +1 -0
  142. package/dist/{index-NvGBiO8H.d.ts → index-ZuWx52WG.d.ts} +266 -495
  143. package/dist/index-ZuWx52WG.d.ts.map +1 -0
  144. package/dist/index.d.ts +2256 -2165
  145. package/dist/index.d.ts.map +1 -1
  146. package/dist/index.js +514 -1192
  147. package/dist/index.js.map +1 -1
  148. package/dist/{integrity-BV1ZDf42.js → integrity-DOd9Q86O.js} +7 -12
  149. package/dist/integrity-DOd9Q86O.js.map +1 -0
  150. package/dist/{integrity-BvnVARVl.js → integrity-W4CADcFc.js} +3 -7
  151. package/dist/{integrity-BvnVARVl.js.map → integrity-W4CADcFc.js.map} +1 -1
  152. package/dist/{integrity-CCAxGVK6.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
  153. package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
  154. package/dist/{internal-CRHwS9iz.js → internal-BTK91Bqt.js} +3 -3
  155. package/dist/{internal-CRHwS9iz.js.map → internal-BTK91Bqt.js.map} +1 -1
  156. package/dist/interval-Dq9gt7jx.js +219 -0
  157. package/dist/interval-Dq9gt7jx.js.map +1 -0
  158. package/dist/interval-ohuOJurX.d.ts +47 -0
  159. package/dist/interval-ohuOJurX.d.ts.map +1 -0
  160. package/dist/{journal-DvKCvjty.js → journal-CR4QikYp.js} +110 -110
  161. package/dist/journal-CR4QikYp.js.map +1 -0
  162. package/dist/json-BhdsJaOQ.js +111 -0
  163. package/dist/json-BhdsJaOQ.js.map +1 -0
  164. package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BkXMIWA4.js} +3 -3
  165. package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
  166. package/dist/{judge-gate-B1qYnkpi.js → judge-gate-Co5tuh57.js} +4 -27
  167. package/dist/judge-gate-Co5tuh57.js.map +1 -0
  168. package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-DsL0yFcb.d.ts} +9 -10
  169. package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
  170. package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-Gy4fXixd.js} +88 -17
  171. package/dist/kind-factory-Gy4fXixd.js.map +1 -0
  172. package/dist/ledger-core/index.d.ts +2 -2
  173. package/dist/ledger-core/index.js +4 -4
  174. package/dist/{ledger-core-B2hahzrt.js → ledger-core-DrJhbms9.js} +5 -84
  175. package/dist/ledger-core-DrJhbms9.js.map +1 -0
  176. package/dist/{llm-client-DQveoxSM.js → llm-client-BshCREn6.js} +90 -219
  177. package/dist/llm-client-BshCREn6.js.map +1 -0
  178. package/dist/{llm-judge-PXV7Xljl.js → llm-judge-C8jMTNNG.js} +501 -931
  179. package/dist/llm-judge-C8jMTNNG.js.map +1 -0
  180. package/dist/matrix/index.d.ts +4 -4
  181. package/dist/matrix/index.js +3 -2
  182. package/dist/{matrix-_BV9-Lk5.js → matrix-B8l-Nsxd.js} +7 -96
  183. package/dist/matrix-B8l-Nsxd.js.map +1 -0
  184. package/dist/{matrix-CHMiebFx.d.ts → matrix-DHe-dwVJ.d.ts} +10 -19
  185. package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
  186. package/dist/{paired-tests-jbwPdSz5.js → mean-BbA_JCzW.js} +45 -5
  187. package/dist/mean-BbA_JCzW.js.map +1 -0
  188. package/dist/meta-eval/index.d.ts +6 -6
  189. package/dist/meta-eval/index.js +10 -10
  190. package/dist/meta-eval/index.js.map +1 -1
  191. package/dist/metrics-BKaPBtIG.js.map +1 -1
  192. package/dist/mint-BRZXuwRi.d.ts +81 -0
  193. package/dist/mint-BRZXuwRi.d.ts.map +1 -0
  194. package/dist/{mint-Csor1BLJ.js → mint-zszyYlpA.js} +4 -4
  195. package/dist/{mint-Csor1BLJ.js.map → mint-zszyYlpA.js.map} +1 -1
  196. package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
  197. package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
  198. package/dist/{multiplicity-D9cjmStq.js → multiplicity-Sg0U7h8j.js} +2 -2
  199. package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
  200. package/dist/multishot/golden/index.d.ts +1 -23
  201. package/dist/multishot/golden/index.d.ts.map +1 -1
  202. package/dist/multishot/golden/index.js +5 -8
  203. package/dist/multishot/golden/index.js.map +1 -1
  204. package/dist/multishot/index.d.ts +3 -12
  205. package/dist/multishot/index.d.ts.map +1 -1
  206. package/dist/multishot/index.js +4 -3
  207. package/dist/multishot/index.js.map +1 -1
  208. package/dist/nonnegative-number-DvdMxg_f.js +9 -0
  209. package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
  210. package/dist/openapi.json +114 -10
  211. package/dist/{otel-spans-C8YIJWy1.js → otel-spans-CdN909k1.js} +3 -3
  212. package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-CdN909k1.js.map} +1 -1
  213. package/dist/{paired-arms-DzKn0TfY.js → paired-arms-Ct4LTKc1.js} +32 -9
  214. package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
  215. package/dist/{paired-binary-Q36bhn2R.js → paired-binary-B_nejA44.js} +3 -36
  216. package/dist/paired-binary-B_nejA44.js.map +1 -0
  217. package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
  218. package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
  219. package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
  220. package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
  221. package/dist/{search-kernel-BztlERLv.js → pareto-BzsxidFD.js} +883 -861
  222. package/dist/pareto-BzsxidFD.js.map +1 -0
  223. package/dist/pipelines/index.d.ts +16 -19
  224. package/dist/pipelines/index.d.ts.map +1 -1
  225. package/dist/pipelines/index.js +4 -68
  226. package/dist/pipelines/index.js.map +1 -1
  227. package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-N5nU4_Ly.js} +3 -3
  228. package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
  229. package/dist/predicate-FdKmElA_.js +38 -0
  230. package/dist/predicate-FdKmElA_.js.map +1 -0
  231. package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-BFgGrEMP.js} +3 -3
  232. package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
  233. package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-BWG2DQIk.d.ts} +4 -22
  234. package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
  235. package/dist/profile-cell.d.ts +2 -2
  236. package/dist/profile-cell.js +2 -2
  237. package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-DaY95yU6.d.ts} +105 -106
  238. package/dist/promotion-gate-DaY95yU6.d.ts.map +1 -0
  239. package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-6A8osGr_.js} +4 -4
  240. package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
  241. package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-BbWidvUk.js} +3 -3
  242. package/dist/proposal-findings-BbWidvUk.js.map +1 -0
  243. package/dist/{query-B1D1qp2Z.js → query-B_RZ36an.js} +5 -5
  244. package/dist/query-B_RZ36an.js.map +1 -0
  245. package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
  246. package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
  247. package/dist/{random-Dn5fPWkt.js → random-DDDcC7YU.js} +2 -2
  248. package/dist/{random-Dn5fPWkt.js.map → random-DDDcC7YU.js.map} +1 -1
  249. package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-75w4FWBj.js} +2 -2
  250. package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
  251. package/dist/{redact-Cveoq6fJ.d.ts → redact-DXdlFb_U.d.ts} +4 -21
  252. package/dist/redact-DXdlFb_U.d.ts.map +1 -0
  253. package/dist/{redact-Dk1ZdJEm.js → redact-mtkTiJVk.js} +92 -98
  254. package/dist/redact-mtkTiJVk.js.map +1 -0
  255. package/dist/{registry-JYfPxcVz.d.ts → registry-ssNoHzwD.d.ts} +5 -5
  256. package/dist/{registry-JYfPxcVz.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
  257. package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CYIFH-fp.js} +6 -11
  258. package/dist/release-confidence-CYIFH-fp.js.map +1 -0
  259. package/dist/{report-command-y8H6RSX8.js → render-C-w2EZ6p.js} +331 -554
  260. package/dist/render-C-w2EZ6p.js.map +1 -0
  261. package/dist/reporting-html-Kvlc8oMt.js +100 -0
  262. package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
  263. package/dist/reporting.d.ts +6 -6
  264. package/dist/reporting.js +8 -8
  265. package/dist/{researcher-f6mZhh58.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
  266. package/dist/{researcher-f6mZhh58.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
  267. package/dist/reward-DrZ71vnf.d.ts +55 -0
  268. package/dist/reward-DrZ71vnf.d.ts.map +1 -0
  269. package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-ByuETaIb.js} +3 -3
  270. package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
  271. package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
  272. package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
  273. package/dist/reward-nw2xZGZG.js.map +1 -1
  274. package/dist/rl.d.ts +117 -295
  275. package/dist/rl.d.ts.map +1 -1
  276. package/dist/rl.js +27 -747
  277. package/dist/rl.js.map +1 -1
  278. package/dist/rollout/index.d.ts +5 -4
  279. package/dist/rollout/index.js +7 -8
  280. package/dist/{rollout-D9bKKBnZ.js → rollout-D6C50mfA.js} +88 -13
  281. package/dist/rollout-D6C50mfA.js.map +1 -0
  282. package/dist/round-decimal-COrKglMh.js +10 -0
  283. package/dist/round-decimal-COrKglMh.js.map +1 -0
  284. package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
  285. package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
  286. package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
  287. package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
  288. package/dist/{run-record-D1Hw-Jml.d.ts → run-record-CaxbLq6Y.d.ts} +5 -11
  289. package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
  290. package/dist/{run-record-ES07wfz9.js → run-record-CysKLB93.js} +3 -5
  291. package/dist/run-record-CysKLB93.js.map +1 -0
  292. package/dist/{run-record-DlTZ_lqe.js → run-record-DhEJgoPz.js} +4 -4
  293. package/dist/{run-record-DlTZ_lqe.js.map → run-record-DhEJgoPz.js.map} +1 -1
  294. package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
  295. package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
  296. package/dist/schema-Bt2vLFm3.js.map +1 -0
  297. package/dist/schema-C-ziyJD7.d.ts.map +1 -1
  298. package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
  299. package/dist/schema-D6EVKzTB.d.ts.map +1 -0
  300. package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
  301. package/dist/schema-DxcKTzbi.js.map +1 -0
  302. package/dist/search/index.d.ts +39 -53
  303. package/dist/search/index.d.ts.map +1 -1
  304. package/dist/search/index.js +4 -4
  305. package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DxqjBxA-.d.ts} +18 -31
  306. package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
  307. package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-TUvB5gWH.js} +189 -199
  308. package/dist/search-ledger-TUvB5gWH.js.map +1 -0
  309. package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-UKCO2r8a.js} +4 -4
  310. package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
  311. package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
  312. package/dist/{search-shipper-er4pK5_Y.js → search-shipper-Cw2z3-td.js} +4 -4
  313. package/dist/search-shipper-Cw2z3-td.js.map +1 -0
  314. package/dist/{search-summary-CBvz3u_S.js → search-summary-DQIvJsSB.js} +5 -5
  315. package/dist/{search-summary-CBvz3u_S.js.map → search-summary-DQIvJsSB.js.map} +1 -1
  316. package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
  317. package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
  318. package/dist/{sequential-D-1PPCmM.js → sequential-CimQrbjv.js} +5 -5
  319. package/dist/sequential-CimQrbjv.js.map +1 -0
  320. package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
  321. package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
  322. package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DodI6RoF.js} +2 -2
  323. package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
  324. package/dist/{server-DbcC7xMl.js → server-DoVbqqtJ.js} +101 -101
  325. package/dist/server-DoVbqqtJ.js.map +1 -0
  326. package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-U58yf8N6.js} +2 -2
  327. package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
  328. package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-kyL94-ri.js} +428 -91
  329. package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
  330. package/dist/{spans-D9_ehFuk.d.ts → spans-CVngGij3.d.ts} +2 -2
  331. package/dist/{spans-D9_ehFuk.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
  332. package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-Bksv92Th.js} +3 -3
  333. package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
  334. package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
  335. package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
  336. package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
  337. package/dist/steps-DlepJPwD.d.ts.map +1 -0
  338. package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
  339. package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
  340. package/dist/{store-otlp-DtB4Slsr.js → store-otlp-BteDAbHK.js} +135 -132
  341. package/dist/store-otlp-BteDAbHK.js.map +1 -0
  342. package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-D5l_OfTJ.js} +9 -9
  343. package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
  344. package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DZS0HWVf.d.ts} +157 -165
  345. package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
  346. package/dist/storyboard/index.d.ts +4 -11
  347. package/dist/storyboard/index.d.ts.map +1 -1
  348. package/dist/storyboard/index.js +21 -19
  349. package/dist/storyboard/index.js.map +1 -1
  350. package/dist/{summary-report-DtdRucM0.js → summary-report-BsXJoYX5.js} +588 -598
  351. package/dist/summary-report-BsXJoYX5.js.map +1 -0
  352. package/dist/{summary-report-C-CtItCN.d.ts → summary-report-DM7tlml1.d.ts} +16 -20
  353. package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
  354. package/dist/supervisor-run/index.d.ts +189 -153
  355. package/dist/supervisor-run/index.d.ts.map +1 -1
  356. package/dist/supervisor-run/index.js +58 -450
  357. package/dist/supervisor-run/index.js.map +1 -1
  358. package/dist/{systemone-CpTLAlqb.d.ts → systemone-Bp_Ac0p1.d.ts} +6 -6
  359. package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
  360. package/dist/systemone-protocol.js +1 -1
  361. package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
  362. package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
  363. package/dist/{systemone-review-BZWfrtq1.js → systemone-review-DzdOJ8I6.js} +3 -3
  364. package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
  365. package/dist/systemone.d.ts +5 -5
  366. package/dist/systemone.js +5 -5
  367. package/dist/systemone.js.map +1 -1
  368. package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-BMsblaDh.js} +3 -3
  369. package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
  370. package/dist/{terminal-record-BHocwFw1.js → terminal-record-78g6E4vh.js} +17 -13
  371. package/dist/terminal-record-78g6E4vh.js.map +1 -0
  372. package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
  373. package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
  374. package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-CkjzWmeY.d.ts} +53 -56
  375. package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
  376. package/dist/{tool-waste-D52PqU9Q.js → tool-waste-DLsjS_p-.js} +5 -5
  377. package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
  378. package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-xxwgsuMi.d.ts} +5 -10
  379. package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
  380. package/dist/trace-repair/index.d.ts +15 -212
  381. package/dist/trace-repair/index.d.ts.map +1 -1
  382. package/dist/trace-repair/index.js +865 -987
  383. package/dist/trace-repair/index.js.map +1 -1
  384. package/dist/traces.d.ts +76 -86
  385. package/dist/traces.d.ts.map +1 -1
  386. package/dist/traces.js +13 -13
  387. package/dist/traces.js.map +1 -1
  388. package/dist/trajectory-replay/index.d.ts +3 -117
  389. package/dist/trajectory-replay/index.d.ts.map +1 -1
  390. package/dist/trajectory-replay/index.js +98 -56
  391. package/dist/trajectory-replay/index.js.map +1 -1
  392. package/dist/{tree-rXLW3M_4.js → tree-BGRglkKd.js} +2355 -2414
  393. package/dist/tree-BGRglkKd.js.map +1 -0
  394. package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
  395. package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
  396. package/dist/{types-oKRhQAde.d.ts → types-CW8ewATZ.d.ts} +5 -114
  397. package/dist/types-CW8ewATZ.d.ts.map +1 -0
  398. package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
  399. package/dist/types-CmXY0PQf.d.ts.map +1 -0
  400. package/dist/{types-DBZ7keha.d.ts → types-D-mgRzlh.d.ts} +9 -72
  401. package/dist/types-D-mgRzlh.d.ts.map +1 -0
  402. package/dist/{types-B4C5nFw6.d.ts → types-D8GhJoHk.d.ts} +79 -75
  403. package/dist/types-D8GhJoHk.d.ts.map +1 -0
  404. package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
  405. package/dist/types-DqFMNTgK.js.map +1 -0
  406. package/dist/{types-CfZMIzB_.d.ts → types-DtMbFw6U.d.ts} +3 -3
  407. package/dist/{types-CfZMIzB_.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
  408. package/dist/{verdict-DhPoXApk.js → verdict-BGQmxQ8A.js} +3 -3
  409. package/dist/verdict-BGQmxQ8A.js.map +1 -0
  410. package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
  411. package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
  412. package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-CL_7tePD.js} +2 -2
  413. package/dist/verdict-cache-CL_7tePD.js.map +1 -0
  414. package/dist/wire/index.d.ts +32 -411
  415. package/dist/wire/index.d.ts.map +1 -1
  416. package/dist/wire/index.js +2 -2
  417. package/docs/changes/jev-evidence-review.md +1 -1
  418. package/docs/changes/jev-product-adoption.md +1 -1
  419. package/docs/changes/jev-recorded-evidence.md +1 -1
  420. package/docs/design/mlbenchmarks-book-review.md +1 -1
  421. package/docs/design/mlbenchmarks-review/probes.mts +23 -31
  422. package/docs/design/statistics-decisions.md +17 -17
  423. package/docs/design.md +1 -1
  424. package/docs/evaluation-integrity.md +1 -1
  425. package/docs/experiment.md +1 -1
  426. package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
  427. package/docs/public-api.md +138 -529
  428. package/docs/rollout.md +1 -1
  429. package/docs/trace-repair-admission.md +1 -1
  430. package/docs/wire-protocol.md +1 -1
  431. package/package.json +1 -1
  432. package/dist/active-curriculum-B_Lc9kfO.js +0 -61
  433. package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
  434. package/dist/agent-profile-CIA2Atnt.d.ts.map +0 -1
  435. package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
  436. package/dist/agent-profile-cell-BYc0bxsL.js.map +0 -1
  437. package/dist/agreement-irr-DsTURxdr.js.map +0 -1
  438. package/dist/analyze-runs-3xpby3p2.js.map +0 -1
  439. package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
  440. package/dist/ast-DJ6Bc7ab.js +0 -558
  441. package/dist/ast-DJ6Bc7ab.js.map +0 -1
  442. package/dist/ast-hI-vjW6J.d.ts +0 -457
  443. package/dist/ast-hI-vjW6J.d.ts.map +0 -1
  444. package/dist/baseline-Dm8966du.js +0 -259
  445. package/dist/baseline-Dm8966du.js.map +0 -1
  446. package/dist/benchmark-BywPeAUo.js +0 -562
  447. package/dist/benchmark-BywPeAUo.js.map +0 -1
  448. package/dist/benchmark-command-CByl9dxG.js.map +0 -1
  449. package/dist/campaign-BQ-Jujuj.js.map +0 -1
  450. package/dist/chat-json-call-BrKsyspg.js.map +0 -1
  451. package/dist/claim-BXcZw-jo.js.map +0 -1
  452. package/dist/claude-jsonl-dZir12m6.js +0 -86
  453. package/dist/claude-jsonl-dZir12m6.js.map +0 -1
  454. package/dist/client-BzL9niqi.d.ts.map +0 -1
  455. package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
  456. package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
  457. package/dist/counterfactual-CPaWN99-.d.ts +0 -60
  458. package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
  459. package/dist/counterfactual-DtMWvn7D.js +0 -98
  460. package/dist/counterfactual-DtMWvn7D.js.map +0 -1
  461. package/dist/default-registry-BrWh62zf.d.ts.map +0 -1
  462. package/dist/default-registry-Cn1LQ2Ve.js.map +0 -1
  463. package/dist/define-agent-eval-CEWkX5wZ.d.ts.map +0 -1
  464. package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
  465. package/dist/dspy-rlm-engine-D-KX_e64.js.map +0 -1
  466. package/dist/effect-sizes-DiH8MGOH.js +0 -82
  467. package/dist/effect-sizes-DiH8MGOH.js.map +0 -1
  468. package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
  469. package/dist/engine-C_zNWfM8.d.ts.map +0 -1
  470. package/dist/equal-terms-Blx9Zk5I.js.map +0 -1
  471. package/dist/evaluation-Cn2dRZIz.d.ts.map +0 -1
  472. package/dist/exporters-X4xwla8A.js.map +0 -1
  473. package/dist/external-optimizer-process-nnsY5D_4.js.map +0 -1
  474. package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
  475. package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
  476. package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
  477. package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
  478. package/dist/index-CPJJr98o.d.ts.map +0 -1
  479. package/dist/index-Ch9VxCpK.d.ts.map +0 -1
  480. package/dist/index-DHKEAv3s.d.ts.map +0 -1
  481. package/dist/index-DlH0Y9gI.d.ts.map +0 -1
  482. package/dist/index-NvGBiO8H.d.ts.map +0 -1
  483. package/dist/integrity-BV1ZDf42.js.map +0 -1
  484. package/dist/journal-DvKCvjty.js.map +0 -1
  485. package/dist/judge-gate-B1qYnkpi.js.map +0 -1
  486. package/dist/judge-gate-DNcUrRFA.d.ts.map +0 -1
  487. package/dist/kind-factory-CbZ-6tvu.js.map +0 -1
  488. package/dist/ledger-core-B2hahzrt.js.map +0 -1
  489. package/dist/llm-client-DQveoxSM.js.map +0 -1
  490. package/dist/llm-judge-PXV7Xljl.js.map +0 -1
  491. package/dist/matrix-CHMiebFx.d.ts.map +0 -1
  492. package/dist/matrix-_BV9-Lk5.js.map +0 -1
  493. package/dist/mint-DCGoBYhB.d.ts +0 -155
  494. package/dist/mint-DCGoBYhB.d.ts.map +0 -1
  495. package/dist/normal-DsJZtrJc.js +0 -25
  496. package/dist/normal-DsJZtrJc.js.map +0 -1
  497. package/dist/paired-arms-DzKn0TfY.js.map +0 -1
  498. package/dist/paired-binary-Q36bhn2R.js.map +0 -1
  499. package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
  500. package/dist/paired-tests-jbwPdSz5.js.map +0 -1
  501. package/dist/prime-protocol-CEE4BR9N.d.ts.map +0 -1
  502. package/dist/prime-protocol-DcAK9pC8.js.map +0 -1
  503. package/dist/produced-state-ZCtDBxAE.js.map +0 -1
  504. package/dist/proposal-findings-BZiMTzAF.js.map +0 -1
  505. package/dist/query-B1D1qp2Z.js.map +0 -1
  506. package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
  507. package/dist/redact-Dk1ZdJEm.js.map +0 -1
  508. package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
  509. package/dist/release-confidence-Dbe9AS_X.js.map +0 -1
  510. package/dist/report-command-y8H6RSX8.js.map +0 -1
  511. package/dist/reward-hacking-DHA7c1l8.d.ts.map +0 -1
  512. package/dist/rollout-D9bKKBnZ.js.map +0 -1
  513. package/dist/run-record-D1Hw-Jml.d.ts.map +0 -1
  514. package/dist/run-record-ES07wfz9.js.map +0 -1
  515. package/dist/schema-2g7trUwH.d.ts.map +0 -1
  516. package/dist/schema-BuiujZ-g.js.map +0 -1
  517. package/dist/schema-CSf6qWgZ.js.map +0 -1
  518. package/dist/search-kernel-BztlERLv.js.map +0 -1
  519. package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
  520. package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
  521. package/dist/search-ledger-file-BqOo-bVo.js.map +0 -1
  522. package/dist/search-shipper-er4pK5_Y.js.map +0 -1
  523. package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
  524. package/dist/sequential-D-1PPCmM.js.map +0 -1
  525. package/dist/server-DbcC7xMl.js.map +0 -1
  526. package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
  527. package/dist/statistical-heldout-1Gbmex7v.js.map +0 -1
  528. package/dist/statistical-heldout-BeR4bOdg.d.ts.map +0 -1
  529. package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
  530. package/dist/store-otlp-DtB4Slsr.js.map +0 -1
  531. package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
  532. package/dist/store-tool-spans-fZIigrpP.js.map +0 -1
  533. package/dist/student-t-BQeKJx2N.js +0 -39
  534. package/dist/student-t-BQeKJx2N.js.map +0 -1
  535. package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
  536. package/dist/summary-report-DtdRucM0.js.map +0 -1
  537. package/dist/task-failure-attributes-DngaLdE4.js.map +0 -1
  538. package/dist/terminal-record-BHocwFw1.js.map +0 -1
  539. package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
  540. package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
  541. package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
  542. package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
  543. package/dist/tree-rXLW3M_4.js.map +0 -1
  544. package/dist/types-B4C5nFw6.d.ts.map +0 -1
  545. package/dist/types-BqkVmHfv.d.ts.map +0 -1
  546. package/dist/types-DBZ7keha.d.ts.map +0 -1
  547. package/dist/types-IXD2HJzu.js.map +0 -1
  548. package/dist/types-oKRhQAde.d.ts.map +0 -1
  549. package/dist/usage-receipt-Cx3xNeyB.js +0 -76
  550. package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
  551. package/dist/verdict-DhPoXApk.js.map +0 -1
  552. package/dist/verdict-cache-BQEsVy9n.js.map +0 -1
  553. package/docs/design/self-improvement-evidence-audit.md +0 -263
@@ -1,29 +1,30 @@
1
- import { a as hashCanonical } from "../canonical-BsZVgBhW.js";
2
- import { r as pairedBootstrap } from "../paired-tests-jbwPdSz5.js";
3
- import { t as decidePairedPromotion } from "../paired-promotion-decision-BN6IQTiu.js";
4
- import { a as heldoutSignificance } from "../statistical-heldout-1Gbmex7v.js";
5
- import { n as summarizeExecution, t as analyzeRuns } from "../analyze-runs-3xpby3p2.js";
6
- import { a as parseRunRecordSafe } from "../run-record-DlTZ_lqe.js";
7
- import { K as defaultProductionGate, a as transientDispatchFailure, c as compareOptimizationMethods, h as runImprovementLoop, nt as runEval, rt as runCampaign, t as llmJudge } from "../llm-judge-PXV7Xljl.js";
1
+ import { a as hashCanonical } from "../canonical-Dq2hOQ8L.js";
2
+ import "../run-record-DhEJgoPz.js";
3
+ import { r as summarizeExecution, t as analyzeRuns } from "../analyze-runs-DqPJaJ4X.js";
4
+ import { i as pairedBootstrap } from "../mean-BbA_JCzW.js";
5
+ import { t as decidePairedPromotion } from "../paired-promotion-decision-D0SHb-ZL.js";
6
+ import { a as heldoutSignificance } from "../statistical-heldout-Bksv92Th.js";
7
+ import { B as defaultProductionGate, Q as runCampaign, Z as runEval, i as transientDispatchFailure, o as clamp01, p as runImprovementLoop, t as llmJudge } from "../llm-judge-C8jMTNNG.js";
8
8
  import { i as isModelPriced, n as estimateCost } from "../metrics-BKaPBtIG.js";
9
- import { i as CostLedger } from "../cost-ledger-D2FnJPBF.js";
10
- import { G as campaignSplitDigest } from "../campaign-evidence-OcUciD-N.js";
11
- import { h as mapConcurrentRange } from "../journal-DvKCvjty.js";
12
- import { F as inMemoryCampaignStorage, P as fsCampaignStorage } from "../external-optimizer-subprocess-DcerlZSG.js";
13
- import { o as makeProposalFinding } from "../types-IXD2HJzu.js";
14
- import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-Ch3MWX6k.js";
15
- import { t as buildDefaultAnalystRegistry } from "../default-registry-Cn1LQ2Ve.js";
16
- import { t as extractUsage } from "../extract-usage-BrnzRJ6H.js";
17
- import { t as createChatClient } from "../chat-client-DthPOYGd.js";
9
+ import { t as CostLedger } from "../cost-ledger-D0352Z4d.js";
10
+ import { z as campaignSplitDigest } from "../campaign-evidence-CQYw8_0H.js";
11
+ import { o as mapConcurrentRange } from "../journal-CR4QikYp.js";
12
+ import { M as fsCampaignStorage, N as inMemoryCampaignStorage } from "../external-optimizer-subprocess-XIHBOSKt.js";
13
+ import { a as makeProposalFinding } from "../types-DqFMNTgK.js";
14
+ import { n as isRecord } from "../json-BhdsJaOQ.js";
15
+ import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-BDxNlP3D.js";
16
+ import { t as buildDefaultAnalystRegistry } from "../default-registry-ClYjq3QE.js";
17
+ import { t as extractUsage } from "../extract-usage-BZLU01kR.js";
18
+ import { t as createChatClient } from "../chat-client-ldqxYYTN.js";
18
19
  import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-DUcMj-3D.js";
19
- import { c as heldOutGate, g as runReferenceEquivalenceJudge, h as createReferenceEquivalenceJudge, l as composeGate, m as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, p as REFERENCE_EQUIVALENCE_INPUT_LIMITS, t as skillOptOptimizationMethod, u as externalTextOptimizationMethod } from "../skillopt-optimization-method-CCtzralq.js";
20
- import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-CWW35UWq.js";
21
- import { t as fromOtelSpans } from "../otel-spans-C8YIJWy1.js";
20
+ import { c as composeGate, l as externalTextOptimizationMethod, n as compareOptimizationMethods, r as gepaOptimizationMethod, s as heldOutGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-kyL94-ri.js";
21
+ import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-6A8osGr_.js";
22
+ import { t as fromOtelSpans } from "../otel-spans-CdN909k1.js";
22
23
  import { createHash } from "node:crypto";
23
24
  import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
24
25
  import { createReadStream } from "node:fs";
25
- import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
26
- import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
26
+ import "node:path";
27
+ import "node:fs/promises";
27
28
  import { readSessionInput, summarizeSessionInput } from "@tangle-network/harness-sessions";
28
29
  //#region src/contract/fixed-spend.ts
29
30
  function addFixedSpend(left, right) {
@@ -37,6 +38,189 @@ function addFixedSpend(left, right) {
37
38
  costProvenance: left.costProvenance === "observed" && right.costProvenance === "observed" ? "observed" : "estimated"
38
39
  };
39
40
  }
41
+ /** Sum both arms in receipt order while retaining the observed/estimated provenance. */
42
+ function pairedFixedSpendReceipt(measurements, model, usageOf) {
43
+ const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, usageOf(measurement.baseline)), usageOf(measurement.candidate)), {
44
+ inputTokens: 0,
45
+ outputTokens: 0,
46
+ cachedInputTokens: 0,
47
+ reasoningTokens: 0,
48
+ modelCalls: 0,
49
+ costUsdNanos: 0,
50
+ costProvenance: "observed"
51
+ });
52
+ const costUsd = usage.costUsdNanos / 1e9;
53
+ return {
54
+ model,
55
+ inputTokens: usage.inputTokens,
56
+ outputTokens: usage.outputTokens,
57
+ cachedTokens: usage.cachedInputTokens,
58
+ reasoningTokens: usage.reasoningTokens,
59
+ ...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
60
+ };
61
+ }
62
+ //#endregion
63
+ //#region src/contract/candidate-experiment-evidence.ts
64
+ /** Verify signed candidate execution receipts and read their measured usage. */
65
+ const candidateExecutionEvidenceAdapter = {
66
+ score: (evidence) => evidence.receipt.benchmarkResult.material.score,
67
+ dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
68
+ costUsd: costFromEvidence,
69
+ costProvenance: (evidence) => combinedUsage(evidence).costProvenance,
70
+ latencyMs: latencyFromEvidence,
71
+ completed: completedSuccessfully,
72
+ passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
73
+ };
74
+ function verifyMeasurement(experiment, input, index) {
75
+ const suite = experiment.benchmark.suite;
76
+ const taskIndex = Math.floor(index / suite.reps);
77
+ const repetition = index % suite.reps;
78
+ const task = experiment.benchmark.tasks[taskIndex];
79
+ const seed = suite.seeds[index];
80
+ if (!task || seed === void 0) throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
81
+ const baseline = verifyExecutionEvidence(input.baseline);
82
+ const candidate = verifyExecutionEvidence(input.candidate);
83
+ for (const [arm, evidence] of [["baseline", baseline], ["candidate", candidate]]) {
84
+ const bundle = experiment[arm];
85
+ const materialization = evidence.materializationReceipt;
86
+ const runCell = materialization.executionPlan.material.runCell;
87
+ verifySelfAddressed(runCell, "candidate run cell");
88
+ if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
89
+ verifyTaskOutcome(task, evidence, index, arm);
90
+ }
91
+ const baselinePlan = baseline.materializationReceipt.executionPlan.material;
92
+ const candidatePlan = candidate.materializationReceipt.executionPlan.material;
93
+ if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
94
+ return {
95
+ baseline,
96
+ candidate
97
+ };
98
+ }
99
+ function verifyExecutionEvidence(input) {
100
+ const evidence = candidateExecutionEvidenceSchema.parse(input);
101
+ verifySelfAddressed(evidence, "candidate execution evidence");
102
+ verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
103
+ verifySelfAddressed(evidence.materializationReceipt.profileActivation, "candidate profile activation");
104
+ verifyMaterialAddressed(evidence.materializationReceipt.profileActivation.profilePlan, "candidate profile plan");
105
+ verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
106
+ verifySelfAddressed(evidence.receipt, "candidate run receipt");
107
+ verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
108
+ verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
109
+ verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
110
+ return evidence;
111
+ }
112
+ function verifySelfAddressed(document, label) {
113
+ if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) throw new Error(`${label} digest is invalid`);
114
+ }
115
+ function verifyTaskOutcome(task, evidence, index, arm) {
116
+ const outcome = evidence.receipt.taskOutcome.material.outcome;
117
+ const result = evidence.receipt.benchmarkResult.material;
118
+ const prefix = `candidate experiment measurement ${index} ${arm}`;
119
+ if (result.evidence.sha256 === task.grader.artifact.sha256) throw new Error(`${prefix} reused grader bytes as grading evidence`);
120
+ const usage = combinedUsage(evidence);
121
+ const usageChecks = [
122
+ [
123
+ usage.modelCalls,
124
+ task.limits.maxModelCalls,
125
+ "model calls"
126
+ ],
127
+ [
128
+ usage.inputTokens,
129
+ task.limits.maxInputTokens,
130
+ "input tokens"
131
+ ],
132
+ [
133
+ usage.outputTokens,
134
+ task.limits.maxOutputTokens,
135
+ "output tokens"
136
+ ],
137
+ [
138
+ usage.costUsdNanos,
139
+ Math.round(task.limits.maxCostUsd * 1e9),
140
+ "cost"
141
+ ]
142
+ ];
143
+ for (const [actual, maximum, label] of usageChecks) if (actual > maximum) throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
144
+ if (outcome.kind !== task.outcome.kind) throw new Error(`${prefix} returned an outcome outside the signed task contract`);
145
+ if (task.outcome.kind === "output") {
146
+ if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) throw new Error(`${prefix} changed the signed output contract`);
147
+ return;
148
+ }
149
+ const repository = task.repository;
150
+ if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) throw new Error(`${prefix} did not start from the signed repository state`);
151
+ }
152
+ function verifyStableProfileMaterialization(measurements) {
153
+ for (const arm of ["baseline", "candidate"]) {
154
+ const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
155
+ if (!expected) throw new Error("candidate experiment contains no profile materialization");
156
+ const expectedDigest = canonicalCandidateDigest({
157
+ profilePlanDigest: expected.profilePlan.digest,
158
+ files: expected.files
159
+ });
160
+ for (const [index, measurement] of measurements.entries()) {
161
+ const activation = measurement[arm].materializationReceipt.profileActivation;
162
+ if (canonicalCandidateDigest({
163
+ profilePlanDigest: activation.profilePlan.digest,
164
+ files: activation.files
165
+ }) !== expectedDigest) throw new Error(`candidate experiment measurement ${index} ${arm} materialized a different profile`);
166
+ }
167
+ }
168
+ }
169
+ function completedSuccessfully(evidence) {
170
+ const termination = evidence.receipt.termination;
171
+ return termination.kind === "exit" && termination.exitCode === 0;
172
+ }
173
+ function verifyMaterialAddressed(evidence, label) {
174
+ if (canonicalCandidateDigest(evidence.material) !== evidence.digest) throw new Error(`${label} digest is invalid`);
175
+ }
176
+ function costFromEvidence(evidence) {
177
+ return combinedUsage(evidence).costUsdNanos / 1e9;
178
+ }
179
+ function latencyFromEvidence(evidence) {
180
+ return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
181
+ }
182
+ function combinedUsage(evidence) {
183
+ const candidate = evidence.receipt.modelSettlement.material.usage;
184
+ const grader = evidence.receipt.benchmarkResult.material.grading.usage;
185
+ return addFixedSpend(candidate, grader);
186
+ }
187
+ function candidateMeasurementCostReceipt(measurements, model) {
188
+ return pairedFixedSpendReceipt(measurements, model, combinedUsage);
189
+ }
190
+ //#endregion
191
+ //#region src/contract/measured-comparison-record.ts
192
+ /** Keep published timing and cost accounting identical across experiment domains. */
193
+ function comparisonEvaluation(options, evaluation) {
194
+ return {
195
+ generationsExplored: options.generationsExplored ?? 0,
196
+ preparation: options.preparation,
197
+ measurement: {
198
+ wallDurationMs: options.measurement.wallDurationMs,
199
+ workDurationMs: evaluation.measurementWorkDurationMs,
200
+ cost: evaluation.measurementCost
201
+ },
202
+ total: {
203
+ wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
204
+ cost: evaluation.totalCost
205
+ }
206
+ };
207
+ }
208
+ /** Replay only the original inputs; derived timing and statistics are recomputed. */
209
+ function comparisonRecomputationInput(comparison) {
210
+ return {
211
+ experiment: comparison.experiment,
212
+ measurements: comparison.measurements,
213
+ runId: comparison.provenance.runId,
214
+ ...comparison.candidate ? { candidate: comparison.candidate } : {},
215
+ generationsExplored: comparison.evaluation.generationsExplored,
216
+ preparation: comparison.evaluation.preparation,
217
+ measurement: {
218
+ wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
219
+ cost: comparison.evaluation.measurement.cost
220
+ },
221
+ ...comparison.metadata ? { metadata: comparison.metadata } : {}
222
+ };
223
+ }
40
224
  //#endregion
41
225
  //#region src/contract/concurrent-map.ts
42
226
  /** Map both arms while bounding the actual executions, not the number of pairs. */
@@ -116,104 +300,67 @@ function assertNonNegativeFinite(value, label) {
116
300
  if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be a non-negative finite number`);
117
301
  }
118
302
  //#endregion
119
- //#region src/contract/measured-comparison.ts
120
- /** Content-address one task before any measured execution can see it. */
121
- function sealCandidateBenchmarkTask(material) {
122
- return agentCandidateBenchmarkTaskSchema.parse({
123
- ...material,
124
- digest: canonicalCandidateDigest(material)
125
- });
126
- }
127
- /** Freeze task order, repetitions, and every seed before either arm runs. */
128
- function sealCandidateBenchmarkSuite(options) {
129
- for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
130
- const material = {
131
- kind: "agent-candidate-benchmark-suite",
132
- digestAlgorithm: "rfc8785-sha256",
133
- taskDigests: options.tasks.map((task) => task.digest),
134
- reps: options.reps,
135
- seeds: options.seeds
303
+ //#region src/contract/paired-measurement-input.ts
304
+ function projectPairedMeasurement(measurement, index, adapter) {
305
+ if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) throw new Error(`paired measurement ${index} requires a cell id`);
306
+ return {
307
+ cellId: measurement.cellId,
308
+ baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
309
+ candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
136
310
  };
311
+ }
312
+ function projectRun(run, adapter, label) {
313
+ const suppliedDimensions = adapter.dimensions(run);
314
+ if (!Array.isArray(suppliedDimensions)) throw new Error(`${label} dimensions must be an array`);
315
+ const dimensions = /* @__PURE__ */ new Map();
316
+ for (const dimension of suppliedDimensions) {
317
+ if (typeof dimension.name !== "string" || !dimension.name.trim()) throw new Error(`${label} contains an unnamed dimension`);
318
+ if (dimensions.has(dimension.name)) throw new Error(`${label} repeats dimension '${dimension.name}'`);
319
+ dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
320
+ }
321
+ const completed = adapter.completed(run);
322
+ const passed = adapter.passed(run);
323
+ if (typeof completed !== "boolean" || typeof passed !== "boolean") throw new Error(`${label} completion and pass values must be booleans`);
324
+ const costProvenance = adapter.costProvenance(run);
325
+ if (costProvenance !== "observed" && costProvenance !== "estimated") throw new Error(`${label} cost provenance must be observed or estimated`);
137
326
  return {
138
- suite: agentCandidateBenchmarkSuiteSchema.parse({
139
- ...material,
140
- digest: canonicalCandidateDigest(material)
141
- }),
142
- tasks: options.tasks
327
+ score: finiteMeasurement(adapter.score(run), `${label} score`),
328
+ dimensions,
329
+ costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
330
+ costProvenance,
331
+ latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
332
+ completed,
333
+ passed
143
334
  };
144
335
  }
145
- /** Freeze both complete agent states and their exact held-out work. */
146
- function sealCandidateExperiment(material) {
147
- return verifyCandidateExperiment(agentCandidateExperimentSchema.parse({
148
- ...material,
149
- digest: canonicalCandidateDigest(material)
150
- }));
336
+ function sharedProjectedDimensions(measurements) {
337
+ const expected = [...measurements[0].baseline.dimensions.keys()];
338
+ for (const [index, measurement] of measurements.entries()) for (const [arm, run] of [["baseline", measurement.baseline], ["candidate", measurement.candidate]]) {
339
+ const actual = [...run.dimensions.keys()];
340
+ if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
341
+ }
342
+ return expected;
151
343
  }
152
- function verifyCandidateExperiment(input) {
153
- const experiment = agentCandidateExperimentSchema.parse(input);
154
- verifySelfAddressed(experiment, "candidate experiment");
155
- verifyBundle(experiment.baseline, "baseline bundle");
156
- verifyBundle(experiment.candidate, "candidate bundle");
157
- if (experiment.baseline.digest === experiment.candidate.digest) throw new Error("candidate experiment baseline and candidate bundles are identical");
158
- verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
159
- return experiment;
344
+ function dimensionScore(run, name) {
345
+ const value = run.dimensions.get(name);
346
+ if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
347
+ return value;
160
348
  }
161
- /** Execute each signed cell for both arms. The callback is Runtime's one executor. */
162
- async function runCandidateExperiment(options) {
163
- const experiment = verifyCandidateExperiment(options.experiment);
164
- const { suite, tasks } = experiment.benchmark;
165
- const run = await runPaidPairedMeasurement({
166
- count: suite.taskDigests.length * suite.reps,
167
- maxConcurrency: options.maxConcurrency ?? 2,
168
- label: "candidate experiment",
169
- budgetUsd: experiment.policy.budgetUsd,
170
- ...options.costLedger ? { costLedger: options.costLedger } : {},
171
- maximumCostUsd: candidateMeasurementMaximumCostUsd(experiment),
172
- call: {
173
- callId: `candidate-measurement:${experiment.digest}`,
174
- channel: "measurement",
175
- phase: "heldout",
176
- actor: experiment.digest,
177
- model: candidateMeasurementModel(experiment),
178
- tags: { experimentDigest: experiment.digest }
179
- },
180
- ...options.signal ? { signal: options.signal } : {},
181
- async execute(index, arm, signal) {
182
- const taskIndex = Math.floor(index / suite.reps);
183
- const repetition = index % suite.reps;
184
- const task = tasks[taskIndex];
185
- const seed = suite.seeds[index];
186
- if (!task || seed === void 0) throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
187
- const benchmarkCell = {
188
- suiteDigest: suite.digest,
189
- taskIndex,
190
- repetition
191
- };
192
- return verifyExecutionEvidence(await options.execute({
193
- experiment,
194
- arm,
195
- bundle: experiment[arm],
196
- task,
197
- benchmarkCell,
198
- seed,
199
- signal
200
- }));
201
- },
202
- receipt(measurements) {
203
- return candidateMeasurementCostReceipt(measurements, candidateMeasurementModel(experiment));
204
- }
205
- });
206
- return {
207
- measurements: run.measurements.map((measurement, index) => verifyMeasurement(experiment, measurement, index)),
208
- measurement: {
209
- wallDurationMs: run.wallDurationMs,
210
- cost: {
211
- usd: run.cost.usd,
212
- provenance: run.cost.kind
213
- }
214
- }
215
- };
349
+ function finiteMeasurement(value, label) {
350
+ if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
351
+ return value;
352
+ }
353
+ function nonNegativeMeasurement(value, label) {
354
+ if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
355
+ return value;
356
+ }
357
+ function assertKnownCost(cost, label) {
358
+ if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new Error(`${label} must be a non-negative number`);
359
+ if (cost.provenance !== "observed" && cost.provenance !== "estimated") throw new Error(`${label} provenance must be observed or estimated`);
216
360
  }
361
+ //#endregion
362
+ //#region src/contract/paired-measurement-evaluation.ts
363
+ /** Shared significance, regression, precision, and budget decisions on paired receipts. */
217
364
  /**
218
365
  * Calculate the shared paired decision from any complete receipt shape.
219
366
  *
@@ -449,6 +596,164 @@ function significanceReason(significance, deltaThreshold) {
449
596
  if (decision.exactTestVetoes && decision.mcnemar !== null) return `McNemar's exact test refuses at threshold ${deltaThreshold}: b=${decision.mcnemar.b}, c=${decision.mcnemar.c}, p=${decision.mcnemar.pValue}`;
450
597
  return `paired ${decision.statistic} interval lower bound ${decision.low} did not clear ${deltaThreshold}.${decision.methodDetail}`;
451
598
  }
599
+ function measuredEstimate(baseline, candidate, options) {
600
+ const bootstrap = pairedBootstrap(baseline, candidate, {
601
+ confidence: options.confidence,
602
+ resamples: options.resamples,
603
+ statistic: "mean",
604
+ seed: options.seed
605
+ });
606
+ const baselineMean = mean(baseline);
607
+ const candidateMean = mean(candidate);
608
+ const delta = candidateMean - baselineMean;
609
+ return {
610
+ baseline: baselineMean,
611
+ candidate: candidateMean,
612
+ delta,
613
+ confidenceInterval: {
614
+ level: bootstrap.confidence,
615
+ lower: Math.min(bootstrap.low, delta),
616
+ upper: Math.max(bootstrap.high, delta),
617
+ method: "paired-bootstrap",
618
+ statistic: "mean",
619
+ resamples: bootstrap.resamples
620
+ },
621
+ n: bootstrap.n
622
+ };
623
+ }
624
+ function observedPairedPrecision(options) {
625
+ const n = options.baselineScores.length;
626
+ const lowerMargin = Math.max(0, options.delta - options.lowerBound);
627
+ const minimumDetectableDelta = options.deltaThreshold + lowerMargin;
628
+ const scaleAssumed = [...options.baselineScores, ...options.candidateScores].every((score) => score >= -.001 && score <= 1.001);
629
+ const headroom = Math.max(0, 1 - mean(options.baselineScores));
630
+ const enoughRuns = n >= options.minProductiveRuns;
631
+ const attainable = !scaleAssumed || minimumDetectableDelta <= headroom;
632
+ const sufficient = enoughRuns && attainable;
633
+ const scorerNote = options.sharedScorerChannel ? " The same scorer evaluated both arms, so scorer bias is not measured." : "";
634
+ if (!enoughRuns) return {
635
+ sufficient,
636
+ minimumDetectableDelta,
637
+ scaleAssumed,
638
+ reason: `only ${n} paired runs; ${options.minProductiveRuns} required.${scorerNote}`
639
+ };
640
+ if (!attainable) return {
641
+ sufficient,
642
+ minimumDetectableDelta,
643
+ scaleAssumed,
644
+ reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)}, above the remaining score headroom ${headroom.toFixed(3)}.${scorerNote}`
645
+ };
646
+ return {
647
+ sufficient,
648
+ minimumDetectableDelta,
649
+ scaleAssumed,
650
+ reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)} at ${options.confidence.toFixed(2)} confidence from ${n} runs.${scorerNote}`
651
+ };
652
+ }
653
+ function mean(values) {
654
+ if (values.length === 0) throw new Error("candidate experiment requires measured values");
655
+ return values.reduce((sum, value) => sum + value, 0) / values.length;
656
+ }
657
+ //#endregion
658
+ //#region src/contract/candidate-experiment.ts
659
+ /** Seal held-out experiments, execute their signed cells, and publish verified comparisons. */
660
+ /** Content-address one task before any measured execution can see it. */
661
+ function sealCandidateBenchmarkTask(material) {
662
+ return agentCandidateBenchmarkTaskSchema.parse({
663
+ ...material,
664
+ digest: canonicalCandidateDigest(material)
665
+ });
666
+ }
667
+ /** Freeze task order, repetitions, and every seed before either arm runs. */
668
+ function sealCandidateBenchmarkSuite(options) {
669
+ for (const task of options.tasks) verifyCandidateBenchmarkTask(task);
670
+ const material = {
671
+ kind: "agent-candidate-benchmark-suite",
672
+ digestAlgorithm: "rfc8785-sha256",
673
+ taskDigests: options.tasks.map((task) => task.digest),
674
+ reps: options.reps,
675
+ seeds: options.seeds
676
+ };
677
+ return {
678
+ suite: agentCandidateBenchmarkSuiteSchema.parse({
679
+ ...material,
680
+ digest: canonicalCandidateDigest(material)
681
+ }),
682
+ tasks: options.tasks
683
+ };
684
+ }
685
+ /** Freeze both complete agent states and their exact held-out work. */
686
+ function sealCandidateExperiment(material) {
687
+ return verifyCandidateExperiment(agentCandidateExperimentSchema.parse({
688
+ ...material,
689
+ digest: canonicalCandidateDigest(material)
690
+ }));
691
+ }
692
+ function verifyCandidateExperiment(input) {
693
+ const experiment = agentCandidateExperimentSchema.parse(input);
694
+ verifySelfAddressed(experiment, "candidate experiment");
695
+ verifyBundle(experiment.baseline, "baseline bundle");
696
+ verifyBundle(experiment.candidate, "candidate bundle");
697
+ if (experiment.baseline.digest === experiment.candidate.digest) throw new Error("candidate experiment baseline and candidate bundles are identical");
698
+ verifyCandidateBenchmarkSuiteInputs(experiment.benchmark);
699
+ return experiment;
700
+ }
701
+ /** Execute each signed cell for both arms. The callback is Runtime's one executor. */
702
+ async function runCandidateExperiment(options) {
703
+ const experiment = verifyCandidateExperiment(options.experiment);
704
+ const { suite, tasks } = experiment.benchmark;
705
+ const run = await runPaidPairedMeasurement({
706
+ count: suite.taskDigests.length * suite.reps,
707
+ maxConcurrency: options.maxConcurrency ?? 2,
708
+ label: "candidate experiment",
709
+ budgetUsd: experiment.policy.budgetUsd,
710
+ ...options.costLedger ? { costLedger: options.costLedger } : {},
711
+ maximumCostUsd: candidateMeasurementMaximumCostUsd(experiment),
712
+ call: {
713
+ callId: `candidate-measurement:${experiment.digest}`,
714
+ channel: "measurement",
715
+ phase: "heldout",
716
+ actor: experiment.digest,
717
+ model: candidateMeasurementModel(experiment),
718
+ tags: { experimentDigest: experiment.digest }
719
+ },
720
+ ...options.signal ? { signal: options.signal } : {},
721
+ async execute(index, arm, signal) {
722
+ const taskIndex = Math.floor(index / suite.reps);
723
+ const repetition = index % suite.reps;
724
+ const task = tasks[taskIndex];
725
+ const seed = suite.seeds[index];
726
+ if (!task || seed === void 0) throw new Error(`candidate experiment cell ${index} has no signed task or seed`);
727
+ const benchmarkCell = {
728
+ suiteDigest: suite.digest,
729
+ taskIndex,
730
+ repetition
731
+ };
732
+ return verifyExecutionEvidence(await options.execute({
733
+ experiment,
734
+ arm,
735
+ bundle: experiment[arm],
736
+ task,
737
+ benchmarkCell,
738
+ seed,
739
+ signal
740
+ }));
741
+ },
742
+ receipt(measurements) {
743
+ return candidateMeasurementCostReceipt(measurements, candidateMeasurementModel(experiment));
744
+ }
745
+ });
746
+ return {
747
+ measurements: run.measurements.map((measurement, index) => verifyMeasurement(experiment, measurement, index)),
748
+ measurement: {
749
+ wallDurationMs: run.wallDurationMs,
750
+ cost: {
751
+ usd: run.cost.usd,
752
+ provenance: run.cost.kind
753
+ }
754
+ }
755
+ };
756
+ }
452
757
  /** Build the only publishable comparison: paired statistics over Runtime receipts. */
453
758
  function measuredComparisonFromCandidateExperiment(options) {
454
759
  const experiment = verifyCandidateExperiment(options.experiment);
@@ -489,19 +794,7 @@ function measuredComparisonFromCandidateExperiment(options) {
489
794
  candidateContentHash: experiment.candidate.digest
490
795
  },
491
796
  diff,
492
- evaluation: {
493
- generationsExplored: options.generationsExplored ?? 0,
494
- preparation: options.preparation,
495
- measurement: {
496
- wallDurationMs: options.measurement.wallDurationMs,
497
- workDurationMs: evaluation.measurementWorkDurationMs,
498
- cost: evaluation.measurementCost
499
- },
500
- total: {
501
- wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
502
- cost: evaluation.totalCost
503
- }
504
- },
797
+ evaluation: comparisonEvaluation(options, evaluation),
505
798
  ...options.metadata ? { metadata: options.metadata } : {}
506
799
  });
507
800
  const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
@@ -519,19 +812,7 @@ function measuredComparisonFromCandidateExperiment(options) {
519
812
  /** Recompute every statistic and decision from the signed experiment receipts. */
520
813
  function verifyCandidateExperimentComparison(input) {
521
814
  const comparison = agentImprovementMeasuredComparisonSchema.parse(input);
522
- const recomputed = measuredComparisonFromCandidateExperiment({
523
- experiment: comparison.experiment,
524
- measurements: comparison.measurements,
525
- runId: comparison.provenance.runId,
526
- ...comparison.candidate ? { candidate: comparison.candidate } : {},
527
- generationsExplored: comparison.evaluation.generationsExplored,
528
- preparation: comparison.evaluation.preparation,
529
- measurement: {
530
- wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
531
- cost: comparison.evaluation.measurement.cost
532
- },
533
- ...comparison.metadata ? { metadata: comparison.metadata } : {}
534
- });
815
+ const recomputed = measuredComparisonFromCandidateExperiment(comparisonRecomputationInput(comparison));
535
816
  if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("candidate experiment comparison does not match its Runtime receipts");
536
817
  return comparison;
537
818
  }
@@ -544,34 +825,10 @@ function candidateMeasurementModel(experiment) {
544
825
  const models = [...new Set(experiment.benchmark.tasks.map((task) => task.model.model))];
545
826
  return models.length === 1 ? models[0] : "multiple-models";
546
827
  }
547
- function candidateMeasurementCostReceipt(measurements, model) {
548
- const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, combinedUsage(measurement.baseline)), combinedUsage(measurement.candidate)), {
549
- inputTokens: 0,
550
- outputTokens: 0,
551
- cachedInputTokens: 0,
552
- reasoningTokens: 0,
553
- modelCalls: 0,
554
- costUsdNanos: 0,
555
- costProvenance: "observed"
556
- });
557
- const costUsd = usage.costUsdNanos / 1e9;
558
- return {
559
- model,
560
- inputTokens: usage.inputTokens,
561
- outputTokens: usage.outputTokens,
562
- cachedTokens: usage.cachedInputTokens,
563
- reasoningTokens: usage.reasoningTokens,
564
- ...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
565
- };
566
- }
567
828
  function assertPhaseAccounting(accounting, label) {
568
829
  if (!Number.isFinite(accounting.wallDurationMs) || accounting.wallDurationMs < 0) throw new Error(`${label} wall duration must be a non-negative number`);
569
830
  assertKnownCost(accounting.cost, `${label} cost`);
570
831
  }
571
- function assertKnownCost(cost, label) {
572
- if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new Error(`${label} must be a non-negative number`);
573
- if (cost.provenance !== "observed" && cost.provenance !== "estimated") throw new Error(`${label} provenance must be observed or estimated`);
574
- }
575
832
  function deriveCandidateBundleDiff(experiment) {
576
833
  const changed = [
577
834
  "profile",
@@ -625,235 +882,6 @@ function verifyBundle(input, label) {
625
882
  verifySelfAddressed(bundle, label);
626
883
  return bundle;
627
884
  }
628
- function verifyMeasurement(experiment, input, index) {
629
- const suite = experiment.benchmark.suite;
630
- const taskIndex = Math.floor(index / suite.reps);
631
- const repetition = index % suite.reps;
632
- const task = experiment.benchmark.tasks[taskIndex];
633
- const seed = suite.seeds[index];
634
- if (!task || seed === void 0) throw new Error(`candidate experiment measurement ${index} is outside the signed suite`);
635
- const baseline = verifyExecutionEvidence(input.baseline);
636
- const candidate = verifyExecutionEvidence(input.candidate);
637
- for (const [arm, evidence] of [["baseline", baseline], ["candidate", candidate]]) {
638
- const bundle = experiment[arm];
639
- const materialization = evidence.materializationReceipt;
640
- const runCell = materialization.executionPlan.material.runCell;
641
- verifySelfAddressed(runCell, "candidate run cell");
642
- if (runCell.experimentDigest !== experiment.digest || runCell.arm !== arm || runCell.bundleDigest !== bundle.digest || runCell.suiteDigest !== suite.digest || runCell.taskDigest !== task.digest || runCell.taskIndex !== taskIndex || runCell.repetition !== repetition || runCell.seed !== seed || runCell.attempt > task.attempt.maxAttempts || materialization.bundleDigest !== bundle.digest || materialization.benchmark.suite.digest !== suite.digest || materialization.benchmark.task.digest !== task.digest || materialization.codeKind !== bundle.code.kind || materialization.profileActivation.profilePlan.material.sourceProfileDigest !== canonicalCandidateDigest(bundle.profile) || evidence.receipt.runCellDigest !== runCell.digest || JSON.stringify(materialization.resolvedModel) !== JSON.stringify(task.model)) throw new Error(`candidate experiment measurement ${index} substituted its ${arm} arm`);
643
- verifyTaskOutcome(task, evidence, index, arm);
644
- }
645
- const baselinePlan = baseline.materializationReceipt.executionPlan.material;
646
- const candidatePlan = candidate.materializationReceipt.executionPlan.material;
647
- if (baselinePlan.executionId === candidatePlan.executionId || baselinePlan.runCell.digest === candidatePlan.runCell.digest || baseline.materializationReceipt.digest === candidate.materializationReceipt.digest || baseline.receipt.digest === candidate.receipt.digest || baseline.digest === candidate.digest) throw new Error(`candidate experiment measurement ${index} reused one execution across arms`);
648
- return {
649
- baseline,
650
- candidate
651
- };
652
- }
653
- function verifyExecutionEvidence(input) {
654
- const evidence = candidateExecutionEvidenceSchema.parse(input);
655
- verifySelfAddressed(evidence, "candidate execution evidence");
656
- verifySelfAddressed(evidence.materializationReceipt, "candidate materialization receipt");
657
- verifySelfAddressed(evidence.materializationReceipt.profileActivation, "candidate profile activation");
658
- verifyMaterialAddressed(evidence.materializationReceipt.profileActivation.profilePlan, "candidate profile plan");
659
- verifyMaterialAddressed(evidence.materializationReceipt.executionPlan, "candidate execution plan");
660
- verifySelfAddressed(evidence.receipt, "candidate run receipt");
661
- verifyMaterialAddressed(evidence.receipt.modelSettlement, "candidate model settlement");
662
- verifyMaterialAddressed(evidence.receipt.taskOutcome, "candidate task outcome");
663
- verifyMaterialAddressed(evidence.receipt.benchmarkResult, "candidate benchmark result");
664
- return evidence;
665
- }
666
- function verifySelfAddressed(document, label) {
667
- if (canonicalCandidateDigest(omitTopLevelDigest(document)) !== document.digest) throw new Error(`${label} digest is invalid`);
668
- }
669
- function verifyTaskOutcome(task, evidence, index, arm) {
670
- const outcome = evidence.receipt.taskOutcome.material.outcome;
671
- const result = evidence.receipt.benchmarkResult.material;
672
- const prefix = `candidate experiment measurement ${index} ${arm}`;
673
- if (result.evidence.sha256 === task.grader.artifact.sha256) throw new Error(`${prefix} reused grader bytes as grading evidence`);
674
- const usage = combinedUsage(evidence);
675
- const usageChecks = [
676
- [
677
- usage.modelCalls,
678
- task.limits.maxModelCalls,
679
- "model calls"
680
- ],
681
- [
682
- usage.inputTokens,
683
- task.limits.maxInputTokens,
684
- "input tokens"
685
- ],
686
- [
687
- usage.outputTokens,
688
- task.limits.maxOutputTokens,
689
- "output tokens"
690
- ],
691
- [
692
- usage.costUsdNanos,
693
- Math.round(task.limits.maxCostUsd * 1e9),
694
- "cost"
695
- ]
696
- ];
697
- for (const [actual, maximum, label] of usageChecks) if (actual > maximum) throw new Error(`${prefix} ${label} ${actual} exceeds the signed limit ${maximum}`);
698
- if (outcome.kind !== task.outcome.kind) throw new Error(`${prefix} returned an outcome outside the signed task contract`);
699
- if (task.outcome.kind === "output") {
700
- if (outcome.kind !== "output" || outcome.spec.mediaType !== task.outcome.mediaType || outcome.spec.maxBytes !== task.outcome.maxBytes) throw new Error(`${prefix} changed the signed output contract`);
701
- return;
702
- }
703
- const repository = task.repository;
704
- if (outcome.kind !== "workspace" || repository === void 0 || outcome.baseRepository.identity !== repository.identity || outcome.baseRepository.rootIdentity !== repository.rootIdentity || outcome.baseRepository.commit !== repository.baseCommit || outcome.baseRepository.tree !== repository.baseTree) throw new Error(`${prefix} did not start from the signed repository state`);
705
- }
706
- function verifyStableProfileMaterialization(measurements) {
707
- for (const arm of ["baseline", "candidate"]) {
708
- const expected = measurements[0]?.[arm].materializationReceipt.profileActivation;
709
- if (!expected) throw new Error("candidate experiment contains no profile materialization");
710
- const expectedDigest = canonicalCandidateDigest({
711
- profilePlanDigest: expected.profilePlan.digest,
712
- files: expected.files
713
- });
714
- for (const [index, measurement] of measurements.entries()) {
715
- const activation = measurement[arm].materializationReceipt.profileActivation;
716
- if (canonicalCandidateDigest({
717
- profilePlanDigest: activation.profilePlan.digest,
718
- files: activation.files
719
- }) !== expectedDigest) throw new Error(`candidate experiment measurement ${index} ${arm} materialized a different profile`);
720
- }
721
- }
722
- }
723
- function completedSuccessfully(evidence) {
724
- const termination = evidence.receipt.termination;
725
- return termination.kind === "exit" && termination.exitCode === 0;
726
- }
727
- function verifyMaterialAddressed(evidence, label) {
728
- if (canonicalCandidateDigest(evidence.material) !== evidence.digest) throw new Error(`${label} digest is invalid`);
729
- }
730
- function projectPairedMeasurement(measurement, index, adapter) {
731
- if (typeof measurement.cellId !== "string" || !measurement.cellId.trim()) throw new Error(`paired measurement ${index} requires a cell id`);
732
- return {
733
- cellId: measurement.cellId,
734
- baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`),
735
- candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`)
736
- };
737
- }
738
- function projectRun(run, adapter, label) {
739
- const suppliedDimensions = adapter.dimensions(run);
740
- if (!Array.isArray(suppliedDimensions)) throw new Error(`${label} dimensions must be an array`);
741
- const dimensions = /* @__PURE__ */ new Map();
742
- for (const dimension of suppliedDimensions) {
743
- if (typeof dimension.name !== "string" || !dimension.name.trim()) throw new Error(`${label} contains an unnamed dimension`);
744
- if (dimensions.has(dimension.name)) throw new Error(`${label} repeats dimension '${dimension.name}'`);
745
- dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`));
746
- }
747
- const completed = adapter.completed(run);
748
- const passed = adapter.passed(run);
749
- if (typeof completed !== "boolean" || typeof passed !== "boolean") throw new Error(`${label} completion and pass values must be booleans`);
750
- const costProvenance = adapter.costProvenance(run);
751
- if (costProvenance !== "observed" && costProvenance !== "estimated") throw new Error(`${label} cost provenance must be observed or estimated`);
752
- return {
753
- score: finiteMeasurement(adapter.score(run), `${label} score`),
754
- dimensions,
755
- costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`),
756
- costProvenance,
757
- latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`),
758
- completed,
759
- passed
760
- };
761
- }
762
- function sharedProjectedDimensions(measurements) {
763
- const expected = [...measurements[0].baseline.dimensions.keys()];
764
- for (const [index, measurement] of measurements.entries()) for (const [arm, run] of [["baseline", measurement.baseline], ["candidate", measurement.candidate]]) {
765
- const actual = [...run.dimensions.keys()];
766
- if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`);
767
- }
768
- return expected;
769
- }
770
- function dimensionScore(run, name) {
771
- const value = run.dimensions.get(name);
772
- if (value === void 0) throw new Error(`paired measurement is missing dimension '${name}'`);
773
- return value;
774
- }
775
- function measuredEstimate(baseline, candidate, options) {
776
- const bootstrap = pairedBootstrap(baseline, candidate, {
777
- confidence: options.confidence,
778
- resamples: options.resamples,
779
- statistic: "mean",
780
- seed: options.seed
781
- });
782
- const baselineMean = mean(baseline);
783
- const candidateMean = mean(candidate);
784
- const delta = candidateMean - baselineMean;
785
- return {
786
- baseline: baselineMean,
787
- candidate: candidateMean,
788
- delta,
789
- confidenceInterval: {
790
- level: bootstrap.confidence,
791
- lower: Math.min(bootstrap.low, delta),
792
- upper: Math.max(bootstrap.high, delta),
793
- method: "paired-bootstrap",
794
- statistic: "mean",
795
- resamples: bootstrap.resamples
796
- },
797
- n: bootstrap.n
798
- };
799
- }
800
- function observedPairedPrecision(options) {
801
- const n = options.baselineScores.length;
802
- const lowerMargin = Math.max(0, options.delta - options.lowerBound);
803
- const minimumDetectableDelta = options.deltaThreshold + lowerMargin;
804
- const scaleAssumed = [...options.baselineScores, ...options.candidateScores].every((score) => score >= -.001 && score <= 1.001);
805
- const headroom = Math.max(0, 1 - mean(options.baselineScores));
806
- const enoughRuns = n >= options.minProductiveRuns;
807
- const attainable = !scaleAssumed || minimumDetectableDelta <= headroom;
808
- const sufficient = enoughRuns && attainable;
809
- const scorerNote = options.sharedScorerChannel ? " The same scorer evaluated both arms, so scorer bias is not measured." : "";
810
- if (!enoughRuns) return {
811
- sufficient,
812
- minimumDetectableDelta,
813
- scaleAssumed,
814
- reason: `only ${n} paired runs; ${options.minProductiveRuns} required.${scorerNote}`
815
- };
816
- if (!attainable) return {
817
- sufficient,
818
- minimumDetectableDelta,
819
- scaleAssumed,
820
- reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)}, above the remaining score headroom ${headroom.toFixed(3)}.${scorerNote}`
821
- };
822
- return {
823
- sufficient,
824
- minimumDetectableDelta,
825
- scaleAssumed,
826
- reason: `observed paired uncertainty gives a minimum detectable delta of ${minimumDetectableDelta.toFixed(3)} at ${options.confidence.toFixed(2)} confidence from ${n} runs.${scorerNote}`
827
- };
828
- }
829
- function finiteMeasurement(value, label) {
830
- if (!Number.isFinite(value)) throw new Error(`${label} must be finite`);
831
- return value;
832
- }
833
- function nonNegativeMeasurement(value, label) {
834
- if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`);
835
- return value;
836
- }
837
- const candidateExecutionEvidenceAdapter = {
838
- score: (evidence) => evidence.receipt.benchmarkResult.material.score,
839
- dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions,
840
- costUsd: costFromEvidence,
841
- costProvenance: (evidence) => combinedUsage(evidence).costProvenance,
842
- latencyMs: latencyFromEvidence,
843
- completed: completedSuccessfully,
844
- passed: (evidence) => evidence.receipt.benchmarkResult.material.passed
845
- };
846
- function costFromEvidence(evidence) {
847
- return combinedUsage(evidence).costUsdNanos / 1e9;
848
- }
849
- function latencyFromEvidence(evidence) {
850
- return evidence.receipt.timing.durationMs + evidence.receipt.benchmarkResult.material.grading.timing.durationMs;
851
- }
852
- function combinedUsage(evidence) {
853
- const candidate = evidence.receipt.modelSettlement.material.usage;
854
- const grader = evidence.receipt.benchmarkResult.material.grading.usage;
855
- return addFixedSpend(candidate, grader);
856
- }
857
885
  function cellIds(experiment) {
858
886
  const { suite, tasks } = experiment.benchmark;
859
887
  return suite.seeds.map((_, index) => {
@@ -862,10 +890,6 @@ function cellIds(experiment) {
862
890
  return `${tasks[taskIndex]?.scenario.id ?? taskIndex}:${repetition}`;
863
891
  });
864
892
  }
865
- function mean(values) {
866
- if (values.length === 0) throw new Error("candidate experiment requires measured values");
867
- return values.reduce((sum, value) => sum + value, 0) / values.length;
868
- }
869
893
  //#endregion
870
894
  //#region src/contract/profile-measured-comparison.ts
871
895
  /** Content-address one held-out profile task before either state can execute it. */
@@ -991,19 +1015,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
991
1015
  candidateContentHash: experiment.candidate.stateDigest
992
1016
  },
993
1017
  diff: canonicalCandidateJson(experiment.change),
994
- evaluation: {
995
- generationsExplored: options.generationsExplored ?? 0,
996
- preparation: options.preparation,
997
- measurement: {
998
- wallDurationMs: options.measurement.wallDurationMs,
999
- workDurationMs: evaluation.measurementWorkDurationMs,
1000
- cost: evaluation.measurementCost
1001
- },
1002
- total: {
1003
- wallDurationMs: options.preparation.wallDurationMs + options.measurement.wallDurationMs,
1004
- cost: evaluation.totalCost
1005
- }
1006
- },
1018
+ evaluation: comparisonEvaluation(options, evaluation),
1007
1019
  ...options.metadata ? { metadata: options.metadata } : {}
1008
1020
  });
1009
1021
  const { recordDigest: _recordDigest, ...provenance } = provisional.provenance;
@@ -1021,19 +1033,7 @@ function measuredComparisonFromAgentProfileImprovementExperiment(options) {
1021
1033
  /** Recompute a profile comparison from the exact sealed experiment and receipts. */
1022
1034
  function verifyAgentProfileImprovementExperimentComparison(input) {
1023
1035
  const comparison = agentProfileImprovementMeasuredComparisonSchema.parse(input);
1024
- const recomputed = measuredComparisonFromAgentProfileImprovementExperiment({
1025
- experiment: comparison.experiment,
1026
- measurements: comparison.measurements,
1027
- runId: comparison.provenance.runId,
1028
- ...comparison.candidate ? { candidate: comparison.candidate } : {},
1029
- generationsExplored: comparison.evaluation.generationsExplored,
1030
- preparation: comparison.evaluation.preparation,
1031
- measurement: {
1032
- wallDurationMs: comparison.evaluation.measurement.wallDurationMs,
1033
- cost: comparison.evaluation.measurement.cost
1034
- },
1035
- ...comparison.metadata ? { metadata: comparison.metadata } : {}
1036
- });
1036
+ const recomputed = measuredComparisonFromAgentProfileImprovementExperiment(comparisonRecomputationInput(comparison));
1037
1037
  if (canonicalCandidateDigest(recomputed) !== canonicalCandidateDigest(comparison)) throw new Error("profile improvement comparison does not match its Runtime receipts");
1038
1038
  return comparison;
1039
1039
  }
@@ -1048,24 +1048,7 @@ function profileMeasurementModel(experiment) {
1048
1048
  return models.length === 1 ? models[0] : "multiple-models";
1049
1049
  }
1050
1050
  function profileMeasurementCostReceipt(measurements, model) {
1051
- const usage = measurements.reduce((sum, measurement) => addFixedSpend(addFixedSpend(sum, combinedProfileUsage(measurement.baseline)), combinedProfileUsage(measurement.candidate)), {
1052
- inputTokens: 0,
1053
- outputTokens: 0,
1054
- cachedInputTokens: 0,
1055
- reasoningTokens: 0,
1056
- modelCalls: 0,
1057
- costUsdNanos: 0,
1058
- costProvenance: "observed"
1059
- });
1060
- const costUsd = usage.costUsdNanos / 1e9;
1061
- return {
1062
- model,
1063
- inputTokens: usage.inputTokens,
1064
- outputTokens: usage.outputTokens,
1065
- cachedTokens: usage.cachedInputTokens,
1066
- reasoningTokens: usage.reasoningTokens,
1067
- ...usage.costProvenance === "observed" ? { actualCostUsd: costUsd } : { estimatedCostUsd: costUsd }
1068
- };
1051
+ return pairedFixedSpendReceipt(measurements, model, combinedProfileUsage);
1069
1052
  }
1070
1053
  function combinedProfileUsage(receipt) {
1071
1054
  return addFixedSpend(receipt.usage, receipt.grading.usage);
@@ -1146,186 +1129,10 @@ const profileReceiptAdapter = {
1146
1129
  passed: (receipt) => receipt.grading.passed
1147
1130
  };
1148
1131
  //#endregion
1149
- //#region src/contract/intake/run-record-dir.ts
1150
- /**
1151
- * # `intake/run-record-dir` — load a directory or file of `RunRecord`s.
1152
- *
1153
- * The on-disk counterpart to the in-memory intake adapters: point it at a
1154
- * single `.json` (array) / `.jsonl` (one record per line) file or at a
1155
- * directory of such files, and it returns the substrate-canonical
1156
- * `RunRecord[]` ready for `analyzeRuns({ runs })`.
1157
- *
1158
- * Validation is at the boundary: each parsed object goes through
1159
- * `parseRunRecordSafe`. By default an invalid record fails loud with its
1160
- * file + index; pass `onInvalid: 'collect'` to keep the valid records and
1161
- * receive the rejects as structured diagnostics instead.
1162
- */
1163
- const ANALYSIS_ARTIFACT$1 = "analysis.json";
1164
- function defaultInclude(fileName) {
1165
- if (fileName === ANALYSIS_ARTIFACT$1) return false;
1166
- return fileName.endsWith(".json") || fileName.endsWith(".jsonl");
1167
- }
1168
- /**
1169
- * Resolve a file or directory path into validated `RunRecord[]`.
1170
- *
1171
- * A `.json` file must parse to a top-level array; a `.jsonl` file is one
1172
- * record per non-empty line. Directories are read shallowly by default
1173
- * (set `recursive` to descend); the `analysis.json` output artifact is
1174
- * always excluded.
1175
- */
1176
- async function fromRunRecordDir(path, options = {}) {
1177
- const onInvalid = options.onInvalid ?? "throw";
1178
- const include = options.include ?? defaultInclude;
1179
- const filePaths = (await stat(path)).isDirectory() ? await collectFiles(path, include, options.recursive ?? false) : [path];
1180
- const runs = [];
1181
- const rejected = [];
1182
- for (const file of filePaths) {
1183
- const raw = await parseRecordFile(file);
1184
- for (const { index, value } of raw) {
1185
- const parsed = parseRunRecordSafe(value);
1186
- if (parsed.ok) {
1187
- runs.push(parsed.value);
1188
- continue;
1189
- }
1190
- const rejection = {
1191
- file,
1192
- index,
1193
- reason: parsed.error.message
1194
- };
1195
- if (onInvalid === "throw") throw new Error(`fromRunRecordDir: invalid RunRecord in '${file}' at index ${index}: ${parsed.error.message}`);
1196
- rejected.push(rejection);
1197
- }
1198
- }
1199
- return {
1200
- runs,
1201
- rejected,
1202
- files: filePaths
1203
- };
1204
- }
1205
- /** Read a single `.json` / `.jsonl` file into `{ index, value }` pairs. A
1206
- * malformed JSONL line throws with its line number rather than being skipped —
1207
- * silent line-dropping is how corpora quietly shrink. */
1208
- async function parseRecordFile(file) {
1209
- const trimmed = (await readFile(file, "utf8")).trim();
1210
- if (trimmed.length === 0) return [];
1211
- if (trimmed.startsWith("[")) {
1212
- const parsed = JSON.parse(trimmed);
1213
- if (!Array.isArray(parsed)) throw new Error(`fromRunRecordDir: file '${file}' did not parse to an array`);
1214
- return parsed.map((value, index) => ({
1215
- index,
1216
- value
1217
- }));
1218
- }
1219
- const out = [];
1220
- const lines = trimmed.split("\n");
1221
- for (let i = 0; i < lines.length; i++) {
1222
- const line = lines[i].trim();
1223
- if (line.length === 0) continue;
1224
- try {
1225
- out.push({
1226
- index: i,
1227
- value: JSON.parse(line)
1228
- });
1229
- } catch (err) {
1230
- throw new Error(`fromRunRecordDir: file '${file}' line ${i + 1} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
1231
- }
1232
- }
1233
- return out;
1234
- }
1235
- /** Sorted file list under a directory, filtered by `include`. Sorted so the
1236
- * resulting `RunRecord` order — and any downstream fingerprint — is stable
1237
- * across filesystems. */
1238
- async function collectFiles(dir, include, recursive) {
1239
- const entries = await readdir(dir, { withFileTypes: true });
1240
- const files = [];
1241
- const subdirs = [];
1242
- for (const entry of entries) {
1243
- if (entry.isDirectory()) {
1244
- if (recursive) subdirs.push(join(dir, entry.name));
1245
- continue;
1246
- }
1247
- if (include(entry.name)) files.push(join(dir, entry.name));
1248
- }
1249
- files.sort();
1250
- subdirs.sort();
1251
- for (const sub of subdirs) files.push(...await collectFiles(sub, include, recursive));
1252
- return files;
1253
- }
1254
- //#endregion
1255
- //#region src/contract/eval-reporting-suite.ts
1256
- /**
1257
- * # `evalReportingSuite` — one call from runs (or a run dir) to `analysis.json`.
1258
- *
1259
- * A thin wrapper over the analysis primitive (`analyzeRuns`) and the on-disk
1260
- * intake adapter (`fromRunRecordDir`). It does NOT reimplement any statistics,
1261
- * distributions, or clustering — it resolves the input into validated
1262
- * `RunRecord[]`, calls `analyzeRuns` with the options you'd pass it directly,
1263
- * wraps the result in a small provenance envelope, and (optionally) writes a
1264
- * single `analysis.json` artifact.
1265
- *
1266
- * ```ts
1267
- * // From a directory of run files, write ./runs/analysis.json:
1268
- * const suite = await evalReportingSuite('./runs', { write: true })
1269
- * // From records already in memory, no write:
1270
- * const suite = await evalReportingSuite(records, { analyze: { decisionThreshold: 0.03 } })
1271
- * suite.report // the InsightReport — distributions, paired lift, findings rollup
1272
- * ```
1273
- */
1274
- const ANALYSIS_ARTIFACT = "analysis.json";
1275
- /**
1276
- * Resolve runs (or a run dir/file), run `analyzeRuns`, and optionally persist a
1277
- * single `analysis.json`. The only analysis logic lives in `analyzeRuns`; this
1278
- * function is composition + I/O.
1279
- */
1280
- async function evalReportingSuite(input, options = {}) {
1281
- const fromPath = typeof input === "string";
1282
- let runs;
1283
- let files = [];
1284
- let rejected = [];
1285
- if (fromPath) {
1286
- const loaded = await fromRunRecordDir(input, options.load);
1287
- runs = loaded.runs;
1288
- files = loaded.files;
1289
- rejected = loaded.rejected;
1290
- } else runs = input;
1291
- if (runs.length === 0) throw new Error(fromPath ? `evalReportingSuite: no RunRecords found at '${input}'` : "evalReportingSuite: no RunRecords to analyze");
1292
- const result = {
1293
- report: await analyzeRuns({
1294
- ...options.analyze,
1295
- runs
1296
- }),
1297
- provenance: {
1298
- generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
1299
- runCount: runs.length,
1300
- sourcePath: fromPath ? input : null,
1301
- files,
1302
- rejected
1303
- },
1304
- writtenTo: null
1305
- };
1306
- const target = resolveWriteTarget(options.write, fromPath ? input : null);
1307
- if (target) {
1308
- await mkdir(dirname(target), { recursive: true });
1309
- await writeFile(target, `${JSON.stringify(result, null, 2)}\n`, "utf8");
1310
- result.writtenTo = target;
1311
- }
1312
- return result;
1313
- }
1314
- /** Resolve where (if anywhere) to write `analysis.json`. Returns null when
1315
- * writing is disabled. Throws on `write: true` with in-memory input — there is
1316
- * no directory to anchor the artifact to, and silently inventing `cwd` would
1317
- * scatter files. */
1318
- function resolveWriteTarget(write, sourcePath) {
1319
- if (!write) return null;
1320
- if (typeof write === "string") return write.endsWith("/") || !write.endsWith(".json") && !write.endsWith(".jsonl") ? join(write, ANALYSIS_ARTIFACT) : write;
1321
- if (sourcePath === null) throw new Error("evalReportingSuite: write:true needs a source path to anchor analysis.json — pass an explicit output path when analyzing in-memory records");
1322
- return sourcePath.endsWith(".json") || sourcePath.endsWith(".jsonl") ? join(dirname(sourcePath), ANALYSIS_ARTIFACT) : join(sourcePath, ANALYSIS_ARTIFACT);
1323
- }
1324
- //#endregion
1325
- //#region src/contract/intake/agent-trace.ts
1326
- function rangeLines(r) {
1327
- return Math.max(0, r.end_line - r.start_line + 1);
1328
- }
1132
+ //#region src/contract/intake/agent-trace.ts
1133
+ function rangeLines(r) {
1134
+ return Math.max(0, r.end_line - r.start_line + 1);
1135
+ }
1329
1136
  /**
1330
1137
  * Build a commit → provenance index from Agent Trace records. Multiple records
1331
1138
  * for the same revision are merged. Records without `vcs.revision` are skipped
@@ -1402,6 +1209,94 @@ function partitionRunsByAuthoringModel(runs, index) {
1402
1209
  };
1403
1210
  }
1404
1211
  //#endregion
1212
+ //#region src/contract/intake/code-agent-fields.ts
1213
+ function record(value) {
1214
+ return isRecord(value) ? value : null;
1215
+ }
1216
+ function stringField(obj, key) {
1217
+ const value = obj[key];
1218
+ return typeof value === "string" && value.length > 0 ? value : void 0;
1219
+ }
1220
+ function numberField(obj, key) {
1221
+ const value = obj[key];
1222
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
1223
+ }
1224
+ function timestampMs(value) {
1225
+ if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
1226
+ if (typeof value === "string" && value.length > 0) {
1227
+ const parsed = Date.parse(value);
1228
+ return Number.isFinite(parsed) ? parsed : void 0;
1229
+ }
1230
+ }
1231
+ //#endregion
1232
+ //#region src/contract/intake/code-agent-identity.ts
1233
+ function sessionIdFromEntries(source, entries) {
1234
+ for (const entry of entries) {
1235
+ if (source === "pi" && entry.type === "session") {
1236
+ const id = stringField(entry, "id");
1237
+ if (id) return id;
1238
+ }
1239
+ if (source === "codex") {
1240
+ const threadId = stringField(entry, "thread_id");
1241
+ if (threadId) return threadId;
1242
+ const payload = record(entry.payload);
1243
+ const id = payload ? stringField(payload, "id") : void 0;
1244
+ if (id) return id;
1245
+ }
1246
+ const payload = record(record(entry.message)?.payload);
1247
+ const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
1248
+ if (sessionId) return sessionId;
1249
+ }
1250
+ }
1251
+ function modelFromEntries(source, entries) {
1252
+ if (source === "pi") return summarizeSessionInput("pi", { records: entries }).lastServedModel ?? void 0;
1253
+ let providerModel;
1254
+ for (const entry of entries) {
1255
+ if (source === "codex") {
1256
+ const payload = record(entry.payload);
1257
+ const model = payload ? stringField(payload, "model") : void 0;
1258
+ if (model) return model;
1259
+ providerModel = providerModel ?? (payload ? stringField(payload, "model_provider") : void 0);
1260
+ }
1261
+ const openCodeModel = stringField(entry, "modelID");
1262
+ if (openCodeModel) {
1263
+ const provider = stringField(entry, "providerID");
1264
+ return provider ? `${provider}/${openCodeModel}` : openCodeModel;
1265
+ }
1266
+ const modelObject = record(entry.model);
1267
+ const nestedModel = stringField(modelObject ?? {}, "modelID");
1268
+ if (nestedModel) {
1269
+ const provider = stringField(modelObject ?? {}, "providerID");
1270
+ return provider ? `${provider}/${nestedModel}` : nestedModel;
1271
+ }
1272
+ const message = record(entry.message);
1273
+ const model = message ? stringField(message, "model") : void 0;
1274
+ if (model) return model;
1275
+ }
1276
+ return providerModel;
1277
+ }
1278
+ function cwdFromEntries(entries) {
1279
+ for (const entry of entries) {
1280
+ const path = record(entry.path);
1281
+ const cwd = stringField(entry, "cwd") ?? stringField(path ?? {}, "cwd") ?? stringField(path ?? {}, "root") ?? (record(entry.payload) ? stringField(record(entry.payload), "cwd") : void 0);
1282
+ if (cwd) return cwd;
1283
+ }
1284
+ }
1285
+ function fallbackSessionId$1(source, sourcePath) {
1286
+ return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(7, 27);
1287
+ }
1288
+ function withSnapshot(model) {
1289
+ const trimmed = model.trim() || "unknown";
1290
+ if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) return trimmed;
1291
+ return `${trimmed}@observed-local`;
1292
+ }
1293
+ function hashString(value) {
1294
+ return `sha256:${createHash("sha256").update(value).digest("hex")}`;
1295
+ }
1296
+ function stableSeed(value) {
1297
+ return createHash("sha256").update(value).digest().readUInt32BE(0);
1298
+ }
1299
+ //#endregion
1405
1300
  //#region src/contract/intake/code-agent-observation.ts
1406
1301
  /**
1407
1302
  * Where a session came from. `pi` is a native Pi session (`~/.pi/agent/sessions/*.jsonl`), read
@@ -1414,12 +1309,12 @@ function partitionRunsByAuthoringModel(runs, index) {
1414
1309
  * stay out of this projection; callers retain the source JSONL as evidence.
1415
1310
  */
1416
1311
  function observeCodeAgentSession(options) {
1417
- const entries = options.entries.filter(isRecord$1);
1312
+ const entries = options.entries.filter(isRecord);
1418
1313
  const projection = projectionFor(options.source, entries);
1419
1314
  const terminal = terminalWithExecution(projection, options.execution);
1420
1315
  return {
1421
1316
  source: options.source,
1422
- sessionId: sessionIdFromEntries$1(options.source, entries) ?? fallbackSessionId$1(options.source, options.sourcePath),
1317
+ sessionId: sessionIdFromEntries(options.source, entries) ?? fallbackSessionId(options.source, options.sourcePath),
1423
1318
  finalText: projection.finalText,
1424
1319
  terminal,
1425
1320
  actions: projection.actions
@@ -1443,16 +1338,16 @@ function codexProjection(entries) {
1443
1338
  let terminal = "unknown";
1444
1339
  let explicitTerminal = false;
1445
1340
  for (const entry of entries) {
1446
- const entryType = stringField$1(entry, "type");
1447
- const payload = record$1(entry.payload) ?? {};
1448
- const payloadType = stringField$1(payload, "type");
1449
- const timestampMs = timestamp(entry.timestamp);
1341
+ const entryType = stringField(entry, "type");
1342
+ const payload = record(entry.payload) ?? {};
1343
+ const payloadType = stringField(payload, "type");
1344
+ const timestampMs$1 = timestampMs(entry.timestamp);
1450
1345
  const itemEvent = admittedCodexItem(entry, transcriptStream);
1451
1346
  const item = itemEvent?.item ?? null;
1452
1347
  if (itemEvent?.eventType === "item.started" && item) {
1453
- const itemType = stringField$1(item, "type");
1348
+ const itemType = stringField(item, "type");
1454
1349
  if (isCodexActionItem(itemType)) {
1455
- const id = stringField$1(item, "id") ?? `item-${actions.length}`;
1350
+ const id = stringField(item, "id") ?? `item-${actions.length}`;
1456
1351
  const action = actionFor({
1457
1352
  id,
1458
1353
  stepIndex: actions.length,
@@ -1460,7 +1355,7 @@ function codexProjection(entries) {
1460
1355
  surface: codexItemSurface(itemType, item),
1461
1356
  name: codexItemName(itemType, item),
1462
1357
  status: "started",
1463
- timestampMs,
1358
+ timestampMs: timestampMs$1,
1464
1359
  metadata: compactMetadata({
1465
1360
  sourceEventType: itemEvent.eventType,
1466
1361
  itemType
@@ -1471,10 +1366,10 @@ function codexProjection(entries) {
1471
1366
  }
1472
1367
  }
1473
1368
  if (itemEvent?.eventType === "item.completed" && item) {
1474
- const itemType = stringField$1(item, "type");
1475
- if (itemType === "agent_message" || itemType === "message") finalText = nonEmpty(stringField$1(item, "text")) ?? finalText;
1369
+ const itemType = stringField(item, "type");
1370
+ if (itemType === "agent_message" || itemType === "message") finalText = nonEmpty(stringField(item, "text")) ?? finalText;
1476
1371
  if (isCodexActionItem(itemType)) {
1477
- const id = stringField$1(item, "id") ?? `item-${actions.length}`;
1372
+ const id = stringField(item, "id") ?? `item-${actions.length}`;
1478
1373
  const status = codexItemStatus(itemType, item);
1479
1374
  const existing = calls.get(id);
1480
1375
  if (existing) existing.status = status;
@@ -1486,7 +1381,7 @@ function codexProjection(entries) {
1486
1381
  surface: codexItemSurface(itemType, item),
1487
1382
  name: codexItemName(itemType, item),
1488
1383
  status,
1489
- timestampMs,
1384
+ timestampMs: timestampMs$1,
1490
1385
  metadata: compactMetadata({
1491
1386
  sourceEventType: itemEvent.eventType,
1492
1387
  itemType
@@ -1500,8 +1395,8 @@ function codexProjection(entries) {
1500
1395
  if (entryType === "response_item") {
1501
1396
  if (payloadType === "message" || payloadType === "agent_message") finalText = textFromMessagePayload(payload) ?? finalText;
1502
1397
  if (payloadType === "function_call" || payloadType === "custom_tool_call") {
1503
- const id = stringField$1(payload, "call_id") ?? stringField$1(payload, "id") ?? `call-${actions.length}`;
1504
- const name = stringField$1(payload, "name") ?? payloadType;
1398
+ const id = stringField(payload, "call_id") ?? stringField(payload, "id") ?? `call-${actions.length}`;
1399
+ const name = stringField(payload, "name") ?? payloadType;
1505
1400
  const action = actionFor({
1506
1401
  id,
1507
1402
  stepIndex: actions.length,
@@ -1509,7 +1404,7 @@ function codexProjection(entries) {
1509
1404
  surface: surfaceForTool(name),
1510
1405
  name,
1511
1406
  status: "started",
1512
- timestampMs,
1407
+ timestampMs: timestampMs$1,
1513
1408
  metadata: compactMetadata({
1514
1409
  sourceEventType: entryType,
1515
1410
  payloadType
@@ -1519,23 +1414,23 @@ function codexProjection(entries) {
1519
1414
  actions.push(action);
1520
1415
  }
1521
1416
  if (payloadType === "function_call_output" || payloadType === "custom_tool_call_output") {
1522
- const id = stringField$1(payload, "call_id") ?? stringField$1(payload, "id");
1417
+ const id = stringField(payload, "call_id") ?? stringField(payload, "id");
1523
1418
  const existing = id ? calls.get(id) : void 0;
1524
1419
  if (existing) existing.status = looksLikeError(payload.output) ? "failed" : "completed";
1525
1420
  }
1526
1421
  }
1527
1422
  if (entryType === "event_msg") {
1528
- if (payloadType === "agent_message") finalText = nonEmpty(stringField$1(payload, "message")) ?? nonEmpty(stringField$1(payload, "text")) ?? finalText;
1423
+ if (payloadType === "agent_message") finalText = nonEmpty(stringField(payload, "message")) ?? nonEmpty(stringField(payload, "text")) ?? finalText;
1529
1424
  if (payloadType === "patch_apply_end") {
1530
1425
  const patchStatus = typeof payload.success === "boolean" ? payload.success ? "completed" : "failed" : "unknown";
1531
1426
  actions.push(actionFor({
1532
- id: stringField$1(payload, "call_id") ?? `patch-${actions.length}`,
1427
+ id: stringField(payload, "call_id") ?? `patch-${actions.length}`,
1533
1428
  stepIndex: actions.length,
1534
1429
  kind: "patch",
1535
1430
  surface: "code",
1536
1431
  name: "patch",
1537
1432
  status: patchStatus,
1538
- timestampMs,
1433
+ timestampMs: timestampMs$1,
1539
1434
  metadata: compactMetadata({
1540
1435
  sourceEventType: entryType,
1541
1436
  payloadType
@@ -1543,15 +1438,15 @@ function codexProjection(entries) {
1543
1438
  }));
1544
1439
  }
1545
1440
  if (payloadType === "sub_agent_activity") {
1546
- const name = stringField$1(payload, "kind") ?? "subagent";
1441
+ const name = stringField(payload, "kind") ?? "subagent";
1547
1442
  actions.push(actionFor({
1548
- id: stringField$1(payload, "event_id") ?? `subagent-${actions.length}`,
1443
+ id: stringField(payload, "event_id") ?? `subagent-${actions.length}`,
1549
1444
  stepIndex: actions.length,
1550
1445
  kind: "tool",
1551
1446
  surface: "subagent",
1552
1447
  name,
1553
1448
  status: statusFrom(payload),
1554
- timestampMs,
1449
+ timestampMs: timestampMs$1,
1555
1450
  metadata: compactMetadata({
1556
1451
  sourceEventType: entryType,
1557
1452
  payloadType
@@ -1562,20 +1457,20 @@ function codexProjection(entries) {
1562
1457
  const completed = payloadType === "task_complete";
1563
1458
  terminal = completed ? "completed" : "failed";
1564
1459
  explicitTerminal = true;
1565
- actions.push(terminalAction(actions.length, payloadType, completed, timestampMs));
1460
+ actions.push(terminalAction(actions.length, payloadType, completed, timestampMs$1));
1566
1461
  }
1567
1462
  }
1568
1463
  if (entryType === "turn.completed" || entryType === "exec_done") {
1569
1464
  const completed = entry.success !== false;
1570
1465
  terminal = completed ? "completed" : "failed";
1571
1466
  explicitTerminal = true;
1572
- finalText = nonEmpty(stringField$1(entry, "message")) ?? finalText;
1573
- actions.push(terminalAction(actions.length, entryType, completed, timestampMs));
1467
+ finalText = nonEmpty(stringField(entry, "message")) ?? finalText;
1468
+ actions.push(terminalAction(actions.length, entryType, completed, timestampMs$1));
1574
1469
  }
1575
1470
  if (entryType === "turn.failed" || entryType === "error") {
1576
1471
  terminal = "failed";
1577
1472
  explicitTerminal = true;
1578
- actions.push(terminalAction(actions.length, entryType, false, timestampMs));
1473
+ actions.push(terminalAction(actions.length, entryType, false, timestampMs$1));
1579
1474
  }
1580
1475
  }
1581
1476
  return {
@@ -1593,20 +1488,20 @@ function claudeProjection(entries) {
1593
1488
  let terminal = "unknown";
1594
1489
  let explicitTerminal = false;
1595
1490
  for (const entry of entries) {
1596
- const entryType = stringField$1(entry, "type");
1597
- const timestampMs = timestamp(entry.timestamp);
1598
- const message = record$1(entry.message);
1491
+ const entryType = stringField(entry, "type");
1492
+ const timestampMs$2 = timestampMs(entry.timestamp);
1493
+ const message = record(entry.message);
1599
1494
  const content = Array.isArray(message?.content) ? message.content : [];
1600
1495
  if (entryType === "assistant") {
1601
1496
  const text = textBlocks(content);
1602
1497
  if (text !== null) finalText = text;
1603
1498
  for (const value of content) {
1604
- const part = record$1(value);
1499
+ const part = record(value);
1605
1500
  if (!part) continue;
1606
- const partType = stringField$1(part, "type");
1501
+ const partType = stringField(part, "type");
1607
1502
  if (partType === "tool_use") {
1608
- const id = stringField$1(part, "id") ?? `tool-${actions.length}`;
1609
- const name = stringField$1(part, "name") ?? "tool";
1503
+ const id = stringField(part, "id") ?? `tool-${actions.length}`;
1504
+ const name = stringField(part, "name") ?? "tool";
1610
1505
  const action = actionFor({
1611
1506
  id,
1612
1507
  stepIndex: actions.length,
@@ -1614,7 +1509,7 @@ function claudeProjection(entries) {
1614
1509
  surface: surfaceForTool(name),
1615
1510
  name,
1616
1511
  status: "started",
1617
- timestampMs,
1512
+ timestampMs: timestampMs$2,
1618
1513
  metadata: compactMetadata({
1619
1514
  sourceEventType: entryType,
1620
1515
  partType
@@ -1625,17 +1520,17 @@ function claudeProjection(entries) {
1625
1520
  }
1626
1521
  }
1627
1522
  completeClaudeTools(content, calls);
1628
- const stopReason = stringField$1(message ?? {}, "stop_reason");
1523
+ const stopReason = stringField(message ?? {}, "stop_reason");
1629
1524
  if (stopReason === "end_turn") {
1630
1525
  terminal = "completed";
1631
1526
  explicitTerminal = true;
1632
- actions.push(terminalAction(actions.length, stopReason, true, timestampMs));
1527
+ actions.push(terminalAction(actions.length, stopReason, true, timestampMs$2));
1633
1528
  }
1634
1529
  }
1635
1530
  if (entryType === "user") completeClaudeTools(content, calls);
1636
1531
  if (entryType === "tool_use") {
1637
- const id = stringField$1(entry, "id") ?? `tool-${actions.length}`;
1638
- const name = stringField$1(entry, "name") ?? "tool";
1532
+ const id = stringField(entry, "id") ?? `tool-${actions.length}`;
1533
+ const name = stringField(entry, "name") ?? "tool";
1639
1534
  const action = actionFor({
1640
1535
  id,
1641
1536
  stepIndex: actions.length,
@@ -1643,47 +1538,47 @@ function claudeProjection(entries) {
1643
1538
  surface: surfaceForTool(name),
1644
1539
  name,
1645
1540
  status: "started",
1646
- timestampMs,
1541
+ timestampMs: timestampMs$2,
1647
1542
  metadata: { sourceEventType: entryType }
1648
1543
  });
1649
1544
  calls.set(id, action);
1650
1545
  actions.push(action);
1651
1546
  }
1652
1547
  if (entryType === "tool_result") {
1653
- const id = stringField$1(entry, "tool_use_id");
1548
+ const id = stringField(entry, "tool_use_id");
1654
1549
  const action = id ? calls.get(id) : void 0;
1655
1550
  if (action) action.status = entry.is_error === true ? "failed" : "completed";
1656
1551
  }
1657
1552
  if (entryType === "system") {
1658
- const subtype = stringField$1(entry, "subtype");
1659
- const hookName = stringField$1(entry, "hook_name") ?? stringField$1(entry, "hookName");
1553
+ const subtype = stringField(entry, "subtype");
1554
+ const hookName = stringField(entry, "hook_name") ?? stringField(entry, "hookName");
1660
1555
  if (subtype?.toLowerCase().includes("hook") || hookName) actions.push(actionFor({
1661
- id: stringField$1(entry, "uuid") ?? `hook-${actions.length}`,
1556
+ id: stringField(entry, "uuid") ?? `hook-${actions.length}`,
1662
1557
  stepIndex: actions.length,
1663
1558
  kind: "tool",
1664
1559
  surface: "hook",
1665
1560
  name: hookName ?? subtype ?? "hook",
1666
1561
  status: statusFrom(entry),
1667
- timestampMs,
1562
+ timestampMs: timestampMs$2,
1668
1563
  metadata: compactMetadata({
1669
1564
  sourceEventType: entryType,
1670
1565
  subtype
1671
1566
  })
1672
1567
  }));
1673
1568
  }
1674
- if (entryType === "pr-link") actions.push(terminalAction(actions.length, entryType, true, timestampMs));
1569
+ if (entryType === "pr-link") actions.push(terminalAction(actions.length, entryType, true, timestampMs$2));
1675
1570
  if (entryType === "result") {
1676
- resultText = nonEmpty(stringField$1(entry, "result")) ?? resultText;
1677
- const subtype = stringField$1(entry, "subtype");
1571
+ resultText = nonEmpty(stringField(entry, "result")) ?? resultText;
1572
+ const subtype = stringField(entry, "subtype");
1678
1573
  const completed = entry.is_error !== true && !subtype?.toLowerCase().includes("error");
1679
1574
  terminal = completed ? "completed" : "failed";
1680
1575
  explicitTerminal = true;
1681
- actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs));
1576
+ actions.push(terminalAction(actions.length, subtype ?? entryType, completed, timestampMs$2));
1682
1577
  }
1683
1578
  if (entryType === "error") {
1684
1579
  terminal = "failed";
1685
1580
  explicitTerminal = true;
1686
- actions.push(terminalAction(actions.length, entryType, false, timestampMs));
1581
+ actions.push(terminalAction(actions.length, entryType, false, timestampMs$2));
1687
1582
  }
1688
1583
  }
1689
1584
  return {
@@ -1695,28 +1590,28 @@ function claudeProjection(entries) {
1695
1590
  }
1696
1591
  function openCodeProjection(entries) {
1697
1592
  const actions = [];
1698
- const assistantMessageIds = new Set(entries.filter((entry) => stringField$1(entry, "role") === "assistant").map((entry) => stringField$1(entry, "id")).filter((id) => id !== void 0));
1593
+ const assistantMessageIds = new Set(entries.filter((entry) => stringField(entry, "role") === "assistant").map((entry) => stringField(entry, "id")).filter((id) => id !== void 0));
1699
1594
  let finalText = null;
1700
1595
  let terminal = "unknown";
1701
1596
  let explicitTerminal = false;
1702
1597
  for (const entry of entries) {
1703
- const entryType = stringField$1(entry, "type");
1704
- const part = record$1(entry.part) ?? entry;
1705
- const partType = stringField$1(part, "type");
1706
- const timestampMs = timestamp(entry.timestamp) ?? timestamp(record$1(entry.time)?.created);
1707
- if (entryType === "text" && record$1(entry.part)) finalText = nonEmpty(stringField$1(part, "text")) ?? finalText;
1708
- else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField$1(part, "messageID") ?? ""))) finalText = nonEmpty(stringField$1(part, "text")) ?? finalText;
1598
+ const entryType = stringField(entry, "type");
1599
+ const part = record(entry.part) ?? entry;
1600
+ const partType = stringField(part, "type");
1601
+ const timestampMs$3 = timestampMs(entry.timestamp) ?? timestampMs(record(entry.time)?.created);
1602
+ if (entryType === "text" && record(entry.part)) finalText = nonEmpty(stringField(part, "text")) ?? finalText;
1603
+ else if (partType === "text" && (assistantMessageIds.size === 0 || assistantMessageIds.has(stringField(part, "messageID") ?? ""))) finalText = nonEmpty(stringField(part, "text")) ?? finalText;
1709
1604
  if (entryType === "tool_use" || partType === "tool") {
1710
- const state = record$1(part.state) ?? {};
1711
- const name = stringField$1(part, "tool") ?? "tool";
1605
+ const state = record(part.state) ?? {};
1606
+ const name = stringField(part, "tool") ?? "tool";
1712
1607
  actions.push(actionFor({
1713
- id: stringField$1(part, "callID") ?? stringField$1(part, "id") ?? `tool-${actions.length}`,
1608
+ id: stringField(part, "callID") ?? stringField(part, "id") ?? `tool-${actions.length}`,
1714
1609
  stepIndex: actions.length,
1715
1610
  kind: "tool",
1716
1611
  surface: surfaceForTool(name),
1717
1612
  name,
1718
1613
  status: statusFrom(state),
1719
- timestampMs,
1614
+ timestampMs: timestampMs$3,
1720
1615
  metadata: compactMetadata({
1721
1616
  sourceEventType: entryType,
1722
1617
  partType
@@ -1724,31 +1619,31 @@ function openCodeProjection(entries) {
1724
1619
  }));
1725
1620
  }
1726
1621
  if (partType === "patch") actions.push(actionFor({
1727
- id: stringField$1(part, "id") ?? `patch-${actions.length}`,
1622
+ id: stringField(part, "id") ?? `patch-${actions.length}`,
1728
1623
  stepIndex: actions.length,
1729
1624
  kind: "patch",
1730
1625
  surface: "code",
1731
1626
  name: "patch",
1732
1627
  status: "completed",
1733
- timestampMs,
1628
+ timestampMs: timestampMs$3,
1734
1629
  metadata: compactMetadata({
1735
1630
  sourceEventType: entryType,
1736
1631
  partType
1737
1632
  })
1738
1633
  }));
1739
- if (stringField$1(entry, "role") === "assistant") {
1740
- const finish = stringField$1(entry, "finish");
1634
+ if (stringField(entry, "role") === "assistant") {
1635
+ const finish = stringField(entry, "finish");
1741
1636
  if (finish === "stop" || finish === "error") {
1742
1637
  const completed = finish === "stop";
1743
1638
  terminal = completed ? "completed" : "failed";
1744
1639
  explicitTerminal = true;
1745
- actions.push(terminalAction(actions.length, finish, completed, timestampMs));
1640
+ actions.push(terminalAction(actions.length, finish, completed, timestampMs$3));
1746
1641
  }
1747
1642
  }
1748
1643
  if (entryType === "error") {
1749
1644
  terminal = "failed";
1750
1645
  explicitTerminal = true;
1751
- actions.push(terminalAction(actions.length, entryType, false, timestampMs));
1646
+ actions.push(terminalAction(actions.length, entryType, false, timestampMs$3));
1752
1647
  }
1753
1648
  }
1754
1649
  return {
@@ -1765,15 +1660,15 @@ function kimiProjection(entries) {
1765
1660
  let terminal = "unknown";
1766
1661
  let explicitTerminal = false;
1767
1662
  for (const entry of entries) {
1768
- const timestampMs = timestamp(entry.timestamp);
1769
- const message = record$1(entry.message);
1770
- const messageType = stringField$1(message ?? {}, "type");
1771
- const payload = record$1(message?.payload) ?? {};
1772
- if (messageType === "ContentPart" && stringField$1(payload, "type") === "text") finalText = nonEmpty(stringField$1(payload, "text")) ?? nonEmpty(stringField$1(payload, "content")) ?? finalText;
1663
+ const timestampMs$4 = timestampMs(entry.timestamp);
1664
+ const message = record(entry.message);
1665
+ const messageType = stringField(message ?? {}, "type");
1666
+ const payload = record(message?.payload) ?? {};
1667
+ if (messageType === "ContentPart" && stringField(payload, "type") === "text") finalText = nonEmpty(stringField(payload, "text")) ?? nonEmpty(stringField(payload, "content")) ?? finalText;
1773
1668
  if (messageType === "ToolCall") {
1774
- const fn = record$1(payload.function) ?? {};
1775
- const id = stringField$1(payload, "id") ?? `tool-${actions.length}`;
1776
- const name = stringField$1(fn, "name") ?? "tool";
1669
+ const fn = record(payload.function) ?? {};
1670
+ const id = stringField(payload, "id") ?? `tool-${actions.length}`;
1671
+ const name = stringField(fn, "name") ?? "tool";
1777
1672
  const action = actionFor({
1778
1673
  id,
1779
1674
  stepIndex: actions.length,
@@ -1781,22 +1676,22 @@ function kimiProjection(entries) {
1781
1676
  surface: surfaceForTool(name),
1782
1677
  name,
1783
1678
  status: "started",
1784
- timestampMs,
1679
+ timestampMs: timestampMs$4,
1785
1680
  metadata: { sourceEventType: messageType }
1786
1681
  });
1787
1682
  calls.set(id, action);
1788
1683
  actions.push(action);
1789
1684
  }
1790
1685
  if (messageType === "ToolResult") {
1791
- const id = stringField$1(payload, "tool_call_id");
1686
+ const id = stringField(payload, "tool_call_id");
1792
1687
  const action = id ? calls.get(id) : void 0;
1793
- if (action) action.status = record$1(payload.return_value)?.is_error === true ? "failed" : "completed";
1688
+ if (action) action.status = record(payload.return_value)?.is_error === true ? "failed" : "completed";
1794
1689
  }
1795
1690
  if (messageType === "TurnEnd" || messageType === "StepInterrupted") {
1796
1691
  const completed = messageType === "TurnEnd";
1797
1692
  terminal = completed ? "completed" : "failed";
1798
1693
  explicitTerminal = true;
1799
- actions.push(terminalAction(actions.length, messageType, completed, timestampMs));
1694
+ actions.push(terminalAction(actions.length, messageType, completed, timestampMs$4));
1800
1695
  }
1801
1696
  }
1802
1697
  return {
@@ -1816,7 +1711,7 @@ function piSessionProjection(entries) {
1816
1711
  surface: surfaceForTool(call.name),
1817
1712
  name: call.name,
1818
1713
  status: call.status === "completed" ? "completed" : call.status === "error" ? "failed" : call.status === "pending" ? "started" : "unknown",
1819
- timestampMs: timestamp(call.startedAt),
1714
+ timestampMs: timestampMs(call.startedAt),
1820
1715
  metadata: { sourceEventType: "pi-tool-call" }
1821
1716
  }));
1822
1717
  const answer = [...session.messages].reverse().find((m) => m.role === "assistant" && m.actor === "agent" && m.parts.some((p) => p.type === "text"));
@@ -1836,17 +1731,17 @@ function piGraphProjection(entries) {
1836
1731
  for (const entry of entries) {
1837
1732
  const nodes = Array.isArray(entry.nodes) ? entry.nodes : [];
1838
1733
  for (const value of nodes) {
1839
- const node = record$1(value);
1840
- const ir = record$1(node?.ir) ?? node;
1841
- const kind = stringField$1(ir ?? {}, "kind");
1734
+ const node = record(value);
1735
+ const ir = record(node?.ir) ?? node;
1736
+ const kind = stringField(ir ?? {}, "kind");
1842
1737
  if (kind === "ToolInvocation") actions.push(actionFor({
1843
- id: stringField$1(ir ?? {}, "id") ?? stringField$1(node ?? {}, "id") ?? `tool-${actions.length}`,
1738
+ id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `tool-${actions.length}`,
1844
1739
  stepIndex: actions.length,
1845
1740
  kind: "tool",
1846
1741
  surface: "tool",
1847
- name: stringField$1(ir ?? {}, "name") ?? "graph-tool",
1742
+ name: stringField(ir ?? {}, "name") ?? "graph-tool",
1848
1743
  status: "started",
1849
- timestampMs: timestamp(ir?.createdAt),
1744
+ timestampMs: timestampMs(ir?.createdAt),
1850
1745
  metadata: {
1851
1746
  sourceEventType: "graph-node",
1852
1747
  graphKind: kind
@@ -1860,13 +1755,13 @@ function piGraphProjection(entries) {
1860
1755
  terminal = "completed";
1861
1756
  explicitTerminal = true;
1862
1757
  actions.push(actionFor({
1863
- id: stringField$1(ir ?? {}, "id") ?? stringField$1(node ?? {}, "id") ?? `completion-${actions.length}`,
1758
+ id: stringField(ir ?? {}, "id") ?? stringField(node ?? {}, "id") ?? `completion-${actions.length}`,
1864
1759
  stepIndex: actions.length,
1865
1760
  kind: "graph-completion",
1866
1761
  surface: "tool",
1867
1762
  name: "complete",
1868
1763
  status: "completed",
1869
- timestampMs: timestamp(ir?.createdAt),
1764
+ timestampMs: timestampMs(ir?.createdAt),
1870
1765
  metadata: {
1871
1766
  sourceEventType: "graph-node",
1872
1767
  graphKind: kind
@@ -1884,9 +1779,9 @@ function piGraphProjection(entries) {
1884
1779
  }
1885
1780
  function completeClaudeTools(content, calls) {
1886
1781
  for (const value of content) {
1887
- const part = record$1(value);
1888
- if (!part || stringField$1(part, "type") !== "tool_result") continue;
1889
- const id = stringField$1(part, "tool_use_id");
1782
+ const part = record(value);
1783
+ if (!part || stringField(part, "type") !== "tool_result") continue;
1784
+ const id = stringField(part, "tool_use_id");
1890
1785
  const action = id ? calls.get(id) : void 0;
1891
1786
  if (action) action.status = part.is_error === true ? "failed" : "completed";
1892
1787
  }
@@ -1900,21 +1795,21 @@ function completeClaudeTools(content, calls) {
1900
1795
  * PascalCase. Read both here so one branch serves both transports.
1901
1796
  */
1902
1797
  function codexItemEvent(entry) {
1903
- const entryType = stringField$1(entry, "type");
1798
+ const entryType = stringField(entry, "type");
1904
1799
  if (entryType === "item.started" || entryType === "item.completed") {
1905
- const item = record$1(entry.item);
1800
+ const item = record(entry.item);
1906
1801
  return item ? {
1907
1802
  eventType: entryType,
1908
1803
  item
1909
1804
  } : null;
1910
1805
  }
1911
1806
  if (entryType !== "event_msg") return null;
1912
- const payload = record$1(entry.payload);
1913
- const payloadType = payload ? stringField$1(payload, "type") : void 0;
1807
+ const payload = record(entry.payload);
1808
+ const payloadType = payload ? stringField(payload, "type") : void 0;
1914
1809
  if (payloadType !== "item_started" && payloadType !== "item_completed") return null;
1915
- const item = record$1(payload?.item);
1810
+ const item = record(payload?.item);
1916
1811
  if (!item) return null;
1917
- const itemType = codexItemType(stringField$1(item, "type"));
1812
+ const itemType = codexItemType(stringField(item, "type"));
1918
1813
  return {
1919
1814
  eventType: payloadType === "item_started" ? "item.started" : "item.completed",
1920
1815
  item: itemType === void 0 ? item : {
@@ -1934,7 +1829,7 @@ function codexItemType(itemType) {
1934
1829
  return itemType.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
1935
1830
  }
1936
1831
  function hasCodexTranscriptStream(entries) {
1937
- return entries.some((entry) => stringField$1(entry, "type") === "response_item");
1832
+ return entries.some((entry) => stringField(entry, "type") === "response_item");
1938
1833
  }
1939
1834
  /**
1940
1835
  * A rollout file records each turn twice: `response_item` holds the model
@@ -1962,7 +1857,7 @@ function admittedCodexItem(entry, transcriptStream) {
1962
1857
  const event = codexItemEvent(entry);
1963
1858
  if (!event) return null;
1964
1859
  if (!transcriptStream) return event;
1965
- return isCodexExecutionOnlyItem(stringField$1(event.item, "type")) ? event : null;
1860
+ return isCodexExecutionOnlyItem(stringField(event.item, "type")) ? event : null;
1966
1861
  }
1967
1862
  function isCodexActionItem(itemType) {
1968
1863
  return itemType === "command_execution" || itemType === "mcp_tool_call" || itemType === "collab_tool_call" || itemType === "web_search" || itemType === "file_change";
@@ -1972,27 +1867,27 @@ function codexItemSurface(itemType, item) {
1972
1867
  if (itemType === "collab_tool_call") return "subagent";
1973
1868
  if (itemType === "web_search") return "web";
1974
1869
  if (itemType === "file_change") return "code";
1975
- return surfaceForTool(stringField$1(item, "name") ?? itemType ?? "tool");
1870
+ return surfaceForTool(stringField(item, "name") ?? itemType ?? "tool");
1976
1871
  }
1977
1872
  function codexItemName(itemType, item) {
1978
- if (itemType === "mcp_tool_call") return [stringField$1(item, "server"), stringField$1(item, "tool")].filter(Boolean).join("/") || "mcp";
1979
- if (itemType === "collab_tool_call") return stringField$1(item, "tool") ?? "subagent";
1873
+ if (itemType === "mcp_tool_call") return [stringField(item, "server"), stringField(item, "tool")].filter(Boolean).join("/") || "mcp";
1874
+ if (itemType === "collab_tool_call") return stringField(item, "tool") ?? "subagent";
1980
1875
  if (itemType === "web_search") return "web_search";
1981
1876
  if (itemType === "file_change") return "file_change";
1982
- return stringField$1(item, "name") ?? itemType ?? "tool";
1877
+ return stringField(item, "name") ?? itemType ?? "tool";
1983
1878
  }
1984
1879
  function codexItemStatus(itemType, item) {
1985
1880
  const status = statusFrom(item);
1986
1881
  if (status === "failed") return status;
1987
1882
  if (itemType === "command_execution") {
1988
- const exitCode = numberField$1(item, "exit_code");
1883
+ const exitCode = numberField(item, "exit_code");
1989
1884
  if (exitCode !== void 0) return exitCode === 0 ? "completed" : "failed";
1990
1885
  }
1991
- if (itemType === "mcp_tool_call" && record$1(item.error)) return "failed";
1886
+ if (itemType === "mcp_tool_call" && record(item.error)) return "failed";
1992
1887
  return status === "unknown" || status === "started" ? "completed" : status;
1993
1888
  }
1994
1889
  function textFromMessagePayload(payload) {
1995
- const direct = nonEmpty(stringField$1(payload, "text"));
1890
+ const direct = nonEmpty(stringField(payload, "text"));
1996
1891
  if (direct) return direct;
1997
1892
  const content = payload.content;
1998
1893
  if (typeof content === "string") return nonEmpty(content);
@@ -2000,7 +1895,7 @@ function textFromMessagePayload(payload) {
2000
1895
  return null;
2001
1896
  }
2002
1897
  function textBlocks(values) {
2003
- return nonEmpty(values.map((value) => record$1(value)).filter((value) => value !== null).filter((value) => stringField$1(value, "type") === "text").map((value) => stringField$1(value, "text")).filter((value) => value !== void 0).join("\n"));
1898
+ return nonEmpty(values.map((value) => record(value)).filter((value) => value !== null).filter((value) => stringField(value, "type") === "text").map((value) => stringField(value, "text")).filter((value) => value !== void 0).join("\n"));
2004
1899
  }
2005
1900
  function terminalWithExecution(projection, execution) {
2006
1901
  if (!execution) return {
@@ -2056,7 +1951,7 @@ function isCodexCollaborationTool(normalized) {
2056
1951
  return /^[a-z_]+_agents?$/.test(verb) || verb === "send_message" || verb === "send_input" || verb === "followup_task";
2057
1952
  }
2058
1953
  function statusFrom(value) {
2059
- const status = (stringField$1(value, "status") ?? "").toLowerCase();
1954
+ const status = (stringField(value, "status") ?? "").toLowerCase();
2060
1955
  if (status === "completed" || status === "success" || status === "succeeded") return "completed";
2061
1956
  if (status === "error" || status === "failed" || status === "declined") return "failed";
2062
1957
  if (status === "running" || status === "started" || status === "in_progress") return "started";
@@ -2065,31 +1960,13 @@ function statusFrom(value) {
2065
1960
  function compactMetadata(input) {
2066
1961
  return Object.fromEntries(Object.entries(input).filter((entry) => entry[1] !== void 0));
2067
1962
  }
2068
- function sessionIdFromEntries$1(source, entries) {
2069
- for (const entry of entries) {
2070
- if (source === "pi" && entry.type === "session") {
2071
- const id = stringField$1(entry, "id");
2072
- if (id) return id;
2073
- }
2074
- if (source === "codex") {
2075
- const threadId = stringField$1(entry, "thread_id");
2076
- if (threadId) return threadId;
2077
- const payload = record$1(entry.payload);
2078
- const id = payload ? stringField$1(payload, "id") : void 0;
2079
- if (id) return id;
2080
- }
2081
- const payload = record$1(record$1(entry.message)?.payload);
2082
- const sessionId = stringField$1(entry, "sessionID") ?? stringField$1(entry, "sessionId") ?? stringField$1(entry, "session_id") ?? stringField$1(payload ?? {}, "session_id");
2083
- if (sessionId) return sessionId;
2084
- }
2085
- }
2086
- function fallbackSessionId$1(source, sourcePath) {
1963
+ function fallbackSessionId(source, sourcePath) {
2087
1964
  return `${source}:${sourcePath ?? "unknown-session"}`;
2088
1965
  }
2089
1966
  function looksLikeError(value) {
2090
- if (isRecord$1(value)) {
1967
+ if (isRecord(value)) {
2091
1968
  if (value.success === false || value.is_error === true) return true;
2092
- const exitCode = numberField$1(value, "exit_code");
1969
+ const exitCode = numberField(value, "exit_code");
2093
1970
  if (exitCode !== void 0 && exitCode !== 0) return true;
2094
1971
  }
2095
1972
  const text = typeof value === "string" ? value : JSON.stringify(value ?? "");
@@ -2098,298 +1975,147 @@ function looksLikeError(value) {
2098
1975
  function nonEmpty(value) {
2099
1976
  return value !== void 0 && value.length > 0 ? value : null;
2100
1977
  }
2101
- function isRecord$1(value) {
2102
- return value !== null && typeof value === "object" && !Array.isArray(value);
1978
+ //#endregion
1979
+ //#region src/contract/intake/code-agent-session-metrics.ts
1980
+ function emptyMetrics(entries) {
1981
+ return {
1982
+ entries,
1983
+ userMessages: 0,
1984
+ assistantMessages: 0,
1985
+ reasoningItems: 0,
1986
+ toolCalls: 0,
1987
+ toolOutputs: 0,
1988
+ toolErrors: 0,
1989
+ unclassifiedErrors: 0,
1990
+ patchAttempts: 0,
1991
+ patchSuccesses: 0,
1992
+ patchFailures: 0,
1993
+ turnsStarted: 0,
1994
+ turnsCompleted: 0,
1995
+ turnsAborted: 0,
1996
+ contextCompactions: 0,
1997
+ mcpCalls: 0,
1998
+ subagentCalls: 0,
1999
+ skillCalls: 0,
2000
+ hookCalls: 0,
2001
+ webCalls: 0,
2002
+ codeActions: 0,
2003
+ prLinks: 0,
2004
+ fileSnapshots: 0,
2005
+ graphNodes: 0,
2006
+ graphEdges: 0,
2007
+ actionCandidates: 0,
2008
+ verificationReports: 0,
2009
+ completionDecisions: 0,
2010
+ reliabilityRows: 0,
2011
+ reliabilityLift: 0,
2012
+ inputTokens: 0,
2013
+ outputTokens: 0,
2014
+ reasoningTokens: 0,
2015
+ cachedTokens: 0,
2016
+ cacheWriteTokens: 0,
2017
+ observedCostUsd: 0,
2018
+ observedCostCaptured: false,
2019
+ wallMs: 0,
2020
+ processScore: 0
2021
+ };
2103
2022
  }
2104
- function record$1(value) {
2105
- return isRecord$1(value) ? value : null;
2023
+ function penalizeErrors(base, metrics) {
2024
+ const operations = Math.max(1, metrics.toolCalls + metrics.patchAttempts);
2025
+ const errorRate = Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations);
2026
+ return clamp01(base * (1 - .5 * errorRate));
2106
2027
  }
2107
- function stringField$1(value, key) {
2108
- const field = value[key];
2109
- return typeof field === "string" && field.length > 0 ? field : void 0;
2028
+ function emptyTokenTotals() {
2029
+ return {
2030
+ input: 0,
2031
+ output: 0,
2032
+ reasoning: 0,
2033
+ cached: 0,
2034
+ cacheWrite: 0
2035
+ };
2110
2036
  }
2111
- function numberField$1(value, key) {
2112
- const field = value[key];
2113
- return typeof field === "number" && Number.isFinite(field) ? field : void 0;
2037
+ function hasTokenTotals(usage) {
2038
+ return usage.input > 0 || usage.output > 0 || usage.reasoning > 0 || usage.cached > 0 || usage.cacheWrite > 0;
2114
2039
  }
2115
- function timestamp(value) {
2116
- if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
2117
- if (typeof value === "string" && value.length > 0) {
2118
- const parsed = Date.parse(value);
2119
- return Number.isFinite(parsed) ? parsed : void 0;
2120
- }
2040
+ function addUsage(metrics, usage) {
2041
+ const parsed = readUsage(usage);
2042
+ metrics.inputTokens += parsed.input;
2043
+ metrics.outputTokens += parsed.output;
2044
+ metrics.reasoningTokens += parsed.reasoning;
2045
+ metrics.cachedTokens += parsed.cached;
2046
+ metrics.cacheWriteTokens += parsed.cacheWrite;
2121
2047
  }
2122
- //#endregion
2123
- //#region src/contract/intake/code-agent-session.ts
2124
- /** The single per-line rule. Both the string path and the file path call this,
2125
- * so malformed-line handling and entry validation cannot drift apart. */
2126
- function readCodeAgentJsonlLine(line, lineNumber) {
2127
- const trimmed = line.trim();
2128
- if (!trimmed) return void 0;
2129
- try {
2130
- return {
2131
- kind: "entry",
2132
- lineNumber,
2133
- entry: JSON.parse(trimmed)
2134
- };
2135
- } catch {
2136
- return {
2137
- kind: "malformed",
2138
- lineNumber
2139
- };
2140
- }
2048
+ function addUsageTo(totals, usage) {
2049
+ const parsed = readUsage(usage);
2050
+ totals.input += parsed.input;
2051
+ totals.output += parsed.output;
2052
+ totals.reasoning += parsed.reasoning;
2053
+ totals.cached += parsed.cached;
2054
+ totals.cacheWrite += parsed.cacheWrite;
2141
2055
  }
2142
- function parseCodeAgentJsonl(jsonl) {
2143
- const entries = [];
2144
- let malformedLines = 0;
2145
- let lineNumber = 0;
2146
- for (const line of jsonl.split("\n")) {
2147
- lineNumber += 1;
2148
- const read = readCodeAgentJsonlLine(line, lineNumber);
2149
- if (!read) continue;
2150
- if (read.kind === "malformed") malformedLines += 1;
2151
- else entries.push(read.entry);
2152
- }
2056
+ function readUsage(usage) {
2057
+ const parsed = extractUsage(usage);
2153
2058
  return {
2154
- entries,
2155
- malformedLines
2059
+ input: parsed?.input ?? 0,
2060
+ output: parsed?.output ?? 0,
2061
+ reasoning: parsed?.reasoning ?? 0,
2062
+ cached: parsed?.cached ?? 0,
2063
+ cacheWrite: parsed?.cacheWrite ?? 0
2156
2064
  };
2157
2065
  }
2158
- /** Reads a transcript one line at a time and never holds the file as a single
2159
- * string. `parseCodeAgentJsonl` needs the whole file in one string, so a
2160
- * session above V8's ~512MB string ceiling throws `ERR_STRING_TOO_LONG` and
2161
- * cannot be ingested at all; the largest real Codex rollout on record is 695MB.
2162
- *
2163
- * Lines break on `\n` only, which is what the string path's `split('\n')` does.
2164
- * `node:readline` also breaks on a bare `\r`, so it is deliberately not used
2165
- * here: a lone carriage return inside a line must stay inside that line for the
2166
- * two paths to report the same malformed count. */
2167
- async function* streamCodeAgentJsonlFile(path) {
2168
- const stream = createReadStream(path, { encoding: "utf8" });
2169
- let pending = "";
2170
- let lineNumber = 0;
2171
- try {
2172
- for await (const chunk of stream) {
2173
- pending += chunk;
2174
- let start = 0;
2175
- for (let at = pending.indexOf("\n"); at !== -1; at = pending.indexOf("\n", start)) {
2176
- lineNumber += 1;
2177
- const read = readCodeAgentJsonlLine(pending.slice(start, at), lineNumber);
2178
- if (read) yield read;
2179
- start = at + 1;
2180
- }
2181
- pending = pending.slice(start);
2066
+ function resolveSessionCost(explicit, metrics, model) {
2067
+ if (explicit) {
2068
+ if (explicit.kind === "uncaptured") {
2069
+ if (explicit.usd !== null) throw new Error("uncaptured cost must have usd: null");
2070
+ return {
2071
+ costUsd: null,
2072
+ costProvenance: explicit
2073
+ };
2182
2074
  }
2183
- } finally {
2184
- stream.destroy();
2075
+ if (explicit.kind === "lower-bound") {
2076
+ if (explicit.usd !== null) throw new Error("lower-bound cost must have usd: null");
2077
+ if (!Number.isFinite(explicit.knownLowerBoundUsd) || explicit.knownLowerBoundUsd <= 0) throw new Error("lower-bound cost must have a finite, positive knownLowerBoundUsd");
2078
+ return {
2079
+ costUsd: null,
2080
+ costProvenance: explicit
2081
+ };
2082
+ }
2083
+ if (!Number.isFinite(explicit.usd) || explicit.usd < 0) throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
2084
+ return {
2085
+ costUsd: explicit.usd,
2086
+ costProvenance: explicit
2087
+ };
2185
2088
  }
2186
- const last = readCodeAgentJsonlLine(pending, lineNumber + 1);
2187
- if (last) yield last;
2188
- }
2189
- /** Streaming counterpart to `parseCodeAgentJsonl` for a transcript on disk.
2190
- * It returns the same shape, so a caller that holds every entry keeps working
2191
- * above the string ceiling. The entry array still grows with the transcript;
2192
- * consume `streamCodeAgentJsonlFile` directly when memory must stay flat. */
2193
- async function parseCodeAgentJsonlFile(path) {
2194
- const entries = [];
2195
- let malformedLines = 0;
2196
- for await (const read of streamCodeAgentJsonlFile(path)) if (read.kind === "malformed") malformedLines += 1;
2197
- else entries.push(read.entry);
2198
- return {
2199
- entries,
2200
- malformedLines
2201
- };
2202
- }
2203
- function fromCodexSession(options) {
2204
- return fromCodeAgentSession("codex", options);
2205
- }
2206
- function fromClaudeCodeSession(options) {
2207
- return fromCodeAgentSession("claude-code", options);
2208
- }
2209
- function fromOpenCodeSession(options) {
2210
- return fromCodeAgentSession("opencode", options);
2211
- }
2212
- function fromKimiCodeSession(options) {
2213
- return fromCodeAgentSession("kimi-code", options);
2214
- }
2215
- /** A native Pi session (`~/.pi/agent/sessions/<cwd>/<ts>_<id>.jsonl`), parsed entries in order. */
2216
- function fromPiSession(options) {
2217
- return fromCodeAgentSession("pi", options);
2218
- }
2219
- /** The graph IR a Pi graph run exports (`nodes[].ir`), which is not a Pi session. */
2220
- function fromPiGraphSession(options) {
2221
- return fromCodeAgentSession("pi-graph", options);
2222
- }
2223
- function fromCodeAgentSession(source, options) {
2224
- const entries = options.entries.filter(isRecord);
2225
- if (entries.length === 0) return {
2226
- runs: [],
2227
- diagnostics: [{
2228
- source,
2229
- sessionId: fallbackSessionId(source, options.sourcePath),
2230
- sourcePath: options.sourcePath,
2231
- entries: 0,
2232
- malformedLines: options.malformedLines ?? 0,
2233
- hasExplicitTerminalSignal: false,
2234
- hasFinalOutput: false,
2235
- hasQualityLabel: false,
2236
- hasTokenUsage: false,
2237
- hasCost: false,
2238
- costKind: "uncaptured",
2239
- warnings: ["no parseable session entries"]
2240
- }],
2241
- metrics: [],
2242
- observations: []
2243
- };
2244
- const metrics = metricsFor(source, entries);
2245
- const sessionId = sessionIdFromEntries(source, entries) ?? fallbackSessionId(source, options.sourcePath);
2246
- const observation = observeCodeAgentSession({
2247
- source,
2248
- entries,
2249
- sourcePath: options.sourcePath,
2250
- execution: options.execution
2251
- });
2252
- applyObservedSurfaceMetrics(metrics, observation);
2253
- const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
2254
- const tokenUsage = {
2255
- input: metrics.inputTokens,
2256
- output: metrics.outputTokens,
2257
- ...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
2258
- ...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
2259
- ...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
2260
- };
2261
- const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
2262
- const qualityScore = finiteScore(options.score);
2263
- const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
2264
- const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
2265
- const configHash = options.configHash ?? hashJson({
2266
- source,
2267
- model,
2268
- sourcePath: options.sourcePath ?? null,
2269
- cwd: cwdFromEntries(entries) ?? null,
2270
- entryCount: entries.length
2271
- });
2272
- const explicitTerminal = observation.terminal.explicit;
2273
- const warnings = diagnosticsFor(metrics, {
2274
- model,
2275
- explicitTerminal,
2276
- malformedLines: options.malformedLines ?? 0,
2277
- hasQualityLabel: qualityScore !== void 0,
2278
- invalidQualityScore,
2279
- costKind: costProvenance.kind
2280
- });
2281
- const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
2282
- const processExitCode = options.execution?.exitCode;
2283
- const splitTag = options.splitTag ?? "holdout";
2284
- return {
2285
- runs: [{
2286
- runId: `${source}:${sessionId}`,
2287
- experimentId: options.experimentId ?? `${source}-local-sessions`,
2288
- candidateId: options.candidateId ?? model,
2289
- seed: options.seed ?? stableSeed(sessionId),
2290
- model,
2291
- promptHash,
2292
- configHash,
2293
- commitSha: options.commitSha ?? "local-session",
2294
- wallMs: metrics.wallMs,
2089
+ if (metrics.observedCostCaptured) {
2090
+ const costProvenance = {
2091
+ kind: "observed",
2092
+ usd: metrics.observedCostUsd
2093
+ };
2094
+ return {
2095
+ costUsd: metrics.observedCostUsd,
2096
+ costProvenance
2097
+ };
2098
+ }
2099
+ if ((metrics.inputTokens > 0 || metrics.outputTokens > 0) && isModelPriced(model)) {
2100
+ const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
2101
+ return {
2295
2102
  costUsd,
2296
- costProvenance,
2297
- tokenUsage,
2298
- terminalOutcome,
2299
- ...terminalOutcome === "failed" ? { terminalFailureReason: processExitCode !== void 0 && processExitCode !== 0 ? `process exited with code ${processExitCode}` : metrics.turnsAborted > 0 ? "turn aborted" : "session reported terminal failure" } : {},
2300
- outcome: {
2301
- ...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
2302
- raw: {
2303
- entries: metrics.entries,
2304
- user_messages: metrics.userMessages,
2305
- assistant_messages: metrics.assistantMessages,
2306
- reasoning_items: metrics.reasoningItems,
2307
- tool_calls: metrics.toolCalls,
2308
- tool_outputs: metrics.toolOutputs,
2309
- tool_errors: metrics.toolErrors,
2310
- execution_error_count: metrics.toolErrors,
2311
- process_error_count: terminalOutcome === "failed" ? 1 : 0,
2312
- unclassified_error_count: metrics.unclassifiedErrors,
2313
- ...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
2314
- patch_attempts: metrics.patchAttempts,
2315
- patch_successes: metrics.patchSuccesses,
2316
- patch_failures: metrics.patchFailures,
2317
- turns_started: metrics.turnsStarted,
2318
- turns_completed: metrics.turnsCompleted,
2319
- turns_aborted: metrics.turnsAborted,
2320
- context_compactions: metrics.contextCompactions,
2321
- mcp_calls: metrics.mcpCalls,
2322
- subagent_calls: metrics.subagentCalls,
2323
- skill_calls: metrics.skillCalls,
2324
- hook_calls: metrics.hookCalls,
2325
- web_calls: metrics.webCalls,
2326
- code_actions: metrics.codeActions,
2327
- pr_links: metrics.prLinks,
2328
- file_snapshots: metrics.fileSnapshots,
2329
- graph_nodes: metrics.graphNodes,
2330
- graph_edges: metrics.graphEdges,
2331
- action_candidates: metrics.actionCandidates,
2332
- verification_reports: metrics.verificationReports,
2333
- completion_decisions: metrics.completionDecisions,
2334
- reliability_rows: metrics.reliabilityRows,
2335
- reliability_lift: metrics.reliabilityLift,
2336
- input_tokens: metrics.inputTokens,
2337
- output_tokens: metrics.outputTokens,
2338
- reasoning_tokens: metrics.reasoningTokens,
2339
- cached_tokens: metrics.cachedTokens,
2340
- cache_write_tokens: metrics.cacheWriteTokens,
2341
- observed_cost_usd: metrics.observedCostUsd,
2342
- observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
2343
- estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
2344
- process_score: metrics.processScore,
2345
- explicit_terminal_signal: explicitTerminal ? 1 : 0,
2346
- quality_label_present: qualityScore !== void 0 ? 1 : 0,
2347
- cost_observed: costProvenance.kind === "observed" ? 1 : 0,
2348
- cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
2349
- cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
2350
- cost_unknown: costProvenance.usd === null ? 1 : 0
2351
- }
2352
- },
2353
- splitTag,
2354
- scenarioId: options.scenarioId ?? sessionId
2355
- }],
2356
- diagnostics: [{
2357
- source,
2358
- sessionId,
2359
- sourcePath: options.sourcePath,
2360
- entries: entries.length,
2361
- malformedLines: options.malformedLines ?? 0,
2362
- hasExplicitTerminalSignal: explicitTerminal,
2363
- hasFinalOutput: observation.finalText !== null,
2364
- hasQualityLabel: qualityScore !== void 0,
2365
- hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
2366
- hasCost: costProvenance.usd !== null,
2367
- costKind: costProvenance.kind,
2368
- warnings
2369
- }],
2370
- metrics: [metrics],
2371
- observations: [{
2372
- ...observation,
2373
- sessionId
2374
- }]
2375
- };
2376
- }
2377
- function terminalOutcomeFromSession(terminal) {
2378
- if (!terminal.explicit) return "unknown";
2379
- if (terminal.status === "completed") return "succeeded";
2380
- if (terminal.status === "failed") return "failed";
2381
- return "unknown";
2382
- }
2383
- function metricsFor(source, entries) {
2384
- switch (source) {
2385
- case "codex": return codexMetrics(entries);
2386
- case "claude-code": return claudeCodeMetrics(entries);
2387
- case "opencode": return openCodeMetrics(entries);
2388
- case "kimi-code": return kimiCodeMetrics(entries);
2389
- case "pi": return piSessionMetrics(entries);
2390
- case "pi-graph": return piGraphMetrics(entries);
2103
+ costProvenance: {
2104
+ kind: "estimated",
2105
+ usd: costUsd
2106
+ }
2107
+ };
2391
2108
  }
2109
+ return {
2110
+ costUsd: null,
2111
+ costProvenance: {
2112
+ kind: "uncaptured",
2113
+ usd: null
2114
+ }
2115
+ };
2392
2116
  }
2117
+ //#endregion
2118
+ //#region src/contract/intake/code-agent-codex-metrics.ts
2393
2119
  function codexMetrics(entries) {
2394
2120
  const metrics = emptyMetrics(entries.length);
2395
2121
  const startedToolIds = /* @__PURE__ */ new Set();
@@ -2408,7 +2134,7 @@ function codexMetrics(entries) {
2408
2134
  if (entryType === "turn.started") metrics.turnsStarted += 1;
2409
2135
  if (entryType === "turn.completed") {
2410
2136
  metrics.turnsCompleted += 1;
2411
- addCodexExecUsage(metrics, record(entry.usage));
2137
+ addUsage(metrics, record(entry.usage));
2412
2138
  }
2413
2139
  if (entryType === "turn.failed") metrics.turnsAborted += 1;
2414
2140
  if (entryType === "error") metrics.unclassifiedErrors += 1;
@@ -2496,14 +2222,6 @@ function codexExecToolFailed(itemType, item) {
2496
2222
  if (itemType === "mcp_tool_call") return status === "failed" || record(item.error) !== null;
2497
2223
  return itemType === "collab_tool_call" && status === "failed";
2498
2224
  }
2499
- function addCodexExecUsage(metrics, usage) {
2500
- const parsed = readUsage(usage);
2501
- metrics.inputTokens += parsed.input;
2502
- metrics.outputTokens += parsed.output;
2503
- metrics.reasoningTokens += parsed.reasoning;
2504
- metrics.cachedTokens += parsed.cached;
2505
- metrics.cacheWriteTokens += parsed.cacheWrite;
2506
- }
2507
2225
  function setCodexCumulativeUsage(metrics, usage) {
2508
2226
  const parsed = readUsage(usage);
2509
2227
  metrics.inputTokens = Math.max(metrics.inputTokens, parsed.input);
@@ -2512,6 +2230,12 @@ function setCodexCumulativeUsage(metrics, usage) {
2512
2230
  metrics.cachedTokens = Math.max(metrics.cachedTokens, parsed.cached);
2513
2231
  metrics.cacheWriteTokens = Math.max(metrics.cacheWriteTokens, parsed.cacheWrite);
2514
2232
  }
2233
+ function codexProcessScore(metrics) {
2234
+ const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
2235
+ return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .5 : 0, metrics);
2236
+ }
2237
+ //#endregion
2238
+ //#region src/contract/intake/code-agent-harness-metrics.ts
2515
2239
  function claudeCodeMetrics(entries) {
2516
2240
  const metrics = emptyMetrics(entries.length);
2517
2241
  let startedAt;
@@ -2708,170 +2432,302 @@ function piGraphMetrics(entries) {
2708
2432
  metrics.turnsCompleted += 1;
2709
2433
  }
2710
2434
  }
2711
- const averageReliability = numberField(entry, "averageNodeReliability");
2712
- if (averageReliability !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, averageReliability);
2713
- const pessimisticPath = numberField(entry, "pessimisticPathEstimate");
2714
- if (pessimisticPath !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, pessimisticPath);
2715
- const rows = Array.isArray(entry.rows) ? entry.rows : [];
2716
- metrics.reliabilityRows += rows.length;
2717
- for (const row of rows) {
2718
- const obj = record(row);
2719
- if (!obj) continue;
2720
- const lift = numberField(obj, "lift");
2721
- if (lift !== void 0) metrics.reliabilityLift = Math.max(metrics.reliabilityLift, lift);
2722
- const validated = numberField(obj, "validatedSuccessEstimate");
2723
- if (validated !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, validated);
2724
- }
2435
+ const averageReliability = numberField(entry, "averageNodeReliability");
2436
+ if (averageReliability !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, averageReliability);
2437
+ const pessimisticPath = numberField(entry, "pessimisticPathEstimate");
2438
+ if (pessimisticPath !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, pessimisticPath);
2439
+ const rows = Array.isArray(entry.rows) ? entry.rows : [];
2440
+ metrics.reliabilityRows += rows.length;
2441
+ for (const row of rows) {
2442
+ const obj = record(row);
2443
+ if (!obj) continue;
2444
+ const lift = numberField(obj, "lift");
2445
+ if (lift !== void 0) metrics.reliabilityLift = Math.max(metrics.reliabilityLift, lift);
2446
+ const validated = numberField(obj, "validatedSuccessEstimate");
2447
+ if (validated !== void 0) bestReliabilityScore = maxOptional(bestReliabilityScore, validated);
2448
+ }
2449
+ }
2450
+ metrics.processScore = bestReliabilityScore !== void 0 ? clamp01(bestReliabilityScore) : metrics.completionDecisions > 0 ? 1 : 0;
2451
+ return metrics;
2452
+ }
2453
+ function claudeProcessScore(metrics) {
2454
+ return penalizeErrors(metrics.prLinks > 0 ? 1 : metrics.assistantMessages > 0 ? .75 : 0, metrics);
2455
+ }
2456
+ function terminalProcessScore(metrics) {
2457
+ const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
2458
+ return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .75 : metrics.toolCalls > 0 ? .5 : 0, metrics);
2459
+ }
2460
+ function maxOptional(current, next) {
2461
+ return current === void 0 ? next : Math.max(current, next);
2462
+ }
2463
+ //#endregion
2464
+ //#region src/contract/intake/code-agent-jsonl.ts
2465
+ /** The single per-line rule. Both the string path and the file path call this,
2466
+ * so malformed-line handling and entry validation cannot drift apart. */
2467
+ function readCodeAgentJsonlLine(line, lineNumber) {
2468
+ const trimmed = line.trim();
2469
+ if (!trimmed) return void 0;
2470
+ try {
2471
+ return {
2472
+ kind: "entry",
2473
+ lineNumber,
2474
+ entry: JSON.parse(trimmed)
2475
+ };
2476
+ } catch {
2477
+ return {
2478
+ kind: "malformed",
2479
+ lineNumber
2480
+ };
2481
+ }
2482
+ }
2483
+ function parseCodeAgentJsonl(jsonl) {
2484
+ const entries = [];
2485
+ let malformedLines = 0;
2486
+ let lineNumber = 0;
2487
+ for (const line of jsonl.split("\n")) {
2488
+ lineNumber += 1;
2489
+ const read = readCodeAgentJsonlLine(line, lineNumber);
2490
+ if (!read) continue;
2491
+ if (read.kind === "malformed") malformedLines += 1;
2492
+ else entries.push(read.entry);
2493
+ }
2494
+ return {
2495
+ entries,
2496
+ malformedLines
2497
+ };
2498
+ }
2499
+ /** Reads a transcript one line at a time and never holds the file as a single
2500
+ * string. `parseCodeAgentJsonl` needs the whole file in one string, so a
2501
+ * session above V8's ~512MB string ceiling throws `ERR_STRING_TOO_LONG` and
2502
+ * cannot be ingested at all; the largest real Codex rollout on record is 695MB.
2503
+ *
2504
+ * Lines break on `\n` only, which is what the string path's `split('\n')` does.
2505
+ * `node:readline` also breaks on a bare `\r`, so it is deliberately not used
2506
+ * here: a lone carriage return inside a line must stay inside that line for the
2507
+ * two paths to report the same malformed count. */
2508
+ async function* streamCodeAgentJsonlFile(path) {
2509
+ const stream = createReadStream(path, { encoding: "utf8" });
2510
+ let pending = "";
2511
+ let lineNumber = 0;
2512
+ try {
2513
+ for await (const chunk of stream) {
2514
+ pending += chunk;
2515
+ let start = 0;
2516
+ for (let at = pending.indexOf("\n"); at !== -1; at = pending.indexOf("\n", start)) {
2517
+ lineNumber += 1;
2518
+ const read = readCodeAgentJsonlLine(pending.slice(start, at), lineNumber);
2519
+ if (read) yield read;
2520
+ start = at + 1;
2521
+ }
2522
+ pending = pending.slice(start);
2523
+ }
2524
+ } finally {
2525
+ stream.destroy();
2725
2526
  }
2726
- metrics.processScore = bestReliabilityScore !== void 0 ? clamp01(bestReliabilityScore) : metrics.completionDecisions > 0 ? 1 : 0;
2727
- return metrics;
2527
+ const last = readCodeAgentJsonlLine(pending, lineNumber + 1);
2528
+ if (last) yield last;
2728
2529
  }
2729
- function emptyMetrics(entries) {
2530
+ /** Streaming counterpart to `parseCodeAgentJsonl` for a transcript on disk.
2531
+ * It returns the same shape, so a caller that holds every entry keeps working
2532
+ * above the string ceiling. The entry array still grows with the transcript;
2533
+ * consume `streamCodeAgentJsonlFile` directly when memory must stay flat. */
2534
+ async function parseCodeAgentJsonlFile(path) {
2535
+ const entries = [];
2536
+ let malformedLines = 0;
2537
+ for await (const read of streamCodeAgentJsonlFile(path)) if (read.kind === "malformed") malformedLines += 1;
2538
+ else entries.push(read.entry);
2730
2539
  return {
2731
2540
  entries,
2732
- userMessages: 0,
2733
- assistantMessages: 0,
2734
- reasoningItems: 0,
2735
- toolCalls: 0,
2736
- toolOutputs: 0,
2737
- toolErrors: 0,
2738
- unclassifiedErrors: 0,
2739
- patchAttempts: 0,
2740
- patchSuccesses: 0,
2741
- patchFailures: 0,
2742
- turnsStarted: 0,
2743
- turnsCompleted: 0,
2744
- turnsAborted: 0,
2745
- contextCompactions: 0,
2746
- mcpCalls: 0,
2747
- subagentCalls: 0,
2748
- skillCalls: 0,
2749
- hookCalls: 0,
2750
- webCalls: 0,
2751
- codeActions: 0,
2752
- prLinks: 0,
2753
- fileSnapshots: 0,
2754
- graphNodes: 0,
2755
- graphEdges: 0,
2756
- actionCandidates: 0,
2757
- verificationReports: 0,
2758
- completionDecisions: 0,
2759
- reliabilityRows: 0,
2760
- reliabilityLift: 0,
2761
- inputTokens: 0,
2762
- outputTokens: 0,
2763
- reasoningTokens: 0,
2764
- cachedTokens: 0,
2765
- cacheWriteTokens: 0,
2766
- observedCostUsd: 0,
2767
- observedCostCaptured: false,
2768
- wallMs: 0,
2769
- processScore: 0
2541
+ malformedLines
2770
2542
  };
2771
2543
  }
2772
- function codexProcessScore(metrics) {
2773
- const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
2774
- return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .5 : 0, metrics);
2544
+ //#endregion
2545
+ //#region src/contract/intake/code-agent-session.ts
2546
+ function fromCodexSession(options) {
2547
+ return fromCodeAgentSession("codex", options);
2775
2548
  }
2776
- function claudeProcessScore(metrics) {
2777
- return penalizeErrors(metrics.prLinks > 0 ? 1 : metrics.assistantMessages > 0 ? .75 : 0, metrics);
2549
+ function fromClaudeCodeSession(options) {
2550
+ return fromCodeAgentSession("claude-code", options);
2778
2551
  }
2779
- function terminalProcessScore(metrics) {
2780
- const terminalTurns = metrics.turnsCompleted + metrics.turnsAborted;
2781
- return penalizeErrors(terminalTurns > 0 ? metrics.turnsCompleted / terminalTurns : metrics.assistantMessages > 0 ? .75 : metrics.toolCalls > 0 ? .5 : 0, metrics);
2552
+ function fromOpenCodeSession(options) {
2553
+ return fromCodeAgentSession("opencode", options);
2782
2554
  }
2783
- function penalizeErrors(base, metrics) {
2784
- const operations = Math.max(1, metrics.toolCalls + metrics.patchAttempts);
2785
- return clamp01(base * (1 - .5 * Math.min(1, (metrics.toolErrors + metrics.patchFailures) / operations)));
2555
+ function fromKimiCodeSession(options) {
2556
+ return fromCodeAgentSession("kimi-code", options);
2786
2557
  }
2787
- function emptyTokenTotals() {
2558
+ /** A native Pi session (`~/.pi/agent/sessions/<cwd>/<ts>_<id>.jsonl`), parsed entries in order. */
2559
+ function fromPiSession(options) {
2560
+ return fromCodeAgentSession("pi", options);
2561
+ }
2562
+ function fromCodeAgentSession(source, options) {
2563
+ const entries = options.entries.filter(isRecord);
2564
+ if (entries.length === 0) return {
2565
+ runs: [],
2566
+ diagnostics: [{
2567
+ source,
2568
+ sessionId: fallbackSessionId$1(source, options.sourcePath),
2569
+ sourcePath: options.sourcePath,
2570
+ entries: 0,
2571
+ malformedLines: options.malformedLines ?? 0,
2572
+ hasExplicitTerminalSignal: false,
2573
+ hasFinalOutput: false,
2574
+ hasQualityLabel: false,
2575
+ hasTokenUsage: false,
2576
+ hasCost: false,
2577
+ costKind: "uncaptured",
2578
+ warnings: ["no parseable session entries"]
2579
+ }],
2580
+ metrics: [],
2581
+ observations: []
2582
+ };
2583
+ const metrics = metricsFor(source, entries);
2584
+ const sessionId = sessionIdFromEntries(source, entries) ?? fallbackSessionId$1(source, options.sourcePath);
2585
+ const observation = observeCodeAgentSession({
2586
+ source,
2587
+ entries,
2588
+ sourcePath: options.sourcePath,
2589
+ execution: options.execution
2590
+ });
2591
+ applyObservedSurfaceMetrics(metrics, observation);
2592
+ const model = withSnapshot(options.model ?? modelFromEntries(source, entries) ?? source);
2593
+ const tokenUsage = {
2594
+ input: metrics.inputTokens,
2595
+ output: metrics.outputTokens,
2596
+ ...metrics.reasoningTokens > 0 ? { reasoning: metrics.reasoningTokens } : {},
2597
+ ...metrics.cachedTokens > 0 ? { cached: metrics.cachedTokens } : {},
2598
+ ...metrics.cacheWriteTokens > 0 ? { cacheWrite: metrics.cacheWriteTokens } : {}
2599
+ };
2600
+ const { costUsd, costProvenance } = resolveSessionCost(options.costProvenance, metrics, model);
2601
+ const qualityScore = finiteScore(options.score);
2602
+ const invalidQualityScore = options.score !== void 0 && qualityScore === void 0;
2603
+ const promptHash = options.promptHash ?? hashString(`prompt:${source}:${firstUserText(entries) ?? ""}`);
2604
+ const configHash = options.configHash ?? hashCanonical({
2605
+ source,
2606
+ model,
2607
+ sourcePath: options.sourcePath ?? null,
2608
+ cwd: cwdFromEntries(entries) ?? null,
2609
+ entryCount: entries.length
2610
+ });
2611
+ const explicitTerminal = observation.terminal.explicit;
2612
+ const warnings = diagnosticsFor(metrics, {
2613
+ model,
2614
+ explicitTerminal,
2615
+ malformedLines: options.malformedLines ?? 0,
2616
+ hasQualityLabel: qualityScore !== void 0,
2617
+ invalidQualityScore,
2618
+ costKind: costProvenance.kind
2619
+ });
2620
+ const terminalOutcome = terminalOutcomeFromSession(observation.terminal);
2621
+ const processExitCode = options.execution?.exitCode;
2622
+ const splitTag = options.splitTag ?? "holdout";
2788
2623
  return {
2789
- input: 0,
2790
- output: 0,
2791
- reasoning: 0,
2792
- cached: 0,
2793
- cacheWrite: 0
2624
+ runs: [{
2625
+ runId: `${source}:${sessionId}`,
2626
+ experimentId: options.experimentId ?? `${source}-local-sessions`,
2627
+ candidateId: options.candidateId ?? model,
2628
+ seed: options.seed ?? stableSeed(sessionId),
2629
+ model,
2630
+ promptHash,
2631
+ configHash,
2632
+ commitSha: options.commitSha ?? "local-session",
2633
+ wallMs: metrics.wallMs,
2634
+ costUsd,
2635
+ costProvenance,
2636
+ tokenUsage,
2637
+ terminalOutcome,
2638
+ ...terminalOutcome === "failed" ? { terminalFailureReason: processExitCode !== void 0 && processExitCode !== 0 ? `process exited with code ${processExitCode}` : metrics.turnsAborted > 0 ? "turn aborted" : "session reported terminal failure" } : {},
2639
+ outcome: {
2640
+ ...qualityScore === void 0 ? {} : splitTag === "holdout" ? { holdoutScore: qualityScore } : { searchScore: qualityScore },
2641
+ raw: {
2642
+ entries: metrics.entries,
2643
+ user_messages: metrics.userMessages,
2644
+ assistant_messages: metrics.assistantMessages,
2645
+ reasoning_items: metrics.reasoningItems,
2646
+ tool_calls: metrics.toolCalls,
2647
+ tool_outputs: metrics.toolOutputs,
2648
+ tool_errors: metrics.toolErrors,
2649
+ execution_error_count: metrics.toolErrors,
2650
+ process_error_count: terminalOutcome === "failed" ? 1 : 0,
2651
+ unclassified_error_count: metrics.unclassifiedErrors,
2652
+ ...processExitCode !== void 0 ? { process_exit_code: processExitCode } : {},
2653
+ patch_attempts: metrics.patchAttempts,
2654
+ patch_successes: metrics.patchSuccesses,
2655
+ patch_failures: metrics.patchFailures,
2656
+ turns_started: metrics.turnsStarted,
2657
+ turns_completed: metrics.turnsCompleted,
2658
+ turns_aborted: metrics.turnsAborted,
2659
+ context_compactions: metrics.contextCompactions,
2660
+ mcp_calls: metrics.mcpCalls,
2661
+ subagent_calls: metrics.subagentCalls,
2662
+ skill_calls: metrics.skillCalls,
2663
+ hook_calls: metrics.hookCalls,
2664
+ web_calls: metrics.webCalls,
2665
+ code_actions: metrics.codeActions,
2666
+ pr_links: metrics.prLinks,
2667
+ file_snapshots: metrics.fileSnapshots,
2668
+ graph_nodes: metrics.graphNodes,
2669
+ graph_edges: metrics.graphEdges,
2670
+ action_candidates: metrics.actionCandidates,
2671
+ verification_reports: metrics.verificationReports,
2672
+ completion_decisions: metrics.completionDecisions,
2673
+ reliability_rows: metrics.reliabilityRows,
2674
+ reliability_lift: metrics.reliabilityLift,
2675
+ input_tokens: metrics.inputTokens,
2676
+ output_tokens: metrics.outputTokens,
2677
+ reasoning_tokens: metrics.reasoningTokens,
2678
+ cached_tokens: metrics.cachedTokens,
2679
+ cache_write_tokens: metrics.cacheWriteTokens,
2680
+ observed_cost_usd: metrics.observedCostUsd,
2681
+ observed_cost_captured: metrics.observedCostCaptured ? 1 : 0,
2682
+ estimated_cost_usd: costProvenance.kind === "estimated" ? costProvenance.usd : 0,
2683
+ process_score: metrics.processScore,
2684
+ explicit_terminal_signal: explicitTerminal ? 1 : 0,
2685
+ quality_label_present: qualityScore !== void 0 ? 1 : 0,
2686
+ cost_observed: costProvenance.kind === "observed" ? 1 : 0,
2687
+ cost_estimated: costProvenance.kind === "estimated" ? 1 : 0,
2688
+ cost_uncaptured: costProvenance.kind === "uncaptured" ? 1 : 0,
2689
+ cost_unknown: costProvenance.usd === null ? 1 : 0
2690
+ }
2691
+ },
2692
+ splitTag,
2693
+ scenarioId: options.scenarioId ?? sessionId
2694
+ }],
2695
+ diagnostics: [{
2696
+ source,
2697
+ sessionId,
2698
+ sourcePath: options.sourcePath,
2699
+ entries: entries.length,
2700
+ malformedLines: options.malformedLines ?? 0,
2701
+ hasExplicitTerminalSignal: explicitTerminal,
2702
+ hasFinalOutput: observation.finalText !== null,
2703
+ hasQualityLabel: qualityScore !== void 0,
2704
+ hasTokenUsage: metrics.inputTokens > 0 || metrics.outputTokens > 0 || metrics.reasoningTokens > 0 || metrics.cachedTokens > 0 || metrics.cacheWriteTokens > 0,
2705
+ hasCost: costProvenance.usd !== null,
2706
+ costKind: costProvenance.kind,
2707
+ warnings
2708
+ }],
2709
+ metrics: [metrics],
2710
+ observations: [{
2711
+ ...observation,
2712
+ sessionId
2713
+ }]
2794
2714
  };
2795
2715
  }
2796
- function hasTokenTotals(usage) {
2797
- return usage.input > 0 || usage.output > 0 || usage.reasoning > 0 || usage.cached > 0 || usage.cacheWrite > 0;
2798
- }
2799
- function addUsage(metrics, usage) {
2800
- const parsed = readUsage(usage);
2801
- metrics.inputTokens += parsed.input;
2802
- metrics.outputTokens += parsed.output;
2803
- metrics.reasoningTokens += parsed.reasoning;
2804
- metrics.cachedTokens += parsed.cached;
2805
- metrics.cacheWriteTokens += parsed.cacheWrite;
2806
- }
2807
- function addUsageTo(totals, usage) {
2808
- const parsed = readUsage(usage);
2809
- totals.input += parsed.input;
2810
- totals.output += parsed.output;
2811
- totals.reasoning += parsed.reasoning;
2812
- totals.cached += parsed.cached;
2813
- totals.cacheWrite += parsed.cacheWrite;
2814
- }
2815
- function readUsage(usage) {
2816
- const parsed = extractUsage(usage);
2817
- return {
2818
- input: parsed?.input ?? 0,
2819
- output: parsed?.output ?? 0,
2820
- reasoning: parsed?.reasoning ?? 0,
2821
- cached: parsed?.cached ?? 0,
2822
- cacheWrite: parsed?.cacheWrite ?? 0
2823
- };
2716
+ function terminalOutcomeFromSession(terminal) {
2717
+ if (!terminal.explicit) return "unknown";
2718
+ if (terminal.status === "completed") return "succeeded";
2719
+ if (terminal.status === "failed") return "failed";
2720
+ return "unknown";
2824
2721
  }
2825
- function resolveSessionCost(explicit, metrics, model) {
2826
- if (explicit) {
2827
- if (explicit.kind === "uncaptured") {
2828
- if (explicit.usd !== null) throw new Error("uncaptured cost must have usd: null");
2829
- return {
2830
- costUsd: null,
2831
- costProvenance: explicit
2832
- };
2833
- }
2834
- if (explicit.kind === "lower-bound") {
2835
- if (explicit.usd !== null) throw new Error("lower-bound cost must have usd: null");
2836
- if (!Number.isFinite(explicit.knownLowerBoundUsd) || explicit.knownLowerBoundUsd <= 0) throw new Error("lower-bound cost must have a finite, positive knownLowerBoundUsd");
2837
- return {
2838
- costUsd: null,
2839
- costProvenance: explicit
2840
- };
2841
- }
2842
- if (!Number.isFinite(explicit.usd) || explicit.usd < 0) throw new Error(`${explicit.kind} cost must be a finite, non-negative USD amount`);
2843
- return {
2844
- costUsd: explicit.usd,
2845
- costProvenance: explicit
2846
- };
2847
- }
2848
- if (metrics.observedCostCaptured) {
2849
- const costProvenance = {
2850
- kind: "observed",
2851
- usd: metrics.observedCostUsd
2852
- };
2853
- return {
2854
- costUsd: metrics.observedCostUsd,
2855
- costProvenance
2856
- };
2857
- }
2858
- if ((metrics.inputTokens > 0 || metrics.outputTokens > 0) && isModelPriced(model)) {
2859
- const costUsd = estimateCost(metrics.inputTokens, metrics.outputTokens, model);
2860
- return {
2861
- costUsd,
2862
- costProvenance: {
2863
- kind: "estimated",
2864
- usd: costUsd
2865
- }
2866
- };
2722
+ function metricsFor(source, entries) {
2723
+ switch (source) {
2724
+ case "codex": return codexMetrics(entries);
2725
+ case "claude-code": return claudeCodeMetrics(entries);
2726
+ case "opencode": return openCodeMetrics(entries);
2727
+ case "kimi-code": return kimiCodeMetrics(entries);
2728
+ case "pi": return piSessionMetrics(entries);
2729
+ case "pi-graph": return piGraphMetrics(entries);
2867
2730
  }
2868
- return {
2869
- costUsd: null,
2870
- costProvenance: {
2871
- kind: "uncaptured",
2872
- usd: null
2873
- }
2874
- };
2875
2731
  }
2876
2732
  function diagnosticsFor(metrics, options) {
2877
2733
  const warnings = [];
@@ -2895,58 +2751,6 @@ function applyObservedSurfaceMetrics(metrics, observation) {
2895
2751
  metrics.webCalls = actions.filter((action) => action.surface === "web").length;
2896
2752
  metrics.codeActions = actions.filter((action) => action.surface === "code").length;
2897
2753
  }
2898
- function sessionIdFromEntries(source, entries) {
2899
- for (const entry of entries) {
2900
- if (source === "pi" && entry.type === "session") {
2901
- const id = stringField(entry, "id");
2902
- if (id) return id;
2903
- }
2904
- if (source === "codex") {
2905
- const threadId = stringField(entry, "thread_id");
2906
- if (threadId) return threadId;
2907
- const payload = record(entry.payload);
2908
- const id = payload ? stringField(payload, "id") : void 0;
2909
- if (id) return id;
2910
- }
2911
- const payload = record(record(entry.message)?.payload);
2912
- const sessionId = stringField(entry, "sessionID") ?? stringField(entry, "sessionId") ?? stringField(entry, "session_id") ?? stringField(payload ?? {}, "session_id");
2913
- if (sessionId) return sessionId;
2914
- }
2915
- }
2916
- function modelFromEntries(source, entries) {
2917
- if (source === "pi") return summarizeSessionInput("pi", { records: entries }).lastServedModel ?? void 0;
2918
- let providerModel;
2919
- for (const entry of entries) {
2920
- if (source === "codex") {
2921
- const payload = record(entry.payload);
2922
- const model = payload ? stringField(payload, "model") : void 0;
2923
- if (model) return model;
2924
- providerModel = providerModel ?? (payload ? stringField(payload, "model_provider") : void 0);
2925
- }
2926
- const openCodeModel = stringField(entry, "modelID");
2927
- if (openCodeModel) {
2928
- const provider = stringField(entry, "providerID");
2929
- return provider ? `${provider}/${openCodeModel}` : openCodeModel;
2930
- }
2931
- const modelObject = record(entry.model);
2932
- const nestedModel = stringField(modelObject ?? {}, "modelID");
2933
- if (nestedModel) {
2934
- const provider = stringField(modelObject ?? {}, "providerID");
2935
- return provider ? `${provider}/${nestedModel}` : nestedModel;
2936
- }
2937
- const message = record(entry.message);
2938
- const model = message ? stringField(message, "model") : void 0;
2939
- if (model) return model;
2940
- }
2941
- return providerModel;
2942
- }
2943
- function cwdFromEntries(entries) {
2944
- for (const entry of entries) {
2945
- const path = record(entry.path);
2946
- const cwd = stringField(entry, "cwd") ?? stringField(path ?? {}, "cwd") ?? stringField(path ?? {}, "root") ?? (record(entry.payload) ? stringField(record(entry.payload), "cwd") : void 0);
2947
- if (cwd) return cwd;
2948
- }
2949
- }
2950
2754
  /** Join the `text` of every part in a `[{ type: 'text', text }]` content list. */
2951
2755
  function textFromContentParts(content) {
2952
2756
  if (!Array.isArray(content)) return void 0;
@@ -2981,245 +2785,9 @@ function firstUserText(entries) {
2981
2785
  }
2982
2786
  }
2983
2787
  }
2984
- function fallbackSessionId(source, sourcePath) {
2985
- return hashString(`${source}:${sourcePath ?? "unknown-session"}`).slice(7, 27);
2986
- }
2987
- function withSnapshot(model) {
2988
- const trimmed = model.trim() || "unknown";
2989
- if (trimmed.includes("@") || /-\d{8}$/.test(trimmed) || /-\d{4}-\d{2}-\d{2}$/.test(trimmed)) return trimmed;
2990
- return `${trimmed}@observed-local`;
2991
- }
2992
- function hashJson(value) {
2993
- return hashCanonical(value);
2994
- }
2995
- function hashString(value) {
2996
- return `sha256:${createHash("sha256").update(value).digest("hex")}`;
2997
- }
2998
- function stableSeed(value) {
2999
- return createHash("sha256").update(value).digest().readUInt32BE(0);
3000
- }
3001
- function maxOptional(current, next) {
3002
- return current === void 0 ? next : Math.max(current, next);
3003
- }
3004
- function clamp01(value) {
3005
- if (!Number.isFinite(value)) return 0;
3006
- return Math.max(0, Math.min(1, value));
3007
- }
3008
2788
  function finiteScore(value) {
3009
2789
  return value !== void 0 && Number.isFinite(value) ? clamp01(value) : void 0;
3010
2790
  }
3011
- function isRecord(value) {
3012
- return value !== null && typeof value === "object" && !Array.isArray(value);
3013
- }
3014
- function record(value) {
3015
- return isRecord(value) ? value : null;
3016
- }
3017
- function stringField(obj, key) {
3018
- const value = obj[key];
3019
- return typeof value === "string" && value.length > 0 ? value : void 0;
3020
- }
3021
- function numberField(obj, key) {
3022
- const value = obj[key];
3023
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
3024
- }
3025
- function timestampMs(value) {
3026
- if (typeof value === "number" && Number.isFinite(value)) return value > 0xe8d4a51000 ? value : value * 1e3;
3027
- if (typeof value === "string" && value.length > 0) {
3028
- const parsed = Date.parse(value);
3029
- return Number.isFinite(parsed) ? parsed : void 0;
3030
- }
3031
- }
3032
- //#endregion
3033
- //#region src/contract/intake/code-agent-store.ts
3034
- const SESSION_SUFFIX = ".jsonl";
3035
- const REJECTION_SAMPLE_LIMIT = 8;
3036
- /**
3037
- * Read every session this run captured under `scope.root` and project them.
3038
- *
3039
- * A store directory that does not exist, or that holds no session the scope attributes, returns
3040
- * `status: 'unavailable'`. That is not an error: a run whose harness store was never synced back
3041
- * has no evidence, and saying so is the correct answer.
3042
- */
3043
- async function observeCodeAgentStore(scope) {
3044
- if (!isAbsolute(scope.root)) throw new TypeError(`observeCodeAgentStore: scope.root must be an absolute path to this run's own store, received ${JSON.stringify(scope.root)}`);
3045
- const workspaceRoot = scope.workspaceRoot === void 0 ? void 0 : resolve(scope.workspaceRoot);
3046
- const paths = await listSessionFiles(resolve(scope.root));
3047
- const sessions = [];
3048
- const byReason = /* @__PURE__ */ new Map();
3049
- const sample = [];
3050
- const reject = (path, reason) => {
3051
- byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
3052
- if (sample.length < REJECTION_SAMPLE_LIMIT) sample.push({
3053
- path,
3054
- reason
3055
- });
3056
- };
3057
- for (const path of paths) {
3058
- let entries;
3059
- try {
3060
- entries = (await parseCodeAgentJsonlFile(path)).entries;
3061
- } catch {
3062
- reject(path, "unreadable");
3063
- continue;
3064
- }
3065
- if (entries.length === 0) {
3066
- reject(path, "no-entries");
3067
- continue;
3068
- }
3069
- const workspace = sessionWorkspace(entries);
3070
- if (workspaceRoot !== void 0) {
3071
- if (workspace === void 0) {
3072
- reject(path, "no-workspace-recorded");
3073
- continue;
3074
- }
3075
- if (!isSameOrBelow(workspaceRoot, workspace)) {
3076
- reject(path, "outside-workspace");
3077
- continue;
3078
- }
3079
- }
3080
- const startedAtMs = sessionStartedAtMs(entries);
3081
- if (scope.window !== void 0) {
3082
- if (startedAtMs === void 0) {
3083
- reject(path, "no-start-recorded");
3084
- continue;
3085
- }
3086
- if (startedAtMs < scope.window.startedAtMs) {
3087
- reject(path, "before-run");
3088
- continue;
3089
- }
3090
- if (scope.window.finishedAtMs !== void 0 && startedAtMs > scope.window.finishedAtMs) {
3091
- reject(path, "after-run");
3092
- continue;
3093
- }
3094
- }
3095
- const observation = observeCodeAgentSession({
3096
- source: scope.source,
3097
- entries,
3098
- sourcePath: path
3099
- });
3100
- sessions.push({
3101
- path,
3102
- sessionId: observation.sessionId,
3103
- ...workspace === void 0 ? {} : { workspace },
3104
- ...startedAtMs === void 0 ? {} : { startedAtMs },
3105
- observation
3106
- });
3107
- }
3108
- const rejections = {
3109
- total: [...byReason.values()].reduce((sum, count) => sum + count, 0),
3110
- byReason: Object.fromEntries(byReason),
3111
- sample
3112
- };
3113
- if (sessions.length === 0) return {
3114
- status: "unavailable",
3115
- scope,
3116
- filesScanned: paths.length,
3117
- reason: unavailableReason(paths.length, rejections),
3118
- rejections
3119
- };
3120
- const bySurface = /* @__PURE__ */ new Map();
3121
- let actions = 0;
3122
- let subagentActions = 0;
3123
- let sessionsWithSubagentActions = 0;
3124
- for (const session of sessions) {
3125
- let sessionSubagents = 0;
3126
- for (const action of session.observation.actions) {
3127
- actions += 1;
3128
- bySurface.set(action.surface, (bySurface.get(action.surface) ?? 0) + 1);
3129
- if (action.surface === "subagent") sessionSubagents += 1;
3130
- }
3131
- subagentActions += sessionSubagents;
3132
- if (sessionSubagents > 0) sessionsWithSubagentActions += 1;
3133
- }
3134
- return {
3135
- status: "observed",
3136
- scope,
3137
- filesScanned: paths.length,
3138
- sessions,
3139
- actions,
3140
- subagentActions,
3141
- sessionsWithSubagentActions,
3142
- bySurface: Object.fromEntries(bySurface),
3143
- rejections
3144
- };
3145
- }
3146
- function unavailableReason(filesScanned, rejections) {
3147
- if (filesScanned === 0) return "the run captured no session files in this store";
3148
- return `no session in this store belongs to the run: ${filesScanned} file(s) scanned, all rejected (${Object.entries(rejections.byReason).map(([reason, count]) => `${count} ${reason}`).sort().join(", ")})`;
3149
- }
3150
- async function listSessionFiles(root) {
3151
- const found = [];
3152
- const walk = async (dir) => {
3153
- let entries;
3154
- try {
3155
- entries = await readdir(dir, { withFileTypes: true });
3156
- } catch {
3157
- return;
3158
- }
3159
- for (const entry of entries) {
3160
- const path = join(dir, entry.name);
3161
- if (entry.isDirectory()) await walk(path);
3162
- else if (entry.isFile() && entry.name.endsWith(SESSION_SUFFIX)) found.push(path);
3163
- }
3164
- };
3165
- await walk(root);
3166
- if (found.length === 0 && await isSessionFile(root)) found.push(root);
3167
- return found.sort();
3168
- }
3169
- async function isSessionFile(path) {
3170
- if (!path.endsWith(SESSION_SUFFIX)) return false;
3171
- try {
3172
- return (await stat(path)).isFile();
3173
- } catch {
3174
- return false;
3175
- }
3176
- }
3177
- /**
3178
- * The working directory the session RECORDED, from the first entry that states one.
3179
- *
3180
- * codex writes it on `session_meta.payload.cwd`; claude-code writes `cwd` on every line. Neither is
3181
- * inferred from the file's own location: where a captured file was copied to says nothing about
3182
- * where the agent ran.
3183
- */
3184
- function sessionWorkspace(entries) {
3185
- for (const entry of entries) {
3186
- const record = plainRecord(entry);
3187
- if (record === void 0) continue;
3188
- const direct = stringOf(record.cwd);
3189
- if (direct !== void 0) return direct;
3190
- const nested = stringOf(plainRecord(record.payload)?.cwd);
3191
- if (nested !== void 0) return nested;
3192
- }
3193
- }
3194
- /** When the session itself says it started, from the first entry carrying a parseable timestamp. */
3195
- function sessionStartedAtMs(entries) {
3196
- for (const entry of entries) {
3197
- const record = plainRecord(entry);
3198
- if (record === void 0) continue;
3199
- const payload = plainRecord(record.payload);
3200
- for (const candidate of [payload?.timestamp, record.timestamp]) {
3201
- const text = stringOf(candidate);
3202
- if (text === void 0) continue;
3203
- const parsed = Date.parse(text);
3204
- if (Number.isFinite(parsed)) return parsed;
3205
- }
3206
- const epochMs = numberOf(record.timestampMs) ?? numberOf(payload?.timestampMs);
3207
- if (epochMs !== void 0) return epochMs;
3208
- }
3209
- }
3210
- function isSameOrBelow(root, candidate) {
3211
- const rel = relative(root, resolve(candidate));
3212
- return rel === "" || rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
3213
- }
3214
- function plainRecord(value) {
3215
- return value !== null && typeof value === "object" && !Array.isArray(value) ? value : void 0;
3216
- }
3217
- function stringOf(value) {
3218
- return typeof value === "string" && value.length > 0 ? value : void 0;
3219
- }
3220
- function numberOf(value) {
3221
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
3222
- }
3223
2791
  //#endregion
3224
2792
  //#region src/contract/intake/feedback-table.ts
3225
2793
  function fromFeedbackTable(opts) {
@@ -3300,6 +2868,6 @@ function fromFeedbackTable(opts) {
3300
2868
  };
3301
2869
  }
3302
2870
  //#endregion
3303
- export { FileSystemOutcomeStore, InMemoryOutcomeStore, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, SelfImproveRunError, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, evalReportingSuite, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiGraphSession, fromPiSession, fromRunRecordDir, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, makeProposalFinding, measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, observeCodeAgentStore, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, parseCodeAgentJsonlFile, partitionRunsByAuthoringModel, runAgentProfileImprovementExperiment, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, streamCodeAgentJsonlFile, summarizeExecution, transientDispatchFailure, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison };
2871
+ export { FileSystemOutcomeStore, InMemoryOutcomeStore, SelfImproveRunError, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, defaultProductionGate, defineAgentEval, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, makeProposalFinding, measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, parseCodeAgentJsonlFile, partitionRunsByAuthoringModel, runAgentProfileImprovementExperiment, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, streamCodeAgentJsonlFile, summarizeExecution, transientDispatchFailure, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison };
3304
2872
 
3305
2873
  //# sourceMappingURL=index.js.map