observability-toolkit-hooks 0.0.0-stage → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (306) hide show
  1. package/CONFIG.md +144 -0
  2. package/LICENSE +95 -0
  3. package/README.md +201 -2
  4. package/dist/handlers/compact.js +38 -0
  5. package/dist/handlers/mcp-status.js +66 -0
  6. package/dist/handlers/notification.js +45 -0
  7. package/dist/handlers/post-tool-agent.js +157 -0
  8. package/dist/handlers/post-tool-changelog-sync.js +148 -0
  9. package/dist/handlers/post-tool-code-structure.js +116 -0
  10. package/dist/handlers/post-tool-edit-ledger.js +15 -0
  11. package/dist/handlers/post-tool-failure.js +83 -0
  12. package/dist/handlers/post-tool-git.js +79 -0
  13. package/dist/handlers/post-tool-mcp.js +105 -0
  14. package/dist/handlers/post-tool-outcome.js +48 -0
  15. package/dist/handlers/post-tool-task-state.js +74 -0
  16. package/dist/handlers/post-tool.js +132 -0
  17. package/dist/handlers/pre-tool-agent.js +192 -0
  18. package/dist/handlers/pre-tool.js +218 -0
  19. package/dist/handlers/session-end.js +39 -0
  20. package/dist/handlers/session-start.js +262 -0
  21. package/dist/handlers/stop-error-reminder.js +195 -0
  22. package/dist/handlers/stop-quality-evaluation.js +58 -0
  23. package/dist/handlers/stop-session-summary.js +62 -0
  24. package/dist/handlers/stop-token-metrics.js +83 -0
  25. package/dist/handlers/stop.js +25 -0
  26. package/dist/handlers/subagent.js +105 -0
  27. package/dist/handlers/tool-event-router.js +39 -0
  28. package/dist/handlers/user-prompt.js +218 -0
  29. package/dist/hook-runner.js +10 -0
  30. package/dist/hook-runner.js.map +7 -0
  31. package/dist/lib/agent/agent-context.js +123 -0
  32. package/dist/lib/agent/agent-evaluations.js +155 -0
  33. package/dist/lib/agent/agent-file-budget.js +65 -0
  34. package/dist/lib/agent/agent-sources.js +165 -0
  35. package/dist/lib/agent/synth-spans.js +51 -0
  36. package/dist/lib/core/bounded-label.js +6 -0
  37. package/dist/lib/core/category-rules.js +11 -0
  38. package/dist/lib/core/child-result.js +19 -0
  39. package/dist/lib/core/config-dir-recovery.js +22 -0
  40. package/dist/lib/core/constants.js +171 -0
  41. package/dist/lib/core/detached-runner.js +55 -0
  42. package/dist/lib/core/file-utils.js +353 -0
  43. package/dist/lib/core/format-utils.js +51 -0
  44. package/dist/lib/core/http-utils.js +74 -0
  45. package/dist/lib/core/http1-fetch.js +14 -0
  46. package/dist/lib/core/identity-key-resolver.js +149 -0
  47. package/dist/lib/core/jsonl-tail.js +207 -0
  48. package/dist/lib/core/load-envrc.js +72 -0
  49. package/dist/lib/core/parsers.js +43 -0
  50. package/dist/lib/core/regex.js +27 -0
  51. package/dist/lib/core/settings-env.js +31 -0
  52. package/dist/lib/core/shell-command.js +122 -0
  53. package/dist/lib/core/units.js +97 -0
  54. package/dist/lib/git/git-cache.js +158 -0
  55. package/dist/lib/git/git-commit.js +189 -0
  56. package/dist/lib/git/git-exec.js +30 -0
  57. package/dist/lib/mcp/mcp-completeness.js +40 -0
  58. package/dist/lib/mcp/mcp-config.js +118 -0
  59. package/dist/lib/mcp/mcp-failure-latch.js +22 -0
  60. package/dist/lib/mcp/mcp-identity-launcher.js +43 -0
  61. package/dist/lib/otel/attributes.js +322 -0
  62. package/dist/lib/otel/evaluation-attrs.js +51 -0
  63. package/dist/lib/otel/file-exporters.js +169 -0
  64. package/dist/lib/otel/host-resource.js +61 -0
  65. package/dist/lib/otel/otel-headers-helper.js +26 -0
  66. package/dist/lib/otel/otel-monitor.js +181 -0
  67. package/dist/lib/otel/otel.js +235 -0
  68. package/dist/lib/otel/otlp-headers.js +101 -0
  69. package/dist/lib/otel/otlp-post.js +122 -0
  70. package/dist/lib/otel/otlp-serialize.js +142 -0
  71. package/dist/lib/otel/otlp-signal.js +2 -0
  72. package/dist/lib/otel/remote-spans.js +142 -0
  73. package/dist/lib/otel/service-identity.js +10 -0
  74. package/dist/lib/otel/telemetry-alerts.js +43 -0
  75. package/dist/lib/otel/token-metrics.js +122 -0
  76. package/dist/lib/otel/trace-context.js +81 -0
  77. package/dist/lib/quality/judge-meta-eval.js +70 -0
  78. package/dist/lib/quality/judge-prompts.js +104 -0
  79. package/dist/lib/quality/judge-providers.js +193 -0
  80. package/dist/lib/quality/judge-transport.js +93 -0
  81. package/dist/lib/quality/llm-judge.js +170 -0
  82. package/dist/lib/quality/quality-budget.js +99 -0
  83. package/dist/lib/quality/quality-degradation.js +156 -0
  84. package/dist/lib/quality/quality-eval-handoff.js +74 -0
  85. package/dist/lib/quality/quality-evaluator.js +178 -0
  86. package/dist/lib/quality/quality-sampler.js +43 -0
  87. package/dist/lib/quality/quality-signals.js +150 -0
  88. package/dist/lib/runtime/categorizers.js +101 -0
  89. package/dist/lib/runtime/hook-input.js +16 -0
  90. package/dist/lib/runtime/hook-output.js +50 -0
  91. package/dist/lib/runtime/output-analyzer.js +160 -0
  92. package/dist/lib/runtime/run-handlers.js +32 -0
  93. package/dist/lib/runtime/tool-input.js +111 -0
  94. package/dist/lib/runtime/tool-output.js +44 -0
  95. package/dist/lib/runtime/unknown-tool.js +32 -0
  96. package/dist/lib/session/edit-ledger.js +29 -0
  97. package/dist/lib/session/forked-skill.js +39 -0
  98. package/dist/lib/session/handoffs.js +77 -0
  99. package/dist/lib/session/session-state.js +169 -0
  100. package/dist/lib/session/skill-rules.js +196 -0
  101. package/dist/lib/session/task-tracker.js +83 -0
  102. package/dist/lib/shipper/shipper-credentials.js +38 -0
  103. package/dist/lib/shipper/shipper-cursor.js +89 -0
  104. package/dist/lib/shipper/shipper-status.js +174 -0
  105. package/dist/lib/shipper/span-shipper.js +272 -0
  106. package/dist/lib/shipper/telemetry-shipper.js +39 -0
  107. package/dist/lib/transcript/agent-transcript-metrics.js +34 -0
  108. package/dist/lib/transcript/context-breakdown.js +134 -0
  109. package/dist/lib/transcript/context-history.js +69 -0
  110. package/dist/lib/transcript/context-tracker.js +226 -0
  111. package/dist/lib/transcript/message-count.js +49 -0
  112. package/dist/lib/transcript/model-ids.js +36 -0
  113. package/dist/lib/transcript/model-pricing.js +153 -0
  114. package/dist/lib/transcript/token-metrics-checkpoint.js +152 -0
  115. package/dist/lib/transcript/tool-call-durations.js +64 -0
  116. package/dist/lib/transcript/transcript-entry.js +67 -0
  117. package/dist/lib/transcript/transcript-fold.js +130 -0
  118. package/dist/lib/transcript/transcript-layout.js +39 -0
  119. package/dist/lib/transcript/transcript-parser.js +117 -0
  120. package/dist/runner/agent-context-4XTCLMLZ.js +23 -0
  121. package/dist/runner/agent-context-4XTCLMLZ.js.map +7 -0
  122. package/dist/runner/agent-file-budget-Q33ESBBE.js +68 -0
  123. package/dist/runner/agent-file-budget-Q33ESBBE.js.map +7 -0
  124. package/dist/runner/agent-transcript-metrics-W5CZZFMI.js +39 -0
  125. package/dist/runner/agent-transcript-metrics-W5CZZFMI.js.map +7 -0
  126. package/dist/runner/chunk-2NTNAJZE.js +49 -0
  127. package/dist/runner/chunk-2NTNAJZE.js.map +7 -0
  128. package/dist/runner/chunk-4KWBQTTG.js +78 -0
  129. package/dist/runner/chunk-4KWBQTTG.js.map +7 -0
  130. package/dist/runner/chunk-52RY6LQT.js +699 -0
  131. package/dist/runner/chunk-52RY6LQT.js.map +7 -0
  132. package/dist/runner/chunk-7OM5CYES.js +8695 -0
  133. package/dist/runner/chunk-7OM5CYES.js.map +7 -0
  134. package/dist/runner/chunk-7PEOKGYR.js +1568 -0
  135. package/dist/runner/chunk-7PEOKGYR.js.map +7 -0
  136. package/dist/runner/chunk-AGSGKELQ.js +126 -0
  137. package/dist/runner/chunk-AGSGKELQ.js.map +7 -0
  138. package/dist/runner/chunk-AVM7PJFW.js +12 -0
  139. package/dist/runner/chunk-AVM7PJFW.js.map +7 -0
  140. package/dist/runner/chunk-C3WHUMU5.js +178 -0
  141. package/dist/runner/chunk-C3WHUMU5.js.map +7 -0
  142. package/dist/runner/chunk-DGB4NSSD.js +157 -0
  143. package/dist/runner/chunk-DGB4NSSD.js.map +7 -0
  144. package/dist/runner/chunk-DNSTWC26.js +151 -0
  145. package/dist/runner/chunk-DNSTWC26.js.map +7 -0
  146. package/dist/runner/chunk-FDJJ3TYU.js +449 -0
  147. package/dist/runner/chunk-FDJJ3TYU.js.map +7 -0
  148. package/dist/runner/chunk-FGKYDQHZ.js +137 -0
  149. package/dist/runner/chunk-FGKYDQHZ.js.map +7 -0
  150. package/dist/runner/chunk-GG2I4TGI.js +582 -0
  151. package/dist/runner/chunk-GG2I4TGI.js.map +7 -0
  152. package/dist/runner/chunk-GNNKUMRF.js +60 -0
  153. package/dist/runner/chunk-GNNKUMRF.js.map +7 -0
  154. package/dist/runner/chunk-I7TD2QVR.js +118 -0
  155. package/dist/runner/chunk-I7TD2QVR.js.map +7 -0
  156. package/dist/runner/chunk-IJ4SLUBV.js +78 -0
  157. package/dist/runner/chunk-IJ4SLUBV.js.map +7 -0
  158. package/dist/runner/chunk-IY7AP7NA.js +32 -0
  159. package/dist/runner/chunk-IY7AP7NA.js.map +7 -0
  160. package/dist/runner/chunk-J3F3Z3EQ.js +177 -0
  161. package/dist/runner/chunk-J3F3Z3EQ.js.map +7 -0
  162. package/dist/runner/chunk-JLEO5OHU.js +104 -0
  163. package/dist/runner/chunk-JLEO5OHU.js.map +7 -0
  164. package/dist/runner/chunk-JRNNLOPR.js +168 -0
  165. package/dist/runner/chunk-JRNNLOPR.js.map +7 -0
  166. package/dist/runner/chunk-KGSA5JYP.js +82 -0
  167. package/dist/runner/chunk-KGSA5JYP.js.map +7 -0
  168. package/dist/runner/chunk-LBCZAYFR.js +125 -0
  169. package/dist/runner/chunk-LBCZAYFR.js.map +7 -0
  170. package/dist/runner/chunk-LHWRXY2G.js +41 -0
  171. package/dist/runner/chunk-LHWRXY2G.js.map +7 -0
  172. package/dist/runner/chunk-LQ2LGUYJ.js +41 -0
  173. package/dist/runner/chunk-LQ2LGUYJ.js.map +7 -0
  174. package/dist/runner/chunk-LYY6RCIE.js +158 -0
  175. package/dist/runner/chunk-LYY6RCIE.js.map +7 -0
  176. package/dist/runner/chunk-M4R4QIQH.js +41 -0
  177. package/dist/runner/chunk-M4R4QIQH.js.map +7 -0
  178. package/dist/runner/chunk-MC75VXMX.js +207 -0
  179. package/dist/runner/chunk-MC75VXMX.js.map +7 -0
  180. package/dist/runner/chunk-NKZU4RRG.js +104 -0
  181. package/dist/runner/chunk-NKZU4RRG.js.map +7 -0
  182. package/dist/runner/chunk-PJHWHJ3K.js +56 -0
  183. package/dist/runner/chunk-PJHWHJ3K.js.map +7 -0
  184. package/dist/runner/chunk-RIGVYJ2O.js +114 -0
  185. package/dist/runner/chunk-RIGVYJ2O.js.map +7 -0
  186. package/dist/runner/chunk-RWRGC7ZK.js +93 -0
  187. package/dist/runner/chunk-RWRGC7ZK.js.map +7 -0
  188. package/dist/runner/chunk-SRMAJQMV.js +54 -0
  189. package/dist/runner/chunk-SRMAJQMV.js.map +7 -0
  190. package/dist/runner/chunk-TRMFD36J.js +47 -0
  191. package/dist/runner/chunk-TRMFD36J.js.map +7 -0
  192. package/dist/runner/chunk-U4IOZYQ3.js +21 -0
  193. package/dist/runner/chunk-U4IOZYQ3.js.map +7 -0
  194. package/dist/runner/chunk-UEXDE3WP.js +83 -0
  195. package/dist/runner/chunk-UEXDE3WP.js.map +7 -0
  196. package/dist/runner/chunk-UW3GQZSM.js +50 -0
  197. package/dist/runner/chunk-UW3GQZSM.js.map +7 -0
  198. package/dist/runner/chunk-V2GSP6VR.js +46 -0
  199. package/dist/runner/chunk-V2GSP6VR.js.map +7 -0
  200. package/dist/runner/chunk-VOV4Y5VU.js +81 -0
  201. package/dist/runner/chunk-VOV4Y5VU.js.map +7 -0
  202. package/dist/runner/chunk-VT5T6OEI.js +65 -0
  203. package/dist/runner/chunk-VT5T6OEI.js.map +7 -0
  204. package/dist/runner/chunk-VUYVFWIN.js +12 -0
  205. package/dist/runner/chunk-VUYVFWIN.js.map +7 -0
  206. package/dist/runner/chunk-VVGOPPEN.js +36 -0
  207. package/dist/runner/chunk-VVGOPPEN.js.map +7 -0
  208. package/dist/runner/chunk-VXFSLM6P.js +14 -0
  209. package/dist/runner/chunk-VXFSLM6P.js.map +7 -0
  210. package/dist/runner/chunk-W3EK7JUT.js +26125 -0
  211. package/dist/runner/chunk-W3EK7JUT.js.map +7 -0
  212. package/dist/runner/chunk-WJHNCOUF.js +52 -0
  213. package/dist/runner/chunk-WJHNCOUF.js.map +7 -0
  214. package/dist/runner/chunk-WU7ANGCW.js +269 -0
  215. package/dist/runner/chunk-WU7ANGCW.js.map +7 -0
  216. package/dist/runner/chunk-X7XLMSZX.js +44 -0
  217. package/dist/runner/chunk-X7XLMSZX.js.map +7 -0
  218. package/dist/runner/chunk-XSO3B4QN.js +124 -0
  219. package/dist/runner/chunk-XSO3B4QN.js.map +7 -0
  220. package/dist/runner/chunk-YGFN3LN2.js +174 -0
  221. package/dist/runner/chunk-YGFN3LN2.js.map +7 -0
  222. package/dist/runner/chunk-YHYDBLVB.js +33 -0
  223. package/dist/runner/chunk-YHYDBLVB.js.map +7 -0
  224. package/dist/runner/compact-36NP3PKT.js +60 -0
  225. package/dist/runner/compact-36NP3PKT.js.map +7 -0
  226. package/dist/runner/getMachineId-bsd-AGENNHN6.js +42 -0
  227. package/dist/runner/getMachineId-bsd-AGENNHN6.js.map +7 -0
  228. package/dist/runner/getMachineId-bsd-I5WZGCL4.js +33 -0
  229. package/dist/runner/getMachineId-bsd-I5WZGCL4.js.map +7 -0
  230. package/dist/runner/getMachineId-bsd-ZXK225P7.js +33 -0
  231. package/dist/runner/getMachineId-bsd-ZXK225P7.js.map +7 -0
  232. package/dist/runner/getMachineId-darwin-47GDLEXL.js +42 -0
  233. package/dist/runner/getMachineId-darwin-47GDLEXL.js.map +7 -0
  234. package/dist/runner/getMachineId-darwin-5CRLMDD6.js +33 -0
  235. package/dist/runner/getMachineId-darwin-5CRLMDD6.js.map +7 -0
  236. package/dist/runner/getMachineId-darwin-ADG4I32N.js +33 -0
  237. package/dist/runner/getMachineId-darwin-ADG4I32N.js.map +7 -0
  238. package/dist/runner/getMachineId-linux-5KVS3T2M.js +27 -0
  239. package/dist/runner/getMachineId-linux-5KVS3T2M.js.map +7 -0
  240. package/dist/runner/getMachineId-linux-GDFB5GSO.js +34 -0
  241. package/dist/runner/getMachineId-linux-GDFB5GSO.js.map +7 -0
  242. package/dist/runner/getMachineId-linux-NQMWCRO3.js +27 -0
  243. package/dist/runner/getMachineId-linux-NQMWCRO3.js.map +7 -0
  244. package/dist/runner/getMachineId-unsupported-5PFIGMVX.js +18 -0
  245. package/dist/runner/getMachineId-unsupported-5PFIGMVX.js.map +7 -0
  246. package/dist/runner/getMachineId-unsupported-DW27K3ZX.js +18 -0
  247. package/dist/runner/getMachineId-unsupported-DW27K3ZX.js.map +7 -0
  248. package/dist/runner/getMachineId-unsupported-XV3FHTHD.js +25 -0
  249. package/dist/runner/getMachineId-unsupported-XV3FHTHD.js.map +7 -0
  250. package/dist/runner/getMachineId-win-5JVMW3EI.js +44 -0
  251. package/dist/runner/getMachineId-win-5JVMW3EI.js.map +7 -0
  252. package/dist/runner/getMachineId-win-SNESQF7R.js +35 -0
  253. package/dist/runner/getMachineId-win-SNESQF7R.js.map +7 -0
  254. package/dist/runner/getMachineId-win-UJYHSBM2.js +35 -0
  255. package/dist/runner/getMachineId-win-UJYHSBM2.js.map +7 -0
  256. package/dist/runner/git-commit-BA53DOJQ.js +30 -0
  257. package/dist/runner/git-commit-BA53DOJQ.js.map +7 -0
  258. package/dist/runner/hook-runner-JHH6PGKE.js +220 -0
  259. package/dist/runner/hook-runner-JHH6PGKE.js.map +7 -0
  260. package/dist/runner/judge-transport-R4TF5LEH.js +91 -0
  261. package/dist/runner/judge-transport-R4TF5LEH.js.map +7 -0
  262. package/dist/runner/mcp-status-QEUH2KGX.js +89 -0
  263. package/dist/runner/mcp-status-QEUH2KGX.js.map +7 -0
  264. package/dist/runner/notification-FKMBCSXF.js +69 -0
  265. package/dist/runner/notification-FKMBCSXF.js.map +7 -0
  266. package/dist/runner/post-tool-3MRHTBKH.js +192 -0
  267. package/dist/runner/post-tool-3MRHTBKH.js.map +7 -0
  268. package/dist/runner/post-tool-agent-BB34MNHI.js +370 -0
  269. package/dist/runner/post-tool-agent-BB34MNHI.js.map +7 -0
  270. package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js +147 -0
  271. package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js.map +7 -0
  272. package/dist/runner/post-tool-code-structure-LE7J2SC2.js +130 -0
  273. package/dist/runner/post-tool-code-structure-LE7J2SC2.js.map +7 -0
  274. package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js +30 -0
  275. package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js.map +7 -0
  276. package/dist/runner/post-tool-failure-XTH2I2Z7.js +111 -0
  277. package/dist/runner/post-tool-failure-XTH2I2Z7.js.map +7 -0
  278. package/dist/runner/post-tool-git-E46HZGZO.js +94 -0
  279. package/dist/runner/post-tool-git-E46HZGZO.js.map +7 -0
  280. package/dist/runner/post-tool-mcp-AMCDY5GH.js +144 -0
  281. package/dist/runner/post-tool-mcp-AMCDY5GH.js.map +7 -0
  282. package/dist/runner/post-tool-task-state-TLDOI33H.js +100 -0
  283. package/dist/runner/post-tool-task-state-TLDOI33H.js.map +7 -0
  284. package/dist/runner/pre-tool-SQE2FZQO.js +245 -0
  285. package/dist/runner/pre-tool-SQE2FZQO.js.map +7 -0
  286. package/dist/runner/pre-tool-agent-OZUPVFUI.js +218 -0
  287. package/dist/runner/pre-tool-agent-OZUPVFUI.js.map +7 -0
  288. package/dist/runner/quality-evaluator-XYNDKDRG.js +901 -0
  289. package/dist/runner/quality-evaluator-XYNDKDRG.js.map +7 -0
  290. package/dist/runner/session-end-3MJTDLLA.js +57 -0
  291. package/dist/runner/session-end-3MJTDLLA.js.map +7 -0
  292. package/dist/runner/session-start-5ZMCLKBS.js +352 -0
  293. package/dist/runner/session-start-5ZMCLKBS.js.map +7 -0
  294. package/dist/runner/shipper-status-MADX6SPG.js +37 -0
  295. package/dist/runner/shipper-status-MADX6SPG.js.map +7 -0
  296. package/dist/runner/span-shipper-4HQDHC47.js +3408 -0
  297. package/dist/runner/span-shipper-4HQDHC47.js.map +7 -0
  298. package/dist/runner/stop-RTKSIYJV.js +596 -0
  299. package/dist/runner/stop-RTKSIYJV.js.map +7 -0
  300. package/dist/runner/subagent-CBCG4OLZ.js +116 -0
  301. package/dist/runner/subagent-CBCG4OLZ.js.map +7 -0
  302. package/dist/runner/token-metrics-AD2RAFGM.js +31 -0
  303. package/dist/runner/token-metrics-AD2RAFGM.js.map +7 -0
  304. package/dist/runner/user-prompt-T5NZ2UJD.js +376 -0
  305. package/dist/runner/user-prompt-T5NZ2UJD.js.map +7 -0
  306. package/package.json +70 -5
@@ -0,0 +1,104 @@
1
+ /**
2
+ * Judge prompt templates and their verdict parser: one contract, since every template ends with
3
+ * the `Score:` / `Explanation:` format `parseJudgeVerdict` reads.
4
+ */
5
+ import { truncate } from '../core/format-utils.js';
6
+ export const JUDGE_METRIC_NAMES = {
7
+ RELEVANCE: 'relevance',
8
+ COHERENCE: 'coherence',
9
+ };
10
+ /** Prompt input caps: context bound and injection surface. */
11
+ const MAX_USER_TEXT_LEN = 500;
12
+ const MAX_ASSISTANT_TEXT_LEN = 2000;
13
+ const MAX_REASON_LEN = 400;
14
+ const MAX_TOOL_RESULTS_IN_PROMPT = 3;
15
+ const JUDGE_SCORE_PATTERN = /Score:\s*(\d+(?:\.\d+)?)/i;
16
+ const JUDGE_EXPLANATION_PATTERN = /Explanation:\s*(.+)/i;
17
+ const SCORE_PARSE_FAILED = 'llm_judge_score_parse_failed';
18
+ /** Score clamped to [0, 1]; reason falls back to the full text. */
19
+ export function parseJudgeVerdict(text) {
20
+ const scoreText = JUDGE_SCORE_PATTERN.exec(text)?.[1];
21
+ if (scoreText === undefined)
22
+ throw new Error(SCORE_PARSE_FAILED);
23
+ const score = Math.max(0, Math.min(1, parseFloat(scoreText)));
24
+ const reason = JUDGE_EXPLANATION_PATTERN.exec(text)?.[1]?.trim() || text.trim();
25
+ return { score, reason };
26
+ }
27
+ export function buildJudgePrompt(metricName, turn) {
28
+ return metricName === JUDGE_METRIC_NAMES.RELEVANCE
29
+ ? buildRelevancePrompt(turn)
30
+ : buildCoherencePrompt(turn);
31
+ }
32
+ export function buildExplanationQualityPrompt(evaluationName, score, reason, userText) {
33
+ const safeUser = truncate(userText, MAX_USER_TEXT_LEN);
34
+ const safeReason = truncate(reason, MAX_REASON_LEN);
35
+ return `You are evaluating the quality of a judge's explanation for an evaluation score.
36
+
37
+ The judge evaluated "${evaluationName}" and gave a score of ${score.toFixed(2)}.
38
+
39
+ Original user request:
40
+ ${safeUser}
41
+
42
+ Judge's explanation:
43
+ ${safeReason}
44
+
45
+ Rate the explanation quality on a scale from 0.0 to 1.0 based on:
46
+ 1. Specificity: Does it reference concrete details from the evaluated output?
47
+ 2. Evidence citation: Does it quote or paraphrase specific passages as evidence?
48
+ 3. Actionability: Does it suggest what could be improved?
49
+
50
+ - 1.0: Highly specific, cites evidence, actionable
51
+ - 0.7: Specific with some evidence, partially actionable
52
+ - 0.5: General but relevant reasoning
53
+ - 0.3: Vague or generic explanation
54
+ - 0.0: No meaningful reasoning provided
55
+
56
+ A concise but specific explanation can score highly.
57
+
58
+ Respond in exactly this format:
59
+ Score: X.XX
60
+ Explanation: [1-2 sentences about the quality of the explanation]`;
61
+ }
62
+ function buildRelevancePrompt(turn) {
63
+ const safeUser = truncate(turn.userText, MAX_USER_TEXT_LEN);
64
+ const safeAssistant = truncate(turn.assistantText, MAX_ASSISTANT_TEXT_LEN);
65
+ const toolContext = turn.toolResults.length > 0
66
+ ? `\nTool results used:\n${turn.toolResults.slice(0, MAX_TOOL_RESULTS_IN_PROMPT).join('\n---\n')}`
67
+ : '';
68
+ return `You are evaluating the relevance of an AI assistant's response to a user's request.
69
+
70
+ User request:
71
+ ${safeUser}
72
+
73
+ Assistant response:
74
+ ${safeAssistant}${toolContext}
75
+
76
+ Rate the relevance of the response on a scale from 0.0 to 1.0:
77
+ - 1.0: Perfectly relevant, directly addresses the user's request
78
+ - 0.7: Mostly relevant with minor tangents
79
+ - 0.5: Partially relevant
80
+ - 0.3: Mostly irrelevant
81
+ - 0.0: Completely irrelevant
82
+
83
+ Respond in exactly this format:
84
+ Score: X.XX
85
+ Explanation: [1-2 sentences citing specific evidence from the response]`;
86
+ }
87
+ function buildCoherencePrompt(turn) {
88
+ const safeAssistant = truncate(turn.assistantText, MAX_ASSISTANT_TEXT_LEN);
89
+ return `You are evaluating the coherence of an AI assistant's response.
90
+
91
+ Assistant response:
92
+ ${safeAssistant}
93
+
94
+ Rate the coherence on a scale from 0.0 to 1.0:
95
+ - 1.0: Well-structured, clear, logically organized
96
+ - 0.7: Mostly coherent with minor issues
97
+ - 0.5: Somewhat coherent but hard to follow
98
+ - 0.3: Mostly incoherent
99
+ - 0.0: Completely incoherent
100
+
101
+ Respond in exactly this format:
102
+ Score: X.XX
103
+ Explanation: [1-2 sentences citing specific evidence from the response]`;
104
+ }
@@ -0,0 +1,193 @@
1
+ /** Judge provider wire formats: endpoints, request bodies, response guards and credential lookup. Orchestration: `llm-judge.ts`. */
2
+ import { isJsonObject } from '../core/file-utils.js';
3
+ import { bearerHeaders, jsonHeaders } from '../core/http-utils.js';
4
+ import { DURATION_MS, TIME_MS } from '../core/units.js';
5
+ import { GENAI_PROVIDER_NAMES } from '../otel/attributes.js';
6
+ import { providerOfModel } from '../transcript/model-ids.js';
7
+ const JUDGE_MAX_TOKENS = 384;
8
+ /** Tuned for Haiku. */
9
+ const JUDGE_TIMEOUT_MS = DURATION_MS.FIFTEEN_SECONDS;
10
+ /** 3× the slowest measured reasoning call (grok-4.6, 18.3 s): an abort costs a budget unit and yields no record. */
11
+ const REASONING_MODEL_TIMEOUT_MS = TIME_MS.MINUTE;
12
+ /** One tick is 1e-10 USD (docs.x.ai/developers/cost-tracking). */
13
+ const XAI_TICKS_PER_USD = 1e10;
14
+ const ANTHROPIC_MESSAGES_URL = 'https://api.anthropic.com/v1/messages';
15
+ const ANTHROPIC_VERSION = '2023-06-01';
16
+ const XAI_CHAT_COMPLETIONS_URL = 'https://api.x.ai/v1/chat/completions';
17
+ const OPENAI_CHAT_COMPLETIONS_URL = 'https://api.openai.com/v1/chat/completions';
18
+ const OPENAI_KEY_PREFIX = 'sk-';
19
+ const RESPONSE_SHAPES = {
20
+ ANTHROPIC: 'Anthropic',
21
+ OPENAI_COMPATIBLE: 'OpenAI-compatible',
22
+ };
23
+ /** Surfaces as `error.type` on the judge span. */
24
+ class JudgeResponseShapeError extends Error {
25
+ constructor(shape) {
26
+ super(`judge response does not match the ${shape} shape`);
27
+ this.name = 'JudgeResponseShapeError';
28
+ }
29
+ }
30
+ function isNumber(value) {
31
+ return typeof value === 'number';
32
+ }
33
+ function isString(value) {
34
+ return typeof value === 'string';
35
+ }
36
+ function isAbsentOr(value, check) {
37
+ return value === undefined || value === null || check(value);
38
+ }
39
+ function isAnthropicContent(value) {
40
+ if (!Array.isArray(value))
41
+ return false;
42
+ const first = value[0];
43
+ return isAbsentOr(first, (block) => isJsonObject(block) && isAbsentOr(block.text, isString));
44
+ }
45
+ function isAnthropicUsage(value) {
46
+ return isJsonObject(value) && isNumber(value.input_tokens) && isNumber(value.output_tokens);
47
+ }
48
+ function isAnthropicResponse(data) {
49
+ return isJsonObject(data)
50
+ && isAbsentOr(data.content, isAnthropicContent)
51
+ && isAbsentOr(data.usage, isAnthropicUsage);
52
+ }
53
+ function asAnthropicResponse(data) {
54
+ if (!isAnthropicResponse(data))
55
+ throw new JudgeResponseShapeError(RESPONSE_SHAPES.ANTHROPIC);
56
+ return data;
57
+ }
58
+ function isOpenAiMessage(value) {
59
+ return isJsonObject(value) && isAbsentOr(value.content, isString);
60
+ }
61
+ function isOpenAiChoices(value) {
62
+ if (!Array.isArray(value))
63
+ return false;
64
+ const first = value[0];
65
+ return isAbsentOr(first, (choice) => isJsonObject(choice) && isAbsentOr(choice.message, isOpenAiMessage));
66
+ }
67
+ /** A string token count would concatenate rather than add and look like a real measurement. */
68
+ function isOpenAiUsage(value) {
69
+ if (!isJsonObject(value))
70
+ return false;
71
+ const isDetails = (details) => isJsonObject(details) && isAbsentOr(details.reasoning_tokens, isNumber);
72
+ return isNumber(value.prompt_tokens)
73
+ && isNumber(value.completion_tokens)
74
+ && isAbsentOr(value.completion_tokens_details, isDetails);
75
+ }
76
+ function isOpenAiCompatibleResponse(data) {
77
+ return isJsonObject(data)
78
+ && isAbsentOr(data.choices, isOpenAiChoices)
79
+ && isAbsentOr(data.usage, isOpenAiUsage)
80
+ && isAbsentOr(data.cost_in_usd_ticks, isNumber);
81
+ }
82
+ function asOpenAiCompatibleResponse(data) {
83
+ if (!isOpenAiCompatibleResponse(data))
84
+ throw new JudgeResponseShapeError(RESPONSE_SHAPES.OPENAI_COMPATIBLE);
85
+ return data;
86
+ }
87
+ /** `outputTokens` is the raw `completion_tokens`, whose meaning differs per provider. */
88
+ function parseOpenAiCompatible(payload) {
89
+ return {
90
+ text: payload.choices?.[0]?.message?.content ?? '',
91
+ inputTokens: payload.usage?.prompt_tokens,
92
+ outputTokens: payload.usage?.completion_tokens,
93
+ // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 0 reasoning tokens maps to undefined
94
+ reasoningTokens: payload.usage?.completion_tokens_details?.reasoning_tokens || undefined,
95
+ };
96
+ }
97
+ function promptBody(model, prompt) {
98
+ return { model, temperature: 0.0, messages: [{ role: 'user', content: prompt }] };
99
+ }
100
+ /** Anthropic and xAI cap output with `max_tokens`. */
101
+ function maxTokensBody(model, prompt) {
102
+ return { ...promptBody(model, prompt), max_tokens: JUDGE_MAX_TOKENS };
103
+ }
104
+ /** GPT-5 rejects `max_tokens` and takes `temperature` only with reasoning `'none'`, which also stops reasoning from eating the cap. */
105
+ function maxCompletionTokensBody(model, prompt) {
106
+ return { ...promptBody(model, prompt), max_completion_tokens: JUDGE_MAX_TOKENS, reasoning_effort: 'none' };
107
+ }
108
+ export const PROVIDER_SPECS = {
109
+ [GENAI_PROVIDER_NAMES.ANTHROPIC]: {
110
+ url: ANTHROPIC_MESSAGES_URL,
111
+ timeoutMs: JUDGE_TIMEOUT_MS,
112
+ // Not ANTHROPIC_API_KEY first: Claude Code reads it and drops subscription auth when set.
113
+ // The backup covers a revoked judge key (JUDGE-KEY-401).
114
+ keyEnvVars: ['LLM_JUDGE_ANTHROPIC_KEY', 'ANTHROPIC_API_KEY'],
115
+ body: maxTokensBody,
116
+ headers: (apiKey) => jsonHeaders({ 'x-api-key': apiKey, 'anthropic-version': ANTHROPIC_VERSION }),
117
+ parse: (data) => {
118
+ const payload = asAnthropicResponse(data);
119
+ return {
120
+ text: payload.content?.[0]?.text ?? '',
121
+ inputTokens: payload.usage?.input_tokens,
122
+ outputTokens: payload.usage?.output_tokens,
123
+ };
124
+ },
125
+ },
126
+ [GENAI_PROVIDER_NAMES.XAI]: {
127
+ url: XAI_CHAT_COMPLETIONS_URL,
128
+ timeoutMs: REASONING_MODEL_TIMEOUT_MS,
129
+ keyEnvVars: ['LLM_JUDGE_XAI_KEY', 'XAI_API_KEY'],
130
+ body: maxTokensBody,
131
+ headers: (apiKey) => jsonHeaders(bearerHeaders(apiKey)),
132
+ parse: (data) => {
133
+ const payload = asOpenAiCompatibleResponse(data);
134
+ const base = parseOpenAiCompatible(payload);
135
+ // A `null` cost is unreported, not $0.
136
+ const ticks = payload.cost_in_usd_ticks ?? undefined;
137
+ return {
138
+ ...base,
139
+ // xAI's `completion_tokens` EXCLUDES reasoning (~18× under-report alone, Gate 2b).
140
+ outputTokens: base.outputTokens === undefined ? undefined : base.outputTokens + (base.reasoningTokens ?? 0),
141
+ actualCostUsd: ticks === undefined ? undefined : ticks / XAI_TICKS_PER_USD,
142
+ };
143
+ },
144
+ },
145
+ [GENAI_PROVIDER_NAMES.OPENAI]: {
146
+ url: OPENAI_CHAT_COMPLETIONS_URL,
147
+ timeoutMs: REASONING_MODEL_TIMEOUT_MS,
148
+ // OPEN_AI_KEY first: OPENAI_API_KEY is a revoked key in Doppler.
149
+ keyEnvVars: ['LLM_JUDGE_OPENAI_KEY', 'OPEN_AI_KEY', 'OPENAI_API_KEY'],
150
+ keyPrefix: OPENAI_KEY_PREFIX,
151
+ body: maxCompletionTokensBody,
152
+ headers: (apiKey) => jsonHeaders(bearerHeaders(apiKey)),
153
+ // OpenAI's `completion_tokens` already INCLUDES reasoning; no per-response cost.
154
+ parse: (data) => parseOpenAiCompatible(asOpenAiCompatibleResponse(data)),
155
+ },
156
+ };
157
+ const JUDGE_PROVIDERS = new Set(Object.values(GENAI_PROVIDER_NAMES));
158
+ function isJudgeProvider(provider) {
159
+ return provider !== undefined && JUDGE_PROVIDERS.has(provider);
160
+ }
161
+ /**
162
+ * Shared provider lookup narrowed to judge providers. No default: routing an unknown model to
163
+ * Anthropic would silently merge Gate 3b's two arms (cf. OBP16).
164
+ */
165
+ export function providerForModel(model) {
166
+ const provider = providerOfModel(model);
167
+ return isJudgeProvider(provider) ? provider : undefined;
168
+ }
169
+ function* keySources(spec) {
170
+ for (const name of spec.keyEnvVars)
171
+ yield process.env[name];
172
+ }
173
+ /**
174
+ * Env vars in precedence order, deduplicated; a 401 moves to the next (JUDGE-KEY-401). Secrets come
175
+ * from the evaluator's `doppler run` wrapper (JDC1), so nothing spawns Doppler in-process.
176
+ */
177
+ export function* judgeApiKeys(provider) {
178
+ const spec = PROVIDER_SPECS[provider];
179
+ const seen = new Set();
180
+ for (const value of keySources(spec)) {
181
+ if (!value || seen.has(value))
182
+ continue;
183
+ if (spec.keyPrefix && !value.startsWith(spec.keyPrefix))
184
+ continue;
185
+ seen.add(value);
186
+ yield value;
187
+ }
188
+ }
189
+ /** The key a call would start with, reading no further; undefined when the provider has none. */
190
+ export function firstJudgeApiKey(provider) {
191
+ const first = judgeApiKeys(provider).next();
192
+ return first.done ? undefined : first.value;
193
+ }
@@ -0,0 +1,93 @@
1
+ /**
2
+ * Judge HTTP boundary: one POST through undici's `RetryAgent` on the HTTP/1.1 pool
3
+ * (NODE-FETCH-HTTP2-DEAD-SESSION). `request`, not `fetch`: a fetch stream body is consumed by the
4
+ * first attempt, so a replay fails on content-length; a string body replays. Imported lazily.
5
+ */
6
+ import { errors, RetryAgent, request } from 'undici';
7
+ import { errorCode, isHttpTimeoutError, isRetryableHttpStatus, isSuccessHttpStatus } from '../core/http-utils.js';
8
+ import { http1Dispatcher } from '../core/http1-fetch.js';
9
+ import { TIME_MS } from '../core/units.js';
10
+ const MAX_RETRY_DELAY_SECONDS = 8;
11
+ /** Vendor-SDK-default exponential backoff; `Retry-After` replaces the delay, same cap. */
12
+ export const JUDGE_RETRY = {
13
+ MAX_RETRIES: 2,
14
+ BASE_DELAY_MS: TIME_MS.SECOND / 2,
15
+ MAX_DELAY_MS: MAX_RETRY_DELAY_SECONDS * TIME_MS.SECOND,
16
+ };
17
+ const RETRY_AFTER_HEADER = 'retry-after';
18
+ /** A `RequestRetryError` (status + headers) for an HTTP status, a transport error otherwise. */
19
+ function asRetryError(error) {
20
+ return error instanceof errors.RequestRetryError ? error : undefined;
21
+ }
22
+ /**
23
+ * Retry reason, or undefined to deliver as-is. Transport errors retry; a timeout does not (it spent
24
+ * the attempt's budget). Statuses follow `isRetryableHttpStatus`.
25
+ */
26
+ function retryReason(error) {
27
+ const statusCode = asRetryError(error)?.statusCode;
28
+ if (statusCode !== undefined)
29
+ return isRetryableHttpStatus(statusCode) ? `http_${statusCode}` : undefined;
30
+ if (isHttpTimeoutError(error))
31
+ return undefined;
32
+ return errorCode(error) ?? error.name;
33
+ }
34
+ /** Seconds or HTTP date; undefined when absent, malformed, past, or `0` (computed backoff applies). */
35
+ function retryAfterMs(error) {
36
+ const header = asRetryError(error)?.headers[RETRY_AFTER_HEADER];
37
+ if (typeof header !== 'string')
38
+ return undefined;
39
+ const seconds = Number(header);
40
+ const ms = Number.isNaN(seconds) ? Date.parse(header) - Date.now() : seconds * TIME_MS.SECOND;
41
+ return ms > 0 ? ms : undefined;
42
+ }
43
+ /** `counter` is 1 on the first failed attempt. */
44
+ function retryDelayMs(policy, counter, retryAfter) {
45
+ return Math.min(policy.MAX_DELAY_MS, retryAfter ?? policy.BASE_DELAY_MS * 2 ** (counter - 1));
46
+ }
47
+ /**
48
+ * undici's `retry` option replaces its default policy outright; undici still owns replay, the failed
49
+ * body, the attempt counter and abort. Under `throwOnError: false`, `callback(error)` delivers the
50
+ * last response instead of throwing.
51
+ */
52
+ function retryPolicy(policy, onRetry) {
53
+ return (error, { state }, callback) => {
54
+ const reason = retryReason(error);
55
+ if (reason === undefined || state.counter > policy.MAX_RETRIES) {
56
+ callback(error);
57
+ return;
58
+ }
59
+ try {
60
+ onRetry(reason);
61
+ }
62
+ catch {
63
+ // The retry proceeds regardless.
64
+ }
65
+ setTimeout(() => callback(null), retryDelayMs(policy, state.counter, retryAfterMs(error)));
66
+ };
67
+ }
68
+ /** Returns the last attempt's response as-is, for the 401 key fallback and status check. */
69
+ export async function judgePost({ url, headers, body, timeoutMs, onRetry, retry = JUDGE_RETRY }) {
70
+ const dispatcher = new RetryAgent(http1Dispatcher(), {
71
+ throwOnError: false,
72
+ methods: ['POST'],
73
+ retry: retryPolicy(retry, onRetry),
74
+ });
75
+ const response = await request(url, {
76
+ method: 'POST',
77
+ headers,
78
+ body,
79
+ dispatcher,
80
+ // Per attempt; one `AbortSignal.timeout` would bound all attempts and backoffs together.
81
+ // undici timers tick at ~1 s: fine for 15 s / 60 s budgets, unreliable below.
82
+ headersTimeout: timeoutMs,
83
+ bodyTimeout: timeoutMs,
84
+ });
85
+ return {
86
+ status: response.statusCode,
87
+ ok: isSuccessHttpStatus(response.statusCode),
88
+ json: () => response.body.json(),
89
+ discard: async () => {
90
+ await response.body.dump().catch(() => undefined);
91
+ },
92
+ };
93
+ }
@@ -0,0 +1,170 @@
1
+ /** LLM-as-Judge client: prompt in, `{ score, reason }` out. Providers: `judge-providers.ts`; prompts: `judge-prompts.ts`; spend: `quality-evaluator.ts`. */
2
+ import { errorTypeName } from '../core/format-utils.js';
3
+ import { errorCode, HTTP_STATUS, isHttpTimeoutError } from '../core/http-utils.js';
4
+ import { TIME_MS, UNITS } from '../core/units.js';
5
+ import { ERROR_TYPE_ATTR, GENAI_AGENT_ATTRIBUTES, GENAI_CLIENT_INFERENCE_DURATION_METRIC, GENAI_CLIENT_OPERATION_SPAN, GENAI_DURATION_BUCKETS_S, GENAI_OPERATION, GENAI_REQUEST_MODEL_ATTR, } from '../otel/attributes.js';
6
+ import { withSpan, recordMetric } from '../otel/otel.js';
7
+ import { recordInferenceCost, recordTokenUsage } from '../otel/token-metrics.js';
8
+ import { usageCostUsd } from '../transcript/model-pricing.js';
9
+ import { buildJudgePrompt, parseJudgeVerdict } from './judge-prompts.js';
10
+ import { judgeApiKeys, PROVIDER_SPECS, providerForModel, } from './judge-providers.js';
11
+ const DEFAULT_JUDGE_MODEL = 'claude-haiku-4-5-20251001';
12
+ /** `QUALITY_JUDGE_MODEL` override, read at module load. */
13
+ export const JUDGE_MODEL = process.env.QUALITY_JUDGE_MODEL || DEFAULT_JUDGE_MODEL;
14
+ const CANARY_SCORE_MIN = 0.2;
15
+ const CANARY_SCORE_SPAN = 0.3;
16
+ const CANARY_REASON = 'canary';
17
+ const CENTS_PER_USD = 100;
18
+ /** One sample per retry, so a 529 storm is visible rather than reading as slow calls. */
19
+ const JUDGE_RETRY_METRIC = 'integritystudio.llm_judge.retries';
20
+ const RETRY_REASON_ATTR = 'integritystudio.llm_judge.retry_reason';
21
+ const REASONING_OUTPUT_TOKENS_ATTR = 'gen_ai.usage.reasoning.output_tokens';
22
+ export class MissingJudgeCredentialError extends Error {
23
+ provider;
24
+ constructor(provider) {
25
+ super(`no api key for judge provider ${provider}`);
26
+ this.provider = provider;
27
+ this.name = 'MissingJudgeCredentialError';
28
+ }
29
+ }
30
+ export class UnknownJudgeModelError extends Error {
31
+ model;
32
+ constructor(model) {
33
+ super(`no judge provider claims model ${model}`);
34
+ this.model = model;
35
+ this.name = 'UnknownJudgeModelError';
36
+ }
37
+ }
38
+ class JudgeHttpError extends Error {
39
+ status;
40
+ constructor(provider, status) {
41
+ super(`${provider} API error: ${status}`);
42
+ this.status = status;
43
+ this.name = 'JudgeHttpError';
44
+ }
45
+ }
46
+ function recordRetry(provider, reason) {
47
+ recordMetric(JUDGE_RETRY_METRIC, 1, {
48
+ [GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider,
49
+ [RETRY_REASON_ATTR]: reason,
50
+ }, { unit: UNITS.RETRY });
51
+ }
52
+ /** A transport error reports its code (`ECONNRESET`, `UND_ERR_SOCKET`); anything else its name. */
53
+ function judgeErrorType(error) {
54
+ if (error instanceof JudgeHttpError)
55
+ return `http_error_${error.status}`;
56
+ if (!(error instanceof Error))
57
+ return 'unknown';
58
+ if (isHttpTimeoutError(error))
59
+ return 'timeout';
60
+ return errorCode(error) ?? errorTypeName(error);
61
+ }
62
+ function elapsedSeconds(startMs) {
63
+ return (Date.now() - startMs) / TIME_MS.SECOND;
64
+ }
65
+ /** Provider-reported, else token-based; undefined for an unknown model rather than a guessed rate (OBP19). */
66
+ function judgeCostUsd(model, usage) {
67
+ if (usage.actualCostUsd !== undefined)
68
+ return usage.actualCostUsd;
69
+ if (usage.inputTokens === undefined || usage.outputTokens === undefined)
70
+ return undefined;
71
+ return usageCostUsd(model, { input: usage.inputTokens, output: usage.outputTokens });
72
+ }
73
+ /** Records the call's token and cost counters and returns the cost, undefined when unknown. */
74
+ function recordJudgeUsage(span, model, provider, usage) {
75
+ const { inputTokens, outputTokens, reasoningTokens } = usage;
76
+ const costUsd = judgeCostUsd(model, usage);
77
+ // Passed in, not re-derived from the model id, so the label is the provider routing chose.
78
+ const labels = {
79
+ model,
80
+ operation: GENAI_OPERATION.EVALUATE,
81
+ attributes: { [GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider },
82
+ };
83
+ if (inputTokens !== undefined && outputTokens !== undefined) {
84
+ recordTokenUsage({ ...labels, inputTokens, outputTokens, reasoningTokens });
85
+ }
86
+ if (costUsd !== undefined)
87
+ recordInferenceCost(labels, costUsd);
88
+ // A span attribute, not a metric label: unbounded cardinality.
89
+ if (reasoningTokens !== undefined)
90
+ span.setAttribute(REASONING_OUTPUT_TOKENS_ATTR, reasoningTokens);
91
+ return costUsd;
92
+ }
93
+ /** Transient failures are retried inside `judgePost`; a 401 is not, so it reaches the key fallback here. */
94
+ async function postWithKeyFallback(post, primaryKey, backupKeys) {
95
+ let response = await post(primaryKey);
96
+ while (response.status === HTTP_STATUS.UNAUTHORIZED) {
97
+ const backup = backupKeys.next();
98
+ if (backup.done)
99
+ break;
100
+ // eslint-disable-next-line no-await-in-loop -- frees the refused response's HTTP/1.1 socket first
101
+ await response.discard();
102
+ // eslint-disable-next-line no-await-in-loop -- backup keys are tried only after the previous key is refused
103
+ response = await post(backup.value);
104
+ }
105
+ return response;
106
+ }
107
+ /** Provider derived from the model id, so a model and a credential cannot be mismatched at a call site. */
108
+ export async function callJudge(prompt, canary, model) {
109
+ if (canary)
110
+ return { score: CANARY_SCORE_MIN + Math.random() * CANARY_SCORE_SPAN, reason: CANARY_REASON };
111
+ const resolvedModel = model ?? JUDGE_MODEL;
112
+ const provider = providerForModel(resolvedModel);
113
+ if (!provider)
114
+ throw new UnknownJudgeModelError(resolvedModel);
115
+ const spec = PROVIDER_SPECS[provider];
116
+ const keys = judgeApiKeys(provider);
117
+ const primary = keys.next();
118
+ if (primary.done)
119
+ throw new MissingJudgeCredentialError(provider);
120
+ const genAiAttributes = {
121
+ [GENAI_AGENT_ATTRIBUTES.OPERATION_NAME]: GENAI_OPERATION.CHAT,
122
+ [GENAI_REQUEST_MODEL_ATTR]: resolvedModel,
123
+ [GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider,
124
+ };
125
+ return withSpan(GENAI_CLIENT_OPERATION_SPAN, genAiAttributes, async (span) => {
126
+ // Lazy keeps undici off Stop's startup; before startMs so it is not counted as API latency.
127
+ const { judgePost } = await import('./judge-transport.js');
128
+ const startMs = Date.now();
129
+ let errorType;
130
+ try {
131
+ const body = JSON.stringify(spec.body(resolvedModel, prompt));
132
+ const post = (apiKey) => judgePost({
133
+ url: spec.url,
134
+ headers: spec.headers(apiKey),
135
+ body,
136
+ timeoutMs: spec.timeoutMs,
137
+ onRetry: (reason) => recordRetry(provider, reason),
138
+ });
139
+ const response = await postWithKeyFallback(post, primary.value, keys);
140
+ if (!response.ok) {
141
+ await response.discard();
142
+ throw new JudgeHttpError(provider, response.status);
143
+ }
144
+ const usage = spec.parse(await response.json());
145
+ const callCostUsd = recordJudgeUsage(span, resolvedModel, provider, usage);
146
+ const { score, reason } = parseJudgeVerdict(usage.text);
147
+ const { traceId, spanId } = span.spanContext();
148
+ const actualCostCents = callCostUsd === undefined ? undefined : callCostUsd * CENTS_PER_USD;
149
+ return { score, reason, actualCostCents, spanContext: { traceId, spanId } };
150
+ }
151
+ catch (error) {
152
+ errorType = judgeErrorType(error);
153
+ span.setAttribute(ERROR_TYPE_ATTR, errorType);
154
+ throw error;
155
+ }
156
+ finally {
157
+ const inferenceAttributes = errorType === undefined
158
+ ? genAiAttributes
159
+ : { ...genAiAttributes, [ERROR_TYPE_ATTR]: errorType };
160
+ // HLS34: the full call including parse and pre-status failures; the only duration emitted for inference.
161
+ recordMetric(GENAI_CLIENT_INFERENCE_DURATION_METRIC, elapsedSeconds(startMs), inferenceAttributes, {
162
+ unit: UNITS.SECOND,
163
+ boundaries: GENAI_DURATION_BUCKETS_S.TOOL,
164
+ });
165
+ }
166
+ });
167
+ }
168
+ export async function callLlmJudge(metricName, turn, canary) {
169
+ return callJudge(buildJudgePrompt(metricName, turn), canary);
170
+ }
@@ -0,0 +1,99 @@
1
+ /** Daily T2 evaluation budget: one UTC-date-stamped file in `qualityStateDir` that rolls over in place (HDF3). */
2
+ import { join } from 'node:path';
3
+ import { FILE_LOCK } from '../core/constants.js';
4
+ import { loadJson, removeFile, tryWithFileLock, writeJson } from '../core/file-utils.js';
5
+ import { utcDateStamp } from '../core/format-utils.js';
6
+ import { qualityStateDir } from '../session/session-state.js';
7
+ /** Longer stale window than `withFileLock` defaults: a batch may hold the counter past 5 s. */
8
+ const BUDGET_LOCK_OPTIONS = {
9
+ maxRetries: FILE_LOCK.BUDGET_MAX_RETRIES,
10
+ retryDelayMs: FILE_LOCK.BUDGET_RETRY_DELAY_MS,
11
+ staleMs: FILE_LOCK.BUDGET_STALE_MS,
12
+ };
13
+ export const DAILY_BUDGET_CENTS = 50;
14
+ export const COST_PER_METRIC_CENTS = 0.75;
15
+ const BUDGET_FILENAME = 'budget.json';
16
+ function getBudgetPath() {
17
+ return join(qualityStateDir(), BUDGET_FILENAME);
18
+ }
19
+ function loadBudget(path) {
20
+ const date = utcDateStamp();
21
+ // null covers missing and malformed; another date is yesterday's spend.
22
+ const parsed = loadJson(path);
23
+ if (parsed?.date !== date) {
24
+ return { date, evaluationsRun: 0, estimatedCostCents: 0 };
25
+ }
26
+ return {
27
+ date,
28
+ evaluationsRun: Number(parsed.evaluationsRun ?? 0),
29
+ estimatedCostCents: Number(parsed.estimatedCostCents ?? 0),
30
+ };
31
+ }
32
+ function saveBudget(path, state) {
33
+ return writeJson(path, state, false, { silent: true });
34
+ }
35
+ /** Snapshot check; use `tryReserveBudget` for correctness under concurrency. */
36
+ export function hasBudget() {
37
+ const state = loadBudget(getBudgetPath());
38
+ return state.estimatedCostCents < DAILY_BUDGET_CENTS;
39
+ }
40
+ /**
41
+ * Locked read-modify-write of today's counters. The callback mutates `state`; returning `false`
42
+ * skips the save. True only when the lock was held, the callback did not decline and the save
43
+ * landed; a throw from the lock or the write reads as false, so every caller fails closed.
44
+ */
45
+ function mutateBudget(fn) {
46
+ try {
47
+ const path = getBudgetPath();
48
+ const outcome = tryWithFileLock(path, () => {
49
+ const state = loadBudget(path);
50
+ if (fn(state) === false)
51
+ return false;
52
+ return saveBudget(path, state);
53
+ }, BUDGET_LOCK_OPTIONS);
54
+ return outcome.locked && outcome.value;
55
+ }
56
+ catch {
57
+ return false;
58
+ }
59
+ }
60
+ /** Locked check-and-reserve. Fails CLOSED: a lock or write failure returns false (HRV5). */
61
+ export function tryReserveBudget(costCents = COST_PER_METRIC_CENTS) {
62
+ return mutateBudget(state => {
63
+ if (state.estimatedCostCents + costCents > DAILY_BUDGET_CENTS)
64
+ return false;
65
+ state.evaluationsRun += 1;
66
+ state.estimatedCostCents += costCents;
67
+ return true;
68
+ });
69
+ }
70
+ /** True-up a reservation to measured cost (OBP19). Best-effort: a lock failure leaves it off by `|actual − reserved|`. */
71
+ export function trueUpBudget(actualCents, reservedCents) {
72
+ const delta = actualCents - reservedCents;
73
+ if (delta === 0)
74
+ return;
75
+ mutateBudget(state => {
76
+ state.estimatedCostCents = Math.max(0, state.estimatedCostCents + delta);
77
+ });
78
+ }
79
+ /** Return a reservation whose call never completed. Best-effort; a lock failure over-counts, the safe direction. */
80
+ export function releaseBudget(costCents = COST_PER_METRIC_CENTS) {
81
+ mutateBudget(state => {
82
+ state.evaluationsRun = Math.max(0, state.evaluationsRun - 1);
83
+ state.estimatedCostCents = Math.max(0, state.estimatedCostCents - costCents);
84
+ });
85
+ }
86
+ /** @internal */
87
+ export function _resetBudgetState() {
88
+ removeFile(getBudgetPath());
89
+ }
90
+ /** Current usage for telemetry. */
91
+ export function getBudgetUsage() {
92
+ const state = loadBudget(getBudgetPath());
93
+ return {
94
+ evaluationsRun: state.evaluationsRun,
95
+ estimatedCostCents: state.estimatedCostCents,
96
+ budgetCents: DAILY_BUDGET_CENTS,
97
+ remaining: Math.max(0, DAILY_BUDGET_CENTS - state.estimatedCostCents),
98
+ };
99
+ }