@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -0,0 +1,1023 @@
1
+ """Langfuse core reporting, official plugin provisioning, and trajectory adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import re
9
+ import shutil
10
+ import tarfile
11
+ import tempfile
12
+ from contextlib import contextmanager
13
+ from dataclasses import dataclass
14
+ from pathlib import Path, PurePosixPath
15
+ from types import SimpleNamespace
16
+ from typing import Any, Callable, Iterator, Mapping
17
+ from urllib.request import Request, urlopen
18
+
19
+ from scaffold.python.pass_at_k import compute_pass_metrics
20
+
21
+
22
+ class LangfuseConfigurationError(RuntimeError):
23
+ """The Langfuse suite cannot start safely with the current configuration."""
24
+
25
+
26
+ class LangfuseReportingError(RuntimeError):
27
+ """A required Langfuse core trace or score could not be written."""
28
+
29
+
30
+ class TrajectoryProvisionError(RuntimeError):
31
+ """An official trajectory plugin could not be safely provisioned."""
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class TrajectoryEvent:
36
+ """A bounded, agent-neutral event parsed from a JSONL transcript."""
37
+
38
+ kind: str
39
+ name: str
40
+ input: Any = None
41
+ output: Any = None
42
+ session_id: str | None = None
43
+ truncated: bool = False
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class TrajectoryProvision:
48
+ """The isolated plugin or hook provision selected for one eval run."""
49
+
50
+ agent: str
51
+ mode: str
52
+ cache_dir: Path
53
+ plugin_path: Path | None = None
54
+ source_repo: str | None = None
55
+ source_ref: str | None = None
56
+ archive_sha256: str | None = None
57
+ content_sha256: str | None = None
58
+
59
+
60
+ _OFFICIAL_PLUGIN_SPECS: dict[str, dict[str, str]] = {
61
+ "claude-code": {
62
+ "repo": "langfuse/Claude-Observability-Plugin",
63
+ "ref": "05b7742829e0b0ad840c8760376ca737fa1eb662",
64
+ "archive_root": ".",
65
+ },
66
+ "codex": {
67
+ "repo": "langfuse/codex-observability-plugin",
68
+ "ref": "96e997cf9c3d4aee7421e849d764b83dcb485462",
69
+ "archive_root": "plugins/tracing",
70
+ },
71
+ }
72
+
73
+ _RUBRIC_SCORE_RE = re.compile(r"\[RUBRIC\]\s+([A-Za-z0-9_.-]+):\s*(0(?:\.\d+)?|1(?:\.0+)?)\b")
74
+ _SUMMARY_K_VALUES = (1, 2, 5)
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class LangfuseConfig:
79
+ public_key: str
80
+ secret_key: str
81
+ base_url: str | None = None
82
+ tracing_environment: str | None = None
83
+
84
+ @classmethod
85
+ def from_environment(cls, environ: Mapping[str, str] | None = None) -> "LangfuseConfig":
86
+ env = environ or os.environ
87
+ missing = [
88
+ name
89
+ for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY")
90
+ if not env.get(name, "").strip()
91
+ ]
92
+ if missing:
93
+ raise LangfuseConfigurationError(
94
+ "Langfuse suite requires "
95
+ + ", ".join(missing)
96
+ + ". Set them in the process environment; they are never persisted by Comet."
97
+ )
98
+ return cls(
99
+ public_key=env["LANGFUSE_PUBLIC_KEY"].strip(),
100
+ secret_key=env["LANGFUSE_SECRET_KEY"].strip(),
101
+ base_url=env.get("LANGFUSE_BASE_URL") or None,
102
+ tracing_environment=env.get("LANGFUSE_TRACING_ENVIRONMENT") or None,
103
+ )
104
+
105
+
106
+ def trajectory_mode(agent: str) -> str:
107
+ return {
108
+ "claude-code": "official-claude-code-plugin",
109
+ "codex": "official-codex-plugin",
110
+ "qoder": "qoder-stop-transcript",
111
+ "codebuddy": "codebuddy-stop-transcript",
112
+ }.get(agent, "best-effort-core-only")
113
+
114
+
115
+ def _sha256_file(path: Path) -> str:
116
+ digest = hashlib.sha256()
117
+ with path.open("rb") as stream:
118
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
119
+ digest.update(chunk)
120
+ return digest.hexdigest()
121
+
122
+
123
+ def _sha256_tree(root: Path) -> str:
124
+ digest = hashlib.sha256()
125
+ for path in sorted(item for item in root.rglob("*") if item.is_file()):
126
+ relative = path.relative_to(root).as_posix()
127
+ digest.update(relative.encode("utf-8"))
128
+ digest.update(b"\0")
129
+ digest.update(path.read_bytes())
130
+ digest.update(b"\0")
131
+ return digest.hexdigest()
132
+
133
+
134
+ def _download_plugin_archive(url: str, destination: Path) -> None:
135
+ request = Request(url, headers={"User-Agent": "Comet eval Langfuse adapter"})
136
+ try:
137
+ with urlopen(request, timeout=90) as response, destination.open("wb") as output:
138
+ shutil.copyfileobj(response, output)
139
+ except Exception as exc:
140
+ raise TrajectoryProvisionError(
141
+ f"Langfuse trajectory plugin download failed: {exc}"
142
+ ) from exc
143
+
144
+
145
+ @contextmanager
146
+ def _provision_lock(path: Path) -> Iterator[None]:
147
+ """Coordinate cache writes across pytest workers and separate processes."""
148
+ path.parent.mkdir(parents=True, exist_ok=True)
149
+ with path.open("a+b") as lock_file:
150
+ if os.name == "nt":
151
+ import msvcrt
152
+
153
+ lock_file.seek(0, os.SEEK_END)
154
+ if lock_file.tell() == 0:
155
+ lock_file.write(b"0")
156
+ lock_file.flush()
157
+ lock_file.seek(0)
158
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
159
+ try:
160
+ yield
161
+ finally:
162
+ lock_file.seek(0)
163
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
164
+ else:
165
+ import fcntl
166
+
167
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
168
+ try:
169
+ yield
170
+ finally:
171
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
172
+
173
+
174
+ def _safe_extract_archive(archive_path: Path, destination: Path) -> Path:
175
+ destination.mkdir(parents=True, exist_ok=True)
176
+ root_name: str | None = None
177
+ try:
178
+ with tarfile.open(archive_path, mode="r:gz") as archive:
179
+ members = archive.getmembers()
180
+ for member in members:
181
+ relative = PurePosixPath(member.name)
182
+ if relative.is_absolute() or ".." in relative.parts:
183
+ raise TrajectoryProvisionError(
184
+ f"Langfuse trajectory plugin archive contains an unsafe path: {member.name}"
185
+ )
186
+ if member.issym() or member.islnk():
187
+ raise TrajectoryProvisionError(
188
+ f"Langfuse trajectory plugin archive contains a link: {member.name}"
189
+ )
190
+ if not root_name and relative.parts:
191
+ root_name = relative.parts[0]
192
+ target = destination.joinpath(*relative.parts).resolve()
193
+ if destination.resolve() not in target.parents and target != destination.resolve():
194
+ raise TrajectoryProvisionError(
195
+ f"Langfuse trajectory plugin archive escapes its cache: {member.name}"
196
+ )
197
+ archive.extractall(destination, filter="data")
198
+ except TrajectoryProvisionError:
199
+ raise
200
+ except (OSError, tarfile.TarError) as exc:
201
+ raise TrajectoryProvisionError(
202
+ f"Langfuse trajectory plugin archive is invalid: {exc}"
203
+ ) from exc
204
+ if not root_name:
205
+ raise TrajectoryProvisionError("Langfuse trajectory plugin archive is empty")
206
+ return destination / root_name
207
+
208
+
209
+ def _read_provisioned_plugin(
210
+ cache_dir: Path, agent: str, spec: Mapping[str, str]
211
+ ) -> TrajectoryProvision:
212
+ metadata_path = cache_dir / "metadata.json"
213
+ plugin_path = cache_dir / "plugin"
214
+ try:
215
+ metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
216
+ actual_content_hash = _sha256_tree(plugin_path)
217
+ except (OSError, ValueError, TypeError) as exc:
218
+ raise TrajectoryProvisionError(
219
+ f"Langfuse trajectory plugin cache is incomplete for {agent}: {cache_dir}"
220
+ ) from exc
221
+ if (
222
+ metadata.get("source_ref") != spec["ref"]
223
+ or metadata.get("content_sha256") != actual_content_hash
224
+ ):
225
+ raise TrajectoryProvisionError(
226
+ f"Langfuse trajectory plugin cache integrity check failed for {agent}: {cache_dir}"
227
+ )
228
+ return TrajectoryProvision(
229
+ agent=agent,
230
+ mode=trajectory_mode(agent),
231
+ cache_dir=cache_dir,
232
+ plugin_path=plugin_path,
233
+ source_repo=metadata.get("source_repo"),
234
+ source_ref=metadata.get("source_ref"),
235
+ archive_sha256=metadata.get("archive_sha256"),
236
+ content_sha256=actual_content_hash,
237
+ )
238
+
239
+
240
+ def provision_trajectory_plugin(
241
+ cache_root: Path,
242
+ agent: str,
243
+ *,
244
+ download: Callable[[str, Path], None] | None = None,
245
+ ) -> TrajectoryProvision:
246
+ """Provision a pinned official plugin without touching user agent config."""
247
+ mode = trajectory_mode(agent)
248
+ if agent not in _OFFICIAL_PLUGIN_SPECS:
249
+ return TrajectoryProvision(agent=agent, mode=mode, cache_dir=cache_root / agent)
250
+
251
+ spec = _OFFICIAL_PLUGIN_SPECS[agent]
252
+ cache_dir = cache_root / agent / spec["ref"]
253
+ cache_dir.parent.mkdir(parents=True, exist_ok=True)
254
+ with _provision_lock(cache_dir.parent / ".provision.lock"):
255
+ if (cache_dir / "metadata.json").is_file() and (cache_dir / "plugin").is_dir():
256
+ return _read_provisioned_plugin(cache_dir, agent, spec)
257
+
258
+ staging = Path(tempfile.mkdtemp(prefix=f".{agent}-", dir=cache_dir.parent))
259
+ try:
260
+ archive_path = staging / "plugin.tar.gz"
261
+ source_url = f"https://codeload.github.com/{spec['repo']}/tar.gz/{spec['ref']}"
262
+ (download or _download_plugin_archive)(source_url, archive_path)
263
+ archive_hash = _sha256_file(archive_path)
264
+ extracted_root = _safe_extract_archive(archive_path, staging / "extracted")
265
+ required_plugin_path = (
266
+ extracted_root
267
+ if spec["archive_root"] == "."
268
+ else extracted_root / spec["archive_root"]
269
+ )
270
+ if not required_plugin_path.is_dir():
271
+ raise TrajectoryProvisionError(
272
+ f"Langfuse trajectory plugin path is missing: {spec['archive_root']}"
273
+ )
274
+ plugin_destination = staging / "plugin"
275
+ shutil.copytree(extracted_root, plugin_destination)
276
+ content_hash = _sha256_tree(plugin_destination)
277
+ metadata = {
278
+ "schema": "comet.eval.langfuse-plugin.v1",
279
+ "agent": agent,
280
+ "source_repo": spec["repo"],
281
+ "source_ref": spec["ref"],
282
+ "archive_sha256": archive_hash,
283
+ "content_sha256": content_hash,
284
+ "mode": mode,
285
+ }
286
+ (staging / "metadata.json").write_text(
287
+ json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
288
+ )
289
+ if cache_dir.exists():
290
+ shutil.rmtree(cache_dir)
291
+ os.replace(staging, cache_dir)
292
+ staging = None # type: ignore[assignment]
293
+ except TrajectoryProvisionError:
294
+ raise
295
+ except Exception as exc:
296
+ raise TrajectoryProvisionError(
297
+ f"Langfuse trajectory plugin provisioning failed for {agent}: {exc}"
298
+ ) from exc
299
+ finally:
300
+ if staging is not None and staging.exists():
301
+ shutil.rmtree(staging, ignore_errors=True)
302
+ return _read_provisioned_plugin(cache_dir, agent, spec)
303
+
304
+
305
+ def _bound_value(value: Any, max_chars: int) -> tuple[Any, bool]:
306
+ if isinstance(value, str):
307
+ return (value[:max_chars], len(value) > max_chars)
308
+ serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
309
+ if len(serialized) <= max_chars:
310
+ return value, False
311
+ return serialized[:max_chars], True
312
+
313
+
314
+ def _message_content(record: Mapping[str, Any]) -> Any:
315
+ message = record.get("message")
316
+ if isinstance(message, Mapping):
317
+ return message.get("content")
318
+ return record.get("content")
319
+
320
+
321
+ def _session_id(record: Mapping[str, Any]) -> str | None:
322
+ value = record.get("sessionId") or record.get("session_id")
323
+ return str(value) if value else None
324
+
325
+
326
+ def parse_agent_transcript(path: Path, *, max_chars: int = 20_000) -> list[TrajectoryEvent]:
327
+ """Parse Qoder/CodeBuddy transcript JSONL without inferring missing events."""
328
+ events: list[TrajectoryEvent] = []
329
+ try:
330
+ lines = path.read_text(encoding="utf-8").splitlines()
331
+ except OSError as exc:
332
+ raise TrajectoryProvisionError(f"Unable to read agent transcript: {path}") from exc
333
+
334
+ for line in lines:
335
+ if not line.strip():
336
+ continue
337
+ try:
338
+ record = json.loads(line)
339
+ except json.JSONDecodeError:
340
+ continue
341
+ if not isinstance(record, Mapping):
342
+ continue
343
+ record_type = str(record.get("type", ""))
344
+ session_id = _session_id(record)
345
+ if record_type == "session_meta":
346
+ data, truncated = _bound_value(record.get("data", {}), max_chars)
347
+ events.append(
348
+ TrajectoryEvent(
349
+ "session",
350
+ "session_meta",
351
+ output=data,
352
+ session_id=session_id,
353
+ truncated=truncated,
354
+ )
355
+ )
356
+ continue
357
+
358
+ content = _message_content(record)
359
+ items = content if isinstance(content, list) else [content]
360
+ for item in items:
361
+ if isinstance(item, Mapping):
362
+ item_type = str(item.get("type", ""))
363
+ if item_type == "tool_use":
364
+ input_value, truncated = _bound_value(item.get("input", {}), max_chars)
365
+ events.append(
366
+ TrajectoryEvent(
367
+ "tool",
368
+ str(item.get("name") or "tool"),
369
+ input=input_value,
370
+ session_id=session_id,
371
+ truncated=truncated,
372
+ )
373
+ )
374
+ continue
375
+ if item_type == "tool_result":
376
+ output, truncated = _bound_value(item.get("content", ""), max_chars)
377
+ events.append(
378
+ TrajectoryEvent(
379
+ "tool_result",
380
+ str(item.get("tool_use_id") or "tool_result"),
381
+ output=output,
382
+ session_id=session_id,
383
+ truncated=truncated,
384
+ )
385
+ )
386
+ continue
387
+ if item_type == "text":
388
+ value, truncated = _bound_value(item.get("text", ""), max_chars)
389
+ events.append(
390
+ TrajectoryEvent(
391
+ "text",
392
+ "assistant",
393
+ output=value,
394
+ session_id=session_id,
395
+ truncated=truncated,
396
+ )
397
+ )
398
+ continue
399
+ elif isinstance(item, str) and item:
400
+ value, truncated = _bound_value(item, max_chars)
401
+ events.append(
402
+ TrajectoryEvent(
403
+ "user" if record_type == "user" else "text",
404
+ "user" if record_type == "user" else "assistant",
405
+ input=value if record_type == "user" else None,
406
+ output=value if record_type != "user" else None,
407
+ session_id=session_id,
408
+ truncated=truncated,
409
+ )
410
+ )
411
+
412
+ if record_type == "progress":
413
+ value, truncated = _bound_value(record.get("data", {}), max_chars)
414
+ events.append(
415
+ TrajectoryEvent(
416
+ "progress", "progress", output=value, session_id=session_id, truncated=truncated
417
+ )
418
+ )
419
+ return events
420
+
421
+
422
+ def render_transcript_hook_script() -> str:
423
+ """Return the no-credential Stop hook used by Qoder and CodeBuddy."""
424
+ return """#!/usr/bin/env python3
425
+ import hashlib
426
+ import json
427
+ import os
428
+ import shutil
429
+ import sys
430
+ import tempfile
431
+
432
+
433
+ def main():
434
+ try:
435
+ payload = json.load(sys.stdin)
436
+ except Exception:
437
+ return 0
438
+ if payload.get("hook_event_name") != "Stop":
439
+ return 0
440
+ transcript = payload.get("transcript_path")
441
+ cwd = payload.get("cwd") or os.getcwd()
442
+ if not transcript or not os.path.isfile(transcript):
443
+ return 0
444
+ role = os.environ.get("COMET_EVAL_AGENT_ROLE", "subject")
445
+ if role not in {"subject", "simulator", "judge"}:
446
+ role = "subject"
447
+ session_id = payload.get("session_id") or payload.get("sessionId")
448
+ if not session_id:
449
+ session_id = hashlib.sha256(os.path.abspath(transcript).encode()).hexdigest()[:16]
450
+ safe_session = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in str(session_id))[:128]
451
+ target_dir = os.path.join(cwd, ".comet", "eval", "langfuse", "trajectories")
452
+ target = os.path.join(target_dir, f"{role}-{safe_session}.jsonl")
453
+ os.makedirs(target_dir, exist_ok=True)
454
+ fd, temporary = tempfile.mkstemp(prefix=".trajectory-", dir=os.path.dirname(target))
455
+ os.close(fd)
456
+ try:
457
+ shutil.copyfile(transcript, temporary)
458
+ os.replace(temporary, target)
459
+ finally:
460
+ try:
461
+ os.unlink(temporary)
462
+ except FileNotFoundError:
463
+ pass
464
+ return 0
465
+
466
+
467
+ if __name__ == "__main__":
468
+ raise SystemExit(main())
469
+ """
470
+
471
+
472
+ def install_transcript_hook(test_dir: Path, agent: str) -> Path | None:
473
+ """Install a project-local, credential-free Stop hook for transcript agents."""
474
+ if agent not in {"qoder", "codebuddy"}:
475
+ return None
476
+ project_dir_name = ".qoder" if agent == "qoder" else ".codebuddy"
477
+ project_dir = test_dir / project_dir_name
478
+ hook_path = project_dir / "hooks" / "langfuse-stop-hook.py"
479
+ hook_path.parent.mkdir(parents=True, exist_ok=True)
480
+ hook_path.write_text(render_transcript_hook_script(), encoding="utf-8")
481
+ settings_path = project_dir / "settings.json"
482
+ settings: dict[str, Any] = {}
483
+ if settings_path.is_file():
484
+ settings = json.loads(settings_path.read_text(encoding="utf-8"))
485
+ hooks = settings.setdefault("hooks", {})
486
+ stop_hooks = hooks.setdefault("Stop", [])
487
+ entry = {
488
+ "hooks": [
489
+ {
490
+ "type": "command",
491
+ "command": f"python3 /workspace/{project_dir_name}/hooks/langfuse-stop-hook.py",
492
+ "timeout": 30,
493
+ }
494
+ ]
495
+ }
496
+ if entry not in stop_hooks:
497
+ stop_hooks.append(entry)
498
+ settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
499
+ return test_dir / ".comet" / "eval" / "langfuse" / "trajectories"
500
+
501
+
502
+ def install_codex_plugin_workspace(test_dir: Path, provision: TrajectoryProvision) -> None:
503
+ """Copy the pinned Codex plugin into the isolated project CODEX_HOME."""
504
+ if provision.agent != "codex" or provision.plugin_path is None:
505
+ return
506
+ source = provision.plugin_path / "plugins" / "tracing"
507
+ version = json.loads((source / ".codex-plugin" / "plugin.json").read_text(encoding="utf-8"))[
508
+ "version"
509
+ ]
510
+ target = (
511
+ test_dir
512
+ / ".codex"
513
+ / "plugins"
514
+ / "cache"
515
+ / "codex-observability-plugin"
516
+ / "tracing"
517
+ / version
518
+ )
519
+ target.parent.mkdir(parents=True, exist_ok=True)
520
+ shutil.copytree(source, target, dirs_exist_ok=True)
521
+ config_path = test_dir / ".codex" / "config.toml"
522
+ existing = config_path.read_text(encoding="utf-8") if config_path.is_file() else ""
523
+ lines = existing.splitlines(keepends=True)
524
+ feature_header = next(
525
+ (index for index, line in enumerate(lines) if line.strip() == "[features]"),
526
+ None,
527
+ )
528
+ if feature_header is None:
529
+ existing = existing.rstrip() + "\n\n[features]\nhooks = true\n"
530
+ else:
531
+ feature_end = next(
532
+ (
533
+ index
534
+ for index in range(feature_header + 1, len(lines))
535
+ if lines[index].lstrip().startswith("[")
536
+ ),
537
+ len(lines),
538
+ )
539
+ feature_lines = lines[feature_header + 1 : feature_end]
540
+ hooks_line = next(
541
+ (
542
+ index
543
+ for index, line in enumerate(feature_lines)
544
+ if line.strip().startswith("hooks =")
545
+ ),
546
+ None,
547
+ )
548
+ if hooks_line is None:
549
+ lines.insert(feature_header + 1 + len(feature_lines), "hooks = true\n")
550
+ else:
551
+ lines[feature_header + 1 + hooks_line] = "hooks = true\n"
552
+ existing = "".join(lines)
553
+ if '[plugins."tracing@codex-observability-plugin"]' not in existing:
554
+ existing = (
555
+ existing.rstrip()
556
+ + '\n\n[plugins."tracing@codex-observability-plugin"]\nenabled = true\n'
557
+ )
558
+ config_path.parent.mkdir(parents=True, exist_ok=True)
559
+ config_path.write_text(existing.lstrip(), encoding="utf-8")
560
+
561
+
562
+ def create_client(
563
+ config: LangfuseConfig | None = None,
564
+ *,
565
+ client_factory: Callable[..., Any] | None = None,
566
+ authenticate: bool = True,
567
+ ) -> Any:
568
+ """Create and authenticate the SDK client only at the run boundary."""
569
+ resolved = config or LangfuseConfig.from_environment()
570
+ if client_factory is None:
571
+ try:
572
+ from langfuse import Langfuse
573
+ except Exception as exc: # pragma: no cover - optional dependency
574
+ raise LangfuseConfigurationError(
575
+ "The Langfuse Python SDK is unavailable. Install the eval langfuse extra."
576
+ ) from exc
577
+ client_factory = Langfuse
578
+
579
+ kwargs: dict[str, str] = {"public_key": resolved.public_key, "secret_key": resolved.secret_key}
580
+ if resolved.base_url:
581
+ kwargs["base_url"] = resolved.base_url
582
+ if resolved.tracing_environment:
583
+ kwargs["environment"] = resolved.tracing_environment
584
+ try:
585
+ client = client_factory(**kwargs)
586
+ if authenticate and hasattr(client, "auth_check") and not client.auth_check():
587
+ raise LangfuseConfigurationError("Langfuse authentication check failed")
588
+ return client
589
+ except LangfuseConfigurationError:
590
+ raise
591
+ except Exception as exc:
592
+ raise LangfuseConfigurationError(f"Langfuse authentication failed: {exc}") from exc
593
+
594
+
595
+ def enable_trajectory_environment(config: LangfuseConfig, agent: str) -> str:
596
+ """Expose credentials to a selected agent without writing them to artifacts."""
597
+ os.environ["TRACE_TO_LANGFUSE"] = "true"
598
+ os.environ["LANGFUSE_PUBLIC_KEY"] = config.public_key
599
+ os.environ["LANGFUSE_SECRET_KEY"] = config.secret_key
600
+ if config.base_url:
601
+ os.environ["LANGFUSE_BASE_URL"] = config.base_url
602
+ if config.tracing_environment:
603
+ os.environ["LANGFUSE_TRACING_ENVIRONMENT"] = config.tracing_environment
604
+ os.environ["LANGFUSE_TRAJECTORY_MODE"] = trajectory_mode(agent)
605
+ os.environ["LANGFUSE_TRAJECTORY_PROVISIONED"] = "false"
606
+ return trajectory_mode(agent)
607
+
608
+
609
+ def _result_payload(result: Any) -> dict[str, Any]:
610
+ passed = list(getattr(result, "checks_passed", []) or [])
611
+ failed = list(getattr(result, "checks_failed", []) or [])
612
+ summary = dict(getattr(result, "events_summary", {}) or {})
613
+ total_checks = len(passed) + len(failed)
614
+ payload: dict[str, Any] = {
615
+ "run_id": getattr(result, "run_id", ""),
616
+ "passed": bool(getattr(result, "passed", False)),
617
+ "checks_passed": len(passed),
618
+ "checks_failed": len(failed),
619
+ "checks_pass_rate": len(passed) / total_checks if total_checks else 0.0,
620
+ "failure_attribution": summary.get("failure_attribution"),
621
+ "num_turns": summary.get("num_turns"),
622
+ "tool_calls": summary.get("tool_calls"),
623
+ "duration_seconds": summary.get("duration_seconds"),
624
+ "total_tokens": summary.get("total_tokens"),
625
+ "total_cost_usd": summary.get("total_cost_usd"),
626
+ "rubric": _rubric_scores(passed),
627
+ }
628
+ for key in (
629
+ "task",
630
+ "treatment",
631
+ "sample",
632
+ "prompt",
633
+ "skill",
634
+ "profile",
635
+ "final_response",
636
+ "quality_gates",
637
+ "execution_identity",
638
+ "case_manifest",
639
+ "interaction",
640
+ "agent",
641
+ "role_models",
642
+ "role_sessions",
643
+ ):
644
+ value = summary.get(key)
645
+ if value is not None:
646
+ payload[key] = _bound_value(value, 20_000)[0]
647
+ experiment_id = os.environ.get("COMET_EVAL_EXPERIMENT_ID")
648
+ if experiment_id:
649
+ payload["experiment_id"] = experiment_id
650
+ payload["quality_gate"] = _quality_gate_result(payload)
651
+ return payload
652
+
653
+
654
+ def result_payload_from_local_report(report: Mapping[str, Any]) -> dict[str, Any]:
655
+ """Rebuild the canonical case payload from a persisted local report."""
656
+ return _result_payload(
657
+ SimpleNamespace(
658
+ run_id=report.get("run_id", ""),
659
+ passed=bool(report.get("passed")),
660
+ checks_passed=list(report.get("checks_passed") or []),
661
+ checks_failed=list(report.get("checks_failed") or []),
662
+ events_summary=dict(report.get("events_summary") or {}),
663
+ )
664
+ )
665
+
666
+
667
+ def _rubric_scores(checks: list[str]) -> dict[str, float]:
668
+ scores: dict[str, float] = {}
669
+ for check in checks:
670
+ match = _RUBRIC_SCORE_RE.search(check)
671
+ if not match:
672
+ continue
673
+ try:
674
+ scores[match.group(1)] = float(match.group(2))
675
+ except ValueError:
676
+ continue
677
+ return scores
678
+
679
+
680
+ def _quality_gate_result(payload: Mapping[str, Any]) -> dict[str, Any]:
681
+ configured = payload.get("quality_gates")
682
+ if not isinstance(configured, Mapping) or not configured:
683
+ return {"status": "not_applicable", "checks": {}}
684
+
685
+ rubric = payload.get("rubric")
686
+ rubric_scores = rubric if isinstance(rubric, Mapping) else {}
687
+ checks: dict[str, bool | None] = {}
688
+ aliases = {
689
+ "minWeightedScore": "weighted_score",
690
+ "minPassRate": "checks_pass_rate",
691
+ "minChecksPassRate": "checks_pass_rate",
692
+ }
693
+ for name, expected in configured.items():
694
+ metric_name = aliases.get(str(name))
695
+ if metric_name is None and str(name).startswith("min"):
696
+ candidate = str(name)[3:]
697
+ metric_name = candidate[:1].lower() + candidate[1:]
698
+ if (
699
+ metric_name is None
700
+ or not isinstance(expected, (int, float))
701
+ or isinstance(expected, bool)
702
+ ):
703
+ checks[str(name)] = None
704
+ continue
705
+ actual = rubric_scores.get(metric_name)
706
+ if metric_name == "checks_pass_rate":
707
+ actual = payload.get("checks_pass_rate")
708
+ checks[str(name)] = (
709
+ isinstance(actual, (int, float))
710
+ and not isinstance(actual, bool)
711
+ and float(actual) >= float(expected)
712
+ )
713
+
714
+ applicable = [value for value in checks.values() if value is not None]
715
+ if not applicable:
716
+ status = "unavailable"
717
+ elif any(value is False for value in applicable):
718
+ status = "failed"
719
+ elif any(value is None for value in checks.values()):
720
+ status = "unavailable"
721
+ else:
722
+ status = "passed"
723
+ return {"status": status, "checks": checks}
724
+
725
+
726
+ class LangfuseRunReporter:
727
+ """Write one strict core trace per task/treatment and score its result."""
728
+
729
+ def __init__(self, client: Any, *, suite: str = "comet-eval-langfuse"):
730
+ self.client = client
731
+ self.suite = suite
732
+ self.cases: list[dict[str, Any]] = []
733
+
734
+ def report_case(
735
+ self,
736
+ task_name: str,
737
+ treatment_name: str,
738
+ run: Callable[[], Any],
739
+ result_loader: Callable[[], Any],
740
+ *,
741
+ metadata: Mapping[str, Any] | None = None,
742
+ trajectory_path: Path | Callable[[], Path | None] | None = None,
743
+ agent: str | None = None,
744
+ ) -> Any:
745
+ input_payload = {
746
+ "task": task_name,
747
+ "treatment": treatment_name,
748
+ "suite": self.suite,
749
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
750
+ **dict(metadata or {}),
751
+ }
752
+ with self.client.start_as_current_observation(
753
+ as_type="span", name="comet.eval.run", input=input_payload
754
+ ) as observation:
755
+ result = None
756
+ run_error: BaseException | None = None
757
+ try:
758
+ result = run()
759
+ except BaseException as exc:
760
+ run_error = exc
761
+ result_loader_error: BaseException | None = None
762
+ try:
763
+ result = result_loader() or result
764
+ except BaseException as exc:
765
+ result_loader_error = exc
766
+ payload = (
767
+ _result_payload(result)
768
+ if result is not None
769
+ else {
770
+ "passed": False,
771
+ "checks_passed": 0,
772
+ "checks_failed": 0,
773
+ "checks_pass_rate": 0.0,
774
+ }
775
+ )
776
+ try:
777
+ resolved_trajectory_path = (
778
+ trajectory_path() if callable(trajectory_path) else trajectory_path
779
+ )
780
+ payload["trajectory"] = self._report_trajectory(
781
+ resolved_trajectory_path,
782
+ agent=agent or str(input_payload.get("agent", "")),
783
+ role_sessions=payload.get("role_sessions") or {},
784
+ )
785
+ except Exception as exc:
786
+ print(f"[langfuse] optional trajectory upload skipped: {exc}", file=os.sys.stderr)
787
+ payload["trajectory"] = {
788
+ "status": "unavailable",
789
+ "mode": trajectory_mode(agent or str(input_payload.get("agent", ""))),
790
+ "reason": str(exc),
791
+ }
792
+ payload["trajectory_status"] = payload["trajectory"].get("status", "unavailable")
793
+ for key in ("prompt", "skill", "profile", "execution_identity", "agent", "sample"):
794
+ if payload.get(key) is not None:
795
+ input_payload[key] = payload[key]
796
+ core_error: BaseException | None = None
797
+ try:
798
+ observation.update(input=input_payload, output=payload)
799
+ self._score(observation, payload)
800
+ except BaseException as exc:
801
+ core_error = exc
802
+ self.cases.append({"task": task_name, "treatment": treatment_name, **payload})
803
+ if run_error is not None:
804
+ if core_error is not None:
805
+ print(
806
+ f"[langfuse] evaluation failed and core upload also failed: {core_error}",
807
+ file=os.sys.stderr,
808
+ )
809
+ raise run_error
810
+ if core_error is not None:
811
+ raise core_error
812
+ if result_loader_error is not None:
813
+ raise result_loader_error
814
+ return result
815
+
816
+ def _report_trajectory(
817
+ self,
818
+ trajectory_path: Path | None,
819
+ *,
820
+ agent: str,
821
+ role_sessions: Mapping[str, Any],
822
+ ) -> dict[str, Any]:
823
+ mode = trajectory_mode(agent)
824
+ if mode.startswith("official-"):
825
+ if os.environ.get("LANGFUSE_TRAJECTORY_PROVISIONED", "").lower() != "true":
826
+ return {
827
+ "status": "unavailable",
828
+ "mode": mode,
829
+ "reason": "plugin_not_provisioned",
830
+ }
831
+ return {
832
+ "status": "provisioned",
833
+ "mode": mode,
834
+ "role_sessions": dict(role_sessions),
835
+ }
836
+ if not trajectory_path or not trajectory_path.exists():
837
+ return {"status": "unavailable", "mode": mode, "reason": "transcript_not_found"}
838
+ try:
839
+ transcript_paths = (
840
+ sorted(trajectory_path.glob("*.jsonl"))
841
+ if trajectory_path.is_dir()
842
+ else [trajectory_path]
843
+ )
844
+ events = [
845
+ event
846
+ for transcript_path in transcript_paths
847
+ for event in parse_agent_transcript(transcript_path)
848
+ ]
849
+ if not events:
850
+ return {"status": "unavailable", "mode": mode, "reason": "transcript_empty"}
851
+ session_ids = sorted({event.session_id for event in events if event.session_id})
852
+ for index, event in enumerate(events):
853
+ observation_type = "generation" if event.kind == "text" else "span"
854
+ with self.client.start_as_current_observation(
855
+ as_type=observation_type,
856
+ name=f"agent.{event.kind}/{event.name}/{index}",
857
+ input=event.input,
858
+ ) as child:
859
+ child.update(
860
+ output=event.output,
861
+ metadata={
862
+ "agent": agent,
863
+ "session_id": event.session_id,
864
+ "truncated": event.truncated,
865
+ },
866
+ )
867
+ return {
868
+ "status": "uploaded",
869
+ "mode": mode,
870
+ "events": len(events),
871
+ "session_ids": session_ids,
872
+ "role_sessions": dict(role_sessions),
873
+ "truncated_events": sum(1 for event in events if event.truncated),
874
+ }
875
+ except Exception as exc:
876
+ print(f"[langfuse] optional trajectory upload skipped: {exc}", file=os.sys.stderr)
877
+ return {"status": "unavailable", "mode": mode, "reason": str(exc)}
878
+
879
+ def _score(self, observation: Any, payload: Mapping[str, Any]) -> None:
880
+ trace_id = getattr(observation, "trace_id", None)
881
+ observation_id = getattr(observation, "id", None)
882
+ if not trace_id:
883
+ raise LangfuseReportingError("Langfuse observation did not expose a trace id")
884
+ try:
885
+ for name, value, data_type in (
886
+ ("task_passed", bool(payload.get("passed")), "BOOLEAN"),
887
+ ("checks_pass_rate", float(payload.get("checks_pass_rate", 0.0)), "NUMERIC"),
888
+ ):
889
+ self.client.create_score(
890
+ name={
891
+ "task_passed": "comet.passed",
892
+ "checks_pass_rate": "comet.checks_pass_rate",
893
+ }[name],
894
+ value=value,
895
+ trace_id=trace_id,
896
+ observation_id=observation_id,
897
+ data_type=data_type,
898
+ )
899
+ for dimension, value in (payload.get("rubric") or {}).items():
900
+ self.client.create_score(
901
+ name=f"comet.rubric.{dimension}",
902
+ value=float(value),
903
+ trace_id=trace_id,
904
+ observation_id=observation_id,
905
+ data_type="NUMERIC",
906
+ )
907
+ except Exception as exc:
908
+ raise LangfuseReportingError(f"Langfuse core score failed: {exc}") from exc
909
+
910
+ def report_summary(self) -> None:
911
+ """Write the suite summary as a final core observation."""
912
+ try:
913
+ with self.client.start_as_current_observation(
914
+ as_type="span",
915
+ name="comet.eval.experiment",
916
+ input={
917
+ "suite": self.suite,
918
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
919
+ "cases": len(self.cases),
920
+ },
921
+ ) as observation:
922
+ passed = sum(1 for case in self.cases if case.get("passed"))
923
+ aggregates = self._aggregates()
924
+ observation.update(
925
+ output={
926
+ "cases": len(self.cases),
927
+ "passed": passed,
928
+ "failed": len(self.cases) - passed,
929
+ "aggregates": aggregates,
930
+ }
931
+ )
932
+ self._score_summary(observation, aggregates)
933
+ except Exception as exc:
934
+ raise LangfuseReportingError(f"Langfuse summary trace failed: {exc}") from exc
935
+
936
+ def _aggregates(self) -> dict[str, Any]:
937
+ groups: dict[str, list[dict[str, Any]]] = {}
938
+ for case in self.cases:
939
+ task = str(case.get("task", "unknown"))
940
+ treatment = str(case.get("treatment", "unknown"))
941
+ groups.setdefault(f"{task}/{treatment}", []).append(case)
942
+
943
+ aggregates: dict[str, Any] = {}
944
+ for key, cases in groups.items():
945
+ outcomes = [bool(case.get("passed")) for case in cases]
946
+ metrics = {str(k): compute_pass_metrics(outcomes, k) for k in _SUMMARY_K_VALUES}
947
+ gate_statuses = [(case.get("quality_gate") or {}).get("status") for case in cases]
948
+ if gate_statuses and all(status == "not_applicable" for status in gate_statuses):
949
+ quality_gate = "not_applicable"
950
+ elif "failed" in gate_statuses:
951
+ quality_gate = "failed"
952
+ elif "unavailable" in gate_statuses:
953
+ quality_gate = "unavailable"
954
+ else:
955
+ quality_gate = "passed"
956
+ aggregates[key] = {
957
+ "task": key.split("/", 1)[0],
958
+ "treatment": key.split("/", 1)[1],
959
+ "runs": len(cases),
960
+ "pass_at_k": {k: metrics[str(k)]["pass_at_k"] for k in _SUMMARY_K_VALUES},
961
+ "pass_power_k": {k: metrics[str(k)]["pass_pow_k"] for k in _SUMMARY_K_VALUES},
962
+ "quality_gate": quality_gate,
963
+ "agent": (
964
+ next(iter({str(case.get("agent")) for case in cases}))
965
+ if len({str(case.get("agent")) for case in cases}) == 1
966
+ else "mixed"
967
+ ),
968
+ }
969
+ return aggregates
970
+
971
+ def _score_summary(self, observation: Any, aggregates: Mapping[str, Any]) -> None:
972
+ trace_id = getattr(observation, "trace_id", None)
973
+ observation_id = getattr(observation, "id", None)
974
+ if not trace_id:
975
+ raise LangfuseReportingError("Langfuse summary observation did not expose a trace id")
976
+ try:
977
+ for aggregate in aggregates.values():
978
+ for k in _SUMMARY_K_VALUES:
979
+ score_metadata = {
980
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
981
+ "agent": aggregate["agent"],
982
+ "task": aggregate["task"],
983
+ "treatment": aggregate["treatment"],
984
+ "k": k,
985
+ }
986
+ self.client.create_score(
987
+ name="comet.pass_at_k",
988
+ value=float(aggregate["pass_at_k"][k]),
989
+ trace_id=trace_id,
990
+ observation_id=observation_id,
991
+ data_type="NUMERIC",
992
+ metadata=score_metadata,
993
+ )
994
+ self.client.create_score(
995
+ name="comet.pass_power_k",
996
+ value=float(aggregate["pass_power_k"][k]),
997
+ trace_id=trace_id,
998
+ observation_id=observation_id,
999
+ data_type="NUMERIC",
1000
+ metadata=score_metadata,
1001
+ )
1002
+ if aggregate["quality_gate"] in {"passed", "failed"}:
1003
+ self.client.create_score(
1004
+ name="comet.quality_gate",
1005
+ value=aggregate["quality_gate"] == "passed",
1006
+ trace_id=trace_id,
1007
+ observation_id=observation_id,
1008
+ data_type="BOOLEAN",
1009
+ metadata={
1010
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
1011
+ "agent": aggregate["agent"],
1012
+ "task": aggregate["task"],
1013
+ "treatment": aggregate["treatment"],
1014
+ },
1015
+ )
1016
+ except Exception as exc:
1017
+ raise LangfuseReportingError(f"Langfuse summary score failed: {exc}") from exc
1018
+
1019
+ def flush(self) -> None:
1020
+ try:
1021
+ self.client.flush()
1022
+ except Exception as exc:
1023
+ raise LangfuseReportingError(f"Langfuse flush failed: {exc}") from exc