@rpamis/comet 0.4.0-beta.16 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (761) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +85 -56
  3. package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.html +8 -0
  4. package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.json +286 -0
  5. package/assets/manifest.json +26 -17
  6. package/assets/skills/comet/SKILL.md +1 -1
  7. package/assets/skills/comet/agents/openai.yaml +3 -0
  8. package/assets/skills/comet/rules/comet-phase-guard.en.md +1 -1
  9. package/assets/skills/comet/rules/comet-phase-guard.md +1 -1
  10. package/assets/skills/comet/rules/comet-workflow-guard.en.md +2 -2
  11. package/assets/skills/comet/rules/comet-workflow-guard.md +2 -2
  12. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  13. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +77 -77
  14. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  15. package/assets/skills/comet/scripts/comet-handoff.mjs +107 -107
  16. package/assets/skills/comet/scripts/comet-hook-guard.mjs +128 -128
  17. package/assets/skills/comet/scripts/comet-hook-router.mjs +143 -141
  18. package/assets/skills/comet/scripts/comet-resume-probe.mjs +100 -100
  19. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  20. package/assets/skills/comet/scripts/comet-state.mjs +118 -118
  21. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +95 -95
  22. package/assets/skills/comet-any/SKILL.md +2 -1
  23. package/assets/skills/comet-any/agents/openai.yaml +5 -0
  24. package/assets/skills/comet-archive/SKILL.md +7 -6
  25. package/assets/skills/comet-archive/agents/openai.yaml +5 -0
  26. package/assets/skills/comet-build/SKILL.md +28 -68
  27. package/assets/skills/comet-build/agents/openai.yaml +5 -0
  28. package/assets/skills/comet-classic/SKILL.md +28 -22
  29. package/assets/skills/comet-classic/agents/openai.yaml +3 -0
  30. package/assets/skills/{comet → comet-classic}/reference/auto-transition.md +1 -1
  31. package/assets/skills/{comet → comet-classic}/reference/comet-yaml-fields.md +3 -3
  32. package/assets/skills/{comet → comet-classic}/reference/context-recovery.md +5 -5
  33. package/assets/skills/{comet → comet-classic}/reference/debug-gate.md +2 -2
  34. package/assets/skills/{comet → comet-classic}/reference/decision-point.md +1 -1
  35. package/assets/skills/{comet → comet-classic}/reference/dirty-worktree.md +1 -1
  36. package/assets/skills/{comet → comet-classic}/reference/file-structure.md +1 -1
  37. package/assets/skills/{comet → comet-classic}/reference/intent-frame.md +1 -1
  38. package/assets/skills/{comet → comet-classic}/reference/scripts.md +4 -2
  39. package/assets/skills/{comet → comet-classic}/reference/subagent-dispatch.md +1 -1
  40. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills/comet-design/SKILL.md +7 -6
  42. package/assets/skills/comet-design/agents/openai.yaml +5 -0
  43. package/assets/skills/comet-hotfix/SKILL.md +10 -9
  44. package/assets/skills/comet-hotfix/agents/openai.yaml +5 -0
  45. package/assets/skills/comet-native/SKILL.md +60 -171
  46. package/assets/skills/comet-native/agents/openai.yaml +3 -0
  47. package/assets/skills/comet-native/reference/artifacts.md +24 -89
  48. package/assets/skills/comet-native/reference/clarification.md +54 -29
  49. package/assets/skills/comet-native/reference/commands.md +30 -173
  50. package/assets/skills/comet-native/reference/recovery.md +38 -88
  51. package/assets/skills/comet-native/reference/workspace.md +23 -0
  52. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +184 -152
  53. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +214 -158
  54. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +132 -123
  55. package/assets/skills/comet-native/scripts/comet-native-init.mjs +134 -124
  56. package/assets/skills/comet-native/scripts/comet-native-new.mjs +165 -158
  57. package/assets/skills/comet-native/scripts/comet-native-next.mjs +183 -152
  58. package/assets/skills/comet-native/scripts/comet-native-root.mjs +152 -147
  59. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +247 -202
  60. package/assets/skills/comet-native/scripts/comet-native-select.mjs +150 -143
  61. package/assets/skills/comet-native/scripts/comet-native-show.mjs +128 -121
  62. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +133 -151
  63. package/assets/skills/comet-native/scripts/comet-native-status.mjs +147 -140
  64. package/assets/skills/comet-open/SKILL.md +31 -11
  65. package/assets/skills/comet-open/agents/openai.yaml +5 -0
  66. package/assets/skills/comet-tweak/SKILL.md +11 -10
  67. package/assets/skills/comet-tweak/agents/openai.yaml +5 -0
  68. package/assets/skills/comet-verify/SKILL.md +9 -8
  69. package/assets/skills/comet-verify/agents/openai.yaml +5 -0
  70. package/assets/skills-zh/comet/SKILL.md +1 -1
  71. package/assets/skills-zh/comet/agents/openai.yaml +3 -0
  72. package/assets/skills-zh/comet-any/SKILL.md +2 -1
  73. package/assets/skills-zh/comet-any/agents/openai.yaml +5 -0
  74. package/assets/skills-zh/comet-archive/SKILL.md +7 -6
  75. package/assets/skills-zh/comet-archive/agents/openai.yaml +5 -0
  76. package/assets/skills-zh/comet-build/SKILL.md +28 -68
  77. package/assets/skills-zh/comet-build/agents/openai.yaml +5 -0
  78. package/assets/skills-zh/comet-classic/SKILL.md +29 -23
  79. package/assets/skills-zh/comet-classic/agents/openai.yaml +3 -0
  80. package/assets/skills-zh/{comet → comet-classic}/reference/auto-transition.md +1 -1
  81. package/assets/skills-zh/{comet → comet-classic}/reference/comet-yaml-fields.md +3 -3
  82. package/assets/skills-zh/{comet → comet-classic}/reference/context-recovery.md +5 -5
  83. package/assets/skills-zh/{comet → comet-classic}/reference/debug-gate.md +2 -2
  84. package/assets/skills-zh/{comet → comet-classic}/reference/decision-point.md +1 -1
  85. package/assets/skills-zh/{comet → comet-classic}/reference/dirty-worktree.md +1 -1
  86. package/assets/skills-zh/{comet → comet-classic}/reference/file-structure.md +1 -1
  87. package/assets/skills-zh/{comet → comet-classic}/reference/intent-frame.md +1 -1
  88. package/assets/skills-zh/{comet → comet-classic}/reference/scripts.md +4 -2
  89. package/assets/skills-zh/{comet → comet-classic}/reference/subagent-dispatch.md +1 -1
  90. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  91. package/assets/skills-zh/comet-design/SKILL.md +7 -6
  92. package/assets/skills-zh/comet-design/agents/openai.yaml +5 -0
  93. package/assets/skills-zh/comet-hotfix/SKILL.md +10 -9
  94. package/assets/skills-zh/comet-hotfix/agents/openai.yaml +5 -0
  95. package/assets/skills-zh/comet-native/SKILL.md +60 -171
  96. package/assets/skills-zh/comet-native/agents/openai.yaml +3 -0
  97. package/assets/skills-zh/comet-native/reference/artifacts.md +24 -89
  98. package/assets/skills-zh/comet-native/reference/clarification.md +40 -15
  99. package/assets/skills-zh/comet-native/reference/commands.md +30 -173
  100. package/assets/skills-zh/comet-native/reference/recovery.md +36 -86
  101. package/assets/skills-zh/comet-native/reference/workspace.md +23 -0
  102. package/assets/skills-zh/comet-open/SKILL.md +31 -11
  103. package/assets/skills-zh/comet-open/agents/openai.yaml +5 -0
  104. package/assets/skills-zh/comet-tweak/SKILL.md +11 -10
  105. package/assets/skills-zh/comet-tweak/agents/openai.yaml +5 -0
  106. package/assets/skills-zh/comet-verify/SKILL.md +9 -8
  107. package/assets/skills-zh/comet-verify/agents/openai.yaml +5 -0
  108. package/bin/fast-runtime-router.js +0 -4
  109. package/dist/app/cli/index.js +20 -1
  110. package/dist/app/cli/index.js.map +1 -1
  111. package/dist/app/commands/classic.d.ts.map +1 -1
  112. package/dist/app/commands/classic.js +2 -0
  113. package/dist/app/commands/classic.js.map +1 -1
  114. package/dist/app/commands/doctor.d.ts.map +1 -1
  115. package/dist/app/commands/doctor.js +8 -0
  116. package/dist/app/commands/doctor.js.map +1 -1
  117. package/dist/app/commands/eval.d.ts +9 -2
  118. package/dist/app/commands/eval.d.ts.map +1 -1
  119. package/dist/app/commands/eval.js +293 -48
  120. package/dist/app/commands/eval.js.map +1 -1
  121. package/dist/app/commands/init.d.ts.map +1 -1
  122. package/dist/app/commands/init.js +11 -2
  123. package/dist/app/commands/init.js.map +1 -1
  124. package/dist/app/commands/update.d.ts.map +1 -1
  125. package/dist/app/commands/update.js +13 -3
  126. package/dist/app/commands/update.js.map +1 -1
  127. package/dist/config/repository-layout.json +0 -8
  128. package/dist/domains/bundle/bundle-platform.d.ts.map +1 -1
  129. package/dist/domains/bundle/bundle-platform.js +8 -0
  130. package/dist/domains/bundle/bundle-platform.js.map +1 -1
  131. package/dist/domains/bundle/types.d.ts +1 -1
  132. package/dist/domains/bundle/types.d.ts.map +1 -1
  133. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  134. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  135. package/dist/domains/comet-classic/classic-cli.js +3 -0
  136. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  137. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  138. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  139. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  140. package/dist/domains/comet-classic/classic-guard.js +2 -2
  141. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  142. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  143. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  144. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  145. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  146. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  147. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  148. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  149. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  150. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  151. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  152. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  153. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  154. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  155. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  156. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  157. package/dist/domains/comet-entry/hook-adapter.js +3 -0
  158. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  159. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  160. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  161. package/dist/domains/comet-entry/hook-router-entry.js +32 -3
  162. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  163. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  164. package/dist/domains/comet-entry/init-workflow.js +13 -5
  165. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  166. package/dist/domains/comet-entry/project-activation.d.ts.map +1 -1
  167. package/dist/domains/comet-entry/project-activation.js +2 -1
  168. package/dist/domains/comet-entry/project-activation.js.map +1 -1
  169. package/dist/domains/comet-entry/resume-probe.d.ts.map +1 -1
  170. package/dist/domains/comet-entry/resume-probe.js +48 -4
  171. package/dist/domains/comet-entry/resume-probe.js.map +1 -1
  172. package/dist/domains/comet-native/index.d.ts +1 -0
  173. package/dist/domains/comet-native/index.d.ts.map +1 -1
  174. package/dist/domains/comet-native/index.js +1 -0
  175. package/dist/domains/comet-native/index.js.map +1 -1
  176. package/dist/domains/comet-native/native-acceptance.d.ts +4 -3
  177. package/dist/domains/comet-native/native-acceptance.d.ts.map +1 -1
  178. package/dist/domains/comet-native/native-acceptance.js +23 -18
  179. package/dist/domains/comet-native/native-acceptance.js.map +1 -1
  180. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  181. package/dist/domains/comet-native/native-archive-command.js +204 -61
  182. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  183. package/dist/domains/comet-native/native-archive-transaction.d.ts.map +1 -1
  184. package/dist/domains/comet-native/native-archive-transaction.js +28 -23
  185. package/dist/domains/comet-native/native-archive-transaction.js.map +1 -1
  186. package/dist/domains/comet-native/native-archive.d.ts +1 -0
  187. package/dist/domains/comet-native/native-archive.d.ts.map +1 -1
  188. package/dist/domains/comet-native/native-archive.js +56 -12
  189. package/dist/domains/comet-native/native-archive.js.map +1 -1
  190. package/dist/domains/comet-native/native-bounded-file.d.ts +32 -2
  191. package/dist/domains/comet-native/native-bounded-file.d.ts.map +1 -1
  192. package/dist/domains/comet-native/native-bounded-file.js +87 -6
  193. package/dist/domains/comet-native/native-bounded-file.js.map +1 -1
  194. package/dist/domains/comet-native/native-build-evidence.d.ts.map +1 -1
  195. package/dist/domains/comet-native/native-build-evidence.js +23 -1
  196. package/dist/domains/comet-native/native-build-evidence.js.map +1 -1
  197. package/dist/domains/comet-native/native-change.d.ts +1 -0
  198. package/dist/domains/comet-native/native-change.d.ts.map +1 -1
  199. package/dist/domains/comet-native/native-change.js +96 -8
  200. package/dist/domains/comet-native/native-change.js.map +1 -1
  201. package/dist/domains/comet-native/native-check-executor.d.ts +36 -0
  202. package/dist/domains/comet-native/native-check-executor.d.ts.map +1 -0
  203. package/dist/domains/comet-native/native-check-executor.js +132 -0
  204. package/dist/domains/comet-native/native-check-executor.js.map +1 -0
  205. package/dist/domains/comet-native/native-check-receipt-model.d.ts +9 -0
  206. package/dist/domains/comet-native/native-check-receipt-model.d.ts.map +1 -1
  207. package/dist/domains/comet-native/native-check-receipt-model.js +70 -8
  208. package/dist/domains/comet-native/native-check-receipt-model.js.map +1 -1
  209. package/dist/domains/comet-native/native-check-receipt-storage.d.ts.map +1 -1
  210. package/dist/domains/comet-native/native-check-receipt-storage.js +9 -8
  211. package/dist/domains/comet-native/native-check-receipt-storage.js.map +1 -1
  212. package/dist/domains/comet-native/native-check-receipt.d.ts.map +1 -1
  213. package/dist/domains/comet-native/native-check-receipt.js +123 -60
  214. package/dist/domains/comet-native/native-check-receipt.js.map +1 -1
  215. package/dist/domains/comet-native/native-checkpoint-command.d.ts.map +1 -1
  216. package/dist/domains/comet-native/native-checkpoint-command.js +0 -6
  217. package/dist/domains/comet-native/native-checkpoint-command.js.map +1 -1
  218. package/dist/domains/comet-native/native-checkpoint-storage.d.ts.map +1 -1
  219. package/dist/domains/comet-native/native-checkpoint-storage.js +27 -19
  220. package/dist/domains/comet-native/native-checkpoint-storage.js.map +1 -1
  221. package/dist/domains/comet-native/native-children.d.ts +49 -0
  222. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  223. package/dist/domains/comet-native/native-children.js +388 -0
  224. package/dist/domains/comet-native/native-children.js.map +1 -0
  225. package/dist/domains/comet-native/native-cli-help.d.ts +6 -0
  226. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -0
  227. package/dist/domains/comet-native/native-cli-help.js +192 -0
  228. package/dist/domains/comet-native/native-cli-help.js.map +1 -0
  229. package/dist/domains/comet-native/native-cli-shared.d.ts +2 -2
  230. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  231. package/dist/domains/comet-native/native-cli-shared.js +46 -24
  232. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  233. package/dist/domains/comet-native/native-cli.d.ts.map +1 -1
  234. package/dist/domains/comet-native/native-cli.js +18 -11
  235. package/dist/domains/comet-native/native-cli.js.map +1 -1
  236. package/dist/domains/comet-native/native-config.d.ts +2 -1
  237. package/dist/domains/comet-native/native-config.d.ts.map +1 -1
  238. package/dist/domains/comet-native/native-config.js +2 -1
  239. package/dist/domains/comet-native/native-config.js.map +1 -1
  240. package/dist/domains/comet-native/native-continuation.d.ts.map +1 -1
  241. package/dist/domains/comet-native/native-continuation.js +175 -147
  242. package/dist/domains/comet-native/native-continuation.js.map +1 -1
  243. package/dist/domains/comet-native/native-diagnostics.d.ts.map +1 -1
  244. package/dist/domains/comet-native/native-diagnostics.js +96 -13
  245. package/dist/domains/comet-native/native-diagnostics.js.map +1 -1
  246. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  247. package/dist/domains/comet-native/native-doctor-command.js +332 -0
  248. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  249. package/dist/domains/comet-native/native-doctor.d.ts +1 -0
  250. package/dist/domains/comet-native/native-doctor.d.ts.map +1 -1
  251. package/dist/domains/comet-native/native-doctor.js +116 -22
  252. package/dist/domains/comet-native/native-doctor.js.map +1 -1
  253. package/dist/domains/comet-native/native-evidence-projection.d.ts +50 -0
  254. package/dist/domains/comet-native/native-evidence-projection.d.ts.map +1 -0
  255. package/dist/domains/comet-native/native-evidence-projection.js +284 -0
  256. package/dist/domains/comet-native/native-evidence-projection.js.map +1 -0
  257. package/dist/domains/comet-native/native-evidence-retention.d.ts.map +1 -1
  258. package/dist/domains/comet-native/native-evidence-retention.js +14 -12
  259. package/dist/domains/comet-native/native-evidence-retention.js.map +1 -1
  260. package/dist/domains/comet-native/native-evidence-storage.d.ts.map +1 -1
  261. package/dist/domains/comet-native/native-evidence-storage.js +14 -11
  262. package/dist/domains/comet-native/native-evidence-storage.js.map +1 -1
  263. package/dist/domains/comet-native/native-findings.d.ts.map +1 -1
  264. package/dist/domains/comet-native/native-findings.js +18 -0
  265. package/dist/domains/comet-native/native-findings.js.map +1 -1
  266. package/dist/domains/comet-native/native-git-provenance.d.ts +16 -0
  267. package/dist/domains/comet-native/native-git-provenance.d.ts.map +1 -0
  268. package/dist/domains/comet-native/native-git-provenance.js +171 -0
  269. package/dist/domains/comet-native/native-git-provenance.js.map +1 -0
  270. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  271. package/dist/domains/comet-native/native-hook-guard.js +199 -19
  272. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  273. package/dist/domains/comet-native/native-init-command.d.ts.map +1 -1
  274. package/dist/domains/comet-native/native-init-command.js +14 -2
  275. package/dist/domains/comet-native/native-init-command.js.map +1 -1
  276. package/dist/domains/comet-native/native-local-execution.d.ts +25 -0
  277. package/dist/domains/comet-native/native-local-execution.d.ts.map +1 -0
  278. package/dist/domains/comet-native/native-local-execution.js +241 -0
  279. package/dist/domains/comet-native/native-local-execution.js.map +1 -0
  280. package/dist/domains/comet-native/native-lock.d.ts +1 -1
  281. package/dist/domains/comet-native/native-lock.d.ts.map +1 -1
  282. package/dist/domains/comet-native/native-lock.js +9 -9
  283. package/dist/domains/comet-native/native-lock.js.map +1 -1
  284. package/dist/domains/comet-native/native-loop-runtime.d.ts +63 -0
  285. package/dist/domains/comet-native/native-loop-runtime.d.ts.map +1 -0
  286. package/dist/domains/comet-native/native-loop-runtime.js +580 -0
  287. package/dist/domains/comet-native/native-loop-runtime.js.map +1 -0
  288. package/dist/domains/comet-native/native-mutation-lock.d.ts +2 -0
  289. package/dist/domains/comet-native/native-mutation-lock.d.ts.map +1 -1
  290. package/dist/domains/comet-native/native-mutation-lock.js +23 -2
  291. package/dist/domains/comet-native/native-mutation-lock.js.map +1 -1
  292. package/dist/domains/comet-native/native-new-command.d.ts.map +1 -1
  293. package/dist/domains/comet-native/native-new-command.js +27 -20
  294. package/dist/domains/comet-native/native-new-command.js.map +1 -1
  295. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  296. package/dist/domains/comet-native/native-next-command.js +193 -62
  297. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  298. package/dist/domains/comet-native/native-paths.d.ts +18 -0
  299. package/dist/domains/comet-native/native-paths.d.ts.map +1 -1
  300. package/dist/domains/comet-native/native-paths.js +125 -13
  301. package/dist/domains/comet-native/native-paths.js.map +1 -1
  302. package/dist/domains/comet-native/native-portable-acceptance.d.ts +28 -0
  303. package/dist/domains/comet-native/native-portable-acceptance.d.ts.map +1 -0
  304. package/dist/domains/comet-native/native-portable-acceptance.js +73 -0
  305. package/dist/domains/comet-native/native-portable-acceptance.js.map +1 -0
  306. package/dist/domains/comet-native/native-portable-archive.d.ts +37 -0
  307. package/dist/domains/comet-native/native-portable-archive.d.ts.map +1 -0
  308. package/dist/domains/comet-native/native-portable-archive.js +492 -0
  309. package/dist/domains/comet-native/native-portable-archive.js.map +1 -0
  310. package/dist/domains/comet-native/native-portable-continuation.d.ts +30 -0
  311. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -0
  312. package/dist/domains/comet-native/native-portable-continuation.js +403 -0
  313. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -0
  314. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts +9 -0
  315. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -0
  316. package/dist/domains/comet-native/native-portable-migration-runtime.js +213 -0
  317. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -0
  318. package/dist/domains/comet-native/native-portable-migration.d.ts +42 -0
  319. package/dist/domains/comet-native/native-portable-migration.d.ts.map +1 -0
  320. package/dist/domains/comet-native/native-portable-migration.js +212 -0
  321. package/dist/domains/comet-native/native-portable-migration.js.map +1 -0
  322. package/dist/domains/comet-native/native-portable-recovery.d.ts +15 -0
  323. package/dist/domains/comet-native/native-portable-recovery.d.ts.map +1 -0
  324. package/dist/domains/comet-native/native-portable-recovery.js +203 -0
  325. package/dist/domains/comet-native/native-portable-recovery.js.map +1 -0
  326. package/dist/domains/comet-native/native-portable-runtime.d.ts +155 -0
  327. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -0
  328. package/dist/domains/comet-native/native-portable-runtime.js +1550 -0
  329. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -0
  330. package/dist/domains/comet-native/native-portable-state.d.ts +30 -0
  331. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -0
  332. package/dist/domains/comet-native/native-portable-state.js +701 -0
  333. package/dist/domains/comet-native/native-portable-state.js.map +1 -0
  334. package/dist/domains/comet-native/native-portable-status.d.ts +65 -0
  335. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -0
  336. package/dist/domains/comet-native/native-portable-status.js +133 -0
  337. package/dist/domains/comet-native/native-portable-status.js.map +1 -0
  338. package/dist/domains/comet-native/native-portable-text.d.ts +8 -0
  339. package/dist/domains/comet-native/native-portable-text.d.ts.map +1 -0
  340. package/dist/domains/comet-native/native-portable-text.js +28 -0
  341. package/dist/domains/comet-native/native-portable-text.js.map +1 -0
  342. package/dist/domains/comet-native/native-portable-transactions.d.ts +41 -0
  343. package/dist/domains/comet-native/native-portable-transactions.d.ts.map +1 -0
  344. package/dist/domains/comet-native/native-portable-transactions.js +204 -0
  345. package/dist/domains/comet-native/native-portable-transactions.js.map +1 -0
  346. package/dist/domains/comet-native/native-portable-types.d.ts +166 -0
  347. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -0
  348. package/dist/domains/comet-native/native-portable-types.js +18 -0
  349. package/dist/domains/comet-native/native-portable-types.js.map +1 -0
  350. package/dist/domains/comet-native/native-repair-integration.js +5 -5
  351. package/dist/domains/comet-native/native-repair-integration.js.map +1 -1
  352. package/dist/domains/comet-native/native-resume-view.d.ts.map +1 -1
  353. package/dist/domains/comet-native/native-resume-view.js +1 -5
  354. package/dist/domains/comet-native/native-resume-view.js.map +1 -1
  355. package/dist/domains/comet-native/native-root-move.d.ts.map +1 -1
  356. package/dist/domains/comet-native/native-root-move.js +4 -20
  357. package/dist/domains/comet-native/native-root-move.js.map +1 -1
  358. package/dist/domains/comet-native/native-run-consistency.d.ts.map +1 -1
  359. package/dist/domains/comet-native/native-run-consistency.js +8 -12
  360. package/dist/domains/comet-native/native-run-consistency.js.map +1 -1
  361. package/dist/domains/comet-native/native-run-store.js +4 -4
  362. package/dist/domains/comet-native/native-run-store.js.map +1 -1
  363. package/dist/domains/comet-native/native-runner-input.d.ts +160 -0
  364. package/dist/domains/comet-native/native-runner-input.d.ts.map +1 -0
  365. package/dist/domains/comet-native/native-runner-input.js +347 -0
  366. package/dist/domains/comet-native/native-runner-input.js.map +1 -0
  367. package/dist/domains/comet-native/native-runner-protocol.d.ts +40 -0
  368. package/dist/domains/comet-native/native-runner-protocol.d.ts.map +1 -0
  369. package/dist/domains/comet-native/native-runner-protocol.js +55 -0
  370. package/dist/domains/comet-native/native-runner-protocol.js.map +1 -0
  371. package/dist/domains/comet-native/native-runtime-package.d.ts +3 -0
  372. package/dist/domains/comet-native/native-runtime-package.d.ts.map +1 -1
  373. package/dist/domains/comet-native/native-runtime-package.js +5 -1
  374. package/dist/domains/comet-native/native-runtime-package.js.map +1 -1
  375. package/dist/domains/comet-native/native-schema-migration.d.ts.map +1 -1
  376. package/dist/domains/comet-native/native-schema-migration.js +30 -27
  377. package/dist/domains/comet-native/native-schema-migration.js.map +1 -1
  378. package/dist/domains/comet-native/native-select-command.d.ts.map +1 -1
  379. package/dist/domains/comet-native/native-select-command.js +8 -4
  380. package/dist/domains/comet-native/native-select-command.js.map +1 -1
  381. package/dist/domains/comet-native/native-selection.d.ts.map +1 -1
  382. package/dist/domains/comet-native/native-selection.js +11 -2
  383. package/dist/domains/comet-native/native-selection.js.map +1 -1
  384. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  385. package/dist/domains/comet-native/native-show-command.js +37 -0
  386. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  387. package/dist/domains/comet-native/native-snapshot.d.ts.map +1 -1
  388. package/dist/domains/comet-native/native-snapshot.js +13 -10
  389. package/dist/domains/comet-native/native-snapshot.js.map +1 -1
  390. package/dist/domains/comet-native/native-spec-command.d.ts.map +1 -1
  391. package/dist/domains/comet-native/native-spec-command.js +14 -28
  392. package/dist/domains/comet-native/native-spec-command.js.map +1 -1
  393. package/dist/domains/comet-native/native-specs.js +3 -3
  394. package/dist/domains/comet-native/native-specs.js.map +1 -1
  395. package/dist/domains/comet-native/native-status-command.d.ts.map +1 -1
  396. package/dist/domains/comet-native/native-status-command.js +7 -17
  397. package/dist/domains/comet-native/native-status-command.js.map +1 -1
  398. package/dist/domains/comet-native/native-status-discovery.d.ts +51 -0
  399. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -0
  400. package/dist/domains/comet-native/native-status-discovery.js +299 -0
  401. package/dist/domains/comet-native/native-status-discovery.js.map +1 -0
  402. package/dist/domains/comet-native/native-trajectory-recovery.d.ts.map +1 -1
  403. package/dist/domains/comet-native/native-trajectory-recovery.js +6 -9
  404. package/dist/domains/comet-native/native-trajectory-recovery.js.map +1 -1
  405. package/dist/domains/comet-native/native-transaction.d.ts.map +1 -1
  406. package/dist/domains/comet-native/native-transaction.js +31 -20
  407. package/dist/domains/comet-native/native-transaction.js.map +1 -1
  408. package/dist/domains/comet-native/native-transition-evidence.d.ts.map +1 -1
  409. package/dist/domains/comet-native/native-transition-evidence.js +1 -0
  410. package/dist/domains/comet-native/native-transition-evidence.js.map +1 -1
  411. package/dist/domains/comet-native/native-transition-journal.d.ts.map +1 -1
  412. package/dist/domains/comet-native/native-transition-journal.js +71 -22
  413. package/dist/domains/comet-native/native-transition-journal.js.map +1 -1
  414. package/dist/domains/comet-native/native-transitions.d.ts.map +1 -1
  415. package/dist/domains/comet-native/native-transitions.js +167 -15
  416. package/dist/domains/comet-native/native-transitions.js.map +1 -1
  417. package/dist/domains/comet-native/native-types.d.ts +36 -1
  418. package/dist/domains/comet-native/native-types.d.ts.map +1 -1
  419. package/dist/domains/comet-native/native-verification-receipt-runtime.d.ts.map +1 -1
  420. package/dist/domains/comet-native/native-verification-receipt-runtime.js +18 -100
  421. package/dist/domains/comet-native/native-verification-receipt-runtime.js.map +1 -1
  422. package/dist/domains/comet-native/native-verification-report-v2.d.ts +12 -0
  423. package/dist/domains/comet-native/native-verification-report-v2.d.ts.map +1 -0
  424. package/dist/domains/comet-native/native-verification-report-v2.js +141 -0
  425. package/dist/domains/comet-native/native-verification-report-v2.js.map +1 -0
  426. package/dist/domains/comet-native/native-verification-runtime.d.ts.map +1 -1
  427. package/dist/domains/comet-native/native-verification-runtime.js +7 -2
  428. package/dist/domains/comet-native/native-verification-runtime.js.map +1 -1
  429. package/dist/domains/comet-native/native-verification-scope.d.ts +23 -0
  430. package/dist/domains/comet-native/native-verification-scope.d.ts.map +1 -1
  431. package/dist/domains/comet-native/native-verification-scope.js +106 -19
  432. package/dist/domains/comet-native/native-verification-scope.js.map +1 -1
  433. package/dist/domains/comet-native/native-verifier-protocol.d.ts +49 -0
  434. package/dist/domains/comet-native/native-verifier-protocol.d.ts.map +1 -0
  435. package/dist/domains/comet-native/native-verifier-protocol.js +157 -0
  436. package/dist/domains/comet-native/native-verifier-protocol.js.map +1 -0
  437. package/dist/domains/comet-native/native-workspace-finish.d.ts +61 -0
  438. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -0
  439. package/dist/domains/comet-native/native-workspace-finish.js +332 -0
  440. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -0
  441. package/dist/domains/comet-native/native-workspace-preparation.d.ts +32 -0
  442. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -0
  443. package/dist/domains/comet-native/native-workspace-preparation.js +324 -0
  444. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -0
  445. package/dist/domains/comet-native/native-workspace.d.ts +9 -1
  446. package/dist/domains/comet-native/native-workspace.d.ts.map +1 -1
  447. package/dist/domains/comet-native/native-workspace.js +109 -11
  448. package/dist/domains/comet-native/native-workspace.js.map +1 -1
  449. package/dist/domains/dashboard/collector.d.ts +1 -1
  450. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  451. package/dist/domains/dashboard/collector.js +102 -87
  452. package/dist/domains/dashboard/collector.js.map +1 -1
  453. package/dist/domains/dashboard/native-adapter.d.ts +155 -126
  454. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  455. package/dist/domains/dashboard/native-adapter.js +165 -268
  456. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  457. package/dist/domains/dashboard/native-collector.d.ts +5 -4
  458. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  459. package/dist/domains/dashboard/native-collector.js +506 -439
  460. package/dist/domains/dashboard/native-collector.js.map +1 -1
  461. package/dist/domains/dashboard/native-legacy-adapter.d.ts +37 -0
  462. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -0
  463. package/dist/domains/dashboard/native-legacy-adapter.js +127 -0
  464. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -0
  465. package/dist/domains/dashboard/server.d.ts.map +1 -1
  466. package/dist/domains/dashboard/server.js +5 -3
  467. package/dist/domains/dashboard/server.js.map +1 -1
  468. package/dist/domains/dashboard/types.d.ts +5 -0
  469. package/dist/domains/dashboard/types.d.ts.map +1 -1
  470. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  471. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  472. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  473. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  474. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-Ca5zba3N.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  475. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  476. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-Bj6HWRwi.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  477. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-B1kXfg5N.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  478. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  479. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-BAJLB5V6.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  480. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  481. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-CwghIMO9.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  482. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-JYluRsP-.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  483. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-CCVPBm0J.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  484. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  485. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-DfYm7pN7.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  486. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  487. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  488. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-BBrkuiAU.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  489. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-DlFhgQHt.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  490. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  491. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  492. package/dist/domains/dashboard/web/assets/demo-BfLsKrIL.js +45 -0
  493. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  494. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-BQahyP-C.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  495. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  496. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  497. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  498. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  499. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  500. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  501. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  502. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-DCxvMAUJ.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  503. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  504. package/dist/domains/dashboard/web/assets/{index-Blit0P5Z.js → index-CCApJ0Yr.js} +1 -1
  505. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  506. package/dist/domains/dashboard/web/assets/index-DhhbseFJ.css +1 -0
  507. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-9qXEkzDe.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  508. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-Dkf5othW.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  509. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-C2_IkO4s.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  510. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-BxWgu4DL.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  511. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  512. package/dist/domains/dashboard/web/assets/{linear-Dkr6p-Ro.js → linear-DXOo6ZRT.js} +1 -1
  513. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  514. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  515. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-C9xyxGfa.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  516. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  517. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  518. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  519. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-Cy6iOcxc.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  520. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  521. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-B_gZONhx.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  522. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  523. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  524. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  525. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  526. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  527. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  528. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  529. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-DvONX2YQ.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  530. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  531. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-cf39WX2o.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  532. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  533. package/dist/domains/dashboard/web/index.html +16 -16
  534. package/dist/domains/dashboard/workspace.d.ts +24 -0
  535. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  536. package/dist/domains/dashboard/workspace.js +87 -0
  537. package/dist/domains/dashboard/workspace.js.map +1 -0
  538. package/dist/domains/eval/index.d.ts +3 -0
  539. package/dist/domains/eval/index.d.ts.map +1 -1
  540. package/dist/domains/eval/index.js +3 -0
  541. package/dist/domains/eval/index.js.map +1 -1
  542. package/dist/domains/eval/standalone-context.d.ts +19 -0
  543. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  544. package/dist/domains/eval/standalone-context.js +107 -0
  545. package/dist/domains/eval/standalone-context.js.map +1 -0
  546. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  547. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  548. package/dist/domains/eval/standalone-static-collect.js +779 -0
  549. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  550. package/dist/domains/eval/user-environment.d.ts +6 -0
  551. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  552. package/dist/domains/eval/user-environment.js +73 -0
  553. package/dist/domains/eval/user-environment.js.map +1 -0
  554. package/dist/domains/integrations/openspec.d.ts +1 -1
  555. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  556. package/dist/domains/integrations/openspec.js +110 -26
  557. package/dist/domains/integrations/openspec.js.map +1 -1
  558. package/dist/domains/integrations/superpowers.d.ts +5 -1
  559. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  560. package/dist/domains/integrations/superpowers.js +21 -2
  561. package/dist/domains/integrations/superpowers.js.map +1 -1
  562. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  563. package/dist/domains/skill/platform-inspect.js +11 -3
  564. package/dist/domains/skill/platform-inspect.js.map +1 -1
  565. package/dist/domains/skill/platform-install.d.ts +8 -2
  566. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  567. package/dist/domains/skill/platform-install.js +115 -34
  568. package/dist/domains/skill/platform-install.js.map +1 -1
  569. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  570. package/dist/domains/skill/project-instructions.js +5 -3
  571. package/dist/domains/skill/project-instructions.js.map +1 -1
  572. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  573. package/dist/domains/skill/uninstall.js +69 -15
  574. package/dist/domains/skill/uninstall.js.map +1 -1
  575. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  576. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  577. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  578. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  579. package/dist/domains/workflow-contract/index.d.ts +3 -0
  580. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  581. package/dist/domains/workflow-contract/index.js +2 -0
  582. package/dist/domains/workflow-contract/index.js.map +1 -1
  583. package/dist/domains/workflow-contract/project-config.d.ts +3 -1
  584. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  585. package/dist/domains/workflow-contract/project-config.js +119 -19
  586. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  587. package/dist/domains/workflow-contract/project-gitignore.d.ts +9 -0
  588. package/dist/domains/workflow-contract/project-gitignore.d.ts.map +1 -0
  589. package/dist/domains/workflow-contract/project-gitignore.js +108 -0
  590. package/dist/domains/workflow-contract/project-gitignore.js.map +1 -0
  591. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  592. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  593. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  594. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  595. package/dist/domains/workflow-contract/types.d.ts +4 -0
  596. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  597. package/dist/platform/install/platforms.d.ts +3 -1
  598. package/dist/platform/install/platforms.d.ts.map +1 -1
  599. package/dist/platform/install/platforms.js +22 -2
  600. package/dist/platform/install/platforms.js.map +1 -1
  601. package/dist/platform/paths/git-worktree.d.ts +8 -1
  602. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  603. package/dist/platform/paths/git-worktree.js +41 -7
  604. package/dist/platform/paths/git-worktree.js.map +1 -1
  605. package/dist/platform/process/external-command.d.ts +13 -0
  606. package/dist/platform/process/external-command.d.ts.map +1 -0
  607. package/dist/platform/process/external-command.js +45 -0
  608. package/dist/platform/process/external-command.js.map +1 -0
  609. package/dist/platform/process/git.d.ts +12 -0
  610. package/dist/platform/process/git.d.ts.map +1 -0
  611. package/dist/platform/process/git.js +86 -0
  612. package/dist/platform/process/git.js.map +1 -0
  613. package/dist/platform/process/spawn-command.d.ts +8 -0
  614. package/dist/platform/process/spawn-command.d.ts.map +1 -0
  615. package/dist/platform/process/spawn-command.js +113 -0
  616. package/dist/platform/process/spawn-command.js.map +1 -0
  617. package/eval/.env.example +115 -0
  618. package/eval/README.md +237 -14
  619. package/eval/langfuse/.env.example +9 -0
  620. package/eval/langfuse/tests/conftest.py +146 -0
  621. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  622. package/eval/langfuse/tests/test_adapter.py +272 -0
  623. package/eval/langsmith/.env.example +16 -0
  624. package/eval/langsmith/tests/conftest.py +10 -1
  625. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  626. package/eval/local/tasks/comet-native-clarification/validation/test_native_clarification.py +9 -21
  627. package/eval/local/tasks/comet-native-clarification-depth/validation/test_native_clarification_depth.py +4 -7
  628. package/eval/local/tasks/comet-native-clarification-modes/validation/test_native_clarification_modes.py +4 -10
  629. package/eval/local/tasks/comet-native-interrupted-transition/instruction.md +1 -1
  630. package/eval/local/tasks/comet-native-interrupted-transition/validation/test_native_interrupted_transition.py +15 -16
  631. package/eval/local/tasks/comet-native-repository-fact/instruction.md +2 -2
  632. package/eval/local/tasks/comet-native-repository-fact/validation/test_native_repository_fact.py +8 -2
  633. package/eval/local/tasks/comet-native-wave-b-decision-resume/instruction.md +8 -4
  634. package/eval/local/tasks/comet-native-wave-b-decision-resume/task.toml +6 -6
  635. package/eval/local/tasks/comet-native-wave-b-decision-resume/validation/test_native_wave_b_decision_resume.py +126 -135
  636. package/eval/local/tasks/comet-native-wave-c-verification-integrity/instruction.md +6 -6
  637. package/eval/local/tasks/comet-native-wave-c-verification-integrity/task.toml +8 -8
  638. package/eval/local/tasks/comet-native-wave-c-verification-integrity/validation/test_native_wave_c_verification_integrity.py +97 -317
  639. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/instruction.md +7 -7
  640. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/task.toml +4 -4
  641. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/validation/test_native_wave_d_stagnation_stop.py +130 -199
  642. package/eval/local/tasks/comet-native-wave-e-parallel-safety/instruction.md +5 -5
  643. package/eval/local/tasks/comet-native-wave-e-parallel-safety/task.toml +8 -8
  644. package/eval/local/tasks/comet-native-wave-e-parallel-safety/validation/test_native_wave_e_parallel_safety.py +135 -239
  645. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  646. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/instruction.md +3 -3
  647. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/task.toml +6 -6
  648. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/validation/test_native_wave_f_dashboard_readonly.py +81 -439
  649. package/eval/local/tasks/comet-native-workflow/instruction.md +5 -4
  650. package/eval/local/tasks/comet-native-workflow/validation/test_native_workflow.py +145 -502
  651. package/eval/local/tasks/index.yaml +5 -5
  652. package/eval/local/tests/conftest.py +701 -80
  653. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  654. package/eval/local/tests/scaffold/test_agents.py +76 -0
  655. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  656. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  657. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  658. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  659. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  660. package/eval/local/tests/scaffold/test_execution.py +364 -0
  661. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  662. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  663. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  664. package/eval/local/tests/scaffold/test_logging.py +162 -0
  665. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  666. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  667. package/eval/local/tests/scaffold/test_native_aligned_eval.py +163 -35
  668. package/eval/local/tests/scaffold/test_native_wave_evaluations.py +75 -2054
  669. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  670. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  671. package/eval/local/tests/scaffold/test_tasks.py +38 -52
  672. package/eval/local/tests/scaffold/test_utils.py +480 -4
  673. package/eval/local/tests/tasks/test_tasks.py +219 -17
  674. package/eval/local/treatments/comet/comet_native_phase1.yaml +7 -4
  675. package/eval/pyproject.toml +9 -4
  676. package/eval/scaffold/__init__.py +6 -0
  677. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  678. package/eval/scaffold/python/__init__.py +128 -20
  679. package/eval/scaffold/python/agents.py +239 -0
  680. package/eval/scaffold/python/aligned_comparison.py +156 -27
  681. package/eval/scaffold/python/auto_tasks.py +700 -0
  682. package/eval/scaffold/python/custom_agents.py +172 -0
  683. package/eval/scaffold/python/eval_context.py +236 -0
  684. package/eval/scaffold/python/eval_schema.py +135 -0
  685. package/eval/scaffold/python/execution.py +544 -0
  686. package/eval/scaffold/python/generated_task_cache.py +230 -0
  687. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  688. package/eval/scaffold/python/inline_tasks.py +118 -0
  689. package/eval/scaffold/python/judge_config.py +158 -37
  690. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  691. package/eval/scaffold/python/llm_judge.py +68 -11
  692. package/eval/scaffold/python/logging.py +157 -17
  693. package/eval/scaffold/python/manifest_tasks.py +69 -0
  694. package/eval/scaffold/python/manifests.py +443 -22
  695. package/eval/scaffold/python/native_eval.py +10 -5
  696. package/eval/scaffold/python/paths.py +11 -1
  697. package/eval/scaffold/python/task_resolution.py +176 -0
  698. package/eval/scaffold/python/tasks.py +17 -4
  699. package/eval/scaffold/python/utils.py +167 -11
  700. package/eval/scaffold/python/validation/core.py +48 -2
  701. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  702. package/eval/scaffold/python/validation/native_v4.py +249 -0
  703. package/eval/scaffold/python/validation/native_wave.py +4 -5
  704. package/eval/scaffold/python/validation/native_workflow.py +199 -98
  705. package/eval/scaffold/python/validation/rubric.py +130 -2
  706. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  707. package/eval/scaffold/shell/docker.sh +534 -33
  708. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  709. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  710. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  711. package/package.json +9 -3
  712. package/assets/skills/comet-native/scripts/comet-native-check.mjs +0 -179
  713. package/assets/skills/comet-native/scripts/comet-native-checkpoint.mjs +0 -179
  714. package/assets/skills/comet-native/scripts/comet-native-evidence.mjs +0 -151
  715. package/assets/skills/comet-native/scripts/comet-native-receipt.mjs +0 -187
  716. package/dist/domains/comet-native/native-check-entry.d.ts +0 -2
  717. package/dist/domains/comet-native/native-check-entry.d.ts.map +0 -1
  718. package/dist/domains/comet-native/native-check-entry.js +0 -4
  719. package/dist/domains/comet-native/native-check-entry.js.map +0 -1
  720. package/dist/domains/comet-native/native-checkpoint-entry.d.ts +0 -2
  721. package/dist/domains/comet-native/native-checkpoint-entry.d.ts.map +0 -1
  722. package/dist/domains/comet-native/native-checkpoint-entry.js +0 -4
  723. package/dist/domains/comet-native/native-checkpoint-entry.js.map +0 -1
  724. package/dist/domains/comet-native/native-evidence-entry.d.ts +0 -2
  725. package/dist/domains/comet-native/native-evidence-entry.d.ts.map +0 -1
  726. package/dist/domains/comet-native/native-evidence-entry.js +0 -4
  727. package/dist/domains/comet-native/native-evidence-entry.js.map +0 -1
  728. package/dist/domains/comet-native/native-receipt-entry.d.ts +0 -2
  729. package/dist/domains/comet-native/native-receipt-entry.d.ts.map +0 -1
  730. package/dist/domains/comet-native/native-receipt-entry.js +0 -4
  731. package/dist/domains/comet-native/native-receipt-entry.js.map +0 -1
  732. package/dist/domains/dashboard/web/assets/arc-ClrSujVb.js +0 -1
  733. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR-B4A_t84A.js +0 -36
  734. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-D72u2953.js +0 -132
  735. package/dist/domains/dashboard/web/assets/channel-B_u-_i2t.js +0 -1
  736. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BA_zgMXZ.js +0 -231
  737. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-BpA7BPgX.js +0 -1
  738. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-BpA7BPgX.js +0 -1
  739. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-B0Bk5oiP.js +0 -4
  740. package/dist/domains/dashboard/web/assets/demo-DZYPLZpF.js +0 -25
  741. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-tvTUiBA2.js +0 -43
  742. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-BZfcK29q.js +0 -10
  743. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-D5bJnW2C.js +0 -24
  744. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-DTwWdzWJ.js +0 -24
  745. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-BbvrdCp9.js +0 -85
  746. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-DgbR7Tcc.js +0 -162
  747. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-wWMOA7F4.js +0 -292
  748. package/dist/domains/dashboard/web/assets/index-Bk1o-4VO.css +0 -1
  749. package/dist/domains/dashboard/web/assets/index-g8vyW0R_.js +0 -257
  750. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  751. package/dist/domains/dashboard/web/assets/mermaid.core-DZUcsoG4.js +0 -301
  752. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-Dn67tdOF.js +0 -30
  753. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  754. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-C3u-4dOk.js +0 -40
  755. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-Xd3ByTVF.js +0 -162
  756. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-CJGHL7Up.js +0 -1
  757. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-rKvBgr6I.js +0 -1
  758. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-B-L4qWy4.js +0 -34
  759. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-Dk1vcm3u.js +0 -7
  760. /package/assets/skills/{comet → comet-classic}/reference/classic-layout.md +0 -0
  761. /package/assets/skills-zh/{comet → comet-classic}/reference/classic-layout.md +0 -0
@@ -0,0 +1,1023 @@
1
+ """Langfuse core reporting, official plugin provisioning, and trajectory adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import re
9
+ import shutil
10
+ import tarfile
11
+ import tempfile
12
+ from contextlib import contextmanager
13
+ from dataclasses import dataclass
14
+ from pathlib import Path, PurePosixPath
15
+ from types import SimpleNamespace
16
+ from typing import Any, Callable, Iterator, Mapping
17
+ from urllib.request import Request, urlopen
18
+
19
+ from scaffold.python.pass_at_k import compute_pass_metrics
20
+
21
+
22
+ class LangfuseConfigurationError(RuntimeError):
23
+ """The Langfuse suite cannot start safely with the current configuration."""
24
+
25
+
26
+ class LangfuseReportingError(RuntimeError):
27
+ """A required Langfuse core trace or score could not be written."""
28
+
29
+
30
+ class TrajectoryProvisionError(RuntimeError):
31
+ """An official trajectory plugin could not be safely provisioned."""
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class TrajectoryEvent:
36
+ """A bounded, agent-neutral event parsed from a JSONL transcript."""
37
+
38
+ kind: str
39
+ name: str
40
+ input: Any = None
41
+ output: Any = None
42
+ session_id: str | None = None
43
+ truncated: bool = False
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class TrajectoryProvision:
48
+ """The isolated plugin or hook provision selected for one eval run."""
49
+
50
+ agent: str
51
+ mode: str
52
+ cache_dir: Path
53
+ plugin_path: Path | None = None
54
+ source_repo: str | None = None
55
+ source_ref: str | None = None
56
+ archive_sha256: str | None = None
57
+ content_sha256: str | None = None
58
+
59
+
60
+ _OFFICIAL_PLUGIN_SPECS: dict[str, dict[str, str]] = {
61
+ "claude-code": {
62
+ "repo": "langfuse/Claude-Observability-Plugin",
63
+ "ref": "05b7742829e0b0ad840c8760376ca737fa1eb662",
64
+ "archive_root": ".",
65
+ },
66
+ "codex": {
67
+ "repo": "langfuse/codex-observability-plugin",
68
+ "ref": "96e997cf9c3d4aee7421e849d764b83dcb485462",
69
+ "archive_root": "plugins/tracing",
70
+ },
71
+ }
72
+
73
+ _RUBRIC_SCORE_RE = re.compile(r"\[RUBRIC\]\s+([A-Za-z0-9_.-]+):\s*(0(?:\.\d+)?|1(?:\.0+)?)\b")
74
+ _SUMMARY_K_VALUES = (1, 2, 5)
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class LangfuseConfig:
79
+ public_key: str
80
+ secret_key: str
81
+ base_url: str | None = None
82
+ tracing_environment: str | None = None
83
+
84
+ @classmethod
85
+ def from_environment(cls, environ: Mapping[str, str] | None = None) -> "LangfuseConfig":
86
+ env = environ or os.environ
87
+ missing = [
88
+ name
89
+ for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY")
90
+ if not env.get(name, "").strip()
91
+ ]
92
+ if missing:
93
+ raise LangfuseConfigurationError(
94
+ "Langfuse suite requires "
95
+ + ", ".join(missing)
96
+ + ". Set them in the process environment; they are never persisted by Comet."
97
+ )
98
+ return cls(
99
+ public_key=env["LANGFUSE_PUBLIC_KEY"].strip(),
100
+ secret_key=env["LANGFUSE_SECRET_KEY"].strip(),
101
+ base_url=env.get("LANGFUSE_BASE_URL") or None,
102
+ tracing_environment=env.get("LANGFUSE_TRACING_ENVIRONMENT") or None,
103
+ )
104
+
105
+
106
+ def trajectory_mode(agent: str) -> str:
107
+ return {
108
+ "claude-code": "official-claude-code-plugin",
109
+ "codex": "official-codex-plugin",
110
+ "qoder": "qoder-stop-transcript",
111
+ "codebuddy": "codebuddy-stop-transcript",
112
+ }.get(agent, "best-effort-core-only")
113
+
114
+
115
+ def _sha256_file(path: Path) -> str:
116
+ digest = hashlib.sha256()
117
+ with path.open("rb") as stream:
118
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
119
+ digest.update(chunk)
120
+ return digest.hexdigest()
121
+
122
+
123
+ def _sha256_tree(root: Path) -> str:
124
+ digest = hashlib.sha256()
125
+ for path in sorted(item for item in root.rglob("*") if item.is_file()):
126
+ relative = path.relative_to(root).as_posix()
127
+ digest.update(relative.encode("utf-8"))
128
+ digest.update(b"\0")
129
+ digest.update(path.read_bytes())
130
+ digest.update(b"\0")
131
+ return digest.hexdigest()
132
+
133
+
134
+ def _download_plugin_archive(url: str, destination: Path) -> None:
135
+ request = Request(url, headers={"User-Agent": "Comet eval Langfuse adapter"})
136
+ try:
137
+ with urlopen(request, timeout=90) as response, destination.open("wb") as output:
138
+ shutil.copyfileobj(response, output)
139
+ except Exception as exc:
140
+ raise TrajectoryProvisionError(
141
+ f"Langfuse trajectory plugin download failed: {exc}"
142
+ ) from exc
143
+
144
+
145
+ @contextmanager
146
+ def _provision_lock(path: Path) -> Iterator[None]:
147
+ """Coordinate cache writes across pytest workers and separate processes."""
148
+ path.parent.mkdir(parents=True, exist_ok=True)
149
+ with path.open("a+b") as lock_file:
150
+ if os.name == "nt":
151
+ import msvcrt
152
+
153
+ lock_file.seek(0, os.SEEK_END)
154
+ if lock_file.tell() == 0:
155
+ lock_file.write(b"0")
156
+ lock_file.flush()
157
+ lock_file.seek(0)
158
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
159
+ try:
160
+ yield
161
+ finally:
162
+ lock_file.seek(0)
163
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
164
+ else:
165
+ import fcntl
166
+
167
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
168
+ try:
169
+ yield
170
+ finally:
171
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
172
+
173
+
174
+ def _safe_extract_archive(archive_path: Path, destination: Path) -> Path:
175
+ destination.mkdir(parents=True, exist_ok=True)
176
+ root_name: str | None = None
177
+ try:
178
+ with tarfile.open(archive_path, mode="r:gz") as archive:
179
+ members = archive.getmembers()
180
+ for member in members:
181
+ relative = PurePosixPath(member.name)
182
+ if relative.is_absolute() or ".." in relative.parts:
183
+ raise TrajectoryProvisionError(
184
+ f"Langfuse trajectory plugin archive contains an unsafe path: {member.name}"
185
+ )
186
+ if member.issym() or member.islnk():
187
+ raise TrajectoryProvisionError(
188
+ f"Langfuse trajectory plugin archive contains a link: {member.name}"
189
+ )
190
+ if not root_name and relative.parts:
191
+ root_name = relative.parts[0]
192
+ target = destination.joinpath(*relative.parts).resolve()
193
+ if destination.resolve() not in target.parents and target != destination.resolve():
194
+ raise TrajectoryProvisionError(
195
+ f"Langfuse trajectory plugin archive escapes its cache: {member.name}"
196
+ )
197
+ archive.extractall(destination, filter="data")
198
+ except TrajectoryProvisionError:
199
+ raise
200
+ except (OSError, tarfile.TarError) as exc:
201
+ raise TrajectoryProvisionError(
202
+ f"Langfuse trajectory plugin archive is invalid: {exc}"
203
+ ) from exc
204
+ if not root_name:
205
+ raise TrajectoryProvisionError("Langfuse trajectory plugin archive is empty")
206
+ return destination / root_name
207
+
208
+
209
+ def _read_provisioned_plugin(
210
+ cache_dir: Path, agent: str, spec: Mapping[str, str]
211
+ ) -> TrajectoryProvision:
212
+ metadata_path = cache_dir / "metadata.json"
213
+ plugin_path = cache_dir / "plugin"
214
+ try:
215
+ metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
216
+ actual_content_hash = _sha256_tree(plugin_path)
217
+ except (OSError, ValueError, TypeError) as exc:
218
+ raise TrajectoryProvisionError(
219
+ f"Langfuse trajectory plugin cache is incomplete for {agent}: {cache_dir}"
220
+ ) from exc
221
+ if (
222
+ metadata.get("source_ref") != spec["ref"]
223
+ or metadata.get("content_sha256") != actual_content_hash
224
+ ):
225
+ raise TrajectoryProvisionError(
226
+ f"Langfuse trajectory plugin cache integrity check failed for {agent}: {cache_dir}"
227
+ )
228
+ return TrajectoryProvision(
229
+ agent=agent,
230
+ mode=trajectory_mode(agent),
231
+ cache_dir=cache_dir,
232
+ plugin_path=plugin_path,
233
+ source_repo=metadata.get("source_repo"),
234
+ source_ref=metadata.get("source_ref"),
235
+ archive_sha256=metadata.get("archive_sha256"),
236
+ content_sha256=actual_content_hash,
237
+ )
238
+
239
+
240
+ def provision_trajectory_plugin(
241
+ cache_root: Path,
242
+ agent: str,
243
+ *,
244
+ download: Callable[[str, Path], None] | None = None,
245
+ ) -> TrajectoryProvision:
246
+ """Provision a pinned official plugin without touching user agent config."""
247
+ mode = trajectory_mode(agent)
248
+ if agent not in _OFFICIAL_PLUGIN_SPECS:
249
+ return TrajectoryProvision(agent=agent, mode=mode, cache_dir=cache_root / agent)
250
+
251
+ spec = _OFFICIAL_PLUGIN_SPECS[agent]
252
+ cache_dir = cache_root / agent / spec["ref"]
253
+ cache_dir.parent.mkdir(parents=True, exist_ok=True)
254
+ with _provision_lock(cache_dir.parent / ".provision.lock"):
255
+ if (cache_dir / "metadata.json").is_file() and (cache_dir / "plugin").is_dir():
256
+ return _read_provisioned_plugin(cache_dir, agent, spec)
257
+
258
+ staging = Path(tempfile.mkdtemp(prefix=f".{agent}-", dir=cache_dir.parent))
259
+ try:
260
+ archive_path = staging / "plugin.tar.gz"
261
+ source_url = f"https://codeload.github.com/{spec['repo']}/tar.gz/{spec['ref']}"
262
+ (download or _download_plugin_archive)(source_url, archive_path)
263
+ archive_hash = _sha256_file(archive_path)
264
+ extracted_root = _safe_extract_archive(archive_path, staging / "extracted")
265
+ required_plugin_path = (
266
+ extracted_root
267
+ if spec["archive_root"] == "."
268
+ else extracted_root / spec["archive_root"]
269
+ )
270
+ if not required_plugin_path.is_dir():
271
+ raise TrajectoryProvisionError(
272
+ f"Langfuse trajectory plugin path is missing: {spec['archive_root']}"
273
+ )
274
+ plugin_destination = staging / "plugin"
275
+ shutil.copytree(extracted_root, plugin_destination)
276
+ content_hash = _sha256_tree(plugin_destination)
277
+ metadata = {
278
+ "schema": "comet.eval.langfuse-plugin.v1",
279
+ "agent": agent,
280
+ "source_repo": spec["repo"],
281
+ "source_ref": spec["ref"],
282
+ "archive_sha256": archive_hash,
283
+ "content_sha256": content_hash,
284
+ "mode": mode,
285
+ }
286
+ (staging / "metadata.json").write_text(
287
+ json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
288
+ )
289
+ if cache_dir.exists():
290
+ shutil.rmtree(cache_dir)
291
+ os.replace(staging, cache_dir)
292
+ staging = None # type: ignore[assignment]
293
+ except TrajectoryProvisionError:
294
+ raise
295
+ except Exception as exc:
296
+ raise TrajectoryProvisionError(
297
+ f"Langfuse trajectory plugin provisioning failed for {agent}: {exc}"
298
+ ) from exc
299
+ finally:
300
+ if staging is not None and staging.exists():
301
+ shutil.rmtree(staging, ignore_errors=True)
302
+ return _read_provisioned_plugin(cache_dir, agent, spec)
303
+
304
+
305
+ def _bound_value(value: Any, max_chars: int) -> tuple[Any, bool]:
306
+ if isinstance(value, str):
307
+ return (value[:max_chars], len(value) > max_chars)
308
+ serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
309
+ if len(serialized) <= max_chars:
310
+ return value, False
311
+ return serialized[:max_chars], True
312
+
313
+
314
+ def _message_content(record: Mapping[str, Any]) -> Any:
315
+ message = record.get("message")
316
+ if isinstance(message, Mapping):
317
+ return message.get("content")
318
+ return record.get("content")
319
+
320
+
321
+ def _session_id(record: Mapping[str, Any]) -> str | None:
322
+ value = record.get("sessionId") or record.get("session_id")
323
+ return str(value) if value else None
324
+
325
+
326
+ def parse_agent_transcript(path: Path, *, max_chars: int = 20_000) -> list[TrajectoryEvent]:
327
+ """Parse Qoder/CodeBuddy transcript JSONL without inferring missing events."""
328
+ events: list[TrajectoryEvent] = []
329
+ try:
330
+ lines = path.read_text(encoding="utf-8").splitlines()
331
+ except OSError as exc:
332
+ raise TrajectoryProvisionError(f"Unable to read agent transcript: {path}") from exc
333
+
334
+ for line in lines:
335
+ if not line.strip():
336
+ continue
337
+ try:
338
+ record = json.loads(line)
339
+ except json.JSONDecodeError:
340
+ continue
341
+ if not isinstance(record, Mapping):
342
+ continue
343
+ record_type = str(record.get("type", ""))
344
+ session_id = _session_id(record)
345
+ if record_type == "session_meta":
346
+ data, truncated = _bound_value(record.get("data", {}), max_chars)
347
+ events.append(
348
+ TrajectoryEvent(
349
+ "session",
350
+ "session_meta",
351
+ output=data,
352
+ session_id=session_id,
353
+ truncated=truncated,
354
+ )
355
+ )
356
+ continue
357
+
358
+ content = _message_content(record)
359
+ items = content if isinstance(content, list) else [content]
360
+ for item in items:
361
+ if isinstance(item, Mapping):
362
+ item_type = str(item.get("type", ""))
363
+ if item_type == "tool_use":
364
+ input_value, truncated = _bound_value(item.get("input", {}), max_chars)
365
+ events.append(
366
+ TrajectoryEvent(
367
+ "tool",
368
+ str(item.get("name") or "tool"),
369
+ input=input_value,
370
+ session_id=session_id,
371
+ truncated=truncated,
372
+ )
373
+ )
374
+ continue
375
+ if item_type == "tool_result":
376
+ output, truncated = _bound_value(item.get("content", ""), max_chars)
377
+ events.append(
378
+ TrajectoryEvent(
379
+ "tool_result",
380
+ str(item.get("tool_use_id") or "tool_result"),
381
+ output=output,
382
+ session_id=session_id,
383
+ truncated=truncated,
384
+ )
385
+ )
386
+ continue
387
+ if item_type == "text":
388
+ value, truncated = _bound_value(item.get("text", ""), max_chars)
389
+ events.append(
390
+ TrajectoryEvent(
391
+ "text",
392
+ "assistant",
393
+ output=value,
394
+ session_id=session_id,
395
+ truncated=truncated,
396
+ )
397
+ )
398
+ continue
399
+ elif isinstance(item, str) and item:
400
+ value, truncated = _bound_value(item, max_chars)
401
+ events.append(
402
+ TrajectoryEvent(
403
+ "user" if record_type == "user" else "text",
404
+ "user" if record_type == "user" else "assistant",
405
+ input=value if record_type == "user" else None,
406
+ output=value if record_type != "user" else None,
407
+ session_id=session_id,
408
+ truncated=truncated,
409
+ )
410
+ )
411
+
412
+ if record_type == "progress":
413
+ value, truncated = _bound_value(record.get("data", {}), max_chars)
414
+ events.append(
415
+ TrajectoryEvent(
416
+ "progress", "progress", output=value, session_id=session_id, truncated=truncated
417
+ )
418
+ )
419
+ return events
420
+
421
+
422
+ def render_transcript_hook_script() -> str:
423
+ """Return the no-credential Stop hook used by Qoder and CodeBuddy."""
424
+ return """#!/usr/bin/env python3
425
+ import hashlib
426
+ import json
427
+ import os
428
+ import shutil
429
+ import sys
430
+ import tempfile
431
+
432
+
433
+ def main():
434
+ try:
435
+ payload = json.load(sys.stdin)
436
+ except Exception:
437
+ return 0
438
+ if payload.get("hook_event_name") != "Stop":
439
+ return 0
440
+ transcript = payload.get("transcript_path")
441
+ cwd = payload.get("cwd") or os.getcwd()
442
+ if not transcript or not os.path.isfile(transcript):
443
+ return 0
444
+ role = os.environ.get("COMET_EVAL_AGENT_ROLE", "subject")
445
+ if role not in {"subject", "simulator", "judge"}:
446
+ role = "subject"
447
+ session_id = payload.get("session_id") or payload.get("sessionId")
448
+ if not session_id:
449
+ session_id = hashlib.sha256(os.path.abspath(transcript).encode()).hexdigest()[:16]
450
+ safe_session = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in str(session_id))[:128]
451
+ target_dir = os.path.join(cwd, ".comet", "eval", "langfuse", "trajectories")
452
+ target = os.path.join(target_dir, f"{role}-{safe_session}.jsonl")
453
+ os.makedirs(target_dir, exist_ok=True)
454
+ fd, temporary = tempfile.mkstemp(prefix=".trajectory-", dir=os.path.dirname(target))
455
+ os.close(fd)
456
+ try:
457
+ shutil.copyfile(transcript, temporary)
458
+ os.replace(temporary, target)
459
+ finally:
460
+ try:
461
+ os.unlink(temporary)
462
+ except FileNotFoundError:
463
+ pass
464
+ return 0
465
+
466
+
467
+ if __name__ == "__main__":
468
+ raise SystemExit(main())
469
+ """
470
+
471
+
472
+ def install_transcript_hook(test_dir: Path, agent: str) -> Path | None:
473
+ """Install a project-local, credential-free Stop hook for transcript agents."""
474
+ if agent not in {"qoder", "codebuddy"}:
475
+ return None
476
+ project_dir_name = ".qoder" if agent == "qoder" else ".codebuddy"
477
+ project_dir = test_dir / project_dir_name
478
+ hook_path = project_dir / "hooks" / "langfuse-stop-hook.py"
479
+ hook_path.parent.mkdir(parents=True, exist_ok=True)
480
+ hook_path.write_text(render_transcript_hook_script(), encoding="utf-8")
481
+ settings_path = project_dir / "settings.json"
482
+ settings: dict[str, Any] = {}
483
+ if settings_path.is_file():
484
+ settings = json.loads(settings_path.read_text(encoding="utf-8"))
485
+ hooks = settings.setdefault("hooks", {})
486
+ stop_hooks = hooks.setdefault("Stop", [])
487
+ entry = {
488
+ "hooks": [
489
+ {
490
+ "type": "command",
491
+ "command": f"python3 /workspace/{project_dir_name}/hooks/langfuse-stop-hook.py",
492
+ "timeout": 30,
493
+ }
494
+ ]
495
+ }
496
+ if entry not in stop_hooks:
497
+ stop_hooks.append(entry)
498
+ settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
499
+ return test_dir / ".comet" / "eval" / "langfuse" / "trajectories"
500
+
501
+
502
+ def install_codex_plugin_workspace(test_dir: Path, provision: TrajectoryProvision) -> None:
503
+ """Copy the pinned Codex plugin into the isolated project CODEX_HOME."""
504
+ if provision.agent != "codex" or provision.plugin_path is None:
505
+ return
506
+ source = provision.plugin_path / "plugins" / "tracing"
507
+ version = json.loads((source / ".codex-plugin" / "plugin.json").read_text(encoding="utf-8"))[
508
+ "version"
509
+ ]
510
+ target = (
511
+ test_dir
512
+ / ".codex"
513
+ / "plugins"
514
+ / "cache"
515
+ / "codex-observability-plugin"
516
+ / "tracing"
517
+ / version
518
+ )
519
+ target.parent.mkdir(parents=True, exist_ok=True)
520
+ shutil.copytree(source, target, dirs_exist_ok=True)
521
+ config_path = test_dir / ".codex" / "config.toml"
522
+ existing = config_path.read_text(encoding="utf-8") if config_path.is_file() else ""
523
+ lines = existing.splitlines(keepends=True)
524
+ feature_header = next(
525
+ (index for index, line in enumerate(lines) if line.strip() == "[features]"),
526
+ None,
527
+ )
528
+ if feature_header is None:
529
+ existing = existing.rstrip() + "\n\n[features]\nhooks = true\n"
530
+ else:
531
+ feature_end = next(
532
+ (
533
+ index
534
+ for index in range(feature_header + 1, len(lines))
535
+ if lines[index].lstrip().startswith("[")
536
+ ),
537
+ len(lines),
538
+ )
539
+ feature_lines = lines[feature_header + 1 : feature_end]
540
+ hooks_line = next(
541
+ (
542
+ index
543
+ for index, line in enumerate(feature_lines)
544
+ if line.strip().startswith("hooks =")
545
+ ),
546
+ None,
547
+ )
548
+ if hooks_line is None:
549
+ lines.insert(feature_header + 1 + len(feature_lines), "hooks = true\n")
550
+ else:
551
+ lines[feature_header + 1 + hooks_line] = "hooks = true\n"
552
+ existing = "".join(lines)
553
+ if '[plugins."tracing@codex-observability-plugin"]' not in existing:
554
+ existing = (
555
+ existing.rstrip()
556
+ + '\n\n[plugins."tracing@codex-observability-plugin"]\nenabled = true\n'
557
+ )
558
+ config_path.parent.mkdir(parents=True, exist_ok=True)
559
+ config_path.write_text(existing.lstrip(), encoding="utf-8")
560
+
561
+
562
+ def create_client(
563
+ config: LangfuseConfig | None = None,
564
+ *,
565
+ client_factory: Callable[..., Any] | None = None,
566
+ authenticate: bool = True,
567
+ ) -> Any:
568
+ """Create and authenticate the SDK client only at the run boundary."""
569
+ resolved = config or LangfuseConfig.from_environment()
570
+ if client_factory is None:
571
+ try:
572
+ from langfuse import Langfuse
573
+ except Exception as exc: # pragma: no cover - optional dependency
574
+ raise LangfuseConfigurationError(
575
+ "The Langfuse Python SDK is unavailable. Install the eval langfuse extra."
576
+ ) from exc
577
+ client_factory = Langfuse
578
+
579
+ kwargs: dict[str, str] = {"public_key": resolved.public_key, "secret_key": resolved.secret_key}
580
+ if resolved.base_url:
581
+ kwargs["base_url"] = resolved.base_url
582
+ if resolved.tracing_environment:
583
+ kwargs["environment"] = resolved.tracing_environment
584
+ try:
585
+ client = client_factory(**kwargs)
586
+ if authenticate and hasattr(client, "auth_check") and not client.auth_check():
587
+ raise LangfuseConfigurationError("Langfuse authentication check failed")
588
+ return client
589
+ except LangfuseConfigurationError:
590
+ raise
591
+ except Exception as exc:
592
+ raise LangfuseConfigurationError(f"Langfuse authentication failed: {exc}") from exc
593
+
594
+
595
+ def enable_trajectory_environment(config: LangfuseConfig, agent: str) -> str:
596
+ """Expose credentials to a selected agent without writing them to artifacts."""
597
+ os.environ["TRACE_TO_LANGFUSE"] = "true"
598
+ os.environ["LANGFUSE_PUBLIC_KEY"] = config.public_key
599
+ os.environ["LANGFUSE_SECRET_KEY"] = config.secret_key
600
+ if config.base_url:
601
+ os.environ["LANGFUSE_BASE_URL"] = config.base_url
602
+ if config.tracing_environment:
603
+ os.environ["LANGFUSE_TRACING_ENVIRONMENT"] = config.tracing_environment
604
+ os.environ["LANGFUSE_TRAJECTORY_MODE"] = trajectory_mode(agent)
605
+ os.environ["LANGFUSE_TRAJECTORY_PROVISIONED"] = "false"
606
+ return trajectory_mode(agent)
607
+
608
+
609
+ def _result_payload(result: Any) -> dict[str, Any]:
610
+ passed = list(getattr(result, "checks_passed", []) or [])
611
+ failed = list(getattr(result, "checks_failed", []) or [])
612
+ summary = dict(getattr(result, "events_summary", {}) or {})
613
+ total_checks = len(passed) + len(failed)
614
+ payload: dict[str, Any] = {
615
+ "run_id": getattr(result, "run_id", ""),
616
+ "passed": bool(getattr(result, "passed", False)),
617
+ "checks_passed": len(passed),
618
+ "checks_failed": len(failed),
619
+ "checks_pass_rate": len(passed) / total_checks if total_checks else 0.0,
620
+ "failure_attribution": summary.get("failure_attribution"),
621
+ "num_turns": summary.get("num_turns"),
622
+ "tool_calls": summary.get("tool_calls"),
623
+ "duration_seconds": summary.get("duration_seconds"),
624
+ "total_tokens": summary.get("total_tokens"),
625
+ "total_cost_usd": summary.get("total_cost_usd"),
626
+ "rubric": _rubric_scores(passed),
627
+ }
628
+ for key in (
629
+ "task",
630
+ "treatment",
631
+ "sample",
632
+ "prompt",
633
+ "skill",
634
+ "profile",
635
+ "final_response",
636
+ "quality_gates",
637
+ "execution_identity",
638
+ "case_manifest",
639
+ "interaction",
640
+ "agent",
641
+ "role_models",
642
+ "role_sessions",
643
+ ):
644
+ value = summary.get(key)
645
+ if value is not None:
646
+ payload[key] = _bound_value(value, 20_000)[0]
647
+ experiment_id = os.environ.get("COMET_EVAL_EXPERIMENT_ID")
648
+ if experiment_id:
649
+ payload["experiment_id"] = experiment_id
650
+ payload["quality_gate"] = _quality_gate_result(payload)
651
+ return payload
652
+
653
+
654
+ def result_payload_from_local_report(report: Mapping[str, Any]) -> dict[str, Any]:
655
+ """Rebuild the canonical case payload from a persisted local report."""
656
+ return _result_payload(
657
+ SimpleNamespace(
658
+ run_id=report.get("run_id", ""),
659
+ passed=bool(report.get("passed")),
660
+ checks_passed=list(report.get("checks_passed") or []),
661
+ checks_failed=list(report.get("checks_failed") or []),
662
+ events_summary=dict(report.get("events_summary") or {}),
663
+ )
664
+ )
665
+
666
+
667
+ def _rubric_scores(checks: list[str]) -> dict[str, float]:
668
+ scores: dict[str, float] = {}
669
+ for check in checks:
670
+ match = _RUBRIC_SCORE_RE.search(check)
671
+ if not match:
672
+ continue
673
+ try:
674
+ scores[match.group(1)] = float(match.group(2))
675
+ except ValueError:
676
+ continue
677
+ return scores
678
+
679
+
680
+ def _quality_gate_result(payload: Mapping[str, Any]) -> dict[str, Any]:
681
+ configured = payload.get("quality_gates")
682
+ if not isinstance(configured, Mapping) or not configured:
683
+ return {"status": "not_applicable", "checks": {}}
684
+
685
+ rubric = payload.get("rubric")
686
+ rubric_scores = rubric if isinstance(rubric, Mapping) else {}
687
+ checks: dict[str, bool | None] = {}
688
+ aliases = {
689
+ "minWeightedScore": "weighted_score",
690
+ "minPassRate": "checks_pass_rate",
691
+ "minChecksPassRate": "checks_pass_rate",
692
+ }
693
+ for name, expected in configured.items():
694
+ metric_name = aliases.get(str(name))
695
+ if metric_name is None and str(name).startswith("min"):
696
+ candidate = str(name)[3:]
697
+ metric_name = candidate[:1].lower() + candidate[1:]
698
+ if (
699
+ metric_name is None
700
+ or not isinstance(expected, (int, float))
701
+ or isinstance(expected, bool)
702
+ ):
703
+ checks[str(name)] = None
704
+ continue
705
+ actual = rubric_scores.get(metric_name)
706
+ if metric_name == "checks_pass_rate":
707
+ actual = payload.get("checks_pass_rate")
708
+ checks[str(name)] = (
709
+ isinstance(actual, (int, float))
710
+ and not isinstance(actual, bool)
711
+ and float(actual) >= float(expected)
712
+ )
713
+
714
+ applicable = [value for value in checks.values() if value is not None]
715
+ if not applicable:
716
+ status = "unavailable"
717
+ elif any(value is False for value in applicable):
718
+ status = "failed"
719
+ elif any(value is None for value in checks.values()):
720
+ status = "unavailable"
721
+ else:
722
+ status = "passed"
723
+ return {"status": status, "checks": checks}
724
+
725
+
726
+ class LangfuseRunReporter:
727
+ """Write one strict core trace per task/treatment and score its result."""
728
+
729
+ def __init__(self, client: Any, *, suite: str = "comet-eval-langfuse"):
730
+ self.client = client
731
+ self.suite = suite
732
+ self.cases: list[dict[str, Any]] = []
733
+
734
+ def report_case(
735
+ self,
736
+ task_name: str,
737
+ treatment_name: str,
738
+ run: Callable[[], Any],
739
+ result_loader: Callable[[], Any],
740
+ *,
741
+ metadata: Mapping[str, Any] | None = None,
742
+ trajectory_path: Path | Callable[[], Path | None] | None = None,
743
+ agent: str | None = None,
744
+ ) -> Any:
745
+ input_payload = {
746
+ "task": task_name,
747
+ "treatment": treatment_name,
748
+ "suite": self.suite,
749
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
750
+ **dict(metadata or {}),
751
+ }
752
+ with self.client.start_as_current_observation(
753
+ as_type="span", name="comet.eval.run", input=input_payload
754
+ ) as observation:
755
+ result = None
756
+ run_error: BaseException | None = None
757
+ try:
758
+ result = run()
759
+ except BaseException as exc:
760
+ run_error = exc
761
+ result_loader_error: BaseException | None = None
762
+ try:
763
+ result = result_loader() or result
764
+ except BaseException as exc:
765
+ result_loader_error = exc
766
+ payload = (
767
+ _result_payload(result)
768
+ if result is not None
769
+ else {
770
+ "passed": False,
771
+ "checks_passed": 0,
772
+ "checks_failed": 0,
773
+ "checks_pass_rate": 0.0,
774
+ }
775
+ )
776
+ try:
777
+ resolved_trajectory_path = (
778
+ trajectory_path() if callable(trajectory_path) else trajectory_path
779
+ )
780
+ payload["trajectory"] = self._report_trajectory(
781
+ resolved_trajectory_path,
782
+ agent=agent or str(input_payload.get("agent", "")),
783
+ role_sessions=payload.get("role_sessions") or {},
784
+ )
785
+ except Exception as exc:
786
+ print(f"[langfuse] optional trajectory upload skipped: {exc}", file=os.sys.stderr)
787
+ payload["trajectory"] = {
788
+ "status": "unavailable",
789
+ "mode": trajectory_mode(agent or str(input_payload.get("agent", ""))),
790
+ "reason": str(exc),
791
+ }
792
+ payload["trajectory_status"] = payload["trajectory"].get("status", "unavailable")
793
+ for key in ("prompt", "skill", "profile", "execution_identity", "agent", "sample"):
794
+ if payload.get(key) is not None:
795
+ input_payload[key] = payload[key]
796
+ core_error: BaseException | None = None
797
+ try:
798
+ observation.update(input=input_payload, output=payload)
799
+ self._score(observation, payload)
800
+ except BaseException as exc:
801
+ core_error = exc
802
+ self.cases.append({"task": task_name, "treatment": treatment_name, **payload})
803
+ if run_error is not None:
804
+ if core_error is not None:
805
+ print(
806
+ f"[langfuse] evaluation failed and core upload also failed: {core_error}",
807
+ file=os.sys.stderr,
808
+ )
809
+ raise run_error
810
+ if core_error is not None:
811
+ raise core_error
812
+ if result_loader_error is not None:
813
+ raise result_loader_error
814
+ return result
815
+
816
+ def _report_trajectory(
817
+ self,
818
+ trajectory_path: Path | None,
819
+ *,
820
+ agent: str,
821
+ role_sessions: Mapping[str, Any],
822
+ ) -> dict[str, Any]:
823
+ mode = trajectory_mode(agent)
824
+ if mode.startswith("official-"):
825
+ if os.environ.get("LANGFUSE_TRAJECTORY_PROVISIONED", "").lower() != "true":
826
+ return {
827
+ "status": "unavailable",
828
+ "mode": mode,
829
+ "reason": "plugin_not_provisioned",
830
+ }
831
+ return {
832
+ "status": "provisioned",
833
+ "mode": mode,
834
+ "role_sessions": dict(role_sessions),
835
+ }
836
+ if not trajectory_path or not trajectory_path.exists():
837
+ return {"status": "unavailable", "mode": mode, "reason": "transcript_not_found"}
838
+ try:
839
+ transcript_paths = (
840
+ sorted(trajectory_path.glob("*.jsonl"))
841
+ if trajectory_path.is_dir()
842
+ else [trajectory_path]
843
+ )
844
+ events = [
845
+ event
846
+ for transcript_path in transcript_paths
847
+ for event in parse_agent_transcript(transcript_path)
848
+ ]
849
+ if not events:
850
+ return {"status": "unavailable", "mode": mode, "reason": "transcript_empty"}
851
+ session_ids = sorted({event.session_id for event in events if event.session_id})
852
+ for index, event in enumerate(events):
853
+ observation_type = "generation" if event.kind == "text" else "span"
854
+ with self.client.start_as_current_observation(
855
+ as_type=observation_type,
856
+ name=f"agent.{event.kind}/{event.name}/{index}",
857
+ input=event.input,
858
+ ) as child:
859
+ child.update(
860
+ output=event.output,
861
+ metadata={
862
+ "agent": agent,
863
+ "session_id": event.session_id,
864
+ "truncated": event.truncated,
865
+ },
866
+ )
867
+ return {
868
+ "status": "uploaded",
869
+ "mode": mode,
870
+ "events": len(events),
871
+ "session_ids": session_ids,
872
+ "role_sessions": dict(role_sessions),
873
+ "truncated_events": sum(1 for event in events if event.truncated),
874
+ }
875
+ except Exception as exc:
876
+ print(f"[langfuse] optional trajectory upload skipped: {exc}", file=os.sys.stderr)
877
+ return {"status": "unavailable", "mode": mode, "reason": str(exc)}
878
+
879
+ def _score(self, observation: Any, payload: Mapping[str, Any]) -> None:
880
+ trace_id = getattr(observation, "trace_id", None)
881
+ observation_id = getattr(observation, "id", None)
882
+ if not trace_id:
883
+ raise LangfuseReportingError("Langfuse observation did not expose a trace id")
884
+ try:
885
+ for name, value, data_type in (
886
+ ("task_passed", bool(payload.get("passed")), "BOOLEAN"),
887
+ ("checks_pass_rate", float(payload.get("checks_pass_rate", 0.0)), "NUMERIC"),
888
+ ):
889
+ self.client.create_score(
890
+ name={
891
+ "task_passed": "comet.passed",
892
+ "checks_pass_rate": "comet.checks_pass_rate",
893
+ }[name],
894
+ value=value,
895
+ trace_id=trace_id,
896
+ observation_id=observation_id,
897
+ data_type=data_type,
898
+ )
899
+ for dimension, value in (payload.get("rubric") or {}).items():
900
+ self.client.create_score(
901
+ name=f"comet.rubric.{dimension}",
902
+ value=float(value),
903
+ trace_id=trace_id,
904
+ observation_id=observation_id,
905
+ data_type="NUMERIC",
906
+ )
907
+ except Exception as exc:
908
+ raise LangfuseReportingError(f"Langfuse core score failed: {exc}") from exc
909
+
910
+ def report_summary(self) -> None:
911
+ """Write the suite summary as a final core observation."""
912
+ try:
913
+ with self.client.start_as_current_observation(
914
+ as_type="span",
915
+ name="comet.eval.experiment",
916
+ input={
917
+ "suite": self.suite,
918
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
919
+ "cases": len(self.cases),
920
+ },
921
+ ) as observation:
922
+ passed = sum(1 for case in self.cases if case.get("passed"))
923
+ aggregates = self._aggregates()
924
+ observation.update(
925
+ output={
926
+ "cases": len(self.cases),
927
+ "passed": passed,
928
+ "failed": len(self.cases) - passed,
929
+ "aggregates": aggregates,
930
+ }
931
+ )
932
+ self._score_summary(observation, aggregates)
933
+ except Exception as exc:
934
+ raise LangfuseReportingError(f"Langfuse summary trace failed: {exc}") from exc
935
+
936
+ def _aggregates(self) -> dict[str, Any]:
937
+ groups: dict[str, list[dict[str, Any]]] = {}
938
+ for case in self.cases:
939
+ task = str(case.get("task", "unknown"))
940
+ treatment = str(case.get("treatment", "unknown"))
941
+ groups.setdefault(f"{task}/{treatment}", []).append(case)
942
+
943
+ aggregates: dict[str, Any] = {}
944
+ for key, cases in groups.items():
945
+ outcomes = [bool(case.get("passed")) for case in cases]
946
+ metrics = {str(k): compute_pass_metrics(outcomes, k) for k in _SUMMARY_K_VALUES}
947
+ gate_statuses = [(case.get("quality_gate") or {}).get("status") for case in cases]
948
+ if gate_statuses and all(status == "not_applicable" for status in gate_statuses):
949
+ quality_gate = "not_applicable"
950
+ elif "failed" in gate_statuses:
951
+ quality_gate = "failed"
952
+ elif "unavailable" in gate_statuses:
953
+ quality_gate = "unavailable"
954
+ else:
955
+ quality_gate = "passed"
956
+ aggregates[key] = {
957
+ "task": key.split("/", 1)[0],
958
+ "treatment": key.split("/", 1)[1],
959
+ "runs": len(cases),
960
+ "pass_at_k": {k: metrics[str(k)]["pass_at_k"] for k in _SUMMARY_K_VALUES},
961
+ "pass_power_k": {k: metrics[str(k)]["pass_pow_k"] for k in _SUMMARY_K_VALUES},
962
+ "quality_gate": quality_gate,
963
+ "agent": (
964
+ next(iter({str(case.get("agent")) for case in cases}))
965
+ if len({str(case.get("agent")) for case in cases}) == 1
966
+ else "mixed"
967
+ ),
968
+ }
969
+ return aggregates
970
+
971
+ def _score_summary(self, observation: Any, aggregates: Mapping[str, Any]) -> None:
972
+ trace_id = getattr(observation, "trace_id", None)
973
+ observation_id = getattr(observation, "id", None)
974
+ if not trace_id:
975
+ raise LangfuseReportingError("Langfuse summary observation did not expose a trace id")
976
+ try:
977
+ for aggregate in aggregates.values():
978
+ for k in _SUMMARY_K_VALUES:
979
+ score_metadata = {
980
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
981
+ "agent": aggregate["agent"],
982
+ "task": aggregate["task"],
983
+ "treatment": aggregate["treatment"],
984
+ "k": k,
985
+ }
986
+ self.client.create_score(
987
+ name="comet.pass_at_k",
988
+ value=float(aggregate["pass_at_k"][k]),
989
+ trace_id=trace_id,
990
+ observation_id=observation_id,
991
+ data_type="NUMERIC",
992
+ metadata=score_metadata,
993
+ )
994
+ self.client.create_score(
995
+ name="comet.pass_power_k",
996
+ value=float(aggregate["pass_power_k"][k]),
997
+ trace_id=trace_id,
998
+ observation_id=observation_id,
999
+ data_type="NUMERIC",
1000
+ metadata=score_metadata,
1001
+ )
1002
+ if aggregate["quality_gate"] in {"passed", "failed"}:
1003
+ self.client.create_score(
1004
+ name="comet.quality_gate",
1005
+ value=aggregate["quality_gate"] == "passed",
1006
+ trace_id=trace_id,
1007
+ observation_id=observation_id,
1008
+ data_type="BOOLEAN",
1009
+ metadata={
1010
+ "experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
1011
+ "agent": aggregate["agent"],
1012
+ "task": aggregate["task"],
1013
+ "treatment": aggregate["treatment"],
1014
+ },
1015
+ )
1016
+ except Exception as exc:
1017
+ raise LangfuseReportingError(f"Langfuse summary score failed: {exc}") from exc
1018
+
1019
+ def flush(self) -> None:
1020
+ try:
1021
+ self.client.flush()
1022
+ except Exception as exc:
1023
+ raise LangfuseReportingError(f"Langfuse flush failed: {exc}") from exc