@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -0,0 +1,263 @@
1
+ """Contract tests for Langfuse trajectory provisioning and transcript adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import json
7
+ import subprocess
8
+ import sys
9
+ import tarfile
10
+ from pathlib import Path
11
+
12
+ import pytest
13
+
14
+ from scaffold.python.langfuse_adapter import (
15
+ LangfuseRunReporter,
16
+ TrajectoryProvisionError,
17
+ TrajectoryProvision,
18
+ install_codex_plugin_workspace,
19
+ install_transcript_hook,
20
+ parse_agent_transcript,
21
+ provision_trajectory_plugin,
22
+ trajectory_mode,
23
+ )
24
+
25
+
26
+ def _archive_bytes(files: dict[str, str]) -> bytes:
27
+ stream = io.BytesIO()
28
+ with tarfile.open(fileobj=stream, mode="w:gz") as archive:
29
+ for name, content in files.items():
30
+ payload = content.encode("utf-8")
31
+ info = tarfile.TarInfo(name)
32
+ info.size = len(payload)
33
+ archive.addfile(info, io.BytesIO(payload))
34
+ return stream.getvalue()
35
+
36
+
37
+ def test_trajectory_modes_cover_all_supported_agent_contracts():
38
+ assert trajectory_mode("claude-code") == "official-claude-code-plugin"
39
+ assert trajectory_mode("codex") == "official-codex-plugin"
40
+ assert trajectory_mode("qoder") == "qoder-stop-transcript"
41
+ assert trajectory_mode("codebuddy") == "codebuddy-stop-transcript"
42
+
43
+
44
+ def test_plugin_provision_is_pinned_hashed_cached_and_credential_free(tmp_path: Path):
45
+ archive = _archive_bytes(
46
+ {
47
+ "langfuse-plugin/plugins/tracing/hooks/hooks.json": '{"hooks": {}}\n',
48
+ "langfuse-plugin/plugins/tracing/dist/index.mjs": "export {};\n",
49
+ }
50
+ )
51
+ calls = []
52
+
53
+ def download(url: str, destination: Path) -> None:
54
+ calls.append(url)
55
+ destination.write_bytes(archive)
56
+
57
+ first = provision_trajectory_plugin(tmp_path, "codex", download=download)
58
+
59
+ assert first.mode == "official-codex-plugin"
60
+ assert first.plugin_path is not None
61
+ assert (first.plugin_path / "plugins" / "tracing" / "hooks" / "hooks.json").is_file()
62
+ metadata = json.loads((first.cache_dir / "metadata.json").read_text(encoding="utf-8"))
63
+ assert metadata["source_ref"]
64
+ assert metadata["archive_sha256"]
65
+ assert metadata["content_sha256"]
66
+ assert "LANGFUSE_SECRET_KEY" not in json.dumps(metadata)
67
+
68
+ second = provision_trajectory_plugin(
69
+ tmp_path,
70
+ "codex",
71
+ download=lambda *_args: pytest.fail("cache hit must not download"),
72
+ )
73
+ assert second.plugin_path == first.plugin_path
74
+ assert len(calls) == 1
75
+
76
+ (first.plugin_path / "plugins" / "tracing" / "dist" / "index.mjs").write_text(
77
+ "tampered\n", encoding="utf-8"
78
+ )
79
+ with pytest.raises(TrajectoryProvisionError, match="integrity"):
80
+ provision_trajectory_plugin(tmp_path, "codex", download=lambda *_args: None)
81
+
82
+
83
+ def test_qoder_and_codebuddy_transcripts_normalize_messages_tools_and_bounds(tmp_path: Path):
84
+ transcript = tmp_path / "session.jsonl"
85
+ transcript.write_text(
86
+ "\n".join(
87
+ [
88
+ json.dumps({"type": "session_meta", "sessionId": "s-1", "data": {"model": "m"}}),
89
+ json.dumps({"type": "user", "message": {"content": "Please inspect"}}),
90
+ json.dumps(
91
+ {
92
+ "type": "assistant",
93
+ "message": {
94
+ "content": [
95
+ {"type": "text", "text": "I will inspect."},
96
+ {
97
+ "type": "tool_use",
98
+ "name": "read_file",
99
+ "input": {"path": "a.py"},
100
+ },
101
+ ]
102
+ },
103
+ }
104
+ ),
105
+ json.dumps(
106
+ {
107
+ "type": "user",
108
+ "message": {
109
+ "content": [
110
+ {"type": "tool_result", "tool_use_id": "t-1", "content": "x" * 100}
111
+ ]
112
+ },
113
+ }
114
+ ),
115
+ ]
116
+ )
117
+ + "\n",
118
+ encoding="utf-8",
119
+ )
120
+
121
+ events = parse_agent_transcript(transcript, max_chars=32)
122
+
123
+ assert [event.kind for event in events] == ["session", "user", "text", "tool", "tool_result"]
124
+ assert events[0].session_id == "s-1"
125
+ assert events[3].name == "read_file"
126
+ assert events[-1].truncated is True
127
+ assert len(events[-1].output) <= 32
128
+
129
+
130
+ def test_transcript_hook_is_project_local_and_preserves_existing_settings(tmp_path: Path):
131
+ settings = tmp_path / ".codebuddy" / "settings.json"
132
+ settings.parent.mkdir(parents=True)
133
+ settings.write_text(
134
+ json.dumps({"custom": True, "hooks": {"Stop": [{"custom": True}]}}), encoding="utf-8"
135
+ )
136
+
137
+ trajectory_path = install_transcript_hook(tmp_path, "codebuddy")
138
+
139
+ assert trajectory_path == tmp_path / ".comet" / "eval" / "langfuse" / "trajectories"
140
+ hook_path = tmp_path / ".codebuddy" / "hooks" / "langfuse-stop-hook.py"
141
+ assert hook_path.is_file()
142
+ hook_source = hook_path.read_text(encoding="utf-8")
143
+ assert "COMET_EVAL_AGENT_ROLE" in hook_source
144
+ assert "hashlib.sha256" in hook_source
145
+ saved = json.loads(settings.read_text(encoding="utf-8"))
146
+ assert saved["custom"] is True
147
+ assert len(saved["hooks"]["Stop"]) == 2
148
+ assert "LANGFUSE_SECRET_KEY" not in settings.read_text(encoding="utf-8")
149
+
150
+ transcript = tmp_path / "transcript.jsonl"
151
+ transcript.write_text(
152
+ '{"type":"assistant","message":{"content":"done"}}\n',
153
+ encoding="utf-8",
154
+ )
155
+ result = subprocess.run(
156
+ [sys.executable, str(hook_path)],
157
+ input=json.dumps(
158
+ {
159
+ "hook_event_name": "Stop",
160
+ "transcript_path": str(transcript),
161
+ "cwd": str(tmp_path),
162
+ }
163
+ ),
164
+ text=True,
165
+ capture_output=True,
166
+ check=False,
167
+ )
168
+ assert result.returncode == 0, result.stderr
169
+ assert list(
170
+ (tmp_path / ".comet" / "eval" / "langfuse" / "trajectories").glob("*.jsonl")
171
+ )
172
+
173
+
174
+ def test_codex_plugin_is_copied_into_the_project_codex_home_cache(tmp_path: Path):
175
+ plugin = tmp_path / "plugin"
176
+ tracing = plugin / "plugins" / "tracing"
177
+ (tracing / ".codex-plugin").mkdir(parents=True)
178
+ (tracing / ".codex-plugin" / "plugin.json").write_text(
179
+ json.dumps({"version": "0.1.0"}), encoding="utf-8"
180
+ )
181
+ (tracing / "dist").mkdir()
182
+ (tracing / "dist" / "index.mjs").write_text("export {};\n", encoding="utf-8")
183
+ provision = TrajectoryProvision(
184
+ agent="codex",
185
+ mode="official-codex-plugin",
186
+ cache_dir=tmp_path,
187
+ plugin_path=plugin,
188
+ )
189
+
190
+ workspace = tmp_path / "workspace"
191
+ (workspace / ".codex").mkdir(parents=True)
192
+ (workspace / ".codex" / "config.toml").write_text(
193
+ '[features]\nother = true\n\n[profiles.default]\nmodel = "x"\n',
194
+ encoding="utf-8",
195
+ )
196
+ install_codex_plugin_workspace(workspace, provision)
197
+
198
+ installed = (
199
+ workspace
200
+ / ".codex"
201
+ / "plugins"
202
+ / "cache"
203
+ / "codex-observability-plugin"
204
+ / "tracing"
205
+ / "0.1.0"
206
+ )
207
+ assert (installed / "dist" / "index.mjs").is_file()
208
+ config = (workspace / ".codex" / "config.toml").read_text(encoding="utf-8")
209
+ assert "hooks = true" in config
210
+ assert 'plugins."tracing@codex-observability-plugin"' in config
211
+ assert config.index("hooks = true") < config.index("[profiles.default]")
212
+
213
+
214
+ class _Observation:
215
+ trace_id = "trace-1"
216
+ id = "observation-1"
217
+
218
+ def __enter__(self):
219
+ return self
220
+
221
+ def __exit__(self, exc_type, exc, tb):
222
+ return False
223
+
224
+ def update(self, **payload):
225
+ self.output = payload.get("output")
226
+
227
+
228
+ class _Client:
229
+ def __init__(self):
230
+ self.observations = []
231
+ self.scores = []
232
+
233
+ def start_as_current_observation(self, **kwargs):
234
+ self.observations.append(kwargs)
235
+ return _Observation()
236
+
237
+ def create_score(self, **kwargs):
238
+ self.scores.append(kwargs)
239
+
240
+
241
+ def test_reporter_maps_qoder_transcript_to_optional_child_observations(tmp_path: Path):
242
+ transcript = tmp_path / "trajectory.jsonl"
243
+ transcript.write_text(
244
+ json.dumps({"type": "assistant", "sessionId": "s-1", "message": {"content": "done"}})
245
+ + "\n",
246
+ encoding="utf-8",
247
+ )
248
+ client = _Client()
249
+ result = type("Result", (), {"passed": True, "checks_passed": ["ok"], "checks_failed": []})()
250
+ reporter = LangfuseRunReporter(client)
251
+
252
+ reporter.report_case(
253
+ "task",
254
+ "CONTROL",
255
+ lambda: None,
256
+ lambda: result,
257
+ metadata={"agent": "qoder"},
258
+ trajectory_path=transcript,
259
+ agent="qoder",
260
+ )
261
+
262
+ assert client.observations[1]["as_type"] == "generation"
263
+ assert client.observations[1]["name"].startswith("agent.text/")
@@ -4,11 +4,17 @@ from __future__ import annotations
4
4
 
5
5
  import inspect
6
6
  import importlib.util
7
+ import os
7
8
  import sys
9
+ import subprocess
8
10
  from pathlib import Path
9
11
  from types import SimpleNamespace
10
12
  import json
11
13
 
14
+ import pytest
15
+
16
+ from scaffold.python.eval_context import resolve_eval_context
17
+
12
18
 
13
19
  def _load_langsmith_conftest():
14
20
  eval_root = Path(__file__).resolve().parents[3]
@@ -95,6 +101,63 @@ def test_langsmith_tracing_auto_builds_default_plugin_cache(monkeypatch, tmp_pat
95
101
  assert module.os.environ["CC_LANGSMITH_PLUGIN_DIR"] == module._plugin_dir_env_value(plugin_dir)
96
102
 
97
103
 
104
+ def test_langsmith_default_plugin_cache_uses_resolved_owner_not_packaged_harness(
105
+ monkeypatch, tmp_path
106
+ ):
107
+ skill = tmp_path / "skill"
108
+ owner = tmp_path / "owner"
109
+ harness_cache = tmp_path / "packaged-harness" / ".cache" / "langsmith-cc-plugin"
110
+ skill.mkdir()
111
+ owner.mkdir()
112
+ (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
113
+ context = resolve_eval_context(skill_path=skill, project_root=owner)
114
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
115
+ module = _load_langsmith_conftest()
116
+ monkeypatch.setattr(module, "DEFAULT_LANGSMITH_PLUGIN_DIR", harness_cache)
117
+ monkeypatch.delenv("CC_LANGSMITH_PLUGIN_DIR", raising=False)
118
+ monkeypatch.setenv("LANGSMITH_TRACING", "true")
119
+ monkeypatch.setenv("TRACE_TO_LANGSMITH", "true")
120
+ calls = []
121
+
122
+ def fake_build(target):
123
+ calls.append(target)
124
+ target.mkdir(parents=True)
125
+ return True
126
+
127
+ monkeypatch.setattr(module, "_build_default_langsmith_plugin", fake_build)
128
+
129
+ resolved = module.provision_langsmith_plugin_dir()
130
+
131
+ expected = owner / ".comet" / "eval" / "cache" / "langsmith" / "langsmith-cc-plugin"
132
+ assert resolved == expected
133
+ assert calls == [expected]
134
+ assert not harness_cache.exists()
135
+
136
+
137
+ def test_langsmith_default_plugin_cache_rejects_an_owner_cache_link_that_escapes(
138
+ monkeypatch, tmp_path
139
+ ):
140
+ skill = tmp_path / "skill"
141
+ owner = tmp_path / "owner"
142
+ outside = tmp_path / "outside"
143
+ skill.mkdir()
144
+ owner.mkdir()
145
+ outside.mkdir()
146
+ (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
147
+ context = resolve_eval_context(skill_path=skill, project_root=owner)
148
+ cache_link = context.artifact_root / "cache"
149
+ cache_link.parent.mkdir(parents=True)
150
+ if os.name == "nt":
151
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(cache_link), str(outside)], check=True)
152
+ else:
153
+ cache_link.symlink_to(outside, target_is_directory=True)
154
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
155
+ module = _load_langsmith_conftest()
156
+
157
+ with pytest.raises(ValueError, match="managed path must stay within its owner root"):
158
+ module._default_langsmith_plugin_dir()
159
+
160
+
98
161
  def test_langsmith_task_wrapper_registers_reexported_local_module():
99
162
  _load_langsmith_task_tests()
100
163
 
@@ -35,6 +35,56 @@ def test_save_artifacts_excludes_nested_git_metadata(tmp_path: Path):
35
35
  assert not (snapshot / ".git").exists()
36
36
 
37
37
 
38
+ def test_save_artifacts_redacts_configured_credentials(tmp_path: Path, monkeypatch):
39
+ from conftest import _save_artifacts
40
+
41
+ monkeypatch.setenv("FIXTURE_AGENT_CREDENTIAL", "artifact-secret-value")
42
+ workspace = tmp_path / "workspace"
43
+ workspace.mkdir()
44
+ (workspace / "result.md").write_text(
45
+ "Authorization: Bearer artifact-secret-value\n", encoding="utf-8"
46
+ )
47
+
48
+ _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
49
+
50
+ snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
51
+ saved = (snapshot / "result.md").read_text(encoding="utf-8")
52
+ assert "artifact-secret-value" not in saved
53
+
54
+
55
+ def test_save_artifacts_skips_non_text_content_instead_of_persisting_secrets(tmp_path: Path):
56
+ from conftest import _save_artifacts
57
+
58
+ workspace = tmp_path / "workspace"
59
+ workspace.mkdir()
60
+ (workspace / "secret.bin").write_bytes(b"artifact-secret-value\x00\xff")
61
+
62
+ _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
63
+
64
+ snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
65
+ assert not (snapshot / "secret.bin").exists()
66
+
67
+
68
+ def test_save_artifacts_uses_selected_agent_root_and_excludes_codebuddy_config(
69
+ tmp_path: Path,
70
+ ):
71
+ from conftest import _save_artifacts
72
+
73
+ workspace = tmp_path / "workspace"
74
+ workspace.mkdir()
75
+ (workspace / "result.md").write_text("ok", encoding="utf-8")
76
+ (workspace / ".codebuddy" / "settings.json").parent.mkdir()
77
+ (workspace / ".codebuddy" / "settings.json").write_text(
78
+ '{"env":{"CODEBUDDY_API_KEY":"secret"}}', encoding="utf-8"
79
+ )
80
+
81
+ _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace, agent="codebuddy")
82
+
83
+ snapshot = tmp_path / "artifacts/comet_full_040_beta_rep1/codebuddy"
84
+ assert (snapshot / "result.md").read_text(encoding="utf-8") == "ok"
85
+ assert not (snapshot / ".codebuddy").exists()
86
+
87
+
38
88
  def test_save_artifacts_excludes_controller_cli_snapshot(tmp_path: Path):
39
89
  from conftest import _save_artifacts
40
90
 
@@ -93,6 +143,97 @@ def test_extract_events_captures_token_usage_and_cost():
93
143
  assert events["model_usage"]["mimo-v2.5-pro"]["costUSD"] == 0.123456
94
144
 
95
145
 
146
+ def test_extract_events_normalizes_codex_turns_and_runtime_skill_evidence():
147
+ stdout = "\n".join(
148
+ [
149
+ json.dumps({"type": "thread.started", "thread_id": "thread-1"}),
150
+ json.dumps(
151
+ {
152
+ "type": "item.completed",
153
+ "item": {
154
+ "type": "command_execution",
155
+ "command": "cat .agents/skills/demo/SKILL.md",
156
+ "aggregated_output": "skill contents",
157
+ },
158
+ }
159
+ ),
160
+ json.dumps(
161
+ {
162
+ "type": "item.completed",
163
+ "item": {
164
+ "type": "file_change",
165
+ "changes": [{"path": "result.md", "kind": "add"}],
166
+ },
167
+ }
168
+ ),
169
+ json.dumps(
170
+ {
171
+ "type": "turn.completed",
172
+ "usage": {"input_tokens": 80, "output_tokens": 20},
173
+ "duration_ms": 900,
174
+ }
175
+ ),
176
+ ]
177
+ )
178
+
179
+ events = extract_events(parse_output(stdout))
180
+
181
+ assert events["role_sessions"]["subject"] == ["thread-1"]
182
+
183
+ assert events["subject_invocations"] == 1
184
+ assert events["num_turns"] == 1
185
+ assert events["input_tokens"] == 80
186
+ assert events["output_tokens"] == 20
187
+ assert events["total_tokens"] == 100
188
+ assert events["duration_seconds"] == 0.9
189
+ assert events["commands_run"] == ["cat .agents/skills/demo/SKILL.md"]
190
+ assert events["files_created"] == ["result.md"]
191
+ assert events["skills_invoked"] == ["demo"]
192
+
193
+
194
+ def test_extract_events_recognizes_codebuddy_runtime_skill_evidence():
195
+ stdout = json.dumps(
196
+ {
197
+ "type": "assistant",
198
+ "message": {
199
+ "content": [
200
+ {
201
+ "type": "tool_use",
202
+ "name": "Bash",
203
+ "input": {"command": "cat .codebuddy/skills/demo/SKILL.md"},
204
+ }
205
+ ]
206
+ },
207
+ }
208
+ )
209
+
210
+ events = extract_events(parse_output(stdout))
211
+
212
+ assert events["skills_invoked"] == ["demo"]
213
+
214
+
215
+ def test_custom_agent_requires_explicit_skill_invocation_events():
216
+ path_only = json.dumps(
217
+ {
218
+ "type": "item.completed",
219
+ "item": {
220
+ "type": "command_execution",
221
+ "command": "cat .agents/skills/demo/SKILL.md",
222
+ },
223
+ }
224
+ )
225
+ inferred = extract_events(parse_output(path_only), agent="fixture-agent")
226
+ assert inferred["skills_invoked"] == []
227
+ assert inferred["skill_invocations"] == []
228
+
229
+ explicit = extract_events(
230
+ parse_output(json.dumps({"type": "skill_invocation", "skill": "demo"})),
231
+ agent="fixture-agent",
232
+ )
233
+ assert explicit["skills_invoked"] == ["demo"]
234
+ assert explicit["skill_invocations"] == ["demo"]
235
+
236
+
96
237
  def test_extract_events_accumulates_duration_across_results():
97
238
  stdout = "\n".join(
98
239
  [
@@ -304,6 +445,27 @@ def test_save_raw_writes_utf8_output(tmp_path: Path):
304
445
  assert "stderr 中文" == stderr_path.read_text(encoding="utf-8")
305
446
 
306
447
 
448
+ def test_saved_eval_outputs_redact_configured_credentials(tmp_path: Path, monkeypatch):
449
+ monkeypatch.setenv("FIXTURE_AGENT_API_KEY", "fixture-secret-value")
450
+ save_raw(
451
+ tmp_path,
452
+ "COMET_FULL_040_BETA",
453
+ 1,
454
+ "Authorization: Bearer fixture-secret-value x-api-key: fixture-secret-value",
455
+ "token=fixture-secret-value",
456
+ )
457
+ save_events(tmp_path, "COMET_FULL_040_BETA", 1, {"token": "fixture-secret-value"})
458
+ save_report(tmp_path, "COMET_FULL_040_BETA", 1, {"credential": "fixture-secret-value"})
459
+
460
+ for path in (
461
+ tmp_path / "raw/COMET_FULL_040_BETA_rep1_stdout.json",
462
+ tmp_path / "raw/COMET_FULL_040_BETA_rep1_stderr.txt",
463
+ tmp_path / "events/COMET_FULL_040_BETA_rep1.json",
464
+ tmp_path / "reports/COMET_FULL_040_BETA_rep1_report.json",
465
+ ):
466
+ assert "fixture-secret-value" not in path.read_text(encoding="utf-8")
467
+
468
+
307
469
  def test_save_artifacts_preserve_stable_treatment_filenames(tmp_path: Path):
308
470
  save_events(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
309
471
  save_raw(tmp_path, "COMET_FULL_040_BETA", 2, "{}", "stderr")
@@ -0,0 +1,76 @@
1
+ """Tests for normalizing manifest-authored tasks into the runner contract."""
2
+
3
+ from pathlib import Path
4
+
5
+ from scaffold.python.manifest_tasks import load_manifest_tasks
6
+ from scaffold.python.manifests import load_eval_manifest
7
+
8
+
9
+ def test_inline_task_uses_generic_environment_and_exposes_normalized_contract(tmp_path: Path):
10
+ package = tmp_path / "my-skill"
11
+ (package / "comet").mkdir(parents=True)
12
+ manifest_path = package / "comet" / "eval.yaml"
13
+ manifest_path.write_text(
14
+ """
15
+ apiVersion: comet.eval/v1alpha1
16
+ kind: SkillEvalManifest
17
+ metadata:
18
+ name: my-skill
19
+ skill:
20
+ name: my-skill
21
+ source: ..
22
+ evaluation:
23
+ tasks:
24
+ - name: inline-task
25
+ prompt: Create result.md.
26
+ expect:
27
+ files: [result.md]
28
+ rubric:
29
+ - The result is useful.
30
+ """,
31
+ encoding="utf-8",
32
+ )
33
+
34
+ tasks = load_manifest_tasks(load_eval_manifest(manifest_path))
35
+
36
+ assert len(tasks) == 1
37
+ task = tasks[0]
38
+ assert task.name == "inline-task"
39
+ assert task.render_prompt() == "Create result.md."
40
+ assert task.environment_dir.is_dir()
41
+ assert task.config.evaluation.expected_artifacts == ["result.md"]
42
+ assert task.config.evaluation.rubric_criteria == ["The result is useful."]
43
+
44
+
45
+ def test_source_task_is_loaded_without_changing_its_task_contract(tmp_path: Path):
46
+ package = tmp_path / "my-skill"
47
+ source = package / "tasks" / "source-task"
48
+ (source / "environment").mkdir(parents=True)
49
+ (source / "task.toml").write_text(
50
+ '[metadata]\nname = "source-task"\n\n[environment]\ndockerfile = "Dockerfile"\n',
51
+ encoding="utf-8",
52
+ )
53
+ (source / "instruction.md").write_text("Source prompt\n", encoding="utf-8")
54
+ (source / "environment" / "Dockerfile").write_text("FROM alpine\n", encoding="utf-8")
55
+ manifest_path = package / "comet" / "eval.yaml"
56
+ manifest_path.parent.mkdir(parents=True)
57
+ manifest_path.write_text(
58
+ """
59
+ apiVersion: comet.eval/v1alpha1
60
+ kind: SkillEvalManifest
61
+ metadata:
62
+ name: my-skill
63
+ skill:
64
+ name: my-skill
65
+ source: ..
66
+ evaluation:
67
+ tasks:
68
+ - source: ./tasks/source-task
69
+ """,
70
+ encoding="utf-8",
71
+ )
72
+
73
+ tasks = load_manifest_tasks(load_eval_manifest(manifest_path))
74
+
75
+ assert [task.name for task in tasks] == ["source-task"]
76
+ assert tasks[0].render_prompt() == "Source prompt\n"