@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -81,8 +81,6 @@ def test_unit_test_detection_keeps_task_runs_as_experiments():
81
81
  assert conftest._is_unit_tests_only(Config()) is False
82
82
 
83
83
 
84
-
85
-
86
84
  def test_experiment_id_uses_explicit_comet_eval_run_id(monkeypatch):
87
85
  monkeypatch.setenv("COMET_EVAL_EXPERIMENT_ID", "comet-eval-1234")
88
86
 
@@ -97,6 +95,125 @@ def test_experiment_id_rejects_unsafe_explicit_value(monkeypatch):
97
95
  conftest._get_or_create_experiment_id("ignored", False)
98
96
 
99
97
 
98
+ def test_auto_generation_loads_environment_before_resolving_cache_identity(
99
+ tmp_path: Path, monkeypatch
100
+ ):
101
+ skill = tmp_path / "skill"
102
+ skill.mkdir()
103
+ (skill / "SKILL.md").write_text("Evaluate me.\n", encoding="utf-8")
104
+ captured = {}
105
+
106
+ def load_environment():
107
+ monkeypatch.setenv("BENCH_CODEX_MODEL", "model-from-dotenv")
108
+
109
+ def generate_manifest(_skill_path, _project_root, **kwargs):
110
+ captured.update(kwargs)
111
+ manifest_path = tmp_path / "generated.yaml"
112
+ manifest_path.write_text(
113
+ """apiVersion: comet.eval/v1alpha1
114
+ kind: SkillEvalManifest
115
+ metadata: {name: generated}
116
+ skill: {name: generated, source: .}
117
+ evaluation:
118
+ tasks:
119
+ - name: generated-one
120
+ prompt: one
121
+ expect: {files: [one.md]}
122
+ - name: generated-two
123
+ prompt: two
124
+ expect: {files: [two.md]}
125
+ """,
126
+ encoding="utf-8",
127
+ )
128
+ return SimpleNamespace(manifest_path=manifest_path)
129
+
130
+ options = {
131
+ "--task": None,
132
+ "--quick": False,
133
+ "--eval-manifest": None,
134
+ "--skill-path": str(skill),
135
+ "--profile": None,
136
+ "--project-root": str(tmp_path),
137
+ "--agent": "codex",
138
+ "--skill-name": None,
139
+ }
140
+
141
+ class Config:
142
+ option = SimpleNamespace(collectonly=False, eval_manifest=None)
143
+
144
+ @staticmethod
145
+ def getoption(name):
146
+ return options[name]
147
+
148
+ monkeypatch.setattr("scaffold.python.utils.load_eval_environment", load_environment)
149
+ monkeypatch.setattr(conftest, "ensure_generated_manifest", generate_manifest)
150
+
151
+ config = Config()
152
+ conftest._ensure_auto_generated_manifest(config)
153
+
154
+ assert captured["agent"] == "codex"
155
+ assert captured["model"] == "model-from-dotenv"
156
+ assert config.option.eval_manifest is None
157
+ assert config._comet_frozen_task_set.source == "generated-cache"
158
+
159
+
160
+ def test_taskless_collect_only_marks_generation_pending_without_agent_or_disk_work(
161
+ tmp_path: Path, monkeypatch
162
+ ):
163
+ skill = tmp_path / "skill"
164
+ skill.mkdir()
165
+ (skill / "SKILL.md").write_text("Evaluate me.\n", encoding="utf-8")
166
+ called = False
167
+
168
+ def fail_generation(*_args, **_kwargs):
169
+ nonlocal called
170
+ called = True
171
+ raise AssertionError("task generation must not run during collect-only")
172
+
173
+ class Config:
174
+ option = SimpleNamespace(collectonly=True, eval_manifest=None, skill_name=None)
175
+
176
+ @staticmethod
177
+ def getoption(name):
178
+ return {
179
+ "--task": None,
180
+ "--quick": False,
181
+ "--eval-manifest": None,
182
+ "--skill-path": str(skill),
183
+ "--profile": None,
184
+ "--project-root": str(tmp_path),
185
+ "--agent": None,
186
+ "--skill-name": None,
187
+ }[name]
188
+
189
+ monkeypatch.setattr(conftest, "ensure_generated_manifest", fail_generation)
190
+ config = Config()
191
+ conftest._ensure_auto_generated_manifest(config)
192
+
193
+ assert called is False
194
+ assert config._comet_frozen_task_set.source == "pending-generation"
195
+ assert config._comet_frozen_task_set.tasks == ()
196
+ assert config._comet_resolution_manifest.skill_path == skill.resolve()
197
+ assert config._comet_resolution_manifest.path == skill / "comet" / "eval.yaml"
198
+ assert config._comet_resolved_tasks == {}
199
+
200
+ from local.tests.tasks import test_tasks
201
+
202
+ monkeypatch.setattr(test_tasks, "load_treatments", lambda: {})
203
+ monkeypatch.setattr(test_tasks, "list_tasks", lambda: [])
204
+ frozen = config._comet_frozen_task_set
205
+ assert test_tasks.generate_test_params(None, None, config) == []
206
+ assert test_tasks.generate_test_params(None, None, config) == []
207
+ assert config._comet_frozen_task_set is frozen
208
+
209
+
210
+ def test_selected_agent_missing_credentials_fails_before_workloads():
211
+ with pytest.raises(pytest.UsageError, match="Credentials for codebuddy not set"):
212
+ conftest._require_agent_credentials("codebuddy", {})
213
+
214
+ conftest._require_agent_credentials("codebuddy", {"CODEBUDDY_API_KEY": "configured"})
215
+
216
+
100
217
  def test_extract_loop_turns_reads_driver_completion_line():
101
218
  stderr = (
102
219
  "[loop] turn 1/4\n"
@@ -113,6 +230,21 @@ def test_extract_loop_turns_reads_driver_completion_line():
113
230
  "deterministic_replies": 1,
114
231
  "completion_signals": 1,
115
232
  "fresh_resume_boundaries": 0,
233
+ "role_sessions": {"subject": [], "simulator": [], "judge": []},
234
+ }
235
+
236
+
237
+ def test_extract_loop_interaction_records_isolated_role_sessions():
238
+ interaction = conftest._extract_loop_interaction(
239
+ "[loop] role-session subject subject-1\n"
240
+ "[loop] role-session simulator simulator-1\n"
241
+ "[loop] role-session subject subject-1\n"
242
+ )
243
+
244
+ assert interaction["role_sessions"] == {
245
+ "subject": ["subject-1"],
246
+ "simulator": ["simulator-1"],
247
+ "judge": [],
116
248
  }
117
249
  assert conftest._extract_loop_turns("ordinary stderr") is None
118
250
 
@@ -383,6 +515,7 @@ def test_dynamic_treatment_config_from_skill_path(tmp_path: Path):
383
515
  "--skill-path": str(skill_dir),
384
516
  "--skill-name": "demo-skill",
385
517
  "--profile": "generic",
518
+ "--quick": True,
386
519
  }
387
520
  return values.get(name)
388
521
 
@@ -552,6 +685,31 @@ def test_setup_test_context_copies_full_skill_package_and_configures_040_hook(
552
685
  ]
553
686
 
554
687
 
688
+ def test_setup_test_context_excludes_comet_runtime_state_from_skill_copy(
689
+ tmp_path: Path, setup_test_context
690
+ ):
691
+ source_dir = tmp_path / "comet-source"
692
+ source_dir.mkdir()
693
+ (source_dir / "SKILL.md").write_text("---\nname: comet\n---\n\nOriginal.", encoding="utf-8")
694
+ runtime_state = source_dir / ".comet" / "eval" / "cache" / "uv"
695
+ runtime_state.mkdir(parents=True)
696
+ (runtime_state / "long-wheel.lock").write_text("cached", encoding="utf-8")
697
+ (source_dir / ".comet" / "eval" / "runs").mkdir(parents=True)
698
+
699
+ setup_test_context(
700
+ skills={
701
+ "comet": {
702
+ "sections": ["---\nname: comet\n---\n\nGenerated."],
703
+ "source_dir": source_dir,
704
+ }
705
+ }
706
+ )
707
+
708
+ installed = tmp_path / ".claude" / "skills" / "comet"
709
+ assert (installed / "SKILL.md").exists()
710
+ assert not (installed / ".comet").exists()
711
+
712
+
555
713
  def test_setup_test_context_configures_039_shell_hook(tmp_path: Path, setup_test_context):
556
714
  source_dir = tmp_path / "comet-source"
557
715
  source_dir.mkdir()
@@ -577,6 +735,49 @@ def test_setup_test_context_configures_039_shell_hook(tmp_path: Path, setup_test
577
735
  assert command == "bash /workspace/.claude/skills/comet/scripts/comet-hook-guard.sh"
578
736
 
579
737
 
738
+ def test_codex_guard_uses_codex_hook_config_while_scripts_remain_under_agents(tmp_path: Path):
739
+ command = "node /workspace/.agents/skills/comet/scripts/comet-hook-guard.mjs"
740
+
741
+ conftest._ensure_agent_pre_tool_hook(tmp_path, "codex", command)
742
+
743
+ hooks_path = tmp_path / ".codex" / "hooks.json"
744
+ hooks = json.loads(hooks_path.read_text(encoding="utf-8"))
745
+ assert hooks["hooks"]["PreToolUse"][0]["hooks"][0]["command"] == command
746
+ assert not (tmp_path / ".agents" / "hooks.json").exists()
747
+
748
+
749
+ def test_transcript_setup_failure_is_best_effort(tmp_path: Path, monkeypatch, capsys):
750
+ monkeypatch.setenv("TRACE_TO_LANGFUSE", "true")
751
+ monkeypatch.delenv("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED", raising=False)
752
+ monkeypatch.setattr(
753
+ conftest,
754
+ "install_transcript_hook",
755
+ lambda *_args: (_ for _ in ()).throw(ValueError("invalid settings")),
756
+ )
757
+
758
+ conftest._ensure_langfuse_trajectory_support(tmp_path, "codebuddy")
759
+
760
+ assert "optional codebuddy transcript setup skipped" in capsys.readouterr().err
761
+
762
+
763
+ def test_setup_test_context_writes_codebuddy_project_memory(
764
+ tmp_path: Path,
765
+ setup_test_context,
766
+ monkeypatch,
767
+ ):
768
+ monkeypatch.setattr(
769
+ conftest,
770
+ "_resolve_eval_agent",
771
+ lambda _config: SimpleNamespace(agent="codebuddy"),
772
+ )
773
+
774
+ setup_test_context(claude_md="Use the CodeBuddy workflow.")
775
+
776
+ memory = tmp_path / ".codebuddy" / "CODEBUDDY.md"
777
+ assert memory.read_text(encoding="utf-8") == "Use the CodeBuddy workflow."
778
+ assert not (tmp_path / "AGENTS.md").exists()
779
+
780
+
580
781
  def test_setup_test_context_copies_dependency_skill_packages_with_scripts(
581
782
  tmp_path: Path, setup_test_context
582
783
  ):
@@ -711,6 +912,38 @@ def test_snapshot_dynamic_skill_package_copies_package_and_node_skills(tmp_path:
711
912
  assert (test_dir / "_eval_target_skills" / "manifest-skill-open" / "SKILL.md").exists()
712
913
 
713
914
 
915
+ def test_snapshot_dynamic_skill_package_excludes_comet_runtime_state(tmp_path: Path):
916
+ source_root = tmp_path / "source"
917
+ package = source_root / "manifest-skill"
918
+ package.mkdir(parents=True)
919
+ (package / "SKILL.md").write_text("---\nname: manifest-skill\n---\n\nBody.", encoding="utf-8")
920
+ package_cache = package / ".comet" / "eval" / "cache" / "uv"
921
+ package_cache.mkdir(parents=True)
922
+ (package_cache / "long-wheel.lock").write_text("cached", encoding="utf-8")
923
+ stage = source_root / "manifest-skill-open"
924
+ stage.mkdir()
925
+ (stage / "SKILL.md").write_text(
926
+ "---\nname: manifest-skill-open\n---\n\nStage.", encoding="utf-8"
927
+ )
928
+ (stage / ".comet" / "eval" / "runs").mkdir(parents=True)
929
+ test_dir = tmp_path / "workspace"
930
+ test_dir.mkdir()
931
+
932
+ relative_package = conftest._snapshot_dynamic_skill_package(
933
+ test_dir,
934
+ {
935
+ "path": str(package),
936
+ "generated_node_skills": ["manifest-skill-open"],
937
+ },
938
+ )
939
+
940
+ assert (test_dir / relative_package / "SKILL.md").exists()
941
+ assert not (test_dir / relative_package / ".comet").exists()
942
+ node_snapshot = test_dir / "_eval_target_skills" / "manifest-skill-open"
943
+ assert (node_snapshot / "SKILL.md").exists()
944
+ assert not (node_snapshot / ".comet").exists()
945
+
946
+
714
947
  def _workflow_overlay_validator():
715
948
  validator_path = (
716
949
  Path(__file__).parents[2]
@@ -879,6 +1112,11 @@ def test_build_report_payload_persists_sample_quality():
879
1112
  "total_tokens": 100,
880
1113
  "total_cost_usd": 0.01,
881
1114
  "case_manifest": {"case_hash": "sha256:demo"},
1115
+ "role_models": {
1116
+ "subject": "subject-model",
1117
+ "simulator": None,
1118
+ "judge": "judge-model",
1119
+ },
882
1120
  },
883
1121
  passed=["[RUBRIC] weighted_score: 1.00"],
884
1122
  failed=[],
@@ -894,6 +1132,11 @@ def test_build_report_payload_persists_sample_quality():
894
1132
  assert report["sample_quality"]["reason_code"] == "valid_signal"
895
1133
  assert report["sample_quality"]["include_in_analysis"] is True
896
1134
  assert report["events_summary"]["case_manifest"] == {"case_hash": "sha256:demo"}
1135
+ assert report["events_summary"]["role_models"] == {
1136
+ "subject": "subject-model",
1137
+ "simulator": None,
1138
+ "judge": "judge-model",
1139
+ }
897
1140
 
898
1141
 
899
1142
  def test_build_report_payload_marks_timeout_as_excluded():
@@ -941,36 +1184,60 @@ def test_prebuild_docker_image_allows_a_cold_image_build_to_run_for_fifteen_minu
941
1184
  ]
942
1185
 
943
1186
 
944
- def test_docker_prebuild_uses_only_the_tasks_selected_by_an_eval_manifest(
945
- tmp_path: Path,
946
- ):
947
- tasks_dir = tmp_path / "tasks"
948
- for task_name in ("authoring-skill-smoke", "workflow-route-conformance", "unrelated"):
949
- environment = tasks_dir / task_name / "environment"
950
- environment.mkdir(parents=True)
951
- (environment / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
952
-
1187
+ def test_resolve_eval_agent_applies_cli_manifest_precedence(tmp_path: Path):
953
1188
  manifest = tmp_path / "eval.yaml"
954
1189
  manifest.write_text(
955
1190
  """
956
1191
  apiVersion: comet.eval/v1alpha1
957
1192
  kind: SkillEvalManifest
958
1193
  metadata:
959
- name: selected-tasks
1194
+ name: selected-agent
960
1195
  skill:
961
- name: selected-tasks
1196
+ name: selected-agent
962
1197
  source: .
963
- evaluation:
964
- recommendedTasks:
965
- - authoring-skill-smoke
966
- - workflow-route-conformance
1198
+ execution:
1199
+ agent: codex
967
1200
  """,
968
1201
  encoding="utf-8",
969
1202
  )
970
1203
 
971
1204
  class Config:
1205
+ def __init__(self, agent):
1206
+ self.agent = agent
1207
+
972
1208
  def getoption(self, name):
973
- return {"--task": None, "--eval-manifest": str(manifest)}[name]
1209
+ return {
1210
+ "--eval-manifest": str(manifest),
1211
+ "--agent": self.agent,
1212
+ }.get(name)
1213
+
1214
+ assert conftest._resolve_eval_agent(Config(None)).agent == "codex"
1215
+ assert conftest._resolve_eval_agent(Config("qoder")).agent == "qoder"
1216
+
1217
+
1218
+ def test_selected_agent_uses_its_native_project_skill_root():
1219
+ assert conftest._agent_project_root("claude-code") == ".claude"
1220
+ assert conftest._agent_project_root("codex") == ".agents"
1221
+ assert conftest._agent_project_root("qoder") == ".qoder"
1222
+ assert conftest._agent_project_root("codebuddy") == ".codebuddy"
1223
+
1224
+
1225
+ def test_docker_prebuild_uses_only_the_tasks_selected_by_an_eval_manifest(
1226
+ tmp_path: Path,
1227
+ ):
1228
+ tasks_dir = tmp_path / "tasks"
1229
+ for task_name in ("authoring-skill-smoke", "workflow-route-conformance", "unrelated"):
1230
+ environment = tasks_dir / task_name / "environment"
1231
+ environment.mkdir(parents=True)
1232
+ (environment / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
1233
+
1234
+ class Config:
1235
+ _comet_frozen_task_set = SimpleNamespace(
1236
+ tasks=(
1237
+ SimpleNamespace(task=SimpleNamespace(environment_dir=tasks_dir / "authoring-skill-smoke" / "environment")),
1238
+ SimpleNamespace(task=SimpleNamespace(environment_dir=tasks_dir / "workflow-route-conformance" / "environment")),
1239
+ )
1240
+ )
974
1241
 
975
1242
  request = SimpleNamespace(config=Config())
976
1243
 
@@ -978,3 +1245,42 @@ evaluation:
978
1245
  tasks_dir / "authoring-skill-smoke" / "environment",
979
1246
  tasks_dir / "workflow-route-conformance" / "environment",
980
1247
  ]
1248
+
1249
+
1250
+ @pytest.mark.parametrize(
1251
+ "source,selected",
1252
+ [
1253
+ ("explicit", ["authoring-skill-smoke"]),
1254
+ ("quick", ["workflow-route-conformance"]),
1255
+ ("authored", ["authoring-skill-smoke"]),
1256
+ ("recommended", ["workflow-route-conformance"]),
1257
+ ("generated-cache", ["authoring-skill-smoke"]),
1258
+ ("source", ["source-custom"]),
1259
+ ],
1260
+ )
1261
+ def test_docker_prebuild_consumes_only_frozen_task_environments(tmp_path: Path, source, selected):
1262
+ tasks_dir = tmp_path / "tasks"
1263
+ environments = {}
1264
+ for name in ("authoring-skill-smoke", "workflow-route-conformance", "source-custom", "unrelated"):
1265
+ environment = tasks_dir / name / "environment"
1266
+ environment.mkdir(parents=True)
1267
+ (environment / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
1268
+ environments[name] = environment
1269
+
1270
+ frozen = SimpleNamespace(
1271
+ source=source,
1272
+ generation=SimpleNamespace(generation_hash="frozen-hash") if source == "generated-cache" else None,
1273
+ tasks=tuple(
1274
+ SimpleNamespace(
1275
+ name=name,
1276
+ provenance="source" if name == "source-custom" else source,
1277
+ task=SimpleNamespace(environment_dir=environments[name]),
1278
+ )
1279
+ for name in [*selected, *selected]
1280
+ ),
1281
+ )
1282
+ request = SimpleNamespace(config=SimpleNamespace(_comet_frozen_task_set=frozen))
1283
+
1284
+ assert conftest._docker_environment_dirs_for_request(request, tasks_dir) == [
1285
+ environments[name] for name in selected
1286
+ ]
@@ -0,0 +1,174 @@
1
+ """Contract tests for explicitly installed custom Eval Agent adapters."""
2
+
3
+ from pathlib import Path
4
+
5
+ import pytest
6
+
7
+ from scaffold.python.agents import (
8
+ get_agent_adapter,
9
+ normalize_skill_invocations,
10
+ validate_agent_capabilities,
11
+ )
12
+ from scaffold.python.custom_agents import discover_custom_agent_specs
13
+
14
+
15
+ def _write_adapter(root: Path, *, name: str = "fixture-agent", **overrides) -> Path:
16
+ directory = root / name
17
+ directory.mkdir(parents=True)
18
+ capabilities = overrides.pop(
19
+ "capabilities",
20
+ {
21
+ "singleTurn": True,
22
+ "resume": True,
23
+ "structuredEvents": True,
24
+ "telemetry": False,
25
+ "skillInvocationEvidence": True,
26
+ },
27
+ )
28
+ runtime = overrides.pop(
29
+ "runtime",
30
+ {"executable": "fixture-agent", "install": {"kind": "none"}},
31
+ )
32
+ credentials = overrides.pop("credentials", ["FIXTURE_AGENT_API_KEY"])
33
+ lines = [
34
+ "apiVersion: comet.eval.agent/v1alpha1",
35
+ "kind: EvalAgentAdapter",
36
+ "metadata:",
37
+ f" id: {name}",
38
+ " version: '1.0.0'",
39
+ "runtime:",
40
+ f" executable: {runtime['executable']}",
41
+ " install:",
42
+ f" kind: {runtime['install']['kind']}",
43
+ "credentials:",
44
+ *[f" - {item}" for item in credentials],
45
+ "modelEnv: FIXTURE_AGENT_MODEL",
46
+ "baseUrlEnv: FIXTURE_AGENT_BASE_URL",
47
+ "capabilities:",
48
+ *[f" {key}: {'true' if value else 'false'}" for key, value in capabilities.items()],
49
+ ]
50
+ for key, value in overrides.items():
51
+ lines.append(f"{key}: {value}")
52
+ path = directory / "adapter.yaml"
53
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
54
+ return path
55
+
56
+
57
+ def test_tracked_fixture_can_be_installed_as_a_custom_adapter(
58
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
59
+ ):
60
+ fixture = Path(__file__).parent / "fixtures" / "custom-agent" / "adapter.yaml"
61
+ destination = tmp_path / "fixture-agent"
62
+ destination.mkdir()
63
+ (destination / "adapter.yaml").write_text(fixture.read_text(encoding="utf-8"), encoding="utf-8")
64
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
65
+
66
+ adapter = get_agent_adapter("fixture-agent")
67
+
68
+ assert adapter.custom is True
69
+ assert adapter.executable == "fixture-agent"
70
+
71
+
72
+ def test_installed_custom_adapter_is_discovered_and_generates_standard_command(
73
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
74
+ ):
75
+ _write_adapter(tmp_path)
76
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
77
+
78
+ specs = discover_custom_agent_specs()
79
+ adapter = get_agent_adapter("fixture-agent")
80
+
81
+ assert list(specs) == ["fixture-agent"]
82
+ assert adapter.custom is True
83
+ assert adapter.executable == "fixture-agent"
84
+ assert adapter.required_credentials == ("FIXTURE_AGENT_API_KEY",)
85
+ assert adapter.supports_telemetry is False
86
+ assert adapter.build_run_command(
87
+ "Do it.", model="fixture-model", role="subject", resume_id="session-1"
88
+ ) == [
89
+ "fixture-agent",
90
+ "-p",
91
+ "Do it.",
92
+ "--output-format",
93
+ "stream-json",
94
+ "--model",
95
+ "fixture-model",
96
+ "--resume",
97
+ "session-1",
98
+ ]
99
+
100
+
101
+ @pytest.mark.parametrize(
102
+ "capability",
103
+ ["single_turn", "resume", "structured_events", "skill_invocation_evidence"],
104
+ )
105
+ def test_missing_required_custom_capability_fails_preflight(
106
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capability: str
107
+ ):
108
+ capabilities = {
109
+ "singleTurn": True,
110
+ "resume": True,
111
+ "structuredEvents": True,
112
+ "telemetry": False,
113
+ "skillInvocationEvidence": True,
114
+ }
115
+ key = {
116
+ "single_turn": "singleTurn",
117
+ "resume": "resume",
118
+ "structured_events": "structuredEvents",
119
+ "skill_invocation_evidence": "skillInvocationEvidence",
120
+ }[capability]
121
+ capabilities[key] = False
122
+ _write_adapter(tmp_path, capabilities=capabilities)
123
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
124
+
125
+ with pytest.raises(ValueError, match=capability):
126
+ validate_agent_capabilities(get_agent_adapter("fixture-agent"))
127
+
128
+
129
+ def test_custom_registry_rejects_malformed_duplicate_and_escape_entries(
130
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
131
+ ):
132
+ _write_adapter(tmp_path, name="first")
133
+ second = _write_adapter(tmp_path, name="second")
134
+ second.write_text(second.read_text(encoding="utf-8").replace("id: second", "id: first"), encoding="utf-8")
135
+ outside = tmp_path.parent / "outside-adapter.yaml"
136
+ outside.write_text("not an adapter\n", encoding="utf-8")
137
+ escaped = tmp_path / "escaped"
138
+ try:
139
+ escaped.symlink_to(outside.parent, target_is_directory=True)
140
+ except OSError:
141
+ pass
142
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
143
+
144
+ with pytest.raises(ValueError, match="duplicate|outside|invalid|match|escapes"):
145
+ discover_custom_agent_specs()
146
+
147
+
148
+ def test_custom_adapter_does_not_infer_skill_invocation_from_artifacts(
149
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
150
+ ):
151
+ _write_adapter(tmp_path)
152
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
153
+ adapter = get_agent_adapter("fixture-agent")
154
+
155
+ assert adapter.has_observable_skill_invocation({"artifacts": ["SKILL.md"]}) is False
156
+ assert adapter.has_observable_skill_invocation({"skill_invocations": ["fixture-agent"]}) is True
157
+
158
+
159
+ def test_custom_adapter_capability_filters_non_observable_skill_evidence(
160
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
161
+ ):
162
+ capabilities = {
163
+ "singleTurn": True,
164
+ "resume": True,
165
+ "structuredEvents": True,
166
+ "telemetry": False,
167
+ "skillInvocationEvidence": False,
168
+ }
169
+ _write_adapter(tmp_path, capabilities=capabilities)
170
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
171
+ adapter = get_agent_adapter("fixture-agent")
172
+ events = {"skills_invoked": ["demo"], "skill_invocations": ["demo"]}
173
+
174
+ assert normalize_skill_invocations(events, adapter=adapter) == []