@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -3,9 +3,9 @@
3
3
  Generates rich experiment logs in logs/experiments/ including:
4
4
  - summary.md: Full markdown report with tables and details
5
5
  - events/: Parsed events from each test run
6
- - raw/: Raw Claude CLI output
6
+ - raw/: Raw selected Agent CLI output
7
7
  - reports/: Per-run validation reports
8
- - artifacts/: Files Claude generated and their execution output
8
+ - artifacts/: Files selected Agent generated and their execution output
9
9
  - metadata.json: Experiment metadata
10
10
 
11
11
  Supports pytest-xdist parallel execution via worker coordination.
@@ -27,7 +27,7 @@ from typing import Any
27
27
 
28
28
  import pytest
29
29
 
30
- from scaffold import run_claude_in_docker, run_python_in_docker, run_shell
30
+ from scaffold import run_agent_in_docker, run_python_in_docker, run_shell
31
31
  from scaffold.python import (
32
32
  ExperimentLogger,
33
33
  TreatmentResult,
@@ -40,9 +40,40 @@ from scaffold.python import (
40
40
  save_report,
41
41
  strip_ansi,
42
42
  )
43
+ from scaffold.python.agents import get_agent_adapter, resolve_agent, validate_agent_capabilities
44
+ from scaffold.python.auto_tasks import AutoTaskError, ensure_generated_manifest, find_project_root
45
+ from scaffold.python.eval_context import (
46
+ EvalContextError,
47
+ context_from_environment,
48
+ resolve_eval_context,
49
+ resolve_managed_path,
50
+ )
51
+ from scaffold.python.execution import (
52
+ ResolvedExecution,
53
+ ResolvedJudge,
54
+ build_agent_environment,
55
+ missing_credentials,
56
+ preflight_credentials,
57
+ redact_sensitive,
58
+ resolve_execution,
59
+ resolve_judge,
60
+ validate_base_url,
61
+ )
62
+ from scaffold.python.generated_task_cache import (
63
+ selected_agent_model as _legacy_selected_agent_model,
64
+ )
65
+ from scaffold.python.langfuse_adapter import (
66
+ install_codex_plugin_workspace,
67
+ install_transcript_hook,
68
+ provision_trajectory_plugin,
69
+ )
43
70
  from scaffold.python.sample_quality import infer_sample_quality
44
71
  from scaffold.python.skill_parser import SCRIPT_EXTENSIONS
45
- from scaffold.python.utils import run_claude_loop_in_docker
72
+ from scaffold.python.utils import (
73
+ check_docker_available,
74
+ run_agent_loop_in_docker,
75
+ run_claude_loop_in_docker,
76
+ )
46
77
  from scaffold.python.aligned_comparison import (
47
78
  EXPECTED_CASE_MATRIX_FILENAME,
48
79
  build_execution_identity,
@@ -56,14 +87,21 @@ def _extract_loop_turns(stderr: str | None) -> int | None:
56
87
  return int(match.group(1)) if match else None
57
88
 
58
89
 
59
- def _extract_loop_interaction(stderr: str | None) -> dict[str, int | None]:
90
+ def _extract_loop_interaction(stderr: str | None) -> dict[str, Any]:
60
91
  source = stderr or ""
92
+ role_sessions = {"subject": [], "simulator": [], "judge": []}
93
+ for role, session_id in re.findall(
94
+ r"\[loop\] role-session (subject|simulator|judge) ([^\s]+)", source
95
+ ):
96
+ if session_id not in role_sessions[role]:
97
+ role_sessions[role].append(session_id)
61
98
  return {
62
99
  "actual_turns": _extract_loop_turns(source),
63
100
  "decision_points": source.count("[loop] decision point detected"),
64
101
  "deterministic_replies": source.count("[loop] deterministic decision reply applied"),
65
102
  "completion_signals": source.count("[loop] workflow completion detected"),
66
103
  "fresh_resume_boundaries": source.count("[loop] fresh resume boundary detected"),
104
+ "role_sessions": role_sessions,
67
105
  }
68
106
 
69
107
 
@@ -212,8 +250,12 @@ EVAL_ROOT = PROJECT_ROOT.parent
212
250
  REPOSITORY_ROOT = EVAL_ROOT.parent
213
251
 
214
252
  # Shared files for xdist worker coordination
215
- XDIST_EXPERIMENT_FILE = PROJECT_ROOT / ".pytest_experiment_id"
216
- DOCKER_BUILD_LOCK = PROJECT_ROOT / ".pytest_docker_build.lock"
253
+ _RUNTIME_CONTEXT = context_from_environment()
254
+ _CONTEXT_LOCK_ROOT = (
255
+ resolve_managed_path(_RUNTIME_CONTEXT, "locks") if _RUNTIME_CONTEXT else PROJECT_ROOT
256
+ )
257
+ XDIST_EXPERIMENT_FILE = _CONTEXT_LOCK_ROOT / ".pytest_experiment_id"
258
+ DOCKER_BUILD_LOCK = _CONTEXT_LOCK_ROOT / ".pytest_docker_build.lock"
217
259
  EXPERIMENT_ID_ENV = "COMET_EVAL_EXPERIMENT_ID"
218
260
  EXPERIMENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
219
261
 
@@ -246,12 +288,305 @@ MODEL_EXECUTION_ENV_KEYS = (
246
288
  "ANTHROPIC_DEFAULT_OPUS_MODEL_NAME",
247
289
  "ANTHROPIC_DEFAULT_SONNET_MODEL_NAME",
248
290
  "CLAUDE_CODE_SUBAGENT_MODEL",
291
+ "CODEBUDDY_BASE_URL",
292
+ "CODEBUDDY_MODEL",
293
+ "CODEBUDDY_SMALL_FAST_MODEL",
294
+ "CODEBUDDY_BIG_SLOW_MODEL",
295
+ "CODEBUDDY_CODE_SUBAGENT_MODEL",
296
+ "BENCH_CODEBUDDY_MODEL",
297
+ "OPENAI_BASE_URL",
298
+ "OPENAI_MODEL",
299
+ "CODEX_BASE_URL",
300
+ "CODEX_MODEL",
301
+ "QODER_BASE_URL",
302
+ "QODER_MODEL",
249
303
  )
250
304
 
251
305
 
252
- def _selected_claude_model() -> str | None:
253
- """Resolve the exact model selector passed to Claude without persisting it."""
254
- return os.environ.get("BENCH_CC_MODEL") or os.environ.get("ANTHROPIC_MODEL")
306
+ def _getoption(config, name: str, default=None):
307
+ try:
308
+ value = config.getoption(name)
309
+ except (AttributeError, KeyError, ValueError):
310
+ return default
311
+ return default if value is None else value
312
+
313
+
314
+ def _manifest_for_config(config):
315
+ manifest_path = _getoption(config, "--eval-manifest")
316
+ if not manifest_path:
317
+ return None
318
+ from scaffold.python.manifests import load_eval_manifest
319
+
320
+ return load_eval_manifest(manifest_path)
321
+
322
+
323
+ def _resolve_eval_execution(config) -> ResolvedExecution:
324
+ manifest = _manifest_for_config(config)
325
+ manifest_execution = manifest.execution if manifest is not None else None
326
+ return resolve_execution(
327
+ cli_agent=_getoption(config, "--agent"),
328
+ cli_model=_getoption(config, "--model"),
329
+ cli_base_url=_getoption(config, "--base-url"),
330
+ manifest=manifest_execution,
331
+ )
332
+
333
+
334
+ def _resolve_eval_judge(config) -> ResolvedJudge | None:
335
+ manifest = _manifest_for_config(config)
336
+ manifest_judge = manifest.judge if manifest is not None else None
337
+ return resolve_judge(
338
+ main=_resolve_eval_execution(config),
339
+ cli_agent=_getoption(config, "--judge-agent"),
340
+ cli_model=_getoption(config, "--judge-model"),
341
+ cli_base_url=_getoption(config, "--judge-base-url"),
342
+ manifest=manifest_judge,
343
+ )
344
+
345
+
346
+ def _selected_agent_model(agent: str) -> str | None:
347
+ if _plugin is not None:
348
+ return _resolve_eval_execution(_plugin.config).model
349
+ return _legacy_selected_agent_model(agent)
350
+
351
+
352
+ def _resolve_eval_agent(config):
353
+ execution = _resolve_eval_execution(config)
354
+ return resolve_agent(execution.agent, None)
355
+
356
+
357
+ def _require_agent_credentials(agent: str, source_env: dict[str, str] | None = None) -> None:
358
+ """Fail setup before workloads when the isolated Agent cannot authenticate."""
359
+ environment = source_env if source_env is not None else os.environ
360
+ missing = missing_credentials(agent, source_env=environment)
361
+ if not missing:
362
+ return
363
+ adapter = get_agent_adapter(agent)
364
+ separator = ", " if adapter.custom else " or "
365
+ raise pytest.UsageError(
366
+ f"Credentials for {agent} not set: {separator.join(missing)}"
367
+ )
368
+
369
+
370
+ def _normal_run_preflight(config) -> None:
371
+ """Validate normal-run dependencies before collection can generate or run tasks."""
372
+
373
+ if _is_unit_tests_only(config) or getattr(config.option, "collectonly", False):
374
+ return
375
+ from scaffold.python.utils import load_eval_environment
376
+
377
+ load_eval_environment()
378
+ main = _resolve_eval_execution(config)
379
+ judge = _resolve_eval_judge(config)
380
+ suite = next(
381
+ (name for name in ("local", "langsmith", "langfuse") if f"{name}/tests/" in " ".join(config.args)),
382
+ "local",
383
+ )
384
+ required_harness = (
385
+ EVAL_ROOT / "pyproject.toml",
386
+ EVAL_ROOT / "schemas" / "comet.eval" / "v1alpha1.schema.json",
387
+ EVAL_ROOT / suite / "tests" / "tasks" / "test_tasks.py",
388
+ )
389
+ missing_harness = [str(path) for path in required_harness if not path.is_file()]
390
+ if missing_harness:
391
+ raise pytest.UsageError(f"Packaged eval harness is incomplete: {', '.join(missing_harness)}")
392
+ manifest = _manifest_for_config(config)
393
+ if manifest is not None and manifest.reporting is not None:
394
+ selected_report_suite = manifest.reporting.suite
395
+ if selected_report_suite and selected_report_suite != suite:
396
+ raise pytest.UsageError(
397
+ f"reporting.suite is {selected_report_suite!r}, but the selected suite is {suite!r}"
398
+ )
399
+ report_config = manifest.reporting.config
400
+ if report_config and not _getoption(config, "--report-config"):
401
+ report_path = Path(report_config)
402
+ if not report_path.is_absolute():
403
+ report_path = manifest.path.parent / report_path
404
+ config.option.report_config = str(report_path.resolve())
405
+ try:
406
+ load_report_output_config(_getoption(config, "--report-config"))
407
+ except (OSError, ValueError) as exc:
408
+ raise pytest.UsageError(f"Eval report configuration is invalid: {exc}") from exc
409
+ try:
410
+ validate_agent_capabilities(get_agent_adapter(main.agent))
411
+ if judge is not None:
412
+ validate_agent_capabilities(get_agent_adapter(judge.agent))
413
+ except ValueError as exc:
414
+ raise pytest.UsageError(str(exc)) from exc
415
+ if main.base_url:
416
+ validate_base_url(main.base_url, field="execution.baseUrl")
417
+ if judge is not None and judge.base_url:
418
+ validate_base_url(judge.base_url, field="judge.baseUrl")
419
+ credential_errors = preflight_credentials(main, judge)
420
+ if credential_errors:
421
+ raise pytest.UsageError("; ".join(credential_errors))
422
+ if shutil.which("uv") is None:
423
+ raise pytest.UsageError("uv is not installed or not in PATH")
424
+ if not check_docker_available():
425
+ raise pytest.UsageError("Docker CLI or daemon is unavailable")
426
+ context = getattr(config, "_comet_eval_context", None)
427
+ if context is None:
428
+ raise pytest.UsageError("Eval context is missing before normal-run preflight")
429
+ artifact_root = Path(context.artifact_root)
430
+ artifact_root.mkdir(parents=True, exist_ok=True)
431
+ try:
432
+ with tempfile.NamedTemporaryFile(dir=artifact_root, prefix=".preflight-", delete=True):
433
+ pass
434
+ except OSError as exc:
435
+ raise pytest.UsageError(f"Eval artifact root is not writable: {artifact_root}") from exc
436
+ config._comet_execution_env = build_agent_environment(main)
437
+ config._comet_execution = main
438
+ config._comet_judge = judge
439
+ os.environ["COMET_EVAL_MAIN_CREDENTIALS"] = ",".join(
440
+ get_agent_adapter(main.agent).required_credentials
441
+ )
442
+ if judge is not None:
443
+ os.environ["BENCH_LLM_JUDGE"] = "1"
444
+ os.environ["BENCH_JUDGE_MODEL"] = judge.model
445
+ os.environ["BENCH_JUDGE_AGENT"] = judge.agent
446
+ if judge.base_url:
447
+ os.environ["BENCH_JUDGE_BASE_URL"] = judge.base_url
448
+ else:
449
+ os.environ.pop("BENCH_JUDGE_BASE_URL", None)
450
+
451
+
452
+ def _agent_project_root(agent: str) -> str:
453
+ return {
454
+ "claude-code": ".claude",
455
+ "codex": ".agents",
456
+ "qoder": ".qoder",
457
+ "codebuddy": ".codebuddy",
458
+ }.get(agent, ".comet-agent")
459
+
460
+
461
+ def _ensure_langfuse_trajectory_support(test_dir: Path, agent: str) -> None:
462
+ """Install only isolated, project-local trajectory support for Langfuse runs."""
463
+ if os.environ.get("TRACE_TO_LANGFUSE", "").lower() != "true":
464
+ return
465
+ if agent in {"qoder", "codebuddy"}:
466
+ try:
467
+ install_transcript_hook(test_dir, agent)
468
+ except Exception as exc:
469
+ if os.environ.get("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED") != "true":
470
+ print(
471
+ f"[langfuse] optional {agent} transcript setup skipped: {exc}",
472
+ file=os.sys.stderr,
473
+ )
474
+ os.environ["LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED"] = "true"
475
+ return
476
+ if agent != "codex":
477
+ return
478
+ cache_root = os.environ.get("LANGFUSE_TRAJECTORY_CACHE_DIR")
479
+ if not cache_root:
480
+ return
481
+ try:
482
+ provision = provision_trajectory_plugin(Path(cache_root), agent)
483
+ install_codex_plugin_workspace(test_dir, provision)
484
+ except Exception as exc:
485
+ if os.environ.get("LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED") != "true":
486
+ print(f"[langfuse] optional Codex trajectory setup skipped: {exc}", file=os.sys.stderr)
487
+ os.environ["LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED"] = "true"
488
+
489
+
490
+ def _ensure_auto_generated_manifest(config, task_filter: str | None = None) -> None:
491
+ """Freeze generated descriptors once while retaining the original manifest authority."""
492
+ if getattr(config, "_comet_frozen_task_set", None) is not None:
493
+ return
494
+ if config.getoption("--task") or task_filter or config.getoption("--quick"):
495
+ return
496
+
497
+ manifest_path = config.getoption("--eval-manifest")
498
+ skill_path = config.getoption("--skill-path")
499
+ manifest = None
500
+ if manifest_path:
501
+ from scaffold.python.manifests import load_eval_manifest
502
+
503
+ manifest = load_eval_manifest(manifest_path)
504
+ if manifest.tasks or manifest.recommended_tasks:
505
+ return
506
+ skill_path = str(manifest.skill_path)
507
+ if not skill_path:
508
+ return
509
+
510
+ if getattr(getattr(config, "option", None), "collectonly", False):
511
+ from scaffold.python.manifests import SkillEvalManifest
512
+ from scaffold.python.task_resolution import ResolvedTaskSet
513
+
514
+ if manifest is None:
515
+ resolved_skill_root = Path(skill_path).expanduser().resolve()
516
+ if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
517
+ resolved_skill_root = resolved_skill_root.parent
518
+ manifest = SkillEvalManifest(
519
+ path=resolved_skill_root / "comet" / "eval.yaml",
520
+ name=resolved_skill_root.name,
521
+ description="",
522
+ skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
523
+ skill_path=resolved_skill_root,
524
+ profile=config.getoption("--profile"),
525
+ )
526
+ config._comet_resolution_manifest = manifest
527
+ config._comet_resolved_tasks = {}
528
+ config._comet_frozen_task_set = ResolvedTaskSet("pending-generation", (), None)
529
+ return
530
+
531
+ # Task generation happens during collection, before session fixtures run.
532
+ # Load the documented local environment now so credentials and the model
533
+ # participating in the cache key match the later execution environment.
534
+ from scaffold.python.utils import load_eval_environment
535
+
536
+ load_eval_environment()
537
+ execution = _resolve_eval_execution(config)
538
+ profile = config.getoption("--profile") or (manifest.profile if manifest else None) or "generic"
539
+ interaction = vars(manifest.interaction) if manifest else {"mode": "none", "max_turns": 12}
540
+ context = getattr(config, "_comet_eval_context", None)
541
+ project_root = (
542
+ context.artifact_owner_root
543
+ if context is not None
544
+ else config.getoption("--project-root") or find_project_root(skill_path)
545
+ )
546
+ try:
547
+ generated = ensure_generated_manifest(
548
+ skill_path,
549
+ project_root,
550
+ agent=execution.agent,
551
+ model=execution.model,
552
+ base_url=execution.base_url,
553
+ environment=build_agent_environment(execution),
554
+ profile=profile,
555
+ interaction=interaction,
556
+ collect_only=bool(getattr(getattr(config, "option", None), "collectonly", False)),
557
+ )
558
+ except AutoTaskError as exc:
559
+ raise pytest.UsageError(str(exc)) from exc
560
+ from scaffold.python.manifest_tasks import load_manifest_tasks
561
+ from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
562
+ from scaffold.python.task_resolution import ResolvedTask, ResolvedTaskSet
563
+
564
+ generated_manifest = load_eval_manifest(generated.manifest_path)
565
+ generated_tasks = tuple(
566
+ ResolvedTask(task.name, "generated", task)
567
+ for task in load_manifest_tasks(generated_manifest)
568
+ )
569
+ if manifest is None:
570
+ resolved_skill_root = Path(skill_path).expanduser().resolve()
571
+ if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
572
+ resolved_skill_root = resolved_skill_root.parent
573
+ manifest = SkillEvalManifest(
574
+ path=resolved_skill_root / "comet" / "eval.yaml",
575
+ name=resolved_skill_root.name,
576
+ description="",
577
+ skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
578
+ skill_path=resolved_skill_root,
579
+ profile=config.getoption("--profile"),
580
+ )
581
+ frozen = ResolvedTaskSet(
582
+ "generated-cache",
583
+ generated_tasks,
584
+ None,
585
+ )
586
+ config._comet_resolution_manifest = manifest
587
+ config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
588
+ config._comet_frozen_task_set = frozen
589
+ config._comet_generated_manifest = generated
255
590
 
256
591
 
257
592
  def _model_execution_config() -> dict[str, str | None]:
@@ -265,11 +600,13 @@ def _capture_execution_identity(test_dir: Path, *, model: str | None, interactio
265
600
  "docker.sh",
266
601
  "execution-identity",
267
602
  str(test_dir),
603
+ "--agent",
604
+ _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code",
268
605
  timeout=300,
269
606
  check=False,
270
607
  )
271
608
  if result.returncode != 0:
272
- raise RuntimeError("Could not verify eval Docker/Claude execution identity")
609
+ raise RuntimeError("Could not verify eval Docker/agent execution identity")
273
610
  try:
274
611
  raw = json.loads(result.stdout)
275
612
  except (TypeError, json.JSONDecodeError) as error:
@@ -532,6 +869,54 @@ def pytest_addoption(parser):
532
869
  default=None,
533
870
  help="Path to comet/eval.yaml",
534
871
  )
872
+ parser.addoption(
873
+ "--agent",
874
+ action="store",
875
+ default=None,
876
+ help="Evaluation agent CLI (built-in or explicitly installed custom adapter)",
877
+ )
878
+ parser.addoption(
879
+ "--model",
880
+ action="store",
881
+ default=None,
882
+ help="Main evaluation model override",
883
+ )
884
+ parser.addoption(
885
+ "--base-url",
886
+ action="store",
887
+ default=None,
888
+ help="Main evaluation API base URL override",
889
+ )
890
+ parser.addoption(
891
+ "--judge-agent",
892
+ action="store",
893
+ default=None,
894
+ help="Independent Judge agent (built-in or explicitly installed custom adapter)",
895
+ )
896
+ parser.addoption(
897
+ "--judge-model",
898
+ action="store",
899
+ default=None,
900
+ help="Independent Judge model",
901
+ )
902
+ parser.addoption(
903
+ "--judge-base-url",
904
+ action="store",
905
+ default=None,
906
+ help="Independent Judge API base URL",
907
+ )
908
+ parser.addoption(
909
+ "--quick",
910
+ action="store_true",
911
+ default=False,
912
+ help="Use generic-skill-smoke and bypass automatic task generation",
913
+ )
914
+ parser.addoption(
915
+ "--project-root",
916
+ action="store",
917
+ default=None,
918
+ help="Project root used for generated eval cache",
919
+ )
535
920
  parser.addoption(
536
921
  "--interaction-mode",
537
922
  action="store",
@@ -564,6 +949,27 @@ def pytest_configure(config):
564
949
  "markers",
565
950
  "eval_case(repetition): controller-owned task/treatment/repetition identity",
566
951
  )
952
+ try:
953
+ context = context_from_environment()
954
+ if context is None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
955
+ context = resolve_eval_context(
956
+ skill_path=config.getoption("--skill-path"),
957
+ manifest_path=config.getoption("--eval-manifest"),
958
+ project_root=config.getoption("--project-root"),
959
+ )
960
+ except EvalContextError as exc:
961
+ raise pytest.UsageError(str(exc)) from exc
962
+ if context is not None:
963
+ config._comet_eval_context = context
964
+ config.option.project_root = str(context.artifact_owner_root)
965
+ if context.manifest_path is not None:
966
+ config.option.eval_manifest = str(context.manifest_path)
967
+ else:
968
+ config.option.skill_path = str(context.skill_root)
969
+ config._comet_execution = _resolve_eval_execution(config)
970
+ config._comet_judge = _resolve_eval_judge(config)
971
+ _normal_run_preflight(config)
972
+ _resolve_eval_agent(config)
567
973
  global _plugin
568
974
  _plugin = ExperimentPlugin(config)
569
975
  config.pluginmanager.register(_plugin, "experiment_plugin")
@@ -772,12 +1178,40 @@ def _get_experiment_name(session) -> str:
772
1178
 
773
1179
 
774
1180
  def _get_dynamic_treatment_config(config):
1181
+ _ensure_auto_generated_manifest(config)
1182
+
1183
+ def generation_hints(manifest):
1184
+ generation_metadata = {}
1185
+ generation_metadata_hash = None
1186
+ generated = getattr(config, "_comet_generated_manifest", None)
1187
+ generation_metadata_path = (
1188
+ generated.metadata_path if generated is not None else manifest.generation_metadata_path
1189
+ )
1190
+ if generation_metadata_path and generation_metadata_path.is_file():
1191
+ generation_metadata_bytes = generation_metadata_path.read_bytes()
1192
+ generation_metadata = json.loads(generation_metadata_bytes)
1193
+ generation_metadata_hash = (
1194
+ "sha256:" + hashlib.sha256(generation_metadata_bytes).hexdigest()
1195
+ )
1196
+ return {
1197
+ "generation_hash": (
1198
+ generated.generation_hash if generated is not None else manifest.generation_hash
1199
+ ),
1200
+ "generation_metadata_path": (
1201
+ str(generation_metadata_path) if generation_metadata_path else None
1202
+ ),
1203
+ "generation_manifest_hash": generation_metadata.get("manifest_hash"),
1204
+ "generation_metadata_hash": generation_metadata_hash,
1205
+ "generation_overhead": generation_metadata.get("generation_overhead"),
1206
+ }
1207
+
775
1208
  manifest_path = config.getoption("--eval-manifest")
776
1209
  if manifest_path:
777
1210
  from scaffold.python.manifests import load_eval_manifest
778
1211
  from scaffold.python.treatments import TreatmentConfig
779
1212
 
780
1213
  manifest = load_eval_manifest(manifest_path)
1214
+ generation = generation_hints(manifest)
781
1215
  node_skills = []
782
1216
  for node_skill in manifest.generated_node_skills:
783
1217
  node_path = manifest.skill_path.parent / node_skill
@@ -799,11 +1233,13 @@ def _get_dynamic_treatment_config(config):
799
1233
  "path": str(manifest.skill_path),
800
1234
  "profile": manifest.profile,
801
1235
  "manifest": str(manifest.path),
1236
+ "execution_agent": manifest.execution_agent,
802
1237
  "baseline_treatments": manifest.baseline_treatments,
803
1238
  "quality_gates": manifest.quality_gates,
804
1239
  "required_output_schemas": manifest.required_output_schemas,
805
1240
  "expected_evidence": manifest.expected_evidence,
806
1241
  "draft_hash": manifest.draft_hash,
1242
+ **generation,
807
1243
  "required_skills": manifest.required_skills,
808
1244
  "expected_artifacts": manifest.expected_artifacts,
809
1245
  "generated_node_skills": manifest.generated_node_skills,
@@ -820,7 +1256,8 @@ def _get_dynamic_treatment_config(config):
820
1256
  if not skill_path:
821
1257
  return None
822
1258
  skill_name = config.getoption("--skill-name") or Path(skill_path).resolve().parent.name
823
- profile = config.getoption("--profile")
1259
+ manifest = getattr(config, "_comet_resolution_manifest", None)
1260
+ profile = config.getoption("--profile") or (manifest.profile if manifest else None)
824
1261
  skill_cfg = {
825
1262
  "name": skill_name,
826
1263
  "source": "path",
@@ -828,6 +1265,9 @@ def _get_dynamic_treatment_config(config):
828
1265
  }
829
1266
  if profile:
830
1267
  skill_cfg["profile"] = profile
1268
+ if manifest:
1269
+ skill_cfg["manifest"] = str(manifest.path)
1270
+ skill_cfg.update(generation_hints(manifest))
831
1271
  from scaffold.python.treatments import TreatmentConfig
832
1272
 
833
1273
  return TreatmentConfig(
@@ -899,14 +1339,15 @@ def _build_eval_claude_md(profile_name: str, treatment_claude_md: str | None = N
899
1339
  return "\n\n".join(section for section in sections if section.strip()) or None
900
1340
 
901
1341
 
902
- def _comet_hook_command(test_dir: Path) -> str | None:
903
- scripts_dir = test_dir / ".claude" / "skills" / "comet" / "scripts"
1342
+ def _comet_hook_command(test_dir: Path, agent: str = "claude-code") -> str | None:
1343
+ project_root = _agent_project_root(agent)
1344
+ scripts_dir = test_dir / project_root / "skills" / "comet" / "scripts"
904
1345
  mjs_hook = scripts_dir / "comet-hook-guard.mjs"
905
1346
  shell_hook = scripts_dir / "comet-hook-guard.sh"
906
1347
  if mjs_hook.exists():
907
- return "node /workspace/.claude/skills/comet/scripts/comet-hook-guard.mjs"
1348
+ return f"node /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.mjs"
908
1349
  if shell_hook.exists():
909
- return "bash /workspace/.claude/skills/comet/scripts/comet-hook-guard.sh"
1350
+ return f"bash /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.sh"
910
1351
  return None
911
1352
 
912
1353
 
@@ -930,6 +1371,34 @@ def _ensure_claude_pre_tool_hook(test_dir: Path, command: str | None) -> None:
930
1371
  settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
931
1372
 
932
1373
 
1374
+ def _ensure_agent_pre_tool_hook(test_dir: Path, agent: str, command: str | None) -> None:
1375
+ if agent == "claude-code":
1376
+ _ensure_claude_pre_tool_hook(test_dir, command)
1377
+ return
1378
+ if not command:
1379
+ return
1380
+
1381
+ project_root = ".codex" if agent == "codex" else _agent_project_root(agent)
1382
+ settings_name = "hooks.json" if agent == "codex" else "settings.json"
1383
+ settings_path = test_dir / project_root / settings_name
1384
+ if settings_path.exists():
1385
+ settings = json.loads(settings_path.read_text(encoding="utf-8"))
1386
+ else:
1387
+ settings = {}
1388
+ hooks = settings.setdefault("hooks", {})
1389
+ pre_tool_use = hooks.setdefault("PreToolUse", [])
1390
+ hook_entry = {
1391
+ "matcher": "Write|Edit|MultiEdit",
1392
+ "hooks": [{"type": "command", "command": command}],
1393
+ }
1394
+ if agent in {"qoder", "codebuddy"}:
1395
+ hook_entry["hooks"][0]["description"] = "Comet workflow guard"
1396
+ if not any(entry == hook_entry for entry in pre_tool_use):
1397
+ pre_tool_use.append(hook_entry)
1398
+ settings_path.parent.mkdir(parents=True, exist_ok=True)
1399
+ settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
1400
+
1401
+
933
1402
  def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
934
1403
  """Get shared experiment ID or create new one."""
935
1404
  requested = os.environ.get(EXPERIMENT_ID_ENV)
@@ -944,16 +1413,22 @@ def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
944
1413
  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
945
1414
  return f"{name}_{timestamp}"
946
1415
 
947
- lock_file = XDIST_EXPERIMENT_FILE.with_suffix(".lock")
1416
+ context = context_from_environment()
1417
+ experiment_file = (
1418
+ resolve_managed_path(context, "locks", ".pytest_experiment_id")
1419
+ if context is not None
1420
+ else XDIST_EXPERIMENT_FILE
1421
+ )
1422
+ lock_file = experiment_file.with_suffix(".lock")
948
1423
 
949
1424
  with file_lock(lock_file):
950
- if XDIST_EXPERIMENT_FILE.exists():
951
- data = json.loads(XDIST_EXPERIMENT_FILE.read_text())
1425
+ if experiment_file.exists():
1426
+ data = json.loads(experiment_file.read_text())
952
1427
  return data["experiment_id"]
953
1428
 
954
1429
  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
955
1430
  experiment_id = f"{name}_{timestamp}"
956
- XDIST_EXPERIMENT_FILE.write_text(
1431
+ experiment_file.write_text(
957
1432
  json.dumps(
958
1433
  {
959
1434
  "experiment_id": experiment_id,
@@ -1024,13 +1499,23 @@ def _snapshot_dynamic_skill_package(test_dir: Path, skill_hints: dict[str, Any])
1024
1499
  snapshot_root = test_dir / "_eval_target_skills"
1025
1500
  snapshot_root.mkdir(parents=True, exist_ok=True)
1026
1501
  package_dest = snapshot_root / package_dir.name
1027
- shutil.copytree(package_dir, package_dest, dirs_exist_ok=True)
1502
+ shutil.copytree(
1503
+ package_dir,
1504
+ package_dest,
1505
+ dirs_exist_ok=True,
1506
+ ignore=shutil.ignore_patterns(".comet"),
1507
+ )
1028
1508
 
1029
1509
  for node_skill in skill_hints.get("generated_node_skills") or []:
1030
1510
  node_source = package_dir.parent / node_skill
1031
1511
  if not (node_source / "SKILL.md").exists():
1032
1512
  continue
1033
- shutil.copytree(node_source, snapshot_root / node_skill, dirs_exist_ok=True)
1513
+ shutil.copytree(
1514
+ node_source,
1515
+ snapshot_root / node_skill,
1516
+ dirs_exist_ok=True,
1517
+ ignore=shutil.ignore_patterns(".comet"),
1518
+ )
1034
1519
 
1035
1520
  return str(package_dest.relative_to(test_dir)).replace("\\", "/")
1036
1521
 
@@ -1039,7 +1524,13 @@ def _cleanup_experiment_coordination():
1039
1524
  """Remove coordination files after experiment."""
1040
1525
  import sys
1041
1526
 
1042
- for f in [XDIST_EXPERIMENT_FILE, XDIST_EXPERIMENT_FILE.with_suffix(".lock")]:
1527
+ context = context_from_environment()
1528
+ experiment_file = (
1529
+ resolve_managed_path(context, "locks", ".pytest_experiment_id")
1530
+ if context is not None
1531
+ else XDIST_EXPERIMENT_FILE
1532
+ )
1533
+ for f in [experiment_file, experiment_file.with_suffix(".lock")]:
1043
1534
  try:
1044
1535
  f.unlink(missing_ok=True)
1045
1536
  except Exception as e:
@@ -1067,8 +1558,8 @@ def worker_id(request):
1067
1558
 
1068
1559
  @pytest.fixture(scope="session", autouse=True)
1069
1560
  def verify_environment(project_root, request):
1070
- """Verify Docker, Claude CLI, uv, bash, and API keys are available."""
1071
- if _is_unit_tests_only(request.config):
1561
+ """Verify the selected agent's Docker execution prerequisites."""
1562
+ if _is_unit_tests_only(request.config) or request.config.option.collectonly:
1072
1563
  return
1073
1564
 
1074
1565
  from scaffold.python.utils import load_eval_environment
@@ -1101,55 +1592,55 @@ def verify_environment(project_root, request):
1101
1592
  if result.returncode != 0:
1102
1593
  pytest.skip("Docker not available")
1103
1594
 
1104
- if not (os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN")):
1105
- pytest.skip("ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN not set")
1106
-
1107
- if shutil.which("claude") is None:
1108
- pytest.skip("Claude CLI not available")
1595
+ selection = _resolve_eval_agent(request.config)
1596
+ _require_agent_credentials(selection.agent)
1109
1597
 
1110
1598
 
1111
1599
  def _docker_environment_dirs_for_request(request, tasks_dir: Path) -> list[Path]:
1112
1600
  """Return only task environments selected by the current eval invocation."""
1113
- if not tasks_dir.exists():
1114
- return []
1115
-
1116
- all_task_names = sorted(task_dir.name for task_dir in tasks_dir.iterdir() if task_dir.is_dir())
1117
- task_filter = request.config.getoption("--task")
1118
- manifest_path = request.config.getoption("--eval-manifest")
1119
- if task_filter:
1120
- task_names = [task_filter]
1121
- elif manifest_path:
1122
- from scaffold.python.manifests import load_eval_manifest
1123
-
1124
- task_names = load_eval_manifest(manifest_path).recommended_tasks or all_task_names
1125
- else:
1126
- task_names = all_task_names
1127
-
1128
- return [
1129
- environment_dir
1130
- for task_name in task_names
1131
- if (environment_dir := tasks_dir / task_name / "environment").is_dir()
1132
- and (environment_dir / "Dockerfile").is_file()
1133
- ]
1601
+ frozen = getattr(request.config, "_comet_frozen_task_set", None)
1602
+ if frozen is None:
1603
+ raise pytest.UsageError("Eval task collection did not produce a frozen task set")
1604
+ selected: list[Path] = []
1605
+ for item in frozen.tasks:
1606
+ environment_dir = item.task.environment_dir
1607
+ if (
1608
+ environment_dir.is_dir()
1609
+ and (environment_dir / "Dockerfile").is_file()
1610
+ and environment_dir not in selected
1611
+ ):
1612
+ selected.append(environment_dir)
1613
+ return selected
1134
1614
 
1135
1615
 
1136
1616
  @pytest.fixture(scope="session", autouse=True)
1137
1617
  def prebuild_docker_image(request):
1138
1618
  """Pre-build Docker image once per session to avoid race conditions."""
1139
- if _is_unit_tests_only(request.config):
1619
+ if _is_unit_tests_only(request.config) or request.config.option.collectonly:
1140
1620
  yield
1141
1621
  return
1142
1622
 
1143
1623
  tasks_dir = PROJECT_ROOT / "tasks"
1144
1624
  for env_dir in _docker_environment_dirs_for_request(request, tasks_dir):
1145
- image = _build_docker_image_with_lock(env_dir)
1625
+ selected_agent = _resolve_eval_agent(request.config).agent
1626
+ image = _build_docker_image_with_lock(
1627
+ env_dir,
1628
+ selected_agent if selected_agent != "claude-code" else None,
1629
+ environment=getattr(request.config, "_comet_execution_env", None),
1630
+ )
1146
1631
  if image:
1147
1632
  print(f"\nPre-built Docker image: {image}")
1148
1633
 
1149
1634
  yield
1150
1635
 
1151
1636
  try:
1152
- DOCKER_BUILD_LOCK.unlink(missing_ok=True)
1637
+ context = context_from_environment()
1638
+ lock_path = (
1639
+ resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
1640
+ if context is not None
1641
+ else DOCKER_BUILD_LOCK
1642
+ )
1643
+ lock_path.unlink(missing_ok=True)
1153
1644
  except Exception:
1154
1645
  pass
1155
1646
 
@@ -1181,11 +1672,17 @@ def setup_test_context(test_dir):
1181
1672
  scripts_dir: Path | None = None,
1182
1673
  source_dir: Path | None = None,
1183
1674
  ) -> None:
1184
- skill_dir = test_dir / ".claude" / "skills" / skill_name
1675
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1676
+ skill_dir = test_dir / _agent_project_root(agent) / "skills" / skill_name
1185
1677
  skill_dir.mkdir(parents=True, exist_ok=True)
1186
1678
 
1187
1679
  if source_dir and source_dir.is_dir():
1188
- shutil.copytree(source_dir, skill_dir, dirs_exist_ok=True)
1680
+ shutil.copytree(
1681
+ source_dir,
1682
+ skill_dir,
1683
+ dirs_exist_ok=True,
1684
+ ignore=shutil.ignore_patterns(".comet"),
1685
+ )
1189
1686
 
1190
1687
  shutil.copyfile(skill_file, skill_dir / "SKILL.md")
1191
1688
 
@@ -1212,13 +1709,38 @@ def setup_test_context(test_dir):
1212
1709
  shutil.copy2(item, dest)
1213
1710
  _copy_current_comet_cli_snapshot(environment_dir, test_dir)
1214
1711
 
1712
+ def _copy_workspace(workspace_dir: Path) -> None:
1713
+ """Copy a bounded project fixture without replacing the task Dockerfile."""
1714
+ for item in workspace_dir.iterdir():
1715
+ if item.name in {"Dockerfile", CURRENT_COMET_CLI_MARKER, TRUSTED_NATIVE_RUNTIME_MARKER}:
1716
+ continue
1717
+ dest = test_dir / item.name
1718
+ if item.is_dir():
1719
+ shutil.copytree(item, dest, dirs_exist_ok=True)
1720
+ else:
1721
+ dest.parent.mkdir(parents=True, exist_ok=True)
1722
+ shutil.copy2(item, dest)
1723
+
1215
1724
  def _write_claude_md(content_file: str) -> None:
1216
- claude_dir = test_dir / ".claude"
1217
- claude_dir.mkdir(parents=True, exist_ok=True)
1218
- shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
1219
- shutil.copyfile(content_file, test_dir / "CLAUDE.md")
1725
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1726
+ if agent == "claude-code":
1727
+ claude_dir = test_dir / ".claude"
1728
+ claude_dir.mkdir(parents=True, exist_ok=True)
1729
+ shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
1730
+ shutil.copyfile(content_file, test_dir / "CLAUDE.md")
1731
+ elif agent == "codebuddy":
1732
+ codebuddy_dir = test_dir / ".codebuddy"
1733
+ codebuddy_dir.mkdir(parents=True, exist_ok=True)
1734
+ shutil.copyfile(content_file, codebuddy_dir / "CODEBUDDY.md")
1735
+ else:
1736
+ shutil.copyfile(content_file, test_dir / "AGENTS.md")
1220
1737
 
1221
- def _setup(skills: dict = None, claude_md: str = None, environment_dir: Path = None):
1738
+ def _setup(
1739
+ skills: dict = None,
1740
+ claude_md: str = None,
1741
+ environment_dir: Path = None,
1742
+ workspace_dir: Path = None,
1743
+ ):
1222
1744
  for skill_name, cfg in (skills or {}).items():
1223
1745
  if not cfg:
1224
1746
  continue
@@ -1251,6 +1773,9 @@ def setup_test_context(test_dir):
1251
1773
  if is_temp_dir and filtered_dir.exists():
1252
1774
  shutil.rmtree(filtered_dir)
1253
1775
 
1776
+ if workspace_dir and workspace_dir.exists():
1777
+ _copy_workspace(workspace_dir)
1778
+
1254
1779
  if environment_dir and environment_dir.exists():
1255
1780
  _copy_environment(environment_dir)
1256
1781
  _copy_trusted_native_runtime_snapshot(environment_dir, test_dir, skills)
@@ -1266,7 +1791,9 @@ def setup_test_context(test_dir):
1266
1791
  finally:
1267
1792
  os.unlink(temp_file)
1268
1793
 
1269
- _ensure_claude_pre_tool_hook(test_dir, _comet_hook_command(test_dir))
1794
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1795
+ _ensure_langfuse_trajectory_support(test_dir, agent)
1796
+ _ensure_agent_pre_tool_hook(test_dir, agent, _comet_hook_command(test_dir, agent))
1270
1797
 
1271
1798
  return test_dir
1272
1799
 
@@ -1275,13 +1802,15 @@ def setup_test_context(test_dir):
1275
1802
 
1276
1803
  @pytest.fixture
1277
1804
  def run_claude(test_dir, experiment_logger, request):
1278
- """Factory fixture to run Claude in Docker and capture artifacts.
1805
+ """Factory fixture to run the selected evaluation agent in Docker.
1279
1806
 
1280
1807
  For tasks using ``interaction.mode=auto_user`` the single-shot ``run-claude``
1281
1808
  is replaced by the multi-turn ``run-claude-loop`` driver, which simulates a
1282
1809
  user replying at the workflow's decision points.
1283
1810
  """
1284
- default_model = _selected_claude_model()
1811
+ execution = _resolve_eval_execution(request.config)
1812
+ selected_agent = execution.agent
1813
+ default_model = execution.model
1285
1814
 
1286
1815
  def _run(
1287
1816
  prompt: str,
@@ -1291,18 +1820,28 @@ def run_claude(test_dir, experiment_logger, request):
1291
1820
  image_id: str | None = None,
1292
1821
  ):
1293
1822
  mdl = model or default_model
1823
+ run_execution = ResolvedExecution(
1824
+ selected_agent,
1825
+ mdl,
1826
+ execution.base_url,
1827
+ execution.sources,
1828
+ )
1829
+ run_environment = build_agent_environment(run_execution)
1294
1830
  if os.environ.get("TRACE_TO_LANGSMITH", "").lower() == "true" and not os.environ.get(
1295
1831
  "CC_LANGSMITH_LOG_FILE"
1296
1832
  ):
1297
1833
  os.environ["CC_LANGSMITH_LOG_FILE"] = "/workspace/langsmith-hook.log"
1298
1834
  use_loop = interaction is not None and interaction.mode == "auto_user"
1299
1835
  if not use_loop:
1300
- result = run_claude_in_docker(
1836
+ result = run_agent_in_docker(
1301
1837
  test_dir,
1302
1838
  prompt,
1839
+ agent=selected_agent,
1303
1840
  timeout=timeout,
1304
1841
  model=mdl,
1842
+ base_url=execution.base_url,
1305
1843
  image_id=image_id,
1844
+ environment=run_environment,
1306
1845
  )
1307
1846
  else:
1308
1847
  task_prompt_file = test_dir / ".eval-task-prompt.txt"
@@ -1342,11 +1881,25 @@ def run_claude(test_dir, experiment_logger, request):
1342
1881
  "--simulator-prompt-file",
1343
1882
  "//workspace/.eval-simulator-prompt.txt",
1344
1883
  ]
1345
- result = run_claude_loop_in_docker(
1346
- test_dir,
1347
- loop_args[2:],
1348
- timeout=timeout + 60,
1349
- )
1884
+ if selected_agent == "claude-code":
1885
+ result = run_claude_loop_in_docker(
1886
+ test_dir,
1887
+ loop_args[2:],
1888
+ timeout=timeout + 60,
1889
+ environment=run_environment,
1890
+ )
1891
+ else:
1892
+ result = run_agent_loop_in_docker(
1893
+ test_dir,
1894
+ [
1895
+ loop_args[2],
1896
+ "--agent",
1897
+ selected_agent,
1898
+ *loop_args[3:],
1899
+ ],
1900
+ timeout=timeout + 60,
1901
+ environment=run_environment,
1902
+ )
1350
1903
  finally:
1351
1904
  task_prompt_file.unlink(missing_ok=True)
1352
1905
  if prompt_file and prompt_file.exists():
@@ -1384,12 +1937,22 @@ def record_result(test_dir, experiment_logger, request):
1384
1937
  if not experiment_logger:
1385
1938
  return
1386
1939
 
1940
+ if _plugin is not None:
1941
+ _plugin.last_test_dir = test_dir
1942
+
1387
1943
  treatment_name = _get_treatment_name(request.node)
1388
1944
  rep = _plugin.run_counter.get(treatment_name, 1) if _plugin else 1
1945
+ events["sample"] = rep
1389
1946
  base_dir = experiment_logger.base_dir
1390
1947
 
1391
1948
  save_events(base_dir, treatment_name, rep, events)
1392
- _save_artifacts(base_dir, treatment_name, rep, test_dir)
1949
+ _save_artifacts(
1950
+ base_dir,
1951
+ treatment_name,
1952
+ rep,
1953
+ test_dir,
1954
+ agent=events.get("agent", "claude-code"),
1955
+ )
1393
1956
  artifact_references = build_eval_artifact_references(base_dir, treatment_name, rep)
1394
1957
 
1395
1958
  scripts_used = _extract_scripts_used(events)
@@ -1439,9 +2002,23 @@ def record_result(test_dir, experiment_logger, request):
1439
2002
  "skill_sources": events.get("skill_sources", []),
1440
2003
  "eval_manifest": events.get("eval_manifest"),
1441
2004
  "case_manifest": events.get("case_manifest"),
2005
+ "eval_generation": events.get("eval_generation"),
1442
2006
  "interaction": events.get("interaction", {}),
1443
2007
  "artifact_references": artifact_references,
1444
2008
  "failure_attribution": failure_attribution,
2009
+ "agent": events.get("agent", "claude-code"),
2010
+ "role_models": events.get("role_models", {}),
2011
+ "role_agents": events.get("role_agents", {}),
2012
+ "telemetry_status": events.get("telemetry_status", "N/A"),
2013
+ "role_sessions": events.get("role_sessions", {}),
2014
+ "task": events.get("task"),
2015
+ "treatment": events.get("treatment", treatment_name),
2016
+ "sample": events.get("sample", rep),
2017
+ "prompt": events.get("prompt"),
2018
+ "skill": events.get("skill"),
2019
+ "final_response": events.get("final_response"),
2020
+ "quality_gates": events.get("quality_gates", {}),
2021
+ "execution_identity": events.get("execution_identity"),
1445
2022
  },
1446
2023
  run_id=run_id,
1447
2024
  ),
@@ -1495,14 +2072,29 @@ def _build_report_payload(
1495
2072
  "model_usage": events.get("model_usage", {}),
1496
2073
  "files_created": events.get("files_created", []),
1497
2074
  "skills_invoked": events.get("skills_invoked", []),
2075
+ "skill_invocations": events.get("skill_invocations", []),
1498
2076
  "scripts_used": scripts_used,
1499
2077
  "profile": events.get("profile"),
1500
2078
  "skill_sources": events.get("skill_sources", []),
1501
2079
  "eval_manifest": events.get("eval_manifest"),
1502
2080
  "case_manifest": events.get("case_manifest"),
2081
+ "eval_generation": events.get("eval_generation"),
1503
2082
  "interaction": events.get("interaction", {}),
1504
2083
  "artifact_references": artifact_references,
1505
2084
  "failure_attribution": failure_attribution,
2085
+ "agent": events.get("agent", "claude-code"),
2086
+ "role_models": events.get("role_models", {}),
2087
+ "role_agents": events.get("role_agents", {}),
2088
+ "telemetry_status": events.get("telemetry_status", "N/A"),
2089
+ "role_sessions": events.get("role_sessions", {}),
2090
+ "task": events.get("task"),
2091
+ "treatment": events.get("treatment", treatment_name),
2092
+ "sample": events.get("sample", rep),
2093
+ "prompt": events.get("prompt"),
2094
+ "skill": events.get("skill"),
2095
+ "final_response": events.get("final_response"),
2096
+ "quality_gates": events.get("quality_gates", {}),
2097
+ "execution_identity": events.get("execution_identity"),
1506
2098
  },
1507
2099
  "timestamp": datetime.now().isoformat(),
1508
2100
  }
@@ -1581,15 +2173,32 @@ def _filter_scripts(scripts_dir: Path, script_filter: str) -> Path | None:
1581
2173
  return temp_dir
1582
2174
 
1583
2175
 
1584
- def _build_docker_image_with_lock(environment_dir: Path) -> str | None:
2176
+ def _build_docker_image_with_lock(
2177
+ environment_dir: Path,
2178
+ agent: str | None = None,
2179
+ *,
2180
+ environment: dict[str, str] | None = None,
2181
+ ) -> str | None:
1585
2182
  """Build Docker image with file locking to prevent race conditions."""
1586
2183
  if not environment_dir or not (environment_dir / "Dockerfile").exists():
1587
2184
  return None
1588
2185
 
1589
- with file_lock(DOCKER_BUILD_LOCK):
1590
- # A cold image build downloads Debian packages and installs the Claude CLI.
2186
+ context = context_from_environment()
2187
+ lock_path = (
2188
+ resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
2189
+ if context is not None
2190
+ else DOCKER_BUILD_LOCK
2191
+ )
2192
+ with file_lock(lock_path):
2193
+ # A cold image build downloads Debian packages and installs the selected CLI.
1591
2194
  # Five minutes is insufficient after cache cleanup or on a proxied connection.
1592
- result = run_shell("docker.sh", "build", str(environment_dir), timeout=900, check=False)
2195
+ command = ["docker.sh", "build", str(environment_dir)]
2196
+ if agent:
2197
+ command.extend(["--agent", agent])
2198
+ run_options = {"timeout": 900, "check": False}
2199
+ if environment is not None:
2200
+ run_options["env"] = environment
2201
+ result = run_shell(*command, **run_options)
1593
2202
  if result.returncode == 0:
1594
2203
  return result.stdout.strip()
1595
2204
  return None
@@ -1664,18 +2273,34 @@ def _extract_scripts_used(events: dict) -> list[str]:
1664
2273
  return [s for s in _get_known_scripts() if s.lower() in all_activity]
1665
2274
 
1666
2275
 
1667
- def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Path):
1668
- """Save Claude's generated files as artifacts."""
2276
+ def _save_artifacts(
2277
+ base_dir: Path,
2278
+ treatment_name: str,
2279
+ rep: int,
2280
+ test_dir: Path,
2281
+ *,
2282
+ agent: str = "claude-code",
2283
+ ):
2284
+ """Save the selected Agent's generated files as artifacts."""
1669
2285
  artifacts_dir = base_dir / "artifacts" / f"{treatment_name.lower()}_rep{rep}"
1670
- claude_dir = artifacts_dir / "claude"
2286
+ agent_dir_name = {
2287
+ "claude-code": "claude",
2288
+ "codex": "codex",
2289
+ "qoder": "qoder",
2290
+ "codebuddy": "codebuddy",
2291
+ }.get(agent, agent)
2292
+ agent_dir = artifacts_dir / agent_dir_name
1671
2293
  execution_dir = artifacts_dir / "execution"
1672
- claude_dir.mkdir(parents=True, exist_ok=True)
2294
+ agent_dir.mkdir(parents=True, exist_ok=True)
1673
2295
  execution_dir.mkdir(parents=True, exist_ok=True)
1674
2296
 
1675
2297
  from scaffold.python.utils import TEST_CONTEXT_FILE, TEST_RESULTS_FILE
1676
2298
 
1677
2299
  exclude_dirs = {
1678
2300
  ".claude",
2301
+ ".agents",
2302
+ ".qoder",
2303
+ ".codebuddy",
1679
2304
  ".git",
1680
2305
  "_eval_current_comet",
1681
2306
  "node_modules",
@@ -1707,9 +2332,20 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
1707
2332
  continue
1708
2333
  try:
1709
2334
  rel_path = item.relative_to(test_dir)
1710
- dest = claude_dir / rel_path
2335
+ dest = agent_dir / rel_path
1711
2336
  dest.parent.mkdir(parents=True, exist_ok=True)
1712
- shutil.copy(item, dest)
2337
+ try:
2338
+ content = item.read_text(encoding="utf-8")
2339
+ except UnicodeDecodeError:
2340
+ import sys
2341
+
2342
+ print(
2343
+ f"Warning: skipped non-text artifact {rel_path}; binary content is not persisted to protect credentials",
2344
+ file=sys.stderr,
2345
+ )
2346
+ continue
2347
+ else:
2348
+ dest.write_text(redact_sensitive(content), encoding="utf-8")
1713
2349
  claude_files.append(item)
1714
2350
  except Exception as e:
1715
2351
  import sys
@@ -1722,7 +2358,7 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
1722
2358
  success, output = run_python_in_docker(test_dir, py_file.name, timeout=300)
1723
2359
  status = "success" if success else "error"
1724
2360
  output_file = execution_dir / f"{py_file.stem}_{status}.txt"
1725
- output_file.write_text(strip_ansi(output))
2361
+ output_file.write_text(redact_sensitive(strip_ansi(output)), encoding="utf-8")
1726
2362
  except Exception as e:
1727
2363
  error_file = execution_dir / f"{py_file.stem}_error.txt"
1728
- error_file.write_text(str(e))
2364
+ error_file.write_text(redact_sensitive(str(e)), encoding="utf-8")