@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -3,9 +3,9 @@
3
3
  Generates rich experiment logs in logs/experiments/ including:
4
4
  - summary.md: Full markdown report with tables and details
5
5
  - events/: Parsed events from each test run
6
- - raw/: Raw Claude CLI output
6
+ - raw/: Raw selected Agent CLI output
7
7
  - reports/: Per-run validation reports
8
- - artifacts/: Files Claude generated and their execution output
8
+ - artifacts/: Files selected Agent generated and their execution output
9
9
  - metadata.json: Experiment metadata
10
10
 
11
11
  Supports pytest-xdist parallel execution via worker coordination.
@@ -27,7 +27,7 @@ from typing import Any
27
27
 
28
28
  import pytest
29
29
 
30
- from scaffold import run_claude_in_docker, run_python_in_docker, run_shell
30
+ from scaffold import run_agent_in_docker, run_python_in_docker, run_shell
31
31
  from scaffold.python import (
32
32
  ExperimentLogger,
33
33
  TreatmentResult,
@@ -40,9 +40,40 @@ from scaffold.python import (
40
40
  save_report,
41
41
  strip_ansi,
42
42
  )
43
+ from scaffold.python.agents import get_agent_adapter, resolve_agent, validate_agent_capabilities
44
+ from scaffold.python.auto_tasks import AutoTaskError, ensure_generated_manifest, find_project_root
45
+ from scaffold.python.eval_context import (
46
+ EvalContextError,
47
+ context_from_environment,
48
+ resolve_eval_context,
49
+ resolve_managed_path,
50
+ )
51
+ from scaffold.python.execution import (
52
+ ResolvedExecution,
53
+ ResolvedJudge,
54
+ build_agent_environment,
55
+ missing_credentials,
56
+ preflight_credentials,
57
+ redact_sensitive,
58
+ resolve_execution,
59
+ resolve_judge,
60
+ validate_base_url,
61
+ )
62
+ from scaffold.python.generated_task_cache import (
63
+ selected_agent_model as _legacy_selected_agent_model,
64
+ )
65
+ from scaffold.python.langfuse_adapter import (
66
+ install_codex_plugin_workspace,
67
+ install_transcript_hook,
68
+ provision_trajectory_plugin,
69
+ )
43
70
  from scaffold.python.sample_quality import infer_sample_quality
44
71
  from scaffold.python.skill_parser import SCRIPT_EXTENSIONS
45
- from scaffold.python.utils import run_claude_loop_in_docker
72
+ from scaffold.python.utils import (
73
+ check_docker_available,
74
+ run_agent_loop_in_docker,
75
+ run_claude_loop_in_docker,
76
+ )
46
77
  from scaffold.python.aligned_comparison import (
47
78
  EXPECTED_CASE_MATRIX_FILENAME,
48
79
  build_execution_identity,
@@ -56,14 +87,21 @@ def _extract_loop_turns(stderr: str | None) -> int | None:
56
87
  return int(match.group(1)) if match else None
57
88
 
58
89
 
59
- def _extract_loop_interaction(stderr: str | None) -> dict[str, int | None]:
90
+ def _extract_loop_interaction(stderr: str | None) -> dict[str, Any]:
60
91
  source = stderr or ""
92
+ role_sessions = {"subject": [], "simulator": [], "judge": []}
93
+ for role, session_id in re.findall(
94
+ r"\[loop\] role-session (subject|simulator|judge) ([^\s]+)", source
95
+ ):
96
+ if session_id not in role_sessions[role]:
97
+ role_sessions[role].append(session_id)
61
98
  return {
62
99
  "actual_turns": _extract_loop_turns(source),
63
100
  "decision_points": source.count("[loop] decision point detected"),
64
101
  "deterministic_replies": source.count("[loop] deterministic decision reply applied"),
65
102
  "completion_signals": source.count("[loop] workflow completion detected"),
66
103
  "fresh_resume_boundaries": source.count("[loop] fresh resume boundary detected"),
104
+ "role_sessions": role_sessions,
67
105
  }
68
106
 
69
107
 
@@ -212,8 +250,12 @@ EVAL_ROOT = PROJECT_ROOT.parent
212
250
  REPOSITORY_ROOT = EVAL_ROOT.parent
213
251
 
214
252
  # Shared files for xdist worker coordination
215
- XDIST_EXPERIMENT_FILE = PROJECT_ROOT / ".pytest_experiment_id"
216
- DOCKER_BUILD_LOCK = PROJECT_ROOT / ".pytest_docker_build.lock"
253
+ _RUNTIME_CONTEXT = context_from_environment()
254
+ _CONTEXT_LOCK_ROOT = (
255
+ resolve_managed_path(_RUNTIME_CONTEXT, "locks") if _RUNTIME_CONTEXT else PROJECT_ROOT
256
+ )
257
+ XDIST_EXPERIMENT_FILE = _CONTEXT_LOCK_ROOT / ".pytest_experiment_id"
258
+ DOCKER_BUILD_LOCK = _CONTEXT_LOCK_ROOT / ".pytest_docker_build.lock"
217
259
  EXPERIMENT_ID_ENV = "COMET_EVAL_EXPERIMENT_ID"
218
260
  EXPERIMENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
219
261
 
@@ -246,12 +288,305 @@ MODEL_EXECUTION_ENV_KEYS = (
246
288
  "ANTHROPIC_DEFAULT_OPUS_MODEL_NAME",
247
289
  "ANTHROPIC_DEFAULT_SONNET_MODEL_NAME",
248
290
  "CLAUDE_CODE_SUBAGENT_MODEL",
291
+ "CODEBUDDY_BASE_URL",
292
+ "CODEBUDDY_MODEL",
293
+ "CODEBUDDY_SMALL_FAST_MODEL",
294
+ "CODEBUDDY_BIG_SLOW_MODEL",
295
+ "CODEBUDDY_CODE_SUBAGENT_MODEL",
296
+ "BENCH_CODEBUDDY_MODEL",
297
+ "OPENAI_BASE_URL",
298
+ "OPENAI_MODEL",
299
+ "CODEX_BASE_URL",
300
+ "CODEX_MODEL",
301
+ "QODER_BASE_URL",
302
+ "QODER_MODEL",
249
303
  )
250
304
 
251
305
 
252
- def _selected_claude_model() -> str | None:
253
- """Resolve the exact model selector passed to Claude without persisting it."""
254
- return os.environ.get("BENCH_CC_MODEL") or os.environ.get("ANTHROPIC_MODEL")
306
+ def _getoption(config, name: str, default=None):
307
+ try:
308
+ value = config.getoption(name)
309
+ except (AttributeError, KeyError, ValueError):
310
+ return default
311
+ return default if value is None else value
312
+
313
+
314
+ def _manifest_for_config(config):
315
+ manifest_path = _getoption(config, "--eval-manifest")
316
+ if not manifest_path:
317
+ return None
318
+ from scaffold.python.manifests import load_eval_manifest
319
+
320
+ return load_eval_manifest(manifest_path)
321
+
322
+
323
+ def _resolve_eval_execution(config) -> ResolvedExecution:
324
+ manifest = _manifest_for_config(config)
325
+ manifest_execution = manifest.execution if manifest is not None else None
326
+ return resolve_execution(
327
+ cli_agent=_getoption(config, "--agent"),
328
+ cli_model=_getoption(config, "--model"),
329
+ cli_base_url=_getoption(config, "--base-url"),
330
+ manifest=manifest_execution,
331
+ )
332
+
333
+
334
+ def _resolve_eval_judge(config) -> ResolvedJudge | None:
335
+ manifest = _manifest_for_config(config)
336
+ manifest_judge = manifest.judge if manifest is not None else None
337
+ return resolve_judge(
338
+ main=_resolve_eval_execution(config),
339
+ cli_agent=_getoption(config, "--judge-agent"),
340
+ cli_model=_getoption(config, "--judge-model"),
341
+ cli_base_url=_getoption(config, "--judge-base-url"),
342
+ manifest=manifest_judge,
343
+ )
344
+
345
+
346
+ def _selected_agent_model(agent: str) -> str | None:
347
+ if _plugin is not None:
348
+ return _resolve_eval_execution(_plugin.config).model
349
+ return _legacy_selected_agent_model(agent)
350
+
351
+
352
+ def _resolve_eval_agent(config):
353
+ execution = _resolve_eval_execution(config)
354
+ return resolve_agent(execution.agent, None)
355
+
356
+
357
+ def _require_agent_credentials(agent: str, source_env: dict[str, str] | None = None) -> None:
358
+ """Fail setup before workloads when the isolated Agent cannot authenticate."""
359
+ environment = source_env if source_env is not None else os.environ
360
+ missing = missing_credentials(agent, source_env=environment)
361
+ if not missing:
362
+ return
363
+ adapter = get_agent_adapter(agent)
364
+ separator = ", " if adapter.custom else " or "
365
+ raise pytest.UsageError(
366
+ f"Credentials for {agent} not set: {separator.join(missing)}"
367
+ )
368
+
369
+
370
+ def _normal_run_preflight(config) -> None:
371
+ """Validate normal-run dependencies before collection can generate or run tasks."""
372
+
373
+ if _is_unit_tests_only(config) or getattr(config.option, "collectonly", False):
374
+ return
375
+ from scaffold.python.utils import load_eval_environment
376
+
377
+ load_eval_environment()
378
+ main = _resolve_eval_execution(config)
379
+ judge = _resolve_eval_judge(config)
380
+ suite = next(
381
+ (name for name in ("local", "langsmith", "langfuse") if f"{name}/tests/" in " ".join(config.args)),
382
+ "local",
383
+ )
384
+ required_harness = (
385
+ EVAL_ROOT / "pyproject.toml",
386
+ EVAL_ROOT / "schemas" / "comet.eval" / "v1alpha1.schema.json",
387
+ EVAL_ROOT / suite / "tests" / "tasks" / "test_tasks.py",
388
+ )
389
+ missing_harness = [str(path) for path in required_harness if not path.is_file()]
390
+ if missing_harness:
391
+ raise pytest.UsageError(f"Packaged eval harness is incomplete: {', '.join(missing_harness)}")
392
+ manifest = _manifest_for_config(config)
393
+ if manifest is not None and manifest.reporting is not None:
394
+ selected_report_suite = manifest.reporting.suite
395
+ if selected_report_suite and selected_report_suite != suite:
396
+ raise pytest.UsageError(
397
+ f"reporting.suite is {selected_report_suite!r}, but the selected suite is {suite!r}"
398
+ )
399
+ report_config = manifest.reporting.config
400
+ if report_config and not _getoption(config, "--report-config"):
401
+ report_path = Path(report_config)
402
+ if not report_path.is_absolute():
403
+ report_path = manifest.path.parent / report_path
404
+ config.option.report_config = str(report_path.resolve())
405
+ try:
406
+ load_report_output_config(_getoption(config, "--report-config"))
407
+ except (OSError, ValueError) as exc:
408
+ raise pytest.UsageError(f"Eval report configuration is invalid: {exc}") from exc
409
+ try:
410
+ validate_agent_capabilities(get_agent_adapter(main.agent))
411
+ if judge is not None:
412
+ validate_agent_capabilities(get_agent_adapter(judge.agent))
413
+ except ValueError as exc:
414
+ raise pytest.UsageError(str(exc)) from exc
415
+ if main.base_url:
416
+ validate_base_url(main.base_url, field="execution.baseUrl")
417
+ if judge is not None and judge.base_url:
418
+ validate_base_url(judge.base_url, field="judge.baseUrl")
419
+ credential_errors = preflight_credentials(main, judge)
420
+ if credential_errors:
421
+ raise pytest.UsageError("; ".join(credential_errors))
422
+ if shutil.which("uv") is None:
423
+ raise pytest.UsageError("uv is not installed or not in PATH")
424
+ if not check_docker_available():
425
+ raise pytest.UsageError("Docker CLI or daemon is unavailable")
426
+ context = getattr(config, "_comet_eval_context", None)
427
+ if context is None:
428
+ raise pytest.UsageError("Eval context is missing before normal-run preflight")
429
+ artifact_root = Path(context.artifact_root)
430
+ artifact_root.mkdir(parents=True, exist_ok=True)
431
+ try:
432
+ with tempfile.NamedTemporaryFile(dir=artifact_root, prefix=".preflight-", delete=True):
433
+ pass
434
+ except OSError as exc:
435
+ raise pytest.UsageError(f"Eval artifact root is not writable: {artifact_root}") from exc
436
+ config._comet_execution_env = build_agent_environment(main)
437
+ config._comet_execution = main
438
+ config._comet_judge = judge
439
+ os.environ["COMET_EVAL_MAIN_CREDENTIALS"] = ",".join(
440
+ get_agent_adapter(main.agent).required_credentials
441
+ )
442
+ if judge is not None:
443
+ os.environ["BENCH_LLM_JUDGE"] = "1"
444
+ os.environ["BENCH_JUDGE_MODEL"] = judge.model
445
+ os.environ["BENCH_JUDGE_AGENT"] = judge.agent
446
+ if judge.base_url:
447
+ os.environ["BENCH_JUDGE_BASE_URL"] = judge.base_url
448
+ else:
449
+ os.environ.pop("BENCH_JUDGE_BASE_URL", None)
450
+
451
+
452
+ def _agent_project_root(agent: str) -> str:
453
+ return {
454
+ "claude-code": ".claude",
455
+ "codex": ".agents",
456
+ "qoder": ".qoder",
457
+ "codebuddy": ".codebuddy",
458
+ }.get(agent, ".comet-agent")
459
+
460
+
461
+ def _ensure_langfuse_trajectory_support(test_dir: Path, agent: str) -> None:
462
+ """Install only isolated, project-local trajectory support for Langfuse runs."""
463
+ if os.environ.get("TRACE_TO_LANGFUSE", "").lower() != "true":
464
+ return
465
+ if agent in {"qoder", "codebuddy"}:
466
+ try:
467
+ install_transcript_hook(test_dir, agent)
468
+ except Exception as exc:
469
+ if os.environ.get("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED") != "true":
470
+ print(
471
+ f"[langfuse] optional {agent} transcript setup skipped: {exc}",
472
+ file=os.sys.stderr,
473
+ )
474
+ os.environ["LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED"] = "true"
475
+ return
476
+ if agent != "codex":
477
+ return
478
+ cache_root = os.environ.get("LANGFUSE_TRAJECTORY_CACHE_DIR")
479
+ if not cache_root:
480
+ return
481
+ try:
482
+ provision = provision_trajectory_plugin(Path(cache_root), agent)
483
+ install_codex_plugin_workspace(test_dir, provision)
484
+ except Exception as exc:
485
+ if os.environ.get("LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED") != "true":
486
+ print(f"[langfuse] optional Codex trajectory setup skipped: {exc}", file=os.sys.stderr)
487
+ os.environ["LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED"] = "true"
488
+
489
+
490
+ def _ensure_auto_generated_manifest(config, task_filter: str | None = None) -> None:
491
+ """Freeze generated descriptors once while retaining the original manifest authority."""
492
+ if getattr(config, "_comet_frozen_task_set", None) is not None:
493
+ return
494
+ if config.getoption("--task") or task_filter or config.getoption("--quick"):
495
+ return
496
+
497
+ manifest_path = config.getoption("--eval-manifest")
498
+ skill_path = config.getoption("--skill-path")
499
+ manifest = None
500
+ if manifest_path:
501
+ from scaffold.python.manifests import load_eval_manifest
502
+
503
+ manifest = load_eval_manifest(manifest_path)
504
+ if manifest.tasks or manifest.recommended_tasks:
505
+ return
506
+ skill_path = str(manifest.skill_path)
507
+ if not skill_path:
508
+ return
509
+
510
+ if getattr(getattr(config, "option", None), "collectonly", False):
511
+ from scaffold.python.manifests import SkillEvalManifest
512
+ from scaffold.python.task_resolution import ResolvedTaskSet
513
+
514
+ if manifest is None:
515
+ resolved_skill_root = Path(skill_path).expanduser().resolve()
516
+ if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
517
+ resolved_skill_root = resolved_skill_root.parent
518
+ manifest = SkillEvalManifest(
519
+ path=resolved_skill_root / "comet" / "eval.yaml",
520
+ name=resolved_skill_root.name,
521
+ description="",
522
+ skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
523
+ skill_path=resolved_skill_root,
524
+ profile=config.getoption("--profile"),
525
+ )
526
+ config._comet_resolution_manifest = manifest
527
+ config._comet_resolved_tasks = {}
528
+ config._comet_frozen_task_set = ResolvedTaskSet("pending-generation", (), None)
529
+ return
530
+
531
+ # Task generation happens during collection, before session fixtures run.
532
+ # Load the documented local environment now so credentials and the model
533
+ # participating in the cache key match the later execution environment.
534
+ from scaffold.python.utils import load_eval_environment
535
+
536
+ load_eval_environment()
537
+ execution = _resolve_eval_execution(config)
538
+ profile = config.getoption("--profile") or (manifest.profile if manifest else None) or "generic"
539
+ interaction = vars(manifest.interaction) if manifest else {"mode": "none", "max_turns": 12}
540
+ context = getattr(config, "_comet_eval_context", None)
541
+ project_root = (
542
+ context.artifact_owner_root
543
+ if context is not None
544
+ else config.getoption("--project-root") or find_project_root(skill_path)
545
+ )
546
+ try:
547
+ generated = ensure_generated_manifest(
548
+ skill_path,
549
+ project_root,
550
+ agent=execution.agent,
551
+ model=execution.model,
552
+ base_url=execution.base_url,
553
+ environment=build_agent_environment(execution),
554
+ profile=profile,
555
+ interaction=interaction,
556
+ collect_only=bool(getattr(getattr(config, "option", None), "collectonly", False)),
557
+ )
558
+ except AutoTaskError as exc:
559
+ raise pytest.UsageError(str(exc)) from exc
560
+ from scaffold.python.manifest_tasks import load_manifest_tasks
561
+ from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
562
+ from scaffold.python.task_resolution import ResolvedTask, ResolvedTaskSet
563
+
564
+ generated_manifest = load_eval_manifest(generated.manifest_path)
565
+ generated_tasks = tuple(
566
+ ResolvedTask(task.name, "generated", task)
567
+ for task in load_manifest_tasks(generated_manifest)
568
+ )
569
+ if manifest is None:
570
+ resolved_skill_root = Path(skill_path).expanduser().resolve()
571
+ if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
572
+ resolved_skill_root = resolved_skill_root.parent
573
+ manifest = SkillEvalManifest(
574
+ path=resolved_skill_root / "comet" / "eval.yaml",
575
+ name=resolved_skill_root.name,
576
+ description="",
577
+ skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
578
+ skill_path=resolved_skill_root,
579
+ profile=config.getoption("--profile"),
580
+ )
581
+ frozen = ResolvedTaskSet(
582
+ "generated-cache",
583
+ generated_tasks,
584
+ None,
585
+ )
586
+ config._comet_resolution_manifest = manifest
587
+ config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
588
+ config._comet_frozen_task_set = frozen
589
+ config._comet_generated_manifest = generated
255
590
 
256
591
 
257
592
  def _model_execution_config() -> dict[str, str | None]:
@@ -265,11 +600,13 @@ def _capture_execution_identity(test_dir: Path, *, model: str | None, interactio
265
600
  "docker.sh",
266
601
  "execution-identity",
267
602
  str(test_dir),
603
+ "--agent",
604
+ _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code",
268
605
  timeout=300,
269
606
  check=False,
270
607
  )
271
608
  if result.returncode != 0:
272
- raise RuntimeError("Could not verify eval Docker/Claude execution identity")
609
+ raise RuntimeError("Could not verify eval Docker/agent execution identity")
273
610
  try:
274
611
  raw = json.loads(result.stdout)
275
612
  except (TypeError, json.JSONDecodeError) as error:
@@ -532,6 +869,54 @@ def pytest_addoption(parser):
532
869
  default=None,
533
870
  help="Path to comet/eval.yaml",
534
871
  )
872
+ parser.addoption(
873
+ "--agent",
874
+ action="store",
875
+ default=None,
876
+ help="Evaluation agent CLI (built-in or explicitly installed custom adapter)",
877
+ )
878
+ parser.addoption(
879
+ "--model",
880
+ action="store",
881
+ default=None,
882
+ help="Main evaluation model override",
883
+ )
884
+ parser.addoption(
885
+ "--base-url",
886
+ action="store",
887
+ default=None,
888
+ help="Main evaluation API base URL override",
889
+ )
890
+ parser.addoption(
891
+ "--judge-agent",
892
+ action="store",
893
+ default=None,
894
+ help="Independent Judge agent (built-in or explicitly installed custom adapter)",
895
+ )
896
+ parser.addoption(
897
+ "--judge-model",
898
+ action="store",
899
+ default=None,
900
+ help="Independent Judge model",
901
+ )
902
+ parser.addoption(
903
+ "--judge-base-url",
904
+ action="store",
905
+ default=None,
906
+ help="Independent Judge API base URL",
907
+ )
908
+ parser.addoption(
909
+ "--quick",
910
+ action="store_true",
911
+ default=False,
912
+ help="Use generic-skill-smoke and bypass automatic task generation",
913
+ )
914
+ parser.addoption(
915
+ "--project-root",
916
+ action="store",
917
+ default=None,
918
+ help="Project root used for generated eval cache",
919
+ )
535
920
  parser.addoption(
536
921
  "--interaction-mode",
537
922
  action="store",
@@ -564,6 +949,27 @@ def pytest_configure(config):
564
949
  "markers",
565
950
  "eval_case(repetition): controller-owned task/treatment/repetition identity",
566
951
  )
952
+ try:
953
+ context = context_from_environment()
954
+ if context is None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
955
+ context = resolve_eval_context(
956
+ skill_path=config.getoption("--skill-path"),
957
+ manifest_path=config.getoption("--eval-manifest"),
958
+ project_root=config.getoption("--project-root"),
959
+ )
960
+ except EvalContextError as exc:
961
+ raise pytest.UsageError(str(exc)) from exc
962
+ if context is not None:
963
+ config._comet_eval_context = context
964
+ config.option.project_root = str(context.artifact_owner_root)
965
+ if context.manifest_path is not None:
966
+ config.option.eval_manifest = str(context.manifest_path)
967
+ else:
968
+ config.option.skill_path = str(context.skill_root)
969
+ config._comet_execution = _resolve_eval_execution(config)
970
+ config._comet_judge = _resolve_eval_judge(config)
971
+ _normal_run_preflight(config)
972
+ _resolve_eval_agent(config)
567
973
  global _plugin
568
974
  _plugin = ExperimentPlugin(config)
569
975
  config.pluginmanager.register(_plugin, "experiment_plugin")
@@ -772,12 +1178,40 @@ def _get_experiment_name(session) -> str:
772
1178
 
773
1179
 
774
1180
  def _get_dynamic_treatment_config(config):
1181
+ _ensure_auto_generated_manifest(config)
1182
+
1183
+ def generation_hints(manifest):
1184
+ generation_metadata = {}
1185
+ generation_metadata_hash = None
1186
+ generated = getattr(config, "_comet_generated_manifest", None)
1187
+ generation_metadata_path = (
1188
+ generated.metadata_path if generated is not None else manifest.generation_metadata_path
1189
+ )
1190
+ if generation_metadata_path and generation_metadata_path.is_file():
1191
+ generation_metadata_bytes = generation_metadata_path.read_bytes()
1192
+ generation_metadata = json.loads(generation_metadata_bytes)
1193
+ generation_metadata_hash = (
1194
+ "sha256:" + hashlib.sha256(generation_metadata_bytes).hexdigest()
1195
+ )
1196
+ return {
1197
+ "generation_hash": (
1198
+ generated.generation_hash if generated is not None else manifest.generation_hash
1199
+ ),
1200
+ "generation_metadata_path": (
1201
+ str(generation_metadata_path) if generation_metadata_path else None
1202
+ ),
1203
+ "generation_manifest_hash": generation_metadata.get("manifest_hash"),
1204
+ "generation_metadata_hash": generation_metadata_hash,
1205
+ "generation_overhead": generation_metadata.get("generation_overhead"),
1206
+ }
1207
+
775
1208
  manifest_path = config.getoption("--eval-manifest")
776
1209
  if manifest_path:
777
1210
  from scaffold.python.manifests import load_eval_manifest
778
1211
  from scaffold.python.treatments import TreatmentConfig
779
1212
 
780
1213
  manifest = load_eval_manifest(manifest_path)
1214
+ generation = generation_hints(manifest)
781
1215
  node_skills = []
782
1216
  for node_skill in manifest.generated_node_skills:
783
1217
  node_path = manifest.skill_path.parent / node_skill
@@ -799,11 +1233,13 @@ def _get_dynamic_treatment_config(config):
799
1233
  "path": str(manifest.skill_path),
800
1234
  "profile": manifest.profile,
801
1235
  "manifest": str(manifest.path),
1236
+ "execution_agent": manifest.execution_agent,
802
1237
  "baseline_treatments": manifest.baseline_treatments,
803
1238
  "quality_gates": manifest.quality_gates,
804
1239
  "required_output_schemas": manifest.required_output_schemas,
805
1240
  "expected_evidence": manifest.expected_evidence,
806
1241
  "draft_hash": manifest.draft_hash,
1242
+ **generation,
807
1243
  "required_skills": manifest.required_skills,
808
1244
  "expected_artifacts": manifest.expected_artifacts,
809
1245
  "generated_node_skills": manifest.generated_node_skills,
@@ -820,7 +1256,8 @@ def _get_dynamic_treatment_config(config):
820
1256
  if not skill_path:
821
1257
  return None
822
1258
  skill_name = config.getoption("--skill-name") or Path(skill_path).resolve().parent.name
823
- profile = config.getoption("--profile")
1259
+ manifest = getattr(config, "_comet_resolution_manifest", None)
1260
+ profile = config.getoption("--profile") or (manifest.profile if manifest else None)
824
1261
  skill_cfg = {
825
1262
  "name": skill_name,
826
1263
  "source": "path",
@@ -828,6 +1265,9 @@ def _get_dynamic_treatment_config(config):
828
1265
  }
829
1266
  if profile:
830
1267
  skill_cfg["profile"] = profile
1268
+ if manifest:
1269
+ skill_cfg["manifest"] = str(manifest.path)
1270
+ skill_cfg.update(generation_hints(manifest))
831
1271
  from scaffold.python.treatments import TreatmentConfig
832
1272
 
833
1273
  return TreatmentConfig(
@@ -899,14 +1339,15 @@ def _build_eval_claude_md(profile_name: str, treatment_claude_md: str | None = N
899
1339
  return "\n\n".join(section for section in sections if section.strip()) or None
900
1340
 
901
1341
 
902
- def _comet_hook_command(test_dir: Path) -> str | None:
903
- scripts_dir = test_dir / ".claude" / "skills" / "comet" / "scripts"
1342
+ def _comet_hook_command(test_dir: Path, agent: str = "claude-code") -> str | None:
1343
+ project_root = _agent_project_root(agent)
1344
+ scripts_dir = test_dir / project_root / "skills" / "comet" / "scripts"
904
1345
  mjs_hook = scripts_dir / "comet-hook-guard.mjs"
905
1346
  shell_hook = scripts_dir / "comet-hook-guard.sh"
906
1347
  if mjs_hook.exists():
907
- return "node /workspace/.claude/skills/comet/scripts/comet-hook-guard.mjs"
1348
+ return f"node /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.mjs"
908
1349
  if shell_hook.exists():
909
- return "bash /workspace/.claude/skills/comet/scripts/comet-hook-guard.sh"
1350
+ return f"bash /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.sh"
910
1351
  return None
911
1352
 
912
1353
 
@@ -930,6 +1371,34 @@ def _ensure_claude_pre_tool_hook(test_dir: Path, command: str | None) -> None:
930
1371
  settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
931
1372
 
932
1373
 
1374
+ def _ensure_agent_pre_tool_hook(test_dir: Path, agent: str, command: str | None) -> None:
1375
+ if agent == "claude-code":
1376
+ _ensure_claude_pre_tool_hook(test_dir, command)
1377
+ return
1378
+ if not command:
1379
+ return
1380
+
1381
+ project_root = ".codex" if agent == "codex" else _agent_project_root(agent)
1382
+ settings_name = "hooks.json" if agent == "codex" else "settings.json"
1383
+ settings_path = test_dir / project_root / settings_name
1384
+ if settings_path.exists():
1385
+ settings = json.loads(settings_path.read_text(encoding="utf-8"))
1386
+ else:
1387
+ settings = {}
1388
+ hooks = settings.setdefault("hooks", {})
1389
+ pre_tool_use = hooks.setdefault("PreToolUse", [])
1390
+ hook_entry = {
1391
+ "matcher": "Write|Edit|MultiEdit",
1392
+ "hooks": [{"type": "command", "command": command}],
1393
+ }
1394
+ if agent in {"qoder", "codebuddy"}:
1395
+ hook_entry["hooks"][0]["description"] = "Comet workflow guard"
1396
+ if not any(entry == hook_entry for entry in pre_tool_use):
1397
+ pre_tool_use.append(hook_entry)
1398
+ settings_path.parent.mkdir(parents=True, exist_ok=True)
1399
+ settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
1400
+
1401
+
933
1402
  def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
934
1403
  """Get shared experiment ID or create new one."""
935
1404
  requested = os.environ.get(EXPERIMENT_ID_ENV)
@@ -944,16 +1413,22 @@ def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
944
1413
  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
945
1414
  return f"{name}_{timestamp}"
946
1415
 
947
- lock_file = XDIST_EXPERIMENT_FILE.with_suffix(".lock")
1416
+ context = context_from_environment()
1417
+ experiment_file = (
1418
+ resolve_managed_path(context, "locks", ".pytest_experiment_id")
1419
+ if context is not None
1420
+ else XDIST_EXPERIMENT_FILE
1421
+ )
1422
+ lock_file = experiment_file.with_suffix(".lock")
948
1423
 
949
1424
  with file_lock(lock_file):
950
- if XDIST_EXPERIMENT_FILE.exists():
951
- data = json.loads(XDIST_EXPERIMENT_FILE.read_text())
1425
+ if experiment_file.exists():
1426
+ data = json.loads(experiment_file.read_text())
952
1427
  return data["experiment_id"]
953
1428
 
954
1429
  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
955
1430
  experiment_id = f"{name}_{timestamp}"
956
- XDIST_EXPERIMENT_FILE.write_text(
1431
+ experiment_file.write_text(
957
1432
  json.dumps(
958
1433
  {
959
1434
  "experiment_id": experiment_id,
@@ -1039,7 +1514,13 @@ def _cleanup_experiment_coordination():
1039
1514
  """Remove coordination files after experiment."""
1040
1515
  import sys
1041
1516
 
1042
- for f in [XDIST_EXPERIMENT_FILE, XDIST_EXPERIMENT_FILE.with_suffix(".lock")]:
1517
+ context = context_from_environment()
1518
+ experiment_file = (
1519
+ resolve_managed_path(context, "locks", ".pytest_experiment_id")
1520
+ if context is not None
1521
+ else XDIST_EXPERIMENT_FILE
1522
+ )
1523
+ for f in [experiment_file, experiment_file.with_suffix(".lock")]:
1043
1524
  try:
1044
1525
  f.unlink(missing_ok=True)
1045
1526
  except Exception as e:
@@ -1067,8 +1548,8 @@ def worker_id(request):
1067
1548
 
1068
1549
  @pytest.fixture(scope="session", autouse=True)
1069
1550
  def verify_environment(project_root, request):
1070
- """Verify Docker, Claude CLI, uv, bash, and API keys are available."""
1071
- if _is_unit_tests_only(request.config):
1551
+ """Verify the selected agent's Docker execution prerequisites."""
1552
+ if _is_unit_tests_only(request.config) or request.config.option.collectonly:
1072
1553
  return
1073
1554
 
1074
1555
  from scaffold.python.utils import load_eval_environment
@@ -1101,55 +1582,55 @@ def verify_environment(project_root, request):
1101
1582
  if result.returncode != 0:
1102
1583
  pytest.skip("Docker not available")
1103
1584
 
1104
- if not (os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN")):
1105
- pytest.skip("ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN not set")
1106
-
1107
- if shutil.which("claude") is None:
1108
- pytest.skip("Claude CLI not available")
1585
+ selection = _resolve_eval_agent(request.config)
1586
+ _require_agent_credentials(selection.agent)
1109
1587
 
1110
1588
 
1111
1589
  def _docker_environment_dirs_for_request(request, tasks_dir: Path) -> list[Path]:
1112
1590
  """Return only task environments selected by the current eval invocation."""
1113
- if not tasks_dir.exists():
1114
- return []
1115
-
1116
- all_task_names = sorted(task_dir.name for task_dir in tasks_dir.iterdir() if task_dir.is_dir())
1117
- task_filter = request.config.getoption("--task")
1118
- manifest_path = request.config.getoption("--eval-manifest")
1119
- if task_filter:
1120
- task_names = [task_filter]
1121
- elif manifest_path:
1122
- from scaffold.python.manifests import load_eval_manifest
1123
-
1124
- task_names = load_eval_manifest(manifest_path).recommended_tasks or all_task_names
1125
- else:
1126
- task_names = all_task_names
1127
-
1128
- return [
1129
- environment_dir
1130
- for task_name in task_names
1131
- if (environment_dir := tasks_dir / task_name / "environment").is_dir()
1132
- and (environment_dir / "Dockerfile").is_file()
1133
- ]
1591
+ frozen = getattr(request.config, "_comet_frozen_task_set", None)
1592
+ if frozen is None:
1593
+ raise pytest.UsageError("Eval task collection did not produce a frozen task set")
1594
+ selected: list[Path] = []
1595
+ for item in frozen.tasks:
1596
+ environment_dir = item.task.environment_dir
1597
+ if (
1598
+ environment_dir.is_dir()
1599
+ and (environment_dir / "Dockerfile").is_file()
1600
+ and environment_dir not in selected
1601
+ ):
1602
+ selected.append(environment_dir)
1603
+ return selected
1134
1604
 
1135
1605
 
1136
1606
  @pytest.fixture(scope="session", autouse=True)
1137
1607
  def prebuild_docker_image(request):
1138
1608
  """Pre-build Docker image once per session to avoid race conditions."""
1139
- if _is_unit_tests_only(request.config):
1609
+ if _is_unit_tests_only(request.config) or request.config.option.collectonly:
1140
1610
  yield
1141
1611
  return
1142
1612
 
1143
1613
  tasks_dir = PROJECT_ROOT / "tasks"
1144
1614
  for env_dir in _docker_environment_dirs_for_request(request, tasks_dir):
1145
- image = _build_docker_image_with_lock(env_dir)
1615
+ selected_agent = _resolve_eval_agent(request.config).agent
1616
+ image = _build_docker_image_with_lock(
1617
+ env_dir,
1618
+ selected_agent if selected_agent != "claude-code" else None,
1619
+ environment=getattr(request.config, "_comet_execution_env", None),
1620
+ )
1146
1621
  if image:
1147
1622
  print(f"\nPre-built Docker image: {image}")
1148
1623
 
1149
1624
  yield
1150
1625
 
1151
1626
  try:
1152
- DOCKER_BUILD_LOCK.unlink(missing_ok=True)
1627
+ context = context_from_environment()
1628
+ lock_path = (
1629
+ resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
1630
+ if context is not None
1631
+ else DOCKER_BUILD_LOCK
1632
+ )
1633
+ lock_path.unlink(missing_ok=True)
1153
1634
  except Exception:
1154
1635
  pass
1155
1636
 
@@ -1181,7 +1662,8 @@ def setup_test_context(test_dir):
1181
1662
  scripts_dir: Path | None = None,
1182
1663
  source_dir: Path | None = None,
1183
1664
  ) -> None:
1184
- skill_dir = test_dir / ".claude" / "skills" / skill_name
1665
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1666
+ skill_dir = test_dir / _agent_project_root(agent) / "skills" / skill_name
1185
1667
  skill_dir.mkdir(parents=True, exist_ok=True)
1186
1668
 
1187
1669
  if source_dir and source_dir.is_dir():
@@ -1212,13 +1694,38 @@ def setup_test_context(test_dir):
1212
1694
  shutil.copy2(item, dest)
1213
1695
  _copy_current_comet_cli_snapshot(environment_dir, test_dir)
1214
1696
 
1697
+ def _copy_workspace(workspace_dir: Path) -> None:
1698
+ """Copy a bounded project fixture without replacing the task Dockerfile."""
1699
+ for item in workspace_dir.iterdir():
1700
+ if item.name in {"Dockerfile", CURRENT_COMET_CLI_MARKER, TRUSTED_NATIVE_RUNTIME_MARKER}:
1701
+ continue
1702
+ dest = test_dir / item.name
1703
+ if item.is_dir():
1704
+ shutil.copytree(item, dest, dirs_exist_ok=True)
1705
+ else:
1706
+ dest.parent.mkdir(parents=True, exist_ok=True)
1707
+ shutil.copy2(item, dest)
1708
+
1215
1709
  def _write_claude_md(content_file: str) -> None:
1216
- claude_dir = test_dir / ".claude"
1217
- claude_dir.mkdir(parents=True, exist_ok=True)
1218
- shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
1219
- shutil.copyfile(content_file, test_dir / "CLAUDE.md")
1710
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1711
+ if agent == "claude-code":
1712
+ claude_dir = test_dir / ".claude"
1713
+ claude_dir.mkdir(parents=True, exist_ok=True)
1714
+ shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
1715
+ shutil.copyfile(content_file, test_dir / "CLAUDE.md")
1716
+ elif agent == "codebuddy":
1717
+ codebuddy_dir = test_dir / ".codebuddy"
1718
+ codebuddy_dir.mkdir(parents=True, exist_ok=True)
1719
+ shutil.copyfile(content_file, codebuddy_dir / "CODEBUDDY.md")
1720
+ else:
1721
+ shutil.copyfile(content_file, test_dir / "AGENTS.md")
1220
1722
 
1221
- def _setup(skills: dict = None, claude_md: str = None, environment_dir: Path = None):
1723
+ def _setup(
1724
+ skills: dict = None,
1725
+ claude_md: str = None,
1726
+ environment_dir: Path = None,
1727
+ workspace_dir: Path = None,
1728
+ ):
1222
1729
  for skill_name, cfg in (skills or {}).items():
1223
1730
  if not cfg:
1224
1731
  continue
@@ -1251,6 +1758,9 @@ def setup_test_context(test_dir):
1251
1758
  if is_temp_dir and filtered_dir.exists():
1252
1759
  shutil.rmtree(filtered_dir)
1253
1760
 
1761
+ if workspace_dir and workspace_dir.exists():
1762
+ _copy_workspace(workspace_dir)
1763
+
1254
1764
  if environment_dir and environment_dir.exists():
1255
1765
  _copy_environment(environment_dir)
1256
1766
  _copy_trusted_native_runtime_snapshot(environment_dir, test_dir, skills)
@@ -1266,7 +1776,9 @@ def setup_test_context(test_dir):
1266
1776
  finally:
1267
1777
  os.unlink(temp_file)
1268
1778
 
1269
- _ensure_claude_pre_tool_hook(test_dir, _comet_hook_command(test_dir))
1779
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1780
+ _ensure_langfuse_trajectory_support(test_dir, agent)
1781
+ _ensure_agent_pre_tool_hook(test_dir, agent, _comet_hook_command(test_dir, agent))
1270
1782
 
1271
1783
  return test_dir
1272
1784
 
@@ -1275,13 +1787,15 @@ def setup_test_context(test_dir):
1275
1787
 
1276
1788
  @pytest.fixture
1277
1789
  def run_claude(test_dir, experiment_logger, request):
1278
- """Factory fixture to run Claude in Docker and capture artifacts.
1790
+ """Factory fixture to run the selected evaluation agent in Docker.
1279
1791
 
1280
1792
  For tasks using ``interaction.mode=auto_user`` the single-shot ``run-claude``
1281
1793
  is replaced by the multi-turn ``run-claude-loop`` driver, which simulates a
1282
1794
  user replying at the workflow's decision points.
1283
1795
  """
1284
- default_model = _selected_claude_model()
1796
+ execution = _resolve_eval_execution(request.config)
1797
+ selected_agent = execution.agent
1798
+ default_model = execution.model
1285
1799
 
1286
1800
  def _run(
1287
1801
  prompt: str,
@@ -1291,18 +1805,28 @@ def run_claude(test_dir, experiment_logger, request):
1291
1805
  image_id: str | None = None,
1292
1806
  ):
1293
1807
  mdl = model or default_model
1808
+ run_execution = ResolvedExecution(
1809
+ selected_agent,
1810
+ mdl,
1811
+ execution.base_url,
1812
+ execution.sources,
1813
+ )
1814
+ run_environment = build_agent_environment(run_execution)
1294
1815
  if os.environ.get("TRACE_TO_LANGSMITH", "").lower() == "true" and not os.environ.get(
1295
1816
  "CC_LANGSMITH_LOG_FILE"
1296
1817
  ):
1297
1818
  os.environ["CC_LANGSMITH_LOG_FILE"] = "/workspace/langsmith-hook.log"
1298
1819
  use_loop = interaction is not None and interaction.mode == "auto_user"
1299
1820
  if not use_loop:
1300
- result = run_claude_in_docker(
1821
+ result = run_agent_in_docker(
1301
1822
  test_dir,
1302
1823
  prompt,
1824
+ agent=selected_agent,
1303
1825
  timeout=timeout,
1304
1826
  model=mdl,
1827
+ base_url=execution.base_url,
1305
1828
  image_id=image_id,
1829
+ environment=run_environment,
1306
1830
  )
1307
1831
  else:
1308
1832
  task_prompt_file = test_dir / ".eval-task-prompt.txt"
@@ -1342,11 +1866,25 @@ def run_claude(test_dir, experiment_logger, request):
1342
1866
  "--simulator-prompt-file",
1343
1867
  "//workspace/.eval-simulator-prompt.txt",
1344
1868
  ]
1345
- result = run_claude_loop_in_docker(
1346
- test_dir,
1347
- loop_args[2:],
1348
- timeout=timeout + 60,
1349
- )
1869
+ if selected_agent == "claude-code":
1870
+ result = run_claude_loop_in_docker(
1871
+ test_dir,
1872
+ loop_args[2:],
1873
+ timeout=timeout + 60,
1874
+ environment=run_environment,
1875
+ )
1876
+ else:
1877
+ result = run_agent_loop_in_docker(
1878
+ test_dir,
1879
+ [
1880
+ loop_args[2],
1881
+ "--agent",
1882
+ selected_agent,
1883
+ *loop_args[3:],
1884
+ ],
1885
+ timeout=timeout + 60,
1886
+ environment=run_environment,
1887
+ )
1350
1888
  finally:
1351
1889
  task_prompt_file.unlink(missing_ok=True)
1352
1890
  if prompt_file and prompt_file.exists():
@@ -1384,12 +1922,22 @@ def record_result(test_dir, experiment_logger, request):
1384
1922
  if not experiment_logger:
1385
1923
  return
1386
1924
 
1925
+ if _plugin is not None:
1926
+ _plugin.last_test_dir = test_dir
1927
+
1387
1928
  treatment_name = _get_treatment_name(request.node)
1388
1929
  rep = _plugin.run_counter.get(treatment_name, 1) if _plugin else 1
1930
+ events["sample"] = rep
1389
1931
  base_dir = experiment_logger.base_dir
1390
1932
 
1391
1933
  save_events(base_dir, treatment_name, rep, events)
1392
- _save_artifacts(base_dir, treatment_name, rep, test_dir)
1934
+ _save_artifacts(
1935
+ base_dir,
1936
+ treatment_name,
1937
+ rep,
1938
+ test_dir,
1939
+ agent=events.get("agent", "claude-code"),
1940
+ )
1393
1941
  artifact_references = build_eval_artifact_references(base_dir, treatment_name, rep)
1394
1942
 
1395
1943
  scripts_used = _extract_scripts_used(events)
@@ -1439,9 +1987,23 @@ def record_result(test_dir, experiment_logger, request):
1439
1987
  "skill_sources": events.get("skill_sources", []),
1440
1988
  "eval_manifest": events.get("eval_manifest"),
1441
1989
  "case_manifest": events.get("case_manifest"),
1990
+ "eval_generation": events.get("eval_generation"),
1442
1991
  "interaction": events.get("interaction", {}),
1443
1992
  "artifact_references": artifact_references,
1444
1993
  "failure_attribution": failure_attribution,
1994
+ "agent": events.get("agent", "claude-code"),
1995
+ "role_models": events.get("role_models", {}),
1996
+ "role_agents": events.get("role_agents", {}),
1997
+ "telemetry_status": events.get("telemetry_status", "N/A"),
1998
+ "role_sessions": events.get("role_sessions", {}),
1999
+ "task": events.get("task"),
2000
+ "treatment": events.get("treatment", treatment_name),
2001
+ "sample": events.get("sample", rep),
2002
+ "prompt": events.get("prompt"),
2003
+ "skill": events.get("skill"),
2004
+ "final_response": events.get("final_response"),
2005
+ "quality_gates": events.get("quality_gates", {}),
2006
+ "execution_identity": events.get("execution_identity"),
1445
2007
  },
1446
2008
  run_id=run_id,
1447
2009
  ),
@@ -1495,14 +2057,29 @@ def _build_report_payload(
1495
2057
  "model_usage": events.get("model_usage", {}),
1496
2058
  "files_created": events.get("files_created", []),
1497
2059
  "skills_invoked": events.get("skills_invoked", []),
2060
+ "skill_invocations": events.get("skill_invocations", []),
1498
2061
  "scripts_used": scripts_used,
1499
2062
  "profile": events.get("profile"),
1500
2063
  "skill_sources": events.get("skill_sources", []),
1501
2064
  "eval_manifest": events.get("eval_manifest"),
1502
2065
  "case_manifest": events.get("case_manifest"),
2066
+ "eval_generation": events.get("eval_generation"),
1503
2067
  "interaction": events.get("interaction", {}),
1504
2068
  "artifact_references": artifact_references,
1505
2069
  "failure_attribution": failure_attribution,
2070
+ "agent": events.get("agent", "claude-code"),
2071
+ "role_models": events.get("role_models", {}),
2072
+ "role_agents": events.get("role_agents", {}),
2073
+ "telemetry_status": events.get("telemetry_status", "N/A"),
2074
+ "role_sessions": events.get("role_sessions", {}),
2075
+ "task": events.get("task"),
2076
+ "treatment": events.get("treatment", treatment_name),
2077
+ "sample": events.get("sample", rep),
2078
+ "prompt": events.get("prompt"),
2079
+ "skill": events.get("skill"),
2080
+ "final_response": events.get("final_response"),
2081
+ "quality_gates": events.get("quality_gates", {}),
2082
+ "execution_identity": events.get("execution_identity"),
1506
2083
  },
1507
2084
  "timestamp": datetime.now().isoformat(),
1508
2085
  }
@@ -1581,15 +2158,32 @@ def _filter_scripts(scripts_dir: Path, script_filter: str) -> Path | None:
1581
2158
  return temp_dir
1582
2159
 
1583
2160
 
1584
- def _build_docker_image_with_lock(environment_dir: Path) -> str | None:
2161
+ def _build_docker_image_with_lock(
2162
+ environment_dir: Path,
2163
+ agent: str | None = None,
2164
+ *,
2165
+ environment: dict[str, str] | None = None,
2166
+ ) -> str | None:
1585
2167
  """Build Docker image with file locking to prevent race conditions."""
1586
2168
  if not environment_dir or not (environment_dir / "Dockerfile").exists():
1587
2169
  return None
1588
2170
 
1589
- with file_lock(DOCKER_BUILD_LOCK):
1590
- # A cold image build downloads Debian packages and installs the Claude CLI.
2171
+ context = context_from_environment()
2172
+ lock_path = (
2173
+ resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
2174
+ if context is not None
2175
+ else DOCKER_BUILD_LOCK
2176
+ )
2177
+ with file_lock(lock_path):
2178
+ # A cold image build downloads Debian packages and installs the selected CLI.
1591
2179
  # Five minutes is insufficient after cache cleanup or on a proxied connection.
1592
- result = run_shell("docker.sh", "build", str(environment_dir), timeout=900, check=False)
2180
+ command = ["docker.sh", "build", str(environment_dir)]
2181
+ if agent:
2182
+ command.extend(["--agent", agent])
2183
+ run_options = {"timeout": 900, "check": False}
2184
+ if environment is not None:
2185
+ run_options["env"] = environment
2186
+ result = run_shell(*command, **run_options)
1593
2187
  if result.returncode == 0:
1594
2188
  return result.stdout.strip()
1595
2189
  return None
@@ -1664,18 +2258,34 @@ def _extract_scripts_used(events: dict) -> list[str]:
1664
2258
  return [s for s in _get_known_scripts() if s.lower() in all_activity]
1665
2259
 
1666
2260
 
1667
- def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Path):
1668
- """Save Claude's generated files as artifacts."""
2261
+ def _save_artifacts(
2262
+ base_dir: Path,
2263
+ treatment_name: str,
2264
+ rep: int,
2265
+ test_dir: Path,
2266
+ *,
2267
+ agent: str = "claude-code",
2268
+ ):
2269
+ """Save the selected Agent's generated files as artifacts."""
1669
2270
  artifacts_dir = base_dir / "artifacts" / f"{treatment_name.lower()}_rep{rep}"
1670
- claude_dir = artifacts_dir / "claude"
2271
+ agent_dir_name = {
2272
+ "claude-code": "claude",
2273
+ "codex": "codex",
2274
+ "qoder": "qoder",
2275
+ "codebuddy": "codebuddy",
2276
+ }.get(agent, agent)
2277
+ agent_dir = artifacts_dir / agent_dir_name
1671
2278
  execution_dir = artifacts_dir / "execution"
1672
- claude_dir.mkdir(parents=True, exist_ok=True)
2279
+ agent_dir.mkdir(parents=True, exist_ok=True)
1673
2280
  execution_dir.mkdir(parents=True, exist_ok=True)
1674
2281
 
1675
2282
  from scaffold.python.utils import TEST_CONTEXT_FILE, TEST_RESULTS_FILE
1676
2283
 
1677
2284
  exclude_dirs = {
1678
2285
  ".claude",
2286
+ ".agents",
2287
+ ".qoder",
2288
+ ".codebuddy",
1679
2289
  ".git",
1680
2290
  "_eval_current_comet",
1681
2291
  "node_modules",
@@ -1707,9 +2317,20 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
1707
2317
  continue
1708
2318
  try:
1709
2319
  rel_path = item.relative_to(test_dir)
1710
- dest = claude_dir / rel_path
2320
+ dest = agent_dir / rel_path
1711
2321
  dest.parent.mkdir(parents=True, exist_ok=True)
1712
- shutil.copy(item, dest)
2322
+ try:
2323
+ content = item.read_text(encoding="utf-8")
2324
+ except UnicodeDecodeError:
2325
+ import sys
2326
+
2327
+ print(
2328
+ f"Warning: skipped non-text artifact {rel_path}; binary content is not persisted to protect credentials",
2329
+ file=sys.stderr,
2330
+ )
2331
+ continue
2332
+ else:
2333
+ dest.write_text(redact_sensitive(content), encoding="utf-8")
1713
2334
  claude_files.append(item)
1714
2335
  except Exception as e:
1715
2336
  import sys
@@ -1722,7 +2343,7 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
1722
2343
  success, output = run_python_in_docker(test_dir, py_file.name, timeout=300)
1723
2344
  status = "success" if success else "error"
1724
2345
  output_file = execution_dir / f"{py_file.stem}_{status}.txt"
1725
- output_file.write_text(strip_ansi(output))
2346
+ output_file.write_text(redact_sensitive(strip_ansi(output)), encoding="utf-8")
1726
2347
  except Exception as e:
1727
2348
  error_file = execution_dir / f"{py_file.stem}_error.txt"
1728
- error_file.write_text(str(e))
2349
+ error_file.write_text(redact_sensitive(str(e)), encoding="utf-8")