@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -20,11 +20,45 @@ def test_import_does_not_read_dotenv(monkeypatch):
20
20
 
21
21
  assert calls == []
22
22
  utils.load_eval_environment()
23
- assert len(calls) == 2
23
+ assert len(calls) == 3
24
24
  monkeypatch.undo()
25
25
  importlib.reload(utils)
26
26
 
27
27
 
28
+ def test_load_eval_environment_prefers_user_file_but_preserves_process_values(
29
+ tmp_path, monkeypatch
30
+ ):
31
+ package = tmp_path / "package"
32
+ suite = tmp_path / "suite"
33
+ user = tmp_path / ".comet" / "eval"
34
+ package.mkdir()
35
+ suite.mkdir()
36
+ user.mkdir(parents=True)
37
+ (package / ".env").write_text(
38
+ "BENCH_MODEL=package\nBENCH_BASE_URL=https://package.example\n",
39
+ encoding="utf-8",
40
+ )
41
+ (suite / ".env").write_text(
42
+ "BENCH_MODEL=suite\nBENCH_BASE_URL=https://suite.example\n",
43
+ encoding="utf-8",
44
+ )
45
+ (user / ".env").write_text(
46
+ "BENCH_MODEL=user\nBENCH_BASE_URL=https://user.example\n",
47
+ encoding="utf-8",
48
+ )
49
+
50
+ monkeypatch.setattr(utils, "EVAL_ROOT", package)
51
+ monkeypatch.setattr(utils, "get_suite_root", lambda: suite)
52
+ monkeypatch.setattr(utils.Path, "home", staticmethod(lambda: tmp_path))
53
+ monkeypatch.setenv("BENCH_MODEL", "process")
54
+ monkeypatch.delenv("BENCH_BASE_URL", raising=False)
55
+
56
+ utils.load_eval_environment()
57
+
58
+ assert os.environ["BENCH_MODEL"] == "process"
59
+ assert os.environ["BENCH_BASE_URL"] == "https://user.example"
60
+
61
+
28
62
  def test_execution_validator_converts_structured_checks(monkeypatch, tmp_path: Path):
29
63
  def fake_run_eval_in_docker(*_args, **_kwargs):
30
64
  return {
@@ -71,6 +105,32 @@ def test_docker_script_failure_preserves_stderr(monkeypatch, tmp_path: Path):
71
105
  assert "validator import failed" in output
72
106
 
73
107
 
108
+ def test_run_command_in_docker_passes_timeout_and_shell_command(monkeypatch, tmp_path: Path):
109
+ captured = {}
110
+
111
+ def fake_run_shell(*args, **kwargs):
112
+ captured["args"] = args
113
+ captured["kwargs"] = kwargs
114
+ return subprocess.CompletedProcess(args=args, returncode=0, stdout="ok", stderr="")
115
+
116
+ monkeypatch.setattr(utils, "check_docker_available", lambda: True)
117
+ monkeypatch.setattr(utils, "run_shell", fake_run_shell)
118
+
119
+ result = utils.run_command_in_docker(tmp_path, "pnpm test", timeout=45)
120
+
121
+ assert result.returncode == 0
122
+ assert captured["args"] == (
123
+ "docker.sh",
124
+ "run-command",
125
+ str(tmp_path),
126
+ "--timeout",
127
+ "45",
128
+ "--",
129
+ "pnpm test",
130
+ )
131
+ assert captured["kwargs"]["timeout"] == 75
132
+
133
+
74
134
  def test_run_shell_decodes_subprocess_output_as_utf8(monkeypatch):
75
135
  captured = {}
76
136
 
@@ -156,6 +216,95 @@ def test_bash_env_bridges_eval_keys_to_wsl(monkeypatch):
156
216
  assert "ANTHROPIC_API_KEY" not in names
157
217
 
158
218
 
219
+ def test_bash_env_bridges_selectable_agent_credentials_to_wsl(monkeypatch):
220
+ monkeypatch.setattr(utils.os, "name", "nt")
221
+ monkeypatch.setattr(
222
+ utils,
223
+ "BASH_EXEC",
224
+ r"C:\Users\BENYM\AppData\Local\Microsoft\WindowsApps\bash.exe",
225
+ )
226
+ monkeypatch.setenv("WSLENV", "")
227
+ monkeypatch.setenv("QODER_PERSONAL_ACCESS_TOKEN", "qoder-token")
228
+
229
+ env = utils._bash_env()
230
+
231
+ assert "QODER_PERSONAL_ACCESS_TOKEN" in env["WSLENV"].split(":")
232
+
233
+
234
+ def test_bash_env_bridges_codebuddy_credentials_and_model_to_wsl(monkeypatch):
235
+ monkeypatch.setattr(utils.os, "name", "nt")
236
+ monkeypatch.setattr(
237
+ utils,
238
+ "BASH_EXEC",
239
+ r"C:\Users\BENYM\AppData\Local\Microsoft\WindowsApps\bash.exe",
240
+ )
241
+ monkeypatch.setenv("WSLENV", "")
242
+ monkeypatch.setenv("CODEBUDDY_API_KEY", "codebuddy-key")
243
+ monkeypatch.setenv("CODEBUDDY_MODEL", "codebuddy-model")
244
+ monkeypatch.setenv("CODEBUDDY_SMALL_FAST_MODEL", "codebuddy-fast-model")
245
+ monkeypatch.setenv("CODEBUDDY_BIG_SLOW_MODEL", "codebuddy-reasoning-model")
246
+ monkeypatch.setenv("CODEBUDDY_CODE_SUBAGENT_MODEL", "codebuddy-subagent-model")
247
+
248
+ env = utils._bash_env()
249
+
250
+ assert "CODEBUDDY_API_KEY" in env["WSLENV"].split(":")
251
+ assert "CODEBUDDY_MODEL" in env["WSLENV"].split(":")
252
+ assert "CODEBUDDY_SMALL_FAST_MODEL" in env["WSLENV"].split(":")
253
+ assert "CODEBUDDY_BIG_SLOW_MODEL" in env["WSLENV"].split(":")
254
+ assert "CODEBUDDY_CODE_SUBAGENT_MODEL" in env["WSLENV"].split(":")
255
+
256
+
257
+ def test_bash_env_bridges_custom_agent_model_and_base_url_to_wsl(monkeypatch):
258
+ monkeypatch.setattr(utils.os, "name", "nt")
259
+ monkeypatch.setattr(
260
+ utils,
261
+ "BASH_EXEC",
262
+ r"C:\Users\BENYM\AppData\Local\Microsoft\WindowsApps\bash.exe",
263
+ )
264
+ monkeypatch.setenv("WSLENV", "")
265
+ monkeypatch.setenv("COMET_EVAL_CUSTOM_MODEL_ENV", "FIXTURE_AGENT_MODEL")
266
+ monkeypatch.setenv("COMET_EVAL_CUSTOM_BASE_URL_ENV", "FIXTURE_AGENT_BASE_URL")
267
+ monkeypatch.setenv("FIXTURE_AGENT_MODEL", "fixture-model")
268
+ monkeypatch.setenv("FIXTURE_AGENT_BASE_URL", "https://fixture.example/v1")
269
+
270
+ env = utils._bash_env()
271
+
272
+ assert "FIXTURE_AGENT_MODEL" in env["WSLENV"].split(":")
273
+ assert "FIXTURE_AGENT_BASE_URL" in env["WSLENV"].split(":")
274
+
275
+
276
+ def test_run_agent_in_docker_builds_the_selected_adapter_command(monkeypatch, tmp_path: Path):
277
+ calls = []
278
+
279
+ def fake_run_shell(script, *args, **kwargs):
280
+ calls.append((script, args, kwargs))
281
+ return subprocess.CompletedProcess([script, *args], 0, "", "")
282
+
283
+ monkeypatch.setattr(utils, "run_shell", fake_run_shell)
284
+ monkeypatch.setattr(utils, "check_docker_available", lambda: True)
285
+
286
+ result = utils.run_agent_in_docker(
287
+ str(tmp_path),
288
+ "inspect the task",
289
+ agent="codex",
290
+ model="test-model",
291
+ timeout=42,
292
+ )
293
+
294
+ assert result.returncode == 0
295
+ assert calls[0][1] == (
296
+ "run-agent",
297
+ str(tmp_path),
298
+ "inspect the task",
299
+ "--agent",
300
+ "codex",
301
+ "--model",
302
+ "test-model",
303
+ "--timeout",
304
+ "42",
305
+ )
306
+
307
+
159
308
  def test_bash_env_bridges_langsmith_hook_log_to_wsl(monkeypatch):
160
309
  monkeypatch.setattr(utils.os, "name", "nt")
161
310
  monkeypatch.setattr(
@@ -200,6 +349,26 @@ def test_claude_loop_timeout_force_removes_its_named_container(monkeypatch, tmp_
200
349
  assert "cleanup-claude-loop" in docker_sh
201
350
 
202
351
 
352
+ def test_agent_loop_timeout_force_removes_its_named_container(monkeypatch, tmp_path: Path):
353
+ calls = []
354
+
355
+ def fake_run_shell(script, *args, **kwargs):
356
+ calls.append((script, args, kwargs))
357
+ if args[0] == "run-agent-loop":
358
+ raise subprocess.TimeoutExpired([script, *args], timeout=42)
359
+ return subprocess.CompletedProcess([script, *args], 0, "", "")
360
+
361
+ monkeypatch.setattr(utils, "run_shell", fake_run_shell)
362
+
363
+ result = utils.run_agent_loop_in_docker(
364
+ str(tmp_path), ["prompt", "--agent", "codex"], timeout=42
365
+ )
366
+
367
+ assert result.returncode == 124
368
+ assert calls[0][1][0] == "run-agent-loop"
369
+ assert calls[1][1] == ("cleanup-agent-loop", str(tmp_path))
370
+
371
+
203
372
  def test_docker_subject_run_uses_controller_verified_immutable_image_identity():
204
373
  docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
205
374
 
@@ -210,6 +379,182 @@ def test_docker_subject_run_uses_controller_verified_immutable_image_identity():
210
379
  assert '"$image_id"' in docker_sh
211
380
 
212
381
 
382
+ def test_docker_script_dispatches_all_supported_agent_clis():
383
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
384
+ runtime_sh = (utils.SHELL_DIR / "run-agent-runtime.sh").read_text(encoding="utf-8")
385
+
386
+ assert "run-agent" in docker_sh
387
+ assert "codex exec" in runtime_sh
388
+ assert "qodercli" in runtime_sh
389
+ assert "QODER_PERSONAL_ACCESS_TOKEN" in docker_sh
390
+ assert "codebuddy" in runtime_sh
391
+ assert "CODEBUDDY_API_KEY" in docker_sh
392
+
393
+ overlay = (utils.SHELL_DIR.parent / "docker" / "agent-overlay.Dockerfile").read_text(
394
+ encoding="utf-8"
395
+ )
396
+ assert "@tencent-ai/codebuddy-code" in overlay
397
+
398
+
399
+ def test_codebuddy_forwards_official_model_variant_environment_variables():
400
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
401
+
402
+ assert "CODEBUDDY_SMALL_FAST_MODEL" in docker_sh
403
+ assert "CODEBUDDY_BIG_SLOW_MODEL" in docker_sh
404
+ assert "CODEBUDDY_CODE_SUBAGENT_MODEL" in docker_sh
405
+
406
+
407
+ def test_codex_commands_use_explicit_openai_base_url_config():
408
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
409
+ loop_sh = (utils.SHELL_DIR / "run-claude-loop.sh").read_text(encoding="utf-8")
410
+
411
+ assert 'base_url = ' in (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
412
+ assert 'model_provider = "comet-eval"' in (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
413
+ assert 'openai_base_url=$OPENAI_BASE_URL' not in docker_sh
414
+ assert 'openai_base_url=$OPENAI_BASE_URL' not in loop_sh
415
+
416
+
417
+ def test_agent_runtime_credentials_use_ephemeral_cli_config_roots():
418
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
419
+ config_sh = (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
420
+
421
+ assert "run-agent-runtime.sh" in docker_sh
422
+ assert "--tmpfs" in docker_sh
423
+ assert "bash //opt/scaffold-shell/run-agent-runtime.sh" in docker_sh
424
+ assert "/home/agent/.codex" in docker_sh
425
+ assert "/home/agent/.qoder" in docker_sh
426
+ assert "/home/agent/.codebuddy" in docker_sh
427
+ assert 'env_key = "OPENAI_API_KEY"' in config_sh
428
+ assert 'model_provider = "comet-eval"' in config_sh
429
+ assert '"apiKeyHelper"' in config_sh
430
+ assert "CODEBUDDY_API_KEY" in config_sh
431
+ assert "QODER_CONFIG_DIR" in config_sh
432
+ assert "config.toml" in config_sh
433
+ assert "settings.json" in config_sh
434
+
435
+
436
+ def test_agent_runtime_config_is_agent_only_and_docker_does_not_inline_secrets():
437
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
438
+ config_sh = (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
439
+ generic_run = docker_sh.split("docker_run() {", 1)[1].split(
440
+ "# Run Claude CLI in Docker", 1
441
+ )[0]
442
+
443
+ assert "COMET_EVAL_CODEBUDDY_CONFIG_DIR" in config_sh
444
+ assert 'ENV_ARGS+=("-e" "$key")' in docker_sh
445
+ assert 'ENV_ARGS+=("-e" "$key=${!key}")' not in docker_sh
446
+ assert "build_agent_runtime_mount_args" not in generic_run
447
+
448
+
449
+ def test_agent_runtime_config_files_never_contain_credential_literals(tmp_path):
450
+ codex_home = utils._to_bash_path(tmp_path / "codex")
451
+ qoder_home = utils._to_bash_path(tmp_path / "qoder")
452
+ codebuddy_home = utils._to_bash_path(tmp_path / "codebuddy")
453
+ script = utils._to_bash_path(utils.SHELL_DIR / "agent-runtime-config.sh")
454
+ env = dict(os.environ)
455
+ env.update(
456
+ {
457
+ "CODEX_HOME": codex_home,
458
+ "QODER_CONFIG_DIR": qoder_home,
459
+ "COMET_EVAL_CODEBUDDY_CONFIG_DIR": codebuddy_home,
460
+ "OPENAI_API_KEY": "codex-sentinel",
461
+ "OPENAI_BASE_URL": "https://codex.example/v1",
462
+ "CODEBUDDY_API_KEY": "codebuddy-sentinel",
463
+ "CODEBUDDY_BASE_URL": "https://codebuddy.example/v1",
464
+ }
465
+ )
466
+ result = subprocess.run(
467
+ [
468
+ utils.BASH_EXEC,
469
+ "-c",
470
+ f'source "{script}"; '
471
+ 'prepare_agent_runtime_config codex "subject-model"; '
472
+ 'prepare_agent_runtime_config qoder "subject-model"; '
473
+ 'prepare_agent_runtime_config codebuddy "subject-model"',
474
+ ],
475
+ env=utils._bash_env(env),
476
+ capture_output=True,
477
+ text=True,
478
+ check=False,
479
+ )
480
+
481
+ assert result.returncode == 0, result.stderr
482
+ config_text = (tmp_path / "codex" / "config.toml").read_text(encoding="utf-8")
483
+ settings_text = (tmp_path / "codebuddy" / "settings.json").read_text(encoding="utf-8")
484
+ helper_text = (tmp_path / "codebuddy" / "api-key-helper.sh").read_text(encoding="utf-8")
485
+
486
+ assert 'env_key = "OPENAI_API_KEY"' in config_text
487
+ assert "codex-sentinel" not in config_text
488
+ assert '"apiKeyHelper"' in settings_text
489
+ assert "codebuddy-sentinel" not in settings_text
490
+ assert "codebuddy-sentinel" not in helper_text
491
+ assert "CODEBUDDY_API_KEY" in helper_text
492
+ assert not list((tmp_path / "qoder").iterdir())
493
+
494
+
495
+ def test_codebuddy_api_key_helper_prefers_auth_token_without_persisting_it(tmp_path):
496
+ codebuddy_home = utils._to_bash_path(tmp_path / "codebuddy")
497
+ script = utils._to_bash_path(utils.SHELL_DIR / "agent-runtime-config.sh")
498
+ env = dict(os.environ)
499
+ env.update(
500
+ {
501
+ "COMET_EVAL_CODEBUDDY_CONFIG_DIR": codebuddy_home,
502
+ "CODEBUDDY_AUTH_TOKEN": "auth-token-sentinel",
503
+ "CODEBUDDY_API_KEY": "api-key-sentinel",
504
+ }
505
+ )
506
+ result = subprocess.run(
507
+ [
508
+ utils.BASH_EXEC,
509
+ "-c",
510
+ f'source "{script}"; '
511
+ 'prepare_agent_runtime_config codebuddy "subject-model"; '
512
+ 'bash "$COMET_EVAL_CODEBUDDY_CONFIG_DIR/api-key-helper.sh"',
513
+ ],
514
+ env=utils._bash_env(env),
515
+ capture_output=True,
516
+ text=True,
517
+ check=False,
518
+ )
519
+
520
+ assert result.returncode == 0, result.stderr
521
+ assert result.stdout == "auth-token-sentinel"
522
+ helper_text = (tmp_path / "codebuddy" / "api-key-helper.sh").read_text(encoding="utf-8")
523
+ assert "auth-token-sentinel" not in helper_text
524
+ assert "api-key-sentinel" not in helper_text
525
+
526
+
527
+ def test_user_authored_validation_commands_receive_no_agent_or_reporting_secrets():
528
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
529
+ command_block = docker_sh.split("docker_run_command() {", 1)[1].split(
530
+ "# Run Python script in Docker", 1
531
+ )[0]
532
+
533
+ assert "build_env_args" not in command_block
534
+ assert "ENV_ARGS" not in command_block
535
+
536
+
537
+ def test_docker_harness_mounts_langfuse_official_plugin_and_enables_trace_image():
538
+ docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
539
+ overlay = (utils.SHELL_DIR.parent / "docker" / "agent-overlay.Dockerfile").read_text(
540
+ encoding="utf-8"
541
+ )
542
+
543
+ assert "LANGFUSE_TRAJECTORY_PLUGIN_DIR" in docker_sh
544
+ assert "build_langfuse_plugin_args" in docker_sh
545
+ assert "comet-langfuse-plugin" in docker_sh
546
+ assert "LANGFUSE_ENABLED" in overlay
547
+
548
+
549
+ def test_interaction_driver_accepts_the_selected_agent():
550
+ loop_sh = (utils.SHELL_DIR / "run-claude-loop.sh").read_text(encoding="utf-8")
551
+
552
+ assert "--agent" in loop_sh
553
+ assert "codex exec" in loop_sh
554
+ assert "qodercli" in loop_sh
555
+ assert "codebuddy" in loop_sh
556
+
557
+
213
558
  def test_docker_harness_has_no_native_review_sidecar_contract():
214
559
  docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
215
560
 
@@ -247,6 +592,44 @@ def _get_image_name(directory: Path) -> str:
247
592
  return result.stdout.strip()
248
593
 
249
594
 
595
+ def _get_custom_image_name(directory: Path, *, version: str) -> str:
596
+ """Resolve a custom-agent image name with the selected install metadata."""
597
+ script = (
598
+ 'source "$1"; '
599
+ 'image=$(get_image_name "$2" fixture-agent) || { echo "image lookup failed"; exit 1; }; '
600
+ 'echo "image=$image"'
601
+ )
602
+ environment = {
603
+ **os.environ,
604
+ "COMET_EVAL_CUSTOM_AGENT_ID": "fixture-agent",
605
+ "COMET_EVAL_CUSTOM_EXECUTABLE": "fixture-agent-cli",
606
+ "COMET_EVAL_CUSTOM_INSTALL_KIND": "npm",
607
+ "COMET_EVAL_CUSTOM_INSTALL_PACKAGE": "fixture-agent-package",
608
+ "COMET_EVAL_CUSTOM_INSTALL_VERSION": version,
609
+ }
610
+ try:
611
+ result = subprocess.run(
612
+ [
613
+ utils.BASH_EXEC,
614
+ "-c",
615
+ script,
616
+ "_",
617
+ utils._to_bash_path(utils.SHELL_DIR / "docker.sh"),
618
+ utils._to_bash_path(directory),
619
+ ],
620
+ env=environment,
621
+ capture_output=True,
622
+ text=True,
623
+ encoding="utf-8",
624
+ errors="replace",
625
+ check=False,
626
+ )
627
+ except FileNotFoundError:
628
+ pytest.skip("bash not available")
629
+ assert result.returncode == 0, f"get_image_name failed (stderr: {result.stderr})"
630
+ return result.stdout.strip()
631
+
632
+
250
633
  def test_get_image_name_falls_back_to_environment_dockerfile(tmp_path: Path):
251
634
  """get_image_name() resolves environment/Dockerfile when dir/Dockerfile is absent.
252
635
 
@@ -291,6 +674,13 @@ def test_get_image_name_prefers_root_dockerfile_over_environment(tmp_path: Path)
291
674
  assert _get_image_name(both) != _get_image_name(env_only)
292
675
 
293
676
 
677
+ def test_custom_image_name_changes_when_install_version_changes(tmp_path: Path):
678
+ (tmp_path / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
679
+ first = _get_custom_image_name(tmp_path, version="1.0.0")
680
+ second = _get_custom_image_name(tmp_path, version="2.0.0")
681
+ assert first != second
682
+
683
+
294
684
  def test_run_claude_fixture_defaults_langsmith_hook_log_path():
295
685
  conftest_py = (Path(__file__).resolve().parents[1] / "conftest.py").read_text(encoding="utf-8")
296
686
 
@@ -310,13 +700,15 @@ def test_claude_loop_applies_plugin_args_to_subject_turns_only():
310
700
  'bash "$SCRIPT_DIR/decision-point.sh" "$RESULT_TEXT"'
311
701
  )
312
702
  assert 'SUBJECT_PROMPT="${FRESH_PROMPT:-$PROMPT}"' in loop_sh
313
- assert 'claude -p "$SUBJECT_PROMPT" "${PLUGIN_ARGS[@]}"' in loop_sh
314
- assert 'claude -p "$USER_REPLY" "${PLUGIN_ARGS[@]}"' in loop_sh
703
+ assert 'run_agent_turn "$SUBJECT_PROMPT" "" "subject"' in loop_sh
704
+ assert 'role_plugin_args=("${PLUGIN_ARGS[@]}")' in loop_sh
705
+ assert 'claude -p "$prompt" "${role_plugin_args[@]}"' in loop_sh
706
+ assert 'run_agent_turn "$USER_REPLY" "$SESSION_ID" "subject"' in loop_sh
315
707
  assert "fresh resume boundary detected" in loop_sh
316
708
  assert 'claude -p "$sim_prompt" "${PLUGIN_ARGS[@]}"' not in loop_sh
317
709
  assert 'if ! rm -f -- "$SIMULATOR_PROMPT_FILE"; then' in loop_sh
318
710
  assert loop_sh.index('rm -f -- "$SIMULATOR_PROMPT_FILE"') < loop_sh.index(
319
- 'RAW=$(claude -p "$SUBJECT_PROMPT"'
711
+ 'RAW=$(run_agent_turn "$SUBJECT_PROMPT"'
320
712
  )
321
713
 
322
714
 
@@ -370,6 +762,90 @@ printf '%s\n' '{"type":"result","subtype":"success","session_id":"session-1","re
370
762
  assert "subject turn 2 failed" in result.stderr
371
763
 
372
764
 
765
+ def test_claude_loop_surfaces_simulator_failure(tmp_path: Path):
766
+ fake_bin = tmp_path / "bin"
767
+ fake_bin.mkdir()
768
+ fake_claude = fake_bin / "claude"
769
+ fake_claude.write_text(
770
+ """#!/usr/bin/env bash
771
+ if [[ "$COMET_EVAL_AGENT_ROLE" == "simulator" ]]; then
772
+ echo "simulator failed" >&2
773
+ exit 44
774
+ fi
775
+ printf '%s\n' '{"type":"system","session_id":"subject-1"}'
776
+ printf '%s\n' '{"type":"result","subtype":"success","session_id":"subject-1","result":"Question: Which option should be used? Recommendation: A."}'
777
+ """,
778
+ encoding="utf-8",
779
+ newline="\n",
780
+ )
781
+ fake_claude.chmod(0o755)
782
+
783
+ env = os.environ.copy()
784
+ env["PATH"] = f"{utils._to_bash_path(fake_bin)}:{env.get('PATH', '')}"
785
+ result = subprocess.run(
786
+ [
787
+ utils.BASH_EXEC,
788
+ utils._to_bash_path(utils.SHELL_DIR / "run-claude-loop.sh"),
789
+ "Implement the requested change.",
790
+ "--max-turns",
791
+ "2",
792
+ ],
793
+ capture_output=True,
794
+ text=True,
795
+ encoding="utf-8",
796
+ errors="replace",
797
+ timeout=10,
798
+ check=False,
799
+ env=env,
800
+ )
801
+
802
+ assert result.returncode == 44
803
+ assert "simulator turn failed (exit 44)" in result.stderr
804
+
805
+
806
+ def test_non_claude_loop_does_not_inherit_anthropic_model(tmp_path: Path):
807
+ fake_bin = tmp_path / "bin"
808
+ fake_bin.mkdir()
809
+ captured_args = tmp_path / "codex-args.txt"
810
+ fake_codex = fake_bin / "codex"
811
+ fake_codex.write_text(
812
+ """#!/usr/bin/env bash
813
+ printf '%s\n' "$*" > "$CAPTURED_CODEX_ARGS"
814
+ printf '%s\n' '{"type":"thread.started","thread_id":"subject-1"}'
815
+ printf '%s\n' '{"type":"result","result":"Workflow completed through all phases and archived."}'
816
+ """,
817
+ encoding="utf-8",
818
+ newline="\n",
819
+ )
820
+ fake_codex.chmod(0o755)
821
+
822
+ env = os.environ.copy()
823
+ env["PATH"] = f"{utils._to_bash_path(fake_bin)}:{env.get('PATH', '')}"
824
+ env["ANTHROPIC_MODEL"] = "claude-only-model"
825
+ env["CAPTURED_CODEX_ARGS"] = utils._to_bash_path(captured_args)
826
+ result = subprocess.run(
827
+ [
828
+ utils.BASH_EXEC,
829
+ utils._to_bash_path(utils.SHELL_DIR / "run-claude-loop.sh"),
830
+ "Implement the requested change.",
831
+ "--agent",
832
+ "codex",
833
+ "--max-turns",
834
+ "1",
835
+ ],
836
+ capture_output=True,
837
+ text=True,
838
+ encoding="utf-8",
839
+ errors="replace",
840
+ timeout=10,
841
+ check=False,
842
+ env=env,
843
+ )
844
+
845
+ assert result.returncode == 0, result.stderr
846
+ assert "claude-only-model" not in captured_args.read_text(encoding="utf-8")
847
+
848
+
373
849
  def test_claude_loop_consumes_deterministic_reply_steps_in_order(tmp_path: Path):
374
850
  fake_bin = tmp_path / "bin"
375
851
  fake_bin.mkdir()