@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -0,0 +1,176 @@
1
+ """The sole, import-safe authority for Eval task selection."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Literal
8
+
9
+ from scaffold.python.eval_context import ResolvedEvalContext
10
+ from scaffold.python.execution import ResolvedExecution, resolve_execution
11
+ from scaffold.python.generated_task_cache import (
12
+ GeneratedCache,
13
+ build_skill_snapshot,
14
+ cache_location,
15
+ generation_hash,
16
+ load_generated_cache,
17
+ selected_agent_model,
18
+ )
19
+ from scaffold.python.manifest_tasks import load_manifest_tasks
20
+ from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
21
+ from scaffold.python.tasks import Task, load_task
22
+
23
+
24
+ TaskSource = Literal[
25
+ "explicit", "quick", "authored", "recommended", "generated-cache", "pending-generation"
26
+ ]
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class GenerationPlan:
31
+ generation_hash: str
32
+ cache_dir: Path
33
+ cached: GeneratedCache | None
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class ResolvedTask:
38
+ name: str
39
+ provenance: Literal["inline", "source", "bundled", "generated"]
40
+ task: Task
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class ResolvedTaskSet:
45
+ source: TaskSource
46
+ tasks: tuple[ResolvedTask, ...]
47
+ generation: GenerationPlan | None
48
+
49
+
50
+ @dataclass(frozen=True)
51
+ class TaskCatalogue:
52
+ bundled: dict[str, ResolvedTask]
53
+ canonical: dict[str, ResolvedTask]
54
+ authored: dict[str, ResolvedTask]
55
+
56
+
57
+ def _bundled(tasks_dir: Path) -> dict[str, ResolvedTask]:
58
+ result: dict[str, ResolvedTask] = {}
59
+ canonical: dict[str, str] = {}
60
+ if not tasks_dir.is_dir():
61
+ return result
62
+ for directory in sorted(path for path in tasks_dir.iterdir() if path.is_dir()):
63
+ if not (directory / "task.toml").is_file() or not (directory / "instruction.md").is_file():
64
+ continue
65
+ task = load_task(directory, tasks_dir=tasks_dir)
66
+ if task.name in canonical:
67
+ raise ValueError(
68
+ f'bundled tasks "{canonical[task.name]}" and "{directory.name}" both declare '
69
+ f'metadata.name "{task.name}"'
70
+ )
71
+ canonical[task.name] = directory.name
72
+ result[directory.name] = ResolvedTask(task.name, "bundled", task)
73
+ return result
74
+
75
+
76
+ def build_task_catalogue(manifest: SkillEvalManifest, tasks_dir: Path) -> TaskCatalogue:
77
+ bundled = _bundled(tasks_dir)
78
+ canonical = {item.name: item for item in bundled.values()}
79
+ authored_tasks = load_manifest_tasks(manifest)
80
+ authored: dict[str, ResolvedTask] = {}
81
+ for index, task in enumerate(authored_tasks):
82
+ source = manifest.tasks[index]
83
+ authored[task.name] = ResolvedTask(task.name, "inline" if source.is_inline else "source", task)
84
+ for index, task_name in enumerate(manifest.recommended_tasks):
85
+ if task_name not in bundled:
86
+ raise ValueError(f'evaluation.recommendedTasks[{index}]: unknown bundled task "{task_name}"')
87
+ if task_name in authored:
88
+ raise ValueError(
89
+ f'evaluation.tasks[{next(i for i, item in enumerate(manifest.tasks) if item.name == task_name)}].name '
90
+ f'conflicts with evaluation.recommendedTasks[{index}]: "{task_name}"'
91
+ )
92
+ for index, spec in enumerate(manifest.tasks):
93
+ if spec.name in canonical:
94
+ bundle = next(key for key, item in bundled.items() if item.name == spec.name)
95
+ raise ValueError(
96
+ f'evaluation.tasks[{index}].name conflicts with bundled task "{bundle}": "{spec.name}"'
97
+ )
98
+ return TaskCatalogue(bundled, canonical, authored)
99
+
100
+
101
+ def _generated_plan(
102
+ context: ResolvedEvalContext,
103
+ manifest: SkillEvalManifest,
104
+ execution: ResolvedExecution | None = None,
105
+ ) -> GenerationPlan:
106
+ snapshot = build_skill_snapshot(context.skill_root)
107
+ resolved_execution = execution or resolve_execution(manifest=manifest.execution)
108
+ selected_agent = resolved_execution.agent
109
+ selected_model = resolved_execution.model or selected_agent_model(selected_agent)
110
+ digest = generation_hash(
111
+ snapshot,
112
+ agent=selected_agent,
113
+ model=selected_model,
114
+ base_url=resolved_execution.base_url,
115
+ profile=manifest.profile or "generic",
116
+ interaction=vars(manifest.interaction),
117
+ )
118
+ cache_dir = cache_location(context.artifact_owner_root, context.skill_root, digest)
119
+ return GenerationPlan(digest, cache_dir, load_generated_cache(cache_dir, digest))
120
+
121
+
122
+ def resolve_task_set(
123
+ context: ResolvedEvalContext,
124
+ manifest: SkillEvalManifest,
125
+ catalogue: TaskCatalogue,
126
+ *,
127
+ explicit_task: str | None = None,
128
+ quick: bool = False,
129
+ static_collect: bool = False,
130
+ execution: ResolvedExecution | None = None,
131
+ ) -> ResolvedTaskSet:
132
+ """Resolve exactly one precedence branch without probing lower branches."""
133
+ if explicit_task:
134
+ selected = catalogue.authored.get(explicit_task) or catalogue.bundled.get(explicit_task) or catalogue.canonical.get(explicit_task)
135
+ if selected is None:
136
+ available = sorted({*catalogue.authored, *catalogue.bundled, *catalogue.canonical})
137
+ raise ValueError(f'Task not found: {explicit_task}. Available: {available}')
138
+ return ResolvedTaskSet("explicit", (selected,), None)
139
+ if quick:
140
+ selected = catalogue.bundled.get("generic-skill-smoke")
141
+ if selected is None:
142
+ raise ValueError('Bundled quick task generic-skill-smoke is unavailable')
143
+ return ResolvedTaskSet("quick", (selected,), None)
144
+ if manifest.tasks:
145
+ return ResolvedTaskSet("authored", tuple(catalogue.authored[item.name] for item in manifest.tasks), None)
146
+ if manifest.recommended_tasks:
147
+ return ResolvedTaskSet("recommended", tuple(catalogue.bundled[item] for item in manifest.recommended_tasks), None)
148
+ plan = _generated_plan(context, manifest, execution)
149
+ if plan.cached is None:
150
+ return ResolvedTaskSet("pending-generation", (), plan)
151
+ generated = load_eval_manifest(plan.cached.manifest_path)
152
+ loaded = load_manifest_tasks(generated)
153
+ names: set[str] = set()
154
+ tasks: list[ResolvedTask] = []
155
+ for index, task in enumerate(loaded):
156
+ if task.name in names:
157
+ first_index = next(
158
+ item_index
159
+ for item_index, item in enumerate(loaded[:index])
160
+ if item.name == task.name
161
+ )
162
+ raise ValueError(
163
+ f'generated.tasks[{index}].name duplicates '
164
+ f'generated.tasks[{first_index}].name: "{task.name}"'
165
+ )
166
+ if task.name in catalogue.canonical:
167
+ bundle = next(
168
+ key for key, item in catalogue.bundled.items() if item.name == task.name
169
+ )
170
+ raise ValueError(
171
+ f'generated.tasks[{index}].name conflicts with bundled task '
172
+ f'"{bundle}": "{task.name}"'
173
+ )
174
+ names.add(task.name)
175
+ tasks.append(ResolvedTask(task.name, "generated", task))
176
+ return ResolvedTaskSet("generated-cache", tuple(tasks), plan)
@@ -148,6 +148,10 @@ class Task:
148
148
  path: Path
149
149
  config: TaskConfig
150
150
  instruction_template: str
151
+ _environment_dir: Path | None = field(default=None, repr=False)
152
+ workspace_dir: Path | None = field(default=None, repr=False)
153
+ manifest_expectations: dict[str, Any] | None = field(default=None, repr=False)
154
+ manifest_path: Path | None = field(default=None, repr=False)
151
155
 
152
156
  @property
153
157
  def name(self) -> str:
@@ -155,7 +159,7 @@ class Task:
155
159
 
156
160
  @property
157
161
  def environment_dir(self) -> Path:
158
- return self.path / "environment"
162
+ return self._environment_dir or self.path / "environment"
159
163
 
160
164
  @property
161
165
  def validation_dir(self) -> Path:
@@ -195,6 +199,11 @@ class Task:
195
199
  The config specifies test_scripts and target_artifacts. The framework
196
200
  auto-builds an execution validator that runs the scripts in Docker.
197
201
  """
202
+ if self.manifest_expectations is not None:
203
+ from scaffold.python.inline_tasks import make_inline_expectations_validator
204
+
205
+ return [make_inline_expectations_validator(self.manifest_expectations)]
206
+
198
207
  vc = self.config.validation
199
208
  if vc.test_scripts:
200
209
  from scaffold.python.utils import make_execution_validator
@@ -212,7 +221,7 @@ class Task:
212
221
  return []
213
222
 
214
223
 
215
- def load_task(name: str, tasks_dir: Path | None = None) -> Task:
224
+ def load_task(name: str | Path, tasks_dir: Path | None = None) -> Task:
216
225
  """Load a task by name.
217
226
 
218
227
  Args:
@@ -222,8 +231,7 @@ def load_task(name: str, tasks_dir: Path | None = None) -> Task:
222
231
  Returns:
223
232
  Task object with config and instruction template
224
233
  """
225
- tasks_dir = tasks_dir or get_tasks_dir()
226
- task_path = tasks_dir / name
234
+ task_path = Path(name) if isinstance(name, Path) else (tasks_dir or get_tasks_dir()) / name
227
235
 
228
236
  if not task_path.exists():
229
237
  raise FileNotFoundError(f"Task not found: {name} (looked in {tasks_dir})")
@@ -323,6 +331,11 @@ def load_task(name: str, tasks_dir: Path | None = None) -> Task:
323
331
  return Task(path=task_path, config=config, instruction_template=instruction_template)
324
332
 
325
333
 
334
+ def load_task_from_path(task_path: Path | str) -> Task:
335
+ """Load an advanced task package referenced by a project eval manifest."""
336
+ return load_task(Path(task_path))
337
+
338
+
326
339
  def list_tasks(tasks_dir: Path | None = None) -> list[str]:
327
340
  """List available task names.
328
341
 
@@ -4,6 +4,7 @@ import json
4
4
  import ntpath
5
5
  import os
6
6
  import random
7
+ import re
7
8
  import shutil
8
9
  import subprocess
9
10
  import time
@@ -11,6 +12,8 @@ from pathlib import Path
11
12
 
12
13
  from dotenv import load_dotenv
13
14
 
15
+ from scaffold.python.agents import validate_agent_id
16
+ from scaffold.python.execution import ResolvedExecution, build_agent_environment
14
17
  from scaffold.python.paths import EVAL_ROOT, get_suite_root
15
18
 
16
19
  TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
@@ -21,8 +24,14 @@ SCAFFOLD_PYTHON_DIR = Path(__file__).parent
21
24
 
22
25
  def load_eval_environment() -> None:
23
26
  """Load eval credentials only at an explicit execution boundary, never on import."""
24
- load_dotenv(EVAL_ROOT / ".env")
25
- load_dotenv(get_suite_root() / ".env", override=True)
27
+ original_environment = dict(os.environ)
28
+ try:
29
+ load_dotenv(EVAL_ROOT / ".env", override=True)
30
+ load_dotenv(get_suite_root() / ".env", override=True)
31
+ load_dotenv(Path.home() / ".comet" / "eval" / ".env", override=True)
32
+ finally:
33
+ for key, value in original_environment.items():
34
+ os.environ[key] = value
26
35
 
27
36
 
28
37
  def _uses_wsl_bash(bash_exec: str) -> bool:
@@ -69,12 +78,28 @@ def _resolve_bash(os_name: str | None = None) -> str:
69
78
 
70
79
  BASH_EXEC = _resolve_bash()
71
80
  WSL_ENV_KEYS = (
81
+ "BENCH_EVAL_AGENT",
72
82
  "OPENAI_API_KEY",
83
+ "OPENAI_BASE_URL",
84
+ "OPENAI_MODEL",
85
+ "QODER_PERSONAL_ACCESS_TOKEN",
86
+ "QODER_BASE_URL",
87
+ "QODER_MODEL",
88
+ "CODEX_API_KEY",
89
+ "CODEX_BASE_URL",
90
+ "CODEX_MODEL",
73
91
  "ANTHROPIC_API_KEY",
74
92
  "LANGSMITH_API_KEY",
75
93
  "LANGSMITH_PROJECT",
76
94
  "LANGSMITH_TRACING",
77
95
  "LANGSMITH_ENDPOINT",
96
+ "LANGFUSE_PUBLIC_KEY",
97
+ "LANGFUSE_SECRET_KEY",
98
+ "LANGFUSE_BASE_URL",
99
+ "LANGFUSE_TRACING_ENVIRONMENT",
100
+ "TRACE_TO_LANGFUSE",
101
+ "LANGFUSE_CODEX_TAGS",
102
+ "LANGFUSE_CODEX_METADATA",
78
103
  "TAVILY_API_KEY",
79
104
  "TRACE_TO_LANGSMITH",
80
105
  "CC_LANGSMITH_API_KEY",
@@ -97,6 +122,31 @@ WSL_ENV_KEYS = (
97
122
  "CLAUDE_CODE_SUBAGENT_MODEL",
98
123
  "BENCH_CC_VERSION",
99
124
  "BENCH_CC_MODEL",
125
+ "BENCH_CODEX_VERSION",
126
+ "BENCH_CODEX_MODEL",
127
+ "BENCH_QODER_VERSION",
128
+ "BENCH_QODER_MODEL",
129
+ "CODEBUDDY_API_KEY",
130
+ "CODEBUDDY_AUTH_TOKEN",
131
+ "CODEBUDDY_BASE_URL",
132
+ "CODEBUDDY_MODEL",
133
+ "CODEBUDDY_SMALL_FAST_MODEL",
134
+ "CODEBUDDY_BIG_SLOW_MODEL",
135
+ "CODEBUDDY_CODE_SUBAGENT_MODEL",
136
+ "CODEBUDDY_CUSTOM_HEADERS",
137
+ "CODEBUDDY_INTERNET_ENVIRONMENT",
138
+ "BENCH_CODEBUDDY_VERSION",
139
+ "BENCH_CODEBUDDY_MODEL",
140
+ "COMET_EVAL_CUSTOM_AGENT_ID",
141
+ "COMET_EVAL_CUSTOM_EXECUTABLE",
142
+ "COMET_EVAL_CUSTOM_CREDENTIALS",
143
+ "COMET_EVAL_CUSTOM_MODEL",
144
+ "COMET_EVAL_CUSTOM_BASE_URL",
145
+ "COMET_EVAL_CUSTOM_MODEL_ENV",
146
+ "COMET_EVAL_CUSTOM_BASE_URL_ENV",
147
+ "COMET_EVAL_CUSTOM_INSTALL_KIND",
148
+ "COMET_EVAL_CUSTOM_INSTALL_PACKAGE",
149
+ "COMET_EVAL_CUSTOM_INSTALL_VERSION",
100
150
  )
101
151
 
102
152
 
@@ -119,20 +169,33 @@ def _to_bash_path(value) -> str:
119
169
  return s
120
170
 
121
171
 
122
- def _bash_env() -> dict[str, str]:
123
- env = os.environ.copy()
172
+ def _bash_env(source_env: dict[str, str] | None = None) -> dict[str, str]:
173
+ env = dict(source_env if source_env is not None else os.environ)
124
174
  if os.name != "nt" or not _uses_wsl_bash(BASH_EXEC):
125
175
  return env
126
176
 
127
177
  existing = [item for item in env.get("WSLENV", "").split(":") if item]
178
+ custom_credentials = [
179
+ key.strip()
180
+ for key in env.get("COMET_EVAL_CUSTOM_CREDENTIALS", "").split(",")
181
+ if key.strip()
182
+ ]
183
+ custom_routing = [
184
+ key.strip()
185
+ for metadata_key in ("COMET_EVAL_CUSTOM_MODEL_ENV", "COMET_EVAL_CUSTOM_BASE_URL_ENV")
186
+ for key in [env.get(metadata_key, "")]
187
+ if re.fullmatch(r"[A-Z][A-Z0-9_]{1,63}", key.strip())
188
+ ]
128
189
  exported = [key for key in WSL_ENV_KEYS if env.get(key)]
190
+ exported.extend(key for key in custom_credentials if env.get(key))
191
+ exported.extend(key for key in custom_routing if env.get(key))
129
192
  merged = list(dict.fromkeys(existing + exported))
130
193
  if merged:
131
194
  env["WSLENV"] = ":".join(merged)
132
195
  return env
133
196
 
134
197
 
135
- def run_shell(script, *args, timeout=None, check=True):
198
+ def run_shell(script, *args, timeout=None, check=True, env=None):
136
199
  cmd = [BASH_EXEC, _to_bash_path(SHELL_DIR / script)] + [_to_bash_path(a) for a in args]
137
200
  return subprocess.run(
138
201
  cmd,
@@ -142,7 +205,7 @@ def run_shell(script, *args, timeout=None, check=True):
142
205
  errors="replace",
143
206
  timeout=timeout,
144
207
  check=check,
145
- env=_bash_env(),
208
+ env=_bash_env(env),
146
209
  )
147
210
 
148
211
 
@@ -153,9 +216,11 @@ def check_docker_available():
153
216
  return False
154
217
 
155
218
 
156
- def build_docker_image(test_dir, force=False, verbose=False):
219
+ def build_docker_image(test_dir, force=False, verbose=False, agent=None):
157
220
  try:
158
221
  args = ["build", str(test_dir)] + (["--force"] if force else [])
222
+ if agent:
223
+ args.extend(["--agent", validate_agent_id(agent)])
159
224
  result = run_shell("docker.sh", *args, timeout=300, check=False)
160
225
  return result.stdout.strip() if result.returncode == 0 else None
161
226
  except subprocess.TimeoutExpired:
@@ -186,7 +251,26 @@ def run_node_in_docker(test_dir, script_name, timeout=120, args=None):
186
251
  return _docker_run_script("run-node", test_dir, script_name, timeout, args)
187
252
 
188
253
 
189
- def run_claude_in_docker(test_dir, prompt, timeout=300, model=None, image_id=None):
254
+ def run_command_in_docker(test_dir, command, timeout=120):
255
+ """Run one user-authored validation command inside the task container."""
256
+ cmd = ["run-command", str(test_dir), "--timeout", str(timeout), "--", command]
257
+ if not check_docker_available():
258
+ return subprocess.CompletedProcess(cmd, 125, "", "Docker not available")
259
+ try:
260
+ return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
261
+ except subprocess.TimeoutExpired:
262
+ return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
263
+
264
+
265
+ def run_claude_in_docker(
266
+ test_dir,
267
+ prompt,
268
+ timeout=300,
269
+ model=None,
270
+ image_id=None,
271
+ base_url=None,
272
+ environment=None,
273
+ ):
190
274
  if not check_docker_available():
191
275
  raise RuntimeError("Docker not available")
192
276
  cmd = ["run-claude", str(test_dir), prompt, "--timeout", str(timeout)]
@@ -194,17 +278,54 @@ def run_claude_in_docker(test_dir, prompt, timeout=300, model=None, image_id=Non
194
278
  cmd.extend(["--model", model])
195
279
  if image_id:
196
280
  cmd.extend(["--image-id", image_id])
281
+ child_env = environment
282
+ if child_env is None and (model or base_url):
283
+ child_env = build_agent_environment(
284
+ ResolvedExecution("claude-code", model, base_url, {}),
285
+ )
197
286
  try:
198
- return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
287
+ return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False, env=child_env)
199
288
  except subprocess.TimeoutExpired:
200
289
  return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
201
290
 
202
291
 
203
- def run_claude_loop_in_docker(test_dir, loop_args, timeout=600):
292
+ def run_agent_in_docker(
293
+ test_dir,
294
+ prompt,
295
+ *,
296
+ agent="claude-code",
297
+ timeout=300,
298
+ model=None,
299
+ base_url=None,
300
+ image_id=None,
301
+ environment=None,
302
+ ):
303
+ """Run one subject, simulator, or judge turn through the selected adapter."""
304
+ agent_id = validate_agent_id(agent)
305
+ if not check_docker_available():
306
+ raise RuntimeError("Docker not available")
307
+ cmd = ["run-agent", str(test_dir), prompt, "--agent", agent_id]
308
+ if model:
309
+ cmd.extend(["--model", model])
310
+ cmd.extend(["--timeout", str(timeout)])
311
+ if image_id:
312
+ cmd.extend(["--image-id", image_id])
313
+ child_env = environment
314
+ if child_env is None and (model or base_url):
315
+ child_env = build_agent_environment(
316
+ ResolvedExecution(agent_id, model, base_url, {}),
317
+ )
318
+ try:
319
+ return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False, env=child_env)
320
+ except subprocess.TimeoutExpired:
321
+ return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
322
+
323
+
324
+ def run_claude_loop_in_docker(test_dir, loop_args, timeout=600, environment=None):
204
325
  """Run the interactive driver and remove its container after host-side timeout."""
205
326
  cmd = ["run-claude-loop", str(test_dir), *loop_args]
206
327
  try:
207
- return run_shell("docker.sh", *cmd, timeout=timeout, check=False)
328
+ return run_shell("docker.sh", *cmd, timeout=timeout, check=False, env=environment)
208
329
  except subprocess.TimeoutExpired as error:
209
330
  cleanup_error = ""
210
331
  try:
@@ -235,6 +356,41 @@ def run_claude_loop_in_docker(test_dir, loop_args, timeout=600):
235
356
  )
236
357
 
237
358
 
359
+ def run_agent_loop_in_docker(test_dir, loop_args, timeout=600, environment=None):
360
+ """Run the shared interactive driver for a non-default evaluation agent."""
361
+ cmd = ["run-agent-loop", str(test_dir), *loop_args]
362
+ try:
363
+ return run_shell("docker.sh", *cmd, timeout=timeout, check=False, env=environment)
364
+ except subprocess.TimeoutExpired as error:
365
+ cleanup_error = ""
366
+ try:
367
+ cleanup = run_shell(
368
+ "docker.sh",
369
+ "cleanup-agent-loop",
370
+ test_dir,
371
+ timeout=30,
372
+ check=False,
373
+ )
374
+ if cleanup.returncode != 0:
375
+ cleanup_error = f"; cleanup failed: {cleanup.stderr or cleanup.stdout}"
376
+ except Exception as cleanup_exception: # pragma: no cover - defensive cleanup
377
+ cleanup_error = f"; cleanup failed: {cleanup_exception}"
378
+ stdout = (
379
+ error.stdout.decode("utf-8", errors="replace")
380
+ if isinstance(error.stdout, bytes)
381
+ else (error.stdout or "")
382
+ )
383
+ stderr = (
384
+ error.stderr.decode("utf-8", errors="replace")
385
+ if isinstance(error.stderr, bytes)
386
+ else (error.stderr or "")
387
+ )
388
+ message = f"Timeout after {timeout}s{cleanup_error}"
389
+ return subprocess.CompletedProcess(
390
+ cmd, 124, stdout, "\n".join(filter(None, (stderr, message)))
391
+ )
392
+
393
+
238
394
  def _copy_scaffold_to_docker(test_dir):
239
395
  py_validation = SCAFFOLD_PYTHON_DIR / "validation"
240
396
 
@@ -20,6 +20,49 @@ TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
20
20
  TEST_RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
21
21
 
22
22
 
23
+ def _normalize_skill_invocations(events: object, outputs: dict | None) -> list[str]:
24
+ """Use the Agent contract when the full scaffold is available.
25
+
26
+ Validator sidecars intentionally copy only a small subset of the scaffold,
27
+ so keep this helper import-local and preserve the historical fallback there.
28
+ """
29
+ try:
30
+ from scaffold.python.agents import normalize_skill_invocations
31
+ except ModuleNotFoundError:
32
+ if not isinstance(events, dict):
33
+ return []
34
+ output_data = outputs if isinstance(outputs, dict) else {}
35
+ event_agent = events.get("agent")
36
+ output_agent = output_data.get("agent")
37
+ agent = output_agent if isinstance(output_agent, str) else event_agent
38
+ if isinstance(agent, str) and agent not in {
39
+ "claude-code",
40
+ "codex",
41
+ "qoder",
42
+ "codebuddy",
43
+ }:
44
+ # Validator sidecars do not carry the host-side adapter registry.
45
+ # A custom Agent must therefore provide explicit invocation events;
46
+ # artifacts or legacy path inference are not sufficient evidence.
47
+ explicit = events.get("skill_invocations")
48
+ return (
49
+ [item for item in explicit if isinstance(item, str)]
50
+ if isinstance(explicit, list)
51
+ else []
52
+ )
53
+ invoked = events.get("skills_invoked")
54
+ return (
55
+ [item for item in invoked if isinstance(item, str)]
56
+ if isinstance(invoked, list)
57
+ else []
58
+ )
59
+ return normalize_skill_invocations(
60
+ events,
61
+ agent=(outputs or {}).get("agent"),
62
+ adapter=(outputs or {}).get("agent_adapter"),
63
+ )
64
+
65
+
23
66
  def load_test_context(path: str = TEST_CONTEXT_FILE) -> dict:
24
67
  """Load test context (run_id, events, langsmith_env, etc.) written by the host.
25
68
 
@@ -141,6 +184,9 @@ def run_validators(
141
184
  Returns:
142
185
  Combined (passed, failed) lists
143
186
  """
187
+ events = outputs.get("events") if isinstance(outputs, dict) else None
188
+ if isinstance(events, dict):
189
+ events["skills_invoked"] = _normalize_skill_invocations(events, outputs)
144
190
  all_passed, all_failed = [], []
145
191
  for validator in validators:
146
192
  passed, failed = validator(test_dir, outputs)
@@ -164,7 +210,7 @@ def check_starter_skill_first(
164
210
  return ["Note: starter skill check skipped (ALL_MAIN_SKILLS)"], []
165
211
 
166
212
  events = outputs.get("events", {}) if outputs else {}
167
- skills_invoked = events.get("skills_invoked", [])
213
+ skills_invoked = _normalize_skill_invocations(events, outputs)
168
214
 
169
215
  # No skills invoked at all (e.g. CONTROL treatment) — nothing to check
170
216
  if not skills_invoked:
@@ -205,7 +251,7 @@ def check_skill_invoked(
205
251
  (passed, failed) lists
206
252
  """
207
253
  events = outputs.get("events", {}) if outputs else {}
208
- skills_invoked = events.get("skills_invoked", [])
254
+ skills_invoked = _normalize_skill_invocations(events, outputs)
209
255
 
210
256
  if skill_name in skills_invoked:
211
257
  return [f"Invoked {skill_name} skill"], []
@@ -7,6 +7,8 @@ import re
7
7
  from pathlib import Path
8
8
  from typing import Any
9
9
 
10
+ from scaffold.python.agents import normalize_skill_invocations
11
+
10
12
  GENERIC_RUBRIC_DIMENSIONS = (
11
13
  "completion",
12
14
  "skill_invocation",
@@ -153,6 +155,13 @@ def _fmt_na(dim: str, reason: str) -> str:
153
155
 
154
156
 
155
157
  def generic_rubric_validator(test_dir: Path, outputs: dict[str, Any]) -> tuple[list[str], list[str]]:
158
+ events = (outputs or {}).get("events")
159
+ if isinstance(events, dict):
160
+ events["skills_invoked"] = normalize_skill_invocations(
161
+ events,
162
+ agent=(outputs or {}).get("agent"),
163
+ adapter=(outputs or {}).get("agent_adapter"),
164
+ )
156
165
  scored: list[tuple[str, float | None, str]] = [
157
166
  ("completion", *_score_completion(outputs)),
158
167
  ("skill_invocation", *_score_skill_invocation(outputs)),
@@ -10,6 +10,8 @@ from typing import Any
10
10
 
11
11
  import yaml
12
12
 
13
+ from scaffold.python.validation.core import _normalize_skill_invocations
14
+
13
15
 
14
16
  def _failure(name: str, reason: str) -> str:
15
17
  return f"{name}: {reason}"
@@ -186,7 +188,10 @@ def validate_native_workflow(
186
188
  passed: list[str] = []
187
189
  failed: list[str] = []
188
190
 
189
- invoked = (outputs.get("events") or {}).get("skills_invoked", []) or []
191
+ events = outputs.get("events") or {}
192
+ invoked = _normalize_skill_invocations(events, outputs)
193
+ if isinstance(events, dict):
194
+ events["skills_invoked"] = invoked
190
195
  unexpected_skills = sorted({skill for skill in invoked if skill != "comet-native"})
191
196
  if "comet-native" in invoked and not unexpected_skills:
192
197
  passed.append("native_skill_invocation")
@@ -39,6 +39,8 @@ import re
39
39
  from pathlib import Path
40
40
  from typing import Any
41
41
 
42
+ from scaffold.python.agents import normalize_skill_invocations
43
+
42
44
  # All nine rubric dimensions, in display order.
43
45
  RUBRIC_DIMENSIONS = (
44
46
  "main_flow",
@@ -747,6 +749,12 @@ def comet_rubric_validator(test_dir: Path, outputs: dict) -> tuple[list[str], li
747
749
  Final weighted score uses dimension weights to aggregate.
748
750
  """
749
751
  events = (outputs or {}).get("events", {}) or {}
752
+ if isinstance(events, dict):
753
+ events["skills_invoked"] = normalize_skill_invocations(
754
+ events,
755
+ agent=(outputs or {}).get("agent"),
756
+ adapter=(outputs or {}).get("agent_adapter"),
757
+ )
750
758
  business_completion = _completion_input(outputs or {}, "business_completion")
751
759
  workflow_completion = _completion_input(outputs or {}, "workflow_completion")
752
760
  is_control = _is_control_business_only(outputs or {})
@@ -880,7 +888,14 @@ def comet_rubric_validator(test_dir: Path, outputs: dict) -> tuple[list[str], li
880
888
  try:
881
889
  from scaffold.python.llm_judge import judge_messages
882
890
 
883
- judge_results = judge_messages(test_dir)
891
+ judge_results = judge_messages(
892
+ test_dir,
893
+ agent=outputs.get("judge_agent") or outputs.get("agent"),
894
+ model=outputs.get("judge_model"),
895
+ base_url=outputs.get("judge_base_url"),
896
+ excluded_credentials=tuple(outputs.get("main_credentials") or ()),
897
+ evidence=outputs,
898
+ )
884
899
  passed.extend(judge_results)
885
900
  if any(
886
901
  result.startswith("[RUBRIC-JUDGE] ") and " status:" not in result