@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -81,8 +81,6 @@ def test_unit_test_detection_keeps_task_runs_as_experiments():
81
81
  assert conftest._is_unit_tests_only(Config()) is False
82
82
 
83
83
 
84
-
85
-
86
84
  def test_experiment_id_uses_explicit_comet_eval_run_id(monkeypatch):
87
85
  monkeypatch.setenv("COMET_EVAL_EXPERIMENT_ID", "comet-eval-1234")
88
86
 
@@ -97,6 +95,125 @@ def test_experiment_id_rejects_unsafe_explicit_value(monkeypatch):
97
95
  conftest._get_or_create_experiment_id("ignored", False)
98
96
 
99
97
 
98
+ def test_auto_generation_loads_environment_before_resolving_cache_identity(
99
+ tmp_path: Path, monkeypatch
100
+ ):
101
+ skill = tmp_path / "skill"
102
+ skill.mkdir()
103
+ (skill / "SKILL.md").write_text("Evaluate me.\n", encoding="utf-8")
104
+ captured = {}
105
+
106
+ def load_environment():
107
+ monkeypatch.setenv("BENCH_CODEX_MODEL", "model-from-dotenv")
108
+
109
+ def generate_manifest(_skill_path, _project_root, **kwargs):
110
+ captured.update(kwargs)
111
+ manifest_path = tmp_path / "generated.yaml"
112
+ manifest_path.write_text(
113
+ """apiVersion: comet.eval/v1alpha1
114
+ kind: SkillEvalManifest
115
+ metadata: {name: generated}
116
+ skill: {name: generated, source: .}
117
+ evaluation:
118
+ tasks:
119
+ - name: generated-one
120
+ prompt: one
121
+ expect: {files: [one.md]}
122
+ - name: generated-two
123
+ prompt: two
124
+ expect: {files: [two.md]}
125
+ """,
126
+ encoding="utf-8",
127
+ )
128
+ return SimpleNamespace(manifest_path=manifest_path)
129
+
130
+ options = {
131
+ "--task": None,
132
+ "--quick": False,
133
+ "--eval-manifest": None,
134
+ "--skill-path": str(skill),
135
+ "--profile": None,
136
+ "--project-root": str(tmp_path),
137
+ "--agent": "codex",
138
+ "--skill-name": None,
139
+ }
140
+
141
+ class Config:
142
+ option = SimpleNamespace(collectonly=False, eval_manifest=None)
143
+
144
+ @staticmethod
145
+ def getoption(name):
146
+ return options[name]
147
+
148
+ monkeypatch.setattr("scaffold.python.utils.load_eval_environment", load_environment)
149
+ monkeypatch.setattr(conftest, "ensure_generated_manifest", generate_manifest)
150
+
151
+ config = Config()
152
+ conftest._ensure_auto_generated_manifest(config)
153
+
154
+ assert captured["agent"] == "codex"
155
+ assert captured["model"] == "model-from-dotenv"
156
+ assert config.option.eval_manifest is None
157
+ assert config._comet_frozen_task_set.source == "generated-cache"
158
+
159
+
160
+ def test_taskless_collect_only_marks_generation_pending_without_agent_or_disk_work(
161
+ tmp_path: Path, monkeypatch
162
+ ):
163
+ skill = tmp_path / "skill"
164
+ skill.mkdir()
165
+ (skill / "SKILL.md").write_text("Evaluate me.\n", encoding="utf-8")
166
+ called = False
167
+
168
+ def fail_generation(*_args, **_kwargs):
169
+ nonlocal called
170
+ called = True
171
+ raise AssertionError("task generation must not run during collect-only")
172
+
173
+ class Config:
174
+ option = SimpleNamespace(collectonly=True, eval_manifest=None, skill_name=None)
175
+
176
+ @staticmethod
177
+ def getoption(name):
178
+ return {
179
+ "--task": None,
180
+ "--quick": False,
181
+ "--eval-manifest": None,
182
+ "--skill-path": str(skill),
183
+ "--profile": None,
184
+ "--project-root": str(tmp_path),
185
+ "--agent": None,
186
+ "--skill-name": None,
187
+ }[name]
188
+
189
+ monkeypatch.setattr(conftest, "ensure_generated_manifest", fail_generation)
190
+ config = Config()
191
+ conftest._ensure_auto_generated_manifest(config)
192
+
193
+ assert called is False
194
+ assert config._comet_frozen_task_set.source == "pending-generation"
195
+ assert config._comet_frozen_task_set.tasks == ()
196
+ assert config._comet_resolution_manifest.skill_path == skill.resolve()
197
+ assert config._comet_resolution_manifest.path == skill / "comet" / "eval.yaml"
198
+ assert config._comet_resolved_tasks == {}
199
+
200
+ from local.tests.tasks import test_tasks
201
+
202
+ monkeypatch.setattr(test_tasks, "load_treatments", lambda: {})
203
+ monkeypatch.setattr(test_tasks, "list_tasks", lambda: [])
204
+ frozen = config._comet_frozen_task_set
205
+ assert test_tasks.generate_test_params(None, None, config) == []
206
+ assert test_tasks.generate_test_params(None, None, config) == []
207
+ assert config._comet_frozen_task_set is frozen
208
+
209
+
210
+ def test_selected_agent_missing_credentials_fails_before_workloads():
211
+ with pytest.raises(pytest.UsageError, match="Credentials for codebuddy not set"):
212
+ conftest._require_agent_credentials("codebuddy", {})
213
+
214
+ conftest._require_agent_credentials("codebuddy", {"CODEBUDDY_API_KEY": "configured"})
215
+
216
+
100
217
  def test_extract_loop_turns_reads_driver_completion_line():
101
218
  stderr = (
102
219
  "[loop] turn 1/4\n"
@@ -113,6 +230,21 @@ def test_extract_loop_turns_reads_driver_completion_line():
113
230
  "deterministic_replies": 1,
114
231
  "completion_signals": 1,
115
232
  "fresh_resume_boundaries": 0,
233
+ "role_sessions": {"subject": [], "simulator": [], "judge": []},
234
+ }
235
+
236
+
237
+ def test_extract_loop_interaction_records_isolated_role_sessions():
238
+ interaction = conftest._extract_loop_interaction(
239
+ "[loop] role-session subject subject-1\n"
240
+ "[loop] role-session simulator simulator-1\n"
241
+ "[loop] role-session subject subject-1\n"
242
+ )
243
+
244
+ assert interaction["role_sessions"] == {
245
+ "subject": ["subject-1"],
246
+ "simulator": ["simulator-1"],
247
+ "judge": [],
116
248
  }
117
249
  assert conftest._extract_loop_turns("ordinary stderr") is None
118
250
 
@@ -383,6 +515,7 @@ def test_dynamic_treatment_config_from_skill_path(tmp_path: Path):
383
515
  "--skill-path": str(skill_dir),
384
516
  "--skill-name": "demo-skill",
385
517
  "--profile": "generic",
518
+ "--quick": True,
386
519
  }
387
520
  return values.get(name)
388
521
 
@@ -577,6 +710,49 @@ def test_setup_test_context_configures_039_shell_hook(tmp_path: Path, setup_test
577
710
  assert command == "bash /workspace/.claude/skills/comet/scripts/comet-hook-guard.sh"
578
711
 
579
712
 
713
+ def test_codex_guard_uses_codex_hook_config_while_scripts_remain_under_agents(tmp_path: Path):
714
+ command = "node /workspace/.agents/skills/comet/scripts/comet-hook-guard.mjs"
715
+
716
+ conftest._ensure_agent_pre_tool_hook(tmp_path, "codex", command)
717
+
718
+ hooks_path = tmp_path / ".codex" / "hooks.json"
719
+ hooks = json.loads(hooks_path.read_text(encoding="utf-8"))
720
+ assert hooks["hooks"]["PreToolUse"][0]["hooks"][0]["command"] == command
721
+ assert not (tmp_path / ".agents" / "hooks.json").exists()
722
+
723
+
724
+ def test_transcript_setup_failure_is_best_effort(tmp_path: Path, monkeypatch, capsys):
725
+ monkeypatch.setenv("TRACE_TO_LANGFUSE", "true")
726
+ monkeypatch.delenv("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED", raising=False)
727
+ monkeypatch.setattr(
728
+ conftest,
729
+ "install_transcript_hook",
730
+ lambda *_args: (_ for _ in ()).throw(ValueError("invalid settings")),
731
+ )
732
+
733
+ conftest._ensure_langfuse_trajectory_support(tmp_path, "codebuddy")
734
+
735
+ assert "optional codebuddy transcript setup skipped" in capsys.readouterr().err
736
+
737
+
738
+ def test_setup_test_context_writes_codebuddy_project_memory(
739
+ tmp_path: Path,
740
+ setup_test_context,
741
+ monkeypatch,
742
+ ):
743
+ monkeypatch.setattr(
744
+ conftest,
745
+ "_resolve_eval_agent",
746
+ lambda _config: SimpleNamespace(agent="codebuddy"),
747
+ )
748
+
749
+ setup_test_context(claude_md="Use the CodeBuddy workflow.")
750
+
751
+ memory = tmp_path / ".codebuddy" / "CODEBUDDY.md"
752
+ assert memory.read_text(encoding="utf-8") == "Use the CodeBuddy workflow."
753
+ assert not (tmp_path / "AGENTS.md").exists()
754
+
755
+
580
756
  def test_setup_test_context_copies_dependency_skill_packages_with_scripts(
581
757
  tmp_path: Path, setup_test_context
582
758
  ):
@@ -879,6 +1055,11 @@ def test_build_report_payload_persists_sample_quality():
879
1055
  "total_tokens": 100,
880
1056
  "total_cost_usd": 0.01,
881
1057
  "case_manifest": {"case_hash": "sha256:demo"},
1058
+ "role_models": {
1059
+ "subject": "subject-model",
1060
+ "simulator": None,
1061
+ "judge": "judge-model",
1062
+ },
882
1063
  },
883
1064
  passed=["[RUBRIC] weighted_score: 1.00"],
884
1065
  failed=[],
@@ -894,6 +1075,11 @@ def test_build_report_payload_persists_sample_quality():
894
1075
  assert report["sample_quality"]["reason_code"] == "valid_signal"
895
1076
  assert report["sample_quality"]["include_in_analysis"] is True
896
1077
  assert report["events_summary"]["case_manifest"] == {"case_hash": "sha256:demo"}
1078
+ assert report["events_summary"]["role_models"] == {
1079
+ "subject": "subject-model",
1080
+ "simulator": None,
1081
+ "judge": "judge-model",
1082
+ }
897
1083
 
898
1084
 
899
1085
  def test_build_report_payload_marks_timeout_as_excluded():
@@ -941,36 +1127,60 @@ def test_prebuild_docker_image_allows_a_cold_image_build_to_run_for_fifteen_minu
941
1127
  ]
942
1128
 
943
1129
 
944
- def test_docker_prebuild_uses_only_the_tasks_selected_by_an_eval_manifest(
945
- tmp_path: Path,
946
- ):
947
- tasks_dir = tmp_path / "tasks"
948
- for task_name in ("authoring-skill-smoke", "workflow-route-conformance", "unrelated"):
949
- environment = tasks_dir / task_name / "environment"
950
- environment.mkdir(parents=True)
951
- (environment / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
952
-
1130
+ def test_resolve_eval_agent_applies_cli_manifest_precedence(tmp_path: Path):
953
1131
  manifest = tmp_path / "eval.yaml"
954
1132
  manifest.write_text(
955
1133
  """
956
1134
  apiVersion: comet.eval/v1alpha1
957
1135
  kind: SkillEvalManifest
958
1136
  metadata:
959
- name: selected-tasks
1137
+ name: selected-agent
960
1138
  skill:
961
- name: selected-tasks
1139
+ name: selected-agent
962
1140
  source: .
963
- evaluation:
964
- recommendedTasks:
965
- - authoring-skill-smoke
966
- - workflow-route-conformance
1141
+ execution:
1142
+ agent: codex
967
1143
  """,
968
1144
  encoding="utf-8",
969
1145
  )
970
1146
 
971
1147
  class Config:
1148
+ def __init__(self, agent):
1149
+ self.agent = agent
1150
+
972
1151
  def getoption(self, name):
973
- return {"--task": None, "--eval-manifest": str(manifest)}[name]
1152
+ return {
1153
+ "--eval-manifest": str(manifest),
1154
+ "--agent": self.agent,
1155
+ }.get(name)
1156
+
1157
+ assert conftest._resolve_eval_agent(Config(None)).agent == "codex"
1158
+ assert conftest._resolve_eval_agent(Config("qoder")).agent == "qoder"
1159
+
1160
+
1161
+ def test_selected_agent_uses_its_native_project_skill_root():
1162
+ assert conftest._agent_project_root("claude-code") == ".claude"
1163
+ assert conftest._agent_project_root("codex") == ".agents"
1164
+ assert conftest._agent_project_root("qoder") == ".qoder"
1165
+ assert conftest._agent_project_root("codebuddy") == ".codebuddy"
1166
+
1167
+
1168
+ def test_docker_prebuild_uses_only_the_tasks_selected_by_an_eval_manifest(
1169
+ tmp_path: Path,
1170
+ ):
1171
+ tasks_dir = tmp_path / "tasks"
1172
+ for task_name in ("authoring-skill-smoke", "workflow-route-conformance", "unrelated"):
1173
+ environment = tasks_dir / task_name / "environment"
1174
+ environment.mkdir(parents=True)
1175
+ (environment / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
1176
+
1177
+ class Config:
1178
+ _comet_frozen_task_set = SimpleNamespace(
1179
+ tasks=(
1180
+ SimpleNamespace(task=SimpleNamespace(environment_dir=tasks_dir / "authoring-skill-smoke" / "environment")),
1181
+ SimpleNamespace(task=SimpleNamespace(environment_dir=tasks_dir / "workflow-route-conformance" / "environment")),
1182
+ )
1183
+ )
974
1184
 
975
1185
  request = SimpleNamespace(config=Config())
976
1186
 
@@ -978,3 +1188,42 @@ evaluation:
978
1188
  tasks_dir / "authoring-skill-smoke" / "environment",
979
1189
  tasks_dir / "workflow-route-conformance" / "environment",
980
1190
  ]
1191
+
1192
+
1193
+ @pytest.mark.parametrize(
1194
+ "source,selected",
1195
+ [
1196
+ ("explicit", ["authoring-skill-smoke"]),
1197
+ ("quick", ["workflow-route-conformance"]),
1198
+ ("authored", ["authoring-skill-smoke"]),
1199
+ ("recommended", ["workflow-route-conformance"]),
1200
+ ("generated-cache", ["authoring-skill-smoke"]),
1201
+ ("source", ["source-custom"]),
1202
+ ],
1203
+ )
1204
+ def test_docker_prebuild_consumes_only_frozen_task_environments(tmp_path: Path, source, selected):
1205
+ tasks_dir = tmp_path / "tasks"
1206
+ environments = {}
1207
+ for name in ("authoring-skill-smoke", "workflow-route-conformance", "source-custom", "unrelated"):
1208
+ environment = tasks_dir / name / "environment"
1209
+ environment.mkdir(parents=True)
1210
+ (environment / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
1211
+ environments[name] = environment
1212
+
1213
+ frozen = SimpleNamespace(
1214
+ source=source,
1215
+ generation=SimpleNamespace(generation_hash="frozen-hash") if source == "generated-cache" else None,
1216
+ tasks=tuple(
1217
+ SimpleNamespace(
1218
+ name=name,
1219
+ provenance="source" if name == "source-custom" else source,
1220
+ task=SimpleNamespace(environment_dir=environments[name]),
1221
+ )
1222
+ for name in [*selected, *selected]
1223
+ ),
1224
+ )
1225
+ request = SimpleNamespace(config=SimpleNamespace(_comet_frozen_task_set=frozen))
1226
+
1227
+ assert conftest._docker_environment_dirs_for_request(request, tasks_dir) == [
1228
+ environments[name] for name in selected
1229
+ ]
@@ -0,0 +1,174 @@
1
+ """Contract tests for explicitly installed custom Eval Agent adapters."""
2
+
3
+ from pathlib import Path
4
+
5
+ import pytest
6
+
7
+ from scaffold.python.agents import (
8
+ get_agent_adapter,
9
+ normalize_skill_invocations,
10
+ validate_agent_capabilities,
11
+ )
12
+ from scaffold.python.custom_agents import discover_custom_agent_specs
13
+
14
+
15
+ def _write_adapter(root: Path, *, name: str = "fixture-agent", **overrides) -> Path:
16
+ directory = root / name
17
+ directory.mkdir(parents=True)
18
+ capabilities = overrides.pop(
19
+ "capabilities",
20
+ {
21
+ "singleTurn": True,
22
+ "resume": True,
23
+ "structuredEvents": True,
24
+ "telemetry": False,
25
+ "skillInvocationEvidence": True,
26
+ },
27
+ )
28
+ runtime = overrides.pop(
29
+ "runtime",
30
+ {"executable": "fixture-agent", "install": {"kind": "none"}},
31
+ )
32
+ credentials = overrides.pop("credentials", ["FIXTURE_AGENT_API_KEY"])
33
+ lines = [
34
+ "apiVersion: comet.eval.agent/v1alpha1",
35
+ "kind: EvalAgentAdapter",
36
+ "metadata:",
37
+ f" id: {name}",
38
+ " version: '1.0.0'",
39
+ "runtime:",
40
+ f" executable: {runtime['executable']}",
41
+ " install:",
42
+ f" kind: {runtime['install']['kind']}",
43
+ "credentials:",
44
+ *[f" - {item}" for item in credentials],
45
+ "modelEnv: FIXTURE_AGENT_MODEL",
46
+ "baseUrlEnv: FIXTURE_AGENT_BASE_URL",
47
+ "capabilities:",
48
+ *[f" {key}: {'true' if value else 'false'}" for key, value in capabilities.items()],
49
+ ]
50
+ for key, value in overrides.items():
51
+ lines.append(f"{key}: {value}")
52
+ path = directory / "adapter.yaml"
53
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
54
+ return path
55
+
56
+
57
+ def test_tracked_fixture_can_be_installed_as_a_custom_adapter(
58
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
59
+ ):
60
+ fixture = Path(__file__).parent / "fixtures" / "custom-agent" / "adapter.yaml"
61
+ destination = tmp_path / "fixture-agent"
62
+ destination.mkdir()
63
+ (destination / "adapter.yaml").write_text(fixture.read_text(encoding="utf-8"), encoding="utf-8")
64
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
65
+
66
+ adapter = get_agent_adapter("fixture-agent")
67
+
68
+ assert adapter.custom is True
69
+ assert adapter.executable == "fixture-agent"
70
+
71
+
72
+ def test_installed_custom_adapter_is_discovered_and_generates_standard_command(
73
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
74
+ ):
75
+ _write_adapter(tmp_path)
76
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
77
+
78
+ specs = discover_custom_agent_specs()
79
+ adapter = get_agent_adapter("fixture-agent")
80
+
81
+ assert list(specs) == ["fixture-agent"]
82
+ assert adapter.custom is True
83
+ assert adapter.executable == "fixture-agent"
84
+ assert adapter.required_credentials == ("FIXTURE_AGENT_API_KEY",)
85
+ assert adapter.supports_telemetry is False
86
+ assert adapter.build_run_command(
87
+ "Do it.", model="fixture-model", role="subject", resume_id="session-1"
88
+ ) == [
89
+ "fixture-agent",
90
+ "-p",
91
+ "Do it.",
92
+ "--output-format",
93
+ "stream-json",
94
+ "--model",
95
+ "fixture-model",
96
+ "--resume",
97
+ "session-1",
98
+ ]
99
+
100
+
101
+ @pytest.mark.parametrize(
102
+ "capability",
103
+ ["single_turn", "resume", "structured_events", "skill_invocation_evidence"],
104
+ )
105
+ def test_missing_required_custom_capability_fails_preflight(
106
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capability: str
107
+ ):
108
+ capabilities = {
109
+ "singleTurn": True,
110
+ "resume": True,
111
+ "structuredEvents": True,
112
+ "telemetry": False,
113
+ "skillInvocationEvidence": True,
114
+ }
115
+ key = {
116
+ "single_turn": "singleTurn",
117
+ "resume": "resume",
118
+ "structured_events": "structuredEvents",
119
+ "skill_invocation_evidence": "skillInvocationEvidence",
120
+ }[capability]
121
+ capabilities[key] = False
122
+ _write_adapter(tmp_path, capabilities=capabilities)
123
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
124
+
125
+ with pytest.raises(ValueError, match=capability):
126
+ validate_agent_capabilities(get_agent_adapter("fixture-agent"))
127
+
128
+
129
+ def test_custom_registry_rejects_malformed_duplicate_and_escape_entries(
130
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
131
+ ):
132
+ _write_adapter(tmp_path, name="first")
133
+ second = _write_adapter(tmp_path, name="second")
134
+ second.write_text(second.read_text(encoding="utf-8").replace("id: second", "id: first"), encoding="utf-8")
135
+ outside = tmp_path.parent / "outside-adapter.yaml"
136
+ outside.write_text("not an adapter\n", encoding="utf-8")
137
+ escaped = tmp_path / "escaped"
138
+ try:
139
+ escaped.symlink_to(outside.parent, target_is_directory=True)
140
+ except OSError:
141
+ pass
142
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
143
+
144
+ with pytest.raises(ValueError, match="duplicate|outside|invalid|match|escapes"):
145
+ discover_custom_agent_specs()
146
+
147
+
148
+ def test_custom_adapter_does_not_infer_skill_invocation_from_artifacts(
149
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
150
+ ):
151
+ _write_adapter(tmp_path)
152
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
153
+ adapter = get_agent_adapter("fixture-agent")
154
+
155
+ assert adapter.has_observable_skill_invocation({"artifacts": ["SKILL.md"]}) is False
156
+ assert adapter.has_observable_skill_invocation({"skill_invocations": ["fixture-agent"]}) is True
157
+
158
+
159
+ def test_custom_adapter_capability_filters_non_observable_skill_evidence(
160
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
161
+ ):
162
+ capabilities = {
163
+ "singleTurn": True,
164
+ "resume": True,
165
+ "structuredEvents": True,
166
+ "telemetry": False,
167
+ "skillInvocationEvidence": False,
168
+ }
169
+ _write_adapter(tmp_path, capabilities=capabilities)
170
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
171
+ adapter = get_agent_adapter("fixture-agent")
172
+ events = {"skills_invoked": ["demo"], "skill_invocations": ["demo"]}
173
+
174
+ assert normalize_skill_invocations(events, adapter=adapter) == []