@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -2,9 +2,12 @@
2
2
 
3
3
  from pathlib import Path
4
4
 
5
+ import re
6
+
5
7
  import pytest
6
8
 
7
9
  from scaffold.python.manifests import load_eval_manifest
10
+ from scaffold.python.manifest_tasks import load_manifest_tasks
8
11
 
9
12
 
10
13
  def test_load_eval_manifest_parses_skill_package_metadata(tmp_path: Path):
@@ -55,6 +58,14 @@ interaction:
55
58
  maxTurns: 8
56
59
  simulatorPrompt: Answer concisely.
57
60
  freshResumeMarker: COLD_RESUME_READY
61
+ execution:
62
+ agent: codex
63
+ model: gpt-main
64
+ baseUrl: https://main.example.com/v1
65
+ judge:
66
+ agent: claude-code
67
+ model: claude-judge
68
+ baseUrl: https://judge.example.com
58
69
  """,
59
70
  encoding="utf-8",
60
71
  )
@@ -81,6 +92,68 @@ interaction:
81
92
  assert manifest.interaction.max_turns == 8
82
93
  assert manifest.interaction.simulator_prompt == "Answer concisely."
83
94
  assert manifest.interaction.fresh_resume_marker == "COLD_RESUME_READY"
95
+ assert manifest.execution_agent == "codex"
96
+ assert manifest.execution.model == "gpt-main"
97
+ assert manifest.execution.base_url == "https://main.example.com/v1"
98
+ assert manifest.judge is not None
99
+ assert manifest.judge.agent == "claude-code"
100
+ assert manifest.judge.model == "claude-judge"
101
+ assert manifest.judge.base_url == "https://judge.example.com"
102
+ assert manifest.raw["execution"]["model"] == "gpt-main"
103
+ assert manifest.raw["judge"]["model"] == "claude-judge"
104
+
105
+
106
+ def test_load_eval_manifest_normalizes_snake_case_aliases(tmp_path: Path):
107
+ package = tmp_path / "my-skill"
108
+ (package / "comet").mkdir(parents=True)
109
+ manifest_path = package / "comet" / "eval.yaml"
110
+ manifest_path.write_text(
111
+ """
112
+ apiVersion: comet.eval/v1alpha1
113
+ kind: SkillEvalManifest
114
+ metadata:
115
+ name: my-skill
116
+ draft_hash: '0000000000000000000000000000000000000000000000000000000000000000'
117
+ generation_hash: generated
118
+ skill:
119
+ name: my-skill
120
+ source: ..
121
+ evaluation:
122
+ recommended_tasks: [generic-skill-smoke]
123
+ required_skills: [my-skill]
124
+ expected_artifacts: [result.md]
125
+ generated_node_skills: [my-skill-open]
126
+ route_conformance:
127
+ task: generic-skill-smoke
128
+ expected_node_order: [open, build]
129
+ interaction:
130
+ max_turns: 4
131
+ simulator_prompt: Answer.
132
+ fresh_resume_marker: READY
133
+ execution:
134
+ base_url: https://main.example.com
135
+ judge:
136
+ model: judge-model
137
+ base_url: https://judge.example.com
138
+ """,
139
+ encoding="utf-8",
140
+ )
141
+
142
+ manifest = load_eval_manifest(manifest_path)
143
+
144
+ assert manifest.draft_hash == "0" * 64
145
+ assert manifest.generation_hash == "generated"
146
+ assert manifest.recommended_tasks == ["generic-skill-smoke"]
147
+ assert manifest.required_skills == ["my-skill"]
148
+ assert manifest.expected_artifacts == ["result.md"]
149
+ assert manifest.generated_node_skills == ["my-skill-open"]
150
+ assert manifest.route_conformance_expected_node_order == ["open", "build"]
151
+ assert manifest.interaction.max_turns == 4
152
+ assert manifest.interaction.simulator_prompt == "Answer."
153
+ assert manifest.interaction.fresh_resume_marker == "READY"
154
+ assert manifest.execution.base_url == "https://main.example.com"
155
+ assert manifest.judge is not None
156
+ assert manifest.judge.base_url == "https://judge.example.com"
84
157
 
85
158
 
86
159
  def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
@@ -94,6 +167,37 @@ def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
94
167
  load_eval_manifest(manifest_path)
95
168
 
96
169
 
170
+ @pytest.mark.parametrize(
171
+ ("section", "message"),
172
+ [
173
+ ("metadata:\n draftHash: one\n draft_hash: two", "metadata.draftHash"),
174
+ (
175
+ "evaluation:\n recommendedTasks: [one]\n recommended_tasks: [two]",
176
+ "evaluation.recommendedTasks",
177
+ ),
178
+ (
179
+ "execution:\n baseUrl: https://main.example\n base_url: https://other.example",
180
+ "execution.baseUrl",
181
+ ),
182
+ ],
183
+ )
184
+ def test_load_eval_manifest_rejects_conflicting_aliases(
185
+ tmp_path: Path, section: str, message: str
186
+ ):
187
+ manifest_path = tmp_path / "eval.yaml"
188
+ manifest_path.write_text(
189
+ "apiVersion: comet.eval/v1alpha1\n"
190
+ "kind: SkillEvalManifest\n"
191
+ "metadata:\n name: demo\n"
192
+ "skill:\n name: demo\n"
193
+ f"{section}\n",
194
+ encoding="utf-8",
195
+ )
196
+
197
+ with pytest.raises(ValueError, match=re.escape(message)):
198
+ load_eval_manifest(manifest_path)
199
+
200
+
97
201
  @pytest.mark.parametrize(
98
202
  ("field_yaml", "message"),
99
203
  [
@@ -117,6 +221,10 @@ def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
117
221
  "metadata:\n name: bad\nskill:\n name: bad\nevaluation:\n expectedEvidence:\n - missing-node\n",
118
222
  "evaluation.expectedEvidence",
119
223
  ),
224
+ (
225
+ "metadata:\n name: bad\nskill:\n name: bad\nexecution:\n agent: gemini\n",
226
+ "Unsupported evaluation agent",
227
+ ),
120
228
  ],
121
229
  )
122
230
  def test_load_eval_manifest_rejects_malformed_structured_fields(
@@ -132,3 +240,291 @@ def test_load_eval_manifest_rejects_malformed_structured_fields(
132
240
 
133
241
  with pytest.raises(ValueError, match=message):
134
242
  load_eval_manifest(manifest_path)
243
+
244
+
245
+ def test_load_eval_manifest_parses_inline_and_source_tasks(tmp_path: Path):
246
+ package = tmp_path / "my-skill"
247
+ package.mkdir()
248
+ (package / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
249
+ fixtures = package / "fixtures"
250
+ fixtures.mkdir()
251
+ (fixtures / "input.txt").write_text("input\n", encoding="utf-8")
252
+
253
+ source_task = package / "tasks" / "advanced"
254
+ (source_task / "environment").mkdir(parents=True)
255
+ (source_task / "validation").mkdir()
256
+ (source_task / "task.toml").write_text(
257
+ '[metadata]\nname = "advanced"\n\n[environment]\ndockerfile = "Dockerfile"\n',
258
+ encoding="utf-8",
259
+ )
260
+ (source_task / "instruction.md").write_text("Advanced task\n", encoding="utf-8")
261
+ (source_task / "environment" / "Dockerfile").write_text("FROM alpine\n", encoding="utf-8")
262
+
263
+ manifest_path = package / "comet" / "eval.yaml"
264
+ manifest_path.parent.mkdir()
265
+ manifest_path.write_text(
266
+ """
267
+ apiVersion: comet.eval/v1alpha1
268
+ kind: SkillEvalManifest
269
+ metadata:
270
+ name: my-skill
271
+ skill:
272
+ name: my-skill
273
+ source: ..
274
+ evaluation:
275
+ tasks:
276
+ - name: create-summary
277
+ prompt: Create summary.md.
278
+ workspace: ./fixtures
279
+ expect:
280
+ files:
281
+ - summary.md
282
+ contains:
283
+ summary.md:
284
+ - "# Summary"
285
+ json:
286
+ - file: summary.json
287
+ path: $.status
288
+ equals: complete
289
+ commands:
290
+ - run: test -f summary.md
291
+ timeout: 30
292
+ rubric:
293
+ - The summary explains the result.
294
+ - name: advanced-alias
295
+ source: ./tasks/advanced
296
+ """,
297
+ encoding="utf-8",
298
+ )
299
+
300
+ manifest = load_eval_manifest(manifest_path)
301
+
302
+ assert [task.name for task in manifest.tasks] == ["create-summary", "advanced-alias"]
303
+ inline = manifest.tasks[0]
304
+ assert inline.prompt == "Create summary.md."
305
+ assert inline.workspace == fixtures.resolve()
306
+ assert inline.expect["files"] == ["summary.md"]
307
+ assert inline.rubric == ["The summary explains the result."]
308
+ assert manifest.tasks[1].source == source_task.resolve()
309
+ assert [task.name for task in load_manifest_tasks(manifest)] == [
310
+ "create-summary",
311
+ "advanced-alias",
312
+ ]
313
+
314
+
315
+ @pytest.mark.parametrize(
316
+ ("task_yaml", "message"),
317
+ [
318
+ (
319
+ "- name: missing-prompt\n expect:\n files: [result.md]\n",
320
+ "prompt",
321
+ ),
322
+ (
323
+ "- name: no-expect\n prompt: Do it.\n",
324
+ "expect: is required",
325
+ ),
326
+ (
327
+ "- name: unsafe\n prompt: Do it.\n expect:\n files: [../secret]\n",
328
+ "evaluation.tasks.*.expect.files",
329
+ ),
330
+ (
331
+ "- name: unsafe\n prompt: Do it.\n workspace: ../outside\n expect:\n files: [result.md]\n",
332
+ "workspace",
333
+ ),
334
+ (
335
+ "- name: unknown\n prompt: Do it.\n expect:\n files: [result.md]\n executable: true\n",
336
+ "unknown field",
337
+ ),
338
+ ],
339
+ )
340
+ def test_load_eval_manifest_rejects_invalid_inline_tasks(
341
+ tmp_path: Path,
342
+ task_yaml: str,
343
+ message: str,
344
+ ):
345
+ package = tmp_path / "my-skill"
346
+ (package / "comet").mkdir(parents=True)
347
+ manifest_path = package / "comet" / "eval.yaml"
348
+ manifest_path.write_text(
349
+ "apiVersion: comet.eval/v1alpha1\n"
350
+ "kind: SkillEvalManifest\n"
351
+ "metadata:\n name: my-skill\n"
352
+ "skill:\n name: my-skill\n source: ..\n"
353
+ "evaluation:\n tasks:\n"
354
+ + "".join(f" {line}\n" for line in task_yaml.rstrip().splitlines()),
355
+ encoding="utf-8",
356
+ )
357
+
358
+ with pytest.raises(ValueError, match=message):
359
+ load_eval_manifest(manifest_path)
360
+
361
+
362
+ def test_load_eval_manifest_rejects_duplicate_authored_task_names(tmp_path: Path):
363
+ package = tmp_path / "my-skill"
364
+ (package / "comet").mkdir(parents=True)
365
+ manifest_path = package / "comet" / "eval.yaml"
366
+ manifest_path.write_text(
367
+ """
368
+ apiVersion: comet.eval/v1alpha1
369
+ kind: SkillEvalManifest
370
+ metadata:
371
+ name: my-skill
372
+ skill:
373
+ name: my-skill
374
+ source: ..
375
+ evaluation:
376
+ tasks:
377
+ - name: duplicate
378
+ prompt: First.
379
+ expect:
380
+ files: [one.txt]
381
+ - name: duplicate
382
+ prompt: Second.
383
+ expect:
384
+ files: [two.txt]
385
+ """,
386
+ encoding="utf-8",
387
+ )
388
+
389
+ with pytest.raises(ValueError, match="duplicates evaluation.tasks"):
390
+ load_eval_manifest(manifest_path)
391
+
392
+
393
+ def test_load_eval_manifest_rejects_duplicate_recommended_task_names(tmp_path: Path):
394
+ manifest_path = tmp_path / "eval.yaml"
395
+ manifest_path.write_text(
396
+ "apiVersion: comet.eval/v1alpha1\n"
397
+ "kind: SkillEvalManifest\n"
398
+ "metadata: {name: my-skill}\n"
399
+ "skill: {name: my-skill, source: .}\n"
400
+ "evaluation:\n"
401
+ " recommendedTasks: [generic-skill-smoke, generic-skill-smoke]\n",
402
+ encoding="utf-8",
403
+ )
404
+
405
+ with pytest.raises(
406
+ ValueError,
407
+ match=re.escape(
408
+ 'evaluation.recommendedTasks[1] duplicates '
409
+ 'evaluation.recommendedTasks[0]: "generic-skill-smoke"'
410
+ ),
411
+ ):
412
+ load_eval_manifest(manifest_path)
413
+
414
+
415
+ @pytest.mark.parametrize(
416
+ ("section", "message"),
417
+ [
418
+ ("execution:\n model: ' '\n", "execution.model"),
419
+ ("execution:\n baseUrl: ftp://main.example.com\n", "execution.baseUrl"),
420
+ ("judge:\n agent: codex\n", "judge.model"),
421
+ ("judge:\n model: ' '\n", "judge.model"),
422
+ (
423
+ "judge:\n agent: unknown-agent\n model: judge-model\n",
424
+ "manifest judge.agent",
425
+ ),
426
+ (
427
+ "judge:\n model: judge-model\n baseUrl: relative/path\n",
428
+ "judge.baseUrl",
429
+ ),
430
+ ],
431
+ )
432
+ def test_manifest_rejects_invalid_preserved_execution_and_judge_fields(
433
+ tmp_path: Path, section: str, message: str
434
+ ):
435
+ manifest_path = tmp_path / "eval.yaml"
436
+ manifest_path.write_text(
437
+ "apiVersion: comet.eval/v1alpha1\n"
438
+ "kind: SkillEvalManifest\n"
439
+ "metadata: {name: my-skill}\n"
440
+ "skill: {name: my-skill, source: .}\n"
441
+ + section,
442
+ encoding="utf-8",
443
+ )
444
+
445
+ with pytest.raises(ValueError, match=re.escape(message)):
446
+ load_eval_manifest(manifest_path)
447
+
448
+
449
+ @pytest.mark.parametrize(
450
+ ("yaml_fragment", "message"),
451
+ [
452
+ ("unknownTopLevel: true\n", "unknownTopLevel: unknown field"),
453
+ ("evaluation:\n unknownFlag: true\n", "evaluation.unknownFlag: unknown field"),
454
+ ("execution:\n unknownFlag: true\n", "execution.unknownFlag: unknown field"),
455
+ (
456
+ "judge:\n model: judge-model\n unknownFlag: true\n",
457
+ "judge.unknownFlag: unknown field",
458
+ ),
459
+ ("reporting:\n unknownFlag: true\n", "reporting.unknownFlag: unknown field"),
460
+ ("interaction:\n unknownFlag: true\n", "interaction.unknownFlag: unknown field"),
461
+ (
462
+ "evaluation:\n"
463
+ " tasks:\n"
464
+ " - name: exact\n"
465
+ " prompt: Write result.md.\n"
466
+ " expect: {files: [result.md]}\n"
467
+ " unknownFlag: true\n",
468
+ "evaluation.tasks[0].unknownFlag: unknown field",
469
+ ),
470
+ ],
471
+ )
472
+ def test_schema_rejects_unknown_fields_with_yaml_paths(tmp_path: Path, yaml_fragment: str, message: str):
473
+ manifest_path = tmp_path / "eval.yaml"
474
+ manifest_path.write_text(
475
+ "apiVersion: comet.eval/v1alpha1\n"
476
+ "kind: SkillEvalManifest\n"
477
+ "metadata:\n"
478
+ " name: my-skill\n"
479
+ "skill:\n"
480
+ " name: my-skill\n"
481
+ " source: .\n"
482
+ + yaml_fragment,
483
+ encoding="utf-8",
484
+ )
485
+
486
+ with pytest.raises(ValueError, match=re.escape(message)):
487
+ load_eval_manifest(manifest_path)
488
+
489
+
490
+ @pytest.mark.parametrize(
491
+ ("fragment", "message"),
492
+ [
493
+ ("interaction: null\n", "interaction: None is not of type 'object'"),
494
+ ("evaluation: []\n", "evaluation: [] is not of type 'object'"),
495
+ ("execution: invalid\n", "execution: 'invalid' is not of type 'object'"),
496
+ ],
497
+ )
498
+ def test_schema_rejects_explicit_non_mapping_sections(
499
+ tmp_path: Path, fragment: str, message: str
500
+ ):
501
+ manifest_path = tmp_path / "eval.yaml"
502
+ manifest_path.write_text(
503
+ "apiVersion: comet.eval/v1alpha1\nkind: SkillEvalManifest\nmetadata: {name: my-skill}\nskill: {name: my-skill, source: .}\n"
504
+ + fragment,
505
+ encoding="utf-8",
506
+ )
507
+ with pytest.raises(ValueError, match=re.escape(message)):
508
+ load_eval_manifest(manifest_path)
509
+
510
+
511
+ def test_schema_declared_interaction_fields_are_normalized(tmp_path: Path):
512
+ manifest_path = tmp_path / "eval.yaml"
513
+ manifest_path.write_text(
514
+ """apiVersion: comet.eval/v1alpha1
515
+ kind: SkillEvalManifest
516
+ metadata: {name: my-skill}
517
+ skill: {name: my-skill, source: .}
518
+ interaction:
519
+ decisionPatterns: [confirm]
520
+ decisionReply: "yes"
521
+ decisionReplies: [first, second]
522
+ continuePrompt: Keep going.
523
+ """,
524
+ encoding="utf-8",
525
+ )
526
+ interaction = load_eval_manifest(manifest_path).interaction
527
+ assert interaction.decision_patterns == ["confirm"]
528
+ assert interaction.decision_reply == "yes"
529
+ assert interaction.decision_replies == ["first", "second"]
530
+ assert interaction.continue_prompt == "Keep going."
@@ -287,6 +287,23 @@ def test_validate_native_workflow_rejects_external_skill_invocation(tmp_path: Pa
287
287
  )
288
288
 
289
289
 
290
+ def test_validate_native_workflow_does_not_infer_custom_agent_skill_invocation(
291
+ tmp_path: Path,
292
+ ):
293
+ _write_native_archive(tmp_path)
294
+
295
+ passed, failed = validate_native_workflow(
296
+ tmp_path,
297
+ {
298
+ "agent": "fixture-agent",
299
+ "events": {"skills_invoked": ["comet-native"]},
300
+ },
301
+ )
302
+
303
+ assert "native_skill_invocation" not in passed
304
+ assert any(item.startswith("native_skill_invocation:") for item in failed)
305
+
306
+
290
307
  def test_adapt_checks_for_native_replaces_classic_contract_but_keeps_business(tmp_path: Path):
291
308
  _write_native_archive(tmp_path)
292
309
 
@@ -422,8 +422,7 @@ def test_comet_profile_recognizes_proxied_openspec_spec_reconciliation(
422
422
  passed, _ = run_profile_rubric("comet-workflow", tmp_path, outputs)
423
423
 
424
424
  assert any(
425
- "[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg
426
- for msg in passed
425
+ "[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg for msg in passed
427
426
  )
428
427
 
429
428
 
@@ -445,8 +444,7 @@ def test_comet_profile_still_recognizes_direct_openspec_spec_reconciliation(tmp_
445
444
  passed, _ = run_profile_rubric("comet-workflow", tmp_path, outputs)
446
445
 
447
446
  assert any(
448
- "[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg
449
- for msg in passed
447
+ "[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg for msg in passed
450
448
  )
451
449
 
452
450
 
@@ -944,6 +942,64 @@ def test_judge_env_maps_independent_provider(monkeypatch):
944
942
  assert "subject-token" not in invocation.env.values()
945
943
 
946
944
 
945
+ def test_judge_env_selects_the_same_agent_with_isolated_credentials(monkeypatch):
946
+ from scaffold.python.judge_config import build_judge_invocation
947
+
948
+ monkeypatch.setenv("OPENAI_API_KEY", "subject-key")
949
+ monkeypatch.setenv("BENCH_JUDGE_MODEL", "judge-model")
950
+ monkeypatch.setenv("BENCH_JUDGE_API_KEY", "judge-key")
951
+
952
+ invocation = build_judge_invocation(agent="codex")
953
+
954
+ assert invocation.agent == "codex"
955
+ assert invocation.env["OPENAI_API_KEY"] == "judge-key"
956
+ assert "ANTHROPIC_API_KEY" not in invocation.env
957
+ assert "subject-key" not in invocation.env.values()
958
+
959
+
960
+ def test_judge_env_maps_codebuddy_credentials_and_endpoint(monkeypatch):
961
+ from scaffold.python.judge_config import build_judge_invocation
962
+
963
+ monkeypatch.setenv("CODEBUDDY_API_KEY", "subject-key")
964
+ monkeypatch.setenv("CODEBUDDY_MODEL", "subject-model")
965
+ monkeypatch.setenv("BENCH_JUDGE_MODEL", "judge-model")
966
+ monkeypatch.setenv("BENCH_JUDGE_API_KEY", "judge-key")
967
+ monkeypatch.setenv("BENCH_JUDGE_AUTH_TOKEN", "judge-token")
968
+ monkeypatch.setenv("BENCH_JUDGE_BASE_URL", "https://judge.example")
969
+
970
+ invocation = build_judge_invocation(agent="codebuddy")
971
+
972
+ assert invocation.agent == "codebuddy"
973
+ assert invocation.env["CODEBUDDY_API_KEY"] == "judge-key"
974
+ assert invocation.env["CODEBUDDY_AUTH_TOKEN"] == "judge-token"
975
+ assert invocation.env["CODEBUDDY_BASE_URL"] == "https://judge.example"
976
+ assert invocation.env["CODEBUDDY_MODEL"] == "judge-model"
977
+ assert invocation.env["COMET_EVAL_AGENT_ROLE"] == "judge"
978
+ assert "subject-key" not in invocation.env.values()
979
+ assert "subject-model" not in invocation.env.values()
980
+
981
+
982
+ def test_codebuddy_judge_extracts_response_text_from_json_stream():
983
+ from scaffold.python.judge_config import _extract_agent_text
984
+
985
+ assert _extract_agent_text('{"response":"[RUBRIC-JUDGE] ok"}') == "[RUBRIC-JUDGE] ok"
986
+
987
+
988
+ def test_judge_session_ids_are_extracted_from_nested_agent_events():
989
+ import json
990
+
991
+ from scaffold.python.judge_config import _extract_agent_session_ids
992
+
993
+ output = "\n".join(
994
+ [
995
+ json.dumps({"type": "thread.started", "thread_id": "judge-thread"}),
996
+ json.dumps({"message": {"sessionId": "judge-session"}}),
997
+ ]
998
+ )
999
+
1000
+ assert _extract_agent_session_ids(output) == ["judge-thread", "judge-session"]
1001
+
1002
+
947
1003
  def test_judge_provider_uses_direct_http_when_base_url_is_configured(monkeypatch):
948
1004
  """Dedicated judge providers should not require Claude CLI compatibility."""
949
1005
  import json
@@ -1034,17 +1090,19 @@ def test_generic_llm_judge_parses_output(tmp_path: Path):
1034
1090
  "[RUBRIC-JUDGE] output_quality: 0.90 - well-structured code\n"
1035
1091
  "[RUBRIC-JUDGE] instruction_adherence: 1.00 - all constraints followed\n"
1036
1092
  )
1037
- outputs = {"completion": {"passed": [], "failed": []}}
1093
+ outputs = {"completion": {"passed": [], "failed": []}, "agent": "codebuddy"}
1038
1094
 
1039
1095
  with patch(
1040
1096
  "scaffold.python.generic_llm_judge._run_judge",
1041
1097
  return_value=mock_output,
1042
- ):
1098
+ ) as run_judge:
1043
1099
  scores = judge_generic_artifacts(tmp_path, outputs)
1044
1100
 
1045
1101
  assert scores["task_completion"] == (0.80, "output present and mostly correct")
1046
1102
  assert scores["output_quality"] == (0.90, "well-structured code")
1047
1103
  assert scores["instruction_adherence"] == (1.00, "all constraints followed")
1104
+ assert run_judge.call_args.kwargs["agent"] == "codebuddy"
1105
+ assert run_judge.call_args.kwargs["evidence"] is outputs
1048
1106
 
1049
1107
 
1050
1108
  def test_generic_llm_judge_parses_custom_dimensions(tmp_path: Path):