@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -0,0 +1,217 @@
1
+ """Tests for standalone Eval target resolution and user-owned state."""
2
+
3
+ import json
4
+ import os
5
+ from pathlib import Path
6
+ import subprocess
7
+
8
+ import pytest
9
+
10
+ from scaffold.python.eval_context import (
11
+ EvalContextError,
12
+ context_from_environment,
13
+ resolve_managed_path,
14
+ resolve_eval_context,
15
+ )
16
+ from scaffold.python.logging import ExperimentLogger
17
+
18
+
19
+ def _write_skill(root: Path, *, manifest_name: str | None = None) -> Path:
20
+ skill = root / "skill"
21
+ skill.mkdir()
22
+ (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
23
+ if manifest_name:
24
+ manifest = skill / "comet" / manifest_name
25
+ manifest.parent.mkdir()
26
+ manifest.write_text(
27
+ "\n".join(
28
+ [
29
+ "apiVersion: comet.eval/v1alpha1",
30
+ "kind: SkillEvalManifest",
31
+ "metadata:",
32
+ " name: demo",
33
+ "skill:",
34
+ " name: demo",
35
+ " source: ..",
36
+ "",
37
+ ]
38
+ ),
39
+ encoding="utf-8",
40
+ )
41
+ return skill
42
+
43
+
44
+ @pytest.mark.parametrize("entry", ["directory", "skill-file"])
45
+ def test_skill_entries_auto_detect_only_the_skill_local_manifest(tmp_path: Path, entry: str):
46
+ skill = _write_skill(tmp_path, manifest_name="eval.yml")
47
+ (tmp_path / "comet").mkdir()
48
+ (tmp_path / "comet" / "eval.yaml").write_text("unrelated", encoding="utf-8")
49
+
50
+ context = resolve_eval_context(
51
+ skill_path=skill if entry == "directory" else skill / "SKILL.md"
52
+ )
53
+
54
+ assert context.skill_root == skill.resolve()
55
+ assert context.manifest_source == "auto-detected"
56
+ assert context.manifest_path == (skill / "comet" / "eval.yml").resolve()
57
+ assert context.artifact_owner_root == skill.resolve()
58
+ assert context.artifact_root == skill / ".comet" / "eval"
59
+
60
+
61
+ def test_auto_detection_prefers_eval_yaml_before_eval_yml(tmp_path: Path):
62
+ skill = _write_skill(tmp_path, manifest_name="eval.yml")
63
+ yaml_manifest = skill / "comet" / "eval.yaml"
64
+ yaml_manifest.write_text("kind: SkillEvalManifest\n", encoding="utf-8")
65
+
66
+ context = resolve_eval_context(skill_path=skill)
67
+
68
+ assert context.manifest_path == yaml_manifest.resolve()
69
+
70
+
71
+ def test_context_synthesizes_without_writing_into_the_skill_and_is_cwd_independent(
72
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
73
+ ):
74
+ skill = _write_skill(tmp_path)
75
+ elsewhere = tmp_path / "elsewhere"
76
+ elsewhere.mkdir()
77
+ monkeypatch.chdir(elsewhere)
78
+
79
+ context = resolve_eval_context(skill_path=Path("..") / "skill")
80
+
81
+ assert context.manifest_source == "synthesized"
82
+ assert context.manifest_path is None
83
+ assert context.base_manifest["skill"]["source"] == str(skill.resolve())
84
+ assert not (skill / "comet").exists()
85
+
86
+
87
+ def test_explicit_manifest_and_project_owner_are_authoritative(tmp_path: Path):
88
+ skill = _write_skill(tmp_path)
89
+ project = tmp_path / "project"
90
+ project.mkdir()
91
+ manifest = tmp_path / "outside" / "eval.yaml"
92
+ manifest.parent.mkdir()
93
+ manifest.write_text(
94
+ "\n".join(
95
+ [
96
+ "apiVersion: comet.eval/v1alpha1",
97
+ "kind: SkillEvalManifest",
98
+ "metadata:",
99
+ " name: demo",
100
+ "skill:",
101
+ " name: demo",
102
+ f" source: {skill.as_posix()}",
103
+ "",
104
+ ]
105
+ ),
106
+ encoding="utf-8",
107
+ )
108
+
109
+ context = resolve_eval_context(manifest_path=manifest, project_root=project)
110
+
111
+ assert context.manifest_source == "explicit"
112
+ assert context.manifest_path == manifest.resolve()
113
+ assert context.skill_root == skill.resolve()
114
+ assert context.artifact_owner_root == project.resolve()
115
+ assert context.artifact_root == project.resolve() / ".comet" / "eval"
116
+
117
+
118
+ def test_context_rejects_manifest_paths_outside_the_skill_package(tmp_path: Path):
119
+ skill = _write_skill(tmp_path)
120
+ outside = tmp_path / "outside"
121
+ outside.mkdir()
122
+ manifest = skill / "comet" / "eval.yaml"
123
+ manifest.parent.mkdir()
124
+ manifest.write_text(
125
+ "kind: SkillEvalManifest\nskill:\n source: ../../outside\n", encoding="utf-8"
126
+ )
127
+
128
+ with pytest.raises(EvalContextError, match="Skill package must contain SKILL.md"):
129
+ resolve_eval_context(manifest_path=manifest)
130
+
131
+
132
+ def test_context_rejects_an_owner_comet_link_that_escapes_the_owner(tmp_path: Path):
133
+ skill = _write_skill(tmp_path)
134
+ owner = tmp_path / "owner"
135
+ outside = tmp_path / "outside"
136
+ owner.mkdir()
137
+ outside.mkdir()
138
+ link = owner / ".comet"
139
+ if os.name == "nt":
140
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(outside)], check=True)
141
+ else:
142
+ link.symlink_to(outside, target_is_directory=True)
143
+
144
+ with pytest.raises(EvalContextError, match="artifact root must stay within its owner root"):
145
+ resolve_eval_context(skill_path=skill, project_root=owner)
146
+
147
+
148
+ @pytest.mark.parametrize(
149
+ ("managed_root", "child"),
150
+ [
151
+ ("cache", "uv"),
152
+ ("runs", "report"),
153
+ ("generated", "demo"),
154
+ ("locks", "generation.lock"),
155
+ ],
156
+ )
157
+ def test_managed_paths_reject_owner_child_links_that_escape(
158
+ tmp_path: Path, managed_root: str, child: str
159
+ ):
160
+ skill = _write_skill(tmp_path)
161
+ owner = tmp_path / "owner"
162
+ outside = tmp_path / "outside"
163
+ owner.mkdir()
164
+ outside.mkdir()
165
+ context = resolve_eval_context(skill_path=skill, project_root=owner)
166
+ link = context.artifact_root / managed_root
167
+ link.parent.mkdir(parents=True)
168
+ if os.name == "nt":
169
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(outside)], check=True)
170
+ else:
171
+ link.symlink_to(outside, target_is_directory=True)
172
+
173
+ with pytest.raises(EvalContextError, match="managed path must stay within its owner root"):
174
+ resolve_managed_path(context, managed_root, child)
175
+
176
+
177
+ def test_environment_context_preserves_user_owned_artifact_root_and_rejects_credentials(
178
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
179
+ ):
180
+ skill = _write_skill(tmp_path)
181
+ context = resolve_eval_context(skill_path=skill)
182
+ payload = context.to_payload()
183
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(payload))
184
+
185
+ assert context_from_environment() == context
186
+
187
+ payload["api_key"] = "secret"
188
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(payload))
189
+ with pytest.raises(EvalContextError, match="credentials"):
190
+ context_from_environment()
191
+
192
+
193
+ def test_run_layout_and_metadata_are_user_owned_and_written_atomically(tmp_path: Path, monkeypatch):
194
+ skill = _write_skill(tmp_path)
195
+ context = resolve_eval_context(skill_path=skill)
196
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
197
+ replacements = []
198
+ import scaffold.python.logging as logging
199
+
200
+ original_replace = logging.os.replace
201
+ monkeypatch.setattr(
202
+ logging.os,
203
+ "replace",
204
+ lambda source, target: (replacements.append((source, target)), original_replace(source, target))[1],
205
+ )
206
+ logger = ExperimentLogger(experiment_name="standalone", experiment_id="run-1")
207
+ logger.finalize()
208
+
209
+ assert logger.base_dir == context.artifact_root / "runs" / "run-1"
210
+ assert {
211
+ path.name
212
+ for path in [logger.events_dir, logger.reports_dir, logger.raw_dir, logger.base_dir / "artifacts"]
213
+ } == {"events", "reports", "raw", "artifacts"}
214
+ assert (logger.base_dir / "summary.md").is_file()
215
+ assert (logger.base_dir / "metadata.json").is_file()
216
+ assert replacements and replacements[-1][1] == logger.base_dir / "metadata.json"
217
+ assert "secret" not in (logger.base_dir / "metadata.json").read_text(encoding="utf-8")
@@ -0,0 +1,364 @@
1
+ """Tests for standalone eval execution and Judge resolution."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import pytest
6
+
7
+ from scaffold.python.execution import (
8
+ build_agent_environment,
9
+ build_judge_environment,
10
+ preflight_credentials,
11
+ resolve_execution,
12
+ resolve_judge,
13
+ )
14
+
15
+
16
+ def _install_fixture_adapter(root, monkeypatch):
17
+ fixture = root / "fixture-agent"
18
+ fixture.mkdir()
19
+ (fixture / "adapter.yaml").write_text(
20
+ """apiVersion: comet.eval.agent/v1alpha1
21
+ kind: EvalAgentAdapter
22
+ metadata:
23
+ id: fixture-agent
24
+ version: 1.0.0
25
+ runtime:
26
+ executable: fixture-agent
27
+ install:
28
+ kind: none
29
+ credentials:
30
+ - FIXTURE_AGENT_API_KEY
31
+ modelEnv: FIXTURE_AGENT_MODEL
32
+ baseUrlEnv: FIXTURE_AGENT_BASE_URL
33
+ capabilities:
34
+ singleTurn: true
35
+ resume: true
36
+ structuredEvents: true
37
+ telemetry: false
38
+ skillInvocationEvidence: true
39
+ """,
40
+ encoding="utf-8",
41
+ )
42
+ monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(root))
43
+
44
+
45
+ def test_main_resolution_uses_cli_then_manifest_then_legacy_environment():
46
+ environment = {
47
+ "BENCH_EVAL_AGENT": "qoder",
48
+ "BENCH_QODER_MODEL": "legacy-qoder",
49
+ "QODER_BASE_URL": "https://legacy.example/v1",
50
+ }
51
+
52
+ resolved = resolve_execution(
53
+ cli_agent="codex",
54
+ cli_model="cli-model",
55
+ cli_base_url="https://cli.example/v1",
56
+ manifest={
57
+ "agent": "claude-code",
58
+ "model": "manifest-model",
59
+ "baseUrl": "https://manifest.example/v1",
60
+ },
61
+ source_env=environment,
62
+ )
63
+
64
+ assert (resolved.agent, resolved.model, resolved.base_url) == (
65
+ "codex",
66
+ "cli-model",
67
+ "https://cli.example/v1",
68
+ )
69
+ assert resolved.sources == {"agent": "cli", "model": "cli", "base_url": "cli"}
70
+
71
+
72
+ def test_main_resolution_falls_back_to_selected_agent_legacy_defaults():
73
+ resolved = resolve_execution(
74
+ manifest={"agent": "codebuddy"},
75
+ source_env={
76
+ "BENCH_CODEBUDDY_MODEL": "legacy-codebuddy",
77
+ "CODEBUDDY_BASE_URL": "https://codebuddy.example/v1",
78
+ },
79
+ )
80
+
81
+ assert resolved.agent == "codebuddy"
82
+ assert resolved.model == "legacy-codebuddy"
83
+ assert resolved.base_url == "https://codebuddy.example/v1"
84
+
85
+
86
+ def test_judge_inherits_only_main_agent_and_requires_independent_model():
87
+ main = resolve_execution(cli_agent="codex", cli_model="subject-model")
88
+
89
+ judge = resolve_judge(main=main, manifest={"model": "judge-model"}, source_env={})
90
+
91
+ assert judge is not None
92
+ assert judge.agent == "codex"
93
+ assert judge.model == "judge-model"
94
+ assert judge.base_url is None
95
+ assert judge.model != main.model
96
+
97
+ with pytest.raises(ValueError, match="BENCH_JUDGE_MODEL"):
98
+ resolve_judge(main=main, manifest={}, source_env={})
99
+
100
+
101
+ def test_judge_cli_and_dedicated_environment_never_inherit_main_provider_values():
102
+ main = resolve_execution(
103
+ cli_agent="claude-code",
104
+ cli_model="subject-model",
105
+ cli_base_url="https://subject.example/v1",
106
+ source_env={"ANTHROPIC_API_KEY": "subject-key"},
107
+ )
108
+ judge = resolve_judge(
109
+ main=main,
110
+ cli_agent="codebuddy",
111
+ cli_model="judge-model",
112
+ cli_base_url="https://judge.example/v1",
113
+ source_env={
114
+ "BENCH_JUDGE_API_KEY": "judge-key",
115
+ "ANTHROPIC_API_KEY": "subject-key",
116
+ "ANTHROPIC_BASE_URL": "https://subject.example/v1",
117
+ },
118
+ )
119
+
120
+ assert judge is not None
121
+ child_env = build_judge_environment(judge, source_env={
122
+ "BENCH_JUDGE_API_KEY": "judge-key",
123
+ "ANTHROPIC_API_KEY": "subject-key",
124
+ "ANTHROPIC_BASE_URL": "https://subject.example/v1",
125
+ })
126
+ assert child_env["CODEBUDDY_API_KEY"] == "judge-key"
127
+ assert child_env["CODEBUDDY_BASE_URL"] == "https://judge.example/v1"
128
+ assert "ANTHROPIC_API_KEY" not in child_env
129
+ assert "ANTHROPIC_BASE_URL" not in child_env
130
+ assert "subject-key" not in child_env.values()
131
+
132
+
133
+ def test_main_environment_maps_model_and_url_through_each_builtin_adapter():
134
+ expected = {
135
+ "claude-code": ("ANTHROPIC_MODEL", "ANTHROPIC_BASE_URL"),
136
+ "codex": ("OPENAI_MODEL", "OPENAI_BASE_URL"),
137
+ "qoder": ("QODER_MODEL", "QODER_BASE_URL"),
138
+ "codebuddy": ("CODEBUDDY_MODEL", "CODEBUDDY_BASE_URL"),
139
+ }
140
+
141
+ for agent, (model_key, url_key) in expected.items():
142
+ resolved = resolve_execution(
143
+ cli_agent=agent,
144
+ cli_model=f"{agent}-model",
145
+ cli_base_url=f"https://{agent}.example/v1",
146
+ )
147
+ child_env = build_agent_environment(
148
+ resolved,
149
+ source_env={"OPENAI_API_KEY": "main-key"},
150
+ )
151
+ assert child_env[model_key] == f"{agent}-model"
152
+ assert child_env[url_key] == f"https://{agent}.example/v1"
153
+
154
+
155
+ def test_common_subject_settings_map_to_each_builtin_agent():
156
+ expected = {
157
+ "claude-code": ("ANTHROPIC_MODEL", "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY", True),
158
+ "codex": ("OPENAI_MODEL", "OPENAI_BASE_URL", "OPENAI_API_KEY", True),
159
+ "qoder": ("QODER_MODEL", "QODER_BASE_URL", "QODER_PERSONAL_ACCESS_TOKEN", False),
160
+ "codebuddy": ("CODEBUDDY_MODEL", "CODEBUDDY_BASE_URL", "CODEBUDDY_API_KEY", True),
161
+ }
162
+ source = {
163
+ "BENCH_MODEL": "bench-model",
164
+ "BENCH_BASE_URL": "https://bench.example/v1",
165
+ "BENCH_API_KEY": "bench-key",
166
+ }
167
+
168
+ for agent, (model_key, url_key, credential_key, uses_common_url) in expected.items():
169
+ resolved = resolve_execution(cli_agent=agent, source_env=source)
170
+ child_env = build_agent_environment(resolved, source_env=source)
171
+
172
+ assert resolved.model == "bench-model"
173
+ assert resolved.base_url == ("https://bench.example/v1" if uses_common_url else None)
174
+ assert child_env[model_key] == "bench-model"
175
+ assert child_env[credential_key] == "bench-key"
176
+ if uses_common_url:
177
+ assert child_env[url_key] == "https://bench.example/v1"
178
+ else:
179
+ assert url_key not in child_env
180
+
181
+
182
+ def test_explicit_subject_credentials_override_common_fallback():
183
+ resolved = resolve_execution(
184
+ cli_agent="claude-code",
185
+ source_env={"BENCH_API_KEY": "fallback-key", "ANTHROPIC_API_KEY": "explicit-key"},
186
+ )
187
+
188
+ child_env = build_agent_environment(
189
+ resolved,
190
+ source_env={"BENCH_API_KEY": "fallback-key", "ANTHROPIC_API_KEY": "explicit-key"},
191
+ )
192
+
193
+ assert child_env["ANTHROPIC_API_KEY"] == "explicit-key"
194
+
195
+
196
+ def test_common_subject_api_key_satisfies_preflight():
197
+ main = resolve_execution(
198
+ cli_agent="codex",
199
+ source_env={"BENCH_MODEL": "bench-model", "BENCH_API_KEY": "bench-key"},
200
+ )
201
+
202
+ assert preflight_credentials(main, None, source_env={"BENCH_API_KEY": "bench-key"}) == []
203
+
204
+
205
+ def test_explicit_agent_model_and_base_url_override_common_fallbacks():
206
+ resolved = resolve_execution(
207
+ cli_agent="codex",
208
+ source_env={
209
+ "BENCH_MODEL": "common-model",
210
+ "BENCH_BASE_URL": "https://common.example/v1",
211
+ "OPENAI_MODEL": "native-model",
212
+ "OPENAI_BASE_URL": "https://native.example/v1",
213
+ },
214
+ )
215
+
216
+ assert resolved.model == "native-model"
217
+ assert resolved.base_url == "https://native.example/v1"
218
+
219
+
220
+ def test_custom_agent_environment_preserves_declared_runtime_contract(
221
+ tmp_path, monkeypatch
222
+ ):
223
+ _install_fixture_adapter(tmp_path, monkeypatch)
224
+
225
+ resolved = resolve_execution(
226
+ cli_agent="fixture-agent",
227
+ cli_model="fixture-model",
228
+ cli_base_url="https://fixture.example/v1",
229
+ )
230
+ child_env = build_agent_environment(
231
+ resolved,
232
+ source_env={"FIXTURE_AGENT_API_KEY": "main-secret"},
233
+ )
234
+
235
+ assert child_env["FIXTURE_AGENT_MODEL"] == "fixture-model"
236
+ assert child_env["FIXTURE_AGENT_BASE_URL"] == "https://fixture.example/v1"
237
+ assert child_env["COMET_EVAL_CUSTOM_AGENT_ID"] == "fixture-agent"
238
+ assert child_env["COMET_EVAL_CUSTOM_CREDENTIALS"] == "FIXTURE_AGENT_API_KEY"
239
+
240
+
241
+ def test_custom_judge_environment_uses_only_dedicated_credentials(tmp_path, monkeypatch):
242
+ _install_fixture_adapter(tmp_path, monkeypatch)
243
+ main = resolve_execution(cli_agent="codex", cli_model="subject-model")
244
+ judge = resolve_judge(
245
+ main=main,
246
+ cli_agent="fixture-agent",
247
+ cli_model="judge-model",
248
+ cli_base_url="https://judge.example/v1",
249
+ source_env={"BENCH_JUDGE_API_KEY": "judge-secret"},
250
+ )
251
+ assert judge is not None
252
+
253
+ child_env = build_judge_environment(
254
+ judge,
255
+ source_env={
256
+ "BENCH_JUDGE_API_KEY": "judge-secret",
257
+ "FIXTURE_AGENT_API_KEY": "main-secret",
258
+ "OPENAI_API_KEY": "subject-secret",
259
+ },
260
+ )
261
+
262
+ assert child_env["FIXTURE_AGENT_API_KEY"] == "judge-secret"
263
+ assert child_env["FIXTURE_AGENT_MODEL"] == "judge-model"
264
+ assert child_env["FIXTURE_AGENT_BASE_URL"] == "https://judge.example/v1"
265
+ assert "main-secret" not in child_env.values()
266
+ assert "subject-secret" not in child_env.values()
267
+
268
+
269
+ def test_custom_main_credentials_require_all_declared_values(tmp_path, monkeypatch):
270
+ _install_fixture_adapter(tmp_path, monkeypatch)
271
+ adapter_path = tmp_path / "fixture-agent" / "adapter.yaml"
272
+ adapter_path.write_text(
273
+ adapter_path.read_text(encoding="utf-8").replace(
274
+ " - FIXTURE_AGENT_API_KEY\n", " - FIXTURE_AGENT_API_KEY\n - FIXTURE_AGENT_AUTH_TOKEN\n"
275
+ ),
276
+ encoding="utf-8",
277
+ )
278
+ main = resolve_execution(cli_agent="fixture-agent", cli_model="subject-model")
279
+
280
+ assert preflight_credentials(
281
+ main,
282
+ None,
283
+ source_env={"FIXTURE_AGENT_API_KEY": "main-secret"},
284
+ ) == ["main credentials missing for fixture-agent: FIXTURE_AGENT_AUTH_TOKEN"]
285
+
286
+ judge = resolve_judge(
287
+ main=main,
288
+ cli_agent="fixture-agent",
289
+ cli_model="judge-model",
290
+ source_env={"BENCH_JUDGE_API_KEY": "judge-secret"},
291
+ )
292
+ assert judge is not None
293
+ assert preflight_credentials(
294
+ main,
295
+ judge,
296
+ source_env={
297
+ "FIXTURE_AGENT_API_KEY": "main-secret",
298
+ "FIXTURE_AGENT_AUTH_TOKEN": "main-token",
299
+ "BENCH_JUDGE_API_KEY": "judge-secret",
300
+ },
301
+ ) == ["judge credentials missing: BENCH_JUDGE_AUTH_TOKEN"]
302
+
303
+
304
+ def test_custom_judge_without_declared_credentials_needs_no_judge_secret(tmp_path, monkeypatch):
305
+ _install_fixture_adapter(tmp_path, monkeypatch)
306
+ adapter_path = tmp_path / "fixture-agent" / "adapter.yaml"
307
+ adapter_path.write_text(
308
+ adapter_path.read_text(encoding="utf-8").replace(
309
+ "credentials:\n - FIXTURE_AGENT_API_KEY\n", "credentials: []\n"
310
+ ),
311
+ encoding="utf-8",
312
+ )
313
+ main = resolve_execution(cli_agent="fixture-agent", cli_model="subject-model")
314
+ judge = resolve_judge(
315
+ main=main,
316
+ cli_agent="fixture-agent",
317
+ cli_model="judge-model",
318
+ source_env={},
319
+ )
320
+
321
+ assert judge is not None
322
+ assert preflight_credentials(main, judge, source_env={}) == []
323
+ child_env = build_judge_environment(judge, source_env={})
324
+ assert "BENCH_JUDGE_API_KEY" not in child_env
325
+
326
+
327
+ def test_judge_legacy_environment_is_independent_from_main_environment():
328
+ main = resolve_execution(
329
+ cli_agent="claude-code",
330
+ source_env={
331
+ "BENCH_CC_MODEL": "subject-model",
332
+ "ANTHROPIC_BASE_URL": "https://subject.example",
333
+ },
334
+ )
335
+ judge = resolve_judge(
336
+ main=main,
337
+ source_env={
338
+ "BENCH_LLM_JUDGE": "1",
339
+ "BENCH_JUDGE_MODEL": "judge-model",
340
+ "BENCH_JUDGE_BASE_URL": "https://judge.example",
341
+ "BENCH_JUDGE_AUTH_TOKEN": "judge-token",
342
+ "ANTHROPIC_BASE_URL": "https://subject.example",
343
+ },
344
+ )
345
+
346
+ assert judge is not None
347
+ assert (judge.model, judge.base_url) == ("judge-model", "https://judge.example")
348
+
349
+
350
+ def test_preflight_reports_missing_main_and_judge_credentials_without_values():
351
+ main = resolve_execution(cli_agent="codex", cli_model="subject-model")
352
+ judge = resolve_judge(
353
+ main=main,
354
+ cli_agent="claude-code",
355
+ cli_model="judge-model",
356
+ source_env={},
357
+ )
358
+
359
+ errors = preflight_credentials(main, judge, source_env={})
360
+
361
+ assert errors == [
362
+ "main credentials missing for codex: OPENAI_API_KEY or CODEX_API_KEY",
363
+ "judge credentials missing: BENCH_JUDGE_API_KEY or BENCH_JUDGE_AUTH_TOKEN",
364
+ ]
@@ -0,0 +1,56 @@
1
+ """Tests for deterministic expectations in inline manifest tasks."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ from scaffold.python import inline_tasks
7
+
8
+
9
+ def test_inline_expectations_check_files_contains_json_and_commands(monkeypatch, tmp_path: Path):
10
+ (tmp_path / "result.md").write_text("# Result\ncomplete\n", encoding="utf-8")
11
+ (tmp_path / "summary.json").write_text(json.dumps({"status": "complete"}), encoding="utf-8")
12
+ commands = []
13
+
14
+ def fake_run_command(test_dir, command, timeout):
15
+ commands.append((test_dir, command, timeout))
16
+ return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()
17
+
18
+ monkeypatch.setattr(inline_tasks, "run_command_in_docker", fake_run_command)
19
+ validator = inline_tasks.make_inline_expectations_validator(
20
+ {
21
+ "files": ["result.md"],
22
+ "contains": {"result.md": ["# Result", "complete"]},
23
+ "json": [{"file": "summary.json", "path": "$.status", "equals": "complete"}],
24
+ "commands": [{"run": "pnpm test", "timeout": 45}],
25
+ }
26
+ )
27
+
28
+ passed, failed = validator(tmp_path, {})
29
+
30
+ assert failed == []
31
+ assert any("File exists: result.md" in check for check in passed)
32
+ assert any("Contains '# Result'" in check for check in passed)
33
+ assert any("JSON $.status equals complete" in check for check in passed)
34
+ assert any("Command succeeded: pnpm test" in check for check in passed)
35
+ assert commands == [(tmp_path, "pnpm test", 45)]
36
+
37
+
38
+ def test_inline_expectations_report_command_failure_without_running_host_shell(
39
+ monkeypatch, tmp_path: Path
40
+ ):
41
+ calls = []
42
+
43
+ def fake_run_command(test_dir, command, timeout):
44
+ calls.append((test_dir, command, timeout))
45
+ return type("Result", (), {"returncode": 1, "stdout": "out", "stderr": "err"})()
46
+
47
+ monkeypatch.setattr(inline_tasks, "run_command_in_docker", fake_run_command)
48
+ validator = inline_tasks.make_inline_expectations_validator(
49
+ {"commands": [{"run": "pnpm test", "timeout": 10}]}
50
+ )
51
+
52
+ passed, failed = validator(tmp_path, {})
53
+
54
+ assert passed == []
55
+ assert failed == ["Command failed: pnpm test (out\nerr)"]
56
+ assert calls == [(tmp_path, "pnpm test", 10)]