@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -33,6 +33,12 @@ from scaffold.python.native_eval import (
33
33
  split_comet_completion_checks as _split_comet_completion_checks,
34
34
  )
35
35
  from scaffold.python.profiles import resolve_profile_name, run_profile_rubric
36
+ from scaffold.python.agents import get_agent_adapter
37
+ from scaffold.python.execution import redact_sensitive
38
+ from scaffold.python.manifest_tasks import load_manifest_tasks
39
+ from scaffold.python.eval_context import resolve_eval_context
40
+ from scaffold.python.task_resolution import build_task_catalogue, resolve_task_set
41
+ from scaffold.python.paths import get_tasks_dir
36
42
  from scaffold.python.tasks import list_tasks, load_task
37
43
  from scaffold.python.treatments import TreatmentConfig, build_treatment_skills, load_treatments
38
44
  from scaffold.python.validation import run_validators
@@ -43,6 +49,67 @@ PYTEST_TIMEOUT = 3000 # 50 minutes total including task-specific runtime and te
43
49
  MANIFEST_DYNAMIC_ONLY_TASKS = {"workflow-overlay-contract"}
44
50
 
45
51
 
52
+ def _manifest_authored_tasks(config):
53
+ if config is None or not config.getoption("--eval-manifest"):
54
+ return {}
55
+ from scaffold.python.manifests import load_eval_manifest
56
+
57
+ manifest = load_eval_manifest(config.getoption("--eval-manifest"))
58
+ return {task.name: task for task in load_manifest_tasks(manifest)}
59
+
60
+
61
+ def _load_eval_task(task_name: str, config=None):
62
+ resolved = getattr(config, "_comet_resolved_tasks", {}) if config is not None else {}
63
+ if task_name in resolved:
64
+ return resolved[task_name]
65
+ authored = _manifest_authored_tasks(config)
66
+ if task_name in authored:
67
+ return authored[task_name]
68
+ return load_task(task_name)
69
+
70
+
71
+ def _resolve_frozen_task_set(config, task_filter: str | None):
72
+ """Resolve once per collection and retain the exact cross-suite task identity."""
73
+ frozen = getattr(config, "_comet_frozen_task_set", None)
74
+ if frozen is not None:
75
+ return frozen
76
+ from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
77
+
78
+ manifest_path = config.getoption("--eval-manifest")
79
+ context = getattr(config, "_comet_eval_context", None)
80
+ if manifest_path:
81
+ manifest = load_eval_manifest(manifest_path)
82
+ context = context or resolve_eval_context(
83
+ manifest_path=manifest.path, project_root=config.getoption("--project-root")
84
+ )
85
+ else:
86
+ if context is None:
87
+ context = resolve_eval_context(
88
+ skill_path=config.getoption("--skill-path"),
89
+ project_root=config.getoption("--project-root"),
90
+ )
91
+ manifest = SkillEvalManifest(
92
+ path=context.skill_root / "comet" / "eval.yaml",
93
+ name=context.skill_root.name,
94
+ description="",
95
+ skill_name=config.getoption("--skill-name") or context.skill_root.name,
96
+ skill_path=context.skill_root,
97
+ profile=config.getoption("--profile"),
98
+ )
99
+ frozen = resolve_task_set(
100
+ context,
101
+ manifest,
102
+ build_task_catalogue(manifest, get_tasks_dir()),
103
+ explicit_task=task_filter,
104
+ quick=bool(config.getoption("--quick")),
105
+ execution=conftest._resolve_eval_execution(config),
106
+ )
107
+ config._comet_frozen_task_set = frozen
108
+ config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
109
+ config._comet_resolution_manifest = manifest
110
+ return frozen
111
+
112
+
46
113
  # =============================================================================
47
114
  # PARAMETRIZE HELPERS
48
115
  # =============================================================================
@@ -73,6 +140,8 @@ def expand_treatment_patterns(patterns: list[str], all_treatments: dict) -> list
73
140
  def generate_test_params(task_filter: str | None, treatment_filter: str | None, config=None):
74
141
  """Generate (task_name, treatment_name) pairs based on filters."""
75
142
  params = []
143
+ if config is not None:
144
+ conftest._ensure_auto_generated_manifest(config, task_filter)
76
145
  all_treatments = load_treatments()
77
146
  all_tasks = list_tasks()
78
147
  dynamic = None
@@ -82,16 +151,24 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
82
151
  if dynamic:
83
152
  all_treatments[dynamic.name] = dynamic
84
153
  manifest_tasks = None
154
+ authored_tasks = {}
85
155
  manifest_baseline_treatments = []
86
- if config is not None and config.getoption("--eval-manifest"):
87
- from scaffold.python.manifests import load_eval_manifest
88
-
89
- manifest = load_eval_manifest(config.getoption("--eval-manifest"))
90
- manifest_tasks = manifest.recommended_tasks
156
+ frozen_selected = False
157
+ if config is not None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
158
+ resolved = _resolve_frozen_task_set(config, task_filter)
159
+ frozen_selected = True
160
+ manifest = config._comet_resolution_manifest
161
+ manifest_tasks = [item.name for item in resolved.tasks]
162
+ authored_tasks = {
163
+ item.name: item.task
164
+ for item in resolved.tasks
165
+ if item.provenance in {"inline", "source"}
166
+ }
91
167
  manifest_baseline_treatments = manifest.baseline_treatments
92
168
 
93
- if task_filter and task_filter not in all_tasks:
94
- raise ValueError(f"Task not found: {task_filter}. Available: {all_tasks}")
169
+ if task_filter and task_filter not in set(all_tasks) | set(authored_tasks):
170
+ available = sorted(set(all_tasks) | set(authored_tasks))
171
+ raise ValueError(f"Task not found: {task_filter}. Available: {available}")
95
172
 
96
173
  treatment_list = []
97
174
  if treatment_filter:
@@ -106,10 +183,18 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
106
183
  elif dynamic:
107
184
  treatment_list = [dynamic.name]
108
185
 
109
- tasks_to_run = [task_filter] if task_filter else (manifest_tasks or all_tasks)
186
+ if task_filter:
187
+ tasks_to_run = [task_filter]
188
+ elif frozen_selected:
189
+ tasks_to_run = list(manifest_tasks or [])
190
+ elif authored_tasks or manifest_tasks:
191
+ tasks_to_run = list(manifest_tasks or authored_tasks)
192
+ else:
193
+ tasks_to_run = all_tasks
194
+ tasks_to_run = list(dict.fromkeys(tasks_to_run))
110
195
 
111
196
  for task_name in tasks_to_run:
112
- task = load_task(task_name)
197
+ task = _load_eval_task(task_name, config)
113
198
  task_treatments = treatment_list
114
199
  if (
115
200
  dynamic
@@ -193,6 +278,46 @@ interaction:
193
278
  ]
194
279
 
195
280
 
281
+ def test_eval_manifest_prefers_authored_tasks_over_recommended_tasks(tmp_path):
282
+ package = tmp_path / "manifest-skill"
283
+ package.mkdir()
284
+ (package / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
285
+ comet_dir = package / "comet"
286
+ comet_dir.mkdir()
287
+ manifest = comet_dir / "eval.yaml"
288
+ manifest.write_text(
289
+ "\n".join(
290
+ [
291
+ "apiVersion: comet.eval/v1alpha1",
292
+ "kind: SkillEvalManifest",
293
+ "metadata:",
294
+ " name: manifest-skill",
295
+ "skill:",
296
+ " name: manifest-skill",
297
+ " source: ..",
298
+ "evaluation:",
299
+ " tasks:",
300
+ " - name: inline-task",
301
+ " prompt: Create result.md.",
302
+ " expect:",
303
+ " files: [result.md]",
304
+ " recommendedTasks:",
305
+ " - generic-skill-smoke",
306
+ ]
307
+ ),
308
+ encoding="utf-8",
309
+ )
310
+
311
+ class Config:
312
+ def getoption(self, name):
313
+ return {"--eval-manifest": str(manifest)}.get(name)
314
+
315
+ params = generate_test_params(None, None, Config())
316
+
317
+ assert [task_name for task_name, _ in params] == ["inline-task"]
318
+ assert all(treatment == "DYNAMIC_SKILL" for _, treatment in params)
319
+
320
+
196
321
  def test_control_comet_workflow_filters_workflow_only_checks():
197
322
  passed, failed = _filter_control_workflow_checks(
198
323
  "comet-workflow",
@@ -310,7 +435,7 @@ def pytest_generate_tests(metafunc):
310
435
  def test_task_treatment(task_name, treatment_name):
311
436
  """Run a task with a treatment and validate results."""
312
437
  fixtures = get_fixtures()
313
- task = load_task(task_name)
438
+ task = _load_eval_task(task_name, fixtures.request_config)
314
439
  treatments = load_treatments()
315
440
  dynamic = conftest._get_dynamic_treatment_config(fixtures.request_config)
316
441
  if dynamic:
@@ -361,8 +486,29 @@ def test_task_treatment(task_name, treatment_name):
361
486
  skills=treatment.skills,
362
487
  claude_md=conftest._build_eval_claude_md(profile_name, treatment.claude_md),
363
488
  environment_dir=task.environment_dir,
489
+ workspace_dir=task.workspace_dir,
364
490
  )
365
- selected_model = conftest._selected_claude_model()
491
+ selected_agent = conftest._resolve_eval_agent(fixtures.request_config).agent
492
+ selected_adapter = get_agent_adapter(selected_agent)
493
+ execution = conftest._resolve_eval_execution(fixtures.request_config)
494
+ selected_model = execution.model
495
+ judge = conftest._resolve_eval_judge(fixtures.request_config)
496
+ role_models = {
497
+ "subject": selected_model,
498
+ "simulator": (
499
+ selected_model
500
+ if interaction.mode == "auto_user"
501
+ and not interaction.decision_reply
502
+ and not interaction.decision_replies
503
+ else None
504
+ ),
505
+ "judge": judge.model if judge is not None else None,
506
+ }
507
+ role_agents = {
508
+ "subject": selected_agent,
509
+ "simulator": selected_agent if interaction.mode == "auto_user" else None,
510
+ "judge": judge.agent if judge is not None else None,
511
+ }
366
512
  captured_execution = conftest._capture_execution_identity(
367
513
  fixtures.test_dir,
368
514
  model=selected_model,
@@ -373,6 +519,11 @@ def test_task_treatment(task_name, treatment_name):
373
519
  task.name,
374
520
  task.path.parent,
375
521
  execution_identity=captured_execution.report_identity,
522
+ manifest_path=task.manifest_path,
523
+ rendered_prompt=prompt if task.manifest_path else None,
524
+ workspace_dir=task.workspace_dir if task.manifest_path else None,
525
+ validation_rules=task.manifest_expectations if task.manifest_path else None,
526
+ environment_dir=task.environment_dir if task.manifest_path else None,
376
527
  )
377
528
  )
378
529
  skill_package_path = conftest._snapshot_dynamic_skill_package(
@@ -387,16 +538,31 @@ def test_task_treatment(task_name, treatment_name):
387
538
  image_id=captured_execution.runtime_image_id,
388
539
  )
389
540
 
390
- events = extract_events(parse_output(result.stdout))
541
+ events = extract_events(parse_output(result.stdout), agent=selected_agent)
542
+ events["telemetry_status"] = "N/A" if not selected_adapter.supports_telemetry else "available"
391
543
  loop_interaction = conftest._extract_loop_interaction(result.stderr)
392
- subject_turns = conftest._extract_subject_turn_evidence(result.stdout)
544
+ for role, session_ids in loop_interaction.get("role_sessions", {}).items():
545
+ target = events["role_sessions"].setdefault(role, [])
546
+ for session_id in session_ids:
547
+ if session_id not in target:
548
+ target.append(session_id)
549
+ subject_turns = conftest._extract_subject_turn_evidence(redact_sensitive(result.stdout))
393
550
  outputs = {
394
551
  "run_id": run_id,
395
552
  "treatment_name": treatment_name,
553
+ "agent": selected_agent,
554
+ "agent_adapter": selected_adapter,
555
+ "main_credentials": selected_adapter.required_credentials,
556
+ "role_models": role_models,
557
+ "role_agents": role_agents,
558
+ "telemetry_status": events["telemetry_status"],
559
+ "judge_agent": judge.agent if judge is not None else None,
560
+ "judge_model": judge.model if judge is not None else None,
561
+ "judge_base_url": judge.base_url if judge is not None else None,
396
562
  "events": events,
397
563
  "profile": profile_name,
398
564
  "skill_sources": skill_sources,
399
- "eval_manifest": eval_manifest,
565
+ "eval_manifest": eval_manifest or skill_hints.get("manifest"),
400
566
  "required_skills": skill_hints.get("required_skills")
401
567
  or task.config.evaluation.required_skills,
402
568
  "expected_artifacts": skill_hints.get("expected_artifacts")
@@ -416,6 +582,19 @@ def test_task_treatment(task_name, treatment_name):
416
582
  "required_output_schemas": skill_hints.get("required_output_schemas") or [],
417
583
  "expected_evidence": skill_hints.get("expected_evidence") or [],
418
584
  "draft_hash": skill_hints.get("draft_hash"),
585
+ "eval_generation": (
586
+ {
587
+ "generation_hash": skill_hints.get("generation_hash"),
588
+ "manifest_path": skill_hints.get("manifest")
589
+ or str(fixtures.request_config.getoption("--eval-manifest")),
590
+ "metadata_path": skill_hints.get("generation_metadata_path"),
591
+ "manifest_hash": skill_hints.get("generation_manifest_hash"),
592
+ "metadata_hash": skill_hints.get("generation_metadata_hash"),
593
+ "overhead": skill_hints.get("generation_overhead"),
594
+ }
595
+ if skill_hints.get("generation_hash")
596
+ else None
597
+ ),
419
598
  "interaction": {
420
599
  "mode": interaction.mode,
421
600
  "max_turns": interaction.max_turns,
@@ -423,12 +602,18 @@ def test_task_treatment(task_name, treatment_name):
423
602
  "subject_turns": subject_turns,
424
603
  },
425
604
  "case_manifest": case_manifest,
605
+ "role_sessions": events["role_sessions"],
426
606
  }
427
607
  events["profile"] = outputs["profile"]
608
+ events["agent"] = selected_agent
609
+ events["role_models"] = role_models
610
+ events["role_agents"] = role_agents
611
+ events["telemetry_status"] = outputs["telemetry_status"]
428
612
  events["skill_sources"] = outputs["skill_sources"]
429
613
  events["eval_manifest"] = outputs["eval_manifest"]
430
614
  events["interaction"] = outputs["interaction"]
431
615
  events["case_manifest"] = case_manifest
616
+ events["eval_generation"] = outputs["eval_generation"]
432
617
 
433
618
  passed, failed = run_validators(validators, fixtures.test_dir, outputs)
434
619
  passed, failed = adapt_checks_for_native(
@@ -459,6 +644,21 @@ def test_task_treatment(task_name, treatment_name):
459
644
  )
460
645
  passed = passed + rubric_passed
461
646
  failed = failed + rubric_failed
647
+ agent_failed = result.returncode != 0
648
+ if agent_failed:
649
+ failed.append(f"Agent execution failed with exit code {result.returncode}")
650
+
651
+ # Keep the run-level context available to thin remote reporting wrappers.
652
+ # These fields are local evidence as well; Langfuse applies its own bounded
653
+ # capture policy before sending them remotely.
654
+ events["task"] = task.name
655
+ events["treatment"] = treatment_name
656
+ events["prompt"] = prompt
657
+ events["skill"] = skill_hints.get("name") or skill_hints.get("path")
658
+ events["final_response"] = subject_turns[-1].get("result") if subject_turns else None
659
+ events["quality_gates"] = outputs["quality_gates"]
660
+ events["execution_identity"] = captured_execution.report_identity
661
+ events["role_sessions"] = outputs["role_sessions"]
462
662
 
463
663
  fixtures.record_result(
464
664
  events,
@@ -470,5 +670,7 @@ def test_task_treatment(task_name, treatment_name):
470
670
  stderr=result.stderr,
471
671
  )
472
672
 
673
+ if agent_failed:
674
+ pytest.fail(f"{selected_agent} execution failed with exit code {result.returncode}")
473
675
  if failed and not _is_observational_baseline_run(treatment_name):
474
676
  pytest.fail(f"Validation failed: {failed}")
@@ -4,6 +4,7 @@ version = "0.1.0"
4
4
  description = "Benchmark evaluation for Comet skill (OpenSpec + Superpowers workflow)"
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
7
+ "jsonschema>=4.23",
7
8
  "pytest>=8.0",
8
9
  "pytest-timeout>=2.2",
9
10
  "pytest-xdist>=3.5",
@@ -23,6 +24,9 @@ langsmith = [
23
24
  "langchain>=0.3",
24
25
  "langchain-openai>=0.2",
25
26
  ]
27
+ langfuse = [
28
+ "langfuse>=4.0,<5",
29
+ ]
26
30
  dev = [
27
31
  "ruff>=0.4",
28
32
  "pre-commit>=3.7",
@@ -34,6 +38,7 @@ testpaths = ["local/tests"]
34
38
  pythonpath = ["."]
35
39
  markers = [
36
40
  "langsmith: marks test as using LangSmith tracing",
41
+ "langfuse: marks test as using Langfuse tracing",
37
42
  ]
38
43
 
39
44
  [tool.ruff]
@@ -18,6 +18,9 @@ from .python import (
18
18
  compose_validators,
19
19
  make_execution_validator,
20
20
  run_claude_in_docker,
21
+ run_agent_in_docker,
22
+ run_agent_loop_in_docker,
23
+ run_command_in_docker,
21
24
  run_eval_in_docker,
22
25
  run_node_in_docker,
23
26
  run_python_in_docker,
@@ -64,4 +67,7 @@ __all__ = [
64
67
  "run_eval_in_docker",
65
68
  "run_node_in_docker",
66
69
  "run_claude_in_docker",
70
+ "run_agent_in_docker",
71
+ "run_agent_loop_in_docker",
72
+ "run_command_in_docker",
67
73
  ]
@@ -0,0 +1,41 @@
1
+ ARG BASE_IMAGE
2
+ FROM ${BASE_IMAGE}
3
+
4
+ USER root
5
+ ARG EVAL_AGENT=claude-code
6
+ ARG CODEX_VERSION=latest
7
+ ARG QODER_VERSION=latest
8
+ ARG CODEBUDDY_VERSION=latest
9
+ ARG CUSTOM_AGENT_ID=
10
+ ARG CUSTOM_AGENT_EXECUTABLE=
11
+ ARG CUSTOM_INSTALL_KIND=none
12
+ ARG CUSTOM_INSTALL_PACKAGE=
13
+ ARG CUSTOM_INSTALL_VERSION=latest
14
+ ARG LANGFUSE_ENABLED=false
15
+
16
+ RUN if [ "$EVAL_AGENT" = "codex" ]; then \
17
+ npm install -g @openai/codex@${CODEX_VERSION}; \
18
+ elif [ "$EVAL_AGENT" = "qoder" ]; then \
19
+ npm install -g @qoder-ai/qodercli@${QODER_VERSION}; \
20
+ elif [ "$EVAL_AGENT" = "codebuddy" ]; then \
21
+ npm install -g @tencent-ai/codebuddy-code@${CODEBUDDY_VERSION}; \
22
+ elif [ -n "$CUSTOM_AGENT_ID" ] && [ "$EVAL_AGENT" = "$CUSTOM_AGENT_ID" ] && [ "$CUSTOM_INSTALL_KIND" = "npm" ]; then \
23
+ npm install -g "${CUSTOM_INSTALL_PACKAGE}@${CUSTOM_INSTALL_VERSION}"; \
24
+ elif [ -n "$CUSTOM_AGENT_ID" ] && [ "$EVAL_AGENT" = "$CUSTOM_AGENT_ID" ] && [ "$CUSTOM_INSTALL_KIND" = "pip" ]; then \
25
+ if [ "$CUSTOM_INSTALL_VERSION" = "latest" ]; then \
26
+ python3 -m pip install --break-system-packages "${CUSTOM_INSTALL_PACKAGE}"; \
27
+ else \
28
+ python3 -m pip install --break-system-packages "${CUSTOM_INSTALL_PACKAGE}==${CUSTOM_INSTALL_VERSION}"; \
29
+ fi; \
30
+ fi
31
+
32
+ # The official Claude hook can use the preinstalled Python SDK, while the
33
+ # official Codex plugin ships its Node bundle. Install the SDK only for an
34
+ # explicit Langfuse suite so Local/LangSmith images stay unchanged.
35
+ RUN if [ "$LANGFUSE_ENABLED" = "true" ]; then \
36
+ apt-get update && apt-get install -y --no-install-recommends python3 python3-pip && \
37
+ python3 -m pip install --break-system-packages --no-cache-dir 'langfuse>=4.0,<5' && \
38
+ rm -rf /var/lib/apt/lists/*; \
39
+ fi
40
+
41
+ USER agent
@@ -14,7 +14,18 @@ from .logging import (
14
14
  save_report,
15
15
  strip_ansi,
16
16
  )
17
- from .manifests import SkillEvalManifest, load_eval_manifest
17
+ from .manifests import ManifestTask, SkillEvalManifest, load_eval_manifest
18
+ from .eval_context import (
19
+ EvalContextError,
20
+ ResolvedEvalContext,
21
+ artifact_root_for_owner,
22
+ assert_artifact_root_is_safe,
23
+ context_from_environment,
24
+ managed_path_for_owner,
25
+ resolve_managed_path,
26
+ resolve_eval_context,
27
+ )
28
+ from .manifest_tasks import load_manifest_tasks
18
29
  from .profiles import (
19
30
  AUTHORING_SKILL_PROFILE,
20
31
  COMET_WORKFLOW_PROFILE,
@@ -28,6 +39,26 @@ from .profiles import (
28
39
  )
29
40
  from .evidence import EvalArtifactReference, build_eval_artifact_references
30
41
  from .attribution import FailureAttribution, classify_failures
42
+ from .agents import (
43
+ AGENT_IDS,
44
+ AgentAdapter,
45
+ AgentSelection,
46
+ get_agent_adapter,
47
+ resolve_agent,
48
+ validate_agent_capabilities,
49
+ )
50
+ from .custom_agents import CustomAgentSpec, discover_custom_agent_specs
51
+ from .execution import (
52
+ ResolvedExecution,
53
+ ResolvedJudge,
54
+ api_identity,
55
+ build_agent_environment,
56
+ build_judge_environment,
57
+ preflight_credentials,
58
+ redact_sensitive,
59
+ resolve_execution,
60
+ resolve_judge,
61
+ )
31
62
  from .sample_quality import (
32
63
  SampleQuality,
33
64
  include_in_analysis,
@@ -54,6 +85,9 @@ from .utils import (
54
85
  read_json_file,
55
86
  retry_with_backoff,
56
87
  run_claude_in_docker,
88
+ run_agent_in_docker,
89
+ run_agent_loop_in_docker,
90
+ run_command_in_docker,
57
91
  run_eval_in_docker,
58
92
  run_node_in_docker,
59
93
  run_python_in_docker,
@@ -73,23 +107,97 @@ from .validation import (
73
107
  )
74
108
 
75
109
  __all__ = [
76
- "NoiseTask", "Treatment", "ValidatorFn", "compose_validators", "run_validators",
77
- "check_file_exists", "check_pattern", "check_no_pattern", "check_skill_invoked",
78
- "check_code_execution", "check_python_execution", "check_typescript_execution",
79
- "run_shell", "check_docker_available", "check_claude_available", "build_docker_image",
80
- "run_python_in_docker", "make_execution_validator", "run_eval_in_docker",
81
- "run_node_in_docker", "run_claude_in_docker", "retry_with_backoff", "read_json_file",
82
- "get_field", "get_nested_field", "normalize_score", "parse_output", "extract_events",
83
- "strip_ansi", "ExperimentLogger", "TreatmentResult", "save_events", "save_raw",
84
- "save_report", "ReportColumn", "bool_column", "quality_column", "default_columns",
85
- "EvalArtifactReference", "build_eval_artifact_references",
86
- "FailureAttribution", "classify_failures",
87
- "SkillEvalManifest", "load_eval_manifest",
88
- "AUTHORING_SKILL_PROFILE", "COMET_WORKFLOW_PROFILE", "GENERIC_PROFILE",
89
- "ProfileSpec", "all_rubric_dimensions", "get_profile", "list_profiles",
90
- "resolve_profile_name", "run_profile_rubric",
91
- "ReportOutputConfig", "load_report_output_config", "preferred_report_path",
92
- "render_markdown_html", "write_report_outputs",
93
- "SampleQuality", "infer_sample_quality", "quality_from_report",
94
- "sample_quality_dict", "include_in_analysis",
110
+ "NoiseTask",
111
+ "Treatment",
112
+ "ValidatorFn",
113
+ "compose_validators",
114
+ "run_validators",
115
+ "check_file_exists",
116
+ "check_pattern",
117
+ "check_no_pattern",
118
+ "check_skill_invoked",
119
+ "check_code_execution",
120
+ "check_python_execution",
121
+ "check_typescript_execution",
122
+ "run_shell",
123
+ "check_docker_available",
124
+ "check_claude_available",
125
+ "build_docker_image",
126
+ "run_python_in_docker",
127
+ "make_execution_validator",
128
+ "run_eval_in_docker",
129
+ "run_node_in_docker",
130
+ "run_claude_in_docker",
131
+ "retry_with_backoff",
132
+ "read_json_file",
133
+ "run_agent_in_docker",
134
+ "run_agent_loop_in_docker",
135
+ "run_command_in_docker",
136
+ "get_field",
137
+ "get_nested_field",
138
+ "normalize_score",
139
+ "parse_output",
140
+ "extract_events",
141
+ "strip_ansi",
142
+ "ExperimentLogger",
143
+ "TreatmentResult",
144
+ "save_events",
145
+ "save_raw",
146
+ "save_report",
147
+ "ReportColumn",
148
+ "bool_column",
149
+ "quality_column",
150
+ "default_columns",
151
+ "EvalArtifactReference",
152
+ "build_eval_artifact_references",
153
+ "FailureAttribution",
154
+ "classify_failures",
155
+ "AGENT_IDS",
156
+ "AgentAdapter",
157
+ "AgentSelection",
158
+ "get_agent_adapter",
159
+ "resolve_agent",
160
+ "validate_agent_capabilities",
161
+ "CustomAgentSpec",
162
+ "discover_custom_agent_specs",
163
+ "ResolvedExecution",
164
+ "ResolvedJudge",
165
+ "resolve_execution",
166
+ "resolve_judge",
167
+ "build_agent_environment",
168
+ "build_judge_environment",
169
+ "preflight_credentials",
170
+ "redact_sensitive",
171
+ "api_identity",
172
+ "ManifestTask",
173
+ "SkillEvalManifest",
174
+ "load_eval_manifest",
175
+ "EvalContextError",
176
+ "ResolvedEvalContext",
177
+ "artifact_root_for_owner",
178
+ "assert_artifact_root_is_safe",
179
+ "context_from_environment",
180
+ "managed_path_for_owner",
181
+ "resolve_managed_path",
182
+ "resolve_eval_context",
183
+ "load_manifest_tasks",
184
+ "AUTHORING_SKILL_PROFILE",
185
+ "COMET_WORKFLOW_PROFILE",
186
+ "GENERIC_PROFILE",
187
+ "ProfileSpec",
188
+ "all_rubric_dimensions",
189
+ "get_profile",
190
+ "list_profiles",
191
+ "resolve_profile_name",
192
+ "run_profile_rubric",
193
+ "ReportOutputConfig",
194
+ "load_report_output_config",
195
+ "preferred_report_path",
196
+ "render_markdown_html",
197
+ "write_report_outputs",
198
+ "SampleQuality",
199
+ "infer_sample_quality",
200
+ "quality_from_report",
201
+ "sample_quality_dict",
202
+ "include_in_analysis",
95
203
  ]