@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -0,0 +1,239 @@
1
+ """Selectable agent CLI contracts used by the evaluation harness."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Literal
7
+
8
+ from scaffold.python.custom_agents import load_custom_agent_spec
9
+
10
+
11
+ AgentId = str
12
+ AgentRole = Literal["subject", "simulator", "judge"]
13
+
14
+ AGENT_IDS: tuple[AgentId, ...] = ("claude-code", "codex", "qoder", "codebuddy")
15
+ DEFAULT_AGENT: AgentId = "claude-code"
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class AgentSelection:
20
+ """The resolved agent and the configuration layer that selected it."""
21
+
22
+ agent: AgentId
23
+ source: Literal["cli", "manifest", "default"]
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class AgentAdapter:
28
+ """Build role-neutral commands for one supported evaluation agent."""
29
+
30
+ id: AgentId
31
+ executable: str
32
+ required_credentials: tuple[str, ...]
33
+ model_env: str = ""
34
+ base_url_env: str = ""
35
+ supports_resume: bool = True
36
+ supports_single_turn: bool = True
37
+ supports_structured_events: bool = True
38
+ supports_telemetry: bool = True
39
+ supports_skill_invocation_evidence: bool = True
40
+ custom: bool = False
41
+ install_kind: str = "none"
42
+ install_package: str | None = None
43
+ install_version: str | None = None
44
+
45
+ def build_version_command(self) -> list[str]:
46
+ return [self.executable, "--version"]
47
+
48
+ def build_run_command(
49
+ self,
50
+ prompt: str,
51
+ *,
52
+ model: str | None,
53
+ role: AgentRole,
54
+ resume_id: str | None = None,
55
+ max_turns: int | None = None,
56
+ ) -> list[str]:
57
+ del role
58
+ command = [self.executable]
59
+ if self.id == "claude-code":
60
+ command.extend(["-p", prompt, "--dangerously-skip-permissions"])
61
+ command.extend(["--output-format", "stream-json", "--verbose"])
62
+ if resume_id:
63
+ command.extend(["--resume", resume_id])
64
+ if max_turns is not None:
65
+ command.extend(["--max-turns", str(max_turns)])
66
+ elif self.id == "codex":
67
+ command.extend(["exec"])
68
+ if resume_id:
69
+ command.extend(["resume", resume_id])
70
+ command.extend(["--json", "--yolo"])
71
+ if model:
72
+ command.extend(["--model", model])
73
+ command.append(prompt)
74
+ elif self.id == "qoder":
75
+ command.extend(["-p", prompt, "--output-format", "stream-json", "--yolo"])
76
+ if resume_id:
77
+ command.extend(["-r", resume_id])
78
+ if model:
79
+ command.extend(["--model", model])
80
+ if max_turns is not None:
81
+ command.extend(["--max-turns", str(max_turns)])
82
+ elif self.custom:
83
+ command.extend(["-p", prompt, "--output-format", "stream-json"])
84
+ if model:
85
+ command.extend(["--model", model])
86
+ if resume_id:
87
+ command.extend(["--resume", resume_id])
88
+ else:
89
+ command.extend(
90
+ [
91
+ "-p",
92
+ prompt,
93
+ "--output-format",
94
+ "stream-json",
95
+ "--dangerously-skip-permissions",
96
+ ]
97
+ )
98
+ if resume_id:
99
+ command.extend(["-r", resume_id])
100
+ if model:
101
+ command.extend(["--model", model])
102
+ if self.id == "claude-code" and model:
103
+ command.extend(["--model", model])
104
+ return command
105
+
106
+ def has_observable_skill_invocation(self, events: object) -> bool:
107
+ """Only explicit invocation events count; artifacts are not evidence."""
108
+ if not self.supports_skill_invocation_evidence or not isinstance(events, dict):
109
+ return False
110
+ invocations = events.get("skill_invocations")
111
+ return isinstance(invocations, list) and bool(invocations)
112
+
113
+
114
+ def validate_agent_id(value: object, *, field: str = "evaluation agent") -> AgentId:
115
+ if value in AGENT_IDS:
116
+ return value
117
+ if isinstance(value, str) and load_custom_agent_spec(value) is not None:
118
+ return value
119
+ if value not in AGENT_IDS:
120
+ supported = ", ".join(AGENT_IDS)
121
+ raise ValueError(
122
+ f"Unsupported evaluation agent {value!r} in {field}. Expected one of: {supported} or an installed custom adapter"
123
+ )
124
+ return value # pragma: no cover
125
+
126
+
127
+ def resolve_agent(
128
+ cli_agent: object | None,
129
+ manifest_agent: object | None,
130
+ ) -> AgentSelection:
131
+ if cli_agent is not None:
132
+ return AgentSelection(validate_agent_id(cli_agent, field="CLI evaluation agent"), "cli")
133
+ if manifest_agent is not None:
134
+ return AgentSelection(
135
+ validate_agent_id(manifest_agent, field="manifest execution.agent"),
136
+ "manifest",
137
+ )
138
+ return AgentSelection(DEFAULT_AGENT, "default")
139
+
140
+
141
+ def get_agent_adapter(agent_id: object) -> AgentAdapter:
142
+ validated = validate_agent_id(agent_id)
143
+ custom = load_custom_agent_spec(validated) if validated not in AGENT_IDS else None
144
+ if custom is not None:
145
+ capabilities = custom.capabilities
146
+ return AgentAdapter(
147
+ id=validated,
148
+ executable=custom.executable,
149
+ required_credentials=custom.credentials,
150
+ model_env=custom.model_env or "",
151
+ base_url_env=custom.base_url_env or "",
152
+ supports_resume=capabilities["resume"],
153
+ supports_single_turn=capabilities["single_turn"],
154
+ supports_structured_events=capabilities["structured_events"],
155
+ supports_telemetry=capabilities["telemetry"],
156
+ supports_skill_invocation_evidence=capabilities["skill_invocation_evidence"],
157
+ custom=True,
158
+ install_kind=custom.install_kind,
159
+ install_package=custom.install_package,
160
+ install_version=custom.install_version,
161
+ )
162
+ executable = {
163
+ "claude-code": "claude",
164
+ "codex": "codex",
165
+ "qoder": "qodercli",
166
+ "codebuddy": "codebuddy",
167
+ }[validated]
168
+ required_credentials = {
169
+ "claude-code": ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"),
170
+ "codex": ("OPENAI_API_KEY", "CODEX_API_KEY"),
171
+ "qoder": ("QODER_PERSONAL_ACCESS_TOKEN",),
172
+ "codebuddy": ("CODEBUDDY_API_KEY", "CODEBUDDY_AUTH_TOKEN"),
173
+ }[validated]
174
+ model_env = {
175
+ "claude-code": "ANTHROPIC_MODEL",
176
+ "codex": "OPENAI_MODEL",
177
+ "qoder": "QODER_MODEL",
178
+ "codebuddy": "CODEBUDDY_MODEL",
179
+ }[validated]
180
+ base_url_env = {
181
+ "claude-code": "ANTHROPIC_BASE_URL",
182
+ "codex": "OPENAI_BASE_URL",
183
+ "qoder": "QODER_BASE_URL",
184
+ "codebuddy": "CODEBUDDY_BASE_URL",
185
+ }[validated]
186
+ return AgentAdapter(validated, executable, required_credentials, model_env, base_url_env)
187
+
188
+
189
+ def normalize_skill_invocations(
190
+ events: object,
191
+ *,
192
+ agent: AgentId | None = None,
193
+ adapter: AgentAdapter | None = None,
194
+ ) -> list[str]:
195
+ """Return Skill names that satisfy the selected Agent's evidence contract.
196
+
197
+ Built-in Agents keep their existing path/tool inference. Custom adapters
198
+ must opt into explicit invocation evidence; otherwise their reported Skill
199
+ names are not treated as observable workflow evidence.
200
+ """
201
+ if not isinstance(events, dict):
202
+ return []
203
+ inferred = events.get("skills_invoked")
204
+ fallback = [item for item in inferred if isinstance(item, str)] if isinstance(inferred, list) else []
205
+ selected = adapter
206
+ if selected is None:
207
+ try:
208
+ selected = get_agent_adapter(agent or DEFAULT_AGENT)
209
+ except (TypeError, ValueError):
210
+ return [] if isinstance(agent, str) and agent not in AGENT_IDS else fallback
211
+ if not selected.custom:
212
+ return fallback
213
+ if not selected.has_observable_skill_invocation(events):
214
+ return []
215
+ explicit = events.get("skill_invocations")
216
+ return [item for item in explicit if isinstance(item, str)] if isinstance(explicit, list) else []
217
+
218
+
219
+ def validate_agent_capabilities(
220
+ adapter: AgentAdapter,
221
+ required: tuple[str, ...] = (
222
+ "single_turn",
223
+ "resume",
224
+ "structured_events",
225
+ "skill_invocation_evidence",
226
+ ),
227
+ ) -> None:
228
+ fields = {
229
+ "single_turn": adapter.supports_single_turn,
230
+ "resume": adapter.supports_resume,
231
+ "structured_events": adapter.supports_structured_events,
232
+ "telemetry": adapter.supports_telemetry,
233
+ "skill_invocation_evidence": adapter.supports_skill_invocation_evidence,
234
+ }
235
+ missing = [name for name in required if not fields.get(name, False)]
236
+ if missing:
237
+ raise ValueError(
238
+ f"Agent {adapter.id} is missing required capabilities: {', '.join(missing)}"
239
+ )
@@ -18,9 +18,10 @@ import statistics
18
18
  from collections import Counter, defaultdict
19
19
  from dataclasses import dataclass
20
20
  from pathlib import Path
21
- from typing import Any, Iterable, Sequence
21
+ from typing import Any, Iterable, Mapping, Sequence
22
22
 
23
23
  from scaffold.python.logging import extract_events, parse_output
24
+ from scaffold.python.agents import AGENT_IDS
24
25
  from scaffold.python.pass_at_k import compute_pass_metrics
25
26
  from scaffold.python.sample_quality import quality_from_report
26
27
 
@@ -64,6 +65,7 @@ _RUNNER_FILES = (
64
65
  "scaffold/shell/run-claude-loop.sh",
65
66
  "scaffold/shell/decision-point.sh",
66
67
  "scaffold/shell/completion-point.sh",
68
+ "scaffold/docker/agent-overlay.Dockerfile",
67
69
  )
68
70
  _CONTROLLER_FILES = (
69
71
  "local/tests/tasks/test_tasks.py",
@@ -73,6 +75,7 @@ _CONTROLLER_FILES = (
73
75
  "scaffold/python/profiles.py",
74
76
  "scaffold/python/manifests.py",
75
77
  "scaffold/python/utils.py",
78
+ "scaffold/python/agents.py",
76
79
  "scaffold/python/logging.py",
77
80
  "scaffold/python/aligned_comparison.py",
78
81
  )
@@ -94,6 +97,7 @@ class CaseManifest:
94
97
  controller_hash: str | None
95
98
  execution_hash: str | None
96
99
  execution_identity: dict[str, str] | None
100
+ authored_hashes: dict[str, str] | None
97
101
  source: str
98
102
  run_bound: bool
99
103
  execution_bound: bool
@@ -236,13 +240,21 @@ def build_execution_identity(
236
240
  ) -> dict[str, str]:
237
241
  """Build the safe, report-bound execution identity.
238
242
 
239
- Docker supplies hashes derived from the immutable image and the actual
240
- ``claude --version`` output. Model and interaction text are hashed by the
241
- controller; only bounded enums and digests enter reports.
243
+ Docker supplies identity derived from the immutable image and the actual
244
+ selected CLI version. Model and private interaction text are hashed by the
245
+ controller; only the bounded CLI version, enums, and digests enter reports.
242
246
  """
243
247
  if docker_identity.get("schema") != EXECUTION_IDENTITY_SCHEMA:
244
248
  raise ValueError("Docker execution identity schema is invalid")
245
- allowed = {"schema", "runtime_image_id", *_EXECUTION_IDENTITY_HASH_KEYS[:4]}
249
+ allowed = {
250
+ "schema",
251
+ "runtime_image_id",
252
+ *_EXECUTION_IDENTITY_HASH_KEYS[:4],
253
+ "agent",
254
+ "agent_cli",
255
+ "agent_cli_version",
256
+ "agent_cli_version_hash",
257
+ }
246
258
  unknown = set(docker_identity) - allowed
247
259
  if unknown:
248
260
  raise ValueError(f"Docker execution identity has unknown keys: {sorted(unknown)}")
@@ -256,6 +268,25 @@ def build_execution_identity(
256
268
  raise ValueError(f"Docker execution identity {key} is invalid")
257
269
  if docker_identity["image_id_hash"] != _sha256_bytes(runtime_image_id.encode("utf-8")):
258
270
  raise ValueError("Docker execution identity image hash does not match its runtime image")
271
+ agent = docker_identity.get("agent", "claude-code")
272
+ agent_cli = docker_identity.get("agent_cli", "claude")
273
+ agent_cli_version_hash = docker_identity.get(
274
+ "agent_cli_version_hash", docker_identity.get("claude_tool_version_hash")
275
+ )
276
+ agent_cli_version = docker_identity.get("agent_cli_version")
277
+ if agent not in AGENT_IDS:
278
+ raise ValueError(f"Docker execution identity agent is invalid: {agent!r}")
279
+ if not isinstance(agent_cli, str) or not agent_cli:
280
+ raise ValueError("Docker execution identity agent_cli is invalid")
281
+ if not _is_hash(agent_cli_version_hash):
282
+ raise ValueError("Docker execution identity agent_cli_version_hash is invalid")
283
+ if agent_cli_version is not None and (
284
+ not isinstance(agent_cli_version, str)
285
+ or not agent_cli_version
286
+ or len(agent_cli_version) > 512
287
+ or any(ord(character) < 32 for character in agent_cli_version)
288
+ ):
289
+ raise ValueError("Docker execution identity agent_cli_version is invalid")
259
290
 
260
291
  mode = getattr(interaction, "mode", None) or "none"
261
292
  if mode not in {"none", "auto_user"}:
@@ -279,7 +310,7 @@ def build_execution_identity(
279
310
  for key, value in sorted((model_config or {}).items())
280
311
  if isinstance(key, str) and isinstance(value, (str, type(None)))
281
312
  }
282
- return {
313
+ identity = {
283
314
  "schema": EXECUTION_IDENTITY_SCHEMA,
284
315
  **{key: docker_identity[key] for key in _EXECUTION_IDENTITY_HASH_KEYS[:4]},
285
316
  "model_selection_hash": _hash_private_value(
@@ -294,6 +325,17 @@ def build_execution_identity(
294
325
  "interaction_mode": mode,
295
326
  "interaction_max_turns": str(max_turns),
296
327
  }
328
+ if "agent" in docker_identity:
329
+ identity.update(
330
+ {
331
+ "agent": agent,
332
+ "agent_cli": agent_cli,
333
+ "agent_cli_version_hash": agent_cli_version_hash,
334
+ }
335
+ )
336
+ if agent_cli_version is not None:
337
+ identity["agent_cli_version"] = agent_cli_version
338
+ return identity
297
339
 
298
340
 
299
341
  def _validated_execution_identity(raw: Any) -> dict[str, str] | None:
@@ -306,10 +348,29 @@ def _validated_execution_identity(raw: Any) -> dict[str, str] | None:
306
348
  "interaction_mode",
307
349
  "interaction_max_turns",
308
350
  }
309
- if set(raw) != allowed or raw.get("schema") != EXECUTION_IDENTITY_SCHEMA:
351
+ new_agent_keys = {"agent", "agent_cli", "agent_cli_version_hash"}
352
+ version_key = {"agent_cli_version"}
353
+ raw_keys = set(raw)
354
+ if raw_keys not in (allowed, allowed | new_agent_keys, allowed | new_agent_keys | version_key):
355
+ return None
356
+ if raw.get("schema") != EXECUTION_IDENTITY_SCHEMA:
310
357
  return None
311
358
  if not all(_is_hash(raw.get(key)) for key in _EXECUTION_IDENTITY_HASH_KEYS):
312
359
  return None
360
+ if new_agent_keys.issubset(raw_keys):
361
+ if raw.get("agent") not in AGENT_IDS:
362
+ return None
363
+ if not isinstance(raw.get("agent_cli"), str) or not raw["agent_cli"]:
364
+ return None
365
+ if not _is_hash(raw.get("agent_cli_version_hash")):
366
+ return None
367
+ if "agent_cli_version" in raw_keys and (
368
+ not isinstance(raw.get("agent_cli_version"), str)
369
+ or not raw["agent_cli_version"]
370
+ or len(raw["agent_cli_version"]) > 512
371
+ or any(ord(character) < 32 for character in raw["agent_cli_version"])
372
+ ):
373
+ return None
313
374
  if raw.get("model_source") not in {"explicit", "runtime-default"}:
314
375
  return None
315
376
  if raw.get("interaction_mode") not in {"none", "auto_user", "custom"}:
@@ -320,7 +381,8 @@ def _validated_execution_identity(raw: Any) -> dict[str, str] | None:
320
381
  return None
321
382
  if not 1 <= max_turns <= 1000 or str(max_turns) != raw.get("interaction_max_turns"):
322
383
  return None
323
- return {key: raw[key] for key in sorted(allowed)}
384
+ identity_keys = raw_keys
385
+ return {key: raw[key] for key in sorted(identity_keys)}
324
386
 
325
387
 
326
388
  def build_case_manifest(
@@ -328,28 +390,73 @@ def build_case_manifest(
328
390
  tasks_dir: Path,
329
391
  *,
330
392
  execution_identity: dict[str, str] | None = None,
393
+ manifest_path: Path | None = None,
394
+ rendered_prompt: str | None = None,
395
+ workspace_dir: Path | None = None,
396
+ validation_rules: Mapping[str, Any] | None = None,
397
+ environment_dir: Path | None = None,
331
398
  ) -> CaseManifest:
332
399
  """Build a manifest for the canonical task and, for new runs, its execution."""
333
400
  task_dir = tasks_dir / task
334
- instruction_path = task_dir / "instruction.md"
335
- task_hash = _hash_tree(task_dir / "task.toml")
336
- instruction_hash = _hash_tree(instruction_path)
337
- validator_components = {
338
- "task_validation": _hash_tree(task_dir / "validation"),
339
- "shared_validation": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "validation"),
340
- "native_adapter": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "native_eval.py"),
341
- }
342
- validator_hash = _sha256_bytes(
343
- json.dumps(validator_components, sort_keys=True, separators=(",", ":")).encode("utf-8")
344
- )
345
- environment_hash = _hash_tree(task_dir / "environment")
346
- data_hash = _hash_tree(task_dir / "data")
347
- instruction_bytes = instruction_path.read_bytes() if instruction_path.is_file() else b"missing"
348
- prompt_components = {
349
- "instruction": _sha256_bytes(b"canonical-task-prompt-v1\0" + instruction_bytes),
350
- "task_renderer": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "tasks.py"),
351
- "native_adapter": validator_components["native_adapter"],
352
- }
401
+ native_adapter_hash = _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "native_eval.py")
402
+ authored_hashes: dict[str, str] | None = None
403
+ if manifest_path is not None:
404
+ prompt_bytes = (rendered_prompt or "").encode("utf-8")
405
+ manifest_hash = _hash_tree(manifest_path)
406
+ workspace_hash = (
407
+ _hash_tree(workspace_dir)
408
+ if workspace_dir is not None
409
+ else _sha256_bytes(b"comet-empty-inline-workspace-v1")
410
+ )
411
+ validation_rule_hash = _hash_private_value(dict(validation_rules or {}))
412
+ instruction_hash = _sha256_bytes(b"inline-task-prompt-v1\0" + prompt_bytes)
413
+ task_hash = _hash_private_value(
414
+ {
415
+ "task": task,
416
+ "manifest_hash": manifest_hash,
417
+ "instruction_hash": instruction_hash,
418
+ "workspace_hash": workspace_hash,
419
+ "validation_rule_hash": validation_rule_hash,
420
+ }
421
+ )
422
+ validator_hash = validation_rule_hash
423
+ environment_hash = (
424
+ _hash_tree(environment_dir) if environment_dir else _sha256_bytes(b"missing")
425
+ )
426
+ data_hash = workspace_hash
427
+ prompt_components = {
428
+ "instruction": instruction_hash,
429
+ "task_renderer": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "tasks.py"),
430
+ "native_adapter": native_adapter_hash,
431
+ }
432
+ authored_hashes = {
433
+ "manifest_hash": manifest_hash,
434
+ "task_definition_hash": task_hash,
435
+ "workspace_hash": workspace_hash,
436
+ "validation_rule_hash": validation_rule_hash,
437
+ }
438
+ else:
439
+ instruction_path = task_dir / "instruction.md"
440
+ task_hash = _hash_tree(task_dir / "task.toml")
441
+ instruction_hash = _hash_tree(instruction_path)
442
+ validator_components = {
443
+ "task_validation": _hash_tree(task_dir / "validation"),
444
+ "shared_validation": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "validation"),
445
+ "native_adapter": native_adapter_hash,
446
+ }
447
+ validator_hash = _sha256_bytes(
448
+ json.dumps(validator_components, sort_keys=True, separators=(",", ":")).encode("utf-8")
449
+ )
450
+ environment_hash = _hash_tree(task_dir / "environment")
451
+ data_hash = _hash_tree(task_dir / "data")
452
+ instruction_bytes = (
453
+ instruction_path.read_bytes() if instruction_path.is_file() else b"missing"
454
+ )
455
+ prompt_components = {
456
+ "instruction": _sha256_bytes(b"canonical-task-prompt-v1\0" + instruction_bytes),
457
+ "task_renderer": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "tasks.py"),
458
+ "native_adapter": native_adapter_hash,
459
+ }
353
460
  prompt_hash = _sha256_bytes(
354
461
  json.dumps(prompt_components, sort_keys=True, separators=(",", ":")).encode("utf-8")
355
462
  )
@@ -403,6 +510,7 @@ def build_case_manifest(
403
510
  controller_hash=controller_hash,
404
511
  execution_hash=execution_hash,
405
512
  execution_identity=validated_identity,
513
+ authored_hashes=authored_hashes,
406
514
  source=source,
407
515
  run_bound=run_bound,
408
516
  execution_bound=execution_bound,
@@ -425,6 +533,8 @@ def case_manifest_payload(manifest: CaseManifest) -> dict[str, Any]:
425
533
  }
426
534
  )
427
535
  payload["execution_identity"] = manifest.execution_identity
536
+ if manifest.authored_hashes is not None:
537
+ payload["authored_hashes"] = manifest.authored_hashes
428
538
  return payload
429
539
 
430
540
 
@@ -442,10 +552,28 @@ def _embedded_case_manifest(report: dict[str, Any], task: str) -> CaseManifest |
442
552
  expected_keys = {"schema", "task", "case_hash", *hashed_parts}
443
553
  if schema == CASE_MANIFEST_SCHEMA:
444
554
  expected_keys.add("execution_identity")
555
+ authored_hashes = raw.get("authored_hashes")
556
+ if authored_hashes is not None:
557
+ expected_keys.add("authored_hashes")
445
558
  if set(raw) != expected_keys:
446
559
  return None
447
560
  if not all(_is_hash(raw.get(key)) for key in hashed_parts):
448
561
  return None
562
+ if authored_hashes is not None and (
563
+ not isinstance(authored_hashes, dict)
564
+ or set(authored_hashes)
565
+ != {
566
+ "manifest_hash",
567
+ "task_definition_hash",
568
+ "workspace_hash",
569
+ "validation_rule_hash",
570
+ }
571
+ or not all(_is_hash(value) for value in authored_hashes.values())
572
+ or authored_hashes["task_definition_hash"] != raw["task_hash"]
573
+ or authored_hashes["workspace_hash"] != raw["data_hash"]
574
+ or authored_hashes["validation_rule_hash"] != raw["validator_hash"]
575
+ ):
576
+ return None
449
577
  case_hash = raw.get("case_hash")
450
578
  if not isinstance(case_hash, str) or not case_hash.startswith("sha256:"):
451
579
  return None
@@ -477,6 +605,7 @@ def _embedded_case_manifest(report: dict[str, Any], task: str) -> CaseManifest |
477
605
  controller_hash=raw.get("controller_hash"),
478
606
  execution_hash=raw.get("execution_hash"),
479
607
  execution_identity=execution_identity,
608
+ authored_hashes=authored_hashes,
480
609
  source="report-bound-v2" if schema == CASE_MANIFEST_SCHEMA else "report-bound-v1",
481
610
  run_bound=True,
482
611
  execution_bound=schema == CASE_MANIFEST_SCHEMA,