@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -0,0 +1,19 @@
1
+ apiVersion: comet.eval.agent/v1alpha1
2
+ kind: EvalAgentAdapter
3
+ metadata:
4
+ id: fixture-agent
5
+ version: 1.0.0
6
+ runtime:
7
+ executable: fixture-agent
8
+ install:
9
+ kind: none
10
+ credentials:
11
+ - FIXTURE_AGENT_API_KEY
12
+ modelEnv: FIXTURE_AGENT_MODEL
13
+ baseUrlEnv: FIXTURE_AGENT_BASE_URL
14
+ capabilities:
15
+ singleTurn: true
16
+ resume: true
17
+ structuredEvents: true
18
+ telemetry: false
19
+ skillInvocationEvidence: true
@@ -0,0 +1,76 @@
1
+ """Tests for selectable evaluation-agent contracts."""
2
+
3
+ import pytest
4
+
5
+ from scaffold.python.agents import (
6
+ AGENT_IDS,
7
+ get_agent_adapter,
8
+ resolve_agent,
9
+ )
10
+
11
+
12
+ def test_resolve_agent_applies_cli_manifest_default_precedence():
13
+ assert resolve_agent("qoder", "codex").agent == "qoder"
14
+ assert resolve_agent(None, "codex").agent == "codex"
15
+ assert resolve_agent(None, None).agent == "claude-code"
16
+
17
+
18
+ @pytest.mark.parametrize("agent", ["gemini", "", "Claude-Code"])
19
+ def test_resolve_agent_rejects_unknown_agent(agent: str):
20
+ with pytest.raises(ValueError, match="Unsupported evaluation agent"):
21
+ resolve_agent(agent, None)
22
+
23
+
24
+ def test_all_supported_agents_expose_role_neutral_run_commands():
25
+ assert AGENT_IDS == ("claude-code", "codex", "qoder", "codebuddy")
26
+
27
+ for agent_id in AGENT_IDS:
28
+ adapter = get_agent_adapter(agent_id)
29
+ command = adapter.build_run_command(
30
+ "inspect the task",
31
+ model="test-model",
32
+ role="subject",
33
+ )
34
+
35
+ assert adapter.id == agent_id
36
+ assert command
37
+ assert "inspect the task" in command
38
+ assert "test-model" in command
39
+
40
+
41
+ def test_codebuddy_adapter_uses_headless_streaming_and_resume_contract():
42
+ adapter = get_agent_adapter("codebuddy")
43
+
44
+ assert adapter.executable == "codebuddy"
45
+ assert adapter.required_credentials == ("CODEBUDDY_API_KEY", "CODEBUDDY_AUTH_TOKEN")
46
+ assert adapter.build_run_command(
47
+ "continue",
48
+ model="codebuddy-model",
49
+ role="subject",
50
+ resume_id="session-123",
51
+ ) == [
52
+ "codebuddy",
53
+ "-p",
54
+ "continue",
55
+ "--output-format",
56
+ "stream-json",
57
+ "--dangerously-skip-permissions",
58
+ "-r",
59
+ "session-123",
60
+ "--model",
61
+ "codebuddy-model",
62
+ ]
63
+
64
+
65
+ def test_adapters_build_resume_commands_with_isolated_session_ids():
66
+ for agent_id in AGENT_IDS:
67
+ adapter = get_agent_adapter(agent_id)
68
+ command = adapter.build_run_command(
69
+ "continue",
70
+ model=None,
71
+ role="simulator",
72
+ resume_id="session-123",
73
+ )
74
+
75
+ assert "session-123" in command
76
+ assert adapter.supports_resume is True
@@ -178,6 +178,39 @@ def test_fallback_case_manifest_never_hashes_dotenv_files(tmp_path: Path):
178
178
  assert before.case_hash == after.case_hash
179
179
 
180
180
 
181
+ def test_inline_case_manifest_hashes_authored_inputs(tmp_path: Path):
182
+ manifest_path = tmp_path / "comet" / "eval.yaml"
183
+ manifest_path.parent.mkdir(parents=True)
184
+ manifest_path.write_text("tasks:\n - name: inline-task\n", encoding="utf-8")
185
+ workspace = tmp_path / "workspace"
186
+ workspace.mkdir()
187
+ (workspace / "input.txt").write_text("fixture\n", encoding="utf-8")
188
+ environment = tmp_path / "environment"
189
+ environment.mkdir()
190
+ (environment / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
191
+
192
+ case = build_case_manifest(
193
+ "inline-task",
194
+ tmp_path,
195
+ manifest_path=manifest_path,
196
+ rendered_prompt="Create result.md.",
197
+ workspace_dir=workspace,
198
+ validation_rules={"files": ["result.md"]},
199
+ environment_dir=environment,
200
+ )
201
+ payload = case_manifest_payload(case)
202
+
203
+ assert payload["authored_hashes"] == {
204
+ "manifest_hash": case.authored_hashes["manifest_hash"],
205
+ "task_definition_hash": case.task_hash,
206
+ "workspace_hash": case.data_hash,
207
+ "validation_rule_hash": case.validator_hash,
208
+ }
209
+ assert case.task_hash.startswith("sha256:")
210
+ assert case.data_hash.startswith("sha256:")
211
+ assert case.validator_hash.startswith("sha256:")
212
+
213
+
181
214
  def test_v2_case_manifest_binds_safe_execution_identity_without_raw_configuration(
182
215
  tmp_path: Path,
183
216
  ):
@@ -234,6 +267,43 @@ def test_v2_case_manifest_binds_safe_execution_identity_without_raw_configuratio
234
267
  assert changed.case_hash != manifest.case_hash
235
268
 
236
269
 
270
+ def test_execution_identity_binds_selected_agent_cli():
271
+ interaction = SimpleNamespace(
272
+ mode="none",
273
+ max_turns=1,
274
+ simulator_prompt=None,
275
+ decision_patterns=[],
276
+ decision_reply=None,
277
+ decision_replies=[],
278
+ continue_prompt="continue",
279
+ fresh_resume_marker=None,
280
+ )
281
+ docker_identity = {
282
+ "schema": EXECUTION_IDENTITY_SCHEMA,
283
+ "runtime_image_id": "sha256:" + "2" * 64,
284
+ "image_id_hash": _digest("sha256:" + "2" * 64),
285
+ "image_repo_digests_hash": _digest("repo-digests"),
286
+ "image_ref_hash": _digest("build-ref"),
287
+ "claude_tool_version_hash": _digest("codex-1.0.0"),
288
+ "agent_cli_version_hash": _digest("codex-1.0.0"),
289
+ "agent_cli_version": "codex-cli 1.0.0",
290
+ "agent": "codex",
291
+ "agent_cli": "codex",
292
+ }
293
+
294
+ identity = build_execution_identity(
295
+ docker_identity,
296
+ model="o3",
297
+ model_config={"OPENAI_MODEL": "o3"},
298
+ interaction=interaction,
299
+ )
300
+
301
+ assert identity["agent"] == "codex"
302
+ assert identity["agent_cli"] == "codex"
303
+ assert identity["agent_cli_version_hash"] == _digest("codex-1.0.0")
304
+ assert identity["agent_cli_version"] == "codex-cli 1.0.0"
305
+
306
+
237
307
  def test_expected_matrix_detects_case_missing_from_both_experiments(tmp_path: Path):
238
308
  candidate, baseline, tasks = _make_experiments(tmp_path)
239
309
  _write_run(
@@ -0,0 +1,350 @@
1
+ """Tests for automatic task generation and its cache contract."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ import pytest
7
+ import yaml
8
+
9
+ from scaffold.python.auto_tasks import (
10
+ AutoTaskError,
11
+ GenerationOutput,
12
+ build_skill_snapshot,
13
+ ensure_generated_manifest,
14
+ )
15
+
16
+
17
+ def _write_skill(tmp_path: Path) -> Path:
18
+ skill = tmp_path / "my-skill"
19
+ (skill / "references").mkdir(parents=True)
20
+ (skill / "SKILL.md").write_text(
21
+ "Use the reference in `references/format.md` and write result.md.\n", encoding="utf-8"
22
+ )
23
+ (skill / "references" / "format.md").write_text("# Format\n", encoding="utf-8")
24
+ (tmp_path / "unrelated.txt").write_text("do not include\n", encoding="utf-8")
25
+ return skill
26
+
27
+
28
+ def _generated_payload():
29
+ return {
30
+ "tasks": [
31
+ {
32
+ "name": "normal-flow",
33
+ "prompt": "Use the Skill to create result.md.",
34
+ "expect": {"files": ["result.md"]},
35
+ },
36
+ {
37
+ "name": "boundary-flow",
38
+ "prompt": "Use the Skill with an invalid input and document recovery.",
39
+ "expect": {"contains": {"result.md": ["recovery"]}},
40
+ "rubric": ["The recovery is clear."],
41
+ },
42
+ ]
43
+ }
44
+
45
+
46
+ def test_skill_snapshot_is_bounded_to_relevant_package_files(tmp_path: Path):
47
+ skill = _write_skill(tmp_path)
48
+
49
+ snapshot = build_skill_snapshot(skill)
50
+
51
+ paths = {item.path for item in snapshot.files}
52
+ assert paths == {"SKILL.md", "references/format.md"}
53
+ assert "unrelated.txt" not in paths
54
+ assert snapshot.content_hash.startswith("sha256:")
55
+
56
+
57
+ def test_generated_manifest_is_cached_and_reused(tmp_path: Path):
58
+ skill = _write_skill(tmp_path)
59
+ calls = []
60
+
61
+ def generate(_prompt):
62
+ calls.append(True)
63
+ return _generated_payload()
64
+
65
+ first = ensure_generated_manifest(
66
+ skill,
67
+ tmp_path,
68
+ agent="claude-code",
69
+ model=None,
70
+ profile="generic",
71
+ interaction={"mode": "none", "max_turns": 12},
72
+ generate=generate,
73
+ )
74
+ second = ensure_generated_manifest(
75
+ skill,
76
+ tmp_path,
77
+ agent="claude-code",
78
+ model=None,
79
+ profile="generic",
80
+ interaction={"mode": "none", "max_turns": 12},
81
+ generate=generate,
82
+ )
83
+
84
+ assert first.manifest_path == second.manifest_path
85
+ assert first.generation_hash == second.generation_hash
86
+ assert calls == [True]
87
+ assert first.manifest_path == (
88
+ tmp_path
89
+ / ".comet"
90
+ / "eval"
91
+ / "generated"
92
+ / "my-skill"
93
+ / first.generation_hash
94
+ / "eval.yaml"
95
+ )
96
+ metadata = json.loads(first.metadata_path.read_text(encoding="utf-8"))
97
+ assert metadata["manifest_hash"].startswith("sha256:")
98
+ assert metadata["generation_overhead"]["attempt_count"] == 1
99
+
100
+
101
+ def test_generated_cache_uses_owner_locks_atomic_replacement_and_redacted_metadata(
102
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
103
+ ):
104
+ skill = _write_skill(tmp_path)
105
+ replacements = []
106
+ import scaffold.python.auto_tasks as auto_tasks
107
+
108
+ original_replace = auto_tasks.os.replace
109
+ monkeypatch.setattr(
110
+ auto_tasks.os,
111
+ "replace",
112
+ lambda source, target: (replacements.append((Path(source), Path(target))), original_replace(source, target))[1],
113
+ )
114
+ result = ensure_generated_manifest(
115
+ skill,
116
+ tmp_path,
117
+ agent="codex",
118
+ model=None,
119
+ profile="generic",
120
+ interaction={"mode": "none", "api_key": "must-not-persist"},
121
+ generate=lambda _prompt: _generated_payload(),
122
+ )
123
+
124
+ metadata = result.metadata_path.read_text(encoding="utf-8")
125
+ lock_dir = tmp_path / ".comet" / "eval" / "locks"
126
+ assert lock_dir.is_dir()
127
+ assert "must-not-persist" not in metadata
128
+ assert "api_key" not in metadata
129
+ assert any(target == result.manifest_path.parent for _, target in replacements)
130
+
131
+
132
+ def test_cached_manifest_hash_mismatch_forces_regeneration(tmp_path: Path):
133
+ skill = _write_skill(tmp_path)
134
+ calls = []
135
+
136
+ def generate(_prompt):
137
+ calls.append(True)
138
+ return _generated_payload()
139
+
140
+ first = ensure_generated_manifest(
141
+ skill,
142
+ tmp_path,
143
+ agent="codex",
144
+ model="gpt-test",
145
+ profile="generic",
146
+ interaction={"mode": "none", "max_turns": 12},
147
+ generate=generate,
148
+ )
149
+ first.manifest_path.write_text("tasks: []\n", encoding="utf-8")
150
+
151
+ second = ensure_generated_manifest(
152
+ skill,
153
+ tmp_path,
154
+ agent="codex",
155
+ model="gpt-test",
156
+ profile="generic",
157
+ interaction={"mode": "none", "max_turns": 12},
158
+ generate=generate,
159
+ )
160
+
161
+ assert calls == [True, True]
162
+ assert "normal-flow" in second.manifest_path.read_text(encoding="utf-8")
163
+
164
+
165
+ @pytest.mark.parametrize("task_count", [1, 5])
166
+ def test_cached_generated_manifest_revalidates_task_count(
167
+ tmp_path: Path, task_count: int
168
+ ):
169
+ skill = _write_skill(tmp_path)
170
+ calls = []
171
+
172
+ def generate(_prompt):
173
+ calls.append(True)
174
+ return _generated_payload()
175
+
176
+ first = ensure_generated_manifest(
177
+ skill,
178
+ tmp_path,
179
+ agent="codex",
180
+ model="gpt-test",
181
+ profile="generic",
182
+ interaction={"mode": "none", "max_turns": 12},
183
+ generate=generate,
184
+ )
185
+ data = yaml.safe_load(first.manifest_path.read_text(encoding="utf-8"))
186
+ tasks = data["evaluation"]["tasks"]
187
+ data["evaluation"]["tasks"] = (tasks * task_count)[:task_count]
188
+ first.manifest_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
189
+ from scaffold.python.generated_task_cache import write_cache_metadata
190
+
191
+ write_cache_metadata(first.manifest_path.parent, first.generation_hash)
192
+
193
+ second = ensure_generated_manifest(
194
+ skill,
195
+ tmp_path,
196
+ agent="codex",
197
+ model="gpt-test",
198
+ profile="generic",
199
+ interaction={"mode": "none", "max_turns": 12},
200
+ generate=generate,
201
+ )
202
+
203
+ assert calls == [True, True]
204
+ assert len(yaml.safe_load(second.manifest_path.read_text(encoding="utf-8"))["evaluation"]["tasks"]) == 2
205
+
206
+
207
+ def test_generation_overhead_is_persisted_in_cache_metadata(tmp_path: Path):
208
+ skill = _write_skill(tmp_path)
209
+
210
+ result = ensure_generated_manifest(
211
+ skill,
212
+ tmp_path,
213
+ agent="codebuddy",
214
+ model="codebuddy-test",
215
+ profile="generic",
216
+ interaction={"mode": "none", "max_turns": 12},
217
+ generate=lambda _prompt: GenerationOutput(
218
+ output=_generated_payload(),
219
+ telemetry={
220
+ "duration_seconds": 0.125,
221
+ "input_tokens": 10,
222
+ "output_tokens": 20,
223
+ "total_tokens": 30,
224
+ "total_cost_usd": 0.01,
225
+ "model": "codebuddy-test",
226
+ },
227
+ ),
228
+ )
229
+
230
+ metadata = json.loads(result.metadata_path.read_text(encoding="utf-8"))
231
+ overhead = metadata["generation_overhead"]
232
+ assert overhead["attempt_count"] == 1
233
+ assert overhead["duration_seconds"] == 0.125
234
+ assert overhead["total_tokens"] == 30
235
+ assert overhead["total_cost_usd"] == 0.01
236
+ assert overhead["attempts"][0]["model"] == "codebuddy-test"
237
+
238
+
239
+ def test_collect_requires_a_valid_cache_and_never_calls_generator(tmp_path: Path):
240
+ skill = _write_skill(tmp_path)
241
+ calls = []
242
+
243
+ with pytest.raises(AutoTaskError, match="run comet eval"):
244
+ ensure_generated_manifest(
245
+ skill,
246
+ tmp_path,
247
+ agent="codex",
248
+ model="gpt-test",
249
+ profile="generic",
250
+ interaction={"mode": "none", "max_turns": 12},
251
+ collect_only=True,
252
+ generate=lambda _prompt: calls.append(True),
253
+ )
254
+
255
+ assert calls == []
256
+
257
+
258
+ def test_invalid_generation_gets_one_repair_attempt(tmp_path: Path):
259
+ skill = _write_skill(tmp_path)
260
+ responses = [{"tasks": [{"name": "only-one", "prompt": "No expect."}]}, _generated_payload()]
261
+
262
+ def generate(_prompt):
263
+ return responses.pop(0)
264
+
265
+ result = ensure_generated_manifest(
266
+ skill,
267
+ tmp_path,
268
+ agent="codebuddy",
269
+ model="codebuddy-test",
270
+ profile="generic",
271
+ interaction={"mode": "none", "max_turns": 12},
272
+ generate=generate,
273
+ )
274
+
275
+ assert result.manifest_path.is_file()
276
+ assert responses == []
277
+
278
+
279
+ def test_invalid_generation_stops_after_one_repair_attempt(tmp_path: Path):
280
+ skill = _write_skill(tmp_path)
281
+ calls = []
282
+
283
+ def generate(_prompt):
284
+ calls.append(True)
285
+ return {"tasks": [{"name": "only-one", "prompt": "No expect. api_key=sk-secret-value"}]}
286
+
287
+ with pytest.raises(AutoTaskError, match="task_generation") as exc_info:
288
+ ensure_generated_manifest(
289
+ skill,
290
+ tmp_path,
291
+ agent="claude-code",
292
+ model=None,
293
+ profile="generic",
294
+ interaction={"mode": "none", "max_turns": 12},
295
+ generate=generate,
296
+ )
297
+
298
+ assert calls == [True, True]
299
+ report_path = Path(str(exc_info.value).split("Partial report: ", 1)[1])
300
+ metadata = json.loads((report_path.parent / "metadata.json").read_text(encoding="utf-8"))
301
+ assert report_path.is_file()
302
+ assert metadata["attribution"] == "task_generation"
303
+ assert metadata["attempt_count"] == 2
304
+ assert metadata["task_denominator"] == 0
305
+ assert metadata["subject_metrics"] == {}
306
+ assert metadata["report_path"] == str(report_path)
307
+ assert "sk-secret-value" not in report_path.read_text(encoding="utf-8")
308
+ assert "sk-secret-value" not in (report_path.parent / "metadata.json").read_text(encoding="utf-8")
309
+
310
+
311
+ def test_html_generation_failure_escapes_generator_controlled_error_text(
312
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
313
+ ):
314
+ skill = _write_skill(tmp_path)
315
+ monkeypatch.setenv("COMET_EVAL_REPORT_HTML", "1")
316
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "environment-secret-value")
317
+
318
+ def generate(_prompt):
319
+ raise ValueError(
320
+ "<script>globalThis.cometEvalInjected = true</script> "
321
+ "api_key=sk-secret-value Authorization: Bearer bearer-secret-value "
322
+ "x-api-key: header-secret-value environment-secret-value"
323
+ )
324
+
325
+ with pytest.raises(AutoTaskError) as exc_info:
326
+ ensure_generated_manifest(
327
+ skill,
328
+ tmp_path,
329
+ agent="claude-code",
330
+ model=None,
331
+ profile="generic",
332
+ interaction={"mode": "none", "max_turns": 12},
333
+ generate=generate,
334
+ )
335
+
336
+ report_path = Path(str(exc_info.value).split("Partial report: ", 1)[1])
337
+ report = report_path.read_text(encoding="utf-8")
338
+ assert report_path.suffix == ".html"
339
+ assert "<script>" not in report
340
+ assert "&lt;script&gt;globalThis.cometEvalInjected = true&lt;/script&gt;" in report
341
+ assert "sk-secret-value" not in report
342
+ metadata = (report_path.parent / "metadata.json").read_text(encoding="utf-8")
343
+ for secret in (
344
+ "sk-secret-value",
345
+ "bearer-secret-value",
346
+ "header-secret-value",
347
+ "environment-secret-value",
348
+ ):
349
+ assert secret not in report
350
+ assert secret not in metadata