@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -0,0 +1,700 @@
1
+ """Automatic, cached generation of #250-compatible eval task manifests."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import html as html_module
7
+ import json
8
+ import os
9
+ import re
10
+ import shutil
11
+ import tempfile
12
+ import time
13
+ from contextlib import contextmanager
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Any, Callable, Iterator, Mapping
17
+
18
+ import yaml
19
+
20
+ from scaffold.python.agents import AgentId, get_agent_adapter, validate_agent_id
21
+ from scaffold.python.eval_context import managed_path_for_owner
22
+ from scaffold.python.generated_task_cache import (
23
+ build_skill_snapshot as _shared_build_skill_snapshot,
24
+ cache_location as _shared_cache_location,
25
+ generation_hash as _shared_generation_hash,
26
+ normalize_interaction,
27
+ validate_generated_manifest,
28
+ )
29
+ from scaffold.python.manifests import load_eval_manifest
30
+ from scaffold.python.utils import run_agent_in_docker
31
+
32
+
33
+ GENERATOR_VERSION = "comet-auto-task-generator.v1"
34
+ TASK_SCHEMA_VERSION = "comet.eval/v1alpha1"
35
+ MAX_SNAPSHOT_FILES = 128
36
+ MAX_SNAPSHOT_BYTES = 512 * 1024
37
+ MAX_GENERATED_TASKS = 4
38
+ MIN_GENERATED_TASKS = 2
39
+
40
+
41
+ class AutoTaskError(ValueError):
42
+ """A task-generation failure that must not be counted as a task failure."""
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class SnapshotFile:
47
+ path: str
48
+ content: str
49
+ content_hash: str
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class SkillSnapshot:
54
+ files: tuple[SnapshotFile, ...]
55
+ content_hash: str
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class GeneratedManifest:
60
+ manifest_path: Path
61
+ metadata_path: Path
62
+ generation_hash: str
63
+ reused: bool = False
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class GenerationOutput:
68
+ """Generator response plus role-local overhead telemetry."""
69
+
70
+ output: object
71
+ telemetry: dict[str, Any]
72
+
73
+
74
+ def _sha256(value: str | bytes) -> str:
75
+ data = value.encode("utf-8") if isinstance(value, str) else value
76
+ return "sha256:" + hashlib.sha256(data).hexdigest()
77
+
78
+
79
+ def _safe_skill_root(skill_path: Path) -> Path:
80
+ root = skill_path.resolve()
81
+ if root.is_file() and root.name == "SKILL.md":
82
+ root = root.parent
83
+ if not root.is_dir() or not (root / "SKILL.md").is_file():
84
+ raise AutoTaskError(f"Skill package must contain SKILL.md: {skill_path}")
85
+ return root
86
+
87
+
88
+ def _inside(root: Path, path: Path) -> bool:
89
+ try:
90
+ path.resolve().relative_to(root.resolve())
91
+ return True
92
+ except ValueError:
93
+ return False
94
+
95
+
96
+ def _add_snapshot_file(root: Path, files: dict[str, SnapshotFile], path: Path) -> None:
97
+ if len(files) >= MAX_SNAPSHOT_FILES or not path.is_file() or not _inside(root, path):
98
+ return
99
+ relative = path.resolve().relative_to(root.resolve()).as_posix()
100
+ if relative in files:
101
+ return
102
+ content = path.read_text(encoding="utf-8", errors="replace")
103
+ if (
104
+ sum(len(item.content.encode("utf-8")) for item in files.values())
105
+ + len(content.encode("utf-8"))
106
+ > MAX_SNAPSHOT_BYTES
107
+ ):
108
+ return
109
+ files[relative] = SnapshotFile(relative, content, _sha256(content))
110
+
111
+
112
+ def build_skill_snapshot(skill_path: Path | str) -> SkillSnapshot:
113
+ """Capture only the Skill and its bounded, relevant package context."""
114
+ snapshot = _shared_build_skill_snapshot(skill_path)
115
+ return SkillSnapshot(
116
+ tuple(SnapshotFile(item.path, item.content, item.content_hash) for item in snapshot.files),
117
+ snapshot.content_hash,
118
+ )
119
+
120
+
121
+ def find_project_root(skill_path: Path | str) -> Path:
122
+ """Find a project root without scanning outside the Skill's ancestors."""
123
+ root = _safe_skill_root(Path(skill_path))
124
+ home = Path.home().resolve()
125
+ for candidate in (root, *root.parents):
126
+ if candidate.resolve() == home:
127
+ continue
128
+ if (candidate / ".comet").is_dir() or (candidate / ".git").exists():
129
+ return candidate
130
+ return root.parent
131
+
132
+
133
+ def _generation_hash(
134
+ snapshot: SkillSnapshot,
135
+ *,
136
+ agent: AgentId,
137
+ model: str | None,
138
+ base_url: str | None = None,
139
+ profile: str,
140
+ interaction: dict[str, Any],
141
+ ) -> str:
142
+ return _shared_generation_hash(
143
+ snapshot,
144
+ agent=agent,
145
+ model=model,
146
+ base_url=base_url,
147
+ profile=profile,
148
+ interaction=interaction,
149
+ )
150
+
151
+
152
+ def _cache_location(project_root: Path, skill_root: Path, generation_hash: str) -> Path:
153
+ return _shared_cache_location(project_root, skill_root, generation_hash)
154
+
155
+
156
+ def _atomic_write_text(path: Path, value: str) -> None:
157
+ path.parent.mkdir(parents=True, exist_ok=True)
158
+ with tempfile.NamedTemporaryFile(
159
+ mode="w",
160
+ encoding="utf-8",
161
+ newline="\n",
162
+ dir=path.parent,
163
+ prefix=f".{path.name}.",
164
+ delete=False,
165
+ ) as temporary:
166
+ temporary.write(value)
167
+ temporary_path = Path(temporary.name)
168
+ try:
169
+ os.replace(temporary_path, path)
170
+ finally:
171
+ temporary_path.unlink(missing_ok=True)
172
+
173
+
174
+ def _write_generation_failure_report(
175
+ project_root: Path,
176
+ *,
177
+ attempts: list[dict[str, Any]],
178
+ error: str,
179
+ environment: Mapping[str, str] | None = None,
180
+ ) -> Path:
181
+ """Persist a user-owned terminal report without manufacturing task results."""
182
+ experiment = os.environ.get("COMET_EVAL_EXPERIMENT_ID") or f"task-generation-{time.time_ns()}"
183
+ report_dir = managed_path_for_owner(project_root, "runs", experiment)
184
+ report_dir.mkdir(parents=True, exist_ok=True)
185
+ safe_error = _redact_sensitive(error, environment)
186
+ safe_attempts = _redact_sensitive(_without_credentials(attempts), environment)
187
+ html = os.environ.get("COMET_EVAL_REPORT_HTML") == "1"
188
+ output_path = report_dir / ("summary.html" if html else "summary.md")
189
+ escaped_error = html_module.escape(str(safe_error)) if html else safe_error
190
+ metadata = {
191
+ "schema": "comet.eval.generation.failure.v1",
192
+ "attribution": "task_generation",
193
+ "category": "task_generation",
194
+ "attempt_count": len(attempts),
195
+ "case_count": 0,
196
+ "task_denominator": 0,
197
+ "subject_metrics": {},
198
+ "generation_overhead": _merge_generation_telemetry(safe_attempts),
199
+ "error": safe_error,
200
+ "report_path": str(output_path),
201
+ "report_output": output_path.name,
202
+ }
203
+ _atomic_write_text(
204
+ report_dir / "metadata.json", json.dumps(metadata, indent=2, sort_keys=True) + "\n"
205
+ )
206
+ _atomic_write_text(
207
+ output_path,
208
+ (
209
+ "<h1>Evaluation stopped during task generation</h1>"
210
+ f"<p>Attempts: {len(attempts)}</p><p>Task denominator: 0</p>"
211
+ f"<p>Error: {escaped_error}</p>"
212
+ if html
213
+ else "# Evaluation stopped during task generation\n\n"
214
+ "Attribution: `task_generation`\n\n"
215
+ f"Attempts: {len(attempts)}\n\nTask denominator: 0\n\nError: {safe_error}\n"
216
+ ),
217
+ )
218
+ return output_path
219
+
220
+
221
+ def _without_credentials(value: Any) -> Any:
222
+ if isinstance(value, dict):
223
+ return {
224
+ key: _without_credentials(item)
225
+ for key, item in value.items()
226
+ if not re.search(
227
+ r"(?:api[_-]?key|auth[_-]?token|access[_-]?token|personal[_-]?access[_-]?token|"
228
+ r"(?:^|[_-])(key|token|secret|password|credential)$)",
229
+ key,
230
+ re.I,
231
+ )
232
+ }
233
+ if isinstance(value, list):
234
+ return [_without_credentials(item) for item in value]
235
+ return value
236
+
237
+
238
+ def _credential_values(source_env: Mapping[str, str] | None = None) -> tuple[str, ...]:
239
+ """Return configured credential values for final report redaction."""
240
+ environment = dict(os.environ)
241
+ if source_env is not None:
242
+ environment.update(source_env)
243
+ custom_names = {
244
+ item.strip()
245
+ for metadata_key in ("COMET_EVAL_CUSTOM_CREDENTIALS", "COMET_EVAL_MAIN_CREDENTIALS")
246
+ for item in environment.get(metadata_key, "").split(",")
247
+ if item.strip()
248
+ }
249
+ values = {
250
+ value.strip()
251
+ for key, value in environment.items()
252
+ if value.strip()
253
+ and (
254
+ key in custom_names
255
+ or re.search(
256
+ r"(?:api[_-]?key|auth[_-]?token|access[_-]?token|personal[_-]?access[_-]?token|"
257
+ r"(?:^|[_-])(key|token|secret|password|credential)$)",
258
+ key,
259
+ re.I,
260
+ )
261
+ )
262
+ }
263
+ return tuple(sorted((value for value in values if len(value) >= 4), key=len, reverse=True))
264
+
265
+
266
+ def _redact_sensitive(value: Any, source_env: Mapping[str, str] | None = None) -> Any:
267
+ """Remove common credential forms and configured values from reports."""
268
+ if isinstance(value, dict):
269
+ return {key: _redact_sensitive(item, source_env) for key, item in value.items()}
270
+ if isinstance(value, list):
271
+ return [_redact_sensitive(item, source_env) for item in value]
272
+ if not isinstance(value, str):
273
+ return value
274
+ redacted = re.sub(
275
+ r"(?i)(authorization\s*:\s*bearer\s+)[^\s,;]+",
276
+ r"\1[REDACTED]",
277
+ value,
278
+ )
279
+ redacted = re.sub(
280
+ r"(?i)(x-api-key\s*:\s*)[^\s,;]+",
281
+ r"\1[REDACTED]",
282
+ redacted,
283
+ )
284
+ redacted = re.sub(
285
+ r"(?i)(api[_-]?key|token|secret|password|credential)\s*[=:]\s*[^\s,;]+",
286
+ r"\1=[REDACTED]",
287
+ redacted,
288
+ )
289
+ redacted = re.sub(
290
+ r"(?i)([?&](?:api[_-]?key|token|access[_-]?token|auth[_-]?token)=)[^&#\s]+",
291
+ r"\1[REDACTED]",
292
+ redacted,
293
+ )
294
+ redacted = re.sub(r"(https?://)[^/@\s:]+:[^/@\s]+@", r"\1[REDACTED]@", redacted)
295
+ for secret in _credential_values(source_env):
296
+ redacted = redacted.replace(secret, "[REDACTED]")
297
+ return redacted
298
+
299
+
300
+ def _extract_json_payload(raw: object) -> dict[str, Any]:
301
+ if isinstance(raw, GenerationOutput):
302
+ raw = raw.output
303
+ if isinstance(raw, dict):
304
+ return raw
305
+ if not isinstance(raw, str):
306
+ raise AutoTaskError("task_generation: generator did not return JSON")
307
+ text = raw.strip()
308
+ if text.startswith("```"):
309
+ text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE)
310
+ candidates = [text, *reversed(text.splitlines())]
311
+ for candidate in candidates:
312
+ try:
313
+ value = json.loads(candidate)
314
+ except json.JSONDecodeError:
315
+ continue
316
+ if isinstance(value, dict):
317
+ stack: list[object] = [value]
318
+ while stack:
319
+ nested = stack.pop()
320
+ if isinstance(nested, dict):
321
+ if isinstance(nested.get("tasks"), list):
322
+ return nested
323
+ stack.extend(nested.values())
324
+ elif isinstance(nested, list):
325
+ stack.extend(nested)
326
+ elif isinstance(nested, str):
327
+ try:
328
+ decoded = json.loads(nested)
329
+ except json.JSONDecodeError:
330
+ continue
331
+ stack.append(decoded)
332
+ raise AutoTaskError("task_generation: generator output was not a JSON task object")
333
+
334
+
335
+ def _validate_generated_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
336
+ tasks = payload.get("tasks")
337
+ if not isinstance(tasks, list) or not MIN_GENERATED_TASKS <= len(tasks) <= MAX_GENERATED_TASKS:
338
+ raise AutoTaskError(
339
+ f"task_generation: generated tasks must contain {MIN_GENERATED_TASKS}-{MAX_GENERATED_TASKS} tasks"
340
+ )
341
+ names: set[str] = set()
342
+ for index, task in enumerate(tasks):
343
+ if not isinstance(task, dict):
344
+ raise AutoTaskError(f"task_generation: tasks[{index}] must be a mapping")
345
+ if set(task) - {"name", "prompt", "workspace", "expect", "rubric"}:
346
+ raise AutoTaskError(f"task_generation: tasks[{index}] contains unsupported fields")
347
+ name = task.get("name")
348
+ if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}", name):
349
+ raise AutoTaskError(f"task_generation: tasks[{index}].name is invalid")
350
+ if name in names:
351
+ raise AutoTaskError(f"task_generation: duplicate task name: {name}")
352
+ names.add(name)
353
+ if not isinstance(task.get("prompt"), str) or not task["prompt"].strip():
354
+ raise AutoTaskError(f"task_generation: tasks[{index}].prompt is required")
355
+ if not isinstance(task.get("expect"), dict):
356
+ raise AutoTaskError(f"task_generation: tasks[{index}].expect is required")
357
+ if task.get("workspace") is not None and not isinstance(task["workspace"], str):
358
+ raise AutoTaskError(f"task_generation: tasks[{index}].workspace must be a path")
359
+ if task.get("rubric") is not None and (
360
+ not isinstance(task["rubric"], list)
361
+ or not all(isinstance(item, str) and item.strip() for item in task["rubric"])
362
+ ):
363
+ raise AutoTaskError(f"task_generation: tasks[{index}].rubric is invalid")
364
+ return tasks
365
+
366
+
367
+ def _reject_generated_bundled_collisions(tasks: list[dict[str, Any]]) -> None:
368
+ """Reject a generated descriptor that would be ambiguous to the normal catalogue."""
369
+ from scaffold.python.paths import get_tasks_dir
370
+ from scaffold.python.tasks import load_task
371
+
372
+ bundled: dict[str, str] = {}
373
+ for directory in get_tasks_dir().iterdir():
374
+ if (directory / "task.toml").is_file() and (directory / "instruction.md").is_file():
375
+ bundled[load_task(directory).name] = directory.name
376
+ for index, task in enumerate(tasks):
377
+ name = task["name"]
378
+ if name in bundled:
379
+ raise AutoTaskError(
380
+ f'generated.tasks[{index}].name conflicts with bundled task "{bundled[name]}": "{name}"'
381
+ )
382
+
383
+
384
+ def _manifest_source(
385
+ skill_root: Path,
386
+ skill_name: str,
387
+ profile: str,
388
+ generation_hash: str,
389
+ tasks: list[dict[str, Any]],
390
+ ) -> str:
391
+ return yaml.safe_dump(
392
+ {
393
+ "apiVersion": TASK_SCHEMA_VERSION,
394
+ "kind": "SkillEvalManifest",
395
+ "metadata": {
396
+ "name": skill_name,
397
+ "generationHash": generation_hash,
398
+ "generationFile": "generation.json",
399
+ },
400
+ "skill": {"name": skill_name, "source": str(skill_root), "profile": profile},
401
+ "evaluation": {"tasks": tasks},
402
+ },
403
+ sort_keys=False,
404
+ allow_unicode=True,
405
+ )
406
+
407
+
408
+ def _generation_prompt(snapshot: SkillSnapshot, *, profile: str, repair: str | None = None) -> str:
409
+ files = "\n\n".join(
410
+ f"FILE: {item.path}\nHASH: {item.content_hash}\n{item.content}" for item in snapshot.files
411
+ )
412
+ repair_text = f"\nRepair the previous output because: {repair}\n" if repair else ""
413
+ return f"""You are Comet's task-generator session. Generate only JSON and no Markdown.
414
+ Produce {MIN_GENERATED_TASKS}-{MAX_GENERATED_TASKS} adaptive Skill evaluation tasks for profile {profile!r}.
415
+ Each task must have a unique name, a concrete prompt, and at least one deterministic #250 expect
416
+ from files, contains, json, or commands. Rubric is optional. Do not invent source task packages.
417
+ Use only paths visible in the bounded Skill snapshot. Return exactly:
418
+ {{"tasks":[{{"name":"...","prompt":"...","expect":{{...}},"rubric":["..."]}}]}}
419
+ {repair_text}
420
+ Bounded Skill snapshot:
421
+ {files}
422
+ """
423
+
424
+
425
+ def _default_generate(
426
+ prompt: str,
427
+ *,
428
+ agent: AgentId,
429
+ model: str | None,
430
+ base_url: str | None = None,
431
+ environment: dict[str, str] | None = None,
432
+ ) -> GenerationOutput:
433
+ from scaffold.python.manifest_tasks import _generic_environment_dir
434
+ from scaffold.python.logging import extract_events, parse_output
435
+
436
+ with tempfile.TemporaryDirectory(prefix="comet-task-generator-") as directory:
437
+ workdir = Path(directory)
438
+ shutil.copytree(_generic_environment_dir(), workdir, dirs_exist_ok=True)
439
+ started = time.monotonic()
440
+ result = run_agent_in_docker(
441
+ workdir,
442
+ prompt,
443
+ agent=agent,
444
+ model=model,
445
+ base_url=base_url,
446
+ environment=environment,
447
+ timeout=300,
448
+ )
449
+ elapsed = time.monotonic() - started
450
+ if result.returncode != 0:
451
+ raise AutoTaskError(
452
+ f"task_generation: generator agent failed ({result.returncode}): "
453
+ f"{(result.stderr or result.stdout or '')[-1000:]}"
454
+ )
455
+ stdout = result.stdout or ""
456
+ events = extract_events(parse_output(stdout), agent=agent)
457
+ return GenerationOutput(
458
+ stdout,
459
+ {
460
+ "duration_seconds": events.get("duration_seconds") or elapsed,
461
+ "input_tokens": events.get("input_tokens"),
462
+ "output_tokens": events.get("output_tokens"),
463
+ "total_tokens": events.get("total_tokens"),
464
+ "total_cost_usd": events.get("total_cost_usd"),
465
+ "model_usage": events.get("model_usage") or {},
466
+ "model": model or "runtime-default",
467
+ "telemetry_status": (
468
+ "N/A" if not get_agent_adapter(agent).supports_telemetry else "available"
469
+ ),
470
+ },
471
+ )
472
+
473
+
474
+ def _manifest_hash_matches(manifest_path: Path, metadata: Mapping[str, Any]) -> bool:
475
+ expected = metadata.get("manifest_hash")
476
+ if not isinstance(expected, str):
477
+ return False
478
+ try:
479
+ return _sha256(manifest_path.read_bytes()) == expected
480
+ except OSError:
481
+ return False
482
+
483
+
484
+ def _load_cached(cache_dir: Path, generation_hash: str) -> GeneratedManifest | None:
485
+ manifest_path = cache_dir / "eval.yaml"
486
+ metadata_path = cache_dir / "generation.json"
487
+ if not manifest_path.is_file() or not metadata_path.is_file():
488
+ return None
489
+ try:
490
+ metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
491
+ if metadata.get("generation_hash") != generation_hash or not _manifest_hash_matches(
492
+ manifest_path, metadata
493
+ ):
494
+ return None
495
+ manifest = validate_generated_manifest(manifest_path)
496
+ _reject_generated_bundled_collisions([{"name": task.name} for task in manifest.tasks])
497
+ except (OSError, ValueError, json.JSONDecodeError, TypeError):
498
+ return None
499
+ return GeneratedManifest(manifest_path, metadata_path, generation_hash, reused=True)
500
+
501
+
502
+ @contextmanager
503
+ def _generation_lock(path: Path) -> Iterator[None]:
504
+ path.parent.mkdir(parents=True, exist_ok=True)
505
+ with path.open("a+b") as lock_file:
506
+ if os.name == "nt":
507
+ import msvcrt
508
+
509
+ if lock_file.seek(0, os.SEEK_END) == 0:
510
+ lock_file.write(b"0")
511
+ lock_file.flush()
512
+ lock_file.seek(0)
513
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
514
+ try:
515
+ yield
516
+ finally:
517
+ lock_file.seek(0)
518
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
519
+ else:
520
+ import fcntl
521
+
522
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
523
+ try:
524
+ yield
525
+ finally:
526
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
527
+
528
+
529
+ def _merge_generation_telemetry(attempts: list[dict[str, Any]]) -> dict[str, Any]:
530
+ def total(key: str) -> int | float | None:
531
+ values = [item[key] for item in attempts if isinstance(item.get(key), (int, float))]
532
+ return sum(values) if values else None
533
+
534
+ merged_models: dict[str, dict[str, int | float]] = {}
535
+ for attempt in attempts:
536
+ for model_name, usage in (attempt.get("model_usage") or {}).items():
537
+ if not isinstance(model_name, str) or not isinstance(usage, dict):
538
+ continue
539
+ target = merged_models.setdefault(model_name, {})
540
+ for key, value in usage.items():
541
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
542
+ target[key] = target.get(key, 0) + value
543
+ return {
544
+ "attempt_count": len(attempts),
545
+ "duration_seconds": total("duration_seconds"),
546
+ "input_tokens": total("input_tokens"),
547
+ "output_tokens": total("output_tokens"),
548
+ "total_tokens": total("total_tokens"),
549
+ "total_cost_usd": total("total_cost_usd"),
550
+ "model_usage": merged_models,
551
+ "telemetry_status": (
552
+ "N/A"
553
+ if any(item.get("telemetry_status") == "N/A" for item in attempts)
554
+ else "available"
555
+ if any(item.get("telemetry_status") == "available" for item in attempts)
556
+ else "N/A"
557
+ ),
558
+ "attempts": attempts,
559
+ }
560
+
561
+
562
+ def ensure_generated_manifest(
563
+ skill_path: Path | str,
564
+ project_root: Path | str,
565
+ *,
566
+ agent: object,
567
+ model: str | None,
568
+ profile: str,
569
+ interaction: dict[str, Any],
570
+ base_url: str | None = None,
571
+ environment: dict[str, str] | None = None,
572
+ collect_only: bool = False,
573
+ generate: Callable[[str], object] | None = None,
574
+ ) -> GeneratedManifest:
575
+ """Return a valid frozen generated manifest, or generate it once and cache it."""
576
+ selected_agent = validate_agent_id(agent, field="task-generator agent")
577
+ skill_root = _safe_skill_root(Path(skill_path))
578
+ snapshot = build_skill_snapshot(skill_root)
579
+ canonical_interaction = normalize_interaction(interaction)
580
+ generation_hash = _generation_hash(
581
+ snapshot,
582
+ agent=selected_agent,
583
+ model=model,
584
+ base_url=base_url,
585
+ profile=profile,
586
+ interaction=canonical_interaction,
587
+ )
588
+ cache_dir = _cache_location(Path(project_root), skill_root, generation_hash)
589
+ cached = _load_cached(cache_dir, generation_hash)
590
+ if cached:
591
+ return cached
592
+ if collect_only:
593
+ raise AutoTaskError(
594
+ "No cached generated eval manifest exists; run comet eval <skill> once to generate and execute tasks"
595
+ )
596
+
597
+ cache_dir.parent.mkdir(parents=True, exist_ok=True)
598
+ lock_path = managed_path_for_owner(
599
+ project_root, "locks", f"generated-{generation_hash}.lock"
600
+ )
601
+ with _generation_lock(lock_path):
602
+ cached = _load_cached(cache_dir, generation_hash)
603
+ if cached:
604
+ return cached
605
+
606
+ generator = generate or (
607
+ lambda prompt: _default_generate(
608
+ prompt,
609
+ agent=selected_agent,
610
+ model=model,
611
+ base_url=base_url,
612
+ environment=environment,
613
+ )
614
+ )
615
+ error: str | None = None
616
+ tasks: list[dict[str, Any]] | None = None
617
+ attempts: list[dict[str, Any]] = []
618
+ for attempt in range(2):
619
+ try:
620
+ generated = generator(_generation_prompt(snapshot, profile=profile, repair=error))
621
+ if isinstance(generated, GenerationOutput):
622
+ raw_output = generated.output
623
+ telemetry = dict(generated.telemetry)
624
+ else:
625
+ raw_output = generated
626
+ telemetry = {}
627
+ telemetry.setdefault("model", model or "runtime-default")
628
+ telemetry.setdefault(
629
+ "telemetry_status",
630
+ "N/A" if not get_agent_adapter(selected_agent).supports_telemetry else "available",
631
+ )
632
+ telemetry["attempt"] = attempt + 1
633
+ attempts.append(telemetry)
634
+ payload = _extract_json_payload(raw_output)
635
+ tasks = _validate_generated_payload(payload)
636
+ _reject_generated_bundled_collisions(tasks)
637
+ manifest_source = _manifest_source(
638
+ skill_root, skill_root.name, profile, generation_hash, tasks
639
+ )
640
+ with tempfile.NamedTemporaryFile(
641
+ mode="w", suffix=".yaml", encoding="utf-8", delete=False
642
+ ) as candidate:
643
+ candidate.write(manifest_source)
644
+ candidate_path = Path(candidate.name)
645
+ try:
646
+ load_eval_manifest(candidate_path)
647
+ finally:
648
+ candidate_path.unlink(missing_ok=True)
649
+ break
650
+ except (AutoTaskError, OSError, ValueError, yaml.YAMLError) as exc:
651
+ error = str(exc)
652
+ tasks = None
653
+ if tasks is None:
654
+ message = error or "task_generation: generated manifest is invalid"
655
+ report_path = _write_generation_failure_report(
656
+ Path(project_root),
657
+ attempts=attempts,
658
+ error=message,
659
+ environment=environment,
660
+ )
661
+ raise AutoTaskError(f"{message}\nPartial report: {report_path}")
662
+
663
+ manifest_source = _manifest_source(
664
+ skill_root, skill_root.name, profile, generation_hash, tasks
665
+ )
666
+ metadata = {
667
+ "schema": "comet.eval.generation.v1",
668
+ "generation_hash": generation_hash,
669
+ "generator_version": GENERATOR_VERSION,
670
+ "task_schema_version": TASK_SCHEMA_VERSION,
671
+ "skill_snapshot_hash": snapshot.content_hash,
672
+ "skill_path": str(skill_root),
673
+ "agent": selected_agent,
674
+ "model": model or "runtime-default",
675
+ "profile": profile,
676
+ "interaction": canonical_interaction,
677
+ "manifest_hash": _sha256(manifest_source),
678
+ "generation_overhead": _merge_generation_telemetry(attempts),
679
+ }
680
+ stage_dir = Path(tempfile.mkdtemp(dir=str(cache_dir.parent), prefix=f".{generation_hash}-"))
681
+ try:
682
+ _atomic_write_text(stage_dir / "eval.yaml", manifest_source)
683
+ _atomic_write_text(
684
+ stage_dir / "generation.json", json.dumps(_without_credentials(metadata), indent=2, sort_keys=True) + "\n"
685
+ )
686
+ if cache_dir.exists():
687
+ retired_dir = cache_dir.with_name(f".{cache_dir.name}.{time.time_ns()}.retired")
688
+ os.replace(cache_dir, retired_dir)
689
+ else:
690
+ retired_dir = None
691
+ os.replace(stage_dir, cache_dir)
692
+ stage_dir = None # type: ignore[assignment]
693
+ if retired_dir is not None:
694
+ shutil.rmtree(retired_dir, ignore_errors=True)
695
+ finally:
696
+ if stage_dir is not None:
697
+ shutil.rmtree(stage_dir, ignore_errors=True)
698
+ return GeneratedManifest(
699
+ cache_dir / "eval.yaml", cache_dir / "generation.json", generation_hash
700
+ )