@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -0,0 +1,779 @@
1
+ import { createHash } from 'crypto';
2
+ import os from 'os';
3
+ import { promises as fs } from 'fs';
4
+ import path from 'path';
5
+ import Ajv from 'ajv';
6
+ import { parse as parseToml } from 'smol-toml';
7
+ import { parse as parseYaml } from 'yaml';
8
+ import { isPathWithin } from './standalone-context.js';
9
+ const AGENT_IDS = ['claude-code', 'codex', 'qoder', 'codebuddy'];
10
+ const CUSTOM_AGENT_ID_RE = /^[a-z][a-z0-9-]{1,31}$/u;
11
+ const ENV_NAME_RE = /^[A-Z][A-Z0-9_]{1,63}$/u;
12
+ const SAFE_TOKEN_RE = /^[A-Za-z0-9._:@/+%-]{1,160}$/u;
13
+ const CUSTOM_CAPABILITIES = [
14
+ 'singleTurn',
15
+ 'resume',
16
+ 'structuredEvents',
17
+ 'telemetry',
18
+ 'skillInvocationEvidence',
19
+ ];
20
+ const BUILTIN_MODEL_ENVS = {
21
+ 'claude-code': ['BENCH_CC_MODEL', 'ANTHROPIC_MODEL', 'BENCH_MODEL'],
22
+ codex: ['BENCH_CODEX_MODEL', 'OPENAI_MODEL', 'CODEX_MODEL', 'BENCH_MODEL'],
23
+ qoder: ['BENCH_QODER_MODEL', 'QODER_MODEL', 'BENCH_MODEL'],
24
+ codebuddy: ['BENCH_CODEBUDDY_MODEL', 'CODEBUDDY_MODEL', 'BENCH_MODEL'],
25
+ };
26
+ const BUILTIN_BASE_URL_ENVS = {
27
+ 'claude-code': ['ANTHROPIC_BASE_URL', 'BENCH_BASE_URL'],
28
+ codex: ['OPENAI_BASE_URL', 'CODEX_BASE_URL', 'BENCH_BASE_URL'],
29
+ qoder: ['QODER_BASE_URL'],
30
+ codebuddy: ['CODEBUDDY_BASE_URL', 'BENCH_BASE_URL'],
31
+ };
32
+ const hash = (value) => `sha256:${createHash('sha256').update(value).digest('hex')}`;
33
+ const codePoint = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
34
+ const canonical = (value) => value === null || typeof value !== 'object'
35
+ ? JSON.stringify(value)
36
+ : Array.isArray(value)
37
+ ? `[${value.map(canonical).join(',')}]`
38
+ : `{${Object.keys(value)
39
+ .sort(codePoint)
40
+ .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`)
41
+ .join(',')}}`;
42
+ const taskName = (value, field) => {
43
+ if (typeof value !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/u.test(value))
44
+ throw new Error(`${field} must be a valid task name`);
45
+ return value;
46
+ };
47
+ function validateAgentId(value, field) {
48
+ if (typeof value !== 'string' ||
49
+ (!AGENT_IDS.includes(value) && !/^[a-z][a-z0-9-]{1,31}$/u.test(value)))
50
+ throw new Error(`Unsupported evaluation agent ${JSON.stringify(value)} in ${field}. Expected a built-in or installed custom adapter`);
51
+ return value;
52
+ }
53
+ /** Read only the explicitly installed adapter metadata needed by static collection. */
54
+ export async function loadInstalledCustomAgent(agentId) {
55
+ if (AGENT_IDS.includes(agentId))
56
+ return null;
57
+ if (!CUSTOM_AGENT_ID_RE.test(agentId))
58
+ return null;
59
+ const configuredRoot = process.env.COMET_EVAL_ADAPTERS_DIR;
60
+ const registry = path.resolve(configuredRoot || path.join(os.homedir(), '.comet', 'eval', 'adapters'));
61
+ let realRegistry;
62
+ try {
63
+ realRegistry = await fs.realpath(registry);
64
+ if (!(await fs.stat(realRegistry)).isDirectory())
65
+ throw new Error('not a directory');
66
+ }
67
+ catch {
68
+ throw new Error(`Custom evaluation agent is not installed: ${agentId}`);
69
+ }
70
+ const candidate = path.join(realRegistry, agentId);
71
+ let realCandidate;
72
+ try {
73
+ realCandidate = await fs.realpath(candidate);
74
+ if (!isPathWithin(realRegistry, realCandidate) ||
75
+ !(await fs.stat(realCandidate)).isDirectory()) {
76
+ throw new Error('invalid adapter directory', {
77
+ cause: new Error('adapter directory escapes registry or is not a directory'),
78
+ });
79
+ }
80
+ }
81
+ catch (error) {
82
+ if (error instanceof Error && error.message === 'invalid adapter directory')
83
+ throw error;
84
+ throw new Error(`Custom evaluation agent is not installed: ${agentId}`, { cause: error });
85
+ }
86
+ let manifestPath;
87
+ try {
88
+ manifestPath = await fs.realpath(path.join(realCandidate, 'adapter.yaml'));
89
+ if (!isPathWithin(realCandidate, manifestPath) || !(await fs.stat(manifestPath)).isFile()) {
90
+ throw new Error('invalid adapter manifest', {
91
+ cause: new Error('adapter manifest escapes directory or is not a file'),
92
+ });
93
+ }
94
+ }
95
+ catch (error) {
96
+ if (error instanceof Error && error.message === 'invalid adapter manifest')
97
+ throw error;
98
+ throw new Error(`Custom evaluation agent ${agentId} is missing adapter.yaml`, { cause: error });
99
+ }
100
+ let data;
101
+ try {
102
+ data = parseYaml(await fs.readFile(manifestPath, 'utf8'));
103
+ }
104
+ catch (cause) {
105
+ throw new Error(`Custom evaluation agent ${agentId} has invalid adapter.yaml`, { cause });
106
+ }
107
+ if (!data || typeof data !== 'object' || Array.isArray(data))
108
+ throw new Error(`Custom evaluation agent ${agentId} adapter.yaml must be a mapping`);
109
+ const adapter = data;
110
+ const metadata = adapter.metadata;
111
+ const allowed = new Set([
112
+ 'apiVersion',
113
+ 'kind',
114
+ 'metadata',
115
+ 'runtime',
116
+ 'credentials',
117
+ 'modelEnv',
118
+ 'baseUrlEnv',
119
+ 'capabilities',
120
+ ]);
121
+ const unknown = Object.keys(adapter).find((key) => !allowed.has(key));
122
+ if (unknown ||
123
+ adapter.apiVersion !== 'comet.eval.agent/v1alpha1' ||
124
+ adapter.kind !== 'EvalAgentAdapter' ||
125
+ !metadata ||
126
+ typeof metadata !== 'object' ||
127
+ Array.isArray(metadata) ||
128
+ metadata.id !== agentId) {
129
+ throw new Error(`Custom evaluation agent ${agentId} has invalid adapter metadata`);
130
+ }
131
+ if (!metadata ||
132
+ typeof metadata.version !== 'string' ||
133
+ !SAFE_TOKEN_RE.test(metadata.version)) {
134
+ throw new Error(`Custom evaluation agent ${agentId} has invalid metadata.version`);
135
+ }
136
+ const runtime = adapter.runtime;
137
+ if (!runtime || typeof runtime !== 'object' || Array.isArray(runtime))
138
+ throw new Error(`Custom evaluation agent ${agentId} runtime must be a mapping`);
139
+ const runtimeData = runtime;
140
+ if (Object.keys(runtimeData).some((key) => !['executable', 'install'].includes(key)))
141
+ throw new Error(`Custom evaluation agent ${agentId} runtime has unknown fields`);
142
+ if (typeof runtimeData.executable !== 'string' || !SAFE_TOKEN_RE.test(runtimeData.executable))
143
+ throw new Error(`Custom evaluation agent ${agentId} runtime.executable is invalid`);
144
+ const install = runtimeData.install ?? { kind: 'none' };
145
+ if (!install || typeof install !== 'object' || Array.isArray(install))
146
+ throw new Error(`Custom evaluation agent ${agentId} runtime.install must be a mapping`);
147
+ const installData = install;
148
+ if (Object.keys(installData).some((key) => !['kind', 'package', 'version'].includes(key)))
149
+ throw new Error(`Custom evaluation agent ${agentId} runtime.install has unknown fields`);
150
+ const installKind = installData.kind ?? 'none';
151
+ if (!['none', 'npm', 'pip'].includes(String(installKind)))
152
+ throw new Error(`Custom evaluation agent ${agentId} runtime.install.kind is invalid`);
153
+ if (installKind === 'none') {
154
+ if (installData.package !== undefined || installData.version !== undefined)
155
+ throw new Error(`Custom evaluation agent ${agentId} runtime.install package requires npm or pip`);
156
+ }
157
+ else if (typeof installData.package !== 'string' ||
158
+ !SAFE_TOKEN_RE.test(installData.package) ||
159
+ (installData.version !== undefined &&
160
+ (typeof installData.version !== 'string' || !SAFE_TOKEN_RE.test(installData.version)))) {
161
+ throw new Error(`Custom evaluation agent ${agentId} runtime.install package is invalid`);
162
+ }
163
+ const credentials = adapter.credentials ?? [];
164
+ if (!Array.isArray(credentials) ||
165
+ credentials.length > 2 ||
166
+ credentials.some((value) => typeof value !== 'string' || !ENV_NAME_RE.test(value)) ||
167
+ new Set(credentials).size !== credentials.length)
168
+ throw new Error(`Custom evaluation agent ${agentId} credentials are invalid`);
169
+ const capabilities = adapter.capabilities;
170
+ if (!capabilities ||
171
+ typeof capabilities !== 'object' ||
172
+ Array.isArray(capabilities) ||
173
+ Object.keys(capabilities).some((key) => !CUSTOM_CAPABILITIES.includes(key)) ||
174
+ CUSTOM_CAPABILITIES.some((key) => !Object.hasOwn(capabilities, key) ||
175
+ typeof capabilities[key] !== 'boolean'))
176
+ throw new Error(`Custom evaluation agent ${agentId} capabilities are invalid`);
177
+ const env = (name) => {
178
+ const value = adapter[name];
179
+ if (value === undefined)
180
+ return undefined;
181
+ if (typeof value !== 'string' || !ENV_NAME_RE.test(value))
182
+ throw new Error(`Custom evaluation agent ${agentId} ${name} must be an environment variable name`);
183
+ return value;
184
+ };
185
+ return { id: agentId, modelEnv: env('modelEnv'), baseUrlEnv: env('baseUrlEnv') };
186
+ }
187
+ function validateModel(value, field, required = false) {
188
+ if (value === undefined) {
189
+ if (required)
190
+ throw new Error(`${field}: is required`);
191
+ return;
192
+ }
193
+ if (typeof value !== 'string' || !value.trim())
194
+ throw new Error(`${field} must be a non-empty string`);
195
+ }
196
+ function validateBaseUrl(data, field) {
197
+ if (Object.hasOwn(data, 'baseUrl') && Object.hasOwn(data, 'base_url'))
198
+ throw new Error(`${field}.baseUrl and ${field}.base_url cannot both be set`);
199
+ const value = data.baseUrl ?? data.base_url;
200
+ if (value === undefined)
201
+ return;
202
+ try {
203
+ if (typeof value !== 'string' || value.trim() !== value)
204
+ throw new Error('invalid');
205
+ const parsed = new URL(value);
206
+ const port = parsed.port ? Number(parsed.port) : undefined;
207
+ if (!['http:', 'https:'].includes(parsed.protocol) ||
208
+ !parsed.hostname ||
209
+ (port !== undefined && (!Number.isInteger(port) || port < 1 || port > 65535)) ||
210
+ parsed.username ||
211
+ parsed.password ||
212
+ parsed.search ||
213
+ parsed.hash)
214
+ throw new Error('invalid');
215
+ }
216
+ catch {
217
+ throw new Error(`${field}.baseUrl: must be a valid absolute http(s) URL`);
218
+ }
219
+ }
220
+ function environmentValue(keys) {
221
+ return keys.map((key) => process.env[key]?.trim()).find((value) => value);
222
+ }
223
+ function judgeFlagEnabled() {
224
+ return ['1', 'true', 'yes', 'on'].includes((process.env.BENCH_LLM_JUDGE ?? '').trim().toLowerCase());
225
+ }
226
+ async function validateExecutionConfig(options, manifest) {
227
+ if (options.agent !== undefined)
228
+ validateAgentId(options.agent, 'CLI evaluation agent');
229
+ validateModel(options.model, 'CLI evaluation model');
230
+ if (options.baseUrl !== undefined)
231
+ validateBaseUrl({ baseUrl: options.baseUrl }, 'CLI evaluation');
232
+ if (options.judgeAgent !== undefined)
233
+ validateAgentId(options.judgeAgent, 'CLI Judge evaluation agent');
234
+ validateModel(options.judgeModel, 'CLI Judge evaluation model');
235
+ if (options.judgeBaseUrl !== undefined)
236
+ validateBaseUrl({ baseUrl: options.judgeBaseUrl }, 'CLI Judge evaluation');
237
+ const execution = manifest?.execution ?? {};
238
+ if (execution.agent !== undefined)
239
+ validateAgentId(execution.agent, 'manifest execution.agent');
240
+ validateModel(execution.model, 'execution.model');
241
+ validateBaseUrl(execution, 'execution');
242
+ if (manifest && Object.hasOwn(manifest, 'judge')) {
243
+ const judge = manifest.judge;
244
+ if (judge.agent !== undefined)
245
+ validateAgentId(judge.agent, 'manifest judge.agent');
246
+ validateModel(judge.model, 'judge.model', true);
247
+ validateBaseUrl(judge, 'judge');
248
+ }
249
+ const configuredAgent = options.agent ??
250
+ (typeof execution.agent === 'string' ? execution.agent : undefined) ??
251
+ (typeof process.env.BENCH_EVAL_AGENT === 'string' && process.env.BENCH_EVAL_AGENT.trim()
252
+ ? process.env.BENCH_EVAL_AGENT.trim()
253
+ : undefined) ??
254
+ 'claude-code';
255
+ const adapter = await loadInstalledCustomAgent(configuredAgent);
256
+ const configuredBaseUrl = options.baseUrl ??
257
+ (typeof execution.baseUrl === 'string'
258
+ ? execution.baseUrl
259
+ : typeof execution.base_url === 'string'
260
+ ? execution.base_url
261
+ : environmentValue(adapter?.baseUrlEnv
262
+ ? [adapter.baseUrlEnv]
263
+ : (BUILTIN_BASE_URL_ENVS[configuredAgent] ?? [])));
264
+ if (configuredBaseUrl !== undefined)
265
+ validateBaseUrl({ baseUrl: configuredBaseUrl }, 'execution');
266
+ if (adapter?.id && configuredBaseUrl && !adapter.baseUrlEnv) {
267
+ throw new Error(`custom Agent ${adapter.id} must declare baseUrlEnv before execution.baseUrl can be used`);
268
+ }
269
+ const judge = manifest?.judge;
270
+ const configuredJudge = options.judgeAgent ??
271
+ (typeof judge?.agent === 'string' ? judge.agent : undefined) ??
272
+ (judge ||
273
+ options.judgeModel !== undefined ||
274
+ options.judgeBaseUrl !== undefined ||
275
+ judgeFlagEnabled()
276
+ ? configuredAgent
277
+ : undefined);
278
+ if (configuredJudge)
279
+ await loadInstalledCustomAgent(configuredJudge);
280
+ const configuredJudgeModel = options.judgeModel ??
281
+ (typeof judge?.model === 'string' ? judge.model : undefined) ??
282
+ environmentValue(['BENCH_JUDGE_MODEL']);
283
+ const judgeEnabled = Boolean(configuredJudge);
284
+ if (judgeEnabled && !configuredJudgeModel) {
285
+ throw new Error(judgeFlagEnabled()
286
+ ? 'BENCH_JUDGE_MODEL is required when BENCH_LLM_JUDGE=1'
287
+ : 'judge.model: is required when LLM-as-Judge is enabled');
288
+ }
289
+ const configuredJudgeBaseUrl = options.judgeBaseUrl ??
290
+ (typeof judge?.baseUrl === 'string'
291
+ ? judge.baseUrl
292
+ : typeof judge?.base_url === 'string'
293
+ ? judge.base_url
294
+ : environmentValue(['BENCH_JUDGE_BASE_URL']));
295
+ if (configuredJudgeBaseUrl !== undefined)
296
+ validateBaseUrl({ baseUrl: configuredJudgeBaseUrl }, 'judge');
297
+ return { agent: configuredAgent, adapter };
298
+ }
299
+ async function file(value) {
300
+ try {
301
+ return (await fs.stat(value)).isFile();
302
+ }
303
+ catch {
304
+ return false;
305
+ }
306
+ }
307
+ function yamlPath(pointer) {
308
+ return !pointer
309
+ ? 'manifest'
310
+ : pointer
311
+ .split('/')
312
+ .slice(1)
313
+ .map((part) => part.replaceAll('~1', '/').replaceAll('~0', '~'))
314
+ .reduce((result, part) => (/^\d+$/u.test(part) ? `${result}[${part}]` : `${result}.${part}`), '')
315
+ .replace(/^\./u, '');
316
+ }
317
+ const ALIAS_PAIRS = [
318
+ ['draftHash', 'draft_hash'],
319
+ ['generationHash', 'generation_hash'],
320
+ ['generationFile', 'generation_file'],
321
+ ['expectedNodeOrder', 'expected_node_order'],
322
+ ['recommendedTasks', 'recommended_tasks'],
323
+ ['baselineTreatments', 'baseline_treatments'],
324
+ ['qualityGates', 'quality_gates'],
325
+ ['requiredOutputSchemas', 'required_output_schemas'],
326
+ ['expectedEvidence', 'expected_evidence'],
327
+ ['requiredSkills', 'required_skills'],
328
+ ['expectedArtifacts', 'expected_artifacts'],
329
+ ['generatedNodeSkills', 'generated_node_skills'],
330
+ ['routeConformance', 'route_conformance'],
331
+ ['maxTurns', 'max_turns'],
332
+ ['simulatorPrompt', 'simulator_prompt'],
333
+ ['freshResumeMarker', 'fresh_resume_marker'],
334
+ ['baseUrl', 'base_url'],
335
+ ];
336
+ function rejectAliasConflicts(value, prefix = '') {
337
+ if (!value || typeof value !== 'object')
338
+ return;
339
+ if (Array.isArray(value)) {
340
+ value.forEach((item, index) => rejectAliasConflicts(item, `${prefix}[${index}]`));
341
+ return;
342
+ }
343
+ const data = value;
344
+ for (const [camel, snake] of ALIAS_PAIRS) {
345
+ if (Object.hasOwn(data, camel) && Object.hasOwn(data, snake)) {
346
+ const field = prefix ? `${prefix}.` : '';
347
+ throw new Error(`${field}${camel} and ${field}${snake} cannot both be set`);
348
+ }
349
+ }
350
+ for (const [key, child] of Object.entries(data)) {
351
+ rejectAliasConflicts(child, prefix ? `${prefix}.${key}` : key);
352
+ }
353
+ }
354
+ function artifact(value, field) {
355
+ if (typeof value !== 'string' || !value.trim())
356
+ throw new Error(`${field} must contain non-empty relative paths`);
357
+ if (path.isAbsolute(value.replaceAll('\\', '/')) ||
358
+ value.replaceAll('\\', '/').split('/').includes('..'))
359
+ throw new Error(`${field} must stay within the task workspace: ${JSON.stringify(value)}`);
360
+ }
361
+ export function validateInlineTask(task, index) {
362
+ const taskLabel = typeof task.name === 'string' ? `task "${task.name}": ` : '';
363
+ const prefix = `${taskLabel}evaluation.tasks[${index}]`;
364
+ taskName(task.name, `${prefix}.name`);
365
+ if (typeof task.prompt !== 'string' || !task.prompt.trim())
366
+ throw new Error(`${prefix}.prompt is required`);
367
+ const expect = task.expect;
368
+ if (!expect || typeof expect !== 'object' || Array.isArray(expect))
369
+ throw new Error(`${prefix}.expect must contain at least one deterministic expect`);
370
+ const data = expect;
371
+ const allowed = new Set(['files', 'contains', 'json', 'commands']);
372
+ const unknown = Object.keys(data).find((key) => !allowed.has(key));
373
+ if (unknown)
374
+ throw new Error(`${prefix}.expect.${unknown}: unknown field`);
375
+ if (!Object.values(data).some((value) => Array.isArray(value)
376
+ ? value.length
377
+ : value && typeof value === 'object' && Object.keys(value).length))
378
+ throw new Error(`${prefix}.expect must contain at least one deterministic expect`);
379
+ if (task.rubric !== undefined &&
380
+ (!Array.isArray(task.rubric) ||
381
+ task.rubric.some((item) => typeof item !== 'string' || !item.trim())))
382
+ throw new Error(`${prefix}.rubric must be a list of non-empty strings`);
383
+ const files = data.files ?? [];
384
+ if (!Array.isArray(files) || files.some((item) => typeof item !== 'string'))
385
+ throw new Error(`${prefix}.expect.files must be a list of strings`);
386
+ files.forEach((item) => artifact(item, `${prefix}.expect.files`));
387
+ const contains = data.contains ?? {};
388
+ if (!contains || typeof contains !== 'object' || Array.isArray(contains))
389
+ throw new Error(`${prefix}.expect.contains must be a mapping`);
390
+ for (const [name, values] of Object.entries(contains)) {
391
+ artifact(name, `${prefix}.expect.contains`);
392
+ if (!Array.isArray(values) || values.some((item) => typeof item !== 'string'))
393
+ throw new Error(`${prefix}.expect.contains.${name} must be a list of strings`);
394
+ }
395
+ const json = data.json ?? [];
396
+ if (!Array.isArray(json))
397
+ throw new Error(`${prefix}.expect.json must be a list of mappings`);
398
+ for (const [i, item] of json.entries()) {
399
+ if (!item || typeof item !== 'object')
400
+ throw new Error(`${prefix}.expect.json must be a list of mappings`);
401
+ const check = item;
402
+ artifact(check.file, `${prefix}.expect.json[${i}].file`);
403
+ if (typeof check.path !== 'string' || !check.path.startsWith('$'))
404
+ throw new Error(`${prefix}.expect.json[${i}].path must start with "$"`);
405
+ if (!Object.hasOwn(check, 'equals'))
406
+ throw new Error(`${prefix}.expect.json[${i}].equals is required`);
407
+ }
408
+ const commands = data.commands ?? [];
409
+ if (!Array.isArray(commands))
410
+ throw new Error(`${prefix}.expect.commands must be a list of mappings`);
411
+ for (const [i, item] of commands.entries()) {
412
+ if (!item || typeof item !== 'object')
413
+ throw new Error(`${prefix}.expect.commands must be a list of mappings`);
414
+ const command = item;
415
+ if (typeof command.run !== 'string' || !command.run.trim())
416
+ throw new Error(`${prefix}.expect.commands[${i}].run is required`);
417
+ const timeout = command.timeout ?? 120;
418
+ if (typeof timeout !== 'number' || !Number.isInteger(timeout) || timeout < 1 || timeout > 3600)
419
+ throw new Error(`${prefix}.expect.commands[${i}].timeout must be 1..3600`);
420
+ }
421
+ }
422
+ async function packagePath(root, value, field) {
423
+ const normalized = value.replaceAll('\\', '/');
424
+ if (!normalized.trim() || path.isAbsolute(normalized) || normalized.split('/').includes('..'))
425
+ throw new Error(`${field} must stay within the Skill package: ${JSON.stringify(value)}`);
426
+ const realRoot = await fs.realpath(root);
427
+ let result;
428
+ try {
429
+ result = await fs.realpath(path.resolve(realRoot, normalized));
430
+ }
431
+ catch {
432
+ throw new Error(`${field} does not exist: ${JSON.stringify(value)}`);
433
+ }
434
+ if (!isPathWithin(realRoot, result))
435
+ throw new Error(`${field} must stay within the Skill package: ${JSON.stringify(value)}`);
436
+ const walk = async (dir) => {
437
+ for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
438
+ const child = path.join(dir, entry.name);
439
+ if (entry.isSymbolicLink()) {
440
+ if (!isPathWithin(realRoot, await fs.realpath(child)))
441
+ throw new Error(`${field} contains a symlink outside the Skill package: ${entry.name}`);
442
+ }
443
+ else if (entry.isDirectory())
444
+ await walk(child);
445
+ }
446
+ };
447
+ if ((await fs.stat(result)).isDirectory())
448
+ await walk(result);
449
+ return result;
450
+ }
451
+ function tomlSection(parsed, name, field) {
452
+ const value = parsed[name];
453
+ if (value === undefined)
454
+ return {};
455
+ if (!value || typeof value !== 'object' || Array.isArray(value))
456
+ throw new Error(`${field}.${name} must be a mapping`);
457
+ return value;
458
+ }
459
+ async function sourceTaskName(root, fallback, field) {
460
+ let parsed;
461
+ try {
462
+ parsed = parseToml(await fs.readFile(path.join(root, 'task.toml'), 'utf8'));
463
+ }
464
+ catch (cause) {
465
+ throw new Error(`${field}.task.toml is invalid TOML`, { cause });
466
+ }
467
+ const metadata = tomlSection(parsed, 'metadata', field);
468
+ const template = tomlSection(parsed, 'template', field);
469
+ const environment = tomlSection(parsed, 'environment', field);
470
+ const validation = tomlSection(parsed, 'validation', field);
471
+ const evaluation = tomlSection(parsed, 'evaluation', field);
472
+ const interaction = tomlSection(parsed, 'interaction', field);
473
+ const setup = tomlSection(parsed, 'setup', field);
474
+ const nativeTerminal = evaluation.native_terminal ?? 'archive';
475
+ if (!['archive', 'active', 'active-blocked'].includes(String(nativeTerminal)))
476
+ throw new Error(`${field}.evaluation.native_terminal: ${String(nativeTerminal)} is invalid`);
477
+ const interactionMode = interaction.mode ?? 'none';
478
+ if (!['none', 'auto_user'].includes(String(interactionMode)))
479
+ throw new Error(`${field}.interaction.mode: ${String(interactionMode)} is invalid`);
480
+ const maxTurns = interaction.max_turns ?? 12;
481
+ if (typeof maxTurns !== 'number' || !Number.isInteger(maxTurns) || maxTurns < 1)
482
+ throw new Error(`${field}.interaction.max_turns must be a positive integer`);
483
+ if (validation.timeout !== undefined) {
484
+ if (typeof validation.timeout !== 'number' ||
485
+ !Number.isInteger(validation.timeout) ||
486
+ validation.timeout < 1 ||
487
+ validation.timeout > 3600)
488
+ throw new Error(`${field}.validation.timeout must be 1..3600`);
489
+ }
490
+ if (template.required !== undefined && !Array.isArray(template.required))
491
+ throw new Error(`${field}.template.required must be a list`);
492
+ if (environment.dockerfile !== undefined && typeof environment.dockerfile !== 'string')
493
+ throw new Error(`${field}.environment.dockerfile must be a string`);
494
+ if (setup.data !== undefined && !Array.isArray(setup.data))
495
+ throw new Error(`${field}.setup.data must be a list`);
496
+ return taskName(metadata.name ?? fallback, field);
497
+ }
498
+ async function snapshot(root) {
499
+ const realRoot = await fs.realpath(root);
500
+ const files = new Map();
501
+ let total = 0;
502
+ const add = async (candidate) => {
503
+ if (files.size >= 128 || !(await file(candidate)))
504
+ return;
505
+ const resolved = await fs.realpath(candidate);
506
+ if (!isPathWithin(realRoot, resolved))
507
+ return;
508
+ const relative = path.relative(realRoot, resolved).replaceAll('\\', '/');
509
+ if (files.has(relative))
510
+ return;
511
+ const content = await fs.readFile(resolved, 'utf8');
512
+ if (total + Buffer.byteLength(content) > 512 * 1024)
513
+ return;
514
+ total += Buffer.byteLength(content);
515
+ files.set(relative, { path: relative, hash: hash(content) });
516
+ };
517
+ const skill = path.join(realRoot, 'SKILL.md');
518
+ await add(skill);
519
+ for (const match of (await fs.readFile(skill, 'utf8')).matchAll(/[`"']([^`"']+)[`"']/gu))
520
+ await add(path.join(realRoot, match[1].replaceAll('\\', '/')));
521
+ for (const name of ['scripts', 'references', 'reference', 'examples', 'templates']) {
522
+ const walk = async (dir) => {
523
+ for (const entry of (await fs.readdir(dir, { withFileTypes: true })).sort((a, b) => codePoint(a.name, b.name))) {
524
+ const child = path.join(dir, entry.name);
525
+ if (entry.isSymbolicLink())
526
+ continue;
527
+ if (entry.isDirectory())
528
+ await walk(child);
529
+ else
530
+ await add(child);
531
+ }
532
+ };
533
+ try {
534
+ await walk(path.join(realRoot, name));
535
+ }
536
+ catch {
537
+ /* optional */
538
+ }
539
+ }
540
+ return hash(canonical([...files.values()].sort((a, b) => codePoint(a.path, b.path))));
541
+ }
542
+ async function managedArtifactPath(context, managedRoot, ...parts) {
543
+ const ownerRoot = await fs.realpath(context.artifactOwnerRoot);
544
+ const artifactRoot = path.resolve(context.artifactRoot);
545
+ const resolvedArtifactRoot = await fs.realpath(artifactRoot).catch(() => artifactRoot);
546
+ if (!isPathWithin(ownerRoot, resolvedArtifactRoot))
547
+ throw new Error('Eval artifact root must stay within its owner root');
548
+ let current = artifactRoot;
549
+ for (const segment of [managedRoot, ...parts]) {
550
+ if (!segment || path.isAbsolute(segment) || segment === '.' || segment === '..')
551
+ throw new Error('Eval managed path components must be relative names');
552
+ current = path.join(current, segment);
553
+ try {
554
+ const resolved = await fs.realpath(current);
555
+ if (!isPathWithin(ownerRoot, resolved) || !isPathWithin(resolvedArtifactRoot, resolved))
556
+ throw new Error('Eval managed path must stay within its owner-local artifact root');
557
+ }
558
+ catch (error) {
559
+ if (error instanceof Error &&
560
+ error.message === 'Eval managed path must stay within its owner-local artifact root')
561
+ throw error;
562
+ if (error.code !== 'ENOENT')
563
+ throw error;
564
+ }
565
+ }
566
+ return current;
567
+ }
568
+ function interaction(manifest) {
569
+ const data = manifest?.interaction &&
570
+ typeof manifest.interaction === 'object' &&
571
+ !Array.isArray(manifest.interaction)
572
+ ? manifest.interaction
573
+ : {};
574
+ return {
575
+ mode: data.mode ?? 'none',
576
+ max_turns: Number(data.maxTurns ?? data.max_turns ?? 12),
577
+ simulator_prompt: data.simulatorPrompt || data.simulator_prompt || null,
578
+ decision_patterns: data.decisionPatterns ?? [],
579
+ decision_reply: data.decisionReply ?? null,
580
+ decision_replies: data.decisionReplies ?? [],
581
+ continue_prompt: data.continuePrompt ?? 'Please continue with the next phase of the workflow.',
582
+ fresh_resume_marker: data.freshResumeMarker || data.fresh_resume_marker || null,
583
+ };
584
+ }
585
+ export async function collectStandaloneTasks(options, context, packageRoot) {
586
+ let manifest;
587
+ let evaluation = {};
588
+ const validate = async (raw) => {
589
+ const value = parseYaml(raw);
590
+ if (!value || typeof value !== 'object' || Array.isArray(value))
591
+ throw new Error('Eval manifest must be a mapping');
592
+ rejectAliasConflicts(value);
593
+ const schema = JSON.parse(await fs.readFile(path.join(packageRoot, 'eval', 'schemas', 'comet.eval', 'v1alpha1.schema.json'), 'utf8'));
594
+ delete schema.$schema;
595
+ const Constructor = Ajv;
596
+ const validator = new Constructor({ allErrors: true, strict: false }).compile(schema);
597
+ if (!validator(value)) {
598
+ const error = validator.errors?.[0];
599
+ const field = error?.params
600
+ ?.additionalProperty;
601
+ const missing = error?.params?.missingProperty;
602
+ const at = yamlPath(error?.instancePath);
603
+ const taskMatch = at.match(/^evaluation\.tasks\[(\d+)\](.*)$/u);
604
+ const taskValue = taskMatch
605
+ ? value.evaluation?.tasks
606
+ : undefined;
607
+ const taskNameValue = taskMatch && Array.isArray(taskValue)
608
+ ? taskValue[Number(taskMatch[1])]?.name
609
+ : undefined;
610
+ const diagnosticAt = taskMatch && typeof taskNameValue === 'string' ? `task "${taskNameValue}": ${at}` : at;
611
+ throw new Error(error?.keyword === 'additionalProperties' && field
612
+ ? `${diagnosticAt === 'manifest' ? field : `${diagnosticAt}.${field}`}: unknown field`
613
+ : error?.keyword === 'required' && missing
614
+ ? `${diagnosticAt === 'manifest' ? missing : `${diagnosticAt}.${missing}`}: is required`
615
+ : error?.keyword === 'pattern' && /(?:baseUrl|base_url)$/u.test(at)
616
+ ? `${diagnosticAt}: must be a valid absolute http(s) URL`
617
+ : `${diagnosticAt}: ${error?.message ?? 'invalid manifest'}`);
618
+ }
619
+ return value;
620
+ };
621
+ if (context.manifestPath) {
622
+ manifest = await validate(await fs.readFile(context.manifestPath, 'utf8'));
623
+ evaluation = manifest.evaluation ?? {};
624
+ }
625
+ const executionConfig = await validateExecutionConfig(options, manifest);
626
+ const bundled = new Set();
627
+ const bundles = new Map();
628
+ for (const entry of await fs.readdir(path.join(packageRoot, 'eval', 'local', 'tasks'), {
629
+ withFileTypes: true,
630
+ })) {
631
+ const root = path.join(packageRoot, 'eval', 'local', 'tasks', entry.name);
632
+ if (entry.isDirectory() &&
633
+ (await file(path.join(root, 'task.toml'))) &&
634
+ (await file(path.join(root, 'instruction.md')))) {
635
+ const name = await sourceTaskName(root, entry.name, `bundled task ${entry.name}`);
636
+ if (bundled.has(name))
637
+ throw new Error(`bundled task name duplicates another bundle: ${name}`);
638
+ bundled.add(name);
639
+ bundles.set(name, entry.name);
640
+ }
641
+ }
642
+ const authored = Array.isArray(evaluation.tasks) ? evaluation.tasks : [];
643
+ const names = [];
644
+ for (const [i, item] of authored.entries()) {
645
+ if (!item || typeof item !== 'object' || Array.isArray(item))
646
+ throw new Error(`evaluation.tasks[${i}] must be a mapping`);
647
+ const task = item;
648
+ if (typeof task.source === 'string') {
649
+ if (Object.keys(task).some((key) => key !== 'source' && key !== 'name'))
650
+ throw new Error(`evaluation.tasks[${i}]: source tasks cannot define inline fields`);
651
+ const source = await packagePath(context.skillRoot, task.source, `evaluation.tasks[${i}].source`);
652
+ if (!(await file(path.join(source, 'task.toml'))) ||
653
+ !(await file(path.join(source, 'instruction.md'))))
654
+ throw new Error(`evaluation.tasks[${i}].source must point to a task package with task.toml and instruction.md`);
655
+ const canonicalSourceName = await sourceTaskName(source, path.basename(source), `evaluation.tasks[${i}].source`);
656
+ names.push(task.name === undefined
657
+ ? canonicalSourceName
658
+ : taskName(task.name, `evaluation.tasks[${i}].name`));
659
+ }
660
+ else {
661
+ validateInlineTask(task, i);
662
+ names.push(task.name);
663
+ }
664
+ if (typeof task.workspace === 'string')
665
+ await packagePath(context.skillRoot, task.workspace, `evaluation.tasks[${i}].workspace`);
666
+ }
667
+ const recommended = Array.isArray(evaluation.recommendedTasks)
668
+ ? evaluation.recommendedTasks
669
+ : Array.isArray(evaluation.recommended_tasks)
670
+ ? evaluation.recommended_tasks
671
+ : [];
672
+ const rec = recommended.map((item, i) => taskName(item, `evaluation.recommendedTasks[${i}]`));
673
+ for (const [i, name] of names.entries()) {
674
+ const first = names.indexOf(name);
675
+ if (first !== i)
676
+ throw new Error(`evaluation.tasks[${i}].name duplicates evaluation.tasks[${first}].name: "${name}"`);
677
+ if (bundled.has(name))
678
+ throw new Error(`evaluation.tasks[${i}].name conflicts with bundled task "${bundles.get(name)}": "${name}"`);
679
+ }
680
+ for (const [i, name] of rec.entries()) {
681
+ const first = rec.indexOf(name);
682
+ if (first !== i)
683
+ throw new Error(`evaluation.recommendedTasks[${i}] duplicates evaluation.recommendedTasks[${first}]: "${name}"`);
684
+ if (!bundled.has(name))
685
+ throw new Error(`evaluation.recommendedTasks[${i}]: unknown bundled task "${name}"`);
686
+ if (names.includes(name))
687
+ throw new Error(`evaluation.tasks[${names.indexOf(name)}].name conflicts with evaluation.recommendedTasks[${i}]: "${name}"`);
688
+ }
689
+ if (options.task) {
690
+ if (!names.includes(options.task) && !bundled.has(options.task))
691
+ throw new Error(`Task not found: ${options.task}`);
692
+ return ['Tasks: explicit', `- ${options.task}`];
693
+ }
694
+ if (options.quick)
695
+ return ['Tasks: quick', '- generic-skill-smoke'];
696
+ if (names.length)
697
+ return ['Tasks: authored', ...names.map((name) => `- ${name}`)];
698
+ if (rec.length)
699
+ return ['Tasks: recommended', ...rec.map((name) => `- ${name}`)];
700
+ try {
701
+ const agent = executionConfig.agent;
702
+ const profile = options.profile ??
703
+ manifest?.skill?.profile ??
704
+ 'generic';
705
+ const execution = manifest?.execution ?? {};
706
+ const configuredModel = execution.model;
707
+ const customModel = executionConfig.adapter?.modelEnv
708
+ ? process.env[executionConfig.adapter.modelEnv]?.trim() || undefined
709
+ : undefined;
710
+ const model = options.model ??
711
+ (typeof configuredModel === 'string' && configuredModel.trim()
712
+ ? configuredModel
713
+ : undefined) ??
714
+ customModel ??
715
+ environmentValue(BUILTIN_MODEL_ENVS[agent] ?? []) ??
716
+ 'runtime-default';
717
+ const customBaseUrl = executionConfig.adapter?.baseUrlEnv
718
+ ? process.env[executionConfig.adapter.baseUrlEnv]?.trim() || undefined
719
+ : undefined;
720
+ const baseUrl = options.baseUrl ??
721
+ (typeof execution.baseUrl === 'string'
722
+ ? execution.baseUrl
723
+ : typeof execution.base_url === 'string'
724
+ ? execution.base_url
725
+ : (customBaseUrl ??
726
+ environmentValue(BUILTIN_BASE_URL_ENVS[agent] ?? []) ??
727
+ 'runtime-default'));
728
+ const generation = createHash('sha256')
729
+ .update(canonical({
730
+ snapshot_hash: await snapshot(context.skillRoot),
731
+ agent,
732
+ model,
733
+ base_url: baseUrl,
734
+ profile,
735
+ interaction: interaction(manifest),
736
+ generator_version: 'comet-auto-task-generator.v1',
737
+ task_schema_version: 'comet.eval/v1alpha1',
738
+ }))
739
+ .digest('hex');
740
+ const safe = path
741
+ .basename(context.skillRoot)
742
+ .replace(/[^A-Za-z0-9._-]+/gu, '-')
743
+ .replace(/^-+|-+$/gu, '') || 'skill';
744
+ const metadataPath = await managedArtifactPath(context, 'generated', safe, generation, 'generation.json');
745
+ const manifestPath = await managedArtifactPath(context, 'generated', safe, generation, 'eval.yaml');
746
+ const [meta, raw] = await Promise.all([
747
+ fs.readFile(metadataPath, 'utf8'),
748
+ fs.readFile(manifestPath, 'utf8'),
749
+ ]);
750
+ if (JSON.parse(meta).generation_hash !== generation ||
751
+ JSON.parse(meta).manifest_hash !== hash(raw))
752
+ throw new Error('cache mismatch');
753
+ const cached = await validate(raw);
754
+ const tasks = cached.evaluation?.tasks;
755
+ if (!Array.isArray(tasks) || tasks.length < 2 || tasks.length > 4)
756
+ throw new Error('cache tasks');
757
+ const cachedNames = tasks.map((task, i) => {
758
+ if (!task || typeof task !== 'object' || Array.isArray(task))
759
+ throw new Error(`cached evaluation.tasks[${i}] must be a mapping`);
760
+ validateInlineTask(task, i);
761
+ return taskName(task.name, `cached evaluation.tasks[${i}].name`);
762
+ });
763
+ if (new Set(cachedNames).size !== cachedNames.length ||
764
+ cachedNames.some((name) => bundled.has(name) || names.includes(name) || rec.includes(name)))
765
+ throw new Error('cache collision');
766
+ return ['Tasks: generated cache', ...cachedNames.map((name) => `- ${name}`)];
767
+ }
768
+ catch (error) {
769
+ if (error instanceof Error &&
770
+ (error.message.startsWith('Eval artifact root') ||
771
+ error.message.startsWith('Eval managed path')))
772
+ throw error;
773
+ return [
774
+ 'Tasks: pending generation',
775
+ 'A normal comet eval run generates and freezes 2-4 tasks before execution.',
776
+ ];
777
+ }
778
+ }
779
+ //# sourceMappingURL=standalone-static-collect.js.map