@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -0,0 +1,230 @@
1
+ """Import-safe generated-task snapshot and cache selection primitives."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import re
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import Any, Mapping
12
+
13
+ from scaffold.python.agents import get_agent_adapter
14
+ from scaffold.python.eval_context import managed_path_for_owner
15
+ from scaffold.python.manifests import load_eval_manifest
16
+
17
+
18
+ GENERATOR_VERSION = "comet-auto-task-generator.v1"
19
+ TASK_SCHEMA_VERSION = "comet.eval/v1alpha1"
20
+ MIN_GENERATED_TASKS = 2
21
+ MAX_GENERATED_TASKS = 4
22
+ MAX_SNAPSHOT_FILES = 128
23
+ MAX_SNAPSHOT_BYTES = 512 * 1024
24
+ DEFAULT_CONTINUE_PROMPT = "Please continue with the next phase of the workflow."
25
+ _MODEL_ENV_KEYS = {
26
+ "claude-code": ("BENCH_CC_MODEL", "ANTHROPIC_MODEL"),
27
+ "codex": ("BENCH_CODEX_MODEL", "OPENAI_MODEL"),
28
+ "qoder": ("BENCH_QODER_MODEL", "QODER_MODEL"),
29
+ "codebuddy": ("BENCH_CODEBUDDY_MODEL", "CODEBUDDY_MODEL"),
30
+ }
31
+
32
+
33
+ @dataclass(frozen=True)
34
+ class SnapshotFile:
35
+ path: str
36
+ content: str
37
+ content_hash: str
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class SkillSnapshot:
42
+ files: tuple[SnapshotFile, ...]
43
+ content_hash: str
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class GeneratedCache:
48
+ manifest_path: Path
49
+ metadata_path: Path
50
+ generation_hash: str
51
+
52
+
53
+ def selected_agent_model(agent: str) -> str | None:
54
+ """Resolve the existing environment-backed model selector for cache identity."""
55
+ adapter = get_agent_adapter(agent)
56
+ keys = (*_MODEL_ENV_KEYS.get(agent, ()), *(key for key in (adapter.model_env,) if key))
57
+ return next(
58
+ (os.environ[key] for key in keys if os.environ.get(key)),
59
+ None,
60
+ )
61
+
62
+
63
+ def normalize_interaction(interaction: Mapping[str, Any]) -> dict[str, Any]:
64
+ """Return the one cross-runtime interaction payload used by generation hashes."""
65
+ return {
66
+ "mode": interaction.get("mode", "none"),
67
+ "max_turns": interaction.get("maxTurns", interaction.get("max_turns", 12)),
68
+ "simulator_prompt": interaction.get("simulatorPrompt")
69
+ or interaction.get("simulator_prompt"),
70
+ "decision_patterns": list(
71
+ interaction.get("decisionPatterns", interaction.get("decision_patterns", [])) or []
72
+ ),
73
+ "decision_reply": interaction.get(
74
+ "decisionReply", interaction.get("decision_reply")
75
+ ),
76
+ "decision_replies": list(
77
+ interaction.get("decisionReplies", interaction.get("decision_replies", [])) or []
78
+ ),
79
+ "continue_prompt": interaction.get(
80
+ "continuePrompt",
81
+ interaction.get("continue_prompt", DEFAULT_CONTINUE_PROMPT),
82
+ ),
83
+ "fresh_resume_marker": interaction.get("freshResumeMarker")
84
+ or interaction.get("fresh_resume_marker"),
85
+ }
86
+
87
+
88
+ def sha256(value: str | bytes) -> str:
89
+ payload = value.encode("utf-8") if isinstance(value, str) else value
90
+ return "sha256:" + hashlib.sha256(payload).hexdigest()
91
+
92
+
93
+ def skill_root(skill_path: Path | str) -> Path:
94
+ root = Path(skill_path).expanduser().resolve()
95
+ if root.is_file() and root.name == "SKILL.md":
96
+ root = root.parent
97
+ if not root.is_dir() or not (root / "SKILL.md").is_file():
98
+ raise ValueError(f"Skill package must contain SKILL.md: {skill_path}")
99
+ return root
100
+
101
+
102
+ def _inside(root: Path, path: Path) -> bool:
103
+ try:
104
+ path.resolve().relative_to(root.resolve())
105
+ return True
106
+ except ValueError:
107
+ return False
108
+
109
+
110
+ def build_skill_snapshot(skill_path: Path | str) -> SkillSnapshot:
111
+ root = skill_root(skill_path)
112
+ selected: dict[str, SnapshotFile] = {}
113
+ total = 0
114
+
115
+ def add(path: Path) -> None:
116
+ nonlocal total
117
+ if len(selected) >= MAX_SNAPSHOT_FILES or not path.is_file() or not _inside(root, path):
118
+ return
119
+ relative = path.resolve().relative_to(root).as_posix()
120
+ if relative in selected:
121
+ return
122
+ content = path.read_text(encoding="utf-8", errors="replace")
123
+ payload = content.encode("utf-8")
124
+ if total + len(payload) > MAX_SNAPSHOT_BYTES:
125
+ return
126
+ total += len(payload)
127
+ selected[relative] = SnapshotFile(relative, content, sha256(content))
128
+
129
+ skill_file = root / "SKILL.md"
130
+ add(skill_file)
131
+ body = skill_file.read_text(encoding="utf-8", errors="replace")
132
+ for reference in re.findall(r"[`\"']([^`\"']+)[`\"']", body):
133
+ add(root / reference.replace("\\", "/"))
134
+ for directory in ("scripts", "references", "reference", "examples", "templates"):
135
+ candidate = root / directory
136
+ if candidate.is_dir():
137
+ for child in sorted(candidate.rglob("*")):
138
+ if not child.is_symlink():
139
+ add(child)
140
+ files = tuple(selected[key] for key in sorted(selected))
141
+ manifest = json.dumps(
142
+ [{"path": item.path, "hash": item.content_hash} for item in files],
143
+ ensure_ascii=False,
144
+ sort_keys=True,
145
+ separators=(",", ":"),
146
+ )
147
+ return SkillSnapshot(files, sha256(manifest))
148
+
149
+
150
+ def generation_hash(
151
+ snapshot: SkillSnapshot,
152
+ *,
153
+ agent: str,
154
+ model: str | None,
155
+ base_url: str | None = None,
156
+ profile: str,
157
+ interaction: Mapping[str, Any],
158
+ ) -> str:
159
+ payload = {
160
+ "snapshot_hash": snapshot.content_hash,
161
+ "agent": agent,
162
+ "model": model or "runtime-default",
163
+ "base_url": base_url or "runtime-default",
164
+ "profile": profile,
165
+ "interaction": normalize_interaction(interaction),
166
+ "generator_version": GENERATOR_VERSION,
167
+ "task_schema_version": TASK_SCHEMA_VERSION,
168
+ }
169
+ return hashlib.sha256(
170
+ json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
171
+ ).hexdigest()
172
+
173
+
174
+ def cache_location(owner_root: Path | str, skill_path: Path | str, cache_hash: str) -> Path:
175
+ root = skill_root(skill_path)
176
+ safe_name = re.sub(r"[^A-Za-z0-9._-]+", "-", root.name).strip("-") or "skill"
177
+ return managed_path_for_owner(owner_root, "generated", safe_name, cache_hash)
178
+
179
+
180
+ def load_generated_cache(cache_dir: Path, cache_hash: str) -> GeneratedCache | None:
181
+ manifest_path = cache_dir / "eval.yaml"
182
+ metadata_path = cache_dir / "generation.json"
183
+ if not manifest_path.is_file() or not metadata_path.is_file():
184
+ return None
185
+ try:
186
+ metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
187
+ if metadata.get("generation_hash") != cache_hash:
188
+ return None
189
+ if metadata.get("manifest_hash") != sha256(manifest_path.read_bytes()):
190
+ return None
191
+ validate_generated_manifest(manifest_path)
192
+ except (OSError, ValueError, TypeError, json.JSONDecodeError):
193
+ return None
194
+ return GeneratedCache(manifest_path, metadata_path, cache_hash)
195
+
196
+
197
+ def validate_generated_manifest(manifest_path: Path | str):
198
+ """Validate the stronger contract that applies to generated cache entries.
199
+
200
+ The public manifest schema intentionally accepts authored and sourced tasks,
201
+ while a generated cache must contain only the bounded inline task snapshot
202
+ produced by the generator. Keeping this check beside cache loading makes
203
+ Python execution and static collection reject the same stale or tampered
204
+ cache before it can be reused.
205
+ """
206
+ manifest = load_eval_manifest(manifest_path)
207
+ tasks = manifest.tasks
208
+ if not MIN_GENERATED_TASKS <= len(tasks) <= MAX_GENERATED_TASKS:
209
+ raise ValueError(
210
+ f"generated evaluation.tasks must contain {MIN_GENERATED_TASKS}-{MAX_GENERATED_TASKS} tasks"
211
+ )
212
+ names: set[str] = set()
213
+ for index, task in enumerate(tasks):
214
+ if not task.is_inline:
215
+ raise ValueError(f"generated evaluation.tasks[{index}] must be an inline task")
216
+ if task.name in names:
217
+ raise ValueError(f'generated evaluation.tasks[{index}].name is duplicated: "{task.name}"')
218
+ names.add(task.name)
219
+ return manifest
220
+
221
+
222
+ def write_cache_metadata(cache_dir: Path, cache_hash: str) -> Path:
223
+ """Test and migration helper for a static cache already written atomically by the generator."""
224
+ manifest_path = cache_dir / "eval.yaml"
225
+ metadata_path = cache_dir / "generation.json"
226
+ metadata_path.write_text(
227
+ json.dumps({"generation_hash": cache_hash, "manifest_hash": sha256(manifest_path.read_bytes())}),
228
+ encoding="utf-8",
229
+ )
230
+ return metadata_path
@@ -6,8 +6,8 @@ produce meaningful output, or just a stub? This module asks a judge LLM to read
6
6
  workspace artifacts and score three generic quality dimensions on a 0.00-1.00
7
7
  scale with a cited reason.
8
8
 
9
- Runs on the host (not in Docker) via the same Claude CLI + proxy used for the
10
- subject agent, so it adds no new dependency. Scores are emitted in the same
9
+ Runs on the host (not in Docker) via the selected agent CLI and its configured
10
+ provider, so it adds no new dependency. Scores are emitted in the same
11
11
  ``[RUBRIC-JUDGE] dim: score - reason`` format the logging layer understands.
12
12
 
13
13
  Usage::
@@ -134,12 +134,9 @@ def _build_generic_judge_prompt(
134
134
 
135
135
  # Build the list of dimensions the judge should output.
136
136
  all_dims = list(GENERIC_JUDGE_DIMENSIONS)
137
- dim_lines = [
138
- f"[RUBRIC-JUDGE] {dim}: <score> - <reason>" for dim in all_dims
139
- ]
137
+ dim_lines = [f"[RUBRIC-JUDGE] {dim}: <score> - <reason>" for dim in all_dims]
140
138
  extra_dim_lines = [
141
- f"[RUBRIC-JUDGE] custom_{i}: <score> - <reason>"
142
- for i in range(len(custom_criteria))
139
+ f"[RUBRIC-JUDGE] custom_{i}: <score> - <reason>" for i in range(len(custom_criteria))
143
140
  ]
144
141
  all_output_lines = "\n".join(dim_lines + extra_dim_lines)
145
142
 
@@ -177,9 +174,25 @@ Baseline validation results:
177
174
  """
178
175
 
179
176
 
180
- def _run_judge(prompt: str, timeout: int = 120) -> str:
177
+ def _run_judge(
178
+ prompt: str,
179
+ timeout: int = 120,
180
+ agent: object | None = None,
181
+ model: object | None = None,
182
+ base_url: object | None = None,
183
+ evidence: dict[str, Any] | None = None,
184
+ excluded_credentials: tuple[str, ...] = (),
185
+ ) -> str:
181
186
  """Call the judge LLM through the configured judge provider."""
182
- return run_judge_prompt(prompt, timeout=timeout)
187
+ return run_judge_prompt(
188
+ prompt,
189
+ timeout=timeout,
190
+ agent=agent,
191
+ model=model,
192
+ base_url=base_url,
193
+ evidence=evidence,
194
+ excluded_credentials=excluded_credentials,
195
+ )
183
196
 
184
197
 
185
198
  def judge_generic_artifacts(
@@ -194,7 +207,15 @@ def judge_generic_artifacts(
194
207
  rule-based scores.
195
208
  """
196
209
  prompt = _build_generic_judge_prompt(test_dir, outputs)
197
- raw = _run_judge(prompt, timeout=timeout)
210
+ raw = _run_judge(
211
+ prompt,
212
+ timeout=timeout,
213
+ agent=outputs.get("judge_agent") or outputs.get("agent"),
214
+ model=outputs.get("judge_model"),
215
+ base_url=outputs.get("judge_base_url"),
216
+ excluded_credentials=tuple(outputs.get("main_credentials") or ()),
217
+ evidence=outputs,
218
+ )
198
219
 
199
220
  # Collect all valid dimension names (standard + custom).
200
221
  custom_criteria = outputs.get("rubric_criteria") or []
@@ -226,12 +247,18 @@ def judge_generic_messages(
226
247
  """Convenience wrapper returning ``[RUBRIC-JUDGE]`` check messages."""
227
248
  out: list[str] = []
228
249
  try:
229
- build_judge_invocation()
250
+ build_judge_invocation(
251
+ agent=outputs.get("judge_agent") or outputs.get("agent"),
252
+ model=outputs.get("judge_model"),
253
+ base_url=outputs.get("judge_base_url"),
254
+ excluded_credentials=tuple(outputs.get("main_credentials") or ()),
255
+ )
230
256
  except ValueError as e:
231
257
  return [f"[RUBRIC-JUDGE] status: skipped - {e}"]
232
258
 
233
- for dim, (score, reason) in judge_generic_artifacts(
234
- test_dir, outputs, timeout=timeout
235
- ).items():
259
+ scores = judge_generic_artifacts(test_dir, outputs, timeout=timeout)
260
+ if not scores:
261
+ return ["[RUBRIC-JUDGE] status: unavailable - no valid Judge scores"]
262
+ for dim, (score, reason) in scores.items():
236
263
  out.append(f"[RUBRIC-JUDGE] {dim}: {score:.2f} - {reason}")
237
264
  return out
@@ -0,0 +1,118 @@
1
+ """Deterministic validators for project-authored inline eval tasks."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from scaffold.python.utils import run_command_in_docker
10
+
11
+
12
+ def _workspace_path(test_dir: Path, value: str) -> Path:
13
+ candidate = test_dir / value
14
+ resolved = candidate.resolve(strict=False)
15
+ try:
16
+ resolved.relative_to(test_dir.resolve())
17
+ except ValueError as exc:
18
+ raise ValueError(f"Task artifact escapes the workspace: {value!r}") from exc
19
+ return candidate
20
+
21
+
22
+ def _json_path(value: Any, path: str) -> Any:
23
+ if path == "$":
24
+ return value
25
+ if not path.startswith("$."):
26
+ raise ValueError(f"Unsupported JSON path: {path}")
27
+ tokens: list[str] = []
28
+ for part in path[2:].replace("[", ".").replace("]", "").split("."):
29
+ if part:
30
+ tokens.append(part)
31
+ current = value
32
+ for token in tokens:
33
+ if isinstance(current, list):
34
+ current = current[int(token)]
35
+ elif isinstance(current, dict):
36
+ current = current[token]
37
+ else:
38
+ raise KeyError(token)
39
+ return current
40
+
41
+
42
+ def _display_output(result: Any) -> str:
43
+ output = "\n".join(
44
+ part for part in (getattr(result, "stdout", ""), getattr(result, "stderr", "")) if part
45
+ )
46
+ return output[-1000:]
47
+
48
+
49
+ def make_inline_expectations_validator(expect: dict[str, Any]):
50
+ """Build a validator for the normalized ``evaluation.tasks.expect`` map."""
51
+
52
+ def validate(test_dir: Path, _outputs: dict) -> tuple[list[str], list[str]]:
53
+ passed: list[str] = []
54
+ failed: list[str] = []
55
+
56
+ for command_spec in expect.get("commands", []):
57
+ command = command_spec["run"]
58
+ result = run_command_in_docker(test_dir, command, command_spec["timeout"])
59
+ if result.returncode == 0:
60
+ passed.append(f"Command succeeded: {command}")
61
+ else:
62
+ failed.append(f"Command failed: {command} ({_display_output(result)})")
63
+
64
+ for artifact in expect.get("files", []):
65
+ path = _workspace_path(test_dir, artifact)
66
+ if any(char in artifact for char in "*?["):
67
+ matches = []
68
+ for item in test_dir.glob(artifact):
69
+ try:
70
+ safe_item = _workspace_path(test_dir, str(item.relative_to(test_dir)))
71
+ except ValueError:
72
+ continue
73
+ if safe_item == item:
74
+ matches.append(item)
75
+ if matches:
76
+ passed.append(f"Files exist: {artifact}")
77
+ else:
78
+ failed.append(f"Files missing: {artifact}")
79
+ elif path.exists():
80
+ passed.append(f"File exists: {artifact}")
81
+ else:
82
+ failed.append(f"File missing: {artifact}")
83
+
84
+ for artifact, values in expect.get("contains", {}).items():
85
+ path = _workspace_path(test_dir, artifact)
86
+ if not path.is_file():
87
+ failed.append(f"Contains checks missing file: {artifact}")
88
+ continue
89
+ content = path.read_text(encoding="utf-8", errors="replace")
90
+ for value in values:
91
+ if value in content:
92
+ passed.append(f"Contains {value!r} in {artifact}")
93
+ else:
94
+ failed.append(f"Missing {value!r} in {artifact}")
95
+
96
+ for check in expect.get("json", []):
97
+ artifact = check["file"]
98
+ path = _workspace_path(test_dir, artifact)
99
+ try:
100
+ actual = _json_path(json.loads(path.read_text(encoding="utf-8")), check["path"])
101
+ except (
102
+ FileNotFoundError,
103
+ json.JSONDecodeError,
104
+ KeyError,
105
+ IndexError,
106
+ TypeError,
107
+ ValueError,
108
+ ) as exc:
109
+ failed.append(f"JSON {check['path']} in {artifact} could not be read: {exc}")
110
+ continue
111
+ if actual == check["equals"]:
112
+ passed.append(f"JSON {check['path']} equals {check['equals']}")
113
+ else:
114
+ failed.append(f"JSON {check['path']} expected {check['equals']!r}, got {actual!r}")
115
+
116
+ return passed, failed
117
+
118
+ return validate