@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -0,0 +1,263 @@
1
+ """Contract tests for Langfuse trajectory provisioning and transcript adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import json
7
+ import subprocess
8
+ import sys
9
+ import tarfile
10
+ from pathlib import Path
11
+
12
+ import pytest
13
+
14
+ from scaffold.python.langfuse_adapter import (
15
+ LangfuseRunReporter,
16
+ TrajectoryProvisionError,
17
+ TrajectoryProvision,
18
+ install_codex_plugin_workspace,
19
+ install_transcript_hook,
20
+ parse_agent_transcript,
21
+ provision_trajectory_plugin,
22
+ trajectory_mode,
23
+ )
24
+
25
+
26
+ def _archive_bytes(files: dict[str, str]) -> bytes:
27
+ stream = io.BytesIO()
28
+ with tarfile.open(fileobj=stream, mode="w:gz") as archive:
29
+ for name, content in files.items():
30
+ payload = content.encode("utf-8")
31
+ info = tarfile.TarInfo(name)
32
+ info.size = len(payload)
33
+ archive.addfile(info, io.BytesIO(payload))
34
+ return stream.getvalue()
35
+
36
+
37
+ def test_trajectory_modes_cover_all_supported_agent_contracts():
38
+ assert trajectory_mode("claude-code") == "official-claude-code-plugin"
39
+ assert trajectory_mode("codex") == "official-codex-plugin"
40
+ assert trajectory_mode("qoder") == "qoder-stop-transcript"
41
+ assert trajectory_mode("codebuddy") == "codebuddy-stop-transcript"
42
+
43
+
44
+ def test_plugin_provision_is_pinned_hashed_cached_and_credential_free(tmp_path: Path):
45
+ archive = _archive_bytes(
46
+ {
47
+ "langfuse-plugin/plugins/tracing/hooks/hooks.json": '{"hooks": {}}\n',
48
+ "langfuse-plugin/plugins/tracing/dist/index.mjs": "export {};\n",
49
+ }
50
+ )
51
+ calls = []
52
+
53
+ def download(url: str, destination: Path) -> None:
54
+ calls.append(url)
55
+ destination.write_bytes(archive)
56
+
57
+ first = provision_trajectory_plugin(tmp_path, "codex", download=download)
58
+
59
+ assert first.mode == "official-codex-plugin"
60
+ assert first.plugin_path is not None
61
+ assert (first.plugin_path / "plugins" / "tracing" / "hooks" / "hooks.json").is_file()
62
+ metadata = json.loads((first.cache_dir / "metadata.json").read_text(encoding="utf-8"))
63
+ assert metadata["source_ref"]
64
+ assert metadata["archive_sha256"]
65
+ assert metadata["content_sha256"]
66
+ assert "LANGFUSE_SECRET_KEY" not in json.dumps(metadata)
67
+
68
+ second = provision_trajectory_plugin(
69
+ tmp_path,
70
+ "codex",
71
+ download=lambda *_args: pytest.fail("cache hit must not download"),
72
+ )
73
+ assert second.plugin_path == first.plugin_path
74
+ assert len(calls) == 1
75
+
76
+ (first.plugin_path / "plugins" / "tracing" / "dist" / "index.mjs").write_text(
77
+ "tampered\n", encoding="utf-8"
78
+ )
79
+ with pytest.raises(TrajectoryProvisionError, match="integrity"):
80
+ provision_trajectory_plugin(tmp_path, "codex", download=lambda *_args: None)
81
+
82
+
83
+ def test_qoder_and_codebuddy_transcripts_normalize_messages_tools_and_bounds(tmp_path: Path):
84
+ transcript = tmp_path / "session.jsonl"
85
+ transcript.write_text(
86
+ "\n".join(
87
+ [
88
+ json.dumps({"type": "session_meta", "sessionId": "s-1", "data": {"model": "m"}}),
89
+ json.dumps({"type": "user", "message": {"content": "Please inspect"}}),
90
+ json.dumps(
91
+ {
92
+ "type": "assistant",
93
+ "message": {
94
+ "content": [
95
+ {"type": "text", "text": "I will inspect."},
96
+ {
97
+ "type": "tool_use",
98
+ "name": "read_file",
99
+ "input": {"path": "a.py"},
100
+ },
101
+ ]
102
+ },
103
+ }
104
+ ),
105
+ json.dumps(
106
+ {
107
+ "type": "user",
108
+ "message": {
109
+ "content": [
110
+ {"type": "tool_result", "tool_use_id": "t-1", "content": "x" * 100}
111
+ ]
112
+ },
113
+ }
114
+ ),
115
+ ]
116
+ )
117
+ + "\n",
118
+ encoding="utf-8",
119
+ )
120
+
121
+ events = parse_agent_transcript(transcript, max_chars=32)
122
+
123
+ assert [event.kind for event in events] == ["session", "user", "text", "tool", "tool_result"]
124
+ assert events[0].session_id == "s-1"
125
+ assert events[3].name == "read_file"
126
+ assert events[-1].truncated is True
127
+ assert len(events[-1].output) <= 32
128
+
129
+
130
+ def test_transcript_hook_is_project_local_and_preserves_existing_settings(tmp_path: Path):
131
+ settings = tmp_path / ".codebuddy" / "settings.json"
132
+ settings.parent.mkdir(parents=True)
133
+ settings.write_text(
134
+ json.dumps({"custom": True, "hooks": {"Stop": [{"custom": True}]}}), encoding="utf-8"
135
+ )
136
+
137
+ trajectory_path = install_transcript_hook(tmp_path, "codebuddy")
138
+
139
+ assert trajectory_path == tmp_path / ".comet" / "eval" / "langfuse" / "trajectories"
140
+ hook_path = tmp_path / ".codebuddy" / "hooks" / "langfuse-stop-hook.py"
141
+ assert hook_path.is_file()
142
+ hook_source = hook_path.read_text(encoding="utf-8")
143
+ assert "COMET_EVAL_AGENT_ROLE" in hook_source
144
+ assert "hashlib.sha256" in hook_source
145
+ saved = json.loads(settings.read_text(encoding="utf-8"))
146
+ assert saved["custom"] is True
147
+ assert len(saved["hooks"]["Stop"]) == 2
148
+ assert "LANGFUSE_SECRET_KEY" not in settings.read_text(encoding="utf-8")
149
+
150
+ transcript = tmp_path / "transcript.jsonl"
151
+ transcript.write_text(
152
+ '{"type":"assistant","message":{"content":"done"}}\n',
153
+ encoding="utf-8",
154
+ )
155
+ result = subprocess.run(
156
+ [sys.executable, str(hook_path)],
157
+ input=json.dumps(
158
+ {
159
+ "hook_event_name": "Stop",
160
+ "transcript_path": str(transcript),
161
+ "cwd": str(tmp_path),
162
+ }
163
+ ),
164
+ text=True,
165
+ capture_output=True,
166
+ check=False,
167
+ )
168
+ assert result.returncode == 0, result.stderr
169
+ assert list(
170
+ (tmp_path / ".comet" / "eval" / "langfuse" / "trajectories").glob("*.jsonl")
171
+ )
172
+
173
+
174
+ def test_codex_plugin_is_copied_into_the_project_codex_home_cache(tmp_path: Path):
175
+ plugin = tmp_path / "plugin"
176
+ tracing = plugin / "plugins" / "tracing"
177
+ (tracing / ".codex-plugin").mkdir(parents=True)
178
+ (tracing / ".codex-plugin" / "plugin.json").write_text(
179
+ json.dumps({"version": "0.1.0"}), encoding="utf-8"
180
+ )
181
+ (tracing / "dist").mkdir()
182
+ (tracing / "dist" / "index.mjs").write_text("export {};\n", encoding="utf-8")
183
+ provision = TrajectoryProvision(
184
+ agent="codex",
185
+ mode="official-codex-plugin",
186
+ cache_dir=tmp_path,
187
+ plugin_path=plugin,
188
+ )
189
+
190
+ workspace = tmp_path / "workspace"
191
+ (workspace / ".codex").mkdir(parents=True)
192
+ (workspace / ".codex" / "config.toml").write_text(
193
+ '[features]\nother = true\n\n[profiles.default]\nmodel = "x"\n',
194
+ encoding="utf-8",
195
+ )
196
+ install_codex_plugin_workspace(workspace, provision)
197
+
198
+ installed = (
199
+ workspace
200
+ / ".codex"
201
+ / "plugins"
202
+ / "cache"
203
+ / "codex-observability-plugin"
204
+ / "tracing"
205
+ / "0.1.0"
206
+ )
207
+ assert (installed / "dist" / "index.mjs").is_file()
208
+ config = (workspace / ".codex" / "config.toml").read_text(encoding="utf-8")
209
+ assert "hooks = true" in config
210
+ assert 'plugins."tracing@codex-observability-plugin"' in config
211
+ assert config.index("hooks = true") < config.index("[profiles.default]")
212
+
213
+
214
+ class _Observation:
215
+ trace_id = "trace-1"
216
+ id = "observation-1"
217
+
218
+ def __enter__(self):
219
+ return self
220
+
221
+ def __exit__(self, exc_type, exc, tb):
222
+ return False
223
+
224
+ def update(self, **payload):
225
+ self.output = payload.get("output")
226
+
227
+
228
+ class _Client:
229
+ def __init__(self):
230
+ self.observations = []
231
+ self.scores = []
232
+
233
+ def start_as_current_observation(self, **kwargs):
234
+ self.observations.append(kwargs)
235
+ return _Observation()
236
+
237
+ def create_score(self, **kwargs):
238
+ self.scores.append(kwargs)
239
+
240
+
241
+ def test_reporter_maps_qoder_transcript_to_optional_child_observations(tmp_path: Path):
242
+ transcript = tmp_path / "trajectory.jsonl"
243
+ transcript.write_text(
244
+ json.dumps({"type": "assistant", "sessionId": "s-1", "message": {"content": "done"}})
245
+ + "\n",
246
+ encoding="utf-8",
247
+ )
248
+ client = _Client()
249
+ result = type("Result", (), {"passed": True, "checks_passed": ["ok"], "checks_failed": []})()
250
+ reporter = LangfuseRunReporter(client)
251
+
252
+ reporter.report_case(
253
+ "task",
254
+ "CONTROL",
255
+ lambda: None,
256
+ lambda: result,
257
+ metadata={"agent": "qoder"},
258
+ trajectory_path=transcript,
259
+ agent="qoder",
260
+ )
261
+
262
+ assert client.observations[1]["as_type"] == "generation"
263
+ assert client.observations[1]["name"].startswith("agent.text/")
@@ -4,11 +4,17 @@ from __future__ import annotations
4
4
 
5
5
  import inspect
6
6
  import importlib.util
7
+ import os
7
8
  import sys
9
+ import subprocess
8
10
  from pathlib import Path
9
11
  from types import SimpleNamespace
10
12
  import json
11
13
 
14
+ import pytest
15
+
16
+ from scaffold.python.eval_context import resolve_eval_context
17
+
12
18
 
13
19
  def _load_langsmith_conftest():
14
20
  eval_root = Path(__file__).resolve().parents[3]
@@ -95,6 +101,63 @@ def test_langsmith_tracing_auto_builds_default_plugin_cache(monkeypatch, tmp_pat
95
101
  assert module.os.environ["CC_LANGSMITH_PLUGIN_DIR"] == module._plugin_dir_env_value(plugin_dir)
96
102
 
97
103
 
104
+ def test_langsmith_default_plugin_cache_uses_resolved_owner_not_packaged_harness(
105
+ monkeypatch, tmp_path
106
+ ):
107
+ skill = tmp_path / "skill"
108
+ owner = tmp_path / "owner"
109
+ harness_cache = tmp_path / "packaged-harness" / ".cache" / "langsmith-cc-plugin"
110
+ skill.mkdir()
111
+ owner.mkdir()
112
+ (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
113
+ context = resolve_eval_context(skill_path=skill, project_root=owner)
114
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
115
+ module = _load_langsmith_conftest()
116
+ monkeypatch.setattr(module, "DEFAULT_LANGSMITH_PLUGIN_DIR", harness_cache)
117
+ monkeypatch.delenv("CC_LANGSMITH_PLUGIN_DIR", raising=False)
118
+ monkeypatch.setenv("LANGSMITH_TRACING", "true")
119
+ monkeypatch.setenv("TRACE_TO_LANGSMITH", "true")
120
+ calls = []
121
+
122
+ def fake_build(target):
123
+ calls.append(target)
124
+ target.mkdir(parents=True)
125
+ return True
126
+
127
+ monkeypatch.setattr(module, "_build_default_langsmith_plugin", fake_build)
128
+
129
+ resolved = module.provision_langsmith_plugin_dir()
130
+
131
+ expected = owner / ".comet" / "eval" / "cache" / "langsmith" / "langsmith-cc-plugin"
132
+ assert resolved == expected
133
+ assert calls == [expected]
134
+ assert not harness_cache.exists()
135
+
136
+
137
+ def test_langsmith_default_plugin_cache_rejects_an_owner_cache_link_that_escapes(
138
+ monkeypatch, tmp_path
139
+ ):
140
+ skill = tmp_path / "skill"
141
+ owner = tmp_path / "owner"
142
+ outside = tmp_path / "outside"
143
+ skill.mkdir()
144
+ owner.mkdir()
145
+ outside.mkdir()
146
+ (skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
147
+ context = resolve_eval_context(skill_path=skill, project_root=owner)
148
+ cache_link = context.artifact_root / "cache"
149
+ cache_link.parent.mkdir(parents=True)
150
+ if os.name == "nt":
151
+ subprocess.run(["cmd", "/c", "mklink", "/J", str(cache_link), str(outside)], check=True)
152
+ else:
153
+ cache_link.symlink_to(outside, target_is_directory=True)
154
+ monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
155
+ module = _load_langsmith_conftest()
156
+
157
+ with pytest.raises(ValueError, match="managed path must stay within its owner root"):
158
+ module._default_langsmith_plugin_dir()
159
+
160
+
98
161
  def test_langsmith_task_wrapper_registers_reexported_local_module():
99
162
  _load_langsmith_task_tests()
100
163
 
@@ -35,6 +35,56 @@ def test_save_artifacts_excludes_nested_git_metadata(tmp_path: Path):
35
35
  assert not (snapshot / ".git").exists()
36
36
 
37
37
 
38
+ def test_save_artifacts_redacts_configured_credentials(tmp_path: Path, monkeypatch):
39
+ from conftest import _save_artifacts
40
+
41
+ monkeypatch.setenv("FIXTURE_AGENT_CREDENTIAL", "artifact-secret-value")
42
+ workspace = tmp_path / "workspace"
43
+ workspace.mkdir()
44
+ (workspace / "result.md").write_text(
45
+ "Authorization: Bearer artifact-secret-value\n", encoding="utf-8"
46
+ )
47
+
48
+ _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
49
+
50
+ snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
51
+ saved = (snapshot / "result.md").read_text(encoding="utf-8")
52
+ assert "artifact-secret-value" not in saved
53
+
54
+
55
+ def test_save_artifacts_skips_non_text_content_instead_of_persisting_secrets(tmp_path: Path):
56
+ from conftest import _save_artifacts
57
+
58
+ workspace = tmp_path / "workspace"
59
+ workspace.mkdir()
60
+ (workspace / "secret.bin").write_bytes(b"artifact-secret-value\x00\xff")
61
+
62
+ _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
63
+
64
+ snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
65
+ assert not (snapshot / "secret.bin").exists()
66
+
67
+
68
+ def test_save_artifacts_uses_selected_agent_root_and_excludes_codebuddy_config(
69
+ tmp_path: Path,
70
+ ):
71
+ from conftest import _save_artifacts
72
+
73
+ workspace = tmp_path / "workspace"
74
+ workspace.mkdir()
75
+ (workspace / "result.md").write_text("ok", encoding="utf-8")
76
+ (workspace / ".codebuddy" / "settings.json").parent.mkdir()
77
+ (workspace / ".codebuddy" / "settings.json").write_text(
78
+ '{"env":{"CODEBUDDY_API_KEY":"secret"}}', encoding="utf-8"
79
+ )
80
+
81
+ _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace, agent="codebuddy")
82
+
83
+ snapshot = tmp_path / "artifacts/comet_full_040_beta_rep1/codebuddy"
84
+ assert (snapshot / "result.md").read_text(encoding="utf-8") == "ok"
85
+ assert not (snapshot / ".codebuddy").exists()
86
+
87
+
38
88
  def test_save_artifacts_excludes_controller_cli_snapshot(tmp_path: Path):
39
89
  from conftest import _save_artifacts
40
90
 
@@ -93,6 +143,97 @@ def test_extract_events_captures_token_usage_and_cost():
93
143
  assert events["model_usage"]["mimo-v2.5-pro"]["costUSD"] == 0.123456
94
144
 
95
145
 
146
+ def test_extract_events_normalizes_codex_turns_and_runtime_skill_evidence():
147
+ stdout = "\n".join(
148
+ [
149
+ json.dumps({"type": "thread.started", "thread_id": "thread-1"}),
150
+ json.dumps(
151
+ {
152
+ "type": "item.completed",
153
+ "item": {
154
+ "type": "command_execution",
155
+ "command": "cat .agents/skills/demo/SKILL.md",
156
+ "aggregated_output": "skill contents",
157
+ },
158
+ }
159
+ ),
160
+ json.dumps(
161
+ {
162
+ "type": "item.completed",
163
+ "item": {
164
+ "type": "file_change",
165
+ "changes": [{"path": "result.md", "kind": "add"}],
166
+ },
167
+ }
168
+ ),
169
+ json.dumps(
170
+ {
171
+ "type": "turn.completed",
172
+ "usage": {"input_tokens": 80, "output_tokens": 20},
173
+ "duration_ms": 900,
174
+ }
175
+ ),
176
+ ]
177
+ )
178
+
179
+ events = extract_events(parse_output(stdout))
180
+
181
+ assert events["role_sessions"]["subject"] == ["thread-1"]
182
+
183
+ assert events["subject_invocations"] == 1
184
+ assert events["num_turns"] == 1
185
+ assert events["input_tokens"] == 80
186
+ assert events["output_tokens"] == 20
187
+ assert events["total_tokens"] == 100
188
+ assert events["duration_seconds"] == 0.9
189
+ assert events["commands_run"] == ["cat .agents/skills/demo/SKILL.md"]
190
+ assert events["files_created"] == ["result.md"]
191
+ assert events["skills_invoked"] == ["demo"]
192
+
193
+
194
+ def test_extract_events_recognizes_codebuddy_runtime_skill_evidence():
195
+ stdout = json.dumps(
196
+ {
197
+ "type": "assistant",
198
+ "message": {
199
+ "content": [
200
+ {
201
+ "type": "tool_use",
202
+ "name": "Bash",
203
+ "input": {"command": "cat .codebuddy/skills/demo/SKILL.md"},
204
+ }
205
+ ]
206
+ },
207
+ }
208
+ )
209
+
210
+ events = extract_events(parse_output(stdout))
211
+
212
+ assert events["skills_invoked"] == ["demo"]
213
+
214
+
215
+ def test_custom_agent_requires_explicit_skill_invocation_events():
216
+ path_only = json.dumps(
217
+ {
218
+ "type": "item.completed",
219
+ "item": {
220
+ "type": "command_execution",
221
+ "command": "cat .agents/skills/demo/SKILL.md",
222
+ },
223
+ }
224
+ )
225
+ inferred = extract_events(parse_output(path_only), agent="fixture-agent")
226
+ assert inferred["skills_invoked"] == []
227
+ assert inferred["skill_invocations"] == []
228
+
229
+ explicit = extract_events(
230
+ parse_output(json.dumps({"type": "skill_invocation", "skill": "demo"})),
231
+ agent="fixture-agent",
232
+ )
233
+ assert explicit["skills_invoked"] == ["demo"]
234
+ assert explicit["skill_invocations"] == ["demo"]
235
+
236
+
96
237
  def test_extract_events_accumulates_duration_across_results():
97
238
  stdout = "\n".join(
98
239
  [
@@ -304,6 +445,27 @@ def test_save_raw_writes_utf8_output(tmp_path: Path):
304
445
  assert "stderr 中文" == stderr_path.read_text(encoding="utf-8")
305
446
 
306
447
 
448
+ def test_saved_eval_outputs_redact_configured_credentials(tmp_path: Path, monkeypatch):
449
+ monkeypatch.setenv("FIXTURE_AGENT_API_KEY", "fixture-secret-value")
450
+ save_raw(
451
+ tmp_path,
452
+ "COMET_FULL_040_BETA",
453
+ 1,
454
+ "Authorization: Bearer fixture-secret-value x-api-key: fixture-secret-value",
455
+ "token=fixture-secret-value",
456
+ )
457
+ save_events(tmp_path, "COMET_FULL_040_BETA", 1, {"token": "fixture-secret-value"})
458
+ save_report(tmp_path, "COMET_FULL_040_BETA", 1, {"credential": "fixture-secret-value"})
459
+
460
+ for path in (
461
+ tmp_path / "raw/COMET_FULL_040_BETA_rep1_stdout.json",
462
+ tmp_path / "raw/COMET_FULL_040_BETA_rep1_stderr.txt",
463
+ tmp_path / "events/COMET_FULL_040_BETA_rep1.json",
464
+ tmp_path / "reports/COMET_FULL_040_BETA_rep1_report.json",
465
+ ):
466
+ assert "fixture-secret-value" not in path.read_text(encoding="utf-8")
467
+
468
+
307
469
  def test_save_artifacts_preserve_stable_treatment_filenames(tmp_path: Path):
308
470
  save_events(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
309
471
  save_raw(tmp_path, "COMET_FULL_040_BETA", 2, "{}", "stderr")
@@ -0,0 +1,76 @@
1
+ """Tests for normalizing manifest-authored tasks into the runner contract."""
2
+
3
+ from pathlib import Path
4
+
5
+ from scaffold.python.manifest_tasks import load_manifest_tasks
6
+ from scaffold.python.manifests import load_eval_manifest
7
+
8
+
9
+ def test_inline_task_uses_generic_environment_and_exposes_normalized_contract(tmp_path: Path):
10
+ package = tmp_path / "my-skill"
11
+ (package / "comet").mkdir(parents=True)
12
+ manifest_path = package / "comet" / "eval.yaml"
13
+ manifest_path.write_text(
14
+ """
15
+ apiVersion: comet.eval/v1alpha1
16
+ kind: SkillEvalManifest
17
+ metadata:
18
+ name: my-skill
19
+ skill:
20
+ name: my-skill
21
+ source: ..
22
+ evaluation:
23
+ tasks:
24
+ - name: inline-task
25
+ prompt: Create result.md.
26
+ expect:
27
+ files: [result.md]
28
+ rubric:
29
+ - The result is useful.
30
+ """,
31
+ encoding="utf-8",
32
+ )
33
+
34
+ tasks = load_manifest_tasks(load_eval_manifest(manifest_path))
35
+
36
+ assert len(tasks) == 1
37
+ task = tasks[0]
38
+ assert task.name == "inline-task"
39
+ assert task.render_prompt() == "Create result.md."
40
+ assert task.environment_dir.is_dir()
41
+ assert task.config.evaluation.expected_artifacts == ["result.md"]
42
+ assert task.config.evaluation.rubric_criteria == ["The result is useful."]
43
+
44
+
45
+ def test_source_task_is_loaded_without_changing_its_task_contract(tmp_path: Path):
46
+ package = tmp_path / "my-skill"
47
+ source = package / "tasks" / "source-task"
48
+ (source / "environment").mkdir(parents=True)
49
+ (source / "task.toml").write_text(
50
+ '[metadata]\nname = "source-task"\n\n[environment]\ndockerfile = "Dockerfile"\n',
51
+ encoding="utf-8",
52
+ )
53
+ (source / "instruction.md").write_text("Source prompt\n", encoding="utf-8")
54
+ (source / "environment" / "Dockerfile").write_text("FROM alpine\n", encoding="utf-8")
55
+ manifest_path = package / "comet" / "eval.yaml"
56
+ manifest_path.parent.mkdir(parents=True)
57
+ manifest_path.write_text(
58
+ """
59
+ apiVersion: comet.eval/v1alpha1
60
+ kind: SkillEvalManifest
61
+ metadata:
62
+ name: my-skill
63
+ skill:
64
+ name: my-skill
65
+ source: ..
66
+ evaluation:
67
+ tasks:
68
+ - source: ./tasks/source-task
69
+ """,
70
+ encoding="utf-8",
71
+ )
72
+
73
+ tasks = load_manifest_tasks(load_eval_manifest(manifest_path))
74
+
75
+ assert [task.name for task in tasks] == ["source-task"]
76
+ assert tasks[0].render_prompt() == "Source prompt\n"