@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -6,8 +6,8 @@ genuinely explore alternatives, or did it just happen to contain the word
6
6
  "risk"? This module asks a judge LLM to read the actual artifacts and score
7
7
  three dimensions on a 0.00–1.00 scale with a cited reason.
8
8
 
9
- Runs on the host (not in Docker) via the same Claude CLI + proxy used for the
10
- subject agent, so it adds no new dependency. Scores are emitted in the same
9
+ Runs on the host (not in Docker) via the selected agent CLI and its configured
10
+ provider, so it adds no new dependency. Scores are emitted in the same
11
11
  ``[RUBRIC-JUDGE] dim: score - reason`` format the logging layer understands.
12
12
 
13
13
  Usage from the rubric validator or rescore tool::
@@ -27,6 +27,7 @@ from scaffold.python.judge_config import build_judge_invocation, run_judge_promp
27
27
  # module is imported outside pytest (e.g. by the rescore tool).
28
28
  try:
29
29
  from dotenv import load_dotenv
30
+
30
31
  load_dotenv(Path(__file__).resolve().parents[3] / ".env")
31
32
  except Exception:
32
33
  pass
@@ -141,12 +142,36 @@ Cite specific content from the artifacts in your reason (<=25 words each).
141
142
  """
142
143
 
143
144
 
144
- def _run_judge(prompt: str, timeout: int = 120) -> str:
145
+ def _run_judge(
146
+ prompt: str,
147
+ timeout: int = 120,
148
+ agent: object | None = None,
149
+ model: object | None = None,
150
+ base_url: object | None = None,
151
+ evidence: dict | None = None,
152
+ excluded_credentials: tuple[str, ...] = (),
153
+ ) -> str:
145
154
  """Call the judge LLM through the configured judge provider."""
146
- return run_judge_prompt(prompt, timeout=timeout)
147
-
148
-
149
- def judge_artifacts(test_dir: Path, timeout: int = 120) -> dict[str, tuple[float, str]]:
155
+ return run_judge_prompt(
156
+ prompt,
157
+ timeout=timeout,
158
+ agent=agent,
159
+ model=model,
160
+ base_url=base_url,
161
+ evidence=evidence,
162
+ excluded_credentials=excluded_credentials,
163
+ )
164
+
165
+
166
+ def judge_artifacts(
167
+ test_dir: Path,
168
+ timeout: int = 120,
169
+ agent: object | None = None,
170
+ model: object | None = None,
171
+ base_url: object | None = None,
172
+ evidence: dict | None = None,
173
+ excluded_credentials: tuple[str, ...] = (),
174
+ ) -> dict[str, tuple[float, str]]:
150
175
  """Score the three qualitative dimensions via an LLM judge.
151
176
 
152
177
  Returns ``{dim: (score, reason)}``. On any failure (model error, no
@@ -157,7 +182,15 @@ def judge_artifacts(test_dir: Path, timeout: int = 120) -> dict[str, tuple[float
157
182
  return {}
158
183
 
159
184
  prompt = _build_judge_prompt(test_dir)
160
- raw = _run_judge(prompt, timeout=timeout)
185
+ raw = _run_judge(
186
+ prompt,
187
+ timeout=timeout,
188
+ agent=agent,
189
+ model=model,
190
+ base_url=base_url,
191
+ evidence=evidence,
192
+ excluded_credentials=excluded_credentials,
193
+ )
161
194
 
162
195
  scores: dict[str, tuple[float, str]] = {}
163
196
  for line in raw.splitlines():
@@ -175,14 +208,38 @@ def judge_artifacts(test_dir: Path, timeout: int = 120) -> dict[str, tuple[float
175
208
  return scores
176
209
 
177
210
 
178
- def judge_messages(test_dir: Path, timeout: int = 120) -> list[str]:
211
+ def judge_messages(
212
+ test_dir: Path,
213
+ timeout: int = 120,
214
+ agent: object | None = None,
215
+ model: object | None = None,
216
+ base_url: object | None = None,
217
+ evidence: dict | None = None,
218
+ excluded_credentials: tuple[str, ...] = (),
219
+ ) -> list[str]:
179
220
  """Convenience wrapper returning ``[RUBRIC-JUDGE]`` check messages."""
180
221
  out: list[str] = []
181
222
  try:
182
- build_judge_invocation()
223
+ build_judge_invocation(
224
+ agent=agent,
225
+ model=model,
226
+ base_url=base_url,
227
+ excluded_credentials=excluded_credentials,
228
+ )
183
229
  except ValueError as e:
184
230
  return [f"[RUBRIC-JUDGE] status: skipped - {e}"]
185
231
 
186
- for dim, (score, reason) in judge_artifacts(test_dir, timeout=timeout).items():
232
+ scores = judge_artifacts(
233
+ test_dir,
234
+ timeout=timeout,
235
+ agent=agent,
236
+ model=model,
237
+ base_url=base_url,
238
+ excluded_credentials=excluded_credentials,
239
+ evidence=evidence,
240
+ )
241
+ if not scores:
242
+ return ["[RUBRIC-JUDGE] status: unavailable - no valid Judge scores"]
243
+ for dim, (score, reason) in scores.items():
187
244
  out.append(f"[RUBRIC-JUDGE] {dim}: {score:.2f} - {reason}")
188
245
  return out
@@ -1,8 +1,10 @@
1
- """Output parsing, event extraction, and experiment logging for Claude CLI."""
1
+ """Output parsing, event extraction, and experiment logging for agent CLIs."""
2
2
 
3
3
  import json
4
4
  import math
5
+ import os
5
6
  import re
7
+ import tempfile
6
8
  from collections.abc import Callable
7
9
  from dataclasses import dataclass, field
8
10
  from datetime import datetime
@@ -10,13 +12,39 @@ from pathlib import Path
10
12
  from typing import Any
11
13
 
12
14
  from scaffold.python.evidence import stable_treatment_name
13
- from scaffold.python.paths import get_logs_dir
15
+ from scaffold.python.execution import redact_sensitive
16
+ from scaffold.python.paths import get_runs_dir
14
17
  from scaffold.python.report_outputs import (
15
18
  ReportOutputConfig,
16
19
  preferred_report_path,
17
20
  write_report_outputs,
18
21
  )
19
22
 
23
+
24
+ def _without_credentials(value: Any) -> Any:
25
+ if isinstance(value, dict):
26
+ return {
27
+ key: _without_credentials(item)
28
+ for key, item in value.items()
29
+ if not re.search(r"(?:api[_-]?key|auth[_-]?token|password|secret|credential)", key, re.I)
30
+ }
31
+ if isinstance(value, list):
32
+ return [_without_credentials(item) for item in value]
33
+ return value
34
+
35
+
36
+ def _atomic_write_json(path: Path, value: dict[str, Any]) -> None:
37
+ path.parent.mkdir(parents=True, exist_ok=True)
38
+ with tempfile.NamedTemporaryFile(
39
+ mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False
40
+ ) as temporary:
41
+ temporary.write(json.dumps(redact_sensitive(_without_credentials(value)), indent=2) + "\n")
42
+ temporary_path = Path(temporary.name)
43
+ try:
44
+ os.replace(temporary_path, path)
45
+ finally:
46
+ temporary_path.unlink(missing_ok=True)
47
+
20
48
  # Regex to strip ANSI escape codes
21
49
  ANSI_ESCAPE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
22
50
 
@@ -35,10 +63,22 @@ def strip_ansi(text: str) -> str:
35
63
  return ANSI_ESCAPE.sub("", text)
36
64
 
37
65
 
38
- def _record_skill_invocation(events: dict[str, Any], skill: str) -> None:
66
+ def _record_skill_invocation(events: dict[str, Any], skill: str, *, explicit: bool = False) -> None:
67
+ if not isinstance(skill, str) or not skill.strip():
68
+ return
39
69
  normalized = _SKILL_ALIASES.get(skill, skill)
40
70
  if normalized not in events["skills_invoked"]:
41
71
  events["skills_invoked"].append(normalized)
72
+ if explicit and normalized not in events["skill_invocations"]:
73
+ events["skill_invocations"].append(normalized)
74
+
75
+
76
+ def _record_skill_path(events: dict[str, Any], value: object) -> None:
77
+ if not isinstance(value, str):
78
+ return
79
+ match = re.search(r"\.(?:claude|agents|qoder|codebuddy)/skills/([^/\\]+)", value)
80
+ if match:
81
+ _record_skill_invocation(events, match.group(1))
42
82
 
43
83
 
44
84
  # =============================================================================
@@ -50,6 +90,7 @@ def parse_output(stdout: str) -> dict[str, Any]:
50
90
  """Parse stream-json output into structured data."""
51
91
  if not stdout:
52
92
  return {"messages": []}
93
+ stdout = redact_sensitive(stdout)
53
94
  messages = []
54
95
  for line in stdout.strip().split("\n"):
55
96
  try:
@@ -59,7 +100,37 @@ def parse_output(stdout: str) -> dict[str, Any]:
59
100
  return {"messages": messages}
60
101
 
61
102
 
62
- def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
103
+ def _record_explicit_skill_value(events: dict[str, Any], value: Any) -> None:
104
+ if isinstance(value, str):
105
+ _record_skill_invocation(events, value, explicit=True)
106
+ elif isinstance(value, dict):
107
+ for key in ("skill", "skill_name", "skillName", "name"):
108
+ candidate = value.get(key)
109
+ if isinstance(candidate, str):
110
+ _record_skill_invocation(events, candidate, explicit=True)
111
+ return
112
+ elif isinstance(value, list):
113
+ for item in value:
114
+ _record_explicit_skill_value(events, item)
115
+
116
+
117
+ def _record_explicit_skill_event(events: dict[str, Any], message: Any) -> None:
118
+ if not isinstance(message, dict):
119
+ return
120
+ event_type = str(message.get("type") or message.get("event") or "").lower()
121
+ if event_type in {"skill_invocation", "skill_invoked", "skill.use", "skill_used"}:
122
+ _record_explicit_skill_value(events, message.get("skill") or message.get("name"))
123
+ for key in ("skill_invocation", "skillInvocation", "skill_invocations", "skillInvocations"):
124
+ if key in message:
125
+ _record_explicit_skill_value(events, message[key])
126
+ item = message.get("item")
127
+ if isinstance(item, dict):
128
+ item_type = str(item.get("type") or item.get("event") or "").lower()
129
+ if item_type in {"skill_invocation", "skill_invoked", "skill.use", "skill_used"}:
130
+ _record_explicit_skill_value(events, item.get("skill") or item.get("name"))
131
+
132
+
133
+ def extract_events(parsed: dict[str, Any], *, agent: str | None = None) -> dict[str, Any]:
63
134
  """Extract events (tool calls, files, etc.) from parsed output."""
64
135
  events = {
65
136
  "tool_calls": [],
@@ -68,6 +139,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
68
139
  "files_modified": [],
69
140
  "commands_run": [],
70
141
  "skills_invoked": [],
142
+ "skill_invocations": [],
71
143
  "duration_seconds": None,
72
144
  "subject_invocations": 0,
73
145
  "num_turns": None,
@@ -78,6 +150,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
78
150
  "total_tokens": None,
79
151
  "total_cost_usd": None,
80
152
  "model_usage": {},
153
+ "role_sessions": {"subject": [], "simulator": [], "judge": []},
81
154
  "peak_context_input_tokens": None,
82
155
  "p95_context_input_tokens": None,
83
156
  "average_context_input_tokens": None,
@@ -129,7 +202,30 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
129
202
  elif key in capacities:
130
203
  target[key] = max(target.get(key, 0), item)
131
204
 
205
+ infer_skill_paths = agent is None or agent in {"claude-code", "codex", "qoder", "codebuddy"}
206
+
207
+ def record_skill_path(value: object) -> None:
208
+ if infer_skill_paths:
209
+ _record_skill_path(events, value)
210
+
132
211
  for msg in parsed.get("messages", []):
212
+ _record_explicit_skill_event(events, msg)
213
+ stack = [msg]
214
+ session_id = None
215
+ while stack and session_id is None:
216
+ item = stack.pop()
217
+ if isinstance(item, dict):
218
+ for key in ("session_id", "sessionId", "thread_id", "threadId"):
219
+ candidate = item.get(key)
220
+ if isinstance(candidate, str) and candidate:
221
+ session_id = candidate
222
+ break
223
+ stack.extend(item.values())
224
+ elif isinstance(item, list):
225
+ stack.extend(item)
226
+ if session_id and session_id not in events["role_sessions"]["subject"]:
227
+ events["role_sessions"]["subject"].append(session_id)
228
+
133
229
  if msg.get("type") == "result":
134
230
  events["subject_invocations"] += 1
135
231
  duration_ms = msg.get("duration_ms")
@@ -137,7 +233,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
137
233
  duration_ms_total += duration_ms
138
234
  duration_observed = True
139
235
 
140
- # A loop concatenates one result per subject invocation. Claude's
236
+ # A loop concatenates one result per subject invocation. Agent
141
237
  # result telemetry is invocation-local, so task totals must add all
142
238
  # initial, answer, and cold-resume invocations instead of retaining
143
239
  # only the final result event.
@@ -150,6 +246,51 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
150
246
  add_metric("total_cost_usd", msg.get("total_cost_usd"))
151
247
  merge_model_usage(msg.get("modelUsage"))
152
248
 
249
+ if msg.get("type") == "turn.completed":
250
+ events["subject_invocations"] += 1
251
+ add_metric("num_turns", 1)
252
+ usage = msg.get("usage") or {}
253
+ add_metric("input_tokens", usage.get("input_tokens", usage.get("inputTokens")))
254
+ add_metric("output_tokens", usage.get("output_tokens", usage.get("outputTokens")))
255
+ add_metric(
256
+ "cache_read_input_tokens",
257
+ usage.get("cached_input_tokens", usage.get("cacheReadInputTokens")),
258
+ )
259
+ duration_ms = msg.get("duration_ms")
260
+ if isinstance(duration_ms, (int, float)) and not isinstance(duration_ms, bool):
261
+ duration_ms_total += duration_ms
262
+ duration_observed = True
263
+
264
+ if msg.get("type") == "item.completed":
265
+ item = msg.get("item") or {}
266
+ if item.get("type") == "command_execution":
267
+ command = item.get("command")
268
+ if isinstance(command, str) and command:
269
+ events["commands_run"].append(command)
270
+ record_skill_path(command)
271
+ tool_call = {"tool": "Bash", "input": {"command": command or ""}}
272
+ if item.get("aggregated_output") is not None:
273
+ tool_call["output"] = item["aggregated_output"]
274
+ events["tool_calls"].append(tool_call)
275
+ elif item.get("type") == "file_change":
276
+ for change in item.get("changes") or []:
277
+ if not isinstance(change, dict):
278
+ continue
279
+ path = change.get("path")
280
+ if not isinstance(path, str) or not path:
281
+ continue
282
+ kind = str(change.get("kind") or "modify").lower()
283
+ if kind in {"add", "create", "created"}:
284
+ events["files_created"].append(path)
285
+ else:
286
+ events["files_modified"].append(path)
287
+ record_skill_path(path)
288
+ elif item.get("type") in {"mcp_tool_call", "tool_call"}:
289
+ tool = item.get("name") or item.get("tool") or "unknown"
290
+ events["tool_calls"].append(
291
+ {"tool": tool, "input": item.get("arguments") or item.get("input") or {}}
292
+ )
293
+
153
294
  if msg.get("type") == "assistant":
154
295
  message = msg.get("message", {})
155
296
  message_id = message.get("id")
@@ -182,18 +323,16 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
182
323
  if tool == "Read" and path:
183
324
  events["files_read"].append(path)
184
325
  # Detect skill reads (e.g., .claude/skills/skill-name/SKILL.md)
185
- if ".claude/skills/" in path and (
186
- m := re.search(r"\.claude/skills/([^/]+)", path)
187
- ):
188
- _record_skill_invocation(events, m.group(1))
326
+ record_skill_path(path)
189
327
  elif tool == "Write" and path:
190
328
  events["files_created"].append(path)
191
329
  elif tool == "Edit" and path:
192
330
  events["files_modified"].append(path)
193
331
  elif tool == "Bash" and inp.get("command"):
194
332
  events["commands_run"].append(inp["command"])
333
+ record_skill_path(inp["command"])
195
334
  elif tool == "Skill" and inp.get("skill"):
196
- _record_skill_invocation(events, inp["skill"])
335
+ _record_skill_invocation(events, inp["skill"], explicit=True)
197
336
 
198
337
  # Capture tool results and match to their tool_use calls
199
338
  if msg.get("type") == "user":
@@ -540,14 +679,15 @@ class ExperimentLogger:
540
679
  self.name = experiment_name or f"experiment_{self.timestamp}"
541
680
  self.experiment_id = f"{self.name}_{self.timestamp}"
542
681
 
543
- self.base_dir = get_logs_dir() / "experiments" / self.experiment_id
682
+ self.base_dir = get_runs_dir() / self.experiment_id
544
683
 
545
684
  # Create subdirectories
546
685
  self.events_dir = self.base_dir / "events"
547
686
  self.reports_dir = self.base_dir / "reports"
548
687
  self.raw_dir = self.base_dir / "raw"
549
688
 
550
- for d in [self.events_dir, self.reports_dir, self.raw_dir]:
689
+ self.artifacts_dir = self.base_dir / "artifacts"
690
+ for d in [self.events_dir, self.reports_dir, self.raw_dir, self.artifacts_dir]:
551
691
  d.mkdir(parents=True, exist_ok=True)
552
692
 
553
693
  self.columns = list(columns) if columns else rubric_columns()
@@ -757,7 +897,7 @@ class ExperimentLogger:
757
897
  self.metadata["report_outputs"] = {name: str(path) for name, path in written.items()}
758
898
 
759
899
  metadata_path = self.base_dir / "metadata.json"
760
- metadata_path.write_text(json.dumps(self.metadata, indent=2), encoding="utf-8")
900
+ _atomic_write_json(metadata_path, self.metadata)
761
901
 
762
902
  print(f"\nExperiment results saved to: {self.base_dir}")
763
903
  if written:
@@ -778,7 +918,7 @@ def save_events(base_dir: Path, treatment_name: str, rep: int, events: dict[str,
778
918
  events_dir = base_dir / "events"
779
919
  events_dir.mkdir(parents=True, exist_ok=True)
780
920
  save_path = events_dir / f"{stable_treatment_name(treatment_name)}_rep{rep}.json"
781
- save_path.write_text(json.dumps(events, indent=2), encoding="utf-8")
921
+ save_path.write_text(json.dumps(redact_sensitive(events), indent=2), encoding="utf-8")
782
922
  return save_path
783
923
 
784
924
 
@@ -788,11 +928,11 @@ def save_raw(base_dir: Path, treatment_name: str, rep: int, stdout: str, stderr:
788
928
  raw_dir.mkdir(parents=True, exist_ok=True)
789
929
  stable_name = stable_treatment_name(treatment_name)
790
930
  stdout_path = raw_dir / f"{stable_name}_rep{rep}_stdout.json"
791
- stdout_path.write_text(stdout, encoding="utf-8")
931
+ stdout_path.write_text(redact_sensitive(stdout), encoding="utf-8")
792
932
 
793
933
  if stderr:
794
934
  stderr_path = raw_dir / f"{stable_name}_rep{rep}_stderr.txt"
795
- stderr_path.write_text(stderr, encoding="utf-8")
935
+ stderr_path.write_text(redact_sensitive(stderr), encoding="utf-8")
796
936
 
797
937
 
798
938
  def save_report(base_dir: Path, treatment_name: str, rep: int, report: dict[str, Any]):
@@ -800,5 +940,5 @@ def save_report(base_dir: Path, treatment_name: str, rep: int, report: dict[str,
800
940
  reports_dir = base_dir / "reports"
801
941
  reports_dir.mkdir(parents=True, exist_ok=True)
802
942
  save_path = reports_dir / f"{stable_treatment_name(treatment_name)}_rep{rep}_report.json"
803
- save_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
943
+ save_path.write_text(json.dumps(redact_sensitive(report), indent=2), encoding="utf-8")
804
944
  return save_path
@@ -0,0 +1,69 @@
1
+ """Normalize project-authored manifest tasks into the existing Task contract."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import replace
6
+ from pathlib import Path
7
+
8
+ from scaffold.python.manifests import ManifestTask, SkillEvalManifest
9
+ from scaffold.python.paths import EVAL_ROOT, get_tasks_dir
10
+ from scaffold.python.tasks import (
11
+ EvaluationConfig,
12
+ Task,
13
+ TaskConfig,
14
+ ValidationConfig,
15
+ load_task_from_path,
16
+ )
17
+
18
+
19
+ def _generic_environment_dir() -> Path:
20
+ candidates = (
21
+ get_tasks_dir() / "generic-skill-smoke" / "environment",
22
+ EVAL_ROOT / "local" / "tasks" / "generic-skill-smoke" / "environment",
23
+ )
24
+ for candidate in candidates:
25
+ if (candidate / "Dockerfile").is_file():
26
+ return candidate
27
+ raise FileNotFoundError("Bundled generic task environment is unavailable")
28
+
29
+
30
+ def _inline_task(manifest: SkillEvalManifest, spec: ManifestTask) -> Task:
31
+ assert spec.prompt is not None
32
+ files = list(spec.expect.get("files", []))
33
+ config = TaskConfig(
34
+ name=spec.name,
35
+ description=f"Inline task authored in {manifest.path}",
36
+ default_treatments=[],
37
+ timeout_sec=900,
38
+ validation=ValidationConfig(target_artifacts=files, timeout=120),
39
+ evaluation=EvaluationConfig(
40
+ profile=manifest.profile or "generic",
41
+ expected_artifacts=files,
42
+ rubric_criteria=list(spec.rubric),
43
+ ),
44
+ interaction=manifest.interaction,
45
+ )
46
+ return Task(
47
+ path=manifest.path.parent,
48
+ config=config,
49
+ instruction_template=spec.prompt,
50
+ _environment_dir=_generic_environment_dir(),
51
+ workspace_dir=spec.workspace,
52
+ manifest_expectations=spec.expect,
53
+ manifest_path=manifest.path,
54
+ )
55
+
56
+
57
+ def load_manifest_tasks(manifest: SkillEvalManifest) -> list[Task]:
58
+ """Load all authored inline/source task entries in manifest order."""
59
+ tasks: list[Task] = []
60
+ for spec in manifest.tasks:
61
+ task = _inline_task(manifest, spec) if spec.is_inline else load_task_from_path(spec.source)
62
+ # ``evaluation.tasks[].name`` is the public identity, including for a
63
+ # sourced package whose internal metadata name differs. Keep the full
64
+ # package semantics loaded by ``load_task_from_path`` while applying
65
+ # the manifest alias consistently to catalogue, selection, and reports.
66
+ if task.name != spec.name:
67
+ task = replace(task, config=replace(task.config, name=spec.name))
68
+ tasks.append(task)
69
+ return tasks