@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +66 -26
  2. package/assets/manifest.json +4 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
  5. package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
  10. package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
  11. package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
  12. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  13. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
  14. package/assets/skills/comet-build/SKILL.md +36 -62
  15. package/assets/skills/comet-classic/SKILL.md +14 -8
  16. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  17. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  18. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  19. package/assets/skills/comet-design/SKILL.md +2 -2
  20. package/assets/skills/comet-native/SKILL.md +18 -15
  21. package/assets/skills/comet-native/reference/artifacts.md +5 -4
  22. package/assets/skills/comet-native/reference/clarification.md +1 -1
  23. package/assets/skills/comet-native/reference/commands.md +4 -4
  24. package/assets/skills/comet-native/reference/recovery.md +1 -1
  25. package/assets/skills/comet-native/reference/workspace.md +4 -4
  26. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
  27. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
  28. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  29. package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
  30. package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
  31. package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
  32. package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
  33. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
  34. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  35. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  36. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
  37. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  38. package/assets/skills/comet-open/SKILL.md +24 -5
  39. package/assets/skills/comet-review/SKILL.md +121 -0
  40. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  41. package/assets/skills-zh/comet-build/SKILL.md +36 -62
  42. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  43. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  44. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  45. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  46. package/assets/skills-zh/comet-design/SKILL.md +2 -2
  47. package/assets/skills-zh/comet-native/SKILL.md +18 -15
  48. package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
  49. package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
  50. package/assets/skills-zh/comet-native/reference/commands.md +4 -4
  51. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  52. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  53. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  54. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  55. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  56. package/dist/app/cli/index.js +20 -1
  57. package/dist/app/cli/index.js.map +1 -1
  58. package/dist/app/commands/classic.d.ts.map +1 -1
  59. package/dist/app/commands/classic.js +2 -0
  60. package/dist/app/commands/classic.js.map +1 -1
  61. package/dist/app/commands/eval.d.ts +9 -2
  62. package/dist/app/commands/eval.d.ts.map +1 -1
  63. package/dist/app/commands/eval.js +293 -48
  64. package/dist/app/commands/eval.js.map +1 -1
  65. package/dist/app/commands/init.d.ts.map +1 -1
  66. package/dist/app/commands/init.js +5 -8
  67. package/dist/app/commands/init.js.map +1 -1
  68. package/dist/app/commands/update.d.ts.map +1 -1
  69. package/dist/app/commands/update.js +5 -6
  70. package/dist/app/commands/update.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  72. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  73. package/dist/domains/comet-classic/classic-cli.js +3 -0
  74. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  75. package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
  76. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
  77. package/dist/domains/comet-classic/classic-command-context.js +10 -0
  78. package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
  79. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  80. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  81. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  82. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  83. package/dist/domains/comet-classic/classic-guard.js +4 -4
  84. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  85. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  86. package/dist/domains/comet-classic/classic-handoff.js +44 -6
  87. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  88. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  89. package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
  90. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  91. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  92. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  93. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  94. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
  95. package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
  96. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
  97. package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
  98. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  99. package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
  100. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  101. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  102. package/dist/domains/comet-classic/classic-state-command.js +39 -13
  103. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  104. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  105. package/dist/domains/comet-classic/classic-validate-command.js +2 -2
  106. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  107. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  108. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  109. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  110. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  111. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  112. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  113. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  114. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  115. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  116. package/dist/domains/comet-entry/hook-adapter.js +5 -1
  117. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  118. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  119. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  120. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  121. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  122. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  123. package/dist/domains/comet-entry/init-workflow.js +13 -5
  124. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  125. package/dist/domains/comet-native/index.d.ts +1 -0
  126. package/dist/domains/comet-native/index.d.ts.map +1 -1
  127. package/dist/domains/comet-native/index.js +1 -0
  128. package/dist/domains/comet-native/index.js.map +1 -1
  129. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
  130. package/dist/domains/comet-native/native-archive-command.js +2 -0
  131. package/dist/domains/comet-native/native-archive-command.js.map +1 -1
  132. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
  133. package/dist/domains/comet-native/native-atomic-file.js +17 -10
  134. package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
  135. package/dist/domains/comet-native/native-children.d.ts +65 -0
  136. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  137. package/dist/domains/comet-native/native-children.js +453 -0
  138. package/dist/domains/comet-native/native-children.js.map +1 -0
  139. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  140. package/dist/domains/comet-native/native-cli-help.js +13 -7
  141. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  142. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
  143. package/dist/domains/comet-native/native-cli-shared.js +18 -5
  144. package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
  145. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  146. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  147. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  148. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  149. package/dist/domains/comet-native/native-hook-guard.js +56 -4
  150. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  151. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  152. package/dist/domains/comet-native/native-next-command.js +179 -20
  153. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  154. package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
  155. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  156. package/dist/domains/comet-native/native-portable-continuation.js +178 -73
  157. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  158. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
  159. package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
  160. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
  161. package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
  162. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  163. package/dist/domains/comet-native/native-portable-runtime.js +168 -4
  164. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  165. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  166. package/dist/domains/comet-native/native-portable-state.js +13 -0
  167. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  168. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  169. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  170. package/dist/domains/comet-native/native-portable-status.js +9 -1
  171. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  172. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  173. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  174. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  175. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  176. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  177. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  178. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  179. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  180. package/dist/domains/comet-native/native-show-command.js +2 -1
  181. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  182. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
  183. package/dist/domains/comet-native/native-status-discovery.js +24 -6
  184. package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
  185. package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
  186. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
  187. package/dist/domains/comet-native/native-workspace-finish.js +86 -35
  188. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
  189. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  190. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  191. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  192. package/dist/domains/dashboard/collector.d.ts +1 -1
  193. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  194. package/dist/domains/dashboard/collector.js +102 -87
  195. package/dist/domains/dashboard/collector.js.map +1 -1
  196. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  197. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  198. package/dist/domains/dashboard/native-adapter.js +11 -0
  199. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  200. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  201. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  202. package/dist/domains/dashboard/native-collector.js +304 -55
  203. package/dist/domains/dashboard/native-collector.js.map +1 -1
  204. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  205. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  206. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  207. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  208. package/dist/domains/dashboard/server.d.ts.map +1 -1
  209. package/dist/domains/dashboard/server.js +5 -3
  210. package/dist/domains/dashboard/server.js.map +1 -1
  211. package/dist/domains/dashboard/types.d.ts +5 -0
  212. package/dist/domains/dashboard/types.d.ts.map +1 -1
  213. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  214. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  215. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  216. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  217. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
  218. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  219. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
  220. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
  221. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  222. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
  223. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  224. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
  225. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
  226. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
  227. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  228. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
  229. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  230. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  231. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
  232. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
  233. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  234. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  235. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  236. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  237. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
  238. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  239. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  240. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  241. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  242. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  243. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  244. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  245. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
  246. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  247. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
  248. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  249. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  250. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
  251. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
  252. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
  253. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
  254. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  255. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
  256. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  257. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  258. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
  259. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  260. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  261. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  262. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
  263. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  264. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
  265. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  266. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  267. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  268. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  269. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  270. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  271. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  272. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
  273. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  274. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
  275. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  276. package/dist/domains/dashboard/web/index.html +2 -2
  277. package/dist/domains/dashboard/workspace.d.ts +27 -0
  278. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  279. package/dist/domains/dashboard/workspace.js +108 -0
  280. package/dist/domains/dashboard/workspace.js.map +1 -0
  281. package/dist/domains/eval/index.d.ts +3 -0
  282. package/dist/domains/eval/index.d.ts.map +1 -1
  283. package/dist/domains/eval/index.js +3 -0
  284. package/dist/domains/eval/index.js.map +1 -1
  285. package/dist/domains/eval/standalone-context.d.ts +19 -0
  286. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  287. package/dist/domains/eval/standalone-context.js +107 -0
  288. package/dist/domains/eval/standalone-context.js.map +1 -0
  289. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  290. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  291. package/dist/domains/eval/standalone-static-collect.js +779 -0
  292. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  293. package/dist/domains/eval/user-environment.d.ts +6 -0
  294. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  295. package/dist/domains/eval/user-environment.js +73 -0
  296. package/dist/domains/eval/user-environment.js.map +1 -0
  297. package/dist/domains/integrations/openspec.d.ts +1 -1
  298. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  299. package/dist/domains/integrations/openspec.js +87 -44
  300. package/dist/domains/integrations/openspec.js.map +1 -1
  301. package/dist/domains/integrations/superpowers.d.ts +14 -1
  302. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  303. package/dist/domains/integrations/superpowers.js +93 -5
  304. package/dist/domains/integrations/superpowers.js.map +1 -1
  305. package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
  306. package/dist/domains/skill/platform-inspect.js +3 -3
  307. package/dist/domains/skill/platform-inspect.js.map +1 -1
  308. package/dist/domains/skill/platform-install.d.ts +4 -2
  309. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  310. package/dist/domains/skill/platform-install.js +16 -10
  311. package/dist/domains/skill/platform-install.js.map +1 -1
  312. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  313. package/dist/domains/skill/project-instructions.js +11 -3
  314. package/dist/domains/skill/project-instructions.js.map +1 -1
  315. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  316. package/dist/domains/skill/uninstall.js +39 -29
  317. package/dist/domains/skill/uninstall.js.map +1 -1
  318. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
  319. package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
  320. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
  321. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  322. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  323. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  324. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  325. package/dist/domains/workflow-contract/index.d.ts +2 -0
  326. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  327. package/dist/domains/workflow-contract/index.js +1 -0
  328. package/dist/domains/workflow-contract/index.js.map +1 -1
  329. package/dist/domains/workflow-contract/project-config.d.ts +4 -2
  330. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  331. package/dist/domains/workflow-contract/project-config.js +174 -2
  332. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  333. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  334. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  335. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  336. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  337. package/dist/domains/workflow-contract/types.d.ts +13 -0
  338. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  339. package/dist/platform/fs/strip-bom.d.ts +10 -0
  340. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  341. package/dist/platform/fs/strip-bom.js +12 -0
  342. package/dist/platform/fs/strip-bom.js.map +1 -0
  343. package/dist/platform/install/platforms.d.ts +11 -0
  344. package/dist/platform/install/platforms.d.ts.map +1 -1
  345. package/dist/platform/install/platforms.js +49 -2
  346. package/dist/platform/install/platforms.js.map +1 -1
  347. package/dist/platform/paths/git-worktree.d.ts +7 -1
  348. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  349. package/dist/platform/paths/git-worktree.js +28 -7
  350. package/dist/platform/paths/git-worktree.js.map +1 -1
  351. package/dist/platform/process/external-command.d.ts +1 -0
  352. package/dist/platform/process/external-command.d.ts.map +1 -1
  353. package/dist/platform/process/external-command.js +2 -1
  354. package/dist/platform/process/external-command.js.map +1 -1
  355. package/eval/.env.example +115 -0
  356. package/eval/README.md +237 -14
  357. package/eval/langfuse/.env.example +9 -0
  358. package/eval/langfuse/tests/conftest.py +146 -0
  359. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  360. package/eval/langfuse/tests/test_adapter.py +272 -0
  361. package/eval/langsmith/.env.example +16 -0
  362. package/eval/langsmith/tests/conftest.py +10 -1
  363. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  364. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  365. package/eval/local/tests/conftest.py +719 -83
  366. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  367. package/eval/local/tests/scaffold/test_agents.py +76 -0
  368. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  369. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  370. package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
  371. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  372. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  373. package/eval/local/tests/scaffold/test_execution.py +364 -0
  374. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  375. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  376. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  377. package/eval/local/tests/scaffold/test_logging.py +162 -0
  378. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  379. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  380. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  381. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  382. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  383. package/eval/local/tests/scaffold/test_utils.py +480 -4
  384. package/eval/local/tests/tasks/test_tasks.py +216 -14
  385. package/eval/pyproject.toml +5 -0
  386. package/eval/scaffold/__init__.py +6 -0
  387. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  388. package/eval/scaffold/python/__init__.py +128 -20
  389. package/eval/scaffold/python/agents.py +239 -0
  390. package/eval/scaffold/python/aligned_comparison.py +156 -27
  391. package/eval/scaffold/python/auto_tasks.py +700 -0
  392. package/eval/scaffold/python/custom_agents.py +172 -0
  393. package/eval/scaffold/python/eval_context.py +236 -0
  394. package/eval/scaffold/python/eval_schema.py +135 -0
  395. package/eval/scaffold/python/execution.py +544 -0
  396. package/eval/scaffold/python/generated_task_cache.py +230 -0
  397. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  398. package/eval/scaffold/python/inline_tasks.py +118 -0
  399. package/eval/scaffold/python/judge_config.py +158 -37
  400. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  401. package/eval/scaffold/python/llm_judge.py +68 -11
  402. package/eval/scaffold/python/logging.py +157 -17
  403. package/eval/scaffold/python/manifest_tasks.py +69 -0
  404. package/eval/scaffold/python/manifests.py +443 -22
  405. package/eval/scaffold/python/paths.py +11 -1
  406. package/eval/scaffold/python/task_resolution.py +176 -0
  407. package/eval/scaffold/python/tasks.py +17 -4
  408. package/eval/scaffold/python/utils.py +167 -11
  409. package/eval/scaffold/python/validation/core.py +48 -2
  410. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  411. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  412. package/eval/scaffold/python/validation/rubric.py +16 -1
  413. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  414. package/eval/scaffold/shell/docker.sh +534 -33
  415. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  416. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  417. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  418. package/package.json +18 -28
  419. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  420. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  421. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  422. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  423. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  424. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  425. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  426. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  427. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  428. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  429. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  430. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  431. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  432. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  433. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  434. package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
  435. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  436. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  437. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  438. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  439. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  440. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  441. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  442. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  443. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  444. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  445. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
  446. package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
@@ -0,0 +1,544 @@
1
+ """Resolve standalone eval execution settings and isolate provider environments."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import re
7
+ from dataclasses import dataclass
8
+ from typing import Any, Mapping
9
+ from urllib.parse import urlsplit
10
+
11
+ from scaffold.python.agents import AgentId, DEFAULT_AGENT, get_agent_adapter, validate_agent_id
12
+
13
+
14
+ _MISSING = object()
15
+ _MODEL_ENV: dict[AgentId, tuple[str, ...]] = {
16
+ "claude-code": ("BENCH_CC_MODEL", "ANTHROPIC_MODEL"),
17
+ "codex": ("BENCH_CODEX_MODEL", "OPENAI_MODEL", "CODEX_MODEL"),
18
+ "qoder": ("BENCH_QODER_MODEL", "QODER_MODEL"),
19
+ "codebuddy": ("BENCH_CODEBUDDY_MODEL", "CODEBUDDY_MODEL"),
20
+ }
21
+ _COMMON_MODEL_AGENTS = frozenset(_MODEL_ENV)
22
+ _BASE_URL_ENV: dict[AgentId, tuple[str, ...]] = {
23
+ "claude-code": ("ANTHROPIC_BASE_URL",),
24
+ "codex": ("OPENAI_BASE_URL", "CODEX_BASE_URL"),
25
+ "qoder": ("QODER_BASE_URL",),
26
+ "codebuddy": ("CODEBUDDY_BASE_URL",),
27
+ }
28
+ _COMMON_BASE_URL_AGENTS = frozenset({"claude-code", "codex", "codebuddy"})
29
+ _MODEL_TARGET_ENV: dict[AgentId, str] = {
30
+ "claude-code": "ANTHROPIC_MODEL",
31
+ "codex": "OPENAI_MODEL",
32
+ "qoder": "QODER_MODEL",
33
+ "codebuddy": "CODEBUDDY_MODEL",
34
+ }
35
+ _BASE_URL_TARGET_ENV: dict[AgentId, str] = {
36
+ "claude-code": "ANTHROPIC_BASE_URL",
37
+ "codex": "OPENAI_BASE_URL",
38
+ "qoder": "QODER_BASE_URL",
39
+ "codebuddy": "CODEBUDDY_BASE_URL",
40
+ }
41
+ _CREDENTIAL_ENV: dict[AgentId, tuple[str, ...]] = {
42
+ "claude-code": ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"),
43
+ "codex": ("OPENAI_API_KEY", "CODEX_API_KEY"),
44
+ "qoder": ("QODER_PERSONAL_ACCESS_TOKEN",),
45
+ "codebuddy": ("CODEBUDDY_API_KEY", "CODEBUDDY_AUTH_TOKEN"),
46
+ }
47
+ _PROVIDER_KEYS = {
48
+ key
49
+ for keys in (*_MODEL_ENV.values(), *_BASE_URL_ENV.values(), *_CREDENTIAL_ENV.values())
50
+ for key in keys
51
+ }
52
+ _MAIN_OVERRIDE_KEYS = {
53
+ "BENCH_EVAL_AGENT",
54
+ "BENCH_CC_MODEL",
55
+ "BENCH_CODEX_MODEL",
56
+ "BENCH_QODER_MODEL",
57
+ "BENCH_CODEBUDDY_MODEL",
58
+ "OPENAI_MODEL",
59
+ "CODEX_MODEL",
60
+ "QODER_MODEL",
61
+ "CODEBUDDY_MODEL",
62
+ "BENCH_BASE_URL",
63
+ "BENCH_API_KEY",
64
+ "BENCH_MODEL",
65
+ }
66
+ _JUDGE_KEYS = {
67
+ "BENCH_LLM_JUDGE",
68
+ "BENCH_JUDGE_MODEL",
69
+ "BENCH_JUDGE_BASE_URL",
70
+ "BENCH_JUDGE_API_KEY",
71
+ "BENCH_JUDGE_AUTH_TOKEN",
72
+ "BENCH_JUDGE_AGENT",
73
+ "COMET_EVAL_MAIN_CREDENTIALS",
74
+ }
75
+ _CREDENTIAL_NAME_RE = re.compile(
76
+ r"(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|PERSONAL_ACCESS_TOKEN|PASSWORD|SECRET|CREDENTIALS?|(?:^|_)(?:KEY|TOKEN))$",
77
+ re.I,
78
+ )
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class ResolvedExecution:
83
+ agent: AgentId
84
+ model: str | None
85
+ base_url: str | None
86
+ sources: dict[str, str]
87
+
88
+
89
+ @dataclass(frozen=True)
90
+ class ResolvedJudge:
91
+ agent: AgentId
92
+ model: str
93
+ base_url: str | None
94
+ sources: dict[str, str]
95
+
96
+
97
+ def _value(source: object, name: str, default: object = _MISSING) -> object:
98
+ if isinstance(source, Mapping):
99
+ if name in source:
100
+ return source[name]
101
+ return default
102
+ value = getattr(source, name, default)
103
+ return value
104
+
105
+
106
+ def _first_non_empty(values: list[object]) -> str | None:
107
+ for value in values:
108
+ if isinstance(value, str) and value.strip():
109
+ return value.strip()
110
+ return None
111
+
112
+
113
+ def validate_base_url(value: object, *, field: str = "baseUrl") -> str | None:
114
+ if value is None:
115
+ return None
116
+ if not isinstance(value, str) or value.strip() != value or not value:
117
+ raise ValueError(f"{field} must be an absolute http(s) URL")
118
+ parsed = urlsplit(value)
119
+ try:
120
+ parsed.port
121
+ except ValueError as exc:
122
+ raise ValueError(f"{field} must be an absolute http(s) URL") from exc
123
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
124
+ raise ValueError(f"{field} must be an absolute http(s) URL")
125
+ if parsed.username or parsed.password or parsed.query or parsed.fragment:
126
+ raise ValueError(f"{field} must not contain credentials, query, or fragment")
127
+ return value
128
+
129
+
130
+ def _declared_credential_names(environment: Mapping[str, str]) -> set[str]:
131
+ names: set[str] = set()
132
+ for key in ("COMET_EVAL_CUSTOM_CREDENTIALS", "COMET_EVAL_MAIN_CREDENTIALS"):
133
+ names.update(
134
+ item.strip()
135
+ for item in environment.get(key, "").split(",")
136
+ if item.strip()
137
+ )
138
+ return names
139
+
140
+
141
+ def _credential_values(environment: Mapping[str, str]) -> tuple[str, ...]:
142
+ declared = _declared_credential_names(environment)
143
+ values = {
144
+ value.strip()
145
+ for key, value in environment.items()
146
+ if value.strip() and (key in declared or _CREDENTIAL_NAME_RE.search(key))
147
+ }
148
+ return tuple(sorted((value for value in values if len(value) >= 4), key=len, reverse=True))
149
+
150
+
151
+ def redact_sensitive(value: Any, source_env: Mapping[str, str] | None = None) -> Any:
152
+ """Redact credential values before eval data reaches user-owned artifacts."""
153
+
154
+ environment = source_env if source_env is not None else os.environ
155
+ declared = _declared_credential_names(environment)
156
+ if isinstance(value, dict):
157
+ return {
158
+ key: "[REDACTED]"
159
+ if key in declared or _CREDENTIAL_NAME_RE.search(str(key))
160
+ else redact_sensitive(item, environment)
161
+ for key, item in value.items()
162
+ }
163
+ if isinstance(value, list):
164
+ return [redact_sensitive(item, environment) for item in value]
165
+ if not isinstance(value, str):
166
+ return value
167
+ redacted = re.sub(
168
+ r"(?i)(authorization\s*:\s*bearer\s+)[^\s,;]+",
169
+ r"\1[REDACTED]",
170
+ value,
171
+ )
172
+ redacted = re.sub(r"(?i)(x-api-key\s*:\s*)[^\s,;]+", r"\1[REDACTED]", redacted)
173
+ redacted = re.sub(
174
+ r"(?i)(api[_-]?key|auth[_-]?token|access[_-]?token|token|secret|password|credential)\s*[=:]\s*[^\s,;]+",
175
+ r"\1=[REDACTED]",
176
+ redacted,
177
+ )
178
+ redacted = re.sub(
179
+ r"(?i)([?&](?:api[_-]?key|token|access[_-]?token|auth[_-]?token)=)[^&#\s]+",
180
+ r"\1[REDACTED]",
181
+ redacted,
182
+ )
183
+ redacted = re.sub(r"(https?://)[^/@\s:]+:[^/@\s]+@", r"\1[REDACTED]@", redacted)
184
+ for secret in _credential_values(environment):
185
+ redacted = redacted.replace(secret, "[REDACTED]")
186
+ return redacted
187
+
188
+
189
+ def _manifest_value(manifest: object | None, name: str) -> object:
190
+ value = _value(manifest, name) if manifest is not None else _MISSING
191
+ if value is not _MISSING:
192
+ return value
193
+ return None
194
+
195
+
196
+ def _model_env_keys(agent: AgentId) -> tuple[str, ...]:
197
+ adapter = get_agent_adapter(agent)
198
+ return (*_MODEL_ENV.get(agent, ()), *(key for key in (adapter.model_env,) if key))
199
+
200
+
201
+ def _base_url_env_keys(agent: AgentId) -> tuple[str, ...]:
202
+ adapter = get_agent_adapter(agent)
203
+ return (*_BASE_URL_ENV.get(agent, ()), *(key for key in (adapter.base_url_env,) if key))
204
+
205
+
206
+ def resolve_execution(
207
+ *,
208
+ cli_agent: object | None = None,
209
+ cli_model: object | None = None,
210
+ cli_base_url: object | None = None,
211
+ manifest: object | None = None,
212
+ source_env: Mapping[str, str] | None = None,
213
+ ) -> ResolvedExecution:
214
+ """Resolve main execution settings using CLI > manifest > legacy env > default."""
215
+
216
+ environment = source_env if source_env is not None else os.environ
217
+ manifest_agent = _manifest_value(manifest, "agent")
218
+ agent_value = _first_non_empty(
219
+ [cli_agent, manifest_agent, environment.get("BENCH_EVAL_AGENT"), DEFAULT_AGENT]
220
+ )
221
+ agent = validate_agent_id(agent_value, field="evaluation agent")
222
+
223
+ manifest_model = _manifest_value(manifest, "model")
224
+ model_candidates: list[object] = [cli_model, manifest_model]
225
+ model_candidates.extend(environment.get(key) for key in _model_env_keys(agent))
226
+ if agent in _COMMON_MODEL_AGENTS:
227
+ model_candidates.append(environment.get("BENCH_MODEL"))
228
+ model = _first_non_empty(model_candidates)
229
+
230
+ manifest_base_url = _manifest_value(manifest, "base_url")
231
+ if manifest_base_url is None:
232
+ manifest_base_url = _manifest_value(manifest, "baseUrl")
233
+ base_url_candidates: list[object] = [cli_base_url, manifest_base_url]
234
+ base_url_candidates.extend(environment.get(key) for key in _base_url_env_keys(agent))
235
+ if agent in _COMMON_BASE_URL_AGENTS:
236
+ base_url_candidates.append(environment.get("BENCH_BASE_URL"))
237
+ base_url = _first_non_empty(base_url_candidates)
238
+ base_url = validate_base_url(base_url, field="execution.baseUrl")
239
+ adapter = get_agent_adapter(agent)
240
+ if adapter.custom and base_url and not adapter.base_url_env:
241
+ raise ValueError(
242
+ f"custom Agent {agent} must declare baseUrlEnv before execution.baseUrl can be used"
243
+ )
244
+
245
+ return ResolvedExecution(
246
+ agent=agent,
247
+ model=model,
248
+ base_url=base_url,
249
+ sources={
250
+ "agent": "cli"
251
+ if _first_non_empty([cli_agent])
252
+ else "manifest"
253
+ if _first_non_empty([manifest_agent])
254
+ else "environment"
255
+ if _first_non_empty([environment.get("BENCH_EVAL_AGENT")])
256
+ else "default",
257
+ "model": "cli"
258
+ if _first_non_empty([cli_model])
259
+ else "manifest"
260
+ if _first_non_empty([manifest_model])
261
+ else "environment"
262
+ if model
263
+ else "default",
264
+ "base_url": "cli"
265
+ if _first_non_empty([cli_base_url])
266
+ else "manifest"
267
+ if _first_non_empty([manifest_base_url])
268
+ else "environment"
269
+ if base_url
270
+ else "default",
271
+ },
272
+ )
273
+
274
+
275
+ def resolve_judge(
276
+ *,
277
+ main: ResolvedExecution,
278
+ cli_agent: object | None = None,
279
+ cli_model: object | None = None,
280
+ cli_base_url: object | None = None,
281
+ manifest: object = None,
282
+ source_env: Mapping[str, str] | None = None,
283
+ ) -> ResolvedJudge | None:
284
+ """Resolve the optional independent Judge configuration.
285
+
286
+ The main model, base URL, and credentials are deliberately not fallbacks
287
+ for Judge. Only the main Agent ID is an allowed implicit inheritance.
288
+ """
289
+
290
+ environment = source_env if source_env is not None else os.environ
291
+ enabled = (
292
+ manifest is not None
293
+ or any(value is not None for value in (cli_agent, cli_model, cli_base_url))
294
+ or str(environment.get("BENCH_LLM_JUDGE", "")).strip().lower()
295
+ in {"1", "true", "yes", "on"}
296
+ )
297
+ if not enabled:
298
+ return None
299
+
300
+ manifest_agent = _manifest_value(manifest, "agent")
301
+ agent_value = _first_non_empty([cli_agent, manifest_agent, main.agent])
302
+ agent = validate_agent_id(agent_value, field="judge evaluation agent")
303
+
304
+ manifest_model = _manifest_value(manifest, "model")
305
+ model = _first_non_empty([cli_model, manifest_model, environment.get("BENCH_JUDGE_MODEL")])
306
+ if model is None:
307
+ if manifest == {} or str(environment.get("BENCH_LLM_JUDGE", "")).strip().lower() in {
308
+ "1",
309
+ "true",
310
+ "yes",
311
+ "on",
312
+ }:
313
+ raise ValueError("BENCH_JUDGE_MODEL is required when BENCH_LLM_JUDGE=1")
314
+ raise ValueError("judge.model is required when LLM-as-Judge is enabled")
315
+
316
+ manifest_base_url = _manifest_value(manifest, "base_url")
317
+ if manifest_base_url is None:
318
+ manifest_base_url = _manifest_value(manifest, "baseUrl")
319
+ base_url = _first_non_empty(
320
+ [cli_base_url, manifest_base_url, environment.get("BENCH_JUDGE_BASE_URL")]
321
+ )
322
+ base_url = validate_base_url(base_url, field="judge.baseUrl")
323
+
324
+ return ResolvedJudge(
325
+ agent=agent,
326
+ model=model,
327
+ base_url=base_url,
328
+ sources={
329
+ "agent": "cli"
330
+ if cli_agent
331
+ else "manifest"
332
+ if manifest_agent
333
+ else "inherited",
334
+ "model": "cli"
335
+ if cli_model
336
+ else "manifest"
337
+ if manifest_model
338
+ else "environment",
339
+ "base_url": "cli"
340
+ if cli_base_url
341
+ else "manifest"
342
+ if manifest_base_url
343
+ else "environment"
344
+ if base_url
345
+ else "default",
346
+ },
347
+ )
348
+
349
+
350
+ def _set_if_value(environment: dict[str, str], key: str, value: str | None) -> None:
351
+ if not key:
352
+ return
353
+ if value:
354
+ environment[key] = value
355
+ else:
356
+ environment.pop(key, None)
357
+
358
+
359
+ def build_agent_environment(
360
+ execution: ResolvedExecution,
361
+ *,
362
+ source_env: Mapping[str, str] | None = None,
363
+ ) -> dict[str, str]:
364
+ """Map resolved main settings to the selected adapter's env contract."""
365
+
366
+ environment = dict(source_env if source_env is not None else os.environ)
367
+ for key in _JUDGE_KEYS:
368
+ environment.pop(key, None)
369
+ for key in tuple(environment):
370
+ if key.startswith("COMET_EVAL_CUSTOM_"):
371
+ environment.pop(key, None)
372
+ adapter = get_agent_adapter(execution.agent)
373
+ _set_if_value(environment, adapter.model_env, execution.model)
374
+ _set_if_value(environment, adapter.base_url_env, execution.base_url)
375
+ if not adapter.custom and not any(
376
+ environment.get(key) for key in _CREDENTIAL_ENV.get(execution.agent, ())
377
+ ):
378
+ credential_keys = _CREDENTIAL_ENV.get(execution.agent, ())
379
+ if credential_keys:
380
+ _set_if_value(environment, credential_keys[0], environment.get("BENCH_API_KEY"))
381
+ if adapter.custom:
382
+ environment["COMET_EVAL_CUSTOM_AGENT_ID"] = adapter.id
383
+ environment["COMET_EVAL_CUSTOM_EXECUTABLE"] = adapter.executable
384
+ environment["COMET_EVAL_CUSTOM_CREDENTIALS"] = ",".join(adapter.required_credentials)
385
+ environment["COMET_EVAL_CUSTOM_MODEL_ENV"] = adapter.model_env
386
+ environment["COMET_EVAL_CUSTOM_BASE_URL_ENV"] = adapter.base_url_env
387
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_MODEL", execution.model)
388
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_BASE_URL", execution.base_url)
389
+ environment["COMET_EVAL_CUSTOM_INSTALL_KIND"] = adapter.install_kind
390
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_PACKAGE", adapter.install_package)
391
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_VERSION", adapter.install_version)
392
+ return environment
393
+
394
+
395
+ def build_judge_environment(
396
+ judge: ResolvedJudge,
397
+ *,
398
+ source_env: Mapping[str, str] | None = None,
399
+ excluded_credentials: tuple[str, ...] = (),
400
+ ) -> dict[str, str]:
401
+ """Create a child environment containing only the dedicated Judge secrets."""
402
+
403
+ source = source_env if source_env is not None else os.environ
404
+ excluded = set(excluded_credentials) | _declared_credential_names(source)
405
+ environment = {
406
+ key: value
407
+ for key, value in source.items()
408
+ if key not in excluded
409
+ and key not in _PROVIDER_KEYS
410
+ and key not in _MAIN_OVERRIDE_KEYS
411
+ and key not in _JUDGE_KEYS
412
+ and not key.startswith("BENCH_JUDGE_")
413
+ and not key.startswith("COMET_EVAL_CUSTOM_")
414
+ and not re.search(
415
+ r"(?:^|_)(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|PERSONAL_ACCESS_TOKEN|MODEL|BASE_URL|KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?)$",
416
+ key,
417
+ )
418
+ }
419
+ api_key = source.get("BENCH_JUDGE_API_KEY", "")
420
+ auth_token = source.get("BENCH_JUDGE_AUTH_TOKEN", "")
421
+ adapter = get_agent_adapter(judge.agent)
422
+ if adapter.custom:
423
+ primary_credential = api_key or auth_token
424
+ for index, key in enumerate(adapter.required_credentials):
425
+ _set_if_value(
426
+ environment,
427
+ key,
428
+ primary_credential if index == 0 else auth_token or api_key,
429
+ )
430
+ elif judge.agent == "claude-code":
431
+ _set_if_value(environment, "ANTHROPIC_API_KEY", api_key)
432
+ _set_if_value(environment, "ANTHROPIC_AUTH_TOKEN", auth_token)
433
+ elif judge.agent == "codex":
434
+ _set_if_value(environment, "OPENAI_API_KEY", api_key)
435
+ _set_if_value(environment, "CODEX_API_KEY", auth_token)
436
+ elif judge.agent == "qoder":
437
+ _set_if_value(environment, "QODER_PERSONAL_ACCESS_TOKEN", auth_token or api_key)
438
+ else:
439
+ _set_if_value(environment, "CODEBUDDY_API_KEY", api_key)
440
+ _set_if_value(environment, "CODEBUDDY_AUTH_TOKEN", auth_token)
441
+ _set_if_value(environment, adapter.model_env, judge.model)
442
+ _set_if_value(environment, adapter.base_url_env, judge.base_url)
443
+ if adapter.custom:
444
+ environment["COMET_EVAL_CUSTOM_AGENT_ID"] = adapter.id
445
+ environment["COMET_EVAL_CUSTOM_EXECUTABLE"] = adapter.executable
446
+ environment["COMET_EVAL_CUSTOM_CREDENTIALS"] = ",".join(adapter.required_credentials)
447
+ environment["COMET_EVAL_CUSTOM_MODEL_ENV"] = adapter.model_env
448
+ environment["COMET_EVAL_CUSTOM_BASE_URL_ENV"] = adapter.base_url_env
449
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_MODEL", judge.model)
450
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_BASE_URL", judge.base_url)
451
+ environment["COMET_EVAL_CUSTOM_INSTALL_KIND"] = adapter.install_kind
452
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_PACKAGE", adapter.install_package)
453
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_VERSION", adapter.install_version)
454
+ if "WSLENV" in environment:
455
+ blocked_wsl = (
456
+ set(excluded)
457
+ | _PROVIDER_KEYS
458
+ | _MAIN_OVERRIDE_KEYS
459
+ | _JUDGE_KEYS
460
+ )
461
+ retained_wsl = [
462
+ item
463
+ for item in environment["WSLENV"].split(":")
464
+ if item
465
+ and item not in blocked_wsl
466
+ and not item.startswith("BENCH_JUDGE_")
467
+ and not item.startswith("COMET_EVAL_CUSTOM_")
468
+ ]
469
+ if retained_wsl:
470
+ environment["WSLENV"] = ":".join(retained_wsl)
471
+ else:
472
+ environment.pop("WSLENV", None)
473
+ environment["COMET_EVAL_AGENT_ROLE"] = "judge"
474
+ return environment
475
+
476
+
477
+ def missing_credentials(
478
+ agent: AgentId,
479
+ *,
480
+ source_env: Mapping[str, str] | None = None,
481
+ judge: bool = False,
482
+ ) -> tuple[str, ...]:
483
+ """Return missing credential names for preflight without exposing values."""
484
+
485
+ environment = source_env if source_env is not None else os.environ
486
+ adapter = get_agent_adapter(agent)
487
+ if judge:
488
+ if not adapter.required_credentials:
489
+ return ()
490
+ if len(adapter.required_credentials) == 1:
491
+ return () if any(
492
+ environment.get(key) for key in ("BENCH_JUDGE_API_KEY", "BENCH_JUDGE_AUTH_TOKEN")
493
+ ) else ("BENCH_JUDGE_API_KEY or BENCH_JUDGE_AUTH_TOKEN",)
494
+ required = (
495
+ "BENCH_JUDGE_API_KEY",
496
+ "BENCH_JUDGE_AUTH_TOKEN",
497
+ )
498
+ return tuple(key for key in required if not environment.get(key))
499
+ keys = adapter.required_credentials
500
+ if adapter.custom:
501
+ return tuple(key for key in keys if not environment.get(key))
502
+ return () if any(environment.get(key) for key in keys) or environment.get("BENCH_API_KEY") else keys
503
+
504
+
505
+ def preflight_credentials(
506
+ main: ResolvedExecution,
507
+ judge: ResolvedJudge | None,
508
+ *,
509
+ source_env: Mapping[str, str] | None = None,
510
+ ) -> list[str]:
511
+ """Return safe, value-free credential errors for normal-run preflight."""
512
+
513
+ errors: list[str] = []
514
+ main_missing = missing_credentials(main.agent, source_env=source_env)
515
+ if main_missing:
516
+ separator = ", " if get_agent_adapter(main.agent).custom else " or "
517
+ errors.append(f"main credentials missing for {main.agent}: {separator.join(main_missing)}")
518
+ if judge is not None:
519
+ judge_missing = missing_credentials(judge.agent, source_env=source_env, judge=True)
520
+ if judge_missing:
521
+ separator = ", " if get_agent_adapter(judge.agent).custom else " or "
522
+ errors.append("judge credentials missing: " + separator.join(judge_missing))
523
+ return errors
524
+
525
+
526
+ def api_identity(base_url: str | None) -> str:
527
+ """Return the user-visible routing identity without exposing a URL."""
528
+
529
+ return "custom" if base_url else "default"
530
+
531
+
532
+ __all__ = [
533
+ "ResolvedExecution",
534
+ "ResolvedJudge",
535
+ "api_identity",
536
+ "build_agent_environment",
537
+ "build_judge_environment",
538
+ "missing_credentials",
539
+ "preflight_credentials",
540
+ "resolve_execution",
541
+ "resolve_judge",
542
+ "redact_sensitive",
543
+ "validate_base_url",
544
+ ]