@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. package/README.md +49 -24
  2. package/assets/manifest.json +2 -1
  3. package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
  4. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
  5. package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
  6. package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
  7. package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
  8. package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
  9. package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
  10. package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
  11. package/assets/skills/comet/scripts/comet-state.mjs +117 -117
  12. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
  13. package/assets/skills/comet-build/SKILL.md +15 -56
  14. package/assets/skills/comet-classic/SKILL.md +14 -8
  15. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
  16. package/assets/skills/comet-classic/reference/scripts.md +3 -1
  17. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  18. package/assets/skills/comet-native/SKILL.md +13 -10
  19. package/assets/skills/comet-native/reference/artifacts.md +3 -2
  20. package/assets/skills/comet-native/reference/commands.md +1 -1
  21. package/assets/skills/comet-native/reference/recovery.md +1 -1
  22. package/assets/skills/comet-native/reference/workspace.md +4 -4
  23. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
  24. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
  25. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
  26. package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
  27. package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
  28. package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
  29. package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
  30. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
  31. package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
  32. package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
  33. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
  34. package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
  35. package/assets/skills/comet-open/SKILL.md +24 -5
  36. package/assets/skills-zh/comet-build/SKILL.md +15 -56
  37. package/assets/skills-zh/comet-classic/SKILL.md +14 -8
  38. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
  39. package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
  40. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  41. package/assets/skills-zh/comet-native/SKILL.md +13 -10
  42. package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
  43. package/assets/skills-zh/comet-native/reference/commands.md +1 -1
  44. package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
  45. package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
  46. package/assets/skills-zh/comet-open/SKILL.md +24 -5
  47. package/dist/app/cli/index.js +20 -1
  48. package/dist/app/cli/index.js.map +1 -1
  49. package/dist/app/commands/classic.d.ts.map +1 -1
  50. package/dist/app/commands/classic.js +2 -0
  51. package/dist/app/commands/classic.js.map +1 -1
  52. package/dist/app/commands/eval.d.ts +9 -2
  53. package/dist/app/commands/eval.d.ts.map +1 -1
  54. package/dist/app/commands/eval.js +293 -48
  55. package/dist/app/commands/eval.js.map +1 -1
  56. package/dist/app/commands/init.d.ts.map +1 -1
  57. package/dist/app/commands/init.js +2 -1
  58. package/dist/app/commands/init.js.map +1 -1
  59. package/dist/app/commands/update.d.ts.map +1 -1
  60. package/dist/app/commands/update.js +4 -1
  61. package/dist/app/commands/update.js.map +1 -1
  62. package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
  63. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  64. package/dist/domains/comet-classic/classic-cli.js +3 -0
  65. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  66. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
  67. package/dist/domains/comet-classic/classic-current-change.js +29 -3
  68. package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
  69. package/dist/domains/comet-classic/classic-guard.js +2 -2
  70. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  71. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  72. package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
  73. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  74. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  75. package/dist/domains/comet-classic/classic-state-command.js +31 -8
  76. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  77. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  78. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  79. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  80. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  81. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  82. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  83. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  84. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  85. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
  86. package/dist/domains/comet-entry/hook-adapter.js +1 -0
  87. package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
  88. package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
  89. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
  90. package/dist/domains/comet-entry/hook-router-entry.js +21 -1
  91. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
  92. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
  93. package/dist/domains/comet-entry/init-workflow.js +13 -5
  94. package/dist/domains/comet-entry/init-workflow.js.map +1 -1
  95. package/dist/domains/comet-native/index.d.ts +1 -0
  96. package/dist/domains/comet-native/index.d.ts.map +1 -1
  97. package/dist/domains/comet-native/index.js +1 -0
  98. package/dist/domains/comet-native/index.js.map +1 -1
  99. package/dist/domains/comet-native/native-children.d.ts +49 -0
  100. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  101. package/dist/domains/comet-native/native-children.js +388 -0
  102. package/dist/domains/comet-native/native-children.js.map +1 -0
  103. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
  104. package/dist/domains/comet-native/native-cli-help.js +3 -3
  105. package/dist/domains/comet-native/native-cli-help.js.map +1 -1
  106. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
  107. package/dist/domains/comet-native/native-doctor-command.js +10 -5
  108. package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
  109. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
  110. package/dist/domains/comet-native/native-hook-guard.js +41 -2
  111. package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
  112. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
  113. package/dist/domains/comet-native/native-next-command.js +85 -5
  114. package/dist/domains/comet-native/native-next-command.js.map +1 -1
  115. package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
  116. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
  117. package/dist/domains/comet-native/native-portable-continuation.js +63 -1
  118. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
  119. package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
  120. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
  121. package/dist/domains/comet-native/native-portable-runtime.js +110 -4
  122. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
  123. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
  124. package/dist/domains/comet-native/native-portable-state.js +13 -0
  125. package/dist/domains/comet-native/native-portable-state.js.map +1 -1
  126. package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
  127. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
  128. package/dist/domains/comet-native/native-portable-status.js +9 -1
  129. package/dist/domains/comet-native/native-portable-status.js.map +1 -1
  130. package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
  131. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
  132. package/dist/domains/comet-native/native-portable-types.js.map +1 -1
  133. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
  134. package/dist/domains/comet-native/native-show-command.js +2 -1
  135. package/dist/domains/comet-native/native-show-command.js.map +1 -1
  136. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
  137. package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
  138. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
  139. package/dist/domains/dashboard/collector.d.ts +1 -1
  140. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  141. package/dist/domains/dashboard/collector.js +102 -87
  142. package/dist/domains/dashboard/collector.js.map +1 -1
  143. package/dist/domains/dashboard/native-adapter.d.ts +20 -0
  144. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
  145. package/dist/domains/dashboard/native-adapter.js +11 -0
  146. package/dist/domains/dashboard/native-adapter.js.map +1 -1
  147. package/dist/domains/dashboard/native-collector.d.ts +1 -0
  148. package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
  149. package/dist/domains/dashboard/native-collector.js +304 -55
  150. package/dist/domains/dashboard/native-collector.js.map +1 -1
  151. package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
  152. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
  153. package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
  154. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
  155. package/dist/domains/dashboard/server.d.ts.map +1 -1
  156. package/dist/domains/dashboard/server.js +5 -3
  157. package/dist/domains/dashboard/server.js.map +1 -1
  158. package/dist/domains/dashboard/types.d.ts +5 -0
  159. package/dist/domains/dashboard/types.d.ts.map +1 -1
  160. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
  161. package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
  162. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
  163. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
  164. package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
  165. package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
  166. package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
  167. package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
  168. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
  169. package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
  170. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
  171. package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
  172. package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
  173. package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
  174. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
  175. package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
  176. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
  177. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
  178. package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
  179. package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
  180. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
  181. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
  182. package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
  183. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
  184. package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
  185. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
  186. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
  187. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
  188. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
  189. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
  190. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
  191. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
  192. package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
  193. package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
  194. package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
  195. package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
  196. package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
  197. package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
  198. package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
  199. package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
  200. package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
  201. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  202. package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
  203. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  204. package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
  205. package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
  206. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
  207. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
  208. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  209. package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
  210. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
  211. package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
  212. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
  213. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
  214. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
  215. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
  216. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
  217. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
  218. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
  219. package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
  220. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
  221. package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
  222. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
  223. package/dist/domains/dashboard/web/index.html +2 -2
  224. package/dist/domains/dashboard/workspace.d.ts +24 -0
  225. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  226. package/dist/domains/dashboard/workspace.js +87 -0
  227. package/dist/domains/dashboard/workspace.js.map +1 -0
  228. package/dist/domains/eval/index.d.ts +3 -0
  229. package/dist/domains/eval/index.d.ts.map +1 -1
  230. package/dist/domains/eval/index.js +3 -0
  231. package/dist/domains/eval/index.js.map +1 -1
  232. package/dist/domains/eval/standalone-context.d.ts +19 -0
  233. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  234. package/dist/domains/eval/standalone-context.js +107 -0
  235. package/dist/domains/eval/standalone-context.js.map +1 -0
  236. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  237. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  238. package/dist/domains/eval/standalone-static-collect.js +779 -0
  239. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  240. package/dist/domains/eval/user-environment.d.ts +6 -0
  241. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  242. package/dist/domains/eval/user-environment.js +73 -0
  243. package/dist/domains/eval/user-environment.js.map +1 -0
  244. package/dist/domains/integrations/openspec.d.ts +1 -1
  245. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  246. package/dist/domains/integrations/openspec.js +110 -26
  247. package/dist/domains/integrations/openspec.js.map +1 -1
  248. package/dist/domains/integrations/superpowers.d.ts +5 -1
  249. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  250. package/dist/domains/integrations/superpowers.js +21 -2
  251. package/dist/domains/integrations/superpowers.js.map +1 -1
  252. package/dist/domains/skill/platform-install.d.ts +1 -1
  253. package/dist/domains/skill/platform-install.js +1 -1
  254. package/dist/domains/skill/project-instructions.d.ts.map +1 -1
  255. package/dist/domains/skill/project-instructions.js +5 -3
  256. package/dist/domains/skill/project-instructions.js.map +1 -1
  257. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  258. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  259. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  260. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  261. package/dist/domains/workflow-contract/index.d.ts +2 -0
  262. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  263. package/dist/domains/workflow-contract/index.js +1 -0
  264. package/dist/domains/workflow-contract/index.js.map +1 -1
  265. package/dist/domains/workflow-contract/project-config.d.ts +2 -2
  266. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
  267. package/dist/domains/workflow-contract/project-config.js +42 -2
  268. package/dist/domains/workflow-contract/project-config.js.map +1 -1
  269. package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
  270. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
  271. package/dist/domains/workflow-contract/protected-project-path.js +83 -0
  272. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
  273. package/dist/domains/workflow-contract/types.d.ts +4 -0
  274. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  275. package/dist/platform/install/platforms.d.ts.map +1 -1
  276. package/dist/platform/install/platforms.js +13 -2
  277. package/dist/platform/install/platforms.js.map +1 -1
  278. package/dist/platform/paths/git-worktree.d.ts +7 -1
  279. package/dist/platform/paths/git-worktree.d.ts.map +1 -1
  280. package/dist/platform/paths/git-worktree.js +28 -7
  281. package/dist/platform/paths/git-worktree.js.map +1 -1
  282. package/eval/.env.example +115 -0
  283. package/eval/README.md +237 -14
  284. package/eval/langfuse/.env.example +9 -0
  285. package/eval/langfuse/tests/conftest.py +146 -0
  286. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  287. package/eval/langfuse/tests/test_adapter.py +272 -0
  288. package/eval/langsmith/.env.example +16 -0
  289. package/eval/langsmith/tests/conftest.py +10 -1
  290. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
  291. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
  292. package/eval/local/tests/conftest.py +701 -80
  293. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  294. package/eval/local/tests/scaffold/test_agents.py +76 -0
  295. package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
  296. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  297. package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
  298. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  299. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  300. package/eval/local/tests/scaffold/test_execution.py +364 -0
  301. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  302. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  303. package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
  304. package/eval/local/tests/scaffold/test_logging.py +162 -0
  305. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  306. package/eval/local/tests/scaffold/test_manifests.py +396 -0
  307. package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
  308. package/eval/local/tests/scaffold/test_profiles.py +64 -6
  309. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  310. package/eval/local/tests/scaffold/test_utils.py +480 -4
  311. package/eval/local/tests/tasks/test_tasks.py +216 -14
  312. package/eval/pyproject.toml +5 -0
  313. package/eval/scaffold/__init__.py +6 -0
  314. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  315. package/eval/scaffold/python/__init__.py +128 -20
  316. package/eval/scaffold/python/agents.py +239 -0
  317. package/eval/scaffold/python/aligned_comparison.py +156 -27
  318. package/eval/scaffold/python/auto_tasks.py +700 -0
  319. package/eval/scaffold/python/custom_agents.py +172 -0
  320. package/eval/scaffold/python/eval_context.py +236 -0
  321. package/eval/scaffold/python/eval_schema.py +135 -0
  322. package/eval/scaffold/python/execution.py +544 -0
  323. package/eval/scaffold/python/generated_task_cache.py +230 -0
  324. package/eval/scaffold/python/generic_llm_judge.py +41 -14
  325. package/eval/scaffold/python/inline_tasks.py +118 -0
  326. package/eval/scaffold/python/judge_config.py +158 -37
  327. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  328. package/eval/scaffold/python/llm_judge.py +68 -11
  329. package/eval/scaffold/python/logging.py +157 -17
  330. package/eval/scaffold/python/manifest_tasks.py +69 -0
  331. package/eval/scaffold/python/manifests.py +443 -22
  332. package/eval/scaffold/python/paths.py +11 -1
  333. package/eval/scaffold/python/task_resolution.py +176 -0
  334. package/eval/scaffold/python/tasks.py +17 -4
  335. package/eval/scaffold/python/utils.py +167 -11
  336. package/eval/scaffold/python/validation/core.py +48 -2
  337. package/eval/scaffold/python/validation/generic_rubric.py +9 -0
  338. package/eval/scaffold/python/validation/native_workflow.py +6 -1
  339. package/eval/scaffold/python/validation/rubric.py +16 -1
  340. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  341. package/eval/scaffold/shell/docker.sh +534 -33
  342. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  343. package/eval/scaffold/shell/run-claude-loop.sh +153 -14
  344. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  345. package/package.json +8 -3
  346. package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
  347. package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
  348. package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
  349. package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
  350. package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
  351. package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
  352. package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
  353. package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
  354. package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
  355. package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
  356. package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
  357. package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
  358. package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
  359. package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
  360. package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
  361. package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
  362. package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
  363. package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
  364. package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
  365. package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
  366. package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
  367. package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
  368. package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
  369. package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
  370. package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
  371. package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
@@ -0,0 +1,544 @@
1
+ """Resolve standalone eval execution settings and isolate provider environments."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import re
7
+ from dataclasses import dataclass
8
+ from typing import Any, Mapping
9
+ from urllib.parse import urlsplit
10
+
11
+ from scaffold.python.agents import AgentId, DEFAULT_AGENT, get_agent_adapter, validate_agent_id
12
+
13
+
14
+ _MISSING = object()
15
+ _MODEL_ENV: dict[AgentId, tuple[str, ...]] = {
16
+ "claude-code": ("BENCH_CC_MODEL", "ANTHROPIC_MODEL"),
17
+ "codex": ("BENCH_CODEX_MODEL", "OPENAI_MODEL", "CODEX_MODEL"),
18
+ "qoder": ("BENCH_QODER_MODEL", "QODER_MODEL"),
19
+ "codebuddy": ("BENCH_CODEBUDDY_MODEL", "CODEBUDDY_MODEL"),
20
+ }
21
+ _COMMON_MODEL_AGENTS = frozenset(_MODEL_ENV)
22
+ _BASE_URL_ENV: dict[AgentId, tuple[str, ...]] = {
23
+ "claude-code": ("ANTHROPIC_BASE_URL",),
24
+ "codex": ("OPENAI_BASE_URL", "CODEX_BASE_URL"),
25
+ "qoder": ("QODER_BASE_URL",),
26
+ "codebuddy": ("CODEBUDDY_BASE_URL",),
27
+ }
28
+ _COMMON_BASE_URL_AGENTS = frozenset({"claude-code", "codex", "codebuddy"})
29
+ _MODEL_TARGET_ENV: dict[AgentId, str] = {
30
+ "claude-code": "ANTHROPIC_MODEL",
31
+ "codex": "OPENAI_MODEL",
32
+ "qoder": "QODER_MODEL",
33
+ "codebuddy": "CODEBUDDY_MODEL",
34
+ }
35
+ _BASE_URL_TARGET_ENV: dict[AgentId, str] = {
36
+ "claude-code": "ANTHROPIC_BASE_URL",
37
+ "codex": "OPENAI_BASE_URL",
38
+ "qoder": "QODER_BASE_URL",
39
+ "codebuddy": "CODEBUDDY_BASE_URL",
40
+ }
41
+ _CREDENTIAL_ENV: dict[AgentId, tuple[str, ...]] = {
42
+ "claude-code": ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"),
43
+ "codex": ("OPENAI_API_KEY", "CODEX_API_KEY"),
44
+ "qoder": ("QODER_PERSONAL_ACCESS_TOKEN",),
45
+ "codebuddy": ("CODEBUDDY_API_KEY", "CODEBUDDY_AUTH_TOKEN"),
46
+ }
47
+ _PROVIDER_KEYS = {
48
+ key
49
+ for keys in (*_MODEL_ENV.values(), *_BASE_URL_ENV.values(), *_CREDENTIAL_ENV.values())
50
+ for key in keys
51
+ }
52
+ _MAIN_OVERRIDE_KEYS = {
53
+ "BENCH_EVAL_AGENT",
54
+ "BENCH_CC_MODEL",
55
+ "BENCH_CODEX_MODEL",
56
+ "BENCH_QODER_MODEL",
57
+ "BENCH_CODEBUDDY_MODEL",
58
+ "OPENAI_MODEL",
59
+ "CODEX_MODEL",
60
+ "QODER_MODEL",
61
+ "CODEBUDDY_MODEL",
62
+ "BENCH_BASE_URL",
63
+ "BENCH_API_KEY",
64
+ "BENCH_MODEL",
65
+ }
66
+ _JUDGE_KEYS = {
67
+ "BENCH_LLM_JUDGE",
68
+ "BENCH_JUDGE_MODEL",
69
+ "BENCH_JUDGE_BASE_URL",
70
+ "BENCH_JUDGE_API_KEY",
71
+ "BENCH_JUDGE_AUTH_TOKEN",
72
+ "BENCH_JUDGE_AGENT",
73
+ "COMET_EVAL_MAIN_CREDENTIALS",
74
+ }
75
+ _CREDENTIAL_NAME_RE = re.compile(
76
+ r"(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|PERSONAL_ACCESS_TOKEN|PASSWORD|SECRET|CREDENTIALS?|(?:^|_)(?:KEY|TOKEN))$",
77
+ re.I,
78
+ )
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class ResolvedExecution:
83
+ agent: AgentId
84
+ model: str | None
85
+ base_url: str | None
86
+ sources: dict[str, str]
87
+
88
+
89
+ @dataclass(frozen=True)
90
+ class ResolvedJudge:
91
+ agent: AgentId
92
+ model: str
93
+ base_url: str | None
94
+ sources: dict[str, str]
95
+
96
+
97
+ def _value(source: object, name: str, default: object = _MISSING) -> object:
98
+ if isinstance(source, Mapping):
99
+ if name in source:
100
+ return source[name]
101
+ return default
102
+ value = getattr(source, name, default)
103
+ return value
104
+
105
+
106
+ def _first_non_empty(values: list[object]) -> str | None:
107
+ for value in values:
108
+ if isinstance(value, str) and value.strip():
109
+ return value.strip()
110
+ return None
111
+
112
+
113
+ def validate_base_url(value: object, *, field: str = "baseUrl") -> str | None:
114
+ if value is None:
115
+ return None
116
+ if not isinstance(value, str) or value.strip() != value or not value:
117
+ raise ValueError(f"{field} must be an absolute http(s) URL")
118
+ parsed = urlsplit(value)
119
+ try:
120
+ parsed.port
121
+ except ValueError as exc:
122
+ raise ValueError(f"{field} must be an absolute http(s) URL") from exc
123
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
124
+ raise ValueError(f"{field} must be an absolute http(s) URL")
125
+ if parsed.username or parsed.password or parsed.query or parsed.fragment:
126
+ raise ValueError(f"{field} must not contain credentials, query, or fragment")
127
+ return value
128
+
129
+
130
+ def _declared_credential_names(environment: Mapping[str, str]) -> set[str]:
131
+ names: set[str] = set()
132
+ for key in ("COMET_EVAL_CUSTOM_CREDENTIALS", "COMET_EVAL_MAIN_CREDENTIALS"):
133
+ names.update(
134
+ item.strip()
135
+ for item in environment.get(key, "").split(",")
136
+ if item.strip()
137
+ )
138
+ return names
139
+
140
+
141
+ def _credential_values(environment: Mapping[str, str]) -> tuple[str, ...]:
142
+ declared = _declared_credential_names(environment)
143
+ values = {
144
+ value.strip()
145
+ for key, value in environment.items()
146
+ if value.strip() and (key in declared or _CREDENTIAL_NAME_RE.search(key))
147
+ }
148
+ return tuple(sorted((value for value in values if len(value) >= 4), key=len, reverse=True))
149
+
150
+
151
+ def redact_sensitive(value: Any, source_env: Mapping[str, str] | None = None) -> Any:
152
+ """Redact credential values before eval data reaches user-owned artifacts."""
153
+
154
+ environment = source_env if source_env is not None else os.environ
155
+ declared = _declared_credential_names(environment)
156
+ if isinstance(value, dict):
157
+ return {
158
+ key: "[REDACTED]"
159
+ if key in declared or _CREDENTIAL_NAME_RE.search(str(key))
160
+ else redact_sensitive(item, environment)
161
+ for key, item in value.items()
162
+ }
163
+ if isinstance(value, list):
164
+ return [redact_sensitive(item, environment) for item in value]
165
+ if not isinstance(value, str):
166
+ return value
167
+ redacted = re.sub(
168
+ r"(?i)(authorization\s*:\s*bearer\s+)[^\s,;]+",
169
+ r"\1[REDACTED]",
170
+ value,
171
+ )
172
+ redacted = re.sub(r"(?i)(x-api-key\s*:\s*)[^\s,;]+", r"\1[REDACTED]", redacted)
173
+ redacted = re.sub(
174
+ r"(?i)(api[_-]?key|auth[_-]?token|access[_-]?token|token|secret|password|credential)\s*[=:]\s*[^\s,;]+",
175
+ r"\1=[REDACTED]",
176
+ redacted,
177
+ )
178
+ redacted = re.sub(
179
+ r"(?i)([?&](?:api[_-]?key|token|access[_-]?token|auth[_-]?token)=)[^&#\s]+",
180
+ r"\1[REDACTED]",
181
+ redacted,
182
+ )
183
+ redacted = re.sub(r"(https?://)[^/@\s:]+:[^/@\s]+@", r"\1[REDACTED]@", redacted)
184
+ for secret in _credential_values(environment):
185
+ redacted = redacted.replace(secret, "[REDACTED]")
186
+ return redacted
187
+
188
+
189
+ def _manifest_value(manifest: object | None, name: str) -> object:
190
+ value = _value(manifest, name) if manifest is not None else _MISSING
191
+ if value is not _MISSING:
192
+ return value
193
+ return None
194
+
195
+
196
+ def _model_env_keys(agent: AgentId) -> tuple[str, ...]:
197
+ adapter = get_agent_adapter(agent)
198
+ return (*_MODEL_ENV.get(agent, ()), *(key for key in (adapter.model_env,) if key))
199
+
200
+
201
+ def _base_url_env_keys(agent: AgentId) -> tuple[str, ...]:
202
+ adapter = get_agent_adapter(agent)
203
+ return (*_BASE_URL_ENV.get(agent, ()), *(key for key in (adapter.base_url_env,) if key))
204
+
205
+
206
+ def resolve_execution(
207
+ *,
208
+ cli_agent: object | None = None,
209
+ cli_model: object | None = None,
210
+ cli_base_url: object | None = None,
211
+ manifest: object | None = None,
212
+ source_env: Mapping[str, str] | None = None,
213
+ ) -> ResolvedExecution:
214
+ """Resolve main execution settings using CLI > manifest > legacy env > default."""
215
+
216
+ environment = source_env if source_env is not None else os.environ
217
+ manifest_agent = _manifest_value(manifest, "agent")
218
+ agent_value = _first_non_empty(
219
+ [cli_agent, manifest_agent, environment.get("BENCH_EVAL_AGENT"), DEFAULT_AGENT]
220
+ )
221
+ agent = validate_agent_id(agent_value, field="evaluation agent")
222
+
223
+ manifest_model = _manifest_value(manifest, "model")
224
+ model_candidates: list[object] = [cli_model, manifest_model]
225
+ model_candidates.extend(environment.get(key) for key in _model_env_keys(agent))
226
+ if agent in _COMMON_MODEL_AGENTS:
227
+ model_candidates.append(environment.get("BENCH_MODEL"))
228
+ model = _first_non_empty(model_candidates)
229
+
230
+ manifest_base_url = _manifest_value(manifest, "base_url")
231
+ if manifest_base_url is None:
232
+ manifest_base_url = _manifest_value(manifest, "baseUrl")
233
+ base_url_candidates: list[object] = [cli_base_url, manifest_base_url]
234
+ base_url_candidates.extend(environment.get(key) for key in _base_url_env_keys(agent))
235
+ if agent in _COMMON_BASE_URL_AGENTS:
236
+ base_url_candidates.append(environment.get("BENCH_BASE_URL"))
237
+ base_url = _first_non_empty(base_url_candidates)
238
+ base_url = validate_base_url(base_url, field="execution.baseUrl")
239
+ adapter = get_agent_adapter(agent)
240
+ if adapter.custom and base_url and not adapter.base_url_env:
241
+ raise ValueError(
242
+ f"custom Agent {agent} must declare baseUrlEnv before execution.baseUrl can be used"
243
+ )
244
+
245
+ return ResolvedExecution(
246
+ agent=agent,
247
+ model=model,
248
+ base_url=base_url,
249
+ sources={
250
+ "agent": "cli"
251
+ if _first_non_empty([cli_agent])
252
+ else "manifest"
253
+ if _first_non_empty([manifest_agent])
254
+ else "environment"
255
+ if _first_non_empty([environment.get("BENCH_EVAL_AGENT")])
256
+ else "default",
257
+ "model": "cli"
258
+ if _first_non_empty([cli_model])
259
+ else "manifest"
260
+ if _first_non_empty([manifest_model])
261
+ else "environment"
262
+ if model
263
+ else "default",
264
+ "base_url": "cli"
265
+ if _first_non_empty([cli_base_url])
266
+ else "manifest"
267
+ if _first_non_empty([manifest_base_url])
268
+ else "environment"
269
+ if base_url
270
+ else "default",
271
+ },
272
+ )
273
+
274
+
275
+ def resolve_judge(
276
+ *,
277
+ main: ResolvedExecution,
278
+ cli_agent: object | None = None,
279
+ cli_model: object | None = None,
280
+ cli_base_url: object | None = None,
281
+ manifest: object = None,
282
+ source_env: Mapping[str, str] | None = None,
283
+ ) -> ResolvedJudge | None:
284
+ """Resolve the optional independent Judge configuration.
285
+
286
+ The main model, base URL, and credentials are deliberately not fallbacks
287
+ for Judge. Only the main Agent ID is an allowed implicit inheritance.
288
+ """
289
+
290
+ environment = source_env if source_env is not None else os.environ
291
+ enabled = (
292
+ manifest is not None
293
+ or any(value is not None for value in (cli_agent, cli_model, cli_base_url))
294
+ or str(environment.get("BENCH_LLM_JUDGE", "")).strip().lower()
295
+ in {"1", "true", "yes", "on"}
296
+ )
297
+ if not enabled:
298
+ return None
299
+
300
+ manifest_agent = _manifest_value(manifest, "agent")
301
+ agent_value = _first_non_empty([cli_agent, manifest_agent, main.agent])
302
+ agent = validate_agent_id(agent_value, field="judge evaluation agent")
303
+
304
+ manifest_model = _manifest_value(manifest, "model")
305
+ model = _first_non_empty([cli_model, manifest_model, environment.get("BENCH_JUDGE_MODEL")])
306
+ if model is None:
307
+ if manifest == {} or str(environment.get("BENCH_LLM_JUDGE", "")).strip().lower() in {
308
+ "1",
309
+ "true",
310
+ "yes",
311
+ "on",
312
+ }:
313
+ raise ValueError("BENCH_JUDGE_MODEL is required when BENCH_LLM_JUDGE=1")
314
+ raise ValueError("judge.model is required when LLM-as-Judge is enabled")
315
+
316
+ manifest_base_url = _manifest_value(manifest, "base_url")
317
+ if manifest_base_url is None:
318
+ manifest_base_url = _manifest_value(manifest, "baseUrl")
319
+ base_url = _first_non_empty(
320
+ [cli_base_url, manifest_base_url, environment.get("BENCH_JUDGE_BASE_URL")]
321
+ )
322
+ base_url = validate_base_url(base_url, field="judge.baseUrl")
323
+
324
+ return ResolvedJudge(
325
+ agent=agent,
326
+ model=model,
327
+ base_url=base_url,
328
+ sources={
329
+ "agent": "cli"
330
+ if cli_agent
331
+ else "manifest"
332
+ if manifest_agent
333
+ else "inherited",
334
+ "model": "cli"
335
+ if cli_model
336
+ else "manifest"
337
+ if manifest_model
338
+ else "environment",
339
+ "base_url": "cli"
340
+ if cli_base_url
341
+ else "manifest"
342
+ if manifest_base_url
343
+ else "environment"
344
+ if base_url
345
+ else "default",
346
+ },
347
+ )
348
+
349
+
350
+ def _set_if_value(environment: dict[str, str], key: str, value: str | None) -> None:
351
+ if not key:
352
+ return
353
+ if value:
354
+ environment[key] = value
355
+ else:
356
+ environment.pop(key, None)
357
+
358
+
359
+ def build_agent_environment(
360
+ execution: ResolvedExecution,
361
+ *,
362
+ source_env: Mapping[str, str] | None = None,
363
+ ) -> dict[str, str]:
364
+ """Map resolved main settings to the selected adapter's env contract."""
365
+
366
+ environment = dict(source_env if source_env is not None else os.environ)
367
+ for key in _JUDGE_KEYS:
368
+ environment.pop(key, None)
369
+ for key in tuple(environment):
370
+ if key.startswith("COMET_EVAL_CUSTOM_"):
371
+ environment.pop(key, None)
372
+ adapter = get_agent_adapter(execution.agent)
373
+ _set_if_value(environment, adapter.model_env, execution.model)
374
+ _set_if_value(environment, adapter.base_url_env, execution.base_url)
375
+ if not adapter.custom and not any(
376
+ environment.get(key) for key in _CREDENTIAL_ENV.get(execution.agent, ())
377
+ ):
378
+ credential_keys = _CREDENTIAL_ENV.get(execution.agent, ())
379
+ if credential_keys:
380
+ _set_if_value(environment, credential_keys[0], environment.get("BENCH_API_KEY"))
381
+ if adapter.custom:
382
+ environment["COMET_EVAL_CUSTOM_AGENT_ID"] = adapter.id
383
+ environment["COMET_EVAL_CUSTOM_EXECUTABLE"] = adapter.executable
384
+ environment["COMET_EVAL_CUSTOM_CREDENTIALS"] = ",".join(adapter.required_credentials)
385
+ environment["COMET_EVAL_CUSTOM_MODEL_ENV"] = adapter.model_env
386
+ environment["COMET_EVAL_CUSTOM_BASE_URL_ENV"] = adapter.base_url_env
387
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_MODEL", execution.model)
388
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_BASE_URL", execution.base_url)
389
+ environment["COMET_EVAL_CUSTOM_INSTALL_KIND"] = adapter.install_kind
390
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_PACKAGE", adapter.install_package)
391
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_VERSION", adapter.install_version)
392
+ return environment
393
+
394
+
395
+ def build_judge_environment(
396
+ judge: ResolvedJudge,
397
+ *,
398
+ source_env: Mapping[str, str] | None = None,
399
+ excluded_credentials: tuple[str, ...] = (),
400
+ ) -> dict[str, str]:
401
+ """Create a child environment containing only the dedicated Judge secrets."""
402
+
403
+ source = source_env if source_env is not None else os.environ
404
+ excluded = set(excluded_credentials) | _declared_credential_names(source)
405
+ environment = {
406
+ key: value
407
+ for key, value in source.items()
408
+ if key not in excluded
409
+ and key not in _PROVIDER_KEYS
410
+ and key not in _MAIN_OVERRIDE_KEYS
411
+ and key not in _JUDGE_KEYS
412
+ and not key.startswith("BENCH_JUDGE_")
413
+ and not key.startswith("COMET_EVAL_CUSTOM_")
414
+ and not re.search(
415
+ r"(?:^|_)(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|PERSONAL_ACCESS_TOKEN|MODEL|BASE_URL|KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?)$",
416
+ key,
417
+ )
418
+ }
419
+ api_key = source.get("BENCH_JUDGE_API_KEY", "")
420
+ auth_token = source.get("BENCH_JUDGE_AUTH_TOKEN", "")
421
+ adapter = get_agent_adapter(judge.agent)
422
+ if adapter.custom:
423
+ primary_credential = api_key or auth_token
424
+ for index, key in enumerate(adapter.required_credentials):
425
+ _set_if_value(
426
+ environment,
427
+ key,
428
+ primary_credential if index == 0 else auth_token or api_key,
429
+ )
430
+ elif judge.agent == "claude-code":
431
+ _set_if_value(environment, "ANTHROPIC_API_KEY", api_key)
432
+ _set_if_value(environment, "ANTHROPIC_AUTH_TOKEN", auth_token)
433
+ elif judge.agent == "codex":
434
+ _set_if_value(environment, "OPENAI_API_KEY", api_key)
435
+ _set_if_value(environment, "CODEX_API_KEY", auth_token)
436
+ elif judge.agent == "qoder":
437
+ _set_if_value(environment, "QODER_PERSONAL_ACCESS_TOKEN", auth_token or api_key)
438
+ else:
439
+ _set_if_value(environment, "CODEBUDDY_API_KEY", api_key)
440
+ _set_if_value(environment, "CODEBUDDY_AUTH_TOKEN", auth_token)
441
+ _set_if_value(environment, adapter.model_env, judge.model)
442
+ _set_if_value(environment, adapter.base_url_env, judge.base_url)
443
+ if adapter.custom:
444
+ environment["COMET_EVAL_CUSTOM_AGENT_ID"] = adapter.id
445
+ environment["COMET_EVAL_CUSTOM_EXECUTABLE"] = adapter.executable
446
+ environment["COMET_EVAL_CUSTOM_CREDENTIALS"] = ",".join(adapter.required_credentials)
447
+ environment["COMET_EVAL_CUSTOM_MODEL_ENV"] = adapter.model_env
448
+ environment["COMET_EVAL_CUSTOM_BASE_URL_ENV"] = adapter.base_url_env
449
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_MODEL", judge.model)
450
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_BASE_URL", judge.base_url)
451
+ environment["COMET_EVAL_CUSTOM_INSTALL_KIND"] = adapter.install_kind
452
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_PACKAGE", adapter.install_package)
453
+ _set_if_value(environment, "COMET_EVAL_CUSTOM_INSTALL_VERSION", adapter.install_version)
454
+ if "WSLENV" in environment:
455
+ blocked_wsl = (
456
+ set(excluded)
457
+ | _PROVIDER_KEYS
458
+ | _MAIN_OVERRIDE_KEYS
459
+ | _JUDGE_KEYS
460
+ )
461
+ retained_wsl = [
462
+ item
463
+ for item in environment["WSLENV"].split(":")
464
+ if item
465
+ and item not in blocked_wsl
466
+ and not item.startswith("BENCH_JUDGE_")
467
+ and not item.startswith("COMET_EVAL_CUSTOM_")
468
+ ]
469
+ if retained_wsl:
470
+ environment["WSLENV"] = ":".join(retained_wsl)
471
+ else:
472
+ environment.pop("WSLENV", None)
473
+ environment["COMET_EVAL_AGENT_ROLE"] = "judge"
474
+ return environment
475
+
476
+
477
+ def missing_credentials(
478
+ agent: AgentId,
479
+ *,
480
+ source_env: Mapping[str, str] | None = None,
481
+ judge: bool = False,
482
+ ) -> tuple[str, ...]:
483
+ """Return missing credential names for preflight without exposing values."""
484
+
485
+ environment = source_env if source_env is not None else os.environ
486
+ adapter = get_agent_adapter(agent)
487
+ if judge:
488
+ if not adapter.required_credentials:
489
+ return ()
490
+ if len(adapter.required_credentials) == 1:
491
+ return () if any(
492
+ environment.get(key) for key in ("BENCH_JUDGE_API_KEY", "BENCH_JUDGE_AUTH_TOKEN")
493
+ ) else ("BENCH_JUDGE_API_KEY or BENCH_JUDGE_AUTH_TOKEN",)
494
+ required = (
495
+ "BENCH_JUDGE_API_KEY",
496
+ "BENCH_JUDGE_AUTH_TOKEN",
497
+ )
498
+ return tuple(key for key in required if not environment.get(key))
499
+ keys = adapter.required_credentials
500
+ if adapter.custom:
501
+ return tuple(key for key in keys if not environment.get(key))
502
+ return () if any(environment.get(key) for key in keys) or environment.get("BENCH_API_KEY") else keys
503
+
504
+
505
+ def preflight_credentials(
506
+ main: ResolvedExecution,
507
+ judge: ResolvedJudge | None,
508
+ *,
509
+ source_env: Mapping[str, str] | None = None,
510
+ ) -> list[str]:
511
+ """Return safe, value-free credential errors for normal-run preflight."""
512
+
513
+ errors: list[str] = []
514
+ main_missing = missing_credentials(main.agent, source_env=source_env)
515
+ if main_missing:
516
+ separator = ", " if get_agent_adapter(main.agent).custom else " or "
517
+ errors.append(f"main credentials missing for {main.agent}: {separator.join(main_missing)}")
518
+ if judge is not None:
519
+ judge_missing = missing_credentials(judge.agent, source_env=source_env, judge=True)
520
+ if judge_missing:
521
+ separator = ", " if get_agent_adapter(judge.agent).custom else " or "
522
+ errors.append("judge credentials missing: " + separator.join(judge_missing))
523
+ return errors
524
+
525
+
526
+ def api_identity(base_url: str | None) -> str:
527
+ """Return the user-visible routing identity without exposing a URL."""
528
+
529
+ return "custom" if base_url else "default"
530
+
531
+
532
+ __all__ = [
533
+ "ResolvedExecution",
534
+ "ResolvedJudge",
535
+ "api_identity",
536
+ "build_agent_environment",
537
+ "build_judge_environment",
538
+ "missing_credentials",
539
+ "preflight_credentials",
540
+ "resolve_execution",
541
+ "resolve_judge",
542
+ "redact_sensitive",
543
+ "validate_base_url",
544
+ ]