@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -26
- package/assets/manifest.json +4 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
- package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
- package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
- package/assets/skills/comet-build/SKILL.md +36 -62
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +2 -2
- package/assets/skills/comet-native/SKILL.md +18 -15
- package/assets/skills/comet-native/reference/artifacts.md +5 -4
- package/assets/skills/comet-native/reference/clarification.md +1 -1
- package/assets/skills/comet-native/reference/commands.md +4 -4
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills/comet-review/SKILL.md +121 -0
- package/assets/skills/comet-review/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +36 -62
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +2 -2
- package/assets/skills-zh/comet-native/SKILL.md +18 -15
- package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
- package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
- package/assets/skills-zh/comet-native/reference/commands.md +4 -4
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-review/SKILL.md +121 -0
- package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +5 -8
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +5 -6
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
- package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.js +10 -0
- package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +4 -4
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.js +44 -6
- package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.js +2 -1
- package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +39 -13
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +2 -2
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +5 -1
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +2 -0
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.js +17 -10
- package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +65 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +453 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +13 -7
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +18 -5
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +56 -4
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +179 -20
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +178 -73
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +168 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
- package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
- package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.js +24 -6
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.js +86 -35
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
- package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +27 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +108 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +87 -44
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +14 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +93 -5
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +3 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +4 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +16 -10
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +11 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +39 -29
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
- package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +4 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +174 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +13 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/fs/strip-bom.d.ts +10 -0
- package/dist/platform/fs/strip-bom.d.ts.map +1 -0
- package/dist/platform/fs/strip-bom.js +12 -0
- package/dist/platform/fs/strip-bom.js.map +1 -0
- package/dist/platform/install/platforms.d.ts +11 -0
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +49 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +1 -0
- package/dist/platform/process/external-command.d.ts.map +1 -1
- package/dist/platform/process/external-command.js +2 -1
- package/dist/platform/process/external-command.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +719 -83
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +18 -28
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
- package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
|
@@ -20,11 +20,45 @@ def test_import_does_not_read_dotenv(monkeypatch):
|
|
|
20
20
|
|
|
21
21
|
assert calls == []
|
|
22
22
|
utils.load_eval_environment()
|
|
23
|
-
assert len(calls) ==
|
|
23
|
+
assert len(calls) == 3
|
|
24
24
|
monkeypatch.undo()
|
|
25
25
|
importlib.reload(utils)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def test_load_eval_environment_prefers_user_file_but_preserves_process_values(
|
|
29
|
+
tmp_path, monkeypatch
|
|
30
|
+
):
|
|
31
|
+
package = tmp_path / "package"
|
|
32
|
+
suite = tmp_path / "suite"
|
|
33
|
+
user = tmp_path / ".comet" / "eval"
|
|
34
|
+
package.mkdir()
|
|
35
|
+
suite.mkdir()
|
|
36
|
+
user.mkdir(parents=True)
|
|
37
|
+
(package / ".env").write_text(
|
|
38
|
+
"BENCH_MODEL=package\nBENCH_BASE_URL=https://package.example\n",
|
|
39
|
+
encoding="utf-8",
|
|
40
|
+
)
|
|
41
|
+
(suite / ".env").write_text(
|
|
42
|
+
"BENCH_MODEL=suite\nBENCH_BASE_URL=https://suite.example\n",
|
|
43
|
+
encoding="utf-8",
|
|
44
|
+
)
|
|
45
|
+
(user / ".env").write_text(
|
|
46
|
+
"BENCH_MODEL=user\nBENCH_BASE_URL=https://user.example\n",
|
|
47
|
+
encoding="utf-8",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
monkeypatch.setattr(utils, "EVAL_ROOT", package)
|
|
51
|
+
monkeypatch.setattr(utils, "get_suite_root", lambda: suite)
|
|
52
|
+
monkeypatch.setattr(utils.Path, "home", staticmethod(lambda: tmp_path))
|
|
53
|
+
monkeypatch.setenv("BENCH_MODEL", "process")
|
|
54
|
+
monkeypatch.delenv("BENCH_BASE_URL", raising=False)
|
|
55
|
+
|
|
56
|
+
utils.load_eval_environment()
|
|
57
|
+
|
|
58
|
+
assert os.environ["BENCH_MODEL"] == "process"
|
|
59
|
+
assert os.environ["BENCH_BASE_URL"] == "https://user.example"
|
|
60
|
+
|
|
61
|
+
|
|
28
62
|
def test_execution_validator_converts_structured_checks(monkeypatch, tmp_path: Path):
|
|
29
63
|
def fake_run_eval_in_docker(*_args, **_kwargs):
|
|
30
64
|
return {
|
|
@@ -71,6 +105,32 @@ def test_docker_script_failure_preserves_stderr(monkeypatch, tmp_path: Path):
|
|
|
71
105
|
assert "validator import failed" in output
|
|
72
106
|
|
|
73
107
|
|
|
108
|
+
def test_run_command_in_docker_passes_timeout_and_shell_command(monkeypatch, tmp_path: Path):
|
|
109
|
+
captured = {}
|
|
110
|
+
|
|
111
|
+
def fake_run_shell(*args, **kwargs):
|
|
112
|
+
captured["args"] = args
|
|
113
|
+
captured["kwargs"] = kwargs
|
|
114
|
+
return subprocess.CompletedProcess(args=args, returncode=0, stdout="ok", stderr="")
|
|
115
|
+
|
|
116
|
+
monkeypatch.setattr(utils, "check_docker_available", lambda: True)
|
|
117
|
+
monkeypatch.setattr(utils, "run_shell", fake_run_shell)
|
|
118
|
+
|
|
119
|
+
result = utils.run_command_in_docker(tmp_path, "pnpm test", timeout=45)
|
|
120
|
+
|
|
121
|
+
assert result.returncode == 0
|
|
122
|
+
assert captured["args"] == (
|
|
123
|
+
"docker.sh",
|
|
124
|
+
"run-command",
|
|
125
|
+
str(tmp_path),
|
|
126
|
+
"--timeout",
|
|
127
|
+
"45",
|
|
128
|
+
"--",
|
|
129
|
+
"pnpm test",
|
|
130
|
+
)
|
|
131
|
+
assert captured["kwargs"]["timeout"] == 75
|
|
132
|
+
|
|
133
|
+
|
|
74
134
|
def test_run_shell_decodes_subprocess_output_as_utf8(monkeypatch):
|
|
75
135
|
captured = {}
|
|
76
136
|
|
|
@@ -156,6 +216,95 @@ def test_bash_env_bridges_eval_keys_to_wsl(monkeypatch):
|
|
|
156
216
|
assert "ANTHROPIC_API_KEY" not in names
|
|
157
217
|
|
|
158
218
|
|
|
219
|
+
def test_bash_env_bridges_selectable_agent_credentials_to_wsl(monkeypatch):
|
|
220
|
+
monkeypatch.setattr(utils.os, "name", "nt")
|
|
221
|
+
monkeypatch.setattr(
|
|
222
|
+
utils,
|
|
223
|
+
"BASH_EXEC",
|
|
224
|
+
r"C:\Users\BENYM\AppData\Local\Microsoft\WindowsApps\bash.exe",
|
|
225
|
+
)
|
|
226
|
+
monkeypatch.setenv("WSLENV", "")
|
|
227
|
+
monkeypatch.setenv("QODER_PERSONAL_ACCESS_TOKEN", "qoder-token")
|
|
228
|
+
|
|
229
|
+
env = utils._bash_env()
|
|
230
|
+
|
|
231
|
+
assert "QODER_PERSONAL_ACCESS_TOKEN" in env["WSLENV"].split(":")
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def test_bash_env_bridges_codebuddy_credentials_and_model_to_wsl(monkeypatch):
|
|
235
|
+
monkeypatch.setattr(utils.os, "name", "nt")
|
|
236
|
+
monkeypatch.setattr(
|
|
237
|
+
utils,
|
|
238
|
+
"BASH_EXEC",
|
|
239
|
+
r"C:\Users\BENYM\AppData\Local\Microsoft\WindowsApps\bash.exe",
|
|
240
|
+
)
|
|
241
|
+
monkeypatch.setenv("WSLENV", "")
|
|
242
|
+
monkeypatch.setenv("CODEBUDDY_API_KEY", "codebuddy-key")
|
|
243
|
+
monkeypatch.setenv("CODEBUDDY_MODEL", "codebuddy-model")
|
|
244
|
+
monkeypatch.setenv("CODEBUDDY_SMALL_FAST_MODEL", "codebuddy-fast-model")
|
|
245
|
+
monkeypatch.setenv("CODEBUDDY_BIG_SLOW_MODEL", "codebuddy-reasoning-model")
|
|
246
|
+
monkeypatch.setenv("CODEBUDDY_CODE_SUBAGENT_MODEL", "codebuddy-subagent-model")
|
|
247
|
+
|
|
248
|
+
env = utils._bash_env()
|
|
249
|
+
|
|
250
|
+
assert "CODEBUDDY_API_KEY" in env["WSLENV"].split(":")
|
|
251
|
+
assert "CODEBUDDY_MODEL" in env["WSLENV"].split(":")
|
|
252
|
+
assert "CODEBUDDY_SMALL_FAST_MODEL" in env["WSLENV"].split(":")
|
|
253
|
+
assert "CODEBUDDY_BIG_SLOW_MODEL" in env["WSLENV"].split(":")
|
|
254
|
+
assert "CODEBUDDY_CODE_SUBAGENT_MODEL" in env["WSLENV"].split(":")
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def test_bash_env_bridges_custom_agent_model_and_base_url_to_wsl(monkeypatch):
|
|
258
|
+
monkeypatch.setattr(utils.os, "name", "nt")
|
|
259
|
+
monkeypatch.setattr(
|
|
260
|
+
utils,
|
|
261
|
+
"BASH_EXEC",
|
|
262
|
+
r"C:\Users\BENYM\AppData\Local\Microsoft\WindowsApps\bash.exe",
|
|
263
|
+
)
|
|
264
|
+
monkeypatch.setenv("WSLENV", "")
|
|
265
|
+
monkeypatch.setenv("COMET_EVAL_CUSTOM_MODEL_ENV", "FIXTURE_AGENT_MODEL")
|
|
266
|
+
monkeypatch.setenv("COMET_EVAL_CUSTOM_BASE_URL_ENV", "FIXTURE_AGENT_BASE_URL")
|
|
267
|
+
monkeypatch.setenv("FIXTURE_AGENT_MODEL", "fixture-model")
|
|
268
|
+
monkeypatch.setenv("FIXTURE_AGENT_BASE_URL", "https://fixture.example/v1")
|
|
269
|
+
|
|
270
|
+
env = utils._bash_env()
|
|
271
|
+
|
|
272
|
+
assert "FIXTURE_AGENT_MODEL" in env["WSLENV"].split(":")
|
|
273
|
+
assert "FIXTURE_AGENT_BASE_URL" in env["WSLENV"].split(":")
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def test_run_agent_in_docker_builds_the_selected_adapter_command(monkeypatch, tmp_path: Path):
|
|
277
|
+
calls = []
|
|
278
|
+
|
|
279
|
+
def fake_run_shell(script, *args, **kwargs):
|
|
280
|
+
calls.append((script, args, kwargs))
|
|
281
|
+
return subprocess.CompletedProcess([script, *args], 0, "", "")
|
|
282
|
+
|
|
283
|
+
monkeypatch.setattr(utils, "run_shell", fake_run_shell)
|
|
284
|
+
monkeypatch.setattr(utils, "check_docker_available", lambda: True)
|
|
285
|
+
|
|
286
|
+
result = utils.run_agent_in_docker(
|
|
287
|
+
str(tmp_path),
|
|
288
|
+
"inspect the task",
|
|
289
|
+
agent="codex",
|
|
290
|
+
model="test-model",
|
|
291
|
+
timeout=42,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
assert result.returncode == 0
|
|
295
|
+
assert calls[0][1] == (
|
|
296
|
+
"run-agent",
|
|
297
|
+
str(tmp_path),
|
|
298
|
+
"inspect the task",
|
|
299
|
+
"--agent",
|
|
300
|
+
"codex",
|
|
301
|
+
"--model",
|
|
302
|
+
"test-model",
|
|
303
|
+
"--timeout",
|
|
304
|
+
"42",
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
|
|
159
308
|
def test_bash_env_bridges_langsmith_hook_log_to_wsl(monkeypatch):
|
|
160
309
|
monkeypatch.setattr(utils.os, "name", "nt")
|
|
161
310
|
monkeypatch.setattr(
|
|
@@ -200,6 +349,26 @@ def test_claude_loop_timeout_force_removes_its_named_container(monkeypatch, tmp_
|
|
|
200
349
|
assert "cleanup-claude-loop" in docker_sh
|
|
201
350
|
|
|
202
351
|
|
|
352
|
+
def test_agent_loop_timeout_force_removes_its_named_container(monkeypatch, tmp_path: Path):
|
|
353
|
+
calls = []
|
|
354
|
+
|
|
355
|
+
def fake_run_shell(script, *args, **kwargs):
|
|
356
|
+
calls.append((script, args, kwargs))
|
|
357
|
+
if args[0] == "run-agent-loop":
|
|
358
|
+
raise subprocess.TimeoutExpired([script, *args], timeout=42)
|
|
359
|
+
return subprocess.CompletedProcess([script, *args], 0, "", "")
|
|
360
|
+
|
|
361
|
+
monkeypatch.setattr(utils, "run_shell", fake_run_shell)
|
|
362
|
+
|
|
363
|
+
result = utils.run_agent_loop_in_docker(
|
|
364
|
+
str(tmp_path), ["prompt", "--agent", "codex"], timeout=42
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
assert result.returncode == 124
|
|
368
|
+
assert calls[0][1][0] == "run-agent-loop"
|
|
369
|
+
assert calls[1][1] == ("cleanup-agent-loop", str(tmp_path))
|
|
370
|
+
|
|
371
|
+
|
|
203
372
|
def test_docker_subject_run_uses_controller_verified_immutable_image_identity():
|
|
204
373
|
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
205
374
|
|
|
@@ -210,6 +379,182 @@ def test_docker_subject_run_uses_controller_verified_immutable_image_identity():
|
|
|
210
379
|
assert '"$image_id"' in docker_sh
|
|
211
380
|
|
|
212
381
|
|
|
382
|
+
def test_docker_script_dispatches_all_supported_agent_clis():
|
|
383
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
384
|
+
runtime_sh = (utils.SHELL_DIR / "run-agent-runtime.sh").read_text(encoding="utf-8")
|
|
385
|
+
|
|
386
|
+
assert "run-agent" in docker_sh
|
|
387
|
+
assert "codex exec" in runtime_sh
|
|
388
|
+
assert "qodercli" in runtime_sh
|
|
389
|
+
assert "QODER_PERSONAL_ACCESS_TOKEN" in docker_sh
|
|
390
|
+
assert "codebuddy" in runtime_sh
|
|
391
|
+
assert "CODEBUDDY_API_KEY" in docker_sh
|
|
392
|
+
|
|
393
|
+
overlay = (utils.SHELL_DIR.parent / "docker" / "agent-overlay.Dockerfile").read_text(
|
|
394
|
+
encoding="utf-8"
|
|
395
|
+
)
|
|
396
|
+
assert "@tencent-ai/codebuddy-code" in overlay
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def test_codebuddy_forwards_official_model_variant_environment_variables():
|
|
400
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
401
|
+
|
|
402
|
+
assert "CODEBUDDY_SMALL_FAST_MODEL" in docker_sh
|
|
403
|
+
assert "CODEBUDDY_BIG_SLOW_MODEL" in docker_sh
|
|
404
|
+
assert "CODEBUDDY_CODE_SUBAGENT_MODEL" in docker_sh
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def test_codex_commands_use_explicit_openai_base_url_config():
|
|
408
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
409
|
+
loop_sh = (utils.SHELL_DIR / "run-claude-loop.sh").read_text(encoding="utf-8")
|
|
410
|
+
|
|
411
|
+
assert 'base_url = ' in (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
|
|
412
|
+
assert 'model_provider = "comet-eval"' in (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
|
|
413
|
+
assert 'openai_base_url=$OPENAI_BASE_URL' not in docker_sh
|
|
414
|
+
assert 'openai_base_url=$OPENAI_BASE_URL' not in loop_sh
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def test_agent_runtime_credentials_use_ephemeral_cli_config_roots():
|
|
418
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
419
|
+
config_sh = (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
|
|
420
|
+
|
|
421
|
+
assert "run-agent-runtime.sh" in docker_sh
|
|
422
|
+
assert "--tmpfs" in docker_sh
|
|
423
|
+
assert "bash //opt/scaffold-shell/run-agent-runtime.sh" in docker_sh
|
|
424
|
+
assert "/home/agent/.codex" in docker_sh
|
|
425
|
+
assert "/home/agent/.qoder" in docker_sh
|
|
426
|
+
assert "/home/agent/.codebuddy" in docker_sh
|
|
427
|
+
assert 'env_key = "OPENAI_API_KEY"' in config_sh
|
|
428
|
+
assert 'model_provider = "comet-eval"' in config_sh
|
|
429
|
+
assert '"apiKeyHelper"' in config_sh
|
|
430
|
+
assert "CODEBUDDY_API_KEY" in config_sh
|
|
431
|
+
assert "QODER_CONFIG_DIR" in config_sh
|
|
432
|
+
assert "config.toml" in config_sh
|
|
433
|
+
assert "settings.json" in config_sh
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def test_agent_runtime_config_is_agent_only_and_docker_does_not_inline_secrets():
|
|
437
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
438
|
+
config_sh = (utils.SHELL_DIR / "agent-runtime-config.sh").read_text(encoding="utf-8")
|
|
439
|
+
generic_run = docker_sh.split("docker_run() {", 1)[1].split(
|
|
440
|
+
"# Run Claude CLI in Docker", 1
|
|
441
|
+
)[0]
|
|
442
|
+
|
|
443
|
+
assert "COMET_EVAL_CODEBUDDY_CONFIG_DIR" in config_sh
|
|
444
|
+
assert 'ENV_ARGS+=("-e" "$key")' in docker_sh
|
|
445
|
+
assert 'ENV_ARGS+=("-e" "$key=${!key}")' not in docker_sh
|
|
446
|
+
assert "build_agent_runtime_mount_args" not in generic_run
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def test_agent_runtime_config_files_never_contain_credential_literals(tmp_path):
|
|
450
|
+
codex_home = utils._to_bash_path(tmp_path / "codex")
|
|
451
|
+
qoder_home = utils._to_bash_path(tmp_path / "qoder")
|
|
452
|
+
codebuddy_home = utils._to_bash_path(tmp_path / "codebuddy")
|
|
453
|
+
script = utils._to_bash_path(utils.SHELL_DIR / "agent-runtime-config.sh")
|
|
454
|
+
env = dict(os.environ)
|
|
455
|
+
env.update(
|
|
456
|
+
{
|
|
457
|
+
"CODEX_HOME": codex_home,
|
|
458
|
+
"QODER_CONFIG_DIR": qoder_home,
|
|
459
|
+
"COMET_EVAL_CODEBUDDY_CONFIG_DIR": codebuddy_home,
|
|
460
|
+
"OPENAI_API_KEY": "codex-sentinel",
|
|
461
|
+
"OPENAI_BASE_URL": "https://codex.example/v1",
|
|
462
|
+
"CODEBUDDY_API_KEY": "codebuddy-sentinel",
|
|
463
|
+
"CODEBUDDY_BASE_URL": "https://codebuddy.example/v1",
|
|
464
|
+
}
|
|
465
|
+
)
|
|
466
|
+
result = subprocess.run(
|
|
467
|
+
[
|
|
468
|
+
utils.BASH_EXEC,
|
|
469
|
+
"-c",
|
|
470
|
+
f'source "{script}"; '
|
|
471
|
+
'prepare_agent_runtime_config codex "subject-model"; '
|
|
472
|
+
'prepare_agent_runtime_config qoder "subject-model"; '
|
|
473
|
+
'prepare_agent_runtime_config codebuddy "subject-model"',
|
|
474
|
+
],
|
|
475
|
+
env=utils._bash_env(env),
|
|
476
|
+
capture_output=True,
|
|
477
|
+
text=True,
|
|
478
|
+
check=False,
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
assert result.returncode == 0, result.stderr
|
|
482
|
+
config_text = (tmp_path / "codex" / "config.toml").read_text(encoding="utf-8")
|
|
483
|
+
settings_text = (tmp_path / "codebuddy" / "settings.json").read_text(encoding="utf-8")
|
|
484
|
+
helper_text = (tmp_path / "codebuddy" / "api-key-helper.sh").read_text(encoding="utf-8")
|
|
485
|
+
|
|
486
|
+
assert 'env_key = "OPENAI_API_KEY"' in config_text
|
|
487
|
+
assert "codex-sentinel" not in config_text
|
|
488
|
+
assert '"apiKeyHelper"' in settings_text
|
|
489
|
+
assert "codebuddy-sentinel" not in settings_text
|
|
490
|
+
assert "codebuddy-sentinel" not in helper_text
|
|
491
|
+
assert "CODEBUDDY_API_KEY" in helper_text
|
|
492
|
+
assert not list((tmp_path / "qoder").iterdir())
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def test_codebuddy_api_key_helper_prefers_auth_token_without_persisting_it(tmp_path):
|
|
496
|
+
codebuddy_home = utils._to_bash_path(tmp_path / "codebuddy")
|
|
497
|
+
script = utils._to_bash_path(utils.SHELL_DIR / "agent-runtime-config.sh")
|
|
498
|
+
env = dict(os.environ)
|
|
499
|
+
env.update(
|
|
500
|
+
{
|
|
501
|
+
"COMET_EVAL_CODEBUDDY_CONFIG_DIR": codebuddy_home,
|
|
502
|
+
"CODEBUDDY_AUTH_TOKEN": "auth-token-sentinel",
|
|
503
|
+
"CODEBUDDY_API_KEY": "api-key-sentinel",
|
|
504
|
+
}
|
|
505
|
+
)
|
|
506
|
+
result = subprocess.run(
|
|
507
|
+
[
|
|
508
|
+
utils.BASH_EXEC,
|
|
509
|
+
"-c",
|
|
510
|
+
f'source "{script}"; '
|
|
511
|
+
'prepare_agent_runtime_config codebuddy "subject-model"; '
|
|
512
|
+
'bash "$COMET_EVAL_CODEBUDDY_CONFIG_DIR/api-key-helper.sh"',
|
|
513
|
+
],
|
|
514
|
+
env=utils._bash_env(env),
|
|
515
|
+
capture_output=True,
|
|
516
|
+
text=True,
|
|
517
|
+
check=False,
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
assert result.returncode == 0, result.stderr
|
|
521
|
+
assert result.stdout == "auth-token-sentinel"
|
|
522
|
+
helper_text = (tmp_path / "codebuddy" / "api-key-helper.sh").read_text(encoding="utf-8")
|
|
523
|
+
assert "auth-token-sentinel" not in helper_text
|
|
524
|
+
assert "api-key-sentinel" not in helper_text
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def test_user_authored_validation_commands_receive_no_agent_or_reporting_secrets():
|
|
528
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
529
|
+
command_block = docker_sh.split("docker_run_command() {", 1)[1].split(
|
|
530
|
+
"# Run Python script in Docker", 1
|
|
531
|
+
)[0]
|
|
532
|
+
|
|
533
|
+
assert "build_env_args" not in command_block
|
|
534
|
+
assert "ENV_ARGS" not in command_block
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def test_docker_harness_mounts_langfuse_official_plugin_and_enables_trace_image():
|
|
538
|
+
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
539
|
+
overlay = (utils.SHELL_DIR.parent / "docker" / "agent-overlay.Dockerfile").read_text(
|
|
540
|
+
encoding="utf-8"
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
assert "LANGFUSE_TRAJECTORY_PLUGIN_DIR" in docker_sh
|
|
544
|
+
assert "build_langfuse_plugin_args" in docker_sh
|
|
545
|
+
assert "comet-langfuse-plugin" in docker_sh
|
|
546
|
+
assert "LANGFUSE_ENABLED" in overlay
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def test_interaction_driver_accepts_the_selected_agent():
|
|
550
|
+
loop_sh = (utils.SHELL_DIR / "run-claude-loop.sh").read_text(encoding="utf-8")
|
|
551
|
+
|
|
552
|
+
assert "--agent" in loop_sh
|
|
553
|
+
assert "codex exec" in loop_sh
|
|
554
|
+
assert "qodercli" in loop_sh
|
|
555
|
+
assert "codebuddy" in loop_sh
|
|
556
|
+
|
|
557
|
+
|
|
213
558
|
def test_docker_harness_has_no_native_review_sidecar_contract():
|
|
214
559
|
docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
|
|
215
560
|
|
|
@@ -247,6 +592,44 @@ def _get_image_name(directory: Path) -> str:
|
|
|
247
592
|
return result.stdout.strip()
|
|
248
593
|
|
|
249
594
|
|
|
595
|
+
def _get_custom_image_name(directory: Path, *, version: str) -> str:
|
|
596
|
+
"""Resolve a custom-agent image name with the selected install metadata."""
|
|
597
|
+
script = (
|
|
598
|
+
'source "$1"; '
|
|
599
|
+
'image=$(get_image_name "$2" fixture-agent) || { echo "image lookup failed"; exit 1; }; '
|
|
600
|
+
'echo "image=$image"'
|
|
601
|
+
)
|
|
602
|
+
environment = {
|
|
603
|
+
**os.environ,
|
|
604
|
+
"COMET_EVAL_CUSTOM_AGENT_ID": "fixture-agent",
|
|
605
|
+
"COMET_EVAL_CUSTOM_EXECUTABLE": "fixture-agent-cli",
|
|
606
|
+
"COMET_EVAL_CUSTOM_INSTALL_KIND": "npm",
|
|
607
|
+
"COMET_EVAL_CUSTOM_INSTALL_PACKAGE": "fixture-agent-package",
|
|
608
|
+
"COMET_EVAL_CUSTOM_INSTALL_VERSION": version,
|
|
609
|
+
}
|
|
610
|
+
try:
|
|
611
|
+
result = subprocess.run(
|
|
612
|
+
[
|
|
613
|
+
utils.BASH_EXEC,
|
|
614
|
+
"-c",
|
|
615
|
+
script,
|
|
616
|
+
"_",
|
|
617
|
+
utils._to_bash_path(utils.SHELL_DIR / "docker.sh"),
|
|
618
|
+
utils._to_bash_path(directory),
|
|
619
|
+
],
|
|
620
|
+
env=environment,
|
|
621
|
+
capture_output=True,
|
|
622
|
+
text=True,
|
|
623
|
+
encoding="utf-8",
|
|
624
|
+
errors="replace",
|
|
625
|
+
check=False,
|
|
626
|
+
)
|
|
627
|
+
except FileNotFoundError:
|
|
628
|
+
pytest.skip("bash not available")
|
|
629
|
+
assert result.returncode == 0, f"get_image_name failed (stderr: {result.stderr})"
|
|
630
|
+
return result.stdout.strip()
|
|
631
|
+
|
|
632
|
+
|
|
250
633
|
def test_get_image_name_falls_back_to_environment_dockerfile(tmp_path: Path):
|
|
251
634
|
"""get_image_name() resolves environment/Dockerfile when dir/Dockerfile is absent.
|
|
252
635
|
|
|
@@ -291,6 +674,13 @@ def test_get_image_name_prefers_root_dockerfile_over_environment(tmp_path: Path)
|
|
|
291
674
|
assert _get_image_name(both) != _get_image_name(env_only)
|
|
292
675
|
|
|
293
676
|
|
|
677
|
+
def test_custom_image_name_changes_when_install_version_changes(tmp_path: Path):
|
|
678
|
+
(tmp_path / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
|
|
679
|
+
first = _get_custom_image_name(tmp_path, version="1.0.0")
|
|
680
|
+
second = _get_custom_image_name(tmp_path, version="2.0.0")
|
|
681
|
+
assert first != second
|
|
682
|
+
|
|
683
|
+
|
|
294
684
|
def test_run_claude_fixture_defaults_langsmith_hook_log_path():
|
|
295
685
|
conftest_py = (Path(__file__).resolve().parents[1] / "conftest.py").read_text(encoding="utf-8")
|
|
296
686
|
|
|
@@ -310,13 +700,15 @@ def test_claude_loop_applies_plugin_args_to_subject_turns_only():
|
|
|
310
700
|
'bash "$SCRIPT_DIR/decision-point.sh" "$RESULT_TEXT"'
|
|
311
701
|
)
|
|
312
702
|
assert 'SUBJECT_PROMPT="${FRESH_PROMPT:-$PROMPT}"' in loop_sh
|
|
313
|
-
assert '
|
|
314
|
-
assert '
|
|
703
|
+
assert 'run_agent_turn "$SUBJECT_PROMPT" "" "subject"' in loop_sh
|
|
704
|
+
assert 'role_plugin_args=("${PLUGIN_ARGS[@]}")' in loop_sh
|
|
705
|
+
assert 'claude -p "$prompt" "${role_plugin_args[@]}"' in loop_sh
|
|
706
|
+
assert 'run_agent_turn "$USER_REPLY" "$SESSION_ID" "subject"' in loop_sh
|
|
315
707
|
assert "fresh resume boundary detected" in loop_sh
|
|
316
708
|
assert 'claude -p "$sim_prompt" "${PLUGIN_ARGS[@]}"' not in loop_sh
|
|
317
709
|
assert 'if ! rm -f -- "$SIMULATOR_PROMPT_FILE"; then' in loop_sh
|
|
318
710
|
assert loop_sh.index('rm -f -- "$SIMULATOR_PROMPT_FILE"') < loop_sh.index(
|
|
319
|
-
'RAW=$(
|
|
711
|
+
'RAW=$(run_agent_turn "$SUBJECT_PROMPT"'
|
|
320
712
|
)
|
|
321
713
|
|
|
322
714
|
|
|
@@ -370,6 +762,90 @@ printf '%s\n' '{"type":"result","subtype":"success","session_id":"session-1","re
|
|
|
370
762
|
assert "subject turn 2 failed" in result.stderr
|
|
371
763
|
|
|
372
764
|
|
|
765
|
+
def test_claude_loop_surfaces_simulator_failure(tmp_path: Path):
|
|
766
|
+
fake_bin = tmp_path / "bin"
|
|
767
|
+
fake_bin.mkdir()
|
|
768
|
+
fake_claude = fake_bin / "claude"
|
|
769
|
+
fake_claude.write_text(
|
|
770
|
+
"""#!/usr/bin/env bash
|
|
771
|
+
if [[ "$COMET_EVAL_AGENT_ROLE" == "simulator" ]]; then
|
|
772
|
+
echo "simulator failed" >&2
|
|
773
|
+
exit 44
|
|
774
|
+
fi
|
|
775
|
+
printf '%s\n' '{"type":"system","session_id":"subject-1"}'
|
|
776
|
+
printf '%s\n' '{"type":"result","subtype":"success","session_id":"subject-1","result":"Question: Which option should be used? Recommendation: A."}'
|
|
777
|
+
""",
|
|
778
|
+
encoding="utf-8",
|
|
779
|
+
newline="\n",
|
|
780
|
+
)
|
|
781
|
+
fake_claude.chmod(0o755)
|
|
782
|
+
|
|
783
|
+
env = os.environ.copy()
|
|
784
|
+
env["PATH"] = f"{utils._to_bash_path(fake_bin)}:{env.get('PATH', '')}"
|
|
785
|
+
result = subprocess.run(
|
|
786
|
+
[
|
|
787
|
+
utils.BASH_EXEC,
|
|
788
|
+
utils._to_bash_path(utils.SHELL_DIR / "run-claude-loop.sh"),
|
|
789
|
+
"Implement the requested change.",
|
|
790
|
+
"--max-turns",
|
|
791
|
+
"2",
|
|
792
|
+
],
|
|
793
|
+
capture_output=True,
|
|
794
|
+
text=True,
|
|
795
|
+
encoding="utf-8",
|
|
796
|
+
errors="replace",
|
|
797
|
+
timeout=10,
|
|
798
|
+
check=False,
|
|
799
|
+
env=env,
|
|
800
|
+
)
|
|
801
|
+
|
|
802
|
+
assert result.returncode == 44
|
|
803
|
+
assert "simulator turn failed (exit 44)" in result.stderr
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def test_non_claude_loop_does_not_inherit_anthropic_model(tmp_path: Path):
|
|
807
|
+
fake_bin = tmp_path / "bin"
|
|
808
|
+
fake_bin.mkdir()
|
|
809
|
+
captured_args = tmp_path / "codex-args.txt"
|
|
810
|
+
fake_codex = fake_bin / "codex"
|
|
811
|
+
fake_codex.write_text(
|
|
812
|
+
"""#!/usr/bin/env bash
|
|
813
|
+
printf '%s\n' "$*" > "$CAPTURED_CODEX_ARGS"
|
|
814
|
+
printf '%s\n' '{"type":"thread.started","thread_id":"subject-1"}'
|
|
815
|
+
printf '%s\n' '{"type":"result","result":"Workflow completed through all phases and archived."}'
|
|
816
|
+
""",
|
|
817
|
+
encoding="utf-8",
|
|
818
|
+
newline="\n",
|
|
819
|
+
)
|
|
820
|
+
fake_codex.chmod(0o755)
|
|
821
|
+
|
|
822
|
+
env = os.environ.copy()
|
|
823
|
+
env["PATH"] = f"{utils._to_bash_path(fake_bin)}:{env.get('PATH', '')}"
|
|
824
|
+
env["ANTHROPIC_MODEL"] = "claude-only-model"
|
|
825
|
+
env["CAPTURED_CODEX_ARGS"] = utils._to_bash_path(captured_args)
|
|
826
|
+
result = subprocess.run(
|
|
827
|
+
[
|
|
828
|
+
utils.BASH_EXEC,
|
|
829
|
+
utils._to_bash_path(utils.SHELL_DIR / "run-claude-loop.sh"),
|
|
830
|
+
"Implement the requested change.",
|
|
831
|
+
"--agent",
|
|
832
|
+
"codex",
|
|
833
|
+
"--max-turns",
|
|
834
|
+
"1",
|
|
835
|
+
],
|
|
836
|
+
capture_output=True,
|
|
837
|
+
text=True,
|
|
838
|
+
encoding="utf-8",
|
|
839
|
+
errors="replace",
|
|
840
|
+
timeout=10,
|
|
841
|
+
check=False,
|
|
842
|
+
env=env,
|
|
843
|
+
)
|
|
844
|
+
|
|
845
|
+
assert result.returncode == 0, result.stderr
|
|
846
|
+
assert "claude-only-model" not in captured_args.read_text(encoding="utf-8")
|
|
847
|
+
|
|
848
|
+
|
|
373
849
|
def test_claude_loop_consumes_deterministic_reply_steps_in_order(tmp_path: Path):
|
|
374
850
|
fake_bin = tmp_path / "bin"
|
|
375
851
|
fake_bin.mkdir()
|