@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -26
- package/assets/manifest.json +4 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
- package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
- package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
- package/assets/skills/comet-build/SKILL.md +36 -62
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +2 -2
- package/assets/skills/comet-native/SKILL.md +18 -15
- package/assets/skills/comet-native/reference/artifacts.md +5 -4
- package/assets/skills/comet-native/reference/clarification.md +1 -1
- package/assets/skills/comet-native/reference/commands.md +4 -4
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills/comet-review/SKILL.md +121 -0
- package/assets/skills/comet-review/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +36 -62
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +2 -2
- package/assets/skills-zh/comet-native/SKILL.md +18 -15
- package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
- package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
- package/assets/skills-zh/comet-native/reference/commands.md +4 -4
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-review/SKILL.md +121 -0
- package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +5 -8
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +5 -6
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
- package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.js +10 -0
- package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +4 -4
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.js +44 -6
- package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.js +2 -1
- package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +39 -13
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +2 -2
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +5 -1
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +2 -0
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.js +17 -10
- package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +65 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +453 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +13 -7
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +18 -5
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +56 -4
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +179 -20
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +178 -73
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +168 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
- package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
- package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.js +24 -6
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.js +86 -35
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
- package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +27 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +108 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +87 -44
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +14 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +93 -5
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +3 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +4 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +16 -10
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +11 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +39 -29
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
- package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +4 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +174 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +13 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/fs/strip-bom.d.ts +10 -0
- package/dist/platform/fs/strip-bom.d.ts.map +1 -0
- package/dist/platform/fs/strip-bom.js +12 -0
- package/dist/platform/fs/strip-bom.js.map +1 -0
- package/dist/platform/install/platforms.d.ts +11 -0
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +49 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +1 -0
- package/dist/platform/process/external-command.d.ts.map +1 -1
- package/dist/platform/process/external-command.js +2 -1
- package/dist/platform/process/external-command.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +719 -83
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +18 -28
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
- package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
Generates rich experiment logs in logs/experiments/ including:
|
|
4
4
|
- summary.md: Full markdown report with tables and details
|
|
5
5
|
- events/: Parsed events from each test run
|
|
6
|
-
- raw/: Raw
|
|
6
|
+
- raw/: Raw selected Agent CLI output
|
|
7
7
|
- reports/: Per-run validation reports
|
|
8
|
-
- artifacts/: Files
|
|
8
|
+
- artifacts/: Files selected Agent generated and their execution output
|
|
9
9
|
- metadata.json: Experiment metadata
|
|
10
10
|
|
|
11
11
|
Supports pytest-xdist parallel execution via worker coordination.
|
|
@@ -27,7 +27,7 @@ from typing import Any
|
|
|
27
27
|
|
|
28
28
|
import pytest
|
|
29
29
|
|
|
30
|
-
from scaffold import
|
|
30
|
+
from scaffold import run_agent_in_docker, run_python_in_docker, run_shell
|
|
31
31
|
from scaffold.python import (
|
|
32
32
|
ExperimentLogger,
|
|
33
33
|
TreatmentResult,
|
|
@@ -40,9 +40,40 @@ from scaffold.python import (
|
|
|
40
40
|
save_report,
|
|
41
41
|
strip_ansi,
|
|
42
42
|
)
|
|
43
|
+
from scaffold.python.agents import get_agent_adapter, resolve_agent, validate_agent_capabilities
|
|
44
|
+
from scaffold.python.auto_tasks import AutoTaskError, ensure_generated_manifest, find_project_root
|
|
45
|
+
from scaffold.python.eval_context import (
|
|
46
|
+
EvalContextError,
|
|
47
|
+
context_from_environment,
|
|
48
|
+
resolve_eval_context,
|
|
49
|
+
resolve_managed_path,
|
|
50
|
+
)
|
|
51
|
+
from scaffold.python.execution import (
|
|
52
|
+
ResolvedExecution,
|
|
53
|
+
ResolvedJudge,
|
|
54
|
+
build_agent_environment,
|
|
55
|
+
missing_credentials,
|
|
56
|
+
preflight_credentials,
|
|
57
|
+
redact_sensitive,
|
|
58
|
+
resolve_execution,
|
|
59
|
+
resolve_judge,
|
|
60
|
+
validate_base_url,
|
|
61
|
+
)
|
|
62
|
+
from scaffold.python.generated_task_cache import (
|
|
63
|
+
selected_agent_model as _legacy_selected_agent_model,
|
|
64
|
+
)
|
|
65
|
+
from scaffold.python.langfuse_adapter import (
|
|
66
|
+
install_codex_plugin_workspace,
|
|
67
|
+
install_transcript_hook,
|
|
68
|
+
provision_trajectory_plugin,
|
|
69
|
+
)
|
|
43
70
|
from scaffold.python.sample_quality import infer_sample_quality
|
|
44
71
|
from scaffold.python.skill_parser import SCRIPT_EXTENSIONS
|
|
45
|
-
from scaffold.python.utils import
|
|
72
|
+
from scaffold.python.utils import (
|
|
73
|
+
check_docker_available,
|
|
74
|
+
run_agent_loop_in_docker,
|
|
75
|
+
run_claude_loop_in_docker,
|
|
76
|
+
)
|
|
46
77
|
from scaffold.python.aligned_comparison import (
|
|
47
78
|
EXPECTED_CASE_MATRIX_FILENAME,
|
|
48
79
|
build_execution_identity,
|
|
@@ -56,14 +87,21 @@ def _extract_loop_turns(stderr: str | None) -> int | None:
|
|
|
56
87
|
return int(match.group(1)) if match else None
|
|
57
88
|
|
|
58
89
|
|
|
59
|
-
def _extract_loop_interaction(stderr: str | None) -> dict[str,
|
|
90
|
+
def _extract_loop_interaction(stderr: str | None) -> dict[str, Any]:
|
|
60
91
|
source = stderr or ""
|
|
92
|
+
role_sessions = {"subject": [], "simulator": [], "judge": []}
|
|
93
|
+
for role, session_id in re.findall(
|
|
94
|
+
r"\[loop\] role-session (subject|simulator|judge) ([^\s]+)", source
|
|
95
|
+
):
|
|
96
|
+
if session_id not in role_sessions[role]:
|
|
97
|
+
role_sessions[role].append(session_id)
|
|
61
98
|
return {
|
|
62
99
|
"actual_turns": _extract_loop_turns(source),
|
|
63
100
|
"decision_points": source.count("[loop] decision point detected"),
|
|
64
101
|
"deterministic_replies": source.count("[loop] deterministic decision reply applied"),
|
|
65
102
|
"completion_signals": source.count("[loop] workflow completion detected"),
|
|
66
103
|
"fresh_resume_boundaries": source.count("[loop] fresh resume boundary detected"),
|
|
104
|
+
"role_sessions": role_sessions,
|
|
67
105
|
}
|
|
68
106
|
|
|
69
107
|
|
|
@@ -212,8 +250,12 @@ EVAL_ROOT = PROJECT_ROOT.parent
|
|
|
212
250
|
REPOSITORY_ROOT = EVAL_ROOT.parent
|
|
213
251
|
|
|
214
252
|
# Shared files for xdist worker coordination
|
|
215
|
-
|
|
216
|
-
|
|
253
|
+
_RUNTIME_CONTEXT = context_from_environment()
|
|
254
|
+
_CONTEXT_LOCK_ROOT = (
|
|
255
|
+
resolve_managed_path(_RUNTIME_CONTEXT, "locks") if _RUNTIME_CONTEXT else PROJECT_ROOT
|
|
256
|
+
)
|
|
257
|
+
XDIST_EXPERIMENT_FILE = _CONTEXT_LOCK_ROOT / ".pytest_experiment_id"
|
|
258
|
+
DOCKER_BUILD_LOCK = _CONTEXT_LOCK_ROOT / ".pytest_docker_build.lock"
|
|
217
259
|
EXPERIMENT_ID_ENV = "COMET_EVAL_EXPERIMENT_ID"
|
|
218
260
|
EXPERIMENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
|
219
261
|
|
|
@@ -246,12 +288,305 @@ MODEL_EXECUTION_ENV_KEYS = (
|
|
|
246
288
|
"ANTHROPIC_DEFAULT_OPUS_MODEL_NAME",
|
|
247
289
|
"ANTHROPIC_DEFAULT_SONNET_MODEL_NAME",
|
|
248
290
|
"CLAUDE_CODE_SUBAGENT_MODEL",
|
|
291
|
+
"CODEBUDDY_BASE_URL",
|
|
292
|
+
"CODEBUDDY_MODEL",
|
|
293
|
+
"CODEBUDDY_SMALL_FAST_MODEL",
|
|
294
|
+
"CODEBUDDY_BIG_SLOW_MODEL",
|
|
295
|
+
"CODEBUDDY_CODE_SUBAGENT_MODEL",
|
|
296
|
+
"BENCH_CODEBUDDY_MODEL",
|
|
297
|
+
"OPENAI_BASE_URL",
|
|
298
|
+
"OPENAI_MODEL",
|
|
299
|
+
"CODEX_BASE_URL",
|
|
300
|
+
"CODEX_MODEL",
|
|
301
|
+
"QODER_BASE_URL",
|
|
302
|
+
"QODER_MODEL",
|
|
249
303
|
)
|
|
250
304
|
|
|
251
305
|
|
|
252
|
-
def
|
|
253
|
-
|
|
254
|
-
|
|
306
|
+
def _getoption(config, name: str, default=None):
|
|
307
|
+
try:
|
|
308
|
+
value = config.getoption(name)
|
|
309
|
+
except (AttributeError, KeyError, ValueError):
|
|
310
|
+
return default
|
|
311
|
+
return default if value is None else value
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _manifest_for_config(config):
|
|
315
|
+
manifest_path = _getoption(config, "--eval-manifest")
|
|
316
|
+
if not manifest_path:
|
|
317
|
+
return None
|
|
318
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
319
|
+
|
|
320
|
+
return load_eval_manifest(manifest_path)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _resolve_eval_execution(config) -> ResolvedExecution:
|
|
324
|
+
manifest = _manifest_for_config(config)
|
|
325
|
+
manifest_execution = manifest.execution if manifest is not None else None
|
|
326
|
+
return resolve_execution(
|
|
327
|
+
cli_agent=_getoption(config, "--agent"),
|
|
328
|
+
cli_model=_getoption(config, "--model"),
|
|
329
|
+
cli_base_url=_getoption(config, "--base-url"),
|
|
330
|
+
manifest=manifest_execution,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _resolve_eval_judge(config) -> ResolvedJudge | None:
|
|
335
|
+
manifest = _manifest_for_config(config)
|
|
336
|
+
manifest_judge = manifest.judge if manifest is not None else None
|
|
337
|
+
return resolve_judge(
|
|
338
|
+
main=_resolve_eval_execution(config),
|
|
339
|
+
cli_agent=_getoption(config, "--judge-agent"),
|
|
340
|
+
cli_model=_getoption(config, "--judge-model"),
|
|
341
|
+
cli_base_url=_getoption(config, "--judge-base-url"),
|
|
342
|
+
manifest=manifest_judge,
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _selected_agent_model(agent: str) -> str | None:
|
|
347
|
+
if _plugin is not None:
|
|
348
|
+
return _resolve_eval_execution(_plugin.config).model
|
|
349
|
+
return _legacy_selected_agent_model(agent)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _resolve_eval_agent(config):
|
|
353
|
+
execution = _resolve_eval_execution(config)
|
|
354
|
+
return resolve_agent(execution.agent, None)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _require_agent_credentials(agent: str, source_env: dict[str, str] | None = None) -> None:
|
|
358
|
+
"""Fail setup before workloads when the isolated Agent cannot authenticate."""
|
|
359
|
+
environment = source_env if source_env is not None else os.environ
|
|
360
|
+
missing = missing_credentials(agent, source_env=environment)
|
|
361
|
+
if not missing:
|
|
362
|
+
return
|
|
363
|
+
adapter = get_agent_adapter(agent)
|
|
364
|
+
separator = ", " if adapter.custom else " or "
|
|
365
|
+
raise pytest.UsageError(
|
|
366
|
+
f"Credentials for {agent} not set: {separator.join(missing)}"
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _normal_run_preflight(config) -> None:
|
|
371
|
+
"""Validate normal-run dependencies before collection can generate or run tasks."""
|
|
372
|
+
|
|
373
|
+
if _is_unit_tests_only(config) or getattr(config.option, "collectonly", False):
|
|
374
|
+
return
|
|
375
|
+
from scaffold.python.utils import load_eval_environment
|
|
376
|
+
|
|
377
|
+
load_eval_environment()
|
|
378
|
+
main = _resolve_eval_execution(config)
|
|
379
|
+
judge = _resolve_eval_judge(config)
|
|
380
|
+
suite = next(
|
|
381
|
+
(name for name in ("local", "langsmith", "langfuse") if f"{name}/tests/" in " ".join(config.args)),
|
|
382
|
+
"local",
|
|
383
|
+
)
|
|
384
|
+
required_harness = (
|
|
385
|
+
EVAL_ROOT / "pyproject.toml",
|
|
386
|
+
EVAL_ROOT / "schemas" / "comet.eval" / "v1alpha1.schema.json",
|
|
387
|
+
EVAL_ROOT / suite / "tests" / "tasks" / "test_tasks.py",
|
|
388
|
+
)
|
|
389
|
+
missing_harness = [str(path) for path in required_harness if not path.is_file()]
|
|
390
|
+
if missing_harness:
|
|
391
|
+
raise pytest.UsageError(f"Packaged eval harness is incomplete: {', '.join(missing_harness)}")
|
|
392
|
+
manifest = _manifest_for_config(config)
|
|
393
|
+
if manifest is not None and manifest.reporting is not None:
|
|
394
|
+
selected_report_suite = manifest.reporting.suite
|
|
395
|
+
if selected_report_suite and selected_report_suite != suite:
|
|
396
|
+
raise pytest.UsageError(
|
|
397
|
+
f"reporting.suite is {selected_report_suite!r}, but the selected suite is {suite!r}"
|
|
398
|
+
)
|
|
399
|
+
report_config = manifest.reporting.config
|
|
400
|
+
if report_config and not _getoption(config, "--report-config"):
|
|
401
|
+
report_path = Path(report_config)
|
|
402
|
+
if not report_path.is_absolute():
|
|
403
|
+
report_path = manifest.path.parent / report_path
|
|
404
|
+
config.option.report_config = str(report_path.resolve())
|
|
405
|
+
try:
|
|
406
|
+
load_report_output_config(_getoption(config, "--report-config"))
|
|
407
|
+
except (OSError, ValueError) as exc:
|
|
408
|
+
raise pytest.UsageError(f"Eval report configuration is invalid: {exc}") from exc
|
|
409
|
+
try:
|
|
410
|
+
validate_agent_capabilities(get_agent_adapter(main.agent))
|
|
411
|
+
if judge is not None:
|
|
412
|
+
validate_agent_capabilities(get_agent_adapter(judge.agent))
|
|
413
|
+
except ValueError as exc:
|
|
414
|
+
raise pytest.UsageError(str(exc)) from exc
|
|
415
|
+
if main.base_url:
|
|
416
|
+
validate_base_url(main.base_url, field="execution.baseUrl")
|
|
417
|
+
if judge is not None and judge.base_url:
|
|
418
|
+
validate_base_url(judge.base_url, field="judge.baseUrl")
|
|
419
|
+
credential_errors = preflight_credentials(main, judge)
|
|
420
|
+
if credential_errors:
|
|
421
|
+
raise pytest.UsageError("; ".join(credential_errors))
|
|
422
|
+
if shutil.which("uv") is None:
|
|
423
|
+
raise pytest.UsageError("uv is not installed or not in PATH")
|
|
424
|
+
if not check_docker_available():
|
|
425
|
+
raise pytest.UsageError("Docker CLI or daemon is unavailable")
|
|
426
|
+
context = getattr(config, "_comet_eval_context", None)
|
|
427
|
+
if context is None:
|
|
428
|
+
raise pytest.UsageError("Eval context is missing before normal-run preflight")
|
|
429
|
+
artifact_root = Path(context.artifact_root)
|
|
430
|
+
artifact_root.mkdir(parents=True, exist_ok=True)
|
|
431
|
+
try:
|
|
432
|
+
with tempfile.NamedTemporaryFile(dir=artifact_root, prefix=".preflight-", delete=True):
|
|
433
|
+
pass
|
|
434
|
+
except OSError as exc:
|
|
435
|
+
raise pytest.UsageError(f"Eval artifact root is not writable: {artifact_root}") from exc
|
|
436
|
+
config._comet_execution_env = build_agent_environment(main)
|
|
437
|
+
config._comet_execution = main
|
|
438
|
+
config._comet_judge = judge
|
|
439
|
+
os.environ["COMET_EVAL_MAIN_CREDENTIALS"] = ",".join(
|
|
440
|
+
get_agent_adapter(main.agent).required_credentials
|
|
441
|
+
)
|
|
442
|
+
if judge is not None:
|
|
443
|
+
os.environ["BENCH_LLM_JUDGE"] = "1"
|
|
444
|
+
os.environ["BENCH_JUDGE_MODEL"] = judge.model
|
|
445
|
+
os.environ["BENCH_JUDGE_AGENT"] = judge.agent
|
|
446
|
+
if judge.base_url:
|
|
447
|
+
os.environ["BENCH_JUDGE_BASE_URL"] = judge.base_url
|
|
448
|
+
else:
|
|
449
|
+
os.environ.pop("BENCH_JUDGE_BASE_URL", None)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _agent_project_root(agent: str) -> str:
|
|
453
|
+
return {
|
|
454
|
+
"claude-code": ".claude",
|
|
455
|
+
"codex": ".agents",
|
|
456
|
+
"qoder": ".qoder",
|
|
457
|
+
"codebuddy": ".codebuddy",
|
|
458
|
+
}.get(agent, ".comet-agent")
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _ensure_langfuse_trajectory_support(test_dir: Path, agent: str) -> None:
|
|
462
|
+
"""Install only isolated, project-local trajectory support for Langfuse runs."""
|
|
463
|
+
if os.environ.get("TRACE_TO_LANGFUSE", "").lower() != "true":
|
|
464
|
+
return
|
|
465
|
+
if agent in {"qoder", "codebuddy"}:
|
|
466
|
+
try:
|
|
467
|
+
install_transcript_hook(test_dir, agent)
|
|
468
|
+
except Exception as exc:
|
|
469
|
+
if os.environ.get("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED") != "true":
|
|
470
|
+
print(
|
|
471
|
+
f"[langfuse] optional {agent} transcript setup skipped: {exc}",
|
|
472
|
+
file=os.sys.stderr,
|
|
473
|
+
)
|
|
474
|
+
os.environ["LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED"] = "true"
|
|
475
|
+
return
|
|
476
|
+
if agent != "codex":
|
|
477
|
+
return
|
|
478
|
+
cache_root = os.environ.get("LANGFUSE_TRAJECTORY_CACHE_DIR")
|
|
479
|
+
if not cache_root:
|
|
480
|
+
return
|
|
481
|
+
try:
|
|
482
|
+
provision = provision_trajectory_plugin(Path(cache_root), agent)
|
|
483
|
+
install_codex_plugin_workspace(test_dir, provision)
|
|
484
|
+
except Exception as exc:
|
|
485
|
+
if os.environ.get("LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED") != "true":
|
|
486
|
+
print(f"[langfuse] optional Codex trajectory setup skipped: {exc}", file=os.sys.stderr)
|
|
487
|
+
os.environ["LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED"] = "true"
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _ensure_auto_generated_manifest(config, task_filter: str | None = None) -> None:
|
|
491
|
+
"""Freeze generated descriptors once while retaining the original manifest authority."""
|
|
492
|
+
if getattr(config, "_comet_frozen_task_set", None) is not None:
|
|
493
|
+
return
|
|
494
|
+
if config.getoption("--task") or task_filter or config.getoption("--quick"):
|
|
495
|
+
return
|
|
496
|
+
|
|
497
|
+
manifest_path = config.getoption("--eval-manifest")
|
|
498
|
+
skill_path = config.getoption("--skill-path")
|
|
499
|
+
manifest = None
|
|
500
|
+
if manifest_path:
|
|
501
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
502
|
+
|
|
503
|
+
manifest = load_eval_manifest(manifest_path)
|
|
504
|
+
if manifest.tasks or manifest.recommended_tasks:
|
|
505
|
+
return
|
|
506
|
+
skill_path = str(manifest.skill_path)
|
|
507
|
+
if not skill_path:
|
|
508
|
+
return
|
|
509
|
+
|
|
510
|
+
if getattr(getattr(config, "option", None), "collectonly", False):
|
|
511
|
+
from scaffold.python.manifests import SkillEvalManifest
|
|
512
|
+
from scaffold.python.task_resolution import ResolvedTaskSet
|
|
513
|
+
|
|
514
|
+
if manifest is None:
|
|
515
|
+
resolved_skill_root = Path(skill_path).expanduser().resolve()
|
|
516
|
+
if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
|
|
517
|
+
resolved_skill_root = resolved_skill_root.parent
|
|
518
|
+
manifest = SkillEvalManifest(
|
|
519
|
+
path=resolved_skill_root / "comet" / "eval.yaml",
|
|
520
|
+
name=resolved_skill_root.name,
|
|
521
|
+
description="",
|
|
522
|
+
skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
|
|
523
|
+
skill_path=resolved_skill_root,
|
|
524
|
+
profile=config.getoption("--profile"),
|
|
525
|
+
)
|
|
526
|
+
config._comet_resolution_manifest = manifest
|
|
527
|
+
config._comet_resolved_tasks = {}
|
|
528
|
+
config._comet_frozen_task_set = ResolvedTaskSet("pending-generation", (), None)
|
|
529
|
+
return
|
|
530
|
+
|
|
531
|
+
# Task generation happens during collection, before session fixtures run.
|
|
532
|
+
# Load the documented local environment now so credentials and the model
|
|
533
|
+
# participating in the cache key match the later execution environment.
|
|
534
|
+
from scaffold.python.utils import load_eval_environment
|
|
535
|
+
|
|
536
|
+
load_eval_environment()
|
|
537
|
+
execution = _resolve_eval_execution(config)
|
|
538
|
+
profile = config.getoption("--profile") or (manifest.profile if manifest else None) or "generic"
|
|
539
|
+
interaction = vars(manifest.interaction) if manifest else {"mode": "none", "max_turns": 12}
|
|
540
|
+
context = getattr(config, "_comet_eval_context", None)
|
|
541
|
+
project_root = (
|
|
542
|
+
context.artifact_owner_root
|
|
543
|
+
if context is not None
|
|
544
|
+
else config.getoption("--project-root") or find_project_root(skill_path)
|
|
545
|
+
)
|
|
546
|
+
try:
|
|
547
|
+
generated = ensure_generated_manifest(
|
|
548
|
+
skill_path,
|
|
549
|
+
project_root,
|
|
550
|
+
agent=execution.agent,
|
|
551
|
+
model=execution.model,
|
|
552
|
+
base_url=execution.base_url,
|
|
553
|
+
environment=build_agent_environment(execution),
|
|
554
|
+
profile=profile,
|
|
555
|
+
interaction=interaction,
|
|
556
|
+
collect_only=bool(getattr(getattr(config, "option", None), "collectonly", False)),
|
|
557
|
+
)
|
|
558
|
+
except AutoTaskError as exc:
|
|
559
|
+
raise pytest.UsageError(str(exc)) from exc
|
|
560
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
561
|
+
from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
|
|
562
|
+
from scaffold.python.task_resolution import ResolvedTask, ResolvedTaskSet
|
|
563
|
+
|
|
564
|
+
generated_manifest = load_eval_manifest(generated.manifest_path)
|
|
565
|
+
generated_tasks = tuple(
|
|
566
|
+
ResolvedTask(task.name, "generated", task)
|
|
567
|
+
for task in load_manifest_tasks(generated_manifest)
|
|
568
|
+
)
|
|
569
|
+
if manifest is None:
|
|
570
|
+
resolved_skill_root = Path(skill_path).expanduser().resolve()
|
|
571
|
+
if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
|
|
572
|
+
resolved_skill_root = resolved_skill_root.parent
|
|
573
|
+
manifest = SkillEvalManifest(
|
|
574
|
+
path=resolved_skill_root / "comet" / "eval.yaml",
|
|
575
|
+
name=resolved_skill_root.name,
|
|
576
|
+
description="",
|
|
577
|
+
skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
|
|
578
|
+
skill_path=resolved_skill_root,
|
|
579
|
+
profile=config.getoption("--profile"),
|
|
580
|
+
)
|
|
581
|
+
frozen = ResolvedTaskSet(
|
|
582
|
+
"generated-cache",
|
|
583
|
+
generated_tasks,
|
|
584
|
+
None,
|
|
585
|
+
)
|
|
586
|
+
config._comet_resolution_manifest = manifest
|
|
587
|
+
config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
|
|
588
|
+
config._comet_frozen_task_set = frozen
|
|
589
|
+
config._comet_generated_manifest = generated
|
|
255
590
|
|
|
256
591
|
|
|
257
592
|
def _model_execution_config() -> dict[str, str | None]:
|
|
@@ -265,11 +600,13 @@ def _capture_execution_identity(test_dir: Path, *, model: str | None, interactio
|
|
|
265
600
|
"docker.sh",
|
|
266
601
|
"execution-identity",
|
|
267
602
|
str(test_dir),
|
|
603
|
+
"--agent",
|
|
604
|
+
_resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code",
|
|
268
605
|
timeout=300,
|
|
269
606
|
check=False,
|
|
270
607
|
)
|
|
271
608
|
if result.returncode != 0:
|
|
272
|
-
raise RuntimeError("Could not verify eval Docker/
|
|
609
|
+
raise RuntimeError("Could not verify eval Docker/agent execution identity")
|
|
273
610
|
try:
|
|
274
611
|
raw = json.loads(result.stdout)
|
|
275
612
|
except (TypeError, json.JSONDecodeError) as error:
|
|
@@ -532,6 +869,54 @@ def pytest_addoption(parser):
|
|
|
532
869
|
default=None,
|
|
533
870
|
help="Path to comet/eval.yaml",
|
|
534
871
|
)
|
|
872
|
+
parser.addoption(
|
|
873
|
+
"--agent",
|
|
874
|
+
action="store",
|
|
875
|
+
default=None,
|
|
876
|
+
help="Evaluation agent CLI (built-in or explicitly installed custom adapter)",
|
|
877
|
+
)
|
|
878
|
+
parser.addoption(
|
|
879
|
+
"--model",
|
|
880
|
+
action="store",
|
|
881
|
+
default=None,
|
|
882
|
+
help="Main evaluation model override",
|
|
883
|
+
)
|
|
884
|
+
parser.addoption(
|
|
885
|
+
"--base-url",
|
|
886
|
+
action="store",
|
|
887
|
+
default=None,
|
|
888
|
+
help="Main evaluation API base URL override",
|
|
889
|
+
)
|
|
890
|
+
parser.addoption(
|
|
891
|
+
"--judge-agent",
|
|
892
|
+
action="store",
|
|
893
|
+
default=None,
|
|
894
|
+
help="Independent Judge agent (built-in or explicitly installed custom adapter)",
|
|
895
|
+
)
|
|
896
|
+
parser.addoption(
|
|
897
|
+
"--judge-model",
|
|
898
|
+
action="store",
|
|
899
|
+
default=None,
|
|
900
|
+
help="Independent Judge model",
|
|
901
|
+
)
|
|
902
|
+
parser.addoption(
|
|
903
|
+
"--judge-base-url",
|
|
904
|
+
action="store",
|
|
905
|
+
default=None,
|
|
906
|
+
help="Independent Judge API base URL",
|
|
907
|
+
)
|
|
908
|
+
parser.addoption(
|
|
909
|
+
"--quick",
|
|
910
|
+
action="store_true",
|
|
911
|
+
default=False,
|
|
912
|
+
help="Use generic-skill-smoke and bypass automatic task generation",
|
|
913
|
+
)
|
|
914
|
+
parser.addoption(
|
|
915
|
+
"--project-root",
|
|
916
|
+
action="store",
|
|
917
|
+
default=None,
|
|
918
|
+
help="Project root used for generated eval cache",
|
|
919
|
+
)
|
|
535
920
|
parser.addoption(
|
|
536
921
|
"--interaction-mode",
|
|
537
922
|
action="store",
|
|
@@ -564,6 +949,27 @@ def pytest_configure(config):
|
|
|
564
949
|
"markers",
|
|
565
950
|
"eval_case(repetition): controller-owned task/treatment/repetition identity",
|
|
566
951
|
)
|
|
952
|
+
try:
|
|
953
|
+
context = context_from_environment()
|
|
954
|
+
if context is None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
|
|
955
|
+
context = resolve_eval_context(
|
|
956
|
+
skill_path=config.getoption("--skill-path"),
|
|
957
|
+
manifest_path=config.getoption("--eval-manifest"),
|
|
958
|
+
project_root=config.getoption("--project-root"),
|
|
959
|
+
)
|
|
960
|
+
except EvalContextError as exc:
|
|
961
|
+
raise pytest.UsageError(str(exc)) from exc
|
|
962
|
+
if context is not None:
|
|
963
|
+
config._comet_eval_context = context
|
|
964
|
+
config.option.project_root = str(context.artifact_owner_root)
|
|
965
|
+
if context.manifest_path is not None:
|
|
966
|
+
config.option.eval_manifest = str(context.manifest_path)
|
|
967
|
+
else:
|
|
968
|
+
config.option.skill_path = str(context.skill_root)
|
|
969
|
+
config._comet_execution = _resolve_eval_execution(config)
|
|
970
|
+
config._comet_judge = _resolve_eval_judge(config)
|
|
971
|
+
_normal_run_preflight(config)
|
|
972
|
+
_resolve_eval_agent(config)
|
|
567
973
|
global _plugin
|
|
568
974
|
_plugin = ExperimentPlugin(config)
|
|
569
975
|
config.pluginmanager.register(_plugin, "experiment_plugin")
|
|
@@ -772,12 +1178,40 @@ def _get_experiment_name(session) -> str:
|
|
|
772
1178
|
|
|
773
1179
|
|
|
774
1180
|
def _get_dynamic_treatment_config(config):
|
|
1181
|
+
_ensure_auto_generated_manifest(config)
|
|
1182
|
+
|
|
1183
|
+
def generation_hints(manifest):
|
|
1184
|
+
generation_metadata = {}
|
|
1185
|
+
generation_metadata_hash = None
|
|
1186
|
+
generated = getattr(config, "_comet_generated_manifest", None)
|
|
1187
|
+
generation_metadata_path = (
|
|
1188
|
+
generated.metadata_path if generated is not None else manifest.generation_metadata_path
|
|
1189
|
+
)
|
|
1190
|
+
if generation_metadata_path and generation_metadata_path.is_file():
|
|
1191
|
+
generation_metadata_bytes = generation_metadata_path.read_bytes()
|
|
1192
|
+
generation_metadata = json.loads(generation_metadata_bytes)
|
|
1193
|
+
generation_metadata_hash = (
|
|
1194
|
+
"sha256:" + hashlib.sha256(generation_metadata_bytes).hexdigest()
|
|
1195
|
+
)
|
|
1196
|
+
return {
|
|
1197
|
+
"generation_hash": (
|
|
1198
|
+
generated.generation_hash if generated is not None else manifest.generation_hash
|
|
1199
|
+
),
|
|
1200
|
+
"generation_metadata_path": (
|
|
1201
|
+
str(generation_metadata_path) if generation_metadata_path else None
|
|
1202
|
+
),
|
|
1203
|
+
"generation_manifest_hash": generation_metadata.get("manifest_hash"),
|
|
1204
|
+
"generation_metadata_hash": generation_metadata_hash,
|
|
1205
|
+
"generation_overhead": generation_metadata.get("generation_overhead"),
|
|
1206
|
+
}
|
|
1207
|
+
|
|
775
1208
|
manifest_path = config.getoption("--eval-manifest")
|
|
776
1209
|
if manifest_path:
|
|
777
1210
|
from scaffold.python.manifests import load_eval_manifest
|
|
778
1211
|
from scaffold.python.treatments import TreatmentConfig
|
|
779
1212
|
|
|
780
1213
|
manifest = load_eval_manifest(manifest_path)
|
|
1214
|
+
generation = generation_hints(manifest)
|
|
781
1215
|
node_skills = []
|
|
782
1216
|
for node_skill in manifest.generated_node_skills:
|
|
783
1217
|
node_path = manifest.skill_path.parent / node_skill
|
|
@@ -799,11 +1233,13 @@ def _get_dynamic_treatment_config(config):
|
|
|
799
1233
|
"path": str(manifest.skill_path),
|
|
800
1234
|
"profile": manifest.profile,
|
|
801
1235
|
"manifest": str(manifest.path),
|
|
1236
|
+
"execution_agent": manifest.execution_agent,
|
|
802
1237
|
"baseline_treatments": manifest.baseline_treatments,
|
|
803
1238
|
"quality_gates": manifest.quality_gates,
|
|
804
1239
|
"required_output_schemas": manifest.required_output_schemas,
|
|
805
1240
|
"expected_evidence": manifest.expected_evidence,
|
|
806
1241
|
"draft_hash": manifest.draft_hash,
|
|
1242
|
+
**generation,
|
|
807
1243
|
"required_skills": manifest.required_skills,
|
|
808
1244
|
"expected_artifacts": manifest.expected_artifacts,
|
|
809
1245
|
"generated_node_skills": manifest.generated_node_skills,
|
|
@@ -820,7 +1256,8 @@ def _get_dynamic_treatment_config(config):
|
|
|
820
1256
|
if not skill_path:
|
|
821
1257
|
return None
|
|
822
1258
|
skill_name = config.getoption("--skill-name") or Path(skill_path).resolve().parent.name
|
|
823
|
-
|
|
1259
|
+
manifest = getattr(config, "_comet_resolution_manifest", None)
|
|
1260
|
+
profile = config.getoption("--profile") or (manifest.profile if manifest else None)
|
|
824
1261
|
skill_cfg = {
|
|
825
1262
|
"name": skill_name,
|
|
826
1263
|
"source": "path",
|
|
@@ -828,6 +1265,9 @@ def _get_dynamic_treatment_config(config):
|
|
|
828
1265
|
}
|
|
829
1266
|
if profile:
|
|
830
1267
|
skill_cfg["profile"] = profile
|
|
1268
|
+
if manifest:
|
|
1269
|
+
skill_cfg["manifest"] = str(manifest.path)
|
|
1270
|
+
skill_cfg.update(generation_hints(manifest))
|
|
831
1271
|
from scaffold.python.treatments import TreatmentConfig
|
|
832
1272
|
|
|
833
1273
|
return TreatmentConfig(
|
|
@@ -899,14 +1339,15 @@ def _build_eval_claude_md(profile_name: str, treatment_claude_md: str | None = N
|
|
|
899
1339
|
return "\n\n".join(section for section in sections if section.strip()) or None
|
|
900
1340
|
|
|
901
1341
|
|
|
902
|
-
def _comet_hook_command(test_dir: Path) -> str | None:
|
|
903
|
-
|
|
1342
|
+
def _comet_hook_command(test_dir: Path, agent: str = "claude-code") -> str | None:
|
|
1343
|
+
project_root = _agent_project_root(agent)
|
|
1344
|
+
scripts_dir = test_dir / project_root / "skills" / "comet" / "scripts"
|
|
904
1345
|
mjs_hook = scripts_dir / "comet-hook-guard.mjs"
|
|
905
1346
|
shell_hook = scripts_dir / "comet-hook-guard.sh"
|
|
906
1347
|
if mjs_hook.exists():
|
|
907
|
-
return "node /workspace
|
|
1348
|
+
return f"node /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.mjs"
|
|
908
1349
|
if shell_hook.exists():
|
|
909
|
-
return "bash /workspace
|
|
1350
|
+
return f"bash /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.sh"
|
|
910
1351
|
return None
|
|
911
1352
|
|
|
912
1353
|
|
|
@@ -930,6 +1371,34 @@ def _ensure_claude_pre_tool_hook(test_dir: Path, command: str | None) -> None:
|
|
|
930
1371
|
settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
|
|
931
1372
|
|
|
932
1373
|
|
|
1374
|
+
def _ensure_agent_pre_tool_hook(test_dir: Path, agent: str, command: str | None) -> None:
|
|
1375
|
+
if agent == "claude-code":
|
|
1376
|
+
_ensure_claude_pre_tool_hook(test_dir, command)
|
|
1377
|
+
return
|
|
1378
|
+
if not command:
|
|
1379
|
+
return
|
|
1380
|
+
|
|
1381
|
+
project_root = ".codex" if agent == "codex" else _agent_project_root(agent)
|
|
1382
|
+
settings_name = "hooks.json" if agent == "codex" else "settings.json"
|
|
1383
|
+
settings_path = test_dir / project_root / settings_name
|
|
1384
|
+
if settings_path.exists():
|
|
1385
|
+
settings = json.loads(settings_path.read_text(encoding="utf-8"))
|
|
1386
|
+
else:
|
|
1387
|
+
settings = {}
|
|
1388
|
+
hooks = settings.setdefault("hooks", {})
|
|
1389
|
+
pre_tool_use = hooks.setdefault("PreToolUse", [])
|
|
1390
|
+
hook_entry = {
|
|
1391
|
+
"matcher": "Write|Edit|MultiEdit",
|
|
1392
|
+
"hooks": [{"type": "command", "command": command}],
|
|
1393
|
+
}
|
|
1394
|
+
if agent in {"qoder", "codebuddy"}:
|
|
1395
|
+
hook_entry["hooks"][0]["description"] = "Comet workflow guard"
|
|
1396
|
+
if not any(entry == hook_entry for entry in pre_tool_use):
|
|
1397
|
+
pre_tool_use.append(hook_entry)
|
|
1398
|
+
settings_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1399
|
+
settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
|
|
1400
|
+
|
|
1401
|
+
|
|
933
1402
|
def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
|
|
934
1403
|
"""Get shared experiment ID or create new one."""
|
|
935
1404
|
requested = os.environ.get(EXPERIMENT_ID_ENV)
|
|
@@ -944,16 +1413,22 @@ def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
|
|
|
944
1413
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
945
1414
|
return f"{name}_{timestamp}"
|
|
946
1415
|
|
|
947
|
-
|
|
1416
|
+
context = context_from_environment()
|
|
1417
|
+
experiment_file = (
|
|
1418
|
+
resolve_managed_path(context, "locks", ".pytest_experiment_id")
|
|
1419
|
+
if context is not None
|
|
1420
|
+
else XDIST_EXPERIMENT_FILE
|
|
1421
|
+
)
|
|
1422
|
+
lock_file = experiment_file.with_suffix(".lock")
|
|
948
1423
|
|
|
949
1424
|
with file_lock(lock_file):
|
|
950
|
-
if
|
|
951
|
-
data = json.loads(
|
|
1425
|
+
if experiment_file.exists():
|
|
1426
|
+
data = json.loads(experiment_file.read_text())
|
|
952
1427
|
return data["experiment_id"]
|
|
953
1428
|
|
|
954
1429
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
955
1430
|
experiment_id = f"{name}_{timestamp}"
|
|
956
|
-
|
|
1431
|
+
experiment_file.write_text(
|
|
957
1432
|
json.dumps(
|
|
958
1433
|
{
|
|
959
1434
|
"experiment_id": experiment_id,
|
|
@@ -1024,13 +1499,23 @@ def _snapshot_dynamic_skill_package(test_dir: Path, skill_hints: dict[str, Any])
|
|
|
1024
1499
|
snapshot_root = test_dir / "_eval_target_skills"
|
|
1025
1500
|
snapshot_root.mkdir(parents=True, exist_ok=True)
|
|
1026
1501
|
package_dest = snapshot_root / package_dir.name
|
|
1027
|
-
shutil.copytree(
|
|
1502
|
+
shutil.copytree(
|
|
1503
|
+
package_dir,
|
|
1504
|
+
package_dest,
|
|
1505
|
+
dirs_exist_ok=True,
|
|
1506
|
+
ignore=shutil.ignore_patterns(".comet"),
|
|
1507
|
+
)
|
|
1028
1508
|
|
|
1029
1509
|
for node_skill in skill_hints.get("generated_node_skills") or []:
|
|
1030
1510
|
node_source = package_dir.parent / node_skill
|
|
1031
1511
|
if not (node_source / "SKILL.md").exists():
|
|
1032
1512
|
continue
|
|
1033
|
-
shutil.copytree(
|
|
1513
|
+
shutil.copytree(
|
|
1514
|
+
node_source,
|
|
1515
|
+
snapshot_root / node_skill,
|
|
1516
|
+
dirs_exist_ok=True,
|
|
1517
|
+
ignore=shutil.ignore_patterns(".comet"),
|
|
1518
|
+
)
|
|
1034
1519
|
|
|
1035
1520
|
return str(package_dest.relative_to(test_dir)).replace("\\", "/")
|
|
1036
1521
|
|
|
@@ -1039,7 +1524,13 @@ def _cleanup_experiment_coordination():
|
|
|
1039
1524
|
"""Remove coordination files after experiment."""
|
|
1040
1525
|
import sys
|
|
1041
1526
|
|
|
1042
|
-
|
|
1527
|
+
context = context_from_environment()
|
|
1528
|
+
experiment_file = (
|
|
1529
|
+
resolve_managed_path(context, "locks", ".pytest_experiment_id")
|
|
1530
|
+
if context is not None
|
|
1531
|
+
else XDIST_EXPERIMENT_FILE
|
|
1532
|
+
)
|
|
1533
|
+
for f in [experiment_file, experiment_file.with_suffix(".lock")]:
|
|
1043
1534
|
try:
|
|
1044
1535
|
f.unlink(missing_ok=True)
|
|
1045
1536
|
except Exception as e:
|
|
@@ -1067,8 +1558,8 @@ def worker_id(request):
|
|
|
1067
1558
|
|
|
1068
1559
|
@pytest.fixture(scope="session", autouse=True)
|
|
1069
1560
|
def verify_environment(project_root, request):
|
|
1070
|
-
"""Verify
|
|
1071
|
-
if _is_unit_tests_only(request.config):
|
|
1561
|
+
"""Verify the selected agent's Docker execution prerequisites."""
|
|
1562
|
+
if _is_unit_tests_only(request.config) or request.config.option.collectonly:
|
|
1072
1563
|
return
|
|
1073
1564
|
|
|
1074
1565
|
from scaffold.python.utils import load_eval_environment
|
|
@@ -1101,55 +1592,55 @@ def verify_environment(project_root, request):
|
|
|
1101
1592
|
if result.returncode != 0:
|
|
1102
1593
|
pytest.skip("Docker not available")
|
|
1103
1594
|
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
if shutil.which("claude") is None:
|
|
1108
|
-
pytest.skip("Claude CLI not available")
|
|
1595
|
+
selection = _resolve_eval_agent(request.config)
|
|
1596
|
+
_require_agent_credentials(selection.agent)
|
|
1109
1597
|
|
|
1110
1598
|
|
|
1111
1599
|
def _docker_environment_dirs_for_request(request, tasks_dir: Path) -> list[Path]:
|
|
1112
1600
|
"""Return only task environments selected by the current eval invocation."""
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
task_names = all_task_names
|
|
1127
|
-
|
|
1128
|
-
return [
|
|
1129
|
-
environment_dir
|
|
1130
|
-
for task_name in task_names
|
|
1131
|
-
if (environment_dir := tasks_dir / task_name / "environment").is_dir()
|
|
1132
|
-
and (environment_dir / "Dockerfile").is_file()
|
|
1133
|
-
]
|
|
1601
|
+
frozen = getattr(request.config, "_comet_frozen_task_set", None)
|
|
1602
|
+
if frozen is None:
|
|
1603
|
+
raise pytest.UsageError("Eval task collection did not produce a frozen task set")
|
|
1604
|
+
selected: list[Path] = []
|
|
1605
|
+
for item in frozen.tasks:
|
|
1606
|
+
environment_dir = item.task.environment_dir
|
|
1607
|
+
if (
|
|
1608
|
+
environment_dir.is_dir()
|
|
1609
|
+
and (environment_dir / "Dockerfile").is_file()
|
|
1610
|
+
and environment_dir not in selected
|
|
1611
|
+
):
|
|
1612
|
+
selected.append(environment_dir)
|
|
1613
|
+
return selected
|
|
1134
1614
|
|
|
1135
1615
|
|
|
1136
1616
|
@pytest.fixture(scope="session", autouse=True)
|
|
1137
1617
|
def prebuild_docker_image(request):
|
|
1138
1618
|
"""Pre-build Docker image once per session to avoid race conditions."""
|
|
1139
|
-
if _is_unit_tests_only(request.config):
|
|
1619
|
+
if _is_unit_tests_only(request.config) or request.config.option.collectonly:
|
|
1140
1620
|
yield
|
|
1141
1621
|
return
|
|
1142
1622
|
|
|
1143
1623
|
tasks_dir = PROJECT_ROOT / "tasks"
|
|
1144
1624
|
for env_dir in _docker_environment_dirs_for_request(request, tasks_dir):
|
|
1145
|
-
|
|
1625
|
+
selected_agent = _resolve_eval_agent(request.config).agent
|
|
1626
|
+
image = _build_docker_image_with_lock(
|
|
1627
|
+
env_dir,
|
|
1628
|
+
selected_agent if selected_agent != "claude-code" else None,
|
|
1629
|
+
environment=getattr(request.config, "_comet_execution_env", None),
|
|
1630
|
+
)
|
|
1146
1631
|
if image:
|
|
1147
1632
|
print(f"\nPre-built Docker image: {image}")
|
|
1148
1633
|
|
|
1149
1634
|
yield
|
|
1150
1635
|
|
|
1151
1636
|
try:
|
|
1152
|
-
|
|
1637
|
+
context = context_from_environment()
|
|
1638
|
+
lock_path = (
|
|
1639
|
+
resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
|
|
1640
|
+
if context is not None
|
|
1641
|
+
else DOCKER_BUILD_LOCK
|
|
1642
|
+
)
|
|
1643
|
+
lock_path.unlink(missing_ok=True)
|
|
1153
1644
|
except Exception:
|
|
1154
1645
|
pass
|
|
1155
1646
|
|
|
@@ -1181,11 +1672,17 @@ def setup_test_context(test_dir):
|
|
|
1181
1672
|
scripts_dir: Path | None = None,
|
|
1182
1673
|
source_dir: Path | None = None,
|
|
1183
1674
|
) -> None:
|
|
1184
|
-
|
|
1675
|
+
agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
|
|
1676
|
+
skill_dir = test_dir / _agent_project_root(agent) / "skills" / skill_name
|
|
1185
1677
|
skill_dir.mkdir(parents=True, exist_ok=True)
|
|
1186
1678
|
|
|
1187
1679
|
if source_dir and source_dir.is_dir():
|
|
1188
|
-
shutil.copytree(
|
|
1680
|
+
shutil.copytree(
|
|
1681
|
+
source_dir,
|
|
1682
|
+
skill_dir,
|
|
1683
|
+
dirs_exist_ok=True,
|
|
1684
|
+
ignore=shutil.ignore_patterns(".comet"),
|
|
1685
|
+
)
|
|
1189
1686
|
|
|
1190
1687
|
shutil.copyfile(skill_file, skill_dir / "SKILL.md")
|
|
1191
1688
|
|
|
@@ -1212,13 +1709,38 @@ def setup_test_context(test_dir):
|
|
|
1212
1709
|
shutil.copy2(item, dest)
|
|
1213
1710
|
_copy_current_comet_cli_snapshot(environment_dir, test_dir)
|
|
1214
1711
|
|
|
1712
|
+
def _copy_workspace(workspace_dir: Path) -> None:
|
|
1713
|
+
"""Copy a bounded project fixture without replacing the task Dockerfile."""
|
|
1714
|
+
for item in workspace_dir.iterdir():
|
|
1715
|
+
if item.name in {"Dockerfile", CURRENT_COMET_CLI_MARKER, TRUSTED_NATIVE_RUNTIME_MARKER}:
|
|
1716
|
+
continue
|
|
1717
|
+
dest = test_dir / item.name
|
|
1718
|
+
if item.is_dir():
|
|
1719
|
+
shutil.copytree(item, dest, dirs_exist_ok=True)
|
|
1720
|
+
else:
|
|
1721
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1722
|
+
shutil.copy2(item, dest)
|
|
1723
|
+
|
|
1215
1724
|
def _write_claude_md(content_file: str) -> None:
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1725
|
+
agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
|
|
1726
|
+
if agent == "claude-code":
|
|
1727
|
+
claude_dir = test_dir / ".claude"
|
|
1728
|
+
claude_dir.mkdir(parents=True, exist_ok=True)
|
|
1729
|
+
shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
|
|
1730
|
+
shutil.copyfile(content_file, test_dir / "CLAUDE.md")
|
|
1731
|
+
elif agent == "codebuddy":
|
|
1732
|
+
codebuddy_dir = test_dir / ".codebuddy"
|
|
1733
|
+
codebuddy_dir.mkdir(parents=True, exist_ok=True)
|
|
1734
|
+
shutil.copyfile(content_file, codebuddy_dir / "CODEBUDDY.md")
|
|
1735
|
+
else:
|
|
1736
|
+
shutil.copyfile(content_file, test_dir / "AGENTS.md")
|
|
1220
1737
|
|
|
1221
|
-
def _setup(
|
|
1738
|
+
def _setup(
|
|
1739
|
+
skills: dict = None,
|
|
1740
|
+
claude_md: str = None,
|
|
1741
|
+
environment_dir: Path = None,
|
|
1742
|
+
workspace_dir: Path = None,
|
|
1743
|
+
):
|
|
1222
1744
|
for skill_name, cfg in (skills or {}).items():
|
|
1223
1745
|
if not cfg:
|
|
1224
1746
|
continue
|
|
@@ -1251,6 +1773,9 @@ def setup_test_context(test_dir):
|
|
|
1251
1773
|
if is_temp_dir and filtered_dir.exists():
|
|
1252
1774
|
shutil.rmtree(filtered_dir)
|
|
1253
1775
|
|
|
1776
|
+
if workspace_dir and workspace_dir.exists():
|
|
1777
|
+
_copy_workspace(workspace_dir)
|
|
1778
|
+
|
|
1254
1779
|
if environment_dir and environment_dir.exists():
|
|
1255
1780
|
_copy_environment(environment_dir)
|
|
1256
1781
|
_copy_trusted_native_runtime_snapshot(environment_dir, test_dir, skills)
|
|
@@ -1266,7 +1791,9 @@ def setup_test_context(test_dir):
|
|
|
1266
1791
|
finally:
|
|
1267
1792
|
os.unlink(temp_file)
|
|
1268
1793
|
|
|
1269
|
-
|
|
1794
|
+
agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
|
|
1795
|
+
_ensure_langfuse_trajectory_support(test_dir, agent)
|
|
1796
|
+
_ensure_agent_pre_tool_hook(test_dir, agent, _comet_hook_command(test_dir, agent))
|
|
1270
1797
|
|
|
1271
1798
|
return test_dir
|
|
1272
1799
|
|
|
@@ -1275,13 +1802,15 @@ def setup_test_context(test_dir):
|
|
|
1275
1802
|
|
|
1276
1803
|
@pytest.fixture
|
|
1277
1804
|
def run_claude(test_dir, experiment_logger, request):
|
|
1278
|
-
"""Factory fixture to run
|
|
1805
|
+
"""Factory fixture to run the selected evaluation agent in Docker.
|
|
1279
1806
|
|
|
1280
1807
|
For tasks using ``interaction.mode=auto_user`` the single-shot ``run-claude``
|
|
1281
1808
|
is replaced by the multi-turn ``run-claude-loop`` driver, which simulates a
|
|
1282
1809
|
user replying at the workflow's decision points.
|
|
1283
1810
|
"""
|
|
1284
|
-
|
|
1811
|
+
execution = _resolve_eval_execution(request.config)
|
|
1812
|
+
selected_agent = execution.agent
|
|
1813
|
+
default_model = execution.model
|
|
1285
1814
|
|
|
1286
1815
|
def _run(
|
|
1287
1816
|
prompt: str,
|
|
@@ -1291,18 +1820,28 @@ def run_claude(test_dir, experiment_logger, request):
|
|
|
1291
1820
|
image_id: str | None = None,
|
|
1292
1821
|
):
|
|
1293
1822
|
mdl = model or default_model
|
|
1823
|
+
run_execution = ResolvedExecution(
|
|
1824
|
+
selected_agent,
|
|
1825
|
+
mdl,
|
|
1826
|
+
execution.base_url,
|
|
1827
|
+
execution.sources,
|
|
1828
|
+
)
|
|
1829
|
+
run_environment = build_agent_environment(run_execution)
|
|
1294
1830
|
if os.environ.get("TRACE_TO_LANGSMITH", "").lower() == "true" and not os.environ.get(
|
|
1295
1831
|
"CC_LANGSMITH_LOG_FILE"
|
|
1296
1832
|
):
|
|
1297
1833
|
os.environ["CC_LANGSMITH_LOG_FILE"] = "/workspace/langsmith-hook.log"
|
|
1298
1834
|
use_loop = interaction is not None and interaction.mode == "auto_user"
|
|
1299
1835
|
if not use_loop:
|
|
1300
|
-
result =
|
|
1836
|
+
result = run_agent_in_docker(
|
|
1301
1837
|
test_dir,
|
|
1302
1838
|
prompt,
|
|
1839
|
+
agent=selected_agent,
|
|
1303
1840
|
timeout=timeout,
|
|
1304
1841
|
model=mdl,
|
|
1842
|
+
base_url=execution.base_url,
|
|
1305
1843
|
image_id=image_id,
|
|
1844
|
+
environment=run_environment,
|
|
1306
1845
|
)
|
|
1307
1846
|
else:
|
|
1308
1847
|
task_prompt_file = test_dir / ".eval-task-prompt.txt"
|
|
@@ -1342,11 +1881,25 @@ def run_claude(test_dir, experiment_logger, request):
|
|
|
1342
1881
|
"--simulator-prompt-file",
|
|
1343
1882
|
"//workspace/.eval-simulator-prompt.txt",
|
|
1344
1883
|
]
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1884
|
+
if selected_agent == "claude-code":
|
|
1885
|
+
result = run_claude_loop_in_docker(
|
|
1886
|
+
test_dir,
|
|
1887
|
+
loop_args[2:],
|
|
1888
|
+
timeout=timeout + 60,
|
|
1889
|
+
environment=run_environment,
|
|
1890
|
+
)
|
|
1891
|
+
else:
|
|
1892
|
+
result = run_agent_loop_in_docker(
|
|
1893
|
+
test_dir,
|
|
1894
|
+
[
|
|
1895
|
+
loop_args[2],
|
|
1896
|
+
"--agent",
|
|
1897
|
+
selected_agent,
|
|
1898
|
+
*loop_args[3:],
|
|
1899
|
+
],
|
|
1900
|
+
timeout=timeout + 60,
|
|
1901
|
+
environment=run_environment,
|
|
1902
|
+
)
|
|
1350
1903
|
finally:
|
|
1351
1904
|
task_prompt_file.unlink(missing_ok=True)
|
|
1352
1905
|
if prompt_file and prompt_file.exists():
|
|
@@ -1384,12 +1937,22 @@ def record_result(test_dir, experiment_logger, request):
|
|
|
1384
1937
|
if not experiment_logger:
|
|
1385
1938
|
return
|
|
1386
1939
|
|
|
1940
|
+
if _plugin is not None:
|
|
1941
|
+
_plugin.last_test_dir = test_dir
|
|
1942
|
+
|
|
1387
1943
|
treatment_name = _get_treatment_name(request.node)
|
|
1388
1944
|
rep = _plugin.run_counter.get(treatment_name, 1) if _plugin else 1
|
|
1945
|
+
events["sample"] = rep
|
|
1389
1946
|
base_dir = experiment_logger.base_dir
|
|
1390
1947
|
|
|
1391
1948
|
save_events(base_dir, treatment_name, rep, events)
|
|
1392
|
-
_save_artifacts(
|
|
1949
|
+
_save_artifacts(
|
|
1950
|
+
base_dir,
|
|
1951
|
+
treatment_name,
|
|
1952
|
+
rep,
|
|
1953
|
+
test_dir,
|
|
1954
|
+
agent=events.get("agent", "claude-code"),
|
|
1955
|
+
)
|
|
1393
1956
|
artifact_references = build_eval_artifact_references(base_dir, treatment_name, rep)
|
|
1394
1957
|
|
|
1395
1958
|
scripts_used = _extract_scripts_used(events)
|
|
@@ -1439,9 +2002,23 @@ def record_result(test_dir, experiment_logger, request):
|
|
|
1439
2002
|
"skill_sources": events.get("skill_sources", []),
|
|
1440
2003
|
"eval_manifest": events.get("eval_manifest"),
|
|
1441
2004
|
"case_manifest": events.get("case_manifest"),
|
|
2005
|
+
"eval_generation": events.get("eval_generation"),
|
|
1442
2006
|
"interaction": events.get("interaction", {}),
|
|
1443
2007
|
"artifact_references": artifact_references,
|
|
1444
2008
|
"failure_attribution": failure_attribution,
|
|
2009
|
+
"agent": events.get("agent", "claude-code"),
|
|
2010
|
+
"role_models": events.get("role_models", {}),
|
|
2011
|
+
"role_agents": events.get("role_agents", {}),
|
|
2012
|
+
"telemetry_status": events.get("telemetry_status", "N/A"),
|
|
2013
|
+
"role_sessions": events.get("role_sessions", {}),
|
|
2014
|
+
"task": events.get("task"),
|
|
2015
|
+
"treatment": events.get("treatment", treatment_name),
|
|
2016
|
+
"sample": events.get("sample", rep),
|
|
2017
|
+
"prompt": events.get("prompt"),
|
|
2018
|
+
"skill": events.get("skill"),
|
|
2019
|
+
"final_response": events.get("final_response"),
|
|
2020
|
+
"quality_gates": events.get("quality_gates", {}),
|
|
2021
|
+
"execution_identity": events.get("execution_identity"),
|
|
1445
2022
|
},
|
|
1446
2023
|
run_id=run_id,
|
|
1447
2024
|
),
|
|
@@ -1495,14 +2072,29 @@ def _build_report_payload(
|
|
|
1495
2072
|
"model_usage": events.get("model_usage", {}),
|
|
1496
2073
|
"files_created": events.get("files_created", []),
|
|
1497
2074
|
"skills_invoked": events.get("skills_invoked", []),
|
|
2075
|
+
"skill_invocations": events.get("skill_invocations", []),
|
|
1498
2076
|
"scripts_used": scripts_used,
|
|
1499
2077
|
"profile": events.get("profile"),
|
|
1500
2078
|
"skill_sources": events.get("skill_sources", []),
|
|
1501
2079
|
"eval_manifest": events.get("eval_manifest"),
|
|
1502
2080
|
"case_manifest": events.get("case_manifest"),
|
|
2081
|
+
"eval_generation": events.get("eval_generation"),
|
|
1503
2082
|
"interaction": events.get("interaction", {}),
|
|
1504
2083
|
"artifact_references": artifact_references,
|
|
1505
2084
|
"failure_attribution": failure_attribution,
|
|
2085
|
+
"agent": events.get("agent", "claude-code"),
|
|
2086
|
+
"role_models": events.get("role_models", {}),
|
|
2087
|
+
"role_agents": events.get("role_agents", {}),
|
|
2088
|
+
"telemetry_status": events.get("telemetry_status", "N/A"),
|
|
2089
|
+
"role_sessions": events.get("role_sessions", {}),
|
|
2090
|
+
"task": events.get("task"),
|
|
2091
|
+
"treatment": events.get("treatment", treatment_name),
|
|
2092
|
+
"sample": events.get("sample", rep),
|
|
2093
|
+
"prompt": events.get("prompt"),
|
|
2094
|
+
"skill": events.get("skill"),
|
|
2095
|
+
"final_response": events.get("final_response"),
|
|
2096
|
+
"quality_gates": events.get("quality_gates", {}),
|
|
2097
|
+
"execution_identity": events.get("execution_identity"),
|
|
1506
2098
|
},
|
|
1507
2099
|
"timestamp": datetime.now().isoformat(),
|
|
1508
2100
|
}
|
|
@@ -1581,15 +2173,32 @@ def _filter_scripts(scripts_dir: Path, script_filter: str) -> Path | None:
|
|
|
1581
2173
|
return temp_dir
|
|
1582
2174
|
|
|
1583
2175
|
|
|
1584
|
-
def _build_docker_image_with_lock(
|
|
2176
|
+
def _build_docker_image_with_lock(
|
|
2177
|
+
environment_dir: Path,
|
|
2178
|
+
agent: str | None = None,
|
|
2179
|
+
*,
|
|
2180
|
+
environment: dict[str, str] | None = None,
|
|
2181
|
+
) -> str | None:
|
|
1585
2182
|
"""Build Docker image with file locking to prevent race conditions."""
|
|
1586
2183
|
if not environment_dir or not (environment_dir / "Dockerfile").exists():
|
|
1587
2184
|
return None
|
|
1588
2185
|
|
|
1589
|
-
|
|
1590
|
-
|
|
2186
|
+
context = context_from_environment()
|
|
2187
|
+
lock_path = (
|
|
2188
|
+
resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
|
|
2189
|
+
if context is not None
|
|
2190
|
+
else DOCKER_BUILD_LOCK
|
|
2191
|
+
)
|
|
2192
|
+
with file_lock(lock_path):
|
|
2193
|
+
# A cold image build downloads Debian packages and installs the selected CLI.
|
|
1591
2194
|
# Five minutes is insufficient after cache cleanup or on a proxied connection.
|
|
1592
|
-
|
|
2195
|
+
command = ["docker.sh", "build", str(environment_dir)]
|
|
2196
|
+
if agent:
|
|
2197
|
+
command.extend(["--agent", agent])
|
|
2198
|
+
run_options = {"timeout": 900, "check": False}
|
|
2199
|
+
if environment is not None:
|
|
2200
|
+
run_options["env"] = environment
|
|
2201
|
+
result = run_shell(*command, **run_options)
|
|
1593
2202
|
if result.returncode == 0:
|
|
1594
2203
|
return result.stdout.strip()
|
|
1595
2204
|
return None
|
|
@@ -1664,18 +2273,34 @@ def _extract_scripts_used(events: dict) -> list[str]:
|
|
|
1664
2273
|
return [s for s in _get_known_scripts() if s.lower() in all_activity]
|
|
1665
2274
|
|
|
1666
2275
|
|
|
1667
|
-
def _save_artifacts(
|
|
1668
|
-
|
|
2276
|
+
def _save_artifacts(
|
|
2277
|
+
base_dir: Path,
|
|
2278
|
+
treatment_name: str,
|
|
2279
|
+
rep: int,
|
|
2280
|
+
test_dir: Path,
|
|
2281
|
+
*,
|
|
2282
|
+
agent: str = "claude-code",
|
|
2283
|
+
):
|
|
2284
|
+
"""Save the selected Agent's generated files as artifacts."""
|
|
1669
2285
|
artifacts_dir = base_dir / "artifacts" / f"{treatment_name.lower()}_rep{rep}"
|
|
1670
|
-
|
|
2286
|
+
agent_dir_name = {
|
|
2287
|
+
"claude-code": "claude",
|
|
2288
|
+
"codex": "codex",
|
|
2289
|
+
"qoder": "qoder",
|
|
2290
|
+
"codebuddy": "codebuddy",
|
|
2291
|
+
}.get(agent, agent)
|
|
2292
|
+
agent_dir = artifacts_dir / agent_dir_name
|
|
1671
2293
|
execution_dir = artifacts_dir / "execution"
|
|
1672
|
-
|
|
2294
|
+
agent_dir.mkdir(parents=True, exist_ok=True)
|
|
1673
2295
|
execution_dir.mkdir(parents=True, exist_ok=True)
|
|
1674
2296
|
|
|
1675
2297
|
from scaffold.python.utils import TEST_CONTEXT_FILE, TEST_RESULTS_FILE
|
|
1676
2298
|
|
|
1677
2299
|
exclude_dirs = {
|
|
1678
2300
|
".claude",
|
|
2301
|
+
".agents",
|
|
2302
|
+
".qoder",
|
|
2303
|
+
".codebuddy",
|
|
1679
2304
|
".git",
|
|
1680
2305
|
"_eval_current_comet",
|
|
1681
2306
|
"node_modules",
|
|
@@ -1707,9 +2332,20 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
|
|
|
1707
2332
|
continue
|
|
1708
2333
|
try:
|
|
1709
2334
|
rel_path = item.relative_to(test_dir)
|
|
1710
|
-
dest =
|
|
2335
|
+
dest = agent_dir / rel_path
|
|
1711
2336
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1712
|
-
|
|
2337
|
+
try:
|
|
2338
|
+
content = item.read_text(encoding="utf-8")
|
|
2339
|
+
except UnicodeDecodeError:
|
|
2340
|
+
import sys
|
|
2341
|
+
|
|
2342
|
+
print(
|
|
2343
|
+
f"Warning: skipped non-text artifact {rel_path}; binary content is not persisted to protect credentials",
|
|
2344
|
+
file=sys.stderr,
|
|
2345
|
+
)
|
|
2346
|
+
continue
|
|
2347
|
+
else:
|
|
2348
|
+
dest.write_text(redact_sensitive(content), encoding="utf-8")
|
|
1713
2349
|
claude_files.append(item)
|
|
1714
2350
|
except Exception as e:
|
|
1715
2351
|
import sys
|
|
@@ -1722,7 +2358,7 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
|
|
|
1722
2358
|
success, output = run_python_in_docker(test_dir, py_file.name, timeout=300)
|
|
1723
2359
|
status = "success" if success else "error"
|
|
1724
2360
|
output_file = execution_dir / f"{py_file.stem}_{status}.txt"
|
|
1725
|
-
output_file.write_text(strip_ansi(output))
|
|
2361
|
+
output_file.write_text(redact_sensitive(strip_ansi(output)), encoding="utf-8")
|
|
1726
2362
|
except Exception as e:
|
|
1727
2363
|
error_file = execution_dir / f"{py_file.stem}_error.txt"
|
|
1728
|
-
error_file.write_text(str(e))
|
|
2364
|
+
error_file.write_text(redact_sensitive(str(e)), encoding="utf-8")
|