@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -26
- package/assets/manifest.json +4 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
- package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
- package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
- package/assets/skills/comet-build/SKILL.md +36 -62
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +2 -2
- package/assets/skills/comet-native/SKILL.md +18 -15
- package/assets/skills/comet-native/reference/artifacts.md +5 -4
- package/assets/skills/comet-native/reference/clarification.md +1 -1
- package/assets/skills/comet-native/reference/commands.md +4 -4
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills/comet-review/SKILL.md +121 -0
- package/assets/skills/comet-review/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +36 -62
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +2 -2
- package/assets/skills-zh/comet-native/SKILL.md +18 -15
- package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
- package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
- package/assets/skills-zh/comet-native/reference/commands.md +4 -4
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-review/SKILL.md +121 -0
- package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +5 -8
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +5 -6
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
- package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.js +10 -0
- package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +4 -4
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.js +44 -6
- package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.js +2 -1
- package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +39 -13
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +2 -2
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +5 -1
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +2 -0
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.js +17 -10
- package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +65 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +453 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +13 -7
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +18 -5
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +56 -4
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +179 -20
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +178 -73
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +168 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
- package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
- package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.js +24 -6
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.js +86 -35
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
- package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +27 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +108 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +87 -44
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +14 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +93 -5
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +3 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +4 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +16 -10
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +11 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +39 -29
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
- package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +4 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +174 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +13 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/fs/strip-bom.d.ts +10 -0
- package/dist/platform/fs/strip-bom.d.ts.map +1 -0
- package/dist/platform/fs/strip-bom.js +12 -0
- package/dist/platform/fs/strip-bom.js.map +1 -0
- package/dist/platform/install/platforms.d.ts +11 -0
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +49 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +1 -0
- package/dist/platform/process/external-command.d.ts.map +1 -1
- package/dist/platform/process/external-command.js +2 -1
- package/dist/platform/process/external-command.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +719 -83
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +18 -28
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
- package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""The sole, import-safe authority for Eval task selection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
from scaffold.python.eval_context import ResolvedEvalContext
|
|
10
|
+
from scaffold.python.execution import ResolvedExecution, resolve_execution
|
|
11
|
+
from scaffold.python.generated_task_cache import (
|
|
12
|
+
GeneratedCache,
|
|
13
|
+
build_skill_snapshot,
|
|
14
|
+
cache_location,
|
|
15
|
+
generation_hash,
|
|
16
|
+
load_generated_cache,
|
|
17
|
+
selected_agent_model,
|
|
18
|
+
)
|
|
19
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
20
|
+
from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
|
|
21
|
+
from scaffold.python.tasks import Task, load_task
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
TaskSource = Literal[
|
|
25
|
+
"explicit", "quick", "authored", "recommended", "generated-cache", "pending-generation"
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class GenerationPlan:
|
|
31
|
+
generation_hash: str
|
|
32
|
+
cache_dir: Path
|
|
33
|
+
cached: GeneratedCache | None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class ResolvedTask:
|
|
38
|
+
name: str
|
|
39
|
+
provenance: Literal["inline", "source", "bundled", "generated"]
|
|
40
|
+
task: Task
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class ResolvedTaskSet:
|
|
45
|
+
source: TaskSource
|
|
46
|
+
tasks: tuple[ResolvedTask, ...]
|
|
47
|
+
generation: GenerationPlan | None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class TaskCatalogue:
|
|
52
|
+
bundled: dict[str, ResolvedTask]
|
|
53
|
+
canonical: dict[str, ResolvedTask]
|
|
54
|
+
authored: dict[str, ResolvedTask]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _bundled(tasks_dir: Path) -> dict[str, ResolvedTask]:
|
|
58
|
+
result: dict[str, ResolvedTask] = {}
|
|
59
|
+
canonical: dict[str, str] = {}
|
|
60
|
+
if not tasks_dir.is_dir():
|
|
61
|
+
return result
|
|
62
|
+
for directory in sorted(path for path in tasks_dir.iterdir() if path.is_dir()):
|
|
63
|
+
if not (directory / "task.toml").is_file() or not (directory / "instruction.md").is_file():
|
|
64
|
+
continue
|
|
65
|
+
task = load_task(directory, tasks_dir=tasks_dir)
|
|
66
|
+
if task.name in canonical:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f'bundled tasks "{canonical[task.name]}" and "{directory.name}" both declare '
|
|
69
|
+
f'metadata.name "{task.name}"'
|
|
70
|
+
)
|
|
71
|
+
canonical[task.name] = directory.name
|
|
72
|
+
result[directory.name] = ResolvedTask(task.name, "bundled", task)
|
|
73
|
+
return result
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def build_task_catalogue(manifest: SkillEvalManifest, tasks_dir: Path) -> TaskCatalogue:
|
|
77
|
+
bundled = _bundled(tasks_dir)
|
|
78
|
+
canonical = {item.name: item for item in bundled.values()}
|
|
79
|
+
authored_tasks = load_manifest_tasks(manifest)
|
|
80
|
+
authored: dict[str, ResolvedTask] = {}
|
|
81
|
+
for index, task in enumerate(authored_tasks):
|
|
82
|
+
source = manifest.tasks[index]
|
|
83
|
+
authored[task.name] = ResolvedTask(task.name, "inline" if source.is_inline else "source", task)
|
|
84
|
+
for index, task_name in enumerate(manifest.recommended_tasks):
|
|
85
|
+
if task_name not in bundled:
|
|
86
|
+
raise ValueError(f'evaluation.recommendedTasks[{index}]: unknown bundled task "{task_name}"')
|
|
87
|
+
if task_name in authored:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f'evaluation.tasks[{next(i for i, item in enumerate(manifest.tasks) if item.name == task_name)}].name '
|
|
90
|
+
f'conflicts with evaluation.recommendedTasks[{index}]: "{task_name}"'
|
|
91
|
+
)
|
|
92
|
+
for index, spec in enumerate(manifest.tasks):
|
|
93
|
+
if spec.name in canonical:
|
|
94
|
+
bundle = next(key for key, item in bundled.items() if item.name == spec.name)
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f'evaluation.tasks[{index}].name conflicts with bundled task "{bundle}": "{spec.name}"'
|
|
97
|
+
)
|
|
98
|
+
return TaskCatalogue(bundled, canonical, authored)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _generated_plan(
|
|
102
|
+
context: ResolvedEvalContext,
|
|
103
|
+
manifest: SkillEvalManifest,
|
|
104
|
+
execution: ResolvedExecution | None = None,
|
|
105
|
+
) -> GenerationPlan:
|
|
106
|
+
snapshot = build_skill_snapshot(context.skill_root)
|
|
107
|
+
resolved_execution = execution or resolve_execution(manifest=manifest.execution)
|
|
108
|
+
selected_agent = resolved_execution.agent
|
|
109
|
+
selected_model = resolved_execution.model or selected_agent_model(selected_agent)
|
|
110
|
+
digest = generation_hash(
|
|
111
|
+
snapshot,
|
|
112
|
+
agent=selected_agent,
|
|
113
|
+
model=selected_model,
|
|
114
|
+
base_url=resolved_execution.base_url,
|
|
115
|
+
profile=manifest.profile or "generic",
|
|
116
|
+
interaction=vars(manifest.interaction),
|
|
117
|
+
)
|
|
118
|
+
cache_dir = cache_location(context.artifact_owner_root, context.skill_root, digest)
|
|
119
|
+
return GenerationPlan(digest, cache_dir, load_generated_cache(cache_dir, digest))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def resolve_task_set(
|
|
123
|
+
context: ResolvedEvalContext,
|
|
124
|
+
manifest: SkillEvalManifest,
|
|
125
|
+
catalogue: TaskCatalogue,
|
|
126
|
+
*,
|
|
127
|
+
explicit_task: str | None = None,
|
|
128
|
+
quick: bool = False,
|
|
129
|
+
static_collect: bool = False,
|
|
130
|
+
execution: ResolvedExecution | None = None,
|
|
131
|
+
) -> ResolvedTaskSet:
|
|
132
|
+
"""Resolve exactly one precedence branch without probing lower branches."""
|
|
133
|
+
if explicit_task:
|
|
134
|
+
selected = catalogue.authored.get(explicit_task) or catalogue.bundled.get(explicit_task) or catalogue.canonical.get(explicit_task)
|
|
135
|
+
if selected is None:
|
|
136
|
+
available = sorted({*catalogue.authored, *catalogue.bundled, *catalogue.canonical})
|
|
137
|
+
raise ValueError(f'Task not found: {explicit_task}. Available: {available}')
|
|
138
|
+
return ResolvedTaskSet("explicit", (selected,), None)
|
|
139
|
+
if quick:
|
|
140
|
+
selected = catalogue.bundled.get("generic-skill-smoke")
|
|
141
|
+
if selected is None:
|
|
142
|
+
raise ValueError('Bundled quick task generic-skill-smoke is unavailable')
|
|
143
|
+
return ResolvedTaskSet("quick", (selected,), None)
|
|
144
|
+
if manifest.tasks:
|
|
145
|
+
return ResolvedTaskSet("authored", tuple(catalogue.authored[item.name] for item in manifest.tasks), None)
|
|
146
|
+
if manifest.recommended_tasks:
|
|
147
|
+
return ResolvedTaskSet("recommended", tuple(catalogue.bundled[item] for item in manifest.recommended_tasks), None)
|
|
148
|
+
plan = _generated_plan(context, manifest, execution)
|
|
149
|
+
if plan.cached is None:
|
|
150
|
+
return ResolvedTaskSet("pending-generation", (), plan)
|
|
151
|
+
generated = load_eval_manifest(plan.cached.manifest_path)
|
|
152
|
+
loaded = load_manifest_tasks(generated)
|
|
153
|
+
names: set[str] = set()
|
|
154
|
+
tasks: list[ResolvedTask] = []
|
|
155
|
+
for index, task in enumerate(loaded):
|
|
156
|
+
if task.name in names:
|
|
157
|
+
first_index = next(
|
|
158
|
+
item_index
|
|
159
|
+
for item_index, item in enumerate(loaded[:index])
|
|
160
|
+
if item.name == task.name
|
|
161
|
+
)
|
|
162
|
+
raise ValueError(
|
|
163
|
+
f'generated.tasks[{index}].name duplicates '
|
|
164
|
+
f'generated.tasks[{first_index}].name: "{task.name}"'
|
|
165
|
+
)
|
|
166
|
+
if task.name in catalogue.canonical:
|
|
167
|
+
bundle = next(
|
|
168
|
+
key for key, item in catalogue.bundled.items() if item.name == task.name
|
|
169
|
+
)
|
|
170
|
+
raise ValueError(
|
|
171
|
+
f'generated.tasks[{index}].name conflicts with bundled task '
|
|
172
|
+
f'"{bundle}": "{task.name}"'
|
|
173
|
+
)
|
|
174
|
+
names.add(task.name)
|
|
175
|
+
tasks.append(ResolvedTask(task.name, "generated", task))
|
|
176
|
+
return ResolvedTaskSet("generated-cache", tuple(tasks), plan)
|
|
@@ -148,6 +148,10 @@ class Task:
|
|
|
148
148
|
path: Path
|
|
149
149
|
config: TaskConfig
|
|
150
150
|
instruction_template: str
|
|
151
|
+
_environment_dir: Path | None = field(default=None, repr=False)
|
|
152
|
+
workspace_dir: Path | None = field(default=None, repr=False)
|
|
153
|
+
manifest_expectations: dict[str, Any] | None = field(default=None, repr=False)
|
|
154
|
+
manifest_path: Path | None = field(default=None, repr=False)
|
|
151
155
|
|
|
152
156
|
@property
|
|
153
157
|
def name(self) -> str:
|
|
@@ -155,7 +159,7 @@ class Task:
|
|
|
155
159
|
|
|
156
160
|
@property
|
|
157
161
|
def environment_dir(self) -> Path:
|
|
158
|
-
return self.path / "environment"
|
|
162
|
+
return self._environment_dir or self.path / "environment"
|
|
159
163
|
|
|
160
164
|
@property
|
|
161
165
|
def validation_dir(self) -> Path:
|
|
@@ -195,6 +199,11 @@ class Task:
|
|
|
195
199
|
The config specifies test_scripts and target_artifacts. The framework
|
|
196
200
|
auto-builds an execution validator that runs the scripts in Docker.
|
|
197
201
|
"""
|
|
202
|
+
if self.manifest_expectations is not None:
|
|
203
|
+
from scaffold.python.inline_tasks import make_inline_expectations_validator
|
|
204
|
+
|
|
205
|
+
return [make_inline_expectations_validator(self.manifest_expectations)]
|
|
206
|
+
|
|
198
207
|
vc = self.config.validation
|
|
199
208
|
if vc.test_scripts:
|
|
200
209
|
from scaffold.python.utils import make_execution_validator
|
|
@@ -212,7 +221,7 @@ class Task:
|
|
|
212
221
|
return []
|
|
213
222
|
|
|
214
223
|
|
|
215
|
-
def load_task(name: str, tasks_dir: Path | None = None) -> Task:
|
|
224
|
+
def load_task(name: str | Path, tasks_dir: Path | None = None) -> Task:
|
|
216
225
|
"""Load a task by name.
|
|
217
226
|
|
|
218
227
|
Args:
|
|
@@ -222,8 +231,7 @@ def load_task(name: str, tasks_dir: Path | None = None) -> Task:
|
|
|
222
231
|
Returns:
|
|
223
232
|
Task object with config and instruction template
|
|
224
233
|
"""
|
|
225
|
-
|
|
226
|
-
task_path = tasks_dir / name
|
|
234
|
+
task_path = Path(name) if isinstance(name, Path) else (tasks_dir or get_tasks_dir()) / name
|
|
227
235
|
|
|
228
236
|
if not task_path.exists():
|
|
229
237
|
raise FileNotFoundError(f"Task not found: {name} (looked in {tasks_dir})")
|
|
@@ -323,6 +331,11 @@ def load_task(name: str, tasks_dir: Path | None = None) -> Task:
|
|
|
323
331
|
return Task(path=task_path, config=config, instruction_template=instruction_template)
|
|
324
332
|
|
|
325
333
|
|
|
334
|
+
def load_task_from_path(task_path: Path | str) -> Task:
|
|
335
|
+
"""Load an advanced task package referenced by a project eval manifest."""
|
|
336
|
+
return load_task(Path(task_path))
|
|
337
|
+
|
|
338
|
+
|
|
326
339
|
def list_tasks(tasks_dir: Path | None = None) -> list[str]:
|
|
327
340
|
"""List available task names.
|
|
328
341
|
|
|
@@ -4,6 +4,7 @@ import json
|
|
|
4
4
|
import ntpath
|
|
5
5
|
import os
|
|
6
6
|
import random
|
|
7
|
+
import re
|
|
7
8
|
import shutil
|
|
8
9
|
import subprocess
|
|
9
10
|
import time
|
|
@@ -11,6 +12,8 @@ from pathlib import Path
|
|
|
11
12
|
|
|
12
13
|
from dotenv import load_dotenv
|
|
13
14
|
|
|
15
|
+
from scaffold.python.agents import validate_agent_id
|
|
16
|
+
from scaffold.python.execution import ResolvedExecution, build_agent_environment
|
|
14
17
|
from scaffold.python.paths import EVAL_ROOT, get_suite_root
|
|
15
18
|
|
|
16
19
|
TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
|
|
@@ -21,8 +24,14 @@ SCAFFOLD_PYTHON_DIR = Path(__file__).parent
|
|
|
21
24
|
|
|
22
25
|
def load_eval_environment() -> None:
|
|
23
26
|
"""Load eval credentials only at an explicit execution boundary, never on import."""
|
|
24
|
-
|
|
25
|
-
|
|
27
|
+
original_environment = dict(os.environ)
|
|
28
|
+
try:
|
|
29
|
+
load_dotenv(EVAL_ROOT / ".env", override=True)
|
|
30
|
+
load_dotenv(get_suite_root() / ".env", override=True)
|
|
31
|
+
load_dotenv(Path.home() / ".comet" / "eval" / ".env", override=True)
|
|
32
|
+
finally:
|
|
33
|
+
for key, value in original_environment.items():
|
|
34
|
+
os.environ[key] = value
|
|
26
35
|
|
|
27
36
|
|
|
28
37
|
def _uses_wsl_bash(bash_exec: str) -> bool:
|
|
@@ -69,12 +78,28 @@ def _resolve_bash(os_name: str | None = None) -> str:
|
|
|
69
78
|
|
|
70
79
|
BASH_EXEC = _resolve_bash()
|
|
71
80
|
WSL_ENV_KEYS = (
|
|
81
|
+
"BENCH_EVAL_AGENT",
|
|
72
82
|
"OPENAI_API_KEY",
|
|
83
|
+
"OPENAI_BASE_URL",
|
|
84
|
+
"OPENAI_MODEL",
|
|
85
|
+
"QODER_PERSONAL_ACCESS_TOKEN",
|
|
86
|
+
"QODER_BASE_URL",
|
|
87
|
+
"QODER_MODEL",
|
|
88
|
+
"CODEX_API_KEY",
|
|
89
|
+
"CODEX_BASE_URL",
|
|
90
|
+
"CODEX_MODEL",
|
|
73
91
|
"ANTHROPIC_API_KEY",
|
|
74
92
|
"LANGSMITH_API_KEY",
|
|
75
93
|
"LANGSMITH_PROJECT",
|
|
76
94
|
"LANGSMITH_TRACING",
|
|
77
95
|
"LANGSMITH_ENDPOINT",
|
|
96
|
+
"LANGFUSE_PUBLIC_KEY",
|
|
97
|
+
"LANGFUSE_SECRET_KEY",
|
|
98
|
+
"LANGFUSE_BASE_URL",
|
|
99
|
+
"LANGFUSE_TRACING_ENVIRONMENT",
|
|
100
|
+
"TRACE_TO_LANGFUSE",
|
|
101
|
+
"LANGFUSE_CODEX_TAGS",
|
|
102
|
+
"LANGFUSE_CODEX_METADATA",
|
|
78
103
|
"TAVILY_API_KEY",
|
|
79
104
|
"TRACE_TO_LANGSMITH",
|
|
80
105
|
"CC_LANGSMITH_API_KEY",
|
|
@@ -97,6 +122,31 @@ WSL_ENV_KEYS = (
|
|
|
97
122
|
"CLAUDE_CODE_SUBAGENT_MODEL",
|
|
98
123
|
"BENCH_CC_VERSION",
|
|
99
124
|
"BENCH_CC_MODEL",
|
|
125
|
+
"BENCH_CODEX_VERSION",
|
|
126
|
+
"BENCH_CODEX_MODEL",
|
|
127
|
+
"BENCH_QODER_VERSION",
|
|
128
|
+
"BENCH_QODER_MODEL",
|
|
129
|
+
"CODEBUDDY_API_KEY",
|
|
130
|
+
"CODEBUDDY_AUTH_TOKEN",
|
|
131
|
+
"CODEBUDDY_BASE_URL",
|
|
132
|
+
"CODEBUDDY_MODEL",
|
|
133
|
+
"CODEBUDDY_SMALL_FAST_MODEL",
|
|
134
|
+
"CODEBUDDY_BIG_SLOW_MODEL",
|
|
135
|
+
"CODEBUDDY_CODE_SUBAGENT_MODEL",
|
|
136
|
+
"CODEBUDDY_CUSTOM_HEADERS",
|
|
137
|
+
"CODEBUDDY_INTERNET_ENVIRONMENT",
|
|
138
|
+
"BENCH_CODEBUDDY_VERSION",
|
|
139
|
+
"BENCH_CODEBUDDY_MODEL",
|
|
140
|
+
"COMET_EVAL_CUSTOM_AGENT_ID",
|
|
141
|
+
"COMET_EVAL_CUSTOM_EXECUTABLE",
|
|
142
|
+
"COMET_EVAL_CUSTOM_CREDENTIALS",
|
|
143
|
+
"COMET_EVAL_CUSTOM_MODEL",
|
|
144
|
+
"COMET_EVAL_CUSTOM_BASE_URL",
|
|
145
|
+
"COMET_EVAL_CUSTOM_MODEL_ENV",
|
|
146
|
+
"COMET_EVAL_CUSTOM_BASE_URL_ENV",
|
|
147
|
+
"COMET_EVAL_CUSTOM_INSTALL_KIND",
|
|
148
|
+
"COMET_EVAL_CUSTOM_INSTALL_PACKAGE",
|
|
149
|
+
"COMET_EVAL_CUSTOM_INSTALL_VERSION",
|
|
100
150
|
)
|
|
101
151
|
|
|
102
152
|
|
|
@@ -119,20 +169,33 @@ def _to_bash_path(value) -> str:
|
|
|
119
169
|
return s
|
|
120
170
|
|
|
121
171
|
|
|
122
|
-
def _bash_env() -> dict[str, str]:
|
|
123
|
-
env = os.environ
|
|
172
|
+
def _bash_env(source_env: dict[str, str] | None = None) -> dict[str, str]:
|
|
173
|
+
env = dict(source_env if source_env is not None else os.environ)
|
|
124
174
|
if os.name != "nt" or not _uses_wsl_bash(BASH_EXEC):
|
|
125
175
|
return env
|
|
126
176
|
|
|
127
177
|
existing = [item for item in env.get("WSLENV", "").split(":") if item]
|
|
178
|
+
custom_credentials = [
|
|
179
|
+
key.strip()
|
|
180
|
+
for key in env.get("COMET_EVAL_CUSTOM_CREDENTIALS", "").split(",")
|
|
181
|
+
if key.strip()
|
|
182
|
+
]
|
|
183
|
+
custom_routing = [
|
|
184
|
+
key.strip()
|
|
185
|
+
for metadata_key in ("COMET_EVAL_CUSTOM_MODEL_ENV", "COMET_EVAL_CUSTOM_BASE_URL_ENV")
|
|
186
|
+
for key in [env.get(metadata_key, "")]
|
|
187
|
+
if re.fullmatch(r"[A-Z][A-Z0-9_]{1,63}", key.strip())
|
|
188
|
+
]
|
|
128
189
|
exported = [key for key in WSL_ENV_KEYS if env.get(key)]
|
|
190
|
+
exported.extend(key for key in custom_credentials if env.get(key))
|
|
191
|
+
exported.extend(key for key in custom_routing if env.get(key))
|
|
129
192
|
merged = list(dict.fromkeys(existing + exported))
|
|
130
193
|
if merged:
|
|
131
194
|
env["WSLENV"] = ":".join(merged)
|
|
132
195
|
return env
|
|
133
196
|
|
|
134
197
|
|
|
135
|
-
def run_shell(script, *args, timeout=None, check=True):
|
|
198
|
+
def run_shell(script, *args, timeout=None, check=True, env=None):
|
|
136
199
|
cmd = [BASH_EXEC, _to_bash_path(SHELL_DIR / script)] + [_to_bash_path(a) for a in args]
|
|
137
200
|
return subprocess.run(
|
|
138
201
|
cmd,
|
|
@@ -142,7 +205,7 @@ def run_shell(script, *args, timeout=None, check=True):
|
|
|
142
205
|
errors="replace",
|
|
143
206
|
timeout=timeout,
|
|
144
207
|
check=check,
|
|
145
|
-
env=_bash_env(),
|
|
208
|
+
env=_bash_env(env),
|
|
146
209
|
)
|
|
147
210
|
|
|
148
211
|
|
|
@@ -153,9 +216,11 @@ def check_docker_available():
|
|
|
153
216
|
return False
|
|
154
217
|
|
|
155
218
|
|
|
156
|
-
def build_docker_image(test_dir, force=False, verbose=False):
|
|
219
|
+
def build_docker_image(test_dir, force=False, verbose=False, agent=None):
|
|
157
220
|
try:
|
|
158
221
|
args = ["build", str(test_dir)] + (["--force"] if force else [])
|
|
222
|
+
if agent:
|
|
223
|
+
args.extend(["--agent", validate_agent_id(agent)])
|
|
159
224
|
result = run_shell("docker.sh", *args, timeout=300, check=False)
|
|
160
225
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
161
226
|
except subprocess.TimeoutExpired:
|
|
@@ -186,7 +251,26 @@ def run_node_in_docker(test_dir, script_name, timeout=120, args=None):
|
|
|
186
251
|
return _docker_run_script("run-node", test_dir, script_name, timeout, args)
|
|
187
252
|
|
|
188
253
|
|
|
189
|
-
def
|
|
254
|
+
def run_command_in_docker(test_dir, command, timeout=120):
|
|
255
|
+
"""Run one user-authored validation command inside the task container."""
|
|
256
|
+
cmd = ["run-command", str(test_dir), "--timeout", str(timeout), "--", command]
|
|
257
|
+
if not check_docker_available():
|
|
258
|
+
return subprocess.CompletedProcess(cmd, 125, "", "Docker not available")
|
|
259
|
+
try:
|
|
260
|
+
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
|
|
261
|
+
except subprocess.TimeoutExpired:
|
|
262
|
+
return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def run_claude_in_docker(
|
|
266
|
+
test_dir,
|
|
267
|
+
prompt,
|
|
268
|
+
timeout=300,
|
|
269
|
+
model=None,
|
|
270
|
+
image_id=None,
|
|
271
|
+
base_url=None,
|
|
272
|
+
environment=None,
|
|
273
|
+
):
|
|
190
274
|
if not check_docker_available():
|
|
191
275
|
raise RuntimeError("Docker not available")
|
|
192
276
|
cmd = ["run-claude", str(test_dir), prompt, "--timeout", str(timeout)]
|
|
@@ -194,17 +278,54 @@ def run_claude_in_docker(test_dir, prompt, timeout=300, model=None, image_id=Non
|
|
|
194
278
|
cmd.extend(["--model", model])
|
|
195
279
|
if image_id:
|
|
196
280
|
cmd.extend(["--image-id", image_id])
|
|
281
|
+
child_env = environment
|
|
282
|
+
if child_env is None and (model or base_url):
|
|
283
|
+
child_env = build_agent_environment(
|
|
284
|
+
ResolvedExecution("claude-code", model, base_url, {}),
|
|
285
|
+
)
|
|
197
286
|
try:
|
|
198
|
-
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
|
|
287
|
+
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False, env=child_env)
|
|
199
288
|
except subprocess.TimeoutExpired:
|
|
200
289
|
return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
|
|
201
290
|
|
|
202
291
|
|
|
203
|
-
def
|
|
292
|
+
def run_agent_in_docker(
|
|
293
|
+
test_dir,
|
|
294
|
+
prompt,
|
|
295
|
+
*,
|
|
296
|
+
agent="claude-code",
|
|
297
|
+
timeout=300,
|
|
298
|
+
model=None,
|
|
299
|
+
base_url=None,
|
|
300
|
+
image_id=None,
|
|
301
|
+
environment=None,
|
|
302
|
+
):
|
|
303
|
+
"""Run one subject, simulator, or judge turn through the selected adapter."""
|
|
304
|
+
agent_id = validate_agent_id(agent)
|
|
305
|
+
if not check_docker_available():
|
|
306
|
+
raise RuntimeError("Docker not available")
|
|
307
|
+
cmd = ["run-agent", str(test_dir), prompt, "--agent", agent_id]
|
|
308
|
+
if model:
|
|
309
|
+
cmd.extend(["--model", model])
|
|
310
|
+
cmd.extend(["--timeout", str(timeout)])
|
|
311
|
+
if image_id:
|
|
312
|
+
cmd.extend(["--image-id", image_id])
|
|
313
|
+
child_env = environment
|
|
314
|
+
if child_env is None and (model or base_url):
|
|
315
|
+
child_env = build_agent_environment(
|
|
316
|
+
ResolvedExecution(agent_id, model, base_url, {}),
|
|
317
|
+
)
|
|
318
|
+
try:
|
|
319
|
+
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False, env=child_env)
|
|
320
|
+
except subprocess.TimeoutExpired:
|
|
321
|
+
return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def run_claude_loop_in_docker(test_dir, loop_args, timeout=600, environment=None):
|
|
204
325
|
"""Run the interactive driver and remove its container after host-side timeout."""
|
|
205
326
|
cmd = ["run-claude-loop", str(test_dir), *loop_args]
|
|
206
327
|
try:
|
|
207
|
-
return run_shell("docker.sh", *cmd, timeout=timeout, check=False)
|
|
328
|
+
return run_shell("docker.sh", *cmd, timeout=timeout, check=False, env=environment)
|
|
208
329
|
except subprocess.TimeoutExpired as error:
|
|
209
330
|
cleanup_error = ""
|
|
210
331
|
try:
|
|
@@ -235,6 +356,41 @@ def run_claude_loop_in_docker(test_dir, loop_args, timeout=600):
|
|
|
235
356
|
)
|
|
236
357
|
|
|
237
358
|
|
|
359
|
+
def run_agent_loop_in_docker(test_dir, loop_args, timeout=600, environment=None):
|
|
360
|
+
"""Run the shared interactive driver for a non-default evaluation agent."""
|
|
361
|
+
cmd = ["run-agent-loop", str(test_dir), *loop_args]
|
|
362
|
+
try:
|
|
363
|
+
return run_shell("docker.sh", *cmd, timeout=timeout, check=False, env=environment)
|
|
364
|
+
except subprocess.TimeoutExpired as error:
|
|
365
|
+
cleanup_error = ""
|
|
366
|
+
try:
|
|
367
|
+
cleanup = run_shell(
|
|
368
|
+
"docker.sh",
|
|
369
|
+
"cleanup-agent-loop",
|
|
370
|
+
test_dir,
|
|
371
|
+
timeout=30,
|
|
372
|
+
check=False,
|
|
373
|
+
)
|
|
374
|
+
if cleanup.returncode != 0:
|
|
375
|
+
cleanup_error = f"; cleanup failed: {cleanup.stderr or cleanup.stdout}"
|
|
376
|
+
except Exception as cleanup_exception: # pragma: no cover - defensive cleanup
|
|
377
|
+
cleanup_error = f"; cleanup failed: {cleanup_exception}"
|
|
378
|
+
stdout = (
|
|
379
|
+
error.stdout.decode("utf-8", errors="replace")
|
|
380
|
+
if isinstance(error.stdout, bytes)
|
|
381
|
+
else (error.stdout or "")
|
|
382
|
+
)
|
|
383
|
+
stderr = (
|
|
384
|
+
error.stderr.decode("utf-8", errors="replace")
|
|
385
|
+
if isinstance(error.stderr, bytes)
|
|
386
|
+
else (error.stderr or "")
|
|
387
|
+
)
|
|
388
|
+
message = f"Timeout after {timeout}s{cleanup_error}"
|
|
389
|
+
return subprocess.CompletedProcess(
|
|
390
|
+
cmd, 124, stdout, "\n".join(filter(None, (stderr, message)))
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
|
|
238
394
|
def _copy_scaffold_to_docker(test_dir):
|
|
239
395
|
py_validation = SCAFFOLD_PYTHON_DIR / "validation"
|
|
240
396
|
|
|
@@ -20,6 +20,49 @@ TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
|
|
|
20
20
|
TEST_RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
def _normalize_skill_invocations(events: object, outputs: dict | None) -> list[str]:
|
|
24
|
+
"""Use the Agent contract when the full scaffold is available.
|
|
25
|
+
|
|
26
|
+
Validator sidecars intentionally copy only a small subset of the scaffold,
|
|
27
|
+
so keep this helper import-local and preserve the historical fallback there.
|
|
28
|
+
"""
|
|
29
|
+
try:
|
|
30
|
+
from scaffold.python.agents import normalize_skill_invocations
|
|
31
|
+
except ModuleNotFoundError:
|
|
32
|
+
if not isinstance(events, dict):
|
|
33
|
+
return []
|
|
34
|
+
output_data = outputs if isinstance(outputs, dict) else {}
|
|
35
|
+
event_agent = events.get("agent")
|
|
36
|
+
output_agent = output_data.get("agent")
|
|
37
|
+
agent = output_agent if isinstance(output_agent, str) else event_agent
|
|
38
|
+
if isinstance(agent, str) and agent not in {
|
|
39
|
+
"claude-code",
|
|
40
|
+
"codex",
|
|
41
|
+
"qoder",
|
|
42
|
+
"codebuddy",
|
|
43
|
+
}:
|
|
44
|
+
# Validator sidecars do not carry the host-side adapter registry.
|
|
45
|
+
# A custom Agent must therefore provide explicit invocation events;
|
|
46
|
+
# artifacts or legacy path inference are not sufficient evidence.
|
|
47
|
+
explicit = events.get("skill_invocations")
|
|
48
|
+
return (
|
|
49
|
+
[item for item in explicit if isinstance(item, str)]
|
|
50
|
+
if isinstance(explicit, list)
|
|
51
|
+
else []
|
|
52
|
+
)
|
|
53
|
+
invoked = events.get("skills_invoked")
|
|
54
|
+
return (
|
|
55
|
+
[item for item in invoked if isinstance(item, str)]
|
|
56
|
+
if isinstance(invoked, list)
|
|
57
|
+
else []
|
|
58
|
+
)
|
|
59
|
+
return normalize_skill_invocations(
|
|
60
|
+
events,
|
|
61
|
+
agent=(outputs or {}).get("agent"),
|
|
62
|
+
adapter=(outputs or {}).get("agent_adapter"),
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
23
66
|
def load_test_context(path: str = TEST_CONTEXT_FILE) -> dict:
|
|
24
67
|
"""Load test context (run_id, events, langsmith_env, etc.) written by the host.
|
|
25
68
|
|
|
@@ -141,6 +184,9 @@ def run_validators(
|
|
|
141
184
|
Returns:
|
|
142
185
|
Combined (passed, failed) lists
|
|
143
186
|
"""
|
|
187
|
+
events = outputs.get("events") if isinstance(outputs, dict) else None
|
|
188
|
+
if isinstance(events, dict):
|
|
189
|
+
events["skills_invoked"] = _normalize_skill_invocations(events, outputs)
|
|
144
190
|
all_passed, all_failed = [], []
|
|
145
191
|
for validator in validators:
|
|
146
192
|
passed, failed = validator(test_dir, outputs)
|
|
@@ -164,7 +210,7 @@ def check_starter_skill_first(
|
|
|
164
210
|
return ["Note: starter skill check skipped (ALL_MAIN_SKILLS)"], []
|
|
165
211
|
|
|
166
212
|
events = outputs.get("events", {}) if outputs else {}
|
|
167
|
-
skills_invoked = events
|
|
213
|
+
skills_invoked = _normalize_skill_invocations(events, outputs)
|
|
168
214
|
|
|
169
215
|
# No skills invoked at all (e.g. CONTROL treatment) — nothing to check
|
|
170
216
|
if not skills_invoked:
|
|
@@ -205,7 +251,7 @@ def check_skill_invoked(
|
|
|
205
251
|
(passed, failed) lists
|
|
206
252
|
"""
|
|
207
253
|
events = outputs.get("events", {}) if outputs else {}
|
|
208
|
-
skills_invoked = events
|
|
254
|
+
skills_invoked = _normalize_skill_invocations(events, outputs)
|
|
209
255
|
|
|
210
256
|
if skill_name in skills_invoked:
|
|
211
257
|
return [f"Invoked {skill_name} skill"], []
|
|
@@ -7,6 +7,8 @@ import re
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
|
+
from scaffold.python.agents import normalize_skill_invocations
|
|
11
|
+
|
|
10
12
|
GENERIC_RUBRIC_DIMENSIONS = (
|
|
11
13
|
"completion",
|
|
12
14
|
"skill_invocation",
|
|
@@ -153,6 +155,13 @@ def _fmt_na(dim: str, reason: str) -> str:
|
|
|
153
155
|
|
|
154
156
|
|
|
155
157
|
def generic_rubric_validator(test_dir: Path, outputs: dict[str, Any]) -> tuple[list[str], list[str]]:
|
|
158
|
+
events = (outputs or {}).get("events")
|
|
159
|
+
if isinstance(events, dict):
|
|
160
|
+
events["skills_invoked"] = normalize_skill_invocations(
|
|
161
|
+
events,
|
|
162
|
+
agent=(outputs or {}).get("agent"),
|
|
163
|
+
adapter=(outputs or {}).get("agent_adapter"),
|
|
164
|
+
)
|
|
156
165
|
scored: list[tuple[str, float | None, str]] = [
|
|
157
166
|
("completion", *_score_completion(outputs)),
|
|
158
167
|
("skill_invocation", *_score_skill_invocation(outputs)),
|
|
@@ -10,6 +10,8 @@ from typing import Any
|
|
|
10
10
|
|
|
11
11
|
import yaml
|
|
12
12
|
|
|
13
|
+
from scaffold.python.validation.core import _normalize_skill_invocations
|
|
14
|
+
|
|
13
15
|
|
|
14
16
|
def _failure(name: str, reason: str) -> str:
|
|
15
17
|
return f"{name}: {reason}"
|
|
@@ -186,7 +188,10 @@ def validate_native_workflow(
|
|
|
186
188
|
passed: list[str] = []
|
|
187
189
|
failed: list[str] = []
|
|
188
190
|
|
|
189
|
-
|
|
191
|
+
events = outputs.get("events") or {}
|
|
192
|
+
invoked = _normalize_skill_invocations(events, outputs)
|
|
193
|
+
if isinstance(events, dict):
|
|
194
|
+
events["skills_invoked"] = invoked
|
|
190
195
|
unexpected_skills = sorted({skill for skill in invoked if skill != "comet-native"})
|
|
191
196
|
if "comet-native" in invoked and not unexpected_skills:
|
|
192
197
|
passed.append("native_skill_invocation")
|
|
@@ -39,6 +39,8 @@ import re
|
|
|
39
39
|
from pathlib import Path
|
|
40
40
|
from typing import Any
|
|
41
41
|
|
|
42
|
+
from scaffold.python.agents import normalize_skill_invocations
|
|
43
|
+
|
|
42
44
|
# All nine rubric dimensions, in display order.
|
|
43
45
|
RUBRIC_DIMENSIONS = (
|
|
44
46
|
"main_flow",
|
|
@@ -747,6 +749,12 @@ def comet_rubric_validator(test_dir: Path, outputs: dict) -> tuple[list[str], li
|
|
|
747
749
|
Final weighted score uses dimension weights to aggregate.
|
|
748
750
|
"""
|
|
749
751
|
events = (outputs or {}).get("events", {}) or {}
|
|
752
|
+
if isinstance(events, dict):
|
|
753
|
+
events["skills_invoked"] = normalize_skill_invocations(
|
|
754
|
+
events,
|
|
755
|
+
agent=(outputs or {}).get("agent"),
|
|
756
|
+
adapter=(outputs or {}).get("agent_adapter"),
|
|
757
|
+
)
|
|
750
758
|
business_completion = _completion_input(outputs or {}, "business_completion")
|
|
751
759
|
workflow_completion = _completion_input(outputs or {}, "workflow_completion")
|
|
752
760
|
is_control = _is_control_business_only(outputs or {})
|
|
@@ -880,7 +888,14 @@ def comet_rubric_validator(test_dir: Path, outputs: dict) -> tuple[list[str], li
|
|
|
880
888
|
try:
|
|
881
889
|
from scaffold.python.llm_judge import judge_messages
|
|
882
890
|
|
|
883
|
-
judge_results = judge_messages(
|
|
891
|
+
judge_results = judge_messages(
|
|
892
|
+
test_dir,
|
|
893
|
+
agent=outputs.get("judge_agent") or outputs.get("agent"),
|
|
894
|
+
model=outputs.get("judge_model"),
|
|
895
|
+
base_url=outputs.get("judge_base_url"),
|
|
896
|
+
excluded_credentials=tuple(outputs.get("main_credentials") or ()),
|
|
897
|
+
evidence=outputs,
|
|
898
|
+
)
|
|
884
899
|
passed.extend(judge_results)
|
|
885
900
|
if any(
|
|
886
901
|
result.startswith("[RUBRIC-JUDGE] ") and " status:" not in result
|