@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -26
- package/assets/manifest.json +4 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
- package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
- package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
- package/assets/skills/comet-build/SKILL.md +36 -62
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +2 -2
- package/assets/skills/comet-native/SKILL.md +18 -15
- package/assets/skills/comet-native/reference/artifacts.md +5 -4
- package/assets/skills/comet-native/reference/clarification.md +1 -1
- package/assets/skills/comet-native/reference/commands.md +4 -4
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills/comet-review/SKILL.md +121 -0
- package/assets/skills/comet-review/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +36 -62
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +2 -2
- package/assets/skills-zh/comet-native/SKILL.md +18 -15
- package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
- package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
- package/assets/skills-zh/comet-native/reference/commands.md +4 -4
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-review/SKILL.md +121 -0
- package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +5 -8
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +5 -6
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
- package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.js +10 -0
- package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +4 -4
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.js +44 -6
- package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.js +2 -1
- package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +39 -13
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +2 -2
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +5 -1
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +2 -0
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.js +17 -10
- package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +65 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +453 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +13 -7
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +18 -5
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +56 -4
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +179 -20
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +178 -73
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +168 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
- package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
- package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.js +24 -6
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.js +86 -35
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
- package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +27 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +108 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +87 -44
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +14 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +93 -5
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +3 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +4 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +16 -10
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +11 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +39 -29
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
- package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +4 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +174 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +13 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/fs/strip-bom.d.ts +10 -0
- package/dist/platform/fs/strip-bom.d.ts.map +1 -0
- package/dist/platform/fs/strip-bom.js +12 -0
- package/dist/platform/fs/strip-bom.js.map +1 -0
- package/dist/platform/install/platforms.d.ts +11 -0
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +49 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +1 -0
- package/dist/platform/process/external-command.d.ts.map +1 -1
- package/dist/platform/process/external-command.js +2 -1
- package/dist/platform/process/external-command.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +719 -83
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +18 -28
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
- package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
|
@@ -33,6 +33,12 @@ from scaffold.python.native_eval import (
|
|
|
33
33
|
split_comet_completion_checks as _split_comet_completion_checks,
|
|
34
34
|
)
|
|
35
35
|
from scaffold.python.profiles import resolve_profile_name, run_profile_rubric
|
|
36
|
+
from scaffold.python.agents import get_agent_adapter
|
|
37
|
+
from scaffold.python.execution import redact_sensitive
|
|
38
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
39
|
+
from scaffold.python.eval_context import resolve_eval_context
|
|
40
|
+
from scaffold.python.task_resolution import build_task_catalogue, resolve_task_set
|
|
41
|
+
from scaffold.python.paths import get_tasks_dir
|
|
36
42
|
from scaffold.python.tasks import list_tasks, load_task
|
|
37
43
|
from scaffold.python.treatments import TreatmentConfig, build_treatment_skills, load_treatments
|
|
38
44
|
from scaffold.python.validation import run_validators
|
|
@@ -43,6 +49,67 @@ PYTEST_TIMEOUT = 3000 # 50 minutes total including task-specific runtime and te
|
|
|
43
49
|
MANIFEST_DYNAMIC_ONLY_TASKS = {"workflow-overlay-contract"}
|
|
44
50
|
|
|
45
51
|
|
|
52
|
+
def _manifest_authored_tasks(config):
|
|
53
|
+
if config is None or not config.getoption("--eval-manifest"):
|
|
54
|
+
return {}
|
|
55
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
56
|
+
|
|
57
|
+
manifest = load_eval_manifest(config.getoption("--eval-manifest"))
|
|
58
|
+
return {task.name: task for task in load_manifest_tasks(manifest)}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _load_eval_task(task_name: str, config=None):
|
|
62
|
+
resolved = getattr(config, "_comet_resolved_tasks", {}) if config is not None else {}
|
|
63
|
+
if task_name in resolved:
|
|
64
|
+
return resolved[task_name]
|
|
65
|
+
authored = _manifest_authored_tasks(config)
|
|
66
|
+
if task_name in authored:
|
|
67
|
+
return authored[task_name]
|
|
68
|
+
return load_task(task_name)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _resolve_frozen_task_set(config, task_filter: str | None):
|
|
72
|
+
"""Resolve once per collection and retain the exact cross-suite task identity."""
|
|
73
|
+
frozen = getattr(config, "_comet_frozen_task_set", None)
|
|
74
|
+
if frozen is not None:
|
|
75
|
+
return frozen
|
|
76
|
+
from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
|
|
77
|
+
|
|
78
|
+
manifest_path = config.getoption("--eval-manifest")
|
|
79
|
+
context = getattr(config, "_comet_eval_context", None)
|
|
80
|
+
if manifest_path:
|
|
81
|
+
manifest = load_eval_manifest(manifest_path)
|
|
82
|
+
context = context or resolve_eval_context(
|
|
83
|
+
manifest_path=manifest.path, project_root=config.getoption("--project-root")
|
|
84
|
+
)
|
|
85
|
+
else:
|
|
86
|
+
if context is None:
|
|
87
|
+
context = resolve_eval_context(
|
|
88
|
+
skill_path=config.getoption("--skill-path"),
|
|
89
|
+
project_root=config.getoption("--project-root"),
|
|
90
|
+
)
|
|
91
|
+
manifest = SkillEvalManifest(
|
|
92
|
+
path=context.skill_root / "comet" / "eval.yaml",
|
|
93
|
+
name=context.skill_root.name,
|
|
94
|
+
description="",
|
|
95
|
+
skill_name=config.getoption("--skill-name") or context.skill_root.name,
|
|
96
|
+
skill_path=context.skill_root,
|
|
97
|
+
profile=config.getoption("--profile"),
|
|
98
|
+
)
|
|
99
|
+
frozen = resolve_task_set(
|
|
100
|
+
context,
|
|
101
|
+
manifest,
|
|
102
|
+
build_task_catalogue(manifest, get_tasks_dir()),
|
|
103
|
+
explicit_task=task_filter,
|
|
104
|
+
quick=bool(config.getoption("--quick")),
|
|
105
|
+
execution=conftest._resolve_eval_execution(config),
|
|
106
|
+
)
|
|
107
|
+
config._comet_frozen_task_set = frozen
|
|
108
|
+
config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
|
|
109
|
+
config._comet_resolution_manifest = manifest
|
|
110
|
+
return frozen
|
|
111
|
+
|
|
112
|
+
|
|
46
113
|
# =============================================================================
|
|
47
114
|
# PARAMETRIZE HELPERS
|
|
48
115
|
# =============================================================================
|
|
@@ -73,6 +140,8 @@ def expand_treatment_patterns(patterns: list[str], all_treatments: dict) -> list
|
|
|
73
140
|
def generate_test_params(task_filter: str | None, treatment_filter: str | None, config=None):
|
|
74
141
|
"""Generate (task_name, treatment_name) pairs based on filters."""
|
|
75
142
|
params = []
|
|
143
|
+
if config is not None:
|
|
144
|
+
conftest._ensure_auto_generated_manifest(config, task_filter)
|
|
76
145
|
all_treatments = load_treatments()
|
|
77
146
|
all_tasks = list_tasks()
|
|
78
147
|
dynamic = None
|
|
@@ -82,16 +151,24 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
|
|
|
82
151
|
if dynamic:
|
|
83
152
|
all_treatments[dynamic.name] = dynamic
|
|
84
153
|
manifest_tasks = None
|
|
154
|
+
authored_tasks = {}
|
|
85
155
|
manifest_baseline_treatments = []
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
156
|
+
frozen_selected = False
|
|
157
|
+
if config is not None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
|
|
158
|
+
resolved = _resolve_frozen_task_set(config, task_filter)
|
|
159
|
+
frozen_selected = True
|
|
160
|
+
manifest = config._comet_resolution_manifest
|
|
161
|
+
manifest_tasks = [item.name for item in resolved.tasks]
|
|
162
|
+
authored_tasks = {
|
|
163
|
+
item.name: item.task
|
|
164
|
+
for item in resolved.tasks
|
|
165
|
+
if item.provenance in {"inline", "source"}
|
|
166
|
+
}
|
|
91
167
|
manifest_baseline_treatments = manifest.baseline_treatments
|
|
92
168
|
|
|
93
|
-
if task_filter and task_filter not in all_tasks:
|
|
94
|
-
|
|
169
|
+
if task_filter and task_filter not in set(all_tasks) | set(authored_tasks):
|
|
170
|
+
available = sorted(set(all_tasks) | set(authored_tasks))
|
|
171
|
+
raise ValueError(f"Task not found: {task_filter}. Available: {available}")
|
|
95
172
|
|
|
96
173
|
treatment_list = []
|
|
97
174
|
if treatment_filter:
|
|
@@ -106,10 +183,18 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
|
|
|
106
183
|
elif dynamic:
|
|
107
184
|
treatment_list = [dynamic.name]
|
|
108
185
|
|
|
109
|
-
|
|
186
|
+
if task_filter:
|
|
187
|
+
tasks_to_run = [task_filter]
|
|
188
|
+
elif frozen_selected:
|
|
189
|
+
tasks_to_run = list(manifest_tasks or [])
|
|
190
|
+
elif authored_tasks or manifest_tasks:
|
|
191
|
+
tasks_to_run = list(manifest_tasks or authored_tasks)
|
|
192
|
+
else:
|
|
193
|
+
tasks_to_run = all_tasks
|
|
194
|
+
tasks_to_run = list(dict.fromkeys(tasks_to_run))
|
|
110
195
|
|
|
111
196
|
for task_name in tasks_to_run:
|
|
112
|
-
task =
|
|
197
|
+
task = _load_eval_task(task_name, config)
|
|
113
198
|
task_treatments = treatment_list
|
|
114
199
|
if (
|
|
115
200
|
dynamic
|
|
@@ -193,6 +278,46 @@ interaction:
|
|
|
193
278
|
]
|
|
194
279
|
|
|
195
280
|
|
|
281
|
+
def test_eval_manifest_prefers_authored_tasks_over_recommended_tasks(tmp_path):
|
|
282
|
+
package = tmp_path / "manifest-skill"
|
|
283
|
+
package.mkdir()
|
|
284
|
+
(package / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
|
|
285
|
+
comet_dir = package / "comet"
|
|
286
|
+
comet_dir.mkdir()
|
|
287
|
+
manifest = comet_dir / "eval.yaml"
|
|
288
|
+
manifest.write_text(
|
|
289
|
+
"\n".join(
|
|
290
|
+
[
|
|
291
|
+
"apiVersion: comet.eval/v1alpha1",
|
|
292
|
+
"kind: SkillEvalManifest",
|
|
293
|
+
"metadata:",
|
|
294
|
+
" name: manifest-skill",
|
|
295
|
+
"skill:",
|
|
296
|
+
" name: manifest-skill",
|
|
297
|
+
" source: ..",
|
|
298
|
+
"evaluation:",
|
|
299
|
+
" tasks:",
|
|
300
|
+
" - name: inline-task",
|
|
301
|
+
" prompt: Create result.md.",
|
|
302
|
+
" expect:",
|
|
303
|
+
" files: [result.md]",
|
|
304
|
+
" recommendedTasks:",
|
|
305
|
+
" - generic-skill-smoke",
|
|
306
|
+
]
|
|
307
|
+
),
|
|
308
|
+
encoding="utf-8",
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
class Config:
|
|
312
|
+
def getoption(self, name):
|
|
313
|
+
return {"--eval-manifest": str(manifest)}.get(name)
|
|
314
|
+
|
|
315
|
+
params = generate_test_params(None, None, Config())
|
|
316
|
+
|
|
317
|
+
assert [task_name for task_name, _ in params] == ["inline-task"]
|
|
318
|
+
assert all(treatment == "DYNAMIC_SKILL" for _, treatment in params)
|
|
319
|
+
|
|
320
|
+
|
|
196
321
|
def test_control_comet_workflow_filters_workflow_only_checks():
|
|
197
322
|
passed, failed = _filter_control_workflow_checks(
|
|
198
323
|
"comet-workflow",
|
|
@@ -310,7 +435,7 @@ def pytest_generate_tests(metafunc):
|
|
|
310
435
|
def test_task_treatment(task_name, treatment_name):
|
|
311
436
|
"""Run a task with a treatment and validate results."""
|
|
312
437
|
fixtures = get_fixtures()
|
|
313
|
-
task =
|
|
438
|
+
task = _load_eval_task(task_name, fixtures.request_config)
|
|
314
439
|
treatments = load_treatments()
|
|
315
440
|
dynamic = conftest._get_dynamic_treatment_config(fixtures.request_config)
|
|
316
441
|
if dynamic:
|
|
@@ -361,8 +486,29 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
361
486
|
skills=treatment.skills,
|
|
362
487
|
claude_md=conftest._build_eval_claude_md(profile_name, treatment.claude_md),
|
|
363
488
|
environment_dir=task.environment_dir,
|
|
489
|
+
workspace_dir=task.workspace_dir,
|
|
364
490
|
)
|
|
365
|
-
|
|
491
|
+
selected_agent = conftest._resolve_eval_agent(fixtures.request_config).agent
|
|
492
|
+
selected_adapter = get_agent_adapter(selected_agent)
|
|
493
|
+
execution = conftest._resolve_eval_execution(fixtures.request_config)
|
|
494
|
+
selected_model = execution.model
|
|
495
|
+
judge = conftest._resolve_eval_judge(fixtures.request_config)
|
|
496
|
+
role_models = {
|
|
497
|
+
"subject": selected_model,
|
|
498
|
+
"simulator": (
|
|
499
|
+
selected_model
|
|
500
|
+
if interaction.mode == "auto_user"
|
|
501
|
+
and not interaction.decision_reply
|
|
502
|
+
and not interaction.decision_replies
|
|
503
|
+
else None
|
|
504
|
+
),
|
|
505
|
+
"judge": judge.model if judge is not None else None,
|
|
506
|
+
}
|
|
507
|
+
role_agents = {
|
|
508
|
+
"subject": selected_agent,
|
|
509
|
+
"simulator": selected_agent if interaction.mode == "auto_user" else None,
|
|
510
|
+
"judge": judge.agent if judge is not None else None,
|
|
511
|
+
}
|
|
366
512
|
captured_execution = conftest._capture_execution_identity(
|
|
367
513
|
fixtures.test_dir,
|
|
368
514
|
model=selected_model,
|
|
@@ -373,6 +519,11 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
373
519
|
task.name,
|
|
374
520
|
task.path.parent,
|
|
375
521
|
execution_identity=captured_execution.report_identity,
|
|
522
|
+
manifest_path=task.manifest_path,
|
|
523
|
+
rendered_prompt=prompt if task.manifest_path else None,
|
|
524
|
+
workspace_dir=task.workspace_dir if task.manifest_path else None,
|
|
525
|
+
validation_rules=task.manifest_expectations if task.manifest_path else None,
|
|
526
|
+
environment_dir=task.environment_dir if task.manifest_path else None,
|
|
376
527
|
)
|
|
377
528
|
)
|
|
378
529
|
skill_package_path = conftest._snapshot_dynamic_skill_package(
|
|
@@ -387,16 +538,31 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
387
538
|
image_id=captured_execution.runtime_image_id,
|
|
388
539
|
)
|
|
389
540
|
|
|
390
|
-
events = extract_events(parse_output(result.stdout))
|
|
541
|
+
events = extract_events(parse_output(result.stdout), agent=selected_agent)
|
|
542
|
+
events["telemetry_status"] = "N/A" if not selected_adapter.supports_telemetry else "available"
|
|
391
543
|
loop_interaction = conftest._extract_loop_interaction(result.stderr)
|
|
392
|
-
|
|
544
|
+
for role, session_ids in loop_interaction.get("role_sessions", {}).items():
|
|
545
|
+
target = events["role_sessions"].setdefault(role, [])
|
|
546
|
+
for session_id in session_ids:
|
|
547
|
+
if session_id not in target:
|
|
548
|
+
target.append(session_id)
|
|
549
|
+
subject_turns = conftest._extract_subject_turn_evidence(redact_sensitive(result.stdout))
|
|
393
550
|
outputs = {
|
|
394
551
|
"run_id": run_id,
|
|
395
552
|
"treatment_name": treatment_name,
|
|
553
|
+
"agent": selected_agent,
|
|
554
|
+
"agent_adapter": selected_adapter,
|
|
555
|
+
"main_credentials": selected_adapter.required_credentials,
|
|
556
|
+
"role_models": role_models,
|
|
557
|
+
"role_agents": role_agents,
|
|
558
|
+
"telemetry_status": events["telemetry_status"],
|
|
559
|
+
"judge_agent": judge.agent if judge is not None else None,
|
|
560
|
+
"judge_model": judge.model if judge is not None else None,
|
|
561
|
+
"judge_base_url": judge.base_url if judge is not None else None,
|
|
396
562
|
"events": events,
|
|
397
563
|
"profile": profile_name,
|
|
398
564
|
"skill_sources": skill_sources,
|
|
399
|
-
"eval_manifest": eval_manifest,
|
|
565
|
+
"eval_manifest": eval_manifest or skill_hints.get("manifest"),
|
|
400
566
|
"required_skills": skill_hints.get("required_skills")
|
|
401
567
|
or task.config.evaluation.required_skills,
|
|
402
568
|
"expected_artifacts": skill_hints.get("expected_artifacts")
|
|
@@ -416,6 +582,19 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
416
582
|
"required_output_schemas": skill_hints.get("required_output_schemas") or [],
|
|
417
583
|
"expected_evidence": skill_hints.get("expected_evidence") or [],
|
|
418
584
|
"draft_hash": skill_hints.get("draft_hash"),
|
|
585
|
+
"eval_generation": (
|
|
586
|
+
{
|
|
587
|
+
"generation_hash": skill_hints.get("generation_hash"),
|
|
588
|
+
"manifest_path": skill_hints.get("manifest")
|
|
589
|
+
or str(fixtures.request_config.getoption("--eval-manifest")),
|
|
590
|
+
"metadata_path": skill_hints.get("generation_metadata_path"),
|
|
591
|
+
"manifest_hash": skill_hints.get("generation_manifest_hash"),
|
|
592
|
+
"metadata_hash": skill_hints.get("generation_metadata_hash"),
|
|
593
|
+
"overhead": skill_hints.get("generation_overhead"),
|
|
594
|
+
}
|
|
595
|
+
if skill_hints.get("generation_hash")
|
|
596
|
+
else None
|
|
597
|
+
),
|
|
419
598
|
"interaction": {
|
|
420
599
|
"mode": interaction.mode,
|
|
421
600
|
"max_turns": interaction.max_turns,
|
|
@@ -423,12 +602,18 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
423
602
|
"subject_turns": subject_turns,
|
|
424
603
|
},
|
|
425
604
|
"case_manifest": case_manifest,
|
|
605
|
+
"role_sessions": events["role_sessions"],
|
|
426
606
|
}
|
|
427
607
|
events["profile"] = outputs["profile"]
|
|
608
|
+
events["agent"] = selected_agent
|
|
609
|
+
events["role_models"] = role_models
|
|
610
|
+
events["role_agents"] = role_agents
|
|
611
|
+
events["telemetry_status"] = outputs["telemetry_status"]
|
|
428
612
|
events["skill_sources"] = outputs["skill_sources"]
|
|
429
613
|
events["eval_manifest"] = outputs["eval_manifest"]
|
|
430
614
|
events["interaction"] = outputs["interaction"]
|
|
431
615
|
events["case_manifest"] = case_manifest
|
|
616
|
+
events["eval_generation"] = outputs["eval_generation"]
|
|
432
617
|
|
|
433
618
|
passed, failed = run_validators(validators, fixtures.test_dir, outputs)
|
|
434
619
|
passed, failed = adapt_checks_for_native(
|
|
@@ -459,6 +644,21 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
459
644
|
)
|
|
460
645
|
passed = passed + rubric_passed
|
|
461
646
|
failed = failed + rubric_failed
|
|
647
|
+
agent_failed = result.returncode != 0
|
|
648
|
+
if agent_failed:
|
|
649
|
+
failed.append(f"Agent execution failed with exit code {result.returncode}")
|
|
650
|
+
|
|
651
|
+
# Keep the run-level context available to thin remote reporting wrappers.
|
|
652
|
+
# These fields are local evidence as well; Langfuse applies its own bounded
|
|
653
|
+
# capture policy before sending them remotely.
|
|
654
|
+
events["task"] = task.name
|
|
655
|
+
events["treatment"] = treatment_name
|
|
656
|
+
events["prompt"] = prompt
|
|
657
|
+
events["skill"] = skill_hints.get("name") or skill_hints.get("path")
|
|
658
|
+
events["final_response"] = subject_turns[-1].get("result") if subject_turns else None
|
|
659
|
+
events["quality_gates"] = outputs["quality_gates"]
|
|
660
|
+
events["execution_identity"] = captured_execution.report_identity
|
|
661
|
+
events["role_sessions"] = outputs["role_sessions"]
|
|
462
662
|
|
|
463
663
|
fixtures.record_result(
|
|
464
664
|
events,
|
|
@@ -470,5 +670,7 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
470
670
|
stderr=result.stderr,
|
|
471
671
|
)
|
|
472
672
|
|
|
673
|
+
if agent_failed:
|
|
674
|
+
pytest.fail(f"{selected_agent} execution failed with exit code {result.returncode}")
|
|
473
675
|
if failed and not _is_observational_baseline_run(treatment_name):
|
|
474
676
|
pytest.fail(f"Validation failed: {failed}")
|
package/eval/pyproject.toml
CHANGED
|
@@ -4,6 +4,7 @@ version = "0.1.0"
|
|
|
4
4
|
description = "Benchmark evaluation for Comet skill (OpenSpec + Superpowers workflow)"
|
|
5
5
|
requires-python = ">=3.11"
|
|
6
6
|
dependencies = [
|
|
7
|
+
"jsonschema>=4.23",
|
|
7
8
|
"pytest>=8.0",
|
|
8
9
|
"pytest-timeout>=2.2",
|
|
9
10
|
"pytest-xdist>=3.5",
|
|
@@ -23,6 +24,9 @@ langsmith = [
|
|
|
23
24
|
"langchain>=0.3",
|
|
24
25
|
"langchain-openai>=0.2",
|
|
25
26
|
]
|
|
27
|
+
langfuse = [
|
|
28
|
+
"langfuse>=4.0,<5",
|
|
29
|
+
]
|
|
26
30
|
dev = [
|
|
27
31
|
"ruff>=0.4",
|
|
28
32
|
"pre-commit>=3.7",
|
|
@@ -34,6 +38,7 @@ testpaths = ["local/tests"]
|
|
|
34
38
|
pythonpath = ["."]
|
|
35
39
|
markers = [
|
|
36
40
|
"langsmith: marks test as using LangSmith tracing",
|
|
41
|
+
"langfuse: marks test as using Langfuse tracing",
|
|
37
42
|
]
|
|
38
43
|
|
|
39
44
|
[tool.ruff]
|
|
@@ -18,6 +18,9 @@ from .python import (
|
|
|
18
18
|
compose_validators,
|
|
19
19
|
make_execution_validator,
|
|
20
20
|
run_claude_in_docker,
|
|
21
|
+
run_agent_in_docker,
|
|
22
|
+
run_agent_loop_in_docker,
|
|
23
|
+
run_command_in_docker,
|
|
21
24
|
run_eval_in_docker,
|
|
22
25
|
run_node_in_docker,
|
|
23
26
|
run_python_in_docker,
|
|
@@ -64,4 +67,7 @@ __all__ = [
|
|
|
64
67
|
"run_eval_in_docker",
|
|
65
68
|
"run_node_in_docker",
|
|
66
69
|
"run_claude_in_docker",
|
|
70
|
+
"run_agent_in_docker",
|
|
71
|
+
"run_agent_loop_in_docker",
|
|
72
|
+
"run_command_in_docker",
|
|
67
73
|
]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
ARG BASE_IMAGE
|
|
2
|
+
FROM ${BASE_IMAGE}
|
|
3
|
+
|
|
4
|
+
USER root
|
|
5
|
+
ARG EVAL_AGENT=claude-code
|
|
6
|
+
ARG CODEX_VERSION=latest
|
|
7
|
+
ARG QODER_VERSION=latest
|
|
8
|
+
ARG CODEBUDDY_VERSION=latest
|
|
9
|
+
ARG CUSTOM_AGENT_ID=
|
|
10
|
+
ARG CUSTOM_AGENT_EXECUTABLE=
|
|
11
|
+
ARG CUSTOM_INSTALL_KIND=none
|
|
12
|
+
ARG CUSTOM_INSTALL_PACKAGE=
|
|
13
|
+
ARG CUSTOM_INSTALL_VERSION=latest
|
|
14
|
+
ARG LANGFUSE_ENABLED=false
|
|
15
|
+
|
|
16
|
+
RUN if [ "$EVAL_AGENT" = "codex" ]; then \
|
|
17
|
+
npm install -g @openai/codex@${CODEX_VERSION}; \
|
|
18
|
+
elif [ "$EVAL_AGENT" = "qoder" ]; then \
|
|
19
|
+
npm install -g @qoder-ai/qodercli@${QODER_VERSION}; \
|
|
20
|
+
elif [ "$EVAL_AGENT" = "codebuddy" ]; then \
|
|
21
|
+
npm install -g @tencent-ai/codebuddy-code@${CODEBUDDY_VERSION}; \
|
|
22
|
+
elif [ -n "$CUSTOM_AGENT_ID" ] && [ "$EVAL_AGENT" = "$CUSTOM_AGENT_ID" ] && [ "$CUSTOM_INSTALL_KIND" = "npm" ]; then \
|
|
23
|
+
npm install -g "${CUSTOM_INSTALL_PACKAGE}@${CUSTOM_INSTALL_VERSION}"; \
|
|
24
|
+
elif [ -n "$CUSTOM_AGENT_ID" ] && [ "$EVAL_AGENT" = "$CUSTOM_AGENT_ID" ] && [ "$CUSTOM_INSTALL_KIND" = "pip" ]; then \
|
|
25
|
+
if [ "$CUSTOM_INSTALL_VERSION" = "latest" ]; then \
|
|
26
|
+
python3 -m pip install --break-system-packages "${CUSTOM_INSTALL_PACKAGE}"; \
|
|
27
|
+
else \
|
|
28
|
+
python3 -m pip install --break-system-packages "${CUSTOM_INSTALL_PACKAGE}==${CUSTOM_INSTALL_VERSION}"; \
|
|
29
|
+
fi; \
|
|
30
|
+
fi
|
|
31
|
+
|
|
32
|
+
# The official Claude hook can use the preinstalled Python SDK, while the
|
|
33
|
+
# official Codex plugin ships its Node bundle. Install the SDK only for an
|
|
34
|
+
# explicit Langfuse suite so Local/LangSmith images stay unchanged.
|
|
35
|
+
RUN if [ "$LANGFUSE_ENABLED" = "true" ]; then \
|
|
36
|
+
apt-get update && apt-get install -y --no-install-recommends python3 python3-pip && \
|
|
37
|
+
python3 -m pip install --break-system-packages --no-cache-dir 'langfuse>=4.0,<5' && \
|
|
38
|
+
rm -rf /var/lib/apt/lists/*; \
|
|
39
|
+
fi
|
|
40
|
+
|
|
41
|
+
USER agent
|
|
@@ -14,7 +14,18 @@ from .logging import (
|
|
|
14
14
|
save_report,
|
|
15
15
|
strip_ansi,
|
|
16
16
|
)
|
|
17
|
-
from .manifests import SkillEvalManifest, load_eval_manifest
|
|
17
|
+
from .manifests import ManifestTask, SkillEvalManifest, load_eval_manifest
|
|
18
|
+
from .eval_context import (
|
|
19
|
+
EvalContextError,
|
|
20
|
+
ResolvedEvalContext,
|
|
21
|
+
artifact_root_for_owner,
|
|
22
|
+
assert_artifact_root_is_safe,
|
|
23
|
+
context_from_environment,
|
|
24
|
+
managed_path_for_owner,
|
|
25
|
+
resolve_managed_path,
|
|
26
|
+
resolve_eval_context,
|
|
27
|
+
)
|
|
28
|
+
from .manifest_tasks import load_manifest_tasks
|
|
18
29
|
from .profiles import (
|
|
19
30
|
AUTHORING_SKILL_PROFILE,
|
|
20
31
|
COMET_WORKFLOW_PROFILE,
|
|
@@ -28,6 +39,26 @@ from .profiles import (
|
|
|
28
39
|
)
|
|
29
40
|
from .evidence import EvalArtifactReference, build_eval_artifact_references
|
|
30
41
|
from .attribution import FailureAttribution, classify_failures
|
|
42
|
+
from .agents import (
|
|
43
|
+
AGENT_IDS,
|
|
44
|
+
AgentAdapter,
|
|
45
|
+
AgentSelection,
|
|
46
|
+
get_agent_adapter,
|
|
47
|
+
resolve_agent,
|
|
48
|
+
validate_agent_capabilities,
|
|
49
|
+
)
|
|
50
|
+
from .custom_agents import CustomAgentSpec, discover_custom_agent_specs
|
|
51
|
+
from .execution import (
|
|
52
|
+
ResolvedExecution,
|
|
53
|
+
ResolvedJudge,
|
|
54
|
+
api_identity,
|
|
55
|
+
build_agent_environment,
|
|
56
|
+
build_judge_environment,
|
|
57
|
+
preflight_credentials,
|
|
58
|
+
redact_sensitive,
|
|
59
|
+
resolve_execution,
|
|
60
|
+
resolve_judge,
|
|
61
|
+
)
|
|
31
62
|
from .sample_quality import (
|
|
32
63
|
SampleQuality,
|
|
33
64
|
include_in_analysis,
|
|
@@ -54,6 +85,9 @@ from .utils import (
|
|
|
54
85
|
read_json_file,
|
|
55
86
|
retry_with_backoff,
|
|
56
87
|
run_claude_in_docker,
|
|
88
|
+
run_agent_in_docker,
|
|
89
|
+
run_agent_loop_in_docker,
|
|
90
|
+
run_command_in_docker,
|
|
57
91
|
run_eval_in_docker,
|
|
58
92
|
run_node_in_docker,
|
|
59
93
|
run_python_in_docker,
|
|
@@ -73,23 +107,97 @@ from .validation import (
|
|
|
73
107
|
)
|
|
74
108
|
|
|
75
109
|
__all__ = [
|
|
76
|
-
"NoiseTask",
|
|
77
|
-
"
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
92
|
-
"
|
|
93
|
-
"
|
|
94
|
-
"
|
|
110
|
+
"NoiseTask",
|
|
111
|
+
"Treatment",
|
|
112
|
+
"ValidatorFn",
|
|
113
|
+
"compose_validators",
|
|
114
|
+
"run_validators",
|
|
115
|
+
"check_file_exists",
|
|
116
|
+
"check_pattern",
|
|
117
|
+
"check_no_pattern",
|
|
118
|
+
"check_skill_invoked",
|
|
119
|
+
"check_code_execution",
|
|
120
|
+
"check_python_execution",
|
|
121
|
+
"check_typescript_execution",
|
|
122
|
+
"run_shell",
|
|
123
|
+
"check_docker_available",
|
|
124
|
+
"check_claude_available",
|
|
125
|
+
"build_docker_image",
|
|
126
|
+
"run_python_in_docker",
|
|
127
|
+
"make_execution_validator",
|
|
128
|
+
"run_eval_in_docker",
|
|
129
|
+
"run_node_in_docker",
|
|
130
|
+
"run_claude_in_docker",
|
|
131
|
+
"retry_with_backoff",
|
|
132
|
+
"read_json_file",
|
|
133
|
+
"run_agent_in_docker",
|
|
134
|
+
"run_agent_loop_in_docker",
|
|
135
|
+
"run_command_in_docker",
|
|
136
|
+
"get_field",
|
|
137
|
+
"get_nested_field",
|
|
138
|
+
"normalize_score",
|
|
139
|
+
"parse_output",
|
|
140
|
+
"extract_events",
|
|
141
|
+
"strip_ansi",
|
|
142
|
+
"ExperimentLogger",
|
|
143
|
+
"TreatmentResult",
|
|
144
|
+
"save_events",
|
|
145
|
+
"save_raw",
|
|
146
|
+
"save_report",
|
|
147
|
+
"ReportColumn",
|
|
148
|
+
"bool_column",
|
|
149
|
+
"quality_column",
|
|
150
|
+
"default_columns",
|
|
151
|
+
"EvalArtifactReference",
|
|
152
|
+
"build_eval_artifact_references",
|
|
153
|
+
"FailureAttribution",
|
|
154
|
+
"classify_failures",
|
|
155
|
+
"AGENT_IDS",
|
|
156
|
+
"AgentAdapter",
|
|
157
|
+
"AgentSelection",
|
|
158
|
+
"get_agent_adapter",
|
|
159
|
+
"resolve_agent",
|
|
160
|
+
"validate_agent_capabilities",
|
|
161
|
+
"CustomAgentSpec",
|
|
162
|
+
"discover_custom_agent_specs",
|
|
163
|
+
"ResolvedExecution",
|
|
164
|
+
"ResolvedJudge",
|
|
165
|
+
"resolve_execution",
|
|
166
|
+
"resolve_judge",
|
|
167
|
+
"build_agent_environment",
|
|
168
|
+
"build_judge_environment",
|
|
169
|
+
"preflight_credentials",
|
|
170
|
+
"redact_sensitive",
|
|
171
|
+
"api_identity",
|
|
172
|
+
"ManifestTask",
|
|
173
|
+
"SkillEvalManifest",
|
|
174
|
+
"load_eval_manifest",
|
|
175
|
+
"EvalContextError",
|
|
176
|
+
"ResolvedEvalContext",
|
|
177
|
+
"artifact_root_for_owner",
|
|
178
|
+
"assert_artifact_root_is_safe",
|
|
179
|
+
"context_from_environment",
|
|
180
|
+
"managed_path_for_owner",
|
|
181
|
+
"resolve_managed_path",
|
|
182
|
+
"resolve_eval_context",
|
|
183
|
+
"load_manifest_tasks",
|
|
184
|
+
"AUTHORING_SKILL_PROFILE",
|
|
185
|
+
"COMET_WORKFLOW_PROFILE",
|
|
186
|
+
"GENERIC_PROFILE",
|
|
187
|
+
"ProfileSpec",
|
|
188
|
+
"all_rubric_dimensions",
|
|
189
|
+
"get_profile",
|
|
190
|
+
"list_profiles",
|
|
191
|
+
"resolve_profile_name",
|
|
192
|
+
"run_profile_rubric",
|
|
193
|
+
"ReportOutputConfig",
|
|
194
|
+
"load_report_output_config",
|
|
195
|
+
"preferred_report_path",
|
|
196
|
+
"render_markdown_html",
|
|
197
|
+
"write_report_outputs",
|
|
198
|
+
"SampleQuality",
|
|
199
|
+
"infer_sample_quality",
|
|
200
|
+
"quality_from_report",
|
|
201
|
+
"sample_quality_dict",
|
|
202
|
+
"include_in_analysis",
|
|
95
203
|
]
|