@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -26
- package/assets/manifest.json +4 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
- package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
- package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
- package/assets/skills/comet-build/SKILL.md +36 -62
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +2 -2
- package/assets/skills/comet-native/SKILL.md +18 -15
- package/assets/skills/comet-native/reference/artifacts.md +5 -4
- package/assets/skills/comet-native/reference/clarification.md +1 -1
- package/assets/skills/comet-native/reference/commands.md +4 -4
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills/comet-review/SKILL.md +121 -0
- package/assets/skills/comet-review/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +36 -62
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +2 -2
- package/assets/skills-zh/comet-native/SKILL.md +18 -15
- package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
- package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
- package/assets/skills-zh/comet-native/reference/commands.md +4 -4
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-review/SKILL.md +121 -0
- package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +5 -8
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +5 -6
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
- package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.js +10 -0
- package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +4 -4
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.js +44 -6
- package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.js +2 -1
- package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +39 -13
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +2 -2
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +5 -1
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +2 -0
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.js +17 -10
- package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +65 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +453 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +13 -7
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +18 -5
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +56 -4
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +179 -20
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +178 -73
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +168 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
- package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
- package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.js +24 -6
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.js +86 -35
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
- package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +27 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +108 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +87 -44
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +14 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +93 -5
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +3 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +4 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +16 -10
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +11 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +39 -29
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
- package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +4 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +174 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +13 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/fs/strip-bom.d.ts +10 -0
- package/dist/platform/fs/strip-bom.d.ts.map +1 -0
- package/dist/platform/fs/strip-bom.js +12 -0
- package/dist/platform/fs/strip-bom.js.map +1 -0
- package/dist/platform/install/platforms.d.ts +11 -0
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +49 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +1 -0
- package/dist/platform/process/external-command.d.ts.map +1 -1
- package/dist/platform/process/external-command.js +2 -1
- package/dist/platform/process/external-command.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +719 -83
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +18 -28
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
- package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
|
@@ -6,8 +6,8 @@ genuinely explore alternatives, or did it just happen to contain the word
|
|
|
6
6
|
"risk"? This module asks a judge LLM to read the actual artifacts and score
|
|
7
7
|
three dimensions on a 0.00–1.00 scale with a cited reason.
|
|
8
8
|
|
|
9
|
-
Runs on the host (not in Docker) via the
|
|
10
|
-
|
|
9
|
+
Runs on the host (not in Docker) via the selected agent CLI and its configured
|
|
10
|
+
provider, so it adds no new dependency. Scores are emitted in the same
|
|
11
11
|
``[RUBRIC-JUDGE] dim: score - reason`` format the logging layer understands.
|
|
12
12
|
|
|
13
13
|
Usage from the rubric validator or rescore tool::
|
|
@@ -27,6 +27,7 @@ from scaffold.python.judge_config import build_judge_invocation, run_judge_promp
|
|
|
27
27
|
# module is imported outside pytest (e.g. by the rescore tool).
|
|
28
28
|
try:
|
|
29
29
|
from dotenv import load_dotenv
|
|
30
|
+
|
|
30
31
|
load_dotenv(Path(__file__).resolve().parents[3] / ".env")
|
|
31
32
|
except Exception:
|
|
32
33
|
pass
|
|
@@ -141,12 +142,36 @@ Cite specific content from the artifacts in your reason (<=25 words each).
|
|
|
141
142
|
"""
|
|
142
143
|
|
|
143
144
|
|
|
144
|
-
def _run_judge(
|
|
145
|
+
def _run_judge(
|
|
146
|
+
prompt: str,
|
|
147
|
+
timeout: int = 120,
|
|
148
|
+
agent: object | None = None,
|
|
149
|
+
model: object | None = None,
|
|
150
|
+
base_url: object | None = None,
|
|
151
|
+
evidence: dict | None = None,
|
|
152
|
+
excluded_credentials: tuple[str, ...] = (),
|
|
153
|
+
) -> str:
|
|
145
154
|
"""Call the judge LLM through the configured judge provider."""
|
|
146
|
-
return run_judge_prompt(
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
155
|
+
return run_judge_prompt(
|
|
156
|
+
prompt,
|
|
157
|
+
timeout=timeout,
|
|
158
|
+
agent=agent,
|
|
159
|
+
model=model,
|
|
160
|
+
base_url=base_url,
|
|
161
|
+
evidence=evidence,
|
|
162
|
+
excluded_credentials=excluded_credentials,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def judge_artifacts(
|
|
167
|
+
test_dir: Path,
|
|
168
|
+
timeout: int = 120,
|
|
169
|
+
agent: object | None = None,
|
|
170
|
+
model: object | None = None,
|
|
171
|
+
base_url: object | None = None,
|
|
172
|
+
evidence: dict | None = None,
|
|
173
|
+
excluded_credentials: tuple[str, ...] = (),
|
|
174
|
+
) -> dict[str, tuple[float, str]]:
|
|
150
175
|
"""Score the three qualitative dimensions via an LLM judge.
|
|
151
176
|
|
|
152
177
|
Returns ``{dim: (score, reason)}``. On any failure (model error, no
|
|
@@ -157,7 +182,15 @@ def judge_artifacts(test_dir: Path, timeout: int = 120) -> dict[str, tuple[float
|
|
|
157
182
|
return {}
|
|
158
183
|
|
|
159
184
|
prompt = _build_judge_prompt(test_dir)
|
|
160
|
-
raw = _run_judge(
|
|
185
|
+
raw = _run_judge(
|
|
186
|
+
prompt,
|
|
187
|
+
timeout=timeout,
|
|
188
|
+
agent=agent,
|
|
189
|
+
model=model,
|
|
190
|
+
base_url=base_url,
|
|
191
|
+
evidence=evidence,
|
|
192
|
+
excluded_credentials=excluded_credentials,
|
|
193
|
+
)
|
|
161
194
|
|
|
162
195
|
scores: dict[str, tuple[float, str]] = {}
|
|
163
196
|
for line in raw.splitlines():
|
|
@@ -175,14 +208,38 @@ def judge_artifacts(test_dir: Path, timeout: int = 120) -> dict[str, tuple[float
|
|
|
175
208
|
return scores
|
|
176
209
|
|
|
177
210
|
|
|
178
|
-
def judge_messages(
|
|
211
|
+
def judge_messages(
|
|
212
|
+
test_dir: Path,
|
|
213
|
+
timeout: int = 120,
|
|
214
|
+
agent: object | None = None,
|
|
215
|
+
model: object | None = None,
|
|
216
|
+
base_url: object | None = None,
|
|
217
|
+
evidence: dict | None = None,
|
|
218
|
+
excluded_credentials: tuple[str, ...] = (),
|
|
219
|
+
) -> list[str]:
|
|
179
220
|
"""Convenience wrapper returning ``[RUBRIC-JUDGE]`` check messages."""
|
|
180
221
|
out: list[str] = []
|
|
181
222
|
try:
|
|
182
|
-
build_judge_invocation(
|
|
223
|
+
build_judge_invocation(
|
|
224
|
+
agent=agent,
|
|
225
|
+
model=model,
|
|
226
|
+
base_url=base_url,
|
|
227
|
+
excluded_credentials=excluded_credentials,
|
|
228
|
+
)
|
|
183
229
|
except ValueError as e:
|
|
184
230
|
return [f"[RUBRIC-JUDGE] status: skipped - {e}"]
|
|
185
231
|
|
|
186
|
-
|
|
232
|
+
scores = judge_artifacts(
|
|
233
|
+
test_dir,
|
|
234
|
+
timeout=timeout,
|
|
235
|
+
agent=agent,
|
|
236
|
+
model=model,
|
|
237
|
+
base_url=base_url,
|
|
238
|
+
excluded_credentials=excluded_credentials,
|
|
239
|
+
evidence=evidence,
|
|
240
|
+
)
|
|
241
|
+
if not scores:
|
|
242
|
+
return ["[RUBRIC-JUDGE] status: unavailable - no valid Judge scores"]
|
|
243
|
+
for dim, (score, reason) in scores.items():
|
|
187
244
|
out.append(f"[RUBRIC-JUDGE] {dim}: {score:.2f} - {reason}")
|
|
188
245
|
return out
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
"""Output parsing, event extraction, and experiment logging for
|
|
1
|
+
"""Output parsing, event extraction, and experiment logging for agent CLIs."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import math
|
|
5
|
+
import os
|
|
5
6
|
import re
|
|
7
|
+
import tempfile
|
|
6
8
|
from collections.abc import Callable
|
|
7
9
|
from dataclasses import dataclass, field
|
|
8
10
|
from datetime import datetime
|
|
@@ -10,13 +12,39 @@ from pathlib import Path
|
|
|
10
12
|
from typing import Any
|
|
11
13
|
|
|
12
14
|
from scaffold.python.evidence import stable_treatment_name
|
|
13
|
-
from scaffold.python.
|
|
15
|
+
from scaffold.python.execution import redact_sensitive
|
|
16
|
+
from scaffold.python.paths import get_runs_dir
|
|
14
17
|
from scaffold.python.report_outputs import (
|
|
15
18
|
ReportOutputConfig,
|
|
16
19
|
preferred_report_path,
|
|
17
20
|
write_report_outputs,
|
|
18
21
|
)
|
|
19
22
|
|
|
23
|
+
|
|
24
|
+
def _without_credentials(value: Any) -> Any:
|
|
25
|
+
if isinstance(value, dict):
|
|
26
|
+
return {
|
|
27
|
+
key: _without_credentials(item)
|
|
28
|
+
for key, item in value.items()
|
|
29
|
+
if not re.search(r"(?:api[_-]?key|auth[_-]?token|password|secret|credential)", key, re.I)
|
|
30
|
+
}
|
|
31
|
+
if isinstance(value, list):
|
|
32
|
+
return [_without_credentials(item) for item in value]
|
|
33
|
+
return value
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _atomic_write_json(path: Path, value: dict[str, Any]) -> None:
|
|
37
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
with tempfile.NamedTemporaryFile(
|
|
39
|
+
mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False
|
|
40
|
+
) as temporary:
|
|
41
|
+
temporary.write(json.dumps(redact_sensitive(_without_credentials(value)), indent=2) + "\n")
|
|
42
|
+
temporary_path = Path(temporary.name)
|
|
43
|
+
try:
|
|
44
|
+
os.replace(temporary_path, path)
|
|
45
|
+
finally:
|
|
46
|
+
temporary_path.unlink(missing_ok=True)
|
|
47
|
+
|
|
20
48
|
# Regex to strip ANSI escape codes
|
|
21
49
|
ANSI_ESCAPE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
|
|
22
50
|
|
|
@@ -35,10 +63,22 @@ def strip_ansi(text: str) -> str:
|
|
|
35
63
|
return ANSI_ESCAPE.sub("", text)
|
|
36
64
|
|
|
37
65
|
|
|
38
|
-
def _record_skill_invocation(events: dict[str, Any], skill: str) -> None:
|
|
66
|
+
def _record_skill_invocation(events: dict[str, Any], skill: str, *, explicit: bool = False) -> None:
|
|
67
|
+
if not isinstance(skill, str) or not skill.strip():
|
|
68
|
+
return
|
|
39
69
|
normalized = _SKILL_ALIASES.get(skill, skill)
|
|
40
70
|
if normalized not in events["skills_invoked"]:
|
|
41
71
|
events["skills_invoked"].append(normalized)
|
|
72
|
+
if explicit and normalized not in events["skill_invocations"]:
|
|
73
|
+
events["skill_invocations"].append(normalized)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _record_skill_path(events: dict[str, Any], value: object) -> None:
|
|
77
|
+
if not isinstance(value, str):
|
|
78
|
+
return
|
|
79
|
+
match = re.search(r"\.(?:claude|agents|qoder|codebuddy)/skills/([^/\\]+)", value)
|
|
80
|
+
if match:
|
|
81
|
+
_record_skill_invocation(events, match.group(1))
|
|
42
82
|
|
|
43
83
|
|
|
44
84
|
# =============================================================================
|
|
@@ -50,6 +90,7 @@ def parse_output(stdout: str) -> dict[str, Any]:
|
|
|
50
90
|
"""Parse stream-json output into structured data."""
|
|
51
91
|
if not stdout:
|
|
52
92
|
return {"messages": []}
|
|
93
|
+
stdout = redact_sensitive(stdout)
|
|
53
94
|
messages = []
|
|
54
95
|
for line in stdout.strip().split("\n"):
|
|
55
96
|
try:
|
|
@@ -59,7 +100,37 @@ def parse_output(stdout: str) -> dict[str, Any]:
|
|
|
59
100
|
return {"messages": messages}
|
|
60
101
|
|
|
61
102
|
|
|
62
|
-
def
|
|
103
|
+
def _record_explicit_skill_value(events: dict[str, Any], value: Any) -> None:
|
|
104
|
+
if isinstance(value, str):
|
|
105
|
+
_record_skill_invocation(events, value, explicit=True)
|
|
106
|
+
elif isinstance(value, dict):
|
|
107
|
+
for key in ("skill", "skill_name", "skillName", "name"):
|
|
108
|
+
candidate = value.get(key)
|
|
109
|
+
if isinstance(candidate, str):
|
|
110
|
+
_record_skill_invocation(events, candidate, explicit=True)
|
|
111
|
+
return
|
|
112
|
+
elif isinstance(value, list):
|
|
113
|
+
for item in value:
|
|
114
|
+
_record_explicit_skill_value(events, item)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _record_explicit_skill_event(events: dict[str, Any], message: Any) -> None:
|
|
118
|
+
if not isinstance(message, dict):
|
|
119
|
+
return
|
|
120
|
+
event_type = str(message.get("type") or message.get("event") or "").lower()
|
|
121
|
+
if event_type in {"skill_invocation", "skill_invoked", "skill.use", "skill_used"}:
|
|
122
|
+
_record_explicit_skill_value(events, message.get("skill") or message.get("name"))
|
|
123
|
+
for key in ("skill_invocation", "skillInvocation", "skill_invocations", "skillInvocations"):
|
|
124
|
+
if key in message:
|
|
125
|
+
_record_explicit_skill_value(events, message[key])
|
|
126
|
+
item = message.get("item")
|
|
127
|
+
if isinstance(item, dict):
|
|
128
|
+
item_type = str(item.get("type") or item.get("event") or "").lower()
|
|
129
|
+
if item_type in {"skill_invocation", "skill_invoked", "skill.use", "skill_used"}:
|
|
130
|
+
_record_explicit_skill_value(events, item.get("skill") or item.get("name"))
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def extract_events(parsed: dict[str, Any], *, agent: str | None = None) -> dict[str, Any]:
|
|
63
134
|
"""Extract events (tool calls, files, etc.) from parsed output."""
|
|
64
135
|
events = {
|
|
65
136
|
"tool_calls": [],
|
|
@@ -68,6 +139,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
|
68
139
|
"files_modified": [],
|
|
69
140
|
"commands_run": [],
|
|
70
141
|
"skills_invoked": [],
|
|
142
|
+
"skill_invocations": [],
|
|
71
143
|
"duration_seconds": None,
|
|
72
144
|
"subject_invocations": 0,
|
|
73
145
|
"num_turns": None,
|
|
@@ -78,6 +150,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
|
78
150
|
"total_tokens": None,
|
|
79
151
|
"total_cost_usd": None,
|
|
80
152
|
"model_usage": {},
|
|
153
|
+
"role_sessions": {"subject": [], "simulator": [], "judge": []},
|
|
81
154
|
"peak_context_input_tokens": None,
|
|
82
155
|
"p95_context_input_tokens": None,
|
|
83
156
|
"average_context_input_tokens": None,
|
|
@@ -129,7 +202,30 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
|
129
202
|
elif key in capacities:
|
|
130
203
|
target[key] = max(target.get(key, 0), item)
|
|
131
204
|
|
|
205
|
+
infer_skill_paths = agent is None or agent in {"claude-code", "codex", "qoder", "codebuddy"}
|
|
206
|
+
|
|
207
|
+
def record_skill_path(value: object) -> None:
|
|
208
|
+
if infer_skill_paths:
|
|
209
|
+
_record_skill_path(events, value)
|
|
210
|
+
|
|
132
211
|
for msg in parsed.get("messages", []):
|
|
212
|
+
_record_explicit_skill_event(events, msg)
|
|
213
|
+
stack = [msg]
|
|
214
|
+
session_id = None
|
|
215
|
+
while stack and session_id is None:
|
|
216
|
+
item = stack.pop()
|
|
217
|
+
if isinstance(item, dict):
|
|
218
|
+
for key in ("session_id", "sessionId", "thread_id", "threadId"):
|
|
219
|
+
candidate = item.get(key)
|
|
220
|
+
if isinstance(candidate, str) and candidate:
|
|
221
|
+
session_id = candidate
|
|
222
|
+
break
|
|
223
|
+
stack.extend(item.values())
|
|
224
|
+
elif isinstance(item, list):
|
|
225
|
+
stack.extend(item)
|
|
226
|
+
if session_id and session_id not in events["role_sessions"]["subject"]:
|
|
227
|
+
events["role_sessions"]["subject"].append(session_id)
|
|
228
|
+
|
|
133
229
|
if msg.get("type") == "result":
|
|
134
230
|
events["subject_invocations"] += 1
|
|
135
231
|
duration_ms = msg.get("duration_ms")
|
|
@@ -137,7 +233,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
|
137
233
|
duration_ms_total += duration_ms
|
|
138
234
|
duration_observed = True
|
|
139
235
|
|
|
140
|
-
# A loop concatenates one result per subject invocation.
|
|
236
|
+
# A loop concatenates one result per subject invocation. Agent
|
|
141
237
|
# result telemetry is invocation-local, so task totals must add all
|
|
142
238
|
# initial, answer, and cold-resume invocations instead of retaining
|
|
143
239
|
# only the final result event.
|
|
@@ -150,6 +246,51 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
|
150
246
|
add_metric("total_cost_usd", msg.get("total_cost_usd"))
|
|
151
247
|
merge_model_usage(msg.get("modelUsage"))
|
|
152
248
|
|
|
249
|
+
if msg.get("type") == "turn.completed":
|
|
250
|
+
events["subject_invocations"] += 1
|
|
251
|
+
add_metric("num_turns", 1)
|
|
252
|
+
usage = msg.get("usage") or {}
|
|
253
|
+
add_metric("input_tokens", usage.get("input_tokens", usage.get("inputTokens")))
|
|
254
|
+
add_metric("output_tokens", usage.get("output_tokens", usage.get("outputTokens")))
|
|
255
|
+
add_metric(
|
|
256
|
+
"cache_read_input_tokens",
|
|
257
|
+
usage.get("cached_input_tokens", usage.get("cacheReadInputTokens")),
|
|
258
|
+
)
|
|
259
|
+
duration_ms = msg.get("duration_ms")
|
|
260
|
+
if isinstance(duration_ms, (int, float)) and not isinstance(duration_ms, bool):
|
|
261
|
+
duration_ms_total += duration_ms
|
|
262
|
+
duration_observed = True
|
|
263
|
+
|
|
264
|
+
if msg.get("type") == "item.completed":
|
|
265
|
+
item = msg.get("item") or {}
|
|
266
|
+
if item.get("type") == "command_execution":
|
|
267
|
+
command = item.get("command")
|
|
268
|
+
if isinstance(command, str) and command:
|
|
269
|
+
events["commands_run"].append(command)
|
|
270
|
+
record_skill_path(command)
|
|
271
|
+
tool_call = {"tool": "Bash", "input": {"command": command or ""}}
|
|
272
|
+
if item.get("aggregated_output") is not None:
|
|
273
|
+
tool_call["output"] = item["aggregated_output"]
|
|
274
|
+
events["tool_calls"].append(tool_call)
|
|
275
|
+
elif item.get("type") == "file_change":
|
|
276
|
+
for change in item.get("changes") or []:
|
|
277
|
+
if not isinstance(change, dict):
|
|
278
|
+
continue
|
|
279
|
+
path = change.get("path")
|
|
280
|
+
if not isinstance(path, str) or not path:
|
|
281
|
+
continue
|
|
282
|
+
kind = str(change.get("kind") or "modify").lower()
|
|
283
|
+
if kind in {"add", "create", "created"}:
|
|
284
|
+
events["files_created"].append(path)
|
|
285
|
+
else:
|
|
286
|
+
events["files_modified"].append(path)
|
|
287
|
+
record_skill_path(path)
|
|
288
|
+
elif item.get("type") in {"mcp_tool_call", "tool_call"}:
|
|
289
|
+
tool = item.get("name") or item.get("tool") or "unknown"
|
|
290
|
+
events["tool_calls"].append(
|
|
291
|
+
{"tool": tool, "input": item.get("arguments") or item.get("input") or {}}
|
|
292
|
+
)
|
|
293
|
+
|
|
153
294
|
if msg.get("type") == "assistant":
|
|
154
295
|
message = msg.get("message", {})
|
|
155
296
|
message_id = message.get("id")
|
|
@@ -182,18 +323,16 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
|
|
|
182
323
|
if tool == "Read" and path:
|
|
183
324
|
events["files_read"].append(path)
|
|
184
325
|
# Detect skill reads (e.g., .claude/skills/skill-name/SKILL.md)
|
|
185
|
-
|
|
186
|
-
m := re.search(r"\.claude/skills/([^/]+)", path)
|
|
187
|
-
):
|
|
188
|
-
_record_skill_invocation(events, m.group(1))
|
|
326
|
+
record_skill_path(path)
|
|
189
327
|
elif tool == "Write" and path:
|
|
190
328
|
events["files_created"].append(path)
|
|
191
329
|
elif tool == "Edit" and path:
|
|
192
330
|
events["files_modified"].append(path)
|
|
193
331
|
elif tool == "Bash" and inp.get("command"):
|
|
194
332
|
events["commands_run"].append(inp["command"])
|
|
333
|
+
record_skill_path(inp["command"])
|
|
195
334
|
elif tool == "Skill" and inp.get("skill"):
|
|
196
|
-
_record_skill_invocation(events, inp["skill"])
|
|
335
|
+
_record_skill_invocation(events, inp["skill"], explicit=True)
|
|
197
336
|
|
|
198
337
|
# Capture tool results and match to their tool_use calls
|
|
199
338
|
if msg.get("type") == "user":
|
|
@@ -540,14 +679,15 @@ class ExperimentLogger:
|
|
|
540
679
|
self.name = experiment_name or f"experiment_{self.timestamp}"
|
|
541
680
|
self.experiment_id = f"{self.name}_{self.timestamp}"
|
|
542
681
|
|
|
543
|
-
self.base_dir =
|
|
682
|
+
self.base_dir = get_runs_dir() / self.experiment_id
|
|
544
683
|
|
|
545
684
|
# Create subdirectories
|
|
546
685
|
self.events_dir = self.base_dir / "events"
|
|
547
686
|
self.reports_dir = self.base_dir / "reports"
|
|
548
687
|
self.raw_dir = self.base_dir / "raw"
|
|
549
688
|
|
|
550
|
-
|
|
689
|
+
self.artifacts_dir = self.base_dir / "artifacts"
|
|
690
|
+
for d in [self.events_dir, self.reports_dir, self.raw_dir, self.artifacts_dir]:
|
|
551
691
|
d.mkdir(parents=True, exist_ok=True)
|
|
552
692
|
|
|
553
693
|
self.columns = list(columns) if columns else rubric_columns()
|
|
@@ -757,7 +897,7 @@ class ExperimentLogger:
|
|
|
757
897
|
self.metadata["report_outputs"] = {name: str(path) for name, path in written.items()}
|
|
758
898
|
|
|
759
899
|
metadata_path = self.base_dir / "metadata.json"
|
|
760
|
-
metadata_path
|
|
900
|
+
_atomic_write_json(metadata_path, self.metadata)
|
|
761
901
|
|
|
762
902
|
print(f"\nExperiment results saved to: {self.base_dir}")
|
|
763
903
|
if written:
|
|
@@ -778,7 +918,7 @@ def save_events(base_dir: Path, treatment_name: str, rep: int, events: dict[str,
|
|
|
778
918
|
events_dir = base_dir / "events"
|
|
779
919
|
events_dir.mkdir(parents=True, exist_ok=True)
|
|
780
920
|
save_path = events_dir / f"{stable_treatment_name(treatment_name)}_rep{rep}.json"
|
|
781
|
-
save_path.write_text(json.dumps(events, indent=2), encoding="utf-8")
|
|
921
|
+
save_path.write_text(json.dumps(redact_sensitive(events), indent=2), encoding="utf-8")
|
|
782
922
|
return save_path
|
|
783
923
|
|
|
784
924
|
|
|
@@ -788,11 +928,11 @@ def save_raw(base_dir: Path, treatment_name: str, rep: int, stdout: str, stderr:
|
|
|
788
928
|
raw_dir.mkdir(parents=True, exist_ok=True)
|
|
789
929
|
stable_name = stable_treatment_name(treatment_name)
|
|
790
930
|
stdout_path = raw_dir / f"{stable_name}_rep{rep}_stdout.json"
|
|
791
|
-
stdout_path.write_text(stdout, encoding="utf-8")
|
|
931
|
+
stdout_path.write_text(redact_sensitive(stdout), encoding="utf-8")
|
|
792
932
|
|
|
793
933
|
if stderr:
|
|
794
934
|
stderr_path = raw_dir / f"{stable_name}_rep{rep}_stderr.txt"
|
|
795
|
-
stderr_path.write_text(stderr, encoding="utf-8")
|
|
935
|
+
stderr_path.write_text(redact_sensitive(stderr), encoding="utf-8")
|
|
796
936
|
|
|
797
937
|
|
|
798
938
|
def save_report(base_dir: Path, treatment_name: str, rep: int, report: dict[str, Any]):
|
|
@@ -800,5 +940,5 @@ def save_report(base_dir: Path, treatment_name: str, rep: int, report: dict[str,
|
|
|
800
940
|
reports_dir = base_dir / "reports"
|
|
801
941
|
reports_dir.mkdir(parents=True, exist_ok=True)
|
|
802
942
|
save_path = reports_dir / f"{stable_treatment_name(treatment_name)}_rep{rep}_report.json"
|
|
803
|
-
save_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
|
|
943
|
+
save_path.write_text(json.dumps(redact_sensitive(report), indent=2), encoding="utf-8")
|
|
804
944
|
return save_path
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Normalize project-authored manifest tasks into the existing Task contract."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from scaffold.python.manifests import ManifestTask, SkillEvalManifest
|
|
9
|
+
from scaffold.python.paths import EVAL_ROOT, get_tasks_dir
|
|
10
|
+
from scaffold.python.tasks import (
|
|
11
|
+
EvaluationConfig,
|
|
12
|
+
Task,
|
|
13
|
+
TaskConfig,
|
|
14
|
+
ValidationConfig,
|
|
15
|
+
load_task_from_path,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _generic_environment_dir() -> Path:
|
|
20
|
+
candidates = (
|
|
21
|
+
get_tasks_dir() / "generic-skill-smoke" / "environment",
|
|
22
|
+
EVAL_ROOT / "local" / "tasks" / "generic-skill-smoke" / "environment",
|
|
23
|
+
)
|
|
24
|
+
for candidate in candidates:
|
|
25
|
+
if (candidate / "Dockerfile").is_file():
|
|
26
|
+
return candidate
|
|
27
|
+
raise FileNotFoundError("Bundled generic task environment is unavailable")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _inline_task(manifest: SkillEvalManifest, spec: ManifestTask) -> Task:
|
|
31
|
+
assert spec.prompt is not None
|
|
32
|
+
files = list(spec.expect.get("files", []))
|
|
33
|
+
config = TaskConfig(
|
|
34
|
+
name=spec.name,
|
|
35
|
+
description=f"Inline task authored in {manifest.path}",
|
|
36
|
+
default_treatments=[],
|
|
37
|
+
timeout_sec=900,
|
|
38
|
+
validation=ValidationConfig(target_artifacts=files, timeout=120),
|
|
39
|
+
evaluation=EvaluationConfig(
|
|
40
|
+
profile=manifest.profile or "generic",
|
|
41
|
+
expected_artifacts=files,
|
|
42
|
+
rubric_criteria=list(spec.rubric),
|
|
43
|
+
),
|
|
44
|
+
interaction=manifest.interaction,
|
|
45
|
+
)
|
|
46
|
+
return Task(
|
|
47
|
+
path=manifest.path.parent,
|
|
48
|
+
config=config,
|
|
49
|
+
instruction_template=spec.prompt,
|
|
50
|
+
_environment_dir=_generic_environment_dir(),
|
|
51
|
+
workspace_dir=spec.workspace,
|
|
52
|
+
manifest_expectations=spec.expect,
|
|
53
|
+
manifest_path=manifest.path,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def load_manifest_tasks(manifest: SkillEvalManifest) -> list[Task]:
|
|
58
|
+
"""Load all authored inline/source task entries in manifest order."""
|
|
59
|
+
tasks: list[Task] = []
|
|
60
|
+
for spec in manifest.tasks:
|
|
61
|
+
task = _inline_task(manifest, spec) if spec.is_inline else load_task_from_path(spec.source)
|
|
62
|
+
# ``evaluation.tasks[].name`` is the public identity, including for a
|
|
63
|
+
# sourced package whose internal metadata name differs. Keep the full
|
|
64
|
+
# package semantics loaded by ``load_task_from_path`` while applying
|
|
65
|
+
# the manifest alias consistently to catalogue, selection, and reports.
|
|
66
|
+
if task.name != spec.name:
|
|
67
|
+
task = replace(task, config=replace(task.config, name=spec.name))
|
|
68
|
+
tasks.append(task)
|
|
69
|
+
return tasks
|