@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -26
- package/assets/manifest.json +4 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +120 -120
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +79 -79
- package/assets/skills/comet/scripts/comet-guard.mjs +134 -134
- package/assets/skills/comet/scripts/comet-handoff.mjs +106 -106
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-intent.mjs +6 -6
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +78 -78
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +96 -96
- package/assets/skills/comet-build/SKILL.md +36 -62
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +2 -2
- package/assets/skills/comet-native/SKILL.md +18 -15
- package/assets/skills/comet-native/reference/artifacts.md +5 -4
- package/assets/skills/comet-native/reference/clarification.md +1 -1
- package/assets/skills/comet-native/reference/commands.md +4 -4
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +116 -115
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +148 -148
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +121 -121
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +108 -108
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +157 -156
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +89 -89
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills/comet-review/SKILL.md +121 -0
- package/assets/skills/comet-review/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +36 -62
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +2 -2
- package/assets/skills-zh/comet-native/SKILL.md +18 -15
- package/assets/skills-zh/comet-native/reference/artifacts.md +5 -4
- package/assets/skills-zh/comet-native/reference/clarification.md +1 -1
- package/assets/skills-zh/comet-native/reference/commands.md +4 -4
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-review/SKILL.md +121 -0
- package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +5 -8
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +5 -6
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.d.ts +9 -0
- package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-command-context.js +10 -0
- package/dist/domains/comet-classic/classic-command-context.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +4 -4
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-handoff.js +44 -6
- package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +16 -2
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-intent-command.js +2 -1
- package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.js +4 -3
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +6 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +58 -9
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +39 -13
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +2 -2
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +5 -1
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +2 -0
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-atomic-file.js +17 -10
- package/dist/domains/comet-native/native-atomic-file.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +65 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +453 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +13 -7
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +18 -5
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +56 -4
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +179 -20
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +22 -9
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +178 -73
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-migration-runtime.js +9 -4
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +19 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +168 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
- package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
- package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.js +24 -6
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.d.ts +6 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-finish.js +86 -35
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-ayV208_D.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-Bn5o8p3H.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-BnF1SI6B.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-Cuvy639H.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-CfREXlEl.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-DWIdpYTb.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-BnqPcfZB.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-BVDhdJyW.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-C-nglwHC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-C8TMmrbP.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-B3VkUMck.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-DQY_Ba00.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-BLv71GRj.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
- package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-CNgNNG9l.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-D9k1fTO1.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-CHijUnPl.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-DfWlntDq.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-o6hemX-z.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-COMQF9Yy.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-C3fGdiMa.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-D_UMpS5e.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-CeyA2KbI.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-CWqdSzJ-.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +27 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +108 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +87 -44
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +14 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +93 -5
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +3 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +4 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +16 -10
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +11 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +39 -29
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -1
- package/dist/domains/workflow-contract/contained-atomic-write.js +18 -10
- package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +4 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +174 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +13 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/fs/strip-bom.d.ts +10 -0
- package/dist/platform/fs/strip-bom.d.ts.map +1 -0
- package/dist/platform/fs/strip-bom.js +12 -0
- package/dist/platform/fs/strip-bom.js.map +1 -0
- package/dist/platform/install/platforms.d.ts +11 -0
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +49 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +1 -0
- package/dist/platform/process/external-command.d.ts.map +1 -1
- package/dist/platform/process/external-command.js +2 -1
- package/dist/platform/process/external-command.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +719 -83
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +324 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +18 -28
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-BFwKBVTJ.css +0 -1
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
- package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml +0 -6
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
apiVersion: comet.eval.agent/v1alpha1
|
|
2
|
+
kind: EvalAgentAdapter
|
|
3
|
+
metadata:
|
|
4
|
+
id: fixture-agent
|
|
5
|
+
version: 1.0.0
|
|
6
|
+
runtime:
|
|
7
|
+
executable: fixture-agent
|
|
8
|
+
install:
|
|
9
|
+
kind: none
|
|
10
|
+
credentials:
|
|
11
|
+
- FIXTURE_AGENT_API_KEY
|
|
12
|
+
modelEnv: FIXTURE_AGENT_MODEL
|
|
13
|
+
baseUrlEnv: FIXTURE_AGENT_BASE_URL
|
|
14
|
+
capabilities:
|
|
15
|
+
singleTurn: true
|
|
16
|
+
resume: true
|
|
17
|
+
structuredEvents: true
|
|
18
|
+
telemetry: false
|
|
19
|
+
skillInvocationEvidence: true
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Tests for selectable evaluation-agent contracts."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from scaffold.python.agents import (
|
|
6
|
+
AGENT_IDS,
|
|
7
|
+
get_agent_adapter,
|
|
8
|
+
resolve_agent,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_resolve_agent_applies_cli_manifest_default_precedence():
|
|
13
|
+
assert resolve_agent("qoder", "codex").agent == "qoder"
|
|
14
|
+
assert resolve_agent(None, "codex").agent == "codex"
|
|
15
|
+
assert resolve_agent(None, None).agent == "claude-code"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.mark.parametrize("agent", ["gemini", "", "Claude-Code"])
|
|
19
|
+
def test_resolve_agent_rejects_unknown_agent(agent: str):
|
|
20
|
+
with pytest.raises(ValueError, match="Unsupported evaluation agent"):
|
|
21
|
+
resolve_agent(agent, None)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_all_supported_agents_expose_role_neutral_run_commands():
|
|
25
|
+
assert AGENT_IDS == ("claude-code", "codex", "qoder", "codebuddy")
|
|
26
|
+
|
|
27
|
+
for agent_id in AGENT_IDS:
|
|
28
|
+
adapter = get_agent_adapter(agent_id)
|
|
29
|
+
command = adapter.build_run_command(
|
|
30
|
+
"inspect the task",
|
|
31
|
+
model="test-model",
|
|
32
|
+
role="subject",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
assert adapter.id == agent_id
|
|
36
|
+
assert command
|
|
37
|
+
assert "inspect the task" in command
|
|
38
|
+
assert "test-model" in command
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_codebuddy_adapter_uses_headless_streaming_and_resume_contract():
|
|
42
|
+
adapter = get_agent_adapter("codebuddy")
|
|
43
|
+
|
|
44
|
+
assert adapter.executable == "codebuddy"
|
|
45
|
+
assert adapter.required_credentials == ("CODEBUDDY_API_KEY", "CODEBUDDY_AUTH_TOKEN")
|
|
46
|
+
assert adapter.build_run_command(
|
|
47
|
+
"continue",
|
|
48
|
+
model="codebuddy-model",
|
|
49
|
+
role="subject",
|
|
50
|
+
resume_id="session-123",
|
|
51
|
+
) == [
|
|
52
|
+
"codebuddy",
|
|
53
|
+
"-p",
|
|
54
|
+
"continue",
|
|
55
|
+
"--output-format",
|
|
56
|
+
"stream-json",
|
|
57
|
+
"--dangerously-skip-permissions",
|
|
58
|
+
"-r",
|
|
59
|
+
"session-123",
|
|
60
|
+
"--model",
|
|
61
|
+
"codebuddy-model",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_adapters_build_resume_commands_with_isolated_session_ids():
|
|
66
|
+
for agent_id in AGENT_IDS:
|
|
67
|
+
adapter = get_agent_adapter(agent_id)
|
|
68
|
+
command = adapter.build_run_command(
|
|
69
|
+
"continue",
|
|
70
|
+
model=None,
|
|
71
|
+
role="simulator",
|
|
72
|
+
resume_id="session-123",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
assert "session-123" in command
|
|
76
|
+
assert adapter.supports_resume is True
|
|
@@ -178,6 +178,39 @@ def test_fallback_case_manifest_never_hashes_dotenv_files(tmp_path: Path):
|
|
|
178
178
|
assert before.case_hash == after.case_hash
|
|
179
179
|
|
|
180
180
|
|
|
181
|
+
def test_inline_case_manifest_hashes_authored_inputs(tmp_path: Path):
|
|
182
|
+
manifest_path = tmp_path / "comet" / "eval.yaml"
|
|
183
|
+
manifest_path.parent.mkdir(parents=True)
|
|
184
|
+
manifest_path.write_text("tasks:\n - name: inline-task\n", encoding="utf-8")
|
|
185
|
+
workspace = tmp_path / "workspace"
|
|
186
|
+
workspace.mkdir()
|
|
187
|
+
(workspace / "input.txt").write_text("fixture\n", encoding="utf-8")
|
|
188
|
+
environment = tmp_path / "environment"
|
|
189
|
+
environment.mkdir()
|
|
190
|
+
(environment / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
|
|
191
|
+
|
|
192
|
+
case = build_case_manifest(
|
|
193
|
+
"inline-task",
|
|
194
|
+
tmp_path,
|
|
195
|
+
manifest_path=manifest_path,
|
|
196
|
+
rendered_prompt="Create result.md.",
|
|
197
|
+
workspace_dir=workspace,
|
|
198
|
+
validation_rules={"files": ["result.md"]},
|
|
199
|
+
environment_dir=environment,
|
|
200
|
+
)
|
|
201
|
+
payload = case_manifest_payload(case)
|
|
202
|
+
|
|
203
|
+
assert payload["authored_hashes"] == {
|
|
204
|
+
"manifest_hash": case.authored_hashes["manifest_hash"],
|
|
205
|
+
"task_definition_hash": case.task_hash,
|
|
206
|
+
"workspace_hash": case.data_hash,
|
|
207
|
+
"validation_rule_hash": case.validator_hash,
|
|
208
|
+
}
|
|
209
|
+
assert case.task_hash.startswith("sha256:")
|
|
210
|
+
assert case.data_hash.startswith("sha256:")
|
|
211
|
+
assert case.validator_hash.startswith("sha256:")
|
|
212
|
+
|
|
213
|
+
|
|
181
214
|
def test_v2_case_manifest_binds_safe_execution_identity_without_raw_configuration(
|
|
182
215
|
tmp_path: Path,
|
|
183
216
|
):
|
|
@@ -234,6 +267,43 @@ def test_v2_case_manifest_binds_safe_execution_identity_without_raw_configuratio
|
|
|
234
267
|
assert changed.case_hash != manifest.case_hash
|
|
235
268
|
|
|
236
269
|
|
|
270
|
+
def test_execution_identity_binds_selected_agent_cli():
|
|
271
|
+
interaction = SimpleNamespace(
|
|
272
|
+
mode="none",
|
|
273
|
+
max_turns=1,
|
|
274
|
+
simulator_prompt=None,
|
|
275
|
+
decision_patterns=[],
|
|
276
|
+
decision_reply=None,
|
|
277
|
+
decision_replies=[],
|
|
278
|
+
continue_prompt="continue",
|
|
279
|
+
fresh_resume_marker=None,
|
|
280
|
+
)
|
|
281
|
+
docker_identity = {
|
|
282
|
+
"schema": EXECUTION_IDENTITY_SCHEMA,
|
|
283
|
+
"runtime_image_id": "sha256:" + "2" * 64,
|
|
284
|
+
"image_id_hash": _digest("sha256:" + "2" * 64),
|
|
285
|
+
"image_repo_digests_hash": _digest("repo-digests"),
|
|
286
|
+
"image_ref_hash": _digest("build-ref"),
|
|
287
|
+
"claude_tool_version_hash": _digest("codex-1.0.0"),
|
|
288
|
+
"agent_cli_version_hash": _digest("codex-1.0.0"),
|
|
289
|
+
"agent_cli_version": "codex-cli 1.0.0",
|
|
290
|
+
"agent": "codex",
|
|
291
|
+
"agent_cli": "codex",
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
identity = build_execution_identity(
|
|
295
|
+
docker_identity,
|
|
296
|
+
model="o3",
|
|
297
|
+
model_config={"OPENAI_MODEL": "o3"},
|
|
298
|
+
interaction=interaction,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
assert identity["agent"] == "codex"
|
|
302
|
+
assert identity["agent_cli"] == "codex"
|
|
303
|
+
assert identity["agent_cli_version_hash"] == _digest("codex-1.0.0")
|
|
304
|
+
assert identity["agent_cli_version"] == "codex-cli 1.0.0"
|
|
305
|
+
|
|
306
|
+
|
|
237
307
|
def test_expected_matrix_detects_case_missing_from_both_experiments(tmp_path: Path):
|
|
238
308
|
candidate, baseline, tasks = _make_experiments(tmp_path)
|
|
239
309
|
_write_run(
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""Tests for automatic task generation and its cache contract."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
from scaffold.python.auto_tasks import (
|
|
10
|
+
AutoTaskError,
|
|
11
|
+
GenerationOutput,
|
|
12
|
+
build_skill_snapshot,
|
|
13
|
+
ensure_generated_manifest,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _write_skill(tmp_path: Path) -> Path:
|
|
18
|
+
skill = tmp_path / "my-skill"
|
|
19
|
+
(skill / "references").mkdir(parents=True)
|
|
20
|
+
(skill / "SKILL.md").write_text(
|
|
21
|
+
"Use the reference in `references/format.md` and write result.md.\n", encoding="utf-8"
|
|
22
|
+
)
|
|
23
|
+
(skill / "references" / "format.md").write_text("# Format\n", encoding="utf-8")
|
|
24
|
+
(tmp_path / "unrelated.txt").write_text("do not include\n", encoding="utf-8")
|
|
25
|
+
return skill
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _generated_payload():
|
|
29
|
+
return {
|
|
30
|
+
"tasks": [
|
|
31
|
+
{
|
|
32
|
+
"name": "normal-flow",
|
|
33
|
+
"prompt": "Use the Skill to create result.md.",
|
|
34
|
+
"expect": {"files": ["result.md"]},
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"name": "boundary-flow",
|
|
38
|
+
"prompt": "Use the Skill with an invalid input and document recovery.",
|
|
39
|
+
"expect": {"contains": {"result.md": ["recovery"]}},
|
|
40
|
+
"rubric": ["The recovery is clear."],
|
|
41
|
+
},
|
|
42
|
+
]
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_skill_snapshot_is_bounded_to_relevant_package_files(tmp_path: Path):
|
|
47
|
+
skill = _write_skill(tmp_path)
|
|
48
|
+
|
|
49
|
+
snapshot = build_skill_snapshot(skill)
|
|
50
|
+
|
|
51
|
+
paths = {item.path for item in snapshot.files}
|
|
52
|
+
assert paths == {"SKILL.md", "references/format.md"}
|
|
53
|
+
assert "unrelated.txt" not in paths
|
|
54
|
+
assert snapshot.content_hash.startswith("sha256:")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_generated_manifest_is_cached_and_reused(tmp_path: Path):
|
|
58
|
+
skill = _write_skill(tmp_path)
|
|
59
|
+
calls = []
|
|
60
|
+
|
|
61
|
+
def generate(_prompt):
|
|
62
|
+
calls.append(True)
|
|
63
|
+
return _generated_payload()
|
|
64
|
+
|
|
65
|
+
first = ensure_generated_manifest(
|
|
66
|
+
skill,
|
|
67
|
+
tmp_path,
|
|
68
|
+
agent="claude-code",
|
|
69
|
+
model=None,
|
|
70
|
+
profile="generic",
|
|
71
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
72
|
+
generate=generate,
|
|
73
|
+
)
|
|
74
|
+
second = ensure_generated_manifest(
|
|
75
|
+
skill,
|
|
76
|
+
tmp_path,
|
|
77
|
+
agent="claude-code",
|
|
78
|
+
model=None,
|
|
79
|
+
profile="generic",
|
|
80
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
81
|
+
generate=generate,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
assert first.manifest_path == second.manifest_path
|
|
85
|
+
assert first.generation_hash == second.generation_hash
|
|
86
|
+
assert calls == [True]
|
|
87
|
+
assert first.manifest_path == (
|
|
88
|
+
tmp_path
|
|
89
|
+
/ ".comet"
|
|
90
|
+
/ "eval"
|
|
91
|
+
/ "generated"
|
|
92
|
+
/ "my-skill"
|
|
93
|
+
/ first.generation_hash
|
|
94
|
+
/ "eval.yaml"
|
|
95
|
+
)
|
|
96
|
+
metadata = json.loads(first.metadata_path.read_text(encoding="utf-8"))
|
|
97
|
+
assert metadata["manifest_hash"].startswith("sha256:")
|
|
98
|
+
assert metadata["generation_overhead"]["attempt_count"] == 1
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_generated_cache_uses_owner_locks_atomic_replacement_and_redacted_metadata(
|
|
102
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
103
|
+
):
|
|
104
|
+
skill = _write_skill(tmp_path)
|
|
105
|
+
replacements = []
|
|
106
|
+
import scaffold.python.auto_tasks as auto_tasks
|
|
107
|
+
|
|
108
|
+
original_replace = auto_tasks.os.replace
|
|
109
|
+
monkeypatch.setattr(
|
|
110
|
+
auto_tasks.os,
|
|
111
|
+
"replace",
|
|
112
|
+
lambda source, target: (replacements.append((Path(source), Path(target))), original_replace(source, target))[1],
|
|
113
|
+
)
|
|
114
|
+
result = ensure_generated_manifest(
|
|
115
|
+
skill,
|
|
116
|
+
tmp_path,
|
|
117
|
+
agent="codex",
|
|
118
|
+
model=None,
|
|
119
|
+
profile="generic",
|
|
120
|
+
interaction={"mode": "none", "api_key": "must-not-persist"},
|
|
121
|
+
generate=lambda _prompt: _generated_payload(),
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
metadata = result.metadata_path.read_text(encoding="utf-8")
|
|
125
|
+
lock_dir = tmp_path / ".comet" / "eval" / "locks"
|
|
126
|
+
assert lock_dir.is_dir()
|
|
127
|
+
assert "must-not-persist" not in metadata
|
|
128
|
+
assert "api_key" not in metadata
|
|
129
|
+
assert any(target == result.manifest_path.parent for _, target in replacements)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_cached_manifest_hash_mismatch_forces_regeneration(tmp_path: Path):
|
|
133
|
+
skill = _write_skill(tmp_path)
|
|
134
|
+
calls = []
|
|
135
|
+
|
|
136
|
+
def generate(_prompt):
|
|
137
|
+
calls.append(True)
|
|
138
|
+
return _generated_payload()
|
|
139
|
+
|
|
140
|
+
first = ensure_generated_manifest(
|
|
141
|
+
skill,
|
|
142
|
+
tmp_path,
|
|
143
|
+
agent="codex",
|
|
144
|
+
model="gpt-test",
|
|
145
|
+
profile="generic",
|
|
146
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
147
|
+
generate=generate,
|
|
148
|
+
)
|
|
149
|
+
first.manifest_path.write_text("tasks: []\n", encoding="utf-8")
|
|
150
|
+
|
|
151
|
+
second = ensure_generated_manifest(
|
|
152
|
+
skill,
|
|
153
|
+
tmp_path,
|
|
154
|
+
agent="codex",
|
|
155
|
+
model="gpt-test",
|
|
156
|
+
profile="generic",
|
|
157
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
158
|
+
generate=generate,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
assert calls == [True, True]
|
|
162
|
+
assert "normal-flow" in second.manifest_path.read_text(encoding="utf-8")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@pytest.mark.parametrize("task_count", [1, 5])
|
|
166
|
+
def test_cached_generated_manifest_revalidates_task_count(
|
|
167
|
+
tmp_path: Path, task_count: int
|
|
168
|
+
):
|
|
169
|
+
skill = _write_skill(tmp_path)
|
|
170
|
+
calls = []
|
|
171
|
+
|
|
172
|
+
def generate(_prompt):
|
|
173
|
+
calls.append(True)
|
|
174
|
+
return _generated_payload()
|
|
175
|
+
|
|
176
|
+
first = ensure_generated_manifest(
|
|
177
|
+
skill,
|
|
178
|
+
tmp_path,
|
|
179
|
+
agent="codex",
|
|
180
|
+
model="gpt-test",
|
|
181
|
+
profile="generic",
|
|
182
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
183
|
+
generate=generate,
|
|
184
|
+
)
|
|
185
|
+
data = yaml.safe_load(first.manifest_path.read_text(encoding="utf-8"))
|
|
186
|
+
tasks = data["evaluation"]["tasks"]
|
|
187
|
+
data["evaluation"]["tasks"] = (tasks * task_count)[:task_count]
|
|
188
|
+
first.manifest_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
|
|
189
|
+
from scaffold.python.generated_task_cache import write_cache_metadata
|
|
190
|
+
|
|
191
|
+
write_cache_metadata(first.manifest_path.parent, first.generation_hash)
|
|
192
|
+
|
|
193
|
+
second = ensure_generated_manifest(
|
|
194
|
+
skill,
|
|
195
|
+
tmp_path,
|
|
196
|
+
agent="codex",
|
|
197
|
+
model="gpt-test",
|
|
198
|
+
profile="generic",
|
|
199
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
200
|
+
generate=generate,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
assert calls == [True, True]
|
|
204
|
+
assert len(yaml.safe_load(second.manifest_path.read_text(encoding="utf-8"))["evaluation"]["tasks"]) == 2
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_generation_overhead_is_persisted_in_cache_metadata(tmp_path: Path):
|
|
208
|
+
skill = _write_skill(tmp_path)
|
|
209
|
+
|
|
210
|
+
result = ensure_generated_manifest(
|
|
211
|
+
skill,
|
|
212
|
+
tmp_path,
|
|
213
|
+
agent="codebuddy",
|
|
214
|
+
model="codebuddy-test",
|
|
215
|
+
profile="generic",
|
|
216
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
217
|
+
generate=lambda _prompt: GenerationOutput(
|
|
218
|
+
output=_generated_payload(),
|
|
219
|
+
telemetry={
|
|
220
|
+
"duration_seconds": 0.125,
|
|
221
|
+
"input_tokens": 10,
|
|
222
|
+
"output_tokens": 20,
|
|
223
|
+
"total_tokens": 30,
|
|
224
|
+
"total_cost_usd": 0.01,
|
|
225
|
+
"model": "codebuddy-test",
|
|
226
|
+
},
|
|
227
|
+
),
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
metadata = json.loads(result.metadata_path.read_text(encoding="utf-8"))
|
|
231
|
+
overhead = metadata["generation_overhead"]
|
|
232
|
+
assert overhead["attempt_count"] == 1
|
|
233
|
+
assert overhead["duration_seconds"] == 0.125
|
|
234
|
+
assert overhead["total_tokens"] == 30
|
|
235
|
+
assert overhead["total_cost_usd"] == 0.01
|
|
236
|
+
assert overhead["attempts"][0]["model"] == "codebuddy-test"
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def test_collect_requires_a_valid_cache_and_never_calls_generator(tmp_path: Path):
|
|
240
|
+
skill = _write_skill(tmp_path)
|
|
241
|
+
calls = []
|
|
242
|
+
|
|
243
|
+
with pytest.raises(AutoTaskError, match="run comet eval"):
|
|
244
|
+
ensure_generated_manifest(
|
|
245
|
+
skill,
|
|
246
|
+
tmp_path,
|
|
247
|
+
agent="codex",
|
|
248
|
+
model="gpt-test",
|
|
249
|
+
profile="generic",
|
|
250
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
251
|
+
collect_only=True,
|
|
252
|
+
generate=lambda _prompt: calls.append(True),
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
assert calls == []
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def test_invalid_generation_gets_one_repair_attempt(tmp_path: Path):
|
|
259
|
+
skill = _write_skill(tmp_path)
|
|
260
|
+
responses = [{"tasks": [{"name": "only-one", "prompt": "No expect."}]}, _generated_payload()]
|
|
261
|
+
|
|
262
|
+
def generate(_prompt):
|
|
263
|
+
return responses.pop(0)
|
|
264
|
+
|
|
265
|
+
result = ensure_generated_manifest(
|
|
266
|
+
skill,
|
|
267
|
+
tmp_path,
|
|
268
|
+
agent="codebuddy",
|
|
269
|
+
model="codebuddy-test",
|
|
270
|
+
profile="generic",
|
|
271
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
272
|
+
generate=generate,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
assert result.manifest_path.is_file()
|
|
276
|
+
assert responses == []
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def test_invalid_generation_stops_after_one_repair_attempt(tmp_path: Path):
|
|
280
|
+
skill = _write_skill(tmp_path)
|
|
281
|
+
calls = []
|
|
282
|
+
|
|
283
|
+
def generate(_prompt):
|
|
284
|
+
calls.append(True)
|
|
285
|
+
return {"tasks": [{"name": "only-one", "prompt": "No expect. api_key=sk-secret-value"}]}
|
|
286
|
+
|
|
287
|
+
with pytest.raises(AutoTaskError, match="task_generation") as exc_info:
|
|
288
|
+
ensure_generated_manifest(
|
|
289
|
+
skill,
|
|
290
|
+
tmp_path,
|
|
291
|
+
agent="claude-code",
|
|
292
|
+
model=None,
|
|
293
|
+
profile="generic",
|
|
294
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
295
|
+
generate=generate,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
assert calls == [True, True]
|
|
299
|
+
report_path = Path(str(exc_info.value).split("Partial report: ", 1)[1])
|
|
300
|
+
metadata = json.loads((report_path.parent / "metadata.json").read_text(encoding="utf-8"))
|
|
301
|
+
assert report_path.is_file()
|
|
302
|
+
assert metadata["attribution"] == "task_generation"
|
|
303
|
+
assert metadata["attempt_count"] == 2
|
|
304
|
+
assert metadata["task_denominator"] == 0
|
|
305
|
+
assert metadata["subject_metrics"] == {}
|
|
306
|
+
assert metadata["report_path"] == str(report_path)
|
|
307
|
+
assert "sk-secret-value" not in report_path.read_text(encoding="utf-8")
|
|
308
|
+
assert "sk-secret-value" not in (report_path.parent / "metadata.json").read_text(encoding="utf-8")
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def test_html_generation_failure_escapes_generator_controlled_error_text(
|
|
312
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
313
|
+
):
|
|
314
|
+
skill = _write_skill(tmp_path)
|
|
315
|
+
monkeypatch.setenv("COMET_EVAL_REPORT_HTML", "1")
|
|
316
|
+
monkeypatch.setenv("ANTHROPIC_API_KEY", "environment-secret-value")
|
|
317
|
+
|
|
318
|
+
def generate(_prompt):
|
|
319
|
+
raise ValueError(
|
|
320
|
+
"<script>globalThis.cometEvalInjected = true</script> "
|
|
321
|
+
"api_key=sk-secret-value Authorization: Bearer bearer-secret-value "
|
|
322
|
+
"x-api-key: header-secret-value environment-secret-value"
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
with pytest.raises(AutoTaskError) as exc_info:
|
|
326
|
+
ensure_generated_manifest(
|
|
327
|
+
skill,
|
|
328
|
+
tmp_path,
|
|
329
|
+
agent="claude-code",
|
|
330
|
+
model=None,
|
|
331
|
+
profile="generic",
|
|
332
|
+
interaction={"mode": "none", "max_turns": 12},
|
|
333
|
+
generate=generate,
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
report_path = Path(str(exc_info.value).split("Partial report: ", 1)[1])
|
|
337
|
+
report = report_path.read_text(encoding="utf-8")
|
|
338
|
+
assert report_path.suffix == ".html"
|
|
339
|
+
assert "<script>" not in report
|
|
340
|
+
assert "<script>globalThis.cometEvalInjected = true</script>" in report
|
|
341
|
+
assert "sk-secret-value" not in report
|
|
342
|
+
metadata = (report_path.parent / "metadata.json").read_text(encoding="utf-8")
|
|
343
|
+
for secret in (
|
|
344
|
+
"sk-secret-value",
|
|
345
|
+
"bearer-secret-value",
|
|
346
|
+
"header-secret-value",
|
|
347
|
+
"environment-secret-value",
|
|
348
|
+
):
|
|
349
|
+
assert secret not in report
|
|
350
|
+
assert secret not in metadata
|