@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""Selectable agent CLI contracts used by the evaluation harness."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
from scaffold.python.custom_agents import load_custom_agent_spec
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
AgentId = str
|
|
12
|
+
AgentRole = Literal["subject", "simulator", "judge"]
|
|
13
|
+
|
|
14
|
+
AGENT_IDS: tuple[AgentId, ...] = ("claude-code", "codex", "qoder", "codebuddy")
|
|
15
|
+
DEFAULT_AGENT: AgentId = "claude-code"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class AgentSelection:
|
|
20
|
+
"""The resolved agent and the configuration layer that selected it."""
|
|
21
|
+
|
|
22
|
+
agent: AgentId
|
|
23
|
+
source: Literal["cli", "manifest", "default"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class AgentAdapter:
|
|
28
|
+
"""Build role-neutral commands for one supported evaluation agent."""
|
|
29
|
+
|
|
30
|
+
id: AgentId
|
|
31
|
+
executable: str
|
|
32
|
+
required_credentials: tuple[str, ...]
|
|
33
|
+
model_env: str = ""
|
|
34
|
+
base_url_env: str = ""
|
|
35
|
+
supports_resume: bool = True
|
|
36
|
+
supports_single_turn: bool = True
|
|
37
|
+
supports_structured_events: bool = True
|
|
38
|
+
supports_telemetry: bool = True
|
|
39
|
+
supports_skill_invocation_evidence: bool = True
|
|
40
|
+
custom: bool = False
|
|
41
|
+
install_kind: str = "none"
|
|
42
|
+
install_package: str | None = None
|
|
43
|
+
install_version: str | None = None
|
|
44
|
+
|
|
45
|
+
def build_version_command(self) -> list[str]:
|
|
46
|
+
return [self.executable, "--version"]
|
|
47
|
+
|
|
48
|
+
def build_run_command(
|
|
49
|
+
self,
|
|
50
|
+
prompt: str,
|
|
51
|
+
*,
|
|
52
|
+
model: str | None,
|
|
53
|
+
role: AgentRole,
|
|
54
|
+
resume_id: str | None = None,
|
|
55
|
+
max_turns: int | None = None,
|
|
56
|
+
) -> list[str]:
|
|
57
|
+
del role
|
|
58
|
+
command = [self.executable]
|
|
59
|
+
if self.id == "claude-code":
|
|
60
|
+
command.extend(["-p", prompt, "--dangerously-skip-permissions"])
|
|
61
|
+
command.extend(["--output-format", "stream-json", "--verbose"])
|
|
62
|
+
if resume_id:
|
|
63
|
+
command.extend(["--resume", resume_id])
|
|
64
|
+
if max_turns is not None:
|
|
65
|
+
command.extend(["--max-turns", str(max_turns)])
|
|
66
|
+
elif self.id == "codex":
|
|
67
|
+
command.extend(["exec"])
|
|
68
|
+
if resume_id:
|
|
69
|
+
command.extend(["resume", resume_id])
|
|
70
|
+
command.extend(["--json", "--yolo"])
|
|
71
|
+
if model:
|
|
72
|
+
command.extend(["--model", model])
|
|
73
|
+
command.append(prompt)
|
|
74
|
+
elif self.id == "qoder":
|
|
75
|
+
command.extend(["-p", prompt, "--output-format", "stream-json", "--yolo"])
|
|
76
|
+
if resume_id:
|
|
77
|
+
command.extend(["-r", resume_id])
|
|
78
|
+
if model:
|
|
79
|
+
command.extend(["--model", model])
|
|
80
|
+
if max_turns is not None:
|
|
81
|
+
command.extend(["--max-turns", str(max_turns)])
|
|
82
|
+
elif self.custom:
|
|
83
|
+
command.extend(["-p", prompt, "--output-format", "stream-json"])
|
|
84
|
+
if model:
|
|
85
|
+
command.extend(["--model", model])
|
|
86
|
+
if resume_id:
|
|
87
|
+
command.extend(["--resume", resume_id])
|
|
88
|
+
else:
|
|
89
|
+
command.extend(
|
|
90
|
+
[
|
|
91
|
+
"-p",
|
|
92
|
+
prompt,
|
|
93
|
+
"--output-format",
|
|
94
|
+
"stream-json",
|
|
95
|
+
"--dangerously-skip-permissions",
|
|
96
|
+
]
|
|
97
|
+
)
|
|
98
|
+
if resume_id:
|
|
99
|
+
command.extend(["-r", resume_id])
|
|
100
|
+
if model:
|
|
101
|
+
command.extend(["--model", model])
|
|
102
|
+
if self.id == "claude-code" and model:
|
|
103
|
+
command.extend(["--model", model])
|
|
104
|
+
return command
|
|
105
|
+
|
|
106
|
+
def has_observable_skill_invocation(self, events: object) -> bool:
|
|
107
|
+
"""Only explicit invocation events count; artifacts are not evidence."""
|
|
108
|
+
if not self.supports_skill_invocation_evidence or not isinstance(events, dict):
|
|
109
|
+
return False
|
|
110
|
+
invocations = events.get("skill_invocations")
|
|
111
|
+
return isinstance(invocations, list) and bool(invocations)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def validate_agent_id(value: object, *, field: str = "evaluation agent") -> AgentId:
|
|
115
|
+
if value in AGENT_IDS:
|
|
116
|
+
return value
|
|
117
|
+
if isinstance(value, str) and load_custom_agent_spec(value) is not None:
|
|
118
|
+
return value
|
|
119
|
+
if value not in AGENT_IDS:
|
|
120
|
+
supported = ", ".join(AGENT_IDS)
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"Unsupported evaluation agent {value!r} in {field}. Expected one of: {supported} or an installed custom adapter"
|
|
123
|
+
)
|
|
124
|
+
return value # pragma: no cover
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def resolve_agent(
|
|
128
|
+
cli_agent: object | None,
|
|
129
|
+
manifest_agent: object | None,
|
|
130
|
+
) -> AgentSelection:
|
|
131
|
+
if cli_agent is not None:
|
|
132
|
+
return AgentSelection(validate_agent_id(cli_agent, field="CLI evaluation agent"), "cli")
|
|
133
|
+
if manifest_agent is not None:
|
|
134
|
+
return AgentSelection(
|
|
135
|
+
validate_agent_id(manifest_agent, field="manifest execution.agent"),
|
|
136
|
+
"manifest",
|
|
137
|
+
)
|
|
138
|
+
return AgentSelection(DEFAULT_AGENT, "default")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def get_agent_adapter(agent_id: object) -> AgentAdapter:
|
|
142
|
+
validated = validate_agent_id(agent_id)
|
|
143
|
+
custom = load_custom_agent_spec(validated) if validated not in AGENT_IDS else None
|
|
144
|
+
if custom is not None:
|
|
145
|
+
capabilities = custom.capabilities
|
|
146
|
+
return AgentAdapter(
|
|
147
|
+
id=validated,
|
|
148
|
+
executable=custom.executable,
|
|
149
|
+
required_credentials=custom.credentials,
|
|
150
|
+
model_env=custom.model_env or "",
|
|
151
|
+
base_url_env=custom.base_url_env or "",
|
|
152
|
+
supports_resume=capabilities["resume"],
|
|
153
|
+
supports_single_turn=capabilities["single_turn"],
|
|
154
|
+
supports_structured_events=capabilities["structured_events"],
|
|
155
|
+
supports_telemetry=capabilities["telemetry"],
|
|
156
|
+
supports_skill_invocation_evidence=capabilities["skill_invocation_evidence"],
|
|
157
|
+
custom=True,
|
|
158
|
+
install_kind=custom.install_kind,
|
|
159
|
+
install_package=custom.install_package,
|
|
160
|
+
install_version=custom.install_version,
|
|
161
|
+
)
|
|
162
|
+
executable = {
|
|
163
|
+
"claude-code": "claude",
|
|
164
|
+
"codex": "codex",
|
|
165
|
+
"qoder": "qodercli",
|
|
166
|
+
"codebuddy": "codebuddy",
|
|
167
|
+
}[validated]
|
|
168
|
+
required_credentials = {
|
|
169
|
+
"claude-code": ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"),
|
|
170
|
+
"codex": ("OPENAI_API_KEY", "CODEX_API_KEY"),
|
|
171
|
+
"qoder": ("QODER_PERSONAL_ACCESS_TOKEN",),
|
|
172
|
+
"codebuddy": ("CODEBUDDY_API_KEY", "CODEBUDDY_AUTH_TOKEN"),
|
|
173
|
+
}[validated]
|
|
174
|
+
model_env = {
|
|
175
|
+
"claude-code": "ANTHROPIC_MODEL",
|
|
176
|
+
"codex": "OPENAI_MODEL",
|
|
177
|
+
"qoder": "QODER_MODEL",
|
|
178
|
+
"codebuddy": "CODEBUDDY_MODEL",
|
|
179
|
+
}[validated]
|
|
180
|
+
base_url_env = {
|
|
181
|
+
"claude-code": "ANTHROPIC_BASE_URL",
|
|
182
|
+
"codex": "OPENAI_BASE_URL",
|
|
183
|
+
"qoder": "QODER_BASE_URL",
|
|
184
|
+
"codebuddy": "CODEBUDDY_BASE_URL",
|
|
185
|
+
}[validated]
|
|
186
|
+
return AgentAdapter(validated, executable, required_credentials, model_env, base_url_env)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def normalize_skill_invocations(
|
|
190
|
+
events: object,
|
|
191
|
+
*,
|
|
192
|
+
agent: AgentId | None = None,
|
|
193
|
+
adapter: AgentAdapter | None = None,
|
|
194
|
+
) -> list[str]:
|
|
195
|
+
"""Return Skill names that satisfy the selected Agent's evidence contract.
|
|
196
|
+
|
|
197
|
+
Built-in Agents keep their existing path/tool inference. Custom adapters
|
|
198
|
+
must opt into explicit invocation evidence; otherwise their reported Skill
|
|
199
|
+
names are not treated as observable workflow evidence.
|
|
200
|
+
"""
|
|
201
|
+
if not isinstance(events, dict):
|
|
202
|
+
return []
|
|
203
|
+
inferred = events.get("skills_invoked")
|
|
204
|
+
fallback = [item for item in inferred if isinstance(item, str)] if isinstance(inferred, list) else []
|
|
205
|
+
selected = adapter
|
|
206
|
+
if selected is None:
|
|
207
|
+
try:
|
|
208
|
+
selected = get_agent_adapter(agent or DEFAULT_AGENT)
|
|
209
|
+
except (TypeError, ValueError):
|
|
210
|
+
return [] if isinstance(agent, str) and agent not in AGENT_IDS else fallback
|
|
211
|
+
if not selected.custom:
|
|
212
|
+
return fallback
|
|
213
|
+
if not selected.has_observable_skill_invocation(events):
|
|
214
|
+
return []
|
|
215
|
+
explicit = events.get("skill_invocations")
|
|
216
|
+
return [item for item in explicit if isinstance(item, str)] if isinstance(explicit, list) else []
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def validate_agent_capabilities(
|
|
220
|
+
adapter: AgentAdapter,
|
|
221
|
+
required: tuple[str, ...] = (
|
|
222
|
+
"single_turn",
|
|
223
|
+
"resume",
|
|
224
|
+
"structured_events",
|
|
225
|
+
"skill_invocation_evidence",
|
|
226
|
+
),
|
|
227
|
+
) -> None:
|
|
228
|
+
fields = {
|
|
229
|
+
"single_turn": adapter.supports_single_turn,
|
|
230
|
+
"resume": adapter.supports_resume,
|
|
231
|
+
"structured_events": adapter.supports_structured_events,
|
|
232
|
+
"telemetry": adapter.supports_telemetry,
|
|
233
|
+
"skill_invocation_evidence": adapter.supports_skill_invocation_evidence,
|
|
234
|
+
}
|
|
235
|
+
missing = [name for name in required if not fields.get(name, False)]
|
|
236
|
+
if missing:
|
|
237
|
+
raise ValueError(
|
|
238
|
+
f"Agent {adapter.id} is missing required capabilities: {', '.join(missing)}"
|
|
239
|
+
)
|
|
@@ -18,9 +18,10 @@ import statistics
|
|
|
18
18
|
from collections import Counter, defaultdict
|
|
19
19
|
from dataclasses import dataclass
|
|
20
20
|
from pathlib import Path
|
|
21
|
-
from typing import Any, Iterable, Sequence
|
|
21
|
+
from typing import Any, Iterable, Mapping, Sequence
|
|
22
22
|
|
|
23
23
|
from scaffold.python.logging import extract_events, parse_output
|
|
24
|
+
from scaffold.python.agents import AGENT_IDS
|
|
24
25
|
from scaffold.python.pass_at_k import compute_pass_metrics
|
|
25
26
|
from scaffold.python.sample_quality import quality_from_report
|
|
26
27
|
|
|
@@ -64,6 +65,7 @@ _RUNNER_FILES = (
|
|
|
64
65
|
"scaffold/shell/run-claude-loop.sh",
|
|
65
66
|
"scaffold/shell/decision-point.sh",
|
|
66
67
|
"scaffold/shell/completion-point.sh",
|
|
68
|
+
"scaffold/docker/agent-overlay.Dockerfile",
|
|
67
69
|
)
|
|
68
70
|
_CONTROLLER_FILES = (
|
|
69
71
|
"local/tests/tasks/test_tasks.py",
|
|
@@ -73,6 +75,7 @@ _CONTROLLER_FILES = (
|
|
|
73
75
|
"scaffold/python/profiles.py",
|
|
74
76
|
"scaffold/python/manifests.py",
|
|
75
77
|
"scaffold/python/utils.py",
|
|
78
|
+
"scaffold/python/agents.py",
|
|
76
79
|
"scaffold/python/logging.py",
|
|
77
80
|
"scaffold/python/aligned_comparison.py",
|
|
78
81
|
)
|
|
@@ -94,6 +97,7 @@ class CaseManifest:
|
|
|
94
97
|
controller_hash: str | None
|
|
95
98
|
execution_hash: str | None
|
|
96
99
|
execution_identity: dict[str, str] | None
|
|
100
|
+
authored_hashes: dict[str, str] | None
|
|
97
101
|
source: str
|
|
98
102
|
run_bound: bool
|
|
99
103
|
execution_bound: bool
|
|
@@ -236,13 +240,21 @@ def build_execution_identity(
|
|
|
236
240
|
) -> dict[str, str]:
|
|
237
241
|
"""Build the safe, report-bound execution identity.
|
|
238
242
|
|
|
239
|
-
Docker supplies
|
|
240
|
-
|
|
241
|
-
controller; only bounded enums and digests enter reports.
|
|
243
|
+
Docker supplies identity derived from the immutable image and the actual
|
|
244
|
+
selected CLI version. Model and private interaction text are hashed by the
|
|
245
|
+
controller; only the bounded CLI version, enums, and digests enter reports.
|
|
242
246
|
"""
|
|
243
247
|
if docker_identity.get("schema") != EXECUTION_IDENTITY_SCHEMA:
|
|
244
248
|
raise ValueError("Docker execution identity schema is invalid")
|
|
245
|
-
allowed = {
|
|
249
|
+
allowed = {
|
|
250
|
+
"schema",
|
|
251
|
+
"runtime_image_id",
|
|
252
|
+
*_EXECUTION_IDENTITY_HASH_KEYS[:4],
|
|
253
|
+
"agent",
|
|
254
|
+
"agent_cli",
|
|
255
|
+
"agent_cli_version",
|
|
256
|
+
"agent_cli_version_hash",
|
|
257
|
+
}
|
|
246
258
|
unknown = set(docker_identity) - allowed
|
|
247
259
|
if unknown:
|
|
248
260
|
raise ValueError(f"Docker execution identity has unknown keys: {sorted(unknown)}")
|
|
@@ -256,6 +268,25 @@ def build_execution_identity(
|
|
|
256
268
|
raise ValueError(f"Docker execution identity {key} is invalid")
|
|
257
269
|
if docker_identity["image_id_hash"] != _sha256_bytes(runtime_image_id.encode("utf-8")):
|
|
258
270
|
raise ValueError("Docker execution identity image hash does not match its runtime image")
|
|
271
|
+
agent = docker_identity.get("agent", "claude-code")
|
|
272
|
+
agent_cli = docker_identity.get("agent_cli", "claude")
|
|
273
|
+
agent_cli_version_hash = docker_identity.get(
|
|
274
|
+
"agent_cli_version_hash", docker_identity.get("claude_tool_version_hash")
|
|
275
|
+
)
|
|
276
|
+
agent_cli_version = docker_identity.get("agent_cli_version")
|
|
277
|
+
if agent not in AGENT_IDS:
|
|
278
|
+
raise ValueError(f"Docker execution identity agent is invalid: {agent!r}")
|
|
279
|
+
if not isinstance(agent_cli, str) or not agent_cli:
|
|
280
|
+
raise ValueError("Docker execution identity agent_cli is invalid")
|
|
281
|
+
if not _is_hash(agent_cli_version_hash):
|
|
282
|
+
raise ValueError("Docker execution identity agent_cli_version_hash is invalid")
|
|
283
|
+
if agent_cli_version is not None and (
|
|
284
|
+
not isinstance(agent_cli_version, str)
|
|
285
|
+
or not agent_cli_version
|
|
286
|
+
or len(agent_cli_version) > 512
|
|
287
|
+
or any(ord(character) < 32 for character in agent_cli_version)
|
|
288
|
+
):
|
|
289
|
+
raise ValueError("Docker execution identity agent_cli_version is invalid")
|
|
259
290
|
|
|
260
291
|
mode = getattr(interaction, "mode", None) or "none"
|
|
261
292
|
if mode not in {"none", "auto_user"}:
|
|
@@ -279,7 +310,7 @@ def build_execution_identity(
|
|
|
279
310
|
for key, value in sorted((model_config or {}).items())
|
|
280
311
|
if isinstance(key, str) and isinstance(value, (str, type(None)))
|
|
281
312
|
}
|
|
282
|
-
|
|
313
|
+
identity = {
|
|
283
314
|
"schema": EXECUTION_IDENTITY_SCHEMA,
|
|
284
315
|
**{key: docker_identity[key] for key in _EXECUTION_IDENTITY_HASH_KEYS[:4]},
|
|
285
316
|
"model_selection_hash": _hash_private_value(
|
|
@@ -294,6 +325,17 @@ def build_execution_identity(
|
|
|
294
325
|
"interaction_mode": mode,
|
|
295
326
|
"interaction_max_turns": str(max_turns),
|
|
296
327
|
}
|
|
328
|
+
if "agent" in docker_identity:
|
|
329
|
+
identity.update(
|
|
330
|
+
{
|
|
331
|
+
"agent": agent,
|
|
332
|
+
"agent_cli": agent_cli,
|
|
333
|
+
"agent_cli_version_hash": agent_cli_version_hash,
|
|
334
|
+
}
|
|
335
|
+
)
|
|
336
|
+
if agent_cli_version is not None:
|
|
337
|
+
identity["agent_cli_version"] = agent_cli_version
|
|
338
|
+
return identity
|
|
297
339
|
|
|
298
340
|
|
|
299
341
|
def _validated_execution_identity(raw: Any) -> dict[str, str] | None:
|
|
@@ -306,10 +348,29 @@ def _validated_execution_identity(raw: Any) -> dict[str, str] | None:
|
|
|
306
348
|
"interaction_mode",
|
|
307
349
|
"interaction_max_turns",
|
|
308
350
|
}
|
|
309
|
-
|
|
351
|
+
new_agent_keys = {"agent", "agent_cli", "agent_cli_version_hash"}
|
|
352
|
+
version_key = {"agent_cli_version"}
|
|
353
|
+
raw_keys = set(raw)
|
|
354
|
+
if raw_keys not in (allowed, allowed | new_agent_keys, allowed | new_agent_keys | version_key):
|
|
355
|
+
return None
|
|
356
|
+
if raw.get("schema") != EXECUTION_IDENTITY_SCHEMA:
|
|
310
357
|
return None
|
|
311
358
|
if not all(_is_hash(raw.get(key)) for key in _EXECUTION_IDENTITY_HASH_KEYS):
|
|
312
359
|
return None
|
|
360
|
+
if new_agent_keys.issubset(raw_keys):
|
|
361
|
+
if raw.get("agent") not in AGENT_IDS:
|
|
362
|
+
return None
|
|
363
|
+
if not isinstance(raw.get("agent_cli"), str) or not raw["agent_cli"]:
|
|
364
|
+
return None
|
|
365
|
+
if not _is_hash(raw.get("agent_cli_version_hash")):
|
|
366
|
+
return None
|
|
367
|
+
if "agent_cli_version" in raw_keys and (
|
|
368
|
+
not isinstance(raw.get("agent_cli_version"), str)
|
|
369
|
+
or not raw["agent_cli_version"]
|
|
370
|
+
or len(raw["agent_cli_version"]) > 512
|
|
371
|
+
or any(ord(character) < 32 for character in raw["agent_cli_version"])
|
|
372
|
+
):
|
|
373
|
+
return None
|
|
313
374
|
if raw.get("model_source") not in {"explicit", "runtime-default"}:
|
|
314
375
|
return None
|
|
315
376
|
if raw.get("interaction_mode") not in {"none", "auto_user", "custom"}:
|
|
@@ -320,7 +381,8 @@ def _validated_execution_identity(raw: Any) -> dict[str, str] | None:
|
|
|
320
381
|
return None
|
|
321
382
|
if not 1 <= max_turns <= 1000 or str(max_turns) != raw.get("interaction_max_turns"):
|
|
322
383
|
return None
|
|
323
|
-
|
|
384
|
+
identity_keys = raw_keys
|
|
385
|
+
return {key: raw[key] for key in sorted(identity_keys)}
|
|
324
386
|
|
|
325
387
|
|
|
326
388
|
def build_case_manifest(
|
|
@@ -328,28 +390,73 @@ def build_case_manifest(
|
|
|
328
390
|
tasks_dir: Path,
|
|
329
391
|
*,
|
|
330
392
|
execution_identity: dict[str, str] | None = None,
|
|
393
|
+
manifest_path: Path | None = None,
|
|
394
|
+
rendered_prompt: str | None = None,
|
|
395
|
+
workspace_dir: Path | None = None,
|
|
396
|
+
validation_rules: Mapping[str, Any] | None = None,
|
|
397
|
+
environment_dir: Path | None = None,
|
|
331
398
|
) -> CaseManifest:
|
|
332
399
|
"""Build a manifest for the canonical task and, for new runs, its execution."""
|
|
333
400
|
task_dir = tasks_dir / task
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
401
|
+
native_adapter_hash = _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "native_eval.py")
|
|
402
|
+
authored_hashes: dict[str, str] | None = None
|
|
403
|
+
if manifest_path is not None:
|
|
404
|
+
prompt_bytes = (rendered_prompt or "").encode("utf-8")
|
|
405
|
+
manifest_hash = _hash_tree(manifest_path)
|
|
406
|
+
workspace_hash = (
|
|
407
|
+
_hash_tree(workspace_dir)
|
|
408
|
+
if workspace_dir is not None
|
|
409
|
+
else _sha256_bytes(b"comet-empty-inline-workspace-v1")
|
|
410
|
+
)
|
|
411
|
+
validation_rule_hash = _hash_private_value(dict(validation_rules or {}))
|
|
412
|
+
instruction_hash = _sha256_bytes(b"inline-task-prompt-v1\0" + prompt_bytes)
|
|
413
|
+
task_hash = _hash_private_value(
|
|
414
|
+
{
|
|
415
|
+
"task": task,
|
|
416
|
+
"manifest_hash": manifest_hash,
|
|
417
|
+
"instruction_hash": instruction_hash,
|
|
418
|
+
"workspace_hash": workspace_hash,
|
|
419
|
+
"validation_rule_hash": validation_rule_hash,
|
|
420
|
+
}
|
|
421
|
+
)
|
|
422
|
+
validator_hash = validation_rule_hash
|
|
423
|
+
environment_hash = (
|
|
424
|
+
_hash_tree(environment_dir) if environment_dir else _sha256_bytes(b"missing")
|
|
425
|
+
)
|
|
426
|
+
data_hash = workspace_hash
|
|
427
|
+
prompt_components = {
|
|
428
|
+
"instruction": instruction_hash,
|
|
429
|
+
"task_renderer": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "tasks.py"),
|
|
430
|
+
"native_adapter": native_adapter_hash,
|
|
431
|
+
}
|
|
432
|
+
authored_hashes = {
|
|
433
|
+
"manifest_hash": manifest_hash,
|
|
434
|
+
"task_definition_hash": task_hash,
|
|
435
|
+
"workspace_hash": workspace_hash,
|
|
436
|
+
"validation_rule_hash": validation_rule_hash,
|
|
437
|
+
}
|
|
438
|
+
else:
|
|
439
|
+
instruction_path = task_dir / "instruction.md"
|
|
440
|
+
task_hash = _hash_tree(task_dir / "task.toml")
|
|
441
|
+
instruction_hash = _hash_tree(instruction_path)
|
|
442
|
+
validator_components = {
|
|
443
|
+
"task_validation": _hash_tree(task_dir / "validation"),
|
|
444
|
+
"shared_validation": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "validation"),
|
|
445
|
+
"native_adapter": native_adapter_hash,
|
|
446
|
+
}
|
|
447
|
+
validator_hash = _sha256_bytes(
|
|
448
|
+
json.dumps(validator_components, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
449
|
+
)
|
|
450
|
+
environment_hash = _hash_tree(task_dir / "environment")
|
|
451
|
+
data_hash = _hash_tree(task_dir / "data")
|
|
452
|
+
instruction_bytes = (
|
|
453
|
+
instruction_path.read_bytes() if instruction_path.is_file() else b"missing"
|
|
454
|
+
)
|
|
455
|
+
prompt_components = {
|
|
456
|
+
"instruction": _sha256_bytes(b"canonical-task-prompt-v1\0" + instruction_bytes),
|
|
457
|
+
"task_renderer": _hash_tree(_EVAL_ROOT / "scaffold" / "python" / "tasks.py"),
|
|
458
|
+
"native_adapter": native_adapter_hash,
|
|
459
|
+
}
|
|
353
460
|
prompt_hash = _sha256_bytes(
|
|
354
461
|
json.dumps(prompt_components, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
355
462
|
)
|
|
@@ -403,6 +510,7 @@ def build_case_manifest(
|
|
|
403
510
|
controller_hash=controller_hash,
|
|
404
511
|
execution_hash=execution_hash,
|
|
405
512
|
execution_identity=validated_identity,
|
|
513
|
+
authored_hashes=authored_hashes,
|
|
406
514
|
source=source,
|
|
407
515
|
run_bound=run_bound,
|
|
408
516
|
execution_bound=execution_bound,
|
|
@@ -425,6 +533,8 @@ def case_manifest_payload(manifest: CaseManifest) -> dict[str, Any]:
|
|
|
425
533
|
}
|
|
426
534
|
)
|
|
427
535
|
payload["execution_identity"] = manifest.execution_identity
|
|
536
|
+
if manifest.authored_hashes is not None:
|
|
537
|
+
payload["authored_hashes"] = manifest.authored_hashes
|
|
428
538
|
return payload
|
|
429
539
|
|
|
430
540
|
|
|
@@ -442,10 +552,28 @@ def _embedded_case_manifest(report: dict[str, Any], task: str) -> CaseManifest |
|
|
|
442
552
|
expected_keys = {"schema", "task", "case_hash", *hashed_parts}
|
|
443
553
|
if schema == CASE_MANIFEST_SCHEMA:
|
|
444
554
|
expected_keys.add("execution_identity")
|
|
555
|
+
authored_hashes = raw.get("authored_hashes")
|
|
556
|
+
if authored_hashes is not None:
|
|
557
|
+
expected_keys.add("authored_hashes")
|
|
445
558
|
if set(raw) != expected_keys:
|
|
446
559
|
return None
|
|
447
560
|
if not all(_is_hash(raw.get(key)) for key in hashed_parts):
|
|
448
561
|
return None
|
|
562
|
+
if authored_hashes is not None and (
|
|
563
|
+
not isinstance(authored_hashes, dict)
|
|
564
|
+
or set(authored_hashes)
|
|
565
|
+
!= {
|
|
566
|
+
"manifest_hash",
|
|
567
|
+
"task_definition_hash",
|
|
568
|
+
"workspace_hash",
|
|
569
|
+
"validation_rule_hash",
|
|
570
|
+
}
|
|
571
|
+
or not all(_is_hash(value) for value in authored_hashes.values())
|
|
572
|
+
or authored_hashes["task_definition_hash"] != raw["task_hash"]
|
|
573
|
+
or authored_hashes["workspace_hash"] != raw["data_hash"]
|
|
574
|
+
or authored_hashes["validation_rule_hash"] != raw["validator_hash"]
|
|
575
|
+
):
|
|
576
|
+
return None
|
|
449
577
|
case_hash = raw.get("case_hash")
|
|
450
578
|
if not isinstance(case_hash, str) or not case_hash.startswith("sha256:"):
|
|
451
579
|
return None
|
|
@@ -477,6 +605,7 @@ def _embedded_case_manifest(report: dict[str, Any], task: str) -> CaseManifest |
|
|
|
477
605
|
controller_hash=raw.get("controller_hash"),
|
|
478
606
|
execution_hash=raw.get("execution_hash"),
|
|
479
607
|
execution_identity=execution_identity,
|
|
608
|
+
authored_hashes=authored_hashes,
|
|
480
609
|
source="report-bound-v2" if schema == CASE_MANIFEST_SCHEMA else "report-bound-v1",
|
|
481
610
|
run_bound=True,
|
|
482
611
|
execution_bound=schema == CASE_MANIFEST_SCHEMA,
|