@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -81,8 +81,6 @@ def test_unit_test_detection_keeps_task_runs_as_experiments():
|
|
|
81
81
|
assert conftest._is_unit_tests_only(Config()) is False
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
|
|
85
|
-
|
|
86
84
|
def test_experiment_id_uses_explicit_comet_eval_run_id(monkeypatch):
|
|
87
85
|
monkeypatch.setenv("COMET_EVAL_EXPERIMENT_ID", "comet-eval-1234")
|
|
88
86
|
|
|
@@ -97,6 +95,125 @@ def test_experiment_id_rejects_unsafe_explicit_value(monkeypatch):
|
|
|
97
95
|
conftest._get_or_create_experiment_id("ignored", False)
|
|
98
96
|
|
|
99
97
|
|
|
98
|
+
def test_auto_generation_loads_environment_before_resolving_cache_identity(
|
|
99
|
+
tmp_path: Path, monkeypatch
|
|
100
|
+
):
|
|
101
|
+
skill = tmp_path / "skill"
|
|
102
|
+
skill.mkdir()
|
|
103
|
+
(skill / "SKILL.md").write_text("Evaluate me.\n", encoding="utf-8")
|
|
104
|
+
captured = {}
|
|
105
|
+
|
|
106
|
+
def load_environment():
|
|
107
|
+
monkeypatch.setenv("BENCH_CODEX_MODEL", "model-from-dotenv")
|
|
108
|
+
|
|
109
|
+
def generate_manifest(_skill_path, _project_root, **kwargs):
|
|
110
|
+
captured.update(kwargs)
|
|
111
|
+
manifest_path = tmp_path / "generated.yaml"
|
|
112
|
+
manifest_path.write_text(
|
|
113
|
+
"""apiVersion: comet.eval/v1alpha1
|
|
114
|
+
kind: SkillEvalManifest
|
|
115
|
+
metadata: {name: generated}
|
|
116
|
+
skill: {name: generated, source: .}
|
|
117
|
+
evaluation:
|
|
118
|
+
tasks:
|
|
119
|
+
- name: generated-one
|
|
120
|
+
prompt: one
|
|
121
|
+
expect: {files: [one.md]}
|
|
122
|
+
- name: generated-two
|
|
123
|
+
prompt: two
|
|
124
|
+
expect: {files: [two.md]}
|
|
125
|
+
""",
|
|
126
|
+
encoding="utf-8",
|
|
127
|
+
)
|
|
128
|
+
return SimpleNamespace(manifest_path=manifest_path)
|
|
129
|
+
|
|
130
|
+
options = {
|
|
131
|
+
"--task": None,
|
|
132
|
+
"--quick": False,
|
|
133
|
+
"--eval-manifest": None,
|
|
134
|
+
"--skill-path": str(skill),
|
|
135
|
+
"--profile": None,
|
|
136
|
+
"--project-root": str(tmp_path),
|
|
137
|
+
"--agent": "codex",
|
|
138
|
+
"--skill-name": None,
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
class Config:
|
|
142
|
+
option = SimpleNamespace(collectonly=False, eval_manifest=None)
|
|
143
|
+
|
|
144
|
+
@staticmethod
|
|
145
|
+
def getoption(name):
|
|
146
|
+
return options[name]
|
|
147
|
+
|
|
148
|
+
monkeypatch.setattr("scaffold.python.utils.load_eval_environment", load_environment)
|
|
149
|
+
monkeypatch.setattr(conftest, "ensure_generated_manifest", generate_manifest)
|
|
150
|
+
|
|
151
|
+
config = Config()
|
|
152
|
+
conftest._ensure_auto_generated_manifest(config)
|
|
153
|
+
|
|
154
|
+
assert captured["agent"] == "codex"
|
|
155
|
+
assert captured["model"] == "model-from-dotenv"
|
|
156
|
+
assert config.option.eval_manifest is None
|
|
157
|
+
assert config._comet_frozen_task_set.source == "generated-cache"
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def test_taskless_collect_only_marks_generation_pending_without_agent_or_disk_work(
|
|
161
|
+
tmp_path: Path, monkeypatch
|
|
162
|
+
):
|
|
163
|
+
skill = tmp_path / "skill"
|
|
164
|
+
skill.mkdir()
|
|
165
|
+
(skill / "SKILL.md").write_text("Evaluate me.\n", encoding="utf-8")
|
|
166
|
+
called = False
|
|
167
|
+
|
|
168
|
+
def fail_generation(*_args, **_kwargs):
|
|
169
|
+
nonlocal called
|
|
170
|
+
called = True
|
|
171
|
+
raise AssertionError("task generation must not run during collect-only")
|
|
172
|
+
|
|
173
|
+
class Config:
|
|
174
|
+
option = SimpleNamespace(collectonly=True, eval_manifest=None, skill_name=None)
|
|
175
|
+
|
|
176
|
+
@staticmethod
|
|
177
|
+
def getoption(name):
|
|
178
|
+
return {
|
|
179
|
+
"--task": None,
|
|
180
|
+
"--quick": False,
|
|
181
|
+
"--eval-manifest": None,
|
|
182
|
+
"--skill-path": str(skill),
|
|
183
|
+
"--profile": None,
|
|
184
|
+
"--project-root": str(tmp_path),
|
|
185
|
+
"--agent": None,
|
|
186
|
+
"--skill-name": None,
|
|
187
|
+
}[name]
|
|
188
|
+
|
|
189
|
+
monkeypatch.setattr(conftest, "ensure_generated_manifest", fail_generation)
|
|
190
|
+
config = Config()
|
|
191
|
+
conftest._ensure_auto_generated_manifest(config)
|
|
192
|
+
|
|
193
|
+
assert called is False
|
|
194
|
+
assert config._comet_frozen_task_set.source == "pending-generation"
|
|
195
|
+
assert config._comet_frozen_task_set.tasks == ()
|
|
196
|
+
assert config._comet_resolution_manifest.skill_path == skill.resolve()
|
|
197
|
+
assert config._comet_resolution_manifest.path == skill / "comet" / "eval.yaml"
|
|
198
|
+
assert config._comet_resolved_tasks == {}
|
|
199
|
+
|
|
200
|
+
from local.tests.tasks import test_tasks
|
|
201
|
+
|
|
202
|
+
monkeypatch.setattr(test_tasks, "load_treatments", lambda: {})
|
|
203
|
+
monkeypatch.setattr(test_tasks, "list_tasks", lambda: [])
|
|
204
|
+
frozen = config._comet_frozen_task_set
|
|
205
|
+
assert test_tasks.generate_test_params(None, None, config) == []
|
|
206
|
+
assert test_tasks.generate_test_params(None, None, config) == []
|
|
207
|
+
assert config._comet_frozen_task_set is frozen
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def test_selected_agent_missing_credentials_fails_before_workloads():
|
|
211
|
+
with pytest.raises(pytest.UsageError, match="Credentials for codebuddy not set"):
|
|
212
|
+
conftest._require_agent_credentials("codebuddy", {})
|
|
213
|
+
|
|
214
|
+
conftest._require_agent_credentials("codebuddy", {"CODEBUDDY_API_KEY": "configured"})
|
|
215
|
+
|
|
216
|
+
|
|
100
217
|
def test_extract_loop_turns_reads_driver_completion_line():
|
|
101
218
|
stderr = (
|
|
102
219
|
"[loop] turn 1/4\n"
|
|
@@ -113,6 +230,21 @@ def test_extract_loop_turns_reads_driver_completion_line():
|
|
|
113
230
|
"deterministic_replies": 1,
|
|
114
231
|
"completion_signals": 1,
|
|
115
232
|
"fresh_resume_boundaries": 0,
|
|
233
|
+
"role_sessions": {"subject": [], "simulator": [], "judge": []},
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def test_extract_loop_interaction_records_isolated_role_sessions():
|
|
238
|
+
interaction = conftest._extract_loop_interaction(
|
|
239
|
+
"[loop] role-session subject subject-1\n"
|
|
240
|
+
"[loop] role-session simulator simulator-1\n"
|
|
241
|
+
"[loop] role-session subject subject-1\n"
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
assert interaction["role_sessions"] == {
|
|
245
|
+
"subject": ["subject-1"],
|
|
246
|
+
"simulator": ["simulator-1"],
|
|
247
|
+
"judge": [],
|
|
116
248
|
}
|
|
117
249
|
assert conftest._extract_loop_turns("ordinary stderr") is None
|
|
118
250
|
|
|
@@ -383,6 +515,7 @@ def test_dynamic_treatment_config_from_skill_path(tmp_path: Path):
|
|
|
383
515
|
"--skill-path": str(skill_dir),
|
|
384
516
|
"--skill-name": "demo-skill",
|
|
385
517
|
"--profile": "generic",
|
|
518
|
+
"--quick": True,
|
|
386
519
|
}
|
|
387
520
|
return values.get(name)
|
|
388
521
|
|
|
@@ -577,6 +710,49 @@ def test_setup_test_context_configures_039_shell_hook(tmp_path: Path, setup_test
|
|
|
577
710
|
assert command == "bash /workspace/.claude/skills/comet/scripts/comet-hook-guard.sh"
|
|
578
711
|
|
|
579
712
|
|
|
713
|
+
def test_codex_guard_uses_codex_hook_config_while_scripts_remain_under_agents(tmp_path: Path):
|
|
714
|
+
command = "node /workspace/.agents/skills/comet/scripts/comet-hook-guard.mjs"
|
|
715
|
+
|
|
716
|
+
conftest._ensure_agent_pre_tool_hook(tmp_path, "codex", command)
|
|
717
|
+
|
|
718
|
+
hooks_path = tmp_path / ".codex" / "hooks.json"
|
|
719
|
+
hooks = json.loads(hooks_path.read_text(encoding="utf-8"))
|
|
720
|
+
assert hooks["hooks"]["PreToolUse"][0]["hooks"][0]["command"] == command
|
|
721
|
+
assert not (tmp_path / ".agents" / "hooks.json").exists()
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def test_transcript_setup_failure_is_best_effort(tmp_path: Path, monkeypatch, capsys):
|
|
725
|
+
monkeypatch.setenv("TRACE_TO_LANGFUSE", "true")
|
|
726
|
+
monkeypatch.delenv("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED", raising=False)
|
|
727
|
+
monkeypatch.setattr(
|
|
728
|
+
conftest,
|
|
729
|
+
"install_transcript_hook",
|
|
730
|
+
lambda *_args: (_ for _ in ()).throw(ValueError("invalid settings")),
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
conftest._ensure_langfuse_trajectory_support(tmp_path, "codebuddy")
|
|
734
|
+
|
|
735
|
+
assert "optional codebuddy transcript setup skipped" in capsys.readouterr().err
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def test_setup_test_context_writes_codebuddy_project_memory(
|
|
739
|
+
tmp_path: Path,
|
|
740
|
+
setup_test_context,
|
|
741
|
+
monkeypatch,
|
|
742
|
+
):
|
|
743
|
+
monkeypatch.setattr(
|
|
744
|
+
conftest,
|
|
745
|
+
"_resolve_eval_agent",
|
|
746
|
+
lambda _config: SimpleNamespace(agent="codebuddy"),
|
|
747
|
+
)
|
|
748
|
+
|
|
749
|
+
setup_test_context(claude_md="Use the CodeBuddy workflow.")
|
|
750
|
+
|
|
751
|
+
memory = tmp_path / ".codebuddy" / "CODEBUDDY.md"
|
|
752
|
+
assert memory.read_text(encoding="utf-8") == "Use the CodeBuddy workflow."
|
|
753
|
+
assert not (tmp_path / "AGENTS.md").exists()
|
|
754
|
+
|
|
755
|
+
|
|
580
756
|
def test_setup_test_context_copies_dependency_skill_packages_with_scripts(
|
|
581
757
|
tmp_path: Path, setup_test_context
|
|
582
758
|
):
|
|
@@ -879,6 +1055,11 @@ def test_build_report_payload_persists_sample_quality():
|
|
|
879
1055
|
"total_tokens": 100,
|
|
880
1056
|
"total_cost_usd": 0.01,
|
|
881
1057
|
"case_manifest": {"case_hash": "sha256:demo"},
|
|
1058
|
+
"role_models": {
|
|
1059
|
+
"subject": "subject-model",
|
|
1060
|
+
"simulator": None,
|
|
1061
|
+
"judge": "judge-model",
|
|
1062
|
+
},
|
|
882
1063
|
},
|
|
883
1064
|
passed=["[RUBRIC] weighted_score: 1.00"],
|
|
884
1065
|
failed=[],
|
|
@@ -894,6 +1075,11 @@ def test_build_report_payload_persists_sample_quality():
|
|
|
894
1075
|
assert report["sample_quality"]["reason_code"] == "valid_signal"
|
|
895
1076
|
assert report["sample_quality"]["include_in_analysis"] is True
|
|
896
1077
|
assert report["events_summary"]["case_manifest"] == {"case_hash": "sha256:demo"}
|
|
1078
|
+
assert report["events_summary"]["role_models"] == {
|
|
1079
|
+
"subject": "subject-model",
|
|
1080
|
+
"simulator": None,
|
|
1081
|
+
"judge": "judge-model",
|
|
1082
|
+
}
|
|
897
1083
|
|
|
898
1084
|
|
|
899
1085
|
def test_build_report_payload_marks_timeout_as_excluded():
|
|
@@ -941,36 +1127,60 @@ def test_prebuild_docker_image_allows_a_cold_image_build_to_run_for_fifteen_minu
|
|
|
941
1127
|
]
|
|
942
1128
|
|
|
943
1129
|
|
|
944
|
-
def
|
|
945
|
-
tmp_path: Path,
|
|
946
|
-
):
|
|
947
|
-
tasks_dir = tmp_path / "tasks"
|
|
948
|
-
for task_name in ("authoring-skill-smoke", "workflow-route-conformance", "unrelated"):
|
|
949
|
-
environment = tasks_dir / task_name / "environment"
|
|
950
|
-
environment.mkdir(parents=True)
|
|
951
|
-
(environment / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
|
|
952
|
-
|
|
1130
|
+
def test_resolve_eval_agent_applies_cli_manifest_precedence(tmp_path: Path):
|
|
953
1131
|
manifest = tmp_path / "eval.yaml"
|
|
954
1132
|
manifest.write_text(
|
|
955
1133
|
"""
|
|
956
1134
|
apiVersion: comet.eval/v1alpha1
|
|
957
1135
|
kind: SkillEvalManifest
|
|
958
1136
|
metadata:
|
|
959
|
-
name: selected-
|
|
1137
|
+
name: selected-agent
|
|
960
1138
|
skill:
|
|
961
|
-
name: selected-
|
|
1139
|
+
name: selected-agent
|
|
962
1140
|
source: .
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
- authoring-skill-smoke
|
|
966
|
-
- workflow-route-conformance
|
|
1141
|
+
execution:
|
|
1142
|
+
agent: codex
|
|
967
1143
|
""",
|
|
968
1144
|
encoding="utf-8",
|
|
969
1145
|
)
|
|
970
1146
|
|
|
971
1147
|
class Config:
|
|
1148
|
+
def __init__(self, agent):
|
|
1149
|
+
self.agent = agent
|
|
1150
|
+
|
|
972
1151
|
def getoption(self, name):
|
|
973
|
-
return {
|
|
1152
|
+
return {
|
|
1153
|
+
"--eval-manifest": str(manifest),
|
|
1154
|
+
"--agent": self.agent,
|
|
1155
|
+
}.get(name)
|
|
1156
|
+
|
|
1157
|
+
assert conftest._resolve_eval_agent(Config(None)).agent == "codex"
|
|
1158
|
+
assert conftest._resolve_eval_agent(Config("qoder")).agent == "qoder"
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def test_selected_agent_uses_its_native_project_skill_root():
|
|
1162
|
+
assert conftest._agent_project_root("claude-code") == ".claude"
|
|
1163
|
+
assert conftest._agent_project_root("codex") == ".agents"
|
|
1164
|
+
assert conftest._agent_project_root("qoder") == ".qoder"
|
|
1165
|
+
assert conftest._agent_project_root("codebuddy") == ".codebuddy"
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
def test_docker_prebuild_uses_only_the_tasks_selected_by_an_eval_manifest(
|
|
1169
|
+
tmp_path: Path,
|
|
1170
|
+
):
|
|
1171
|
+
tasks_dir = tmp_path / "tasks"
|
|
1172
|
+
for task_name in ("authoring-skill-smoke", "workflow-route-conformance", "unrelated"):
|
|
1173
|
+
environment = tasks_dir / task_name / "environment"
|
|
1174
|
+
environment.mkdir(parents=True)
|
|
1175
|
+
(environment / "Dockerfile").write_text("FROM python:3.11-slim\n", encoding="utf-8")
|
|
1176
|
+
|
|
1177
|
+
class Config:
|
|
1178
|
+
_comet_frozen_task_set = SimpleNamespace(
|
|
1179
|
+
tasks=(
|
|
1180
|
+
SimpleNamespace(task=SimpleNamespace(environment_dir=tasks_dir / "authoring-skill-smoke" / "environment")),
|
|
1181
|
+
SimpleNamespace(task=SimpleNamespace(environment_dir=tasks_dir / "workflow-route-conformance" / "environment")),
|
|
1182
|
+
)
|
|
1183
|
+
)
|
|
974
1184
|
|
|
975
1185
|
request = SimpleNamespace(config=Config())
|
|
976
1186
|
|
|
@@ -978,3 +1188,42 @@ evaluation:
|
|
|
978
1188
|
tasks_dir / "authoring-skill-smoke" / "environment",
|
|
979
1189
|
tasks_dir / "workflow-route-conformance" / "environment",
|
|
980
1190
|
]
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
@pytest.mark.parametrize(
|
|
1194
|
+
"source,selected",
|
|
1195
|
+
[
|
|
1196
|
+
("explicit", ["authoring-skill-smoke"]),
|
|
1197
|
+
("quick", ["workflow-route-conformance"]),
|
|
1198
|
+
("authored", ["authoring-skill-smoke"]),
|
|
1199
|
+
("recommended", ["workflow-route-conformance"]),
|
|
1200
|
+
("generated-cache", ["authoring-skill-smoke"]),
|
|
1201
|
+
("source", ["source-custom"]),
|
|
1202
|
+
],
|
|
1203
|
+
)
|
|
1204
|
+
def test_docker_prebuild_consumes_only_frozen_task_environments(tmp_path: Path, source, selected):
|
|
1205
|
+
tasks_dir = tmp_path / "tasks"
|
|
1206
|
+
environments = {}
|
|
1207
|
+
for name in ("authoring-skill-smoke", "workflow-route-conformance", "source-custom", "unrelated"):
|
|
1208
|
+
environment = tasks_dir / name / "environment"
|
|
1209
|
+
environment.mkdir(parents=True)
|
|
1210
|
+
(environment / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
|
|
1211
|
+
environments[name] = environment
|
|
1212
|
+
|
|
1213
|
+
frozen = SimpleNamespace(
|
|
1214
|
+
source=source,
|
|
1215
|
+
generation=SimpleNamespace(generation_hash="frozen-hash") if source == "generated-cache" else None,
|
|
1216
|
+
tasks=tuple(
|
|
1217
|
+
SimpleNamespace(
|
|
1218
|
+
name=name,
|
|
1219
|
+
provenance="source" if name == "source-custom" else source,
|
|
1220
|
+
task=SimpleNamespace(environment_dir=environments[name]),
|
|
1221
|
+
)
|
|
1222
|
+
for name in [*selected, *selected]
|
|
1223
|
+
),
|
|
1224
|
+
)
|
|
1225
|
+
request = SimpleNamespace(config=SimpleNamespace(_comet_frozen_task_set=frozen))
|
|
1226
|
+
|
|
1227
|
+
assert conftest._docker_environment_dirs_for_request(request, tasks_dir) == [
|
|
1228
|
+
environments[name] for name in selected
|
|
1229
|
+
]
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Contract tests for explicitly installed custom Eval Agent adapters."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from scaffold.python.agents import (
|
|
8
|
+
get_agent_adapter,
|
|
9
|
+
normalize_skill_invocations,
|
|
10
|
+
validate_agent_capabilities,
|
|
11
|
+
)
|
|
12
|
+
from scaffold.python.custom_agents import discover_custom_agent_specs
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _write_adapter(root: Path, *, name: str = "fixture-agent", **overrides) -> Path:
|
|
16
|
+
directory = root / name
|
|
17
|
+
directory.mkdir(parents=True)
|
|
18
|
+
capabilities = overrides.pop(
|
|
19
|
+
"capabilities",
|
|
20
|
+
{
|
|
21
|
+
"singleTurn": True,
|
|
22
|
+
"resume": True,
|
|
23
|
+
"structuredEvents": True,
|
|
24
|
+
"telemetry": False,
|
|
25
|
+
"skillInvocationEvidence": True,
|
|
26
|
+
},
|
|
27
|
+
)
|
|
28
|
+
runtime = overrides.pop(
|
|
29
|
+
"runtime",
|
|
30
|
+
{"executable": "fixture-agent", "install": {"kind": "none"}},
|
|
31
|
+
)
|
|
32
|
+
credentials = overrides.pop("credentials", ["FIXTURE_AGENT_API_KEY"])
|
|
33
|
+
lines = [
|
|
34
|
+
"apiVersion: comet.eval.agent/v1alpha1",
|
|
35
|
+
"kind: EvalAgentAdapter",
|
|
36
|
+
"metadata:",
|
|
37
|
+
f" id: {name}",
|
|
38
|
+
" version: '1.0.0'",
|
|
39
|
+
"runtime:",
|
|
40
|
+
f" executable: {runtime['executable']}",
|
|
41
|
+
" install:",
|
|
42
|
+
f" kind: {runtime['install']['kind']}",
|
|
43
|
+
"credentials:",
|
|
44
|
+
*[f" - {item}" for item in credentials],
|
|
45
|
+
"modelEnv: FIXTURE_AGENT_MODEL",
|
|
46
|
+
"baseUrlEnv: FIXTURE_AGENT_BASE_URL",
|
|
47
|
+
"capabilities:",
|
|
48
|
+
*[f" {key}: {'true' if value else 'false'}" for key, value in capabilities.items()],
|
|
49
|
+
]
|
|
50
|
+
for key, value in overrides.items():
|
|
51
|
+
lines.append(f"{key}: {value}")
|
|
52
|
+
path = directory / "adapter.yaml"
|
|
53
|
+
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
54
|
+
return path
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_tracked_fixture_can_be_installed_as_a_custom_adapter(
|
|
58
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
59
|
+
):
|
|
60
|
+
fixture = Path(__file__).parent / "fixtures" / "custom-agent" / "adapter.yaml"
|
|
61
|
+
destination = tmp_path / "fixture-agent"
|
|
62
|
+
destination.mkdir()
|
|
63
|
+
(destination / "adapter.yaml").write_text(fixture.read_text(encoding="utf-8"), encoding="utf-8")
|
|
64
|
+
monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
|
|
65
|
+
|
|
66
|
+
adapter = get_agent_adapter("fixture-agent")
|
|
67
|
+
|
|
68
|
+
assert adapter.custom is True
|
|
69
|
+
assert adapter.executable == "fixture-agent"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_installed_custom_adapter_is_discovered_and_generates_standard_command(
|
|
73
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
74
|
+
):
|
|
75
|
+
_write_adapter(tmp_path)
|
|
76
|
+
monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
|
|
77
|
+
|
|
78
|
+
specs = discover_custom_agent_specs()
|
|
79
|
+
adapter = get_agent_adapter("fixture-agent")
|
|
80
|
+
|
|
81
|
+
assert list(specs) == ["fixture-agent"]
|
|
82
|
+
assert adapter.custom is True
|
|
83
|
+
assert adapter.executable == "fixture-agent"
|
|
84
|
+
assert adapter.required_credentials == ("FIXTURE_AGENT_API_KEY",)
|
|
85
|
+
assert adapter.supports_telemetry is False
|
|
86
|
+
assert adapter.build_run_command(
|
|
87
|
+
"Do it.", model="fixture-model", role="subject", resume_id="session-1"
|
|
88
|
+
) == [
|
|
89
|
+
"fixture-agent",
|
|
90
|
+
"-p",
|
|
91
|
+
"Do it.",
|
|
92
|
+
"--output-format",
|
|
93
|
+
"stream-json",
|
|
94
|
+
"--model",
|
|
95
|
+
"fixture-model",
|
|
96
|
+
"--resume",
|
|
97
|
+
"session-1",
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@pytest.mark.parametrize(
|
|
102
|
+
"capability",
|
|
103
|
+
["single_turn", "resume", "structured_events", "skill_invocation_evidence"],
|
|
104
|
+
)
|
|
105
|
+
def test_missing_required_custom_capability_fails_preflight(
|
|
106
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capability: str
|
|
107
|
+
):
|
|
108
|
+
capabilities = {
|
|
109
|
+
"singleTurn": True,
|
|
110
|
+
"resume": True,
|
|
111
|
+
"structuredEvents": True,
|
|
112
|
+
"telemetry": False,
|
|
113
|
+
"skillInvocationEvidence": True,
|
|
114
|
+
}
|
|
115
|
+
key = {
|
|
116
|
+
"single_turn": "singleTurn",
|
|
117
|
+
"resume": "resume",
|
|
118
|
+
"structured_events": "structuredEvents",
|
|
119
|
+
"skill_invocation_evidence": "skillInvocationEvidence",
|
|
120
|
+
}[capability]
|
|
121
|
+
capabilities[key] = False
|
|
122
|
+
_write_adapter(tmp_path, capabilities=capabilities)
|
|
123
|
+
monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
|
|
124
|
+
|
|
125
|
+
with pytest.raises(ValueError, match=capability):
|
|
126
|
+
validate_agent_capabilities(get_agent_adapter("fixture-agent"))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_custom_registry_rejects_malformed_duplicate_and_escape_entries(
|
|
130
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
131
|
+
):
|
|
132
|
+
_write_adapter(tmp_path, name="first")
|
|
133
|
+
second = _write_adapter(tmp_path, name="second")
|
|
134
|
+
second.write_text(second.read_text(encoding="utf-8").replace("id: second", "id: first"), encoding="utf-8")
|
|
135
|
+
outside = tmp_path.parent / "outside-adapter.yaml"
|
|
136
|
+
outside.write_text("not an adapter\n", encoding="utf-8")
|
|
137
|
+
escaped = tmp_path / "escaped"
|
|
138
|
+
try:
|
|
139
|
+
escaped.symlink_to(outside.parent, target_is_directory=True)
|
|
140
|
+
except OSError:
|
|
141
|
+
pass
|
|
142
|
+
monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
|
|
143
|
+
|
|
144
|
+
with pytest.raises(ValueError, match="duplicate|outside|invalid|match|escapes"):
|
|
145
|
+
discover_custom_agent_specs()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def test_custom_adapter_does_not_infer_skill_invocation_from_artifacts(
|
|
149
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
150
|
+
):
|
|
151
|
+
_write_adapter(tmp_path)
|
|
152
|
+
monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
|
|
153
|
+
adapter = get_agent_adapter("fixture-agent")
|
|
154
|
+
|
|
155
|
+
assert adapter.has_observable_skill_invocation({"artifacts": ["SKILL.md"]}) is False
|
|
156
|
+
assert adapter.has_observable_skill_invocation({"skill_invocations": ["fixture-agent"]}) is True
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def test_custom_adapter_capability_filters_non_observable_skill_evidence(
|
|
160
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
161
|
+
):
|
|
162
|
+
capabilities = {
|
|
163
|
+
"singleTurn": True,
|
|
164
|
+
"resume": True,
|
|
165
|
+
"structuredEvents": True,
|
|
166
|
+
"telemetry": False,
|
|
167
|
+
"skillInvocationEvidence": False,
|
|
168
|
+
}
|
|
169
|
+
_write_adapter(tmp_path, capabilities=capabilities)
|
|
170
|
+
monkeypatch.setenv("COMET_EVAL_ADAPTERS_DIR", str(tmp_path))
|
|
171
|
+
adapter = get_agent_adapter("fixture-agent")
|
|
172
|
+
events = {"skills_invoked": ["demo"], "skill_invocations": ["demo"]}
|
|
173
|
+
|
|
174
|
+
assert normalize_skill_invocations(events, adapter=adapter) == []
|