@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -2,9 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
|
+
import re
|
|
6
|
+
|
|
5
7
|
import pytest
|
|
6
8
|
|
|
7
9
|
from scaffold.python.manifests import load_eval_manifest
|
|
10
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
8
11
|
|
|
9
12
|
|
|
10
13
|
def test_load_eval_manifest_parses_skill_package_metadata(tmp_path: Path):
|
|
@@ -55,6 +58,14 @@ interaction:
|
|
|
55
58
|
maxTurns: 8
|
|
56
59
|
simulatorPrompt: Answer concisely.
|
|
57
60
|
freshResumeMarker: COLD_RESUME_READY
|
|
61
|
+
execution:
|
|
62
|
+
agent: codex
|
|
63
|
+
model: gpt-main
|
|
64
|
+
baseUrl: https://main.example.com/v1
|
|
65
|
+
judge:
|
|
66
|
+
agent: claude-code
|
|
67
|
+
model: claude-judge
|
|
68
|
+
baseUrl: https://judge.example.com
|
|
58
69
|
""",
|
|
59
70
|
encoding="utf-8",
|
|
60
71
|
)
|
|
@@ -81,6 +92,68 @@ interaction:
|
|
|
81
92
|
assert manifest.interaction.max_turns == 8
|
|
82
93
|
assert manifest.interaction.simulator_prompt == "Answer concisely."
|
|
83
94
|
assert manifest.interaction.fresh_resume_marker == "COLD_RESUME_READY"
|
|
95
|
+
assert manifest.execution_agent == "codex"
|
|
96
|
+
assert manifest.execution.model == "gpt-main"
|
|
97
|
+
assert manifest.execution.base_url == "https://main.example.com/v1"
|
|
98
|
+
assert manifest.judge is not None
|
|
99
|
+
assert manifest.judge.agent == "claude-code"
|
|
100
|
+
assert manifest.judge.model == "claude-judge"
|
|
101
|
+
assert manifest.judge.base_url == "https://judge.example.com"
|
|
102
|
+
assert manifest.raw["execution"]["model"] == "gpt-main"
|
|
103
|
+
assert manifest.raw["judge"]["model"] == "claude-judge"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_load_eval_manifest_normalizes_snake_case_aliases(tmp_path: Path):
|
|
107
|
+
package = tmp_path / "my-skill"
|
|
108
|
+
(package / "comet").mkdir(parents=True)
|
|
109
|
+
manifest_path = package / "comet" / "eval.yaml"
|
|
110
|
+
manifest_path.write_text(
|
|
111
|
+
"""
|
|
112
|
+
apiVersion: comet.eval/v1alpha1
|
|
113
|
+
kind: SkillEvalManifest
|
|
114
|
+
metadata:
|
|
115
|
+
name: my-skill
|
|
116
|
+
draft_hash: '0000000000000000000000000000000000000000000000000000000000000000'
|
|
117
|
+
generation_hash: generated
|
|
118
|
+
skill:
|
|
119
|
+
name: my-skill
|
|
120
|
+
source: ..
|
|
121
|
+
evaluation:
|
|
122
|
+
recommended_tasks: [generic-skill-smoke]
|
|
123
|
+
required_skills: [my-skill]
|
|
124
|
+
expected_artifacts: [result.md]
|
|
125
|
+
generated_node_skills: [my-skill-open]
|
|
126
|
+
route_conformance:
|
|
127
|
+
task: generic-skill-smoke
|
|
128
|
+
expected_node_order: [open, build]
|
|
129
|
+
interaction:
|
|
130
|
+
max_turns: 4
|
|
131
|
+
simulator_prompt: Answer.
|
|
132
|
+
fresh_resume_marker: READY
|
|
133
|
+
execution:
|
|
134
|
+
base_url: https://main.example.com
|
|
135
|
+
judge:
|
|
136
|
+
model: judge-model
|
|
137
|
+
base_url: https://judge.example.com
|
|
138
|
+
""",
|
|
139
|
+
encoding="utf-8",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
manifest = load_eval_manifest(manifest_path)
|
|
143
|
+
|
|
144
|
+
assert manifest.draft_hash == "0" * 64
|
|
145
|
+
assert manifest.generation_hash == "generated"
|
|
146
|
+
assert manifest.recommended_tasks == ["generic-skill-smoke"]
|
|
147
|
+
assert manifest.required_skills == ["my-skill"]
|
|
148
|
+
assert manifest.expected_artifacts == ["result.md"]
|
|
149
|
+
assert manifest.generated_node_skills == ["my-skill-open"]
|
|
150
|
+
assert manifest.route_conformance_expected_node_order == ["open", "build"]
|
|
151
|
+
assert manifest.interaction.max_turns == 4
|
|
152
|
+
assert manifest.interaction.simulator_prompt == "Answer."
|
|
153
|
+
assert manifest.interaction.fresh_resume_marker == "READY"
|
|
154
|
+
assert manifest.execution.base_url == "https://main.example.com"
|
|
155
|
+
assert manifest.judge is not None
|
|
156
|
+
assert manifest.judge.base_url == "https://judge.example.com"
|
|
84
157
|
|
|
85
158
|
|
|
86
159
|
def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
|
|
@@ -94,6 +167,37 @@ def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
|
|
|
94
167
|
load_eval_manifest(manifest_path)
|
|
95
168
|
|
|
96
169
|
|
|
170
|
+
@pytest.mark.parametrize(
|
|
171
|
+
("section", "message"),
|
|
172
|
+
[
|
|
173
|
+
("metadata:\n draftHash: one\n draft_hash: two", "metadata.draftHash"),
|
|
174
|
+
(
|
|
175
|
+
"evaluation:\n recommendedTasks: [one]\n recommended_tasks: [two]",
|
|
176
|
+
"evaluation.recommendedTasks",
|
|
177
|
+
),
|
|
178
|
+
(
|
|
179
|
+
"execution:\n baseUrl: https://main.example\n base_url: https://other.example",
|
|
180
|
+
"execution.baseUrl",
|
|
181
|
+
),
|
|
182
|
+
],
|
|
183
|
+
)
|
|
184
|
+
def test_load_eval_manifest_rejects_conflicting_aliases(
|
|
185
|
+
tmp_path: Path, section: str, message: str
|
|
186
|
+
):
|
|
187
|
+
manifest_path = tmp_path / "eval.yaml"
|
|
188
|
+
manifest_path.write_text(
|
|
189
|
+
"apiVersion: comet.eval/v1alpha1\n"
|
|
190
|
+
"kind: SkillEvalManifest\n"
|
|
191
|
+
"metadata:\n name: demo\n"
|
|
192
|
+
"skill:\n name: demo\n"
|
|
193
|
+
f"{section}\n",
|
|
194
|
+
encoding="utf-8",
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
with pytest.raises(ValueError, match=re.escape(message)):
|
|
198
|
+
load_eval_manifest(manifest_path)
|
|
199
|
+
|
|
200
|
+
|
|
97
201
|
@pytest.mark.parametrize(
|
|
98
202
|
("field_yaml", "message"),
|
|
99
203
|
[
|
|
@@ -117,6 +221,10 @@ def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
|
|
|
117
221
|
"metadata:\n name: bad\nskill:\n name: bad\nevaluation:\n expectedEvidence:\n - missing-node\n",
|
|
118
222
|
"evaluation.expectedEvidence",
|
|
119
223
|
),
|
|
224
|
+
(
|
|
225
|
+
"metadata:\n name: bad\nskill:\n name: bad\nexecution:\n agent: gemini\n",
|
|
226
|
+
"Unsupported evaluation agent",
|
|
227
|
+
),
|
|
120
228
|
],
|
|
121
229
|
)
|
|
122
230
|
def test_load_eval_manifest_rejects_malformed_structured_fields(
|
|
@@ -132,3 +240,291 @@ def test_load_eval_manifest_rejects_malformed_structured_fields(
|
|
|
132
240
|
|
|
133
241
|
with pytest.raises(ValueError, match=message):
|
|
134
242
|
load_eval_manifest(manifest_path)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def test_load_eval_manifest_parses_inline_and_source_tasks(tmp_path: Path):
|
|
246
|
+
package = tmp_path / "my-skill"
|
|
247
|
+
package.mkdir()
|
|
248
|
+
(package / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
|
|
249
|
+
fixtures = package / "fixtures"
|
|
250
|
+
fixtures.mkdir()
|
|
251
|
+
(fixtures / "input.txt").write_text("input\n", encoding="utf-8")
|
|
252
|
+
|
|
253
|
+
source_task = package / "tasks" / "advanced"
|
|
254
|
+
(source_task / "environment").mkdir(parents=True)
|
|
255
|
+
(source_task / "validation").mkdir()
|
|
256
|
+
(source_task / "task.toml").write_text(
|
|
257
|
+
'[metadata]\nname = "advanced"\n\n[environment]\ndockerfile = "Dockerfile"\n',
|
|
258
|
+
encoding="utf-8",
|
|
259
|
+
)
|
|
260
|
+
(source_task / "instruction.md").write_text("Advanced task\n", encoding="utf-8")
|
|
261
|
+
(source_task / "environment" / "Dockerfile").write_text("FROM alpine\n", encoding="utf-8")
|
|
262
|
+
|
|
263
|
+
manifest_path = package / "comet" / "eval.yaml"
|
|
264
|
+
manifest_path.parent.mkdir()
|
|
265
|
+
manifest_path.write_text(
|
|
266
|
+
"""
|
|
267
|
+
apiVersion: comet.eval/v1alpha1
|
|
268
|
+
kind: SkillEvalManifest
|
|
269
|
+
metadata:
|
|
270
|
+
name: my-skill
|
|
271
|
+
skill:
|
|
272
|
+
name: my-skill
|
|
273
|
+
source: ..
|
|
274
|
+
evaluation:
|
|
275
|
+
tasks:
|
|
276
|
+
- name: create-summary
|
|
277
|
+
prompt: Create summary.md.
|
|
278
|
+
workspace: ./fixtures
|
|
279
|
+
expect:
|
|
280
|
+
files:
|
|
281
|
+
- summary.md
|
|
282
|
+
contains:
|
|
283
|
+
summary.md:
|
|
284
|
+
- "# Summary"
|
|
285
|
+
json:
|
|
286
|
+
- file: summary.json
|
|
287
|
+
path: $.status
|
|
288
|
+
equals: complete
|
|
289
|
+
commands:
|
|
290
|
+
- run: test -f summary.md
|
|
291
|
+
timeout: 30
|
|
292
|
+
rubric:
|
|
293
|
+
- The summary explains the result.
|
|
294
|
+
- name: advanced-alias
|
|
295
|
+
source: ./tasks/advanced
|
|
296
|
+
""",
|
|
297
|
+
encoding="utf-8",
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
manifest = load_eval_manifest(manifest_path)
|
|
301
|
+
|
|
302
|
+
assert [task.name for task in manifest.tasks] == ["create-summary", "advanced-alias"]
|
|
303
|
+
inline = manifest.tasks[0]
|
|
304
|
+
assert inline.prompt == "Create summary.md."
|
|
305
|
+
assert inline.workspace == fixtures.resolve()
|
|
306
|
+
assert inline.expect["files"] == ["summary.md"]
|
|
307
|
+
assert inline.rubric == ["The summary explains the result."]
|
|
308
|
+
assert manifest.tasks[1].source == source_task.resolve()
|
|
309
|
+
assert [task.name for task in load_manifest_tasks(manifest)] == [
|
|
310
|
+
"create-summary",
|
|
311
|
+
"advanced-alias",
|
|
312
|
+
]
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@pytest.mark.parametrize(
|
|
316
|
+
("task_yaml", "message"),
|
|
317
|
+
[
|
|
318
|
+
(
|
|
319
|
+
"- name: missing-prompt\n expect:\n files: [result.md]\n",
|
|
320
|
+
"prompt",
|
|
321
|
+
),
|
|
322
|
+
(
|
|
323
|
+
"- name: no-expect\n prompt: Do it.\n",
|
|
324
|
+
"expect: is required",
|
|
325
|
+
),
|
|
326
|
+
(
|
|
327
|
+
"- name: unsafe\n prompt: Do it.\n expect:\n files: [../secret]\n",
|
|
328
|
+
"evaluation.tasks.*.expect.files",
|
|
329
|
+
),
|
|
330
|
+
(
|
|
331
|
+
"- name: unsafe\n prompt: Do it.\n workspace: ../outside\n expect:\n files: [result.md]\n",
|
|
332
|
+
"workspace",
|
|
333
|
+
),
|
|
334
|
+
(
|
|
335
|
+
"- name: unknown\n prompt: Do it.\n expect:\n files: [result.md]\n executable: true\n",
|
|
336
|
+
"unknown field",
|
|
337
|
+
),
|
|
338
|
+
],
|
|
339
|
+
)
|
|
340
|
+
def test_load_eval_manifest_rejects_invalid_inline_tasks(
|
|
341
|
+
tmp_path: Path,
|
|
342
|
+
task_yaml: str,
|
|
343
|
+
message: str,
|
|
344
|
+
):
|
|
345
|
+
package = tmp_path / "my-skill"
|
|
346
|
+
(package / "comet").mkdir(parents=True)
|
|
347
|
+
manifest_path = package / "comet" / "eval.yaml"
|
|
348
|
+
manifest_path.write_text(
|
|
349
|
+
"apiVersion: comet.eval/v1alpha1\n"
|
|
350
|
+
"kind: SkillEvalManifest\n"
|
|
351
|
+
"metadata:\n name: my-skill\n"
|
|
352
|
+
"skill:\n name: my-skill\n source: ..\n"
|
|
353
|
+
"evaluation:\n tasks:\n"
|
|
354
|
+
+ "".join(f" {line}\n" for line in task_yaml.rstrip().splitlines()),
|
|
355
|
+
encoding="utf-8",
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
with pytest.raises(ValueError, match=message):
|
|
359
|
+
load_eval_manifest(manifest_path)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def test_load_eval_manifest_rejects_duplicate_authored_task_names(tmp_path: Path):
|
|
363
|
+
package = tmp_path / "my-skill"
|
|
364
|
+
(package / "comet").mkdir(parents=True)
|
|
365
|
+
manifest_path = package / "comet" / "eval.yaml"
|
|
366
|
+
manifest_path.write_text(
|
|
367
|
+
"""
|
|
368
|
+
apiVersion: comet.eval/v1alpha1
|
|
369
|
+
kind: SkillEvalManifest
|
|
370
|
+
metadata:
|
|
371
|
+
name: my-skill
|
|
372
|
+
skill:
|
|
373
|
+
name: my-skill
|
|
374
|
+
source: ..
|
|
375
|
+
evaluation:
|
|
376
|
+
tasks:
|
|
377
|
+
- name: duplicate
|
|
378
|
+
prompt: First.
|
|
379
|
+
expect:
|
|
380
|
+
files: [one.txt]
|
|
381
|
+
- name: duplicate
|
|
382
|
+
prompt: Second.
|
|
383
|
+
expect:
|
|
384
|
+
files: [two.txt]
|
|
385
|
+
""",
|
|
386
|
+
encoding="utf-8",
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
with pytest.raises(ValueError, match="duplicates evaluation.tasks"):
|
|
390
|
+
load_eval_manifest(manifest_path)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def test_load_eval_manifest_rejects_duplicate_recommended_task_names(tmp_path: Path):
|
|
394
|
+
manifest_path = tmp_path / "eval.yaml"
|
|
395
|
+
manifest_path.write_text(
|
|
396
|
+
"apiVersion: comet.eval/v1alpha1\n"
|
|
397
|
+
"kind: SkillEvalManifest\n"
|
|
398
|
+
"metadata: {name: my-skill}\n"
|
|
399
|
+
"skill: {name: my-skill, source: .}\n"
|
|
400
|
+
"evaluation:\n"
|
|
401
|
+
" recommendedTasks: [generic-skill-smoke, generic-skill-smoke]\n",
|
|
402
|
+
encoding="utf-8",
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
with pytest.raises(
|
|
406
|
+
ValueError,
|
|
407
|
+
match=re.escape(
|
|
408
|
+
'evaluation.recommendedTasks[1] duplicates '
|
|
409
|
+
'evaluation.recommendedTasks[0]: "generic-skill-smoke"'
|
|
410
|
+
),
|
|
411
|
+
):
|
|
412
|
+
load_eval_manifest(manifest_path)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
@pytest.mark.parametrize(
|
|
416
|
+
("section", "message"),
|
|
417
|
+
[
|
|
418
|
+
("execution:\n model: ' '\n", "execution.model"),
|
|
419
|
+
("execution:\n baseUrl: ftp://main.example.com\n", "execution.baseUrl"),
|
|
420
|
+
("judge:\n agent: codex\n", "judge.model"),
|
|
421
|
+
("judge:\n model: ' '\n", "judge.model"),
|
|
422
|
+
(
|
|
423
|
+
"judge:\n agent: unknown-agent\n model: judge-model\n",
|
|
424
|
+
"manifest judge.agent",
|
|
425
|
+
),
|
|
426
|
+
(
|
|
427
|
+
"judge:\n model: judge-model\n baseUrl: relative/path\n",
|
|
428
|
+
"judge.baseUrl",
|
|
429
|
+
),
|
|
430
|
+
],
|
|
431
|
+
)
|
|
432
|
+
def test_manifest_rejects_invalid_preserved_execution_and_judge_fields(
|
|
433
|
+
tmp_path: Path, section: str, message: str
|
|
434
|
+
):
|
|
435
|
+
manifest_path = tmp_path / "eval.yaml"
|
|
436
|
+
manifest_path.write_text(
|
|
437
|
+
"apiVersion: comet.eval/v1alpha1\n"
|
|
438
|
+
"kind: SkillEvalManifest\n"
|
|
439
|
+
"metadata: {name: my-skill}\n"
|
|
440
|
+
"skill: {name: my-skill, source: .}\n"
|
|
441
|
+
+ section,
|
|
442
|
+
encoding="utf-8",
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
with pytest.raises(ValueError, match=re.escape(message)):
|
|
446
|
+
load_eval_manifest(manifest_path)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
@pytest.mark.parametrize(
|
|
450
|
+
("yaml_fragment", "message"),
|
|
451
|
+
[
|
|
452
|
+
("unknownTopLevel: true\n", "unknownTopLevel: unknown field"),
|
|
453
|
+
("evaluation:\n unknownFlag: true\n", "evaluation.unknownFlag: unknown field"),
|
|
454
|
+
("execution:\n unknownFlag: true\n", "execution.unknownFlag: unknown field"),
|
|
455
|
+
(
|
|
456
|
+
"judge:\n model: judge-model\n unknownFlag: true\n",
|
|
457
|
+
"judge.unknownFlag: unknown field",
|
|
458
|
+
),
|
|
459
|
+
("reporting:\n unknownFlag: true\n", "reporting.unknownFlag: unknown field"),
|
|
460
|
+
("interaction:\n unknownFlag: true\n", "interaction.unknownFlag: unknown field"),
|
|
461
|
+
(
|
|
462
|
+
"evaluation:\n"
|
|
463
|
+
" tasks:\n"
|
|
464
|
+
" - name: exact\n"
|
|
465
|
+
" prompt: Write result.md.\n"
|
|
466
|
+
" expect: {files: [result.md]}\n"
|
|
467
|
+
" unknownFlag: true\n",
|
|
468
|
+
"evaluation.tasks[0].unknownFlag: unknown field",
|
|
469
|
+
),
|
|
470
|
+
],
|
|
471
|
+
)
|
|
472
|
+
def test_schema_rejects_unknown_fields_with_yaml_paths(tmp_path: Path, yaml_fragment: str, message: str):
|
|
473
|
+
manifest_path = tmp_path / "eval.yaml"
|
|
474
|
+
manifest_path.write_text(
|
|
475
|
+
"apiVersion: comet.eval/v1alpha1\n"
|
|
476
|
+
"kind: SkillEvalManifest\n"
|
|
477
|
+
"metadata:\n"
|
|
478
|
+
" name: my-skill\n"
|
|
479
|
+
"skill:\n"
|
|
480
|
+
" name: my-skill\n"
|
|
481
|
+
" source: .\n"
|
|
482
|
+
+ yaml_fragment,
|
|
483
|
+
encoding="utf-8",
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
with pytest.raises(ValueError, match=re.escape(message)):
|
|
487
|
+
load_eval_manifest(manifest_path)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
@pytest.mark.parametrize(
|
|
491
|
+
("fragment", "message"),
|
|
492
|
+
[
|
|
493
|
+
("interaction: null\n", "interaction: None is not of type 'object'"),
|
|
494
|
+
("evaluation: []\n", "evaluation: [] is not of type 'object'"),
|
|
495
|
+
("execution: invalid\n", "execution: 'invalid' is not of type 'object'"),
|
|
496
|
+
],
|
|
497
|
+
)
|
|
498
|
+
def test_schema_rejects_explicit_non_mapping_sections(
|
|
499
|
+
tmp_path: Path, fragment: str, message: str
|
|
500
|
+
):
|
|
501
|
+
manifest_path = tmp_path / "eval.yaml"
|
|
502
|
+
manifest_path.write_text(
|
|
503
|
+
"apiVersion: comet.eval/v1alpha1\nkind: SkillEvalManifest\nmetadata: {name: my-skill}\nskill: {name: my-skill, source: .}\n"
|
|
504
|
+
+ fragment,
|
|
505
|
+
encoding="utf-8",
|
|
506
|
+
)
|
|
507
|
+
with pytest.raises(ValueError, match=re.escape(message)):
|
|
508
|
+
load_eval_manifest(manifest_path)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def test_schema_declared_interaction_fields_are_normalized(tmp_path: Path):
|
|
512
|
+
manifest_path = tmp_path / "eval.yaml"
|
|
513
|
+
manifest_path.write_text(
|
|
514
|
+
"""apiVersion: comet.eval/v1alpha1
|
|
515
|
+
kind: SkillEvalManifest
|
|
516
|
+
metadata: {name: my-skill}
|
|
517
|
+
skill: {name: my-skill, source: .}
|
|
518
|
+
interaction:
|
|
519
|
+
decisionPatterns: [confirm]
|
|
520
|
+
decisionReply: "yes"
|
|
521
|
+
decisionReplies: [first, second]
|
|
522
|
+
continuePrompt: Keep going.
|
|
523
|
+
""",
|
|
524
|
+
encoding="utf-8",
|
|
525
|
+
)
|
|
526
|
+
interaction = load_eval_manifest(manifest_path).interaction
|
|
527
|
+
assert interaction.decision_patterns == ["confirm"]
|
|
528
|
+
assert interaction.decision_reply == "yes"
|
|
529
|
+
assert interaction.decision_replies == ["first", "second"]
|
|
530
|
+
assert interaction.continue_prompt == "Keep going."
|
|
@@ -287,6 +287,23 @@ def test_validate_native_workflow_rejects_external_skill_invocation(tmp_path: Pa
|
|
|
287
287
|
)
|
|
288
288
|
|
|
289
289
|
|
|
290
|
+
def test_validate_native_workflow_does_not_infer_custom_agent_skill_invocation(
|
|
291
|
+
tmp_path: Path,
|
|
292
|
+
):
|
|
293
|
+
_write_native_archive(tmp_path)
|
|
294
|
+
|
|
295
|
+
passed, failed = validate_native_workflow(
|
|
296
|
+
tmp_path,
|
|
297
|
+
{
|
|
298
|
+
"agent": "fixture-agent",
|
|
299
|
+
"events": {"skills_invoked": ["comet-native"]},
|
|
300
|
+
},
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
assert "native_skill_invocation" not in passed
|
|
304
|
+
assert any(item.startswith("native_skill_invocation:") for item in failed)
|
|
305
|
+
|
|
306
|
+
|
|
290
307
|
def test_adapt_checks_for_native_replaces_classic_contract_but_keeps_business(tmp_path: Path):
|
|
291
308
|
_write_native_archive(tmp_path)
|
|
292
309
|
|
|
@@ -422,8 +422,7 @@ def test_comet_profile_recognizes_proxied_openspec_spec_reconciliation(
|
|
|
422
422
|
passed, _ = run_profile_rubric("comet-workflow", tmp_path, outputs)
|
|
423
423
|
|
|
424
424
|
assert any(
|
|
425
|
-
"[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg
|
|
426
|
-
for msg in passed
|
|
425
|
+
"[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg for msg in passed
|
|
427
426
|
)
|
|
428
427
|
|
|
429
428
|
|
|
@@ -445,8 +444,7 @@ def test_comet_profile_still_recognizes_direct_openspec_spec_reconciliation(tmp_
|
|
|
445
444
|
passed, _ = run_profile_rubric("comet-workflow", tmp_path, outputs)
|
|
446
445
|
|
|
447
446
|
assert any(
|
|
448
|
-
"[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg
|
|
449
|
-
for msg in passed
|
|
447
|
+
"[RUBRIC] spec_drift: 1.00 - spec_written=True spec_synced=True" in msg for msg in passed
|
|
450
448
|
)
|
|
451
449
|
|
|
452
450
|
|
|
@@ -944,6 +942,64 @@ def test_judge_env_maps_independent_provider(monkeypatch):
|
|
|
944
942
|
assert "subject-token" not in invocation.env.values()
|
|
945
943
|
|
|
946
944
|
|
|
945
|
+
def test_judge_env_selects_the_same_agent_with_isolated_credentials(monkeypatch):
|
|
946
|
+
from scaffold.python.judge_config import build_judge_invocation
|
|
947
|
+
|
|
948
|
+
monkeypatch.setenv("OPENAI_API_KEY", "subject-key")
|
|
949
|
+
monkeypatch.setenv("BENCH_JUDGE_MODEL", "judge-model")
|
|
950
|
+
monkeypatch.setenv("BENCH_JUDGE_API_KEY", "judge-key")
|
|
951
|
+
|
|
952
|
+
invocation = build_judge_invocation(agent="codex")
|
|
953
|
+
|
|
954
|
+
assert invocation.agent == "codex"
|
|
955
|
+
assert invocation.env["OPENAI_API_KEY"] == "judge-key"
|
|
956
|
+
assert "ANTHROPIC_API_KEY" not in invocation.env
|
|
957
|
+
assert "subject-key" not in invocation.env.values()
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def test_judge_env_maps_codebuddy_credentials_and_endpoint(monkeypatch):
|
|
961
|
+
from scaffold.python.judge_config import build_judge_invocation
|
|
962
|
+
|
|
963
|
+
monkeypatch.setenv("CODEBUDDY_API_KEY", "subject-key")
|
|
964
|
+
monkeypatch.setenv("CODEBUDDY_MODEL", "subject-model")
|
|
965
|
+
monkeypatch.setenv("BENCH_JUDGE_MODEL", "judge-model")
|
|
966
|
+
monkeypatch.setenv("BENCH_JUDGE_API_KEY", "judge-key")
|
|
967
|
+
monkeypatch.setenv("BENCH_JUDGE_AUTH_TOKEN", "judge-token")
|
|
968
|
+
monkeypatch.setenv("BENCH_JUDGE_BASE_URL", "https://judge.example")
|
|
969
|
+
|
|
970
|
+
invocation = build_judge_invocation(agent="codebuddy")
|
|
971
|
+
|
|
972
|
+
assert invocation.agent == "codebuddy"
|
|
973
|
+
assert invocation.env["CODEBUDDY_API_KEY"] == "judge-key"
|
|
974
|
+
assert invocation.env["CODEBUDDY_AUTH_TOKEN"] == "judge-token"
|
|
975
|
+
assert invocation.env["CODEBUDDY_BASE_URL"] == "https://judge.example"
|
|
976
|
+
assert invocation.env["CODEBUDDY_MODEL"] == "judge-model"
|
|
977
|
+
assert invocation.env["COMET_EVAL_AGENT_ROLE"] == "judge"
|
|
978
|
+
assert "subject-key" not in invocation.env.values()
|
|
979
|
+
assert "subject-model" not in invocation.env.values()
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def test_codebuddy_judge_extracts_response_text_from_json_stream():
|
|
983
|
+
from scaffold.python.judge_config import _extract_agent_text
|
|
984
|
+
|
|
985
|
+
assert _extract_agent_text('{"response":"[RUBRIC-JUDGE] ok"}') == "[RUBRIC-JUDGE] ok"
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def test_judge_session_ids_are_extracted_from_nested_agent_events():
|
|
989
|
+
import json
|
|
990
|
+
|
|
991
|
+
from scaffold.python.judge_config import _extract_agent_session_ids
|
|
992
|
+
|
|
993
|
+
output = "\n".join(
|
|
994
|
+
[
|
|
995
|
+
json.dumps({"type": "thread.started", "thread_id": "judge-thread"}),
|
|
996
|
+
json.dumps({"message": {"sessionId": "judge-session"}}),
|
|
997
|
+
]
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
assert _extract_agent_session_ids(output) == ["judge-thread", "judge-session"]
|
|
1001
|
+
|
|
1002
|
+
|
|
947
1003
|
def test_judge_provider_uses_direct_http_when_base_url_is_configured(monkeypatch):
|
|
948
1004
|
"""Dedicated judge providers should not require Claude CLI compatibility."""
|
|
949
1005
|
import json
|
|
@@ -1034,17 +1090,19 @@ def test_generic_llm_judge_parses_output(tmp_path: Path):
|
|
|
1034
1090
|
"[RUBRIC-JUDGE] output_quality: 0.90 - well-structured code\n"
|
|
1035
1091
|
"[RUBRIC-JUDGE] instruction_adherence: 1.00 - all constraints followed\n"
|
|
1036
1092
|
)
|
|
1037
|
-
outputs = {"completion": {"passed": [], "failed": []}}
|
|
1093
|
+
outputs = {"completion": {"passed": [], "failed": []}, "agent": "codebuddy"}
|
|
1038
1094
|
|
|
1039
1095
|
with patch(
|
|
1040
1096
|
"scaffold.python.generic_llm_judge._run_judge",
|
|
1041
1097
|
return_value=mock_output,
|
|
1042
|
-
):
|
|
1098
|
+
) as run_judge:
|
|
1043
1099
|
scores = judge_generic_artifacts(tmp_path, outputs)
|
|
1044
1100
|
|
|
1045
1101
|
assert scores["task_completion"] == (0.80, "output present and mostly correct")
|
|
1046
1102
|
assert scores["output_quality"] == (0.90, "well-structured code")
|
|
1047
1103
|
assert scores["instruction_adherence"] == (1.00, "all constraints followed")
|
|
1104
|
+
assert run_judge.call_args.kwargs["agent"] == "codebuddy"
|
|
1105
|
+
assert run_judge.call_args.kwargs["evidence"] is outputs
|
|
1048
1106
|
|
|
1049
1107
|
|
|
1050
1108
|
def test_generic_llm_judge_parses_custom_dimensions(tmp_path: Path):
|