@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
package/eval/README.md
CHANGED
|
@@ -23,16 +23,24 @@ eval/
|
|
|
23
23
|
│ ├── treatments/ # Treatment 配置(注入哪些 Skill)
|
|
24
24
|
│ ├── tests/ # pytest 测试入口
|
|
25
25
|
│ └── logs/ # 评估结果日志
|
|
26
|
-
|
|
26
|
+
├── langsmith/ # LangSmith 评估入口(可选)
|
|
27
|
+
└── langfuse/ # Langfuse 评估入口(可选)
|
|
27
28
|
```
|
|
28
29
|
|
|
29
30
|
## 环境配置
|
|
30
31
|
|
|
32
|
+
普通 npm 用户不需要进入 Comet 源码目录。首次运行 `comet eval` 时,CLI 会在用户目录自动创建完整配置模板:
|
|
33
|
+
|
|
34
|
+
- Windows:`%USERPROFILE%\.comet\eval\.env`
|
|
35
|
+
- macOS/Linux:`~/.comet/eval/.env`
|
|
36
|
+
|
|
37
|
+
模板包含主任务(Bench)、独立 Judge、Claude Code、Codex、Qoder、CodeBuddy、LangSmith 和 Langfuse 的全部可配置参数。CLI 只会创建缺失文件,不会覆盖已有 `.env`;当前进程环境变量优先于文件内容。源码仓库下的 `eval/.env` 仅用于 Comet harness 开发和测试,不是普通用户配置入口。
|
|
38
|
+
|
|
31
39
|
### 前置依赖
|
|
32
40
|
|
|
33
41
|
- Python 3.11+
|
|
34
42
|
- Docker
|
|
35
|
-
- Claude Code CLI
|
|
43
|
+
- 一个受支持的评估 Agent CLI:Claude Code、Codex、Qoder 或 CodeBuddy(运行时会在 Docker 镜像中准备对应 CLI)
|
|
36
44
|
- `uv`(推荐)或 pip
|
|
37
45
|
|
|
38
46
|
### 安装
|
|
@@ -44,20 +52,32 @@ uv sync
|
|
|
44
52
|
|
|
45
53
|
### 环境变量
|
|
46
54
|
|
|
47
|
-
|
|
55
|
+
已发布的 `comet eval` 不需要用户拉取源码。评估运行会先读取当前 shell 的环境变量,再按需加载用户目录下、与自定义 Agent 适配器并列的 `~/.comet/eval/.env`(Windows:`%USERPROFILE%\.comet\eval\.env`)。录屏或演示时可以不创建 `.env`,把密钥放在系统环境变量里。
|
|
48
56
|
|
|
49
|
-
|
|
57
|
+
如需本地文件配置,可参考 `.env.example`,创建用户级文件:
|
|
50
58
|
|
|
51
59
|
```bash
|
|
52
|
-
|
|
60
|
+
mkdir -p ~/.comet/eval
|
|
61
|
+
cp .env.example ~/.comet/eval/.env
|
|
53
62
|
```
|
|
54
63
|
|
|
55
|
-
|
|
64
|
+
进程环境变量优先于用户 `.env`;显式 CLI 或 manifest 配置优先于环境默认值。空值不会覆盖已有的有效进程变量。
|
|
56
65
|
|
|
57
66
|
| 变量 | 必填 | 说明 |
|
|
58
67
|
| ----------------------------------------------------------------- | ---- | --------------------------------------------------------------------------------------------------- |
|
|
59
|
-
| `
|
|
60
|
-
| `
|
|
68
|
+
| `BENCH_EVAL_AGENT` | ❌ | 未通过 CLI 或 manifest 指定时的默认 Agent |
|
|
69
|
+
| `BENCH_API_KEY` / `BENCH_BASE_URL` / `BENCH_MODEL` | ✅\* | 主任务统一凭据、API 地址和模型;会映射到所选 Agent 的原生配置 |
|
|
70
|
+
| `BENCH_JUDGE_AGENT` / `BENCH_JUDGE_API_KEY` / `BENCH_JUDGE_BASE_URL` / `BENCH_JUDGE_MODEL` | Judge | LLM-as-judge 的独立 Agent、凭据、API 地址和模型;不会继承主任务配置 |
|
|
71
|
+
| `ANTHROPIC_API_KEY` | ✅\* | Claude Code 原生 API 密钥;显式设置时优先于 `BENCH_API_KEY` |
|
|
72
|
+
| `BENCH_CC_MODEL` / `BENCH_CC_VERSION` | ❌ | Claude 模型和 CLI 版本覆盖(默认用 CLI 配置) |
|
|
73
|
+
| `OPENAI_API_KEY` / `CODEX_API_KEY` | Codex | 使用 `--agent codex` 时的 Codex 认证 |
|
|
74
|
+
| `OPENAI_BASE_URL` / `OPENAI_MODEL` / `CODEX_BASE_URL` / `CODEX_MODEL` / `BENCH_CODEX_VERSION` | ❌ | Codex 原生地址、模型和 CLI 版本覆盖 |
|
|
75
|
+
| `QODER_PERSONAL_ACCESS_TOKEN` | Qoder | 使用 `--agent qoder` 时的 Qoder 认证 |
|
|
76
|
+
| `QODER_BASE_URL` / `QODER_MODEL` / `BENCH_QODER_VERSION` | ❌ | Qoder 原生地址、模型和 CLI 版本覆盖 |
|
|
77
|
+
| `CODEBUDDY_API_KEY` / `CODEBUDDY_AUTH_TOKEN` | CodeBuddy | 使用 `--agent codebuddy` 时的 CodeBuddy 认证 |
|
|
78
|
+
| `CODEBUDDY_BASE_URL` / `CODEBUDDY_MODEL` / `BENCH_CODEBUDDY_VERSION` | ❌ | CodeBuddy API endpoint、模型和 CLI 版本覆盖 |
|
|
79
|
+
| `CODEBUDDY_SMALL_FAST_MODEL` / `CODEBUDDY_BIG_SLOW_MODEL` / `CODEBUDDY_CODE_SUBAGENT_MODEL` | ❌ | CodeBuddy 的 fast、reasoning 和 code-subagent 模型覆盖 |
|
|
80
|
+
| `CODEBUDDY_CUSTOM_HEADERS` / `CODEBUDDY_INTERNET_ENVIRONMENT` | ❌ | CodeBuddy 的自定义请求头和联网环境配置 |
|
|
61
81
|
| `BENCH_SIMULATOR_PROMPT_FILE` | ❌ | 显式覆盖任务级用户模拟器提示词;任务未配置时回退到 `eval/simulator-instruction.md` |
|
|
62
82
|
| `ANTHROPIC_AUTH_TOKEN` / `ANTHROPIC_BASE_URL` / `ANTHROPIC_MODEL` | ❌ | 用 Anthropic 兼容代理时的认证与模型配置;设置 `ANTHROPIC_AUTH_TOKEN` 后可不设置 `ANTHROPIC_API_KEY` |
|
|
63
83
|
| `BENCH_LLM_JUDGE` | ❌ | 设为 `1` 启用 LLM-as-judge 评分 |
|
|
@@ -66,8 +86,50 @@ cp .env.example .env
|
|
|
66
86
|
| `BENCH_JUDGE_API_KEY` | ❌ | Judge 专用 Anthropic API key;如果设置 `BENCH_JUDGE_AUTH_TOKEN`,优先使用 auth token |
|
|
67
87
|
| `LANGSMITH_API_KEY` | ❌ | 仅 LangSmith 套件需要 |
|
|
68
88
|
| `LANGSMITH_PROJECT` / `LANGSMITH_TRACING` | ❌ | LangSmith 套件项目名与追踪开关;Claude Code 插件变量会从这组配置自动派生 |
|
|
89
|
+
| `TRACE_TO_LANGSMITH` | ❌ | 兼容旧配置的 LangSmith 追踪开关 |
|
|
90
|
+
| `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` | Langfuse | `--suite langfuse` 运行模式的项目凭据;只从当前进程读取,不写入报告或 manifest |
|
|
91
|
+
| `LANGFUSE_BASE_URL` / `LANGFUSE_TRACING_ENVIRONMENT` | ❌ | Langfuse 区域地址与 trace 环境标签 |
|
|
92
|
+
| `COMET_EVAL_ADAPTERS_DIR` | ❌ | 覆盖自定义 Agent 适配器注册目录;默认是用户目录下的 `~/.comet/eval/adapters` |
|
|
93
|
+
|
|
94
|
+
`*` 主任务至少需要 `BENCH_API_KEY` 或所选 Agent 的原生凭据之一。Claude Code、Codex 和 CodeBuddy 支持 `BENCH_BASE_URL`;Qoder 使用其官方支持的认证和服务地址配置。
|
|
95
|
+
|
|
96
|
+
安全边界:Eval 不会把 API key 写入发布包、manifest、报告或 Skill 工作区。Docker 运行 Agent 时,Codex、Qoder、CodeBuddy 使用容器内独立的配置根;Codex 的 `config.toml` 只引用 `env_key`,CodeBuddy 的 `settings.json` 只使用 `apiKeyHelper`,实际密钥只存在于本次容器进程环境中,运行结束后随临时配置目录销毁。
|
|
97
|
+
|
|
98
|
+
`comet eval` 默认使用 `claude-code`。可以通过 CLI 或 manifest 选择执行 Agent;CLI 优先于
|
|
99
|
+
manifest,未指定时回退到 Claude Code:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
comet eval ./generated-skill/comet/eval.yaml --agent codex
|
|
103
|
+
comet eval ./generated-skill/comet/eval.yaml --agent qoder
|
|
104
|
+
comet eval ./generated-skill/comet/eval.yaml --agent codebuddy
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
CodeBuddy 的自定义模型使用其 OpenAI-compatible 配置:`CODEBUDDY_API_KEY`、
|
|
108
|
+
`CODEBUDDY_BASE_URL` 和 provider 接受的 `CODEBUDDY_MODEL`,也可以直接用 `--model`
|
|
109
|
+
覆盖当前运行的模型。宿主机的 `~/.codebuddy/models.json` 只用于注册模型;Eval 不挂载它,
|
|
110
|
+
而是在容器内生成临时 `settings.json`,通过 `apiKeyHelper` 从进程环境读取凭据。不要把
|
|
111
|
+
Claude Code 的 Anthropic 地址或 Claude 专用模型 ID 直接复用给 CodeBuddy。
|
|
112
|
+
|
|
113
|
+
manifest 可声明默认 Agent:
|
|
114
|
+
|
|
115
|
+
```yaml
|
|
116
|
+
execution:
|
|
117
|
+
agent: codex
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
四种 Agent 共用 subject、auto_user simulator 和可选 Judge 的执行/报告契约;缺失的 token、成本或
|
|
121
|
+
上下文遥测会在报告中保留为 `N/A`,而不是用估算值补齐。
|
|
122
|
+
|
|
123
|
+
### Langfuse 套件
|
|
124
|
+
|
|
125
|
+
使用 `--suite langfuse` 时,local runner 的任务、Agent、Docker、validator 和本地报告保持不变,额外把每个 task/treatment 的核心结果写入 Langfuse,并在结束时 flush:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-... \
|
|
129
|
+
comet eval ./generated-skill/comet/eval.yaml --suite langfuse --html
|
|
130
|
+
```
|
|
69
131
|
|
|
70
|
-
|
|
132
|
+
`comet eval` 会自动选择 Langfuse optional extra。Claude Code 和 Codex 会在隔离的 `.comet/eval/langfuse/plugins/` 缓存中准备带完整性元数据的固定版本官方插件;Qoder 和 CodeBuddy 使用项目级 Stop hook 读取 JSONL transcript,转换失败时只降级详细轨迹,不影响本地评分和核心 trace。`--collect --suite langfuse` 不初始化 SDK、不联网、不下载插件。
|
|
71
133
|
|
|
72
134
|
## 快速开始
|
|
73
135
|
|
|
@@ -94,6 +156,132 @@ Eval 报告同时保留 raw set 和 analysis set。Raw set 包含所有 run,
|
|
|
94
156
|
|
|
95
157
|
如果关键 treatment 的 clean data 不足,报告会输出 `Insufficient clean data` 或 `Inconclusive due to data quality`,而不是给出误导性的胜负结论。
|
|
96
158
|
|
|
159
|
+
### 独立评估任意 Skill
|
|
160
|
+
|
|
161
|
+
独立评估不依赖 `comet-any`。只要有一个 Skill 目录,就可以直接运行;没有
|
|
162
|
+
`comet/eval.yaml` 时,普通运行会自动生成并缓存 2–4 个任务:
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
comet eval ./my-skill
|
|
166
|
+
comet eval ./my-skill --agent codebuddy --model subject-model
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
主模型和 LLM-as-judge 是两套独立配置。Judge 只有在显式启用后才会运行,至少需要自己的
|
|
170
|
+
`BENCH_JUDGE_MODEL` 和 `BENCH_JUDGE_API_KEY`/`BENCH_JUDGE_AUTH_TOKEN`;它可以使用不同的
|
|
171
|
+
Agent、模型和 API 地址:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
BENCH_LLM_JUDGE=1 \
|
|
175
|
+
BENCH_JUDGE_MODEL=judge-model \
|
|
176
|
+
BENCH_JUDGE_BASE_URL=https://judge.example/v1 \
|
|
177
|
+
BENCH_JUDGE_API_KEY=... \
|
|
178
|
+
comet eval ./my-skill --model subject-model --base-url https://subject.example/v1
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
也可以全部通过命令行或 manifest 配置:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
comet eval ./my-skill \
|
|
185
|
+
--agent codebuddy --model subject-model --base-url https://subject.example/v1 \
|
|
186
|
+
--judge-agent claude-code --judge-model judge-model \
|
|
187
|
+
--judge-base-url https://judge.example/v1
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
如果只想检查任务、配置和快照,不执行 Agent、Docker、插件或网络请求,使用:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
comet eval ./my-skill --collect
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### 扩展自定义 Agent
|
|
197
|
+
|
|
198
|
+
非预定义的 Agent 不需要修改 Comet 源码,也不需要把适配器放进 npm 包。用户在自己的配置目录
|
|
199
|
+
中显式注册即可,Eval 不会仅因为某个可执行文件在 `PATH` 中就自动发现它:
|
|
200
|
+
|
|
201
|
+
| 系统 | 默认适配器目录 |
|
|
202
|
+
| --- | --- |
|
|
203
|
+
| macOS/Linux | `~/.comet/eval/adapters/<agent-id>/adapter.yaml` |
|
|
204
|
+
| Windows | `%USERPROFILE%\\.comet\\eval\\adapters\\<agent-id>\\adapter.yaml` |
|
|
205
|
+
|
|
206
|
+
也可以在用户级 `~/.comet/eval/.env`(Windows 为 `%USERPROFILE%\\.comet\\eval\\.env`)中设置
|
|
207
|
+
`COMET_EVAL_ADAPTERS_DIR`,把整个适配器注册表放到其他位置。`<agent-id>` 必须是小写字母开头、
|
|
208
|
+
只包含小写字母/数字/连字符、长度 2–32 的标识符。
|
|
209
|
+
|
|
210
|
+
创建 `<agent-id>/adapter.yaml`,例如:
|
|
211
|
+
|
|
212
|
+
```yaml
|
|
213
|
+
apiVersion: comet.eval.agent/v1alpha1
|
|
214
|
+
kind: EvalAgentAdapter
|
|
215
|
+
metadata: { id: my-agent, version: 1.0.0 }
|
|
216
|
+
runtime:
|
|
217
|
+
executable: my-agent
|
|
218
|
+
install: { kind: npm, package: my-agent, version: latest }
|
|
219
|
+
credentials: [MY_AGENT_API_KEY]
|
|
220
|
+
modelEnv: MY_AGENT_MODEL
|
|
221
|
+
baseUrlEnv: MY_AGENT_BASE_URL
|
|
222
|
+
capabilities:
|
|
223
|
+
singleTurn: true
|
|
224
|
+
resume: true
|
|
225
|
+
structuredEvents: true
|
|
226
|
+
telemetry: false
|
|
227
|
+
skillInvocationEvidence: true
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
字段含义:
|
|
231
|
+
|
|
232
|
+
- `runtime.executable` 是容器内实际启动的命令;npm/pip 包必须暴露这个可执行入口。
|
|
233
|
+
`runtime.install.kind` 可为 `npm`、`pip` 或 `none`;使用 `npm`/`pip` 时 Eval 会在本次 Docker
|
|
234
|
+
镜像构建中安装对应包,`none` 表示命令已在基础镜像中准备好。
|
|
235
|
+
- `credentials` 只能填写最多两个环境变量名,不要填写密钥值。主 Agent 运行时,只有这些声明的
|
|
236
|
+
环境变量会被转发到容器;Judge 运行时,`BENCH_JUDGE_API_KEY` 映射到第一个凭据名,
|
|
237
|
+
`BENCH_JUDGE_AUTH_TOKEN` 映射到第二个凭据名(如果声明了第二个)。
|
|
238
|
+
- `modelEnv` 和 `baseUrlEnv` 是可选的环境变量名。主 Agent 可通过这些变量,或 CLI 的
|
|
239
|
+
`--model`/`--base-url`,或 manifest 的 `execution.model`/`execution.baseUrl` 配置;自定义 Agent
|
|
240
|
+
不会自动把 `BENCH_MODEL`、`BENCH_BASE_URL` 或 `BENCH_API_KEY` 映射到自定义变量。
|
|
241
|
+
- `capabilities` 描述适配器的真实能力。`resume` 用于 `auto_user` 多轮交互;`structuredEvents`
|
|
242
|
+
要求 CLI 输出 JSON Lines;`skillInvocationEvidence` 表示输出中能提供明确的 Skill 调用事件;
|
|
243
|
+
`telemetry: false` 时报告中的 token/cost 可以是 `N/A`。运行真实评估至少需要
|
|
244
|
+
`singleTurn`、`resume`、`structuredEvents` 和 `skillInvocationEvidence` 为 `true`。
|
|
245
|
+
|
|
246
|
+
自定义 CLI 必须接受 Eval 生成的调用形态:
|
|
247
|
+
|
|
248
|
+
```text
|
|
249
|
+
<executable> -p "<prompt>" --output-format stream-json --model <model> [--resume <session-id>]
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
并把每条结构化事件写到 stdout。若要让 rubric 认定 Skill 被调用,输出中需要有类似下面的 JSONL
|
|
253
|
+
事件(普通文件产物或路径推断不算自定义 Agent 的调用证据):
|
|
254
|
+
|
|
255
|
+
```json
|
|
256
|
+
{"type":"skill_invocation","skill":"my-skill"}
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
在用户级 `.env` 中填写适配器声明的变量,然后运行:
|
|
260
|
+
|
|
261
|
+
```dotenv
|
|
262
|
+
BENCH_EVAL_AGENT=my-agent
|
|
263
|
+
MY_AGENT_API_KEY=your-subject-key
|
|
264
|
+
MY_AGENT_MODEL=your-model
|
|
265
|
+
MY_AGENT_BASE_URL=https://your-provider.example/v1
|
|
266
|
+
|
|
267
|
+
# 如果让它作为独立 Judge:
|
|
268
|
+
# BENCH_LLM_JUDGE=1
|
|
269
|
+
# BENCH_JUDGE_AGENT=my-agent
|
|
270
|
+
# BENCH_JUDGE_API_KEY=your-judge-key
|
|
271
|
+
# BENCH_JUDGE_MODEL=your-judge-model
|
|
272
|
+
# BENCH_JUDGE_BASE_URL=https://your-judge-provider.example/v1
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
```bash
|
|
276
|
+
comet eval ./my-skill --collect --agent my-agent
|
|
277
|
+
comet eval ./my-skill --quick --agent my-agent --model your-model
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
`--collect` 会校验适配器和配置但不会启动 Agent、Docker 或联网;确认通过后再运行真实评估。
|
|
281
|
+
如果你的 CLI 不能兼容上述参数和 JSONL 事件契约,需要提供一个 wrapper CLI,将它转换为该契约;
|
|
282
|
+
当前 `adapter.yaml` 不支持自定义参数模板或直接挂载宿主机 Agent 配置目录。适配器文件可以分发给
|
|
283
|
+
团队成员,但真实凭据必须留在每个人自己的用户级 `.env` 或当前 shell 中。
|
|
284
|
+
|
|
97
285
|
### 评估任意 Skill
|
|
98
286
|
|
|
99
287
|
```bash
|
|
@@ -111,23 +299,29 @@ uv run pytest local/tests/tasks/test_tasks.py \
|
|
|
111
299
|
--eval-manifest=/path/to/my-skill/comet/eval.yaml -v
|
|
112
300
|
```
|
|
113
301
|
|
|
114
|
-
|
|
302
|
+
如果 manifest 没有 `evaluation.tasks` 或 `recommendedTasks`,普通运行会基于 Skill 的受限快照自动生成 2–4 个任务,并按 Agent、profile、交互配置和快照 hash 缓存到项目的 `.comet/eval/generated/`。缓存 manifest 会写入生成 hash 和元数据,方便复现;`--collect` 只接受已有缓存,不会在收集阶段调用生成 Agent。需要快速冒烟时显式使用:
|
|
115
303
|
|
|
116
|
-
|
|
304
|
+
```bash
|
|
305
|
+
comet eval ./my-skill --quick
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
## 容器中的 Agent 实验工作区
|
|
309
|
+
|
|
310
|
+
每次实验都会先在宿主机创建一个临时隔离目录,然后把它挂载到所选 Agent 的 Docker 容器的 `/workspace`。Agent 的当前工作目录就是 `/workspace`;双 agent 交互模式还会只读挂载 scaffold shell 到 `/opt/scaffold-shell`。
|
|
117
311
|
|
|
118
312
|
典型结构如下(以 `COMET_FULL_040_BETA` 为例,展示 0.4.0 beta 的 Comet Skill + 完整 dependency snapshot;实际任务产物会随 task/treatment 变化):
|
|
119
313
|
|
|
120
314
|
```text
|
|
121
|
-
/workspace #
|
|
315
|
+
/workspace # 所选 Agent 的当前工作目录;宿主机临时隔离目录挂载点
|
|
122
316
|
|-- Dockerfile # 当前 task 的 Docker 构建文件,来自 task/environment/
|
|
123
317
|
|-- requirements.txt # Python 依赖文件;仅 task 提供时存在
|
|
124
318
|
|-- package.json # Node 依赖文件;仅 task 提供时存在
|
|
125
319
|
|-- package-lock.json # Node 锁文件;仅 task 提供时存在
|
|
126
320
|
|-- tsconfig.json # TypeScript 配置;仅 task 提供时存在
|
|
127
|
-
|-- CLAUDE.md #
|
|
321
|
+
|-- CLAUDE.md # Claude 项目级指令;其他 Agent 使用 AGENTS.md
|
|
128
322
|
|-- .eval-task-prompt.txt # auto_user 模式下的任务提示词临时文件,运行后清理
|
|
129
323
|
|-- .eval-simulator-prompt.txt # auto_user 模式下的用户模拟器提示词临时文件,运行后清理
|
|
130
|
-
|-- .claude
|
|
324
|
+
|-- .claude / .agents / .qoder / .codebuddy # 所选 Agent 的本地配置和 Skill 安装目录
|
|
131
325
|
| |-- CLAUDE.md # Claude 专用项目指令副本
|
|
132
326
|
| |-- settings.json # Claude Code settings;包含 Comet PreToolUse hook
|
|
133
327
|
| `-- skills # 当前 treatment 注入的完整 Skill 包
|
|
@@ -229,6 +423,28 @@ uv run pytest local/tests/tasks/test_tasks.py \
|
|
|
229
423
|
|
|
230
424
|
## 定义自己的 Task
|
|
231
425
|
|
|
426
|
+
除了把任务注册到 `eval/local/tasks/`,项目也可以在自己的 `comet/eval.yaml` 中声明 inline task 或引用 Skill 包内的 task package。项目任务会和 `recommendedTasks` 合并;同名任务会直接拒绝,避免覆盖内置任务。
|
|
427
|
+
|
|
428
|
+
```yaml
|
|
429
|
+
evaluation:
|
|
430
|
+
tasks:
|
|
431
|
+
- name: writes-summary
|
|
432
|
+
prompt: Create summary.md with a short result.
|
|
433
|
+
workspace: fixtures/summary
|
|
434
|
+
expect:
|
|
435
|
+
files: [summary.md]
|
|
436
|
+
contains:
|
|
437
|
+
summary.md: ["# Summary"]
|
|
438
|
+
commands:
|
|
439
|
+
- run: test -s summary.md
|
|
440
|
+
timeout: 30
|
|
441
|
+
rubric:
|
|
442
|
+
- The summary is specific and actionable.
|
|
443
|
+
- source: tasks/advanced
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
`workspace`、`source` 和所有期望产物都必须留在 Skill 包/任务工作区内;`files`、`contains`、`json` 和 `commands` 检查在隔离 Docker 工作区中执行。`--collect` 只做解析和发现,不运行 Agent 或用户命令。
|
|
447
|
+
|
|
232
448
|
每个任务是 `local/tasks/` 下的一个目录,包含三个部分:
|
|
233
449
|
|
|
234
450
|
```text
|
|
@@ -531,6 +747,13 @@ uv run pytest local/tests/tasks/test_tasks.py --count=5 -v
|
|
|
531
747
|
| `--task=<name>` | 指定任务 |
|
|
532
748
|
| `--treatment=<name>` | 指定 treatment(逗号分隔多个) |
|
|
533
749
|
| `--profile=<name>` | 覆盖评估 profile |
|
|
750
|
+
| `--agent=<id>` | 主 Agent;可用内置 Agent 或已安装的自定义适配器 |
|
|
751
|
+
| `--model=<name>` | 主模型覆盖 |
|
|
752
|
+
| `--base-url=<url>` | 主模型 API 地址覆盖 |
|
|
753
|
+
| `--judge-agent=<id>` | 独立 Judge Agent |
|
|
754
|
+
| `--judge-model=<name>` | 独立 Judge 模型 |
|
|
755
|
+
| `--judge-base-url=<url>` | 独立 Judge API 地址 |
|
|
756
|
+
| `--collect` | 只解析/发现,不执行工作负载 |
|
|
534
757
|
| `--skill-path=<path>` | 注入本地 Skill 路径 |
|
|
535
758
|
| `--skill-name=<name>` | 注入的 Skill 名称 |
|
|
536
759
|
| `--count=<n>` | 重复运行次数 |
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Required for `comet eval --suite langfuse` run mode.
|
|
2
|
+
LANGFUSE_PUBLIC_KEY=
|
|
3
|
+
LANGFUSE_SECRET_KEY=
|
|
4
|
+
LANGFUSE_BASE_URL=https://cloud.langfuse.com
|
|
5
|
+
LANGFUSE_TRACING_ENVIRONMENT=ci
|
|
6
|
+
|
|
7
|
+
# Claude Code and Codex official integrations are enabled by the suite through
|
|
8
|
+
# TRACE_TO_LANGFUSE=true. Install their official plugins in the agent image or
|
|
9
|
+
# project configuration before a run; collect mode never installs or contacts them.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Langfuse suite configuration with collect-safe credential handling."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
from dotenv import load_dotenv
|
|
12
|
+
|
|
13
|
+
from scaffold.python.eval_context import (
|
|
14
|
+
ResolvedEvalContext,
|
|
15
|
+
context_from_environment,
|
|
16
|
+
managed_path_for_owner,
|
|
17
|
+
resolve_managed_path,
|
|
18
|
+
)
|
|
19
|
+
from scaffold.python.paths import get_runs_dir
|
|
20
|
+
|
|
21
|
+
from scaffold.python.langfuse_adapter import (
|
|
22
|
+
LangfuseConfig,
|
|
23
|
+
LangfuseRunReporter,
|
|
24
|
+
TrajectoryProvisionError,
|
|
25
|
+
create_client,
|
|
26
|
+
enable_trajectory_environment,
|
|
27
|
+
provision_trajectory_plugin,
|
|
28
|
+
result_payload_from_local_report,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
LANGFUSE_ROOT = Path(__file__).resolve().parents[1]
|
|
32
|
+
EVAL_ROOT = LANGFUSE_ROOT.parent
|
|
33
|
+
LOCAL_ROOT = EVAL_ROOT / "local"
|
|
34
|
+
_LANGFUSE_CLIENT = None
|
|
35
|
+
_LANGFUSE_REPORTER: LangfuseRunReporter | None = None
|
|
36
|
+
_LANGFUSE_CONFIG: LangfuseConfig | None = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def default_trajectory_cache_root(context: ResolvedEvalContext) -> Path:
|
|
40
|
+
"""Place default trajectory assets in the resolved owner cache."""
|
|
41
|
+
return resolve_managed_path(context, "cache", "langfuse", "plugins")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def resolve_trajectory_cache_root(
|
|
45
|
+
context: ResolvedEvalContext | None, project_root: Path
|
|
46
|
+
) -> Path:
|
|
47
|
+
"""Keep CLI-resolved trajectory state owner-local; retain direct harness overrides."""
|
|
48
|
+
if context is not None:
|
|
49
|
+
return default_trajectory_cache_root(context)
|
|
50
|
+
explicit_cache = os.environ.get("LANGFUSE_TRAJECTORY_CACHE_DIR")
|
|
51
|
+
if explicit_cache:
|
|
52
|
+
return Path(explicit_cache)
|
|
53
|
+
return managed_path_for_owner(project_root, "cache", "langfuse", "plugins")
|
|
54
|
+
|
|
55
|
+
os.environ.setdefault("BENCH_SUITE_ROOT", str(LANGFUSE_ROOT))
|
|
56
|
+
os.environ.setdefault("BENCH_TASKS_DIR", str(LOCAL_ROOT / "tasks"))
|
|
57
|
+
os.environ.setdefault("BENCH_TREATMENTS_DIR", str(LOCAL_ROOT / "treatments"))
|
|
58
|
+
os.environ.setdefault("BENCH_SKILLS_DIR", str(LOCAL_ROOT / "skills"))
|
|
59
|
+
os.environ.setdefault("BENCH_LOGS_DIR", str(LANGFUSE_ROOT / "logs"))
|
|
60
|
+
os.environ.setdefault("LANGFUSE_TEST_SUITE", "comet-skill-eval")
|
|
61
|
+
|
|
62
|
+
load_dotenv(EVAL_ROOT / ".env")
|
|
63
|
+
load_dotenv(LANGFUSE_ROOT / ".env", override=True)
|
|
64
|
+
|
|
65
|
+
_LOCAL_CONFTEST = LOCAL_ROOT / "tests" / "conftest.py"
|
|
66
|
+
_spec = importlib.util.spec_from_file_location("_comet_local_conftest", _LOCAL_CONFTEST)
|
|
67
|
+
_local_conftest = importlib.util.module_from_spec(_spec)
|
|
68
|
+
assert _spec and _spec.loader
|
|
69
|
+
_spec.loader.exec_module(_local_conftest)
|
|
70
|
+
|
|
71
|
+
for _name in dir(_local_conftest):
|
|
72
|
+
if not _name.startswith("__") and _name not in globals():
|
|
73
|
+
globals()[_name] = getattr(_local_conftest, _name)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _is_adapter_tests_only(config) -> bool:
|
|
77
|
+
args = [arg for arg in (config.args or []) if not arg.startswith("-")]
|
|
78
|
+
return bool(args) and all("test_adapter.py" in arg for arg in args)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def pytest_configure(config):
|
|
82
|
+
"""Register the local plugin, then authenticate before any Agent run."""
|
|
83
|
+
if _is_adapter_tests_only(config):
|
|
84
|
+
_local_conftest._is_unit_tests_only = lambda _config: True
|
|
85
|
+
_local_conftest.pytest_configure(config)
|
|
86
|
+
if (
|
|
87
|
+
config.option.collectonly
|
|
88
|
+
or _local_conftest._is_unit_tests_only(config)
|
|
89
|
+
or _is_adapter_tests_only(config)
|
|
90
|
+
):
|
|
91
|
+
return
|
|
92
|
+
|
|
93
|
+
global _LANGFUSE_CLIENT, _LANGFUSE_CONFIG, _LANGFUSE_REPORTER
|
|
94
|
+
_LANGFUSE_CONFIG = LangfuseConfig.from_environment()
|
|
95
|
+
selected_agent = _local_conftest._resolve_eval_agent(config).agent
|
|
96
|
+
enable_trajectory_environment(_LANGFUSE_CONFIG, selected_agent)
|
|
97
|
+
_LANGFUSE_CLIENT = create_client(_LANGFUSE_CONFIG)
|
|
98
|
+
_LANGFUSE_REPORTER = LangfuseRunReporter(_LANGFUSE_CLIENT)
|
|
99
|
+
context = getattr(config, "_comet_eval_context", None) or context_from_environment()
|
|
100
|
+
project_root = Path(config.getoption("--project-root") or EVAL_ROOT.parent)
|
|
101
|
+
cache_root = resolve_trajectory_cache_root(context, project_root)
|
|
102
|
+
os.environ["LANGFUSE_TRAJECTORY_CACHE_DIR"] = str(cache_root)
|
|
103
|
+
try:
|
|
104
|
+
provision = provision_trajectory_plugin(cache_root, selected_agent)
|
|
105
|
+
if provision.plugin_path is not None:
|
|
106
|
+
os.environ["LANGFUSE_TRAJECTORY_PLUGIN_DIR"] = str(provision.plugin_path)
|
|
107
|
+
os.environ["LANGFUSE_TRAJECTORY_PROVISIONED"] = "true"
|
|
108
|
+
except TrajectoryProvisionError as exc:
|
|
109
|
+
os.environ["LANGFUSE_TRAJECTORY_PROVISIONED"] = "false"
|
|
110
|
+
print(f"[langfuse] optional trajectory provisioning unavailable: {exc}", file=os.sys.stderr)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def pytest_sessionfinish(session, exitstatus):
|
|
114
|
+
"""Flush the summary trace after all task cases have completed."""
|
|
115
|
+
if _LANGFUSE_REPORTER is None:
|
|
116
|
+
return
|
|
117
|
+
try:
|
|
118
|
+
if hasattr(session.config, "workerinput"):
|
|
119
|
+
_LANGFUSE_REPORTER.flush()
|
|
120
|
+
return
|
|
121
|
+
if (getattr(session.config.option, "numprocesses", None) or 0) > 0:
|
|
122
|
+
experiment_id = os.environ.get("COMET_EVAL_EXPERIMENT_ID", "")
|
|
123
|
+
reports_dir = get_runs_dir() / experiment_id / "reports"
|
|
124
|
+
cases = []
|
|
125
|
+
for report_path in sorted(reports_dir.glob("*_report.json")):
|
|
126
|
+
report = json.loads(report_path.read_text(encoding="utf-8"))
|
|
127
|
+
cases.append(result_payload_from_local_report(report))
|
|
128
|
+
_LANGFUSE_REPORTER.cases = cases
|
|
129
|
+
_LANGFUSE_REPORTER.report_summary()
|
|
130
|
+
_LANGFUSE_REPORTER.flush()
|
|
131
|
+
except Exception as exc:
|
|
132
|
+
session.exitstatus = 1
|
|
133
|
+
print(f"[langfuse] required summary reporting failed: {exc}", file=os.sys.stderr)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@pytest.fixture(scope="session", autouse=True)
|
|
137
|
+
def verify_langfuse_environment(request):
|
|
138
|
+
"""Keep collection offline; runtime authentication is done in pytest_configure."""
|
|
139
|
+
if (
|
|
140
|
+
request.config.option.collectonly
|
|
141
|
+
or _local_conftest._is_unit_tests_only(request.config)
|
|
142
|
+
or _is_adapter_tests_only(request.config)
|
|
143
|
+
):
|
|
144
|
+
return
|
|
145
|
+
if _LANGFUSE_REPORTER is None:
|
|
146
|
+
raise pytest.UsageError("Langfuse reporter was not initialized before Agent execution")
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Langfuse wrapper around the shared local task runner."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
import conftest
|
|
12
|
+
|
|
13
|
+
_LOCAL_TEST_TASKS = (
|
|
14
|
+
Path(__file__).resolve().parents[3] / "local" / "tests" / "tasks" / "test_tasks.py"
|
|
15
|
+
)
|
|
16
|
+
_spec = importlib.util.spec_from_file_location("_comet_local_test_tasks", _LOCAL_TEST_TASKS)
|
|
17
|
+
_local_test_tasks = importlib.util.module_from_spec(_spec)
|
|
18
|
+
assert _spec and _spec.loader
|
|
19
|
+
sys.modules[_spec.name] = _local_test_tasks
|
|
20
|
+
_spec.loader.exec_module(_local_test_tasks)
|
|
21
|
+
|
|
22
|
+
pytest_generate_tests = _local_test_tasks.pytest_generate_tests
|
|
23
|
+
for _name in dir(_local_test_tasks):
|
|
24
|
+
if _name.startswith("test_") and _name != "test_task_treatment":
|
|
25
|
+
globals()[_name] = getattr(_local_test_tasks, _name)
|
|
26
|
+
|
|
27
|
+
PYTEST_TIMEOUT = _local_test_tasks.PYTEST_TIMEOUT
|
|
28
|
+
_run_local_task_treatment = _local_test_tasks.test_task_treatment
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _latest_result(treatment_name):
|
|
32
|
+
candidates = [
|
|
33
|
+
getattr(_local_test_tasks, "conftest", None),
|
|
34
|
+
sys.modules.get("conftest"),
|
|
35
|
+
sys.modules.get("_comet_local_conftest"),
|
|
36
|
+
]
|
|
37
|
+
seen = set()
|
|
38
|
+
for module in candidates:
|
|
39
|
+
plugin = getattr(module, "_plugin", None) if module else None
|
|
40
|
+
logger = getattr(plugin, "logger", None) if plugin else None
|
|
41
|
+
if logger is None or id(logger) in seen:
|
|
42
|
+
continue
|
|
43
|
+
seen.add(id(logger))
|
|
44
|
+
runs = logger.results.get(treatment_name) or []
|
|
45
|
+
if runs:
|
|
46
|
+
return runs[-1]
|
|
47
|
+
|
|
48
|
+
logs_dir = Path(conftest.os.environ.get("BENCH_LOGS_DIR", ""))
|
|
49
|
+
reports_root = logs_dir / "experiments"
|
|
50
|
+
if not reports_root.exists():
|
|
51
|
+
return None
|
|
52
|
+
suffix = f"-{treatment_name}-"
|
|
53
|
+
for report_file in sorted(
|
|
54
|
+
reports_root.glob("*/reports/*_report.json"),
|
|
55
|
+
key=lambda path: path.stat().st_mtime,
|
|
56
|
+
reverse=True,
|
|
57
|
+
):
|
|
58
|
+
try:
|
|
59
|
+
report = json.loads(report_file.read_text(encoding="utf-8"))
|
|
60
|
+
except Exception:
|
|
61
|
+
continue
|
|
62
|
+
if report.get("name") == treatment_name or suffix in report.get("name", ""):
|
|
63
|
+
return conftest.TreatmentResult(
|
|
64
|
+
name=report.get("name", treatment_name),
|
|
65
|
+
passed=report.get("passed", False),
|
|
66
|
+
checks_passed=report.get("checks_passed", []),
|
|
67
|
+
checks_failed=report.get("checks_failed", []),
|
|
68
|
+
events_summary=report.get("events_summary", {}),
|
|
69
|
+
run_id=report.get("run_id", ""),
|
|
70
|
+
)
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _latest_trajectory_path():
|
|
75
|
+
candidates = [
|
|
76
|
+
getattr(_local_test_tasks, "conftest", None),
|
|
77
|
+
sys.modules.get("conftest"),
|
|
78
|
+
sys.modules.get("_comet_local_conftest"),
|
|
79
|
+
]
|
|
80
|
+
for module in candidates:
|
|
81
|
+
plugin = getattr(module, "_plugin", None) if module else None
|
|
82
|
+
test_dir = getattr(plugin, "last_test_dir", None) if plugin else None
|
|
83
|
+
if test_dir:
|
|
84
|
+
path = Path(test_dir) / ".comet" / "eval" / "langfuse" / "trajectories"
|
|
85
|
+
if path.is_dir() and any(path.glob("*.jsonl")):
|
|
86
|
+
return path
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@pytest.mark.timeout(PYTEST_TIMEOUT)
|
|
91
|
+
@pytest.mark.langfuse
|
|
92
|
+
def test_task_treatment(task_name, treatment_name):
|
|
93
|
+
"""Run the local case and publish strict core results to Langfuse."""
|
|
94
|
+
reporter = getattr(conftest, "_LANGFUSE_REPORTER", None)
|
|
95
|
+
if reporter is None:
|
|
96
|
+
return _run_local_task_treatment(task_name, treatment_name)
|
|
97
|
+
plugin = getattr(conftest._local_conftest, "_plugin", None) or getattr(
|
|
98
|
+
conftest, "_plugin", None
|
|
99
|
+
)
|
|
100
|
+
agent = conftest._resolve_eval_agent(plugin.config).agent if plugin else "claude-code"
|
|
101
|
+
return reporter.report_case(
|
|
102
|
+
task_name,
|
|
103
|
+
treatment_name,
|
|
104
|
+
lambda: _run_local_task_treatment(task_name, treatment_name),
|
|
105
|
+
lambda: _latest_result(treatment_name),
|
|
106
|
+
metadata={"agent": agent},
|
|
107
|
+
trajectory_path=_latest_trajectory_path,
|
|
108
|
+
agent=agent,
|
|
109
|
+
)
|