@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -0,0 +1,1023 @@
|
|
|
1
|
+
"""Langfuse core reporting, official plugin provisioning, and trajectory adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import shutil
|
|
10
|
+
import tarfile
|
|
11
|
+
import tempfile
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path, PurePosixPath
|
|
15
|
+
from types import SimpleNamespace
|
|
16
|
+
from typing import Any, Callable, Iterator, Mapping
|
|
17
|
+
from urllib.request import Request, urlopen
|
|
18
|
+
|
|
19
|
+
from scaffold.python.pass_at_k import compute_pass_metrics
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LangfuseConfigurationError(RuntimeError):
|
|
23
|
+
"""The Langfuse suite cannot start safely with the current configuration."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LangfuseReportingError(RuntimeError):
|
|
27
|
+
"""A required Langfuse core trace or score could not be written."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TrajectoryProvisionError(RuntimeError):
|
|
31
|
+
"""An official trajectory plugin could not be safely provisioned."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class TrajectoryEvent:
|
|
36
|
+
"""A bounded, agent-neutral event parsed from a JSONL transcript."""
|
|
37
|
+
|
|
38
|
+
kind: str
|
|
39
|
+
name: str
|
|
40
|
+
input: Any = None
|
|
41
|
+
output: Any = None
|
|
42
|
+
session_id: str | None = None
|
|
43
|
+
truncated: bool = False
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class TrajectoryProvision:
|
|
48
|
+
"""The isolated plugin or hook provision selected for one eval run."""
|
|
49
|
+
|
|
50
|
+
agent: str
|
|
51
|
+
mode: str
|
|
52
|
+
cache_dir: Path
|
|
53
|
+
plugin_path: Path | None = None
|
|
54
|
+
source_repo: str | None = None
|
|
55
|
+
source_ref: str | None = None
|
|
56
|
+
archive_sha256: str | None = None
|
|
57
|
+
content_sha256: str | None = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_OFFICIAL_PLUGIN_SPECS: dict[str, dict[str, str]] = {
|
|
61
|
+
"claude-code": {
|
|
62
|
+
"repo": "langfuse/Claude-Observability-Plugin",
|
|
63
|
+
"ref": "05b7742829e0b0ad840c8760376ca737fa1eb662",
|
|
64
|
+
"archive_root": ".",
|
|
65
|
+
},
|
|
66
|
+
"codex": {
|
|
67
|
+
"repo": "langfuse/codex-observability-plugin",
|
|
68
|
+
"ref": "96e997cf9c3d4aee7421e849d764b83dcb485462",
|
|
69
|
+
"archive_root": "plugins/tracing",
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
_RUBRIC_SCORE_RE = re.compile(r"\[RUBRIC\]\s+([A-Za-z0-9_.-]+):\s*(0(?:\.\d+)?|1(?:\.0+)?)\b")
|
|
74
|
+
_SUMMARY_K_VALUES = (1, 2, 5)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class LangfuseConfig:
|
|
79
|
+
public_key: str
|
|
80
|
+
secret_key: str
|
|
81
|
+
base_url: str | None = None
|
|
82
|
+
tracing_environment: str | None = None
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def from_environment(cls, environ: Mapping[str, str] | None = None) -> "LangfuseConfig":
|
|
86
|
+
env = environ or os.environ
|
|
87
|
+
missing = [
|
|
88
|
+
name
|
|
89
|
+
for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY")
|
|
90
|
+
if not env.get(name, "").strip()
|
|
91
|
+
]
|
|
92
|
+
if missing:
|
|
93
|
+
raise LangfuseConfigurationError(
|
|
94
|
+
"Langfuse suite requires "
|
|
95
|
+
+ ", ".join(missing)
|
|
96
|
+
+ ". Set them in the process environment; they are never persisted by Comet."
|
|
97
|
+
)
|
|
98
|
+
return cls(
|
|
99
|
+
public_key=env["LANGFUSE_PUBLIC_KEY"].strip(),
|
|
100
|
+
secret_key=env["LANGFUSE_SECRET_KEY"].strip(),
|
|
101
|
+
base_url=env.get("LANGFUSE_BASE_URL") or None,
|
|
102
|
+
tracing_environment=env.get("LANGFUSE_TRACING_ENVIRONMENT") or None,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def trajectory_mode(agent: str) -> str:
|
|
107
|
+
return {
|
|
108
|
+
"claude-code": "official-claude-code-plugin",
|
|
109
|
+
"codex": "official-codex-plugin",
|
|
110
|
+
"qoder": "qoder-stop-transcript",
|
|
111
|
+
"codebuddy": "codebuddy-stop-transcript",
|
|
112
|
+
}.get(agent, "best-effort-core-only")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _sha256_file(path: Path) -> str:
|
|
116
|
+
digest = hashlib.sha256()
|
|
117
|
+
with path.open("rb") as stream:
|
|
118
|
+
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
119
|
+
digest.update(chunk)
|
|
120
|
+
return digest.hexdigest()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _sha256_tree(root: Path) -> str:
|
|
124
|
+
digest = hashlib.sha256()
|
|
125
|
+
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
|
126
|
+
relative = path.relative_to(root).as_posix()
|
|
127
|
+
digest.update(relative.encode("utf-8"))
|
|
128
|
+
digest.update(b"\0")
|
|
129
|
+
digest.update(path.read_bytes())
|
|
130
|
+
digest.update(b"\0")
|
|
131
|
+
return digest.hexdigest()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _download_plugin_archive(url: str, destination: Path) -> None:
|
|
135
|
+
request = Request(url, headers={"User-Agent": "Comet eval Langfuse adapter"})
|
|
136
|
+
try:
|
|
137
|
+
with urlopen(request, timeout=90) as response, destination.open("wb") as output:
|
|
138
|
+
shutil.copyfileobj(response, output)
|
|
139
|
+
except Exception as exc:
|
|
140
|
+
raise TrajectoryProvisionError(
|
|
141
|
+
f"Langfuse trajectory plugin download failed: {exc}"
|
|
142
|
+
) from exc
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@contextmanager
|
|
146
|
+
def _provision_lock(path: Path) -> Iterator[None]:
|
|
147
|
+
"""Coordinate cache writes across pytest workers and separate processes."""
|
|
148
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
149
|
+
with path.open("a+b") as lock_file:
|
|
150
|
+
if os.name == "nt":
|
|
151
|
+
import msvcrt
|
|
152
|
+
|
|
153
|
+
lock_file.seek(0, os.SEEK_END)
|
|
154
|
+
if lock_file.tell() == 0:
|
|
155
|
+
lock_file.write(b"0")
|
|
156
|
+
lock_file.flush()
|
|
157
|
+
lock_file.seek(0)
|
|
158
|
+
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
|
159
|
+
try:
|
|
160
|
+
yield
|
|
161
|
+
finally:
|
|
162
|
+
lock_file.seek(0)
|
|
163
|
+
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
|
164
|
+
else:
|
|
165
|
+
import fcntl
|
|
166
|
+
|
|
167
|
+
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
|
168
|
+
try:
|
|
169
|
+
yield
|
|
170
|
+
finally:
|
|
171
|
+
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _safe_extract_archive(archive_path: Path, destination: Path) -> Path:
|
|
175
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
176
|
+
root_name: str | None = None
|
|
177
|
+
try:
|
|
178
|
+
with tarfile.open(archive_path, mode="r:gz") as archive:
|
|
179
|
+
members = archive.getmembers()
|
|
180
|
+
for member in members:
|
|
181
|
+
relative = PurePosixPath(member.name)
|
|
182
|
+
if relative.is_absolute() or ".." in relative.parts:
|
|
183
|
+
raise TrajectoryProvisionError(
|
|
184
|
+
f"Langfuse trajectory plugin archive contains an unsafe path: {member.name}"
|
|
185
|
+
)
|
|
186
|
+
if member.issym() or member.islnk():
|
|
187
|
+
raise TrajectoryProvisionError(
|
|
188
|
+
f"Langfuse trajectory plugin archive contains a link: {member.name}"
|
|
189
|
+
)
|
|
190
|
+
if not root_name and relative.parts:
|
|
191
|
+
root_name = relative.parts[0]
|
|
192
|
+
target = destination.joinpath(*relative.parts).resolve()
|
|
193
|
+
if destination.resolve() not in target.parents and target != destination.resolve():
|
|
194
|
+
raise TrajectoryProvisionError(
|
|
195
|
+
f"Langfuse trajectory plugin archive escapes its cache: {member.name}"
|
|
196
|
+
)
|
|
197
|
+
archive.extractall(destination, filter="data")
|
|
198
|
+
except TrajectoryProvisionError:
|
|
199
|
+
raise
|
|
200
|
+
except (OSError, tarfile.TarError) as exc:
|
|
201
|
+
raise TrajectoryProvisionError(
|
|
202
|
+
f"Langfuse trajectory plugin archive is invalid: {exc}"
|
|
203
|
+
) from exc
|
|
204
|
+
if not root_name:
|
|
205
|
+
raise TrajectoryProvisionError("Langfuse trajectory plugin archive is empty")
|
|
206
|
+
return destination / root_name
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _read_provisioned_plugin(
|
|
210
|
+
cache_dir: Path, agent: str, spec: Mapping[str, str]
|
|
211
|
+
) -> TrajectoryProvision:
|
|
212
|
+
metadata_path = cache_dir / "metadata.json"
|
|
213
|
+
plugin_path = cache_dir / "plugin"
|
|
214
|
+
try:
|
|
215
|
+
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
216
|
+
actual_content_hash = _sha256_tree(plugin_path)
|
|
217
|
+
except (OSError, ValueError, TypeError) as exc:
|
|
218
|
+
raise TrajectoryProvisionError(
|
|
219
|
+
f"Langfuse trajectory plugin cache is incomplete for {agent}: {cache_dir}"
|
|
220
|
+
) from exc
|
|
221
|
+
if (
|
|
222
|
+
metadata.get("source_ref") != spec["ref"]
|
|
223
|
+
or metadata.get("content_sha256") != actual_content_hash
|
|
224
|
+
):
|
|
225
|
+
raise TrajectoryProvisionError(
|
|
226
|
+
f"Langfuse trajectory plugin cache integrity check failed for {agent}: {cache_dir}"
|
|
227
|
+
)
|
|
228
|
+
return TrajectoryProvision(
|
|
229
|
+
agent=agent,
|
|
230
|
+
mode=trajectory_mode(agent),
|
|
231
|
+
cache_dir=cache_dir,
|
|
232
|
+
plugin_path=plugin_path,
|
|
233
|
+
source_repo=metadata.get("source_repo"),
|
|
234
|
+
source_ref=metadata.get("source_ref"),
|
|
235
|
+
archive_sha256=metadata.get("archive_sha256"),
|
|
236
|
+
content_sha256=actual_content_hash,
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def provision_trajectory_plugin(
|
|
241
|
+
cache_root: Path,
|
|
242
|
+
agent: str,
|
|
243
|
+
*,
|
|
244
|
+
download: Callable[[str, Path], None] | None = None,
|
|
245
|
+
) -> TrajectoryProvision:
|
|
246
|
+
"""Provision a pinned official plugin without touching user agent config."""
|
|
247
|
+
mode = trajectory_mode(agent)
|
|
248
|
+
if agent not in _OFFICIAL_PLUGIN_SPECS:
|
|
249
|
+
return TrajectoryProvision(agent=agent, mode=mode, cache_dir=cache_root / agent)
|
|
250
|
+
|
|
251
|
+
spec = _OFFICIAL_PLUGIN_SPECS[agent]
|
|
252
|
+
cache_dir = cache_root / agent / spec["ref"]
|
|
253
|
+
cache_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
254
|
+
with _provision_lock(cache_dir.parent / ".provision.lock"):
|
|
255
|
+
if (cache_dir / "metadata.json").is_file() and (cache_dir / "plugin").is_dir():
|
|
256
|
+
return _read_provisioned_plugin(cache_dir, agent, spec)
|
|
257
|
+
|
|
258
|
+
staging = Path(tempfile.mkdtemp(prefix=f".{agent}-", dir=cache_dir.parent))
|
|
259
|
+
try:
|
|
260
|
+
archive_path = staging / "plugin.tar.gz"
|
|
261
|
+
source_url = f"https://codeload.github.com/{spec['repo']}/tar.gz/{spec['ref']}"
|
|
262
|
+
(download or _download_plugin_archive)(source_url, archive_path)
|
|
263
|
+
archive_hash = _sha256_file(archive_path)
|
|
264
|
+
extracted_root = _safe_extract_archive(archive_path, staging / "extracted")
|
|
265
|
+
required_plugin_path = (
|
|
266
|
+
extracted_root
|
|
267
|
+
if spec["archive_root"] == "."
|
|
268
|
+
else extracted_root / spec["archive_root"]
|
|
269
|
+
)
|
|
270
|
+
if not required_plugin_path.is_dir():
|
|
271
|
+
raise TrajectoryProvisionError(
|
|
272
|
+
f"Langfuse trajectory plugin path is missing: {spec['archive_root']}"
|
|
273
|
+
)
|
|
274
|
+
plugin_destination = staging / "plugin"
|
|
275
|
+
shutil.copytree(extracted_root, plugin_destination)
|
|
276
|
+
content_hash = _sha256_tree(plugin_destination)
|
|
277
|
+
metadata = {
|
|
278
|
+
"schema": "comet.eval.langfuse-plugin.v1",
|
|
279
|
+
"agent": agent,
|
|
280
|
+
"source_repo": spec["repo"],
|
|
281
|
+
"source_ref": spec["ref"],
|
|
282
|
+
"archive_sha256": archive_hash,
|
|
283
|
+
"content_sha256": content_hash,
|
|
284
|
+
"mode": mode,
|
|
285
|
+
}
|
|
286
|
+
(staging / "metadata.json").write_text(
|
|
287
|
+
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
288
|
+
)
|
|
289
|
+
if cache_dir.exists():
|
|
290
|
+
shutil.rmtree(cache_dir)
|
|
291
|
+
os.replace(staging, cache_dir)
|
|
292
|
+
staging = None # type: ignore[assignment]
|
|
293
|
+
except TrajectoryProvisionError:
|
|
294
|
+
raise
|
|
295
|
+
except Exception as exc:
|
|
296
|
+
raise TrajectoryProvisionError(
|
|
297
|
+
f"Langfuse trajectory plugin provisioning failed for {agent}: {exc}"
|
|
298
|
+
) from exc
|
|
299
|
+
finally:
|
|
300
|
+
if staging is not None and staging.exists():
|
|
301
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
302
|
+
return _read_provisioned_plugin(cache_dir, agent, spec)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _bound_value(value: Any, max_chars: int) -> tuple[Any, bool]:
|
|
306
|
+
if isinstance(value, str):
|
|
307
|
+
return (value[:max_chars], len(value) > max_chars)
|
|
308
|
+
serialized = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
|
|
309
|
+
if len(serialized) <= max_chars:
|
|
310
|
+
return value, False
|
|
311
|
+
return serialized[:max_chars], True
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _message_content(record: Mapping[str, Any]) -> Any:
|
|
315
|
+
message = record.get("message")
|
|
316
|
+
if isinstance(message, Mapping):
|
|
317
|
+
return message.get("content")
|
|
318
|
+
return record.get("content")
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _session_id(record: Mapping[str, Any]) -> str | None:
|
|
322
|
+
value = record.get("sessionId") or record.get("session_id")
|
|
323
|
+
return str(value) if value else None
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def parse_agent_transcript(path: Path, *, max_chars: int = 20_000) -> list[TrajectoryEvent]:
|
|
327
|
+
"""Parse Qoder/CodeBuddy transcript JSONL without inferring missing events."""
|
|
328
|
+
events: list[TrajectoryEvent] = []
|
|
329
|
+
try:
|
|
330
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
331
|
+
except OSError as exc:
|
|
332
|
+
raise TrajectoryProvisionError(f"Unable to read agent transcript: {path}") from exc
|
|
333
|
+
|
|
334
|
+
for line in lines:
|
|
335
|
+
if not line.strip():
|
|
336
|
+
continue
|
|
337
|
+
try:
|
|
338
|
+
record = json.loads(line)
|
|
339
|
+
except json.JSONDecodeError:
|
|
340
|
+
continue
|
|
341
|
+
if not isinstance(record, Mapping):
|
|
342
|
+
continue
|
|
343
|
+
record_type = str(record.get("type", ""))
|
|
344
|
+
session_id = _session_id(record)
|
|
345
|
+
if record_type == "session_meta":
|
|
346
|
+
data, truncated = _bound_value(record.get("data", {}), max_chars)
|
|
347
|
+
events.append(
|
|
348
|
+
TrajectoryEvent(
|
|
349
|
+
"session",
|
|
350
|
+
"session_meta",
|
|
351
|
+
output=data,
|
|
352
|
+
session_id=session_id,
|
|
353
|
+
truncated=truncated,
|
|
354
|
+
)
|
|
355
|
+
)
|
|
356
|
+
continue
|
|
357
|
+
|
|
358
|
+
content = _message_content(record)
|
|
359
|
+
items = content if isinstance(content, list) else [content]
|
|
360
|
+
for item in items:
|
|
361
|
+
if isinstance(item, Mapping):
|
|
362
|
+
item_type = str(item.get("type", ""))
|
|
363
|
+
if item_type == "tool_use":
|
|
364
|
+
input_value, truncated = _bound_value(item.get("input", {}), max_chars)
|
|
365
|
+
events.append(
|
|
366
|
+
TrajectoryEvent(
|
|
367
|
+
"tool",
|
|
368
|
+
str(item.get("name") or "tool"),
|
|
369
|
+
input=input_value,
|
|
370
|
+
session_id=session_id,
|
|
371
|
+
truncated=truncated,
|
|
372
|
+
)
|
|
373
|
+
)
|
|
374
|
+
continue
|
|
375
|
+
if item_type == "tool_result":
|
|
376
|
+
output, truncated = _bound_value(item.get("content", ""), max_chars)
|
|
377
|
+
events.append(
|
|
378
|
+
TrajectoryEvent(
|
|
379
|
+
"tool_result",
|
|
380
|
+
str(item.get("tool_use_id") or "tool_result"),
|
|
381
|
+
output=output,
|
|
382
|
+
session_id=session_id,
|
|
383
|
+
truncated=truncated,
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
continue
|
|
387
|
+
if item_type == "text":
|
|
388
|
+
value, truncated = _bound_value(item.get("text", ""), max_chars)
|
|
389
|
+
events.append(
|
|
390
|
+
TrajectoryEvent(
|
|
391
|
+
"text",
|
|
392
|
+
"assistant",
|
|
393
|
+
output=value,
|
|
394
|
+
session_id=session_id,
|
|
395
|
+
truncated=truncated,
|
|
396
|
+
)
|
|
397
|
+
)
|
|
398
|
+
continue
|
|
399
|
+
elif isinstance(item, str) and item:
|
|
400
|
+
value, truncated = _bound_value(item, max_chars)
|
|
401
|
+
events.append(
|
|
402
|
+
TrajectoryEvent(
|
|
403
|
+
"user" if record_type == "user" else "text",
|
|
404
|
+
"user" if record_type == "user" else "assistant",
|
|
405
|
+
input=value if record_type == "user" else None,
|
|
406
|
+
output=value if record_type != "user" else None,
|
|
407
|
+
session_id=session_id,
|
|
408
|
+
truncated=truncated,
|
|
409
|
+
)
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
if record_type == "progress":
|
|
413
|
+
value, truncated = _bound_value(record.get("data", {}), max_chars)
|
|
414
|
+
events.append(
|
|
415
|
+
TrajectoryEvent(
|
|
416
|
+
"progress", "progress", output=value, session_id=session_id, truncated=truncated
|
|
417
|
+
)
|
|
418
|
+
)
|
|
419
|
+
return events
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def render_transcript_hook_script() -> str:
|
|
423
|
+
"""Return the no-credential Stop hook used by Qoder and CodeBuddy."""
|
|
424
|
+
return """#!/usr/bin/env python3
|
|
425
|
+
import hashlib
|
|
426
|
+
import json
|
|
427
|
+
import os
|
|
428
|
+
import shutil
|
|
429
|
+
import sys
|
|
430
|
+
import tempfile
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def main():
|
|
434
|
+
try:
|
|
435
|
+
payload = json.load(sys.stdin)
|
|
436
|
+
except Exception:
|
|
437
|
+
return 0
|
|
438
|
+
if payload.get("hook_event_name") != "Stop":
|
|
439
|
+
return 0
|
|
440
|
+
transcript = payload.get("transcript_path")
|
|
441
|
+
cwd = payload.get("cwd") or os.getcwd()
|
|
442
|
+
if not transcript or not os.path.isfile(transcript):
|
|
443
|
+
return 0
|
|
444
|
+
role = os.environ.get("COMET_EVAL_AGENT_ROLE", "subject")
|
|
445
|
+
if role not in {"subject", "simulator", "judge"}:
|
|
446
|
+
role = "subject"
|
|
447
|
+
session_id = payload.get("session_id") or payload.get("sessionId")
|
|
448
|
+
if not session_id:
|
|
449
|
+
session_id = hashlib.sha256(os.path.abspath(transcript).encode()).hexdigest()[:16]
|
|
450
|
+
safe_session = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in str(session_id))[:128]
|
|
451
|
+
target_dir = os.path.join(cwd, ".comet", "eval", "langfuse", "trajectories")
|
|
452
|
+
target = os.path.join(target_dir, f"{role}-{safe_session}.jsonl")
|
|
453
|
+
os.makedirs(target_dir, exist_ok=True)
|
|
454
|
+
fd, temporary = tempfile.mkstemp(prefix=".trajectory-", dir=os.path.dirname(target))
|
|
455
|
+
os.close(fd)
|
|
456
|
+
try:
|
|
457
|
+
shutil.copyfile(transcript, temporary)
|
|
458
|
+
os.replace(temporary, target)
|
|
459
|
+
finally:
|
|
460
|
+
try:
|
|
461
|
+
os.unlink(temporary)
|
|
462
|
+
except FileNotFoundError:
|
|
463
|
+
pass
|
|
464
|
+
return 0
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
if __name__ == "__main__":
|
|
468
|
+
raise SystemExit(main())
|
|
469
|
+
"""
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def install_transcript_hook(test_dir: Path, agent: str) -> Path | None:
|
|
473
|
+
"""Install a project-local, credential-free Stop hook for transcript agents."""
|
|
474
|
+
if agent not in {"qoder", "codebuddy"}:
|
|
475
|
+
return None
|
|
476
|
+
project_dir_name = ".qoder" if agent == "qoder" else ".codebuddy"
|
|
477
|
+
project_dir = test_dir / project_dir_name
|
|
478
|
+
hook_path = project_dir / "hooks" / "langfuse-stop-hook.py"
|
|
479
|
+
hook_path.parent.mkdir(parents=True, exist_ok=True)
|
|
480
|
+
hook_path.write_text(render_transcript_hook_script(), encoding="utf-8")
|
|
481
|
+
settings_path = project_dir / "settings.json"
|
|
482
|
+
settings: dict[str, Any] = {}
|
|
483
|
+
if settings_path.is_file():
|
|
484
|
+
settings = json.loads(settings_path.read_text(encoding="utf-8"))
|
|
485
|
+
hooks = settings.setdefault("hooks", {})
|
|
486
|
+
stop_hooks = hooks.setdefault("Stop", [])
|
|
487
|
+
entry = {
|
|
488
|
+
"hooks": [
|
|
489
|
+
{
|
|
490
|
+
"type": "command",
|
|
491
|
+
"command": f"python3 /workspace/{project_dir_name}/hooks/langfuse-stop-hook.py",
|
|
492
|
+
"timeout": 30,
|
|
493
|
+
}
|
|
494
|
+
]
|
|
495
|
+
}
|
|
496
|
+
if entry not in stop_hooks:
|
|
497
|
+
stop_hooks.append(entry)
|
|
498
|
+
settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
|
|
499
|
+
return test_dir / ".comet" / "eval" / "langfuse" / "trajectories"
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def install_codex_plugin_workspace(test_dir: Path, provision: TrajectoryProvision) -> None:
|
|
503
|
+
"""Copy the pinned Codex plugin into the isolated project CODEX_HOME."""
|
|
504
|
+
if provision.agent != "codex" or provision.plugin_path is None:
|
|
505
|
+
return
|
|
506
|
+
source = provision.plugin_path / "plugins" / "tracing"
|
|
507
|
+
version = json.loads((source / ".codex-plugin" / "plugin.json").read_text(encoding="utf-8"))[
|
|
508
|
+
"version"
|
|
509
|
+
]
|
|
510
|
+
target = (
|
|
511
|
+
test_dir
|
|
512
|
+
/ ".codex"
|
|
513
|
+
/ "plugins"
|
|
514
|
+
/ "cache"
|
|
515
|
+
/ "codex-observability-plugin"
|
|
516
|
+
/ "tracing"
|
|
517
|
+
/ version
|
|
518
|
+
)
|
|
519
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
520
|
+
shutil.copytree(source, target, dirs_exist_ok=True)
|
|
521
|
+
config_path = test_dir / ".codex" / "config.toml"
|
|
522
|
+
existing = config_path.read_text(encoding="utf-8") if config_path.is_file() else ""
|
|
523
|
+
lines = existing.splitlines(keepends=True)
|
|
524
|
+
feature_header = next(
|
|
525
|
+
(index for index, line in enumerate(lines) if line.strip() == "[features]"),
|
|
526
|
+
None,
|
|
527
|
+
)
|
|
528
|
+
if feature_header is None:
|
|
529
|
+
existing = existing.rstrip() + "\n\n[features]\nhooks = true\n"
|
|
530
|
+
else:
|
|
531
|
+
feature_end = next(
|
|
532
|
+
(
|
|
533
|
+
index
|
|
534
|
+
for index in range(feature_header + 1, len(lines))
|
|
535
|
+
if lines[index].lstrip().startswith("[")
|
|
536
|
+
),
|
|
537
|
+
len(lines),
|
|
538
|
+
)
|
|
539
|
+
feature_lines = lines[feature_header + 1 : feature_end]
|
|
540
|
+
hooks_line = next(
|
|
541
|
+
(
|
|
542
|
+
index
|
|
543
|
+
for index, line in enumerate(feature_lines)
|
|
544
|
+
if line.strip().startswith("hooks =")
|
|
545
|
+
),
|
|
546
|
+
None,
|
|
547
|
+
)
|
|
548
|
+
if hooks_line is None:
|
|
549
|
+
lines.insert(feature_header + 1 + len(feature_lines), "hooks = true\n")
|
|
550
|
+
else:
|
|
551
|
+
lines[feature_header + 1 + hooks_line] = "hooks = true\n"
|
|
552
|
+
existing = "".join(lines)
|
|
553
|
+
if '[plugins."tracing@codex-observability-plugin"]' not in existing:
|
|
554
|
+
existing = (
|
|
555
|
+
existing.rstrip()
|
|
556
|
+
+ '\n\n[plugins."tracing@codex-observability-plugin"]\nenabled = true\n'
|
|
557
|
+
)
|
|
558
|
+
config_path.parent.mkdir(parents=True, exist_ok=True)
|
|
559
|
+
config_path.write_text(existing.lstrip(), encoding="utf-8")
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def create_client(
|
|
563
|
+
config: LangfuseConfig | None = None,
|
|
564
|
+
*,
|
|
565
|
+
client_factory: Callable[..., Any] | None = None,
|
|
566
|
+
authenticate: bool = True,
|
|
567
|
+
) -> Any:
|
|
568
|
+
"""Create and authenticate the SDK client only at the run boundary."""
|
|
569
|
+
resolved = config or LangfuseConfig.from_environment()
|
|
570
|
+
if client_factory is None:
|
|
571
|
+
try:
|
|
572
|
+
from langfuse import Langfuse
|
|
573
|
+
except Exception as exc: # pragma: no cover - optional dependency
|
|
574
|
+
raise LangfuseConfigurationError(
|
|
575
|
+
"The Langfuse Python SDK is unavailable. Install the eval langfuse extra."
|
|
576
|
+
) from exc
|
|
577
|
+
client_factory = Langfuse
|
|
578
|
+
|
|
579
|
+
kwargs: dict[str, str] = {"public_key": resolved.public_key, "secret_key": resolved.secret_key}
|
|
580
|
+
if resolved.base_url:
|
|
581
|
+
kwargs["base_url"] = resolved.base_url
|
|
582
|
+
if resolved.tracing_environment:
|
|
583
|
+
kwargs["environment"] = resolved.tracing_environment
|
|
584
|
+
try:
|
|
585
|
+
client = client_factory(**kwargs)
|
|
586
|
+
if authenticate and hasattr(client, "auth_check") and not client.auth_check():
|
|
587
|
+
raise LangfuseConfigurationError("Langfuse authentication check failed")
|
|
588
|
+
return client
|
|
589
|
+
except LangfuseConfigurationError:
|
|
590
|
+
raise
|
|
591
|
+
except Exception as exc:
|
|
592
|
+
raise LangfuseConfigurationError(f"Langfuse authentication failed: {exc}") from exc
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def enable_trajectory_environment(config: LangfuseConfig, agent: str) -> str:
|
|
596
|
+
"""Expose credentials to a selected agent without writing them to artifacts."""
|
|
597
|
+
os.environ["TRACE_TO_LANGFUSE"] = "true"
|
|
598
|
+
os.environ["LANGFUSE_PUBLIC_KEY"] = config.public_key
|
|
599
|
+
os.environ["LANGFUSE_SECRET_KEY"] = config.secret_key
|
|
600
|
+
if config.base_url:
|
|
601
|
+
os.environ["LANGFUSE_BASE_URL"] = config.base_url
|
|
602
|
+
if config.tracing_environment:
|
|
603
|
+
os.environ["LANGFUSE_TRACING_ENVIRONMENT"] = config.tracing_environment
|
|
604
|
+
os.environ["LANGFUSE_TRAJECTORY_MODE"] = trajectory_mode(agent)
|
|
605
|
+
os.environ["LANGFUSE_TRAJECTORY_PROVISIONED"] = "false"
|
|
606
|
+
return trajectory_mode(agent)
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _result_payload(result: Any) -> dict[str, Any]:
|
|
610
|
+
passed = list(getattr(result, "checks_passed", []) or [])
|
|
611
|
+
failed = list(getattr(result, "checks_failed", []) or [])
|
|
612
|
+
summary = dict(getattr(result, "events_summary", {}) or {})
|
|
613
|
+
total_checks = len(passed) + len(failed)
|
|
614
|
+
payload: dict[str, Any] = {
|
|
615
|
+
"run_id": getattr(result, "run_id", ""),
|
|
616
|
+
"passed": bool(getattr(result, "passed", False)),
|
|
617
|
+
"checks_passed": len(passed),
|
|
618
|
+
"checks_failed": len(failed),
|
|
619
|
+
"checks_pass_rate": len(passed) / total_checks if total_checks else 0.0,
|
|
620
|
+
"failure_attribution": summary.get("failure_attribution"),
|
|
621
|
+
"num_turns": summary.get("num_turns"),
|
|
622
|
+
"tool_calls": summary.get("tool_calls"),
|
|
623
|
+
"duration_seconds": summary.get("duration_seconds"),
|
|
624
|
+
"total_tokens": summary.get("total_tokens"),
|
|
625
|
+
"total_cost_usd": summary.get("total_cost_usd"),
|
|
626
|
+
"rubric": _rubric_scores(passed),
|
|
627
|
+
}
|
|
628
|
+
for key in (
|
|
629
|
+
"task",
|
|
630
|
+
"treatment",
|
|
631
|
+
"sample",
|
|
632
|
+
"prompt",
|
|
633
|
+
"skill",
|
|
634
|
+
"profile",
|
|
635
|
+
"final_response",
|
|
636
|
+
"quality_gates",
|
|
637
|
+
"execution_identity",
|
|
638
|
+
"case_manifest",
|
|
639
|
+
"interaction",
|
|
640
|
+
"agent",
|
|
641
|
+
"role_models",
|
|
642
|
+
"role_sessions",
|
|
643
|
+
):
|
|
644
|
+
value = summary.get(key)
|
|
645
|
+
if value is not None:
|
|
646
|
+
payload[key] = _bound_value(value, 20_000)[0]
|
|
647
|
+
experiment_id = os.environ.get("COMET_EVAL_EXPERIMENT_ID")
|
|
648
|
+
if experiment_id:
|
|
649
|
+
payload["experiment_id"] = experiment_id
|
|
650
|
+
payload["quality_gate"] = _quality_gate_result(payload)
|
|
651
|
+
return payload
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def result_payload_from_local_report(report: Mapping[str, Any]) -> dict[str, Any]:
|
|
655
|
+
"""Rebuild the canonical case payload from a persisted local report."""
|
|
656
|
+
return _result_payload(
|
|
657
|
+
SimpleNamespace(
|
|
658
|
+
run_id=report.get("run_id", ""),
|
|
659
|
+
passed=bool(report.get("passed")),
|
|
660
|
+
checks_passed=list(report.get("checks_passed") or []),
|
|
661
|
+
checks_failed=list(report.get("checks_failed") or []),
|
|
662
|
+
events_summary=dict(report.get("events_summary") or {}),
|
|
663
|
+
)
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def _rubric_scores(checks: list[str]) -> dict[str, float]:
|
|
668
|
+
scores: dict[str, float] = {}
|
|
669
|
+
for check in checks:
|
|
670
|
+
match = _RUBRIC_SCORE_RE.search(check)
|
|
671
|
+
if not match:
|
|
672
|
+
continue
|
|
673
|
+
try:
|
|
674
|
+
scores[match.group(1)] = float(match.group(2))
|
|
675
|
+
except ValueError:
|
|
676
|
+
continue
|
|
677
|
+
return scores
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def _quality_gate_result(payload: Mapping[str, Any]) -> dict[str, Any]:
|
|
681
|
+
configured = payload.get("quality_gates")
|
|
682
|
+
if not isinstance(configured, Mapping) or not configured:
|
|
683
|
+
return {"status": "not_applicable", "checks": {}}
|
|
684
|
+
|
|
685
|
+
rubric = payload.get("rubric")
|
|
686
|
+
rubric_scores = rubric if isinstance(rubric, Mapping) else {}
|
|
687
|
+
checks: dict[str, bool | None] = {}
|
|
688
|
+
aliases = {
|
|
689
|
+
"minWeightedScore": "weighted_score",
|
|
690
|
+
"minPassRate": "checks_pass_rate",
|
|
691
|
+
"minChecksPassRate": "checks_pass_rate",
|
|
692
|
+
}
|
|
693
|
+
for name, expected in configured.items():
|
|
694
|
+
metric_name = aliases.get(str(name))
|
|
695
|
+
if metric_name is None and str(name).startswith("min"):
|
|
696
|
+
candidate = str(name)[3:]
|
|
697
|
+
metric_name = candidate[:1].lower() + candidate[1:]
|
|
698
|
+
if (
|
|
699
|
+
metric_name is None
|
|
700
|
+
or not isinstance(expected, (int, float))
|
|
701
|
+
or isinstance(expected, bool)
|
|
702
|
+
):
|
|
703
|
+
checks[str(name)] = None
|
|
704
|
+
continue
|
|
705
|
+
actual = rubric_scores.get(metric_name)
|
|
706
|
+
if metric_name == "checks_pass_rate":
|
|
707
|
+
actual = payload.get("checks_pass_rate")
|
|
708
|
+
checks[str(name)] = (
|
|
709
|
+
isinstance(actual, (int, float))
|
|
710
|
+
and not isinstance(actual, bool)
|
|
711
|
+
and float(actual) >= float(expected)
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
applicable = [value for value in checks.values() if value is not None]
|
|
715
|
+
if not applicable:
|
|
716
|
+
status = "unavailable"
|
|
717
|
+
elif any(value is False for value in applicable):
|
|
718
|
+
status = "failed"
|
|
719
|
+
elif any(value is None for value in checks.values()):
|
|
720
|
+
status = "unavailable"
|
|
721
|
+
else:
|
|
722
|
+
status = "passed"
|
|
723
|
+
return {"status": status, "checks": checks}
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
class LangfuseRunReporter:
|
|
727
|
+
"""Write one strict core trace per task/treatment and score its result."""
|
|
728
|
+
|
|
729
|
+
def __init__(self, client: Any, *, suite: str = "comet-eval-langfuse"):
|
|
730
|
+
self.client = client
|
|
731
|
+
self.suite = suite
|
|
732
|
+
self.cases: list[dict[str, Any]] = []
|
|
733
|
+
|
|
734
|
+
def report_case(
|
|
735
|
+
self,
|
|
736
|
+
task_name: str,
|
|
737
|
+
treatment_name: str,
|
|
738
|
+
run: Callable[[], Any],
|
|
739
|
+
result_loader: Callable[[], Any],
|
|
740
|
+
*,
|
|
741
|
+
metadata: Mapping[str, Any] | None = None,
|
|
742
|
+
trajectory_path: Path | Callable[[], Path | None] | None = None,
|
|
743
|
+
agent: str | None = None,
|
|
744
|
+
) -> Any:
|
|
745
|
+
input_payload = {
|
|
746
|
+
"task": task_name,
|
|
747
|
+
"treatment": treatment_name,
|
|
748
|
+
"suite": self.suite,
|
|
749
|
+
"experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
|
|
750
|
+
**dict(metadata or {}),
|
|
751
|
+
}
|
|
752
|
+
with self.client.start_as_current_observation(
|
|
753
|
+
as_type="span", name="comet.eval.run", input=input_payload
|
|
754
|
+
) as observation:
|
|
755
|
+
result = None
|
|
756
|
+
run_error: BaseException | None = None
|
|
757
|
+
try:
|
|
758
|
+
result = run()
|
|
759
|
+
except BaseException as exc:
|
|
760
|
+
run_error = exc
|
|
761
|
+
result_loader_error: BaseException | None = None
|
|
762
|
+
try:
|
|
763
|
+
result = result_loader() or result
|
|
764
|
+
except BaseException as exc:
|
|
765
|
+
result_loader_error = exc
|
|
766
|
+
payload = (
|
|
767
|
+
_result_payload(result)
|
|
768
|
+
if result is not None
|
|
769
|
+
else {
|
|
770
|
+
"passed": False,
|
|
771
|
+
"checks_passed": 0,
|
|
772
|
+
"checks_failed": 0,
|
|
773
|
+
"checks_pass_rate": 0.0,
|
|
774
|
+
}
|
|
775
|
+
)
|
|
776
|
+
try:
|
|
777
|
+
resolved_trajectory_path = (
|
|
778
|
+
trajectory_path() if callable(trajectory_path) else trajectory_path
|
|
779
|
+
)
|
|
780
|
+
payload["trajectory"] = self._report_trajectory(
|
|
781
|
+
resolved_trajectory_path,
|
|
782
|
+
agent=agent or str(input_payload.get("agent", "")),
|
|
783
|
+
role_sessions=payload.get("role_sessions") or {},
|
|
784
|
+
)
|
|
785
|
+
except Exception as exc:
|
|
786
|
+
print(f"[langfuse] optional trajectory upload skipped: {exc}", file=os.sys.stderr)
|
|
787
|
+
payload["trajectory"] = {
|
|
788
|
+
"status": "unavailable",
|
|
789
|
+
"mode": trajectory_mode(agent or str(input_payload.get("agent", ""))),
|
|
790
|
+
"reason": str(exc),
|
|
791
|
+
}
|
|
792
|
+
payload["trajectory_status"] = payload["trajectory"].get("status", "unavailable")
|
|
793
|
+
for key in ("prompt", "skill", "profile", "execution_identity", "agent", "sample"):
|
|
794
|
+
if payload.get(key) is not None:
|
|
795
|
+
input_payload[key] = payload[key]
|
|
796
|
+
core_error: BaseException | None = None
|
|
797
|
+
try:
|
|
798
|
+
observation.update(input=input_payload, output=payload)
|
|
799
|
+
self._score(observation, payload)
|
|
800
|
+
except BaseException as exc:
|
|
801
|
+
core_error = exc
|
|
802
|
+
self.cases.append({"task": task_name, "treatment": treatment_name, **payload})
|
|
803
|
+
if run_error is not None:
|
|
804
|
+
if core_error is not None:
|
|
805
|
+
print(
|
|
806
|
+
f"[langfuse] evaluation failed and core upload also failed: {core_error}",
|
|
807
|
+
file=os.sys.stderr,
|
|
808
|
+
)
|
|
809
|
+
raise run_error
|
|
810
|
+
if core_error is not None:
|
|
811
|
+
raise core_error
|
|
812
|
+
if result_loader_error is not None:
|
|
813
|
+
raise result_loader_error
|
|
814
|
+
return result
|
|
815
|
+
|
|
816
|
+
def _report_trajectory(
|
|
817
|
+
self,
|
|
818
|
+
trajectory_path: Path | None,
|
|
819
|
+
*,
|
|
820
|
+
agent: str,
|
|
821
|
+
role_sessions: Mapping[str, Any],
|
|
822
|
+
) -> dict[str, Any]:
|
|
823
|
+
mode = trajectory_mode(agent)
|
|
824
|
+
if mode.startswith("official-"):
|
|
825
|
+
if os.environ.get("LANGFUSE_TRAJECTORY_PROVISIONED", "").lower() != "true":
|
|
826
|
+
return {
|
|
827
|
+
"status": "unavailable",
|
|
828
|
+
"mode": mode,
|
|
829
|
+
"reason": "plugin_not_provisioned",
|
|
830
|
+
}
|
|
831
|
+
return {
|
|
832
|
+
"status": "provisioned",
|
|
833
|
+
"mode": mode,
|
|
834
|
+
"role_sessions": dict(role_sessions),
|
|
835
|
+
}
|
|
836
|
+
if not trajectory_path or not trajectory_path.exists():
|
|
837
|
+
return {"status": "unavailable", "mode": mode, "reason": "transcript_not_found"}
|
|
838
|
+
try:
|
|
839
|
+
transcript_paths = (
|
|
840
|
+
sorted(trajectory_path.glob("*.jsonl"))
|
|
841
|
+
if trajectory_path.is_dir()
|
|
842
|
+
else [trajectory_path]
|
|
843
|
+
)
|
|
844
|
+
events = [
|
|
845
|
+
event
|
|
846
|
+
for transcript_path in transcript_paths
|
|
847
|
+
for event in parse_agent_transcript(transcript_path)
|
|
848
|
+
]
|
|
849
|
+
if not events:
|
|
850
|
+
return {"status": "unavailable", "mode": mode, "reason": "transcript_empty"}
|
|
851
|
+
session_ids = sorted({event.session_id for event in events if event.session_id})
|
|
852
|
+
for index, event in enumerate(events):
|
|
853
|
+
observation_type = "generation" if event.kind == "text" else "span"
|
|
854
|
+
with self.client.start_as_current_observation(
|
|
855
|
+
as_type=observation_type,
|
|
856
|
+
name=f"agent.{event.kind}/{event.name}/{index}",
|
|
857
|
+
input=event.input,
|
|
858
|
+
) as child:
|
|
859
|
+
child.update(
|
|
860
|
+
output=event.output,
|
|
861
|
+
metadata={
|
|
862
|
+
"agent": agent,
|
|
863
|
+
"session_id": event.session_id,
|
|
864
|
+
"truncated": event.truncated,
|
|
865
|
+
},
|
|
866
|
+
)
|
|
867
|
+
return {
|
|
868
|
+
"status": "uploaded",
|
|
869
|
+
"mode": mode,
|
|
870
|
+
"events": len(events),
|
|
871
|
+
"session_ids": session_ids,
|
|
872
|
+
"role_sessions": dict(role_sessions),
|
|
873
|
+
"truncated_events": sum(1 for event in events if event.truncated),
|
|
874
|
+
}
|
|
875
|
+
except Exception as exc:
|
|
876
|
+
print(f"[langfuse] optional trajectory upload skipped: {exc}", file=os.sys.stderr)
|
|
877
|
+
return {"status": "unavailable", "mode": mode, "reason": str(exc)}
|
|
878
|
+
|
|
879
|
+
def _score(self, observation: Any, payload: Mapping[str, Any]) -> None:
|
|
880
|
+
trace_id = getattr(observation, "trace_id", None)
|
|
881
|
+
observation_id = getattr(observation, "id", None)
|
|
882
|
+
if not trace_id:
|
|
883
|
+
raise LangfuseReportingError("Langfuse observation did not expose a trace id")
|
|
884
|
+
try:
|
|
885
|
+
for name, value, data_type in (
|
|
886
|
+
("task_passed", bool(payload.get("passed")), "BOOLEAN"),
|
|
887
|
+
("checks_pass_rate", float(payload.get("checks_pass_rate", 0.0)), "NUMERIC"),
|
|
888
|
+
):
|
|
889
|
+
self.client.create_score(
|
|
890
|
+
name={
|
|
891
|
+
"task_passed": "comet.passed",
|
|
892
|
+
"checks_pass_rate": "comet.checks_pass_rate",
|
|
893
|
+
}[name],
|
|
894
|
+
value=value,
|
|
895
|
+
trace_id=trace_id,
|
|
896
|
+
observation_id=observation_id,
|
|
897
|
+
data_type=data_type,
|
|
898
|
+
)
|
|
899
|
+
for dimension, value in (payload.get("rubric") or {}).items():
|
|
900
|
+
self.client.create_score(
|
|
901
|
+
name=f"comet.rubric.{dimension}",
|
|
902
|
+
value=float(value),
|
|
903
|
+
trace_id=trace_id,
|
|
904
|
+
observation_id=observation_id,
|
|
905
|
+
data_type="NUMERIC",
|
|
906
|
+
)
|
|
907
|
+
except Exception as exc:
|
|
908
|
+
raise LangfuseReportingError(f"Langfuse core score failed: {exc}") from exc
|
|
909
|
+
|
|
910
|
+
def report_summary(self) -> None:
|
|
911
|
+
"""Write the suite summary as a final core observation."""
|
|
912
|
+
try:
|
|
913
|
+
with self.client.start_as_current_observation(
|
|
914
|
+
as_type="span",
|
|
915
|
+
name="comet.eval.experiment",
|
|
916
|
+
input={
|
|
917
|
+
"suite": self.suite,
|
|
918
|
+
"experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
|
|
919
|
+
"cases": len(self.cases),
|
|
920
|
+
},
|
|
921
|
+
) as observation:
|
|
922
|
+
passed = sum(1 for case in self.cases if case.get("passed"))
|
|
923
|
+
aggregates = self._aggregates()
|
|
924
|
+
observation.update(
|
|
925
|
+
output={
|
|
926
|
+
"cases": len(self.cases),
|
|
927
|
+
"passed": passed,
|
|
928
|
+
"failed": len(self.cases) - passed,
|
|
929
|
+
"aggregates": aggregates,
|
|
930
|
+
}
|
|
931
|
+
)
|
|
932
|
+
self._score_summary(observation, aggregates)
|
|
933
|
+
except Exception as exc:
|
|
934
|
+
raise LangfuseReportingError(f"Langfuse summary trace failed: {exc}") from exc
|
|
935
|
+
|
|
936
|
+
def _aggregates(self) -> dict[str, Any]:
|
|
937
|
+
groups: dict[str, list[dict[str, Any]]] = {}
|
|
938
|
+
for case in self.cases:
|
|
939
|
+
task = str(case.get("task", "unknown"))
|
|
940
|
+
treatment = str(case.get("treatment", "unknown"))
|
|
941
|
+
groups.setdefault(f"{task}/{treatment}", []).append(case)
|
|
942
|
+
|
|
943
|
+
aggregates: dict[str, Any] = {}
|
|
944
|
+
for key, cases in groups.items():
|
|
945
|
+
outcomes = [bool(case.get("passed")) for case in cases]
|
|
946
|
+
metrics = {str(k): compute_pass_metrics(outcomes, k) for k in _SUMMARY_K_VALUES}
|
|
947
|
+
gate_statuses = [(case.get("quality_gate") or {}).get("status") for case in cases]
|
|
948
|
+
if gate_statuses and all(status == "not_applicable" for status in gate_statuses):
|
|
949
|
+
quality_gate = "not_applicable"
|
|
950
|
+
elif "failed" in gate_statuses:
|
|
951
|
+
quality_gate = "failed"
|
|
952
|
+
elif "unavailable" in gate_statuses:
|
|
953
|
+
quality_gate = "unavailable"
|
|
954
|
+
else:
|
|
955
|
+
quality_gate = "passed"
|
|
956
|
+
aggregates[key] = {
|
|
957
|
+
"task": key.split("/", 1)[0],
|
|
958
|
+
"treatment": key.split("/", 1)[1],
|
|
959
|
+
"runs": len(cases),
|
|
960
|
+
"pass_at_k": {k: metrics[str(k)]["pass_at_k"] for k in _SUMMARY_K_VALUES},
|
|
961
|
+
"pass_power_k": {k: metrics[str(k)]["pass_pow_k"] for k in _SUMMARY_K_VALUES},
|
|
962
|
+
"quality_gate": quality_gate,
|
|
963
|
+
"agent": (
|
|
964
|
+
next(iter({str(case.get("agent")) for case in cases}))
|
|
965
|
+
if len({str(case.get("agent")) for case in cases}) == 1
|
|
966
|
+
else "mixed"
|
|
967
|
+
),
|
|
968
|
+
}
|
|
969
|
+
return aggregates
|
|
970
|
+
|
|
971
|
+
def _score_summary(self, observation: Any, aggregates: Mapping[str, Any]) -> None:
|
|
972
|
+
trace_id = getattr(observation, "trace_id", None)
|
|
973
|
+
observation_id = getattr(observation, "id", None)
|
|
974
|
+
if not trace_id:
|
|
975
|
+
raise LangfuseReportingError("Langfuse summary observation did not expose a trace id")
|
|
976
|
+
try:
|
|
977
|
+
for aggregate in aggregates.values():
|
|
978
|
+
for k in _SUMMARY_K_VALUES:
|
|
979
|
+
score_metadata = {
|
|
980
|
+
"experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
|
|
981
|
+
"agent": aggregate["agent"],
|
|
982
|
+
"task": aggregate["task"],
|
|
983
|
+
"treatment": aggregate["treatment"],
|
|
984
|
+
"k": k,
|
|
985
|
+
}
|
|
986
|
+
self.client.create_score(
|
|
987
|
+
name="comet.pass_at_k",
|
|
988
|
+
value=float(aggregate["pass_at_k"][k]),
|
|
989
|
+
trace_id=trace_id,
|
|
990
|
+
observation_id=observation_id,
|
|
991
|
+
data_type="NUMERIC",
|
|
992
|
+
metadata=score_metadata,
|
|
993
|
+
)
|
|
994
|
+
self.client.create_score(
|
|
995
|
+
name="comet.pass_power_k",
|
|
996
|
+
value=float(aggregate["pass_power_k"][k]),
|
|
997
|
+
trace_id=trace_id,
|
|
998
|
+
observation_id=observation_id,
|
|
999
|
+
data_type="NUMERIC",
|
|
1000
|
+
metadata=score_metadata,
|
|
1001
|
+
)
|
|
1002
|
+
if aggregate["quality_gate"] in {"passed", "failed"}:
|
|
1003
|
+
self.client.create_score(
|
|
1004
|
+
name="comet.quality_gate",
|
|
1005
|
+
value=aggregate["quality_gate"] == "passed",
|
|
1006
|
+
trace_id=trace_id,
|
|
1007
|
+
observation_id=observation_id,
|
|
1008
|
+
data_type="BOOLEAN",
|
|
1009
|
+
metadata={
|
|
1010
|
+
"experiment_id": os.environ.get("COMET_EVAL_EXPERIMENT_ID"),
|
|
1011
|
+
"agent": aggregate["agent"],
|
|
1012
|
+
"task": aggregate["task"],
|
|
1013
|
+
"treatment": aggregate["treatment"],
|
|
1014
|
+
},
|
|
1015
|
+
)
|
|
1016
|
+
except Exception as exc:
|
|
1017
|
+
raise LangfuseReportingError(f"Langfuse summary score failed: {exc}") from exc
|
|
1018
|
+
|
|
1019
|
+
def flush(self) -> None:
|
|
1020
|
+
try:
|
|
1021
|
+
self.client.flush()
|
|
1022
|
+
except Exception as exc:
|
|
1023
|
+
raise LangfuseReportingError(f"Langfuse flush failed: {exc}") from exc
|