@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -33,6 +33,12 @@ from scaffold.python.native_eval import (
|
|
|
33
33
|
split_comet_completion_checks as _split_comet_completion_checks,
|
|
34
34
|
)
|
|
35
35
|
from scaffold.python.profiles import resolve_profile_name, run_profile_rubric
|
|
36
|
+
from scaffold.python.agents import get_agent_adapter
|
|
37
|
+
from scaffold.python.execution import redact_sensitive
|
|
38
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
39
|
+
from scaffold.python.eval_context import resolve_eval_context
|
|
40
|
+
from scaffold.python.task_resolution import build_task_catalogue, resolve_task_set
|
|
41
|
+
from scaffold.python.paths import get_tasks_dir
|
|
36
42
|
from scaffold.python.tasks import list_tasks, load_task
|
|
37
43
|
from scaffold.python.treatments import TreatmentConfig, build_treatment_skills, load_treatments
|
|
38
44
|
from scaffold.python.validation import run_validators
|
|
@@ -43,6 +49,67 @@ PYTEST_TIMEOUT = 3000 # 50 minutes total including task-specific runtime and te
|
|
|
43
49
|
MANIFEST_DYNAMIC_ONLY_TASKS = {"workflow-overlay-contract"}
|
|
44
50
|
|
|
45
51
|
|
|
52
|
+
def _manifest_authored_tasks(config):
|
|
53
|
+
if config is None or not config.getoption("--eval-manifest"):
|
|
54
|
+
return {}
|
|
55
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
56
|
+
|
|
57
|
+
manifest = load_eval_manifest(config.getoption("--eval-manifest"))
|
|
58
|
+
return {task.name: task for task in load_manifest_tasks(manifest)}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _load_eval_task(task_name: str, config=None):
|
|
62
|
+
resolved = getattr(config, "_comet_resolved_tasks", {}) if config is not None else {}
|
|
63
|
+
if task_name in resolved:
|
|
64
|
+
return resolved[task_name]
|
|
65
|
+
authored = _manifest_authored_tasks(config)
|
|
66
|
+
if task_name in authored:
|
|
67
|
+
return authored[task_name]
|
|
68
|
+
return load_task(task_name)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _resolve_frozen_task_set(config, task_filter: str | None):
|
|
72
|
+
"""Resolve once per collection and retain the exact cross-suite task identity."""
|
|
73
|
+
frozen = getattr(config, "_comet_frozen_task_set", None)
|
|
74
|
+
if frozen is not None:
|
|
75
|
+
return frozen
|
|
76
|
+
from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
|
|
77
|
+
|
|
78
|
+
manifest_path = config.getoption("--eval-manifest")
|
|
79
|
+
context = getattr(config, "_comet_eval_context", None)
|
|
80
|
+
if manifest_path:
|
|
81
|
+
manifest = load_eval_manifest(manifest_path)
|
|
82
|
+
context = context or resolve_eval_context(
|
|
83
|
+
manifest_path=manifest.path, project_root=config.getoption("--project-root")
|
|
84
|
+
)
|
|
85
|
+
else:
|
|
86
|
+
if context is None:
|
|
87
|
+
context = resolve_eval_context(
|
|
88
|
+
skill_path=config.getoption("--skill-path"),
|
|
89
|
+
project_root=config.getoption("--project-root"),
|
|
90
|
+
)
|
|
91
|
+
manifest = SkillEvalManifest(
|
|
92
|
+
path=context.skill_root / "comet" / "eval.yaml",
|
|
93
|
+
name=context.skill_root.name,
|
|
94
|
+
description="",
|
|
95
|
+
skill_name=config.getoption("--skill-name") or context.skill_root.name,
|
|
96
|
+
skill_path=context.skill_root,
|
|
97
|
+
profile=config.getoption("--profile"),
|
|
98
|
+
)
|
|
99
|
+
frozen = resolve_task_set(
|
|
100
|
+
context,
|
|
101
|
+
manifest,
|
|
102
|
+
build_task_catalogue(manifest, get_tasks_dir()),
|
|
103
|
+
explicit_task=task_filter,
|
|
104
|
+
quick=bool(config.getoption("--quick")),
|
|
105
|
+
execution=conftest._resolve_eval_execution(config),
|
|
106
|
+
)
|
|
107
|
+
config._comet_frozen_task_set = frozen
|
|
108
|
+
config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
|
|
109
|
+
config._comet_resolution_manifest = manifest
|
|
110
|
+
return frozen
|
|
111
|
+
|
|
112
|
+
|
|
46
113
|
# =============================================================================
|
|
47
114
|
# PARAMETRIZE HELPERS
|
|
48
115
|
# =============================================================================
|
|
@@ -73,6 +140,8 @@ def expand_treatment_patterns(patterns: list[str], all_treatments: dict) -> list
|
|
|
73
140
|
def generate_test_params(task_filter: str | None, treatment_filter: str | None, config=None):
|
|
74
141
|
"""Generate (task_name, treatment_name) pairs based on filters."""
|
|
75
142
|
params = []
|
|
143
|
+
if config is not None:
|
|
144
|
+
conftest._ensure_auto_generated_manifest(config, task_filter)
|
|
76
145
|
all_treatments = load_treatments()
|
|
77
146
|
all_tasks = list_tasks()
|
|
78
147
|
dynamic = None
|
|
@@ -82,16 +151,24 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
|
|
|
82
151
|
if dynamic:
|
|
83
152
|
all_treatments[dynamic.name] = dynamic
|
|
84
153
|
manifest_tasks = None
|
|
154
|
+
authored_tasks = {}
|
|
85
155
|
manifest_baseline_treatments = []
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
156
|
+
frozen_selected = False
|
|
157
|
+
if config is not None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
|
|
158
|
+
resolved = _resolve_frozen_task_set(config, task_filter)
|
|
159
|
+
frozen_selected = True
|
|
160
|
+
manifest = config._comet_resolution_manifest
|
|
161
|
+
manifest_tasks = [item.name for item in resolved.tasks]
|
|
162
|
+
authored_tasks = {
|
|
163
|
+
item.name: item.task
|
|
164
|
+
for item in resolved.tasks
|
|
165
|
+
if item.provenance in {"inline", "source"}
|
|
166
|
+
}
|
|
91
167
|
manifest_baseline_treatments = manifest.baseline_treatments
|
|
92
168
|
|
|
93
|
-
if task_filter and task_filter not in all_tasks:
|
|
94
|
-
|
|
169
|
+
if task_filter and task_filter not in set(all_tasks) | set(authored_tasks):
|
|
170
|
+
available = sorted(set(all_tasks) | set(authored_tasks))
|
|
171
|
+
raise ValueError(f"Task not found: {task_filter}. Available: {available}")
|
|
95
172
|
|
|
96
173
|
treatment_list = []
|
|
97
174
|
if treatment_filter:
|
|
@@ -106,10 +183,18 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
|
|
|
106
183
|
elif dynamic:
|
|
107
184
|
treatment_list = [dynamic.name]
|
|
108
185
|
|
|
109
|
-
|
|
186
|
+
if task_filter:
|
|
187
|
+
tasks_to_run = [task_filter]
|
|
188
|
+
elif frozen_selected:
|
|
189
|
+
tasks_to_run = list(manifest_tasks or [])
|
|
190
|
+
elif authored_tasks or manifest_tasks:
|
|
191
|
+
tasks_to_run = list(manifest_tasks or authored_tasks)
|
|
192
|
+
else:
|
|
193
|
+
tasks_to_run = all_tasks
|
|
194
|
+
tasks_to_run = list(dict.fromkeys(tasks_to_run))
|
|
110
195
|
|
|
111
196
|
for task_name in tasks_to_run:
|
|
112
|
-
task =
|
|
197
|
+
task = _load_eval_task(task_name, config)
|
|
113
198
|
task_treatments = treatment_list
|
|
114
199
|
if (
|
|
115
200
|
dynamic
|
|
@@ -193,6 +278,46 @@ interaction:
|
|
|
193
278
|
]
|
|
194
279
|
|
|
195
280
|
|
|
281
|
+
def test_eval_manifest_prefers_authored_tasks_over_recommended_tasks(tmp_path):
|
|
282
|
+
package = tmp_path / "manifest-skill"
|
|
283
|
+
package.mkdir()
|
|
284
|
+
(package / "SKILL.md").write_text("# Skill\n", encoding="utf-8")
|
|
285
|
+
comet_dir = package / "comet"
|
|
286
|
+
comet_dir.mkdir()
|
|
287
|
+
manifest = comet_dir / "eval.yaml"
|
|
288
|
+
manifest.write_text(
|
|
289
|
+
"\n".join(
|
|
290
|
+
[
|
|
291
|
+
"apiVersion: comet.eval/v1alpha1",
|
|
292
|
+
"kind: SkillEvalManifest",
|
|
293
|
+
"metadata:",
|
|
294
|
+
" name: manifest-skill",
|
|
295
|
+
"skill:",
|
|
296
|
+
" name: manifest-skill",
|
|
297
|
+
" source: ..",
|
|
298
|
+
"evaluation:",
|
|
299
|
+
" tasks:",
|
|
300
|
+
" - name: inline-task",
|
|
301
|
+
" prompt: Create result.md.",
|
|
302
|
+
" expect:",
|
|
303
|
+
" files: [result.md]",
|
|
304
|
+
" recommendedTasks:",
|
|
305
|
+
" - generic-skill-smoke",
|
|
306
|
+
]
|
|
307
|
+
),
|
|
308
|
+
encoding="utf-8",
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
class Config:
|
|
312
|
+
def getoption(self, name):
|
|
313
|
+
return {"--eval-manifest": str(manifest)}.get(name)
|
|
314
|
+
|
|
315
|
+
params = generate_test_params(None, None, Config())
|
|
316
|
+
|
|
317
|
+
assert [task_name for task_name, _ in params] == ["inline-task"]
|
|
318
|
+
assert all(treatment == "DYNAMIC_SKILL" for _, treatment in params)
|
|
319
|
+
|
|
320
|
+
|
|
196
321
|
def test_control_comet_workflow_filters_workflow_only_checks():
|
|
197
322
|
passed, failed = _filter_control_workflow_checks(
|
|
198
323
|
"comet-workflow",
|
|
@@ -310,7 +435,7 @@ def pytest_generate_tests(metafunc):
|
|
|
310
435
|
def test_task_treatment(task_name, treatment_name):
|
|
311
436
|
"""Run a task with a treatment and validate results."""
|
|
312
437
|
fixtures = get_fixtures()
|
|
313
|
-
task =
|
|
438
|
+
task = _load_eval_task(task_name, fixtures.request_config)
|
|
314
439
|
treatments = load_treatments()
|
|
315
440
|
dynamic = conftest._get_dynamic_treatment_config(fixtures.request_config)
|
|
316
441
|
if dynamic:
|
|
@@ -361,8 +486,29 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
361
486
|
skills=treatment.skills,
|
|
362
487
|
claude_md=conftest._build_eval_claude_md(profile_name, treatment.claude_md),
|
|
363
488
|
environment_dir=task.environment_dir,
|
|
489
|
+
workspace_dir=task.workspace_dir,
|
|
364
490
|
)
|
|
365
|
-
|
|
491
|
+
selected_agent = conftest._resolve_eval_agent(fixtures.request_config).agent
|
|
492
|
+
selected_adapter = get_agent_adapter(selected_agent)
|
|
493
|
+
execution = conftest._resolve_eval_execution(fixtures.request_config)
|
|
494
|
+
selected_model = execution.model
|
|
495
|
+
judge = conftest._resolve_eval_judge(fixtures.request_config)
|
|
496
|
+
role_models = {
|
|
497
|
+
"subject": selected_model,
|
|
498
|
+
"simulator": (
|
|
499
|
+
selected_model
|
|
500
|
+
if interaction.mode == "auto_user"
|
|
501
|
+
and not interaction.decision_reply
|
|
502
|
+
and not interaction.decision_replies
|
|
503
|
+
else None
|
|
504
|
+
),
|
|
505
|
+
"judge": judge.model if judge is not None else None,
|
|
506
|
+
}
|
|
507
|
+
role_agents = {
|
|
508
|
+
"subject": selected_agent,
|
|
509
|
+
"simulator": selected_agent if interaction.mode == "auto_user" else None,
|
|
510
|
+
"judge": judge.agent if judge is not None else None,
|
|
511
|
+
}
|
|
366
512
|
captured_execution = conftest._capture_execution_identity(
|
|
367
513
|
fixtures.test_dir,
|
|
368
514
|
model=selected_model,
|
|
@@ -373,6 +519,11 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
373
519
|
task.name,
|
|
374
520
|
task.path.parent,
|
|
375
521
|
execution_identity=captured_execution.report_identity,
|
|
522
|
+
manifest_path=task.manifest_path,
|
|
523
|
+
rendered_prompt=prompt if task.manifest_path else None,
|
|
524
|
+
workspace_dir=task.workspace_dir if task.manifest_path else None,
|
|
525
|
+
validation_rules=task.manifest_expectations if task.manifest_path else None,
|
|
526
|
+
environment_dir=task.environment_dir if task.manifest_path else None,
|
|
376
527
|
)
|
|
377
528
|
)
|
|
378
529
|
skill_package_path = conftest._snapshot_dynamic_skill_package(
|
|
@@ -387,16 +538,31 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
387
538
|
image_id=captured_execution.runtime_image_id,
|
|
388
539
|
)
|
|
389
540
|
|
|
390
|
-
events = extract_events(parse_output(result.stdout))
|
|
541
|
+
events = extract_events(parse_output(result.stdout), agent=selected_agent)
|
|
542
|
+
events["telemetry_status"] = "N/A" if not selected_adapter.supports_telemetry else "available"
|
|
391
543
|
loop_interaction = conftest._extract_loop_interaction(result.stderr)
|
|
392
|
-
|
|
544
|
+
for role, session_ids in loop_interaction.get("role_sessions", {}).items():
|
|
545
|
+
target = events["role_sessions"].setdefault(role, [])
|
|
546
|
+
for session_id in session_ids:
|
|
547
|
+
if session_id not in target:
|
|
548
|
+
target.append(session_id)
|
|
549
|
+
subject_turns = conftest._extract_subject_turn_evidence(redact_sensitive(result.stdout))
|
|
393
550
|
outputs = {
|
|
394
551
|
"run_id": run_id,
|
|
395
552
|
"treatment_name": treatment_name,
|
|
553
|
+
"agent": selected_agent,
|
|
554
|
+
"agent_adapter": selected_adapter,
|
|
555
|
+
"main_credentials": selected_adapter.required_credentials,
|
|
556
|
+
"role_models": role_models,
|
|
557
|
+
"role_agents": role_agents,
|
|
558
|
+
"telemetry_status": events["telemetry_status"],
|
|
559
|
+
"judge_agent": judge.agent if judge is not None else None,
|
|
560
|
+
"judge_model": judge.model if judge is not None else None,
|
|
561
|
+
"judge_base_url": judge.base_url if judge is not None else None,
|
|
396
562
|
"events": events,
|
|
397
563
|
"profile": profile_name,
|
|
398
564
|
"skill_sources": skill_sources,
|
|
399
|
-
"eval_manifest": eval_manifest,
|
|
565
|
+
"eval_manifest": eval_manifest or skill_hints.get("manifest"),
|
|
400
566
|
"required_skills": skill_hints.get("required_skills")
|
|
401
567
|
or task.config.evaluation.required_skills,
|
|
402
568
|
"expected_artifacts": skill_hints.get("expected_artifacts")
|
|
@@ -416,6 +582,19 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
416
582
|
"required_output_schemas": skill_hints.get("required_output_schemas") or [],
|
|
417
583
|
"expected_evidence": skill_hints.get("expected_evidence") or [],
|
|
418
584
|
"draft_hash": skill_hints.get("draft_hash"),
|
|
585
|
+
"eval_generation": (
|
|
586
|
+
{
|
|
587
|
+
"generation_hash": skill_hints.get("generation_hash"),
|
|
588
|
+
"manifest_path": skill_hints.get("manifest")
|
|
589
|
+
or str(fixtures.request_config.getoption("--eval-manifest")),
|
|
590
|
+
"metadata_path": skill_hints.get("generation_metadata_path"),
|
|
591
|
+
"manifest_hash": skill_hints.get("generation_manifest_hash"),
|
|
592
|
+
"metadata_hash": skill_hints.get("generation_metadata_hash"),
|
|
593
|
+
"overhead": skill_hints.get("generation_overhead"),
|
|
594
|
+
}
|
|
595
|
+
if skill_hints.get("generation_hash")
|
|
596
|
+
else None
|
|
597
|
+
),
|
|
419
598
|
"interaction": {
|
|
420
599
|
"mode": interaction.mode,
|
|
421
600
|
"max_turns": interaction.max_turns,
|
|
@@ -423,12 +602,18 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
423
602
|
"subject_turns": subject_turns,
|
|
424
603
|
},
|
|
425
604
|
"case_manifest": case_manifest,
|
|
605
|
+
"role_sessions": events["role_sessions"],
|
|
426
606
|
}
|
|
427
607
|
events["profile"] = outputs["profile"]
|
|
608
|
+
events["agent"] = selected_agent
|
|
609
|
+
events["role_models"] = role_models
|
|
610
|
+
events["role_agents"] = role_agents
|
|
611
|
+
events["telemetry_status"] = outputs["telemetry_status"]
|
|
428
612
|
events["skill_sources"] = outputs["skill_sources"]
|
|
429
613
|
events["eval_manifest"] = outputs["eval_manifest"]
|
|
430
614
|
events["interaction"] = outputs["interaction"]
|
|
431
615
|
events["case_manifest"] = case_manifest
|
|
616
|
+
events["eval_generation"] = outputs["eval_generation"]
|
|
432
617
|
|
|
433
618
|
passed, failed = run_validators(validators, fixtures.test_dir, outputs)
|
|
434
619
|
passed, failed = adapt_checks_for_native(
|
|
@@ -459,6 +644,21 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
459
644
|
)
|
|
460
645
|
passed = passed + rubric_passed
|
|
461
646
|
failed = failed + rubric_failed
|
|
647
|
+
agent_failed = result.returncode != 0
|
|
648
|
+
if agent_failed:
|
|
649
|
+
failed.append(f"Agent execution failed with exit code {result.returncode}")
|
|
650
|
+
|
|
651
|
+
# Keep the run-level context available to thin remote reporting wrappers.
|
|
652
|
+
# These fields are local evidence as well; Langfuse applies its own bounded
|
|
653
|
+
# capture policy before sending them remotely.
|
|
654
|
+
events["task"] = task.name
|
|
655
|
+
events["treatment"] = treatment_name
|
|
656
|
+
events["prompt"] = prompt
|
|
657
|
+
events["skill"] = skill_hints.get("name") or skill_hints.get("path")
|
|
658
|
+
events["final_response"] = subject_turns[-1].get("result") if subject_turns else None
|
|
659
|
+
events["quality_gates"] = outputs["quality_gates"]
|
|
660
|
+
events["execution_identity"] = captured_execution.report_identity
|
|
661
|
+
events["role_sessions"] = outputs["role_sessions"]
|
|
462
662
|
|
|
463
663
|
fixtures.record_result(
|
|
464
664
|
events,
|
|
@@ -470,5 +670,7 @@ def test_task_treatment(task_name, treatment_name):
|
|
|
470
670
|
stderr=result.stderr,
|
|
471
671
|
)
|
|
472
672
|
|
|
673
|
+
if agent_failed:
|
|
674
|
+
pytest.fail(f"{selected_agent} execution failed with exit code {result.returncode}")
|
|
473
675
|
if failed and not _is_observational_baseline_run(treatment_name):
|
|
474
676
|
pytest.fail(f"Validation failed: {failed}")
|
package/eval/pyproject.toml
CHANGED
|
@@ -4,6 +4,7 @@ version = "0.1.0"
|
|
|
4
4
|
description = "Benchmark evaluation for Comet skill (OpenSpec + Superpowers workflow)"
|
|
5
5
|
requires-python = ">=3.11"
|
|
6
6
|
dependencies = [
|
|
7
|
+
"jsonschema>=4.23",
|
|
7
8
|
"pytest>=8.0",
|
|
8
9
|
"pytest-timeout>=2.2",
|
|
9
10
|
"pytest-xdist>=3.5",
|
|
@@ -23,6 +24,9 @@ langsmith = [
|
|
|
23
24
|
"langchain>=0.3",
|
|
24
25
|
"langchain-openai>=0.2",
|
|
25
26
|
]
|
|
27
|
+
langfuse = [
|
|
28
|
+
"langfuse>=4.0,<5",
|
|
29
|
+
]
|
|
26
30
|
dev = [
|
|
27
31
|
"ruff>=0.4",
|
|
28
32
|
"pre-commit>=3.7",
|
|
@@ -34,6 +38,7 @@ testpaths = ["local/tests"]
|
|
|
34
38
|
pythonpath = ["."]
|
|
35
39
|
markers = [
|
|
36
40
|
"langsmith: marks test as using LangSmith tracing",
|
|
41
|
+
"langfuse: marks test as using Langfuse tracing",
|
|
37
42
|
]
|
|
38
43
|
|
|
39
44
|
[tool.ruff]
|
|
@@ -18,6 +18,9 @@ from .python import (
|
|
|
18
18
|
compose_validators,
|
|
19
19
|
make_execution_validator,
|
|
20
20
|
run_claude_in_docker,
|
|
21
|
+
run_agent_in_docker,
|
|
22
|
+
run_agent_loop_in_docker,
|
|
23
|
+
run_command_in_docker,
|
|
21
24
|
run_eval_in_docker,
|
|
22
25
|
run_node_in_docker,
|
|
23
26
|
run_python_in_docker,
|
|
@@ -64,4 +67,7 @@ __all__ = [
|
|
|
64
67
|
"run_eval_in_docker",
|
|
65
68
|
"run_node_in_docker",
|
|
66
69
|
"run_claude_in_docker",
|
|
70
|
+
"run_agent_in_docker",
|
|
71
|
+
"run_agent_loop_in_docker",
|
|
72
|
+
"run_command_in_docker",
|
|
67
73
|
]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
ARG BASE_IMAGE
|
|
2
|
+
FROM ${BASE_IMAGE}
|
|
3
|
+
|
|
4
|
+
USER root
|
|
5
|
+
ARG EVAL_AGENT=claude-code
|
|
6
|
+
ARG CODEX_VERSION=latest
|
|
7
|
+
ARG QODER_VERSION=latest
|
|
8
|
+
ARG CODEBUDDY_VERSION=latest
|
|
9
|
+
ARG CUSTOM_AGENT_ID=
|
|
10
|
+
ARG CUSTOM_AGENT_EXECUTABLE=
|
|
11
|
+
ARG CUSTOM_INSTALL_KIND=none
|
|
12
|
+
ARG CUSTOM_INSTALL_PACKAGE=
|
|
13
|
+
ARG CUSTOM_INSTALL_VERSION=latest
|
|
14
|
+
ARG LANGFUSE_ENABLED=false
|
|
15
|
+
|
|
16
|
+
RUN if [ "$EVAL_AGENT" = "codex" ]; then \
|
|
17
|
+
npm install -g @openai/codex@${CODEX_VERSION}; \
|
|
18
|
+
elif [ "$EVAL_AGENT" = "qoder" ]; then \
|
|
19
|
+
npm install -g @qoder-ai/qodercli@${QODER_VERSION}; \
|
|
20
|
+
elif [ "$EVAL_AGENT" = "codebuddy" ]; then \
|
|
21
|
+
npm install -g @tencent-ai/codebuddy-code@${CODEBUDDY_VERSION}; \
|
|
22
|
+
elif [ -n "$CUSTOM_AGENT_ID" ] && [ "$EVAL_AGENT" = "$CUSTOM_AGENT_ID" ] && [ "$CUSTOM_INSTALL_KIND" = "npm" ]; then \
|
|
23
|
+
npm install -g "${CUSTOM_INSTALL_PACKAGE}@${CUSTOM_INSTALL_VERSION}"; \
|
|
24
|
+
elif [ -n "$CUSTOM_AGENT_ID" ] && [ "$EVAL_AGENT" = "$CUSTOM_AGENT_ID" ] && [ "$CUSTOM_INSTALL_KIND" = "pip" ]; then \
|
|
25
|
+
if [ "$CUSTOM_INSTALL_VERSION" = "latest" ]; then \
|
|
26
|
+
python3 -m pip install --break-system-packages "${CUSTOM_INSTALL_PACKAGE}"; \
|
|
27
|
+
else \
|
|
28
|
+
python3 -m pip install --break-system-packages "${CUSTOM_INSTALL_PACKAGE}==${CUSTOM_INSTALL_VERSION}"; \
|
|
29
|
+
fi; \
|
|
30
|
+
fi
|
|
31
|
+
|
|
32
|
+
# The official Claude hook can use the preinstalled Python SDK, while the
|
|
33
|
+
# official Codex plugin ships its Node bundle. Install the SDK only for an
|
|
34
|
+
# explicit Langfuse suite so Local/LangSmith images stay unchanged.
|
|
35
|
+
RUN if [ "$LANGFUSE_ENABLED" = "true" ]; then \
|
|
36
|
+
apt-get update && apt-get install -y --no-install-recommends python3 python3-pip && \
|
|
37
|
+
python3 -m pip install --break-system-packages --no-cache-dir 'langfuse>=4.0,<5' && \
|
|
38
|
+
rm -rf /var/lib/apt/lists/*; \
|
|
39
|
+
fi
|
|
40
|
+
|
|
41
|
+
USER agent
|
|
@@ -14,7 +14,18 @@ from .logging import (
|
|
|
14
14
|
save_report,
|
|
15
15
|
strip_ansi,
|
|
16
16
|
)
|
|
17
|
-
from .manifests import SkillEvalManifest, load_eval_manifest
|
|
17
|
+
from .manifests import ManifestTask, SkillEvalManifest, load_eval_manifest
|
|
18
|
+
from .eval_context import (
|
|
19
|
+
EvalContextError,
|
|
20
|
+
ResolvedEvalContext,
|
|
21
|
+
artifact_root_for_owner,
|
|
22
|
+
assert_artifact_root_is_safe,
|
|
23
|
+
context_from_environment,
|
|
24
|
+
managed_path_for_owner,
|
|
25
|
+
resolve_managed_path,
|
|
26
|
+
resolve_eval_context,
|
|
27
|
+
)
|
|
28
|
+
from .manifest_tasks import load_manifest_tasks
|
|
18
29
|
from .profiles import (
|
|
19
30
|
AUTHORING_SKILL_PROFILE,
|
|
20
31
|
COMET_WORKFLOW_PROFILE,
|
|
@@ -28,6 +39,26 @@ from .profiles import (
|
|
|
28
39
|
)
|
|
29
40
|
from .evidence import EvalArtifactReference, build_eval_artifact_references
|
|
30
41
|
from .attribution import FailureAttribution, classify_failures
|
|
42
|
+
from .agents import (
|
|
43
|
+
AGENT_IDS,
|
|
44
|
+
AgentAdapter,
|
|
45
|
+
AgentSelection,
|
|
46
|
+
get_agent_adapter,
|
|
47
|
+
resolve_agent,
|
|
48
|
+
validate_agent_capabilities,
|
|
49
|
+
)
|
|
50
|
+
from .custom_agents import CustomAgentSpec, discover_custom_agent_specs
|
|
51
|
+
from .execution import (
|
|
52
|
+
ResolvedExecution,
|
|
53
|
+
ResolvedJudge,
|
|
54
|
+
api_identity,
|
|
55
|
+
build_agent_environment,
|
|
56
|
+
build_judge_environment,
|
|
57
|
+
preflight_credentials,
|
|
58
|
+
redact_sensitive,
|
|
59
|
+
resolve_execution,
|
|
60
|
+
resolve_judge,
|
|
61
|
+
)
|
|
31
62
|
from .sample_quality import (
|
|
32
63
|
SampleQuality,
|
|
33
64
|
include_in_analysis,
|
|
@@ -54,6 +85,9 @@ from .utils import (
|
|
|
54
85
|
read_json_file,
|
|
55
86
|
retry_with_backoff,
|
|
56
87
|
run_claude_in_docker,
|
|
88
|
+
run_agent_in_docker,
|
|
89
|
+
run_agent_loop_in_docker,
|
|
90
|
+
run_command_in_docker,
|
|
57
91
|
run_eval_in_docker,
|
|
58
92
|
run_node_in_docker,
|
|
59
93
|
run_python_in_docker,
|
|
@@ -73,23 +107,97 @@ from .validation import (
|
|
|
73
107
|
)
|
|
74
108
|
|
|
75
109
|
__all__ = [
|
|
76
|
-
"NoiseTask",
|
|
77
|
-
"
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
92
|
-
"
|
|
93
|
-
"
|
|
94
|
-
"
|
|
110
|
+
"NoiseTask",
|
|
111
|
+
"Treatment",
|
|
112
|
+
"ValidatorFn",
|
|
113
|
+
"compose_validators",
|
|
114
|
+
"run_validators",
|
|
115
|
+
"check_file_exists",
|
|
116
|
+
"check_pattern",
|
|
117
|
+
"check_no_pattern",
|
|
118
|
+
"check_skill_invoked",
|
|
119
|
+
"check_code_execution",
|
|
120
|
+
"check_python_execution",
|
|
121
|
+
"check_typescript_execution",
|
|
122
|
+
"run_shell",
|
|
123
|
+
"check_docker_available",
|
|
124
|
+
"check_claude_available",
|
|
125
|
+
"build_docker_image",
|
|
126
|
+
"run_python_in_docker",
|
|
127
|
+
"make_execution_validator",
|
|
128
|
+
"run_eval_in_docker",
|
|
129
|
+
"run_node_in_docker",
|
|
130
|
+
"run_claude_in_docker",
|
|
131
|
+
"retry_with_backoff",
|
|
132
|
+
"read_json_file",
|
|
133
|
+
"run_agent_in_docker",
|
|
134
|
+
"run_agent_loop_in_docker",
|
|
135
|
+
"run_command_in_docker",
|
|
136
|
+
"get_field",
|
|
137
|
+
"get_nested_field",
|
|
138
|
+
"normalize_score",
|
|
139
|
+
"parse_output",
|
|
140
|
+
"extract_events",
|
|
141
|
+
"strip_ansi",
|
|
142
|
+
"ExperimentLogger",
|
|
143
|
+
"TreatmentResult",
|
|
144
|
+
"save_events",
|
|
145
|
+
"save_raw",
|
|
146
|
+
"save_report",
|
|
147
|
+
"ReportColumn",
|
|
148
|
+
"bool_column",
|
|
149
|
+
"quality_column",
|
|
150
|
+
"default_columns",
|
|
151
|
+
"EvalArtifactReference",
|
|
152
|
+
"build_eval_artifact_references",
|
|
153
|
+
"FailureAttribution",
|
|
154
|
+
"classify_failures",
|
|
155
|
+
"AGENT_IDS",
|
|
156
|
+
"AgentAdapter",
|
|
157
|
+
"AgentSelection",
|
|
158
|
+
"get_agent_adapter",
|
|
159
|
+
"resolve_agent",
|
|
160
|
+
"validate_agent_capabilities",
|
|
161
|
+
"CustomAgentSpec",
|
|
162
|
+
"discover_custom_agent_specs",
|
|
163
|
+
"ResolvedExecution",
|
|
164
|
+
"ResolvedJudge",
|
|
165
|
+
"resolve_execution",
|
|
166
|
+
"resolve_judge",
|
|
167
|
+
"build_agent_environment",
|
|
168
|
+
"build_judge_environment",
|
|
169
|
+
"preflight_credentials",
|
|
170
|
+
"redact_sensitive",
|
|
171
|
+
"api_identity",
|
|
172
|
+
"ManifestTask",
|
|
173
|
+
"SkillEvalManifest",
|
|
174
|
+
"load_eval_manifest",
|
|
175
|
+
"EvalContextError",
|
|
176
|
+
"ResolvedEvalContext",
|
|
177
|
+
"artifact_root_for_owner",
|
|
178
|
+
"assert_artifact_root_is_safe",
|
|
179
|
+
"context_from_environment",
|
|
180
|
+
"managed_path_for_owner",
|
|
181
|
+
"resolve_managed_path",
|
|
182
|
+
"resolve_eval_context",
|
|
183
|
+
"load_manifest_tasks",
|
|
184
|
+
"AUTHORING_SKILL_PROFILE",
|
|
185
|
+
"COMET_WORKFLOW_PROFILE",
|
|
186
|
+
"GENERIC_PROFILE",
|
|
187
|
+
"ProfileSpec",
|
|
188
|
+
"all_rubric_dimensions",
|
|
189
|
+
"get_profile",
|
|
190
|
+
"list_profiles",
|
|
191
|
+
"resolve_profile_name",
|
|
192
|
+
"run_profile_rubric",
|
|
193
|
+
"ReportOutputConfig",
|
|
194
|
+
"load_report_output_config",
|
|
195
|
+
"preferred_report_path",
|
|
196
|
+
"render_markdown_html",
|
|
197
|
+
"write_report_outputs",
|
|
198
|
+
"SampleQuality",
|
|
199
|
+
"infer_sample_quality",
|
|
200
|
+
"quality_from_report",
|
|
201
|
+
"sample_quality_dict",
|
|
202
|
+
"include_in_analysis",
|
|
95
203
|
]
|