@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -2,16 +2,59 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import copy
|
|
5
6
|
import re
|
|
6
7
|
from dataclasses import dataclass, field
|
|
7
8
|
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
from urllib.parse import urlsplit
|
|
8
11
|
|
|
9
12
|
import yaml
|
|
10
13
|
|
|
14
|
+
from scaffold.python.agents import validate_agent_id
|
|
15
|
+
from scaffold.python.eval_schema import validate_manifest_schema
|
|
11
16
|
from scaffold.python.tasks import InteractionConfig
|
|
12
17
|
|
|
13
18
|
|
|
14
19
|
SHA256_HEX_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
20
|
+
TASK_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$")
|
|
21
|
+
INLINE_EXPECT_KEYS = {"files", "contains", "json", "commands"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class ManifestTask:
|
|
26
|
+
"""One project-authored task reference from ``evaluation.tasks``."""
|
|
27
|
+
|
|
28
|
+
name: str
|
|
29
|
+
source: Path | None = None
|
|
30
|
+
prompt: str | None = None
|
|
31
|
+
workspace: Path | None = None
|
|
32
|
+
expect: dict[str, Any] = field(default_factory=dict)
|
|
33
|
+
rubric: list[str] = field(default_factory=list)
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def is_inline(self) -> bool:
|
|
37
|
+
return self.source is None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class ManifestExecutionConfig:
|
|
42
|
+
agent: str | None = None
|
|
43
|
+
model: str | None = None
|
|
44
|
+
base_url: str | None = None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class ManifestJudgeConfig:
|
|
49
|
+
agent: str | None
|
|
50
|
+
model: str
|
|
51
|
+
base_url: str | None = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class ManifestReportingConfig:
|
|
56
|
+
suite: str | None = None
|
|
57
|
+
config: str | None = None
|
|
15
58
|
|
|
16
59
|
|
|
17
60
|
@dataclass(frozen=True)
|
|
@@ -23,17 +66,28 @@ class SkillEvalManifest:
|
|
|
23
66
|
skill_path: Path
|
|
24
67
|
profile: str | None = None
|
|
25
68
|
draft_hash: str | None = None
|
|
69
|
+
generation_hash: str | None = None
|
|
70
|
+
generation_metadata_path: Path | None = None
|
|
26
71
|
recommended_tasks: list[str] = field(default_factory=list)
|
|
27
72
|
baseline_treatments: list[str] = field(default_factory=list)
|
|
28
73
|
quality_gates: dict = field(default_factory=dict)
|
|
29
74
|
required_output_schemas: list[str] = field(default_factory=list)
|
|
30
75
|
expected_evidence: list[dict] = field(default_factory=list)
|
|
31
76
|
required_skills: list[str] = field(default_factory=list)
|
|
32
|
-
expected_artifacts: list[str] = field(default_factory=list)
|
|
77
|
+
expected_artifacts: list[str | dict[str, Any]] = field(default_factory=list)
|
|
33
78
|
generated_node_skills: list[str] = field(default_factory=list)
|
|
34
79
|
route_conformance_task: str | None = None
|
|
35
80
|
route_conformance_expected_node_order: list[str] = field(default_factory=list)
|
|
36
81
|
interaction: InteractionConfig = field(default_factory=InteractionConfig)
|
|
82
|
+
execution: ManifestExecutionConfig = field(default_factory=ManifestExecutionConfig)
|
|
83
|
+
judge: ManifestJudgeConfig | None = None
|
|
84
|
+
reporting: ManifestReportingConfig | None = None
|
|
85
|
+
tasks: list[ManifestTask] = field(default_factory=list)
|
|
86
|
+
raw: dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def execution_agent(self) -> str | None:
|
|
90
|
+
return self.execution.agent
|
|
37
91
|
|
|
38
92
|
|
|
39
93
|
def _require_mapping(data: dict, field_name: str) -> dict:
|
|
@@ -43,8 +97,17 @@ def _require_mapping(data: dict, field_name: str) -> dict:
|
|
|
43
97
|
return value
|
|
44
98
|
|
|
45
99
|
|
|
100
|
+
def _alias_value(data: dict, camel_name: str, snake_name: str, field_prefix: str) -> Any:
|
|
101
|
+
"""Read one public alias pair without silently discarding configuration."""
|
|
102
|
+
if camel_name in data and snake_name in data:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"{field_prefix}.{camel_name} and {field_prefix}.{snake_name} cannot both be set"
|
|
105
|
+
)
|
|
106
|
+
return data.get(camel_name, data.get(snake_name))
|
|
107
|
+
|
|
108
|
+
|
|
46
109
|
def _optional_string_list(data: dict, camel_name: str, snake_name: str) -> list[str]:
|
|
47
|
-
value = data
|
|
110
|
+
value = _alias_value(data, camel_name, snake_name, "evaluation")
|
|
48
111
|
if value is None:
|
|
49
112
|
return []
|
|
50
113
|
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
@@ -53,7 +116,7 @@ def _optional_string_list(data: dict, camel_name: str, snake_name: str) -> list[
|
|
|
53
116
|
|
|
54
117
|
|
|
55
118
|
def _optional_mapping(data: dict, camel_name: str, snake_name: str) -> dict:
|
|
56
|
-
value = data
|
|
119
|
+
value = _alias_value(data, camel_name, snake_name, "evaluation")
|
|
57
120
|
if value is None:
|
|
58
121
|
return {}
|
|
59
122
|
if not isinstance(value, dict):
|
|
@@ -62,7 +125,7 @@ def _optional_mapping(data: dict, camel_name: str, snake_name: str) -> dict:
|
|
|
62
125
|
|
|
63
126
|
|
|
64
127
|
def _optional_dict_list(data: dict, camel_name: str, snake_name: str) -> list[dict]:
|
|
65
|
-
value = data
|
|
128
|
+
value = _alias_value(data, camel_name, snake_name, "evaluation")
|
|
66
129
|
if value is None:
|
|
67
130
|
return []
|
|
68
131
|
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
|
|
@@ -71,38 +134,377 @@ def _optional_dict_list(data: dict, camel_name: str, snake_name: str) -> list[di
|
|
|
71
134
|
|
|
72
135
|
|
|
73
136
|
def _optional_draft_hash(metadata: dict) -> str | None:
|
|
74
|
-
value = metadata
|
|
137
|
+
value = _alias_value(metadata, "draftHash", "draft_hash", "metadata")
|
|
75
138
|
if value is None:
|
|
76
139
|
return None
|
|
140
|
+
if value == "<current-bundle-hash>":
|
|
141
|
+
return None
|
|
77
142
|
if not isinstance(value, str) or not SHA256_HEX_RE.match(value):
|
|
78
143
|
raise ValueError("Expected metadata.draftHash to be 64 lowercase hex characters")
|
|
79
144
|
return value
|
|
80
145
|
|
|
81
146
|
|
|
147
|
+
def _optional_model(data: dict, field_name: str, *, required: bool = False) -> str | None:
|
|
148
|
+
value = data.get("model")
|
|
149
|
+
if value is None:
|
|
150
|
+
if required:
|
|
151
|
+
raise ValueError(f"{field_name}: is required")
|
|
152
|
+
return None
|
|
153
|
+
if not isinstance(value, str) or not value.strip():
|
|
154
|
+
raise ValueError(f"{field_name} must be a non-empty string")
|
|
155
|
+
return value
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _optional_base_url(data: dict, field_prefix: str) -> str | None:
|
|
159
|
+
value = _alias_value(data, "baseUrl", "base_url", field_prefix)
|
|
160
|
+
if value is None:
|
|
161
|
+
return None
|
|
162
|
+
if not isinstance(value, str):
|
|
163
|
+
raise ValueError(f"{field_prefix}.baseUrl: must be a valid absolute http(s) URL")
|
|
164
|
+
try:
|
|
165
|
+
parsed = urlsplit(value)
|
|
166
|
+
parsed.port
|
|
167
|
+
invalid = (
|
|
168
|
+
value.strip() != value
|
|
169
|
+
or parsed.scheme not in {"http", "https"}
|
|
170
|
+
or not parsed.hostname
|
|
171
|
+
or any(character.isspace() for character in parsed.netloc)
|
|
172
|
+
or parsed.username is not None
|
|
173
|
+
or parsed.password is not None
|
|
174
|
+
or bool(parsed.query)
|
|
175
|
+
or bool(parsed.fragment)
|
|
176
|
+
)
|
|
177
|
+
except ValueError:
|
|
178
|
+
invalid = True
|
|
179
|
+
if invalid:
|
|
180
|
+
raise ValueError(f"{field_prefix}.baseUrl: must be a valid absolute http(s) URL")
|
|
181
|
+
return value
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _execution_config(data: dict) -> ManifestExecutionConfig:
|
|
185
|
+
agent = data.get("agent")
|
|
186
|
+
if agent is not None:
|
|
187
|
+
agent = validate_agent_id(agent, field="manifest execution.agent")
|
|
188
|
+
return ManifestExecutionConfig(
|
|
189
|
+
agent=agent,
|
|
190
|
+
model=_optional_model(data, "execution.model"),
|
|
191
|
+
base_url=_optional_base_url(data, "execution"),
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _judge_config(data: dict[str, Any] | None) -> ManifestJudgeConfig | None:
|
|
196
|
+
if data is None:
|
|
197
|
+
return None
|
|
198
|
+
agent = data.get("agent")
|
|
199
|
+
if agent is not None:
|
|
200
|
+
agent = validate_agent_id(agent, field="manifest judge.agent")
|
|
201
|
+
model = _optional_model(data, "judge.model", required=True)
|
|
202
|
+
assert model is not None
|
|
203
|
+
return ManifestJudgeConfig(
|
|
204
|
+
agent=agent,
|
|
205
|
+
model=model,
|
|
206
|
+
base_url=_optional_base_url(data, "judge"),
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _reporting_config(data: dict[str, Any] | None) -> ManifestReportingConfig | None:
|
|
211
|
+
if data is None:
|
|
212
|
+
return None
|
|
213
|
+
suite = data.get("suite")
|
|
214
|
+
if suite is not None and suite not in {"local", "langsmith", "langfuse"}:
|
|
215
|
+
raise ValueError("reporting.suite must be local, langsmith, or langfuse")
|
|
216
|
+
config = data.get("config")
|
|
217
|
+
if config is not None and (not isinstance(config, str) or not config.strip()):
|
|
218
|
+
raise ValueError("reporting.config must be a non-empty path")
|
|
219
|
+
return ManifestReportingConfig(suite=suite, config=config)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _resolve_package_path(
|
|
223
|
+
value: object,
|
|
224
|
+
*,
|
|
225
|
+
package_root: Path,
|
|
226
|
+
field_name: str,
|
|
227
|
+
must_exist: bool,
|
|
228
|
+
) -> Path:
|
|
229
|
+
if not isinstance(value, str) or not value.strip():
|
|
230
|
+
raise ValueError(f"Expected {field_name} to be a non-empty relative path")
|
|
231
|
+
normalized = value.replace("\\", "/")
|
|
232
|
+
path = Path(normalized)
|
|
233
|
+
if path.is_absolute() or ".." in path.parts:
|
|
234
|
+
raise ValueError(f"{field_name} must stay within the Skill package: {value!r}")
|
|
235
|
+
resolved = (package_root / path).resolve()
|
|
236
|
+
try:
|
|
237
|
+
resolved.relative_to(package_root.resolve())
|
|
238
|
+
except ValueError as exc:
|
|
239
|
+
raise ValueError(f"{field_name} must stay within the Skill package: {value!r}") from exc
|
|
240
|
+
if must_exist and not resolved.exists():
|
|
241
|
+
raise ValueError(f"{field_name} does not exist: {value!r}")
|
|
242
|
+
if resolved.is_dir():
|
|
243
|
+
package_resolved = package_root.resolve()
|
|
244
|
+
for child in resolved.rglob("*"):
|
|
245
|
+
if not child.is_symlink():
|
|
246
|
+
continue
|
|
247
|
+
target = child.resolve()
|
|
248
|
+
try:
|
|
249
|
+
target.relative_to(package_resolved)
|
|
250
|
+
except ValueError as exc:
|
|
251
|
+
raise ValueError(
|
|
252
|
+
f"{field_name} contains a symlink outside the Skill package: {child.name!r}"
|
|
253
|
+
) from exc
|
|
254
|
+
return resolved
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _validate_relative_artifact(value: object, field_name: str) -> str:
|
|
258
|
+
if not isinstance(value, str) or not value.strip():
|
|
259
|
+
raise ValueError(f"{field_name} must contain non-empty relative paths")
|
|
260
|
+
normalized = value.replace("\\", "/")
|
|
261
|
+
path = Path(normalized)
|
|
262
|
+
if path.is_absolute() or ".." in path.parts:
|
|
263
|
+
raise ValueError(f"{field_name} must stay within the task workspace: {value!r}")
|
|
264
|
+
return normalized
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _parse_inline_expect(expect: object, task_name: str) -> dict[str, Any]:
|
|
268
|
+
field_prefix = f"evaluation.tasks.{task_name}.expect"
|
|
269
|
+
if expect is None:
|
|
270
|
+
raise ValueError(f"{field_prefix} must contain at least one deterministic expect")
|
|
271
|
+
if not isinstance(expect, dict):
|
|
272
|
+
raise ValueError(f"{field_prefix} must be a mapping")
|
|
273
|
+
unknown = set(expect) - INLINE_EXPECT_KEYS
|
|
274
|
+
if unknown:
|
|
275
|
+
raise ValueError(f"{field_prefix} has unknown fields: {sorted(unknown)}")
|
|
276
|
+
if not any(expect.get(key) for key in INLINE_EXPECT_KEYS):
|
|
277
|
+
raise ValueError(f"{field_prefix} must contain at least one deterministic expect")
|
|
278
|
+
|
|
279
|
+
files = expect.get("files", [])
|
|
280
|
+
if not isinstance(files, list) or not all(isinstance(item, str) for item in files):
|
|
281
|
+
raise ValueError(f"{field_prefix}.files must be a list of strings")
|
|
282
|
+
normalized_files = [
|
|
283
|
+
_validate_relative_artifact(item, f"{field_prefix}.files") for item in files
|
|
284
|
+
]
|
|
285
|
+
|
|
286
|
+
contains = expect.get("contains", {})
|
|
287
|
+
if not isinstance(contains, dict):
|
|
288
|
+
raise ValueError(f"{field_prefix}.contains must be a mapping")
|
|
289
|
+
normalized_contains: dict[str, list[str]] = {}
|
|
290
|
+
for file_name, values in contains.items():
|
|
291
|
+
safe_file = _validate_relative_artifact(file_name, f"{field_prefix}.contains")
|
|
292
|
+
if not isinstance(values, list) or not all(isinstance(item, str) for item in values):
|
|
293
|
+
raise ValueError(f"{field_prefix}.contains.{file_name} must be a list of strings")
|
|
294
|
+
normalized_contains[safe_file] = list(values)
|
|
295
|
+
|
|
296
|
+
json_checks = expect.get("json", [])
|
|
297
|
+
if not isinstance(json_checks, list) or not all(isinstance(item, dict) for item in json_checks):
|
|
298
|
+
raise ValueError(f"{field_prefix}.json must be a list of mappings")
|
|
299
|
+
normalized_json = []
|
|
300
|
+
for index, item in enumerate(json_checks):
|
|
301
|
+
file_name = _validate_relative_artifact(
|
|
302
|
+
item.get("file"), f"{field_prefix}.json[{index}].file"
|
|
303
|
+
)
|
|
304
|
+
json_path = item.get("path")
|
|
305
|
+
if not isinstance(json_path, str) or not json_path.startswith("$"):
|
|
306
|
+
raise ValueError(f"{field_prefix}.json[{index}].path must start with '$'")
|
|
307
|
+
if "equals" not in item:
|
|
308
|
+
raise ValueError(f"{field_prefix}.json[{index}].equals is required")
|
|
309
|
+
normalized_json.append({"file": file_name, "path": json_path, "equals": item["equals"]})
|
|
310
|
+
|
|
311
|
+
commands = expect.get("commands", [])
|
|
312
|
+
if not isinstance(commands, list) or not all(isinstance(item, dict) for item in commands):
|
|
313
|
+
raise ValueError(f"{field_prefix}.commands must be a list of mappings")
|
|
314
|
+
normalized_commands = []
|
|
315
|
+
for index, item in enumerate(commands):
|
|
316
|
+
command = item.get("run")
|
|
317
|
+
if not isinstance(command, str) or not command.strip():
|
|
318
|
+
raise ValueError(f"{field_prefix}.commands[{index}].run is required")
|
|
319
|
+
timeout = item.get("timeout", 120)
|
|
320
|
+
if isinstance(timeout, bool) or not isinstance(timeout, int) or not 1 <= timeout <= 3600:
|
|
321
|
+
raise ValueError(f"{field_prefix}.commands[{index}].timeout must be 1..3600")
|
|
322
|
+
normalized_commands.append({"run": command, "timeout": timeout})
|
|
323
|
+
|
|
324
|
+
return {
|
|
325
|
+
"files": normalized_files,
|
|
326
|
+
"contains": normalized_contains,
|
|
327
|
+
"json": normalized_json,
|
|
328
|
+
"commands": normalized_commands,
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _source_task_name(source: Path, task_name: object) -> str:
|
|
333
|
+
name = task_name
|
|
334
|
+
if name is None:
|
|
335
|
+
try:
|
|
336
|
+
from scaffold.python.tasks import load_task_from_path
|
|
337
|
+
|
|
338
|
+
name = load_task_from_path(source).name
|
|
339
|
+
except (FileNotFoundError, OSError, ValueError, KeyError, TypeError, OverflowError) as exc:
|
|
340
|
+
raise ValueError(f"evaluation.tasks source task.toml is invalid: {source}") from exc
|
|
341
|
+
else:
|
|
342
|
+
# Aliases still point at a normal task package; validate it with the
|
|
343
|
+
# same loader used by the execution path before applying the alias.
|
|
344
|
+
name = task_name
|
|
345
|
+
try:
|
|
346
|
+
from scaffold.python.tasks import load_task_from_path
|
|
347
|
+
|
|
348
|
+
load_task_from_path(source)
|
|
349
|
+
except (FileNotFoundError, OSError, ValueError, KeyError, TypeError, OverflowError) as exc:
|
|
350
|
+
raise ValueError(f"evaluation.tasks source task.toml is invalid: {source}") from exc
|
|
351
|
+
if not isinstance(name, str) or not TASK_NAME_RE.fullmatch(name):
|
|
352
|
+
raise ValueError(f"Invalid evaluation task name: {name!r}")
|
|
353
|
+
return name
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _parse_manifest_tasks(evaluation: dict, package_root: Path) -> list[ManifestTask]:
|
|
357
|
+
raw_tasks = evaluation.get("tasks") or []
|
|
358
|
+
if not isinstance(raw_tasks, list):
|
|
359
|
+
raise ValueError("Expected evaluation.tasks to be a list")
|
|
360
|
+
tasks: list[ManifestTask] = []
|
|
361
|
+
names: set[str] = set()
|
|
362
|
+
for index, raw in enumerate(raw_tasks):
|
|
363
|
+
field_prefix = f"evaluation.tasks[{index}]"
|
|
364
|
+
if not isinstance(raw, dict):
|
|
365
|
+
raise ValueError(f"{field_prefix} must be a mapping")
|
|
366
|
+
source_value = raw.get("source")
|
|
367
|
+
if source_value is not None:
|
|
368
|
+
if set(raw) - {"source", "name"}:
|
|
369
|
+
raise ValueError(f"{field_prefix} source tasks cannot define inline fields")
|
|
370
|
+
source = _resolve_package_path(
|
|
371
|
+
source_value,
|
|
372
|
+
package_root=package_root,
|
|
373
|
+
field_name=f"{field_prefix}.source",
|
|
374
|
+
must_exist=True,
|
|
375
|
+
)
|
|
376
|
+
if (
|
|
377
|
+
not source.is_dir()
|
|
378
|
+
or not (source / "task.toml").is_file()
|
|
379
|
+
or not (source / "instruction.md").is_file()
|
|
380
|
+
):
|
|
381
|
+
raise ValueError(
|
|
382
|
+
f"{field_prefix}.source must point to a task package with task.toml and instruction.md"
|
|
383
|
+
)
|
|
384
|
+
name = _source_task_name(source, raw.get("name"))
|
|
385
|
+
task = ManifestTask(name=name, source=source)
|
|
386
|
+
else:
|
|
387
|
+
unknown = set(raw) - {"name", "prompt", "workspace", "expect", "rubric"}
|
|
388
|
+
if unknown:
|
|
389
|
+
raise ValueError(f"{field_prefix} has unknown fields: {sorted(unknown)}")
|
|
390
|
+
name = raw.get("name")
|
|
391
|
+
if not isinstance(name, str) or not TASK_NAME_RE.fullmatch(name):
|
|
392
|
+
raise ValueError(f"{field_prefix}.name must be a valid task name")
|
|
393
|
+
prompt = raw.get("prompt")
|
|
394
|
+
if not isinstance(prompt, str) or not prompt.strip():
|
|
395
|
+
raise ValueError(f"{field_prefix}.prompt is required")
|
|
396
|
+
workspace_value = raw.get("workspace")
|
|
397
|
+
workspace = (
|
|
398
|
+
_resolve_package_path(
|
|
399
|
+
workspace_value,
|
|
400
|
+
package_root=package_root,
|
|
401
|
+
field_name=f"{field_prefix}.workspace",
|
|
402
|
+
must_exist=True,
|
|
403
|
+
)
|
|
404
|
+
if workspace_value is not None
|
|
405
|
+
else None
|
|
406
|
+
)
|
|
407
|
+
task = ManifestTask(
|
|
408
|
+
name=name,
|
|
409
|
+
prompt=prompt,
|
|
410
|
+
workspace=workspace,
|
|
411
|
+
expect=_parse_inline_expect(raw.get("expect"), name),
|
|
412
|
+
rubric=raw.get("rubric", []),
|
|
413
|
+
)
|
|
414
|
+
if not isinstance(task.rubric, list) or not all(
|
|
415
|
+
isinstance(item, str) and item.strip() for item in task.rubric
|
|
416
|
+
):
|
|
417
|
+
raise ValueError(f"{field_prefix}.rubric must be a list of non-empty strings")
|
|
418
|
+
if task.name in names:
|
|
419
|
+
first_index = next(item_index for item_index, item in enumerate(tasks) if item.name == task.name)
|
|
420
|
+
raise ValueError(
|
|
421
|
+
f'evaluation.tasks[{index}].name duplicates evaluation.tasks[{first_index}].name: "{task.name}"'
|
|
422
|
+
)
|
|
423
|
+
names.add(task.name)
|
|
424
|
+
tasks.append(task)
|
|
425
|
+
return tasks
|
|
426
|
+
|
|
427
|
+
|
|
82
428
|
def load_eval_manifest(path: Path | str) -> SkillEvalManifest:
|
|
83
429
|
manifest_path = Path(path).expanduser().resolve()
|
|
84
|
-
data = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
|
|
430
|
+
data = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
|
|
431
|
+
if not isinstance(data, dict):
|
|
432
|
+
raise ValueError("manifest: must be a mapping")
|
|
85
433
|
if data.get("apiVersion") != "comet.eval/v1alpha1":
|
|
86
434
|
raise ValueError("Expected apiVersion comet.eval/v1alpha1")
|
|
87
435
|
if data.get("kind") != "SkillEvalManifest":
|
|
88
436
|
raise ValueError("Expected kind SkillEvalManifest")
|
|
89
|
-
|
|
437
|
+
validate_manifest_schema(data)
|
|
90
438
|
metadata = _require_mapping(data, "metadata")
|
|
91
439
|
skill = _require_mapping(data, "skill")
|
|
92
440
|
evaluation = data.get("evaluation") or {}
|
|
441
|
+
if not isinstance(evaluation, dict):
|
|
442
|
+
raise ValueError("Expected evaluation to be a mapping")
|
|
93
443
|
interaction_data = data.get("interaction") or {}
|
|
444
|
+
if not isinstance(interaction_data, dict):
|
|
445
|
+
raise ValueError("Expected interaction to be a mapping")
|
|
446
|
+
execution_data = data.get("execution") or {}
|
|
447
|
+
if not isinstance(execution_data, dict):
|
|
448
|
+
raise ValueError("Expected execution to be a mapping")
|
|
449
|
+
judge_data = data.get("judge") if "judge" in data else None
|
|
450
|
+
if judge_data is not None and not isinstance(judge_data, dict):
|
|
451
|
+
raise ValueError("Expected judge to be a mapping")
|
|
452
|
+
reporting_data = data.get("reporting") if "reporting" in data else None
|
|
453
|
+
if reporting_data is not None and not isinstance(reporting_data, dict):
|
|
454
|
+
raise ValueError("Expected reporting to be a mapping")
|
|
94
455
|
skill_source = skill.get("source", "..")
|
|
95
|
-
|
|
456
|
+
if not isinstance(skill_source, str):
|
|
457
|
+
raise ValueError("Expected skill.source to be a relative path")
|
|
458
|
+
skill_path = (manifest_path.parent / skill_source).resolve()
|
|
459
|
+
package_root = skill_path
|
|
460
|
+
route_conformance = (
|
|
461
|
+
_alias_value(evaluation, "routeConformance", "route_conformance", "evaluation") or {}
|
|
462
|
+
)
|
|
463
|
+
if not isinstance(route_conformance, dict):
|
|
464
|
+
raise ValueError("Expected evaluation.routeConformance to be a mapping")
|
|
465
|
+
recommended_tasks = _optional_string_list(evaluation, "recommendedTasks", "recommended_tasks")
|
|
466
|
+
seen_recommended: dict[str, int] = {}
|
|
467
|
+
for index, task_name in enumerate(recommended_tasks):
|
|
468
|
+
if task_name in seen_recommended:
|
|
469
|
+
raise ValueError(
|
|
470
|
+
f'evaluation.recommendedTasks[{index}] duplicates '
|
|
471
|
+
f'evaluation.recommendedTasks[{seen_recommended[task_name]}]: "{task_name}"'
|
|
472
|
+
)
|
|
473
|
+
seen_recommended[task_name] = index
|
|
96
474
|
|
|
475
|
+
interaction_mode = interaction_data.get("mode", "none")
|
|
476
|
+
if interaction_mode not in {"none", "auto_user"}:
|
|
477
|
+
raise ValueError("interaction.mode must be none or auto_user")
|
|
478
|
+
max_turns = _alias_value(interaction_data, "maxTurns", "max_turns", "interaction")
|
|
479
|
+
if max_turns is None:
|
|
480
|
+
max_turns = 12
|
|
481
|
+
if isinstance(max_turns, bool) or not isinstance(max_turns, int) or max_turns < 1:
|
|
482
|
+
raise ValueError("interaction.maxTurns must be a positive integer")
|
|
483
|
+
|
|
484
|
+
generation_hash_value = _alias_value(
|
|
485
|
+
metadata, "generationHash", "generation_hash", "metadata"
|
|
486
|
+
)
|
|
487
|
+
generation_file_value = _alias_value(
|
|
488
|
+
metadata, "generationFile", "generation_file", "metadata"
|
|
489
|
+
)
|
|
490
|
+
expected_artifacts_value = _alias_value(
|
|
491
|
+
evaluation, "expectedArtifacts", "expected_artifacts", "evaluation"
|
|
492
|
+
)
|
|
97
493
|
return SkillEvalManifest(
|
|
98
494
|
path=manifest_path,
|
|
99
495
|
name=str(metadata.get("name") or skill.get("name")),
|
|
100
496
|
description=str(metadata.get("description") or ""),
|
|
101
497
|
skill_name=str(skill.get("name") or metadata.get("name")),
|
|
102
|
-
skill_path=
|
|
498
|
+
skill_path=skill_path,
|
|
103
499
|
profile=skill.get("profile"),
|
|
104
500
|
draft_hash=_optional_draft_hash(metadata),
|
|
105
|
-
|
|
501
|
+
generation_hash=generation_hash_value if isinstance(generation_hash_value, str) else None,
|
|
502
|
+
generation_metadata_path=(
|
|
503
|
+
(manifest_path.parent / generation_file_value).resolve()
|
|
504
|
+
if isinstance(generation_file_value, str)
|
|
505
|
+
else None
|
|
506
|
+
),
|
|
507
|
+
recommended_tasks=recommended_tasks,
|
|
106
508
|
baseline_treatments=_optional_string_list(
|
|
107
509
|
evaluation, "baselineTreatments", "baseline_treatments"
|
|
108
510
|
),
|
|
@@ -110,22 +512,41 @@ def load_eval_manifest(path: Path | str) -> SkillEvalManifest:
|
|
|
110
512
|
required_output_schemas=_optional_string_list(
|
|
111
513
|
evaluation, "requiredOutputSchemas", "required_output_schemas"
|
|
112
514
|
),
|
|
113
|
-
expected_evidence=_optional_dict_list(
|
|
114
|
-
|
|
515
|
+
expected_evidence=_optional_dict_list(evaluation, "expectedEvidence", "expected_evidence"),
|
|
516
|
+
required_skills=_optional_string_list(evaluation, "requiredSkills", "required_skills"),
|
|
517
|
+
expected_artifacts=list(expected_artifacts_value or []),
|
|
518
|
+
generated_node_skills=_optional_string_list(
|
|
519
|
+
evaluation, "generatedNodeSkills", "generated_node_skills"
|
|
115
520
|
),
|
|
116
|
-
required_skills=list(evaluation.get("requiredSkills") or []),
|
|
117
|
-
expected_artifacts=list(evaluation.get("expectedArtifacts") or []),
|
|
118
|
-
generated_node_skills=list(evaluation.get("generatedNodeSkills") or []),
|
|
119
521
|
route_conformance_task=route_conformance.get("task"),
|
|
120
522
|
route_conformance_expected_node_order=list(
|
|
121
|
-
|
|
523
|
+
_alias_value(
|
|
524
|
+
route_conformance,
|
|
525
|
+
"expectedNodeOrder",
|
|
526
|
+
"expected_node_order",
|
|
527
|
+
"evaluation.routeConformance",
|
|
528
|
+
)
|
|
529
|
+
or []
|
|
122
530
|
),
|
|
123
531
|
interaction=InteractionConfig(
|
|
124
|
-
mode=
|
|
125
|
-
max_turns=
|
|
126
|
-
simulator_prompt=
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
532
|
+
mode=interaction_mode,
|
|
533
|
+
max_turns=max_turns,
|
|
534
|
+
simulator_prompt=_alias_value(
|
|
535
|
+
interaction_data, "simulatorPrompt", "simulator_prompt", "interaction"
|
|
536
|
+
),
|
|
537
|
+
decision_patterns=list(interaction_data.get("decisionPatterns") or []),
|
|
538
|
+
decision_reply=interaction_data.get("decisionReply"),
|
|
539
|
+
decision_replies=list(interaction_data.get("decisionReplies") or []),
|
|
540
|
+
continue_prompt=interaction_data.get(
|
|
541
|
+
"continuePrompt", "Please continue with the next phase of the workflow."
|
|
542
|
+
),
|
|
543
|
+
fresh_resume_marker=_alias_value(
|
|
544
|
+
interaction_data, "freshResumeMarker", "fresh_resume_marker", "interaction"
|
|
545
|
+
),
|
|
130
546
|
),
|
|
547
|
+
execution=_execution_config(execution_data),
|
|
548
|
+
judge=_judge_config(judge_data),
|
|
549
|
+
reporting=_reporting_config(reporting_data),
|
|
550
|
+
tasks=_parse_manifest_tasks(evaluation, package_root),
|
|
551
|
+
raw=copy.deepcopy(data),
|
|
131
552
|
)
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
import os
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
|
|
6
|
+
from scaffold.python.eval_context import context_from_environment, resolve_managed_path
|
|
7
|
+
|
|
6
8
|
|
|
7
9
|
EVAL_ROOT = Path(__file__).resolve().parents[2]
|
|
8
|
-
SUITE_NAMES = {"local", "langsmith"}
|
|
10
|
+
SUITE_NAMES = {"local", "langsmith", "langfuse"}
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
def get_suite_root() -> Path:
|
|
@@ -54,3 +56,11 @@ def get_skills_dir() -> Path:
|
|
|
54
56
|
def get_logs_dir() -> Path:
|
|
55
57
|
configured = os.environ.get("BENCH_LOGS_DIR")
|
|
56
58
|
return Path(configured).resolve() if configured else get_suite_root() / "logs"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_runs_dir() -> Path:
|
|
62
|
+
"""Return the run directory, preferring CLI-resolved user-owned state."""
|
|
63
|
+
context = context_from_environment()
|
|
64
|
+
if context is not None:
|
|
65
|
+
return resolve_managed_path(context, "runs")
|
|
66
|
+
return get_logs_dir() / "experiments"
|