@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Import-safe generated-task snapshot and cache selection primitives."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Mapping
|
|
12
|
+
|
|
13
|
+
from scaffold.python.agents import get_agent_adapter
|
|
14
|
+
from scaffold.python.eval_context import managed_path_for_owner
|
|
15
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
GENERATOR_VERSION = "comet-auto-task-generator.v1"
|
|
19
|
+
TASK_SCHEMA_VERSION = "comet.eval/v1alpha1"
|
|
20
|
+
MIN_GENERATED_TASKS = 2
|
|
21
|
+
MAX_GENERATED_TASKS = 4
|
|
22
|
+
MAX_SNAPSHOT_FILES = 128
|
|
23
|
+
MAX_SNAPSHOT_BYTES = 512 * 1024
|
|
24
|
+
DEFAULT_CONTINUE_PROMPT = "Please continue with the next phase of the workflow."
|
|
25
|
+
_MODEL_ENV_KEYS = {
|
|
26
|
+
"claude-code": ("BENCH_CC_MODEL", "ANTHROPIC_MODEL"),
|
|
27
|
+
"codex": ("BENCH_CODEX_MODEL", "OPENAI_MODEL"),
|
|
28
|
+
"qoder": ("BENCH_QODER_MODEL", "QODER_MODEL"),
|
|
29
|
+
"codebuddy": ("BENCH_CODEBUDDY_MODEL", "CODEBUDDY_MODEL"),
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class SnapshotFile:
|
|
35
|
+
path: str
|
|
36
|
+
content: str
|
|
37
|
+
content_hash: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class SkillSnapshot:
|
|
42
|
+
files: tuple[SnapshotFile, ...]
|
|
43
|
+
content_hash: str
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class GeneratedCache:
|
|
48
|
+
manifest_path: Path
|
|
49
|
+
metadata_path: Path
|
|
50
|
+
generation_hash: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def selected_agent_model(agent: str) -> str | None:
|
|
54
|
+
"""Resolve the existing environment-backed model selector for cache identity."""
|
|
55
|
+
adapter = get_agent_adapter(agent)
|
|
56
|
+
keys = (*_MODEL_ENV_KEYS.get(agent, ()), *(key for key in (adapter.model_env,) if key))
|
|
57
|
+
return next(
|
|
58
|
+
(os.environ[key] for key in keys if os.environ.get(key)),
|
|
59
|
+
None,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def normalize_interaction(interaction: Mapping[str, Any]) -> dict[str, Any]:
|
|
64
|
+
"""Return the one cross-runtime interaction payload used by generation hashes."""
|
|
65
|
+
return {
|
|
66
|
+
"mode": interaction.get("mode", "none"),
|
|
67
|
+
"max_turns": interaction.get("maxTurns", interaction.get("max_turns", 12)),
|
|
68
|
+
"simulator_prompt": interaction.get("simulatorPrompt")
|
|
69
|
+
or interaction.get("simulator_prompt"),
|
|
70
|
+
"decision_patterns": list(
|
|
71
|
+
interaction.get("decisionPatterns", interaction.get("decision_patterns", [])) or []
|
|
72
|
+
),
|
|
73
|
+
"decision_reply": interaction.get(
|
|
74
|
+
"decisionReply", interaction.get("decision_reply")
|
|
75
|
+
),
|
|
76
|
+
"decision_replies": list(
|
|
77
|
+
interaction.get("decisionReplies", interaction.get("decision_replies", [])) or []
|
|
78
|
+
),
|
|
79
|
+
"continue_prompt": interaction.get(
|
|
80
|
+
"continuePrompt",
|
|
81
|
+
interaction.get("continue_prompt", DEFAULT_CONTINUE_PROMPT),
|
|
82
|
+
),
|
|
83
|
+
"fresh_resume_marker": interaction.get("freshResumeMarker")
|
|
84
|
+
or interaction.get("fresh_resume_marker"),
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def sha256(value: str | bytes) -> str:
|
|
89
|
+
payload = value.encode("utf-8") if isinstance(value, str) else value
|
|
90
|
+
return "sha256:" + hashlib.sha256(payload).hexdigest()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def skill_root(skill_path: Path | str) -> Path:
|
|
94
|
+
root = Path(skill_path).expanduser().resolve()
|
|
95
|
+
if root.is_file() and root.name == "SKILL.md":
|
|
96
|
+
root = root.parent
|
|
97
|
+
if not root.is_dir() or not (root / "SKILL.md").is_file():
|
|
98
|
+
raise ValueError(f"Skill package must contain SKILL.md: {skill_path}")
|
|
99
|
+
return root
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _inside(root: Path, path: Path) -> bool:
|
|
103
|
+
try:
|
|
104
|
+
path.resolve().relative_to(root.resolve())
|
|
105
|
+
return True
|
|
106
|
+
except ValueError:
|
|
107
|
+
return False
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def build_skill_snapshot(skill_path: Path | str) -> SkillSnapshot:
|
|
111
|
+
root = skill_root(skill_path)
|
|
112
|
+
selected: dict[str, SnapshotFile] = {}
|
|
113
|
+
total = 0
|
|
114
|
+
|
|
115
|
+
def add(path: Path) -> None:
|
|
116
|
+
nonlocal total
|
|
117
|
+
if len(selected) >= MAX_SNAPSHOT_FILES or not path.is_file() or not _inside(root, path):
|
|
118
|
+
return
|
|
119
|
+
relative = path.resolve().relative_to(root).as_posix()
|
|
120
|
+
if relative in selected:
|
|
121
|
+
return
|
|
122
|
+
content = path.read_text(encoding="utf-8", errors="replace")
|
|
123
|
+
payload = content.encode("utf-8")
|
|
124
|
+
if total + len(payload) > MAX_SNAPSHOT_BYTES:
|
|
125
|
+
return
|
|
126
|
+
total += len(payload)
|
|
127
|
+
selected[relative] = SnapshotFile(relative, content, sha256(content))
|
|
128
|
+
|
|
129
|
+
skill_file = root / "SKILL.md"
|
|
130
|
+
add(skill_file)
|
|
131
|
+
body = skill_file.read_text(encoding="utf-8", errors="replace")
|
|
132
|
+
for reference in re.findall(r"[`\"']([^`\"']+)[`\"']", body):
|
|
133
|
+
add(root / reference.replace("\\", "/"))
|
|
134
|
+
for directory in ("scripts", "references", "reference", "examples", "templates"):
|
|
135
|
+
candidate = root / directory
|
|
136
|
+
if candidate.is_dir():
|
|
137
|
+
for child in sorted(candidate.rglob("*")):
|
|
138
|
+
if not child.is_symlink():
|
|
139
|
+
add(child)
|
|
140
|
+
files = tuple(selected[key] for key in sorted(selected))
|
|
141
|
+
manifest = json.dumps(
|
|
142
|
+
[{"path": item.path, "hash": item.content_hash} for item in files],
|
|
143
|
+
ensure_ascii=False,
|
|
144
|
+
sort_keys=True,
|
|
145
|
+
separators=(",", ":"),
|
|
146
|
+
)
|
|
147
|
+
return SkillSnapshot(files, sha256(manifest))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def generation_hash(
|
|
151
|
+
snapshot: SkillSnapshot,
|
|
152
|
+
*,
|
|
153
|
+
agent: str,
|
|
154
|
+
model: str | None,
|
|
155
|
+
base_url: str | None = None,
|
|
156
|
+
profile: str,
|
|
157
|
+
interaction: Mapping[str, Any],
|
|
158
|
+
) -> str:
|
|
159
|
+
payload = {
|
|
160
|
+
"snapshot_hash": snapshot.content_hash,
|
|
161
|
+
"agent": agent,
|
|
162
|
+
"model": model or "runtime-default",
|
|
163
|
+
"base_url": base_url or "runtime-default",
|
|
164
|
+
"profile": profile,
|
|
165
|
+
"interaction": normalize_interaction(interaction),
|
|
166
|
+
"generator_version": GENERATOR_VERSION,
|
|
167
|
+
"task_schema_version": TASK_SCHEMA_VERSION,
|
|
168
|
+
}
|
|
169
|
+
return hashlib.sha256(
|
|
170
|
+
json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
|
|
171
|
+
).hexdigest()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def cache_location(owner_root: Path | str, skill_path: Path | str, cache_hash: str) -> Path:
|
|
175
|
+
root = skill_root(skill_path)
|
|
176
|
+
safe_name = re.sub(r"[^A-Za-z0-9._-]+", "-", root.name).strip("-") or "skill"
|
|
177
|
+
return managed_path_for_owner(owner_root, "generated", safe_name, cache_hash)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def load_generated_cache(cache_dir: Path, cache_hash: str) -> GeneratedCache | None:
|
|
181
|
+
manifest_path = cache_dir / "eval.yaml"
|
|
182
|
+
metadata_path = cache_dir / "generation.json"
|
|
183
|
+
if not manifest_path.is_file() or not metadata_path.is_file():
|
|
184
|
+
return None
|
|
185
|
+
try:
|
|
186
|
+
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
187
|
+
if metadata.get("generation_hash") != cache_hash:
|
|
188
|
+
return None
|
|
189
|
+
if metadata.get("manifest_hash") != sha256(manifest_path.read_bytes()):
|
|
190
|
+
return None
|
|
191
|
+
validate_generated_manifest(manifest_path)
|
|
192
|
+
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
|
193
|
+
return None
|
|
194
|
+
return GeneratedCache(manifest_path, metadata_path, cache_hash)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def validate_generated_manifest(manifest_path: Path | str):
|
|
198
|
+
"""Validate the stronger contract that applies to generated cache entries.
|
|
199
|
+
|
|
200
|
+
The public manifest schema intentionally accepts authored and sourced tasks,
|
|
201
|
+
while a generated cache must contain only the bounded inline task snapshot
|
|
202
|
+
produced by the generator. Keeping this check beside cache loading makes
|
|
203
|
+
Python execution and static collection reject the same stale or tampered
|
|
204
|
+
cache before it can be reused.
|
|
205
|
+
"""
|
|
206
|
+
manifest = load_eval_manifest(manifest_path)
|
|
207
|
+
tasks = manifest.tasks
|
|
208
|
+
if not MIN_GENERATED_TASKS <= len(tasks) <= MAX_GENERATED_TASKS:
|
|
209
|
+
raise ValueError(
|
|
210
|
+
f"generated evaluation.tasks must contain {MIN_GENERATED_TASKS}-{MAX_GENERATED_TASKS} tasks"
|
|
211
|
+
)
|
|
212
|
+
names: set[str] = set()
|
|
213
|
+
for index, task in enumerate(tasks):
|
|
214
|
+
if not task.is_inline:
|
|
215
|
+
raise ValueError(f"generated evaluation.tasks[{index}] must be an inline task")
|
|
216
|
+
if task.name in names:
|
|
217
|
+
raise ValueError(f'generated evaluation.tasks[{index}].name is duplicated: "{task.name}"')
|
|
218
|
+
names.add(task.name)
|
|
219
|
+
return manifest
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def write_cache_metadata(cache_dir: Path, cache_hash: str) -> Path:
|
|
223
|
+
"""Test and migration helper for a static cache already written atomically by the generator."""
|
|
224
|
+
manifest_path = cache_dir / "eval.yaml"
|
|
225
|
+
metadata_path = cache_dir / "generation.json"
|
|
226
|
+
metadata_path.write_text(
|
|
227
|
+
json.dumps({"generation_hash": cache_hash, "manifest_hash": sha256(manifest_path.read_bytes())}),
|
|
228
|
+
encoding="utf-8",
|
|
229
|
+
)
|
|
230
|
+
return metadata_path
|
|
@@ -6,8 +6,8 @@ produce meaningful output, or just a stub? This module asks a judge LLM to read
|
|
|
6
6
|
workspace artifacts and score three generic quality dimensions on a 0.00-1.00
|
|
7
7
|
scale with a cited reason.
|
|
8
8
|
|
|
9
|
-
Runs on the host (not in Docker) via the
|
|
10
|
-
|
|
9
|
+
Runs on the host (not in Docker) via the selected agent CLI and its configured
|
|
10
|
+
provider, so it adds no new dependency. Scores are emitted in the same
|
|
11
11
|
``[RUBRIC-JUDGE] dim: score - reason`` format the logging layer understands.
|
|
12
12
|
|
|
13
13
|
Usage::
|
|
@@ -134,12 +134,9 @@ def _build_generic_judge_prompt(
|
|
|
134
134
|
|
|
135
135
|
# Build the list of dimensions the judge should output.
|
|
136
136
|
all_dims = list(GENERIC_JUDGE_DIMENSIONS)
|
|
137
|
-
dim_lines = [
|
|
138
|
-
f"[RUBRIC-JUDGE] {dim}: <score> - <reason>" for dim in all_dims
|
|
139
|
-
]
|
|
137
|
+
dim_lines = [f"[RUBRIC-JUDGE] {dim}: <score> - <reason>" for dim in all_dims]
|
|
140
138
|
extra_dim_lines = [
|
|
141
|
-
f"[RUBRIC-JUDGE] custom_{i}: <score> - <reason>"
|
|
142
|
-
for i in range(len(custom_criteria))
|
|
139
|
+
f"[RUBRIC-JUDGE] custom_{i}: <score> - <reason>" for i in range(len(custom_criteria))
|
|
143
140
|
]
|
|
144
141
|
all_output_lines = "\n".join(dim_lines + extra_dim_lines)
|
|
145
142
|
|
|
@@ -177,9 +174,25 @@ Baseline validation results:
|
|
|
177
174
|
"""
|
|
178
175
|
|
|
179
176
|
|
|
180
|
-
def _run_judge(
|
|
177
|
+
def _run_judge(
|
|
178
|
+
prompt: str,
|
|
179
|
+
timeout: int = 120,
|
|
180
|
+
agent: object | None = None,
|
|
181
|
+
model: object | None = None,
|
|
182
|
+
base_url: object | None = None,
|
|
183
|
+
evidence: dict[str, Any] | None = None,
|
|
184
|
+
excluded_credentials: tuple[str, ...] = (),
|
|
185
|
+
) -> str:
|
|
181
186
|
"""Call the judge LLM through the configured judge provider."""
|
|
182
|
-
return run_judge_prompt(
|
|
187
|
+
return run_judge_prompt(
|
|
188
|
+
prompt,
|
|
189
|
+
timeout=timeout,
|
|
190
|
+
agent=agent,
|
|
191
|
+
model=model,
|
|
192
|
+
base_url=base_url,
|
|
193
|
+
evidence=evidence,
|
|
194
|
+
excluded_credentials=excluded_credentials,
|
|
195
|
+
)
|
|
183
196
|
|
|
184
197
|
|
|
185
198
|
def judge_generic_artifacts(
|
|
@@ -194,7 +207,15 @@ def judge_generic_artifacts(
|
|
|
194
207
|
rule-based scores.
|
|
195
208
|
"""
|
|
196
209
|
prompt = _build_generic_judge_prompt(test_dir, outputs)
|
|
197
|
-
raw = _run_judge(
|
|
210
|
+
raw = _run_judge(
|
|
211
|
+
prompt,
|
|
212
|
+
timeout=timeout,
|
|
213
|
+
agent=outputs.get("judge_agent") or outputs.get("agent"),
|
|
214
|
+
model=outputs.get("judge_model"),
|
|
215
|
+
base_url=outputs.get("judge_base_url"),
|
|
216
|
+
excluded_credentials=tuple(outputs.get("main_credentials") or ()),
|
|
217
|
+
evidence=outputs,
|
|
218
|
+
)
|
|
198
219
|
|
|
199
220
|
# Collect all valid dimension names (standard + custom).
|
|
200
221
|
custom_criteria = outputs.get("rubric_criteria") or []
|
|
@@ -226,12 +247,18 @@ def judge_generic_messages(
|
|
|
226
247
|
"""Convenience wrapper returning ``[RUBRIC-JUDGE]`` check messages."""
|
|
227
248
|
out: list[str] = []
|
|
228
249
|
try:
|
|
229
|
-
build_judge_invocation(
|
|
250
|
+
build_judge_invocation(
|
|
251
|
+
agent=outputs.get("judge_agent") or outputs.get("agent"),
|
|
252
|
+
model=outputs.get("judge_model"),
|
|
253
|
+
base_url=outputs.get("judge_base_url"),
|
|
254
|
+
excluded_credentials=tuple(outputs.get("main_credentials") or ()),
|
|
255
|
+
)
|
|
230
256
|
except ValueError as e:
|
|
231
257
|
return [f"[RUBRIC-JUDGE] status: skipped - {e}"]
|
|
232
258
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
259
|
+
scores = judge_generic_artifacts(test_dir, outputs, timeout=timeout)
|
|
260
|
+
if not scores:
|
|
261
|
+
return ["[RUBRIC-JUDGE] status: unavailable - no valid Judge scores"]
|
|
262
|
+
for dim, (score, reason) in scores.items():
|
|
236
263
|
out.append(f"[RUBRIC-JUDGE] {dim}: {score:.2f} - {reason}")
|
|
237
264
|
return out
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Deterministic validators for project-authored inline eval tasks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from scaffold.python.utils import run_command_in_docker
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _workspace_path(test_dir: Path, value: str) -> Path:
|
|
13
|
+
candidate = test_dir / value
|
|
14
|
+
resolved = candidate.resolve(strict=False)
|
|
15
|
+
try:
|
|
16
|
+
resolved.relative_to(test_dir.resolve())
|
|
17
|
+
except ValueError as exc:
|
|
18
|
+
raise ValueError(f"Task artifact escapes the workspace: {value!r}") from exc
|
|
19
|
+
return candidate
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _json_path(value: Any, path: str) -> Any:
|
|
23
|
+
if path == "$":
|
|
24
|
+
return value
|
|
25
|
+
if not path.startswith("$."):
|
|
26
|
+
raise ValueError(f"Unsupported JSON path: {path}")
|
|
27
|
+
tokens: list[str] = []
|
|
28
|
+
for part in path[2:].replace("[", ".").replace("]", "").split("."):
|
|
29
|
+
if part:
|
|
30
|
+
tokens.append(part)
|
|
31
|
+
current = value
|
|
32
|
+
for token in tokens:
|
|
33
|
+
if isinstance(current, list):
|
|
34
|
+
current = current[int(token)]
|
|
35
|
+
elif isinstance(current, dict):
|
|
36
|
+
current = current[token]
|
|
37
|
+
else:
|
|
38
|
+
raise KeyError(token)
|
|
39
|
+
return current
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _display_output(result: Any) -> str:
|
|
43
|
+
output = "\n".join(
|
|
44
|
+
part for part in (getattr(result, "stdout", ""), getattr(result, "stderr", "")) if part
|
|
45
|
+
)
|
|
46
|
+
return output[-1000:]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def make_inline_expectations_validator(expect: dict[str, Any]):
|
|
50
|
+
"""Build a validator for the normalized ``evaluation.tasks.expect`` map."""
|
|
51
|
+
|
|
52
|
+
def validate(test_dir: Path, _outputs: dict) -> tuple[list[str], list[str]]:
|
|
53
|
+
passed: list[str] = []
|
|
54
|
+
failed: list[str] = []
|
|
55
|
+
|
|
56
|
+
for command_spec in expect.get("commands", []):
|
|
57
|
+
command = command_spec["run"]
|
|
58
|
+
result = run_command_in_docker(test_dir, command, command_spec["timeout"])
|
|
59
|
+
if result.returncode == 0:
|
|
60
|
+
passed.append(f"Command succeeded: {command}")
|
|
61
|
+
else:
|
|
62
|
+
failed.append(f"Command failed: {command} ({_display_output(result)})")
|
|
63
|
+
|
|
64
|
+
for artifact in expect.get("files", []):
|
|
65
|
+
path = _workspace_path(test_dir, artifact)
|
|
66
|
+
if any(char in artifact for char in "*?["):
|
|
67
|
+
matches = []
|
|
68
|
+
for item in test_dir.glob(artifact):
|
|
69
|
+
try:
|
|
70
|
+
safe_item = _workspace_path(test_dir, str(item.relative_to(test_dir)))
|
|
71
|
+
except ValueError:
|
|
72
|
+
continue
|
|
73
|
+
if safe_item == item:
|
|
74
|
+
matches.append(item)
|
|
75
|
+
if matches:
|
|
76
|
+
passed.append(f"Files exist: {artifact}")
|
|
77
|
+
else:
|
|
78
|
+
failed.append(f"Files missing: {artifact}")
|
|
79
|
+
elif path.exists():
|
|
80
|
+
passed.append(f"File exists: {artifact}")
|
|
81
|
+
else:
|
|
82
|
+
failed.append(f"File missing: {artifact}")
|
|
83
|
+
|
|
84
|
+
for artifact, values in expect.get("contains", {}).items():
|
|
85
|
+
path = _workspace_path(test_dir, artifact)
|
|
86
|
+
if not path.is_file():
|
|
87
|
+
failed.append(f"Contains checks missing file: {artifact}")
|
|
88
|
+
continue
|
|
89
|
+
content = path.read_text(encoding="utf-8", errors="replace")
|
|
90
|
+
for value in values:
|
|
91
|
+
if value in content:
|
|
92
|
+
passed.append(f"Contains {value!r} in {artifact}")
|
|
93
|
+
else:
|
|
94
|
+
failed.append(f"Missing {value!r} in {artifact}")
|
|
95
|
+
|
|
96
|
+
for check in expect.get("json", []):
|
|
97
|
+
artifact = check["file"]
|
|
98
|
+
path = _workspace_path(test_dir, artifact)
|
|
99
|
+
try:
|
|
100
|
+
actual = _json_path(json.loads(path.read_text(encoding="utf-8")), check["path"])
|
|
101
|
+
except (
|
|
102
|
+
FileNotFoundError,
|
|
103
|
+
json.JSONDecodeError,
|
|
104
|
+
KeyError,
|
|
105
|
+
IndexError,
|
|
106
|
+
TypeError,
|
|
107
|
+
ValueError,
|
|
108
|
+
) as exc:
|
|
109
|
+
failed.append(f"JSON {check['path']} in {artifact} could not be read: {exc}")
|
|
110
|
+
continue
|
|
111
|
+
if actual == check["equals"]:
|
|
112
|
+
passed.append(f"JSON {check['path']} equals {check['equals']}")
|
|
113
|
+
else:
|
|
114
|
+
failed.append(f"JSON {check['path']} expected {check['equals']!r}, got {actual!r}")
|
|
115
|
+
|
|
116
|
+
return passed, failed
|
|
117
|
+
|
|
118
|
+
return validate
|