@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -0,0 +1,700 @@
|
|
|
1
|
+
"""Automatic, cached generation of #250-compatible eval task manifests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import html as html_module
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import shutil
|
|
11
|
+
import tempfile
|
|
12
|
+
import time
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Callable, Iterator, Mapping
|
|
17
|
+
|
|
18
|
+
import yaml
|
|
19
|
+
|
|
20
|
+
from scaffold.python.agents import AgentId, get_agent_adapter, validate_agent_id
|
|
21
|
+
from scaffold.python.eval_context import managed_path_for_owner
|
|
22
|
+
from scaffold.python.generated_task_cache import (
|
|
23
|
+
build_skill_snapshot as _shared_build_skill_snapshot,
|
|
24
|
+
cache_location as _shared_cache_location,
|
|
25
|
+
generation_hash as _shared_generation_hash,
|
|
26
|
+
normalize_interaction,
|
|
27
|
+
validate_generated_manifest,
|
|
28
|
+
)
|
|
29
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
30
|
+
from scaffold.python.utils import run_agent_in_docker
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
GENERATOR_VERSION = "comet-auto-task-generator.v1"
|
|
34
|
+
TASK_SCHEMA_VERSION = "comet.eval/v1alpha1"
|
|
35
|
+
MAX_SNAPSHOT_FILES = 128
|
|
36
|
+
MAX_SNAPSHOT_BYTES = 512 * 1024
|
|
37
|
+
MAX_GENERATED_TASKS = 4
|
|
38
|
+
MIN_GENERATED_TASKS = 2
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class AutoTaskError(ValueError):
|
|
42
|
+
"""A task-generation failure that must not be counted as a task failure."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class SnapshotFile:
|
|
47
|
+
path: str
|
|
48
|
+
content: str
|
|
49
|
+
content_hash: str
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class SkillSnapshot:
|
|
54
|
+
files: tuple[SnapshotFile, ...]
|
|
55
|
+
content_hash: str
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class GeneratedManifest:
|
|
60
|
+
manifest_path: Path
|
|
61
|
+
metadata_path: Path
|
|
62
|
+
generation_hash: str
|
|
63
|
+
reused: bool = False
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class GenerationOutput:
|
|
68
|
+
"""Generator response plus role-local overhead telemetry."""
|
|
69
|
+
|
|
70
|
+
output: object
|
|
71
|
+
telemetry: dict[str, Any]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _sha256(value: str | bytes) -> str:
|
|
75
|
+
data = value.encode("utf-8") if isinstance(value, str) else value
|
|
76
|
+
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _safe_skill_root(skill_path: Path) -> Path:
|
|
80
|
+
root = skill_path.resolve()
|
|
81
|
+
if root.is_file() and root.name == "SKILL.md":
|
|
82
|
+
root = root.parent
|
|
83
|
+
if not root.is_dir() or not (root / "SKILL.md").is_file():
|
|
84
|
+
raise AutoTaskError(f"Skill package must contain SKILL.md: {skill_path}")
|
|
85
|
+
return root
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _inside(root: Path, path: Path) -> bool:
|
|
89
|
+
try:
|
|
90
|
+
path.resolve().relative_to(root.resolve())
|
|
91
|
+
return True
|
|
92
|
+
except ValueError:
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _add_snapshot_file(root: Path, files: dict[str, SnapshotFile], path: Path) -> None:
|
|
97
|
+
if len(files) >= MAX_SNAPSHOT_FILES or not path.is_file() or not _inside(root, path):
|
|
98
|
+
return
|
|
99
|
+
relative = path.resolve().relative_to(root.resolve()).as_posix()
|
|
100
|
+
if relative in files:
|
|
101
|
+
return
|
|
102
|
+
content = path.read_text(encoding="utf-8", errors="replace")
|
|
103
|
+
if (
|
|
104
|
+
sum(len(item.content.encode("utf-8")) for item in files.values())
|
|
105
|
+
+ len(content.encode("utf-8"))
|
|
106
|
+
> MAX_SNAPSHOT_BYTES
|
|
107
|
+
):
|
|
108
|
+
return
|
|
109
|
+
files[relative] = SnapshotFile(relative, content, _sha256(content))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_skill_snapshot(skill_path: Path | str) -> SkillSnapshot:
|
|
113
|
+
"""Capture only the Skill and its bounded, relevant package context."""
|
|
114
|
+
snapshot = _shared_build_skill_snapshot(skill_path)
|
|
115
|
+
return SkillSnapshot(
|
|
116
|
+
tuple(SnapshotFile(item.path, item.content, item.content_hash) for item in snapshot.files),
|
|
117
|
+
snapshot.content_hash,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def find_project_root(skill_path: Path | str) -> Path:
|
|
122
|
+
"""Find a project root without scanning outside the Skill's ancestors."""
|
|
123
|
+
root = _safe_skill_root(Path(skill_path))
|
|
124
|
+
home = Path.home().resolve()
|
|
125
|
+
for candidate in (root, *root.parents):
|
|
126
|
+
if candidate.resolve() == home:
|
|
127
|
+
continue
|
|
128
|
+
if (candidate / ".comet").is_dir() or (candidate / ".git").exists():
|
|
129
|
+
return candidate
|
|
130
|
+
return root.parent
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _generation_hash(
|
|
134
|
+
snapshot: SkillSnapshot,
|
|
135
|
+
*,
|
|
136
|
+
agent: AgentId,
|
|
137
|
+
model: str | None,
|
|
138
|
+
base_url: str | None = None,
|
|
139
|
+
profile: str,
|
|
140
|
+
interaction: dict[str, Any],
|
|
141
|
+
) -> str:
|
|
142
|
+
return _shared_generation_hash(
|
|
143
|
+
snapshot,
|
|
144
|
+
agent=agent,
|
|
145
|
+
model=model,
|
|
146
|
+
base_url=base_url,
|
|
147
|
+
profile=profile,
|
|
148
|
+
interaction=interaction,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _cache_location(project_root: Path, skill_root: Path, generation_hash: str) -> Path:
|
|
153
|
+
return _shared_cache_location(project_root, skill_root, generation_hash)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _atomic_write_text(path: Path, value: str) -> None:
|
|
157
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
158
|
+
with tempfile.NamedTemporaryFile(
|
|
159
|
+
mode="w",
|
|
160
|
+
encoding="utf-8",
|
|
161
|
+
newline="\n",
|
|
162
|
+
dir=path.parent,
|
|
163
|
+
prefix=f".{path.name}.",
|
|
164
|
+
delete=False,
|
|
165
|
+
) as temporary:
|
|
166
|
+
temporary.write(value)
|
|
167
|
+
temporary_path = Path(temporary.name)
|
|
168
|
+
try:
|
|
169
|
+
os.replace(temporary_path, path)
|
|
170
|
+
finally:
|
|
171
|
+
temporary_path.unlink(missing_ok=True)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _write_generation_failure_report(
|
|
175
|
+
project_root: Path,
|
|
176
|
+
*,
|
|
177
|
+
attempts: list[dict[str, Any]],
|
|
178
|
+
error: str,
|
|
179
|
+
environment: Mapping[str, str] | None = None,
|
|
180
|
+
) -> Path:
|
|
181
|
+
"""Persist a user-owned terminal report without manufacturing task results."""
|
|
182
|
+
experiment = os.environ.get("COMET_EVAL_EXPERIMENT_ID") or f"task-generation-{time.time_ns()}"
|
|
183
|
+
report_dir = managed_path_for_owner(project_root, "runs", experiment)
|
|
184
|
+
report_dir.mkdir(parents=True, exist_ok=True)
|
|
185
|
+
safe_error = _redact_sensitive(error, environment)
|
|
186
|
+
safe_attempts = _redact_sensitive(_without_credentials(attempts), environment)
|
|
187
|
+
html = os.environ.get("COMET_EVAL_REPORT_HTML") == "1"
|
|
188
|
+
output_path = report_dir / ("summary.html" if html else "summary.md")
|
|
189
|
+
escaped_error = html_module.escape(str(safe_error)) if html else safe_error
|
|
190
|
+
metadata = {
|
|
191
|
+
"schema": "comet.eval.generation.failure.v1",
|
|
192
|
+
"attribution": "task_generation",
|
|
193
|
+
"category": "task_generation",
|
|
194
|
+
"attempt_count": len(attempts),
|
|
195
|
+
"case_count": 0,
|
|
196
|
+
"task_denominator": 0,
|
|
197
|
+
"subject_metrics": {},
|
|
198
|
+
"generation_overhead": _merge_generation_telemetry(safe_attempts),
|
|
199
|
+
"error": safe_error,
|
|
200
|
+
"report_path": str(output_path),
|
|
201
|
+
"report_output": output_path.name,
|
|
202
|
+
}
|
|
203
|
+
_atomic_write_text(
|
|
204
|
+
report_dir / "metadata.json", json.dumps(metadata, indent=2, sort_keys=True) + "\n"
|
|
205
|
+
)
|
|
206
|
+
_atomic_write_text(
|
|
207
|
+
output_path,
|
|
208
|
+
(
|
|
209
|
+
"<h1>Evaluation stopped during task generation</h1>"
|
|
210
|
+
f"<p>Attempts: {len(attempts)}</p><p>Task denominator: 0</p>"
|
|
211
|
+
f"<p>Error: {escaped_error}</p>"
|
|
212
|
+
if html
|
|
213
|
+
else "# Evaluation stopped during task generation\n\n"
|
|
214
|
+
"Attribution: `task_generation`\n\n"
|
|
215
|
+
f"Attempts: {len(attempts)}\n\nTask denominator: 0\n\nError: {safe_error}\n"
|
|
216
|
+
),
|
|
217
|
+
)
|
|
218
|
+
return output_path
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _without_credentials(value: Any) -> Any:
|
|
222
|
+
if isinstance(value, dict):
|
|
223
|
+
return {
|
|
224
|
+
key: _without_credentials(item)
|
|
225
|
+
for key, item in value.items()
|
|
226
|
+
if not re.search(
|
|
227
|
+
r"(?:api[_-]?key|auth[_-]?token|access[_-]?token|personal[_-]?access[_-]?token|"
|
|
228
|
+
r"(?:^|[_-])(key|token|secret|password|credential)$)",
|
|
229
|
+
key,
|
|
230
|
+
re.I,
|
|
231
|
+
)
|
|
232
|
+
}
|
|
233
|
+
if isinstance(value, list):
|
|
234
|
+
return [_without_credentials(item) for item in value]
|
|
235
|
+
return value
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _credential_values(source_env: Mapping[str, str] | None = None) -> tuple[str, ...]:
|
|
239
|
+
"""Return configured credential values for final report redaction."""
|
|
240
|
+
environment = dict(os.environ)
|
|
241
|
+
if source_env is not None:
|
|
242
|
+
environment.update(source_env)
|
|
243
|
+
custom_names = {
|
|
244
|
+
item.strip()
|
|
245
|
+
for metadata_key in ("COMET_EVAL_CUSTOM_CREDENTIALS", "COMET_EVAL_MAIN_CREDENTIALS")
|
|
246
|
+
for item in environment.get(metadata_key, "").split(",")
|
|
247
|
+
if item.strip()
|
|
248
|
+
}
|
|
249
|
+
values = {
|
|
250
|
+
value.strip()
|
|
251
|
+
for key, value in environment.items()
|
|
252
|
+
if value.strip()
|
|
253
|
+
and (
|
|
254
|
+
key in custom_names
|
|
255
|
+
or re.search(
|
|
256
|
+
r"(?:api[_-]?key|auth[_-]?token|access[_-]?token|personal[_-]?access[_-]?token|"
|
|
257
|
+
r"(?:^|[_-])(key|token|secret|password|credential)$)",
|
|
258
|
+
key,
|
|
259
|
+
re.I,
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
}
|
|
263
|
+
return tuple(sorted((value for value in values if len(value) >= 4), key=len, reverse=True))
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _redact_sensitive(value: Any, source_env: Mapping[str, str] | None = None) -> Any:
|
|
267
|
+
"""Remove common credential forms and configured values from reports."""
|
|
268
|
+
if isinstance(value, dict):
|
|
269
|
+
return {key: _redact_sensitive(item, source_env) for key, item in value.items()}
|
|
270
|
+
if isinstance(value, list):
|
|
271
|
+
return [_redact_sensitive(item, source_env) for item in value]
|
|
272
|
+
if not isinstance(value, str):
|
|
273
|
+
return value
|
|
274
|
+
redacted = re.sub(
|
|
275
|
+
r"(?i)(authorization\s*:\s*bearer\s+)[^\s,;]+",
|
|
276
|
+
r"\1[REDACTED]",
|
|
277
|
+
value,
|
|
278
|
+
)
|
|
279
|
+
redacted = re.sub(
|
|
280
|
+
r"(?i)(x-api-key\s*:\s*)[^\s,;]+",
|
|
281
|
+
r"\1[REDACTED]",
|
|
282
|
+
redacted,
|
|
283
|
+
)
|
|
284
|
+
redacted = re.sub(
|
|
285
|
+
r"(?i)(api[_-]?key|token|secret|password|credential)\s*[=:]\s*[^\s,;]+",
|
|
286
|
+
r"\1=[REDACTED]",
|
|
287
|
+
redacted,
|
|
288
|
+
)
|
|
289
|
+
redacted = re.sub(
|
|
290
|
+
r"(?i)([?&](?:api[_-]?key|token|access[_-]?token|auth[_-]?token)=)[^&#\s]+",
|
|
291
|
+
r"\1[REDACTED]",
|
|
292
|
+
redacted,
|
|
293
|
+
)
|
|
294
|
+
redacted = re.sub(r"(https?://)[^/@\s:]+:[^/@\s]+@", r"\1[REDACTED]@", redacted)
|
|
295
|
+
for secret in _credential_values(source_env):
|
|
296
|
+
redacted = redacted.replace(secret, "[REDACTED]")
|
|
297
|
+
return redacted
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _extract_json_payload(raw: object) -> dict[str, Any]:
|
|
301
|
+
if isinstance(raw, GenerationOutput):
|
|
302
|
+
raw = raw.output
|
|
303
|
+
if isinstance(raw, dict):
|
|
304
|
+
return raw
|
|
305
|
+
if not isinstance(raw, str):
|
|
306
|
+
raise AutoTaskError("task_generation: generator did not return JSON")
|
|
307
|
+
text = raw.strip()
|
|
308
|
+
if text.startswith("```"):
|
|
309
|
+
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE)
|
|
310
|
+
candidates = [text, *reversed(text.splitlines())]
|
|
311
|
+
for candidate in candidates:
|
|
312
|
+
try:
|
|
313
|
+
value = json.loads(candidate)
|
|
314
|
+
except json.JSONDecodeError:
|
|
315
|
+
continue
|
|
316
|
+
if isinstance(value, dict):
|
|
317
|
+
stack: list[object] = [value]
|
|
318
|
+
while stack:
|
|
319
|
+
nested = stack.pop()
|
|
320
|
+
if isinstance(nested, dict):
|
|
321
|
+
if isinstance(nested.get("tasks"), list):
|
|
322
|
+
return nested
|
|
323
|
+
stack.extend(nested.values())
|
|
324
|
+
elif isinstance(nested, list):
|
|
325
|
+
stack.extend(nested)
|
|
326
|
+
elif isinstance(nested, str):
|
|
327
|
+
try:
|
|
328
|
+
decoded = json.loads(nested)
|
|
329
|
+
except json.JSONDecodeError:
|
|
330
|
+
continue
|
|
331
|
+
stack.append(decoded)
|
|
332
|
+
raise AutoTaskError("task_generation: generator output was not a JSON task object")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _validate_generated_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
|
336
|
+
tasks = payload.get("tasks")
|
|
337
|
+
if not isinstance(tasks, list) or not MIN_GENERATED_TASKS <= len(tasks) <= MAX_GENERATED_TASKS:
|
|
338
|
+
raise AutoTaskError(
|
|
339
|
+
f"task_generation: generated tasks must contain {MIN_GENERATED_TASKS}-{MAX_GENERATED_TASKS} tasks"
|
|
340
|
+
)
|
|
341
|
+
names: set[str] = set()
|
|
342
|
+
for index, task in enumerate(tasks):
|
|
343
|
+
if not isinstance(task, dict):
|
|
344
|
+
raise AutoTaskError(f"task_generation: tasks[{index}] must be a mapping")
|
|
345
|
+
if set(task) - {"name", "prompt", "workspace", "expect", "rubric"}:
|
|
346
|
+
raise AutoTaskError(f"task_generation: tasks[{index}] contains unsupported fields")
|
|
347
|
+
name = task.get("name")
|
|
348
|
+
if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}", name):
|
|
349
|
+
raise AutoTaskError(f"task_generation: tasks[{index}].name is invalid")
|
|
350
|
+
if name in names:
|
|
351
|
+
raise AutoTaskError(f"task_generation: duplicate task name: {name}")
|
|
352
|
+
names.add(name)
|
|
353
|
+
if not isinstance(task.get("prompt"), str) or not task["prompt"].strip():
|
|
354
|
+
raise AutoTaskError(f"task_generation: tasks[{index}].prompt is required")
|
|
355
|
+
if not isinstance(task.get("expect"), dict):
|
|
356
|
+
raise AutoTaskError(f"task_generation: tasks[{index}].expect is required")
|
|
357
|
+
if task.get("workspace") is not None and not isinstance(task["workspace"], str):
|
|
358
|
+
raise AutoTaskError(f"task_generation: tasks[{index}].workspace must be a path")
|
|
359
|
+
if task.get("rubric") is not None and (
|
|
360
|
+
not isinstance(task["rubric"], list)
|
|
361
|
+
or not all(isinstance(item, str) and item.strip() for item in task["rubric"])
|
|
362
|
+
):
|
|
363
|
+
raise AutoTaskError(f"task_generation: tasks[{index}].rubric is invalid")
|
|
364
|
+
return tasks
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _reject_generated_bundled_collisions(tasks: list[dict[str, Any]]) -> None:
|
|
368
|
+
"""Reject a generated descriptor that would be ambiguous to the normal catalogue."""
|
|
369
|
+
from scaffold.python.paths import get_tasks_dir
|
|
370
|
+
from scaffold.python.tasks import load_task
|
|
371
|
+
|
|
372
|
+
bundled: dict[str, str] = {}
|
|
373
|
+
for directory in get_tasks_dir().iterdir():
|
|
374
|
+
if (directory / "task.toml").is_file() and (directory / "instruction.md").is_file():
|
|
375
|
+
bundled[load_task(directory).name] = directory.name
|
|
376
|
+
for index, task in enumerate(tasks):
|
|
377
|
+
name = task["name"]
|
|
378
|
+
if name in bundled:
|
|
379
|
+
raise AutoTaskError(
|
|
380
|
+
f'generated.tasks[{index}].name conflicts with bundled task "{bundled[name]}": "{name}"'
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _manifest_source(
|
|
385
|
+
skill_root: Path,
|
|
386
|
+
skill_name: str,
|
|
387
|
+
profile: str,
|
|
388
|
+
generation_hash: str,
|
|
389
|
+
tasks: list[dict[str, Any]],
|
|
390
|
+
) -> str:
|
|
391
|
+
return yaml.safe_dump(
|
|
392
|
+
{
|
|
393
|
+
"apiVersion": TASK_SCHEMA_VERSION,
|
|
394
|
+
"kind": "SkillEvalManifest",
|
|
395
|
+
"metadata": {
|
|
396
|
+
"name": skill_name,
|
|
397
|
+
"generationHash": generation_hash,
|
|
398
|
+
"generationFile": "generation.json",
|
|
399
|
+
},
|
|
400
|
+
"skill": {"name": skill_name, "source": str(skill_root), "profile": profile},
|
|
401
|
+
"evaluation": {"tasks": tasks},
|
|
402
|
+
},
|
|
403
|
+
sort_keys=False,
|
|
404
|
+
allow_unicode=True,
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _generation_prompt(snapshot: SkillSnapshot, *, profile: str, repair: str | None = None) -> str:
|
|
409
|
+
files = "\n\n".join(
|
|
410
|
+
f"FILE: {item.path}\nHASH: {item.content_hash}\n{item.content}" for item in snapshot.files
|
|
411
|
+
)
|
|
412
|
+
repair_text = f"\nRepair the previous output because: {repair}\n" if repair else ""
|
|
413
|
+
return f"""You are Comet's task-generator session. Generate only JSON and no Markdown.
|
|
414
|
+
Produce {MIN_GENERATED_TASKS}-{MAX_GENERATED_TASKS} adaptive Skill evaluation tasks for profile {profile!r}.
|
|
415
|
+
Each task must have a unique name, a concrete prompt, and at least one deterministic #250 expect
|
|
416
|
+
from files, contains, json, or commands. Rubric is optional. Do not invent source task packages.
|
|
417
|
+
Use only paths visible in the bounded Skill snapshot. Return exactly:
|
|
418
|
+
{{"tasks":[{{"name":"...","prompt":"...","expect":{{...}},"rubric":["..."]}}]}}
|
|
419
|
+
{repair_text}
|
|
420
|
+
Bounded Skill snapshot:
|
|
421
|
+
{files}
|
|
422
|
+
"""
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _default_generate(
|
|
426
|
+
prompt: str,
|
|
427
|
+
*,
|
|
428
|
+
agent: AgentId,
|
|
429
|
+
model: str | None,
|
|
430
|
+
base_url: str | None = None,
|
|
431
|
+
environment: dict[str, str] | None = None,
|
|
432
|
+
) -> GenerationOutput:
|
|
433
|
+
from scaffold.python.manifest_tasks import _generic_environment_dir
|
|
434
|
+
from scaffold.python.logging import extract_events, parse_output
|
|
435
|
+
|
|
436
|
+
with tempfile.TemporaryDirectory(prefix="comet-task-generator-") as directory:
|
|
437
|
+
workdir = Path(directory)
|
|
438
|
+
shutil.copytree(_generic_environment_dir(), workdir, dirs_exist_ok=True)
|
|
439
|
+
started = time.monotonic()
|
|
440
|
+
result = run_agent_in_docker(
|
|
441
|
+
workdir,
|
|
442
|
+
prompt,
|
|
443
|
+
agent=agent,
|
|
444
|
+
model=model,
|
|
445
|
+
base_url=base_url,
|
|
446
|
+
environment=environment,
|
|
447
|
+
timeout=300,
|
|
448
|
+
)
|
|
449
|
+
elapsed = time.monotonic() - started
|
|
450
|
+
if result.returncode != 0:
|
|
451
|
+
raise AutoTaskError(
|
|
452
|
+
f"task_generation: generator agent failed ({result.returncode}): "
|
|
453
|
+
f"{(result.stderr or result.stdout or '')[-1000:]}"
|
|
454
|
+
)
|
|
455
|
+
stdout = result.stdout or ""
|
|
456
|
+
events = extract_events(parse_output(stdout), agent=agent)
|
|
457
|
+
return GenerationOutput(
|
|
458
|
+
stdout,
|
|
459
|
+
{
|
|
460
|
+
"duration_seconds": events.get("duration_seconds") or elapsed,
|
|
461
|
+
"input_tokens": events.get("input_tokens"),
|
|
462
|
+
"output_tokens": events.get("output_tokens"),
|
|
463
|
+
"total_tokens": events.get("total_tokens"),
|
|
464
|
+
"total_cost_usd": events.get("total_cost_usd"),
|
|
465
|
+
"model_usage": events.get("model_usage") or {},
|
|
466
|
+
"model": model or "runtime-default",
|
|
467
|
+
"telemetry_status": (
|
|
468
|
+
"N/A" if not get_agent_adapter(agent).supports_telemetry else "available"
|
|
469
|
+
),
|
|
470
|
+
},
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _manifest_hash_matches(manifest_path: Path, metadata: Mapping[str, Any]) -> bool:
|
|
475
|
+
expected = metadata.get("manifest_hash")
|
|
476
|
+
if not isinstance(expected, str):
|
|
477
|
+
return False
|
|
478
|
+
try:
|
|
479
|
+
return _sha256(manifest_path.read_bytes()) == expected
|
|
480
|
+
except OSError:
|
|
481
|
+
return False
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _load_cached(cache_dir: Path, generation_hash: str) -> GeneratedManifest | None:
|
|
485
|
+
manifest_path = cache_dir / "eval.yaml"
|
|
486
|
+
metadata_path = cache_dir / "generation.json"
|
|
487
|
+
if not manifest_path.is_file() or not metadata_path.is_file():
|
|
488
|
+
return None
|
|
489
|
+
try:
|
|
490
|
+
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
491
|
+
if metadata.get("generation_hash") != generation_hash or not _manifest_hash_matches(
|
|
492
|
+
manifest_path, metadata
|
|
493
|
+
):
|
|
494
|
+
return None
|
|
495
|
+
manifest = validate_generated_manifest(manifest_path)
|
|
496
|
+
_reject_generated_bundled_collisions([{"name": task.name} for task in manifest.tasks])
|
|
497
|
+
except (OSError, ValueError, json.JSONDecodeError, TypeError):
|
|
498
|
+
return None
|
|
499
|
+
return GeneratedManifest(manifest_path, metadata_path, generation_hash, reused=True)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
@contextmanager
|
|
503
|
+
def _generation_lock(path: Path) -> Iterator[None]:
|
|
504
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
505
|
+
with path.open("a+b") as lock_file:
|
|
506
|
+
if os.name == "nt":
|
|
507
|
+
import msvcrt
|
|
508
|
+
|
|
509
|
+
if lock_file.seek(0, os.SEEK_END) == 0:
|
|
510
|
+
lock_file.write(b"0")
|
|
511
|
+
lock_file.flush()
|
|
512
|
+
lock_file.seek(0)
|
|
513
|
+
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
|
514
|
+
try:
|
|
515
|
+
yield
|
|
516
|
+
finally:
|
|
517
|
+
lock_file.seek(0)
|
|
518
|
+
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
|
519
|
+
else:
|
|
520
|
+
import fcntl
|
|
521
|
+
|
|
522
|
+
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
|
523
|
+
try:
|
|
524
|
+
yield
|
|
525
|
+
finally:
|
|
526
|
+
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _merge_generation_telemetry(attempts: list[dict[str, Any]]) -> dict[str, Any]:
|
|
530
|
+
def total(key: str) -> int | float | None:
|
|
531
|
+
values = [item[key] for item in attempts if isinstance(item.get(key), (int, float))]
|
|
532
|
+
return sum(values) if values else None
|
|
533
|
+
|
|
534
|
+
merged_models: dict[str, dict[str, int | float]] = {}
|
|
535
|
+
for attempt in attempts:
|
|
536
|
+
for model_name, usage in (attempt.get("model_usage") or {}).items():
|
|
537
|
+
if not isinstance(model_name, str) or not isinstance(usage, dict):
|
|
538
|
+
continue
|
|
539
|
+
target = merged_models.setdefault(model_name, {})
|
|
540
|
+
for key, value in usage.items():
|
|
541
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
542
|
+
target[key] = target.get(key, 0) + value
|
|
543
|
+
return {
|
|
544
|
+
"attempt_count": len(attempts),
|
|
545
|
+
"duration_seconds": total("duration_seconds"),
|
|
546
|
+
"input_tokens": total("input_tokens"),
|
|
547
|
+
"output_tokens": total("output_tokens"),
|
|
548
|
+
"total_tokens": total("total_tokens"),
|
|
549
|
+
"total_cost_usd": total("total_cost_usd"),
|
|
550
|
+
"model_usage": merged_models,
|
|
551
|
+
"telemetry_status": (
|
|
552
|
+
"N/A"
|
|
553
|
+
if any(item.get("telemetry_status") == "N/A" for item in attempts)
|
|
554
|
+
else "available"
|
|
555
|
+
if any(item.get("telemetry_status") == "available" for item in attempts)
|
|
556
|
+
else "N/A"
|
|
557
|
+
),
|
|
558
|
+
"attempts": attempts,
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def ensure_generated_manifest(
|
|
563
|
+
skill_path: Path | str,
|
|
564
|
+
project_root: Path | str,
|
|
565
|
+
*,
|
|
566
|
+
agent: object,
|
|
567
|
+
model: str | None,
|
|
568
|
+
profile: str,
|
|
569
|
+
interaction: dict[str, Any],
|
|
570
|
+
base_url: str | None = None,
|
|
571
|
+
environment: dict[str, str] | None = None,
|
|
572
|
+
collect_only: bool = False,
|
|
573
|
+
generate: Callable[[str], object] | None = None,
|
|
574
|
+
) -> GeneratedManifest:
|
|
575
|
+
"""Return a valid frozen generated manifest, or generate it once and cache it."""
|
|
576
|
+
selected_agent = validate_agent_id(agent, field="task-generator agent")
|
|
577
|
+
skill_root = _safe_skill_root(Path(skill_path))
|
|
578
|
+
snapshot = build_skill_snapshot(skill_root)
|
|
579
|
+
canonical_interaction = normalize_interaction(interaction)
|
|
580
|
+
generation_hash = _generation_hash(
|
|
581
|
+
snapshot,
|
|
582
|
+
agent=selected_agent,
|
|
583
|
+
model=model,
|
|
584
|
+
base_url=base_url,
|
|
585
|
+
profile=profile,
|
|
586
|
+
interaction=canonical_interaction,
|
|
587
|
+
)
|
|
588
|
+
cache_dir = _cache_location(Path(project_root), skill_root, generation_hash)
|
|
589
|
+
cached = _load_cached(cache_dir, generation_hash)
|
|
590
|
+
if cached:
|
|
591
|
+
return cached
|
|
592
|
+
if collect_only:
|
|
593
|
+
raise AutoTaskError(
|
|
594
|
+
"No cached generated eval manifest exists; run comet eval <skill> once to generate and execute tasks"
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
cache_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
598
|
+
lock_path = managed_path_for_owner(
|
|
599
|
+
project_root, "locks", f"generated-{generation_hash}.lock"
|
|
600
|
+
)
|
|
601
|
+
with _generation_lock(lock_path):
|
|
602
|
+
cached = _load_cached(cache_dir, generation_hash)
|
|
603
|
+
if cached:
|
|
604
|
+
return cached
|
|
605
|
+
|
|
606
|
+
generator = generate or (
|
|
607
|
+
lambda prompt: _default_generate(
|
|
608
|
+
prompt,
|
|
609
|
+
agent=selected_agent,
|
|
610
|
+
model=model,
|
|
611
|
+
base_url=base_url,
|
|
612
|
+
environment=environment,
|
|
613
|
+
)
|
|
614
|
+
)
|
|
615
|
+
error: str | None = None
|
|
616
|
+
tasks: list[dict[str, Any]] | None = None
|
|
617
|
+
attempts: list[dict[str, Any]] = []
|
|
618
|
+
for attempt in range(2):
|
|
619
|
+
try:
|
|
620
|
+
generated = generator(_generation_prompt(snapshot, profile=profile, repair=error))
|
|
621
|
+
if isinstance(generated, GenerationOutput):
|
|
622
|
+
raw_output = generated.output
|
|
623
|
+
telemetry = dict(generated.telemetry)
|
|
624
|
+
else:
|
|
625
|
+
raw_output = generated
|
|
626
|
+
telemetry = {}
|
|
627
|
+
telemetry.setdefault("model", model or "runtime-default")
|
|
628
|
+
telemetry.setdefault(
|
|
629
|
+
"telemetry_status",
|
|
630
|
+
"N/A" if not get_agent_adapter(selected_agent).supports_telemetry else "available",
|
|
631
|
+
)
|
|
632
|
+
telemetry["attempt"] = attempt + 1
|
|
633
|
+
attempts.append(telemetry)
|
|
634
|
+
payload = _extract_json_payload(raw_output)
|
|
635
|
+
tasks = _validate_generated_payload(payload)
|
|
636
|
+
_reject_generated_bundled_collisions(tasks)
|
|
637
|
+
manifest_source = _manifest_source(
|
|
638
|
+
skill_root, skill_root.name, profile, generation_hash, tasks
|
|
639
|
+
)
|
|
640
|
+
with tempfile.NamedTemporaryFile(
|
|
641
|
+
mode="w", suffix=".yaml", encoding="utf-8", delete=False
|
|
642
|
+
) as candidate:
|
|
643
|
+
candidate.write(manifest_source)
|
|
644
|
+
candidate_path = Path(candidate.name)
|
|
645
|
+
try:
|
|
646
|
+
load_eval_manifest(candidate_path)
|
|
647
|
+
finally:
|
|
648
|
+
candidate_path.unlink(missing_ok=True)
|
|
649
|
+
break
|
|
650
|
+
except (AutoTaskError, OSError, ValueError, yaml.YAMLError) as exc:
|
|
651
|
+
error = str(exc)
|
|
652
|
+
tasks = None
|
|
653
|
+
if tasks is None:
|
|
654
|
+
message = error or "task_generation: generated manifest is invalid"
|
|
655
|
+
report_path = _write_generation_failure_report(
|
|
656
|
+
Path(project_root),
|
|
657
|
+
attempts=attempts,
|
|
658
|
+
error=message,
|
|
659
|
+
environment=environment,
|
|
660
|
+
)
|
|
661
|
+
raise AutoTaskError(f"{message}\nPartial report: {report_path}")
|
|
662
|
+
|
|
663
|
+
manifest_source = _manifest_source(
|
|
664
|
+
skill_root, skill_root.name, profile, generation_hash, tasks
|
|
665
|
+
)
|
|
666
|
+
metadata = {
|
|
667
|
+
"schema": "comet.eval.generation.v1",
|
|
668
|
+
"generation_hash": generation_hash,
|
|
669
|
+
"generator_version": GENERATOR_VERSION,
|
|
670
|
+
"task_schema_version": TASK_SCHEMA_VERSION,
|
|
671
|
+
"skill_snapshot_hash": snapshot.content_hash,
|
|
672
|
+
"skill_path": str(skill_root),
|
|
673
|
+
"agent": selected_agent,
|
|
674
|
+
"model": model or "runtime-default",
|
|
675
|
+
"profile": profile,
|
|
676
|
+
"interaction": canonical_interaction,
|
|
677
|
+
"manifest_hash": _sha256(manifest_source),
|
|
678
|
+
"generation_overhead": _merge_generation_telemetry(attempts),
|
|
679
|
+
}
|
|
680
|
+
stage_dir = Path(tempfile.mkdtemp(dir=str(cache_dir.parent), prefix=f".{generation_hash}-"))
|
|
681
|
+
try:
|
|
682
|
+
_atomic_write_text(stage_dir / "eval.yaml", manifest_source)
|
|
683
|
+
_atomic_write_text(
|
|
684
|
+
stage_dir / "generation.json", json.dumps(_without_credentials(metadata), indent=2, sort_keys=True) + "\n"
|
|
685
|
+
)
|
|
686
|
+
if cache_dir.exists():
|
|
687
|
+
retired_dir = cache_dir.with_name(f".{cache_dir.name}.{time.time_ns()}.retired")
|
|
688
|
+
os.replace(cache_dir, retired_dir)
|
|
689
|
+
else:
|
|
690
|
+
retired_dir = None
|
|
691
|
+
os.replace(stage_dir, cache_dir)
|
|
692
|
+
stage_dir = None # type: ignore[assignment]
|
|
693
|
+
if retired_dir is not None:
|
|
694
|
+
shutil.rmtree(retired_dir, ignore_errors=True)
|
|
695
|
+
finally:
|
|
696
|
+
if stage_dir is not None:
|
|
697
|
+
shutil.rmtree(stage_dir, ignore_errors=True)
|
|
698
|
+
return GeneratedManifest(
|
|
699
|
+
cache_dir / "eval.yaml", cache_dir / "generation.json", generation_hash
|
|
700
|
+
)
|