@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Contract tests for Langfuse trajectory provisioning and transcript adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tarfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
from scaffold.python.langfuse_adapter import (
|
|
15
|
+
LangfuseRunReporter,
|
|
16
|
+
TrajectoryProvisionError,
|
|
17
|
+
TrajectoryProvision,
|
|
18
|
+
install_codex_plugin_workspace,
|
|
19
|
+
install_transcript_hook,
|
|
20
|
+
parse_agent_transcript,
|
|
21
|
+
provision_trajectory_plugin,
|
|
22
|
+
trajectory_mode,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _archive_bytes(files: dict[str, str]) -> bytes:
|
|
27
|
+
stream = io.BytesIO()
|
|
28
|
+
with tarfile.open(fileobj=stream, mode="w:gz") as archive:
|
|
29
|
+
for name, content in files.items():
|
|
30
|
+
payload = content.encode("utf-8")
|
|
31
|
+
info = tarfile.TarInfo(name)
|
|
32
|
+
info.size = len(payload)
|
|
33
|
+
archive.addfile(info, io.BytesIO(payload))
|
|
34
|
+
return stream.getvalue()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_trajectory_modes_cover_all_supported_agent_contracts():
|
|
38
|
+
assert trajectory_mode("claude-code") == "official-claude-code-plugin"
|
|
39
|
+
assert trajectory_mode("codex") == "official-codex-plugin"
|
|
40
|
+
assert trajectory_mode("qoder") == "qoder-stop-transcript"
|
|
41
|
+
assert trajectory_mode("codebuddy") == "codebuddy-stop-transcript"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_plugin_provision_is_pinned_hashed_cached_and_credential_free(tmp_path: Path):
|
|
45
|
+
archive = _archive_bytes(
|
|
46
|
+
{
|
|
47
|
+
"langfuse-plugin/plugins/tracing/hooks/hooks.json": '{"hooks": {}}\n',
|
|
48
|
+
"langfuse-plugin/plugins/tracing/dist/index.mjs": "export {};\n",
|
|
49
|
+
}
|
|
50
|
+
)
|
|
51
|
+
calls = []
|
|
52
|
+
|
|
53
|
+
def download(url: str, destination: Path) -> None:
|
|
54
|
+
calls.append(url)
|
|
55
|
+
destination.write_bytes(archive)
|
|
56
|
+
|
|
57
|
+
first = provision_trajectory_plugin(tmp_path, "codex", download=download)
|
|
58
|
+
|
|
59
|
+
assert first.mode == "official-codex-plugin"
|
|
60
|
+
assert first.plugin_path is not None
|
|
61
|
+
assert (first.plugin_path / "plugins" / "tracing" / "hooks" / "hooks.json").is_file()
|
|
62
|
+
metadata = json.loads((first.cache_dir / "metadata.json").read_text(encoding="utf-8"))
|
|
63
|
+
assert metadata["source_ref"]
|
|
64
|
+
assert metadata["archive_sha256"]
|
|
65
|
+
assert metadata["content_sha256"]
|
|
66
|
+
assert "LANGFUSE_SECRET_KEY" not in json.dumps(metadata)
|
|
67
|
+
|
|
68
|
+
second = provision_trajectory_plugin(
|
|
69
|
+
tmp_path,
|
|
70
|
+
"codex",
|
|
71
|
+
download=lambda *_args: pytest.fail("cache hit must not download"),
|
|
72
|
+
)
|
|
73
|
+
assert second.plugin_path == first.plugin_path
|
|
74
|
+
assert len(calls) == 1
|
|
75
|
+
|
|
76
|
+
(first.plugin_path / "plugins" / "tracing" / "dist" / "index.mjs").write_text(
|
|
77
|
+
"tampered\n", encoding="utf-8"
|
|
78
|
+
)
|
|
79
|
+
with pytest.raises(TrajectoryProvisionError, match="integrity"):
|
|
80
|
+
provision_trajectory_plugin(tmp_path, "codex", download=lambda *_args: None)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_qoder_and_codebuddy_transcripts_normalize_messages_tools_and_bounds(tmp_path: Path):
|
|
84
|
+
transcript = tmp_path / "session.jsonl"
|
|
85
|
+
transcript.write_text(
|
|
86
|
+
"\n".join(
|
|
87
|
+
[
|
|
88
|
+
json.dumps({"type": "session_meta", "sessionId": "s-1", "data": {"model": "m"}}),
|
|
89
|
+
json.dumps({"type": "user", "message": {"content": "Please inspect"}}),
|
|
90
|
+
json.dumps(
|
|
91
|
+
{
|
|
92
|
+
"type": "assistant",
|
|
93
|
+
"message": {
|
|
94
|
+
"content": [
|
|
95
|
+
{"type": "text", "text": "I will inspect."},
|
|
96
|
+
{
|
|
97
|
+
"type": "tool_use",
|
|
98
|
+
"name": "read_file",
|
|
99
|
+
"input": {"path": "a.py"},
|
|
100
|
+
},
|
|
101
|
+
]
|
|
102
|
+
},
|
|
103
|
+
}
|
|
104
|
+
),
|
|
105
|
+
json.dumps(
|
|
106
|
+
{
|
|
107
|
+
"type": "user",
|
|
108
|
+
"message": {
|
|
109
|
+
"content": [
|
|
110
|
+
{"type": "tool_result", "tool_use_id": "t-1", "content": "x" * 100}
|
|
111
|
+
]
|
|
112
|
+
},
|
|
113
|
+
}
|
|
114
|
+
),
|
|
115
|
+
]
|
|
116
|
+
)
|
|
117
|
+
+ "\n",
|
|
118
|
+
encoding="utf-8",
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
events = parse_agent_transcript(transcript, max_chars=32)
|
|
122
|
+
|
|
123
|
+
assert [event.kind for event in events] == ["session", "user", "text", "tool", "tool_result"]
|
|
124
|
+
assert events[0].session_id == "s-1"
|
|
125
|
+
assert events[3].name == "read_file"
|
|
126
|
+
assert events[-1].truncated is True
|
|
127
|
+
assert len(events[-1].output) <= 32
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_transcript_hook_is_project_local_and_preserves_existing_settings(tmp_path: Path):
|
|
131
|
+
settings = tmp_path / ".codebuddy" / "settings.json"
|
|
132
|
+
settings.parent.mkdir(parents=True)
|
|
133
|
+
settings.write_text(
|
|
134
|
+
json.dumps({"custom": True, "hooks": {"Stop": [{"custom": True}]}}), encoding="utf-8"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
trajectory_path = install_transcript_hook(tmp_path, "codebuddy")
|
|
138
|
+
|
|
139
|
+
assert trajectory_path == tmp_path / ".comet" / "eval" / "langfuse" / "trajectories"
|
|
140
|
+
hook_path = tmp_path / ".codebuddy" / "hooks" / "langfuse-stop-hook.py"
|
|
141
|
+
assert hook_path.is_file()
|
|
142
|
+
hook_source = hook_path.read_text(encoding="utf-8")
|
|
143
|
+
assert "COMET_EVAL_AGENT_ROLE" in hook_source
|
|
144
|
+
assert "hashlib.sha256" in hook_source
|
|
145
|
+
saved = json.loads(settings.read_text(encoding="utf-8"))
|
|
146
|
+
assert saved["custom"] is True
|
|
147
|
+
assert len(saved["hooks"]["Stop"]) == 2
|
|
148
|
+
assert "LANGFUSE_SECRET_KEY" not in settings.read_text(encoding="utf-8")
|
|
149
|
+
|
|
150
|
+
transcript = tmp_path / "transcript.jsonl"
|
|
151
|
+
transcript.write_text(
|
|
152
|
+
'{"type":"assistant","message":{"content":"done"}}\n',
|
|
153
|
+
encoding="utf-8",
|
|
154
|
+
)
|
|
155
|
+
result = subprocess.run(
|
|
156
|
+
[sys.executable, str(hook_path)],
|
|
157
|
+
input=json.dumps(
|
|
158
|
+
{
|
|
159
|
+
"hook_event_name": "Stop",
|
|
160
|
+
"transcript_path": str(transcript),
|
|
161
|
+
"cwd": str(tmp_path),
|
|
162
|
+
}
|
|
163
|
+
),
|
|
164
|
+
text=True,
|
|
165
|
+
capture_output=True,
|
|
166
|
+
check=False,
|
|
167
|
+
)
|
|
168
|
+
assert result.returncode == 0, result.stderr
|
|
169
|
+
assert list(
|
|
170
|
+
(tmp_path / ".comet" / "eval" / "langfuse" / "trajectories").glob("*.jsonl")
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def test_codex_plugin_is_copied_into_the_project_codex_home_cache(tmp_path: Path):
|
|
175
|
+
plugin = tmp_path / "plugin"
|
|
176
|
+
tracing = plugin / "plugins" / "tracing"
|
|
177
|
+
(tracing / ".codex-plugin").mkdir(parents=True)
|
|
178
|
+
(tracing / ".codex-plugin" / "plugin.json").write_text(
|
|
179
|
+
json.dumps({"version": "0.1.0"}), encoding="utf-8"
|
|
180
|
+
)
|
|
181
|
+
(tracing / "dist").mkdir()
|
|
182
|
+
(tracing / "dist" / "index.mjs").write_text("export {};\n", encoding="utf-8")
|
|
183
|
+
provision = TrajectoryProvision(
|
|
184
|
+
agent="codex",
|
|
185
|
+
mode="official-codex-plugin",
|
|
186
|
+
cache_dir=tmp_path,
|
|
187
|
+
plugin_path=plugin,
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
workspace = tmp_path / "workspace"
|
|
191
|
+
(workspace / ".codex").mkdir(parents=True)
|
|
192
|
+
(workspace / ".codex" / "config.toml").write_text(
|
|
193
|
+
'[features]\nother = true\n\n[profiles.default]\nmodel = "x"\n',
|
|
194
|
+
encoding="utf-8",
|
|
195
|
+
)
|
|
196
|
+
install_codex_plugin_workspace(workspace, provision)
|
|
197
|
+
|
|
198
|
+
installed = (
|
|
199
|
+
workspace
|
|
200
|
+
/ ".codex"
|
|
201
|
+
/ "plugins"
|
|
202
|
+
/ "cache"
|
|
203
|
+
/ "codex-observability-plugin"
|
|
204
|
+
/ "tracing"
|
|
205
|
+
/ "0.1.0"
|
|
206
|
+
)
|
|
207
|
+
assert (installed / "dist" / "index.mjs").is_file()
|
|
208
|
+
config = (workspace / ".codex" / "config.toml").read_text(encoding="utf-8")
|
|
209
|
+
assert "hooks = true" in config
|
|
210
|
+
assert 'plugins."tracing@codex-observability-plugin"' in config
|
|
211
|
+
assert config.index("hooks = true") < config.index("[profiles.default]")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class _Observation:
|
|
215
|
+
trace_id = "trace-1"
|
|
216
|
+
id = "observation-1"
|
|
217
|
+
|
|
218
|
+
def __enter__(self):
|
|
219
|
+
return self
|
|
220
|
+
|
|
221
|
+
def __exit__(self, exc_type, exc, tb):
|
|
222
|
+
return False
|
|
223
|
+
|
|
224
|
+
def update(self, **payload):
|
|
225
|
+
self.output = payload.get("output")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class _Client:
|
|
229
|
+
def __init__(self):
|
|
230
|
+
self.observations = []
|
|
231
|
+
self.scores = []
|
|
232
|
+
|
|
233
|
+
def start_as_current_observation(self, **kwargs):
|
|
234
|
+
self.observations.append(kwargs)
|
|
235
|
+
return _Observation()
|
|
236
|
+
|
|
237
|
+
def create_score(self, **kwargs):
|
|
238
|
+
self.scores.append(kwargs)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def test_reporter_maps_qoder_transcript_to_optional_child_observations(tmp_path: Path):
|
|
242
|
+
transcript = tmp_path / "trajectory.jsonl"
|
|
243
|
+
transcript.write_text(
|
|
244
|
+
json.dumps({"type": "assistant", "sessionId": "s-1", "message": {"content": "done"}})
|
|
245
|
+
+ "\n",
|
|
246
|
+
encoding="utf-8",
|
|
247
|
+
)
|
|
248
|
+
client = _Client()
|
|
249
|
+
result = type("Result", (), {"passed": True, "checks_passed": ["ok"], "checks_failed": []})()
|
|
250
|
+
reporter = LangfuseRunReporter(client)
|
|
251
|
+
|
|
252
|
+
reporter.report_case(
|
|
253
|
+
"task",
|
|
254
|
+
"CONTROL",
|
|
255
|
+
lambda: None,
|
|
256
|
+
lambda: result,
|
|
257
|
+
metadata={"agent": "qoder"},
|
|
258
|
+
trajectory_path=transcript,
|
|
259
|
+
agent="qoder",
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
assert client.observations[1]["as_type"] == "generation"
|
|
263
|
+
assert client.observations[1]["name"].startswith("agent.text/")
|
|
@@ -4,11 +4,17 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import inspect
|
|
6
6
|
import importlib.util
|
|
7
|
+
import os
|
|
7
8
|
import sys
|
|
9
|
+
import subprocess
|
|
8
10
|
from pathlib import Path
|
|
9
11
|
from types import SimpleNamespace
|
|
10
12
|
import json
|
|
11
13
|
|
|
14
|
+
import pytest
|
|
15
|
+
|
|
16
|
+
from scaffold.python.eval_context import resolve_eval_context
|
|
17
|
+
|
|
12
18
|
|
|
13
19
|
def _load_langsmith_conftest():
|
|
14
20
|
eval_root = Path(__file__).resolve().parents[3]
|
|
@@ -95,6 +101,63 @@ def test_langsmith_tracing_auto_builds_default_plugin_cache(monkeypatch, tmp_pat
|
|
|
95
101
|
assert module.os.environ["CC_LANGSMITH_PLUGIN_DIR"] == module._plugin_dir_env_value(plugin_dir)
|
|
96
102
|
|
|
97
103
|
|
|
104
|
+
def test_langsmith_default_plugin_cache_uses_resolved_owner_not_packaged_harness(
|
|
105
|
+
monkeypatch, tmp_path
|
|
106
|
+
):
|
|
107
|
+
skill = tmp_path / "skill"
|
|
108
|
+
owner = tmp_path / "owner"
|
|
109
|
+
harness_cache = tmp_path / "packaged-harness" / ".cache" / "langsmith-cc-plugin"
|
|
110
|
+
skill.mkdir()
|
|
111
|
+
owner.mkdir()
|
|
112
|
+
(skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
|
|
113
|
+
context = resolve_eval_context(skill_path=skill, project_root=owner)
|
|
114
|
+
monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
|
|
115
|
+
module = _load_langsmith_conftest()
|
|
116
|
+
monkeypatch.setattr(module, "DEFAULT_LANGSMITH_PLUGIN_DIR", harness_cache)
|
|
117
|
+
monkeypatch.delenv("CC_LANGSMITH_PLUGIN_DIR", raising=False)
|
|
118
|
+
monkeypatch.setenv("LANGSMITH_TRACING", "true")
|
|
119
|
+
monkeypatch.setenv("TRACE_TO_LANGSMITH", "true")
|
|
120
|
+
calls = []
|
|
121
|
+
|
|
122
|
+
def fake_build(target):
|
|
123
|
+
calls.append(target)
|
|
124
|
+
target.mkdir(parents=True)
|
|
125
|
+
return True
|
|
126
|
+
|
|
127
|
+
monkeypatch.setattr(module, "_build_default_langsmith_plugin", fake_build)
|
|
128
|
+
|
|
129
|
+
resolved = module.provision_langsmith_plugin_dir()
|
|
130
|
+
|
|
131
|
+
expected = owner / ".comet" / "eval" / "cache" / "langsmith" / "langsmith-cc-plugin"
|
|
132
|
+
assert resolved == expected
|
|
133
|
+
assert calls == [expected]
|
|
134
|
+
assert not harness_cache.exists()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_langsmith_default_plugin_cache_rejects_an_owner_cache_link_that_escapes(
|
|
138
|
+
monkeypatch, tmp_path
|
|
139
|
+
):
|
|
140
|
+
skill = tmp_path / "skill"
|
|
141
|
+
owner = tmp_path / "owner"
|
|
142
|
+
outside = tmp_path / "outside"
|
|
143
|
+
skill.mkdir()
|
|
144
|
+
owner.mkdir()
|
|
145
|
+
outside.mkdir()
|
|
146
|
+
(skill / "SKILL.md").write_text("# Demo\n", encoding="utf-8")
|
|
147
|
+
context = resolve_eval_context(skill_path=skill, project_root=owner)
|
|
148
|
+
cache_link = context.artifact_root / "cache"
|
|
149
|
+
cache_link.parent.mkdir(parents=True)
|
|
150
|
+
if os.name == "nt":
|
|
151
|
+
subprocess.run(["cmd", "/c", "mklink", "/J", str(cache_link), str(outside)], check=True)
|
|
152
|
+
else:
|
|
153
|
+
cache_link.symlink_to(outside, target_is_directory=True)
|
|
154
|
+
monkeypatch.setenv("COMET_EVAL_CONTEXT", json.dumps(context.to_payload()))
|
|
155
|
+
module = _load_langsmith_conftest()
|
|
156
|
+
|
|
157
|
+
with pytest.raises(ValueError, match="managed path must stay within its owner root"):
|
|
158
|
+
module._default_langsmith_plugin_dir()
|
|
159
|
+
|
|
160
|
+
|
|
98
161
|
def test_langsmith_task_wrapper_registers_reexported_local_module():
|
|
99
162
|
_load_langsmith_task_tests()
|
|
100
163
|
|
|
@@ -35,6 +35,56 @@ def test_save_artifacts_excludes_nested_git_metadata(tmp_path: Path):
|
|
|
35
35
|
assert not (snapshot / ".git").exists()
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
def test_save_artifacts_redacts_configured_credentials(tmp_path: Path, monkeypatch):
|
|
39
|
+
from conftest import _save_artifacts
|
|
40
|
+
|
|
41
|
+
monkeypatch.setenv("FIXTURE_AGENT_CREDENTIAL", "artifact-secret-value")
|
|
42
|
+
workspace = tmp_path / "workspace"
|
|
43
|
+
workspace.mkdir()
|
|
44
|
+
(workspace / "result.md").write_text(
|
|
45
|
+
"Authorization: Bearer artifact-secret-value\n", encoding="utf-8"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
_save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
|
|
49
|
+
|
|
50
|
+
snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
|
|
51
|
+
saved = (snapshot / "result.md").read_text(encoding="utf-8")
|
|
52
|
+
assert "artifact-secret-value" not in saved
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_save_artifacts_skips_non_text_content_instead_of_persisting_secrets(tmp_path: Path):
|
|
56
|
+
from conftest import _save_artifacts
|
|
57
|
+
|
|
58
|
+
workspace = tmp_path / "workspace"
|
|
59
|
+
workspace.mkdir()
|
|
60
|
+
(workspace / "secret.bin").write_bytes(b"artifact-secret-value\x00\xff")
|
|
61
|
+
|
|
62
|
+
_save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
|
|
63
|
+
|
|
64
|
+
snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
|
|
65
|
+
assert not (snapshot / "secret.bin").exists()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_save_artifacts_uses_selected_agent_root_and_excludes_codebuddy_config(
|
|
69
|
+
tmp_path: Path,
|
|
70
|
+
):
|
|
71
|
+
from conftest import _save_artifacts
|
|
72
|
+
|
|
73
|
+
workspace = tmp_path / "workspace"
|
|
74
|
+
workspace.mkdir()
|
|
75
|
+
(workspace / "result.md").write_text("ok", encoding="utf-8")
|
|
76
|
+
(workspace / ".codebuddy" / "settings.json").parent.mkdir()
|
|
77
|
+
(workspace / ".codebuddy" / "settings.json").write_text(
|
|
78
|
+
'{"env":{"CODEBUDDY_API_KEY":"secret"}}', encoding="utf-8"
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
_save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace, agent="codebuddy")
|
|
82
|
+
|
|
83
|
+
snapshot = tmp_path / "artifacts/comet_full_040_beta_rep1/codebuddy"
|
|
84
|
+
assert (snapshot / "result.md").read_text(encoding="utf-8") == "ok"
|
|
85
|
+
assert not (snapshot / ".codebuddy").exists()
|
|
86
|
+
|
|
87
|
+
|
|
38
88
|
def test_save_artifacts_excludes_controller_cli_snapshot(tmp_path: Path):
|
|
39
89
|
from conftest import _save_artifacts
|
|
40
90
|
|
|
@@ -93,6 +143,97 @@ def test_extract_events_captures_token_usage_and_cost():
|
|
|
93
143
|
assert events["model_usage"]["mimo-v2.5-pro"]["costUSD"] == 0.123456
|
|
94
144
|
|
|
95
145
|
|
|
146
|
+
def test_extract_events_normalizes_codex_turns_and_runtime_skill_evidence():
|
|
147
|
+
stdout = "\n".join(
|
|
148
|
+
[
|
|
149
|
+
json.dumps({"type": "thread.started", "thread_id": "thread-1"}),
|
|
150
|
+
json.dumps(
|
|
151
|
+
{
|
|
152
|
+
"type": "item.completed",
|
|
153
|
+
"item": {
|
|
154
|
+
"type": "command_execution",
|
|
155
|
+
"command": "cat .agents/skills/demo/SKILL.md",
|
|
156
|
+
"aggregated_output": "skill contents",
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
),
|
|
160
|
+
json.dumps(
|
|
161
|
+
{
|
|
162
|
+
"type": "item.completed",
|
|
163
|
+
"item": {
|
|
164
|
+
"type": "file_change",
|
|
165
|
+
"changes": [{"path": "result.md", "kind": "add"}],
|
|
166
|
+
},
|
|
167
|
+
}
|
|
168
|
+
),
|
|
169
|
+
json.dumps(
|
|
170
|
+
{
|
|
171
|
+
"type": "turn.completed",
|
|
172
|
+
"usage": {"input_tokens": 80, "output_tokens": 20},
|
|
173
|
+
"duration_ms": 900,
|
|
174
|
+
}
|
|
175
|
+
),
|
|
176
|
+
]
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
events = extract_events(parse_output(stdout))
|
|
180
|
+
|
|
181
|
+
assert events["role_sessions"]["subject"] == ["thread-1"]
|
|
182
|
+
|
|
183
|
+
assert events["subject_invocations"] == 1
|
|
184
|
+
assert events["num_turns"] == 1
|
|
185
|
+
assert events["input_tokens"] == 80
|
|
186
|
+
assert events["output_tokens"] == 20
|
|
187
|
+
assert events["total_tokens"] == 100
|
|
188
|
+
assert events["duration_seconds"] == 0.9
|
|
189
|
+
assert events["commands_run"] == ["cat .agents/skills/demo/SKILL.md"]
|
|
190
|
+
assert events["files_created"] == ["result.md"]
|
|
191
|
+
assert events["skills_invoked"] == ["demo"]
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def test_extract_events_recognizes_codebuddy_runtime_skill_evidence():
|
|
195
|
+
stdout = json.dumps(
|
|
196
|
+
{
|
|
197
|
+
"type": "assistant",
|
|
198
|
+
"message": {
|
|
199
|
+
"content": [
|
|
200
|
+
{
|
|
201
|
+
"type": "tool_use",
|
|
202
|
+
"name": "Bash",
|
|
203
|
+
"input": {"command": "cat .codebuddy/skills/demo/SKILL.md"},
|
|
204
|
+
}
|
|
205
|
+
]
|
|
206
|
+
},
|
|
207
|
+
}
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
events = extract_events(parse_output(stdout))
|
|
211
|
+
|
|
212
|
+
assert events["skills_invoked"] == ["demo"]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def test_custom_agent_requires_explicit_skill_invocation_events():
|
|
216
|
+
path_only = json.dumps(
|
|
217
|
+
{
|
|
218
|
+
"type": "item.completed",
|
|
219
|
+
"item": {
|
|
220
|
+
"type": "command_execution",
|
|
221
|
+
"command": "cat .agents/skills/demo/SKILL.md",
|
|
222
|
+
},
|
|
223
|
+
}
|
|
224
|
+
)
|
|
225
|
+
inferred = extract_events(parse_output(path_only), agent="fixture-agent")
|
|
226
|
+
assert inferred["skills_invoked"] == []
|
|
227
|
+
assert inferred["skill_invocations"] == []
|
|
228
|
+
|
|
229
|
+
explicit = extract_events(
|
|
230
|
+
parse_output(json.dumps({"type": "skill_invocation", "skill": "demo"})),
|
|
231
|
+
agent="fixture-agent",
|
|
232
|
+
)
|
|
233
|
+
assert explicit["skills_invoked"] == ["demo"]
|
|
234
|
+
assert explicit["skill_invocations"] == ["demo"]
|
|
235
|
+
|
|
236
|
+
|
|
96
237
|
def test_extract_events_accumulates_duration_across_results():
|
|
97
238
|
stdout = "\n".join(
|
|
98
239
|
[
|
|
@@ -304,6 +445,27 @@ def test_save_raw_writes_utf8_output(tmp_path: Path):
|
|
|
304
445
|
assert "stderr 中文" == stderr_path.read_text(encoding="utf-8")
|
|
305
446
|
|
|
306
447
|
|
|
448
|
+
def test_saved_eval_outputs_redact_configured_credentials(tmp_path: Path, monkeypatch):
|
|
449
|
+
monkeypatch.setenv("FIXTURE_AGENT_API_KEY", "fixture-secret-value")
|
|
450
|
+
save_raw(
|
|
451
|
+
tmp_path,
|
|
452
|
+
"COMET_FULL_040_BETA",
|
|
453
|
+
1,
|
|
454
|
+
"Authorization: Bearer fixture-secret-value x-api-key: fixture-secret-value",
|
|
455
|
+
"token=fixture-secret-value",
|
|
456
|
+
)
|
|
457
|
+
save_events(tmp_path, "COMET_FULL_040_BETA", 1, {"token": "fixture-secret-value"})
|
|
458
|
+
save_report(tmp_path, "COMET_FULL_040_BETA", 1, {"credential": "fixture-secret-value"})
|
|
459
|
+
|
|
460
|
+
for path in (
|
|
461
|
+
tmp_path / "raw/COMET_FULL_040_BETA_rep1_stdout.json",
|
|
462
|
+
tmp_path / "raw/COMET_FULL_040_BETA_rep1_stderr.txt",
|
|
463
|
+
tmp_path / "events/COMET_FULL_040_BETA_rep1.json",
|
|
464
|
+
tmp_path / "reports/COMET_FULL_040_BETA_rep1_report.json",
|
|
465
|
+
):
|
|
466
|
+
assert "fixture-secret-value" not in path.read_text(encoding="utf-8")
|
|
467
|
+
|
|
468
|
+
|
|
307
469
|
def test_save_artifacts_preserve_stable_treatment_filenames(tmp_path: Path):
|
|
308
470
|
save_events(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
|
|
309
471
|
save_raw(tmp_path, "COMET_FULL_040_BETA", 2, "{}", "stderr")
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Tests for normalizing manifest-authored tasks into the runner contract."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
6
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_inline_task_uses_generic_environment_and_exposes_normalized_contract(tmp_path: Path):
|
|
10
|
+
package = tmp_path / "my-skill"
|
|
11
|
+
(package / "comet").mkdir(parents=True)
|
|
12
|
+
manifest_path = package / "comet" / "eval.yaml"
|
|
13
|
+
manifest_path.write_text(
|
|
14
|
+
"""
|
|
15
|
+
apiVersion: comet.eval/v1alpha1
|
|
16
|
+
kind: SkillEvalManifest
|
|
17
|
+
metadata:
|
|
18
|
+
name: my-skill
|
|
19
|
+
skill:
|
|
20
|
+
name: my-skill
|
|
21
|
+
source: ..
|
|
22
|
+
evaluation:
|
|
23
|
+
tasks:
|
|
24
|
+
- name: inline-task
|
|
25
|
+
prompt: Create result.md.
|
|
26
|
+
expect:
|
|
27
|
+
files: [result.md]
|
|
28
|
+
rubric:
|
|
29
|
+
- The result is useful.
|
|
30
|
+
""",
|
|
31
|
+
encoding="utf-8",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
tasks = load_manifest_tasks(load_eval_manifest(manifest_path))
|
|
35
|
+
|
|
36
|
+
assert len(tasks) == 1
|
|
37
|
+
task = tasks[0]
|
|
38
|
+
assert task.name == "inline-task"
|
|
39
|
+
assert task.render_prompt() == "Create result.md."
|
|
40
|
+
assert task.environment_dir.is_dir()
|
|
41
|
+
assert task.config.evaluation.expected_artifacts == ["result.md"]
|
|
42
|
+
assert task.config.evaluation.rubric_criteria == ["The result is useful."]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_source_task_is_loaded_without_changing_its_task_contract(tmp_path: Path):
|
|
46
|
+
package = tmp_path / "my-skill"
|
|
47
|
+
source = package / "tasks" / "source-task"
|
|
48
|
+
(source / "environment").mkdir(parents=True)
|
|
49
|
+
(source / "task.toml").write_text(
|
|
50
|
+
'[metadata]\nname = "source-task"\n\n[environment]\ndockerfile = "Dockerfile"\n',
|
|
51
|
+
encoding="utf-8",
|
|
52
|
+
)
|
|
53
|
+
(source / "instruction.md").write_text("Source prompt\n", encoding="utf-8")
|
|
54
|
+
(source / "environment" / "Dockerfile").write_text("FROM alpine\n", encoding="utf-8")
|
|
55
|
+
manifest_path = package / "comet" / "eval.yaml"
|
|
56
|
+
manifest_path.parent.mkdir(parents=True)
|
|
57
|
+
manifest_path.write_text(
|
|
58
|
+
"""
|
|
59
|
+
apiVersion: comet.eval/v1alpha1
|
|
60
|
+
kind: SkillEvalManifest
|
|
61
|
+
metadata:
|
|
62
|
+
name: my-skill
|
|
63
|
+
skill:
|
|
64
|
+
name: my-skill
|
|
65
|
+
source: ..
|
|
66
|
+
evaluation:
|
|
67
|
+
tasks:
|
|
68
|
+
- source: ./tasks/source-task
|
|
69
|
+
""",
|
|
70
|
+
encoding="utf-8",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
tasks = load_manifest_tasks(load_eval_manifest(manifest_path))
|
|
74
|
+
|
|
75
|
+
assert [task.name for task in tasks] == ["source-task"]
|
|
76
|
+
assert tasks[0].render_prompt() == "Source prompt\n"
|