@rpamis/comet 0.4.0-beta.17 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -24
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +78 -78
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +109 -109
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +130 -130
- package/assets/skills/comet/scripts/comet-hook-router.mjs +115 -115
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +85 -85
- package/assets/skills/comet/scripts/comet-runtime.mjs +188 -187
- package/assets/skills/comet/scripts/comet-state.mjs +117 -117
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +71 -71
- package/assets/skills/comet-build/SKILL.md +15 -56
- package/assets/skills/comet-classic/SKILL.md +14 -8
- package/assets/skills/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills/comet-classic/reference/scripts.md +3 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-native/SKILL.md +13 -10
- package/assets/skills/comet-native/reference/artifacts.md +3 -2
- package/assets/skills/comet-native/reference/commands.md +1 -1
- package/assets/skills/comet-native/reference/recovery.md +1 -1
- package/assets/skills/comet-native/reference/workspace.md +4 -4
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +97 -97
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +146 -146
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +100 -100
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +85 -85
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +115 -115
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +118 -118
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +107 -107
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +154 -154
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +133 -133
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +109 -109
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +87 -87
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +109 -109
- package/assets/skills/comet-open/SKILL.md +24 -5
- package/assets/skills-zh/comet-build/SKILL.md +15 -56
- package/assets/skills-zh/comet-classic/SKILL.md +14 -8
- package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +1 -1
- package/assets/skills-zh/comet-classic/reference/scripts.md +3 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-native/SKILL.md +13 -10
- package/assets/skills-zh/comet-native/reference/artifacts.md +3 -2
- package/assets/skills-zh/comet-native/reference/commands.md +1 -1
- package/assets/skills-zh/comet-native/reference/recovery.md +1 -1
- package/assets/skills-zh/comet-native/reference/workspace.md +4 -4
- package/assets/skills-zh/comet-open/SKILL.md +24 -5
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +2 -1
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +4 -1
- package/dist/app/commands/update.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +1 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +21 -1
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-help.js +3 -3
- package/dist/domains/comet-native/native-cli-help.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +10 -5
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +41 -2
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +85 -5
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.d.ts +3 -2
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-continuation.js +63 -1
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.d.ts +5 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-runtime.js +110 -4
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-state.js +13 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -1
- package/dist/domains/comet-native/native-portable-status.d.ts +3 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-status.js +9 -1
- package/dist/domains/comet-native/native-portable-status.js.map +1 -1
- package/dist/domains/comet-native/native-portable-types.d.ts +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-portable-types.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +2 -1
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace-preparation.js +192 -133
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +20 -0
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +11 -0
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +1 -0
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +304 -55
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +12 -5
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.js +12 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -1
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-BL17BIb8.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-XWLMOyd4.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-4drjXzHQ.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-CQXqBSqQ.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-pMreYMW7.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-vQyyh3eG.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-B_DDS2IK.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-Cug8J1Qa.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-IQKP-hgj.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-CvvRO4J2.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/{demo-CeIDHUFJ.js → demo-BfLsKrIL.js} +26 -5
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-CN_LYqvT.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-CaWEk3PM.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-BNyOfAhu.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/{index-BFwKBVTJ.css → index-DhhbseFJ.css} +1 -1
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-BK07mNG0.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-mWnmaWfZ.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-BPy-P7xx.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-3Jq9c_lo.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-kQyEW-4G.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-CyyurUpv.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-CpayUMjH.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-Bi8YpfnA.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-aHrrtV-b.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-RVQbcySh.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +2 -2
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +1 -1
- package/dist/domains/skill/platform-install.js +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +2 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +1 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +2 -2
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +42 -2
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +13 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +7 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +28 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +17 -0
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +216 -14
- package/eval/pyproject.toml +5 -0
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_workflow.py +6 -1
- package/eval/scaffold/python/validation/rubric.py +16 -1
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +8 -3
- package/dist/domains/dashboard/web/assets/arc-CkuYweSh.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR--tTGrKam.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-DLKYOixs.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-Cg__FSMr.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BUkV_rb7.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-CsC-pvu9.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-DeOQo7bP.js +0 -4
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-IQd_jQwB.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-Dllkdt0m.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-DK36WkNN.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-iqUnMYEh.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-Bc64lzy8.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-UfOqef9j.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-D-u6Aauc.js +0 -292
- package/dist/domains/dashboard/web/assets/index-COZ_03uK.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-COBBjtVH.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CJro_XKk.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-Wx60WwHe.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BiMsRFbc.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BrURMY0R.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-BpEE-yEg.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-DGB16bvR.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-DMzzwDVX.js +0 -7
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
# ./docker.sh build <directory> [--force]
|
|
8
8
|
# ./docker.sh run <directory> <command...>
|
|
9
9
|
# ./docker.sh run-python <directory> <script.py> [args...]
|
|
10
|
-
# ./docker.sh run-
|
|
10
|
+
# ./docker.sh run-agent <directory> <prompt> --agent AGENT [--model MODEL] [--timeout SECONDS]
|
|
11
11
|
|
|
12
12
|
set -euo pipefail
|
|
13
13
|
|
|
@@ -40,6 +40,11 @@ IMAGE_PREFIX="skillbench"
|
|
|
40
40
|
|
|
41
41
|
# Claude Code version (configurable via env var, defaults to "latest")
|
|
42
42
|
CLAUDE_CODE_VERSION="${BENCH_CC_VERSION:-latest}"
|
|
43
|
+
CODEX_VERSION="${BENCH_CODEX_VERSION:-latest}"
|
|
44
|
+
QODER_VERSION="${BENCH_QODER_VERSION:-latest}"
|
|
45
|
+
CODEBUDDY_VERSION="${BENCH_CODEBUDDY_VERSION:-latest}"
|
|
46
|
+
DEFAULT_AGENT="${BENCH_EVAL_AGENT:-claude-code}"
|
|
47
|
+
LANGFUSE_ENABLED="${TRACE_TO_LANGFUSE:-false}"
|
|
43
48
|
|
|
44
49
|
# Cross-platform timeout command (macOS uses gtimeout from coreutils)
|
|
45
50
|
TIMEOUT_CMD=""
|
|
@@ -52,11 +57,35 @@ fi
|
|
|
52
57
|
# API keys to pass through to containers
|
|
53
58
|
ENV_KEYS=(
|
|
54
59
|
OPENAI_API_KEY
|
|
60
|
+
OPENAI_BASE_URL
|
|
61
|
+
OPENAI_MODEL
|
|
62
|
+
CODEX_API_KEY
|
|
63
|
+
CODEX_BASE_URL
|
|
64
|
+
CODEX_MODEL
|
|
65
|
+
QODER_PERSONAL_ACCESS_TOKEN
|
|
66
|
+
QODER_BASE_URL
|
|
67
|
+
QODER_MODEL
|
|
68
|
+
CODEBUDDY_API_KEY
|
|
69
|
+
CODEBUDDY_AUTH_TOKEN
|
|
70
|
+
CODEBUDDY_BASE_URL
|
|
71
|
+
CODEBUDDY_MODEL
|
|
72
|
+
CODEBUDDY_SMALL_FAST_MODEL
|
|
73
|
+
CODEBUDDY_BIG_SLOW_MODEL
|
|
74
|
+
CODEBUDDY_CODE_SUBAGENT_MODEL
|
|
75
|
+
CODEBUDDY_CUSTOM_HEADERS
|
|
76
|
+
CODEBUDDY_INTERNET_ENVIRONMENT
|
|
55
77
|
ANTHROPIC_API_KEY
|
|
56
78
|
LANGSMITH_API_KEY
|
|
57
79
|
LANGSMITH_PROJECT
|
|
58
80
|
LANGSMITH_TRACING
|
|
59
81
|
LANGSMITH_ENDPOINT
|
|
82
|
+
LANGFUSE_PUBLIC_KEY
|
|
83
|
+
LANGFUSE_SECRET_KEY
|
|
84
|
+
LANGFUSE_BASE_URL
|
|
85
|
+
LANGFUSE_TRACING_ENVIRONMENT
|
|
86
|
+
TRACE_TO_LANGFUSE
|
|
87
|
+
LANGFUSE_CODEX_TAGS
|
|
88
|
+
LANGFUSE_CODEX_METADATA
|
|
60
89
|
TAVILY_API_KEY
|
|
61
90
|
# Claude Code LangSmith tracing (official langsmith-tracing plugin)
|
|
62
91
|
TRACE_TO_LANGSMITH
|
|
@@ -81,8 +110,64 @@ ENV_KEYS=(
|
|
|
81
110
|
ANTHROPIC_DEFAULT_OPUS_MODEL_NAME
|
|
82
111
|
ANTHROPIC_DEFAULT_SONNET_MODEL_NAME
|
|
83
112
|
CLAUDE_CODE_SUBAGENT_MODEL
|
|
113
|
+
COMET_EVAL_CUSTOM_AGENT_ID
|
|
114
|
+
COMET_EVAL_CUSTOM_EXECUTABLE
|
|
115
|
+
COMET_EVAL_CUSTOM_CREDENTIALS
|
|
116
|
+
COMET_EVAL_CUSTOM_MODEL
|
|
117
|
+
COMET_EVAL_CUSTOM_BASE_URL
|
|
118
|
+
COMET_EVAL_CUSTOM_MODEL_ENV
|
|
119
|
+
COMET_EVAL_CUSTOM_BASE_URL_ENV
|
|
120
|
+
COMET_EVAL_CUSTOM_INSTALL_KIND
|
|
121
|
+
COMET_EVAL_CUSTOM_INSTALL_PACKAGE
|
|
122
|
+
COMET_EVAL_CUSTOM_INSTALL_VERSION
|
|
84
123
|
)
|
|
85
124
|
|
|
125
|
+
validate_agent() {
|
|
126
|
+
case "$1" in
|
|
127
|
+
claude-code|codex|qoder|codebuddy) return 0 ;;
|
|
128
|
+
*)
|
|
129
|
+
if [[ "${COMET_EVAL_CUSTOM_AGENT_ID:-}" == "$1" && -n "${COMET_EVAL_CUSTOM_EXECUTABLE:-}" ]]; then
|
|
130
|
+
return 0
|
|
131
|
+
fi
|
|
132
|
+
echo "ERROR: Unsupported evaluation agent: $1" >&2
|
|
133
|
+
return 1
|
|
134
|
+
;;
|
|
135
|
+
esac
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
agent_executable() {
|
|
139
|
+
case "$1" in
|
|
140
|
+
claude-code) printf '%s' "claude" ;;
|
|
141
|
+
codex) printf '%s' "codex" ;;
|
|
142
|
+
qoder) printf '%s' "qodercli" ;;
|
|
143
|
+
codebuddy) printf '%s' "codebuddy" ;;
|
|
144
|
+
*)
|
|
145
|
+
validate_agent "$1" || return 1
|
|
146
|
+
printf '%s' "${COMET_EVAL_CUSTOM_EXECUTABLE}"
|
|
147
|
+
;;
|
|
148
|
+
esac
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
agent_version() {
|
|
152
|
+
case "$1" in
|
|
153
|
+
claude-code) printf '%s' "$CLAUDE_CODE_VERSION" ;;
|
|
154
|
+
codex) printf '%s' "$CODEX_VERSION" ;;
|
|
155
|
+
qoder) printf '%s' "$QODER_VERSION" ;;
|
|
156
|
+
codebuddy) printf '%s' "$CODEBUDDY_VERSION" ;;
|
|
157
|
+
*)
|
|
158
|
+
validate_agent "$1" || return 1
|
|
159
|
+
local custom_identity
|
|
160
|
+
custom_identity=$(printf '%s\n%s\n%s\n%s' \
|
|
161
|
+
"${COMET_EVAL_CUSTOM_EXECUTABLE:-}" \
|
|
162
|
+
"${COMET_EVAL_CUSTOM_INSTALL_KIND:-none}" \
|
|
163
|
+
"${COMET_EVAL_CUSTOM_INSTALL_PACKAGE:-}" \
|
|
164
|
+
"${COMET_EVAL_CUSTOM_INSTALL_VERSION:-latest}")
|
|
165
|
+
custom_identity=$(sha256_text "$custom_identity") || return 1
|
|
166
|
+
printf 'custom-%s' "${custom_identity:0:12}"
|
|
167
|
+
;;
|
|
168
|
+
esac
|
|
169
|
+
}
|
|
170
|
+
|
|
86
171
|
# =============================================================================
|
|
87
172
|
# DOCKER CHECKS
|
|
88
173
|
# =============================================================================
|
|
@@ -135,7 +220,7 @@ get_dockerfile_hash() {
|
|
|
135
220
|
}
|
|
136
221
|
|
|
137
222
|
# Get image name for a directory (based on build context hash + Claude Code version)
|
|
138
|
-
|
|
223
|
+
get_base_image_name() {
|
|
139
224
|
local dir="$1"
|
|
140
225
|
local hash
|
|
141
226
|
hash=$(get_dockerfile_hash "$dir") || return 1
|
|
@@ -147,6 +232,36 @@ get_image_name() {
|
|
|
147
232
|
fi
|
|
148
233
|
}
|
|
149
234
|
|
|
235
|
+
get_image_name() {
|
|
236
|
+
local dir="$1"
|
|
237
|
+
local agent="${2:-$DEFAULT_AGENT}"
|
|
238
|
+
validate_agent "$agent" || return 1
|
|
239
|
+
local base
|
|
240
|
+
base=$(get_base_image_name "$dir") || return 1
|
|
241
|
+
local trace_suffix=""
|
|
242
|
+
if [[ "${TRACE_TO_LANGFUSE:-}" == "true" ]]; then
|
|
243
|
+
trace_suffix="-lf"
|
|
244
|
+
fi
|
|
245
|
+
if [[ "$agent" == "claude-code" && "$trace_suffix" == "" ]]; then
|
|
246
|
+
echo "$base"
|
|
247
|
+
else
|
|
248
|
+
local overlay_file overlay_hash
|
|
249
|
+
overlay_file="$SCRIPT_DIR/../docker/agent-overlay.Dockerfile"
|
|
250
|
+
if [[ -f "$overlay_file" ]]; then
|
|
251
|
+
local overlay_contents
|
|
252
|
+
overlay_contents=$(cat "$overlay_file")
|
|
253
|
+
if command -v md5 &> /dev/null; then
|
|
254
|
+
overlay_hash=$(printf '%s' "$overlay_contents" | md5 -q | cut -c1-8)
|
|
255
|
+
else
|
|
256
|
+
overlay_hash=$(printf '%s' "$overlay_contents" | md5sum | cut -c1-8)
|
|
257
|
+
fi
|
|
258
|
+
else
|
|
259
|
+
overlay_hash="missing"
|
|
260
|
+
fi
|
|
261
|
+
echo "${base}-${agent}-$(agent_version "$agent")-o${overlay_hash}${trace_suffix}"
|
|
262
|
+
fi
|
|
263
|
+
}
|
|
264
|
+
|
|
150
265
|
# Check if image exists
|
|
151
266
|
image_exists() {
|
|
152
267
|
local image_name="$1"
|
|
@@ -167,6 +282,8 @@ sha256_text() {
|
|
|
167
282
|
resolve_runtime_image() {
|
|
168
283
|
local dir="$1"
|
|
169
284
|
local expected_image_id="${2:-}"
|
|
285
|
+
local agent="${3:-$DEFAULT_AGENT}"
|
|
286
|
+
validate_agent "$agent" || return 1
|
|
170
287
|
if [[ -n "$expected_image_id" ]]; then
|
|
171
288
|
if [[ ! "$expected_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
|
172
289
|
echo "ERROR: Invalid immutable Docker image ID" >&2
|
|
@@ -181,7 +298,7 @@ resolve_runtime_image() {
|
|
|
181
298
|
fi
|
|
182
299
|
|
|
183
300
|
local image_name image_id
|
|
184
|
-
image_name=$(docker_build "$dir") || return 1
|
|
301
|
+
image_name=$(docker_build "$dir" "" "$agent") || return 1
|
|
185
302
|
image_id=$(docker image inspect --format '{{.Id}}' "$image_name") || return 1
|
|
186
303
|
if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
|
187
304
|
echo "ERROR: Docker returned an invalid immutable image ID" >&2
|
|
@@ -192,24 +309,43 @@ resolve_runtime_image() {
|
|
|
192
309
|
|
|
193
310
|
docker_execution_identity() {
|
|
194
311
|
local dir="$1"
|
|
195
|
-
local
|
|
196
|
-
|
|
312
|
+
local agent="${2:-$DEFAULT_AGENT}"
|
|
313
|
+
validate_agent "$agent" || return 1
|
|
314
|
+
local image_name image_id repo_digests agent_version_output
|
|
315
|
+
image_name=$(docker_build "$dir" "" "$agent") || return 1
|
|
197
316
|
image_id=$(docker image inspect --format '{{.Id}}' "$image_name") || return 1
|
|
198
317
|
if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
|
199
318
|
echo "ERROR: Docker returned an invalid immutable image ID" >&2
|
|
200
319
|
return 1
|
|
201
320
|
fi
|
|
202
321
|
repo_digests=$(docker image inspect --format '{{json .RepoDigests}}' "$image_id") || return 1
|
|
203
|
-
|
|
204
|
-
|
|
322
|
+
local executable
|
|
323
|
+
executable=$(agent_executable "$agent")
|
|
324
|
+
if [[ "$agent" == "claude-code" && "${TRACE_TO_LANGFUSE:-}" != "true" ]]; then
|
|
325
|
+
agent_version_output=$(docker run --rm "$image_id" claude --version 2>/dev/null)
|
|
326
|
+
else
|
|
327
|
+
agent_version_output=$(docker run --rm "$image_id" "$executable" --version 2>/dev/null)
|
|
328
|
+
fi || {
|
|
329
|
+
echo "ERROR: Cannot verify $agent CLI version in benchmark image" >&2
|
|
205
330
|
return 1
|
|
206
331
|
}
|
|
207
|
-
|
|
332
|
+
local agent_version_json
|
|
333
|
+
agent_version_json=$(printf '%s' "$agent_version_output" | node -e '
|
|
334
|
+
let value = "";
|
|
335
|
+
process.stdin.setEncoding("utf8");
|
|
336
|
+
process.stdin.on("data", chunk => value += chunk);
|
|
337
|
+
process.stdin.on("end", () => process.stdout.write(JSON.stringify(value.trim())));
|
|
338
|
+
') || return 1
|
|
339
|
+
printf '{"schema":"comet.eval.execution-identity.v1","runtime_image_id":"%s","agent":"%s","agent_cli":"%s","agent_cli_version":%s,"image_id_hash":"sha256:%s","image_repo_digests_hash":"sha256:%s","image_ref_hash":"sha256:%s","agent_cli_version_hash":"sha256:%s","claude_tool_version_hash":"sha256:%s"}\n' \
|
|
208
340
|
"$image_id" \
|
|
341
|
+
"$agent" \
|
|
342
|
+
"$executable" \
|
|
343
|
+
"$agent_version_json" \
|
|
209
344
|
"$(sha256_text "$image_id")" \
|
|
210
345
|
"$(sha256_text "$repo_digests")" \
|
|
211
346
|
"$(sha256_text "$image_name")" \
|
|
212
|
-
"$(sha256_text "$
|
|
347
|
+
"$(sha256_text "$agent_version_output")" \
|
|
348
|
+
"$(sha256_text "$agent_version_output")"
|
|
213
349
|
}
|
|
214
350
|
|
|
215
351
|
# =============================================================================
|
|
@@ -222,6 +358,12 @@ docker_execution_identity() {
|
|
|
222
358
|
docker_build() {
|
|
223
359
|
local dir="$1"
|
|
224
360
|
local force="${2:-}"
|
|
361
|
+
local agent="${3:-$DEFAULT_AGENT}"
|
|
362
|
+
if [[ "$force" == "--agent" ]]; then
|
|
363
|
+
agent="${3:-$DEFAULT_AGENT}"
|
|
364
|
+
force=""
|
|
365
|
+
fi
|
|
366
|
+
validate_agent "$agent" || return 1
|
|
225
367
|
|
|
226
368
|
local dockerfile="$dir/Dockerfile"
|
|
227
369
|
if [[ ! -f "$dockerfile" ]]; then
|
|
@@ -234,8 +376,9 @@ docker_build() {
|
|
|
234
376
|
return 1
|
|
235
377
|
fi
|
|
236
378
|
|
|
237
|
-
local image_name
|
|
238
|
-
|
|
379
|
+
local base_image_name image_name
|
|
380
|
+
base_image_name=$(get_base_image_name "$dir") || return 1
|
|
381
|
+
image_name=$(get_image_name "$dir" "$agent") || return 1
|
|
239
382
|
|
|
240
383
|
# Check cache unless forced
|
|
241
384
|
if [[ "$force" != "--force" ]] && image_exists "$image_name"; then
|
|
@@ -243,17 +386,51 @@ docker_build() {
|
|
|
243
386
|
return 0
|
|
244
387
|
fi
|
|
245
388
|
|
|
246
|
-
# Build image (pass Claude Code version as build arg)
|
|
389
|
+
# Build the task image first (pass Claude Code version as build arg).
|
|
247
390
|
local windir windockerfile
|
|
248
391
|
windir=$(_winpath "$dir")
|
|
249
392
|
windockerfile=$(_winpath "$dockerfile")
|
|
393
|
+
if [[ "$agent" == "claude-code" && "${TRACE_TO_LANGFUSE:-}" != "true" ]]; then
|
|
394
|
+
if docker build -t "$image_name" \
|
|
395
|
+
--build-arg CLAUDE_CODE_VERSION="$CLAUDE_CODE_VERSION" \
|
|
396
|
+
-f "$windockerfile" "$windir" >&2; then
|
|
397
|
+
echo "$image_name"
|
|
398
|
+
return 0
|
|
399
|
+
else
|
|
400
|
+
echo "ERROR: Build failed" >&2
|
|
401
|
+
return 1
|
|
402
|
+
fi
|
|
403
|
+
fi
|
|
404
|
+
|
|
405
|
+
if [[ "$force" == "--force" ]] || ! image_exists "$base_image_name"; then
|
|
406
|
+
if ! docker build -t "$base_image_name" \
|
|
407
|
+
--build-arg CLAUDE_CODE_VERSION="$CLAUDE_CODE_VERSION" \
|
|
408
|
+
-f "$windockerfile" "$windir" >&2; then
|
|
409
|
+
echo "ERROR: Base image build failed" >&2
|
|
410
|
+
return 1
|
|
411
|
+
fi
|
|
412
|
+
fi
|
|
413
|
+
|
|
414
|
+
local overlay_file overlay_context
|
|
415
|
+
overlay_file="$SCRIPT_DIR/../docker/agent-overlay.Dockerfile"
|
|
416
|
+
overlay_context=$(_winpath "$(cd "$(dirname "$overlay_file")" && pwd)")
|
|
250
417
|
if docker build -t "$image_name" \
|
|
251
|
-
--build-arg
|
|
252
|
-
-
|
|
418
|
+
--build-arg BASE_IMAGE="$base_image_name" \
|
|
419
|
+
--build-arg EVAL_AGENT="$agent" \
|
|
420
|
+
--build-arg CODEX_VERSION="$CODEX_VERSION" \
|
|
421
|
+
--build-arg QODER_VERSION="$QODER_VERSION" \
|
|
422
|
+
--build-arg CODEBUDDY_VERSION="$CODEBUDDY_VERSION" \
|
|
423
|
+
--build-arg CUSTOM_AGENT_ID="${COMET_EVAL_CUSTOM_AGENT_ID:-}" \
|
|
424
|
+
--build-arg CUSTOM_AGENT_EXECUTABLE="${COMET_EVAL_CUSTOM_EXECUTABLE:-}" \
|
|
425
|
+
--build-arg CUSTOM_INSTALL_KIND="${COMET_EVAL_CUSTOM_INSTALL_KIND:-none}" \
|
|
426
|
+
--build-arg CUSTOM_INSTALL_PACKAGE="${COMET_EVAL_CUSTOM_INSTALL_PACKAGE:-}" \
|
|
427
|
+
--build-arg CUSTOM_INSTALL_VERSION="${COMET_EVAL_CUSTOM_INSTALL_VERSION:-latest}" \
|
|
428
|
+
--build-arg LANGFUSE_ENABLED="${TRACE_TO_LANGFUSE:-false}" \
|
|
429
|
+
-f "$(_winpath "$overlay_file")" "$overlay_context" >&2; then
|
|
253
430
|
echo "$image_name"
|
|
254
431
|
return 0
|
|
255
432
|
else
|
|
256
|
-
echo "ERROR:
|
|
433
|
+
echo "ERROR: Agent overlay build failed" >&2
|
|
257
434
|
return 1
|
|
258
435
|
fi
|
|
259
436
|
}
|
|
@@ -268,11 +445,46 @@ build_env_args() {
|
|
|
268
445
|
ENV_ARGS=()
|
|
269
446
|
for key in "${ENV_KEYS[@]}"; do
|
|
270
447
|
if [[ -n "${!key:-}" ]]; then
|
|
271
|
-
|
|
448
|
+
# Let Docker inherit the value from this process instead of putting
|
|
449
|
+
# the secret literal in the docker command's argument list.
|
|
450
|
+
ENV_ARGS+=("-e" "$key")
|
|
451
|
+
fi
|
|
452
|
+
done
|
|
453
|
+
if [[ -n "${COMET_EVAL_CUSTOM_CREDENTIALS:-}" ]]; then
|
|
454
|
+
local custom_key
|
|
455
|
+
IFS=',' read -r -a custom_keys <<< "${COMET_EVAL_CUSTOM_CREDENTIALS}"
|
|
456
|
+
for custom_key in "${custom_keys[@]}"; do
|
|
457
|
+
if [[ "$custom_key" =~ ^[A-Z][A-Z0-9_]{1,63}$ ]] && [[ -n "${!custom_key:-}" ]]; then
|
|
458
|
+
ENV_ARGS+=("-e" "$custom_key")
|
|
459
|
+
fi
|
|
460
|
+
done
|
|
461
|
+
fi
|
|
462
|
+
local metadata_key custom_routing_key
|
|
463
|
+
for metadata_key in COMET_EVAL_CUSTOM_MODEL_ENV COMET_EVAL_CUSTOM_BASE_URL_ENV; do
|
|
464
|
+
custom_routing_key="${!metadata_key:-}"
|
|
465
|
+
if [[ "$custom_routing_key" =~ ^[A-Z][A-Z0-9_]{1,63}$ ]] && [[ -n "${!custom_routing_key:-}" ]]; then
|
|
466
|
+
ENV_ARGS+=("-e" "$custom_routing_key")
|
|
272
467
|
fi
|
|
273
468
|
done
|
|
274
469
|
}
|
|
275
470
|
|
|
471
|
+
build_agent_runtime_mount_args() {
|
|
472
|
+
RUNTIME_CONFIG_MOUNT_ARGS=()
|
|
473
|
+
RUNTIME_CONFIG_TMPFS_ARGS=(
|
|
474
|
+
--tmpfs "//home/agent/.codex:rw,nosuid,nodev"
|
|
475
|
+
--tmpfs "//home/agent/.qoder:rw,nosuid,nodev"
|
|
476
|
+
--tmpfs "//home/agent/.codebuddy:rw,nosuid,nodev"
|
|
477
|
+
)
|
|
478
|
+
local shell_host
|
|
479
|
+
shell_host=$(_winpath "$SCRIPT_DIR")
|
|
480
|
+
RUNTIME_CONFIG_MOUNT_ARGS=("-v" "$shell_host://opt/scaffold-shell:ro")
|
|
481
|
+
# Agent auth/config roots are container-local. The generated files use
|
|
482
|
+
# env_key/apiKeyHelper references, never credential literals.
|
|
483
|
+
ENV_ARGS+=("-e" "CODEX_HOME=/home/agent/.codex")
|
|
484
|
+
ENV_ARGS+=("-e" "QODER_CONFIG_DIR=/home/agent/.qoder")
|
|
485
|
+
ENV_ARGS+=("-e" "COMET_EVAL_CODEBUDDY_CONFIG_DIR=/home/agent/.codebuddy")
|
|
486
|
+
}
|
|
487
|
+
|
|
276
488
|
# Build mount + CLI args for the official LangSmith Claude Code plugin.
|
|
277
489
|
# Populates PLUGIN_MOUNT_ARGS (docker -v) and PLUGIN_CLI_ARGS (claude --plugin-dir).
|
|
278
490
|
# Activated only when tracing is on and a prebuilt plugin dir is provided via
|
|
@@ -297,6 +509,30 @@ build_plugin_args() {
|
|
|
297
509
|
PLUGIN_CLI_ARGS=("--plugin-dir" "//opt/langsmith-cc-plugin")
|
|
298
510
|
}
|
|
299
511
|
|
|
512
|
+
# Build mount + CLI args for the isolated official Langfuse Agent plugin.
|
|
513
|
+
# The host-side Langfuse suite provisions this directory. No user-global Agent
|
|
514
|
+
# configuration is touched and credentials remain environment-only.
|
|
515
|
+
build_langfuse_plugin_args() {
|
|
516
|
+
LANGFUSE_PLUGIN_MOUNT_ARGS=()
|
|
517
|
+
LANGFUSE_PLUGIN_CLI_ARGS=()
|
|
518
|
+
if [[ "${TRACE_TO_LANGFUSE:-}" != "true" ]]; then
|
|
519
|
+
return 0
|
|
520
|
+
fi
|
|
521
|
+
if [[ -z "${LANGFUSE_TRAJECTORY_PLUGIN_DIR:-}" ]]; then
|
|
522
|
+
return 0
|
|
523
|
+
fi
|
|
524
|
+
if [[ ! -d "${LANGFUSE_TRAJECTORY_PLUGIN_DIR}" ]]; then
|
|
525
|
+
echo "WARN: LANGFUSE_TRAJECTORY_PLUGIN_DIR is not a directory; skipping official trajectory plugin" >&2
|
|
526
|
+
return 0
|
|
527
|
+
fi
|
|
528
|
+
local plugin_host
|
|
529
|
+
plugin_host=$(_winpath "${LANGFUSE_TRAJECTORY_PLUGIN_DIR}")
|
|
530
|
+
LANGFUSE_PLUGIN_MOUNT_ARGS=("-v" "$plugin_host://opt/comet-langfuse-plugin:ro")
|
|
531
|
+
if [[ "${1:-}" == "claude-code" ]]; then
|
|
532
|
+
LANGFUSE_PLUGIN_CLI_ARGS=("--plugin-dir" "//opt/comet-langfuse-plugin")
|
|
533
|
+
fi
|
|
534
|
+
}
|
|
535
|
+
|
|
300
536
|
# Keep controller-built oracle snapshots immutable inside both agent and validator containers.
|
|
301
537
|
# Nested read-only binds override the writable /workspace parent mount when present.
|
|
302
538
|
build_trusted_oracle_mount_args() {
|
|
@@ -343,6 +579,54 @@ docker_run() {
|
|
|
343
579
|
"${cmd[@]}"
|
|
344
580
|
}
|
|
345
581
|
|
|
582
|
+
# Run one user-authored validation command inside the task container.
|
|
583
|
+
# Usage: docker_run_command <directory> --timeout SECONDS -- <shell command>
|
|
584
|
+
docker_run_command() {
|
|
585
|
+
local dir="$1"
|
|
586
|
+
shift
|
|
587
|
+
local timeout="120"
|
|
588
|
+
local command=()
|
|
589
|
+
while [[ $# -gt 0 ]]; do
|
|
590
|
+
case "$1" in
|
|
591
|
+
--timeout)
|
|
592
|
+
timeout="$2"
|
|
593
|
+
shift 2
|
|
594
|
+
;;
|
|
595
|
+
--)
|
|
596
|
+
shift
|
|
597
|
+
command=("$@")
|
|
598
|
+
break
|
|
599
|
+
;;
|
|
600
|
+
*)
|
|
601
|
+
command+=("$1")
|
|
602
|
+
shift
|
|
603
|
+
;;
|
|
604
|
+
esac
|
|
605
|
+
done
|
|
606
|
+
if [[ ${#command[@]} -eq 0 ]]; then
|
|
607
|
+
echo "ERROR: validation command is required" >&2
|
|
608
|
+
return 2
|
|
609
|
+
fi
|
|
610
|
+
|
|
611
|
+
local image_name
|
|
612
|
+
image_name=$(docker_build "$dir") || return 1
|
|
613
|
+
build_trusted_oracle_mount_args "$dir"
|
|
614
|
+
|
|
615
|
+
local windir
|
|
616
|
+
windir=$(_winpath "$dir")
|
|
617
|
+
local docker_command=(docker run --rm
|
|
618
|
+
-v "$windir://workspace"
|
|
619
|
+
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"}
|
|
620
|
+
-w //workspace
|
|
621
|
+
"$image_name"
|
|
622
|
+
bash -lc "${command[*]}")
|
|
623
|
+
if [[ -n "$TIMEOUT_CMD" ]]; then
|
|
624
|
+
$TIMEOUT_CMD "$timeout" "${docker_command[@]}"
|
|
625
|
+
else
|
|
626
|
+
"${docker_command[@]}"
|
|
627
|
+
fi
|
|
628
|
+
}
|
|
629
|
+
|
|
346
630
|
# Run Python script in Docker
|
|
347
631
|
# Usage: docker_run_python <directory> <script.py> [args...]
|
|
348
632
|
docker_run_python() {
|
|
@@ -403,22 +687,17 @@ docker_run_claude() {
|
|
|
403
687
|
image_id=$(resolve_runtime_image "$dir" "$expected_image_id") || return 1
|
|
404
688
|
|
|
405
689
|
build_env_args
|
|
690
|
+
build_agent_runtime_mount_args
|
|
406
691
|
build_plugin_args
|
|
692
|
+
build_langfuse_plugin_args "claude-code"
|
|
407
693
|
build_trusted_oracle_mount_args "$dir"
|
|
408
694
|
|
|
409
|
-
local cmd=(
|
|
410
|
-
claude -p "$prompt"
|
|
411
|
-
--dangerously-skip-permissions
|
|
412
|
-
--output-format stream-json
|
|
413
|
-
--verbose
|
|
414
|
-
)
|
|
415
|
-
|
|
695
|
+
local cmd=(bash //opt/scaffold-shell/run-agent-runtime.sh claude-code "$model" "$prompt" --)
|
|
416
696
|
if [[ ${#PLUGIN_CLI_ARGS[@]} -gt 0 ]]; then
|
|
417
697
|
cmd+=("${PLUGIN_CLI_ARGS[@]}")
|
|
418
698
|
fi
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
cmd+=(--model "$model")
|
|
699
|
+
if [[ ${#LANGFUSE_PLUGIN_CLI_ARGS[@]} -gt 0 ]]; then
|
|
700
|
+
cmd+=("${LANGFUSE_PLUGIN_CLI_ARGS[@]}")
|
|
422
701
|
fi
|
|
423
702
|
|
|
424
703
|
local windir
|
|
@@ -427,8 +706,11 @@ docker_run_claude() {
|
|
|
427
706
|
if [[ -n "$TIMEOUT_CMD" ]]; then
|
|
428
707
|
$TIMEOUT_CMD "$timeout" docker run --rm \
|
|
429
708
|
-v "$windir://workspace" \
|
|
709
|
+
"${RUNTIME_CONFIG_MOUNT_ARGS[@]}" \
|
|
710
|
+
"${RUNTIME_CONFIG_TMPFS_ARGS[@]}" \
|
|
430
711
|
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
|
|
431
712
|
${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
|
|
713
|
+
${LANGFUSE_PLUGIN_MOUNT_ARGS[@]+"${LANGFUSE_PLUGIN_MOUNT_ARGS[@]}"} \
|
|
432
714
|
-w //workspace \
|
|
433
715
|
"${ENV_ARGS[@]}" \
|
|
434
716
|
"$image_id" \
|
|
@@ -436,8 +718,11 @@ docker_run_claude() {
|
|
|
436
718
|
else
|
|
437
719
|
docker run --rm \
|
|
438
720
|
-v "$windir://workspace" \
|
|
721
|
+
"${RUNTIME_CONFIG_MOUNT_ARGS[@]}" \
|
|
722
|
+
"${RUNTIME_CONFIG_TMPFS_ARGS[@]}" \
|
|
439
723
|
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
|
|
440
724
|
${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
|
|
725
|
+
${LANGFUSE_PLUGIN_MOUNT_ARGS[@]+"${LANGFUSE_PLUGIN_MOUNT_ARGS[@]}"} \
|
|
441
726
|
-w //workspace \
|
|
442
727
|
"${ENV_ARGS[@]}" \
|
|
443
728
|
"$image_id" \
|
|
@@ -445,6 +730,93 @@ docker_run_claude() {
|
|
|
445
730
|
fi
|
|
446
731
|
}
|
|
447
732
|
|
|
733
|
+
# Run a selected evaluation agent CLI in Docker.
|
|
734
|
+
# Usage: docker_run_agent <directory> <prompt> --agent AGENT [--model MODEL] [--timeout SECONDS]
|
|
735
|
+
docker_run_agent() {
|
|
736
|
+
local dir="$1"
|
|
737
|
+
local prompt="$2"
|
|
738
|
+
shift 2
|
|
739
|
+
|
|
740
|
+
local agent="$DEFAULT_AGENT"
|
|
741
|
+
local model=""
|
|
742
|
+
local timeout="300"
|
|
743
|
+
local expected_image_id=""
|
|
744
|
+
|
|
745
|
+
while [[ $# -gt 0 ]]; do
|
|
746
|
+
case "$1" in
|
|
747
|
+
--agent)
|
|
748
|
+
agent="$2"
|
|
749
|
+
shift 2
|
|
750
|
+
;;
|
|
751
|
+
--model)
|
|
752
|
+
model="$2"
|
|
753
|
+
shift 2
|
|
754
|
+
;;
|
|
755
|
+
--timeout)
|
|
756
|
+
timeout="$2"
|
|
757
|
+
shift 2
|
|
758
|
+
;;
|
|
759
|
+
--image-id)
|
|
760
|
+
expected_image_id="$2"
|
|
761
|
+
shift 2
|
|
762
|
+
;;
|
|
763
|
+
*)
|
|
764
|
+
shift
|
|
765
|
+
;;
|
|
766
|
+
esac
|
|
767
|
+
done
|
|
768
|
+
validate_agent "$agent" || return 1
|
|
769
|
+
|
|
770
|
+
local image_id
|
|
771
|
+
image_id=$(resolve_runtime_image "$dir" "$expected_image_id" "$agent") || return 1
|
|
772
|
+
|
|
773
|
+
build_env_args
|
|
774
|
+
build_agent_runtime_mount_args
|
|
775
|
+
if [[ "$agent" == "claude-code" ]]; then
|
|
776
|
+
build_plugin_args
|
|
777
|
+
else
|
|
778
|
+
PLUGIN_MOUNT_ARGS=()
|
|
779
|
+
PLUGIN_CLI_ARGS=()
|
|
780
|
+
fi
|
|
781
|
+
build_langfuse_plugin_args "$agent"
|
|
782
|
+
build_trusted_oracle_mount_args "$dir"
|
|
783
|
+
AGENT_COMMAND=(bash //opt/scaffold-shell/run-agent-runtime.sh "$agent" "$model" "$prompt" --)
|
|
784
|
+
if [[ "$agent" == "claude-code" && ${#PLUGIN_CLI_ARGS[@]} -gt 0 ]]; then
|
|
785
|
+
AGENT_COMMAND+=("${PLUGIN_CLI_ARGS[@]}")
|
|
786
|
+
fi
|
|
787
|
+
if [[ "$agent" == "claude-code" && ${#LANGFUSE_PLUGIN_CLI_ARGS[@]} -gt 0 ]]; then
|
|
788
|
+
AGENT_COMMAND+=("${LANGFUSE_PLUGIN_CLI_ARGS[@]}")
|
|
789
|
+
fi
|
|
790
|
+
|
|
791
|
+
local windir
|
|
792
|
+
windir=$(_winpath "$dir")
|
|
793
|
+
if [[ -n "$TIMEOUT_CMD" ]]; then
|
|
794
|
+
$TIMEOUT_CMD "$timeout" docker run --rm \
|
|
795
|
+
-v "$windir://workspace" \
|
|
796
|
+
"${RUNTIME_CONFIG_MOUNT_ARGS[@]}" \
|
|
797
|
+
"${RUNTIME_CONFIG_TMPFS_ARGS[@]}" \
|
|
798
|
+
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
|
|
799
|
+
${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
|
|
800
|
+
${LANGFUSE_PLUGIN_MOUNT_ARGS[@]+"${LANGFUSE_PLUGIN_MOUNT_ARGS[@]}"} \
|
|
801
|
+
-w //workspace \
|
|
802
|
+
"${ENV_ARGS[@]}" \
|
|
803
|
+
"$image_id" \
|
|
804
|
+
"${AGENT_COMMAND[@]}"
|
|
805
|
+
else
|
|
806
|
+
docker run --rm \
|
|
807
|
+
-v "$windir://workspace" \
|
|
808
|
+
"${RUNTIME_CONFIG_MOUNT_ARGS[@]}" \
|
|
809
|
+
"${RUNTIME_CONFIG_TMPFS_ARGS[@]}" \
|
|
810
|
+
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
|
|
811
|
+
${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
|
|
812
|
+
${LANGFUSE_PLUGIN_MOUNT_ARGS[@]+"${LANGFUSE_PLUGIN_MOUNT_ARGS[@]}"} \
|
|
813
|
+
-w //workspace \
|
|
814
|
+
"${ENV_ARGS[@]}" \
|
|
815
|
+
"$image_id" \
|
|
816
|
+
"${AGENT_COMMAND[@]}"
|
|
817
|
+
fi
|
|
818
|
+
}
|
|
819
|
+
|
|
448
820
|
# Run the multi-turn interactive claude driver in Docker.
|
|
449
821
|
# Usage: docker_run_claude_loop <directory> <prompt> [--max-turns N] [--model MODEL]
|
|
450
822
|
# Mounts the scaffold shell dir so run-claude-loop.sh is available inside.
|
|
@@ -473,15 +845,16 @@ docker_run_claude_loop() {
|
|
|
473
845
|
done
|
|
474
846
|
|
|
475
847
|
local image_id
|
|
476
|
-
image_id=$(resolve_runtime_image "$dir" "$expected_image_id") || return 1
|
|
848
|
+
image_id=$(resolve_runtime_image "$dir" "$expected_image_id" "claude-code") || return 1
|
|
477
849
|
|
|
478
850
|
build_env_args
|
|
851
|
+
build_agent_runtime_mount_args
|
|
479
852
|
build_plugin_args
|
|
853
|
+
build_langfuse_plugin_args "claude-code"
|
|
480
854
|
build_trusted_oracle_mount_args "$dir"
|
|
481
855
|
|
|
482
|
-
local windir
|
|
856
|
+
local windir
|
|
483
857
|
windir=$(_winpath "$dir")
|
|
484
|
-
shell_dir=$(_winpath "$SCRIPT_DIR")
|
|
485
858
|
local container_name
|
|
486
859
|
container_name="comet-eval-loop-$(sha256_text "$dir" | cut -c1-24)"
|
|
487
860
|
|
|
@@ -494,14 +867,85 @@ docker_run_claude_loop() {
|
|
|
494
867
|
# at /opt/scaffold-shell/ inside the container.
|
|
495
868
|
docker run --rm --name "$container_name" \
|
|
496
869
|
-v "$windir://workspace" \
|
|
870
|
+
"${RUNTIME_CONFIG_MOUNT_ARGS[@]}" \
|
|
871
|
+
"${RUNTIME_CONFIG_TMPFS_ARGS[@]}" \
|
|
872
|
+
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
|
|
873
|
+
${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
|
|
874
|
+
${LANGFUSE_PLUGIN_MOUNT_ARGS[@]+"${LANGFUSE_PLUGIN_MOUNT_ARGS[@]}"} \
|
|
875
|
+
-w //workspace \
|
|
876
|
+
"${ENV_ARGS[@]}" \
|
|
877
|
+
"$image_id" \
|
|
878
|
+
bash //opt/scaffold-shell/run-claude-loop.sh "$prompt" \
|
|
879
|
+
${PLUGIN_CLI_ARGS[@]+"${PLUGIN_CLI_ARGS[@]}"} \
|
|
880
|
+
${LANGFUSE_PLUGIN_CLI_ARGS[@]+"${LANGFUSE_PLUGIN_CLI_ARGS[@]}"} \
|
|
881
|
+
"${loop_args[@]}"
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
# Run the shared multi-turn driver with a selected agent CLI.
|
|
885
|
+
docker_run_agent_loop() {
|
|
886
|
+
local dir="$1"
|
|
887
|
+
local prompt="$2"
|
|
888
|
+
shift 2
|
|
889
|
+
|
|
890
|
+
local agent="$DEFAULT_AGENT"
|
|
891
|
+
local expected_image_id=""
|
|
892
|
+
local loop_args=()
|
|
893
|
+
while [[ $# -gt 0 ]]; do
|
|
894
|
+
case "$1" in
|
|
895
|
+
--agent)
|
|
896
|
+
agent="$2"
|
|
897
|
+
shift 2
|
|
898
|
+
;;
|
|
899
|
+
--image-id)
|
|
900
|
+
expected_image_id="$2"
|
|
901
|
+
shift 2
|
|
902
|
+
;;
|
|
903
|
+
--max-turns|--model|--simulator-prompt-file|--decision-reply|--decision-reply-step|--continue-prompt|--decision-pattern|--fresh-resume-marker)
|
|
904
|
+
loop_args+=("$1" "$2")
|
|
905
|
+
shift 2
|
|
906
|
+
;;
|
|
907
|
+
*)
|
|
908
|
+
loop_args+=("$1")
|
|
909
|
+
shift
|
|
910
|
+
;;
|
|
911
|
+
esac
|
|
912
|
+
done
|
|
913
|
+
validate_agent "$agent" || return 1
|
|
914
|
+
|
|
915
|
+
local image_id
|
|
916
|
+
image_id=$(resolve_runtime_image "$dir" "$expected_image_id" "$agent") || return 1
|
|
917
|
+
|
|
918
|
+
build_env_args
|
|
919
|
+
build_agent_runtime_mount_args
|
|
920
|
+
if [[ "$agent" == "claude-code" ]]; then
|
|
921
|
+
build_plugin_args
|
|
922
|
+
else
|
|
923
|
+
PLUGIN_MOUNT_ARGS=()
|
|
924
|
+
PLUGIN_CLI_ARGS=()
|
|
925
|
+
fi
|
|
926
|
+
build_langfuse_plugin_args "$agent"
|
|
927
|
+
build_trusted_oracle_mount_args "$dir"
|
|
928
|
+
|
|
929
|
+
local windir
|
|
930
|
+
windir=$(_winpath "$dir")
|
|
931
|
+
local container_name
|
|
932
|
+
container_name="comet-eval-agent-loop-$(sha256_text "$dir" | cut -c1-24)"
|
|
933
|
+
docker rm -f "$container_name" &> /dev/null || true
|
|
934
|
+
|
|
935
|
+
docker run --rm --name "$container_name" \
|
|
936
|
+
-v "$windir://workspace" \
|
|
937
|
+
"${RUNTIME_CONFIG_MOUNT_ARGS[@]}" \
|
|
938
|
+
"${RUNTIME_CONFIG_TMPFS_ARGS[@]}" \
|
|
497
939
|
${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
|
|
498
|
-
-v "$shell_dir://opt/scaffold-shell:ro" \
|
|
499
940
|
${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
|
|
941
|
+
${LANGFUSE_PLUGIN_MOUNT_ARGS[@]+"${LANGFUSE_PLUGIN_MOUNT_ARGS[@]}"} \
|
|
500
942
|
-w //workspace \
|
|
501
943
|
"${ENV_ARGS[@]}" \
|
|
502
944
|
"$image_id" \
|
|
503
945
|
bash //opt/scaffold-shell/run-claude-loop.sh "$prompt" \
|
|
946
|
+
--agent "$agent" \
|
|
504
947
|
${PLUGIN_CLI_ARGS[@]+"${PLUGIN_CLI_ARGS[@]}"} \
|
|
948
|
+
${LANGFUSE_PLUGIN_CLI_ARGS[@]+"${LANGFUSE_PLUGIN_CLI_ARGS[@]}"} \
|
|
505
949
|
"${loop_args[@]}"
|
|
506
950
|
}
|
|
507
951
|
|
|
@@ -512,6 +956,13 @@ cleanup_claude_loop() {
|
|
|
512
956
|
docker rm -f "$container_name" &> /dev/null || true
|
|
513
957
|
}
|
|
514
958
|
|
|
959
|
+
cleanup_agent_loop() {
|
|
960
|
+
local dir="$1"
|
|
961
|
+
local container_name
|
|
962
|
+
container_name="comet-eval-agent-loop-$(sha256_text "$dir" | cut -c1-24)"
|
|
963
|
+
docker rm -f "$container_name" &> /dev/null || true
|
|
964
|
+
}
|
|
965
|
+
|
|
515
966
|
# =============================================================================
|
|
516
967
|
# CLI MODE
|
|
517
968
|
# =============================================================================
|
|
@@ -530,18 +981,35 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
|
530
981
|
;;
|
|
531
982
|
build)
|
|
532
983
|
dir="${1:-}"
|
|
533
|
-
force="
|
|
984
|
+
force=""
|
|
985
|
+
agent="$DEFAULT_AGENT"
|
|
534
986
|
if [[ -z "$dir" ]]; then
|
|
535
987
|
die "Usage: $0 build <directory> [--force]"
|
|
536
988
|
fi
|
|
537
|
-
|
|
989
|
+
shift
|
|
990
|
+
while [[ $# -gt 0 ]]; do
|
|
991
|
+
case "$1" in
|
|
992
|
+
--force) force="--force"; shift ;;
|
|
993
|
+
--agent) agent="${2:-}"; shift 2 ;;
|
|
994
|
+
*) shift ;;
|
|
995
|
+
esac
|
|
996
|
+
done
|
|
997
|
+
docker_build "$(realpath "$dir")" "$force" "$agent"
|
|
538
998
|
;;
|
|
539
999
|
execution-identity)
|
|
540
1000
|
dir="${1:-}"
|
|
541
1001
|
if [[ -z "$dir" ]]; then
|
|
542
1002
|
die "Usage: $0 execution-identity <directory>"
|
|
543
1003
|
fi
|
|
544
|
-
|
|
1004
|
+
shift
|
|
1005
|
+
agent="$DEFAULT_AGENT"
|
|
1006
|
+
while [[ $# -gt 0 ]]; do
|
|
1007
|
+
case "$1" in
|
|
1008
|
+
--agent) agent="${2:-}"; shift 2 ;;
|
|
1009
|
+
*) shift ;;
|
|
1010
|
+
esac
|
|
1011
|
+
done
|
|
1012
|
+
docker_execution_identity "$(realpath "$dir")" "$agent"
|
|
545
1013
|
;;
|
|
546
1014
|
run)
|
|
547
1015
|
dir="${1:-}"
|
|
@@ -551,6 +1019,14 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
|
551
1019
|
shift
|
|
552
1020
|
docker_run "$(realpath "$dir")" "$@"
|
|
553
1021
|
;;
|
|
1022
|
+
run-command)
|
|
1023
|
+
dir="${1:-}"
|
|
1024
|
+
if [[ -z "$dir" ]]; then
|
|
1025
|
+
die "Usage: $0 run-command <directory> --timeout SECONDS -- <command>"
|
|
1026
|
+
fi
|
|
1027
|
+
shift
|
|
1028
|
+
docker_run_command "$(realpath "$dir")" "$@"
|
|
1029
|
+
;;
|
|
554
1030
|
run-python)
|
|
555
1031
|
dir="${1:-}"
|
|
556
1032
|
script="${2:-}"
|
|
@@ -578,6 +1054,15 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
|
578
1054
|
shift 2
|
|
579
1055
|
docker_run_claude "$(realpath "$dir")" "$prompt" "$@"
|
|
580
1056
|
;;
|
|
1057
|
+
run-agent)
|
|
1058
|
+
dir="${1:-}"
|
|
1059
|
+
prompt="${2:-}"
|
|
1060
|
+
if [[ -z "$dir" || -z "$prompt" ]]; then
|
|
1061
|
+
die "Usage: $0 run-agent <directory> <prompt> --agent AGENT [--model MODEL] [--timeout SECONDS]"
|
|
1062
|
+
fi
|
|
1063
|
+
shift 2
|
|
1064
|
+
docker_run_agent "$(realpath "$dir")" "$prompt" "$@"
|
|
1065
|
+
;;
|
|
581
1066
|
run-claude-loop)
|
|
582
1067
|
dir="${1:-}"
|
|
583
1068
|
prompt="${2:-}"
|
|
@@ -594,6 +1079,22 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
|
|
594
1079
|
fi
|
|
595
1080
|
cleanup_claude_loop "$(realpath "$dir")"
|
|
596
1081
|
;;
|
|
1082
|
+
run-agent-loop)
|
|
1083
|
+
dir="${1:-}"
|
|
1084
|
+
prompt="${2:-}"
|
|
1085
|
+
if [[ -z "$dir" || -z "$prompt" ]]; then
|
|
1086
|
+
die "Usage: $0 run-agent-loop <directory> <prompt> --agent AGENT [--max-turns N]"
|
|
1087
|
+
fi
|
|
1088
|
+
shift 2
|
|
1089
|
+
docker_run_agent_loop "$(realpath "$dir")" "$prompt" "$@"
|
|
1090
|
+
;;
|
|
1091
|
+
cleanup-agent-loop)
|
|
1092
|
+
dir="${1:-}"
|
|
1093
|
+
if [[ -z "$dir" ]]; then
|
|
1094
|
+
die "Usage: $0 cleanup-agent-loop <directory>"
|
|
1095
|
+
fi
|
|
1096
|
+
cleanup_agent_loop "$(realpath "$dir")"
|
|
1097
|
+
;;
|
|
597
1098
|
help|*)
|
|
598
1099
|
cat <<EOF
|
|
599
1100
|
Docker utilities for skill benchmarks
|