@rpamis/comet 0.4.0-beta.16 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +85 -56
- package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.html +8 -0
- package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.json +286 -0
- package/assets/manifest.json +26 -17
- package/assets/skills/comet/SKILL.md +1 -1
- package/assets/skills/comet/agents/openai.yaml +3 -0
- package/assets/skills/comet/rules/comet-phase-guard.en.md +1 -1
- package/assets/skills/comet/rules/comet-phase-guard.md +1 -1
- package/assets/skills/comet/rules/comet-workflow-guard.en.md +2 -2
- package/assets/skills/comet/rules/comet-workflow-guard.md +2 -2
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +77 -77
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +107 -107
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +128 -128
- package/assets/skills/comet/scripts/comet-hook-router.mjs +143 -141
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +100 -100
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +118 -118
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +95 -95
- package/assets/skills/comet-any/SKILL.md +2 -1
- package/assets/skills/comet-any/agents/openai.yaml +5 -0
- package/assets/skills/comet-archive/SKILL.md +7 -6
- package/assets/skills/comet-archive/agents/openai.yaml +5 -0
- package/assets/skills/comet-build/SKILL.md +28 -68
- package/assets/skills/comet-build/agents/openai.yaml +5 -0
- package/assets/skills/comet-classic/SKILL.md +28 -22
- package/assets/skills/comet-classic/agents/openai.yaml +3 -0
- package/assets/skills/{comet → comet-classic}/reference/auto-transition.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/comet-yaml-fields.md +3 -3
- package/assets/skills/{comet → comet-classic}/reference/context-recovery.md +5 -5
- package/assets/skills/{comet → comet-classic}/reference/debug-gate.md +2 -2
- package/assets/skills/{comet → comet-classic}/reference/decision-point.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/dirty-worktree.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/file-structure.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/intent-frame.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/scripts.md +4 -2
- package/assets/skills/{comet → comet-classic}/reference/subagent-dispatch.md +1 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +7 -6
- package/assets/skills/comet-design/agents/openai.yaml +5 -0
- package/assets/skills/comet-hotfix/SKILL.md +10 -9
- package/assets/skills/comet-hotfix/agents/openai.yaml +5 -0
- package/assets/skills/comet-native/SKILL.md +60 -171
- package/assets/skills/comet-native/agents/openai.yaml +3 -0
- package/assets/skills/comet-native/reference/artifacts.md +24 -89
- package/assets/skills/comet-native/reference/clarification.md +54 -29
- package/assets/skills/comet-native/reference/commands.md +30 -173
- package/assets/skills/comet-native/reference/recovery.md +38 -88
- package/assets/skills/comet-native/reference/workspace.md +23 -0
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +184 -152
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +214 -158
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +132 -123
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +134 -124
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +165 -158
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +183 -152
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +152 -147
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +247 -202
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +150 -143
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +128 -121
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +133 -151
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +147 -140
- package/assets/skills/comet-open/SKILL.md +31 -11
- package/assets/skills/comet-open/agents/openai.yaml +5 -0
- package/assets/skills/comet-tweak/SKILL.md +11 -10
- package/assets/skills/comet-tweak/agents/openai.yaml +5 -0
- package/assets/skills/comet-verify/SKILL.md +9 -8
- package/assets/skills/comet-verify/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet/SKILL.md +1 -1
- package/assets/skills-zh/comet/agents/openai.yaml +3 -0
- package/assets/skills-zh/comet-any/SKILL.md +2 -1
- package/assets/skills-zh/comet-any/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-archive/SKILL.md +7 -6
- package/assets/skills-zh/comet-archive/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +28 -68
- package/assets/skills-zh/comet-build/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-classic/SKILL.md +29 -23
- package/assets/skills-zh/comet-classic/agents/openai.yaml +3 -0
- package/assets/skills-zh/{comet → comet-classic}/reference/auto-transition.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/comet-yaml-fields.md +3 -3
- package/assets/skills-zh/{comet → comet-classic}/reference/context-recovery.md +5 -5
- package/assets/skills-zh/{comet → comet-classic}/reference/debug-gate.md +2 -2
- package/assets/skills-zh/{comet → comet-classic}/reference/decision-point.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/dirty-worktree.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/file-structure.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/intent-frame.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/scripts.md +4 -2
- package/assets/skills-zh/{comet → comet-classic}/reference/subagent-dispatch.md +1 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +7 -6
- package/assets/skills-zh/comet-design/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-hotfix/SKILL.md +10 -9
- package/assets/skills-zh/comet-hotfix/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-native/SKILL.md +60 -171
- package/assets/skills-zh/comet-native/agents/openai.yaml +3 -0
- package/assets/skills-zh/comet-native/reference/artifacts.md +24 -89
- package/assets/skills-zh/comet-native/reference/clarification.md +40 -15
- package/assets/skills-zh/comet-native/reference/commands.md +30 -173
- package/assets/skills-zh/comet-native/reference/recovery.md +36 -86
- package/assets/skills-zh/comet-native/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-open/SKILL.md +31 -11
- package/assets/skills-zh/comet-open/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-tweak/SKILL.md +11 -10
- package/assets/skills-zh/comet-tweak/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-verify/SKILL.md +9 -8
- package/assets/skills-zh/comet-verify/agents/openai.yaml +5 -0
- package/bin/fast-runtime-router.js +0 -4
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/doctor.d.ts.map +1 -1
- package/dist/app/commands/doctor.js +8 -0
- package/dist/app/commands/doctor.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +11 -2
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +13 -3
- package/dist/app/commands/update.js.map +1 -1
- package/dist/config/repository-layout.json +0 -8
- package/dist/domains/bundle/bundle-platform.d.ts.map +1 -1
- package/dist/domains/bundle/bundle-platform.js +8 -0
- package/dist/domains/bundle/bundle-platform.js.map +1 -1
- package/dist/domains/bundle/types.d.ts +1 -1
- package/dist/domains/bundle/types.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +3 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +32 -3
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-entry/project-activation.d.ts.map +1 -1
- package/dist/domains/comet-entry/project-activation.js +2 -1
- package/dist/domains/comet-entry/project-activation.js.map +1 -1
- package/dist/domains/comet-entry/resume-probe.d.ts.map +1 -1
- package/dist/domains/comet-entry/resume-probe.js +48 -4
- package/dist/domains/comet-entry/resume-probe.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-acceptance.d.ts +4 -3
- package/dist/domains/comet-native/native-acceptance.d.ts.map +1 -1
- package/dist/domains/comet-native/native-acceptance.js +23 -18
- package/dist/domains/comet-native/native-acceptance.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +204 -61
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-archive-transaction.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-transaction.js +28 -23
- package/dist/domains/comet-native/native-archive-transaction.js.map +1 -1
- package/dist/domains/comet-native/native-archive.d.ts +1 -0
- package/dist/domains/comet-native/native-archive.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive.js +56 -12
- package/dist/domains/comet-native/native-archive.js.map +1 -1
- package/dist/domains/comet-native/native-bounded-file.d.ts +32 -2
- package/dist/domains/comet-native/native-bounded-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-bounded-file.js +87 -6
- package/dist/domains/comet-native/native-bounded-file.js.map +1 -1
- package/dist/domains/comet-native/native-build-evidence.d.ts.map +1 -1
- package/dist/domains/comet-native/native-build-evidence.js +23 -1
- package/dist/domains/comet-native/native-build-evidence.js.map +1 -1
- package/dist/domains/comet-native/native-change.d.ts +1 -0
- package/dist/domains/comet-native/native-change.d.ts.map +1 -1
- package/dist/domains/comet-native/native-change.js +96 -8
- package/dist/domains/comet-native/native-change.js.map +1 -1
- package/dist/domains/comet-native/native-check-executor.d.ts +36 -0
- package/dist/domains/comet-native/native-check-executor.d.ts.map +1 -0
- package/dist/domains/comet-native/native-check-executor.js +132 -0
- package/dist/domains/comet-native/native-check-executor.js.map +1 -0
- package/dist/domains/comet-native/native-check-receipt-model.d.ts +9 -0
- package/dist/domains/comet-native/native-check-receipt-model.d.ts.map +1 -1
- package/dist/domains/comet-native/native-check-receipt-model.js +70 -8
- package/dist/domains/comet-native/native-check-receipt-model.js.map +1 -1
- package/dist/domains/comet-native/native-check-receipt-storage.d.ts.map +1 -1
- package/dist/domains/comet-native/native-check-receipt-storage.js +9 -8
- package/dist/domains/comet-native/native-check-receipt-storage.js.map +1 -1
- package/dist/domains/comet-native/native-check-receipt.d.ts.map +1 -1
- package/dist/domains/comet-native/native-check-receipt.js +123 -60
- package/dist/domains/comet-native/native-check-receipt.js.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-command.js +0 -6
- package/dist/domains/comet-native/native-checkpoint-command.js.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-storage.d.ts.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-storage.js +27 -19
- package/dist/domains/comet-native/native-checkpoint-storage.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts +6 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -0
- package/dist/domains/comet-native/native-cli-help.js +192 -0
- package/dist/domains/comet-native/native-cli-help.js.map +1 -0
- package/dist/domains/comet-native/native-cli-shared.d.ts +2 -2
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +46 -24
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-cli.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli.js +18 -11
- package/dist/domains/comet-native/native-cli.js.map +1 -1
- package/dist/domains/comet-native/native-config.d.ts +2 -1
- package/dist/domains/comet-native/native-config.d.ts.map +1 -1
- package/dist/domains/comet-native/native-config.js +2 -1
- package/dist/domains/comet-native/native-config.js.map +1 -1
- package/dist/domains/comet-native/native-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-continuation.js +175 -147
- package/dist/domains/comet-native/native-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-diagnostics.d.ts.map +1 -1
- package/dist/domains/comet-native/native-diagnostics.js +96 -13
- package/dist/domains/comet-native/native-diagnostics.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +332 -0
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-doctor.d.ts +1 -0
- package/dist/domains/comet-native/native-doctor.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor.js +116 -22
- package/dist/domains/comet-native/native-doctor.js.map +1 -1
- package/dist/domains/comet-native/native-evidence-projection.d.ts +50 -0
- package/dist/domains/comet-native/native-evidence-projection.d.ts.map +1 -0
- package/dist/domains/comet-native/native-evidence-projection.js +284 -0
- package/dist/domains/comet-native/native-evidence-projection.js.map +1 -0
- package/dist/domains/comet-native/native-evidence-retention.d.ts.map +1 -1
- package/dist/domains/comet-native/native-evidence-retention.js +14 -12
- package/dist/domains/comet-native/native-evidence-retention.js.map +1 -1
- package/dist/domains/comet-native/native-evidence-storage.d.ts.map +1 -1
- package/dist/domains/comet-native/native-evidence-storage.js +14 -11
- package/dist/domains/comet-native/native-evidence-storage.js.map +1 -1
- package/dist/domains/comet-native/native-findings.d.ts.map +1 -1
- package/dist/domains/comet-native/native-findings.js +18 -0
- package/dist/domains/comet-native/native-findings.js.map +1 -1
- package/dist/domains/comet-native/native-git-provenance.d.ts +16 -0
- package/dist/domains/comet-native/native-git-provenance.d.ts.map +1 -0
- package/dist/domains/comet-native/native-git-provenance.js +171 -0
- package/dist/domains/comet-native/native-git-provenance.js.map +1 -0
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +199 -19
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-init-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-init-command.js +14 -2
- package/dist/domains/comet-native/native-init-command.js.map +1 -1
- package/dist/domains/comet-native/native-local-execution.d.ts +25 -0
- package/dist/domains/comet-native/native-local-execution.d.ts.map +1 -0
- package/dist/domains/comet-native/native-local-execution.js +241 -0
- package/dist/domains/comet-native/native-local-execution.js.map +1 -0
- package/dist/domains/comet-native/native-lock.d.ts +1 -1
- package/dist/domains/comet-native/native-lock.d.ts.map +1 -1
- package/dist/domains/comet-native/native-lock.js +9 -9
- package/dist/domains/comet-native/native-lock.js.map +1 -1
- package/dist/domains/comet-native/native-loop-runtime.d.ts +63 -0
- package/dist/domains/comet-native/native-loop-runtime.d.ts.map +1 -0
- package/dist/domains/comet-native/native-loop-runtime.js +580 -0
- package/dist/domains/comet-native/native-loop-runtime.js.map +1 -0
- package/dist/domains/comet-native/native-mutation-lock.d.ts +2 -0
- package/dist/domains/comet-native/native-mutation-lock.d.ts.map +1 -1
- package/dist/domains/comet-native/native-mutation-lock.js +23 -2
- package/dist/domains/comet-native/native-mutation-lock.js.map +1 -1
- package/dist/domains/comet-native/native-new-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-new-command.js +27 -20
- package/dist/domains/comet-native/native-new-command.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +193 -62
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-paths.d.ts +18 -0
- package/dist/domains/comet-native/native-paths.d.ts.map +1 -1
- package/dist/domains/comet-native/native-paths.js +125 -13
- package/dist/domains/comet-native/native-paths.js.map +1 -1
- package/dist/domains/comet-native/native-portable-acceptance.d.ts +28 -0
- package/dist/domains/comet-native/native-portable-acceptance.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-acceptance.js +73 -0
- package/dist/domains/comet-native/native-portable-acceptance.js.map +1 -0
- package/dist/domains/comet-native/native-portable-archive.d.ts +37 -0
- package/dist/domains/comet-native/native-portable-archive.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-archive.js +492 -0
- package/dist/domains/comet-native/native-portable-archive.js.map +1 -0
- package/dist/domains/comet-native/native-portable-continuation.d.ts +30 -0
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-continuation.js +403 -0
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts +9 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.js +213 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -0
- package/dist/domains/comet-native/native-portable-migration.d.ts +42 -0
- package/dist/domains/comet-native/native-portable-migration.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-migration.js +212 -0
- package/dist/domains/comet-native/native-portable-migration.js.map +1 -0
- package/dist/domains/comet-native/native-portable-recovery.d.ts +15 -0
- package/dist/domains/comet-native/native-portable-recovery.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-recovery.js +203 -0
- package/dist/domains/comet-native/native-portable-recovery.js.map +1 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts +155 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-runtime.js +1550 -0
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -0
- package/dist/domains/comet-native/native-portable-state.d.ts +30 -0
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-state.js +701 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -0
- package/dist/domains/comet-native/native-portable-status.d.ts +65 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-status.js +133 -0
- package/dist/domains/comet-native/native-portable-status.js.map +1 -0
- package/dist/domains/comet-native/native-portable-text.d.ts +8 -0
- package/dist/domains/comet-native/native-portable-text.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-text.js +28 -0
- package/dist/domains/comet-native/native-portable-text.js.map +1 -0
- package/dist/domains/comet-native/native-portable-transactions.d.ts +41 -0
- package/dist/domains/comet-native/native-portable-transactions.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-transactions.js +204 -0
- package/dist/domains/comet-native/native-portable-transactions.js.map +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts +166 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-types.js +18 -0
- package/dist/domains/comet-native/native-portable-types.js.map +1 -0
- package/dist/domains/comet-native/native-repair-integration.js +5 -5
- package/dist/domains/comet-native/native-repair-integration.js.map +1 -1
- package/dist/domains/comet-native/native-resume-view.d.ts.map +1 -1
- package/dist/domains/comet-native/native-resume-view.js +1 -5
- package/dist/domains/comet-native/native-resume-view.js.map +1 -1
- package/dist/domains/comet-native/native-root-move.d.ts.map +1 -1
- package/dist/domains/comet-native/native-root-move.js +4 -20
- package/dist/domains/comet-native/native-root-move.js.map +1 -1
- package/dist/domains/comet-native/native-run-consistency.d.ts.map +1 -1
- package/dist/domains/comet-native/native-run-consistency.js +8 -12
- package/dist/domains/comet-native/native-run-consistency.js.map +1 -1
- package/dist/domains/comet-native/native-run-store.js +4 -4
- package/dist/domains/comet-native/native-run-store.js.map +1 -1
- package/dist/domains/comet-native/native-runner-input.d.ts +160 -0
- package/dist/domains/comet-native/native-runner-input.d.ts.map +1 -0
- package/dist/domains/comet-native/native-runner-input.js +347 -0
- package/dist/domains/comet-native/native-runner-input.js.map +1 -0
- package/dist/domains/comet-native/native-runner-protocol.d.ts +40 -0
- package/dist/domains/comet-native/native-runner-protocol.d.ts.map +1 -0
- package/dist/domains/comet-native/native-runner-protocol.js +55 -0
- package/dist/domains/comet-native/native-runner-protocol.js.map +1 -0
- package/dist/domains/comet-native/native-runtime-package.d.ts +3 -0
- package/dist/domains/comet-native/native-runtime-package.d.ts.map +1 -1
- package/dist/domains/comet-native/native-runtime-package.js +5 -1
- package/dist/domains/comet-native/native-runtime-package.js.map +1 -1
- package/dist/domains/comet-native/native-schema-migration.d.ts.map +1 -1
- package/dist/domains/comet-native/native-schema-migration.js +30 -27
- package/dist/domains/comet-native/native-schema-migration.js.map +1 -1
- package/dist/domains/comet-native/native-select-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-select-command.js +8 -4
- package/dist/domains/comet-native/native-select-command.js.map +1 -1
- package/dist/domains/comet-native/native-selection.d.ts.map +1 -1
- package/dist/domains/comet-native/native-selection.js +11 -2
- package/dist/domains/comet-native/native-selection.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +37 -0
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-snapshot.d.ts.map +1 -1
- package/dist/domains/comet-native/native-snapshot.js +13 -10
- package/dist/domains/comet-native/native-snapshot.js.map +1 -1
- package/dist/domains/comet-native/native-spec-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-spec-command.js +14 -28
- package/dist/domains/comet-native/native-spec-command.js.map +1 -1
- package/dist/domains/comet-native/native-specs.js +3 -3
- package/dist/domains/comet-native/native-specs.js.map +1 -1
- package/dist/domains/comet-native/native-status-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-command.js +7 -17
- package/dist/domains/comet-native/native-status-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts +51 -0
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -0
- package/dist/domains/comet-native/native-status-discovery.js +299 -0
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -0
- package/dist/domains/comet-native/native-trajectory-recovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-trajectory-recovery.js +6 -9
- package/dist/domains/comet-native/native-trajectory-recovery.js.map +1 -1
- package/dist/domains/comet-native/native-transaction.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transaction.js +31 -20
- package/dist/domains/comet-native/native-transaction.js.map +1 -1
- package/dist/domains/comet-native/native-transition-evidence.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transition-evidence.js +1 -0
- package/dist/domains/comet-native/native-transition-evidence.js.map +1 -1
- package/dist/domains/comet-native/native-transition-journal.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transition-journal.js +71 -22
- package/dist/domains/comet-native/native-transition-journal.js.map +1 -1
- package/dist/domains/comet-native/native-transitions.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transitions.js +167 -15
- package/dist/domains/comet-native/native-transitions.js.map +1 -1
- package/dist/domains/comet-native/native-types.d.ts +36 -1
- package/dist/domains/comet-native/native-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-receipt-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-receipt-runtime.js +18 -100
- package/dist/domains/comet-native/native-verification-receipt-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-verification-report-v2.d.ts +12 -0
- package/dist/domains/comet-native/native-verification-report-v2.d.ts.map +1 -0
- package/dist/domains/comet-native/native-verification-report-v2.js +141 -0
- package/dist/domains/comet-native/native-verification-report-v2.js.map +1 -0
- package/dist/domains/comet-native/native-verification-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-runtime.js +7 -2
- package/dist/domains/comet-native/native-verification-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-verification-scope.d.ts +23 -0
- package/dist/domains/comet-native/native-verification-scope.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-scope.js +106 -19
- package/dist/domains/comet-native/native-verification-scope.js.map +1 -1
- package/dist/domains/comet-native/native-verifier-protocol.d.ts +49 -0
- package/dist/domains/comet-native/native-verifier-protocol.d.ts.map +1 -0
- package/dist/domains/comet-native/native-verifier-protocol.js +157 -0
- package/dist/domains/comet-native/native-verifier-protocol.js.map +1 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts +61 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-workspace-finish.js +332 -0
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -0
- package/dist/domains/comet-native/native-workspace-preparation.d.ts +32 -0
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -0
- package/dist/domains/comet-native/native-workspace-preparation.js +324 -0
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -0
- package/dist/domains/comet-native/native-workspace.d.ts +9 -1
- package/dist/domains/comet-native/native-workspace.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace.js +109 -11
- package/dist/domains/comet-native/native-workspace.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +155 -126
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +165 -268
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +5 -4
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +506 -439
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +37 -0
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -0
- package/dist/domains/dashboard/native-legacy-adapter.js +127 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -0
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-Ca5zba3N.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-Bj6HWRwi.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-B1kXfg5N.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-BAJLB5V6.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-CwghIMO9.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-JYluRsP-.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-CCVPBm0J.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-DfYm7pN7.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-BBrkuiAU.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-DlFhgQHt.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/demo-BfLsKrIL.js +45 -0
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-BQahyP-C.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-DCxvMAUJ.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-Blit0P5Z.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/index-DhhbseFJ.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-9qXEkzDe.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-Dkf5othW.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-C2_IkO4s.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-BxWgu4DL.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-Dkr6p-Ro.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-C9xyxGfa.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-Cy6iOcxc.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-B_gZONhx.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-DvONX2YQ.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-cf39WX2o.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +16 -16
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +11 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +8 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +115 -34
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +69 -15
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +3 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +2 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +3 -1
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +119 -19
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/project-gitignore.d.ts +9 -0
- package/dist/domains/workflow-contract/project-gitignore.d.ts.map +1 -0
- package/dist/domains/workflow-contract/project-gitignore.js +108 -0
- package/dist/domains/workflow-contract/project-gitignore.js.map +1 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts +3 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +22 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +8 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +41 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +13 -0
- package/dist/platform/process/external-command.d.ts.map +1 -0
- package/dist/platform/process/external-command.js +45 -0
- package/dist/platform/process/external-command.js.map +1 -0
- package/dist/platform/process/git.d.ts +12 -0
- package/dist/platform/process/git.d.ts.map +1 -0
- package/dist/platform/process/git.js +86 -0
- package/dist/platform/process/git.js.map +1 -0
- package/dist/platform/process/spawn-command.d.ts +8 -0
- package/dist/platform/process/spawn-command.d.ts.map +1 -0
- package/dist/platform/process/spawn-command.js +113 -0
- package/dist/platform/process/spawn-command.js.map +1 -0
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-clarification/validation/test_native_clarification.py +9 -21
- package/eval/local/tasks/comet-native-clarification-depth/validation/test_native_clarification_depth.py +4 -7
- package/eval/local/tasks/comet-native-clarification-modes/validation/test_native_clarification_modes.py +4 -10
- package/eval/local/tasks/comet-native-interrupted-transition/instruction.md +1 -1
- package/eval/local/tasks/comet-native-interrupted-transition/validation/test_native_interrupted_transition.py +15 -16
- package/eval/local/tasks/comet-native-repository-fact/instruction.md +2 -2
- package/eval/local/tasks/comet-native-repository-fact/validation/test_native_repository_fact.py +8 -2
- package/eval/local/tasks/comet-native-wave-b-decision-resume/instruction.md +8 -4
- package/eval/local/tasks/comet-native-wave-b-decision-resume/task.toml +6 -6
- package/eval/local/tasks/comet-native-wave-b-decision-resume/validation/test_native_wave_b_decision_resume.py +126 -135
- package/eval/local/tasks/comet-native-wave-c-verification-integrity/instruction.md +6 -6
- package/eval/local/tasks/comet-native-wave-c-verification-integrity/task.toml +8 -8
- package/eval/local/tasks/comet-native-wave-c-verification-integrity/validation/test_native_wave_c_verification_integrity.py +97 -317
- package/eval/local/tasks/comet-native-wave-d-stagnation-stop/instruction.md +7 -7
- package/eval/local/tasks/comet-native-wave-d-stagnation-stop/task.toml +4 -4
- package/eval/local/tasks/comet-native-wave-d-stagnation-stop/validation/test_native_wave_d_stagnation_stop.py +130 -199
- package/eval/local/tasks/comet-native-wave-e-parallel-safety/instruction.md +5 -5
- package/eval/local/tasks/comet-native-wave-e-parallel-safety/task.toml +8 -8
- package/eval/local/tasks/comet-native-wave-e-parallel-safety/validation/test_native_wave_e_parallel_safety.py +135 -239
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/instruction.md +3 -3
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/task.toml +6 -6
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/validation/test_native_wave_f_dashboard_readonly.py +81 -439
- package/eval/local/tasks/comet-native-workflow/instruction.md +5 -4
- package/eval/local/tasks/comet-native-workflow/validation/test_native_workflow.py +145 -502
- package/eval/local/tasks/index.yaml +5 -5
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +163 -35
- package/eval/local/tests/scaffold/test_native_wave_evaluations.py +75 -2054
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_tasks.py +38 -52
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +219 -17
- package/eval/local/treatments/comet/comet_native_phase1.yaml +7 -4
- package/eval/pyproject.toml +9 -4
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/native_eval.py +10 -5
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_v4.py +249 -0
- package/eval/scaffold/python/validation/native_wave.py +4 -5
- package/eval/scaffold/python/validation/native_workflow.py +199 -98
- package/eval/scaffold/python/validation/rubric.py +130 -2
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +9 -3
- package/assets/skills/comet-native/scripts/comet-native-check.mjs +0 -179
- package/assets/skills/comet-native/scripts/comet-native-checkpoint.mjs +0 -179
- package/assets/skills/comet-native/scripts/comet-native-evidence.mjs +0 -151
- package/assets/skills/comet-native/scripts/comet-native-receipt.mjs +0 -187
- package/dist/domains/comet-native/native-check-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-check-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-check-entry.js +0 -4
- package/dist/domains/comet-native/native-check-entry.js.map +0 -1
- package/dist/domains/comet-native/native-checkpoint-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-checkpoint-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-checkpoint-entry.js +0 -4
- package/dist/domains/comet-native/native-checkpoint-entry.js.map +0 -1
- package/dist/domains/comet-native/native-evidence-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-evidence-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-evidence-entry.js +0 -4
- package/dist/domains/comet-native/native-evidence-entry.js.map +0 -1
- package/dist/domains/comet-native/native-receipt-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-receipt-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-receipt-entry.js +0 -4
- package/dist/domains/comet-native/native-receipt-entry.js.map +0 -1
- package/dist/domains/dashboard/web/assets/arc-ClrSujVb.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR-B4A_t84A.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-D72u2953.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-B_u-_i2t.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BA_zgMXZ.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-BpA7BPgX.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-BpA7BPgX.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-B0Bk5oiP.js +0 -4
- package/dist/domains/dashboard/web/assets/demo-DZYPLZpF.js +0 -25
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-tvTUiBA2.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-BZfcK29q.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-D5bJnW2C.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-DTwWdzWJ.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-BbvrdCp9.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-DgbR7Tcc.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-wWMOA7F4.js +0 -292
- package/dist/domains/dashboard/web/assets/index-Bk1o-4VO.css +0 -1
- package/dist/domains/dashboard/web/assets/index-g8vyW0R_.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-DZUcsoG4.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-Dn67tdOF.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-C3u-4dOk.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-Xd3ByTVF.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-CJGHL7Up.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-rKvBgr6I.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-B-L4qWy4.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-Dk1vcm3u.js +0 -7
- /package/assets/skills/{comet → comet-classic}/reference/classic-layout.md +0 -0
- /package/assets/skills-zh/{comet → comet-classic}/reference/classic-layout.md +0 -0
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
Generates rich experiment logs in logs/experiments/ including:
|
|
4
4
|
- summary.md: Full markdown report with tables and details
|
|
5
5
|
- events/: Parsed events from each test run
|
|
6
|
-
- raw/: Raw
|
|
6
|
+
- raw/: Raw selected Agent CLI output
|
|
7
7
|
- reports/: Per-run validation reports
|
|
8
|
-
- artifacts/: Files
|
|
8
|
+
- artifacts/: Files selected Agent generated and their execution output
|
|
9
9
|
- metadata.json: Experiment metadata
|
|
10
10
|
|
|
11
11
|
Supports pytest-xdist parallel execution via worker coordination.
|
|
@@ -27,7 +27,7 @@ from typing import Any
|
|
|
27
27
|
|
|
28
28
|
import pytest
|
|
29
29
|
|
|
30
|
-
from scaffold import
|
|
30
|
+
from scaffold import run_agent_in_docker, run_python_in_docker, run_shell
|
|
31
31
|
from scaffold.python import (
|
|
32
32
|
ExperimentLogger,
|
|
33
33
|
TreatmentResult,
|
|
@@ -40,9 +40,40 @@ from scaffold.python import (
|
|
|
40
40
|
save_report,
|
|
41
41
|
strip_ansi,
|
|
42
42
|
)
|
|
43
|
+
from scaffold.python.agents import get_agent_adapter, resolve_agent, validate_agent_capabilities
|
|
44
|
+
from scaffold.python.auto_tasks import AutoTaskError, ensure_generated_manifest, find_project_root
|
|
45
|
+
from scaffold.python.eval_context import (
|
|
46
|
+
EvalContextError,
|
|
47
|
+
context_from_environment,
|
|
48
|
+
resolve_eval_context,
|
|
49
|
+
resolve_managed_path,
|
|
50
|
+
)
|
|
51
|
+
from scaffold.python.execution import (
|
|
52
|
+
ResolvedExecution,
|
|
53
|
+
ResolvedJudge,
|
|
54
|
+
build_agent_environment,
|
|
55
|
+
missing_credentials,
|
|
56
|
+
preflight_credentials,
|
|
57
|
+
redact_sensitive,
|
|
58
|
+
resolve_execution,
|
|
59
|
+
resolve_judge,
|
|
60
|
+
validate_base_url,
|
|
61
|
+
)
|
|
62
|
+
from scaffold.python.generated_task_cache import (
|
|
63
|
+
selected_agent_model as _legacy_selected_agent_model,
|
|
64
|
+
)
|
|
65
|
+
from scaffold.python.langfuse_adapter import (
|
|
66
|
+
install_codex_plugin_workspace,
|
|
67
|
+
install_transcript_hook,
|
|
68
|
+
provision_trajectory_plugin,
|
|
69
|
+
)
|
|
43
70
|
from scaffold.python.sample_quality import infer_sample_quality
|
|
44
71
|
from scaffold.python.skill_parser import SCRIPT_EXTENSIONS
|
|
45
|
-
from scaffold.python.utils import
|
|
72
|
+
from scaffold.python.utils import (
|
|
73
|
+
check_docker_available,
|
|
74
|
+
run_agent_loop_in_docker,
|
|
75
|
+
run_claude_loop_in_docker,
|
|
76
|
+
)
|
|
46
77
|
from scaffold.python.aligned_comparison import (
|
|
47
78
|
EXPECTED_CASE_MATRIX_FILENAME,
|
|
48
79
|
build_execution_identity,
|
|
@@ -56,14 +87,21 @@ def _extract_loop_turns(stderr: str | None) -> int | None:
|
|
|
56
87
|
return int(match.group(1)) if match else None
|
|
57
88
|
|
|
58
89
|
|
|
59
|
-
def _extract_loop_interaction(stderr: str | None) -> dict[str,
|
|
90
|
+
def _extract_loop_interaction(stderr: str | None) -> dict[str, Any]:
|
|
60
91
|
source = stderr or ""
|
|
92
|
+
role_sessions = {"subject": [], "simulator": [], "judge": []}
|
|
93
|
+
for role, session_id in re.findall(
|
|
94
|
+
r"\[loop\] role-session (subject|simulator|judge) ([^\s]+)", source
|
|
95
|
+
):
|
|
96
|
+
if session_id not in role_sessions[role]:
|
|
97
|
+
role_sessions[role].append(session_id)
|
|
61
98
|
return {
|
|
62
99
|
"actual_turns": _extract_loop_turns(source),
|
|
63
100
|
"decision_points": source.count("[loop] decision point detected"),
|
|
64
101
|
"deterministic_replies": source.count("[loop] deterministic decision reply applied"),
|
|
65
102
|
"completion_signals": source.count("[loop] workflow completion detected"),
|
|
66
103
|
"fresh_resume_boundaries": source.count("[loop] fresh resume boundary detected"),
|
|
104
|
+
"role_sessions": role_sessions,
|
|
67
105
|
}
|
|
68
106
|
|
|
69
107
|
|
|
@@ -212,8 +250,12 @@ EVAL_ROOT = PROJECT_ROOT.parent
|
|
|
212
250
|
REPOSITORY_ROOT = EVAL_ROOT.parent
|
|
213
251
|
|
|
214
252
|
# Shared files for xdist worker coordination
|
|
215
|
-
|
|
216
|
-
|
|
253
|
+
_RUNTIME_CONTEXT = context_from_environment()
|
|
254
|
+
_CONTEXT_LOCK_ROOT = (
|
|
255
|
+
resolve_managed_path(_RUNTIME_CONTEXT, "locks") if _RUNTIME_CONTEXT else PROJECT_ROOT
|
|
256
|
+
)
|
|
257
|
+
XDIST_EXPERIMENT_FILE = _CONTEXT_LOCK_ROOT / ".pytest_experiment_id"
|
|
258
|
+
DOCKER_BUILD_LOCK = _CONTEXT_LOCK_ROOT / ".pytest_docker_build.lock"
|
|
217
259
|
EXPERIMENT_ID_ENV = "COMET_EVAL_EXPERIMENT_ID"
|
|
218
260
|
EXPERIMENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
|
219
261
|
|
|
@@ -246,12 +288,305 @@ MODEL_EXECUTION_ENV_KEYS = (
|
|
|
246
288
|
"ANTHROPIC_DEFAULT_OPUS_MODEL_NAME",
|
|
247
289
|
"ANTHROPIC_DEFAULT_SONNET_MODEL_NAME",
|
|
248
290
|
"CLAUDE_CODE_SUBAGENT_MODEL",
|
|
291
|
+
"CODEBUDDY_BASE_URL",
|
|
292
|
+
"CODEBUDDY_MODEL",
|
|
293
|
+
"CODEBUDDY_SMALL_FAST_MODEL",
|
|
294
|
+
"CODEBUDDY_BIG_SLOW_MODEL",
|
|
295
|
+
"CODEBUDDY_CODE_SUBAGENT_MODEL",
|
|
296
|
+
"BENCH_CODEBUDDY_MODEL",
|
|
297
|
+
"OPENAI_BASE_URL",
|
|
298
|
+
"OPENAI_MODEL",
|
|
299
|
+
"CODEX_BASE_URL",
|
|
300
|
+
"CODEX_MODEL",
|
|
301
|
+
"QODER_BASE_URL",
|
|
302
|
+
"QODER_MODEL",
|
|
249
303
|
)
|
|
250
304
|
|
|
251
305
|
|
|
252
|
-
def
|
|
253
|
-
|
|
254
|
-
|
|
306
|
+
def _getoption(config, name: str, default=None):
|
|
307
|
+
try:
|
|
308
|
+
value = config.getoption(name)
|
|
309
|
+
except (AttributeError, KeyError, ValueError):
|
|
310
|
+
return default
|
|
311
|
+
return default if value is None else value
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _manifest_for_config(config):
|
|
315
|
+
manifest_path = _getoption(config, "--eval-manifest")
|
|
316
|
+
if not manifest_path:
|
|
317
|
+
return None
|
|
318
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
319
|
+
|
|
320
|
+
return load_eval_manifest(manifest_path)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _resolve_eval_execution(config) -> ResolvedExecution:
|
|
324
|
+
manifest = _manifest_for_config(config)
|
|
325
|
+
manifest_execution = manifest.execution if manifest is not None else None
|
|
326
|
+
return resolve_execution(
|
|
327
|
+
cli_agent=_getoption(config, "--agent"),
|
|
328
|
+
cli_model=_getoption(config, "--model"),
|
|
329
|
+
cli_base_url=_getoption(config, "--base-url"),
|
|
330
|
+
manifest=manifest_execution,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _resolve_eval_judge(config) -> ResolvedJudge | None:
|
|
335
|
+
manifest = _manifest_for_config(config)
|
|
336
|
+
manifest_judge = manifest.judge if manifest is not None else None
|
|
337
|
+
return resolve_judge(
|
|
338
|
+
main=_resolve_eval_execution(config),
|
|
339
|
+
cli_agent=_getoption(config, "--judge-agent"),
|
|
340
|
+
cli_model=_getoption(config, "--judge-model"),
|
|
341
|
+
cli_base_url=_getoption(config, "--judge-base-url"),
|
|
342
|
+
manifest=manifest_judge,
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _selected_agent_model(agent: str) -> str | None:
|
|
347
|
+
if _plugin is not None:
|
|
348
|
+
return _resolve_eval_execution(_plugin.config).model
|
|
349
|
+
return _legacy_selected_agent_model(agent)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _resolve_eval_agent(config):
|
|
353
|
+
execution = _resolve_eval_execution(config)
|
|
354
|
+
return resolve_agent(execution.agent, None)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _require_agent_credentials(agent: str, source_env: dict[str, str] | None = None) -> None:
|
|
358
|
+
"""Fail setup before workloads when the isolated Agent cannot authenticate."""
|
|
359
|
+
environment = source_env if source_env is not None else os.environ
|
|
360
|
+
missing = missing_credentials(agent, source_env=environment)
|
|
361
|
+
if not missing:
|
|
362
|
+
return
|
|
363
|
+
adapter = get_agent_adapter(agent)
|
|
364
|
+
separator = ", " if adapter.custom else " or "
|
|
365
|
+
raise pytest.UsageError(
|
|
366
|
+
f"Credentials for {agent} not set: {separator.join(missing)}"
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _normal_run_preflight(config) -> None:
|
|
371
|
+
"""Validate normal-run dependencies before collection can generate or run tasks."""
|
|
372
|
+
|
|
373
|
+
if _is_unit_tests_only(config) or getattr(config.option, "collectonly", False):
|
|
374
|
+
return
|
|
375
|
+
from scaffold.python.utils import load_eval_environment
|
|
376
|
+
|
|
377
|
+
load_eval_environment()
|
|
378
|
+
main = _resolve_eval_execution(config)
|
|
379
|
+
judge = _resolve_eval_judge(config)
|
|
380
|
+
suite = next(
|
|
381
|
+
(name for name in ("local", "langsmith", "langfuse") if f"{name}/tests/" in " ".join(config.args)),
|
|
382
|
+
"local",
|
|
383
|
+
)
|
|
384
|
+
required_harness = (
|
|
385
|
+
EVAL_ROOT / "pyproject.toml",
|
|
386
|
+
EVAL_ROOT / "schemas" / "comet.eval" / "v1alpha1.schema.json",
|
|
387
|
+
EVAL_ROOT / suite / "tests" / "tasks" / "test_tasks.py",
|
|
388
|
+
)
|
|
389
|
+
missing_harness = [str(path) for path in required_harness if not path.is_file()]
|
|
390
|
+
if missing_harness:
|
|
391
|
+
raise pytest.UsageError(f"Packaged eval harness is incomplete: {', '.join(missing_harness)}")
|
|
392
|
+
manifest = _manifest_for_config(config)
|
|
393
|
+
if manifest is not None and manifest.reporting is not None:
|
|
394
|
+
selected_report_suite = manifest.reporting.suite
|
|
395
|
+
if selected_report_suite and selected_report_suite != suite:
|
|
396
|
+
raise pytest.UsageError(
|
|
397
|
+
f"reporting.suite is {selected_report_suite!r}, but the selected suite is {suite!r}"
|
|
398
|
+
)
|
|
399
|
+
report_config = manifest.reporting.config
|
|
400
|
+
if report_config and not _getoption(config, "--report-config"):
|
|
401
|
+
report_path = Path(report_config)
|
|
402
|
+
if not report_path.is_absolute():
|
|
403
|
+
report_path = manifest.path.parent / report_path
|
|
404
|
+
config.option.report_config = str(report_path.resolve())
|
|
405
|
+
try:
|
|
406
|
+
load_report_output_config(_getoption(config, "--report-config"))
|
|
407
|
+
except (OSError, ValueError) as exc:
|
|
408
|
+
raise pytest.UsageError(f"Eval report configuration is invalid: {exc}") from exc
|
|
409
|
+
try:
|
|
410
|
+
validate_agent_capabilities(get_agent_adapter(main.agent))
|
|
411
|
+
if judge is not None:
|
|
412
|
+
validate_agent_capabilities(get_agent_adapter(judge.agent))
|
|
413
|
+
except ValueError as exc:
|
|
414
|
+
raise pytest.UsageError(str(exc)) from exc
|
|
415
|
+
if main.base_url:
|
|
416
|
+
validate_base_url(main.base_url, field="execution.baseUrl")
|
|
417
|
+
if judge is not None and judge.base_url:
|
|
418
|
+
validate_base_url(judge.base_url, field="judge.baseUrl")
|
|
419
|
+
credential_errors = preflight_credentials(main, judge)
|
|
420
|
+
if credential_errors:
|
|
421
|
+
raise pytest.UsageError("; ".join(credential_errors))
|
|
422
|
+
if shutil.which("uv") is None:
|
|
423
|
+
raise pytest.UsageError("uv is not installed or not in PATH")
|
|
424
|
+
if not check_docker_available():
|
|
425
|
+
raise pytest.UsageError("Docker CLI or daemon is unavailable")
|
|
426
|
+
context = getattr(config, "_comet_eval_context", None)
|
|
427
|
+
if context is None:
|
|
428
|
+
raise pytest.UsageError("Eval context is missing before normal-run preflight")
|
|
429
|
+
artifact_root = Path(context.artifact_root)
|
|
430
|
+
artifact_root.mkdir(parents=True, exist_ok=True)
|
|
431
|
+
try:
|
|
432
|
+
with tempfile.NamedTemporaryFile(dir=artifact_root, prefix=".preflight-", delete=True):
|
|
433
|
+
pass
|
|
434
|
+
except OSError as exc:
|
|
435
|
+
raise pytest.UsageError(f"Eval artifact root is not writable: {artifact_root}") from exc
|
|
436
|
+
config._comet_execution_env = build_agent_environment(main)
|
|
437
|
+
config._comet_execution = main
|
|
438
|
+
config._comet_judge = judge
|
|
439
|
+
os.environ["COMET_EVAL_MAIN_CREDENTIALS"] = ",".join(
|
|
440
|
+
get_agent_adapter(main.agent).required_credentials
|
|
441
|
+
)
|
|
442
|
+
if judge is not None:
|
|
443
|
+
os.environ["BENCH_LLM_JUDGE"] = "1"
|
|
444
|
+
os.environ["BENCH_JUDGE_MODEL"] = judge.model
|
|
445
|
+
os.environ["BENCH_JUDGE_AGENT"] = judge.agent
|
|
446
|
+
if judge.base_url:
|
|
447
|
+
os.environ["BENCH_JUDGE_BASE_URL"] = judge.base_url
|
|
448
|
+
else:
|
|
449
|
+
os.environ.pop("BENCH_JUDGE_BASE_URL", None)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _agent_project_root(agent: str) -> str:
|
|
453
|
+
return {
|
|
454
|
+
"claude-code": ".claude",
|
|
455
|
+
"codex": ".agents",
|
|
456
|
+
"qoder": ".qoder",
|
|
457
|
+
"codebuddy": ".codebuddy",
|
|
458
|
+
}.get(agent, ".comet-agent")
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _ensure_langfuse_trajectory_support(test_dir: Path, agent: str) -> None:
|
|
462
|
+
"""Install only isolated, project-local trajectory support for Langfuse runs."""
|
|
463
|
+
if os.environ.get("TRACE_TO_LANGFUSE", "").lower() != "true":
|
|
464
|
+
return
|
|
465
|
+
if agent in {"qoder", "codebuddy"}:
|
|
466
|
+
try:
|
|
467
|
+
install_transcript_hook(test_dir, agent)
|
|
468
|
+
except Exception as exc:
|
|
469
|
+
if os.environ.get("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED") != "true":
|
|
470
|
+
print(
|
|
471
|
+
f"[langfuse] optional {agent} transcript setup skipped: {exc}",
|
|
472
|
+
file=os.sys.stderr,
|
|
473
|
+
)
|
|
474
|
+
os.environ["LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED"] = "true"
|
|
475
|
+
return
|
|
476
|
+
if agent != "codex":
|
|
477
|
+
return
|
|
478
|
+
cache_root = os.environ.get("LANGFUSE_TRAJECTORY_CACHE_DIR")
|
|
479
|
+
if not cache_root:
|
|
480
|
+
return
|
|
481
|
+
try:
|
|
482
|
+
provision = provision_trajectory_plugin(Path(cache_root), agent)
|
|
483
|
+
install_codex_plugin_workspace(test_dir, provision)
|
|
484
|
+
except Exception as exc:
|
|
485
|
+
if os.environ.get("LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED") != "true":
|
|
486
|
+
print(f"[langfuse] optional Codex trajectory setup skipped: {exc}", file=os.sys.stderr)
|
|
487
|
+
os.environ["LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED"] = "true"
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _ensure_auto_generated_manifest(config, task_filter: str | None = None) -> None:
|
|
491
|
+
"""Freeze generated descriptors once while retaining the original manifest authority."""
|
|
492
|
+
if getattr(config, "_comet_frozen_task_set", None) is not None:
|
|
493
|
+
return
|
|
494
|
+
if config.getoption("--task") or task_filter or config.getoption("--quick"):
|
|
495
|
+
return
|
|
496
|
+
|
|
497
|
+
manifest_path = config.getoption("--eval-manifest")
|
|
498
|
+
skill_path = config.getoption("--skill-path")
|
|
499
|
+
manifest = None
|
|
500
|
+
if manifest_path:
|
|
501
|
+
from scaffold.python.manifests import load_eval_manifest
|
|
502
|
+
|
|
503
|
+
manifest = load_eval_manifest(manifest_path)
|
|
504
|
+
if manifest.tasks or manifest.recommended_tasks:
|
|
505
|
+
return
|
|
506
|
+
skill_path = str(manifest.skill_path)
|
|
507
|
+
if not skill_path:
|
|
508
|
+
return
|
|
509
|
+
|
|
510
|
+
if getattr(getattr(config, "option", None), "collectonly", False):
|
|
511
|
+
from scaffold.python.manifests import SkillEvalManifest
|
|
512
|
+
from scaffold.python.task_resolution import ResolvedTaskSet
|
|
513
|
+
|
|
514
|
+
if manifest is None:
|
|
515
|
+
resolved_skill_root = Path(skill_path).expanduser().resolve()
|
|
516
|
+
if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
|
|
517
|
+
resolved_skill_root = resolved_skill_root.parent
|
|
518
|
+
manifest = SkillEvalManifest(
|
|
519
|
+
path=resolved_skill_root / "comet" / "eval.yaml",
|
|
520
|
+
name=resolved_skill_root.name,
|
|
521
|
+
description="",
|
|
522
|
+
skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
|
|
523
|
+
skill_path=resolved_skill_root,
|
|
524
|
+
profile=config.getoption("--profile"),
|
|
525
|
+
)
|
|
526
|
+
config._comet_resolution_manifest = manifest
|
|
527
|
+
config._comet_resolved_tasks = {}
|
|
528
|
+
config._comet_frozen_task_set = ResolvedTaskSet("pending-generation", (), None)
|
|
529
|
+
return
|
|
530
|
+
|
|
531
|
+
# Task generation happens during collection, before session fixtures run.
|
|
532
|
+
# Load the documented local environment now so credentials and the model
|
|
533
|
+
# participating in the cache key match the later execution environment.
|
|
534
|
+
from scaffold.python.utils import load_eval_environment
|
|
535
|
+
|
|
536
|
+
load_eval_environment()
|
|
537
|
+
execution = _resolve_eval_execution(config)
|
|
538
|
+
profile = config.getoption("--profile") or (manifest.profile if manifest else None) or "generic"
|
|
539
|
+
interaction = vars(manifest.interaction) if manifest else {"mode": "none", "max_turns": 12}
|
|
540
|
+
context = getattr(config, "_comet_eval_context", None)
|
|
541
|
+
project_root = (
|
|
542
|
+
context.artifact_owner_root
|
|
543
|
+
if context is not None
|
|
544
|
+
else config.getoption("--project-root") or find_project_root(skill_path)
|
|
545
|
+
)
|
|
546
|
+
try:
|
|
547
|
+
generated = ensure_generated_manifest(
|
|
548
|
+
skill_path,
|
|
549
|
+
project_root,
|
|
550
|
+
agent=execution.agent,
|
|
551
|
+
model=execution.model,
|
|
552
|
+
base_url=execution.base_url,
|
|
553
|
+
environment=build_agent_environment(execution),
|
|
554
|
+
profile=profile,
|
|
555
|
+
interaction=interaction,
|
|
556
|
+
collect_only=bool(getattr(getattr(config, "option", None), "collectonly", False)),
|
|
557
|
+
)
|
|
558
|
+
except AutoTaskError as exc:
|
|
559
|
+
raise pytest.UsageError(str(exc)) from exc
|
|
560
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
561
|
+
from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
|
|
562
|
+
from scaffold.python.task_resolution import ResolvedTask, ResolvedTaskSet
|
|
563
|
+
|
|
564
|
+
generated_manifest = load_eval_manifest(generated.manifest_path)
|
|
565
|
+
generated_tasks = tuple(
|
|
566
|
+
ResolvedTask(task.name, "generated", task)
|
|
567
|
+
for task in load_manifest_tasks(generated_manifest)
|
|
568
|
+
)
|
|
569
|
+
if manifest is None:
|
|
570
|
+
resolved_skill_root = Path(skill_path).expanduser().resolve()
|
|
571
|
+
if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
|
|
572
|
+
resolved_skill_root = resolved_skill_root.parent
|
|
573
|
+
manifest = SkillEvalManifest(
|
|
574
|
+
path=resolved_skill_root / "comet" / "eval.yaml",
|
|
575
|
+
name=resolved_skill_root.name,
|
|
576
|
+
description="",
|
|
577
|
+
skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
|
|
578
|
+
skill_path=resolved_skill_root,
|
|
579
|
+
profile=config.getoption("--profile"),
|
|
580
|
+
)
|
|
581
|
+
frozen = ResolvedTaskSet(
|
|
582
|
+
"generated-cache",
|
|
583
|
+
generated_tasks,
|
|
584
|
+
None,
|
|
585
|
+
)
|
|
586
|
+
config._comet_resolution_manifest = manifest
|
|
587
|
+
config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
|
|
588
|
+
config._comet_frozen_task_set = frozen
|
|
589
|
+
config._comet_generated_manifest = generated
|
|
255
590
|
|
|
256
591
|
|
|
257
592
|
def _model_execution_config() -> dict[str, str | None]:
|
|
@@ -265,11 +600,13 @@ def _capture_execution_identity(test_dir: Path, *, model: str | None, interactio
|
|
|
265
600
|
"docker.sh",
|
|
266
601
|
"execution-identity",
|
|
267
602
|
str(test_dir),
|
|
603
|
+
"--agent",
|
|
604
|
+
_resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code",
|
|
268
605
|
timeout=300,
|
|
269
606
|
check=False,
|
|
270
607
|
)
|
|
271
608
|
if result.returncode != 0:
|
|
272
|
-
raise RuntimeError("Could not verify eval Docker/
|
|
609
|
+
raise RuntimeError("Could not verify eval Docker/agent execution identity")
|
|
273
610
|
try:
|
|
274
611
|
raw = json.loads(result.stdout)
|
|
275
612
|
except (TypeError, json.JSONDecodeError) as error:
|
|
@@ -532,6 +869,54 @@ def pytest_addoption(parser):
|
|
|
532
869
|
default=None,
|
|
533
870
|
help="Path to comet/eval.yaml",
|
|
534
871
|
)
|
|
872
|
+
parser.addoption(
|
|
873
|
+
"--agent",
|
|
874
|
+
action="store",
|
|
875
|
+
default=None,
|
|
876
|
+
help="Evaluation agent CLI (built-in or explicitly installed custom adapter)",
|
|
877
|
+
)
|
|
878
|
+
parser.addoption(
|
|
879
|
+
"--model",
|
|
880
|
+
action="store",
|
|
881
|
+
default=None,
|
|
882
|
+
help="Main evaluation model override",
|
|
883
|
+
)
|
|
884
|
+
parser.addoption(
|
|
885
|
+
"--base-url",
|
|
886
|
+
action="store",
|
|
887
|
+
default=None,
|
|
888
|
+
help="Main evaluation API base URL override",
|
|
889
|
+
)
|
|
890
|
+
parser.addoption(
|
|
891
|
+
"--judge-agent",
|
|
892
|
+
action="store",
|
|
893
|
+
default=None,
|
|
894
|
+
help="Independent Judge agent (built-in or explicitly installed custom adapter)",
|
|
895
|
+
)
|
|
896
|
+
parser.addoption(
|
|
897
|
+
"--judge-model",
|
|
898
|
+
action="store",
|
|
899
|
+
default=None,
|
|
900
|
+
help="Independent Judge model",
|
|
901
|
+
)
|
|
902
|
+
parser.addoption(
|
|
903
|
+
"--judge-base-url",
|
|
904
|
+
action="store",
|
|
905
|
+
default=None,
|
|
906
|
+
help="Independent Judge API base URL",
|
|
907
|
+
)
|
|
908
|
+
parser.addoption(
|
|
909
|
+
"--quick",
|
|
910
|
+
action="store_true",
|
|
911
|
+
default=False,
|
|
912
|
+
help="Use generic-skill-smoke and bypass automatic task generation",
|
|
913
|
+
)
|
|
914
|
+
parser.addoption(
|
|
915
|
+
"--project-root",
|
|
916
|
+
action="store",
|
|
917
|
+
default=None,
|
|
918
|
+
help="Project root used for generated eval cache",
|
|
919
|
+
)
|
|
535
920
|
parser.addoption(
|
|
536
921
|
"--interaction-mode",
|
|
537
922
|
action="store",
|
|
@@ -564,6 +949,27 @@ def pytest_configure(config):
|
|
|
564
949
|
"markers",
|
|
565
950
|
"eval_case(repetition): controller-owned task/treatment/repetition identity",
|
|
566
951
|
)
|
|
952
|
+
try:
|
|
953
|
+
context = context_from_environment()
|
|
954
|
+
if context is None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
|
|
955
|
+
context = resolve_eval_context(
|
|
956
|
+
skill_path=config.getoption("--skill-path"),
|
|
957
|
+
manifest_path=config.getoption("--eval-manifest"),
|
|
958
|
+
project_root=config.getoption("--project-root"),
|
|
959
|
+
)
|
|
960
|
+
except EvalContextError as exc:
|
|
961
|
+
raise pytest.UsageError(str(exc)) from exc
|
|
962
|
+
if context is not None:
|
|
963
|
+
config._comet_eval_context = context
|
|
964
|
+
config.option.project_root = str(context.artifact_owner_root)
|
|
965
|
+
if context.manifest_path is not None:
|
|
966
|
+
config.option.eval_manifest = str(context.manifest_path)
|
|
967
|
+
else:
|
|
968
|
+
config.option.skill_path = str(context.skill_root)
|
|
969
|
+
config._comet_execution = _resolve_eval_execution(config)
|
|
970
|
+
config._comet_judge = _resolve_eval_judge(config)
|
|
971
|
+
_normal_run_preflight(config)
|
|
972
|
+
_resolve_eval_agent(config)
|
|
567
973
|
global _plugin
|
|
568
974
|
_plugin = ExperimentPlugin(config)
|
|
569
975
|
config.pluginmanager.register(_plugin, "experiment_plugin")
|
|
@@ -772,12 +1178,40 @@ def _get_experiment_name(session) -> str:
|
|
|
772
1178
|
|
|
773
1179
|
|
|
774
1180
|
def _get_dynamic_treatment_config(config):
|
|
1181
|
+
_ensure_auto_generated_manifest(config)
|
|
1182
|
+
|
|
1183
|
+
def generation_hints(manifest):
|
|
1184
|
+
generation_metadata = {}
|
|
1185
|
+
generation_metadata_hash = None
|
|
1186
|
+
generated = getattr(config, "_comet_generated_manifest", None)
|
|
1187
|
+
generation_metadata_path = (
|
|
1188
|
+
generated.metadata_path if generated is not None else manifest.generation_metadata_path
|
|
1189
|
+
)
|
|
1190
|
+
if generation_metadata_path and generation_metadata_path.is_file():
|
|
1191
|
+
generation_metadata_bytes = generation_metadata_path.read_bytes()
|
|
1192
|
+
generation_metadata = json.loads(generation_metadata_bytes)
|
|
1193
|
+
generation_metadata_hash = (
|
|
1194
|
+
"sha256:" + hashlib.sha256(generation_metadata_bytes).hexdigest()
|
|
1195
|
+
)
|
|
1196
|
+
return {
|
|
1197
|
+
"generation_hash": (
|
|
1198
|
+
generated.generation_hash if generated is not None else manifest.generation_hash
|
|
1199
|
+
),
|
|
1200
|
+
"generation_metadata_path": (
|
|
1201
|
+
str(generation_metadata_path) if generation_metadata_path else None
|
|
1202
|
+
),
|
|
1203
|
+
"generation_manifest_hash": generation_metadata.get("manifest_hash"),
|
|
1204
|
+
"generation_metadata_hash": generation_metadata_hash,
|
|
1205
|
+
"generation_overhead": generation_metadata.get("generation_overhead"),
|
|
1206
|
+
}
|
|
1207
|
+
|
|
775
1208
|
manifest_path = config.getoption("--eval-manifest")
|
|
776
1209
|
if manifest_path:
|
|
777
1210
|
from scaffold.python.manifests import load_eval_manifest
|
|
778
1211
|
from scaffold.python.treatments import TreatmentConfig
|
|
779
1212
|
|
|
780
1213
|
manifest = load_eval_manifest(manifest_path)
|
|
1214
|
+
generation = generation_hints(manifest)
|
|
781
1215
|
node_skills = []
|
|
782
1216
|
for node_skill in manifest.generated_node_skills:
|
|
783
1217
|
node_path = manifest.skill_path.parent / node_skill
|
|
@@ -799,11 +1233,13 @@ def _get_dynamic_treatment_config(config):
|
|
|
799
1233
|
"path": str(manifest.skill_path),
|
|
800
1234
|
"profile": manifest.profile,
|
|
801
1235
|
"manifest": str(manifest.path),
|
|
1236
|
+
"execution_agent": manifest.execution_agent,
|
|
802
1237
|
"baseline_treatments": manifest.baseline_treatments,
|
|
803
1238
|
"quality_gates": manifest.quality_gates,
|
|
804
1239
|
"required_output_schemas": manifest.required_output_schemas,
|
|
805
1240
|
"expected_evidence": manifest.expected_evidence,
|
|
806
1241
|
"draft_hash": manifest.draft_hash,
|
|
1242
|
+
**generation,
|
|
807
1243
|
"required_skills": manifest.required_skills,
|
|
808
1244
|
"expected_artifacts": manifest.expected_artifacts,
|
|
809
1245
|
"generated_node_skills": manifest.generated_node_skills,
|
|
@@ -820,7 +1256,8 @@ def _get_dynamic_treatment_config(config):
|
|
|
820
1256
|
if not skill_path:
|
|
821
1257
|
return None
|
|
822
1258
|
skill_name = config.getoption("--skill-name") or Path(skill_path).resolve().parent.name
|
|
823
|
-
|
|
1259
|
+
manifest = getattr(config, "_comet_resolution_manifest", None)
|
|
1260
|
+
profile = config.getoption("--profile") or (manifest.profile if manifest else None)
|
|
824
1261
|
skill_cfg = {
|
|
825
1262
|
"name": skill_name,
|
|
826
1263
|
"source": "path",
|
|
@@ -828,6 +1265,9 @@ def _get_dynamic_treatment_config(config):
|
|
|
828
1265
|
}
|
|
829
1266
|
if profile:
|
|
830
1267
|
skill_cfg["profile"] = profile
|
|
1268
|
+
if manifest:
|
|
1269
|
+
skill_cfg["manifest"] = str(manifest.path)
|
|
1270
|
+
skill_cfg.update(generation_hints(manifest))
|
|
831
1271
|
from scaffold.python.treatments import TreatmentConfig
|
|
832
1272
|
|
|
833
1273
|
return TreatmentConfig(
|
|
@@ -899,14 +1339,15 @@ def _build_eval_claude_md(profile_name: str, treatment_claude_md: str | None = N
|
|
|
899
1339
|
return "\n\n".join(section for section in sections if section.strip()) or None
|
|
900
1340
|
|
|
901
1341
|
|
|
902
|
-
def _comet_hook_command(test_dir: Path) -> str | None:
|
|
903
|
-
|
|
1342
|
+
def _comet_hook_command(test_dir: Path, agent: str = "claude-code") -> str | None:
|
|
1343
|
+
project_root = _agent_project_root(agent)
|
|
1344
|
+
scripts_dir = test_dir / project_root / "skills" / "comet" / "scripts"
|
|
904
1345
|
mjs_hook = scripts_dir / "comet-hook-guard.mjs"
|
|
905
1346
|
shell_hook = scripts_dir / "comet-hook-guard.sh"
|
|
906
1347
|
if mjs_hook.exists():
|
|
907
|
-
return "node /workspace
|
|
1348
|
+
return f"node /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.mjs"
|
|
908
1349
|
if shell_hook.exists():
|
|
909
|
-
return "bash /workspace
|
|
1350
|
+
return f"bash /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.sh"
|
|
910
1351
|
return None
|
|
911
1352
|
|
|
912
1353
|
|
|
@@ -930,6 +1371,34 @@ def _ensure_claude_pre_tool_hook(test_dir: Path, command: str | None) -> None:
|
|
|
930
1371
|
settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
|
|
931
1372
|
|
|
932
1373
|
|
|
1374
|
+
def _ensure_agent_pre_tool_hook(test_dir: Path, agent: str, command: str | None) -> None:
|
|
1375
|
+
if agent == "claude-code":
|
|
1376
|
+
_ensure_claude_pre_tool_hook(test_dir, command)
|
|
1377
|
+
return
|
|
1378
|
+
if not command:
|
|
1379
|
+
return
|
|
1380
|
+
|
|
1381
|
+
project_root = ".codex" if agent == "codex" else _agent_project_root(agent)
|
|
1382
|
+
settings_name = "hooks.json" if agent == "codex" else "settings.json"
|
|
1383
|
+
settings_path = test_dir / project_root / settings_name
|
|
1384
|
+
if settings_path.exists():
|
|
1385
|
+
settings = json.loads(settings_path.read_text(encoding="utf-8"))
|
|
1386
|
+
else:
|
|
1387
|
+
settings = {}
|
|
1388
|
+
hooks = settings.setdefault("hooks", {})
|
|
1389
|
+
pre_tool_use = hooks.setdefault("PreToolUse", [])
|
|
1390
|
+
hook_entry = {
|
|
1391
|
+
"matcher": "Write|Edit|MultiEdit",
|
|
1392
|
+
"hooks": [{"type": "command", "command": command}],
|
|
1393
|
+
}
|
|
1394
|
+
if agent in {"qoder", "codebuddy"}:
|
|
1395
|
+
hook_entry["hooks"][0]["description"] = "Comet workflow guard"
|
|
1396
|
+
if not any(entry == hook_entry for entry in pre_tool_use):
|
|
1397
|
+
pre_tool_use.append(hook_entry)
|
|
1398
|
+
settings_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1399
|
+
settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
|
|
1400
|
+
|
|
1401
|
+
|
|
933
1402
|
def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
|
|
934
1403
|
"""Get shared experiment ID or create new one."""
|
|
935
1404
|
requested = os.environ.get(EXPERIMENT_ID_ENV)
|
|
@@ -944,16 +1413,22 @@ def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
|
|
|
944
1413
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
945
1414
|
return f"{name}_{timestamp}"
|
|
946
1415
|
|
|
947
|
-
|
|
1416
|
+
context = context_from_environment()
|
|
1417
|
+
experiment_file = (
|
|
1418
|
+
resolve_managed_path(context, "locks", ".pytest_experiment_id")
|
|
1419
|
+
if context is not None
|
|
1420
|
+
else XDIST_EXPERIMENT_FILE
|
|
1421
|
+
)
|
|
1422
|
+
lock_file = experiment_file.with_suffix(".lock")
|
|
948
1423
|
|
|
949
1424
|
with file_lock(lock_file):
|
|
950
|
-
if
|
|
951
|
-
data = json.loads(
|
|
1425
|
+
if experiment_file.exists():
|
|
1426
|
+
data = json.loads(experiment_file.read_text())
|
|
952
1427
|
return data["experiment_id"]
|
|
953
1428
|
|
|
954
1429
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
955
1430
|
experiment_id = f"{name}_{timestamp}"
|
|
956
|
-
|
|
1431
|
+
experiment_file.write_text(
|
|
957
1432
|
json.dumps(
|
|
958
1433
|
{
|
|
959
1434
|
"experiment_id": experiment_id,
|
|
@@ -1039,7 +1514,13 @@ def _cleanup_experiment_coordination():
|
|
|
1039
1514
|
"""Remove coordination files after experiment."""
|
|
1040
1515
|
import sys
|
|
1041
1516
|
|
|
1042
|
-
|
|
1517
|
+
context = context_from_environment()
|
|
1518
|
+
experiment_file = (
|
|
1519
|
+
resolve_managed_path(context, "locks", ".pytest_experiment_id")
|
|
1520
|
+
if context is not None
|
|
1521
|
+
else XDIST_EXPERIMENT_FILE
|
|
1522
|
+
)
|
|
1523
|
+
for f in [experiment_file, experiment_file.with_suffix(".lock")]:
|
|
1043
1524
|
try:
|
|
1044
1525
|
f.unlink(missing_ok=True)
|
|
1045
1526
|
except Exception as e:
|
|
@@ -1067,8 +1548,8 @@ def worker_id(request):
|
|
|
1067
1548
|
|
|
1068
1549
|
@pytest.fixture(scope="session", autouse=True)
|
|
1069
1550
|
def verify_environment(project_root, request):
|
|
1070
|
-
"""Verify
|
|
1071
|
-
if _is_unit_tests_only(request.config):
|
|
1551
|
+
"""Verify the selected agent's Docker execution prerequisites."""
|
|
1552
|
+
if _is_unit_tests_only(request.config) or request.config.option.collectonly:
|
|
1072
1553
|
return
|
|
1073
1554
|
|
|
1074
1555
|
from scaffold.python.utils import load_eval_environment
|
|
@@ -1101,55 +1582,55 @@ def verify_environment(project_root, request):
|
|
|
1101
1582
|
if result.returncode != 0:
|
|
1102
1583
|
pytest.skip("Docker not available")
|
|
1103
1584
|
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
if shutil.which("claude") is None:
|
|
1108
|
-
pytest.skip("Claude CLI not available")
|
|
1585
|
+
selection = _resolve_eval_agent(request.config)
|
|
1586
|
+
_require_agent_credentials(selection.agent)
|
|
1109
1587
|
|
|
1110
1588
|
|
|
1111
1589
|
def _docker_environment_dirs_for_request(request, tasks_dir: Path) -> list[Path]:
|
|
1112
1590
|
"""Return only task environments selected by the current eval invocation."""
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
task_names = all_task_names
|
|
1127
|
-
|
|
1128
|
-
return [
|
|
1129
|
-
environment_dir
|
|
1130
|
-
for task_name in task_names
|
|
1131
|
-
if (environment_dir := tasks_dir / task_name / "environment").is_dir()
|
|
1132
|
-
and (environment_dir / "Dockerfile").is_file()
|
|
1133
|
-
]
|
|
1591
|
+
frozen = getattr(request.config, "_comet_frozen_task_set", None)
|
|
1592
|
+
if frozen is None:
|
|
1593
|
+
raise pytest.UsageError("Eval task collection did not produce a frozen task set")
|
|
1594
|
+
selected: list[Path] = []
|
|
1595
|
+
for item in frozen.tasks:
|
|
1596
|
+
environment_dir = item.task.environment_dir
|
|
1597
|
+
if (
|
|
1598
|
+
environment_dir.is_dir()
|
|
1599
|
+
and (environment_dir / "Dockerfile").is_file()
|
|
1600
|
+
and environment_dir not in selected
|
|
1601
|
+
):
|
|
1602
|
+
selected.append(environment_dir)
|
|
1603
|
+
return selected
|
|
1134
1604
|
|
|
1135
1605
|
|
|
1136
1606
|
@pytest.fixture(scope="session", autouse=True)
|
|
1137
1607
|
def prebuild_docker_image(request):
|
|
1138
1608
|
"""Pre-build Docker image once per session to avoid race conditions."""
|
|
1139
|
-
if _is_unit_tests_only(request.config):
|
|
1609
|
+
if _is_unit_tests_only(request.config) or request.config.option.collectonly:
|
|
1140
1610
|
yield
|
|
1141
1611
|
return
|
|
1142
1612
|
|
|
1143
1613
|
tasks_dir = PROJECT_ROOT / "tasks"
|
|
1144
1614
|
for env_dir in _docker_environment_dirs_for_request(request, tasks_dir):
|
|
1145
|
-
|
|
1615
|
+
selected_agent = _resolve_eval_agent(request.config).agent
|
|
1616
|
+
image = _build_docker_image_with_lock(
|
|
1617
|
+
env_dir,
|
|
1618
|
+
selected_agent if selected_agent != "claude-code" else None,
|
|
1619
|
+
environment=getattr(request.config, "_comet_execution_env", None),
|
|
1620
|
+
)
|
|
1146
1621
|
if image:
|
|
1147
1622
|
print(f"\nPre-built Docker image: {image}")
|
|
1148
1623
|
|
|
1149
1624
|
yield
|
|
1150
1625
|
|
|
1151
1626
|
try:
|
|
1152
|
-
|
|
1627
|
+
context = context_from_environment()
|
|
1628
|
+
lock_path = (
|
|
1629
|
+
resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
|
|
1630
|
+
if context is not None
|
|
1631
|
+
else DOCKER_BUILD_LOCK
|
|
1632
|
+
)
|
|
1633
|
+
lock_path.unlink(missing_ok=True)
|
|
1153
1634
|
except Exception:
|
|
1154
1635
|
pass
|
|
1155
1636
|
|
|
@@ -1181,7 +1662,8 @@ def setup_test_context(test_dir):
|
|
|
1181
1662
|
scripts_dir: Path | None = None,
|
|
1182
1663
|
source_dir: Path | None = None,
|
|
1183
1664
|
) -> None:
|
|
1184
|
-
|
|
1665
|
+
agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
|
|
1666
|
+
skill_dir = test_dir / _agent_project_root(agent) / "skills" / skill_name
|
|
1185
1667
|
skill_dir.mkdir(parents=True, exist_ok=True)
|
|
1186
1668
|
|
|
1187
1669
|
if source_dir and source_dir.is_dir():
|
|
@@ -1212,13 +1694,38 @@ def setup_test_context(test_dir):
|
|
|
1212
1694
|
shutil.copy2(item, dest)
|
|
1213
1695
|
_copy_current_comet_cli_snapshot(environment_dir, test_dir)
|
|
1214
1696
|
|
|
1697
|
+
def _copy_workspace(workspace_dir: Path) -> None:
|
|
1698
|
+
"""Copy a bounded project fixture without replacing the task Dockerfile."""
|
|
1699
|
+
for item in workspace_dir.iterdir():
|
|
1700
|
+
if item.name in {"Dockerfile", CURRENT_COMET_CLI_MARKER, TRUSTED_NATIVE_RUNTIME_MARKER}:
|
|
1701
|
+
continue
|
|
1702
|
+
dest = test_dir / item.name
|
|
1703
|
+
if item.is_dir():
|
|
1704
|
+
shutil.copytree(item, dest, dirs_exist_ok=True)
|
|
1705
|
+
else:
|
|
1706
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1707
|
+
shutil.copy2(item, dest)
|
|
1708
|
+
|
|
1215
1709
|
def _write_claude_md(content_file: str) -> None:
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1710
|
+
agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
|
|
1711
|
+
if agent == "claude-code":
|
|
1712
|
+
claude_dir = test_dir / ".claude"
|
|
1713
|
+
claude_dir.mkdir(parents=True, exist_ok=True)
|
|
1714
|
+
shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
|
|
1715
|
+
shutil.copyfile(content_file, test_dir / "CLAUDE.md")
|
|
1716
|
+
elif agent == "codebuddy":
|
|
1717
|
+
codebuddy_dir = test_dir / ".codebuddy"
|
|
1718
|
+
codebuddy_dir.mkdir(parents=True, exist_ok=True)
|
|
1719
|
+
shutil.copyfile(content_file, codebuddy_dir / "CODEBUDDY.md")
|
|
1720
|
+
else:
|
|
1721
|
+
shutil.copyfile(content_file, test_dir / "AGENTS.md")
|
|
1220
1722
|
|
|
1221
|
-
def _setup(
|
|
1723
|
+
def _setup(
|
|
1724
|
+
skills: dict = None,
|
|
1725
|
+
claude_md: str = None,
|
|
1726
|
+
environment_dir: Path = None,
|
|
1727
|
+
workspace_dir: Path = None,
|
|
1728
|
+
):
|
|
1222
1729
|
for skill_name, cfg in (skills or {}).items():
|
|
1223
1730
|
if not cfg:
|
|
1224
1731
|
continue
|
|
@@ -1251,6 +1758,9 @@ def setup_test_context(test_dir):
|
|
|
1251
1758
|
if is_temp_dir and filtered_dir.exists():
|
|
1252
1759
|
shutil.rmtree(filtered_dir)
|
|
1253
1760
|
|
|
1761
|
+
if workspace_dir and workspace_dir.exists():
|
|
1762
|
+
_copy_workspace(workspace_dir)
|
|
1763
|
+
|
|
1254
1764
|
if environment_dir and environment_dir.exists():
|
|
1255
1765
|
_copy_environment(environment_dir)
|
|
1256
1766
|
_copy_trusted_native_runtime_snapshot(environment_dir, test_dir, skills)
|
|
@@ -1266,7 +1776,9 @@ def setup_test_context(test_dir):
|
|
|
1266
1776
|
finally:
|
|
1267
1777
|
os.unlink(temp_file)
|
|
1268
1778
|
|
|
1269
|
-
|
|
1779
|
+
agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
|
|
1780
|
+
_ensure_langfuse_trajectory_support(test_dir, agent)
|
|
1781
|
+
_ensure_agent_pre_tool_hook(test_dir, agent, _comet_hook_command(test_dir, agent))
|
|
1270
1782
|
|
|
1271
1783
|
return test_dir
|
|
1272
1784
|
|
|
@@ -1275,13 +1787,15 @@ def setup_test_context(test_dir):
|
|
|
1275
1787
|
|
|
1276
1788
|
@pytest.fixture
|
|
1277
1789
|
def run_claude(test_dir, experiment_logger, request):
|
|
1278
|
-
"""Factory fixture to run
|
|
1790
|
+
"""Factory fixture to run the selected evaluation agent in Docker.
|
|
1279
1791
|
|
|
1280
1792
|
For tasks using ``interaction.mode=auto_user`` the single-shot ``run-claude``
|
|
1281
1793
|
is replaced by the multi-turn ``run-claude-loop`` driver, which simulates a
|
|
1282
1794
|
user replying at the workflow's decision points.
|
|
1283
1795
|
"""
|
|
1284
|
-
|
|
1796
|
+
execution = _resolve_eval_execution(request.config)
|
|
1797
|
+
selected_agent = execution.agent
|
|
1798
|
+
default_model = execution.model
|
|
1285
1799
|
|
|
1286
1800
|
def _run(
|
|
1287
1801
|
prompt: str,
|
|
@@ -1291,18 +1805,28 @@ def run_claude(test_dir, experiment_logger, request):
|
|
|
1291
1805
|
image_id: str | None = None,
|
|
1292
1806
|
):
|
|
1293
1807
|
mdl = model or default_model
|
|
1808
|
+
run_execution = ResolvedExecution(
|
|
1809
|
+
selected_agent,
|
|
1810
|
+
mdl,
|
|
1811
|
+
execution.base_url,
|
|
1812
|
+
execution.sources,
|
|
1813
|
+
)
|
|
1814
|
+
run_environment = build_agent_environment(run_execution)
|
|
1294
1815
|
if os.environ.get("TRACE_TO_LANGSMITH", "").lower() == "true" and not os.environ.get(
|
|
1295
1816
|
"CC_LANGSMITH_LOG_FILE"
|
|
1296
1817
|
):
|
|
1297
1818
|
os.environ["CC_LANGSMITH_LOG_FILE"] = "/workspace/langsmith-hook.log"
|
|
1298
1819
|
use_loop = interaction is not None and interaction.mode == "auto_user"
|
|
1299
1820
|
if not use_loop:
|
|
1300
|
-
result =
|
|
1821
|
+
result = run_agent_in_docker(
|
|
1301
1822
|
test_dir,
|
|
1302
1823
|
prompt,
|
|
1824
|
+
agent=selected_agent,
|
|
1303
1825
|
timeout=timeout,
|
|
1304
1826
|
model=mdl,
|
|
1827
|
+
base_url=execution.base_url,
|
|
1305
1828
|
image_id=image_id,
|
|
1829
|
+
environment=run_environment,
|
|
1306
1830
|
)
|
|
1307
1831
|
else:
|
|
1308
1832
|
task_prompt_file = test_dir / ".eval-task-prompt.txt"
|
|
@@ -1342,11 +1866,25 @@ def run_claude(test_dir, experiment_logger, request):
|
|
|
1342
1866
|
"--simulator-prompt-file",
|
|
1343
1867
|
"//workspace/.eval-simulator-prompt.txt",
|
|
1344
1868
|
]
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1869
|
+
if selected_agent == "claude-code":
|
|
1870
|
+
result = run_claude_loop_in_docker(
|
|
1871
|
+
test_dir,
|
|
1872
|
+
loop_args[2:],
|
|
1873
|
+
timeout=timeout + 60,
|
|
1874
|
+
environment=run_environment,
|
|
1875
|
+
)
|
|
1876
|
+
else:
|
|
1877
|
+
result = run_agent_loop_in_docker(
|
|
1878
|
+
test_dir,
|
|
1879
|
+
[
|
|
1880
|
+
loop_args[2],
|
|
1881
|
+
"--agent",
|
|
1882
|
+
selected_agent,
|
|
1883
|
+
*loop_args[3:],
|
|
1884
|
+
],
|
|
1885
|
+
timeout=timeout + 60,
|
|
1886
|
+
environment=run_environment,
|
|
1887
|
+
)
|
|
1350
1888
|
finally:
|
|
1351
1889
|
task_prompt_file.unlink(missing_ok=True)
|
|
1352
1890
|
if prompt_file and prompt_file.exists():
|
|
@@ -1384,12 +1922,22 @@ def record_result(test_dir, experiment_logger, request):
|
|
|
1384
1922
|
if not experiment_logger:
|
|
1385
1923
|
return
|
|
1386
1924
|
|
|
1925
|
+
if _plugin is not None:
|
|
1926
|
+
_plugin.last_test_dir = test_dir
|
|
1927
|
+
|
|
1387
1928
|
treatment_name = _get_treatment_name(request.node)
|
|
1388
1929
|
rep = _plugin.run_counter.get(treatment_name, 1) if _plugin else 1
|
|
1930
|
+
events["sample"] = rep
|
|
1389
1931
|
base_dir = experiment_logger.base_dir
|
|
1390
1932
|
|
|
1391
1933
|
save_events(base_dir, treatment_name, rep, events)
|
|
1392
|
-
_save_artifacts(
|
|
1934
|
+
_save_artifacts(
|
|
1935
|
+
base_dir,
|
|
1936
|
+
treatment_name,
|
|
1937
|
+
rep,
|
|
1938
|
+
test_dir,
|
|
1939
|
+
agent=events.get("agent", "claude-code"),
|
|
1940
|
+
)
|
|
1393
1941
|
artifact_references = build_eval_artifact_references(base_dir, treatment_name, rep)
|
|
1394
1942
|
|
|
1395
1943
|
scripts_used = _extract_scripts_used(events)
|
|
@@ -1439,9 +1987,23 @@ def record_result(test_dir, experiment_logger, request):
|
|
|
1439
1987
|
"skill_sources": events.get("skill_sources", []),
|
|
1440
1988
|
"eval_manifest": events.get("eval_manifest"),
|
|
1441
1989
|
"case_manifest": events.get("case_manifest"),
|
|
1990
|
+
"eval_generation": events.get("eval_generation"),
|
|
1442
1991
|
"interaction": events.get("interaction", {}),
|
|
1443
1992
|
"artifact_references": artifact_references,
|
|
1444
1993
|
"failure_attribution": failure_attribution,
|
|
1994
|
+
"agent": events.get("agent", "claude-code"),
|
|
1995
|
+
"role_models": events.get("role_models", {}),
|
|
1996
|
+
"role_agents": events.get("role_agents", {}),
|
|
1997
|
+
"telemetry_status": events.get("telemetry_status", "N/A"),
|
|
1998
|
+
"role_sessions": events.get("role_sessions", {}),
|
|
1999
|
+
"task": events.get("task"),
|
|
2000
|
+
"treatment": events.get("treatment", treatment_name),
|
|
2001
|
+
"sample": events.get("sample", rep),
|
|
2002
|
+
"prompt": events.get("prompt"),
|
|
2003
|
+
"skill": events.get("skill"),
|
|
2004
|
+
"final_response": events.get("final_response"),
|
|
2005
|
+
"quality_gates": events.get("quality_gates", {}),
|
|
2006
|
+
"execution_identity": events.get("execution_identity"),
|
|
1445
2007
|
},
|
|
1446
2008
|
run_id=run_id,
|
|
1447
2009
|
),
|
|
@@ -1495,14 +2057,29 @@ def _build_report_payload(
|
|
|
1495
2057
|
"model_usage": events.get("model_usage", {}),
|
|
1496
2058
|
"files_created": events.get("files_created", []),
|
|
1497
2059
|
"skills_invoked": events.get("skills_invoked", []),
|
|
2060
|
+
"skill_invocations": events.get("skill_invocations", []),
|
|
1498
2061
|
"scripts_used": scripts_used,
|
|
1499
2062
|
"profile": events.get("profile"),
|
|
1500
2063
|
"skill_sources": events.get("skill_sources", []),
|
|
1501
2064
|
"eval_manifest": events.get("eval_manifest"),
|
|
1502
2065
|
"case_manifest": events.get("case_manifest"),
|
|
2066
|
+
"eval_generation": events.get("eval_generation"),
|
|
1503
2067
|
"interaction": events.get("interaction", {}),
|
|
1504
2068
|
"artifact_references": artifact_references,
|
|
1505
2069
|
"failure_attribution": failure_attribution,
|
|
2070
|
+
"agent": events.get("agent", "claude-code"),
|
|
2071
|
+
"role_models": events.get("role_models", {}),
|
|
2072
|
+
"role_agents": events.get("role_agents", {}),
|
|
2073
|
+
"telemetry_status": events.get("telemetry_status", "N/A"),
|
|
2074
|
+
"role_sessions": events.get("role_sessions", {}),
|
|
2075
|
+
"task": events.get("task"),
|
|
2076
|
+
"treatment": events.get("treatment", treatment_name),
|
|
2077
|
+
"sample": events.get("sample", rep),
|
|
2078
|
+
"prompt": events.get("prompt"),
|
|
2079
|
+
"skill": events.get("skill"),
|
|
2080
|
+
"final_response": events.get("final_response"),
|
|
2081
|
+
"quality_gates": events.get("quality_gates", {}),
|
|
2082
|
+
"execution_identity": events.get("execution_identity"),
|
|
1506
2083
|
},
|
|
1507
2084
|
"timestamp": datetime.now().isoformat(),
|
|
1508
2085
|
}
|
|
@@ -1581,15 +2158,32 @@ def _filter_scripts(scripts_dir: Path, script_filter: str) -> Path | None:
|
|
|
1581
2158
|
return temp_dir
|
|
1582
2159
|
|
|
1583
2160
|
|
|
1584
|
-
def _build_docker_image_with_lock(
|
|
2161
|
+
def _build_docker_image_with_lock(
|
|
2162
|
+
environment_dir: Path,
|
|
2163
|
+
agent: str | None = None,
|
|
2164
|
+
*,
|
|
2165
|
+
environment: dict[str, str] | None = None,
|
|
2166
|
+
) -> str | None:
|
|
1585
2167
|
"""Build Docker image with file locking to prevent race conditions."""
|
|
1586
2168
|
if not environment_dir or not (environment_dir / "Dockerfile").exists():
|
|
1587
2169
|
return None
|
|
1588
2170
|
|
|
1589
|
-
|
|
1590
|
-
|
|
2171
|
+
context = context_from_environment()
|
|
2172
|
+
lock_path = (
|
|
2173
|
+
resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
|
|
2174
|
+
if context is not None
|
|
2175
|
+
else DOCKER_BUILD_LOCK
|
|
2176
|
+
)
|
|
2177
|
+
with file_lock(lock_path):
|
|
2178
|
+
# A cold image build downloads Debian packages and installs the selected CLI.
|
|
1591
2179
|
# Five minutes is insufficient after cache cleanup or on a proxied connection.
|
|
1592
|
-
|
|
2180
|
+
command = ["docker.sh", "build", str(environment_dir)]
|
|
2181
|
+
if agent:
|
|
2182
|
+
command.extend(["--agent", agent])
|
|
2183
|
+
run_options = {"timeout": 900, "check": False}
|
|
2184
|
+
if environment is not None:
|
|
2185
|
+
run_options["env"] = environment
|
|
2186
|
+
result = run_shell(*command, **run_options)
|
|
1593
2187
|
if result.returncode == 0:
|
|
1594
2188
|
return result.stdout.strip()
|
|
1595
2189
|
return None
|
|
@@ -1664,18 +2258,34 @@ def _extract_scripts_used(events: dict) -> list[str]:
|
|
|
1664
2258
|
return [s for s in _get_known_scripts() if s.lower() in all_activity]
|
|
1665
2259
|
|
|
1666
2260
|
|
|
1667
|
-
def _save_artifacts(
|
|
1668
|
-
|
|
2261
|
+
def _save_artifacts(
|
|
2262
|
+
base_dir: Path,
|
|
2263
|
+
treatment_name: str,
|
|
2264
|
+
rep: int,
|
|
2265
|
+
test_dir: Path,
|
|
2266
|
+
*,
|
|
2267
|
+
agent: str = "claude-code",
|
|
2268
|
+
):
|
|
2269
|
+
"""Save the selected Agent's generated files as artifacts."""
|
|
1669
2270
|
artifacts_dir = base_dir / "artifacts" / f"{treatment_name.lower()}_rep{rep}"
|
|
1670
|
-
|
|
2271
|
+
agent_dir_name = {
|
|
2272
|
+
"claude-code": "claude",
|
|
2273
|
+
"codex": "codex",
|
|
2274
|
+
"qoder": "qoder",
|
|
2275
|
+
"codebuddy": "codebuddy",
|
|
2276
|
+
}.get(agent, agent)
|
|
2277
|
+
agent_dir = artifacts_dir / agent_dir_name
|
|
1671
2278
|
execution_dir = artifacts_dir / "execution"
|
|
1672
|
-
|
|
2279
|
+
agent_dir.mkdir(parents=True, exist_ok=True)
|
|
1673
2280
|
execution_dir.mkdir(parents=True, exist_ok=True)
|
|
1674
2281
|
|
|
1675
2282
|
from scaffold.python.utils import TEST_CONTEXT_FILE, TEST_RESULTS_FILE
|
|
1676
2283
|
|
|
1677
2284
|
exclude_dirs = {
|
|
1678
2285
|
".claude",
|
|
2286
|
+
".agents",
|
|
2287
|
+
".qoder",
|
|
2288
|
+
".codebuddy",
|
|
1679
2289
|
".git",
|
|
1680
2290
|
"_eval_current_comet",
|
|
1681
2291
|
"node_modules",
|
|
@@ -1707,9 +2317,20 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
|
|
|
1707
2317
|
continue
|
|
1708
2318
|
try:
|
|
1709
2319
|
rel_path = item.relative_to(test_dir)
|
|
1710
|
-
dest =
|
|
2320
|
+
dest = agent_dir / rel_path
|
|
1711
2321
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1712
|
-
|
|
2322
|
+
try:
|
|
2323
|
+
content = item.read_text(encoding="utf-8")
|
|
2324
|
+
except UnicodeDecodeError:
|
|
2325
|
+
import sys
|
|
2326
|
+
|
|
2327
|
+
print(
|
|
2328
|
+
f"Warning: skipped non-text artifact {rel_path}; binary content is not persisted to protect credentials",
|
|
2329
|
+
file=sys.stderr,
|
|
2330
|
+
)
|
|
2331
|
+
continue
|
|
2332
|
+
else:
|
|
2333
|
+
dest.write_text(redact_sensitive(content), encoding="utf-8")
|
|
1713
2334
|
claude_files.append(item)
|
|
1714
2335
|
except Exception as e:
|
|
1715
2336
|
import sys
|
|
@@ -1722,7 +2343,7 @@ def _save_artifacts(base_dir: Path, treatment_name: str, rep: int, test_dir: Pat
|
|
|
1722
2343
|
success, output = run_python_in_docker(test_dir, py_file.name, timeout=300)
|
|
1723
2344
|
status = "success" if success else "error"
|
|
1724
2345
|
output_file = execution_dir / f"{py_file.stem}_{status}.txt"
|
|
1725
|
-
output_file.write_text(strip_ansi(output))
|
|
2346
|
+
output_file.write_text(redact_sensitive(strip_ansi(output)), encoding="utf-8")
|
|
1726
2347
|
except Exception as e:
|
|
1727
2348
|
error_file = execution_dir / f"{py_file.stem}_error.txt"
|
|
1728
|
-
error_file.write_text(str(e))
|
|
2349
|
+
error_file.write_text(redact_sensitive(str(e)), encoding="utf-8")
|