@rpamis/comet 0.4.0-beta.16 → 0.4.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +85 -56
- package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.html +8 -0
- package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.json +286 -0
- package/assets/manifest.json +26 -17
- package/assets/skills/comet/SKILL.md +1 -1
- package/assets/skills/comet/agents/openai.yaml +3 -0
- package/assets/skills/comet/rules/comet-phase-guard.en.md +1 -1
- package/assets/skills/comet/rules/comet-phase-guard.md +1 -1
- package/assets/skills/comet/rules/comet-workflow-guard.en.md +2 -2
- package/assets/skills/comet/rules/comet-workflow-guard.md +2 -2
- package/assets/skills/comet/scripts/comet-archive.mjs +128 -128
- package/assets/skills/comet/scripts/comet-entry-runtime.mjs +77 -77
- package/assets/skills/comet/scripts/comet-guard.mjs +144 -144
- package/assets/skills/comet/scripts/comet-handoff.mjs +107 -107
- package/assets/skills/comet/scripts/comet-hook-guard.mjs +128 -128
- package/assets/skills/comet/scripts/comet-hook-router.mjs +143 -141
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +100 -100
- package/assets/skills/comet/scripts/comet-runtime.mjs +194 -193
- package/assets/skills/comet/scripts/comet-state.mjs +118 -118
- package/assets/skills/comet/scripts/comet-yaml-validate.mjs +95 -95
- package/assets/skills/comet-any/SKILL.md +2 -1
- package/assets/skills/comet-any/agents/openai.yaml +5 -0
- package/assets/skills/comet-archive/SKILL.md +7 -6
- package/assets/skills/comet-archive/agents/openai.yaml +5 -0
- package/assets/skills/comet-build/SKILL.md +28 -68
- package/assets/skills/comet-build/agents/openai.yaml +5 -0
- package/assets/skills/comet-classic/SKILL.md +28 -22
- package/assets/skills/comet-classic/agents/openai.yaml +3 -0
- package/assets/skills/{comet → comet-classic}/reference/auto-transition.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/comet-yaml-fields.md +3 -3
- package/assets/skills/{comet → comet-classic}/reference/context-recovery.md +5 -5
- package/assets/skills/{comet → comet-classic}/reference/debug-gate.md +2 -2
- package/assets/skills/{comet → comet-classic}/reference/decision-point.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/dirty-worktree.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/file-structure.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/intent-frame.md +1 -1
- package/assets/skills/{comet → comet-classic}/reference/scripts.md +4 -2
- package/assets/skills/{comet → comet-classic}/reference/subagent-dispatch.md +1 -1
- package/assets/skills/comet-classic/reference/workspace.md +23 -0
- package/assets/skills/comet-design/SKILL.md +7 -6
- package/assets/skills/comet-design/agents/openai.yaml +5 -0
- package/assets/skills/comet-hotfix/SKILL.md +10 -9
- package/assets/skills/comet-hotfix/agents/openai.yaml +5 -0
- package/assets/skills/comet-native/SKILL.md +60 -171
- package/assets/skills/comet-native/agents/openai.yaml +3 -0
- package/assets/skills/comet-native/reference/artifacts.md +24 -89
- package/assets/skills/comet-native/reference/clarification.md +54 -29
- package/assets/skills/comet-native/reference/commands.md +30 -173
- package/assets/skills/comet-native/reference/recovery.md +38 -88
- package/assets/skills/comet-native/reference/workspace.md +23 -0
- package/assets/skills/comet-native/scripts/comet-native-archive.mjs +184 -152
- package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +214 -158
- package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +132 -123
- package/assets/skills/comet-native/scripts/comet-native-init.mjs +134 -124
- package/assets/skills/comet-native/scripts/comet-native-new.mjs +165 -158
- package/assets/skills/comet-native/scripts/comet-native-next.mjs +183 -152
- package/assets/skills/comet-native/scripts/comet-native-root.mjs +152 -147
- package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +247 -202
- package/assets/skills/comet-native/scripts/comet-native-select.mjs +150 -143
- package/assets/skills/comet-native/scripts/comet-native-show.mjs +128 -121
- package/assets/skills/comet-native/scripts/comet-native-spec.mjs +133 -151
- package/assets/skills/comet-native/scripts/comet-native-status.mjs +147 -140
- package/assets/skills/comet-open/SKILL.md +31 -11
- package/assets/skills/comet-open/agents/openai.yaml +5 -0
- package/assets/skills/comet-tweak/SKILL.md +11 -10
- package/assets/skills/comet-tweak/agents/openai.yaml +5 -0
- package/assets/skills/comet-verify/SKILL.md +9 -8
- package/assets/skills/comet-verify/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet/SKILL.md +1 -1
- package/assets/skills-zh/comet/agents/openai.yaml +3 -0
- package/assets/skills-zh/comet-any/SKILL.md +2 -1
- package/assets/skills-zh/comet-any/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-archive/SKILL.md +7 -6
- package/assets/skills-zh/comet-archive/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-build/SKILL.md +28 -68
- package/assets/skills-zh/comet-build/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-classic/SKILL.md +29 -23
- package/assets/skills-zh/comet-classic/agents/openai.yaml +3 -0
- package/assets/skills-zh/{comet → comet-classic}/reference/auto-transition.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/comet-yaml-fields.md +3 -3
- package/assets/skills-zh/{comet → comet-classic}/reference/context-recovery.md +5 -5
- package/assets/skills-zh/{comet → comet-classic}/reference/debug-gate.md +2 -2
- package/assets/skills-zh/{comet → comet-classic}/reference/decision-point.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/dirty-worktree.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/file-structure.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/intent-frame.md +1 -1
- package/assets/skills-zh/{comet → comet-classic}/reference/scripts.md +4 -2
- package/assets/skills-zh/{comet → comet-classic}/reference/subagent-dispatch.md +1 -1
- package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-design/SKILL.md +7 -6
- package/assets/skills-zh/comet-design/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-hotfix/SKILL.md +10 -9
- package/assets/skills-zh/comet-hotfix/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-native/SKILL.md +60 -171
- package/assets/skills-zh/comet-native/agents/openai.yaml +3 -0
- package/assets/skills-zh/comet-native/reference/artifacts.md +24 -89
- package/assets/skills-zh/comet-native/reference/clarification.md +40 -15
- package/assets/skills-zh/comet-native/reference/commands.md +30 -173
- package/assets/skills-zh/comet-native/reference/recovery.md +36 -86
- package/assets/skills-zh/comet-native/reference/workspace.md +23 -0
- package/assets/skills-zh/comet-open/SKILL.md +31 -11
- package/assets/skills-zh/comet-open/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-tweak/SKILL.md +11 -10
- package/assets/skills-zh/comet-tweak/agents/openai.yaml +5 -0
- package/assets/skills-zh/comet-verify/SKILL.md +9 -8
- package/assets/skills-zh/comet-verify/agents/openai.yaml +5 -0
- package/bin/fast-runtime-router.js +0 -4
- package/dist/app/cli/index.js +20 -1
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts.map +1 -1
- package/dist/app/commands/classic.js +2 -0
- package/dist/app/commands/classic.js.map +1 -1
- package/dist/app/commands/doctor.d.ts.map +1 -1
- package/dist/app/commands/doctor.js +8 -0
- package/dist/app/commands/doctor.js.map +1 -1
- package/dist/app/commands/eval.d.ts +9 -2
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +293 -48
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +11 -2
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +13 -3
- package/dist/app/commands/update.js.map +1 -1
- package/dist/config/repository-layout.json +0 -8
- package/dist/domains/bundle/bundle-platform.d.ts.map +1 -1
- package/dist/domains/bundle/bundle-platform.js +8 -0
- package/dist/domains/bundle/bundle-platform.js.map +1 -1
- package/dist/domains/bundle/types.d.ts +1 -1
- package/dist/domains/bundle/types.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-current-change.js +29 -3
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +2 -2
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +14 -1
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +31 -8
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
- package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
- package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-workspace.js +315 -0
- package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
- package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-adapter.js +3 -0
- package/dist/domains/comet-entry/hook-adapter.js.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.d.ts +2 -0
- package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -1
- package/dist/domains/comet-entry/hook-router-entry.js +32 -3
- package/dist/domains/comet-entry/hook-router-entry.js.map +1 -1
- package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -1
- package/dist/domains/comet-entry/init-workflow.js +13 -5
- package/dist/domains/comet-entry/init-workflow.js.map +1 -1
- package/dist/domains/comet-entry/project-activation.d.ts.map +1 -1
- package/dist/domains/comet-entry/project-activation.js +2 -1
- package/dist/domains/comet-entry/project-activation.js.map +1 -1
- package/dist/domains/comet-entry/resume-probe.d.ts.map +1 -1
- package/dist/domains/comet-entry/resume-probe.js +48 -4
- package/dist/domains/comet-entry/resume-probe.js.map +1 -1
- package/dist/domains/comet-native/index.d.ts +1 -0
- package/dist/domains/comet-native/index.d.ts.map +1 -1
- package/dist/domains/comet-native/index.js +1 -0
- package/dist/domains/comet-native/index.js.map +1 -1
- package/dist/domains/comet-native/native-acceptance.d.ts +4 -3
- package/dist/domains/comet-native/native-acceptance.d.ts.map +1 -1
- package/dist/domains/comet-native/native-acceptance.js +23 -18
- package/dist/domains/comet-native/native-acceptance.js.map +1 -1
- package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-command.js +204 -61
- package/dist/domains/comet-native/native-archive-command.js.map +1 -1
- package/dist/domains/comet-native/native-archive-transaction.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive-transaction.js +28 -23
- package/dist/domains/comet-native/native-archive-transaction.js.map +1 -1
- package/dist/domains/comet-native/native-archive.d.ts +1 -0
- package/dist/domains/comet-native/native-archive.d.ts.map +1 -1
- package/dist/domains/comet-native/native-archive.js +56 -12
- package/dist/domains/comet-native/native-archive.js.map +1 -1
- package/dist/domains/comet-native/native-bounded-file.d.ts +32 -2
- package/dist/domains/comet-native/native-bounded-file.d.ts.map +1 -1
- package/dist/domains/comet-native/native-bounded-file.js +87 -6
- package/dist/domains/comet-native/native-bounded-file.js.map +1 -1
- package/dist/domains/comet-native/native-build-evidence.d.ts.map +1 -1
- package/dist/domains/comet-native/native-build-evidence.js +23 -1
- package/dist/domains/comet-native/native-build-evidence.js.map +1 -1
- package/dist/domains/comet-native/native-change.d.ts +1 -0
- package/dist/domains/comet-native/native-change.d.ts.map +1 -1
- package/dist/domains/comet-native/native-change.js +96 -8
- package/dist/domains/comet-native/native-change.js.map +1 -1
- package/dist/domains/comet-native/native-check-executor.d.ts +36 -0
- package/dist/domains/comet-native/native-check-executor.d.ts.map +1 -0
- package/dist/domains/comet-native/native-check-executor.js +132 -0
- package/dist/domains/comet-native/native-check-executor.js.map +1 -0
- package/dist/domains/comet-native/native-check-receipt-model.d.ts +9 -0
- package/dist/domains/comet-native/native-check-receipt-model.d.ts.map +1 -1
- package/dist/domains/comet-native/native-check-receipt-model.js +70 -8
- package/dist/domains/comet-native/native-check-receipt-model.js.map +1 -1
- package/dist/domains/comet-native/native-check-receipt-storage.d.ts.map +1 -1
- package/dist/domains/comet-native/native-check-receipt-storage.js +9 -8
- package/dist/domains/comet-native/native-check-receipt-storage.js.map +1 -1
- package/dist/domains/comet-native/native-check-receipt.d.ts.map +1 -1
- package/dist/domains/comet-native/native-check-receipt.js +123 -60
- package/dist/domains/comet-native/native-check-receipt.js.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-command.js +0 -6
- package/dist/domains/comet-native/native-checkpoint-command.js.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-storage.d.ts.map +1 -1
- package/dist/domains/comet-native/native-checkpoint-storage.js +27 -19
- package/dist/domains/comet-native/native-checkpoint-storage.js.map +1 -1
- package/dist/domains/comet-native/native-children.d.ts +49 -0
- package/dist/domains/comet-native/native-children.d.ts.map +1 -0
- package/dist/domains/comet-native/native-children.js +388 -0
- package/dist/domains/comet-native/native-children.js.map +1 -0
- package/dist/domains/comet-native/native-cli-help.d.ts +6 -0
- package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -0
- package/dist/domains/comet-native/native-cli-help.js +192 -0
- package/dist/domains/comet-native/native-cli-help.js.map +1 -0
- package/dist/domains/comet-native/native-cli-shared.d.ts +2 -2
- package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli-shared.js +46 -24
- package/dist/domains/comet-native/native-cli-shared.js.map +1 -1
- package/dist/domains/comet-native/native-cli.d.ts.map +1 -1
- package/dist/domains/comet-native/native-cli.js +18 -11
- package/dist/domains/comet-native/native-cli.js.map +1 -1
- package/dist/domains/comet-native/native-config.d.ts +2 -1
- package/dist/domains/comet-native/native-config.d.ts.map +1 -1
- package/dist/domains/comet-native/native-config.js +2 -1
- package/dist/domains/comet-native/native-config.js.map +1 -1
- package/dist/domains/comet-native/native-continuation.d.ts.map +1 -1
- package/dist/domains/comet-native/native-continuation.js +175 -147
- package/dist/domains/comet-native/native-continuation.js.map +1 -1
- package/dist/domains/comet-native/native-diagnostics.d.ts.map +1 -1
- package/dist/domains/comet-native/native-diagnostics.js +96 -13
- package/dist/domains/comet-native/native-diagnostics.js.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor-command.js +332 -0
- package/dist/domains/comet-native/native-doctor-command.js.map +1 -1
- package/dist/domains/comet-native/native-doctor.d.ts +1 -0
- package/dist/domains/comet-native/native-doctor.d.ts.map +1 -1
- package/dist/domains/comet-native/native-doctor.js +116 -22
- package/dist/domains/comet-native/native-doctor.js.map +1 -1
- package/dist/domains/comet-native/native-evidence-projection.d.ts +50 -0
- package/dist/domains/comet-native/native-evidence-projection.d.ts.map +1 -0
- package/dist/domains/comet-native/native-evidence-projection.js +284 -0
- package/dist/domains/comet-native/native-evidence-projection.js.map +1 -0
- package/dist/domains/comet-native/native-evidence-retention.d.ts.map +1 -1
- package/dist/domains/comet-native/native-evidence-retention.js +14 -12
- package/dist/domains/comet-native/native-evidence-retention.js.map +1 -1
- package/dist/domains/comet-native/native-evidence-storage.d.ts.map +1 -1
- package/dist/domains/comet-native/native-evidence-storage.js +14 -11
- package/dist/domains/comet-native/native-evidence-storage.js.map +1 -1
- package/dist/domains/comet-native/native-findings.d.ts.map +1 -1
- package/dist/domains/comet-native/native-findings.js +18 -0
- package/dist/domains/comet-native/native-findings.js.map +1 -1
- package/dist/domains/comet-native/native-git-provenance.d.ts +16 -0
- package/dist/domains/comet-native/native-git-provenance.d.ts.map +1 -0
- package/dist/domains/comet-native/native-git-provenance.js +171 -0
- package/dist/domains/comet-native/native-git-provenance.js.map +1 -0
- package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-native/native-hook-guard.js +199 -19
- package/dist/domains/comet-native/native-hook-guard.js.map +1 -1
- package/dist/domains/comet-native/native-init-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-init-command.js +14 -2
- package/dist/domains/comet-native/native-init-command.js.map +1 -1
- package/dist/domains/comet-native/native-local-execution.d.ts +25 -0
- package/dist/domains/comet-native/native-local-execution.d.ts.map +1 -0
- package/dist/domains/comet-native/native-local-execution.js +241 -0
- package/dist/domains/comet-native/native-local-execution.js.map +1 -0
- package/dist/domains/comet-native/native-lock.d.ts +1 -1
- package/dist/domains/comet-native/native-lock.d.ts.map +1 -1
- package/dist/domains/comet-native/native-lock.js +9 -9
- package/dist/domains/comet-native/native-lock.js.map +1 -1
- package/dist/domains/comet-native/native-loop-runtime.d.ts +63 -0
- package/dist/domains/comet-native/native-loop-runtime.d.ts.map +1 -0
- package/dist/domains/comet-native/native-loop-runtime.js +580 -0
- package/dist/domains/comet-native/native-loop-runtime.js.map +1 -0
- package/dist/domains/comet-native/native-mutation-lock.d.ts +2 -0
- package/dist/domains/comet-native/native-mutation-lock.d.ts.map +1 -1
- package/dist/domains/comet-native/native-mutation-lock.js +23 -2
- package/dist/domains/comet-native/native-mutation-lock.js.map +1 -1
- package/dist/domains/comet-native/native-new-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-new-command.js +27 -20
- package/dist/domains/comet-native/native-new-command.js.map +1 -1
- package/dist/domains/comet-native/native-next-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-next-command.js +193 -62
- package/dist/domains/comet-native/native-next-command.js.map +1 -1
- package/dist/domains/comet-native/native-paths.d.ts +18 -0
- package/dist/domains/comet-native/native-paths.d.ts.map +1 -1
- package/dist/domains/comet-native/native-paths.js +125 -13
- package/dist/domains/comet-native/native-paths.js.map +1 -1
- package/dist/domains/comet-native/native-portable-acceptance.d.ts +28 -0
- package/dist/domains/comet-native/native-portable-acceptance.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-acceptance.js +73 -0
- package/dist/domains/comet-native/native-portable-acceptance.js.map +1 -0
- package/dist/domains/comet-native/native-portable-archive.d.ts +37 -0
- package/dist/domains/comet-native/native-portable-archive.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-archive.js +492 -0
- package/dist/domains/comet-native/native-portable-archive.js.map +1 -0
- package/dist/domains/comet-native/native-portable-continuation.d.ts +30 -0
- package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-continuation.js +403 -0
- package/dist/domains/comet-native/native-portable-continuation.js.map +1 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts +9 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.js +213 -0
- package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -0
- package/dist/domains/comet-native/native-portable-migration.d.ts +42 -0
- package/dist/domains/comet-native/native-portable-migration.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-migration.js +212 -0
- package/dist/domains/comet-native/native-portable-migration.js.map +1 -0
- package/dist/domains/comet-native/native-portable-recovery.d.ts +15 -0
- package/dist/domains/comet-native/native-portable-recovery.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-recovery.js +203 -0
- package/dist/domains/comet-native/native-portable-recovery.js.map +1 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts +155 -0
- package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-runtime.js +1550 -0
- package/dist/domains/comet-native/native-portable-runtime.js.map +1 -0
- package/dist/domains/comet-native/native-portable-state.d.ts +30 -0
- package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-state.js +701 -0
- package/dist/domains/comet-native/native-portable-state.js.map +1 -0
- package/dist/domains/comet-native/native-portable-status.d.ts +65 -0
- package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-status.js +133 -0
- package/dist/domains/comet-native/native-portable-status.js.map +1 -0
- package/dist/domains/comet-native/native-portable-text.d.ts +8 -0
- package/dist/domains/comet-native/native-portable-text.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-text.js +28 -0
- package/dist/domains/comet-native/native-portable-text.js.map +1 -0
- package/dist/domains/comet-native/native-portable-transactions.d.ts +41 -0
- package/dist/domains/comet-native/native-portable-transactions.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-transactions.js +204 -0
- package/dist/domains/comet-native/native-portable-transactions.js.map +1 -0
- package/dist/domains/comet-native/native-portable-types.d.ts +166 -0
- package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -0
- package/dist/domains/comet-native/native-portable-types.js +18 -0
- package/dist/domains/comet-native/native-portable-types.js.map +1 -0
- package/dist/domains/comet-native/native-repair-integration.js +5 -5
- package/dist/domains/comet-native/native-repair-integration.js.map +1 -1
- package/dist/domains/comet-native/native-resume-view.d.ts.map +1 -1
- package/dist/domains/comet-native/native-resume-view.js +1 -5
- package/dist/domains/comet-native/native-resume-view.js.map +1 -1
- package/dist/domains/comet-native/native-root-move.d.ts.map +1 -1
- package/dist/domains/comet-native/native-root-move.js +4 -20
- package/dist/domains/comet-native/native-root-move.js.map +1 -1
- package/dist/domains/comet-native/native-run-consistency.d.ts.map +1 -1
- package/dist/domains/comet-native/native-run-consistency.js +8 -12
- package/dist/domains/comet-native/native-run-consistency.js.map +1 -1
- package/dist/domains/comet-native/native-run-store.js +4 -4
- package/dist/domains/comet-native/native-run-store.js.map +1 -1
- package/dist/domains/comet-native/native-runner-input.d.ts +160 -0
- package/dist/domains/comet-native/native-runner-input.d.ts.map +1 -0
- package/dist/domains/comet-native/native-runner-input.js +347 -0
- package/dist/domains/comet-native/native-runner-input.js.map +1 -0
- package/dist/domains/comet-native/native-runner-protocol.d.ts +40 -0
- package/dist/domains/comet-native/native-runner-protocol.d.ts.map +1 -0
- package/dist/domains/comet-native/native-runner-protocol.js +55 -0
- package/dist/domains/comet-native/native-runner-protocol.js.map +1 -0
- package/dist/domains/comet-native/native-runtime-package.d.ts +3 -0
- package/dist/domains/comet-native/native-runtime-package.d.ts.map +1 -1
- package/dist/domains/comet-native/native-runtime-package.js +5 -1
- package/dist/domains/comet-native/native-runtime-package.js.map +1 -1
- package/dist/domains/comet-native/native-schema-migration.d.ts.map +1 -1
- package/dist/domains/comet-native/native-schema-migration.js +30 -27
- package/dist/domains/comet-native/native-schema-migration.js.map +1 -1
- package/dist/domains/comet-native/native-select-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-select-command.js +8 -4
- package/dist/domains/comet-native/native-select-command.js.map +1 -1
- package/dist/domains/comet-native/native-selection.d.ts.map +1 -1
- package/dist/domains/comet-native/native-selection.js +11 -2
- package/dist/domains/comet-native/native-selection.js.map +1 -1
- package/dist/domains/comet-native/native-show-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-show-command.js +37 -0
- package/dist/domains/comet-native/native-show-command.js.map +1 -1
- package/dist/domains/comet-native/native-snapshot.d.ts.map +1 -1
- package/dist/domains/comet-native/native-snapshot.js +13 -10
- package/dist/domains/comet-native/native-snapshot.js.map +1 -1
- package/dist/domains/comet-native/native-spec-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-spec-command.js +14 -28
- package/dist/domains/comet-native/native-spec-command.js.map +1 -1
- package/dist/domains/comet-native/native-specs.js +3 -3
- package/dist/domains/comet-native/native-specs.js.map +1 -1
- package/dist/domains/comet-native/native-status-command.d.ts.map +1 -1
- package/dist/domains/comet-native/native-status-command.js +7 -17
- package/dist/domains/comet-native/native-status-command.js.map +1 -1
- package/dist/domains/comet-native/native-status-discovery.d.ts +51 -0
- package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -0
- package/dist/domains/comet-native/native-status-discovery.js +299 -0
- package/dist/domains/comet-native/native-status-discovery.js.map +1 -0
- package/dist/domains/comet-native/native-trajectory-recovery.d.ts.map +1 -1
- package/dist/domains/comet-native/native-trajectory-recovery.js +6 -9
- package/dist/domains/comet-native/native-trajectory-recovery.js.map +1 -1
- package/dist/domains/comet-native/native-transaction.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transaction.js +31 -20
- package/dist/domains/comet-native/native-transaction.js.map +1 -1
- package/dist/domains/comet-native/native-transition-evidence.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transition-evidence.js +1 -0
- package/dist/domains/comet-native/native-transition-evidence.js.map +1 -1
- package/dist/domains/comet-native/native-transition-journal.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transition-journal.js +71 -22
- package/dist/domains/comet-native/native-transition-journal.js.map +1 -1
- package/dist/domains/comet-native/native-transitions.d.ts.map +1 -1
- package/dist/domains/comet-native/native-transitions.js +167 -15
- package/dist/domains/comet-native/native-transitions.js.map +1 -1
- package/dist/domains/comet-native/native-types.d.ts +36 -1
- package/dist/domains/comet-native/native-types.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-receipt-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-receipt-runtime.js +18 -100
- package/dist/domains/comet-native/native-verification-receipt-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-verification-report-v2.d.ts +12 -0
- package/dist/domains/comet-native/native-verification-report-v2.d.ts.map +1 -0
- package/dist/domains/comet-native/native-verification-report-v2.js +141 -0
- package/dist/domains/comet-native/native-verification-report-v2.js.map +1 -0
- package/dist/domains/comet-native/native-verification-runtime.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-runtime.js +7 -2
- package/dist/domains/comet-native/native-verification-runtime.js.map +1 -1
- package/dist/domains/comet-native/native-verification-scope.d.ts +23 -0
- package/dist/domains/comet-native/native-verification-scope.d.ts.map +1 -1
- package/dist/domains/comet-native/native-verification-scope.js +106 -19
- package/dist/domains/comet-native/native-verification-scope.js.map +1 -1
- package/dist/domains/comet-native/native-verifier-protocol.d.ts +49 -0
- package/dist/domains/comet-native/native-verifier-protocol.d.ts.map +1 -0
- package/dist/domains/comet-native/native-verifier-protocol.js +157 -0
- package/dist/domains/comet-native/native-verifier-protocol.js.map +1 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts +61 -0
- package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -0
- package/dist/domains/comet-native/native-workspace-finish.js +332 -0
- package/dist/domains/comet-native/native-workspace-finish.js.map +1 -0
- package/dist/domains/comet-native/native-workspace-preparation.d.ts +32 -0
- package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -0
- package/dist/domains/comet-native/native-workspace-preparation.js +324 -0
- package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -0
- package/dist/domains/comet-native/native-workspace.d.ts +9 -1
- package/dist/domains/comet-native/native-workspace.d.ts.map +1 -1
- package/dist/domains/comet-native/native-workspace.js +109 -11
- package/dist/domains/comet-native/native-workspace.js.map +1 -1
- package/dist/domains/dashboard/collector.d.ts +1 -1
- package/dist/domains/dashboard/collector.d.ts.map +1 -1
- package/dist/domains/dashboard/collector.js +102 -87
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/native-adapter.d.ts +155 -126
- package/dist/domains/dashboard/native-adapter.d.ts.map +1 -1
- package/dist/domains/dashboard/native-adapter.js +165 -268
- package/dist/domains/dashboard/native-adapter.js.map +1 -1
- package/dist/domains/dashboard/native-collector.d.ts +5 -4
- package/dist/domains/dashboard/native-collector.d.ts.map +1 -1
- package/dist/domains/dashboard/native-collector.js +506 -439
- package/dist/domains/dashboard/native-collector.js.map +1 -1
- package/dist/domains/dashboard/native-legacy-adapter.d.ts +37 -0
- package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -0
- package/dist/domains/dashboard/native-legacy-adapter.js +127 -0
- package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -0
- package/dist/domains/dashboard/server.d.ts.map +1 -1
- package/dist/domains/dashboard/server.js +5 -3
- package/dist/domains/dashboard/server.js.map +1 -1
- package/dist/domains/dashboard/types.d.ts +5 -0
- package/dist/domains/dashboard/types.d.ts.map +1 -1
- package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-B5cFR6hS.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-CVQDtMsu.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-C0v0kULK.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-DpHLuxu4.js +132 -0
- package/dist/domains/dashboard/web/assets/{c4Diagram-AAUBKEIU-Ca5zba3N.js → c4Diagram-5PPSVZJV-BHnFxUs1.js} +1 -1
- package/dist/domains/dashboard/web/assets/channel-eM0x3m1G.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-ND2GUHAM-Bj6HWRwi.js → chunk-2GRJ4B5K-CGC9gh5n.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-QZHKN3VN-B1kXfg5N.js → chunk-2Q5K7J3B-l4puLU_O.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ls9jQHZc.js +231 -0
- package/dist/domains/dashboard/web/assets/{chunk-FMBD7UC4-BAJLB5V6.js → chunk-5VM5RSS4-BVavmTEq.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-Dungt2g9.js +88 -0
- package/dist/domains/dashboard/web/assets/{chunk-727SXJPM-CwghIMO9.js → chunk-GF5L2VYU-BidXO_ZP.js} +5 -5
- package/dist/domains/dashboard/web/assets/{chunk-4BX2VUAB-JYluRsP-.js → chunk-JWPE2WC7-57L5Up5X.js} +1 -1
- package/dist/domains/dashboard/web/assets/{chunk-2J33WTMH-CCVPBm0J.js → chunk-KBJHAD2P-C5u5vbwr.js} +1 -1
- package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-CvLz21I2.js +1 -0
- package/dist/domains/dashboard/web/assets/{chunk-55IACEB6-DfYm7pN7.js → chunk-XXDRQBXY-D5nJpckG.js} +1 -1
- package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CEFhao2K.js +1 -0
- package/dist/domains/dashboard/web/assets/{cose-bilkent-S5V4N54A-BBrkuiAU.js → cose-bilkent-JH36ORCC-Dp0JAGyE.js} +1 -1
- package/dist/domains/dashboard/web/assets/{cynefin-VYW2F7L2-DlFhgQHt.js → cynefin-VYW2F7L2-EjC54l1g.js} +2 -2
- package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-CpYrW3w3.js +62 -0
- package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-h5K39rCk.js +4 -0
- package/dist/domains/dashboard/web/assets/demo-BfLsKrIL.js +45 -0
- package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-CKQj9O4H.js +30 -0
- package/dist/domains/dashboard/web/assets/{diagram-KO2AKTUF-BQahyP-C.js → diagram-B4RE2ZJO-DXRIYcnW.js} +3 -3
- package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-wWutQYmv.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-EgXLJQYi.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CsQRXDQZ.js +41 -0
- package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-gm328Eho.js +1 -0
- package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-BbbMVUx_.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-1Y7scCns.js +156 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-BAoBqczv.js +292 -0
- package/dist/domains/dashboard/web/assets/{gitGraphDiagram-PVQCEYII-DCxvMAUJ.js → gitGraphDiagram-DS77QQ5N-CMUAyEzl.js} +3 -3
- package/dist/domains/dashboard/web/assets/{graph--OzhPTMs.js → graph-DOmOIIwC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{index-Blit0P5Z.js → index-CCApJ0Yr.js} +1 -1
- package/dist/domains/dashboard/web/assets/index-DWdm_XBi.js +257 -0
- package/dist/domains/dashboard/web/assets/index-DhhbseFJ.css +1 -0
- package/dist/domains/dashboard/web/assets/{infoDiagram-5YYISTIA-9qXEkzDe.js → infoDiagram-6WML65LV-BaCdtYYC.js} +1 -1
- package/dist/domains/dashboard/web/assets/{ishikawaDiagram-YF4QCWOH-Dkf5othW.js → ishikawaDiagram-WSZJBQD7-DQQlNnjU.js} +4 -4
- package/dist/domains/dashboard/web/assets/{journeyDiagram-JHISSGLW-C2_IkO4s.js → journeyDiagram-NVQOT4AX-DW9wUMqn.js} +1 -1
- package/dist/domains/dashboard/web/assets/{kanban-definition-UN3LZRKU-BxWgu4DL.js → kanban-definition-27J2QSJJ-C_ITaZ0q.js} +7 -7
- package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
- package/dist/domains/dashboard/web/assets/{linear-Dkr6p-Ro.js → linear-DXOo6ZRT.js} +1 -1
- package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-ol-0NI97.js +306 -0
- package/dist/domains/dashboard/web/assets/{mindmap-definition-RKZ34NQL-C9xyxGfa.js → mindmap-definition-FAOFIHXS-CHF_HOLV.js} +1 -1
- package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-wQEvkNF8.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-C3H3kiIP.js +39 -0
- package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
- package/dist/domains/dashboard/web/assets/{quadrantDiagram-W4KKPZXB-Cy6iOcxc.js → quadrantDiagram-CIZ2JOQS-S0FdVzJb.js} +4 -4
- package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-DCDXDaVg.js +1 -0
- package/dist/domains/dashboard/web/assets/{requirementDiagram-4Y6WPE33-B_gZONhx.js → requirementDiagram-LRYGKXZP-BW8J4gO_.js} +1 -1
- package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-DQQBmLoy.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-2tDcli0Y.js +162 -0
- package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-4sEZLBKb.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-OgPb8pyR.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-CiDNPr4P.js +1 -0
- package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-DR7lwtxd.js +2 -0
- package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-DggHFP9q.js +8 -0
- package/dist/domains/dashboard/web/assets/{timeline-definition-PNZ67QCA-DvONX2YQ.js → timeline-definition-Z64GVDOM-DNIJVlQV.js} +4 -4
- package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-CClhw4QZ.js +34 -0
- package/dist/domains/dashboard/web/assets/{wardleyDiagram-YWT4CUSO-cf39WX2o.js → wardleyDiagram-T6FBY63Y-DzR3aFPS.js} +3 -3
- package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DZhq0ZRj.js +7 -0
- package/dist/domains/dashboard/web/index.html +16 -16
- package/dist/domains/dashboard/workspace.d.ts +24 -0
- package/dist/domains/dashboard/workspace.d.ts.map +1 -0
- package/dist/domains/dashboard/workspace.js +87 -0
- package/dist/domains/dashboard/workspace.js.map +1 -0
- package/dist/domains/eval/index.d.ts +3 -0
- package/dist/domains/eval/index.d.ts.map +1 -1
- package/dist/domains/eval/index.js +3 -0
- package/dist/domains/eval/index.js.map +1 -1
- package/dist/domains/eval/standalone-context.d.ts +19 -0
- package/dist/domains/eval/standalone-context.d.ts.map +1 -0
- package/dist/domains/eval/standalone-context.js +107 -0
- package/dist/domains/eval/standalone-context.js.map +1 -0
- package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
- package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
- package/dist/domains/eval/standalone-static-collect.js +779 -0
- package/dist/domains/eval/standalone-static-collect.js.map +1 -0
- package/dist/domains/eval/user-environment.d.ts +6 -0
- package/dist/domains/eval/user-environment.d.ts.map +1 -0
- package/dist/domains/eval/user-environment.js +73 -0
- package/dist/domains/eval/user-environment.js.map +1 -0
- package/dist/domains/integrations/openspec.d.ts +1 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +110 -26
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/integrations/superpowers.d.ts +5 -1
- package/dist/domains/integrations/superpowers.d.ts.map +1 -1
- package/dist/domains/integrations/superpowers.js +21 -2
- package/dist/domains/integrations/superpowers.js.map +1 -1
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -1
- package/dist/domains/skill/platform-inspect.js +11 -3
- package/dist/domains/skill/platform-inspect.js.map +1 -1
- package/dist/domains/skill/platform-install.d.ts +8 -2
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +115 -34
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts.map +1 -1
- package/dist/domains/skill/project-instructions.js +5 -3
- package/dist/domains/skill/project-instructions.js.map +1 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +69 -15
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
- package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
- package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
- package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
- package/dist/domains/workflow-contract/index.d.ts +3 -0
- package/dist/domains/workflow-contract/index.d.ts.map +1 -1
- package/dist/domains/workflow-contract/index.js +2 -0
- package/dist/domains/workflow-contract/index.js.map +1 -1
- package/dist/domains/workflow-contract/project-config.d.ts +3 -1
- package/dist/domains/workflow-contract/project-config.d.ts.map +1 -1
- package/dist/domains/workflow-contract/project-config.js +119 -19
- package/dist/domains/workflow-contract/project-config.js.map +1 -1
- package/dist/domains/workflow-contract/project-gitignore.d.ts +9 -0
- package/dist/domains/workflow-contract/project-gitignore.d.ts.map +1 -0
- package/dist/domains/workflow-contract/project-gitignore.js +108 -0
- package/dist/domains/workflow-contract/project-gitignore.js.map +1 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts +13 -0
- package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -1
- package/dist/domains/workflow-contract/protected-project-path.js +83 -0
- package/dist/domains/workflow-contract/protected-project-path.js.map +1 -1
- package/dist/domains/workflow-contract/types.d.ts +4 -0
- package/dist/domains/workflow-contract/types.d.ts.map +1 -1
- package/dist/platform/install/platforms.d.ts +3 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +22 -2
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/paths/git-worktree.d.ts +8 -1
- package/dist/platform/paths/git-worktree.d.ts.map +1 -1
- package/dist/platform/paths/git-worktree.js +41 -7
- package/dist/platform/paths/git-worktree.js.map +1 -1
- package/dist/platform/process/external-command.d.ts +13 -0
- package/dist/platform/process/external-command.d.ts.map +1 -0
- package/dist/platform/process/external-command.js +45 -0
- package/dist/platform/process/external-command.js.map +1 -0
- package/dist/platform/process/git.d.ts +12 -0
- package/dist/platform/process/git.d.ts.map +1 -0
- package/dist/platform/process/git.js +86 -0
- package/dist/platform/process/git.js.map +1 -0
- package/dist/platform/process/spawn-command.d.ts +8 -0
- package/dist/platform/process/spawn-command.d.ts.map +1 -0
- package/dist/platform/process/spawn-command.js +113 -0
- package/dist/platform/process/spawn-command.js.map +1 -0
- package/eval/.env.example +115 -0
- package/eval/README.md +237 -14
- package/eval/langfuse/.env.example +9 -0
- package/eval/langfuse/tests/conftest.py +146 -0
- package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
- package/eval/langfuse/tests/test_adapter.py +272 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/tests/conftest.py +10 -1
- package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +4 -2
- package/eval/local/tasks/comet-native-clarification/validation/test_native_clarification.py +9 -21
- package/eval/local/tasks/comet-native-clarification-depth/validation/test_native_clarification_depth.py +4 -7
- package/eval/local/tasks/comet-native-clarification-modes/validation/test_native_clarification_modes.py +4 -10
- package/eval/local/tasks/comet-native-interrupted-transition/instruction.md +1 -1
- package/eval/local/tasks/comet-native-interrupted-transition/validation/test_native_interrupted_transition.py +15 -16
- package/eval/local/tasks/comet-native-repository-fact/instruction.md +2 -2
- package/eval/local/tasks/comet-native-repository-fact/validation/test_native_repository_fact.py +8 -2
- package/eval/local/tasks/comet-native-wave-b-decision-resume/instruction.md +8 -4
- package/eval/local/tasks/comet-native-wave-b-decision-resume/task.toml +6 -6
- package/eval/local/tasks/comet-native-wave-b-decision-resume/validation/test_native_wave_b_decision_resume.py +126 -135
- package/eval/local/tasks/comet-native-wave-c-verification-integrity/instruction.md +6 -6
- package/eval/local/tasks/comet-native-wave-c-verification-integrity/task.toml +8 -8
- package/eval/local/tasks/comet-native-wave-c-verification-integrity/validation/test_native_wave_c_verification_integrity.py +97 -317
- package/eval/local/tasks/comet-native-wave-d-stagnation-stop/instruction.md +7 -7
- package/eval/local/tasks/comet-native-wave-d-stagnation-stop/task.toml +4 -4
- package/eval/local/tasks/comet-native-wave-d-stagnation-stop/validation/test_native_wave_d_stagnation_stop.py +130 -199
- package/eval/local/tasks/comet-native-wave-e-parallel-safety/instruction.md +5 -5
- package/eval/local/tasks/comet-native-wave-e-parallel-safety/task.toml +8 -8
- package/eval/local/tasks/comet-native-wave-e-parallel-safety/validation/test_native_wave_e_parallel_safety.py +135 -239
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +2 -2
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/instruction.md +3 -3
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/task.toml +6 -6
- package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/validation/test_native_wave_f_dashboard_readonly.py +81 -439
- package/eval/local/tasks/comet-native-workflow/instruction.md +5 -4
- package/eval/local/tasks/comet-native-workflow/validation/test_native_workflow.py +145 -502
- package/eval/local/tasks/index.yaml +5 -5
- package/eval/local/tests/conftest.py +701 -80
- package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
- package/eval/local/tests/scaffold/test_agents.py +76 -0
- package/eval/local/tests/scaffold/test_aligned_comparison.py +70 -0
- package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +267 -18
- package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
- package/eval/local/tests/scaffold/test_eval_context.py +217 -0
- package/eval/local/tests/scaffold/test_execution.py +364 -0
- package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
- package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +63 -0
- package/eval/local/tests/scaffold/test_logging.py +162 -0
- package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
- package/eval/local/tests/scaffold/test_manifests.py +396 -0
- package/eval/local/tests/scaffold/test_native_aligned_eval.py +163 -35
- package/eval/local/tests/scaffold/test_native_wave_evaluations.py +75 -2054
- package/eval/local/tests/scaffold/test_profiles.py +64 -6
- package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
- package/eval/local/tests/scaffold/test_tasks.py +38 -52
- package/eval/local/tests/scaffold/test_utils.py +480 -4
- package/eval/local/tests/tasks/test_tasks.py +219 -17
- package/eval/local/treatments/comet/comet_native_phase1.yaml +7 -4
- package/eval/pyproject.toml +9 -4
- package/eval/scaffold/__init__.py +6 -0
- package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
- package/eval/scaffold/python/__init__.py +128 -20
- package/eval/scaffold/python/agents.py +239 -0
- package/eval/scaffold/python/aligned_comparison.py +156 -27
- package/eval/scaffold/python/auto_tasks.py +700 -0
- package/eval/scaffold/python/custom_agents.py +172 -0
- package/eval/scaffold/python/eval_context.py +236 -0
- package/eval/scaffold/python/eval_schema.py +135 -0
- package/eval/scaffold/python/execution.py +544 -0
- package/eval/scaffold/python/generated_task_cache.py +230 -0
- package/eval/scaffold/python/generic_llm_judge.py +41 -14
- package/eval/scaffold/python/inline_tasks.py +118 -0
- package/eval/scaffold/python/judge_config.py +158 -37
- package/eval/scaffold/python/langfuse_adapter.py +1023 -0
- package/eval/scaffold/python/llm_judge.py +68 -11
- package/eval/scaffold/python/logging.py +157 -17
- package/eval/scaffold/python/manifest_tasks.py +69 -0
- package/eval/scaffold/python/manifests.py +443 -22
- package/eval/scaffold/python/native_eval.py +10 -5
- package/eval/scaffold/python/paths.py +11 -1
- package/eval/scaffold/python/task_resolution.py +176 -0
- package/eval/scaffold/python/tasks.py +17 -4
- package/eval/scaffold/python/utils.py +167 -11
- package/eval/scaffold/python/validation/core.py +48 -2
- package/eval/scaffold/python/validation/generic_rubric.py +9 -0
- package/eval/scaffold/python/validation/native_v4.py +249 -0
- package/eval/scaffold/python/validation/native_wave.py +4 -5
- package/eval/scaffold/python/validation/native_workflow.py +199 -98
- package/eval/scaffold/python/validation/rubric.py +130 -2
- package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
- package/eval/scaffold/shell/docker.sh +534 -33
- package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
- package/eval/scaffold/shell/run-claude-loop.sh +153 -14
- package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
- package/package.json +9 -3
- package/assets/skills/comet-native/scripts/comet-native-check.mjs +0 -179
- package/assets/skills/comet-native/scripts/comet-native-checkpoint.mjs +0 -179
- package/assets/skills/comet-native/scripts/comet-native-evidence.mjs +0 -151
- package/assets/skills/comet-native/scripts/comet-native-receipt.mjs +0 -187
- package/dist/domains/comet-native/native-check-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-check-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-check-entry.js +0 -4
- package/dist/domains/comet-native/native-check-entry.js.map +0 -1
- package/dist/domains/comet-native/native-checkpoint-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-checkpoint-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-checkpoint-entry.js +0 -4
- package/dist/domains/comet-native/native-checkpoint-entry.js.map +0 -1
- package/dist/domains/comet-native/native-evidence-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-evidence-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-evidence-entry.js +0 -4
- package/dist/domains/comet-native/native-evidence-entry.js.map +0 -1
- package/dist/domains/comet-native/native-receipt-entry.d.ts +0 -2
- package/dist/domains/comet-native/native-receipt-entry.d.ts.map +0 -1
- package/dist/domains/comet-native/native-receipt-entry.js +0 -4
- package/dist/domains/comet-native/native-receipt-entry.js.map +0 -1
- package/dist/domains/dashboard/web/assets/arc-ClrSujVb.js +0 -1
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR-B4A_t84A.js +0 -36
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-D72u2953.js +0 -132
- package/dist/domains/dashboard/web/assets/channel-B_u-_i2t.js +0 -1
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BA_zgMXZ.js +0 -231
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-BpA7BPgX.js +0 -1
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-BpA7BPgX.js +0 -1
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-B0Bk5oiP.js +0 -4
- package/dist/domains/dashboard/web/assets/demo-DZYPLZpF.js +0 -25
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-tvTUiBA2.js +0 -43
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-BZfcK29q.js +0 -10
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-D5bJnW2C.js +0 -24
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-DTwWdzWJ.js +0 -24
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-BbvrdCp9.js +0 -85
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-DgbR7Tcc.js +0 -162
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-wWMOA7F4.js +0 -292
- package/dist/domains/dashboard/web/assets/index-Bk1o-4VO.css +0 -1
- package/dist/domains/dashboard/web/assets/index-g8vyW0R_.js +0 -257
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +0 -1
- package/dist/domains/dashboard/web/assets/mermaid.core-DZUcsoG4.js +0 -301
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-Dn67tdOF.js +0 -30
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +0 -3
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-C3u-4dOk.js +0 -40
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-Xd3ByTVF.js +0 -162
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-CJGHL7Up.js +0 -1
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-rKvBgr6I.js +0 -1
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-B-L4qWy4.js +0 -34
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-Dk1vcm3u.js +0 -7
- /package/assets/skills/{comet → comet-classic}/reference/classic-layout.md +0 -0
- /package/assets/skills-zh/{comet → comet-classic}/reference/classic-layout.md +0 -0
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""The sole, import-safe authority for Eval task selection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
from scaffold.python.eval_context import ResolvedEvalContext
|
|
10
|
+
from scaffold.python.execution import ResolvedExecution, resolve_execution
|
|
11
|
+
from scaffold.python.generated_task_cache import (
|
|
12
|
+
GeneratedCache,
|
|
13
|
+
build_skill_snapshot,
|
|
14
|
+
cache_location,
|
|
15
|
+
generation_hash,
|
|
16
|
+
load_generated_cache,
|
|
17
|
+
selected_agent_model,
|
|
18
|
+
)
|
|
19
|
+
from scaffold.python.manifest_tasks import load_manifest_tasks
|
|
20
|
+
from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
|
|
21
|
+
from scaffold.python.tasks import Task, load_task
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
TaskSource = Literal[
|
|
25
|
+
"explicit", "quick", "authored", "recommended", "generated-cache", "pending-generation"
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class GenerationPlan:
|
|
31
|
+
generation_hash: str
|
|
32
|
+
cache_dir: Path
|
|
33
|
+
cached: GeneratedCache | None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class ResolvedTask:
|
|
38
|
+
name: str
|
|
39
|
+
provenance: Literal["inline", "source", "bundled", "generated"]
|
|
40
|
+
task: Task
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class ResolvedTaskSet:
|
|
45
|
+
source: TaskSource
|
|
46
|
+
tasks: tuple[ResolvedTask, ...]
|
|
47
|
+
generation: GenerationPlan | None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class TaskCatalogue:
|
|
52
|
+
bundled: dict[str, ResolvedTask]
|
|
53
|
+
canonical: dict[str, ResolvedTask]
|
|
54
|
+
authored: dict[str, ResolvedTask]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _bundled(tasks_dir: Path) -> dict[str, ResolvedTask]:
|
|
58
|
+
result: dict[str, ResolvedTask] = {}
|
|
59
|
+
canonical: dict[str, str] = {}
|
|
60
|
+
if not tasks_dir.is_dir():
|
|
61
|
+
return result
|
|
62
|
+
for directory in sorted(path for path in tasks_dir.iterdir() if path.is_dir()):
|
|
63
|
+
if not (directory / "task.toml").is_file() or not (directory / "instruction.md").is_file():
|
|
64
|
+
continue
|
|
65
|
+
task = load_task(directory, tasks_dir=tasks_dir)
|
|
66
|
+
if task.name in canonical:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f'bundled tasks "{canonical[task.name]}" and "{directory.name}" both declare '
|
|
69
|
+
f'metadata.name "{task.name}"'
|
|
70
|
+
)
|
|
71
|
+
canonical[task.name] = directory.name
|
|
72
|
+
result[directory.name] = ResolvedTask(task.name, "bundled", task)
|
|
73
|
+
return result
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def build_task_catalogue(manifest: SkillEvalManifest, tasks_dir: Path) -> TaskCatalogue:
|
|
77
|
+
bundled = _bundled(tasks_dir)
|
|
78
|
+
canonical = {item.name: item for item in bundled.values()}
|
|
79
|
+
authored_tasks = load_manifest_tasks(manifest)
|
|
80
|
+
authored: dict[str, ResolvedTask] = {}
|
|
81
|
+
for index, task in enumerate(authored_tasks):
|
|
82
|
+
source = manifest.tasks[index]
|
|
83
|
+
authored[task.name] = ResolvedTask(task.name, "inline" if source.is_inline else "source", task)
|
|
84
|
+
for index, task_name in enumerate(manifest.recommended_tasks):
|
|
85
|
+
if task_name not in bundled:
|
|
86
|
+
raise ValueError(f'evaluation.recommendedTasks[{index}]: unknown bundled task "{task_name}"')
|
|
87
|
+
if task_name in authored:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f'evaluation.tasks[{next(i for i, item in enumerate(manifest.tasks) if item.name == task_name)}].name '
|
|
90
|
+
f'conflicts with evaluation.recommendedTasks[{index}]: "{task_name}"'
|
|
91
|
+
)
|
|
92
|
+
for index, spec in enumerate(manifest.tasks):
|
|
93
|
+
if spec.name in canonical:
|
|
94
|
+
bundle = next(key for key, item in bundled.items() if item.name == spec.name)
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f'evaluation.tasks[{index}].name conflicts with bundled task "{bundle}": "{spec.name}"'
|
|
97
|
+
)
|
|
98
|
+
return TaskCatalogue(bundled, canonical, authored)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _generated_plan(
|
|
102
|
+
context: ResolvedEvalContext,
|
|
103
|
+
manifest: SkillEvalManifest,
|
|
104
|
+
execution: ResolvedExecution | None = None,
|
|
105
|
+
) -> GenerationPlan:
|
|
106
|
+
snapshot = build_skill_snapshot(context.skill_root)
|
|
107
|
+
resolved_execution = execution or resolve_execution(manifest=manifest.execution)
|
|
108
|
+
selected_agent = resolved_execution.agent
|
|
109
|
+
selected_model = resolved_execution.model or selected_agent_model(selected_agent)
|
|
110
|
+
digest = generation_hash(
|
|
111
|
+
snapshot,
|
|
112
|
+
agent=selected_agent,
|
|
113
|
+
model=selected_model,
|
|
114
|
+
base_url=resolved_execution.base_url,
|
|
115
|
+
profile=manifest.profile or "generic",
|
|
116
|
+
interaction=vars(manifest.interaction),
|
|
117
|
+
)
|
|
118
|
+
cache_dir = cache_location(context.artifact_owner_root, context.skill_root, digest)
|
|
119
|
+
return GenerationPlan(digest, cache_dir, load_generated_cache(cache_dir, digest))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def resolve_task_set(
|
|
123
|
+
context: ResolvedEvalContext,
|
|
124
|
+
manifest: SkillEvalManifest,
|
|
125
|
+
catalogue: TaskCatalogue,
|
|
126
|
+
*,
|
|
127
|
+
explicit_task: str | None = None,
|
|
128
|
+
quick: bool = False,
|
|
129
|
+
static_collect: bool = False,
|
|
130
|
+
execution: ResolvedExecution | None = None,
|
|
131
|
+
) -> ResolvedTaskSet:
|
|
132
|
+
"""Resolve exactly one precedence branch without probing lower branches."""
|
|
133
|
+
if explicit_task:
|
|
134
|
+
selected = catalogue.authored.get(explicit_task) or catalogue.bundled.get(explicit_task) or catalogue.canonical.get(explicit_task)
|
|
135
|
+
if selected is None:
|
|
136
|
+
available = sorted({*catalogue.authored, *catalogue.bundled, *catalogue.canonical})
|
|
137
|
+
raise ValueError(f'Task not found: {explicit_task}. Available: {available}')
|
|
138
|
+
return ResolvedTaskSet("explicit", (selected,), None)
|
|
139
|
+
if quick:
|
|
140
|
+
selected = catalogue.bundled.get("generic-skill-smoke")
|
|
141
|
+
if selected is None:
|
|
142
|
+
raise ValueError('Bundled quick task generic-skill-smoke is unavailable')
|
|
143
|
+
return ResolvedTaskSet("quick", (selected,), None)
|
|
144
|
+
if manifest.tasks:
|
|
145
|
+
return ResolvedTaskSet("authored", tuple(catalogue.authored[item.name] for item in manifest.tasks), None)
|
|
146
|
+
if manifest.recommended_tasks:
|
|
147
|
+
return ResolvedTaskSet("recommended", tuple(catalogue.bundled[item] for item in manifest.recommended_tasks), None)
|
|
148
|
+
plan = _generated_plan(context, manifest, execution)
|
|
149
|
+
if plan.cached is None:
|
|
150
|
+
return ResolvedTaskSet("pending-generation", (), plan)
|
|
151
|
+
generated = load_eval_manifest(plan.cached.manifest_path)
|
|
152
|
+
loaded = load_manifest_tasks(generated)
|
|
153
|
+
names: set[str] = set()
|
|
154
|
+
tasks: list[ResolvedTask] = []
|
|
155
|
+
for index, task in enumerate(loaded):
|
|
156
|
+
if task.name in names:
|
|
157
|
+
first_index = next(
|
|
158
|
+
item_index
|
|
159
|
+
for item_index, item in enumerate(loaded[:index])
|
|
160
|
+
if item.name == task.name
|
|
161
|
+
)
|
|
162
|
+
raise ValueError(
|
|
163
|
+
f'generated.tasks[{index}].name duplicates '
|
|
164
|
+
f'generated.tasks[{first_index}].name: "{task.name}"'
|
|
165
|
+
)
|
|
166
|
+
if task.name in catalogue.canonical:
|
|
167
|
+
bundle = next(
|
|
168
|
+
key for key, item in catalogue.bundled.items() if item.name == task.name
|
|
169
|
+
)
|
|
170
|
+
raise ValueError(
|
|
171
|
+
f'generated.tasks[{index}].name conflicts with bundled task '
|
|
172
|
+
f'"{bundle}": "{task.name}"'
|
|
173
|
+
)
|
|
174
|
+
names.add(task.name)
|
|
175
|
+
tasks.append(ResolvedTask(task.name, "generated", task))
|
|
176
|
+
return ResolvedTaskSet("generated-cache", tuple(tasks), plan)
|
|
@@ -148,6 +148,10 @@ class Task:
|
|
|
148
148
|
path: Path
|
|
149
149
|
config: TaskConfig
|
|
150
150
|
instruction_template: str
|
|
151
|
+
_environment_dir: Path | None = field(default=None, repr=False)
|
|
152
|
+
workspace_dir: Path | None = field(default=None, repr=False)
|
|
153
|
+
manifest_expectations: dict[str, Any] | None = field(default=None, repr=False)
|
|
154
|
+
manifest_path: Path | None = field(default=None, repr=False)
|
|
151
155
|
|
|
152
156
|
@property
|
|
153
157
|
def name(self) -> str:
|
|
@@ -155,7 +159,7 @@ class Task:
|
|
|
155
159
|
|
|
156
160
|
@property
|
|
157
161
|
def environment_dir(self) -> Path:
|
|
158
|
-
return self.path / "environment"
|
|
162
|
+
return self._environment_dir or self.path / "environment"
|
|
159
163
|
|
|
160
164
|
@property
|
|
161
165
|
def validation_dir(self) -> Path:
|
|
@@ -195,6 +199,11 @@ class Task:
|
|
|
195
199
|
The config specifies test_scripts and target_artifacts. The framework
|
|
196
200
|
auto-builds an execution validator that runs the scripts in Docker.
|
|
197
201
|
"""
|
|
202
|
+
if self.manifest_expectations is not None:
|
|
203
|
+
from scaffold.python.inline_tasks import make_inline_expectations_validator
|
|
204
|
+
|
|
205
|
+
return [make_inline_expectations_validator(self.manifest_expectations)]
|
|
206
|
+
|
|
198
207
|
vc = self.config.validation
|
|
199
208
|
if vc.test_scripts:
|
|
200
209
|
from scaffold.python.utils import make_execution_validator
|
|
@@ -212,7 +221,7 @@ class Task:
|
|
|
212
221
|
return []
|
|
213
222
|
|
|
214
223
|
|
|
215
|
-
def load_task(name: str, tasks_dir: Path | None = None) -> Task:
|
|
224
|
+
def load_task(name: str | Path, tasks_dir: Path | None = None) -> Task:
|
|
216
225
|
"""Load a task by name.
|
|
217
226
|
|
|
218
227
|
Args:
|
|
@@ -222,8 +231,7 @@ def load_task(name: str, tasks_dir: Path | None = None) -> Task:
|
|
|
222
231
|
Returns:
|
|
223
232
|
Task object with config and instruction template
|
|
224
233
|
"""
|
|
225
|
-
|
|
226
|
-
task_path = tasks_dir / name
|
|
234
|
+
task_path = Path(name) if isinstance(name, Path) else (tasks_dir or get_tasks_dir()) / name
|
|
227
235
|
|
|
228
236
|
if not task_path.exists():
|
|
229
237
|
raise FileNotFoundError(f"Task not found: {name} (looked in {tasks_dir})")
|
|
@@ -323,6 +331,11 @@ def load_task(name: str, tasks_dir: Path | None = None) -> Task:
|
|
|
323
331
|
return Task(path=task_path, config=config, instruction_template=instruction_template)
|
|
324
332
|
|
|
325
333
|
|
|
334
|
+
def load_task_from_path(task_path: Path | str) -> Task:
|
|
335
|
+
"""Load an advanced task package referenced by a project eval manifest."""
|
|
336
|
+
return load_task(Path(task_path))
|
|
337
|
+
|
|
338
|
+
|
|
326
339
|
def list_tasks(tasks_dir: Path | None = None) -> list[str]:
|
|
327
340
|
"""List available task names.
|
|
328
341
|
|
|
@@ -4,6 +4,7 @@ import json
|
|
|
4
4
|
import ntpath
|
|
5
5
|
import os
|
|
6
6
|
import random
|
|
7
|
+
import re
|
|
7
8
|
import shutil
|
|
8
9
|
import subprocess
|
|
9
10
|
import time
|
|
@@ -11,6 +12,8 @@ from pathlib import Path
|
|
|
11
12
|
|
|
12
13
|
from dotenv import load_dotenv
|
|
13
14
|
|
|
15
|
+
from scaffold.python.agents import validate_agent_id
|
|
16
|
+
from scaffold.python.execution import ResolvedExecution, build_agent_environment
|
|
14
17
|
from scaffold.python.paths import EVAL_ROOT, get_suite_root
|
|
15
18
|
|
|
16
19
|
TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
|
|
@@ -21,8 +24,14 @@ SCAFFOLD_PYTHON_DIR = Path(__file__).parent
|
|
|
21
24
|
|
|
22
25
|
def load_eval_environment() -> None:
|
|
23
26
|
"""Load eval credentials only at an explicit execution boundary, never on import."""
|
|
24
|
-
|
|
25
|
-
|
|
27
|
+
original_environment = dict(os.environ)
|
|
28
|
+
try:
|
|
29
|
+
load_dotenv(EVAL_ROOT / ".env", override=True)
|
|
30
|
+
load_dotenv(get_suite_root() / ".env", override=True)
|
|
31
|
+
load_dotenv(Path.home() / ".comet" / "eval" / ".env", override=True)
|
|
32
|
+
finally:
|
|
33
|
+
for key, value in original_environment.items():
|
|
34
|
+
os.environ[key] = value
|
|
26
35
|
|
|
27
36
|
|
|
28
37
|
def _uses_wsl_bash(bash_exec: str) -> bool:
|
|
@@ -69,12 +78,28 @@ def _resolve_bash(os_name: str | None = None) -> str:
|
|
|
69
78
|
|
|
70
79
|
BASH_EXEC = _resolve_bash()
|
|
71
80
|
WSL_ENV_KEYS = (
|
|
81
|
+
"BENCH_EVAL_AGENT",
|
|
72
82
|
"OPENAI_API_KEY",
|
|
83
|
+
"OPENAI_BASE_URL",
|
|
84
|
+
"OPENAI_MODEL",
|
|
85
|
+
"QODER_PERSONAL_ACCESS_TOKEN",
|
|
86
|
+
"QODER_BASE_URL",
|
|
87
|
+
"QODER_MODEL",
|
|
88
|
+
"CODEX_API_KEY",
|
|
89
|
+
"CODEX_BASE_URL",
|
|
90
|
+
"CODEX_MODEL",
|
|
73
91
|
"ANTHROPIC_API_KEY",
|
|
74
92
|
"LANGSMITH_API_KEY",
|
|
75
93
|
"LANGSMITH_PROJECT",
|
|
76
94
|
"LANGSMITH_TRACING",
|
|
77
95
|
"LANGSMITH_ENDPOINT",
|
|
96
|
+
"LANGFUSE_PUBLIC_KEY",
|
|
97
|
+
"LANGFUSE_SECRET_KEY",
|
|
98
|
+
"LANGFUSE_BASE_URL",
|
|
99
|
+
"LANGFUSE_TRACING_ENVIRONMENT",
|
|
100
|
+
"TRACE_TO_LANGFUSE",
|
|
101
|
+
"LANGFUSE_CODEX_TAGS",
|
|
102
|
+
"LANGFUSE_CODEX_METADATA",
|
|
78
103
|
"TAVILY_API_KEY",
|
|
79
104
|
"TRACE_TO_LANGSMITH",
|
|
80
105
|
"CC_LANGSMITH_API_KEY",
|
|
@@ -97,6 +122,31 @@ WSL_ENV_KEYS = (
|
|
|
97
122
|
"CLAUDE_CODE_SUBAGENT_MODEL",
|
|
98
123
|
"BENCH_CC_VERSION",
|
|
99
124
|
"BENCH_CC_MODEL",
|
|
125
|
+
"BENCH_CODEX_VERSION",
|
|
126
|
+
"BENCH_CODEX_MODEL",
|
|
127
|
+
"BENCH_QODER_VERSION",
|
|
128
|
+
"BENCH_QODER_MODEL",
|
|
129
|
+
"CODEBUDDY_API_KEY",
|
|
130
|
+
"CODEBUDDY_AUTH_TOKEN",
|
|
131
|
+
"CODEBUDDY_BASE_URL",
|
|
132
|
+
"CODEBUDDY_MODEL",
|
|
133
|
+
"CODEBUDDY_SMALL_FAST_MODEL",
|
|
134
|
+
"CODEBUDDY_BIG_SLOW_MODEL",
|
|
135
|
+
"CODEBUDDY_CODE_SUBAGENT_MODEL",
|
|
136
|
+
"CODEBUDDY_CUSTOM_HEADERS",
|
|
137
|
+
"CODEBUDDY_INTERNET_ENVIRONMENT",
|
|
138
|
+
"BENCH_CODEBUDDY_VERSION",
|
|
139
|
+
"BENCH_CODEBUDDY_MODEL",
|
|
140
|
+
"COMET_EVAL_CUSTOM_AGENT_ID",
|
|
141
|
+
"COMET_EVAL_CUSTOM_EXECUTABLE",
|
|
142
|
+
"COMET_EVAL_CUSTOM_CREDENTIALS",
|
|
143
|
+
"COMET_EVAL_CUSTOM_MODEL",
|
|
144
|
+
"COMET_EVAL_CUSTOM_BASE_URL",
|
|
145
|
+
"COMET_EVAL_CUSTOM_MODEL_ENV",
|
|
146
|
+
"COMET_EVAL_CUSTOM_BASE_URL_ENV",
|
|
147
|
+
"COMET_EVAL_CUSTOM_INSTALL_KIND",
|
|
148
|
+
"COMET_EVAL_CUSTOM_INSTALL_PACKAGE",
|
|
149
|
+
"COMET_EVAL_CUSTOM_INSTALL_VERSION",
|
|
100
150
|
)
|
|
101
151
|
|
|
102
152
|
|
|
@@ -119,20 +169,33 @@ def _to_bash_path(value) -> str:
|
|
|
119
169
|
return s
|
|
120
170
|
|
|
121
171
|
|
|
122
|
-
def _bash_env() -> dict[str, str]:
|
|
123
|
-
env = os.environ
|
|
172
|
+
def _bash_env(source_env: dict[str, str] | None = None) -> dict[str, str]:
|
|
173
|
+
env = dict(source_env if source_env is not None else os.environ)
|
|
124
174
|
if os.name != "nt" or not _uses_wsl_bash(BASH_EXEC):
|
|
125
175
|
return env
|
|
126
176
|
|
|
127
177
|
existing = [item for item in env.get("WSLENV", "").split(":") if item]
|
|
178
|
+
custom_credentials = [
|
|
179
|
+
key.strip()
|
|
180
|
+
for key in env.get("COMET_EVAL_CUSTOM_CREDENTIALS", "").split(",")
|
|
181
|
+
if key.strip()
|
|
182
|
+
]
|
|
183
|
+
custom_routing = [
|
|
184
|
+
key.strip()
|
|
185
|
+
for metadata_key in ("COMET_EVAL_CUSTOM_MODEL_ENV", "COMET_EVAL_CUSTOM_BASE_URL_ENV")
|
|
186
|
+
for key in [env.get(metadata_key, "")]
|
|
187
|
+
if re.fullmatch(r"[A-Z][A-Z0-9_]{1,63}", key.strip())
|
|
188
|
+
]
|
|
128
189
|
exported = [key for key in WSL_ENV_KEYS if env.get(key)]
|
|
190
|
+
exported.extend(key for key in custom_credentials if env.get(key))
|
|
191
|
+
exported.extend(key for key in custom_routing if env.get(key))
|
|
129
192
|
merged = list(dict.fromkeys(existing + exported))
|
|
130
193
|
if merged:
|
|
131
194
|
env["WSLENV"] = ":".join(merged)
|
|
132
195
|
return env
|
|
133
196
|
|
|
134
197
|
|
|
135
|
-
def run_shell(script, *args, timeout=None, check=True):
|
|
198
|
+
def run_shell(script, *args, timeout=None, check=True, env=None):
|
|
136
199
|
cmd = [BASH_EXEC, _to_bash_path(SHELL_DIR / script)] + [_to_bash_path(a) for a in args]
|
|
137
200
|
return subprocess.run(
|
|
138
201
|
cmd,
|
|
@@ -142,7 +205,7 @@ def run_shell(script, *args, timeout=None, check=True):
|
|
|
142
205
|
errors="replace",
|
|
143
206
|
timeout=timeout,
|
|
144
207
|
check=check,
|
|
145
|
-
env=_bash_env(),
|
|
208
|
+
env=_bash_env(env),
|
|
146
209
|
)
|
|
147
210
|
|
|
148
211
|
|
|
@@ -153,9 +216,11 @@ def check_docker_available():
|
|
|
153
216
|
return False
|
|
154
217
|
|
|
155
218
|
|
|
156
|
-
def build_docker_image(test_dir, force=False, verbose=False):
|
|
219
|
+
def build_docker_image(test_dir, force=False, verbose=False, agent=None):
|
|
157
220
|
try:
|
|
158
221
|
args = ["build", str(test_dir)] + (["--force"] if force else [])
|
|
222
|
+
if agent:
|
|
223
|
+
args.extend(["--agent", validate_agent_id(agent)])
|
|
159
224
|
result = run_shell("docker.sh", *args, timeout=300, check=False)
|
|
160
225
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
161
226
|
except subprocess.TimeoutExpired:
|
|
@@ -186,7 +251,26 @@ def run_node_in_docker(test_dir, script_name, timeout=120, args=None):
|
|
|
186
251
|
return _docker_run_script("run-node", test_dir, script_name, timeout, args)
|
|
187
252
|
|
|
188
253
|
|
|
189
|
-
def
|
|
254
|
+
def run_command_in_docker(test_dir, command, timeout=120):
|
|
255
|
+
"""Run one user-authored validation command inside the task container."""
|
|
256
|
+
cmd = ["run-command", str(test_dir), "--timeout", str(timeout), "--", command]
|
|
257
|
+
if not check_docker_available():
|
|
258
|
+
return subprocess.CompletedProcess(cmd, 125, "", "Docker not available")
|
|
259
|
+
try:
|
|
260
|
+
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
|
|
261
|
+
except subprocess.TimeoutExpired:
|
|
262
|
+
return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def run_claude_in_docker(
|
|
266
|
+
test_dir,
|
|
267
|
+
prompt,
|
|
268
|
+
timeout=300,
|
|
269
|
+
model=None,
|
|
270
|
+
image_id=None,
|
|
271
|
+
base_url=None,
|
|
272
|
+
environment=None,
|
|
273
|
+
):
|
|
190
274
|
if not check_docker_available():
|
|
191
275
|
raise RuntimeError("Docker not available")
|
|
192
276
|
cmd = ["run-claude", str(test_dir), prompt, "--timeout", str(timeout)]
|
|
@@ -194,17 +278,54 @@ def run_claude_in_docker(test_dir, prompt, timeout=300, model=None, image_id=Non
|
|
|
194
278
|
cmd.extend(["--model", model])
|
|
195
279
|
if image_id:
|
|
196
280
|
cmd.extend(["--image-id", image_id])
|
|
281
|
+
child_env = environment
|
|
282
|
+
if child_env is None and (model or base_url):
|
|
283
|
+
child_env = build_agent_environment(
|
|
284
|
+
ResolvedExecution("claude-code", model, base_url, {}),
|
|
285
|
+
)
|
|
197
286
|
try:
|
|
198
|
-
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
|
|
287
|
+
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False, env=child_env)
|
|
199
288
|
except subprocess.TimeoutExpired:
|
|
200
289
|
return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
|
|
201
290
|
|
|
202
291
|
|
|
203
|
-
def
|
|
292
|
+
def run_agent_in_docker(
|
|
293
|
+
test_dir,
|
|
294
|
+
prompt,
|
|
295
|
+
*,
|
|
296
|
+
agent="claude-code",
|
|
297
|
+
timeout=300,
|
|
298
|
+
model=None,
|
|
299
|
+
base_url=None,
|
|
300
|
+
image_id=None,
|
|
301
|
+
environment=None,
|
|
302
|
+
):
|
|
303
|
+
"""Run one subject, simulator, or judge turn through the selected adapter."""
|
|
304
|
+
agent_id = validate_agent_id(agent)
|
|
305
|
+
if not check_docker_available():
|
|
306
|
+
raise RuntimeError("Docker not available")
|
|
307
|
+
cmd = ["run-agent", str(test_dir), prompt, "--agent", agent_id]
|
|
308
|
+
if model:
|
|
309
|
+
cmd.extend(["--model", model])
|
|
310
|
+
cmd.extend(["--timeout", str(timeout)])
|
|
311
|
+
if image_id:
|
|
312
|
+
cmd.extend(["--image-id", image_id])
|
|
313
|
+
child_env = environment
|
|
314
|
+
if child_env is None and (model or base_url):
|
|
315
|
+
child_env = build_agent_environment(
|
|
316
|
+
ResolvedExecution(agent_id, model, base_url, {}),
|
|
317
|
+
)
|
|
318
|
+
try:
|
|
319
|
+
return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False, env=child_env)
|
|
320
|
+
except subprocess.TimeoutExpired:
|
|
321
|
+
return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def run_claude_loop_in_docker(test_dir, loop_args, timeout=600, environment=None):
|
|
204
325
|
"""Run the interactive driver and remove its container after host-side timeout."""
|
|
205
326
|
cmd = ["run-claude-loop", str(test_dir), *loop_args]
|
|
206
327
|
try:
|
|
207
|
-
return run_shell("docker.sh", *cmd, timeout=timeout, check=False)
|
|
328
|
+
return run_shell("docker.sh", *cmd, timeout=timeout, check=False, env=environment)
|
|
208
329
|
except subprocess.TimeoutExpired as error:
|
|
209
330
|
cleanup_error = ""
|
|
210
331
|
try:
|
|
@@ -235,6 +356,41 @@ def run_claude_loop_in_docker(test_dir, loop_args, timeout=600):
|
|
|
235
356
|
)
|
|
236
357
|
|
|
237
358
|
|
|
359
|
+
def run_agent_loop_in_docker(test_dir, loop_args, timeout=600, environment=None):
|
|
360
|
+
"""Run the shared interactive driver for a non-default evaluation agent."""
|
|
361
|
+
cmd = ["run-agent-loop", str(test_dir), *loop_args]
|
|
362
|
+
try:
|
|
363
|
+
return run_shell("docker.sh", *cmd, timeout=timeout, check=False, env=environment)
|
|
364
|
+
except subprocess.TimeoutExpired as error:
|
|
365
|
+
cleanup_error = ""
|
|
366
|
+
try:
|
|
367
|
+
cleanup = run_shell(
|
|
368
|
+
"docker.sh",
|
|
369
|
+
"cleanup-agent-loop",
|
|
370
|
+
test_dir,
|
|
371
|
+
timeout=30,
|
|
372
|
+
check=False,
|
|
373
|
+
)
|
|
374
|
+
if cleanup.returncode != 0:
|
|
375
|
+
cleanup_error = f"; cleanup failed: {cleanup.stderr or cleanup.stdout}"
|
|
376
|
+
except Exception as cleanup_exception: # pragma: no cover - defensive cleanup
|
|
377
|
+
cleanup_error = f"; cleanup failed: {cleanup_exception}"
|
|
378
|
+
stdout = (
|
|
379
|
+
error.stdout.decode("utf-8", errors="replace")
|
|
380
|
+
if isinstance(error.stdout, bytes)
|
|
381
|
+
else (error.stdout or "")
|
|
382
|
+
)
|
|
383
|
+
stderr = (
|
|
384
|
+
error.stderr.decode("utf-8", errors="replace")
|
|
385
|
+
if isinstance(error.stderr, bytes)
|
|
386
|
+
else (error.stderr or "")
|
|
387
|
+
)
|
|
388
|
+
message = f"Timeout after {timeout}s{cleanup_error}"
|
|
389
|
+
return subprocess.CompletedProcess(
|
|
390
|
+
cmd, 124, stdout, "\n".join(filter(None, (stderr, message)))
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
|
|
238
394
|
def _copy_scaffold_to_docker(test_dir):
|
|
239
395
|
py_validation = SCAFFOLD_PYTHON_DIR / "validation"
|
|
240
396
|
|
|
@@ -20,6 +20,49 @@ TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
|
|
|
20
20
|
TEST_RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
def _normalize_skill_invocations(events: object, outputs: dict | None) -> list[str]:
|
|
24
|
+
"""Use the Agent contract when the full scaffold is available.
|
|
25
|
+
|
|
26
|
+
Validator sidecars intentionally copy only a small subset of the scaffold,
|
|
27
|
+
so keep this helper import-local and preserve the historical fallback there.
|
|
28
|
+
"""
|
|
29
|
+
try:
|
|
30
|
+
from scaffold.python.agents import normalize_skill_invocations
|
|
31
|
+
except ModuleNotFoundError:
|
|
32
|
+
if not isinstance(events, dict):
|
|
33
|
+
return []
|
|
34
|
+
output_data = outputs if isinstance(outputs, dict) else {}
|
|
35
|
+
event_agent = events.get("agent")
|
|
36
|
+
output_agent = output_data.get("agent")
|
|
37
|
+
agent = output_agent if isinstance(output_agent, str) else event_agent
|
|
38
|
+
if isinstance(agent, str) and agent not in {
|
|
39
|
+
"claude-code",
|
|
40
|
+
"codex",
|
|
41
|
+
"qoder",
|
|
42
|
+
"codebuddy",
|
|
43
|
+
}:
|
|
44
|
+
# Validator sidecars do not carry the host-side adapter registry.
|
|
45
|
+
# A custom Agent must therefore provide explicit invocation events;
|
|
46
|
+
# artifacts or legacy path inference are not sufficient evidence.
|
|
47
|
+
explicit = events.get("skill_invocations")
|
|
48
|
+
return (
|
|
49
|
+
[item for item in explicit if isinstance(item, str)]
|
|
50
|
+
if isinstance(explicit, list)
|
|
51
|
+
else []
|
|
52
|
+
)
|
|
53
|
+
invoked = events.get("skills_invoked")
|
|
54
|
+
return (
|
|
55
|
+
[item for item in invoked if isinstance(item, str)]
|
|
56
|
+
if isinstance(invoked, list)
|
|
57
|
+
else []
|
|
58
|
+
)
|
|
59
|
+
return normalize_skill_invocations(
|
|
60
|
+
events,
|
|
61
|
+
agent=(outputs or {}).get("agent"),
|
|
62
|
+
adapter=(outputs or {}).get("agent_adapter"),
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
23
66
|
def load_test_context(path: str = TEST_CONTEXT_FILE) -> dict:
|
|
24
67
|
"""Load test context (run_id, events, langsmith_env, etc.) written by the host.
|
|
25
68
|
|
|
@@ -141,6 +184,9 @@ def run_validators(
|
|
|
141
184
|
Returns:
|
|
142
185
|
Combined (passed, failed) lists
|
|
143
186
|
"""
|
|
187
|
+
events = outputs.get("events") if isinstance(outputs, dict) else None
|
|
188
|
+
if isinstance(events, dict):
|
|
189
|
+
events["skills_invoked"] = _normalize_skill_invocations(events, outputs)
|
|
144
190
|
all_passed, all_failed = [], []
|
|
145
191
|
for validator in validators:
|
|
146
192
|
passed, failed = validator(test_dir, outputs)
|
|
@@ -164,7 +210,7 @@ def check_starter_skill_first(
|
|
|
164
210
|
return ["Note: starter skill check skipped (ALL_MAIN_SKILLS)"], []
|
|
165
211
|
|
|
166
212
|
events = outputs.get("events", {}) if outputs else {}
|
|
167
|
-
skills_invoked = events
|
|
213
|
+
skills_invoked = _normalize_skill_invocations(events, outputs)
|
|
168
214
|
|
|
169
215
|
# No skills invoked at all (e.g. CONTROL treatment) — nothing to check
|
|
170
216
|
if not skills_invoked:
|
|
@@ -205,7 +251,7 @@ def check_skill_invoked(
|
|
|
205
251
|
(passed, failed) lists
|
|
206
252
|
"""
|
|
207
253
|
events = outputs.get("events", {}) if outputs else {}
|
|
208
|
-
skills_invoked = events
|
|
254
|
+
skills_invoked = _normalize_skill_invocations(events, outputs)
|
|
209
255
|
|
|
210
256
|
if skill_name in skills_invoked:
|
|
211
257
|
return [f"Invoked {skill_name} skill"], []
|
|
@@ -7,6 +7,8 @@ import re
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
|
+
from scaffold.python.agents import normalize_skill_invocations
|
|
11
|
+
|
|
10
12
|
GENERIC_RUBRIC_DIMENSIONS = (
|
|
11
13
|
"completion",
|
|
12
14
|
"skill_invocation",
|
|
@@ -153,6 +155,13 @@ def _fmt_na(dim: str, reason: str) -> str:
|
|
|
153
155
|
|
|
154
156
|
|
|
155
157
|
def generic_rubric_validator(test_dir: Path, outputs: dict[str, Any]) -> tuple[list[str], list[str]]:
|
|
158
|
+
events = (outputs or {}).get("events")
|
|
159
|
+
if isinstance(events, dict):
|
|
160
|
+
events["skills_invoked"] = normalize_skill_invocations(
|
|
161
|
+
events,
|
|
162
|
+
agent=(outputs or {}).get("agent"),
|
|
163
|
+
adapter=(outputs or {}).get("agent_adapter"),
|
|
164
|
+
)
|
|
156
165
|
scored: list[tuple[str, float | None, str]] = [
|
|
157
166
|
("completion", *_score_completion(outputs)),
|
|
158
167
|
("skill_invocation", *_score_skill_invocation(outputs)),
|