okstra 0.174.0 → 0.176.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -9
- package/bin/okstra +8 -4
- package/dist/cli-registry.d.mts +16 -0
- package/dist/cli-registry.mjs +560 -0
- package/dist/cli-registry.mjs.map +1 -0
- package/dist/commands/execute/agent-prompt.d.mts +1 -0
- package/{src → dist}/commands/execute/agent-prompt.mjs +10 -11
- package/dist/commands/execute/agent-prompt.mjs.map +1 -0
- package/dist/commands/execute/codex-dispatch.d.mts +3 -0
- package/dist/commands/execute/codex-dispatch.mjs +6 -0
- package/dist/commands/execute/codex-dispatch.mjs.map +1 -0
- package/dist/commands/execute/codex-run.d.mts +3 -0
- package/{src → dist}/commands/execute/codex-run.mjs +30 -37
- package/dist/commands/execute/codex-run.mjs.map +1 -0
- package/dist/commands/execute/convergence.d.mts +1 -0
- package/{src → dist}/commands/execute/convergence.mjs +15 -16
- package/dist/commands/execute/convergence.mjs.map +1 -0
- package/dist/commands/execute/error-log.d.mts +1 -0
- package/{src → dist}/commands/execute/error-log.mjs +2 -3
- package/dist/commands/execute/error-log.mjs.map +1 -0
- package/dist/commands/execute/git-reconcile.d.mts +1 -0
- package/{src → dist}/commands/execute/git-reconcile.mjs +10 -11
- package/dist/commands/execute/git-reconcile.mjs.map +1 -0
- package/dist/commands/execute/handoff.d.mts +1 -0
- package/{src → dist}/commands/execute/handoff.mjs +10 -11
- package/dist/commands/execute/handoff.mjs.map +1 -0
- package/dist/commands/execute/incremental-carry.d.mts +1 -0
- package/{src → dist}/commands/execute/incremental-carry.mjs +2 -3
- package/dist/commands/execute/incremental-carry.mjs.map +1 -0
- package/dist/commands/execute/incremental-scope.d.mts +1 -0
- package/{src → dist}/commands/execute/incremental-scope.mjs +2 -3
- package/dist/commands/execute/incremental-scope.mjs.map +1 -0
- package/dist/commands/execute/integrate-stages.d.mts +1 -0
- package/{src → dist}/commands/execute/integrate-stages.mjs +10 -11
- package/dist/commands/execute/integrate-stages.mjs.map +1 -0
- package/dist/commands/execute/pane-title.d.mts +1 -0
- package/dist/commands/execute/pane-title.mjs +20 -0
- package/dist/commands/execute/pane-title.mjs.map +1 -0
- package/dist/commands/execute/plan-items.d.mts +1 -0
- package/dist/commands/execute/plan-items.mjs +9 -0
- package/dist/commands/execute/plan-items.mjs.map +1 -0
- package/dist/commands/execute/plan-validate.d.mts +1 -0
- package/dist/commands/execute/plan-validate.mjs +68 -0
- package/dist/commands/execute/plan-validate.mjs.map +1 -0
- package/dist/commands/execute/plan-verify.d.mts +1 -0
- package/{src → dist}/commands/execute/plan-verify.mjs +13 -16
- package/dist/commands/execute/plan-verify.mjs.map +1 -0
- package/dist/commands/execute/render-bundle.d.mts +3 -0
- package/{src → dist}/commands/execute/render-bundle.mjs +65 -61
- package/dist/commands/execute/render-bundle.mjs.map +1 -0
- package/dist/commands/execute/run.d.mts +24 -0
- package/dist/commands/execute/run.mjs +196 -0
- package/dist/commands/execute/run.mjs.map +1 -0
- package/dist/commands/execute/spawn-followups.d.mts +1 -0
- package/{src → dist}/commands/execute/spawn-followups.mjs +6 -7
- package/dist/commands/execute/spawn-followups.mjs.map +1 -0
- package/dist/commands/execute/team.d.mts +3 -0
- package/{src → dist}/commands/execute/team.mjs +29 -36
- package/dist/commands/execute/team.mjs.map +1 -0
- package/dist/commands/execute/token-usage.d.mts +1 -0
- package/{src → dist}/commands/execute/token-usage.mjs +2 -3
- package/dist/commands/execute/token-usage.mjs.map +1 -0
- package/dist/commands/execute/wizard.d.mts +8 -0
- package/dist/commands/execute/wizard.mjs +191 -0
- package/dist/commands/execute/wizard.mjs.map +1 -0
- package/dist/commands/execute/worker-audit-check.d.mts +1 -0
- package/{src → dist}/commands/execute/worker-audit-check.mjs +11 -12
- package/dist/commands/execute/worker-audit-check.mjs.map +1 -0
- package/dist/commands/execute/worker-dispatch.d.mts +7 -0
- package/{src → dist}/commands/execute/worker-dispatch.mjs +34 -46
- package/dist/commands/execute/worker-dispatch.mjs.map +1 -0
- package/dist/commands/execute/worker-state.d.mts +1 -0
- package/{src → dist}/commands/execute/worker-state.mjs +15 -16
- package/dist/commands/execute/worker-state.mjs.map +1 -0
- package/dist/commands/execute/worktree-lookup.d.mts +1 -0
- package/{src → dist}/commands/execute/worktree-lookup.mjs +38 -37
- package/dist/commands/execute/worktree-lookup.mjs.map +1 -0
- package/dist/commands/execute/worktree-status.d.mts +1 -0
- package/{src → dist}/commands/execute/worktree-status.mjs +52 -46
- package/dist/commands/execute/worktree-status.mjs.map +1 -0
- package/dist/commands/inspect/code-review.d.mts +1 -0
- package/{src → dist}/commands/inspect/code-review.mjs +16 -19
- package/dist/commands/inspect/code-review.mjs.map +1 -0
- package/dist/commands/inspect/container.d.mts +1 -0
- package/{src → dist}/commands/inspect/container.mjs +11 -13
- package/dist/commands/inspect/container.mjs.map +1 -0
- package/dist/commands/inspect/context-cost.d.mts +1 -0
- package/{src → dist}/commands/inspect/context-cost.mjs +11 -13
- package/dist/commands/inspect/context-cost.mjs.map +1 -0
- package/dist/commands/inspect/design-prep.d.mts +1 -0
- package/{src → dist}/commands/inspect/design-prep.mjs +10 -11
- package/dist/commands/inspect/design-prep.mjs.map +1 -0
- package/dist/commands/inspect/error-report.d.mts +1 -0
- package/{src → dist}/commands/inspect/error-report.mjs +11 -13
- package/dist/commands/inspect/error-report.mjs.map +1 -0
- package/dist/commands/inspect/error-zip.d.mts +1 -0
- package/{src → dist}/commands/inspect/error-zip.mjs +11 -12
- package/dist/commands/inspect/error-zip.mjs.map +1 -0
- package/dist/commands/inspect/log-report.d.mts +1 -0
- package/{src → dist}/commands/inspect/log-report.mjs +11 -13
- package/dist/commands/inspect/log-report.mjs.map +1 -0
- package/dist/commands/inspect/profile-show.d.mts +1 -0
- package/{src → dist}/commands/inspect/profile-show.mjs +10 -11
- package/dist/commands/inspect/profile-show.mjs.map +1 -0
- package/dist/commands/inspect/recap.d.mts +1 -0
- package/{src → dist}/commands/inspect/recap.mjs +11 -13
- package/dist/commands/inspect/recap.mjs.map +1 -0
- package/dist/commands/inspect/resolve-task-key.d.mts +1 -0
- package/{src → dist}/commands/inspect/resolve-task-key.mjs +11 -13
- package/dist/commands/inspect/resolve-task-key.mjs.map +1 -0
- package/dist/commands/inspect/rollup.d.mts +1 -0
- package/{src → dist}/commands/inspect/rollup.mjs +11 -13
- package/dist/commands/inspect/rollup.mjs.map +1 -0
- package/dist/commands/inspect/run-audit.d.mts +1 -0
- package/{src → dist}/commands/inspect/run-audit.mjs +11 -12
- package/dist/commands/inspect/run-audit.mjs.map +1 -0
- package/dist/commands/inspect/set-work-status.d.mts +1 -0
- package/{src → dist}/commands/inspect/set-work-status.mjs +11 -13
- package/dist/commands/inspect/set-work-status.mjs.map +1 -0
- package/dist/commands/inspect/stage-map.d.mts +1 -0
- package/dist/commands/inspect/stage-map.mjs +110 -0
- package/dist/commands/inspect/stage-map.mjs.map +1 -0
- package/dist/commands/inspect/task-list.d.mts +1 -0
- package/dist/commands/inspect/task-list.mjs +103 -0
- package/dist/commands/inspect/task-list.mjs.map +1 -0
- package/dist/commands/inspect/task-show.d.mts +1 -0
- package/dist/commands/inspect/task-show.mjs +108 -0
- package/dist/commands/inspect/task-show.mjs.map +1 -0
- package/dist/commands/inspect/time-report.d.mts +1 -0
- package/{src → dist}/commands/inspect/time-report.mjs +11 -13
- package/dist/commands/inspect/time-report.mjs.map +1 -0
- package/dist/commands/inspect/usage-report.d.mts +1 -0
- package/{src → dist}/commands/inspect/usage-report.mjs +11 -13
- package/dist/commands/inspect/usage-report.mjs.map +1 -0
- package/dist/commands/inspect/user-response.d.mts +1 -0
- package/{src → dist}/commands/inspect/user-response.mjs +10 -11
- package/dist/commands/inspect/user-response.mjs.map +1 -0
- package/dist/commands/inspect/worker-liveness.d.mts +1 -0
- package/{src → dist}/commands/inspect/worker-liveness.mjs +11 -13
- package/dist/commands/inspect/worker-liveness.mjs.map +1 -0
- package/dist/commands/lifecycle/check-project.d.mts +21 -0
- package/dist/commands/lifecycle/check-project.mjs +175 -0
- package/dist/commands/lifecycle/check-project.mjs.map +1 -0
- package/dist/commands/lifecycle/config.d.mts +1 -0
- package/dist/commands/lifecycle/config.mjs +369 -0
- package/dist/commands/lifecycle/config.mjs.map +1 -0
- package/dist/commands/lifecycle/doctor.d.mts +10 -0
- package/dist/commands/lifecycle/doctor.mjs +317 -0
- package/dist/commands/lifecycle/doctor.mjs.map +1 -0
- package/dist/commands/lifecycle/install.d.mts +17 -0
- package/dist/commands/lifecycle/install.mjs +1077 -0
- package/dist/commands/lifecycle/install.mjs.map +1 -0
- package/dist/commands/lifecycle/migrate.d.mts +1 -0
- package/{src → dist}/commands/lifecycle/migrate.mjs +10 -11
- package/dist/commands/lifecycle/migrate.mjs.map +1 -0
- package/dist/commands/lifecycle/model.d.mts +1 -0
- package/dist/commands/lifecycle/model.mjs +22 -0
- package/dist/commands/lifecycle/model.mjs.map +1 -0
- package/dist/commands/lifecycle/paths.d.mts +1 -0
- package/dist/commands/lifecycle/paths.mjs +102 -0
- package/dist/commands/lifecycle/paths.mjs.map +1 -0
- package/dist/commands/lifecycle/preflight.d.mts +1 -0
- package/dist/commands/lifecycle/preflight.mjs +124 -0
- package/dist/commands/lifecycle/preflight.mjs.map +1 -0
- package/dist/commands/lifecycle/setup.d.mts +1 -0
- package/dist/commands/lifecycle/setup.mjs +297 -0
- package/dist/commands/lifecycle/setup.mjs.map +1 -0
- package/dist/commands/lifecycle/uninstall.d.mts +4 -0
- package/dist/commands/lifecycle/uninstall.mjs +262 -0
- package/dist/commands/lifecycle/uninstall.mjs.map +1 -0
- package/dist/commands/manager.d.mts +3 -0
- package/{src → dist}/commands/manager.mjs +25 -32
- package/dist/commands/manager.mjs.map +1 -0
- package/dist/commands/memory/memory.d.mts +1 -0
- package/dist/commands/memory/memory.mjs +569 -0
- package/dist/commands/memory/memory.mjs.map +1 -0
- package/dist/commands/pr/pr.d.mts +33 -0
- package/dist/commands/pr/pr.mjs +296 -0
- package/dist/commands/pr/pr.mjs.map +1 -0
- package/dist/commands/report/agent-activity.d.mts +1 -0
- package/{src → dist}/commands/report/agent-activity.mjs +11 -12
- package/dist/commands/report/agent-activity.mjs.map +1 -0
- package/dist/commands/report/finalize.d.mts +4 -0
- package/{src → dist}/commands/report/finalize.mjs +25 -32
- package/dist/commands/report/finalize.mjs.map +1 -0
- package/dist/commands/report/inject-report-index.d.mts +1 -0
- package/{src → dist}/commands/report/inject-report-index.mjs +6 -7
- package/dist/commands/report/inject-report-index.mjs.map +1 -0
- package/dist/commands/report/render-final-report.d.mts +1 -0
- package/{src → dist}/commands/report/render-final-report.mjs +6 -7
- package/dist/commands/report/render-final-report.mjs.map +1 -0
- package/dist/commands/report/render-views.d.mts +1 -0
- package/{src → dist}/commands/report/render-views.mjs +6 -7
- package/dist/commands/report/render-views.mjs.map +1 -0
- package/dist/commands/report/translate.d.mts +1 -0
- package/{src → dist}/commands/report/translate.mjs +6 -7
- package/dist/commands/report/translate.mjs.map +1 -0
- package/dist/lib/helper-scripts.d.mts +3 -0
- package/{src → dist}/lib/helper-scripts.mjs +11 -14
- package/dist/lib/helper-scripts.mjs.map +1 -0
- package/dist/lib/host-registry-client.d.mts +25 -0
- package/dist/lib/host-registry-client.mjs +152 -0
- package/dist/lib/host-registry-client.mjs.map +1 -0
- package/dist/lib/install-assets.d.mts +8 -0
- package/{src → dist}/lib/install-assets.mjs +17 -20
- package/dist/lib/install-assets.mjs.map +1 -0
- package/dist/lib/okstra-dirs.d.mts +4 -0
- package/{src → dist}/lib/okstra-dirs.mjs +3 -6
- package/dist/lib/okstra-dirs.mjs.map +1 -0
- package/dist/lib/paths.d.mts +4 -0
- package/dist/lib/paths.mjs +78 -0
- package/dist/lib/paths.mjs.map +1 -0
- package/dist/lib/proc.d.mts +3 -0
- package/dist/lib/proc.mjs +31 -0
- package/dist/lib/proc.mjs.map +1 -0
- package/dist/lib/python-helper.d.mts +8 -0
- package/dist/lib/python-helper.mjs +123 -0
- package/dist/lib/python-helper.mjs.map +1 -0
- package/dist/lib/runtime-manifest.d.mts +7 -0
- package/dist/lib/runtime-manifest.mjs +73 -0
- package/dist/lib/runtime-manifest.mjs.map +1 -0
- package/dist/lib/skill-catalog.d.mts +4 -0
- package/dist/lib/skill-catalog.mjs +50 -0
- package/dist/lib/skill-catalog.mjs.map +1 -0
- package/dist/lib/types.d.mts +129 -0
- package/dist/lib/types.mjs +2 -0
- package/dist/lib/types.mjs.map +1 -0
- package/dist/lib/version.d.mts +2 -0
- package/dist/lib/version.mjs +27 -0
- package/dist/lib/version.mjs.map +1 -0
- package/docs/architecture/storage-model.md +6 -2
- package/docs/architecture.md +29 -20
- package/docs/cli.md +40 -16
- package/docs/contributor-change-matrix.md +2 -2
- package/docs/for-ai/skills/okstra-inspect.md +2 -3
- package/docs/for-ai/skills/okstra-rollup.md +1 -0
- package/docs/project-structure-overview.md +72 -62
- package/docs/task-process/README.md +12 -8
- package/docs/task-process/common-flow.md +13 -16
- package/docs/task-process/error-analysis.md +9 -10
- package/docs/task-process/final-verification.md +7 -7
- package/docs/task-process/implementation-planning.md +9 -9
- package/docs/task-process/implementation.md +6 -6
- package/docs/task-process/release-handoff.md +8 -7
- package/docs/task-process/requirements-discovery.md +8 -8
- package/package.json +11 -5
- package/runtime/BUILD.json +2 -2
- package/runtime/agents/workers/claude-worker.md +2 -1
- package/runtime/agents/workers/report-writer-worker.md +3 -1
- package/runtime/agents/workers/translator-worker.md +3 -1
- package/runtime/bin/lib/okstra/cli.sh +12 -0
- package/runtime/bin/lib/okstra/globals.sh +3 -0
- package/runtime/bin/lib/okstra/interactive.sh +12 -6
- package/runtime/bin/lib/okstra/usage.sh +7 -6
- package/runtime/bin/okstra-error-log.py +27 -0
- package/runtime/bin/okstra-provider-exec.py +217 -5
- package/runtime/bin/okstra-spawn-followups.py +4 -2
- package/runtime/bin/okstra.sh +10 -0
- package/runtime/prompts/launch.template.md +2 -2
- package/runtime/prompts/lead/context-loader.md +1 -2
- package/runtime/prompts/lead/convergence.md +27 -18
- package/runtime/prompts/lead/okstra-lead-contract.md +6 -5
- package/runtime/prompts/lead/plan-body-verification.md +10 -3
- package/runtime/prompts/lead/report-writer.md +15 -1
- package/runtime/prompts/lead/team-contract.md +17 -13
- package/runtime/prompts/profiles/_implementation-deliverable.md +1 -1
- package/runtime/prompts/profiles/_implementation-executor.md +0 -3
- package/runtime/prompts/profiles/_implementation-verifier.md +3 -7
- package/runtime/prompts/profiles/change-impact-analysis.md +20 -0
- package/runtime/prompts/profiles/error-analysis.md +27 -1
- package/runtime/prompts/profiles/feature-analysis.md +20 -0
- package/runtime/prompts/profiles/final-verification.md +20 -1
- package/runtime/prompts/profiles/implementation-option-selection.md +20 -0
- package/runtime/prompts/profiles/implementation-planning.md +26 -0
- package/runtime/prompts/profiles/implementation.md +19 -0
- package/runtime/prompts/profiles/improvement-discovery.md +21 -1
- package/runtime/prompts/profiles/project-analysis.md +20 -0
- package/runtime/prompts/profiles/release-handoff.md +4 -0
- package/runtime/prompts/profiles/requirements-discovery.md +27 -1
- package/runtime/prompts/wizard/prompts.ko.json +35 -0
- package/runtime/python/okstra_ctl/adapters/dispatch/cli_wrapper.py +8 -0
- package/runtime/python/okstra_ctl/adapters/dispatch/cmux.py +8 -0
- package/runtime/python/okstra_ctl/adapters/dispatch/native_team.py +13 -0
- package/runtime/python/okstra_ctl/adapters/hosts/antigravity/adapter.py +2 -2
- package/runtime/python/okstra_ctl/adapters/hosts/claude-code/adapter.py +25 -5
- package/runtime/python/okstra_ctl/adapters/hosts/codex/adapter.py +2 -2
- package/runtime/python/okstra_ctl/adapters/hosts/external/adapter.py +4 -1
- package/runtime/python/okstra_ctl/adapters/hosts/grok/adapter.py +2 -2
- package/runtime/python/okstra_ctl/adapters/hosts/kimi/adapter.py +2 -2
- package/runtime/python/okstra_ctl/adapters/providers/antigravity/adapter.py +79 -11
- package/runtime/python/okstra_ctl/adapters/providers/claude/adapter.py +95 -25
- package/runtime/python/okstra_ctl/adapters/providers/codex/adapter.py +59 -14
- package/runtime/python/okstra_ctl/adapters/providers/grok/adapter.py +55 -9
- package/runtime/python/okstra_ctl/adapters/providers/kimi/adapter.py +51 -12
- package/runtime/python/okstra_ctl/adapters/runtime/__init__.py +1 -0
- package/runtime/python/okstra_ctl/adapters/runtime/assembly.py +27 -0
- package/runtime/python/okstra_ctl/adapters/runtime/cli_wrapper.py +49 -0
- package/runtime/python/okstra_ctl/adapters/runtime/cmux.py +74 -0
- package/runtime/python/okstra_ctl/agent_activity.py +5 -1
- package/runtime/python/okstra_ctl/agent_invocation.py +608 -14
- package/runtime/python/okstra_ctl/agent_prompt_cli.py +249 -50
- package/runtime/python/okstra_ctl/application/open_worker.py +29 -0
- package/runtime/python/okstra_ctl/application/resolve_assignment.py +53 -12
- package/runtime/python/okstra_ctl/assignment_environment.py +102 -0
- package/runtime/python/okstra_ctl/assignment_resolver.py +790 -0
- package/runtime/python/okstra_ctl/attempt_evidence.py +342 -0
- package/runtime/python/okstra_ctl/convergence.py +409 -3
- package/runtime/python/okstra_ctl/convergence_engine.py +248 -15
- package/runtime/python/okstra_ctl/convergence_store.py +170 -1
- package/runtime/python/okstra_ctl/dispatch_core.py +1204 -155
- package/runtime/python/okstra_ctl/dispatch_state.py +696 -11
- package/runtime/python/okstra_ctl/doctor.py +14 -0
- package/runtime/python/okstra_ctl/domain/host.py +34 -1
- package/runtime/python/okstra_ctl/domain/provider.py +212 -6
- package/runtime/python/okstra_ctl/domain/role.py +125 -0
- package/runtime/python/okstra_ctl/domain/wizard/interaction.py +18 -0
- package/runtime/python/okstra_ctl/domain/worker_exec.py +34 -1
- package/runtime/python/okstra_ctl/domain/worker_role.py +13 -3
- package/runtime/python/okstra_ctl/domain/worker_runtime.py +44 -0
- package/runtime/python/okstra_ctl/domain/worker_stream.py +24 -0
- package/runtime/python/okstra_ctl/error_log_write.py +9 -0
- package/runtime/python/okstra_ctl/error_report.py +21 -1
- package/runtime/python/okstra_ctl/execution_identity.py +529 -0
- package/runtime/python/okstra_ctl/execution_manifest.py +1281 -0
- package/runtime/python/okstra_ctl/execution_mutation_audit.py +619 -0
- package/runtime/python/okstra_ctl/implementation_outcome.py +21 -5
- package/runtime/python/okstra_ctl/index.py +6 -2
- package/runtime/python/okstra_ctl/initial_prompt_materialization.py +43 -3
- package/runtime/python/okstra_ctl/legacy_model_selection.py +578 -0
- package/runtime/python/okstra_ctl/manager_sync.py +4 -1
- package/runtime/python/okstra_ctl/model_cli.py +278 -0
- package/runtime/python/okstra_ctl/model_defaults.py +46 -0
- package/runtime/python/okstra_ctl/model_discovery.py +26 -7
- package/runtime/python/okstra_ctl/model_pool.py +501 -0
- package/runtime/python/okstra_ctl/models.py +47 -27
- package/runtime/python/okstra_ctl/mutation_recovery.py +293 -0
- package/runtime/python/okstra_ctl/next_phase.py +236 -0
- package/runtime/python/okstra_ctl/pane_title.py +151 -0
- package/runtime/python/okstra_ctl/ports/host_model.py +48 -17
- package/runtime/python/okstra_ctl/ports/worker_dispatch.py +4 -0
- package/runtime/python/okstra_ctl/ports/worker_runtime.py +26 -0
- package/runtime/python/okstra_ctl/recap.py +4 -1
- package/runtime/python/okstra_ctl/registry/provider_registry.py +8 -4
- package/runtime/python/okstra_ctl/render.py +124 -57
- package/runtime/python/okstra_ctl/render_final_report.py +19 -0
- package/runtime/python/okstra_ctl/report_contract.py +39 -0
- package/runtime/python/okstra_ctl/report_finalize.py +16 -0
- package/runtime/python/okstra_ctl/report_html/render.py +1 -0
- package/runtime/python/okstra_ctl/role_requirements.py +210 -0
- package/runtime/python/okstra_ctl/rollup.py +4 -1
- package/runtime/python/okstra_ctl/run.py +1210 -258
- package/runtime/python/okstra_ctl/run_index_row.py +7 -1
- package/runtime/python/okstra_ctl/stage_fix_carry.py +17 -1
- package/runtime/python/okstra_ctl/team.py +14 -18
- package/runtime/python/okstra_ctl/time_report.py +38 -4
- package/runtime/python/okstra_ctl/usage_identity.py +86 -0
- package/runtime/python/okstra_ctl/usage_report.py +15 -3
- package/runtime/python/okstra_ctl/wizard.py +1449 -66
- package/runtime/python/okstra_ctl/worker_audit_ledger.py +68 -65
- package/runtime/python/okstra_ctl/worker_prompt_body.py +18 -2
- package/runtime/python/okstra_ctl/worker_prompt_contract.py +40 -9
- package/runtime/python/okstra_ctl/worker_prompt_headers.py +16 -1
- package/runtime/python/okstra_ctl/worker_request.py +7 -1
- package/runtime/python/okstra_ctl/worker_runner.py +130 -5
- package/runtime/python/okstra_ctl/workers.py +7 -5
- package/runtime/python/okstra_ctl/workflow.py +18 -32
- package/runtime/python/okstra_ctl/worktree.py +3 -3
- package/runtime/python/okstra_ctl/worktree_registry.py +5 -4
- package/runtime/python/okstra_ctl/write_policy.py +571 -0
- package/runtime/python/okstra_project/state.py +54 -6
- package/runtime/schemas/convergence-groups-v2.0.schema.json +128 -0
- package/runtime/schemas/execution-manifest-v2.schema.json +340 -0
- package/runtime/schemas/final-report-v2.0.schema.json +68 -5
- package/runtime/skills/okstra-inspect/facets/recap.md +2 -0
- package/runtime/skills/okstra-inspect/facets/report.md +1 -1
- package/runtime/skills/okstra-inspect/facets/status.md +15 -13
- package/runtime/skills/okstra-rollup/SKILL.md +1 -0
- package/runtime/skills/okstra-run/SKILL.md +14 -14
- package/runtime/templates/implementation-worker-preamble.md +3 -18
- package/runtime/templates/project-docs/task-index.template.md +0 -1
- package/runtime/templates/report-writer-prompt-preamble.md +2 -0
- package/runtime/templates/reports/final-report-v2.template.md +7 -1
- package/runtime/templates/reports/html/base.template.html +21 -0
- package/runtime/templates/reports/html/i18n/en.json +4 -0
- package/runtime/templates/reports/html/i18n/ko.json +4 -0
- package/runtime/templates/reports/html/macros/forms.html +5 -4
- package/runtime/templates/reports/html/tasks/final-verification.template.html +1 -1
- package/runtime/templates/reports/html/tasks/implementation.template.html +1 -1
- package/runtime/templates/worker-prompt-preamble.md +6 -35
- package/runtime/validators/lib/fixtures.sh +49 -16
- package/runtime/validators/validate-run.py +486 -134
- package/runtime/validators/validate_session_conformance.py +25 -7
- package/src/cli-registry.mjs +0 -539
- package/src/commands/execute/codex-dispatch.mjs +0 -10
- package/src/commands/execute/plan-items.mjs +0 -9
- package/src/commands/execute/plan-validate.mjs +0 -68
- package/src/commands/execute/run.mjs +0 -208
- package/src/commands/execute/wizard.mjs +0 -167
- package/src/commands/inspect/stage-map.mjs +0 -98
- package/src/commands/inspect/task-list.mjs +0 -97
- package/src/commands/inspect/task-show.mjs +0 -100
- package/src/commands/lifecycle/check-project.mjs +0 -187
- package/src/commands/lifecycle/config.mjs +0 -385
- package/src/commands/lifecycle/doctor.mjs +0 -308
- package/src/commands/lifecycle/install.mjs +0 -1170
- package/src/commands/lifecycle/paths.mjs +0 -104
- package/src/commands/lifecycle/preflight.mjs +0 -125
- package/src/commands/lifecycle/setup.mjs +0 -333
- package/src/commands/lifecycle/uninstall.mjs +0 -279
- package/src/commands/memory/memory.mjs +0 -524
- package/src/commands/pr/pr.mjs +0 -261
- package/src/lib/host-registry-client.mjs +0 -176
- package/src/lib/paths.mjs +0 -86
- package/src/lib/proc.mjs +0 -31
- package/src/lib/python-helper.mjs +0 -142
- package/src/lib/runtime-manifest.mjs +0 -70
- package/src/lib/skill-catalog.mjs +0 -53
- package/src/lib/version.mjs +0 -20
- /package/{src → dist}/commands/pr/default.md +0 -0
|
@@ -9,7 +9,6 @@ from datetime import datetime, timezone
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
from typing import Any, Mapping, Sequence
|
|
11
11
|
|
|
12
|
-
from . import cmux
|
|
13
12
|
from .dispatch_state import (
|
|
14
13
|
BACKEND_CLI_WRAPPER,
|
|
15
14
|
BACKEND_CMUX_PANE,
|
|
@@ -39,10 +38,46 @@ from .dispatch_state import (
|
|
|
39
38
|
TEARDOWN_BEFORE_TERMINAL_REASON,
|
|
40
39
|
utc_now as _utc_now,
|
|
41
40
|
validate_dispatch_prompts as _validate_dispatch_prompts,
|
|
41
|
+
v2_worker_state_key as _v2_worker_state_key,
|
|
42
|
+
worker_execution_identity as _worker_execution_identity,
|
|
42
43
|
worker_jobs_from_file as _worker_jobs_from_file,
|
|
43
44
|
worker_state as _worker_state,
|
|
44
45
|
worktree_path as _worktree_path,
|
|
45
46
|
)
|
|
47
|
+
from .assignment_resolver import (
|
|
48
|
+
AssignmentEnvironment,
|
|
49
|
+
AssignmentResolutionError,
|
|
50
|
+
RoleInstance,
|
|
51
|
+
resolve_model_assignment,
|
|
52
|
+
)
|
|
53
|
+
from .application.open_worker import open_worker
|
|
54
|
+
from .domain.host import HostSessionContext
|
|
55
|
+
from .domain.provider import HostModelBinding, ServedModelAttestation
|
|
56
|
+
from .domain.worker_exec import SERVED_MODEL_MISMATCH_EXIT_CODE
|
|
57
|
+
from .domain.worker_runtime import (
|
|
58
|
+
SURFACE_CMUX_PANE,
|
|
59
|
+
EnvironmentBlocked,
|
|
60
|
+
ProgressEvent,
|
|
61
|
+
RuntimeHandle,
|
|
62
|
+
SurfaceUnavailable,
|
|
63
|
+
WorkerSpawnRequest,
|
|
64
|
+
)
|
|
65
|
+
from .ports.worker_runtime import WorkerRuntimePort
|
|
66
|
+
from .execution_identity import Attempt, Invocation, RoleExecution, model_spec_digest
|
|
67
|
+
from .execution_manifest import (
|
|
68
|
+
finish_attempt_mutation,
|
|
69
|
+
read_execution_manifest,
|
|
70
|
+
record_invocation_attempt,
|
|
71
|
+
)
|
|
72
|
+
from .execution_mutation_audit import (
|
|
73
|
+
ExecutionMutationAudit,
|
|
74
|
+
MutationAuditResult,
|
|
75
|
+
MutationSnapshot,
|
|
76
|
+
assert_compatible_batch,
|
|
77
|
+
)
|
|
78
|
+
from .convergence_store import write_json_atomic
|
|
79
|
+
from .assignment_environment import load_assignment_context
|
|
80
|
+
from .model_pool import ModelPool
|
|
46
81
|
from .error_log_write import append_observed
|
|
47
82
|
from .lead_events import LeadEvent, append_lead_event
|
|
48
83
|
from .initial_prompt_materialization import (
|
|
@@ -52,6 +87,7 @@ from .initial_prompt_materialization import (
|
|
|
52
87
|
PromptDeliveryMode,
|
|
53
88
|
materialize_initial_prompts,
|
|
54
89
|
)
|
|
90
|
+
from .agent_invocation import AgentInvocationError, materialize_retry_invocation
|
|
55
91
|
from .path_hints import hydrate_active_run_context
|
|
56
92
|
from .schema_excerpt import bundle_excerpt_path, excerpt_version_skew
|
|
57
93
|
from .seeding import installed_version
|
|
@@ -72,6 +108,15 @@ from .worker_prompt_headers import (
|
|
|
72
108
|
)
|
|
73
109
|
from .worker_artifact_paths import audit_sidecar_rel
|
|
74
110
|
from .wrapper_status import read_wrapper_status, status_path_for_prompt
|
|
111
|
+
from .worker_request import verifier_extra_dirs
|
|
112
|
+
from .write_policy import (
|
|
113
|
+
WriteEnforcement,
|
|
114
|
+
WritePolicy,
|
|
115
|
+
build_invocation_write_contract,
|
|
116
|
+
planned_paths_from_run_manifest,
|
|
117
|
+
write_enforcement_from_payload,
|
|
118
|
+
write_policy_from_payload,
|
|
119
|
+
)
|
|
75
120
|
|
|
76
121
|
|
|
77
122
|
MAX_WORKER_ATTEMPTS = 2
|
|
@@ -84,6 +129,102 @@ _WRAPPER_TIMEOUT_EXIT_CODE = 124
|
|
|
84
129
|
_WRAPPER_LOG_TAIL_BYTES = 800
|
|
85
130
|
|
|
86
131
|
|
|
132
|
+
@dataclass(frozen=True)
|
|
133
|
+
class RevalidatedBinding:
|
|
134
|
+
role_execution_ref: str
|
|
135
|
+
binding: HostModelBinding
|
|
136
|
+
model_spec_digest: str
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def revalidate_role_execution(
|
|
140
|
+
role_execution: RoleExecution,
|
|
141
|
+
pool: ModelPool,
|
|
142
|
+
host: AssignmentEnvironment,
|
|
143
|
+
) -> RevalidatedBinding:
|
|
144
|
+
"""Recompute one selected model binding without inspecting unrelated rows."""
|
|
145
|
+
if (
|
|
146
|
+
role_execution.model_ref is None
|
|
147
|
+
or role_execution.binding is None
|
|
148
|
+
or role_execution.model_spec_digest is None
|
|
149
|
+
):
|
|
150
|
+
raise DispatchError("model binding changed: role execution is unbound")
|
|
151
|
+
try:
|
|
152
|
+
assignment = resolve_model_assignment(
|
|
153
|
+
instance=RoleInstance(
|
|
154
|
+
role_execution.role,
|
|
155
|
+
role_execution.role,
|
|
156
|
+
role_execution.ordinal,
|
|
157
|
+
),
|
|
158
|
+
model_ref=role_execution.model_ref,
|
|
159
|
+
pool=pool,
|
|
160
|
+
host=HostSessionContext(
|
|
161
|
+
host_id=host.host_descriptor.id,
|
|
162
|
+
entry_mode="spawn-process",
|
|
163
|
+
available_functions=frozenset(),
|
|
164
|
+
interaction_surface="role-execution-dispatch",
|
|
165
|
+
),
|
|
166
|
+
environment=host,
|
|
167
|
+
)
|
|
168
|
+
binding = assignment.binding
|
|
169
|
+
if binding is None:
|
|
170
|
+
raise AssignmentResolutionError("selected model has no worker binding")
|
|
171
|
+
digest = model_spec_digest(pool.resolve(role_execution.model_ref), binding)
|
|
172
|
+
except (ValueError, RuntimeError) as exc:
|
|
173
|
+
raise DispatchError(f"model binding changed: {exc}") from exc
|
|
174
|
+
identity_matches = (
|
|
175
|
+
assignment.provider_id == role_execution.provider
|
|
176
|
+
and assignment.model_id == role_execution.model_id
|
|
177
|
+
and binding == role_execution.binding
|
|
178
|
+
and digest == role_execution.model_spec_digest
|
|
179
|
+
)
|
|
180
|
+
if not identity_matches:
|
|
181
|
+
raise DispatchError("model binding changed for selected role execution")
|
|
182
|
+
return RevalidatedBinding(role_execution.role_execution_ref, binding, digest)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def dispatch_revalidated_role_execution(
|
|
186
|
+
role_execution: RoleExecution,
|
|
187
|
+
pool: ModelPool,
|
|
188
|
+
host: AssignmentEnvironment,
|
|
189
|
+
dispatch,
|
|
190
|
+
):
|
|
191
|
+
"""Run a dispatch callback only after the pure binding gate succeeds."""
|
|
192
|
+
binding = revalidate_role_execution(role_execution, pool, host)
|
|
193
|
+
return dispatch(binding)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def verify_served_model(
|
|
197
|
+
role_execution: RoleExecution,
|
|
198
|
+
attestation: ServedModelAttestation,
|
|
199
|
+
*,
|
|
200
|
+
pool: ModelPool,
|
|
201
|
+
) -> ServedModelAttestation:
|
|
202
|
+
"""Reject observed substitution while preserving unobservable attempts."""
|
|
203
|
+
if attestation.level == "unknown":
|
|
204
|
+
return attestation
|
|
205
|
+
if role_execution.model_ref is None or not attestation.normalized_model_ref:
|
|
206
|
+
raise DispatchError("served model differs from selected model")
|
|
207
|
+
try:
|
|
208
|
+
selected = pool.resolve(role_execution.model_ref)
|
|
209
|
+
observed = pool.resolve(attestation.normalized_model_ref)
|
|
210
|
+
except ValueError as exc:
|
|
211
|
+
raise DispatchError("served model differs from selected model") from exc
|
|
212
|
+
expected_level = "channel" if observed.version_kind == "channel" else "exact"
|
|
213
|
+
if attestation.level != expected_level:
|
|
214
|
+
raise DispatchError("served model attestation level is inconsistent")
|
|
215
|
+
if selected.version_kind == "pinned":
|
|
216
|
+
if str(selected.model_ref) != str(observed.model_ref):
|
|
217
|
+
raise DispatchError("served model differs from selected model")
|
|
218
|
+
return attestation
|
|
219
|
+
if (
|
|
220
|
+
selected.provider_id != observed.provider_id
|
|
221
|
+
or not selected.channel_family
|
|
222
|
+
or selected.channel_family != observed.channel_family
|
|
223
|
+
):
|
|
224
|
+
raise DispatchError("served model channel family differs")
|
|
225
|
+
return attestation
|
|
226
|
+
|
|
227
|
+
|
|
87
228
|
@dataclass(frozen=True)
|
|
88
229
|
class WorkerHandle:
|
|
89
230
|
job: WorkerJob
|
|
@@ -92,6 +233,12 @@ class WorkerHandle:
|
|
|
92
233
|
status_sidecar_path: Path | None
|
|
93
234
|
degraded_from: str
|
|
94
235
|
running_process: subprocess.Popen[str] | None = None
|
|
236
|
+
mutation_snapshot: MutationSnapshot | None = None
|
|
237
|
+
mutation_policies: tuple[WritePolicy, ...] = ()
|
|
238
|
+
mutation_retry_allowed: bool = True
|
|
239
|
+
write_policy_digest: str = ""
|
|
240
|
+
write_enforcement: WriteEnforcement | None = None
|
|
241
|
+
runtime_handle: RuntimeHandle | None = None
|
|
95
242
|
|
|
96
243
|
|
|
97
244
|
@dataclass(frozen=True)
|
|
@@ -115,6 +262,7 @@ class DispatchPlan:
|
|
|
115
262
|
manifest: Mapping[str, Any]
|
|
116
263
|
jobs: tuple[WorkerJob, ...]
|
|
117
264
|
default_backend: str
|
|
265
|
+
runtime_chain: tuple[WorkerRuntimePort, ...] = ()
|
|
118
266
|
|
|
119
267
|
def to_payload(self, dry_run: bool) -> dict[str, Any]:
|
|
120
268
|
payload = {
|
|
@@ -221,8 +369,10 @@ def build_dispatch_plan(
|
|
|
221
369
|
|
|
222
370
|
|
|
223
371
|
def dispatch_plan(plan: DispatchPlan, *, wait: bool = True) -> int:
|
|
372
|
+
plan = _ensure_runtime_chain(plan)
|
|
224
373
|
_validate_report_writer_isolation(plan.jobs)
|
|
225
374
|
_validate_implementation_phase_order(plan.jobs)
|
|
375
|
+
_revalidate_plan_role_executions(plan)
|
|
226
376
|
if wait:
|
|
227
377
|
pane_backends = sorted(
|
|
228
378
|
{
|
|
@@ -253,14 +403,46 @@ def dispatch_cli_wrapper_plan(plan: DispatchPlan) -> int:
|
|
|
253
403
|
"""Start one dependency-free CLI batch before collecting any worker."""
|
|
254
404
|
if any(job.backend != BACKEND_CLI_WRAPPER for job in plan.jobs):
|
|
255
405
|
raise DispatchError("concurrent CLI dispatch requires cli-wrapper jobs only")
|
|
406
|
+
plan = _ensure_runtime_chain(plan)
|
|
256
407
|
_validate_report_writer_isolation(plan.jobs)
|
|
257
408
|
_validate_implementation_phase_order(plan.jobs)
|
|
409
|
+
_revalidate_plan_role_executions(plan)
|
|
258
410
|
_record_dispatch_facts(plan.team_state_path, _dispatch_mode(plan.jobs))
|
|
259
411
|
final_codes = _dispatch_cli_wrapper_batch(plan, plan.jobs)
|
|
260
412
|
return next((final_codes[job.worker_id] for job in plan.jobs
|
|
261
413
|
if final_codes.get(job.worker_id, 0) != 0), 0)
|
|
262
414
|
|
|
263
415
|
|
|
416
|
+
def _revalidate_job_role_execution(plan: DispatchPlan, job: WorkerJob) -> None:
|
|
417
|
+
manifest = read_execution_manifest(plan.manifest_path)
|
|
418
|
+
executions = {
|
|
419
|
+
row.role_execution_ref: row for row in manifest.role_executions
|
|
420
|
+
}
|
|
421
|
+
context = load_assignment_context(
|
|
422
|
+
host_runtime=_require_string(plan.manifest, "leadRuntime"),
|
|
423
|
+
terminal_backend=_require_string(plan.manifest, "terminalBackend"),
|
|
424
|
+
execution_provider_ids=(job.provider,),
|
|
425
|
+
)
|
|
426
|
+
role_execution = executions.get(job.role_execution_ref)
|
|
427
|
+
if (
|
|
428
|
+
role_execution is None
|
|
429
|
+
or role_execution.participant_ref != job.participant_ref
|
|
430
|
+
or role_execution.execution_label != job.execution_label
|
|
431
|
+
):
|
|
432
|
+
raise DispatchError("worker job role execution does not match run manifest")
|
|
433
|
+
revalidate_role_execution(
|
|
434
|
+
role_execution,
|
|
435
|
+
context.pool,
|
|
436
|
+
context.environment,
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _revalidate_plan_role_executions(plan: DispatchPlan) -> None:
|
|
441
|
+
for job in plan.jobs:
|
|
442
|
+
if job.has_execution_identity:
|
|
443
|
+
_revalidate_job_role_execution(plan, job)
|
|
444
|
+
|
|
445
|
+
|
|
264
446
|
def _validate_report_writer_isolation(jobs: Sequence[WorkerJob]) -> None:
|
|
265
447
|
if (
|
|
266
448
|
any(job.worker_id == REPORT_WRITER_WORKER_ID for job in jobs)
|
|
@@ -328,15 +510,60 @@ def _dispatch_cli_wrapper_batch(
|
|
|
328
510
|
"""Run one concurrent batch through all retries before the next dependency."""
|
|
329
511
|
pending = [(job, 1) for job in jobs]
|
|
330
512
|
final_codes: dict[str, int] = {}
|
|
513
|
+
baseline_snapshot: MutationSnapshot | None = None
|
|
331
514
|
while pending:
|
|
332
515
|
active: list[tuple[WorkerHandle, int]] = []
|
|
516
|
+
draft_contracts = tuple(
|
|
517
|
+
_canonical_write_contract(plan, job) if job.has_execution_identity else None
|
|
518
|
+
for job, _attempt in pending
|
|
519
|
+
)
|
|
520
|
+
draft_policies = tuple(
|
|
521
|
+
contract[0] for contract in draft_contracts if contract is not None
|
|
522
|
+
)
|
|
523
|
+
if draft_policies:
|
|
524
|
+
assert_compatible_batch(draft_policies)
|
|
525
|
+
prepared = [
|
|
526
|
+
(_prepare_job_attempt(plan, job, attempt), attempt)
|
|
527
|
+
for job, attempt in pending
|
|
528
|
+
]
|
|
529
|
+
contracts = tuple(
|
|
530
|
+
_persisted_write_contract(plan, job) for job, _ in prepared
|
|
531
|
+
)
|
|
532
|
+
policies = tuple(
|
|
533
|
+
contract[0] for contract in contracts if contract is not None
|
|
534
|
+
)
|
|
535
|
+
needs_batch = any(
|
|
536
|
+
contract is not None and contract[1].mutation_audit == "batch"
|
|
537
|
+
for contract in contracts
|
|
538
|
+
)
|
|
539
|
+
if needs_batch and baseline_snapshot is None:
|
|
540
|
+
baseline_snapshot = _mutation_snapshot(
|
|
541
|
+
plan, policies, tuple(job for job, _ in prepared)
|
|
542
|
+
)
|
|
543
|
+
snapshot = baseline_snapshot if needs_batch else None
|
|
333
544
|
starting_job: WorkerJob | None = None
|
|
334
545
|
starting_attempt = 1
|
|
335
546
|
try:
|
|
336
|
-
for job, attempt in
|
|
547
|
+
for (job, attempt), contract in zip(prepared, contracts, strict=True):
|
|
337
548
|
starting_job = job
|
|
338
549
|
starting_attempt = attempt
|
|
339
|
-
|
|
550
|
+
handle = _spawn_cli_job_nonblocking(plan, job, attempt)
|
|
551
|
+
handle = replace(
|
|
552
|
+
handle,
|
|
553
|
+
mutation_snapshot=snapshot,
|
|
554
|
+
mutation_policies=policies,
|
|
555
|
+
write_policy_digest=contract[0].digest if contract else "",
|
|
556
|
+
write_enforcement=contract[1] if contract else None,
|
|
557
|
+
)
|
|
558
|
+
active.append((handle, attempt))
|
|
559
|
+
_record_dispatch(
|
|
560
|
+
plan.team_state_path, handle, attempt, "running", ""
|
|
561
|
+
)
|
|
562
|
+
_append_event(
|
|
563
|
+
plan,
|
|
564
|
+
"worker-dispatched",
|
|
565
|
+
_attempt_details(job, attempt, handle),
|
|
566
|
+
)
|
|
340
567
|
except (OSError, DispatchError) as exc:
|
|
341
568
|
if starting_job is not None:
|
|
342
569
|
_abort_cli_batch_start(
|
|
@@ -344,28 +571,40 @@ def _dispatch_cli_wrapper_batch(
|
|
|
344
571
|
active,
|
|
345
572
|
starting_job,
|
|
346
573
|
starting_attempt,
|
|
347
|
-
|
|
574
|
+
prepared,
|
|
575
|
+
snapshot,
|
|
576
|
+
policies,
|
|
348
577
|
exc,
|
|
349
578
|
)
|
|
350
579
|
raise
|
|
580
|
+
from .adapters.runtime.assembly import port_for
|
|
581
|
+
|
|
351
582
|
outcomes: list[tuple[WorkerHandle, int, WorkerOutcome]] = []
|
|
352
583
|
for handle, attempt in active:
|
|
353
|
-
|
|
354
|
-
if
|
|
584
|
+
runtime = handle.runtime_handle
|
|
585
|
+
if runtime is None or not runtime.waitable:
|
|
355
586
|
raise DispatchError("concurrent CLI worker process is missing")
|
|
356
|
-
|
|
587
|
+
port = port_for(plan.runtime_chain, runtime.surface)
|
|
588
|
+
returncode = port.wait(runtime)
|
|
357
589
|
completed = subprocess.CompletedProcess(handle.job.command, returncode)
|
|
358
|
-
settled = replace(handle, completed_process=completed
|
|
590
|
+
settled = replace(handle, completed_process=completed)
|
|
359
591
|
outcomes.append((settled, attempt, _outcome_from_completed(settled)))
|
|
360
592
|
|
|
361
593
|
next_round: list[tuple[WorkerJob, int]] = []
|
|
362
594
|
for handle, attempt, outcome in outcomes:
|
|
363
595
|
job = handle.job
|
|
364
|
-
_finish_attempt(
|
|
365
|
-
|
|
596
|
+
retry_allowed = _finish_attempt(
|
|
597
|
+
plan,
|
|
598
|
+
job,
|
|
599
|
+
attempt,
|
|
600
|
+
outcome,
|
|
601
|
+
snapshot=handle.mutation_snapshot,
|
|
602
|
+
policies=handle.mutation_policies,
|
|
603
|
+
)
|
|
604
|
+
if _job_terminal_status(plan, job, attempt) == "completed":
|
|
366
605
|
final_codes[job.worker_id] = 0
|
|
367
606
|
continue
|
|
368
|
-
if _should_retry(outcome, attempt):
|
|
607
|
+
if _should_retry(outcome, attempt) and retry_allowed:
|
|
369
608
|
_append_event(
|
|
370
609
|
plan,
|
|
371
610
|
"worker-retry-scheduled",
|
|
@@ -384,26 +623,27 @@ def _abort_cli_batch_start(
|
|
|
384
623
|
failed_job: WorkerJob,
|
|
385
624
|
failed_attempt: int,
|
|
386
625
|
pending: Sequence[tuple[WorkerJob, int]],
|
|
626
|
+
snapshot: MutationSnapshot | None,
|
|
627
|
+
policies: Sequence[WritePolicy],
|
|
387
628
|
error: Exception,
|
|
388
629
|
) -> None:
|
|
630
|
+
from .adapters.runtime.assembly import port_for
|
|
631
|
+
|
|
389
632
|
reason = f"concurrent CLI batch launch failed: {error}"
|
|
390
633
|
for handle, _attempt in active:
|
|
391
|
-
|
|
392
|
-
if
|
|
393
|
-
|
|
634
|
+
runtime = handle.runtime_handle
|
|
635
|
+
if runtime is None:
|
|
636
|
+
continue
|
|
637
|
+
port_for(plan.runtime_chain, runtime.surface).terminate(runtime)
|
|
394
638
|
for handle, attempt in active:
|
|
395
|
-
|
|
396
|
-
if
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
plan.team_state_path, handle.job.worker_id, "error", reason
|
|
400
|
-
)
|
|
639
|
+
runtime = handle.runtime_handle
|
|
640
|
+
if runtime is not None:
|
|
641
|
+
port_for(plan.runtime_chain, runtime.surface).wait(runtime)
|
|
642
|
+
_transition_job_status(plan, handle.job, "error", reason)
|
|
401
643
|
_update_dispatch_status(
|
|
402
644
|
plan.team_state_path, handle.job, attempt, "error", reason
|
|
403
645
|
)
|
|
404
|
-
|
|
405
|
-
plan.team_state_path, failed_job.worker_id, "error", reason
|
|
406
|
-
)
|
|
646
|
+
_transition_job_status(plan, failed_job, "error", reason)
|
|
407
647
|
if not _update_dispatch_status(
|
|
408
648
|
plan.team_state_path,
|
|
409
649
|
failed_job,
|
|
@@ -411,12 +651,19 @@ def _abort_cli_batch_start(
|
|
|
411
651
|
"error",
|
|
412
652
|
reason,
|
|
413
653
|
):
|
|
654
|
+
failed_contract = _persisted_write_contract(plan, failed_job)
|
|
414
655
|
failed_handle = WorkerHandle(
|
|
415
656
|
failed_job,
|
|
416
657
|
"",
|
|
417
658
|
None,
|
|
418
659
|
status_path_for_prompt(failed_job.prompt_path),
|
|
419
660
|
"",
|
|
661
|
+
mutation_snapshot=snapshot,
|
|
662
|
+
mutation_policies=tuple(policies),
|
|
663
|
+
write_policy_digest=(
|
|
664
|
+
failed_contract[0].digest if failed_contract else ""
|
|
665
|
+
),
|
|
666
|
+
write_enforcement=(failed_contract[1] if failed_contract else None),
|
|
420
667
|
)
|
|
421
668
|
_record_dispatch(
|
|
422
669
|
plan.team_state_path,
|
|
@@ -432,18 +679,21 @@ def _abort_cli_batch_start(
|
|
|
432
679
|
for job, attempt in pending:
|
|
433
680
|
if (job.worker_id, attempt) in closed:
|
|
434
681
|
continue
|
|
435
|
-
|
|
436
|
-
plan.team_state_path, job.worker_id, "error", reason
|
|
437
|
-
)
|
|
682
|
+
_transition_job_status(plan, job, "error", reason)
|
|
438
683
|
if not _update_dispatch_status(
|
|
439
684
|
plan.team_state_path, job, attempt, "error", reason
|
|
440
685
|
):
|
|
686
|
+
contract = _persisted_write_contract(plan, job)
|
|
441
687
|
handle = WorkerHandle(
|
|
442
688
|
job,
|
|
443
689
|
"",
|
|
444
690
|
None,
|
|
445
691
|
status_path_for_prompt(job.prompt_path),
|
|
446
692
|
"",
|
|
693
|
+
mutation_snapshot=snapshot,
|
|
694
|
+
mutation_policies=tuple(policies),
|
|
695
|
+
write_policy_digest=contract[0].digest if contract else "",
|
|
696
|
+
write_enforcement=contract[1] if contract else None,
|
|
447
697
|
)
|
|
448
698
|
_record_dispatch(
|
|
449
699
|
plan.team_state_path,
|
|
@@ -452,6 +702,20 @@ def _abort_cli_batch_start(
|
|
|
452
702
|
"error",
|
|
453
703
|
reason,
|
|
454
704
|
)
|
|
705
|
+
for job, attempt in pending:
|
|
706
|
+
if not job.has_execution_identity:
|
|
707
|
+
continue
|
|
708
|
+
outcome = WorkerOutcome(
|
|
709
|
+
returncode=1,
|
|
710
|
+
missing_completion_paths=_missing_completion_paths(job),
|
|
711
|
+
pane_id="",
|
|
712
|
+
status_sidecar_path=status_path_for_prompt(job.prompt_path),
|
|
713
|
+
timeout=False,
|
|
714
|
+
terminal_stage="launch-failed",
|
|
715
|
+
degraded_from="",
|
|
716
|
+
)
|
|
717
|
+
mutation = _audit_attempt(job, outcome, snapshot, policies)
|
|
718
|
+
_finish_manifest_attempt(plan, job, attempt, mutation, None)
|
|
455
719
|
|
|
456
720
|
|
|
457
721
|
def await_dispatches(
|
|
@@ -497,6 +761,15 @@ def _jobs_from_roster(
|
|
|
497
761
|
options,
|
|
498
762
|
team_state=team_state,
|
|
499
763
|
)
|
|
764
|
+
if selected == ["translator"]:
|
|
765
|
+
return [_translator_job_from_reservation(
|
|
766
|
+
project_root,
|
|
767
|
+
workspace_root,
|
|
768
|
+
manifest,
|
|
769
|
+
team_state,
|
|
770
|
+
active_context,
|
|
771
|
+
options,
|
|
772
|
+
)]
|
|
500
773
|
_mark_roster_skips(project_root, manifest, team_state, selected, requested_workers, options)
|
|
501
774
|
worker_facts, prompt_paths = _materialize_roster_prompts(
|
|
502
775
|
project_root,
|
|
@@ -533,6 +806,18 @@ def _select_workers(
|
|
|
533
806
|
recommended = _string_list(manifest.get("recommendedWorkers"))
|
|
534
807
|
if not recommended:
|
|
535
808
|
raise DispatchError("run manifest has no recommendedWorkers entries")
|
|
809
|
+
if tuple(requested_workers) == ("translator",):
|
|
810
|
+
provider = _canonical_translator_provider(manifest)
|
|
811
|
+
if provider is None:
|
|
812
|
+
raise DispatchError(
|
|
813
|
+
"translator dispatch requires a canonical translator role execution"
|
|
814
|
+
)
|
|
815
|
+
if provider not in options.supported_worker_wrappers:
|
|
816
|
+
raise DispatchError(
|
|
817
|
+
f"unsupported {options.unsupported_worker_label} translator provider: "
|
|
818
|
+
f"{provider}"
|
|
819
|
+
)
|
|
820
|
+
return ["translator"]
|
|
536
821
|
if options.cli_wrapper_assignments_only:
|
|
537
822
|
return _select_cli_wrapper_assignments(
|
|
538
823
|
recommended,
|
|
@@ -564,6 +849,178 @@ def _select_workers(
|
|
|
564
849
|
return selected
|
|
565
850
|
|
|
566
851
|
|
|
852
|
+
def _canonical_translator_provider(manifest: Mapping[str, Any]) -> str | None:
|
|
853
|
+
execution = _canonical_translator_execution(manifest)
|
|
854
|
+
return _string_value(execution.get("provider")) if execution else None
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _canonical_translator_execution(
|
|
858
|
+
manifest: Mapping[str, Any],
|
|
859
|
+
) -> Mapping[str, Any] | None:
|
|
860
|
+
assignments = manifest.get("invocationAssignments")
|
|
861
|
+
assignment = (
|
|
862
|
+
assignments.get("translator") if isinstance(assignments, Mapping) else None
|
|
863
|
+
)
|
|
864
|
+
if not isinstance(assignment, Mapping):
|
|
865
|
+
return None
|
|
866
|
+
provider = _string_value(assignment.get("provider"))
|
|
867
|
+
model_execution = _string_value(assignment.get("modelExecutionValue"))
|
|
868
|
+
runner = _string_value(assignment.get("runner"))
|
|
869
|
+
if not provider or not model_execution or runner != BACKEND_CLI_WRAPPER:
|
|
870
|
+
return None
|
|
871
|
+
role_executions = manifest.get("roleExecutions")
|
|
872
|
+
matches = [
|
|
873
|
+
row
|
|
874
|
+
for row in role_executions
|
|
875
|
+
if isinstance(row, Mapping)
|
|
876
|
+
and row.get("role") == "translator"
|
|
877
|
+
and row.get("provider") == provider
|
|
878
|
+
and isinstance(row.get("binding"), Mapping)
|
|
879
|
+
and row["binding"].get("runner") == runner
|
|
880
|
+
and row["binding"].get("resolvedExecutionValue") == model_execution
|
|
881
|
+
] if isinstance(role_executions, list) else []
|
|
882
|
+
return matches[0] if len(matches) == 1 else None
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
_TRANSLATOR_RESERVATION_KEYS = {
|
|
886
|
+
"schemaVersion", "executionIdentityVersion", "invocationId",
|
|
887
|
+
"assignmentRef", "audience", "dispatchKind", "promptPath",
|
|
888
|
+
"metadataPath", "invocationRef", "participantRef",
|
|
889
|
+
"roleExecutionRef", "dutyId", "attempt",
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def _translator_job_from_reservation(
|
|
894
|
+
project_root: Path,
|
|
895
|
+
workspace_root: Path,
|
|
896
|
+
manifest: Mapping[str, Any],
|
|
897
|
+
team_state: Mapping[str, Any],
|
|
898
|
+
active_context: Mapping[str, Any],
|
|
899
|
+
options: _BuildOptions,
|
|
900
|
+
) -> WorkerJob:
|
|
901
|
+
execution = _canonical_translator_execution(manifest)
|
|
902
|
+
if execution is None:
|
|
903
|
+
raise DispatchError(
|
|
904
|
+
"translator dispatch requires a canonical translator role execution"
|
|
905
|
+
)
|
|
906
|
+
contract = manifest.get("agentContract")
|
|
907
|
+
if not isinstance(contract, Mapping):
|
|
908
|
+
raise DispatchError("translator dispatch requires an agent contract")
|
|
909
|
+
reservation_root = _resolve_required_path(
|
|
910
|
+
project_root, contract, "invocationReservationRootPath"
|
|
911
|
+
)
|
|
912
|
+
dispatched_ids = {
|
|
913
|
+
str(row.get("dispatchId") or "")
|
|
914
|
+
for collection in (
|
|
915
|
+
team_state.get("workerDispatches") or [],
|
|
916
|
+
team_state.get("agentDispatches") or [],
|
|
917
|
+
)
|
|
918
|
+
for row in collection
|
|
919
|
+
if isinstance(row, Mapping)
|
|
920
|
+
}
|
|
921
|
+
candidates: list[Mapping[str, Any]] = []
|
|
922
|
+
for path in sorted(reservation_root.glob("*.json")):
|
|
923
|
+
reservation = _load_json_object(path, "agent invocation reservation")
|
|
924
|
+
if (
|
|
925
|
+
set(reservation) == _TRANSLATOR_RESERVATION_KEYS
|
|
926
|
+
and reservation.get("schemaVersion") == "2.0"
|
|
927
|
+
and reservation.get("executionIdentityVersion") == 2
|
|
928
|
+
and reservation.get("assignmentRef") == "translator"
|
|
929
|
+
and reservation.get("audience") == "translator"
|
|
930
|
+
and reservation.get("dispatchKind") == "translator"
|
|
931
|
+
and reservation.get("dutyId") == "translator"
|
|
932
|
+
and reservation.get("participantRef") == execution.get("participantRef")
|
|
933
|
+
and reservation.get("roleExecutionRef") == execution.get("roleExecutionRef")
|
|
934
|
+
and _build_dispatch_id(
|
|
935
|
+
str(reservation.get("invocationId") or ""),
|
|
936
|
+
int(reservation.get("attempt") or 0),
|
|
937
|
+
) not in dispatched_ids
|
|
938
|
+
):
|
|
939
|
+
candidates.append(reservation)
|
|
940
|
+
if len(candidates) != 1:
|
|
941
|
+
raise DispatchError(
|
|
942
|
+
"translator dispatch requires exactly one canonical invocation reservation"
|
|
943
|
+
)
|
|
944
|
+
reservation = candidates[0]
|
|
945
|
+
prompt_path = _resolve_project_path(
|
|
946
|
+
project_root, _require_string(reservation, "promptPath")
|
|
947
|
+
)
|
|
948
|
+
metadata_path = _resolve_project_path(
|
|
949
|
+
project_root, _require_string(reservation, "metadataPath")
|
|
950
|
+
)
|
|
951
|
+
expected_metadata = prompt_path.with_name(prompt_path.name + ".meta.json")
|
|
952
|
+
if metadata_path != expected_metadata:
|
|
953
|
+
raise DispatchError("translator invocation metadata path is not prompt-adjacent")
|
|
954
|
+
metadata = _load_json_object(metadata_path, "agent invocation metadata")
|
|
955
|
+
invocation = _invocation_job_fields(metadata, prompt_path, manifest)
|
|
956
|
+
expected = {
|
|
957
|
+
"invocationId": invocation["invocation_id"],
|
|
958
|
+
"invocationRef": invocation["invocation_ref"],
|
|
959
|
+
"participantRef": invocation["participant_ref"],
|
|
960
|
+
"roleExecutionRef": invocation["role_execution_ref"],
|
|
961
|
+
"dutyId": invocation["duty_id"],
|
|
962
|
+
"attempt": invocation["attempt"],
|
|
963
|
+
}
|
|
964
|
+
if any(reservation.get(key) != value for key, value in expected.items()):
|
|
965
|
+
raise DispatchError("translator reservation and invocation metadata differ")
|
|
966
|
+
if (
|
|
967
|
+
invocation["audience"] != "translator"
|
|
968
|
+
or invocation["assignment_ref"] != "translator"
|
|
969
|
+
or metadata.get("dispatchKind") != "translator"
|
|
970
|
+
):
|
|
971
|
+
raise DispatchError("translator invocation identity is invalid")
|
|
972
|
+
result_path, worker_result_path = _translator_prompt_outputs(
|
|
973
|
+
project_root, prompt_path
|
|
974
|
+
)
|
|
975
|
+
if result_path == worker_result_path:
|
|
976
|
+
raise DispatchError(
|
|
977
|
+
"translator result and standard worker result paths must differ"
|
|
978
|
+
)
|
|
979
|
+
provider = _require_string(execution, "provider")
|
|
980
|
+
binding = execution.get("binding")
|
|
981
|
+
if not isinstance(binding, Mapping):
|
|
982
|
+
raise DispatchError("translator role execution has no binding")
|
|
983
|
+
return WorkerJob(
|
|
984
|
+
worker_id="translator",
|
|
985
|
+
provider=provider,
|
|
986
|
+
backend=options.default_backend,
|
|
987
|
+
project_root=project_root,
|
|
988
|
+
model_execution_value=_require_string(binding, "resolvedExecutionValue"),
|
|
989
|
+
wrapper_path=_resolve_wrapper(provider, workspace_root, options),
|
|
990
|
+
prompt_path=prompt_path,
|
|
991
|
+
result_path=result_path,
|
|
992
|
+
worker_result_path=worker_result_path,
|
|
993
|
+
completion_paths=(result_path, worker_result_path),
|
|
994
|
+
worktree_path=_worktree_path(manifest, active_context),
|
|
995
|
+
role="translator",
|
|
996
|
+
idle_timeout_seconds=options.idle_timeout_seconds,
|
|
997
|
+
dispatch_kind="translator",
|
|
998
|
+
**invocation,
|
|
999
|
+
)
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def _translator_prompt_outputs(
|
|
1003
|
+
project_root: Path, prompt_path: Path,
|
|
1004
|
+
) -> tuple[Path, Path]:
|
|
1005
|
+
try:
|
|
1006
|
+
lines = prompt_path.read_text(encoding="utf-8").splitlines()
|
|
1007
|
+
except OSError as exc:
|
|
1008
|
+
raise DispatchError(f"cannot read translator prompt: {exc}") from exc
|
|
1009
|
+
values: list[str] = []
|
|
1010
|
+
for header in ("Result Path", "Worker Result Path"):
|
|
1011
|
+
prefix = f"**{header}:** "
|
|
1012
|
+
matches = [line[len(prefix):].strip() for line in lines if line.startswith(prefix)]
|
|
1013
|
+
if len(matches) != 1 or not matches[0]:
|
|
1014
|
+
raise DispatchError(
|
|
1015
|
+
f"translator prompt requires exactly one non-empty {prefix.strip()} header"
|
|
1016
|
+
)
|
|
1017
|
+
values.append(matches[0])
|
|
1018
|
+
return (
|
|
1019
|
+
_resolve_project_path(project_root, values[0]),
|
|
1020
|
+
_resolve_project_path(project_root, values[1]),
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
|
|
567
1024
|
def _select_cli_wrapper_assignments(
|
|
568
1025
|
recommended: Sequence[str],
|
|
569
1026
|
requested_workers: Sequence[str],
|
|
@@ -642,7 +1099,7 @@ def _job_from_roster_worker(
|
|
|
642
1099
|
) -> WorkerJob:
|
|
643
1100
|
result_path = _resolve_project_path(project_root, fact.result_path)
|
|
644
1101
|
metadata = _agent_metadata(prompt_path)
|
|
645
|
-
invocation = _invocation_job_fields(metadata, prompt_path)
|
|
1102
|
+
invocation = _invocation_job_fields(metadata, prompt_path, manifest)
|
|
646
1103
|
return WorkerJob(
|
|
647
1104
|
worker_id=fact.worker_id,
|
|
648
1105
|
provider=fact.provider,
|
|
@@ -677,6 +1134,7 @@ def _agent_metadata(prompt_path: Path) -> Mapping[str, Any] | None:
|
|
|
677
1134
|
def _invocation_job_fields(
|
|
678
1135
|
metadata: Mapping[str, Any] | None,
|
|
679
1136
|
prompt_path: Path,
|
|
1137
|
+
manifest: Mapping[str, Any],
|
|
680
1138
|
) -> dict[str, Any]:
|
|
681
1139
|
if metadata is None:
|
|
682
1140
|
return {}
|
|
@@ -687,7 +1145,7 @@ def _invocation_job_fields(
|
|
|
687
1145
|
host_model_value = assignment.get("hostModelValue")
|
|
688
1146
|
if host_model_value is not None and not isinstance(host_model_value, str):
|
|
689
1147
|
raise DispatchError("agent invocation hostModelValue is invalid")
|
|
690
|
-
|
|
1148
|
+
fields = {
|
|
691
1149
|
"invocation_id": _require_string(metadata, "invocationId"),
|
|
692
1150
|
"audience": _require_string(metadata, "audience"),
|
|
693
1151
|
"assignment_ref": _require_string(metadata, "assignmentRef"),
|
|
@@ -702,6 +1160,37 @@ def _invocation_job_fields(
|
|
|
702
1160
|
"host_model_value": host_model_value,
|
|
703
1161
|
"enforcement_mode": "core-pre-dispatch",
|
|
704
1162
|
}
|
|
1163
|
+
schema = metadata.get("schemaVersion")
|
|
1164
|
+
identity_version = metadata.get("executionIdentityVersion")
|
|
1165
|
+
if schema == 1 and identity_version is None:
|
|
1166
|
+
return fields
|
|
1167
|
+
if schema != "2.0" or identity_version != 2:
|
|
1168
|
+
raise DispatchError("agent invocation metadata mixes v1 and v2 identity")
|
|
1169
|
+
participant_ref = _require_string(metadata, "participantRef")
|
|
1170
|
+
role_execution_ref = _require_string(metadata, "roleExecutionRef")
|
|
1171
|
+
role_executions = manifest.get("roleExecutions")
|
|
1172
|
+
execution = next((
|
|
1173
|
+
row for row in role_executions
|
|
1174
|
+
if isinstance(row, Mapping)
|
|
1175
|
+
and row.get("roleExecutionRef") == role_execution_ref
|
|
1176
|
+
), None) if isinstance(role_executions, list) else None
|
|
1177
|
+
if execution is None or execution.get("participantRef") != participant_ref:
|
|
1178
|
+
raise DispatchError("agent invocation role execution does not match run manifest")
|
|
1179
|
+
execution_label = _require_string(metadata, "executionLabel")
|
|
1180
|
+
if execution.get("executionLabel") != execution_label:
|
|
1181
|
+
raise DispatchError("agent invocation execution label does not match run manifest")
|
|
1182
|
+
attempt = metadata.get("attempt")
|
|
1183
|
+
if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1:
|
|
1184
|
+
raise DispatchError("agent invocation v2 attempt must be positive")
|
|
1185
|
+
fields.update({
|
|
1186
|
+
"participant_ref": participant_ref,
|
|
1187
|
+
"role_execution_ref": role_execution_ref,
|
|
1188
|
+
"execution_label": execution_label,
|
|
1189
|
+
"duty_id": _require_string(metadata, "dutyId"),
|
|
1190
|
+
"invocation_ref": _require_string(metadata, "invocationRef"),
|
|
1191
|
+
"attempt": attempt,
|
|
1192
|
+
})
|
|
1193
|
+
return fields
|
|
705
1194
|
|
|
706
1195
|
|
|
707
1196
|
def _materialize_roster_prompts(
|
|
@@ -775,40 +1264,264 @@ def _jobs_from_file(
|
|
|
775
1264
|
|
|
776
1265
|
|
|
777
1266
|
def _spawn_job(plan: DispatchPlan, job: WorkerJob, attempt: int) -> WorkerHandle:
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
1267
|
+
job = _prepare_job_attempt(plan, job, attempt)
|
|
1268
|
+
contract = _persisted_write_contract(plan, job)
|
|
1269
|
+
policies = (contract[0],) if contract else ()
|
|
1270
|
+
snapshot = None
|
|
1271
|
+
if contract is not None and contract[1].mutation_audit == "batch":
|
|
1272
|
+
snapshot_path = _mutation_snapshot_path(job)
|
|
1273
|
+
if snapshot_path.is_file():
|
|
1274
|
+
snapshot = MutationSnapshot.from_payload(
|
|
1275
|
+
_load_json_object(snapshot_path, "mutation audit snapshot")
|
|
1276
|
+
)
|
|
1277
|
+
else:
|
|
1278
|
+
snapshot = _mutation_snapshot(plan, policies, (job,))
|
|
1279
|
+
handle = replace(
|
|
1280
|
+
_start_job(plan, job),
|
|
1281
|
+
mutation_snapshot=snapshot,
|
|
1282
|
+
mutation_policies=policies,
|
|
1283
|
+
write_policy_digest=contract[0].digest if contract else "",
|
|
1284
|
+
write_enforcement=contract[1] if contract else None,
|
|
781
1285
|
)
|
|
782
|
-
handle = _start_job(plan, job)
|
|
783
1286
|
_record_dispatch(plan.team_state_path, handle, attempt, "running", "")
|
|
784
1287
|
_append_event(plan, "worker-dispatched", _attempt_details(job, attempt, handle))
|
|
785
1288
|
if handle.completed_process is not None:
|
|
786
1289
|
outcome = _outcome_from_completed(handle)
|
|
787
|
-
_finish_attempt(
|
|
1290
|
+
retry_allowed = _finish_attempt(
|
|
1291
|
+
plan,
|
|
1292
|
+
job,
|
|
1293
|
+
attempt,
|
|
1294
|
+
outcome,
|
|
1295
|
+
snapshot=snapshot,
|
|
1296
|
+
policies=policies,
|
|
1297
|
+
)
|
|
1298
|
+
handle = replace(handle, mutation_retry_allowed=retry_allowed)
|
|
788
1299
|
return handle
|
|
789
1300
|
|
|
790
1301
|
|
|
791
1302
|
def _spawn_cli_job_nonblocking(
|
|
792
1303
|
plan: DispatchPlan, job: WorkerJob, attempt: int,
|
|
793
1304
|
) -> WorkerHandle:
|
|
794
|
-
|
|
795
|
-
|
|
1305
|
+
plan = _ensure_runtime_chain(plan)
|
|
1306
|
+
try:
|
|
1307
|
+
runtime = open_worker(
|
|
1308
|
+
_chain_starting_at(plan.runtime_chain, job.backend),
|
|
1309
|
+
lambda surface: _spawn_request(plan, job, surface),
|
|
1310
|
+
)
|
|
1311
|
+
except EnvironmentBlocked as exc:
|
|
1312
|
+
raise DispatchError(str(exc)) from exc
|
|
1313
|
+
except (SurfaceUnavailable, OSError) as exc:
|
|
1314
|
+
raise DispatchError(str(exc)) from exc
|
|
1315
|
+
return WorkerHandle(
|
|
1316
|
+
job,
|
|
1317
|
+
"",
|
|
1318
|
+
None,
|
|
1319
|
+
status_path_for_prompt(job.prompt_path),
|
|
1320
|
+
"",
|
|
1321
|
+
runtime_handle=runtime,
|
|
1322
|
+
)
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _prepare_job_attempt(
|
|
1326
|
+
plan: DispatchPlan, job: WorkerJob, attempt: int
|
|
1327
|
+
) -> WorkerJob:
|
|
1328
|
+
if job.has_execution_identity:
|
|
1329
|
+
_revalidate_job_role_execution(plan, job)
|
|
1330
|
+
job = _job_for_attempt(job, attempt)
|
|
1331
|
+
_record_execution_attempt(plan, job)
|
|
1332
|
+
_transition_job_status(
|
|
1333
|
+
plan,
|
|
1334
|
+
job,
|
|
1335
|
+
"in-progress",
|
|
1336
|
+
"",
|
|
796
1337
|
model_execution_value=job.model_execution_value,
|
|
797
1338
|
)
|
|
798
|
-
|
|
1339
|
+
return job
|
|
1340
|
+
|
|
1341
|
+
|
|
1342
|
+
def _persisted_write_contract(
|
|
1343
|
+
plan: DispatchPlan, job: WorkerJob
|
|
1344
|
+
) -> tuple[WritePolicy, WriteEnforcement] | None:
|
|
1345
|
+
if not job.has_execution_identity:
|
|
1346
|
+
return None
|
|
1347
|
+
manifest = read_execution_manifest(plan.manifest_path)
|
|
1348
|
+
invocation = next(
|
|
1349
|
+
(
|
|
1350
|
+
row for row in manifest.invocations
|
|
1351
|
+
if row.invocation_ref == job.invocation_ref
|
|
1352
|
+
),
|
|
1353
|
+
None,
|
|
1354
|
+
)
|
|
1355
|
+
if invocation is None:
|
|
1356
|
+
raise DispatchError("worker invocation write policy was not persisted")
|
|
799
1357
|
try:
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
job, "", None, status_path_for_prompt(job.prompt_path), "", process
|
|
1358
|
+
enforcement = write_enforcement_from_payload(
|
|
1359
|
+
invocation.write_enforcement
|
|
803
1360
|
)
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
raise
|
|
1361
|
+
policy = replace(
|
|
1362
|
+
write_policy_from_payload(invocation.write_policy),
|
|
1363
|
+
maximum_boundary_precision=enforcement.boundary_precision,
|
|
1364
|
+
)
|
|
1365
|
+
except (TypeError, ValueError) as exc:
|
|
1366
|
+
raise DispatchError(f"persisted worker write policy is invalid: {exc}") from exc
|
|
1367
|
+
if policy.digest != invocation.write_policy_digest:
|
|
1368
|
+
raise DispatchError("persisted worker write policy digest changed")
|
|
1369
|
+
return policy, enforcement
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
def _mutation_snapshot(
|
|
1373
|
+
plan: DispatchPlan,
|
|
1374
|
+
policies: Sequence[WritePolicy],
|
|
1375
|
+
jobs: Sequence[WorkerJob],
|
|
1376
|
+
) -> MutationSnapshot:
|
|
1377
|
+
sidecars = tuple(_mutation_snapshot_path(job) for job in jobs)
|
|
1378
|
+
snapshot = ExecutionMutationAudit().snapshot(
|
|
1379
|
+
policies,
|
|
1380
|
+
orchestrator_paths=(
|
|
1381
|
+
plan.manifest_path,
|
|
1382
|
+
plan.team_state_path,
|
|
1383
|
+
Path(f"{plan.team_state_path}.lock"),
|
|
1384
|
+
plan.lead_events_path,
|
|
1385
|
+
plan.lead_events_path.with_name(
|
|
1386
|
+
f".{plan.lead_events_path.name}.lock"
|
|
1387
|
+
),
|
|
1388
|
+
*sidecars,
|
|
1389
|
+
),
|
|
1390
|
+
)
|
|
1391
|
+
for path in sidecars:
|
|
1392
|
+
write_json_atomic(path, snapshot.to_payload())
|
|
1393
|
+
return snapshot
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def _mutation_snapshot_path(job: WorkerJob) -> Path:
|
|
1397
|
+
return job.prompt_path.with_suffix(job.prompt_path.suffix + ".mutation-audit.json")
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def _record_execution_attempt(plan: DispatchPlan, job: WorkerJob) -> None:
|
|
1401
|
+
task_key = _require_string(plan.manifest, "taskKey")
|
|
1402
|
+
write_policy, write_enforcement = _canonical_write_contract(plan, job)
|
|
1403
|
+
record_invocation_attempt(
|
|
1404
|
+
plan.manifest_path,
|
|
1405
|
+
Invocation(
|
|
1406
|
+
invocation_ref=job.invocation_ref,
|
|
1407
|
+
participant_ref=job.participant_ref,
|
|
1408
|
+
role_execution_ref=job.role_execution_ref,
|
|
1409
|
+
source_invocation_ref=None,
|
|
1410
|
+
recovery_ref=None,
|
|
1411
|
+
duty_id=job.duty_id,
|
|
1412
|
+
dispatch_kind=job.dispatch_kind,
|
|
1413
|
+
round=_dispatch_round(job.dispatch_kind),
|
|
1414
|
+
input_digest=job.prompt_digest,
|
|
1415
|
+
write_policy=write_policy.to_payload(),
|
|
1416
|
+
write_policy_digest=write_policy.digest,
|
|
1417
|
+
write_enforcement=write_enforcement.to_payload(),
|
|
1418
|
+
),
|
|
1419
|
+
Attempt(
|
|
1420
|
+
invocation_ref=job.invocation_ref,
|
|
1421
|
+
attempt=job.attempt,
|
|
1422
|
+
started_at=_utc_now(),
|
|
1423
|
+
finished_at=None,
|
|
1424
|
+
status="started",
|
|
1425
|
+
result_path=None,
|
|
1426
|
+
error_path=None,
|
|
1427
|
+
change_summary={},
|
|
1428
|
+
evidence_seal_ref=None,
|
|
1429
|
+
),
|
|
1430
|
+
task_key=task_key,
|
|
1431
|
+
)
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
def _canonical_write_contract(
|
|
1435
|
+
plan: DispatchPlan, job: WorkerJob
|
|
1436
|
+
) -> tuple[WritePolicy, WriteEnforcement]:
|
|
1437
|
+
manifest = read_execution_manifest(plan.manifest_path)
|
|
1438
|
+
execution = next(
|
|
1439
|
+
(
|
|
1440
|
+
row for row in manifest.role_executions
|
|
1441
|
+
if row.role_execution_ref == job.role_execution_ref
|
|
1442
|
+
),
|
|
1443
|
+
None,
|
|
1444
|
+
)
|
|
1445
|
+
if execution is None or execution.binding is None:
|
|
1446
|
+
raise DispatchError("worker write policy has no role execution binding")
|
|
1447
|
+
capability = execution.binding.worker_write_capability
|
|
1448
|
+
if capability is None:
|
|
1449
|
+
raise DispatchError("worker write policy has no runner write capability")
|
|
1450
|
+
artifacts = _worker_artifact_paths(plan, job)
|
|
1451
|
+
worktree = Path(job.worktree_path) if job.worktree_path else None
|
|
1452
|
+
try:
|
|
1453
|
+
return build_invocation_write_contract(
|
|
1454
|
+
role=execution.role,
|
|
1455
|
+
project_root=plan.project_root,
|
|
1456
|
+
worktree=worktree,
|
|
1457
|
+
artifact_paths=artifacts,
|
|
1458
|
+
maximum_precision=capability.max_boundary_precision,
|
|
1459
|
+
planned_paths=(
|
|
1460
|
+
planned_paths_from_run_manifest(plan.project_root, plan.manifest)
|
|
1461
|
+
if execution.role == "implementer"
|
|
1462
|
+
else ()
|
|
1463
|
+
),
|
|
1464
|
+
auxiliary_roots=verifier_extra_dirs(execution.role),
|
|
1465
|
+
validated_auxiliary_roots=verifier_extra_dirs(execution.role),
|
|
1466
|
+
)
|
|
1467
|
+
except (OSError, ValueError) as exc:
|
|
1468
|
+
raise DispatchError(f"worker write policy is invalid: {exc}") from exc
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def _worker_artifact_paths(plan: DispatchPlan, job: WorkerJob) -> tuple[Path, ...]:
|
|
1472
|
+
active_context = _load_optional_json(
|
|
1473
|
+
plan.project_root, plan.manifest.get("activeRunContextPath")
|
|
1474
|
+
)
|
|
1475
|
+
paths = {
|
|
1476
|
+
job.result_path,
|
|
1477
|
+
job.worker_result_path,
|
|
1478
|
+
*job.completion_paths,
|
|
1479
|
+
Path(audit_sidecar_rel(str(job.worker_result_path))),
|
|
1480
|
+
status_path_for_prompt(job.prompt_path),
|
|
1481
|
+
job.prompt_path.with_suffix(job.prompt_path.suffix + ".log"),
|
|
1482
|
+
}
|
|
1483
|
+
error_logs = active_context.get("errorLogs")
|
|
1484
|
+
if isinstance(error_logs, Mapping):
|
|
1485
|
+
sidecars = error_logs.get("sidecarsByWorkerId")
|
|
1486
|
+
if isinstance(sidecars, Mapping):
|
|
1487
|
+
value = _string_value(sidecars.get(job.worker_id))
|
|
1488
|
+
if value:
|
|
1489
|
+
paths.add(_resolve_project_path(plan.project_root, value))
|
|
1490
|
+
return tuple(sorted(paths, key=str))
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
|
|
1495
|
+
def _job_for_attempt(job: WorkerJob, attempt: int) -> WorkerJob:
|
|
1496
|
+
if attempt == job.attempt:
|
|
1497
|
+
return job
|
|
1498
|
+
try:
|
|
1499
|
+
retry = materialize_retry_invocation(
|
|
1500
|
+
job.prompt_metadata_path,
|
|
1501
|
+
project_root=job.project_root,
|
|
1502
|
+
attempt=attempt,
|
|
1503
|
+
)
|
|
1504
|
+
except AgentInvocationError as exc:
|
|
1505
|
+
raise DispatchError(f"cannot materialize retry invocation: {exc}") from exc
|
|
1506
|
+
if retry.invocation_ref != job.invocation_ref:
|
|
1507
|
+
raise DispatchError("retry invocation reference changed")
|
|
1508
|
+
return replace(
|
|
1509
|
+
job,
|
|
1510
|
+
prompt_path=retry.prompt_path,
|
|
1511
|
+
prompt_metadata_path=retry.metadata_path,
|
|
1512
|
+
invocation_id=retry.invocation_id,
|
|
1513
|
+
attempt=retry.attempt,
|
|
1514
|
+
)
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
def _dispatch_round(dispatch_kind: str) -> int:
|
|
1518
|
+
prefix = "reverify-r"
|
|
1519
|
+
if not dispatch_kind.startswith(prefix):
|
|
1520
|
+
return 1
|
|
1521
|
+
suffix = dispatch_kind[len(prefix):]
|
|
1522
|
+
if not suffix.isdigit() or int(suffix) < 1:
|
|
1523
|
+
raise DispatchError(f"invalid reverify dispatch kind: {dispatch_kind}")
|
|
1524
|
+
return int(suffix)
|
|
812
1525
|
|
|
813
1526
|
|
|
814
1527
|
def _dispatch_job_with_retry(plan: DispatchPlan, job: WorkerJob) -> int:
|
|
@@ -817,9 +1530,9 @@ def _dispatch_job_with_retry(plan: DispatchPlan, job: WorkerJob) -> int:
|
|
|
817
1530
|
if handle.completed_process is None:
|
|
818
1531
|
return await_dispatches(plan, timeout_seconds=None)
|
|
819
1532
|
outcome = _outcome_from_completed(handle)
|
|
820
|
-
if
|
|
1533
|
+
if _job_terminal_status(plan, job, attempt) == "completed":
|
|
821
1534
|
return 0
|
|
822
|
-
if _should_retry(outcome, attempt):
|
|
1535
|
+
if _should_retry(outcome, attempt) and handle.mutation_retry_allowed:
|
|
823
1536
|
_append_event(plan, "worker-retry-scheduled", _retry_details(job, attempt, outcome))
|
|
824
1537
|
job = _job_for_next_attempt(job)
|
|
825
1538
|
continue
|
|
@@ -827,31 +1540,66 @@ def _dispatch_job_with_retry(plan: DispatchPlan, job: WorkerJob) -> int:
|
|
|
827
1540
|
return 1
|
|
828
1541
|
|
|
829
1542
|
|
|
1543
|
+
def _ensure_runtime_chain(plan: DispatchPlan) -> DispatchPlan:
|
|
1544
|
+
if plan.runtime_chain:
|
|
1545
|
+
return plan
|
|
1546
|
+
from .adapters.runtime.assembly import runtime_chain as build_chain
|
|
1547
|
+
return replace(plan, runtime_chain=build_chain(plan.default_backend))
|
|
1548
|
+
|
|
1549
|
+
|
|
1550
|
+
def _spawn_request(
|
|
1551
|
+
plan: DispatchPlan, job: WorkerJob, surface: str
|
|
1552
|
+
) -> WorkerSpawnRequest:
|
|
1553
|
+
targeted = replace(job, backend=surface)
|
|
1554
|
+
title = (
|
|
1555
|
+
targeted.execution_label
|
|
1556
|
+
if targeted.has_execution_identity
|
|
1557
|
+
else f"{targeted.worker_id}-worker"
|
|
1558
|
+
)
|
|
1559
|
+
return WorkerSpawnRequest(
|
|
1560
|
+
command=tuple(targeted.command),
|
|
1561
|
+
cwd=plan.project_root,
|
|
1562
|
+
title=title,
|
|
1563
|
+
owned_surface_ids=_opened_cmux_surfaces(plan.team_state_path),
|
|
1564
|
+
)
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
def _chain_starting_at(
|
|
1568
|
+
chain: tuple[WorkerRuntimePort, ...], surface: str
|
|
1569
|
+
) -> tuple[WorkerRuntimePort, ...]:
|
|
1570
|
+
# A retry job is already cli-wrapper; do not walk back up to cmux-pane.
|
|
1571
|
+
for index, port in enumerate(chain):
|
|
1572
|
+
if port.surface == surface:
|
|
1573
|
+
return chain[index:]
|
|
1574
|
+
raise DispatchError(f"unsupported worker backend: {surface}")
|
|
1575
|
+
|
|
1576
|
+
|
|
830
1577
|
def _start_job(plan: DispatchPlan, job: WorkerJob) -> WorkerHandle:
|
|
831
|
-
|
|
832
|
-
return _run_cli_wrapper(plan, job, "")
|
|
833
|
-
if job.backend == BACKEND_CMUX_PANE:
|
|
834
|
-
return _start_cmux_or_degrade(plan, job)
|
|
835
|
-
raise DispatchError(f"unsupported worker backend: {job.backend}")
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
def _start_cmux_or_degrade(plan: DispatchPlan, job: WorkerJob) -> WorkerHandle:
|
|
839
|
-
workspace = cmux.resolve_lead_workspace()
|
|
840
|
-
if not workspace:
|
|
841
|
-
_refuse_when_cmux_is_walled_off()
|
|
842
|
-
return _run_cli_wrapper(plan, job, BACKEND_CMUX_PANE)
|
|
1578
|
+
plan = _ensure_runtime_chain(plan)
|
|
843
1579
|
try:
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
command=job.command,
|
|
848
|
-
title=f"{job.worker_id}-worker",
|
|
849
|
-
owned_surface_ids=_opened_cmux_surfaces(plan.team_state_path),
|
|
1580
|
+
runtime = open_worker(
|
|
1581
|
+
_chain_starting_at(plan.runtime_chain, job.backend),
|
|
1582
|
+
lambda surface: _spawn_request(plan, job, surface),
|
|
850
1583
|
)
|
|
851
|
-
except
|
|
852
|
-
|
|
1584
|
+
except EnvironmentBlocked as exc:
|
|
1585
|
+
raise DispatchError(str(exc)) from exc
|
|
1586
|
+
except SurfaceUnavailable as exc:
|
|
1587
|
+
raise DispatchError(str(exc)) from exc
|
|
1588
|
+
degraded_from = job.backend if runtime.surface != job.backend else ""
|
|
1589
|
+
settled = replace(job, backend=runtime.surface) if degraded_from else job
|
|
1590
|
+
completed = None
|
|
1591
|
+
if runtime.waitable:
|
|
1592
|
+
from .adapters.runtime.assembly import port_for
|
|
1593
|
+
returncode = port_for(plan.runtime_chain, runtime.surface).wait(runtime)
|
|
1594
|
+
completed = subprocess.CompletedProcess(list(settled.command), returncode)
|
|
1595
|
+
pane_id = runtime.identifier if runtime.surface == SURFACE_CMUX_PANE else ""
|
|
853
1596
|
return WorkerHandle(
|
|
854
|
-
|
|
1597
|
+
settled,
|
|
1598
|
+
pane_id,
|
|
1599
|
+
completed,
|
|
1600
|
+
status_path_for_prompt(settled.prompt_path),
|
|
1601
|
+
degraded_from,
|
|
1602
|
+
runtime_handle=runtime,
|
|
855
1603
|
)
|
|
856
1604
|
|
|
857
1605
|
|
|
@@ -877,42 +1625,6 @@ def _opened_cmux_surfaces(team_state_path: Path) -> tuple[str, ...]:
|
|
|
877
1625
|
return tuple(opened)
|
|
878
1626
|
|
|
879
1627
|
|
|
880
|
-
def _refuse_when_cmux_is_walled_off() -> None:
|
|
881
|
-
"""Stop rather than degrade when a sandbox stands between okstra and cmux.
|
|
882
|
-
|
|
883
|
-
prepare records `cmux-pane` only after reaching cmux, so losing it by
|
|
884
|
-
dispatch time means something came between. When the socket is present but
|
|
885
|
-
connecting is denied, that something is a sandbox around the lead — and the
|
|
886
|
-
same sandbox hides the worker CLIs' own config, so the fallback this would
|
|
887
|
-
otherwise take is already dead. Degrading here spends every worker's retry
|
|
888
|
-
budget on the same wall and leaves the user with three failures and no
|
|
889
|
-
explanation, which is exactly what it looks like when nothing happens.
|
|
890
|
-
"""
|
|
891
|
-
reason = cmux.unreachable_reason()
|
|
892
|
-
if reason not in (cmux.LOST_ENVIRONMENT, cmux.LOST_DENIED):
|
|
893
|
-
return
|
|
894
|
-
observed = (
|
|
895
|
-
"this process has no cmux environment left — a sandbox sanitized it"
|
|
896
|
-
if reason == cmux.LOST_ENVIRONMENT
|
|
897
|
-
else "cmux is running but this process may not connect to its socket"
|
|
898
|
-
)
|
|
899
|
-
raise DispatchError(
|
|
900
|
-
f"this run was prepared for cmux panes, but {observed}. The same "
|
|
901
|
-
"sandbox blocks worker CLIs from their own config, so falling back to "
|
|
902
|
-
"blocking workers would fail too. Relaunch the lead outside a sandbox "
|
|
903
|
-
"(codex: `codex -s danger-full-access`)."
|
|
904
|
-
)
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
def _run_cli_wrapper(plan: DispatchPlan, job: WorkerJob, degraded_from: str) -> WorkerHandle:
|
|
908
|
-
# Whatever the job was planned as, this is a cli-wrapper dispatch: no pane
|
|
909
|
-
# took it, so the worker's stdout is read by whoever called the lead. The
|
|
910
|
-
# record already says so (`_dispatch_record`), and the command must agree.
|
|
911
|
-
job = replace(job, backend=BACKEND_CLI_WRAPPER)
|
|
912
|
-
completed = subprocess.run(job.command, cwd=plan.project_root, text=True)
|
|
913
|
-
return WorkerHandle(job, "", completed, status_path_for_prompt(job.prompt_path), degraded_from)
|
|
914
|
-
|
|
915
|
-
|
|
916
1628
|
def _outcome_from_completed(handle: WorkerHandle) -> WorkerOutcome:
|
|
917
1629
|
completed = handle.completed_process
|
|
918
1630
|
if completed is None:
|
|
@@ -964,16 +1676,17 @@ def _advance_running_dispatches(plan: DispatchPlan, records: Sequence[Mapping[st
|
|
|
964
1676
|
if status is None or not status.is_terminal:
|
|
965
1677
|
continue
|
|
966
1678
|
outcome = _outcome_from_status(record, status)
|
|
967
|
-
|
|
1679
|
+
retry_allowed = _finish_record(plan, record, outcome)
|
|
1680
|
+
if (
|
|
1681
|
+
_should_retry(outcome, int(record.get("attempt", 1)))
|
|
1682
|
+
and retry_allowed
|
|
1683
|
+
):
|
|
968
1684
|
_retry_from_record(plan, record, outcome)
|
|
969
|
-
continue
|
|
970
|
-
_finish_record(plan, record, outcome)
|
|
971
1685
|
|
|
972
1686
|
|
|
973
1687
|
def _retry_from_record(
|
|
974
1688
|
plan: DispatchPlan, record: Mapping[str, Any], outcome: WorkerOutcome
|
|
975
1689
|
) -> None:
|
|
976
|
-
worker_id = _require_string(record, "workerId")
|
|
977
1690
|
attempt = int(record.get("attempt", 1))
|
|
978
1691
|
job = _job_from_record(plan.project_root, record)
|
|
979
1692
|
reason = "required worker artifact was not produced"
|
|
@@ -981,7 +1694,7 @@ def _retry_from_record(
|
|
|
981
1694
|
# `team-contract` counts the first attempt's failure as a recorded
|
|
982
1695
|
# `cli-failure`; a retry that succeeds settles `completed` and would
|
|
983
1696
|
# otherwise leave no trace that anything had to be re-run.
|
|
984
|
-
details
|
|
1697
|
+
details = _event_execution_identity(job, attempt)
|
|
985
1698
|
details["errorLogAppend"] = _record_wrapper_failure(
|
|
986
1699
|
plan, job, attempt, outcome, reason
|
|
987
1700
|
)
|
|
@@ -991,14 +1704,37 @@ def _retry_from_record(
|
|
|
991
1704
|
_spawn_job(plan, _job_for_next_attempt(job), attempt + 1)
|
|
992
1705
|
|
|
993
1706
|
|
|
994
|
-
def _finish_attempt(
|
|
1707
|
+
def _finish_attempt(
|
|
1708
|
+
plan: DispatchPlan,
|
|
1709
|
+
job: WorkerJob,
|
|
1710
|
+
attempt: int,
|
|
1711
|
+
outcome: WorkerOutcome,
|
|
1712
|
+
*,
|
|
1713
|
+
snapshot: MutationSnapshot | None = None,
|
|
1714
|
+
policies: Sequence[WritePolicy] = (),
|
|
1715
|
+
) -> bool:
|
|
1716
|
+
mutation = _audit_attempt(job, outcome, snapshot, policies)
|
|
1717
|
+
if mutation is not None and mutation.status in {
|
|
1718
|
+
"contract-failed-unattributed",
|
|
1719
|
+
"mutation-present-unresolved",
|
|
1720
|
+
}:
|
|
1721
|
+
reason = "; ".join(mutation.violations) or mutation.status
|
|
1722
|
+
_transition_job_status(plan, job, "error", reason)
|
|
1723
|
+
_update_dispatch_status(
|
|
1724
|
+
plan.team_state_path, job, attempt, "error", reason
|
|
1725
|
+
)
|
|
1726
|
+
details = _failure_details(job, attempt, outcome, reason)
|
|
1727
|
+
details["mutationAudit"] = mutation.change_summary()
|
|
1728
|
+
_append_event(plan, "worker-failed", details)
|
|
1729
|
+
_finish_manifest_attempt(plan, job, attempt, mutation, None)
|
|
1730
|
+
return False
|
|
995
1731
|
settlement = _settle(plan, job, attempt, outcome)
|
|
996
1732
|
if settlement.completed:
|
|
997
1733
|
result_link = _link_result(plan, job, attempt)
|
|
998
1734
|
post_process = _post_process_report_writer_result(plan, job)
|
|
999
1735
|
if not post_process["ok"]:
|
|
1000
1736
|
reason = _require_string(post_process, "reason")
|
|
1001
|
-
|
|
1737
|
+
_transition_job_status(plan, job, "error", reason)
|
|
1002
1738
|
_update_dispatch_status(
|
|
1003
1739
|
plan.team_state_path,
|
|
1004
1740
|
job,
|
|
@@ -1015,10 +1751,9 @@ def _finish_attempt(plan: DispatchPlan, job: WorkerJob, attempt: int, outcome: W
|
|
|
1015
1751
|
)
|
|
1016
1752
|
details["postProcessing"] = post_process["steps"]
|
|
1017
1753
|
_append_event(plan, "worker-failed", details)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
)
|
|
1754
|
+
_finish_manifest_attempt(plan, job, attempt, mutation, None)
|
|
1755
|
+
return False
|
|
1756
|
+
_transition_job_status(plan, job, "completed", "")
|
|
1022
1757
|
_update_dispatch_status(
|
|
1023
1758
|
plan.team_state_path,
|
|
1024
1759
|
job,
|
|
@@ -1031,16 +1766,148 @@ def _finish_attempt(plan: DispatchPlan, job: WorkerJob, attempt: int, outcome: W
|
|
|
1031
1766
|
details["resultLink"] = result_link
|
|
1032
1767
|
if settlement.error_log_append is not None:
|
|
1033
1768
|
details["errorLogAppend"] = settlement.error_log_append
|
|
1769
|
+
if mutation is not None:
|
|
1770
|
+
details["mutationAudit"] = mutation.change_summary()
|
|
1034
1771
|
_append_event(plan, "worker-result-collected", details)
|
|
1035
|
-
|
|
1772
|
+
_finish_manifest_attempt(
|
|
1773
|
+
plan, job, attempt, mutation, job.worker_result_path
|
|
1774
|
+
)
|
|
1775
|
+
return False
|
|
1036
1776
|
reason = settlement.reason
|
|
1037
1777
|
status = "timeout" if outcome.timeout else "error"
|
|
1038
|
-
|
|
1778
|
+
_transition_job_status(plan, job, status, reason)
|
|
1039
1779
|
_update_dispatch_status(plan.team_state_path, job, attempt, status, reason)
|
|
1040
1780
|
details = _failure_details(job, attempt, outcome, reason)
|
|
1041
1781
|
if settlement.error_log_append is not None:
|
|
1042
1782
|
details["errorLogAppend"] = settlement.error_log_append
|
|
1783
|
+
if mutation is not None:
|
|
1784
|
+
details["mutationAudit"] = mutation.change_summary()
|
|
1043
1785
|
_append_event(plan, "worker-failed", details)
|
|
1786
|
+
_finish_manifest_attempt(plan, job, attempt, mutation, None)
|
|
1787
|
+
return mutation.retry_allowed if mutation is not None else True
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def _audit_attempt(
|
|
1791
|
+
job: WorkerJob,
|
|
1792
|
+
outcome: WorkerOutcome,
|
|
1793
|
+
snapshot: MutationSnapshot | None,
|
|
1794
|
+
policies: Sequence[WritePolicy],
|
|
1795
|
+
) -> MutationAuditResult | None:
|
|
1796
|
+
if not job.has_execution_identity or not policies:
|
|
1797
|
+
return None
|
|
1798
|
+
succeeded = outcome.returncode == 0 and not outcome.timeout
|
|
1799
|
+
result_present = not outcome.missing_completion_paths
|
|
1800
|
+
if snapshot is None:
|
|
1801
|
+
status = "ok" if succeeded and result_present else "failed-no-mutation"
|
|
1802
|
+
return MutationAuditResult(
|
|
1803
|
+
status=status,
|
|
1804
|
+
changed_paths=(),
|
|
1805
|
+
source_changed=False,
|
|
1806
|
+
git_changed=False,
|
|
1807
|
+
attribution="call",
|
|
1808
|
+
retry_allowed=status == "failed-no-mutation",
|
|
1809
|
+
violations=(),
|
|
1810
|
+
before_digest="",
|
|
1811
|
+
after_digest="",
|
|
1812
|
+
git_projection={},
|
|
1813
|
+
)
|
|
1814
|
+
return ExecutionMutationAudit().compare(
|
|
1815
|
+
snapshot,
|
|
1816
|
+
policies,
|
|
1817
|
+
out_of_plan_edits=_out_of_plan_edit_paths(job.result_path),
|
|
1818
|
+
attempt_succeeded=succeeded,
|
|
1819
|
+
result_present=result_present,
|
|
1820
|
+
)
|
|
1821
|
+
|
|
1822
|
+
|
|
1823
|
+
def _out_of_plan_edit_paths(result_path: Path) -> tuple[str, ...]:
|
|
1824
|
+
if not result_path.is_file() or result_path.suffix != ".json":
|
|
1825
|
+
return ()
|
|
1826
|
+
try:
|
|
1827
|
+
payload = json.loads(result_path.read_text(encoding="utf-8"))
|
|
1828
|
+
except (OSError, UnicodeError, json.JSONDecodeError):
|
|
1829
|
+
return ()
|
|
1830
|
+
implementation = payload.get("implementation") if isinstance(payload, Mapping) else None
|
|
1831
|
+
rows = implementation.get("outOfPlanEdits") if isinstance(implementation, Mapping) else None
|
|
1832
|
+
if not isinstance(rows, list):
|
|
1833
|
+
return ()
|
|
1834
|
+
return tuple(
|
|
1835
|
+
str(row["file"])
|
|
1836
|
+
for row in rows
|
|
1837
|
+
if isinstance(row, Mapping)
|
|
1838
|
+
and isinstance(row.get("file"), str)
|
|
1839
|
+
and row["file"]
|
|
1840
|
+
)
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
def _finish_manifest_attempt(
|
|
1844
|
+
plan: DispatchPlan,
|
|
1845
|
+
job: WorkerJob,
|
|
1846
|
+
attempt: int,
|
|
1847
|
+
mutation: MutationAuditResult | None,
|
|
1848
|
+
result_path: Path | None,
|
|
1849
|
+
) -> None:
|
|
1850
|
+
if mutation is None:
|
|
1851
|
+
return
|
|
1852
|
+
manifest = read_execution_manifest(plan.manifest_path)
|
|
1853
|
+
existing = next(
|
|
1854
|
+
(
|
|
1855
|
+
row for row in manifest.attempts
|
|
1856
|
+
if row.invocation_ref == job.invocation_ref
|
|
1857
|
+
and row.attempt == attempt
|
|
1858
|
+
),
|
|
1859
|
+
None,
|
|
1860
|
+
)
|
|
1861
|
+
expected_result = str(result_path) if result_path is not None else None
|
|
1862
|
+
if existing is not None and existing.finished_at is not None:
|
|
1863
|
+
if (
|
|
1864
|
+
existing.status == mutation.status
|
|
1865
|
+
and existing.result_path == expected_result
|
|
1866
|
+
and dict(existing.change_summary) == mutation.change_summary()
|
|
1867
|
+
):
|
|
1868
|
+
return
|
|
1869
|
+
raise DispatchError("attempt terminal mutation result drift")
|
|
1870
|
+
finish_attempt_mutation(
|
|
1871
|
+
plan.manifest_path,
|
|
1872
|
+
invocation_ref=job.invocation_ref,
|
|
1873
|
+
attempt=attempt,
|
|
1874
|
+
finished_at=_utc_now(),
|
|
1875
|
+
status=mutation.status,
|
|
1876
|
+
result_path=expected_result,
|
|
1877
|
+
error_path=None,
|
|
1878
|
+
change_summary=mutation.change_summary(),
|
|
1879
|
+
task_key=_require_string(plan.manifest, "taskKey"),
|
|
1880
|
+
)
|
|
1881
|
+
_seal_finished_attempt(plan, job, attempt, mutation)
|
|
1882
|
+
|
|
1883
|
+
|
|
1884
|
+
def _seal_finished_attempt(plan, job, attempt, mutation) -> None:
|
|
1885
|
+
from .attempt_evidence import TerminalObservation, finalize_attempt
|
|
1886
|
+
|
|
1887
|
+
events = job.prompt_path.with_name(job.prompt_path.name + ".host-events.jsonl")
|
|
1888
|
+
if not events.is_file() or not job.has_execution_identity:
|
|
1889
|
+
return
|
|
1890
|
+
artifacts = tuple(
|
|
1891
|
+
path for path in (job.result_path, job.worker_result_path) if path
|
|
1892
|
+
)
|
|
1893
|
+
try:
|
|
1894
|
+
finalize_attempt(
|
|
1895
|
+
plan.manifest_path,
|
|
1896
|
+
_build_dispatch_id(job.invocation_ref, attempt),
|
|
1897
|
+
TerminalObservation(
|
|
1898
|
+
status=mutation.status,
|
|
1899
|
+
source_diff_digest=mutation.change_summary().get("beforeDigest", ""),
|
|
1900
|
+
git_projection_digest=mutation.change_summary().get("afterDigest", ""),
|
|
1901
|
+
changed_paths=tuple(mutation.changed_paths),
|
|
1902
|
+
artifact_paths=artifacts,
|
|
1903
|
+
host_event_stream_path=events,
|
|
1904
|
+
project_root=plan.project_root,
|
|
1905
|
+
worktree=Path(job.worktree_path) if job.worktree_path else None,
|
|
1906
|
+
),
|
|
1907
|
+
task_key=_require_string(plan.manifest, "taskKey"),
|
|
1908
|
+
)
|
|
1909
|
+
except (OSError, ValueError):
|
|
1910
|
+
return
|
|
1044
1911
|
|
|
1045
1912
|
|
|
1046
1913
|
def _link_result(
|
|
@@ -1080,9 +1947,34 @@ def _link_result(
|
|
|
1080
1947
|
return link
|
|
1081
1948
|
|
|
1082
1949
|
|
|
1083
|
-
def _finish_record(
|
|
1950
|
+
def _finish_record(
|
|
1951
|
+
plan: DispatchPlan,
|
|
1952
|
+
record: Mapping[str, Any],
|
|
1953
|
+
outcome: WorkerOutcome,
|
|
1954
|
+
) -> bool:
|
|
1084
1955
|
job = _job_from_record(plan.project_root, record)
|
|
1085
|
-
|
|
1956
|
+
snapshot = _snapshot_from_record(record)
|
|
1957
|
+
contract = _persisted_write_contract(plan, job)
|
|
1958
|
+
policies = (contract[0],) if contract is not None else ()
|
|
1959
|
+
return _finish_attempt(
|
|
1960
|
+
plan,
|
|
1961
|
+
job,
|
|
1962
|
+
int(record.get("attempt", 1)),
|
|
1963
|
+
outcome,
|
|
1964
|
+
snapshot=snapshot,
|
|
1965
|
+
policies=policies,
|
|
1966
|
+
)
|
|
1967
|
+
|
|
1968
|
+
|
|
1969
|
+
def _snapshot_from_record(record: Mapping[str, Any]) -> MutationSnapshot | None:
|
|
1970
|
+
value = _string_value(record.get("mutationAuditSnapshotPath"))
|
|
1971
|
+
if not value:
|
|
1972
|
+
return None
|
|
1973
|
+
payload = _load_json_object(Path(value), "mutation audit snapshot")
|
|
1974
|
+
try:
|
|
1975
|
+
return MutationSnapshot.from_payload(payload)
|
|
1976
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
1977
|
+
raise DispatchError("mutation audit snapshot is invalid") from exc
|
|
1086
1978
|
|
|
1087
1979
|
|
|
1088
1980
|
@dataclass(frozen=True)
|
|
@@ -1120,7 +2012,11 @@ def _settle(
|
|
|
1120
2012
|
reason = _failure_reason(outcome)
|
|
1121
2013
|
completed = False
|
|
1122
2014
|
note = ""
|
|
1123
|
-
if
|
|
2015
|
+
if (
|
|
2016
|
+
outcome.returncode != SERVED_MODEL_MISMATCH_EXIT_CODE
|
|
2017
|
+
and not outcome.timeout
|
|
2018
|
+
and not outcome.missing_completion_paths
|
|
2019
|
+
):
|
|
1124
2020
|
audit_failures = _audit_sidecar_failures(job)
|
|
1125
2021
|
if audit_failures:
|
|
1126
2022
|
reason = (
|
|
@@ -1214,6 +2110,11 @@ def _record_wrapper_failure(
|
|
|
1214
2110
|
message=f"attempt {attempt}: {message}",
|
|
1215
2111
|
stderr_excerpt=_wrapper_log_tail(job),
|
|
1216
2112
|
context=None,
|
|
2113
|
+
identity=(
|
|
2114
|
+
_job_stored_identity(job, attempt)
|
|
2115
|
+
if job.has_execution_identity
|
|
2116
|
+
else None
|
|
2117
|
+
),
|
|
1217
2118
|
)
|
|
1218
2119
|
except (OSError, ValueError, TypeError, WorkerPromptHeaderError) as exc:
|
|
1219
2120
|
result["reason"] = f"{type(exc).__name__}: {exc}"
|
|
@@ -1222,6 +2123,17 @@ def _record_wrapper_failure(
|
|
|
1222
2123
|
return result
|
|
1223
2124
|
|
|
1224
2125
|
|
|
2126
|
+
def _job_stored_identity(job: WorkerJob, attempt: int) -> dict[str, Any]:
|
|
2127
|
+
"""Copy the job's stored execution refs. Do not infer them from labels."""
|
|
2128
|
+
return {
|
|
2129
|
+
"participantRef": job.participant_ref,
|
|
2130
|
+
"roleExecutionRef": job.role_execution_ref,
|
|
2131
|
+
"invocationRef": job.invocation_ref,
|
|
2132
|
+
"attempt": job.attempt or attempt,
|
|
2133
|
+
"executionLabel": job.execution_label,
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
|
|
1225
2137
|
def _error_log_agent(worker_id: str) -> str:
|
|
1226
2138
|
"""The error log's `--agent` enum value for a worker id.
|
|
1227
2139
|
|
|
@@ -1300,9 +2212,87 @@ def _post_process_report_writer_result(
|
|
|
1300
2212
|
return {"ok": False, "reason": str(exc), "steps": []}
|
|
1301
2213
|
|
|
1302
2214
|
|
|
1303
|
-
def
|
|
1304
|
-
|
|
1305
|
-
|
|
2215
|
+
def _projectionless_translator(plan: DispatchPlan, job: WorkerJob) -> bool:
|
|
2216
|
+
if not (
|
|
2217
|
+
job.has_execution_identity
|
|
2218
|
+
and job.role == "translator"
|
|
2219
|
+
and job.worker_id == "translator"
|
|
2220
|
+
and job.audience == "translator"
|
|
2221
|
+
and job.assignment_ref == "translator"
|
|
2222
|
+
and job.duty_id == "translator"
|
|
2223
|
+
and job.dispatch_kind == "translator"
|
|
2224
|
+
):
|
|
2225
|
+
return False
|
|
2226
|
+
execution = _canonical_translator_execution(plan.manifest)
|
|
2227
|
+
return bool(
|
|
2228
|
+
execution
|
|
2229
|
+
and execution.get("roleExecutionRef") == job.role_execution_ref
|
|
2230
|
+
and execution.get("participantRef") == job.participant_ref
|
|
2231
|
+
and execution.get("executionLabel") == job.execution_label
|
|
2232
|
+
and execution.get("provider") == job.provider
|
|
2233
|
+
)
|
|
2234
|
+
|
|
2235
|
+
|
|
2236
|
+
def _transition_job_status(
|
|
2237
|
+
plan: DispatchPlan,
|
|
2238
|
+
job: WorkerJob,
|
|
2239
|
+
status: str,
|
|
2240
|
+
reason: str,
|
|
2241
|
+
*,
|
|
2242
|
+
model_execution_value: str = "",
|
|
2243
|
+
) -> None:
|
|
2244
|
+
if not _projectionless_translator(plan, job):
|
|
2245
|
+
_transition_worker_status(
|
|
2246
|
+
plan.team_state_path,
|
|
2247
|
+
job.worker_id,
|
|
2248
|
+
status,
|
|
2249
|
+
reason,
|
|
2250
|
+
model_execution_value=model_execution_value,
|
|
2251
|
+
)
|
|
2252
|
+
return
|
|
2253
|
+
team_state = _load_json_object(plan.team_state_path, "team-state")
|
|
2254
|
+
workers = team_state.get("workers")
|
|
2255
|
+
if not isinstance(workers, list):
|
|
2256
|
+
raise DispatchError("team-state workers must be an array")
|
|
2257
|
+
if not any(
|
|
2258
|
+
isinstance(row, Mapping) and row.get("workerId") == job.worker_id
|
|
2259
|
+
for row in workers
|
|
2260
|
+
):
|
|
2261
|
+
return
|
|
2262
|
+
_transition_worker_status(
|
|
2263
|
+
plan.team_state_path,
|
|
2264
|
+
job.worker_id,
|
|
2265
|
+
status,
|
|
2266
|
+
reason,
|
|
2267
|
+
model_execution_value=model_execution_value,
|
|
2268
|
+
)
|
|
2269
|
+
|
|
2270
|
+
|
|
2271
|
+
def _job_terminal_status(
|
|
2272
|
+
plan: DispatchPlan, job: WorkerJob, attempt: int,
|
|
2273
|
+
) -> str:
|
|
2274
|
+
team_state = _load_json_object(plan.team_state_path, "team-state")
|
|
2275
|
+
workers = team_state.get("workers")
|
|
2276
|
+
if not isinstance(workers, list):
|
|
2277
|
+
raise DispatchError("team-state workers must be an array")
|
|
2278
|
+
matches = [
|
|
2279
|
+
row for row in workers
|
|
2280
|
+
if isinstance(row, Mapping) and row.get("workerId") == job.worker_id
|
|
2281
|
+
]
|
|
2282
|
+
if len(matches) == 1:
|
|
2283
|
+
return _string_value(matches[0].get("status"))
|
|
2284
|
+
if matches or not _projectionless_translator(plan, job):
|
|
2285
|
+
raise DispatchError(f"team-state has no unique workerId={job.worker_id}")
|
|
2286
|
+
dispatches = team_state.get("workerDispatches")
|
|
2287
|
+
records = [
|
|
2288
|
+
row for row in dispatches
|
|
2289
|
+
if isinstance(row, Mapping)
|
|
2290
|
+
and row.get("promptPath") == str(job.prompt_path)
|
|
2291
|
+
and row.get("attempt") == attempt
|
|
2292
|
+
] if isinstance(dispatches, list) else []
|
|
2293
|
+
if len(records) != 1:
|
|
2294
|
+
raise DispatchError("translator dispatch has no unique status record")
|
|
2295
|
+
return _string_value(records[0].get("status"))
|
|
1306
2296
|
|
|
1307
2297
|
|
|
1308
2298
|
def _record_dispatch(
|
|
@@ -1317,15 +2307,31 @@ def _record_dispatch(
|
|
|
1317
2307
|
handle.pane_id,
|
|
1318
2308
|
handle.degraded_from,
|
|
1319
2309
|
reason,
|
|
2310
|
+
mutation_snapshot_path=(
|
|
2311
|
+
_mutation_snapshot_path(handle.job)
|
|
2312
|
+
if handle.mutation_snapshot is not None
|
|
2313
|
+
else None
|
|
2314
|
+
),
|
|
2315
|
+
write_policy_digest=handle.write_policy_digest,
|
|
2316
|
+
write_enforcement=handle.write_enforcement,
|
|
1320
2317
|
),
|
|
1321
2318
|
)
|
|
1322
2319
|
|
|
1323
2320
|
|
|
1324
2321
|
def _dispatch_record(
|
|
1325
|
-
job: WorkerJob,
|
|
2322
|
+
job: WorkerJob,
|
|
2323
|
+
attempt: int,
|
|
2324
|
+
status: str,
|
|
2325
|
+
pane_id: str,
|
|
2326
|
+
degraded_from: str,
|
|
2327
|
+
reason: str = "",
|
|
2328
|
+
*,
|
|
2329
|
+
mutation_snapshot_path: Path | None = None,
|
|
2330
|
+
write_policy_digest: str = "",
|
|
2331
|
+
write_enforcement: WriteEnforcement | None = None,
|
|
1326
2332
|
) -> dict[str, Any]:
|
|
2333
|
+
has_execution_identity = job.has_execution_identity
|
|
1327
2334
|
record = {
|
|
1328
|
-
"workerId": job.worker_id,
|
|
1329
2335
|
"role": job.role,
|
|
1330
2336
|
"kind": job.dispatch_kind,
|
|
1331
2337
|
"attempt": attempt,
|
|
@@ -1363,6 +2369,27 @@ def _dispatch_record(
|
|
|
1363
2369
|
"hostModelValue": job.host_model_value,
|
|
1364
2370
|
"enforcementMode": job.enforcement_mode,
|
|
1365
2371
|
})
|
|
2372
|
+
if has_execution_identity:
|
|
2373
|
+
record.update({
|
|
2374
|
+
"schemaVersion": "2.0",
|
|
2375
|
+
"executionIdentityVersion": 2,
|
|
2376
|
+
"participantRef": job.participant_ref,
|
|
2377
|
+
"roleExecutionRef": job.role_execution_ref,
|
|
2378
|
+
"executionLabel": job.execution_label,
|
|
2379
|
+
"dutyId": job.duty_id,
|
|
2380
|
+
"invocationRef": job.invocation_ref,
|
|
2381
|
+
"writePolicyDigest": write_policy_digest,
|
|
2382
|
+
"writeEnforcement": (
|
|
2383
|
+
write_enforcement.to_payload()
|
|
2384
|
+
if write_enforcement is not None
|
|
2385
|
+
else None
|
|
2386
|
+
),
|
|
2387
|
+
"mutationAuditSnapshotPath": (
|
|
2388
|
+
str(mutation_snapshot_path) if mutation_snapshot_path else ""
|
|
2389
|
+
),
|
|
2390
|
+
})
|
|
2391
|
+
else:
|
|
2392
|
+
record["workerId"] = job.worker_id
|
|
1366
2393
|
return record
|
|
1367
2394
|
|
|
1368
2395
|
|
|
@@ -1487,31 +2514,33 @@ _SIDEBAR_LEVELS = {
|
|
|
1487
2514
|
|
|
1488
2515
|
|
|
1489
2516
|
def _relay_to_sidebar(plan: DispatchPlan, event_type: str, details: Mapping[str, Any]) -> None:
|
|
1490
|
-
"""Mirror a dispatch event onto the
|
|
2517
|
+
"""Mirror a dispatch event onto the lead surface.
|
|
1491
2518
|
|
|
1492
|
-
A long run is mostly silence, and the
|
|
2519
|
+
A long run is mostly silence, and the lead surface is the one still
|
|
1493
2520
|
visible after the user scrolls away or switches workspaces. A failed worker
|
|
1494
2521
|
additionally raises a notification, because that is the event whose cost
|
|
1495
|
-
grows the longer it goes unnoticed.
|
|
2522
|
+
grows the longer it goes unnoticed. The lead port is `runtime_chain[0]`,
|
|
2523
|
+
so a worker that degraded to the wrapper still lands on a cmux run's
|
|
2524
|
+
sidebar. A cli-wrapper lead is a no-op.
|
|
1496
2525
|
"""
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
return
|
|
1502
|
-
workspace = cmux.resolve_lead_workspace()
|
|
1503
|
-
worker_id = str(details.get("workerId", "") or "worker")
|
|
1504
|
-
cmux.sidebar_log(
|
|
1505
|
-
workspace,
|
|
1506
|
-
f"{worker_id}: {event_type.removeprefix('worker-')}",
|
|
1507
|
-
level=_SIDEBAR_LEVELS.get(event_type, "info"),
|
|
2526
|
+
plan = _ensure_runtime_chain(plan)
|
|
2527
|
+
lead = plan.runtime_chain[0]
|
|
2528
|
+
worker_id = str(
|
|
2529
|
+
details.get("executionLabel") or details.get("workerId") or "worker"
|
|
1508
2530
|
)
|
|
2531
|
+
notify_title = None
|
|
2532
|
+
notify_body = None
|
|
1509
2533
|
if event_type == "worker-failed":
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
2534
|
+
notify_title = f"okstra — {_require_string(plan.manifest, 'taskType')}"
|
|
2535
|
+
notify_body = f"{worker_id} failed: {details.get('reason', 'no reason recorded')}"
|
|
2536
|
+
lead.notify(
|
|
2537
|
+
ProgressEvent(
|
|
2538
|
+
message=f"{worker_id}: {event_type.removeprefix('worker-')}",
|
|
2539
|
+
level=_SIDEBAR_LEVELS.get(event_type, "info"),
|
|
2540
|
+
notify_title=notify_title,
|
|
2541
|
+
notify_body=notify_body,
|
|
1514
2542
|
)
|
|
2543
|
+
)
|
|
1515
2544
|
|
|
1516
2545
|
|
|
1517
2546
|
def _append_event(plan: DispatchPlan, event_type: str, details: Mapping[str, Any]) -> None:
|
|
@@ -1531,13 +2560,16 @@ def _append_event(plan: DispatchPlan, event_type: str, details: Mapping[str, Any
|
|
|
1531
2560
|
|
|
1532
2561
|
|
|
1533
2562
|
def _attempt_details(job: WorkerJob, attempt: int, handle: WorkerHandle) -> dict[str, Any]:
|
|
1534
|
-
return {
|
|
2563
|
+
return {
|
|
2564
|
+
**_event_details(job, handle),
|
|
2565
|
+
**_event_execution_identity(job, attempt),
|
|
2566
|
+
"maxAttempts": MAX_WORKER_ATTEMPTS,
|
|
2567
|
+
}
|
|
1535
2568
|
|
|
1536
2569
|
|
|
1537
2570
|
def _event_details(job: WorkerJob, handle: WorkerHandle) -> dict[str, Any]:
|
|
1538
|
-
|
|
2571
|
+
details = {
|
|
1539
2572
|
"dispatchMode": BACKEND_CLI_WRAPPER if handle.degraded_from else job.backend,
|
|
1540
|
-
"workerId": job.worker_id,
|
|
1541
2573
|
"promptPath": str(job.prompt_path),
|
|
1542
2574
|
"resultPath": str(job.result_path),
|
|
1543
2575
|
"workerResultPath": str(job.worker_result_path),
|
|
@@ -1546,12 +2578,25 @@ def _event_details(job: WorkerJob, handle: WorkerHandle) -> dict[str, Any]:
|
|
|
1546
2578
|
"paneId": handle.pane_id,
|
|
1547
2579
|
"degradedFrom": handle.degraded_from,
|
|
1548
2580
|
}
|
|
2581
|
+
return details
|
|
2582
|
+
|
|
2583
|
+
|
|
2584
|
+
def _event_execution_identity(job: WorkerJob, attempt: int) -> dict[str, Any]:
|
|
2585
|
+
if job.has_execution_identity:
|
|
2586
|
+
return {
|
|
2587
|
+
"participantRef": job.participant_ref,
|
|
2588
|
+
"roleExecutionRef": job.role_execution_ref,
|
|
2589
|
+
"executionLabel": job.execution_label,
|
|
2590
|
+
"dutyId": job.duty_id,
|
|
2591
|
+
"invocationRef": job.invocation_ref,
|
|
2592
|
+
"attempt": attempt,
|
|
2593
|
+
}
|
|
2594
|
+
return {"workerId": job.worker_id, "attempt": attempt}
|
|
1549
2595
|
|
|
1550
2596
|
|
|
1551
2597
|
def _result_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> dict[str, Any]:
|
|
1552
2598
|
return {
|
|
1553
|
-
|
|
1554
|
-
"attempt": attempt,
|
|
2599
|
+
**_event_execution_identity(job, attempt),
|
|
1555
2600
|
"dispatchMode": BACKEND_CLI_WRAPPER if outcome.degraded_from else job.backend,
|
|
1556
2601
|
"missingCompletionPaths": [],
|
|
1557
2602
|
# A collected result can still come from a wrapper that exited non-zero
|
|
@@ -1570,8 +2615,7 @@ def _failure_details(
|
|
|
1570
2615
|
exit_code: int | None = None,
|
|
1571
2616
|
) -> dict[str, Any]:
|
|
1572
2617
|
return {
|
|
1573
|
-
|
|
1574
|
-
"attempt": attempt,
|
|
2618
|
+
**_event_execution_identity(job, attempt),
|
|
1575
2619
|
"maxAttempts": MAX_WORKER_ATTEMPTS,
|
|
1576
2620
|
"dispatchMode": BACKEND_CLI_WRAPPER if outcome.degraded_from else job.backend,
|
|
1577
2621
|
"exitCode": outcome.returncode if exit_code is None else exit_code,
|
|
@@ -1587,8 +2631,7 @@ def _failure_details(
|
|
|
1587
2631
|
|
|
1588
2632
|
def _retry_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> dict[str, Any]:
|
|
1589
2633
|
return {
|
|
1590
|
-
|
|
1591
|
-
"attempt": attempt,
|
|
2634
|
+
**_event_execution_identity(job, attempt),
|
|
1592
2635
|
"missingCompletionPaths": [str(path) for path in outcome.missing_completion_paths],
|
|
1593
2636
|
"reason": "required worker artifact was not produced",
|
|
1594
2637
|
}
|
|
@@ -1607,8 +2650,13 @@ def _outcome_from_status(record: Mapping[str, Any], status) -> WorkerOutcome:
|
|
|
1607
2650
|
|
|
1608
2651
|
|
|
1609
2652
|
def _job_from_record(project_root: Path, record: Mapping[str, Any]) -> WorkerJob:
|
|
2653
|
+
identity = _worker_execution_identity(record)
|
|
1610
2654
|
return WorkerJob(
|
|
1611
|
-
worker_id=
|
|
2655
|
+
worker_id=(
|
|
2656
|
+
_require_string(record, "workerId")
|
|
2657
|
+
if identity is None
|
|
2658
|
+
else _v2_worker_state_key(record)
|
|
2659
|
+
),
|
|
1612
2660
|
provider=_require_string(record, "provider"),
|
|
1613
2661
|
backend=BACKEND_CLI_WRAPPER,
|
|
1614
2662
|
project_root=project_root,
|
|
@@ -1643,6 +2691,7 @@ def _job_from_record(project_root: Path, record: Mapping[str, Any]) -> WorkerJob
|
|
|
1643
2691
|
else None
|
|
1644
2692
|
),
|
|
1645
2693
|
enforcement_mode=_string_value(record.get("enforcementMode")),
|
|
2694
|
+
**(identity or {}),
|
|
1646
2695
|
)
|
|
1647
2696
|
|
|
1648
2697
|
|