oh-my-codex 0.20.1 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +7 -6
- package/Cargo.toml +1 -1
- package/README.md +36 -1
- package/crates/omx-runtime/Cargo.toml +1 -0
- package/crates/omx-runtime/src/main.rs +25 -2
- package/crates/omx-runtime/tests/execution.rs +219 -0
- package/crates/omx-runtime-core/src/dispatch.rs +96 -10
- package/crates/omx-runtime-core/src/engine.rs +839 -29
- package/crates/omx-runtime-core/src/lib.rs +8 -0
- package/dist/adapt/__tests__/hermes.test.js.map +1 -1
- package/dist/adapt/contracts.d.ts.map +1 -1
- package/dist/agents/__tests__/definitions.test.js +81 -0
- package/dist/agents/__tests__/definitions.test.js.map +1 -1
- package/dist/agents/__tests__/native-config.test.js +25 -0
- package/dist/agents/__tests__/native-config.test.js.map +1 -1
- package/dist/agents/native-config.d.ts +2 -1
- package/dist/agents/native-config.d.ts.map +1 -1
- package/dist/agents/native-config.js.map +1 -1
- package/dist/auth/__tests__/hotswap.test.d.ts +2 -0
- package/dist/auth/__tests__/hotswap.test.d.ts.map +1 -0
- package/dist/auth/__tests__/hotswap.test.js +183 -0
- package/dist/auth/__tests__/hotswap.test.js.map +1 -0
- package/dist/auth/hotswap.d.ts +6 -0
- package/dist/auth/hotswap.d.ts.map +1 -1
- package/dist/auth/hotswap.js +102 -19
- package/dist/auth/hotswap.js.map +1 -1
- package/dist/auth/paths.js.map +1 -1
- package/dist/auth/storage.js.map +1 -1
- package/dist/autopilot/__tests__/ralplan-gate.test.js +5 -5
- package/dist/autopilot/__tests__/ralplan-gate.test.js.map +1 -1
- package/dist/autopilot/completion-gate.js.map +1 -1
- package/dist/autopilot/fsm.d.ts +1 -1
- package/dist/autopilot/fsm.d.ts.map +1 -1
- package/dist/autoresearch/goal.js.map +1 -1
- package/dist/autoresearch/runtime.js.map +1 -1
- package/dist/capabilities/lockfile.js.map +1 -1
- package/dist/catalog/reader.js.map +1 -1
- package/dist/catalog/skill-mirror.js.map +1 -1
- package/dist/cli/__tests__/adapt-help.test.js.map +1 -1
- package/dist/cli/__tests__/agents-init.test.js.map +1 -1
- package/dist/cli/__tests__/agents.test.js.map +1 -1
- package/dist/cli/__tests__/api.test.js.map +1 -1
- package/dist/cli/__tests__/ask.test.js.map +1 -1
- package/dist/cli/__tests__/auth.test.js +60 -6
- package/dist/cli/__tests__/auth.test.js.map +1 -1
- package/dist/cli/__tests__/autoresearch-guided.test.js.map +1 -1
- package/dist/cli/__tests__/autoresearch.test.js.map +1 -1
- package/dist/cli/__tests__/capabilities.test.js.map +1 -1
- package/dist/cli/__tests__/codex-plugin-layout.test.js +80 -0
- package/dist/cli/__tests__/codex-plugin-layout.test.js.map +1 -1
- package/dist/cli/__tests__/doctor-context-window-warning.test.js.map +1 -1
- package/dist/cli/__tests__/doctor-invalid-config.test.js +47 -4
- package/dist/cli/__tests__/doctor-invalid-config.test.js.map +1 -1
- package/dist/cli/__tests__/doctor-team.test.js.map +1 -1
- package/dist/cli/__tests__/doctor-warning-copy.test.js +752 -15
- package/dist/cli/__tests__/doctor-warning-copy.test.js.map +1 -1
- package/dist/cli/__tests__/exec.test.js +136 -0
- package/dist/cli/__tests__/exec.test.js.map +1 -1
- package/dist/cli/__tests__/imagegen-continuation.test.js.map +1 -1
- package/dist/cli/__tests__/index.test.js +473 -14
- package/dist/cli/__tests__/index.test.js.map +1 -1
- package/dist/cli/__tests__/launch-fallback.test.js +336 -4
- package/dist/cli/__tests__/launch-fallback.test.js.map +1 -1
- package/dist/cli/__tests__/mcp-parity.test.js +87 -12
- package/dist/cli/__tests__/mcp-parity.test.js.map +1 -1
- package/dist/cli/__tests__/native-hook-claim-journal.test.d.ts +2 -0
- package/dist/cli/__tests__/native-hook-claim-journal.test.d.ts.map +1 -0
- package/dist/cli/__tests__/native-hook-claim-journal.test.js +206 -0
- package/dist/cli/__tests__/native-hook-claim-journal.test.js.map +1 -0
- package/dist/cli/__tests__/nested-help-routing.test.js.map +1 -1
- package/dist/cli/__tests__/question.test.js.map +1 -1
- package/dist/cli/__tests__/ralph-deslop-contract.test.js.map +1 -1
- package/dist/cli/__tests__/ralph-goal-mode-contract.test.js +6 -0
- package/dist/cli/__tests__/ralph-goal-mode-contract.test.js.map +1 -1
- package/dist/cli/__tests__/ralph-prd-deep-interview.test.js.map +1 -1
- package/dist/cli/__tests__/ralph-prd-smoke.test.js.map +1 -1
- package/dist/cli/__tests__/ralph.test.js +8 -3
- package/dist/cli/__tests__/ralph.test.js.map +1 -1
- package/dist/cli/__tests__/ralplan-bootstrap-3181.test.d.ts +2 -0
- package/dist/cli/__tests__/ralplan-bootstrap-3181.test.d.ts.map +1 -0
- package/dist/cli/__tests__/ralplan-bootstrap-3181.test.js +72 -0
- package/dist/cli/__tests__/ralplan-bootstrap-3181.test.js.map +1 -0
- package/dist/cli/__tests__/ralplan.test.d.ts +2 -0
- package/dist/cli/__tests__/ralplan.test.d.ts.map +1 -0
- package/dist/cli/__tests__/ralplan.test.js +55 -0
- package/dist/cli/__tests__/ralplan.test.js.map +1 -0
- package/dist/cli/__tests__/resume.test.js +17 -11
- package/dist/cli/__tests__/resume.test.js.map +1 -1
- package/dist/cli/__tests__/session-scoped-runtime.test.js +25 -0
- package/dist/cli/__tests__/session-scoped-runtime.test.js.map +1 -1
- package/dist/cli/__tests__/session-search-help.test.js +25 -1
- package/dist/cli/__tests__/session-search-help.test.js.map +1 -1
- package/dist/cli/__tests__/session-search.test.js.map +1 -1
- package/dist/cli/__tests__/setup-agents-overwrite.test.js +79 -0
- package/dist/cli/__tests__/setup-agents-overwrite.test.js.map +1 -1
- package/dist/cli/__tests__/setup-gh-star.test.js.map +1 -1
- package/dist/cli/__tests__/setup-hooks-shared-ownership.test.js +77 -3
- package/dist/cli/__tests__/setup-hooks-shared-ownership.test.js.map +1 -1
- package/dist/cli/__tests__/setup-hooks-trust-e2e.test.d.ts +2 -0
- package/dist/cli/__tests__/setup-hooks-trust-e2e.test.d.ts.map +1 -0
- package/dist/cli/__tests__/setup-hooks-trust-e2e.test.js +285 -0
- package/dist/cli/__tests__/setup-hooks-trust-e2e.test.js.map +1 -0
- package/dist/cli/__tests__/setup-install-mode.test.js +2579 -294
- package/dist/cli/__tests__/setup-install-mode.test.js.map +1 -1
- package/dist/cli/__tests__/setup-preferences.test.d.ts +2 -0
- package/dist/cli/__tests__/setup-preferences.test.d.ts.map +1 -0
- package/dist/cli/__tests__/setup-preferences.test.js +79 -0
- package/dist/cli/__tests__/setup-preferences.test.js.map +1 -0
- package/dist/cli/__tests__/setup-refresh.test.js +33 -2
- package/dist/cli/__tests__/setup-refresh.test.js.map +1 -1
- package/dist/cli/__tests__/setup-scope.test.js +46 -3
- package/dist/cli/__tests__/setup-scope.test.js.map +1 -1
- package/dist/cli/__tests__/sparkshell-cli.test.js.map +1 -1
- package/dist/cli/__tests__/state.test.js +37 -0
- package/dist/cli/__tests__/state.test.js.map +1 -1
- package/dist/cli/__tests__/team.test.js +105 -7
- package/dist/cli/__tests__/team.test.js.map +1 -1
- package/dist/cli/__tests__/ultragoal.test.js +61 -1
- package/dist/cli/__tests__/ultragoal.test.js.map +1 -1
- package/dist/cli/__tests__/uninstall.test.js +1817 -24
- package/dist/cli/__tests__/uninstall.test.js.map +1 -1
- package/dist/cli/__tests__/update.test.js +58 -0
- package/dist/cli/__tests__/update.test.js.map +1 -1
- package/dist/cli/__tests__/url.test.js.map +1 -1
- package/dist/cli/__tests__/windows-popup-loop-contract.test.js +2 -1
- package/dist/cli/__tests__/windows-popup-loop-contract.test.js.map +1 -1
- package/dist/cli/adapt.js.map +1 -1
- package/dist/cli/agents-init.js.map +1 -1
- package/dist/cli/agents.js.map +1 -1
- package/dist/cli/api.js.map +1 -1
- package/dist/cli/ask.d.ts +1 -1
- package/dist/cli/ask.d.ts.map +1 -1
- package/dist/cli/ask.js.map +1 -1
- package/dist/cli/auth.js.map +1 -1
- package/dist/cli/autoresearch-guided.js.map +1 -1
- package/dist/cli/autoresearch-intake.js.map +1 -1
- package/dist/cli/capabilities.js.map +1 -1
- package/dist/cli/cleanup.js.map +1 -1
- package/dist/cli/codex-feature-probe.js.map +1 -1
- package/dist/cli/codex-home.js.map +1 -1
- package/dist/cli/doctor.d.ts +21 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +607 -130
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/explore.js.map +1 -1
- package/dist/cli/index.d.ts +14 -3
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +245 -57
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/mcp-parity.js +2 -2
- package/dist/cli/mcp-parity.js.map +1 -1
- package/dist/cli/mcp-serve.js.map +1 -1
- package/dist/cli/mission.js.map +1 -1
- package/dist/cli/native-assets.js.map +1 -1
- package/dist/cli/native-hook-claim-journal.d.ts +29 -0
- package/dist/cli/native-hook-claim-journal.d.ts.map +1 -0
- package/dist/cli/native-hook-claim-journal.js +252 -0
- package/dist/cli/native-hook-claim-journal.js.map +1 -0
- package/dist/cli/omx.js.map +1 -1
- package/dist/cli/plugin-marketplace.js.map +1 -1
- package/dist/cli/project-runtime-codex-homes.js.map +1 -1
- package/dist/cli/question.js.map +1 -1
- package/dist/cli/ralph.d.ts.map +1 -1
- package/dist/cli/ralph.js +3 -2
- package/dist/cli/ralph.js.map +1 -1
- package/dist/cli/ralplan.d.ts +10 -0
- package/dist/cli/ralplan.d.ts.map +1 -0
- package/dist/cli/ralplan.js +110 -0
- package/dist/cli/ralplan.js.map +1 -0
- package/dist/cli/session-search.d.ts +5 -0
- package/dist/cli/session-search.d.ts.map +1 -1
- package/dist/cli/session-search.js +87 -0
- package/dist/cli/session-search.js.map +1 -1
- package/dist/cli/setup-preferences.d.ts +11 -0
- package/dist/cli/setup-preferences.d.ts.map +1 -1
- package/dist/cli/setup-preferences.js +32 -0
- package/dist/cli/setup-preferences.js.map +1 -1
- package/dist/cli/setup.d.ts +58 -0
- package/dist/cli/setup.d.ts.map +1 -1
- package/dist/cli/setup.js +1450 -398
- package/dist/cli/setup.js.map +1 -1
- package/dist/cli/sparkshell.js.map +1 -1
- package/dist/cli/star-prompt.js.map +1 -1
- package/dist/cli/state.d.ts.map +1 -1
- package/dist/cli/state.js +2 -1
- package/dist/cli/state.js.map +1 -1
- package/dist/cli/team.js.map +1 -1
- package/dist/cli/ultragoal.d.ts +5 -1
- package/dist/cli/ultragoal.d.ts.map +1 -1
- package/dist/cli/ultragoal.js +34 -2
- package/dist/cli/ultragoal.js.map +1 -1
- package/dist/cli/uninstall.d.ts +17 -0
- package/dist/cli/uninstall.d.ts.map +1 -1
- package/dist/cli/uninstall.js +986 -95
- package/dist/cli/uninstall.js.map +1 -1
- package/dist/cli/update.d.ts.map +1 -1
- package/dist/cli/update.js +8 -1
- package/dist/cli/update.js.map +1 -1
- package/dist/cli/url.js.map +1 -1
- package/dist/compat/__tests__/doctor-contract.test.js +5 -1
- package/dist/compat/__tests__/doctor-contract.test.js.map +1 -1
- package/dist/compat/__tests__/rust-runtime-compat.test.js +1 -1
- package/dist/compat/__tests__/rust-runtime-compat.test.js.map +1 -1
- package/dist/config/__tests__/codex-hooks.test.js +1287 -11
- package/dist/config/__tests__/codex-hooks.test.js.map +1 -1
- package/dist/config/__tests__/generator-idempotent.test.js +1179 -32
- package/dist/config/__tests__/generator-idempotent.test.js.map +1 -1
- package/dist/config/__tests__/generator-notify.test.js +6 -1
- package/dist/config/__tests__/generator-notify.test.js.map +1 -1
- package/dist/config/__tests__/models.test.js +67 -3
- package/dist/config/__tests__/models.test.js.map +1 -1
- package/dist/config/codex-feature-flags.d.ts.map +1 -1
- package/dist/config/codex-feature-flags.js.map +1 -1
- package/dist/config/codex-hooks.d.ts +124 -1
- package/dist/config/codex-hooks.d.ts.map +1 -1
- package/dist/config/codex-hooks.js +2142 -265
- package/dist/config/codex-hooks.js.map +1 -1
- package/dist/config/commit-lore-guard.js.map +1 -1
- package/dist/config/generator.d.ts +23 -34
- package/dist/config/generator.d.ts.map +1 -1
- package/dist/config/generator.js +1480 -246
- package/dist/config/generator.js.map +1 -1
- package/dist/config/mcp-registry.js.map +1 -1
- package/dist/config/models.d.ts +14 -4
- package/dist/config/models.d.ts.map +1 -1
- package/dist/config/models.js +15 -1
- package/dist/config/models.js.map +1 -1
- package/dist/config/omx-first-party-mcp.js.map +1 -1
- package/dist/config/team-mode.d.ts.map +1 -1
- package/dist/document-refresh/__tests__/enforcer.test.js.map +1 -1
- package/dist/document-refresh/enforcer.js.map +1 -1
- package/dist/exec/followup.js.map +1 -1
- package/dist/goal-workflows/artifacts.js.map +1 -1
- package/dist/hooks/__tests__/agents-overlay.test.js +7 -4
- package/dist/hooks/__tests__/agents-overlay.test.js.map +1 -1
- package/dist/hooks/__tests__/analyze-routing-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/analyze-skill-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/anti-slop-workflow.test.js.map +1 -1
- package/dist/hooks/__tests__/autopilot-skill-contract.test.js +9 -3
- package/dist/hooks/__tests__/autopilot-skill-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/code-review-skill-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/consensus-execution-handoff.test.js.map +1 -1
- package/dist/hooks/__tests__/deep-interview-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/design-skill.test.js.map +1 -1
- package/dist/hooks/__tests__/keyword-detector.test.js +1376 -6
- package/dist/hooks/__tests__/keyword-detector.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-fallback-watcher.test.js +252 -543
- package/dist/hooks/__tests__/notify-fallback-watcher.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-all-workers-idle.test.js +38 -4
- package/dist/hooks/__tests__/notify-hook-all-workers-idle.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-auto-nudge.test.js +765 -10
- package/dist/hooks/__tests__/notify-hook-auto-nudge.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-cross-worktree-heartbeat.test.js +2 -0
- package/dist/hooks/__tests__/notify-hook-cross-worktree-heartbeat.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-managed-tmux.test.js +142 -1
- package/dist/hooks/__tests__/notify-hook-managed-tmux.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-modules.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-non-omx-guard.test.js +48 -2
- package/dist/hooks/__tests__/notify-hook-non-omx-guard.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-ralph-resume.test.js +61 -44
- package/dist/hooks/__tests__/notify-hook-ralph-resume.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-regression-205.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-session-idle-dedupe.test.js +3 -2
- package/dist/hooks/__tests__/notify-hook-session-idle-dedupe.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-session-scope.test.js +243 -4
- package/dist/hooks/__tests__/notify-hook-session-scope.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-team-dispatch.test.js +531 -12
- package/dist/hooks/__tests__/notify-hook-team-dispatch.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-team-leader-nudge.test.js +163 -24
- package/dist/hooks/__tests__/notify-hook-team-leader-nudge.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-team-tmux-guard.test.js +482 -32
- package/dist/hooks/__tests__/notify-hook-team-tmux-guard.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-tmux-heal.test.js +68 -1
- package/dist/hooks/__tests__/notify-hook-tmux-heal.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-tmux-scrollback.test.js +7 -1
- package/dist/hooks/__tests__/notify-hook-tmux-scrollback.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-visual-verdict.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-worker-idle.test.js +43 -4
- package/dist/hooks/__tests__/notify-hook-worker-idle.test.js.map +1 -1
- package/dist/hooks/__tests__/openclaw-setup-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/pre-context-gate-skills.test.js.map +1 -1
- package/dist/hooks/__tests__/prometheus-strict-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/prompt-guidance-fragments.test.js.map +1 -1
- package/dist/hooks/__tests__/prompt-guidance-test-helpers.js.map +1 -1
- package/dist/hooks/__tests__/prompt-session-provenance.test.d.ts +2 -0
- package/dist/hooks/__tests__/prompt-session-provenance.test.d.ts.map +1 -0
- package/dist/hooks/__tests__/prompt-session-provenance.test.js +82 -0
- package/dist/hooks/__tests__/prompt-session-provenance.test.js.map +1 -0
- package/dist/hooks/__tests__/session.test.js +878 -11
- package/dist/hooks/__tests__/session.test.js.map +1 -1
- package/dist/hooks/__tests__/skill-catalog-hygiene.test.js.map +1 -1
- package/dist/hooks/__tests__/team-runtime-gating-docs-contract.test.js.map +1 -1
- package/dist/hooks/__tests__/visual-ralph-skill.test.js.map +1 -1
- package/dist/hooks/__tests__/visual-verdict-loop.test.js.map +1 -1
- package/dist/hooks/agents-overlay.js +5 -5
- package/dist/hooks/agents-overlay.js.map +1 -1
- package/dist/hooks/code-simplifier/index.js.map +1 -1
- package/dist/hooks/explore-routing.js.map +1 -1
- package/dist/hooks/extensibility/dispatcher.js.map +1 -1
- package/dist/hooks/extensibility/events.js.map +1 -1
- package/dist/hooks/extensibility/loader.js.map +1 -1
- package/dist/hooks/extensibility/plugin-runner-stdin.js.map +1 -1
- package/dist/hooks/extensibility/sdk/logging.js.map +1 -1
- package/dist/hooks/keyword-detector.d.ts +33 -7
- package/dist/hooks/keyword-detector.d.ts.map +1 -1
- package/dist/hooks/keyword-detector.js +2228 -119
- package/dist/hooks/keyword-detector.js.map +1 -1
- package/dist/hooks/keyword-registry.d.ts +12 -0
- package/dist/hooks/keyword-registry.d.ts.map +1 -1
- package/dist/hooks/keyword-registry.js +41 -0
- package/dist/hooks/keyword-registry.js.map +1 -1
- package/dist/hooks/prompt-session-provenance.d.ts +87 -0
- package/dist/hooks/prompt-session-provenance.d.ts.map +1 -0
- package/dist/hooks/prompt-session-provenance.js +180 -0
- package/dist/hooks/prompt-session-provenance.js.map +1 -0
- package/dist/hooks/session.d.ts +163 -34
- package/dist/hooks/session.d.ts.map +1 -1
- package/dist/hooks/session.js +1395 -317
- package/dist/hooks/session.js.map +1 -1
- package/dist/hooks/task-size-detector.js.map +1 -1
- package/dist/hud/__tests__/state.test.js.map +1 -1
- package/dist/hud/authority.js.map +1 -1
- package/dist/hud/constants.js.map +1 -1
- package/dist/hud/index.js.map +1 -1
- package/dist/hud/reconcile.js.map +1 -1
- package/dist/hud/render.js.map +1 -1
- package/dist/hud/state.js.map +1 -1
- package/dist/hud/tmux.js.map +1 -1
- package/dist/leader/__tests__/contract.test.js +153 -1
- package/dist/leader/__tests__/contract.test.js.map +1 -1
- package/dist/leader/contract.d.ts +42 -5
- package/dist/leader/contract.d.ts.map +1 -1
- package/dist/leader/contract.js +184 -3
- package/dist/leader/contract.js.map +1 -1
- package/dist/mcp/__tests__/bootstrap.test.js.map +1 -1
- package/dist/mcp/__tests__/code-intel-server.test.js +31 -3
- package/dist/mcp/__tests__/code-intel-server.test.js.map +1 -1
- package/dist/mcp/__tests__/server-lifecycle.test.js +15 -2
- package/dist/mcp/__tests__/server-lifecycle.test.js.map +1 -1
- package/dist/mcp/__tests__/state-paths.test.js +98 -1
- package/dist/mcp/__tests__/state-paths.test.js.map +1 -1
- package/dist/mcp/__tests__/wiki-server.test.js +18 -0
- package/dist/mcp/__tests__/wiki-server.test.js.map +1 -1
- package/dist/mcp/bootstrap.js.map +1 -1
- package/dist/mcp/code-intel-server.d.ts +58 -58
- package/dist/mcp/code-intel-server.d.ts.map +1 -1
- package/dist/mcp/code-intel-server.js +13 -3
- package/dist/mcp/code-intel-server.js.map +1 -1
- package/dist/mcp/hermes-bridge.js.map +1 -1
- package/dist/mcp/hermes-server.d.ts +70 -70
- package/dist/mcp/hermes-server.d.ts.map +1 -1
- package/dist/mcp/lifecycle-telemetry.js.map +1 -1
- package/dist/mcp/memory-server.d.ts +34 -34
- package/dist/mcp/memory-server.d.ts.map +1 -1
- package/dist/mcp/memory-server.js.map +1 -1
- package/dist/mcp/state-paths.d.ts +14 -0
- package/dist/mcp/state-paths.d.ts.map +1 -1
- package/dist/mcp/state-paths.js +96 -20
- package/dist/mcp/state-paths.js.map +1 -1
- package/dist/mcp/state-server.d.ts +47 -47
- package/dist/mcp/state-server.d.ts.map +1 -1
- package/dist/mcp/trace-server.d.ts +4 -4
- package/dist/mcp/trace-server.d.ts.map +1 -1
- package/dist/mcp/trace-server.js.map +1 -1
- package/dist/mcp/wiki-server.d.ts +34 -23
- package/dist/mcp/wiki-server.d.ts.map +1 -1
- package/dist/mcp/wiki-server.js +9 -2
- package/dist/mcp/wiki-server.js.map +1 -1
- package/dist/modes/__tests__/base-session-scope.test.js +53 -1
- package/dist/modes/__tests__/base-session-scope.test.js.map +1 -1
- package/dist/modes/__tests__/base-tmux-pane.test.js +4 -4
- package/dist/modes/__tests__/base-tmux-pane.test.js.map +1 -1
- package/dist/modes/base.d.ts.map +1 -1
- package/dist/modes/base.js +5 -5
- package/dist/modes/base.js.map +1 -1
- package/dist/notifications/__tests__/lifecycle-dedupe.test.js.map +1 -1
- package/dist/notifications/__tests__/reply-listener.test.js.map +1 -1
- package/dist/notifications/__tests__/session-idle-tail-dedupe.test.js +177 -85
- package/dist/notifications/__tests__/session-idle-tail-dedupe.test.js.map +1 -1
- package/dist/notifications/__tests__/session-registry.test.js.map +1 -1
- package/dist/notifications/__tests__/template-engine.test.js.map +1 -1
- package/dist/notifications/__tests__/verbosity.test.js.map +1 -1
- package/dist/notifications/http-client.js.map +1 -1
- package/dist/notifications/index.d.ts +4 -1
- package/dist/notifications/index.d.ts.map +1 -1
- package/dist/notifications/index.js +3 -9
- package/dist/notifications/index.js.map +1 -1
- package/dist/notifications/lifecycle-dedupe.js.map +1 -1
- package/dist/notifications/reply-listener.d.ts.map +1 -1
- package/dist/notifications/reply-listener.js.map +1 -1
- package/dist/notifications/session-registry.js.map +1 -1
- package/dist/notifications/session-status.js.map +1 -1
- package/dist/notifications/temp-contract.d.ts.map +1 -1
- package/dist/notifications/temp-contract.js +4 -0
- package/dist/notifications/temp-contract.js.map +1 -1
- package/dist/notifications/tmux-detector.js.map +1 -1
- package/dist/notifications/tmux.js.map +1 -1
- package/dist/openclaw/config.js.map +1 -1
- package/dist/openclaw/dispatcher.js.map +1 -1
- package/dist/pipeline/__tests__/orchestrator.test.js +4 -4
- package/dist/pipeline/__tests__/orchestrator.test.js.map +1 -1
- package/dist/pipeline/__tests__/stages.test.js +16 -13
- package/dist/pipeline/__tests__/stages.test.js.map +1 -1
- package/dist/pipeline/stages/code-review.js.map +1 -1
- package/dist/pipeline/stages/ralph-verify.js.map +1 -1
- package/dist/pipeline/stages/ralplan.js.map +1 -1
- package/dist/pipeline/stages/team-exec.js.map +1 -1
- package/dist/pipeline/stages/ultragoal.js.map +1 -1
- package/dist/pipeline/stages/ultraqa.js.map +1 -1
- package/dist/planning/__tests__/approved-execution-lifecycle-matrix.test.js.map +1 -1
- package/dist/planning/artifact-names.js.map +1 -1
- package/dist/planning/artifacts.js.map +1 -1
- package/dist/question/__tests__/state.test.js.map +1 -1
- package/dist/question/__tests__/ui.test.js.map +1 -1
- package/dist/question/autopilot-wait.js.map +1 -1
- package/dist/question/client.d.ts.map +1 -1
- package/dist/question/client.js.map +1 -1
- package/dist/question/deep-interview.js.map +1 -1
- package/dist/question/events.js.map +1 -1
- package/dist/question/renderer.js.map +1 -1
- package/dist/question/state.d.ts.map +1 -1
- package/dist/question/state.js.map +1 -1
- package/dist/question/ui.js.map +1 -1
- package/dist/ralph/contract.d.ts +1 -1
- package/dist/ralph/contract.d.ts.map +1 -1
- package/dist/ralplan/__tests__/consensus-gate.test.js +527 -8
- package/dist/ralplan/__tests__/consensus-gate.test.js.map +1 -1
- package/dist/ralplan/__tests__/documented-leader-preflight.test.d.ts +2 -0
- package/dist/ralplan/__tests__/documented-leader-preflight.test.d.ts.map +1 -0
- package/dist/ralplan/__tests__/documented-leader-preflight.test.js +46 -0
- package/dist/ralplan/__tests__/documented-leader-preflight.test.js.map +1 -0
- package/dist/ralplan/__tests__/runtime.test.js +116 -14
- package/dist/ralplan/__tests__/runtime.test.js.map +1 -1
- package/dist/ralplan/consensus-gate.d.ts +3 -3
- package/dist/ralplan/consensus-gate.d.ts.map +1 -1
- package/dist/ralplan/consensus-gate.js +173 -70
- package/dist/ralplan/consensus-gate.js.map +1 -1
- package/dist/ralplan/documented-leader-preflight.d.ts +34 -0
- package/dist/ralplan/documented-leader-preflight.d.ts.map +1 -0
- package/dist/ralplan/documented-leader-preflight.js +58 -0
- package/dist/ralplan/documented-leader-preflight.js.map +1 -0
- package/dist/ralplan/runtime.d.ts +3 -2
- package/dist/ralplan/runtime.d.ts.map +1 -1
- package/dist/ralplan/runtime.js +12 -9
- package/dist/ralplan/runtime.js.map +1 -1
- package/dist/runtime/__tests__/bridge.test.js +42 -1
- package/dist/runtime/__tests__/bridge.test.js.map +1 -1
- package/dist/runtime/bridge.d.ts +12 -0
- package/dist/runtime/bridge.d.ts.map +1 -1
- package/dist/runtime/bridge.js +52 -1
- package/dist/runtime/bridge.js.map +1 -1
- package/dist/runtime/process-tree.js.map +1 -1
- package/dist/runtime/run-loop.js.map +1 -1
- package/dist/runtime/run-outcome.d.ts +3 -3
- package/dist/runtime/run-outcome.d.ts.map +1 -1
- package/dist/runtime/run-outcome.js.map +1 -1
- package/dist/runtime/run-state.js.map +1 -1
- package/dist/scripts/__tests__/codex-native-hook.test.js +6657 -995
- package/dist/scripts/__tests__/codex-native-hook.test.js.map +1 -1
- package/dist/scripts/__tests__/fixtures/issue-3138-installed-smoke.d.ts +2 -0
- package/dist/scripts/__tests__/fixtures/issue-3138-installed-smoke.d.ts.map +1 -0
- package/dist/scripts/__tests__/fixtures/issue-3138-installed-smoke.js +634 -0
- package/dist/scripts/__tests__/fixtures/issue-3138-installed-smoke.js.map +1 -0
- package/dist/scripts/__tests__/generate-release-body.test.js.map +1 -1
- package/dist/scripts/__tests__/hook-derived-watcher.test.js.map +1 -1
- package/dist/scripts/__tests__/notify-dispatcher.test.js.map +1 -1
- package/dist/scripts/__tests__/notify-fallback-delivery.test.d.ts +2 -0
- package/dist/scripts/__tests__/notify-fallback-delivery.test.d.ts.map +1 -0
- package/dist/scripts/__tests__/notify-fallback-delivery.test.js +174 -0
- package/dist/scripts/__tests__/notify-fallback-delivery.test.js.map +1 -0
- package/dist/scripts/__tests__/notify-state-io.test.js +57 -1
- package/dist/scripts/__tests__/notify-state-io.test.js.map +1 -1
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.d.ts +2 -0
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.d.ts.map +1 -0
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.js +108 -0
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.js.map +1 -0
- package/dist/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.d.ts +2 -0
- package/dist/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.d.ts.map +1 -0
- package/dist/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.js +78 -0
- package/dist/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.js.map +1 -0
- package/dist/scripts/__tests__/role-intent-durable-recovery-3181.test.d.ts +2 -0
- package/dist/scripts/__tests__/role-intent-durable-recovery-3181.test.d.ts.map +1 -0
- package/dist/scripts/__tests__/role-intent-durable-recovery-3181.test.js +55 -0
- package/dist/scripts/__tests__/role-intent-durable-recovery-3181.test.js.map +1 -0
- package/dist/scripts/__tests__/run-test-files.test.js.map +1 -1
- package/dist/scripts/__tests__/smoke-packed-install.test.js +1100 -4
- package/dist/scripts/__tests__/smoke-packed-install.test.js.map +1 -1
- package/dist/scripts/__tests__/test-reply-listener-live.test.js.map +1 -1
- package/dist/scripts/build-api.js.map +1 -1
- package/dist/scripts/build-explore-harness.js.map +1 -1
- package/dist/scripts/build-sparkshell.js.map +1 -1
- package/dist/scripts/cleanup-explore-harness.js.map +1 -1
- package/dist/scripts/codex-execution-surface.js.map +1 -1
- package/dist/scripts/codex-native-hook.d.ts +4 -1
- package/dist/scripts/codex-native-hook.d.ts.map +1 -1
- package/dist/scripts/codex-native-hook.js +11577 -1184
- package/dist/scripts/codex-native-hook.js.map +1 -1
- package/dist/scripts/generate-native-release-manifest.js.map +1 -1
- package/dist/scripts/generate-release-body.js.map +1 -1
- package/dist/scripts/hook-payload-guard.d.ts.map +1 -1
- package/dist/scripts/hook-payload-guard.js.map +1 -1
- package/dist/scripts/notify-fallback-delivery.d.ts +48 -0
- package/dist/scripts/notify-fallback-delivery.d.ts.map +1 -0
- package/dist/scripts/notify-fallback-delivery.js +419 -0
- package/dist/scripts/notify-fallback-delivery.js.map +1 -0
- package/dist/scripts/notify-fallback-watcher.js +218 -138
- package/dist/scripts/notify-fallback-watcher.js.map +1 -1
- package/dist/scripts/notify-hook/auto-nudge.d.ts +18 -9
- package/dist/scripts/notify-hook/auto-nudge.d.ts.map +1 -1
- package/dist/scripts/notify-hook/auto-nudge.js +112 -15
- package/dist/scripts/notify-hook/auto-nudge.js.map +1 -1
- package/dist/scripts/notify-hook/managed-tmux.d.ts +23 -0
- package/dist/scripts/notify-hook/managed-tmux.d.ts.map +1 -1
- package/dist/scripts/notify-hook/managed-tmux.js +122 -12
- package/dist/scripts/notify-hook/managed-tmux.js.map +1 -1
- package/dist/scripts/notify-hook/orchestration-intent.d.ts +7 -7
- package/dist/scripts/notify-hook/orchestration-intent.d.ts.map +1 -1
- package/dist/scripts/notify-hook/ralph-session-resume.d.ts +5 -2
- package/dist/scripts/notify-hook/ralph-session-resume.d.ts.map +1 -1
- package/dist/scripts/notify-hook/ralph-session-resume.js +33 -37
- package/dist/scripts/notify-hook/ralph-session-resume.js.map +1 -1
- package/dist/scripts/notify-hook/state-io.d.ts +17 -0
- package/dist/scripts/notify-hook/state-io.d.ts.map +1 -1
- package/dist/scripts/notify-hook/state-io.js +73 -11
- package/dist/scripts/notify-hook/state-io.js.map +1 -1
- package/dist/scripts/notify-hook/team-dispatch.d.ts +10 -1
- package/dist/scripts/notify-hook/team-dispatch.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-dispatch.js +234 -79
- package/dist/scripts/notify-hook/team-dispatch.js.map +1 -1
- package/dist/scripts/notify-hook/team-leader-nudge.d.ts +2 -2
- package/dist/scripts/notify-hook/team-leader-nudge.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-leader-nudge.js +39 -13
- package/dist/scripts/notify-hook/team-leader-nudge.js.map +1 -1
- package/dist/scripts/notify-hook/team-tmux-guard.d.ts +16 -8
- package/dist/scripts/notify-hook/team-tmux-guard.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-tmux-guard.js +365 -44
- package/dist/scripts/notify-hook/team-tmux-guard.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker-posttooluse.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker-stop.d.ts +1 -1
- package/dist/scripts/notify-hook/team-worker-stop.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker-stop.js +19 -18
- package/dist/scripts/notify-hook/team-worker-stop.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker.d.ts +5 -2
- package/dist/scripts/notify-hook/team-worker.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker.js +37 -28
- package/dist/scripts/notify-hook/team-worker.js.map +1 -1
- package/dist/scripts/notify-hook/tmux-injection.d.ts +8 -1
- package/dist/scripts/notify-hook/tmux-injection.d.ts.map +1 -1
- package/dist/scripts/notify-hook/tmux-injection.js +53 -18
- package/dist/scripts/notify-hook/tmux-injection.js.map +1 -1
- package/dist/scripts/notify-hook/visual-verdict.d.ts +1 -1
- package/dist/scripts/notify-hook/visual-verdict.d.ts.map +1 -1
- package/dist/scripts/notify-hook/visual-verdict.js +15 -13
- package/dist/scripts/notify-hook/visual-verdict.js.map +1 -1
- package/dist/scripts/notify-hook.d.ts +24 -1
- package/dist/scripts/notify-hook.d.ts.map +1 -1
- package/dist/scripts/notify-hook.js +267 -90
- package/dist/scripts/notify-hook.js.map +1 -1
- package/dist/scripts/postinstall.js.map +1 -1
- package/dist/scripts/prompt-inventory.js.map +1 -1
- package/dist/scripts/smoke-packed-install.d.ts +927 -5
- package/dist/scripts/smoke-packed-install.d.ts.map +1 -1
- package/dist/scripts/smoke-packed-install.js +4317 -13
- package/dist/scripts/smoke-packed-install.js.map +1 -1
- package/dist/scripts/sync-plugin-mirror.js.map +1 -1
- package/dist/scripts/test-reply-listener-live.js.map +1 -1
- package/dist/scripts/test-sparkshell.js.map +1 -1
- package/dist/scripts/verify-native-agents.js.map +1 -1
- package/dist/session-history/friction.js.map +1 -1
- package/dist/session-history/search.js.map +1 -1
- package/dist/sidecar/__tests__/boundary.test.js.map +1 -1
- package/dist/sidecar/collector.js.map +1 -1
- package/dist/sidecar/index.js.map +1 -1
- package/dist/sidecar/render.js.map +1 -1
- package/dist/sidecar/tmux.js.map +1 -1
- package/dist/state/__tests__/mode-state-context.test.js +54 -0
- package/dist/state/__tests__/mode-state-context.test.js.map +1 -1
- package/dist/state/__tests__/operations.test.js +66 -15
- package/dist/state/__tests__/operations.test.js.map +1 -1
- package/dist/state/__tests__/skill-active.test.js +19 -1
- package/dist/state/__tests__/skill-active.test.js.map +1 -1
- package/dist/state/__tests__/workflow-transition.test.js +61 -0
- package/dist/state/__tests__/workflow-transition.test.js.map +1 -1
- package/dist/state/mode-state-context.d.ts +5 -0
- package/dist/state/mode-state-context.d.ts.map +1 -1
- package/dist/state/mode-state-context.js +59 -0
- package/dist/state/mode-state-context.js.map +1 -1
- package/dist/state/operations.d.ts +1 -1
- package/dist/state/operations.d.ts.map +1 -1
- package/dist/state/operations.js +49 -55
- package/dist/state/operations.js.map +1 -1
- package/dist/state/paths.d.ts +1 -1
- package/dist/state/paths.d.ts.map +1 -1
- package/dist/state/paths.js +1 -1
- package/dist/state/paths.js.map +1 -1
- package/dist/state/skill-active.d.ts +13 -1
- package/dist/state/skill-active.d.ts.map +1 -1
- package/dist/state/skill-active.js +43 -4
- package/dist/state/skill-active.js.map +1 -1
- package/dist/state/terminal-normalization.js.map +1 -1
- package/dist/state/workflow-transition-reconcile.d.ts.map +1 -1
- package/dist/state/workflow-transition-reconcile.js +20 -4
- package/dist/state/workflow-transition-reconcile.js.map +1 -1
- package/dist/state/workflow-transition.d.ts +1 -1
- package/dist/state/workflow-transition.d.ts.map +1 -1
- package/dist/state/workflow-transition.js.map +1 -1
- package/dist/subagents/__tests__/leader-bootstrap-3181.test.d.ts +2 -0
- package/dist/subagents/__tests__/leader-bootstrap-3181.test.d.ts.map +1 -0
- package/dist/subagents/__tests__/leader-bootstrap-3181.test.js +52 -0
- package/dist/subagents/__tests__/leader-bootstrap-3181.test.js.map +1 -0
- package/dist/subagents/__tests__/tracker.test.js +1550 -10
- package/dist/subagents/__tests__/tracker.test.js.map +1 -1
- package/dist/subagents/role-routing-marker.d.ts +10 -0
- package/dist/subagents/role-routing-marker.d.ts.map +1 -0
- package/dist/subagents/role-routing-marker.js +136 -0
- package/dist/subagents/role-routing-marker.js.map +1 -0
- package/dist/subagents/tracker.d.ts +91 -0
- package/dist/subagents/tracker.d.ts.map +1 -1
- package/dist/subagents/tracker.js +806 -62
- package/dist/subagents/tracker.js.map +1 -1
- package/dist/team/__tests__/api-interop.test.js +91 -1
- package/dist/team/__tests__/api-interop.test.js.map +1 -1
- package/dist/team/__tests__/approved-execution.test.js.map +1 -1
- package/dist/team/__tests__/delivery-e2e-smoke.test.js +52 -10
- package/dist/team/__tests__/delivery-e2e-smoke.test.js.map +1 -1
- package/dist/team/__tests__/idle-nudge.test.js +45 -15
- package/dist/team/__tests__/idle-nudge.test.js.map +1 -1
- package/dist/team/__tests__/mcp-comm.test.js +93 -27
- package/dist/team/__tests__/mcp-comm.test.js.map +1 -1
- package/dist/team/__tests__/model-contract.test.js +341 -4
- package/dist/team/__tests__/model-contract.test.js.map +1 -1
- package/dist/team/__tests__/role-router.test.js.map +1 -1
- package/dist/team/__tests__/runtime-cli.test.js +9 -0
- package/dist/team/__tests__/runtime-cli.test.js.map +1 -1
- package/dist/team/__tests__/runtime.test.js +3624 -355
- package/dist/team/__tests__/runtime.test.js.map +1 -1
- package/dist/team/__tests__/scaling.test.js +2516 -118
- package/dist/team/__tests__/scaling.test.js.map +1 -1
- package/dist/team/__tests__/state-root.test.js.map +1 -1
- package/dist/team/__tests__/state.test.js +804 -36
- package/dist/team/__tests__/state.test.js.map +1 -1
- package/dist/team/__tests__/team-ops-contract.test.js +6 -0
- package/dist/team/__tests__/team-ops-contract.test.js.map +1 -1
- package/dist/team/__tests__/tmux-session.test.js +3664 -969
- package/dist/team/__tests__/tmux-session.test.js.map +1 -1
- package/dist/team/__tests__/tmux-test-fixture.js.map +1 -1
- package/dist/team/__tests__/worker-runtime-identity.test.js +38 -2
- package/dist/team/__tests__/worker-runtime-identity.test.js.map +1 -1
- package/dist/team/__tests__/worktree.test.js +6 -0
- package/dist/team/__tests__/worktree.test.js.map +1 -1
- package/dist/team/api-interop.d.ts +2 -2
- package/dist/team/api-interop.d.ts.map +1 -1
- package/dist/team/api-interop.js +15 -9
- package/dist/team/api-interop.js.map +1 -1
- package/dist/team/commit-hygiene.js.map +1 -1
- package/dist/team/contracts.d.ts +5 -5
- package/dist/team/contracts.d.ts.map +1 -1
- package/dist/team/delivery-log.js.map +1 -1
- package/dist/team/exact-pane.d.ts +23 -0
- package/dist/team/exact-pane.d.ts.map +1 -0
- package/dist/team/exact-pane.js +133 -0
- package/dist/team/exact-pane.js.map +1 -0
- package/dist/team/followup-planner.js.map +1 -1
- package/dist/team/goal-workflow.js.map +1 -1
- package/dist/team/idle-nudge.d.ts +12 -1
- package/dist/team/idle-nudge.d.ts.map +1 -1
- package/dist/team/idle-nudge.js +41 -14
- package/dist/team/idle-nudge.js.map +1 -1
- package/dist/team/model-contract.d.ts +18 -2
- package/dist/team/model-contract.d.ts.map +1 -1
- package/dist/team/model-contract.js +314 -35
- package/dist/team/model-contract.js.map +1 -1
- package/dist/team/orchestrator.js.map +1 -1
- package/dist/team/pane-status.js.map +1 -1
- package/dist/team/phase-controller.js.map +1 -1
- package/dist/team/reminder-intents.d.ts +1 -1
- package/dist/team/reminder-intents.d.ts.map +1 -1
- package/dist/team/reminder-intents.js.map +1 -1
- package/dist/team/runtime-cli.js.map +1 -1
- package/dist/team/runtime.d.ts +1 -0
- package/dist/team/runtime.d.ts.map +1 -1
- package/dist/team/runtime.js +1326 -231
- package/dist/team/runtime.js.map +1 -1
- package/dist/team/scaling.d.ts +22 -8
- package/dist/team/scaling.d.ts.map +1 -1
- package/dist/team/scaling.js +1463 -376
- package/dist/team/scaling.js.map +1 -1
- package/dist/team/state/dispatch-lock.js.map +1 -1
- package/dist/team/state/dispatch.d.ts.map +1 -1
- package/dist/team/state/dispatch.js +26 -8
- package/dist/team/state/dispatch.js.map +1 -1
- package/dist/team/state/events.js.map +1 -1
- package/dist/team/state/locks.d.ts.map +1 -1
- package/dist/team/state/locks.js +61 -13
- package/dist/team/state/locks.js.map +1 -1
- package/dist/team/state/tasks.d.ts.map +1 -1
- package/dist/team/state/tasks.js +4 -3
- package/dist/team/state/tasks.js.map +1 -1
- package/dist/team/state-root.js.map +1 -1
- package/dist/team/state.d.ts +67 -1
- package/dist/team/state.d.ts.map +1 -1
- package/dist/team/state.js +636 -129
- package/dist/team/state.js.map +1 -1
- package/dist/team/team-identity.d.ts.map +1 -1
- package/dist/team/team-identity.js.map +1 -1
- package/dist/team/team-ops.d.ts +5 -0
- package/dist/team/team-ops.d.ts.map +1 -1
- package/dist/team/team-ops.js +4 -0
- package/dist/team/team-ops.js.map +1 -1
- package/dist/team/tmux-session.d.ts +125 -23
- package/dist/team/tmux-session.d.ts.map +1 -1
- package/dist/team/tmux-session.js +1340 -333
- package/dist/team/tmux-session.js.map +1 -1
- package/dist/team/worker-bootstrap.js.map +1 -1
- package/dist/team/worktree.d.ts.map +1 -1
- package/dist/team/worktree.js +4 -0
- package/dist/team/worktree.js.map +1 -1
- package/dist/ultragoal/__tests__/artifacts.test.js.map +1 -1
- package/dist/ultragoal/__tests__/docs-contract.test.js.map +1 -1
- package/dist/ultragoal/artifacts.d.ts.map +1 -1
- package/dist/ultragoal/artifacts.js +4 -1
- package/dist/ultragoal/artifacts.js.map +1 -1
- package/dist/url-reader/index.js.map +1 -1
- package/dist/utils/__tests__/file-durability.test.d.ts +2 -0
- package/dist/utils/__tests__/file-durability.test.d.ts.map +1 -0
- package/dist/utils/__tests__/file-durability.test.js +57 -0
- package/dist/utils/__tests__/file-durability.test.js.map +1 -0
- package/dist/utils/agents-model-table.js.map +1 -1
- package/dist/utils/file-durability.d.ts +15 -0
- package/dist/utils/file-durability.d.ts.map +1 -0
- package/dist/utils/file-durability.js +37 -0
- package/dist/utils/file-durability.js.map +1 -0
- package/dist/utils/package.js.map +1 -1
- package/dist/utils/paths.js.map +1 -1
- package/dist/utils/platform-command.js.map +1 -1
- package/dist/utils/repo-deps.js.map +1 -1
- package/dist/utils/version.js.map +1 -1
- package/dist/verification/__tests__/ci-rust-gates.test.js +1 -1
- package/dist/verification/__tests__/dev-merge-issue-close-workflow.test.js +3 -0
- package/dist/verification/__tests__/dev-merge-issue-close-workflow.test.js.map +1 -1
- package/dist/verification/verifier.js.map +1 -1
- package/dist/visual/constants.d.ts +1 -1
- package/dist/visual/constants.d.ts.map +1 -1
- package/dist/vscode/__tests__/index.test.js.map +1 -1
- package/dist/vscode/index.js.map +1 -1
- package/dist/wiki/__tests__/cjk-tokenize.test.js.map +1 -1
- package/dist/wiki/__tests__/lint.test.js.map +1 -1
- package/dist/wiki/__tests__/query.test.js.map +1 -1
- package/dist/wiki/__tests__/storage.test.js.map +1 -1
- package/dist/wiki/__tests__/test-helpers.d.ts.map +1 -1
- package/dist/wiki/lint.d.ts +4 -1
- package/dist/wiki/lint.d.ts.map +1 -1
- package/dist/wiki/lint.js +3 -2
- package/dist/wiki/lint.js.map +1 -1
- package/dist/wiki/query.js.map +1 -1
- package/dist/wiki/storage.js.map +1 -1
- package/package.json +2 -2
- package/plugins/oh-my-codex/.codex-plugin/plugin.json +1 -1
- package/plugins/oh-my-codex/hooks/codex-native-hook.mjs +24 -0
- package/plugins/oh-my-codex/skills/omx-setup/SKILL.md +10 -5
- package/plugins/oh-my-codex/skills/ralph/SKILL.md +6 -2
- package/plugins/oh-my-codex/skills/ralplan/SKILL.md +6 -2
- package/plugins/oh-my-codex/skills/team/SKILL.md +13 -9
- package/skills/omx-setup/SKILL.md +10 -5
- package/skills/ralph/SKILL.md +6 -2
- package/skills/ralplan/SKILL.md +6 -2
- package/skills/team/SKILL.md +13 -9
- package/src/scripts/__tests__/codex-native-hook.test.ts +20048 -10827
- package/src/scripts/__tests__/fixtures/issue-3138-installed-smoke.ts +734 -0
- package/src/scripts/__tests__/notify-fallback-delivery.test.ts +177 -0
- package/src/scripts/__tests__/notify-state-io.test.ts +73 -0
- package/src/scripts/__tests__/reasoning-artifact-contract.test.ts +129 -0
- package/src/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.ts +78 -0
- package/src/scripts/__tests__/role-intent-durable-recovery-3181.test.ts +60 -0
- package/src/scripts/__tests__/smoke-packed-install.test.ts +1309 -4
- package/src/scripts/codex-native-hook.ts +11429 -1163
- package/src/scripts/notify-fallback-delivery.ts +416 -0
- package/src/scripts/notify-fallback-watcher.ts +216 -155
- package/src/scripts/notify-hook/auto-nudge.ts +113 -14
- package/src/scripts/notify-hook/managed-tmux.ts +149 -12
- package/src/scripts/notify-hook/ralph-session-resume.ts +37 -36
- package/src/scripts/notify-hook/state-io.ts +100 -13
- package/src/scripts/notify-hook/team-dispatch.ts +242 -75
- package/src/scripts/notify-hook/team-leader-nudge.ts +40 -18
- package/src/scripts/notify-hook/team-tmux-guard.ts +366 -39
- package/src/scripts/notify-hook/team-worker-stop.ts +20 -16
- package/src/scripts/notify-hook/team-worker.ts +39 -26
- package/src/scripts/notify-hook/tmux-injection.ts +66 -23
- package/src/scripts/notify-hook/visual-verdict.ts +22 -12
- package/src/scripts/notify-hook.ts +336 -90
- package/src/scripts/smoke-packed-install.ts +5154 -149
|
@@ -2,15 +2,18 @@ import { describe, it } from 'node:test';
|
|
|
2
2
|
import assert from 'node:assert/strict';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { execFileSync } from 'node:child_process';
|
|
5
|
-
import { mkdtemp, rm, readFile, writeFile, mkdir, chmod, readdir } from 'fs/promises';
|
|
6
|
-
import { join, relative } from 'path';
|
|
5
|
+
import { mkdtemp, rm, readFile, writeFile, mkdir, chmod, readdir, symlink } from 'fs/promises';
|
|
6
|
+
import { join, posix, relative, win32 } from 'path';
|
|
7
7
|
import { tmpdir } from 'os';
|
|
8
8
|
import { existsSync, readFileSync } from 'fs';
|
|
9
|
-
import { initTeamState, createTask, readTask, readTeamConfig, saveTeamConfig, readWorkerStatus, writeWorkerStatus, withScalingLock, DEFAULT_MAX_WORKERS, } from '../state.js';
|
|
10
|
-
import { isScalingEnabled, scaleUp, scaleDown } from '../scaling.js';
|
|
9
|
+
import { initTeamState, createTask, claimTask, readTask, readTeamConfig, saveTeamConfig, readWorkerStatus, writeWorkerStatus, withScalingLock, recoverTeamMembershipTaskTransaction, commitTeamMembershipTaskTransaction, withTeamTaskBarrier, DEFAULT_MAX_WORKERS, listDispatchRequests, readTeamManifestV2, } from '../state.js';
|
|
10
|
+
import { isSameOrInsidePath, isScalingEnabled, reconcileScaleDownCleanupDebt, scaleUp, scaleDown } from '../scaling.js';
|
|
11
11
|
import { resolveCanonicalTeamStateRoot } from '../state-root.js';
|
|
12
12
|
import { resolvePersistedApprovedTeamExecutionContinuityState, writePersistedApprovedTeamExecutionBinding, } from '../approved-execution.js';
|
|
13
|
+
import { TEAM_WORKER_INHERITED_MODEL_ENV } from '../model-contract.js';
|
|
14
|
+
import { buildWorkerProcessLaunchSpec } from '../tmux-session.js';
|
|
13
15
|
delete process.env.OMX_TEAM_STATE_ROOT;
|
|
16
|
+
process.env.OMX_RUNTIME_BRIDGE = '0';
|
|
14
17
|
async function initCommittedGitRepo(cwd) {
|
|
15
18
|
execFileSync('git', ['init'], { cwd, stdio: 'pipe' });
|
|
16
19
|
execFileSync('git', ['config', 'user.name', 'OMX Test'], { cwd, stdio: 'pipe' });
|
|
@@ -33,6 +36,11 @@ function computeGitBlobSha1(content) {
|
|
|
33
36
|
const header = Buffer.from(`blob ${buffer.length}\0`, 'utf-8');
|
|
34
37
|
return createHash('sha1').update(header).update(buffer).digest('hex');
|
|
35
38
|
}
|
|
39
|
+
function withoutConfigGeneration(bytes) {
|
|
40
|
+
const value = JSON.parse(bytes);
|
|
41
|
+
delete value.config_generation;
|
|
42
|
+
return value;
|
|
43
|
+
}
|
|
36
44
|
function canonicalContextPackRelativePath(slug) {
|
|
37
45
|
return `.omx/context/context-20260507T120000Z-${slug}.json`;
|
|
38
46
|
}
|
|
@@ -145,6 +153,7 @@ async function writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath) {
|
|
|
145
153
|
'set -eu',
|
|
146
154
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
147
155
|
'case "${1:-}" in',
|
|
156
|
+
' show-option) echo "team:scale-up" ;;',
|
|
148
157
|
' -V)',
|
|
149
158
|
' echo "tmux 3.2a"',
|
|
150
159
|
' ;;',
|
|
@@ -152,10 +161,28 @@ async function writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath) {
|
|
|
152
161
|
' echo "%31"',
|
|
153
162
|
' ;;',
|
|
154
163
|
' list-panes)',
|
|
155
|
-
'
|
|
164
|
+
' case "$*" in',
|
|
165
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
166
|
+
" printf '%s\t%s\t%s\n' '%11' '0' '42411'",
|
|
167
|
+
" printf '%s\t%s\t%s\n' '%21' '0' '42421'",
|
|
168
|
+
` if [ ! -f "${join(fakeBinDir, 'killed-%31')}" ]; then printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'; fi`,
|
|
169
|
+
' ;;',
|
|
170
|
+
' *)',
|
|
171
|
+
' echo "42424"',
|
|
172
|
+
' ;;',
|
|
173
|
+
' esac',
|
|
174
|
+
' ;;',
|
|
175
|
+
' show-option)',
|
|
176
|
+
' case "$*" in',
|
|
177
|
+
' *"@omx_team_pane_owner_id"*) echo "team:scale-up" ;;',
|
|
178
|
+
' *) exit 1 ;;',
|
|
179
|
+
' esac',
|
|
156
180
|
' ;;',
|
|
157
181
|
' send-keys)',
|
|
158
182
|
' ;;',
|
|
183
|
+
' kill-pane)',
|
|
184
|
+
` : > "${join(fakeBinDir, 'killed-')}$3"`,
|
|
185
|
+
' ;;',
|
|
159
186
|
' capture-pane)',
|
|
160
187
|
' echo ""',
|
|
161
188
|
' ;;',
|
|
@@ -173,8 +200,12 @@ async function configureScaleUpTeamForDirectDispatch(teamName, cwd) {
|
|
|
173
200
|
throw new Error(`missing team config for ${teamName}`);
|
|
174
201
|
}
|
|
175
202
|
config.tmux_session = `omx-team-${teamName}`;
|
|
203
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
176
204
|
config.leader_pane_id = '%11';
|
|
205
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
206
|
+
config.leader_pane_pid = 42411;
|
|
177
207
|
config.workers[0].pane_id = '%21';
|
|
208
|
+
config.workers[0].pid = 42421;
|
|
178
209
|
await saveTeamConfig(config, cwd);
|
|
179
210
|
const manifestPath = join(cwd, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
180
211
|
if (!existsSync(manifestPath)) {
|
|
@@ -315,6 +346,60 @@ describe('isScalingEnabled', () => {
|
|
|
315
346
|
assert.equal(isScalingEnabled({ OMX_TEAM_SCALING_ENABLED: ' true ' }), true);
|
|
316
347
|
});
|
|
317
348
|
});
|
|
349
|
+
// ── scale-down cleanup debt path containment ──────────────────────────────────
|
|
350
|
+
describe('scale-down cleanup debt path containment', () => {
|
|
351
|
+
const windowsPathSemantics = {
|
|
352
|
+
relative: win32.relative,
|
|
353
|
+
isAbsolute: win32.isAbsolute,
|
|
354
|
+
sep: win32.sep,
|
|
355
|
+
};
|
|
356
|
+
it('accepts same-root and descendant native Windows canonical paths', () => {
|
|
357
|
+
assert.equal(isSameOrInsidePath('C:\\repo', 'C:\\repo', windowsPathSemantics), true);
|
|
358
|
+
assert.equal(isSameOrInsidePath('C:\\repo\\.omx\\team\\alpha', 'C:\\repo', windowsPathSemantics), true);
|
|
359
|
+
});
|
|
360
|
+
it('accepts mixed-separator, drive-letter, and case-equivalent Windows descendants', () => {
|
|
361
|
+
assert.equal(isSameOrInsidePath('c:/REPO/.omx\\team/alpha', 'C:\\repo', windowsPathSemantics), true);
|
|
362
|
+
});
|
|
363
|
+
it('accepts same-share UNC descendants across host and share casing', () => {
|
|
364
|
+
assert.equal(isSameOrInsidePath('\\\\SERVER\\Share\\repo/.omx\\team\\alpha', '\\\\server\\share\\REPO', windowsPathSemantics), true);
|
|
365
|
+
});
|
|
366
|
+
for (const [name, candidate] of [
|
|
367
|
+
['a sibling prefix collision', 'C:\\repo-other\\worker'],
|
|
368
|
+
['a parent traversal escape', 'C:\\repo\\..\\foreign\\worker'],
|
|
369
|
+
['a foreign same-volume root', 'C:\\foreign\\worker'],
|
|
370
|
+
['an absolute relative result from a foreign drive', 'D:\\foreign\\worker'],
|
|
371
|
+
['a foreign UNC host', '\\\\foreign-server\\share\\repo\\worker'],
|
|
372
|
+
['a foreign UNC share', '\\\\server\\foreign-share\\repo\\worker'],
|
|
373
|
+
['a UNC sibling prefix collision', '\\\\server\\share\\repo-other\\worker'],
|
|
374
|
+
]) {
|
|
375
|
+
it(`rejects ${name} under native Windows path semantics`, () => {
|
|
376
|
+
assert.equal(isSameOrInsidePath(candidate, 'C:\\repo', windowsPathSemantics), false);
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
for (const [name, candidate] of [
|
|
380
|
+
['a foreign UNC host', '\\\\foreign-server\\share\\repo\\worker'],
|
|
381
|
+
['a foreign UNC share', '\\\\server\\foreign-share\\repo\\worker'],
|
|
382
|
+
['a UNC sibling prefix collision', '\\\\server\\share\\repo-other\\worker'],
|
|
383
|
+
]) {
|
|
384
|
+
it(`rejects ${name} under native Windows UNC path semantics`, () => {
|
|
385
|
+
assert.equal(isSameOrInsidePath(candidate, '\\\\server\\share\\repo', windowsPathSemantics), false);
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
const posixPathSemantics = {
|
|
389
|
+
relative: posix.relative,
|
|
390
|
+
isAbsolute: posix.isAbsolute,
|
|
391
|
+
sep: posix.sep,
|
|
392
|
+
};
|
|
393
|
+
it('preserves POSIX same-root and descendant containment', () => {
|
|
394
|
+
assert.equal(isSameOrInsidePath('/repo', '/repo', posixPathSemantics), true);
|
|
395
|
+
assert.equal(isSameOrInsidePath('/repo/.omx/team/alpha', '/repo', posixPathSemantics), true);
|
|
396
|
+
});
|
|
397
|
+
it('rejects POSIX siblings, traversal, and foreign roots', () => {
|
|
398
|
+
assert.equal(isSameOrInsidePath('/repo-other/worker', '/repo', posixPathSemantics), false);
|
|
399
|
+
assert.equal(isSameOrInsidePath('/repo/../foreign/worker', '/repo', posixPathSemantics), false);
|
|
400
|
+
assert.equal(isSameOrInsidePath('/foreign/worker', '/repo', posixPathSemantics), false);
|
|
401
|
+
});
|
|
402
|
+
});
|
|
318
403
|
// ── WorkerStatus draining state ───────────────────────────────────────────────
|
|
319
404
|
describe('WorkerStatus draining state', () => {
|
|
320
405
|
it('writeWorkerStatus writes draining status and readWorkerStatus reads it back', async () => {
|
|
@@ -392,7 +477,7 @@ describe('withScalingLock', () => {
|
|
|
392
477
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-lock-'));
|
|
393
478
|
try {
|
|
394
479
|
await initTeamState('lock-test', 'task', 'executor', 1, cwd);
|
|
395
|
-
const lockDir = join(cwd, '.omx', 'state', 'team', 'lock-test
|
|
480
|
+
const lockDir = join(cwd, '.omx', 'state', '.team-locks', 'lock-test.scaling');
|
|
396
481
|
const result = await withScalingLock('lock-test', cwd, async () => {
|
|
397
482
|
// Lock should exist during execution
|
|
398
483
|
assert.equal(existsSync(lockDir), true);
|
|
@@ -410,7 +495,7 @@ describe('withScalingLock', () => {
|
|
|
410
495
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-lock-err-'));
|
|
411
496
|
try {
|
|
412
497
|
await initTeamState('lock-err', 'task', 'executor', 1, cwd);
|
|
413
|
-
const lockDir = join(cwd, '.omx', 'state', 'team', 'lock-err
|
|
498
|
+
const lockDir = join(cwd, '.omx', 'state', '.team-locks', 'lock-err.scaling');
|
|
414
499
|
await assert.rejects(withScalingLock('lock-err', cwd, async () => {
|
|
415
500
|
throw new Error('test error');
|
|
416
501
|
}), { message: 'test error' });
|
|
@@ -487,6 +572,94 @@ describe('scaleUp', () => {
|
|
|
487
572
|
delete process.env.PATH;
|
|
488
573
|
}
|
|
489
574
|
});
|
|
575
|
+
it('rejects explicit mixed worker policy before scale-up creates worker state or a pane', async () => {
|
|
576
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-explicit-policy-'));
|
|
577
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-explicit-policy-bin-'));
|
|
578
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
579
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
580
|
+
const previousPath = process.env.PATH;
|
|
581
|
+
try {
|
|
582
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
583
|
+
printf '%s\n' "$*" >> "${tmuxLogPath}"
|
|
584
|
+
exit 0
|
|
585
|
+
`);
|
|
586
|
+
await chmod(tmuxStubPath, 0o755);
|
|
587
|
+
await writeFile(tmuxLogPath, '');
|
|
588
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
589
|
+
await initTeamState('scale-up-explicit-policy', 'task', 'executor', 1, cwd);
|
|
590
|
+
const result = await scaleUp('scale-up-explicit-policy', 1, 'executor', [{ subject: 'new task', description: 'new task', owner: 'worker-2' }], cwd, {
|
|
591
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
592
|
+
OMX_TEAM_WORKER_LAUNCH_ARGS: '--dangerously-bypass-approvals-and-sandbox --sandbox workspace-write',
|
|
593
|
+
});
|
|
594
|
+
assert.equal(result.ok, false);
|
|
595
|
+
if (!result.ok) {
|
|
596
|
+
assert.match(result.error, /Invalid OMX_TEAM_WORKER_LAUNCH_ARGS: bypass cannot be combined with direct approval or sandbox policy/);
|
|
597
|
+
}
|
|
598
|
+
const config = await readTeamConfig('scale-up-explicit-policy', cwd);
|
|
599
|
+
assert.equal(config?.workers.length, 1);
|
|
600
|
+
assert.equal(config?.next_worker_index, 2);
|
|
601
|
+
assert.deepEqual(await readScaleUpTaskPayloads('scale-up-explicit-policy', cwd), []);
|
|
602
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-up-explicit-policy', 'workers', 'worker-2')), false);
|
|
603
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, 'scale-up-explicit-policy', 'worker-2')), false);
|
|
604
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
605
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
606
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
607
|
+
}
|
|
608
|
+
finally {
|
|
609
|
+
if (typeof previousPath === 'string')
|
|
610
|
+
process.env.PATH = previousPath;
|
|
611
|
+
else
|
|
612
|
+
delete process.env.PATH;
|
|
613
|
+
await rm(cwd, { recursive: true, force: true });
|
|
614
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
615
|
+
}
|
|
616
|
+
});
|
|
617
|
+
it('rejects Claude and Gemini restrictive config policy before scale-up creates task payloads, worker state, a pane, or process', async () => {
|
|
618
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-restrictive-noncodex-'));
|
|
619
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-restrictive-noncodex-bin-'));
|
|
620
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
621
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
622
|
+
const previousPath = process.env.PATH;
|
|
623
|
+
try {
|
|
624
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
625
|
+
printf '%s\n' "$*" >> "${tmuxLogPath}"
|
|
626
|
+
exit 0
|
|
627
|
+
`);
|
|
628
|
+
await chmod(tmuxStubPath, 0o755);
|
|
629
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
630
|
+
for (const workerCli of ['claude', 'gemini']) {
|
|
631
|
+
const teamName = `scale-up-restrictive-${workerCli}`;
|
|
632
|
+
await writeFile(tmuxLogPath, '');
|
|
633
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
634
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'new task', description: 'new task', owner: 'worker-2' }], cwd, {
|
|
635
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
636
|
+
OMX_TEAM_WORKER_CLI: workerCli,
|
|
637
|
+
OMX_TEAM_WORKER_LAUNCH_ARGS: `--config 'sandbox_mode="workspace-write"'`,
|
|
638
|
+
});
|
|
639
|
+
assert.equal(result.ok, false);
|
|
640
|
+
if (!result.ok) {
|
|
641
|
+
assert.match(result.error, new RegExp(`Selected team worker CLI "${workerCli}" is incompatible with an explicit approval or sandbox policy\\.`));
|
|
642
|
+
}
|
|
643
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
644
|
+
assert.equal(config?.workers.length, 1);
|
|
645
|
+
assert.equal(config?.next_worker_index, 2);
|
|
646
|
+
assert.deepEqual(await readScaleUpTaskPayloads(teamName, cwd), []);
|
|
647
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
648
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), false);
|
|
649
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
650
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
651
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
finally {
|
|
655
|
+
if (typeof previousPath === 'string')
|
|
656
|
+
process.env.PATH = previousPath;
|
|
657
|
+
else
|
|
658
|
+
delete process.env.PATH;
|
|
659
|
+
await rm(cwd, { recursive: true, force: true });
|
|
660
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
661
|
+
}
|
|
662
|
+
});
|
|
490
663
|
it('persists scaled-up task roles in canonical task state and inbox ids', async () => {
|
|
491
664
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-role-'));
|
|
492
665
|
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-role-bin-'));
|
|
@@ -499,6 +672,7 @@ describe('scaleUp', () => {
|
|
|
499
672
|
'set -eu',
|
|
500
673
|
`printf '\%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
501
674
|
'case "${1:-}" in',
|
|
675
|
+
' show-option) echo "team:scale-up-role" ;;',
|
|
502
676
|
' -V)',
|
|
503
677
|
' echo "tmux 3.2a"',
|
|
504
678
|
' ;;',
|
|
@@ -506,10 +680,22 @@ describe('scaleUp', () => {
|
|
|
506
680
|
' echo "%31"',
|
|
507
681
|
' ;;',
|
|
508
682
|
' list-panes)',
|
|
509
|
-
'
|
|
683
|
+
' case "$*" in',
|
|
684
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
685
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
686
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
687
|
+
` if [ ! -f "${join(fakeBinDir, 'scale-up-role-killed-%31')}" ]; then printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'; fi`,
|
|
688
|
+
' ;;',
|
|
689
|
+
' *)',
|
|
690
|
+
' echo "42424"',
|
|
691
|
+
' ;;',
|
|
692
|
+
' esac',
|
|
510
693
|
' ;;',
|
|
511
694
|
' send-keys)',
|
|
512
695
|
' ;;',
|
|
696
|
+
' kill-pane)',
|
|
697
|
+
` : > "${join(fakeBinDir, 'scale-up-role-killed-')}$3"`,
|
|
698
|
+
' ;;',
|
|
513
699
|
' capture-pane)',
|
|
514
700
|
' echo ""',
|
|
515
701
|
' ;;',
|
|
@@ -537,7 +723,9 @@ describe('scaleUp', () => {
|
|
|
537
723
|
return;
|
|
538
724
|
config.tmux_session = 'omx-team-scale-up-role';
|
|
539
725
|
config.leader_pane_id = '%11';
|
|
726
|
+
config.tmux_pane_owner_id = 'team:scale-up-role';
|
|
540
727
|
config.workers[0].pane_id = '%21';
|
|
728
|
+
config.workers[0].pid = 42421;
|
|
541
729
|
await saveTeamConfig(config, cwd);
|
|
542
730
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'scale-up-role', 'manifest.v2.json');
|
|
543
731
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -570,18 +758,111 @@ describe('scaleUp', () => {
|
|
|
570
758
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
571
759
|
}
|
|
572
760
|
});
|
|
573
|
-
it('
|
|
761
|
+
it('covers the scale-up config-policy and no-policy argv matrix', async () => {
|
|
762
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-policy-matrix-'));
|
|
763
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-policy-matrix-bin-'));
|
|
764
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
765
|
+
const capturePath = join(cwd, 'worker-argv.txt');
|
|
766
|
+
const emptyCodexHome = await mkdtemp(join(tmpdir(), 'omx-scale-up-policy-matrix-codex-home-'));
|
|
767
|
+
const previousPath = process.env.PATH;
|
|
768
|
+
const previousArgv = process.argv;
|
|
769
|
+
const previousBypass = process.env.OMX_BYPASS_DEFAULT_SYSTEM_PROMPT;
|
|
770
|
+
try {
|
|
771
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
772
|
+
await writeFile(join(fakeBinDir, 'codex'), `#!/bin/sh
|
|
773
|
+
printf '%s\\n' "$@" > '${capturePath}'
|
|
774
|
+
`);
|
|
775
|
+
await chmod(join(fakeBinDir, 'codex'), 0o755);
|
|
776
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
777
|
+
process.env.OMX_BYPASS_DEFAULT_SYSTEM_PROMPT = '0';
|
|
778
|
+
const runScaleUpCase = async (params) => {
|
|
779
|
+
await writeFile(capturePath, '');
|
|
780
|
+
await initTeamState(params.teamName, 'task', 'executor', 1, cwd);
|
|
781
|
+
await configureScaleUpTeamForDirectDispatch(params.teamName, cwd);
|
|
782
|
+
const result = await scaleUp(params.teamName, 1, 'executor', [{
|
|
783
|
+
subject: 'implement task',
|
|
784
|
+
description: 'implement task',
|
|
785
|
+
owner: 'worker-2',
|
|
786
|
+
...(params.role === undefined ? {} : { role: params.role }),
|
|
787
|
+
}], cwd, {
|
|
788
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
789
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
790
|
+
CODEX_HOME: emptyCodexHome,
|
|
791
|
+
OMX_TEAM_WORKER_LAUNCH_ARGS: params.launchArgs,
|
|
792
|
+
...(params.inheritedModel ? { [TEAM_WORKER_INHERITED_MODEL_ENV]: params.inheritedModel } : {}),
|
|
793
|
+
});
|
|
794
|
+
if (!result.ok)
|
|
795
|
+
assert.fail(result.error);
|
|
796
|
+
const startupScriptPath = workerStartupScriptPath(cwd, params.teamName, 'worker-2');
|
|
797
|
+
const scriptResult = execFileSync('/bin/sh', [startupScriptPath], { encoding: 'utf-8' });
|
|
798
|
+
assert.equal(scriptResult, '');
|
|
799
|
+
return (await readFile(capturePath, 'utf-8')).trim().split('\n');
|
|
800
|
+
};
|
|
801
|
+
process.argv = [
|
|
802
|
+
...previousArgv.filter((arg) => arg !== '--dangerously-bypass-approvals-and-sandbox' && arg !== '--madmax'),
|
|
803
|
+
'--madmax',
|
|
804
|
+
];
|
|
805
|
+
const sandboxArgs = await runScaleUpCase({
|
|
806
|
+
teamName: 'scale-up-sandbox-policy',
|
|
807
|
+
role: 'executor',
|
|
808
|
+
launchArgs: String.raw `--config 'sandbox_mode="workspace-write"' -- 'C:\scale-up\nested\' '--sandbox=read-only' '--madmax'`,
|
|
809
|
+
inheritedModel: 'leader-model',
|
|
810
|
+
});
|
|
811
|
+
const expectedSandboxArgs = [
|
|
812
|
+
'--sandbox', 'workspace-write',
|
|
813
|
+
'-c', 'model_reasoning_effort="medium"',
|
|
814
|
+
'--model', 'leader-model',
|
|
815
|
+
'--', 'C:\\scale-up\\nested\\', '--sandbox=read-only', '--madmax',
|
|
816
|
+
];
|
|
817
|
+
assert.deepEqual(sandboxArgs, expectedSandboxArgs);
|
|
818
|
+
assert.deepEqual(buildWorkerProcessLaunchSpec('initial-policy-team', 1, expectedSandboxArgs, cwd, { CODEX_HOME: emptyCodexHome }, 'codex', undefined, 'executor').args, sandboxArgs);
|
|
819
|
+
const approvalArgs = await runScaleUpCase({
|
|
820
|
+
teamName: 'scale-up-approval-policy',
|
|
821
|
+
role: 'executor',
|
|
822
|
+
launchArgs: '--ask-for-approval=on-request',
|
|
823
|
+
inheritedModel: 'leader-model',
|
|
824
|
+
});
|
|
825
|
+
assert.deepEqual(approvalArgs, [
|
|
826
|
+
'--ask-for-approval', 'on-request',
|
|
827
|
+
'-c', 'model_reasoning_effort="medium"',
|
|
828
|
+
'--model', 'leader-model',
|
|
829
|
+
]);
|
|
830
|
+
process.argv = previousArgv.filter((arg) => arg !== '--dangerously-bypass-approvals-and-sandbox' && arg !== '--madmax');
|
|
831
|
+
const bypass = '--dangerously-bypass-approvals-and-sandbox';
|
|
832
|
+
assert.deepEqual(await runScaleUpCase({ teamName: 'scale-up-execution-default', role: 'executor', launchArgs: '--model policy-model' }), ['-c', 'model_reasoning_effort="medium"', '--model', 'policy-model', bypass]);
|
|
833
|
+
assert.deepEqual(await runScaleUpCase({ teamName: 'scale-up-nonexecution-default', role: 'explore', launchArgs: '--model policy-model' }), ['-c', 'model_reasoning_effort="low"', '--model', 'policy-model']);
|
|
834
|
+
assert.deepEqual(await runScaleUpCase({ teamName: 'scale-up-absent-role-default', launchArgs: '--model policy-model' }), ['-c', 'model_reasoning_effort="medium"', '--model', 'policy-model', bypass]);
|
|
835
|
+
assert.deepEqual(await runScaleUpCase({ teamName: 'scale-up-unknown-role-default', role: 'unknown-role', launchArgs: '--model policy-model' }), ['-c', 'model_reasoning_effort="medium"', '--model', 'policy-model', bypass]);
|
|
836
|
+
}
|
|
837
|
+
finally {
|
|
838
|
+
process.argv = previousArgv;
|
|
839
|
+
if (typeof previousPath === 'string')
|
|
840
|
+
process.env.PATH = previousPath;
|
|
841
|
+
else
|
|
842
|
+
delete process.env.PATH;
|
|
843
|
+
if (typeof previousBypass === 'string')
|
|
844
|
+
process.env.OMX_BYPASS_DEFAULT_SYSTEM_PROMPT = previousBypass;
|
|
845
|
+
else
|
|
846
|
+
delete process.env.OMX_BYPASS_DEFAULT_SYSTEM_PROMPT;
|
|
847
|
+
await rm(emptyCodexHome, { recursive: true, force: true });
|
|
848
|
+
await rm(cwd, { recursive: true, force: true });
|
|
849
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
850
|
+
}
|
|
851
|
+
});
|
|
852
|
+
it('persists a newly created worker when rollback pane proof is unavailable', async () => {
|
|
574
853
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-rollback-'));
|
|
575
854
|
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-rollback-bin-'));
|
|
576
855
|
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
577
856
|
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
578
857
|
const previousPath = process.env.PATH;
|
|
858
|
+
const previousBridge = process.env.OMX_RUNTIME_BRIDGE;
|
|
579
859
|
try {
|
|
580
860
|
await writeFile(tmuxStubPath, [
|
|
581
861
|
'#!/bin/sh',
|
|
582
862
|
'set -eu',
|
|
583
863
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
584
864
|
'case "${1:-}" in',
|
|
865
|
+
' show-option) echo "team:scale-up-owner-tag-rollback" ;;',
|
|
585
866
|
' -V)',
|
|
586
867
|
' echo "tmux 3.2a"',
|
|
587
868
|
' ;;',
|
|
@@ -597,7 +878,18 @@ describe('scaleUp', () => {
|
|
|
597
878
|
' esac',
|
|
598
879
|
' ;;',
|
|
599
880
|
' list-panes)',
|
|
600
|
-
|
|
881
|
+
` count_file="${join(fakeBinDir, 'proof-count')}"`,
|
|
882
|
+
' count=0',
|
|
883
|
+
' if [ -f "$count_file" ]; then count=$(cat "$count_file"); fi',
|
|
884
|
+
' count=$((count + 1))',
|
|
885
|
+
' printf "%s" "$count" > "$count_file"',
|
|
886
|
+
' if [ "$count" -le 5 ]; then',
|
|
887
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
888
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
889
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'",
|
|
890
|
+
' else',
|
|
891
|
+
" printf 'malformed pane snapshot\\n'",
|
|
892
|
+
' fi',
|
|
601
893
|
' ;;',
|
|
602
894
|
' kill-pane|send-keys|capture-pane)',
|
|
603
895
|
' ;;',
|
|
@@ -608,19 +900,155 @@ describe('scaleUp', () => {
|
|
|
608
900
|
await chmod(tmuxStubPath, 0o755);
|
|
609
901
|
await writeFile(tmuxLogPath, '');
|
|
610
902
|
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
903
|
+
process.env.OMX_RUNTIME_BRIDGE = '0';
|
|
611
904
|
await initTeamState('scale-up-owner-tag-rollback', 'task', 'executor', 1, cwd);
|
|
612
905
|
await configureScaleUpTeamForDirectDispatch('scale-up-owner-tag-rollback', cwd);
|
|
906
|
+
const ownerConfig = await readTeamConfig('scale-up-owner-tag-rollback', cwd);
|
|
907
|
+
assert.ok(ownerConfig);
|
|
908
|
+
if (!ownerConfig)
|
|
909
|
+
return;
|
|
910
|
+
ownerConfig.tmux_pane_owner_id = 'team:scale-up-owner-tag-rollback';
|
|
911
|
+
await saveTeamConfig(ownerConfig, cwd);
|
|
613
912
|
const result = await scaleUp('scale-up-owner-tag-rollback', 1, 'executor', [{ subject: 'new work', description: 'new work', owner: 'worker-2' }], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
614
913
|
assert.equal(result.ok, false);
|
|
615
914
|
if (result.ok)
|
|
616
915
|
return;
|
|
617
|
-
assert.match(result.error, /
|
|
916
|
+
assert.match(result.error, /scale_up_rollback_cleanup_debt:pane_owner_unverified:%31/);
|
|
618
917
|
const config = await readTeamConfig('scale-up-owner-tag-rollback', cwd);
|
|
619
|
-
assert.
|
|
620
|
-
assert.equal(
|
|
918
|
+
assert.deepEqual(config?.workers.map((worker) => worker.name), ['worker-1', 'worker-2']);
|
|
919
|
+
assert.equal(config?.workers[1]?.pane_id, '%31');
|
|
920
|
+
assert.equal(config?.next_worker_index, 3);
|
|
921
|
+
assert.equal((await readTask('scale-up-owner-tag-rollback', '1', cwd))?.owner, 'worker-2');
|
|
922
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-up-owner-tag-rollback', 'workers', 'worker-2', 'identity.json')), false);
|
|
923
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-up-owner-tag-rollback', 'workers', 'worker-2', 'inbox.md')), false);
|
|
924
|
+
const dispatch = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'scale-up-owner-tag-rollback', 'dispatch', 'requests.json'), 'utf8'));
|
|
925
|
+
assert.equal(dispatch.some((request) => request.to_worker === 'worker-2'), false);
|
|
926
|
+
assert.equal((await listDispatchRequests('scale-up-owner-tag-rollback', cwd, { to_worker: 'worker-2' })).length, 0);
|
|
621
927
|
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
622
928
|
assert.ok(tmuxCommands.some((command) => (command === 'set-option -p -t %31 @omx_team_pane_owner_id team:scale-up-owner-tag-rollback')));
|
|
623
|
-
assert.
|
|
929
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %31'), false);
|
|
930
|
+
}
|
|
931
|
+
finally {
|
|
932
|
+
if (typeof previousPath === 'string')
|
|
933
|
+
process.env.PATH = previousPath;
|
|
934
|
+
else
|
|
935
|
+
delete process.env.PATH;
|
|
936
|
+
if (typeof previousBridge === 'string')
|
|
937
|
+
process.env.OMX_RUNTIME_BRIDGE = previousBridge;
|
|
938
|
+
else
|
|
939
|
+
delete process.env.OMX_RUNTIME_BRIDGE;
|
|
940
|
+
await rm(cwd, { recursive: true, force: true });
|
|
941
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
942
|
+
}
|
|
943
|
+
});
|
|
944
|
+
it('retains unpinned rollback debt without killing a pane when its first post-split proof is unavailable', async () => {
|
|
945
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-unpinned-rollback-'));
|
|
946
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-unpinned-rollback-bin-'));
|
|
947
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
948
|
+
const splitPath = join(fakeBinDir, 'split-created');
|
|
949
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
950
|
+
const previousPath = process.env.PATH;
|
|
951
|
+
try {
|
|
952
|
+
await writeFile(tmuxStubPath, [
|
|
953
|
+
'#!/bin/sh',
|
|
954
|
+
'set -eu',
|
|
955
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
956
|
+
'case "${1:-}" in',
|
|
957
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
958
|
+
' show-option) echo "team:scale-up" ;;',
|
|
959
|
+
' split-window) : > "' + splitPath + '"; echo "%31" ;;',
|
|
960
|
+
' list-panes)',
|
|
961
|
+
' if [ -f "' + splitPath + '" ]; then printf "malformed pane snapshot\\n"; else',
|
|
962
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
963
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
964
|
+
' fi',
|
|
965
|
+
' ;;',
|
|
966
|
+
'esac',
|
|
967
|
+
'exit 0',
|
|
968
|
+
'',
|
|
969
|
+
].join('\n'));
|
|
970
|
+
await chmod(tmuxStubPath, 0o755);
|
|
971
|
+
await writeFile(tmuxLogPath, '');
|
|
972
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
973
|
+
const teamName = 'scale-up-unpinned-rollback';
|
|
974
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
975
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
976
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
977
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
978
|
+
});
|
|
979
|
+
assert.equal(result.ok, false);
|
|
980
|
+
if (result.ok)
|
|
981
|
+
return;
|
|
982
|
+
assert.match(result.error, /scale_up_rollback_cleanup_debt:pane_pid_unpinned:%31/);
|
|
983
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
984
|
+
assert.equal(config?.workers[1]?.pane_id, '%31');
|
|
985
|
+
assert.equal(config?.workers[1]?.pid, undefined);
|
|
986
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
987
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %31'), false);
|
|
988
|
+
}
|
|
989
|
+
finally {
|
|
990
|
+
if (typeof previousPath === 'string')
|
|
991
|
+
process.env.PATH = previousPath;
|
|
992
|
+
else
|
|
993
|
+
delete process.env.PATH;
|
|
994
|
+
await rm(cwd, { recursive: true, force: true });
|
|
995
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
996
|
+
}
|
|
997
|
+
});
|
|
998
|
+
it('retains the original pane/PID rollback debt when the new pane ID is reused before owner tagging', async () => {
|
|
999
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-reuse-'));
|
|
1000
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-reuse-bin-'));
|
|
1001
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
1002
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1003
|
+
const previousPath = process.env.PATH;
|
|
1004
|
+
try {
|
|
1005
|
+
await writeFile(tmuxStubPath, [
|
|
1006
|
+
'#!/bin/sh',
|
|
1007
|
+
'set -eu',
|
|
1008
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
1009
|
+
'case "${1:-}" in',
|
|
1010
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1011
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
1012
|
+
' split-window) echo "%31" ;;',
|
|
1013
|
+
' list-panes)',
|
|
1014
|
+
` count_file="${join(fakeBinDir, 'proof-count')}"`,
|
|
1015
|
+
' count=0; [ ! -f "$count_file" ] || count=$(cat "$count_file")',
|
|
1016
|
+
' count=$((count + 1)); printf "%s" "$count" > "$count_file"',
|
|
1017
|
+
' if [ "$count" -le 4 ]; then',
|
|
1018
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
1019
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'",
|
|
1020
|
+
' elif [ "$count" -eq 5 ]; then',
|
|
1021
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
1022
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '52424'",
|
|
1023
|
+
' else',
|
|
1024
|
+
" printf 'malformed pane snapshot\\n'",
|
|
1025
|
+
' fi',
|
|
1026
|
+
' ;;',
|
|
1027
|
+
' kill-pane|send-keys|capture-pane) ;;',
|
|
1028
|
+
'esac',
|
|
1029
|
+
'exit 0',
|
|
1030
|
+
'',
|
|
1031
|
+
].join('\n'));
|
|
1032
|
+
await chmod(tmuxStubPath, 0o755);
|
|
1033
|
+
await writeFile(tmuxLogPath, '');
|
|
1034
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1035
|
+
const teamName = 'scale-up-owner-tag-reuse';
|
|
1036
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
1037
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
1038
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
1039
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
1040
|
+
});
|
|
1041
|
+
assert.equal(result.ok, false);
|
|
1042
|
+
if (result.ok)
|
|
1043
|
+
return;
|
|
1044
|
+
assert.match(result.error, /scale_up_rollback_cleanup_debt:pane_owner_unverified:%31/);
|
|
1045
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
1046
|
+
assert.equal(config?.workers[1]?.pane_id, '%31');
|
|
1047
|
+
assert.equal(config?.workers[1]?.pid, 42424);
|
|
1048
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
1049
|
+
assert.equal(tmuxCommands.some((command) => command.includes('set-option -p -t %31 @omx_team_pane_owner_id')), false);
|
|
1050
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), true);
|
|
1051
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %31'), false);
|
|
624
1052
|
}
|
|
625
1053
|
finally {
|
|
626
1054
|
if (typeof previousPath === 'string')
|
|
@@ -945,6 +1373,7 @@ describe('scaleUp', () => {
|
|
|
945
1373
|
'set -eu',
|
|
946
1374
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
947
1375
|
'case "${1:-}" in',
|
|
1376
|
+
' show-option) echo "team:scale-up" ;;',
|
|
948
1377
|
' -V)',
|
|
949
1378
|
' echo "tmux 3.2a"',
|
|
950
1379
|
' ;;',
|
|
@@ -952,7 +1381,16 @@ describe('scaleUp', () => {
|
|
|
952
1381
|
' echo "%31"',
|
|
953
1382
|
' ;;',
|
|
954
1383
|
' list-panes)',
|
|
955
|
-
'
|
|
1384
|
+
' case "$*" in',
|
|
1385
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
1386
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
1387
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
1388
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'",
|
|
1389
|
+
' ;;',
|
|
1390
|
+
' *)',
|
|
1391
|
+
' echo "42424"',
|
|
1392
|
+
' ;;',
|
|
1393
|
+
' esac',
|
|
956
1394
|
' ;;',
|
|
957
1395
|
' send-keys)',
|
|
958
1396
|
' ;;',
|
|
@@ -997,7 +1435,9 @@ describe('scaleUp', () => {
|
|
|
997
1435
|
return;
|
|
998
1436
|
config.tmux_session = 'omx-team-scale-up-project-reasoning';
|
|
999
1437
|
config.leader_pane_id = '%11';
|
|
1438
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1000
1439
|
config.workers[0].pane_id = '%21';
|
|
1440
|
+
config.workers[0].pid = 42421;
|
|
1001
1441
|
await saveTeamConfig(config, cwd);
|
|
1002
1442
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'scale-up-project-reasoning', 'manifest.v2.json');
|
|
1003
1443
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1051,6 +1491,7 @@ describe('scaleUp', () => {
|
|
|
1051
1491
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1052
1492
|
set -eu
|
|
1053
1493
|
case "\${1:-}" in
|
|
1494
|
+
show-option) echo 'team:scale-up' ;;
|
|
1054
1495
|
-V)
|
|
1055
1496
|
echo "tmux 3.2a"
|
|
1056
1497
|
;;
|
|
@@ -1058,11 +1499,23 @@ set -eu
|
|
|
1058
1499
|
echo "%31"
|
|
1059
1500
|
;;
|
|
1060
1501
|
list-panes)
|
|
1061
|
-
|
|
1502
|
+
case "$*" in
|
|
1503
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1504
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1505
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1506
|
+
if [ ! -f "${fakeBinDir}/killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1507
|
+
;;
|
|
1508
|
+
*)
|
|
1509
|
+
echo "42424"
|
|
1510
|
+
;;
|
|
1511
|
+
esac
|
|
1062
1512
|
;;
|
|
1063
1513
|
send-keys)
|
|
1064
1514
|
exit 1
|
|
1065
1515
|
;;
|
|
1516
|
+
kill-pane)
|
|
1517
|
+
: > "${fakeBinDir}/killed-$3"
|
|
1518
|
+
;;
|
|
1066
1519
|
capture-pane)
|
|
1067
1520
|
echo ""
|
|
1068
1521
|
;;
|
|
@@ -1086,7 +1539,9 @@ exit 0
|
|
|
1086
1539
|
return;
|
|
1087
1540
|
config.tmux_session = 'omx-team-rollback-worktree';
|
|
1088
1541
|
config.leader_pane_id = '%11';
|
|
1542
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1089
1543
|
config.workers[0].pane_id = '%21';
|
|
1544
|
+
config.workers[0].pid = 42421;
|
|
1090
1545
|
await saveTeamConfig(config, cwd);
|
|
1091
1546
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'rollback-worktree', 'manifest.v2.json');
|
|
1092
1547
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1101,7 +1556,7 @@ exit 0
|
|
|
1101
1556
|
return;
|
|
1102
1557
|
assert.match(result.error, /scale_up_dispatch_failed:worker-2/);
|
|
1103
1558
|
const workerRootAgents = join(cwd, '.omx', 'team', 'rollback-worktree', 'worktrees', 'worker-2', 'AGENTS.md');
|
|
1104
|
-
assert.
|
|
1559
|
+
await assert.rejects(readFile(workerRootAgents, 'utf-8'), { code: 'ENOENT' });
|
|
1105
1560
|
const backupPath = join(cwd, '.git', 'worktrees', 'worker-2', 'omx', 'root-agents-backup.json');
|
|
1106
1561
|
assert.equal(existsSync(backupPath), false);
|
|
1107
1562
|
}
|
|
@@ -1123,6 +1578,7 @@ exit 0
|
|
|
1123
1578
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1124
1579
|
set -eu
|
|
1125
1580
|
case "\${1:-}" in
|
|
1581
|
+
show-option) echo 'team:scale-up' ;;
|
|
1126
1582
|
-V)
|
|
1127
1583
|
echo "tmux 3.2a"
|
|
1128
1584
|
;;
|
|
@@ -1130,11 +1586,23 @@ set -eu
|
|
|
1130
1586
|
echo "%31"
|
|
1131
1587
|
;;
|
|
1132
1588
|
list-panes)
|
|
1133
|
-
|
|
1589
|
+
case "$*" in
|
|
1590
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1591
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1592
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1593
|
+
if [ ! -f "${fakeBinDir}/canonical-root-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1594
|
+
;;
|
|
1595
|
+
*)
|
|
1596
|
+
echo "42424"
|
|
1597
|
+
;;
|
|
1598
|
+
esac
|
|
1134
1599
|
;;
|
|
1135
1600
|
capture-pane)
|
|
1136
1601
|
echo ""
|
|
1137
1602
|
;;
|
|
1603
|
+
kill-pane)
|
|
1604
|
+
: > "${fakeBinDir}/canonical-root-killed-$3"
|
|
1605
|
+
;;
|
|
1138
1606
|
esac
|
|
1139
1607
|
exit 0
|
|
1140
1608
|
`);
|
|
@@ -1155,7 +1623,9 @@ exit 0
|
|
|
1155
1623
|
return;
|
|
1156
1624
|
config.tmux_session = 'omx-team-canonical-root';
|
|
1157
1625
|
config.leader_pane_id = '%11';
|
|
1626
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1158
1627
|
config.workers[0].pane_id = '%21';
|
|
1628
|
+
config.workers[0].pid = 42421;
|
|
1159
1629
|
await saveTeamConfig(config, cwd);
|
|
1160
1630
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'canonical-root', 'manifest.v2.json');
|
|
1161
1631
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1199,6 +1669,7 @@ exit 0
|
|
|
1199
1669
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1200
1670
|
set -eu
|
|
1201
1671
|
case "\${1:-}" in
|
|
1672
|
+
show-option) echo 'team:scale-up' ;;
|
|
1202
1673
|
-V)
|
|
1203
1674
|
echo "tmux 3.2a"
|
|
1204
1675
|
;;
|
|
@@ -1206,11 +1677,23 @@ case "\${1:-}" in
|
|
|
1206
1677
|
echo "%31"
|
|
1207
1678
|
;;
|
|
1208
1679
|
list-panes)
|
|
1209
|
-
|
|
1680
|
+
case "$*" in
|
|
1681
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1682
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1683
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1684
|
+
if [ ! -f "${fakeBinDir}/frontier-role-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1685
|
+
;;
|
|
1686
|
+
*)
|
|
1687
|
+
echo "42424"
|
|
1688
|
+
;;
|
|
1689
|
+
esac
|
|
1210
1690
|
;;
|
|
1211
1691
|
capture-pane)
|
|
1212
1692
|
echo ""
|
|
1213
1693
|
;;
|
|
1694
|
+
kill-pane)
|
|
1695
|
+
: > "${fakeBinDir}/frontier-role-killed-$3"
|
|
1696
|
+
;;
|
|
1214
1697
|
esac
|
|
1215
1698
|
exit 0
|
|
1216
1699
|
`);
|
|
@@ -1233,7 +1716,9 @@ exit 0
|
|
|
1233
1716
|
return;
|
|
1234
1717
|
config.tmux_session = 'omx-team-frontier-role';
|
|
1235
1718
|
config.leader_pane_id = '%11';
|
|
1719
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1236
1720
|
config.workers[0].pane_id = '%21';
|
|
1721
|
+
config.workers[0].pid = 42421;
|
|
1237
1722
|
await saveTeamConfig(config, cwd);
|
|
1238
1723
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'frontier-role', 'manifest.v2.json');
|
|
1239
1724
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1269,6 +1754,7 @@ exit 0
|
|
|
1269
1754
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1270
1755
|
set -eu
|
|
1271
1756
|
case "\${1:-}" in
|
|
1757
|
+
show-option) echo 'team:scale-up' ;;
|
|
1272
1758
|
-V)
|
|
1273
1759
|
echo "tmux 3.2a"
|
|
1274
1760
|
;;
|
|
@@ -1276,11 +1762,23 @@ case "\${1:-}" in
|
|
|
1276
1762
|
echo "%31"
|
|
1277
1763
|
;;
|
|
1278
1764
|
list-panes)
|
|
1279
|
-
|
|
1765
|
+
case "$*" in
|
|
1766
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1767
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1768
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1769
|
+
if [ ! -f "${fakeBinDir}/mini-tuned-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1770
|
+
;;
|
|
1771
|
+
*)
|
|
1772
|
+
echo "42424"
|
|
1773
|
+
;;
|
|
1774
|
+
esac
|
|
1280
1775
|
;;
|
|
1281
1776
|
capture-pane)
|
|
1282
1777
|
echo ""
|
|
1283
1778
|
;;
|
|
1779
|
+
kill-pane)
|
|
1780
|
+
: > "${fakeBinDir}/mini-tuned-killed-$3"
|
|
1781
|
+
;;
|
|
1284
1782
|
esac
|
|
1285
1783
|
exit 0
|
|
1286
1784
|
`);
|
|
@@ -1301,7 +1799,9 @@ exit 0
|
|
|
1301
1799
|
return;
|
|
1302
1800
|
config.tmux_session = 'omx-team-mini-tuned-root';
|
|
1303
1801
|
config.leader_pane_id = '%11';
|
|
1802
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1304
1803
|
config.workers[0].pane_id = '%21';
|
|
1804
|
+
config.workers[0].pid = 42421;
|
|
1305
1805
|
await saveTeamConfig(config, cwd);
|
|
1306
1806
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'mini-tuned-root', 'manifest.v2.json');
|
|
1307
1807
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1343,6 +1843,7 @@ exit 0
|
|
|
1343
1843
|
set -eu
|
|
1344
1844
|
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
1345
1845
|
case "\${1:-}" in
|
|
1846
|
+
show-option) echo 'team:scale-up' ;;
|
|
1346
1847
|
-V)
|
|
1347
1848
|
echo "tmux 3.2a"
|
|
1348
1849
|
;;
|
|
@@ -1350,11 +1851,23 @@ case "\${1:-}" in
|
|
|
1350
1851
|
echo "%31"
|
|
1351
1852
|
;;
|
|
1352
1853
|
list-panes)
|
|
1353
|
-
|
|
1854
|
+
case "$*" in
|
|
1855
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1856
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1857
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1858
|
+
if [ ! -f "${fakeBinDir}/layout-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1859
|
+
;;
|
|
1860
|
+
*)
|
|
1861
|
+
echo "42424"
|
|
1862
|
+
;;
|
|
1863
|
+
esac
|
|
1354
1864
|
;;
|
|
1355
1865
|
capture-pane)
|
|
1356
1866
|
echo ""
|
|
1357
1867
|
;;
|
|
1868
|
+
kill-pane)
|
|
1869
|
+
: > "${fakeBinDir}/layout-killed-$3"
|
|
1870
|
+
;;
|
|
1358
1871
|
esac
|
|
1359
1872
|
exit 0
|
|
1360
1873
|
`);
|
|
@@ -1368,7 +1881,9 @@ exit 0
|
|
|
1368
1881
|
return;
|
|
1369
1882
|
config.tmux_session = 'omx-team-scale-up-layout';
|
|
1370
1883
|
config.leader_pane_id = '%11';
|
|
1884
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1371
1885
|
config.workers[0].pane_id = '%21';
|
|
1886
|
+
config.workers[0].pid = 42421;
|
|
1372
1887
|
await saveTeamConfig(config, cwd);
|
|
1373
1888
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'scale-up-layout', 'manifest.v2.json');
|
|
1374
1889
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1381,9 +1896,14 @@ exit 0
|
|
|
1381
1896
|
assert.equal(result.ok, true);
|
|
1382
1897
|
if (!result.ok)
|
|
1383
1898
|
return;
|
|
1384
|
-
const
|
|
1385
|
-
|
|
1386
|
-
assert.
|
|
1899
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
1900
|
+
const splitWindowIndex = tmuxCommands.findIndex((command) => command.startsWith('split-window '));
|
|
1901
|
+
assert.ok(splitWindowIndex > 2);
|
|
1902
|
+
assert.match(tmuxCommands[splitWindowIndex - 3], /^list-panes -a -F #\{pane_id\}\t#\{pane_dead\}\t#\{pane_pid\}$/);
|
|
1903
|
+
assert.equal(tmuxCommands[splitWindowIndex - 2], 'show-option -qv -p -t %21 @omx_team_pane_owner_id');
|
|
1904
|
+
assert.match(tmuxCommands[splitWindowIndex - 1], /^list-panes -a -F #\{pane_id\}\t#\{pane_dead\}\t#\{pane_pid\}$/);
|
|
1905
|
+
assert.match(tmuxCommands[splitWindowIndex], /split-window -v -t %21/);
|
|
1906
|
+
assert.doesNotMatch(tmuxCommands.join('\n'), /select-layout .*tiled/);
|
|
1387
1907
|
}
|
|
1388
1908
|
finally {
|
|
1389
1909
|
if (typeof previousPath === 'string')
|
|
@@ -1394,81 +1914,430 @@ exit 0
|
|
|
1394
1914
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1395
1915
|
}
|
|
1396
1916
|
});
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1917
|
+
for (const [name, proofOutput, proofExitStatus, expectedReason] of [
|
|
1918
|
+
['dead persisted worker pane', '%21\t1\t42421\n', 0, 'dead'],
|
|
1919
|
+
['similar persisted worker pane ID', '%210\t0\t424210\n', 0, 'absent'],
|
|
1920
|
+
['malformed global pane PID', '%21\t0\tnot-a-pid\n', 0, 'malformed_snapshot'],
|
|
1921
|
+
['duplicate global pane proof rows', '%21\t0\t42421\n%21\t0\t42422\n', 0, 'malformed_snapshot'],
|
|
1922
|
+
['unavailable global pane query', '', 1, 'query_failed'],
|
|
1923
|
+
]) {
|
|
1924
|
+
it(`fails closed before split-window when exact proof has ${name}`, async () => {
|
|
1925
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-proof-'));
|
|
1926
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-proof-bin-'));
|
|
1927
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
1928
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1929
|
+
const previousPath = process.env.PATH;
|
|
1930
|
+
try {
|
|
1931
|
+
await writeFile(tmuxStubPath, [
|
|
1932
|
+
'#!/bin/sh',
|
|
1933
|
+
'set -eu',
|
|
1934
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
1935
|
+
'case "${1:-}" in',
|
|
1936
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1937
|
+
' -V)',
|
|
1938
|
+
' echo "tmux 3.2a"',
|
|
1939
|
+
' ;;',
|
|
1940
|
+
' list-panes)',
|
|
1941
|
+
` printf '%b' ${JSON.stringify(proofOutput)}`,
|
|
1942
|
+
proofExitStatus === 0 ? ' ;;' : ' exit 1',
|
|
1943
|
+
proofExitStatus === 0 ? '' : ' ;;',
|
|
1944
|
+
' split-window)',
|
|
1945
|
+
' echo "%31"',
|
|
1946
|
+
' ;;',
|
|
1947
|
+
'esac',
|
|
1948
|
+
'exit 0',
|
|
1949
|
+
'',
|
|
1950
|
+
].join('\n'));
|
|
1951
|
+
await chmod(tmuxStubPath, 0o755);
|
|
1952
|
+
await writeFile(tmuxLogPath, '');
|
|
1953
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1954
|
+
const teamName = `split-proof-${expectedReason === 'malformed_snapshot' ? name.includes('duplicate') ? 'duplicate' : 'malformed' : expectedReason.replace('_', '-')}`;
|
|
1955
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
1956
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
1957
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1958
|
+
assert.equal(result.ok, false);
|
|
1959
|
+
if (!result.ok) {
|
|
1960
|
+
assert.equal(result.error, `scale_up_split_target_proof_unavailable:%21:${expectedReason}`);
|
|
1961
|
+
}
|
|
1962
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
1963
|
+
assert.equal(config?.workers.length, 1);
|
|
1964
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
1965
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
1966
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
1967
|
+
}
|
|
1968
|
+
finally {
|
|
1969
|
+
if (typeof previousPath === 'string')
|
|
1970
|
+
process.env.PATH = previousPath;
|
|
1971
|
+
else
|
|
1972
|
+
delete process.env.PATH;
|
|
1973
|
+
await rm(cwd, { recursive: true, force: true });
|
|
1974
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1975
|
+
}
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
1978
|
+
it('migrates a legacy worker PID from an exact owner-bound proof before split-window', async () => {
|
|
1979
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-target-pid-'));
|
|
1980
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-target-pid-bin-'));
|
|
1400
1981
|
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
1401
1982
|
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1402
1983
|
const previousPath = process.env.PATH;
|
|
1403
1984
|
try {
|
|
1404
|
-
await
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
'case "${1:-}" in',
|
|
1409
|
-
' -V)',
|
|
1410
|
-
' echo "tmux 3.2a"',
|
|
1411
|
-
' ;;',
|
|
1412
|
-
' split-window)',
|
|
1413
|
-
' echo "%41"',
|
|
1414
|
-
' ;;',
|
|
1415
|
-
' list-panes)',
|
|
1416
|
-
' echo "45454"',
|
|
1417
|
-
' ;;',
|
|
1418
|
-
' capture-pane)',
|
|
1419
|
-
' echo ""',
|
|
1420
|
-
' ;;',
|
|
1421
|
-
'esac',
|
|
1422
|
-
'exit 0',
|
|
1423
|
-
'',
|
|
1424
|
-
].join('\n'));
|
|
1425
|
-
await chmod(tmuxStubPath, 0o755);
|
|
1426
|
-
await writeFile(tmuxLogPath, '');
|
|
1985
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
1986
|
+
const stub = await readFile(tmuxStubPath, 'utf-8');
|
|
1987
|
+
const configPath = join(cwd, '.omx', 'state', 'team', 'legacy-target-pid', 'config.json');
|
|
1988
|
+
await writeFile(tmuxStubPath, stub.replace(' split-window)', ` split-window)\n grep -q '"pid": 42421' "${configPath}" || exit 23`));
|
|
1427
1989
|
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1428
|
-
const teamName = '
|
|
1429
|
-
await
|
|
1430
|
-
await
|
|
1431
|
-
await
|
|
1432
|
-
leader_cwd: repo,
|
|
1433
|
-
team_state_root: join(repo, '.omx', 'state'),
|
|
1434
|
-
workspace_mode: 'worktree',
|
|
1435
|
-
worktree_mode: { enabled: true, detached: true, name: null },
|
|
1436
|
-
});
|
|
1437
|
-
const config = await readTeamConfig(teamName, repo);
|
|
1990
|
+
const teamName = 'legacy-target-pid';
|
|
1991
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
1992
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
1993
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
1438
1994
|
assert.ok(config);
|
|
1439
1995
|
if (!config)
|
|
1440
1996
|
return;
|
|
1441
|
-
config.
|
|
1442
|
-
config
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
1447
|
-
manifest.policy = {
|
|
1448
|
-
...(manifest.policy ?? {}),
|
|
1449
|
-
dispatch_mode: 'transport_direct',
|
|
1450
|
-
};
|
|
1451
|
-
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
1452
|
-
const result = await scaleUp(teamName, 1, 'executor', [], repo, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1997
|
+
delete config.workers[0].pid;
|
|
1998
|
+
await saveTeamConfig(config, cwd);
|
|
1999
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
2000
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2001
|
+
});
|
|
1453
2002
|
assert.equal(result.ok, true);
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
const worker = updated?.workers.find((entry) => entry.name === 'worker-2');
|
|
1458
|
-
assert.deepEqual(updated?.worktree_mode, { enabled: true, detached: true, name: null });
|
|
1459
|
-
assert.ok(worker?.worktree_path, 'scaled worker should have detached worktree path');
|
|
1460
|
-
assert.equal(worker?.working_dir, worker?.worktree_path);
|
|
1461
|
-
assert.equal(worker?.worktree_detached, true);
|
|
1462
|
-
assert.equal(worker?.worktree_created, true);
|
|
1463
|
-
assert.equal(existsSync(worker?.worktree_path), true);
|
|
1464
|
-
assert.throws(() => execFileSync('git', ['symbolic-ref', '--quiet', '--short', 'HEAD'], { cwd: worker?.worktree_path, stdio: 'pipe' }));
|
|
2003
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers[0]?.pid, 42421);
|
|
2004
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2005
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window ')));
|
|
1465
2006
|
}
|
|
1466
2007
|
finally {
|
|
1467
2008
|
if (typeof previousPath === 'string')
|
|
1468
2009
|
process.env.PATH = previousPath;
|
|
1469
2010
|
else
|
|
1470
2011
|
delete process.env.PATH;
|
|
1471
|
-
await rm(
|
|
2012
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2013
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2014
|
+
}
|
|
2015
|
+
});
|
|
2016
|
+
for (const [name, proofOutput, owner, expectedError] of [
|
|
2017
|
+
['malformed proof', '%21\\t0\\tnot-a-pid\\n', 'team:scale-up', 'scale_up_split_target_proof_unavailable:%21:malformed_snapshot'],
|
|
2018
|
+
['foreign pane', '%210\\t0\\t424210\\n', 'team:scale-up', 'scale_up_split_target_proof_unavailable:%21:absent'],
|
|
2019
|
+
['foreign owner tag', '%21\\t0\\t42421\\n', 'team:foreign', 'scale_up_split_target_owner_changed:%21'],
|
|
2020
|
+
]) {
|
|
2021
|
+
it(`rejects legacy PID migration with ${name} before a scale effect`, async () => {
|
|
2022
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-proof-reject-'));
|
|
2023
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-proof-reject-bin-'));
|
|
2024
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2025
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2026
|
+
const previousPath = process.env.PATH;
|
|
2027
|
+
try {
|
|
2028
|
+
await writeFile(tmuxStubPath, [
|
|
2029
|
+
'#!/bin/sh', 'set -eu', `printf '%s\\n' "$*" >> "${tmuxLogPath}"`, 'case "${1:-}" in',
|
|
2030
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
2031
|
+
` list-panes) printf '%b' ${JSON.stringify(proofOutput)} ;;`,
|
|
2032
|
+
` show-option) echo ${JSON.stringify(owner)} ;;`,
|
|
2033
|
+
'esac', 'exit 0', '',
|
|
2034
|
+
].join('\n'));
|
|
2035
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2036
|
+
await writeFile(tmuxLogPath, '');
|
|
2037
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2038
|
+
const teamName = `legacy-proof-${name.replaceAll(' ', '-')}`;
|
|
2039
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2040
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2041
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
2042
|
+
assert.ok(config);
|
|
2043
|
+
if (!config)
|
|
2044
|
+
return;
|
|
2045
|
+
delete config.workers[0].pid;
|
|
2046
|
+
await saveTeamConfig(config, cwd);
|
|
2047
|
+
assert.deepEqual(await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
2048
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2049
|
+
}), { ok: false, error: expectedError });
|
|
2050
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers[0]?.pid, undefined);
|
|
2051
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2052
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ') || command.startsWith('set-option ')), false);
|
|
2053
|
+
}
|
|
2054
|
+
finally {
|
|
2055
|
+
if (typeof previousPath === 'string')
|
|
2056
|
+
process.env.PATH = previousPath;
|
|
2057
|
+
else
|
|
2058
|
+
delete process.env.PATH;
|
|
2059
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2060
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2061
|
+
}
|
|
2062
|
+
});
|
|
2063
|
+
}
|
|
2064
|
+
it('rejects a PID-reused explicit split target at the first proof before any split or owner tag', async () => {
|
|
2065
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-first-proof-reuse-'));
|
|
2066
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-first-proof-reuse-bin-'));
|
|
2067
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2068
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2069
|
+
const previousPath = process.env.PATH;
|
|
2070
|
+
try {
|
|
2071
|
+
await writeFile(tmuxStubPath, [
|
|
2072
|
+
'#!/bin/sh', 'set -eu', `printf '%s\\n' "$*" >> "${tmuxLogPath}"`, 'case "${1:-}" in', ' show-option) echo "team:scale-up" ;;',
|
|
2073
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2074
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
2075
|
+
" list-panes) printf '%s\\t%s\\t%s\\n' '%21' '0' '52421' ;;",
|
|
2076
|
+
'esac', 'exit 0', '',
|
|
2077
|
+
].join('\n'));
|
|
2078
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2079
|
+
await writeFile(tmuxLogPath, '');
|
|
2080
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2081
|
+
const teamName = 'first-proof-reuse';
|
|
2082
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2083
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2084
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2085
|
+
assert.deepEqual(result, { ok: false, error: 'scale_up_split_target_pid_changed:%21:42421:52421' });
|
|
2086
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2087
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ') || command.startsWith('set-option ')), false);
|
|
2088
|
+
}
|
|
2089
|
+
finally {
|
|
2090
|
+
if (typeof previousPath === 'string')
|
|
2091
|
+
process.env.PATH = previousPath;
|
|
2092
|
+
else
|
|
2093
|
+
delete process.env.PATH;
|
|
2094
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2095
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2096
|
+
}
|
|
2097
|
+
});
|
|
2098
|
+
it('rolls back all preparation when the split target proof is lost immediately before split-window', async () => {
|
|
2099
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-second-split-proof-'));
|
|
2100
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-second-split-proof-bin-'));
|
|
2101
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2102
|
+
const proofCountPath = join(fakeBinDir, 'proof-count');
|
|
2103
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2104
|
+
const previousPath = process.env.PATH;
|
|
2105
|
+
try {
|
|
2106
|
+
await writeFile(tmuxStubPath, [
|
|
2107
|
+
'#!/bin/sh',
|
|
2108
|
+
'set -eu',
|
|
2109
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
2110
|
+
'case "${1:-}" in',
|
|
2111
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2112
|
+
' -V)',
|
|
2113
|
+
' echo "tmux 3.2a"',
|
|
2114
|
+
' ;;',
|
|
2115
|
+
' list-panes)',
|
|
2116
|
+
` count=0; [ ! -f "${proofCountPath}" ] || count=$(cat "${proofCountPath}")`,
|
|
2117
|
+
' count=$((count + 1))',
|
|
2118
|
+
` printf '%s' "$count" > "${proofCountPath}"`,
|
|
2119
|
+
' if [ "$count" -eq 1 ]; then',
|
|
2120
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
2121
|
+
' else',
|
|
2122
|
+
' exit 1',
|
|
2123
|
+
' fi',
|
|
2124
|
+
' ;;',
|
|
2125
|
+
' split-window)',
|
|
2126
|
+
' echo "%31"',
|
|
2127
|
+
' ;;',
|
|
2128
|
+
'esac',
|
|
2129
|
+
'exit 0',
|
|
2130
|
+
'',
|
|
2131
|
+
].join('\n'));
|
|
2132
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2133
|
+
await writeFile(tmuxLogPath, '');
|
|
2134
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2135
|
+
const teamName = 'second-split-proof';
|
|
2136
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2137
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2138
|
+
const teamStateDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
2139
|
+
const configPath = join(teamStateDir, 'config.json');
|
|
2140
|
+
const tasksDir = join(teamStateDir, 'tasks');
|
|
2141
|
+
const workersDir = join(teamStateDir, 'workers');
|
|
2142
|
+
const runtimeDir = join(teamStateDir, 'runtime');
|
|
2143
|
+
const configBefore = await readFile(configPath, 'utf-8');
|
|
2144
|
+
const taskEntriesBefore = (await readdir(tasksDir)).sort();
|
|
2145
|
+
const workerEntriesBefore = (await readdir(workersDir)).sort();
|
|
2146
|
+
const runtimeDirExistedBefore = existsSync(runtimeDir);
|
|
2147
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'must not persist', description: 'proof loss rollback', owner: 'worker-2' }], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2148
|
+
assert.deepEqual(result, {
|
|
2149
|
+
ok: false,
|
|
2150
|
+
error: 'scale_up_split_target_proof_unavailable:%21:query_failed',
|
|
2151
|
+
});
|
|
2152
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2153
|
+
assert.deepEqual(tmuxCommands.filter((command) => command.startsWith('list-panes ')), [
|
|
2154
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2155
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2156
|
+
]);
|
|
2157
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
2158
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
2159
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(configPath, 'utf-8')), withoutConfigGeneration(configBefore));
|
|
2160
|
+
assert.deepEqual((await readdir(tasksDir)).sort(), taskEntriesBefore);
|
|
2161
|
+
assert.deepEqual((await readdir(workersDir)).sort(), workerEntriesBefore);
|
|
2162
|
+
assert.equal(existsSync(join(workersDir, 'worker-2')), false);
|
|
2163
|
+
assert.equal(existsSync(join(cwd, '.omx', 'team', teamName, 'worktrees', 'worker-2')), false);
|
|
2164
|
+
assert.equal(existsSync(runtimeDir), runtimeDirExistedBefore);
|
|
2165
|
+
}
|
|
2166
|
+
finally {
|
|
2167
|
+
if (typeof previousPath === 'string')
|
|
2168
|
+
process.env.PATH = previousPath;
|
|
2169
|
+
else
|
|
2170
|
+
delete process.env.PATH;
|
|
2171
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2172
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2173
|
+
}
|
|
2174
|
+
});
|
|
2175
|
+
it('rejects a split target ID replacement that occurs during owner authorization', async () => {
|
|
2176
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-target-pid-reuse-'));
|
|
2177
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-target-pid-reuse-bin-'));
|
|
2178
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2179
|
+
const proofCountPath = join(fakeBinDir, 'proof-count');
|
|
2180
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2181
|
+
const previousPath = process.env.PATH;
|
|
2182
|
+
try {
|
|
2183
|
+
await writeFile(tmuxStubPath, [
|
|
2184
|
+
'#!/bin/sh',
|
|
2185
|
+
'set -eu',
|
|
2186
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
2187
|
+
'case "${1:-}" in',
|
|
2188
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2189
|
+
' -V)',
|
|
2190
|
+
' echo "tmux 3.2a"',
|
|
2191
|
+
' ;;',
|
|
2192
|
+
' list-panes)',
|
|
2193
|
+
` count=0; [ ! -f "${proofCountPath}" ] || count=$(cat "${proofCountPath}")`,
|
|
2194
|
+
' count=$((count + 1))',
|
|
2195
|
+
` printf '%s' "$count" > "${proofCountPath}"`,
|
|
2196
|
+
' if [ "$count" -eq 1 ]; then',
|
|
2197
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
2198
|
+
' else',
|
|
2199
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '52421'",
|
|
2200
|
+
' fi',
|
|
2201
|
+
' ;;',
|
|
2202
|
+
' split-window)',
|
|
2203
|
+
' echo "%31"',
|
|
2204
|
+
' ;;',
|
|
2205
|
+
'esac',
|
|
2206
|
+
'exit 0',
|
|
2207
|
+
'',
|
|
2208
|
+
].join('\n'));
|
|
2209
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2210
|
+
await writeFile(tmuxLogPath, '');
|
|
2211
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2212
|
+
const teamName = 'split-target-pid-reuse';
|
|
2213
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2214
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2215
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
2216
|
+
assert.ok(config);
|
|
2217
|
+
if (!config)
|
|
2218
|
+
return;
|
|
2219
|
+
config.workers[0].pid = 42421;
|
|
2220
|
+
await saveTeamConfig(config, cwd);
|
|
2221
|
+
const teamStateDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
2222
|
+
const configPath = join(teamStateDir, 'config.json');
|
|
2223
|
+
const tasksDir = join(teamStateDir, 'tasks');
|
|
2224
|
+
const workersDir = join(teamStateDir, 'workers');
|
|
2225
|
+
const runtimeDir = join(teamStateDir, 'runtime');
|
|
2226
|
+
const configBefore = await readFile(configPath, 'utf-8');
|
|
2227
|
+
const taskEntriesBefore = (await readdir(tasksDir)).sort();
|
|
2228
|
+
const workerEntriesBefore = (await readdir(workersDir)).sort();
|
|
2229
|
+
const runtimeDirExistedBefore = existsSync(runtimeDir);
|
|
2230
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'must not persist', description: 'PID reuse rollback', owner: 'worker-2' }], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2231
|
+
assert.deepEqual(result, {
|
|
2232
|
+
ok: false,
|
|
2233
|
+
error: 'scale_up_split_target_pid_changed:%21:42421:52421',
|
|
2234
|
+
});
|
|
2235
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2236
|
+
assert.equal(tmuxCommands.filter((command) => command.startsWith('list-panes ')).length, 2);
|
|
2237
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
2238
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
2239
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(configPath, 'utf-8')), withoutConfigGeneration(configBefore));
|
|
2240
|
+
assert.deepEqual((await readdir(tasksDir)).sort(), taskEntriesBefore);
|
|
2241
|
+
assert.deepEqual((await readdir(workersDir)).sort(), workerEntriesBefore);
|
|
2242
|
+
assert.equal(existsSync(join(workersDir, 'worker-2')), false);
|
|
2243
|
+
assert.equal(existsSync(runtimeDir), runtimeDirExistedBefore);
|
|
2244
|
+
}
|
|
2245
|
+
finally {
|
|
2246
|
+
if (typeof previousPath === 'string')
|
|
2247
|
+
process.env.PATH = previousPath;
|
|
2248
|
+
else
|
|
2249
|
+
delete process.env.PATH;
|
|
2250
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2251
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2252
|
+
}
|
|
2253
|
+
});
|
|
2254
|
+
it('provisions detached worktrees for scaled-up workers from persisted team worktree mode', async () => {
|
|
2255
|
+
const repo = await initRepo();
|
|
2256
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-detached-bin-'));
|
|
2257
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2258
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2259
|
+
const previousPath = process.env.PATH;
|
|
2260
|
+
try {
|
|
2261
|
+
await writeFile(tmuxStubPath, [
|
|
2262
|
+
'#!/bin/sh',
|
|
2263
|
+
'set -eu',
|
|
2264
|
+
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
2265
|
+
'case "${1:-}" in',
|
|
2266
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2267
|
+
' -V)',
|
|
2268
|
+
' echo "tmux 3.2a"',
|
|
2269
|
+
' ;;',
|
|
2270
|
+
' split-window)',
|
|
2271
|
+
' echo "%41"',
|
|
2272
|
+
' ;;',
|
|
2273
|
+
' list-panes)',
|
|
2274
|
+
' case "$*" in',
|
|
2275
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
2276
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '45451'",
|
|
2277
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '45452'",
|
|
2278
|
+
" printf '%s\\t%s\\t%s\\n' '%41' '0' '45454'",
|
|
2279
|
+
' ;;',
|
|
2280
|
+
' *)',
|
|
2281
|
+
' echo "45454"',
|
|
2282
|
+
' ;;',
|
|
2283
|
+
' esac',
|
|
2284
|
+
' ;;',
|
|
2285
|
+
' capture-pane)',
|
|
2286
|
+
' echo ""',
|
|
2287
|
+
' ;;',
|
|
2288
|
+
'esac',
|
|
2289
|
+
'exit 0',
|
|
2290
|
+
'',
|
|
2291
|
+
].join('\n'));
|
|
2292
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2293
|
+
await writeFile(tmuxLogPath, '');
|
|
2294
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2295
|
+
const teamName = 'scale-up-detached-worktree';
|
|
2296
|
+
await mkdir(join(repo, '.omx', 'state', 'team', teamName), { recursive: true });
|
|
2297
|
+
await writeFile(join(repo, '.omx', 'state', 'team', teamName, 'worker-agents.md'), '# Base worker instructions\n');
|
|
2298
|
+
await initTeamState(teamName, 'task', 'executor', 1, repo, DEFAULT_MAX_WORKERS, process.env, {
|
|
2299
|
+
leader_cwd: repo,
|
|
2300
|
+
team_state_root: join(repo, '.omx', 'state'),
|
|
2301
|
+
workspace_mode: 'worktree',
|
|
2302
|
+
worktree_mode: { enabled: true, detached: true, name: null },
|
|
2303
|
+
});
|
|
2304
|
+
const config = await readTeamConfig(teamName, repo);
|
|
2305
|
+
assert.ok(config);
|
|
2306
|
+
if (!config)
|
|
2307
|
+
return;
|
|
2308
|
+
config.tmux_session = `omx-team-${teamName}`;
|
|
2309
|
+
config.leader_pane_id = '%11';
|
|
2310
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
2311
|
+
config.workers[0].pane_id = '%21';
|
|
2312
|
+
config.workers[0].pid = 45452;
|
|
2313
|
+
await saveTeamConfig(config, repo);
|
|
2314
|
+
const manifestPath = join(repo, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
2315
|
+
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
2316
|
+
manifest.policy = {
|
|
2317
|
+
...(manifest.policy ?? {}),
|
|
2318
|
+
dispatch_mode: 'transport_direct',
|
|
2319
|
+
};
|
|
2320
|
+
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
2321
|
+
const result = await scaleUp(teamName, 1, 'executor', [], repo, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2322
|
+
assert.equal(result.ok, true);
|
|
2323
|
+
if (!result.ok)
|
|
2324
|
+
return;
|
|
2325
|
+
const updated = await readTeamConfig(teamName, repo);
|
|
2326
|
+
const worker = updated?.workers.find((entry) => entry.name === 'worker-2');
|
|
2327
|
+
assert.deepEqual(updated?.worktree_mode, { enabled: true, detached: true, name: null });
|
|
2328
|
+
assert.ok(worker?.worktree_path, 'scaled worker should have detached worktree path');
|
|
2329
|
+
assert.equal(worker?.working_dir, worker?.worktree_path);
|
|
2330
|
+
assert.equal(worker?.worktree_detached, true);
|
|
2331
|
+
assert.equal(worker?.worktree_created, true);
|
|
2332
|
+
assert.equal(existsSync(worker?.worktree_path), true);
|
|
2333
|
+
assert.throws(() => execFileSync('git', ['symbolic-ref', '--quiet', '--short', 'HEAD'], { cwd: worker?.worktree_path, stdio: 'pipe' }));
|
|
2334
|
+
}
|
|
2335
|
+
finally {
|
|
2336
|
+
if (typeof previousPath === 'string')
|
|
2337
|
+
process.env.PATH = previousPath;
|
|
2338
|
+
else
|
|
2339
|
+
delete process.env.PATH;
|
|
2340
|
+
await rm(repo, { recursive: true, force: true });
|
|
1472
2341
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1473
2342
|
}
|
|
1474
2343
|
});
|
|
@@ -1484,6 +2353,7 @@ exit 0
|
|
|
1484
2353
|
'set -eu',
|
|
1485
2354
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
1486
2355
|
'case "${1:-}" in',
|
|
2356
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1487
2357
|
' -V)',
|
|
1488
2358
|
' echo "tmux 3.2a"',
|
|
1489
2359
|
' ;;',
|
|
@@ -1491,7 +2361,16 @@ exit 0
|
|
|
1491
2361
|
' echo "%42"',
|
|
1492
2362
|
' ;;',
|
|
1493
2363
|
' list-panes)',
|
|
1494
|
-
'
|
|
2364
|
+
' case "$*" in',
|
|
2365
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
2366
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '46461'",
|
|
2367
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '46462'",
|
|
2368
|
+
" printf '%s\\t%s\\t%s\\n' '%42' '0' '46464'",
|
|
2369
|
+
' ;;',
|
|
2370
|
+
' *)',
|
|
2371
|
+
' echo "46464"',
|
|
2372
|
+
' ;;',
|
|
2373
|
+
' esac',
|
|
1495
2374
|
' ;;',
|
|
1496
2375
|
' capture-pane)',
|
|
1497
2376
|
' echo ""',
|
|
@@ -1519,7 +2398,9 @@ exit 0
|
|
|
1519
2398
|
return;
|
|
1520
2399
|
config.tmux_session = `omx-team-${teamName}`;
|
|
1521
2400
|
config.leader_pane_id = '%11';
|
|
2401
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1522
2402
|
config.workers[0].pane_id = '%21';
|
|
2403
|
+
config.workers[0].pid = 46462;
|
|
1523
2404
|
await saveTeamConfig(config, repo);
|
|
1524
2405
|
const manifestPath = join(repo, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
1525
2406
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1551,40 +2432,347 @@ exit 0
|
|
|
1551
2432
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1552
2433
|
}
|
|
1553
2434
|
});
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
const
|
|
2435
|
+
it('rolls back pre-split worker preparation when worktree-root instructions cannot materialize', async () => {
|
|
2436
|
+
const cwd = await initRepo();
|
|
2437
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-pre-split-root-failure-bin-'));
|
|
2438
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2439
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2440
|
+
const previousPath = process.env.PATH;
|
|
2441
|
+
const teamName = 'pre-split-root-failure';
|
|
2442
|
+
const worktreePath = join(cwd, '.omx', 'team', teamName, 'worktrees', 'worker-2');
|
|
1561
2443
|
try {
|
|
1562
|
-
|
|
2444
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2445
|
+
set -eu
|
|
2446
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
2447
|
+
case "\${1:-}" in
|
|
2448
|
+
show-option) echo 'team:scale-up' ;;
|
|
2449
|
+
-V) echo 'tmux 3.2a' ;;
|
|
2450
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%21' '0' '42421' ;;
|
|
2451
|
+
esac
|
|
2452
|
+
`);
|
|
2453
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2454
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2455
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2456
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2457
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
2458
|
+
assert.ok(config);
|
|
2459
|
+
if (!config)
|
|
2460
|
+
return;
|
|
2461
|
+
config.tmux_session = `omx-team-${teamName}`;
|
|
2462
|
+
config.workers[0].pane_id = '%21';
|
|
2463
|
+
config.workspace_mode = 'worktree';
|
|
2464
|
+
config.worktree_mode = { enabled: true, detached: true, name: null };
|
|
2465
|
+
await saveTeamConfig(config, cwd);
|
|
2466
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktreePath], { cwd, stdio: 'ignore' });
|
|
2467
|
+
await mkdir(join(worktreePath, 'AGENTS.md'));
|
|
2468
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1563
2469
|
assert.equal(result.ok, false);
|
|
1564
|
-
if (
|
|
1565
|
-
|
|
1566
|
-
|
|
2470
|
+
if (result.ok)
|
|
2471
|
+
return;
|
|
2472
|
+
assert.match(result.error, /^scale_up_worker_preparation_failed:worker-2:/);
|
|
2473
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
2474
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), false);
|
|
2475
|
+
assert.equal(existsSync(join(worktreePath, 'AGENTS.md')), true);
|
|
2476
|
+
assert.deepEqual(await readScaleUpTmuxLogCommands(tmuxLogPath), [
|
|
2477
|
+
'-V',
|
|
2478
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2479
|
+
]);
|
|
1567
2480
|
}
|
|
1568
2481
|
finally {
|
|
2482
|
+
if (typeof previousPath === 'string')
|
|
2483
|
+
process.env.PATH = previousPath;
|
|
2484
|
+
else
|
|
2485
|
+
delete process.env.PATH;
|
|
1569
2486
|
await rm(cwd, { recursive: true, force: true });
|
|
2487
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1570
2488
|
}
|
|
1571
2489
|
});
|
|
1572
|
-
it('
|
|
1573
|
-
const cwd = await
|
|
2490
|
+
it('retains only unresolved rollback workers and deletes never-created tasks after an explicit proof-loss phase', async () => {
|
|
2491
|
+
const cwd = await initRepo();
|
|
2492
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-rollback-kill-fail-bin-'));
|
|
2493
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2494
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2495
|
+
const proofLossUsedPath = join(fakeBinDir, 'proof-loss-used');
|
|
2496
|
+
const splitCountPath = join(fakeBinDir, 'split-count');
|
|
2497
|
+
const previousPath = process.env.PATH;
|
|
1574
2498
|
try {
|
|
1575
|
-
await
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
2499
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2500
|
+
set -eu
|
|
2501
|
+
printf '%s\n' "$*" >> "${tmuxLogPath}"
|
|
2502
|
+
case "\${1:-}" in
|
|
2503
|
+
show-option) echo 'team:scale-up' ;;
|
|
2504
|
+
-V) echo 'tmux 3.2a' ;;
|
|
2505
|
+
list-panes)
|
|
2506
|
+
split_count=0; [ ! -f "${splitCountPath}" ] || split_count=$(cat "${splitCountPath}")
|
|
2507
|
+
if [ "$split_count" -ge 2 ] && [ ! -f "${proofLossUsedPath}" ]; then
|
|
2508
|
+
: > "${proofLossUsedPath}"
|
|
2509
|
+
exit 1
|
|
2510
|
+
fi
|
|
2511
|
+
if [ "$split_count" -ge 2 ]; then
|
|
2512
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
2513
|
+
printf '%s\t%s\t%s\n' '%31' '0' '42431'
|
|
2514
|
+
printf '%s\t%s\t%s\n' '%32' '0' '42432'
|
|
2515
|
+
elif [ "$split_count" -eq 1 ]; then
|
|
2516
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
2517
|
+
printf '%s\t%s\t%s\n' '%31' '0' '42431'
|
|
2518
|
+
else
|
|
2519
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
2520
|
+
fi
|
|
2521
|
+
;;
|
|
2522
|
+
split-window)
|
|
2523
|
+
split_count=0; [ ! -f "${splitCountPath}" ] || split_count=$(cat "${splitCountPath}")
|
|
2524
|
+
split_count=$((split_count + 1)); printf '%s' "$split_count" > "${splitCountPath}"
|
|
2525
|
+
if [ "$split_count" -eq 1 ]; then
|
|
2526
|
+
echo '%31'
|
|
2527
|
+
else
|
|
2528
|
+
echo '%32'
|
|
2529
|
+
fi
|
|
2530
|
+
;;
|
|
2531
|
+
kill-pane)
|
|
2532
|
+
case "$3" in
|
|
2533
|
+
%31|%32) exit 0 ;;
|
|
2534
|
+
*) exit 1 ;;
|
|
2535
|
+
esac
|
|
2536
|
+
;;
|
|
2537
|
+
esac
|
|
2538
|
+
`);
|
|
2539
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2540
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2541
|
+
await initTeamState('rollback-kill-fail', 'task', 'executor', 1, cwd);
|
|
2542
|
+
await configureScaleUpTeamForDirectDispatch('rollback-kill-fail', cwd);
|
|
2543
|
+
const rollbackConfig = await readTeamConfig('rollback-kill-fail', cwd);
|
|
2544
|
+
assert.ok(rollbackConfig);
|
|
2545
|
+
if (!rollbackConfig)
|
|
2546
|
+
return;
|
|
2547
|
+
rollbackConfig.workspace_mode = 'worktree';
|
|
2548
|
+
rollbackConfig.worktree_mode = { enabled: true, detached: true, name: null };
|
|
2549
|
+
await saveTeamConfig(rollbackConfig, cwd);
|
|
2550
|
+
const result = await scaleUp('rollback-kill-fail', 3, 'executor', [
|
|
2551
|
+
{ subject: 'first', description: 'created before mixed rollback', owner: 'worker-2' },
|
|
2552
|
+
{ subject: 'second', description: 'created before mixed rollback', owner: 'worker-3' },
|
|
2553
|
+
{ subject: 'third', description: 'never created after proof loss', owner: 'worker-4' },
|
|
2554
|
+
], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2555
|
+
assert.equal(result.ok, false);
|
|
2556
|
+
if (!result.ok)
|
|
2557
|
+
assert.match(result.error, /^scale_up_rollback_cleanup_debt:.*pane_teardown_failed:%31.*pane_teardown_unresolved:%32/);
|
|
2558
|
+
assert.equal(await readFile(splitCountPath, 'utf-8'), '2');
|
|
2559
|
+
const config = await readTeamConfig('rollback-kill-fail', cwd);
|
|
2560
|
+
assert.deepEqual(config?.workers.map((worker) => worker.name), ['worker-1', 'worker-2', 'worker-3']);
|
|
2561
|
+
assert.equal(config?.worker_count, 3);
|
|
2562
|
+
assert.equal(config?.next_worker_index, 4);
|
|
2563
|
+
assert.equal((await readTask('rollback-kill-fail', '1', cwd))?.owner, 'worker-2');
|
|
2564
|
+
assert.equal((await readTask('rollback-kill-fail', '2', cwd))?.owner, 'worker-3');
|
|
2565
|
+
assert.equal(await readTask('rollback-kill-fail', '3', cwd), null);
|
|
2566
|
+
for (const workerName of ['worker-2', 'worker-3']) {
|
|
2567
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'rollback-kill-fail', 'workers', workerName);
|
|
2568
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, 'rollback-kill-fail', workerName)), true, `${workerName} startup script must remain`);
|
|
2569
|
+
assert.equal(existsSync(join(cwd, '.omx', 'team', 'rollback-kill-fail', 'worktrees', workerName)), true, `${workerName} worktree must remain`);
|
|
2570
|
+
if (workerName === 'worker-2') {
|
|
2571
|
+
assert.equal(existsSync(workerDir), true, 'worker-2 materialized state must remain retryable');
|
|
2572
|
+
assert.equal(existsSync(join(workerDir, 'identity.json')), true, 'worker-2 identity must remain');
|
|
2573
|
+
assert.equal(existsSync(join(workerDir, 'inbox.md')), true, 'worker-2 inbox must remain');
|
|
2574
|
+
}
|
|
2575
|
+
else {
|
|
2576
|
+
assert.equal(existsSync(join(workerDir, 'identity.json')), false, 'worker-3 proof failed before identity materialization');
|
|
2577
|
+
assert.equal(existsSync(join(workerDir, 'inbox.md')), false, 'worker-3 proof failed before inbox materialization');
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
// P2: retry execution of recorded cleanup debt is intentionally a follow-up surface.
|
|
2581
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2582
|
+
const mutationCommands = tmuxCommands
|
|
2583
|
+
.filter((command) => command.startsWith('split-window ') || command.startsWith('kill-pane '))
|
|
2584
|
+
.map((command) => command.startsWith('split-window ') ? command.split(' -c ')[0] : command);
|
|
2585
|
+
assert.deepEqual(mutationCommands, [
|
|
2586
|
+
'split-window -v -t %21 -d -P -F #{pane_id}',
|
|
2587
|
+
'split-window -v -t %31 -d -P -F #{pane_id}',
|
|
2588
|
+
'kill-pane -t %31',
|
|
2589
|
+
]);
|
|
2590
|
+
for (const mutationCommand of mutationCommands) {
|
|
2591
|
+
const mutationIndex = tmuxCommands.findIndex((command) => (command === mutationCommand || command.startsWith(`${mutationCommand} -c `)));
|
|
2592
|
+
if (mutationCommand.startsWith('split-window ')) {
|
|
2593
|
+
assert.ok(mutationIndex > 2);
|
|
2594
|
+
assert.equal(tmuxCommands[mutationIndex - 3], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2595
|
+
assert.match(tmuxCommands[mutationIndex - 2], /^show-option -qv -p -t %(?:21|31) @omx_team_pane_owner_id$/);
|
|
2596
|
+
assert.equal(tmuxCommands[mutationIndex - 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2597
|
+
}
|
|
2598
|
+
else {
|
|
2599
|
+
assert.ok(mutationIndex > 2);
|
|
2600
|
+
assert.equal(tmuxCommands[mutationIndex - 3], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2601
|
+
assert.equal(tmuxCommands[mutationIndex - 2], 'show-option -qv -p -t %31 @omx_team_pane_owner_id');
|
|
2602
|
+
assert.equal(tmuxCommands[mutationIndex - 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window -v -t %21 ')));
|
|
2606
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window -v -t %31 ')));
|
|
2607
|
+
assert.ok(tmuxCommands.some((command) => command === 'kill-pane -t %31'));
|
|
2608
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %32'), false);
|
|
2609
|
+
}
|
|
2610
|
+
finally {
|
|
2611
|
+
if (typeof previousPath === 'string')
|
|
2612
|
+
process.env.PATH = previousPath;
|
|
2613
|
+
else
|
|
2614
|
+
delete process.env.PATH;
|
|
2615
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2616
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2617
|
+
}
|
|
2618
|
+
});
|
|
2619
|
+
for (const rollbackFailurePhase of ['rollback-membership-config-persistence', 'rollback-membership-manifest-persistence']) {
|
|
2620
|
+
it(`recovers and raw-verifies original membership after public scaleUp ${rollbackFailurePhase}`, async () => {
|
|
2621
|
+
const cwd = await mkdtemp(join(tmpdir(), `omx-scale-up-${rollbackFailurePhase}-`));
|
|
2622
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), `omx-scale-up-${rollbackFailurePhase}-bin-`));
|
|
2623
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2624
|
+
const previousPath = process.env.PATH;
|
|
2625
|
+
const teamName = `scale-up-${rollbackFailurePhase.replace('rollback-membership-', '').replace('-persistence', '')}`;
|
|
2626
|
+
try {
|
|
2627
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
2628
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2629
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2630
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2631
|
+
const teamDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
2632
|
+
const configPath = join(teamDir, 'config.json');
|
|
2633
|
+
const manifestPath = join(teamDir, 'manifest.v2.json');
|
|
2634
|
+
const originalConfigBytes = await readFile(configPath, 'utf8');
|
|
2635
|
+
const originalManifestBytes = await readFile(manifestPath, 'utf8');
|
|
2636
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'public rollback recovery', description: 'must be removed', owner: 'worker-2' }], cwd, {
|
|
2637
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2638
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2639
|
+
OMX_TEAM_SCALE_UP_INJECT_FAILURE: `finalization,${rollbackFailurePhase}`,
|
|
2640
|
+
});
|
|
2641
|
+
assert.equal(result.ok, false);
|
|
2642
|
+
if (!result.ok)
|
|
2643
|
+
assert.match(result.error, /scale_up_rollback_membership_persistence_failed:.*rollback-persistence-failure/);
|
|
2644
|
+
await readTeamConfig(teamName, cwd);
|
|
2645
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(configPath, 'utf8')), withoutConfigGeneration(originalConfigBytes));
|
|
2646
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(manifestPath, 'utf8')), withoutConfigGeneration(originalManifestBytes));
|
|
2647
|
+
assert.equal(existsSync(join(teamDir, '.membership-task-transaction.json')), false);
|
|
2648
|
+
assert.equal(await readTask(teamName, '1', cwd), null);
|
|
2649
|
+
assert.deepEqual(await listDispatchRequests(teamName, cwd), []);
|
|
2650
|
+
assert.equal(existsSync(join(teamDir, 'workers', 'worker-2')), true);
|
|
2651
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), true);
|
|
2652
|
+
}
|
|
2653
|
+
finally {
|
|
2654
|
+
if (typeof previousPath === 'string')
|
|
2655
|
+
process.env.PATH = previousPath;
|
|
2656
|
+
else
|
|
2657
|
+
delete process.env.PATH;
|
|
2658
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2659
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2660
|
+
}
|
|
2661
|
+
});
|
|
2662
|
+
}
|
|
2663
|
+
for (const phase of ['identity', 'inbox', 'config']) {
|
|
2664
|
+
it(`rolls back live pane and materialized state when ${phase} materialization fails`, async () => {
|
|
2665
|
+
const cwd = await mkdtemp(join(tmpdir(), `omx-scale-up-${phase}-failure-`));
|
|
2666
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), `omx-scale-up-${phase}-failure-bin-`));
|
|
2667
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2668
|
+
const previousPath = process.env.PATH;
|
|
2669
|
+
try {
|
|
2670
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
2671
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2672
|
+
const teamName = `scale-up-${phase}-failure`;
|
|
2673
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2674
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2675
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: `${phase} rollback`, description: 'must be removed', owner: 'worker-2' }], cwd, {
|
|
2676
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2677
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2678
|
+
OMX_TEAM_SCALE_UP_INJECT_FAILURE: phase,
|
|
2679
|
+
});
|
|
2680
|
+
assert.equal(result.ok, false);
|
|
2681
|
+
if (!result.ok)
|
|
2682
|
+
assert.match(result.error, new RegExp(`scale_up_worker_materialization_failed:worker-2:.*${phase}`));
|
|
2683
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers.length, 1);
|
|
2684
|
+
assert.equal(await readTask(teamName, '1', cwd), null);
|
|
2685
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
2686
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), false);
|
|
2687
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2688
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window ')));
|
|
2689
|
+
assert.ok(tmuxCommands.includes('kill-pane -t %31'));
|
|
2690
|
+
}
|
|
2691
|
+
finally {
|
|
2692
|
+
if (typeof previousPath === 'string')
|
|
2693
|
+
process.env.PATH = previousPath;
|
|
2694
|
+
else
|
|
2695
|
+
delete process.env.PATH;
|
|
2696
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2697
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2698
|
+
}
|
|
2699
|
+
});
|
|
2700
|
+
}
|
|
2701
|
+
it('rolls back a named worktree created before injected ensure bookkeeping fails', async () => {
|
|
2702
|
+
const repo = await initRepo();
|
|
2703
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-named-post-create-bin-'));
|
|
2704
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2705
|
+
const previousPath = process.env.PATH;
|
|
2706
|
+
const teamName = 'named-post-create-failure';
|
|
2707
|
+
const branchName = 'feature/named-post-create/worker-2';
|
|
2708
|
+
const worktreePath = join(repo, '.omx', 'team', teamName, 'worktrees', 'worker-2');
|
|
2709
|
+
try {
|
|
2710
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
2711
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2712
|
+
await initTeamState(teamName, 'task', 'executor', 1, repo, undefined, process.env, {
|
|
2713
|
+
workspace_mode: 'worktree',
|
|
2714
|
+
leader_cwd: repo,
|
|
2715
|
+
team_state_root: join(repo, '.omx', 'state'),
|
|
2716
|
+
worktree_mode: { enabled: true, detached: false, name: 'feature/named-post-create' },
|
|
2717
|
+
});
|
|
2718
|
+
await configureScaleUpTeamForDirectDispatch(teamName, repo);
|
|
2719
|
+
const result = await scaleUp(teamName, 1, 'executor', [], repo, {
|
|
2720
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2721
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2722
|
+
OMX_TEAM_SCALE_UP_INJECT_FAILURE: 'worktree-ensure-post-create',
|
|
2723
|
+
});
|
|
2724
|
+
assert.equal(result.ok, false);
|
|
2725
|
+
if (!result.ok)
|
|
2726
|
+
assert.match(result.error, /scale_up_worker_preparation_failed:worker-2:.*worktree-ensure-post-create/);
|
|
2727
|
+
assert.equal(existsSync(worktreePath), false);
|
|
2728
|
+
assert.throws(() => execFileSync('git', ['show-ref', '--verify', '--quiet', `refs/heads/${branchName}`], { cwd: repo }));
|
|
2729
|
+
assert.equal((await readTeamConfig(teamName, repo))?.workers.length, 1);
|
|
2730
|
+
assert.equal(existsSync(join(repo, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
2731
|
+
assert.deepEqual((await readScaleUpTmuxLogCommands(tmuxLogPath)).filter((command) => command.startsWith('split-window ')), []);
|
|
2732
|
+
}
|
|
2733
|
+
finally {
|
|
2734
|
+
if (typeof previousPath === 'string')
|
|
2735
|
+
process.env.PATH = previousPath;
|
|
2736
|
+
else
|
|
2737
|
+
delete process.env.PATH;
|
|
2738
|
+
await rm(repo, { recursive: true, force: true });
|
|
2739
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2740
|
+
}
|
|
2741
|
+
});
|
|
2742
|
+
});
|
|
2743
|
+
describe('scaleDown', () => {
|
|
2744
|
+
it('rejects when scaling is disabled', async () => {
|
|
2745
|
+
await assert.rejects(scaleDown('test', '/tmp', {}, {}), /Dynamic scaling is disabled/);
|
|
2746
|
+
});
|
|
2747
|
+
it('returns error when team not found', async () => {
|
|
2748
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-nf-'));
|
|
2749
|
+
try {
|
|
2750
|
+
const result = await scaleDown('nonexistent', cwd, {}, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2751
|
+
assert.equal(result.ok, false);
|
|
2752
|
+
if (!result.ok) {
|
|
2753
|
+
assert.match(result.error, /not found/);
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2756
|
+
finally {
|
|
2757
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2758
|
+
}
|
|
2759
|
+
});
|
|
2760
|
+
it('returns error when trying to remove all workers', async () => {
|
|
2761
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-all-'));
|
|
2762
|
+
try {
|
|
2763
|
+
await initTeamState('all-test', 'task', 'executor', 1, cwd);
|
|
2764
|
+
const result = await scaleDown('all-test', cwd, { workerNames: ['worker-1'] }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2765
|
+
assert.equal(result.ok, false);
|
|
2766
|
+
if (!result.ok) {
|
|
2767
|
+
assert.match(result.error, /at least 1 must remain/);
|
|
2768
|
+
}
|
|
2769
|
+
}
|
|
2770
|
+
finally {
|
|
2771
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2772
|
+
}
|
|
2773
|
+
});
|
|
2774
|
+
it('returns error for worker not in team', async () => {
|
|
2775
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-miss-'));
|
|
1588
2776
|
try {
|
|
1589
2777
|
await initTeamState('miss-test', 'task', 'executor', 2, cwd);
|
|
1590
2778
|
const result = await scaleDown('miss-test', cwd, { workerNames: ['worker-99'] }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
@@ -1622,6 +2810,71 @@ describe('scaleDown', () => {
|
|
|
1622
2810
|
await rm(cwd, { recursive: true, force: true });
|
|
1623
2811
|
}
|
|
1624
2812
|
});
|
|
2813
|
+
it('serializes a concurrent claim across the canonical scale-down boundary', async () => {
|
|
2814
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-claim-boundary-'));
|
|
2815
|
+
try {
|
|
2816
|
+
await initTeamState('claim-boundary', 'task', 'executor', 2, cwd);
|
|
2817
|
+
const task = await createTask('claim-boundary', {
|
|
2818
|
+
subject: 'boundary task', description: 'must not be claimed by a removed worker', status: 'pending', owner: 'worker-2',
|
|
2819
|
+
}, cwd);
|
|
2820
|
+
const down = scaleDown('claim-boundary', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
2821
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2822
|
+
OMX_TEAM_SCALE_DOWN_BOUNDARY_HOLD_MS: '100',
|
|
2823
|
+
});
|
|
2824
|
+
const lockPath = join(cwd, '.omx', 'state', 'team', 'claim-boundary', 'claims', `task-${task.id}.lock`);
|
|
2825
|
+
for (let attempt = 0; attempt < 50 && !existsSync(lockPath); attempt += 1) {
|
|
2826
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
2827
|
+
}
|
|
2828
|
+
assert.equal(existsSync(lockPath), true);
|
|
2829
|
+
const claim = claimTask('claim-boundary', task.id, 'worker-2', task.version ?? 1, cwd);
|
|
2830
|
+
assert.deepEqual(await down, { ok: true, removedWorkers: ['worker-2'], newWorkerCount: 1 });
|
|
2831
|
+
assert.deepEqual(await claim, { ok: false, error: 'claim_conflict' });
|
|
2832
|
+
const reconciled = await readTask('claim-boundary', task.id, cwd);
|
|
2833
|
+
assert.equal(reconciled?.owner, undefined);
|
|
2834
|
+
assert.equal(reconciled?.claim, undefined);
|
|
2835
|
+
}
|
|
2836
|
+
finally {
|
|
2837
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2838
|
+
}
|
|
2839
|
+
});
|
|
2840
|
+
it('blocks post-snapshot task creation and null-version claims until removed membership is committed', async () => {
|
|
2841
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-create-claim-boundary-'));
|
|
2842
|
+
try {
|
|
2843
|
+
await initTeamState('create-claim-boundary', 'task', 'executor', 2, cwd);
|
|
2844
|
+
const seed = await createTask('create-claim-boundary', {
|
|
2845
|
+
subject: 'snapshot marker', description: 'holds the canonical task lock', status: 'pending', owner: 'worker-2',
|
|
2846
|
+
}, cwd);
|
|
2847
|
+
const postSnapshotMarker = join(cwd, 'post-snapshot-held');
|
|
2848
|
+
const down = scaleDown('create-claim-boundary', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
2849
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2850
|
+
OMX_TEAM_SCALE_DOWN_BOUNDARY_HOLD_MS: '250',
|
|
2851
|
+
OMX_TEAM_SCALE_DOWN_POST_SNAPSHOT_MARKER: postSnapshotMarker,
|
|
2852
|
+
});
|
|
2853
|
+
for (let attempt = 0; attempt < 50 && !existsSync(postSnapshotMarker); attempt += 1) {
|
|
2854
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
2855
|
+
}
|
|
2856
|
+
assert.equal(existsSync(postSnapshotMarker), true);
|
|
2857
|
+
let createSettled = false;
|
|
2858
|
+
let claimSettled = false;
|
|
2859
|
+
const created = createTask('create-claim-boundary', {
|
|
2860
|
+
subject: 'created after snapshot', description: 'must not restore removed membership', status: 'pending', owner: 'worker-2',
|
|
2861
|
+
}, cwd).finally(() => { createSettled = true; });
|
|
2862
|
+
const claim = claimTask('create-claim-boundary', seed.id, 'worker-2', null, cwd)
|
|
2863
|
+
.finally(() => { claimSettled = true; });
|
|
2864
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
2865
|
+
assert.equal(createSettled, false);
|
|
2866
|
+
assert.equal(claimSettled, false);
|
|
2867
|
+
assert.deepEqual(await down, { ok: true, removedWorkers: ['worker-2'], newWorkerCount: 1 });
|
|
2868
|
+
assert.deepEqual(await claim, { ok: false, error: 'worker_not_found' });
|
|
2869
|
+
const task = await created;
|
|
2870
|
+
assert.equal(task.owner, 'worker-2');
|
|
2871
|
+
assert.deepEqual((await readTeamConfig('create-claim-boundary', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
2872
|
+
assert.equal((await readTask('create-claim-boundary', task.id, cwd))?.owner, 'worker-2');
|
|
2873
|
+
}
|
|
2874
|
+
finally {
|
|
2875
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2876
|
+
}
|
|
2877
|
+
});
|
|
1625
2878
|
});
|
|
1626
2879
|
describe('scaleDown worktree AGENTS cleanup', () => {
|
|
1627
2880
|
it('removes generated worktree-root AGENTS during scale-down', async () => {
|
|
@@ -1657,15 +2910,47 @@ describe('scaleDown worktree AGENTS cleanup', () => {
|
|
|
1657
2910
|
});
|
|
1658
2911
|
});
|
|
1659
2912
|
describe('scaleDown teardown hardening', () => {
|
|
1660
|
-
it('scaleDown removes
|
|
2913
|
+
it('scaleDown removes a worker with a pane proven dead without targeting it', async () => {
|
|
1661
2914
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-dead-'));
|
|
2915
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-dead-bin-'));
|
|
2916
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2917
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2918
|
+
const previousPath = process.env.PATH;
|
|
1662
2919
|
try {
|
|
2920
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2921
|
+
set -eu
|
|
2922
|
+
printf '%s\n' "$*" >> "${tmuxLogPath}"
|
|
2923
|
+
case "\${1:-}" in
|
|
2924
|
+
show-option) echo 'team:scale-down' ;;
|
|
2925
|
+
list-panes)
|
|
2926
|
+
case "$*" in
|
|
2927
|
+
*"-a -F #{pane_id}"*)
|
|
2928
|
+
printf '%%405\t1\t4050\n'
|
|
2929
|
+
;;
|
|
2930
|
+
*)
|
|
2931
|
+
exit 1
|
|
2932
|
+
;;
|
|
2933
|
+
esac
|
|
2934
|
+
;;
|
|
2935
|
+
kill-pane)
|
|
2936
|
+
echo "a proven-dead pane must not be targeted" >&2
|
|
2937
|
+
exit 1
|
|
2938
|
+
;;
|
|
2939
|
+
*)
|
|
2940
|
+
exit 0
|
|
2941
|
+
;;
|
|
2942
|
+
esac
|
|
2943
|
+
`);
|
|
2944
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2945
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1663
2946
|
await initTeamState('dead-pane', 'task', 'executor', 2, cwd);
|
|
1664
2947
|
const config = await readTeamConfig('dead-pane', cwd);
|
|
1665
2948
|
assert.ok(config);
|
|
1666
2949
|
if (!config)
|
|
1667
2950
|
return;
|
|
1668
|
-
config.workers[1].pane_id = '%
|
|
2951
|
+
config.workers[1].pane_id = '%405';
|
|
2952
|
+
config.workers[1].pid = 424405;
|
|
2953
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
1669
2954
|
await saveTeamConfig(config, cwd);
|
|
1670
2955
|
const result = await scaleDown('dead-pane', cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
1671
2956
|
assert.equal(result.ok, true);
|
|
@@ -1675,9 +2960,81 @@ describe('scaleDown teardown hardening', () => {
|
|
|
1675
2960
|
const updated = await readTeamConfig('dead-pane', cwd);
|
|
1676
2961
|
assert.ok(updated);
|
|
1677
2962
|
assert.equal(updated?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
2963
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2964
|
+
assert.deepEqual(tmuxCommands, [
|
|
2965
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2966
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2967
|
+
]);
|
|
2968
|
+
}
|
|
2969
|
+
finally {
|
|
2970
|
+
if (typeof previousPath === 'string')
|
|
2971
|
+
process.env.PATH = previousPath;
|
|
2972
|
+
else
|
|
2973
|
+
delete process.env.PATH;
|
|
2974
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2975
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2976
|
+
}
|
|
2977
|
+
});
|
|
2978
|
+
it('retains PID-less rollback debt when a recycled same-Team pane remains live', async () => {
|
|
2979
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-unpinned-recycled-'));
|
|
2980
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-unpinned-recycled-bin-'));
|
|
2981
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2982
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2983
|
+
const previousPath = process.env.PATH;
|
|
2984
|
+
try {
|
|
2985
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2986
|
+
set -eu
|
|
2987
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
2988
|
+
case "\${1:-}" in
|
|
2989
|
+
list-panes) printf '%%405\\t0\\t999405\\n' ;;
|
|
2990
|
+
show-option|kill-pane) echo "PID-less debt must not authorize an effect" >&2; exit 99 ;;
|
|
2991
|
+
esac
|
|
2992
|
+
`);
|
|
2993
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2994
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2995
|
+
await initTeamState('unpinned-recycled', 'task', 'executor', 2, cwd);
|
|
2996
|
+
const config = await readTeamConfig('unpinned-recycled', cwd);
|
|
2997
|
+
assert.ok(config);
|
|
2998
|
+
if (!config)
|
|
2999
|
+
return;
|
|
3000
|
+
config.workers[1].pane_id = '%405';
|
|
3001
|
+
config.workers[1].pid = undefined;
|
|
3002
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3003
|
+
await saveTeamConfig(config, cwd);
|
|
3004
|
+
const initial = await scaleDown('unpinned-recycled', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3005
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
3006
|
+
});
|
|
3007
|
+
assert.deepEqual(initial, {
|
|
3008
|
+
ok: false,
|
|
3009
|
+
error: 'scale_down_cleanup_debt:pane_teardown_unresolved:%405',
|
|
3010
|
+
});
|
|
3011
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'unpinned-recycled', '.scale-down-cleanup-debt.json');
|
|
3012
|
+
const initialDebt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
3013
|
+
assert.deepEqual(initialDebt.unresolved_panes, [{ name: 'worker-2', index: 2, pane_id: '%405', pid: null }]);
|
|
3014
|
+
assert.ok(initialDebt.reasons.includes('%405:legacy_pid_missing_live'));
|
|
3015
|
+
const canonical = await readTeamConfig('unpinned-recycled', cwd);
|
|
3016
|
+
assert.ok(canonical);
|
|
3017
|
+
if (!canonical)
|
|
3018
|
+
return;
|
|
3019
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('unpinned-recycled', cwd, canonical), {
|
|
3020
|
+
ok: false,
|
|
3021
|
+
error: 'scale_down_cleanup_debt_unresolved:%405',
|
|
3022
|
+
});
|
|
3023
|
+
const reconciledDebt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
3024
|
+
assert.ok(reconciledDebt.reasons.includes('%405:legacy_pid_missing_live'));
|
|
3025
|
+
const commands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3026
|
+
assert.deepEqual(commands, [
|
|
3027
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3028
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3029
|
+
]);
|
|
1678
3030
|
}
|
|
1679
3031
|
finally {
|
|
3032
|
+
if (typeof previousPath === 'string')
|
|
3033
|
+
process.env.PATH = previousPath;
|
|
3034
|
+
else
|
|
3035
|
+
delete process.env.PATH;
|
|
1680
3036
|
await rm(cwd, { recursive: true, force: true });
|
|
3037
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1681
3038
|
}
|
|
1682
3039
|
});
|
|
1683
3040
|
it('scaleDown never targets leader or hud panes during teardown', async () => {
|
|
@@ -1690,6 +3047,22 @@ describe('scaleDown teardown hardening', () => {
|
|
|
1690
3047
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1691
3048
|
set -eu
|
|
1692
3049
|
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3050
|
+
case "\${1:-}" in
|
|
3051
|
+
show-option) echo 'team:scale-down' ;;
|
|
3052
|
+
list-panes)
|
|
3053
|
+
case "$*" in
|
|
3054
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
3055
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
3056
|
+
printf '%s\t%s\t%s\n' '%12' '0' '42412'
|
|
3057
|
+
if [ ! -f "${tmuxLogPath}.killed-%13" ]; then printf '%s\t%s\t%s\n' '%13' '0' '42413'; fi
|
|
3058
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3059
|
+
;;
|
|
3060
|
+
esac
|
|
3061
|
+
;;
|
|
3062
|
+
kill-pane)
|
|
3063
|
+
: > "${tmuxLogPath}.killed-$3"
|
|
3064
|
+
;;
|
|
3065
|
+
esac
|
|
1693
3066
|
exit 0
|
|
1694
3067
|
`);
|
|
1695
3068
|
await writeFile(tmuxLogPath, '');
|
|
@@ -1703,16 +3076,1041 @@ exit 0
|
|
|
1703
3076
|
config.leader_pane_id = '%11';
|
|
1704
3077
|
config.hud_pane_id = '%12';
|
|
1705
3078
|
config.workers[0].pane_id = '%11';
|
|
3079
|
+
config.workers[0].pid = 42411;
|
|
1706
3080
|
config.workers[1].pane_id = '%12';
|
|
3081
|
+
config.workers[1].pid = 42412;
|
|
1707
3082
|
config.workers[2].pane_id = '%13';
|
|
3083
|
+
config.workers[2].pid = 42413;
|
|
1708
3084
|
config.workers[3].pane_id = '%14';
|
|
3085
|
+
config.workers[3].pid = 42414;
|
|
3086
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
1709
3087
|
await saveTeamConfig(config, cwd);
|
|
1710
3088
|
const result = await scaleDown('exclusions', cwd, { workerNames: ['worker-1', 'worker-2', 'worker-3'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
1711
|
-
assert.equal(result.ok,
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
assert.
|
|
1715
|
-
|
|
3089
|
+
assert.equal(result.ok, false);
|
|
3090
|
+
if (!result.ok)
|
|
3091
|
+
assert.match(result.error, /pane_teardown_unresolved:%11.*pane_teardown_unresolved:%12/);
|
|
3092
|
+
assert.deepEqual((await readTeamConfig('exclusions', cwd))?.workers.map((worker) => worker.name), ['worker-4']);
|
|
3093
|
+
const tmuxCommands = (await readFile(tmuxLogPath, 'utf-8')).trim().split('\n');
|
|
3094
|
+
assert.deepEqual(tmuxCommands, [
|
|
3095
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3096
|
+
'show-option -qv -p -t %13 @omx_team_pane_owner_id',
|
|
3097
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3098
|
+
'kill-pane -t %13',
|
|
3099
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3100
|
+
]);
|
|
3101
|
+
}
|
|
3102
|
+
finally {
|
|
3103
|
+
if (typeof previousPath === 'string')
|
|
3104
|
+
process.env.PATH = previousPath;
|
|
3105
|
+
else
|
|
3106
|
+
delete process.env.PATH;
|
|
3107
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3108
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3109
|
+
}
|
|
3110
|
+
});
|
|
3111
|
+
it('scaleDown commits forward membership and retains artifacts/debt when pane ID is reused', async () => {
|
|
3112
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-kill-fail-'));
|
|
3113
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-kill-fail-bin-'));
|
|
3114
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3115
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3116
|
+
const previousPath = process.env.PATH;
|
|
3117
|
+
try {
|
|
3118
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3119
|
+
set -eu
|
|
3120
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3121
|
+
case "\${1:-}" in
|
|
3122
|
+
show-option) echo 'team:scale-down' ;;
|
|
3123
|
+
list-panes)
|
|
3124
|
+
if [ -f "${cwd}/pane-reused" ]; then
|
|
3125
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42414'
|
|
3126
|
+
else
|
|
3127
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3128
|
+
fi
|
|
3129
|
+
;;
|
|
3130
|
+
kill-pane)
|
|
3131
|
+
: > "${cwd}/pane-reused"
|
|
3132
|
+
exit 0
|
|
3133
|
+
;;
|
|
3134
|
+
esac
|
|
3135
|
+
exit 0
|
|
3136
|
+
`);
|
|
3137
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3138
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3139
|
+
await initTeamState('kill-fail', 'task', 'executor', 2, cwd);
|
|
3140
|
+
const config = await readTeamConfig('kill-fail', cwd);
|
|
3141
|
+
assert.ok(config);
|
|
3142
|
+
if (!config)
|
|
3143
|
+
return;
|
|
3144
|
+
config.workers[1].pane_id = '%13';
|
|
3145
|
+
config.workers[1].pid = 42413;
|
|
3146
|
+
const worktreePath = join(cwd, '.omx', 'team', 'kill-fail', 'worktrees', 'worker-2');
|
|
3147
|
+
await mkdir(worktreePath, { recursive: true });
|
|
3148
|
+
await writeFile(join(worktreePath, 'keep.txt'), 'retryable worktree');
|
|
3149
|
+
config.workers[1].worktree_path = worktreePath;
|
|
3150
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'kill-fail', 'workers', 'worker-2');
|
|
3151
|
+
await writeFile(join(workerDir, 'identity.json'), '{"worker":"worker-2"}');
|
|
3152
|
+
await writeFile(join(workerDir, 'inbox.md'), 'retryable inbox');
|
|
3153
|
+
await mkdir(join(cwd, '.omx', 'state', 'team', 'kill-fail', 'runtime'), { recursive: true });
|
|
3154
|
+
const startupScriptPath = workerStartupScriptPath(cwd, 'kill-fail', 'worker-2');
|
|
3155
|
+
await writeFile(startupScriptPath, '#!/bin/sh\n');
|
|
3156
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3157
|
+
await saveTeamConfig(config, cwd);
|
|
3158
|
+
const priorStatus = {
|
|
3159
|
+
state: 'idle',
|
|
3160
|
+
reason: 'waiting for assignment',
|
|
3161
|
+
updated_at: '2026-07-14T00:00:00.000Z',
|
|
3162
|
+
};
|
|
3163
|
+
await writeWorkerStatus('kill-fail', 'worker-2', priorStatus, cwd);
|
|
3164
|
+
const result = await scaleDown('kill-fail', cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3165
|
+
assert.equal(result.ok, false);
|
|
3166
|
+
if (!result.ok)
|
|
3167
|
+
assert.match(result.error, /^scale_down_cleanup_debt:pane_teardown_failed:%13/);
|
|
3168
|
+
const committed = await readTeamConfig('kill-fail', cwd);
|
|
3169
|
+
assert.deepEqual(committed?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3170
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'kill-fail', 'workers', 'worker-2')), true);
|
|
3171
|
+
assert.equal(await readFile(join(workerDir, 'identity.json'), 'utf8'), '{"worker":"worker-2"}');
|
|
3172
|
+
assert.equal(await readFile(join(workerDir, 'inbox.md'), 'utf8'), 'retryable inbox');
|
|
3173
|
+
assert.equal(await readFile(join(worktreePath, 'keep.txt'), 'utf8'), 'retryable worktree');
|
|
3174
|
+
assert.equal(await readFile(startupScriptPath, 'utf8'), '#!/bin/sh\n');
|
|
3175
|
+
assert.deepEqual(await readWorkerStatus('kill-fail', 'worker-2', cwd), priorStatus);
|
|
3176
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'kill-fail', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3177
|
+
assert.equal(debt.status, 'unresolved');
|
|
3178
|
+
assert.equal(debt.unresolved_panes[0]?.pane_id, '%13');
|
|
3179
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3180
|
+
assert.deepEqual(tmuxCommands, [
|
|
3181
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3182
|
+
'show-option -qv -p -t %13 @omx_team_pane_owner_id',
|
|
3183
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3184
|
+
'kill-pane -t %13',
|
|
3185
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3186
|
+
]);
|
|
3187
|
+
}
|
|
3188
|
+
finally {
|
|
3189
|
+
if (typeof previousPath === 'string')
|
|
3190
|
+
process.env.PATH = previousPath;
|
|
3191
|
+
else
|
|
3192
|
+
delete process.env.PATH;
|
|
3193
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3194
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3195
|
+
}
|
|
3196
|
+
});
|
|
3197
|
+
it('aborts before pane teardown and preserves exact state when task reconciliation cannot acquire a lock', async () => {
|
|
3198
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-task-reconcile-fail-'));
|
|
3199
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-task-reconcile-fail-bin-'));
|
|
3200
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3201
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3202
|
+
const previousPath = process.env.PATH;
|
|
3203
|
+
try {
|
|
3204
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3205
|
+
set -eu
|
|
3206
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3207
|
+
case "\${1:-}" in
|
|
3208
|
+
show-option) echo 'team:scale-down' ;;
|
|
3209
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%11' '0' '42411' ;;
|
|
3210
|
+
esac
|
|
3211
|
+
`);
|
|
3212
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3213
|
+
await writeFile(tmuxLogPath, '');
|
|
3214
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3215
|
+
await initTeamState('task-reconcile-fail', 'task', 'executor', 2, cwd);
|
|
3216
|
+
const config = await readTeamConfig('task-reconcile-fail', cwd);
|
|
3217
|
+
assert.ok(config);
|
|
3218
|
+
if (!config)
|
|
3219
|
+
return;
|
|
3220
|
+
config.workers[1].pane_id = '%13';
|
|
3221
|
+
config.workers[1].pid = 42413;
|
|
3222
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3223
|
+
await saveTeamConfig(config, cwd);
|
|
3224
|
+
const firstTask = await createTask('task-reconcile-fail', {
|
|
3225
|
+
subject: 'first reconciliation', description: 'must be restored after later failure', status: 'pending', owner: 'worker-2',
|
|
3226
|
+
}, cwd);
|
|
3227
|
+
const firstTaskPath = join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'tasks', `task-${firstTask.id}.json`);
|
|
3228
|
+
const firstTaskRaw = await readFile(firstTaskPath, 'utf-8');
|
|
3229
|
+
const task = await createTask('task-reconcile-fail', {
|
|
3230
|
+
subject: 'locked reconciliation', description: 'must remain exact', status: 'pending', owner: 'worker-2',
|
|
3231
|
+
}, cwd);
|
|
3232
|
+
const taskPath = join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'tasks', `task-${task.id}.json`);
|
|
3233
|
+
const taskRaw = await readFile(taskPath, 'utf-8');
|
|
3234
|
+
const priorConfig = structuredClone(await readTeamConfig('task-reconcile-fail', cwd));
|
|
3235
|
+
await mkdir(join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'claims', `task-${task.id}.lock`));
|
|
3236
|
+
const result = await scaleDown('task-reconcile-fail', cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3237
|
+
assert.deepEqual(result, { ok: false, error: 'scale_down_task_reconciliation_failed:Error: Timed out acquiring task claim lock for task-reconcile-fail/2' });
|
|
3238
|
+
assert.deepEqual(await readTeamConfig('task-reconcile-fail', cwd), priorConfig);
|
|
3239
|
+
assert.equal(await readFile(taskPath, 'utf-8'), taskRaw);
|
|
3240
|
+
assert.equal(await readFile(firstTaskPath, 'utf-8'), firstTaskRaw);
|
|
3241
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'workers', 'worker-2')), true);
|
|
3242
|
+
assert.deepEqual(await readScaleUpTmuxLogCommands(tmuxLogPath), []);
|
|
3243
|
+
}
|
|
3244
|
+
finally {
|
|
3245
|
+
if (typeof previousPath === 'string')
|
|
3246
|
+
process.env.PATH = previousPath;
|
|
3247
|
+
else
|
|
3248
|
+
delete process.env.PATH;
|
|
3249
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3250
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3251
|
+
}
|
|
3252
|
+
});
|
|
3253
|
+
it('commits forward membership and durable debt when scale-down pane proof is unavailable', async () => {
|
|
3254
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-proof-unavailable-'));
|
|
3255
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-proof-unavailable-bin-'));
|
|
3256
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3257
|
+
const previousPath = process.env.PATH;
|
|
3258
|
+
try {
|
|
3259
|
+
await writeFile(tmuxStubPath, '#!/bin/sh\n[ "$1" = list-panes ] && exit 1\nexit 0\n');
|
|
3260
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3261
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3262
|
+
await initTeamState('proof-unavailable', 'task', 'executor', 2, cwd);
|
|
3263
|
+
const config = await readTeamConfig('proof-unavailable', cwd);
|
|
3264
|
+
assert.ok(config);
|
|
3265
|
+
if (!config)
|
|
3266
|
+
return;
|
|
3267
|
+
config.workers[1].pane_id = '%13';
|
|
3268
|
+
config.workers[1].pid = 42413;
|
|
3269
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3270
|
+
await saveTeamConfig(config, cwd);
|
|
3271
|
+
const task = await createTask('proof-unavailable', {
|
|
3272
|
+
subject: 'preserved', description: 'exact proof is unavailable', status: 'pending', owner: 'worker-2',
|
|
3273
|
+
}, cwd);
|
|
3274
|
+
const statusPath = join(cwd, '.omx', 'state', 'team', 'proof-unavailable', 'workers', 'worker-2', 'status.json');
|
|
3275
|
+
const statusRaw = '{"state":"idle", "reason":"preserve bytes", "updated_at":"2026-07-14T00:00:00.000Z"}\n';
|
|
3276
|
+
await writeFile(statusPath, statusRaw);
|
|
3277
|
+
const result = await scaleDown('proof-unavailable', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3278
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
3279
|
+
});
|
|
3280
|
+
assert.equal(result.ok, false);
|
|
3281
|
+
assert.deepEqual((await readTeamConfig('proof-unavailable', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3282
|
+
assert.equal((await readTask('proof-unavailable', task.id, cwd))?.owner, undefined);
|
|
3283
|
+
assert.equal(await readFile(statusPath, 'utf8'), statusRaw);
|
|
3284
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'proof-unavailable', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3285
|
+
assert.equal(debt.status, 'unresolved');
|
|
3286
|
+
assert.equal(debt.unresolved_panes[0]?.pane_id, '%13');
|
|
3287
|
+
}
|
|
3288
|
+
finally {
|
|
3289
|
+
if (typeof previousPath === 'string')
|
|
3290
|
+
process.env.PATH = previousPath;
|
|
3291
|
+
else
|
|
3292
|
+
delete process.env.PATH;
|
|
3293
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3294
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3295
|
+
}
|
|
3296
|
+
});
|
|
3297
|
+
it('converges entirely old or entirely new after a mid-commit scale-down interruption', async () => {
|
|
3298
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-atomic-'));
|
|
3299
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-atomic-bin-'));
|
|
3300
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3301
|
+
const previousPath = process.env.PATH;
|
|
3302
|
+
try {
|
|
3303
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3304
|
+
marker="${tmuxStubPath}.killed"
|
|
3305
|
+
if [ "$1" = list-panes ]; then
|
|
3306
|
+
[ -f "$marker" ] || printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3307
|
+
exit 0
|
|
3308
|
+
fi
|
|
3309
|
+
if [ "$1" = kill-pane ]; then : > "$marker"; exit 0; fi
|
|
3310
|
+
exit 0
|
|
3311
|
+
`);
|
|
3312
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3313
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3314
|
+
await initTeamState('atomic-down', 'task', 'executor', 2, cwd);
|
|
3315
|
+
const config = await readTeamConfig('atomic-down', cwd);
|
|
3316
|
+
assert.ok(config);
|
|
3317
|
+
if (!config)
|
|
3318
|
+
return;
|
|
3319
|
+
config.workers[1].pane_id = '%13';
|
|
3320
|
+
config.workers[1].pid = 42413;
|
|
3321
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3322
|
+
await saveTeamConfig(config, cwd);
|
|
3323
|
+
const first = await createTask('atomic-down', { subject: 'first', description: 'first', status: 'pending', owner: 'worker-2' }, cwd);
|
|
3324
|
+
const second = await createTask('atomic-down', { subject: 'second', description: 'second', status: 'pending', owner: 'worker-2' }, cwd);
|
|
3325
|
+
const result = await scaleDown('atomic-down', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3326
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SCALE_DOWN_INJECT_FAILURE: 'after-first-task-write',
|
|
3327
|
+
});
|
|
3328
|
+
assert.match(result.ok ? '' : result.error, /injected_scale_down_interruption:after-first-task-write/);
|
|
3329
|
+
await recoverTeamMembershipTaskTransaction('atomic-down', cwd);
|
|
3330
|
+
const recoveredConfig = await readTeamConfig('atomic-down', cwd);
|
|
3331
|
+
const recoveredFirst = await readTask('atomic-down', first.id, cwd);
|
|
3332
|
+
const recoveredSecond = await readTask('atomic-down', second.id, cwd);
|
|
3333
|
+
const convergedOld = recoveredConfig?.workers.some((worker) => worker.name === 'worker-2') === true
|
|
3334
|
+
&& recoveredFirst?.owner === 'worker-2'
|
|
3335
|
+
&& recoveredSecond?.owner === 'worker-2';
|
|
3336
|
+
const convergedNew = recoveredConfig?.workers.some((worker) => worker.name === 'worker-2') === false
|
|
3337
|
+
&& recoveredFirst?.owner === undefined
|
|
3338
|
+
&& recoveredSecond?.owner === undefined;
|
|
3339
|
+
assert.equal(convergedOld || convergedNew, true);
|
|
3340
|
+
}
|
|
3341
|
+
finally {
|
|
3342
|
+
if (typeof previousPath === 'string')
|
|
3343
|
+
process.env.PATH = previousPath;
|
|
3344
|
+
else
|
|
3345
|
+
delete process.env.PATH;
|
|
3346
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3347
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3348
|
+
}
|
|
3349
|
+
});
|
|
3350
|
+
it('recovers rollback-persistence failure through public config reads with config, manifest, and tasks converged', async () => {
|
|
3351
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-rollback-recovery-'));
|
|
3352
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-rollback-recovery-bin-'));
|
|
3353
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3354
|
+
const previousPath = process.env.PATH;
|
|
3355
|
+
try {
|
|
3356
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3357
|
+
marker="${tmuxStubPath}.killed"
|
|
3358
|
+
if [ "$1" = list-panes ]; then
|
|
3359
|
+
[ -f "$marker" ] || printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3360
|
+
exit 0
|
|
3361
|
+
fi
|
|
3362
|
+
if [ "$1" = kill-pane ]; then : > "$marker"; exit 0; fi
|
|
3363
|
+
exit 0
|
|
3364
|
+
`);
|
|
3365
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3366
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3367
|
+
await initTeamState('rollback-recovery', 'task', 'executor', 2, cwd);
|
|
3368
|
+
const config = await readTeamConfig('rollback-recovery', cwd);
|
|
3369
|
+
assert.ok(config);
|
|
3370
|
+
if (!config)
|
|
3371
|
+
return;
|
|
3372
|
+
config.workers[1].pane_id = '%13';
|
|
3373
|
+
config.workers[1].pid = 42413;
|
|
3374
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3375
|
+
await saveTeamConfig(config, cwd);
|
|
3376
|
+
const task = await createTask('rollback-recovery', { subject: 'owned', description: 'owned', status: 'pending', owner: 'worker-2' }, cwd);
|
|
3377
|
+
const failed = await scaleDown('rollback-recovery', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3378
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SCALE_DOWN_INJECT_FAILURE: 'rollback-persistence-failure',
|
|
3379
|
+
});
|
|
3380
|
+
assert.match(failed.ok ? '' : failed.error, /rollback-persistence-failure/);
|
|
3381
|
+
const recoveredConfig = await readTeamConfig('rollback-recovery', cwd);
|
|
3382
|
+
const recoveredManifest = await readTeamManifestV2('rollback-recovery', cwd);
|
|
3383
|
+
const recoveredTask = await readTask('rollback-recovery', task.id, cwd);
|
|
3384
|
+
assert.equal(recoveredConfig?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3385
|
+
assert.equal(recoveredManifest?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3386
|
+
assert.equal(recoveredTask?.owner, undefined);
|
|
3387
|
+
}
|
|
3388
|
+
finally {
|
|
3389
|
+
if (typeof previousPath === 'string')
|
|
3390
|
+
process.env.PATH = previousPath;
|
|
3391
|
+
else
|
|
3392
|
+
delete process.env.PATH;
|
|
3393
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3394
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3395
|
+
}
|
|
3396
|
+
});
|
|
3397
|
+
for (const target of ['config', 'manifest']) {
|
|
3398
|
+
it(`publicly recovers a committed scale-up membership rollback after partial ${target} old-generation persistence`, async () => {
|
|
3399
|
+
const cwd = await mkdtemp(join(tmpdir(), `omx-scale-up-rollback-${target}-`));
|
|
3400
|
+
try {
|
|
3401
|
+
const teamName = `rollback-${target}`;
|
|
3402
|
+
await initTeamState(teamName, 'task', 'executor', 2, cwd);
|
|
3403
|
+
const task = await createTask(teamName, {
|
|
3404
|
+
subject: 'rollback owner', description: 'must converge with membership', status: 'pending', owner: 'worker-2',
|
|
3405
|
+
}, cwd);
|
|
3406
|
+
const teamDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
3407
|
+
const configPath = join(teamDir, 'config.json');
|
|
3408
|
+
const manifestPath = join(teamDir, 'manifest.v2.json');
|
|
3409
|
+
const taskPath = join(teamDir, 'tasks', `task-${task.id}.json`);
|
|
3410
|
+
const oldConfig = await readFile(configPath, 'utf8');
|
|
3411
|
+
const oldManifest = await readFile(manifestPath, 'utf8');
|
|
3412
|
+
const oldTask = await readFile(taskPath, 'utf8');
|
|
3413
|
+
const oldConfigValue = JSON.parse(oldConfig);
|
|
3414
|
+
const nextConfig = { ...oldConfigValue, workers: oldConfigValue.workers.filter((worker) => worker.name !== 'worker-2') };
|
|
3415
|
+
const nextManifest = { ...JSON.parse(oldManifest), workers: nextConfig.workers, worker_count: nextConfig.workers.length };
|
|
3416
|
+
const nextTask = { ...JSON.parse(oldTask), owner: undefined, claim: undefined };
|
|
3417
|
+
await assert.rejects(withTeamTaskBarrier(teamName, cwd, () => commitTeamMembershipTaskTransaction(teamName, cwd, {
|
|
3418
|
+
baseGeneration: 0,
|
|
3419
|
+
tasks: [{ taskId: task.id, oldBytes: oldTask, newBytes: JSON.stringify(nextTask, null, 2) }],
|
|
3420
|
+
config: { oldBytes: oldConfig, newBytes: JSON.stringify(nextConfig, null, 2) },
|
|
3421
|
+
manifest: { oldBytes: oldManifest, newBytes: JSON.stringify(nextManifest, null, 2) },
|
|
3422
|
+
recoverToNewOnFailure: true,
|
|
3423
|
+
failRollbackPersistenceAfter: target,
|
|
3424
|
+
})), /rollback-persistence-failure/);
|
|
3425
|
+
assert.equal(existsSync(join(teamDir, '.membership-task-transaction.json')), true);
|
|
3426
|
+
await recoverTeamMembershipTaskTransaction(teamName, cwd);
|
|
3427
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3428
|
+
assert.equal((await readTeamManifestV2(teamName, cwd))?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3429
|
+
assert.equal((await readTask(teamName, task.id, cwd))?.owner, undefined);
|
|
3430
|
+
}
|
|
3431
|
+
finally {
|
|
3432
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3433
|
+
}
|
|
3434
|
+
});
|
|
3435
|
+
}
|
|
3436
|
+
it('discards precommit cleanup debt while removed workers remain canonical', async () => {
|
|
3437
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-precommit-debt-'));
|
|
3438
|
+
try {
|
|
3439
|
+
await initTeamState('precommit-debt', 'task', 'executor', 2, cwd);
|
|
3440
|
+
const config = await readTeamConfig('precommit-debt', cwd);
|
|
3441
|
+
assert.ok(config);
|
|
3442
|
+
if (!config)
|
|
3443
|
+
return;
|
|
3444
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'precommit-debt', '.scale-down-cleanup-debt.json');
|
|
3445
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3446
|
+
schema_version: 1,
|
|
3447
|
+
operation: 'scale_down',
|
|
3448
|
+
status: 'pending_teardown',
|
|
3449
|
+
removed_worker_names: ['worker-2'],
|
|
3450
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
3451
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
3452
|
+
}));
|
|
3453
|
+
const recovered = await reconcileScaleDownCleanupDebt('precommit-debt', cwd, config);
|
|
3454
|
+
assert.deepEqual(recovered, { ok: true });
|
|
3455
|
+
assert.equal(existsSync(debtPath), false);
|
|
3456
|
+
assert.equal((await readTeamConfig('precommit-debt', cwd))?.workers.some((worker) => worker.name === 'worker-2'), true);
|
|
3457
|
+
}
|
|
3458
|
+
finally {
|
|
3459
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3460
|
+
}
|
|
3461
|
+
});
|
|
3462
|
+
it('commits forward state and records unresolved debt when proof is lost after a pane kill', async () => {
|
|
3463
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-success-proof-loss-'));
|
|
3464
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-success-proof-loss-bin-'));
|
|
3465
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3466
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3467
|
+
const proofLossMarkerPath = join(fakeBinDir, 'proof-loss');
|
|
3468
|
+
const previousPath = process.env.PATH;
|
|
3469
|
+
try {
|
|
3470
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3471
|
+
set -eu
|
|
3472
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3473
|
+
case "\${1:-}" in
|
|
3474
|
+
show-option) echo 'team:scale-down' ;;
|
|
3475
|
+
list-panes)
|
|
3476
|
+
if [ -f "${proofLossMarkerPath}.recovery" ]; then
|
|
3477
|
+
if [ ! -f "${proofLossMarkerPath}.killed-13" ]; then printf '%s\t%s\t%s\n' '%13' '0' '42413'; fi
|
|
3478
|
+
if [ ! -f "${proofLossMarkerPath}.killed-14" ]; then printf '%s\t%s\t%s\n' '%14' '0' '42414'; fi
|
|
3479
|
+
else
|
|
3480
|
+
if [ -f "${proofLossMarkerPath}.killed" ]; then exit 1; fi
|
|
3481
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3482
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3483
|
+
fi
|
|
3484
|
+
;;
|
|
3485
|
+
show-option) printf '%s\n' 'team:success-proof-loss' ;;
|
|
3486
|
+
kill-pane)
|
|
3487
|
+
if [ -f "${proofLossMarkerPath}.recovery" ]; then
|
|
3488
|
+
if [ "$3" = '%13' ]; then : > "${proofLossMarkerPath}.killed-13"; fi
|
|
3489
|
+
if [ "$3" = '%14' ]; then : > "${proofLossMarkerPath}.killed-14"; fi
|
|
3490
|
+
elif [ "$3" = '%13' ]; then
|
|
3491
|
+
: > "${proofLossMarkerPath}.killed"
|
|
3492
|
+
fi
|
|
3493
|
+
;;
|
|
3494
|
+
esac
|
|
3495
|
+
exit 0
|
|
3496
|
+
`);
|
|
3497
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3498
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3499
|
+
await initTeamState('success-proof-loss', 'task', 'executor', 3, cwd);
|
|
3500
|
+
const config = await readTeamConfig('success-proof-loss', cwd);
|
|
3501
|
+
assert.ok(config);
|
|
3502
|
+
if (!config)
|
|
3503
|
+
return;
|
|
3504
|
+
config.workers[1].pane_id = '%13';
|
|
3505
|
+
config.workers[1].pid = 42413;
|
|
3506
|
+
config.workers[2].pane_id = '%14';
|
|
3507
|
+
config.workers[2].pid = 42414;
|
|
3508
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3509
|
+
await saveTeamConfig(config, cwd);
|
|
3510
|
+
await writeFile(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', 'workers', 'worker-2', 'identity.json'), JSON.stringify({ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }));
|
|
3511
|
+
await writeFile(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', 'workers', 'worker-3', 'identity.json'), JSON.stringify({ name: 'worker-3', index: 3, pane_id: '%14', pid: 42414 }));
|
|
3512
|
+
const resolvedTask = await createTask('success-proof-loss', {
|
|
3513
|
+
subject: 'resolved owner task', description: 'must be reclaimed', status: 'pending', owner: 'worker-2',
|
|
3514
|
+
}, cwd);
|
|
3515
|
+
const unresolvedTask = await createTask('success-proof-loss', {
|
|
3516
|
+
subject: 'unresolved owner task', description: 'must be preserved', status: 'pending', owner: 'worker-3',
|
|
3517
|
+
}, cwd);
|
|
3518
|
+
const resolvedClaim = await claimTask('success-proof-loss', resolvedTask.id, 'worker-2', resolvedTask.version ?? 1, cwd);
|
|
3519
|
+
assert.equal(resolvedClaim.ok, true);
|
|
3520
|
+
const worker3StatusPath = join(cwd, '.omx', 'state', 'team', 'success-proof-loss', 'workers', 'worker-3', 'status.json');
|
|
3521
|
+
const worker3Raw = '{"reason":"three",\n"updated_at":"2026-07-14T00:00:01.000Z", "state":"idle"}\n';
|
|
3522
|
+
await writeFile(worker3StatusPath, worker3Raw);
|
|
3523
|
+
const result = await scaleDown('success-proof-loss', cwd, { workerNames: ['worker-2', 'worker-3'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3524
|
+
assert.equal(result.ok, false);
|
|
3525
|
+
if (!result.ok)
|
|
3526
|
+
assert.match(result.error, /^scale_down_pane_proof_unavailable:%13:query_failed/);
|
|
3527
|
+
assert.deepEqual((await readTeamConfig('success-proof-loss', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3528
|
+
assert.equal(existsSync(worker3StatusPath), true);
|
|
3529
|
+
assert.equal(await readFile(worker3StatusPath, 'utf8'), worker3Raw);
|
|
3530
|
+
assert.equal((await readTask('success-proof-loss', resolvedTask.id, cwd))?.owner, undefined);
|
|
3531
|
+
assert.equal((await readTask('success-proof-loss', unresolvedTask.id, cwd))?.owner, undefined);
|
|
3532
|
+
const retainedTask = await readTask('success-proof-loss', resolvedTask.id, cwd);
|
|
3533
|
+
assert.equal(retainedTask?.status, 'pending');
|
|
3534
|
+
assert.equal(retainedTask?.claim, undefined);
|
|
3535
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3536
|
+
assert.deepEqual(debt.unresolved_panes.map((pane) => pane.pane_id), ['%13', '%14']);
|
|
3537
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3538
|
+
assert.deepEqual(tmuxCommands.filter((command) => command.startsWith('kill-pane ')), ['kill-pane -t %13']);
|
|
3539
|
+
const killIndex = tmuxCommands.indexOf('kill-pane -t %13');
|
|
3540
|
+
assert.ok(killIndex > 2);
|
|
3541
|
+
assert.equal(tmuxCommands[killIndex - 3], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
3542
|
+
assert.equal(tmuxCommands[killIndex - 2], 'show-option -qv -p -t %13 @omx_team_pane_owner_id');
|
|
3543
|
+
assert.equal(tmuxCommands[killIndex - 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
3544
|
+
assert.equal(tmuxCommands[killIndex + 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
3545
|
+
await writeFile(`${proofLossMarkerPath}.recovery`, '1');
|
|
3546
|
+
await rm(`${proofLossMarkerPath}.killed`, { force: true });
|
|
3547
|
+
const committedConfig = await readTeamConfig('success-proof-loss', cwd);
|
|
3548
|
+
assert.ok(committedConfig);
|
|
3549
|
+
if (!committedConfig)
|
|
3550
|
+
return;
|
|
3551
|
+
const recovered = await reconcileScaleDownCleanupDebt('success-proof-loss', cwd, committedConfig);
|
|
3552
|
+
assert.deepEqual(recovered, { ok: true });
|
|
3553
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', '.scale-down-cleanup-debt.json')), false);
|
|
3554
|
+
const recoveredCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3555
|
+
assert.deepEqual(recoveredCommands.filter((command) => command.startsWith('kill-pane ')), [
|
|
3556
|
+
'kill-pane -t %13',
|
|
3557
|
+
'kill-pane -t %13',
|
|
3558
|
+
'kill-pane -t %14',
|
|
3559
|
+
]);
|
|
3560
|
+
}
|
|
3561
|
+
finally {
|
|
3562
|
+
if (typeof previousPath === 'string')
|
|
3563
|
+
process.env.PATH = previousPath;
|
|
3564
|
+
else
|
|
3565
|
+
delete process.env.PATH;
|
|
3566
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3567
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3568
|
+
}
|
|
3569
|
+
});
|
|
3570
|
+
it('commits forward membership and retains exact debt when a later pane teardown fails', async () => {
|
|
3571
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-gone-kill-fail-'));
|
|
3572
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-gone-kill-fail-bin-'));
|
|
3573
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3574
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3575
|
+
const previousPath = process.env.PATH;
|
|
3576
|
+
try {
|
|
3577
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3578
|
+
set -eu
|
|
3579
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3580
|
+
case "\${1:-}" in
|
|
3581
|
+
show-option) echo 'team:scale-down' ;;
|
|
3582
|
+
list-panes)
|
|
3583
|
+
if [ -f "${tmuxLogPath}.killed-%13" ]; then
|
|
3584
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3585
|
+
else
|
|
3586
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3587
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3588
|
+
fi
|
|
3589
|
+
;;
|
|
3590
|
+
kill-pane)
|
|
3591
|
+
if [ "$3" = '%13' ]; then : > "${tmuxLogPath}.killed-%13"; exit 0; fi
|
|
3592
|
+
exit 1
|
|
3593
|
+
;;
|
|
3594
|
+
esac
|
|
3595
|
+
`);
|
|
3596
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3597
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3598
|
+
await initTeamState('gone-kill-fail', 'task', 'executor', 3, cwd);
|
|
3599
|
+
const config = await readTeamConfig('gone-kill-fail', cwd);
|
|
3600
|
+
assert.ok(config);
|
|
3601
|
+
if (!config)
|
|
3602
|
+
return;
|
|
3603
|
+
config.workers[1].pane_id = '%13';
|
|
3604
|
+
config.workers[1].pid = 42413;
|
|
3605
|
+
config.workers[2].pane_id = '%14';
|
|
3606
|
+
config.workers[2].pid = 42414;
|
|
3607
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3608
|
+
await saveTeamConfig(config, cwd);
|
|
3609
|
+
const resolvedTask = await createTask('gone-kill-fail', {
|
|
3610
|
+
subject: 'gone owner task', description: 'must be reclaimed', status: 'pending', owner: 'worker-2',
|
|
3611
|
+
}, cwd);
|
|
3612
|
+
const unresolvedTask = await createTask('gone-kill-fail', {
|
|
3613
|
+
subject: 'failed owner task', description: 'must be preserved', status: 'pending', owner: 'worker-3',
|
|
3614
|
+
}, cwd);
|
|
3615
|
+
const resolvedClaim = await claimTask('gone-kill-fail', resolvedTask.id, 'worker-2', resolvedTask.version ?? 1, cwd);
|
|
3616
|
+
assert.equal(resolvedClaim.ok, true);
|
|
3617
|
+
const worker3StatusPath = join(cwd, '.omx', 'state', 'team', 'gone-kill-fail', 'workers', 'worker-3', 'status.json');
|
|
3618
|
+
await writeFile(worker3StatusPath, '{\n "state" : "idle", "reason":"three", "updated_at":"2026-07-14T00:00:01.000Z"\n}\n');
|
|
3619
|
+
const result = await scaleDown('gone-kill-fail', cwd, { workerNames: ['worker-2', 'worker-3'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3620
|
+
assert.equal(result.ok, false);
|
|
3621
|
+
if (!result.ok)
|
|
3622
|
+
assert.match(result.error, /^scale_down_cleanup_debt:pane_teardown_failed:%14/);
|
|
3623
|
+
assert.deepEqual((await readTeamConfig('gone-kill-fail', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3624
|
+
assert.equal(existsSync(worker3StatusPath), true);
|
|
3625
|
+
assert.equal(await readFile(worker3StatusPath, 'utf8'), '{\n "state" : "idle", "reason":"three", "updated_at":"2026-07-14T00:00:01.000Z"\n}\n');
|
|
3626
|
+
assert.equal((await readTask('gone-kill-fail', resolvedTask.id, cwd))?.owner, undefined);
|
|
3627
|
+
assert.equal((await readTask('gone-kill-fail', unresolvedTask.id, cwd))?.owner, undefined);
|
|
3628
|
+
const retainedTask = await readTask('gone-kill-fail', resolvedTask.id, cwd);
|
|
3629
|
+
assert.equal(retainedTask?.status, 'pending');
|
|
3630
|
+
assert.equal(retainedTask?.claim, undefined);
|
|
3631
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'gone-kill-fail', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3632
|
+
assert.deepEqual(debt.unresolved_panes.map((pane) => pane.pane_id), ['%14']);
|
|
3633
|
+
assert.deepEqual(await readScaleUpTmuxLogCommands(tmuxLogPath), [
|
|
3634
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3635
|
+
'show-option -qv -p -t %13 @omx_team_pane_owner_id',
|
|
3636
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3637
|
+
'kill-pane -t %13',
|
|
3638
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3639
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3640
|
+
'show-option -qv -p -t %14 @omx_team_pane_owner_id',
|
|
3641
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3642
|
+
'kill-pane -t %14',
|
|
3643
|
+
]);
|
|
3644
|
+
}
|
|
3645
|
+
finally {
|
|
3646
|
+
if (typeof previousPath === 'string')
|
|
3647
|
+
process.env.PATH = previousPath;
|
|
3648
|
+
else
|
|
3649
|
+
delete process.env.PATH;
|
|
3650
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3651
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3652
|
+
}
|
|
3653
|
+
});
|
|
3654
|
+
for (const [name, priorRaw] of [
|
|
3655
|
+
['absent status artifact', undefined],
|
|
3656
|
+
['malformed status artifact', '{"state":\n'],
|
|
3657
|
+
['noncanonical valid status artifact', '{\n "updated_at" : "2026-07-14T00:00:00.000Z",\n "reason":"waiting for assignment", "state" : "idle"\n}\n'],
|
|
3658
|
+
]) {
|
|
3659
|
+
it(`scaleDown restores the exact ${name} after kill-pane failure`, async () => {
|
|
3660
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-status-artifact-'));
|
|
3661
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-status-artifact-bin-'));
|
|
3662
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3663
|
+
const previousPath = process.env.PATH;
|
|
3664
|
+
try {
|
|
3665
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3666
|
+
case "\${1:-}" in
|
|
3667
|
+
show-option) echo 'team:scale-down' ;;
|
|
3668
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%13' '0' '42413' ;;
|
|
3669
|
+
kill-pane) exit 1 ;;
|
|
3670
|
+
esac
|
|
3671
|
+
`);
|
|
3672
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3673
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3674
|
+
await initTeamState(`status-${name.startsWith('absent') ? 'absent' : 'malformed'}`, 'task', 'executor', 2, cwd);
|
|
3675
|
+
const teamName = `status-${name.startsWith('absent') ? 'absent' : 'malformed'}`;
|
|
3676
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
3677
|
+
assert.ok(config);
|
|
3678
|
+
if (!config)
|
|
3679
|
+
return;
|
|
3680
|
+
config.workers[1].pane_id = '%13';
|
|
3681
|
+
config.workers[1].pid = 42413;
|
|
3682
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3683
|
+
await saveTeamConfig(config, cwd);
|
|
3684
|
+
const statusPath = join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2', 'status.json');
|
|
3685
|
+
if (priorRaw === undefined) {
|
|
3686
|
+
await rm(statusPath, { force: true });
|
|
3687
|
+
}
|
|
3688
|
+
else {
|
|
3689
|
+
await writeFile(statusPath, priorRaw);
|
|
3690
|
+
}
|
|
3691
|
+
const result = await scaleDown(teamName, cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3692
|
+
assert.equal(result.ok, false);
|
|
3693
|
+
if (!result.ok)
|
|
3694
|
+
assert.match(result.error, /^scale_down_cleanup_debt:pane_teardown_failed:%13/);
|
|
3695
|
+
assert.equal(existsSync(statusPath), priorRaw !== undefined);
|
|
3696
|
+
if (priorRaw !== undefined)
|
|
3697
|
+
assert.equal(await readFile(statusPath, 'utf8'), priorRaw);
|
|
3698
|
+
}
|
|
3699
|
+
finally {
|
|
3700
|
+
if (typeof previousPath === 'string')
|
|
3701
|
+
process.env.PATH = previousPath;
|
|
3702
|
+
else
|
|
3703
|
+
delete process.env.PATH;
|
|
3704
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3705
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3706
|
+
}
|
|
3707
|
+
});
|
|
3708
|
+
}
|
|
3709
|
+
it('uses canonical membership rather than a stale caller config before consuming precommit debt', async () => {
|
|
3710
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-canonical-debt-'));
|
|
3711
|
+
try {
|
|
3712
|
+
await initTeamState('canonical-debt', 'task', 'executor', 2, cwd);
|
|
3713
|
+
const canonical = await readTeamConfig('canonical-debt', cwd);
|
|
3714
|
+
assert.ok(canonical);
|
|
3715
|
+
if (!canonical)
|
|
3716
|
+
return;
|
|
3717
|
+
const staleCaller = structuredClone(canonical);
|
|
3718
|
+
staleCaller.workers = staleCaller.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3719
|
+
staleCaller.worker_count = staleCaller.workers.length;
|
|
3720
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'canonical-debt', 'workers', 'worker-2');
|
|
3721
|
+
const sentinelPath = join(workerDir, 'caller-must-not-authorize');
|
|
3722
|
+
await writeFile(sentinelPath, 'preserve');
|
|
3723
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'canonical-debt', '.scale-down-cleanup-debt.json');
|
|
3724
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3725
|
+
schema_version: 1,
|
|
3726
|
+
operation: 'scale_down',
|
|
3727
|
+
status: 'pending_teardown',
|
|
3728
|
+
removed_worker_names: ['worker-2'],
|
|
3729
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
3730
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
3731
|
+
}));
|
|
3732
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('canonical-debt', cwd, staleCaller), { ok: true });
|
|
3733
|
+
assert.equal(existsSync(debtPath), false);
|
|
3734
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'preserve');
|
|
3735
|
+
assert.equal((await readTeamConfig('canonical-debt', cwd))?.workers.some((worker) => worker.name === 'worker-2'), true);
|
|
3736
|
+
}
|
|
3737
|
+
finally {
|
|
3738
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3739
|
+
}
|
|
3740
|
+
});
|
|
3741
|
+
it('converges pane-less crash debt through worker status and directory cleanup', async () => {
|
|
3742
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-zero-pane-debt-'));
|
|
3743
|
+
try {
|
|
3744
|
+
await initTeamState('zero-pane-debt', 'task', 'executor', 2, cwd);
|
|
3745
|
+
const config = await readTeamConfig('zero-pane-debt', cwd);
|
|
3746
|
+
assert.ok(config);
|
|
3747
|
+
if (!config)
|
|
3748
|
+
return;
|
|
3749
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3750
|
+
config.worker_count = config.workers.length;
|
|
3751
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3752
|
+
await saveTeamConfig(config, cwd);
|
|
3753
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'zero-pane-debt', 'workers', 'worker-2');
|
|
3754
|
+
await mkdir(workerDir, { recursive: true });
|
|
3755
|
+
await writeFile(join(workerDir, 'status.json'), '{"state":"draining"}');
|
|
3756
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'zero-pane-debt', '.scale-down-cleanup-debt.json');
|
|
3757
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3758
|
+
schema_version: 1,
|
|
3759
|
+
operation: 'scale_down',
|
|
3760
|
+
status: 'resource_cleanup_pending',
|
|
3761
|
+
removed_worker_names: ['worker-2'],
|
|
3762
|
+
workers: [],
|
|
3763
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
3764
|
+
}));
|
|
3765
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('zero-pane-debt', cwd, config), { ok: true });
|
|
3766
|
+
assert.equal(existsSync(workerDir), false);
|
|
3767
|
+
assert.equal(existsSync(debtPath), false);
|
|
3768
|
+
}
|
|
3769
|
+
finally {
|
|
3770
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3771
|
+
}
|
|
3772
|
+
});
|
|
3773
|
+
it('converges resource debt after a crash following successful worktree cleanup', async () => {
|
|
3774
|
+
const repo = await initRepo();
|
|
3775
|
+
try {
|
|
3776
|
+
await initTeamState('resource-retry', 'task', 'executor', 2, repo);
|
|
3777
|
+
const config = await readTeamConfig('resource-retry', repo);
|
|
3778
|
+
assert.ok(config);
|
|
3779
|
+
if (!config)
|
|
3780
|
+
return;
|
|
3781
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3782
|
+
config.worker_count = config.workers.length;
|
|
3783
|
+
await saveTeamConfig(config, repo);
|
|
3784
|
+
const worktreePath = join(repo, '.omx', 'team', 'resource-retry', 'worktrees', 'worker-2');
|
|
3785
|
+
const workerDir = join(repo, '.omx', 'state', 'team', 'resource-retry', 'workers', 'worker-2');
|
|
3786
|
+
await mkdir(workerDir, { recursive: true });
|
|
3787
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktreePath], { cwd: repo, stdio: 'ignore' });
|
|
3788
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'resource-retry', '.scale-down-cleanup-debt.json');
|
|
3789
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3790
|
+
schema_version: 1,
|
|
3791
|
+
operation: 'scale_down',
|
|
3792
|
+
status: 'resource_cleanup_pending',
|
|
3793
|
+
removed_worker_names: ['worker-2'],
|
|
3794
|
+
workers: [],
|
|
3795
|
+
resource_workers: [{
|
|
3796
|
+
name: 'worker-2', worktree_path: worktreePath, worktree_repo_root: repo,
|
|
3797
|
+
worktree_detached: true, worktree_created: true,
|
|
3798
|
+
}],
|
|
3799
|
+
}));
|
|
3800
|
+
// Simulate a crash after git worktree removal but before worker-state
|
|
3801
|
+
// cleanup and debt deletion. The retry must treat the exact absent target
|
|
3802
|
+
// as converged rather than attempting to remove a replacement.
|
|
3803
|
+
execFileSync('git', ['worktree', 'remove', '--force', worktreePath], { cwd: repo, stdio: 'ignore' });
|
|
3804
|
+
assert.equal(existsSync(worktreePath), false);
|
|
3805
|
+
assert.equal(existsSync(workerDir), true);
|
|
3806
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('resource-retry', repo, config), { ok: true });
|
|
3807
|
+
assert.equal(existsSync(worktreePath), false);
|
|
3808
|
+
assert.equal(existsSync(workerDir), false);
|
|
3809
|
+
assert.equal(existsSync(debtPath), false);
|
|
3810
|
+
}
|
|
3811
|
+
finally {
|
|
3812
|
+
await rm(repo, { recursive: true, force: true });
|
|
3813
|
+
}
|
|
3814
|
+
});
|
|
3815
|
+
it('rejects malformed worktree debt without mutating worker resources', async () => {
|
|
3816
|
+
const repo = await initRepo();
|
|
3817
|
+
try {
|
|
3818
|
+
await initTeamState('malformed-resource-debt', 'task', 'executor', 2, repo);
|
|
3819
|
+
const config = await readTeamConfig('malformed-resource-debt', repo);
|
|
3820
|
+
assert.ok(config);
|
|
3821
|
+
if (!config)
|
|
3822
|
+
return;
|
|
3823
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3824
|
+
config.worker_count = config.workers.length;
|
|
3825
|
+
await saveTeamConfig(config, repo);
|
|
3826
|
+
const workerDir = join(repo, '.omx', 'state', 'team', 'malformed-resource-debt', 'workers', 'worker-2');
|
|
3827
|
+
const sentinelPath = join(workerDir, 'sentinel');
|
|
3828
|
+
await mkdir(workerDir, { recursive: true });
|
|
3829
|
+
await writeFile(sentinelPath, 'preserve');
|
|
3830
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'malformed-resource-debt', '.scale-down-cleanup-debt.json');
|
|
3831
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3832
|
+
schema_version: 1,
|
|
3833
|
+
operation: 'scale_down',
|
|
3834
|
+
status: 'resource_cleanup_pending',
|
|
3835
|
+
removed_worker_names: ['worker-2'],
|
|
3836
|
+
workers: [],
|
|
3837
|
+
resource_workers: [{
|
|
3838
|
+
name: 'worker-2',
|
|
3839
|
+
worktree_path: join(repo, '.omx', 'team', 'malformed-resource-debt', 'worktrees', '..', 'worker-2'),
|
|
3840
|
+
worktree_repo_root: repo,
|
|
3841
|
+
worktree_detached: true,
|
|
3842
|
+
worktree_created: true,
|
|
3843
|
+
}],
|
|
3844
|
+
}));
|
|
3845
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('malformed-resource-debt', repo, config), {
|
|
3846
|
+
ok: false,
|
|
3847
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3848
|
+
});
|
|
3849
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'preserve');
|
|
3850
|
+
assert.equal(existsSync(debtPath), true);
|
|
3851
|
+
}
|
|
3852
|
+
finally {
|
|
3853
|
+
await rm(repo, { recursive: true, force: true });
|
|
3854
|
+
}
|
|
3855
|
+
});
|
|
3856
|
+
it('rejects foreign worktree debt without mutating the foreign location', async () => {
|
|
3857
|
+
const repo = await initRepo();
|
|
3858
|
+
const foreign = await mkdtemp(join(tmpdir(), 'omx-scale-down-foreign-'));
|
|
3859
|
+
try {
|
|
3860
|
+
await initTeamState('foreign-resource-debt', 'task', 'executor', 2, repo);
|
|
3861
|
+
const config = await readTeamConfig('foreign-resource-debt', repo);
|
|
3862
|
+
assert.ok(config);
|
|
3863
|
+
if (!config)
|
|
3864
|
+
return;
|
|
3865
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3866
|
+
config.worker_count = config.workers.length;
|
|
3867
|
+
await saveTeamConfig(config, repo);
|
|
3868
|
+
const sentinelPath = join(foreign, 'AGENTS.md');
|
|
3869
|
+
await writeFile(sentinelPath, 'foreign');
|
|
3870
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'foreign-resource-debt', '.scale-down-cleanup-debt.json');
|
|
3871
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3872
|
+
schema_version: 1,
|
|
3873
|
+
operation: 'scale_down',
|
|
3874
|
+
status: 'resource_cleanup_pending',
|
|
3875
|
+
removed_worker_names: ['worker-2'],
|
|
3876
|
+
workers: [],
|
|
3877
|
+
resource_workers: [{
|
|
3878
|
+
name: 'worker-2', worktree_path: foreign, worktree_repo_root: repo,
|
|
3879
|
+
worktree_detached: true, worktree_created: true,
|
|
3880
|
+
}],
|
|
3881
|
+
}));
|
|
3882
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('foreign-resource-debt', repo, config), {
|
|
3883
|
+
ok: false,
|
|
3884
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3885
|
+
});
|
|
3886
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'foreign');
|
|
3887
|
+
assert.equal(existsSync(debtPath), true);
|
|
3888
|
+
}
|
|
3889
|
+
finally {
|
|
3890
|
+
await rm(repo, { recursive: true, force: true });
|
|
3891
|
+
await rm(foreign, { recursive: true, force: true });
|
|
3892
|
+
}
|
|
3893
|
+
});
|
|
3894
|
+
it('rejects symlink-parent worktree debt without mutating the linked location', async () => {
|
|
3895
|
+
const repo = await initRepo();
|
|
3896
|
+
const foreign = await mkdtemp(join(tmpdir(), 'omx-scale-down-symlink-'));
|
|
3897
|
+
try {
|
|
3898
|
+
await initTeamState('symlink-resource-debt', 'task', 'executor', 2, repo);
|
|
3899
|
+
const config = await readTeamConfig('symlink-resource-debt', repo);
|
|
3900
|
+
assert.ok(config);
|
|
3901
|
+
if (!config)
|
|
3902
|
+
return;
|
|
3903
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3904
|
+
config.worker_count = config.workers.length;
|
|
3905
|
+
await saveTeamConfig(config, repo);
|
|
3906
|
+
const expectedWorktreeRoot = join(repo, '.omx', 'team', 'symlink-resource-debt', 'worktrees');
|
|
3907
|
+
await mkdir(join(repo, '.omx', 'team', 'symlink-resource-debt'), { recursive: true });
|
|
3908
|
+
await symlink(foreign, expectedWorktreeRoot);
|
|
3909
|
+
const sentinelPath = join(foreign, 'AGENTS.md');
|
|
3910
|
+
await writeFile(sentinelPath, 'foreign');
|
|
3911
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'symlink-resource-debt', '.scale-down-cleanup-debt.json');
|
|
3912
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3913
|
+
schema_version: 1,
|
|
3914
|
+
operation: 'scale_down',
|
|
3915
|
+
status: 'resource_cleanup_pending',
|
|
3916
|
+
removed_worker_names: ['worker-2'],
|
|
3917
|
+
workers: [],
|
|
3918
|
+
resource_workers: [{
|
|
3919
|
+
name: 'worker-2', worktree_path: join(expectedWorktreeRoot, 'worker-2'), worktree_repo_root: repo,
|
|
3920
|
+
worktree_detached: true, worktree_created: true,
|
|
3921
|
+
}],
|
|
3922
|
+
}));
|
|
3923
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('symlink-resource-debt', repo, config), {
|
|
3924
|
+
ok: false,
|
|
3925
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3926
|
+
});
|
|
3927
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'foreign');
|
|
3928
|
+
assert.equal(existsSync(debtPath), true);
|
|
3929
|
+
}
|
|
3930
|
+
finally {
|
|
3931
|
+
await rm(repo, { recursive: true, force: true });
|
|
3932
|
+
await rm(foreign, { recursive: true, force: true });
|
|
3933
|
+
}
|
|
3934
|
+
});
|
|
3935
|
+
it('rejects cleanup debt that collides with a surviving canonical worker before any tmux effect', async () => {
|
|
3936
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-survivor-collision-'));
|
|
3937
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-survivor-collision-bin-'));
|
|
3938
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3939
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3940
|
+
const previousPath = process.env.PATH;
|
|
3941
|
+
try {
|
|
3942
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3943
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3944
|
+
exit 99
|
|
3945
|
+
`);
|
|
3946
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3947
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3948
|
+
await initTeamState('debt-survivor-collision', 'task', 'executor', 3, cwd);
|
|
3949
|
+
const config = await readTeamConfig('debt-survivor-collision', cwd);
|
|
3950
|
+
assert.ok(config);
|
|
3951
|
+
if (!config)
|
|
3952
|
+
return;
|
|
3953
|
+
config.workers[0].pane_id = '%13';
|
|
3954
|
+
config.workers[0].pid = 42413;
|
|
3955
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2' && worker.name !== 'worker-3');
|
|
3956
|
+
config.worker_count = config.workers.length;
|
|
3957
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3958
|
+
await saveTeamConfig(config, cwd);
|
|
3959
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-survivor-collision', '.scale-down-cleanup-debt.json');
|
|
3960
|
+
const debt = {
|
|
3961
|
+
schema_version: 1,
|
|
3962
|
+
operation: 'scale_down',
|
|
3963
|
+
status: 'pending_teardown',
|
|
3964
|
+
removed_worker_names: ['worker-2', 'worker-3'],
|
|
3965
|
+
workers: [
|
|
3966
|
+
{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42414 },
|
|
3967
|
+
{ name: 'worker-3', index: 3, pane_id: '%14', pid: 42413 },
|
|
3968
|
+
],
|
|
3969
|
+
resource_workers: [{ name: 'worker-2' }, { name: 'worker-3' }],
|
|
3970
|
+
};
|
|
3971
|
+
await writeFile(debtPath, JSON.stringify(debt));
|
|
3972
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-survivor-collision', cwd, config), {
|
|
3973
|
+
ok: false,
|
|
3974
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3975
|
+
});
|
|
3976
|
+
assert.deepEqual(JSON.parse(await readFile(debtPath, 'utf8')), debt);
|
|
3977
|
+
assert.equal(existsSync(tmuxLogPath), false);
|
|
3978
|
+
}
|
|
3979
|
+
finally {
|
|
3980
|
+
if (typeof previousPath === 'string')
|
|
3981
|
+
process.env.PATH = previousPath;
|
|
3982
|
+
else
|
|
3983
|
+
delete process.env.PATH;
|
|
3984
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3985
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3986
|
+
}
|
|
3987
|
+
});
|
|
3988
|
+
for (const [name, livePid, ownerMode, expectedReason] of [
|
|
3989
|
+
['refuses an unrelated current pane occupant', 42413, 'foreign', 'team_owner_mismatch'],
|
|
3990
|
+
['refuses a PID-reused pane even with the Team owner tag', 52413, 'team', 'pane_pid_changed'],
|
|
3991
|
+
['fails closed when Team owner authorization is unavailable', 42413, 'unavailable', 'team_owner_unavailable'],
|
|
3992
|
+
]) {
|
|
3993
|
+
it(name, async () => {
|
|
3994
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-pane-authority-'));
|
|
3995
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-pane-authority-bin-'));
|
|
3996
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3997
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3998
|
+
const previousPath = process.env.PATH;
|
|
3999
|
+
try {
|
|
4000
|
+
const ownerScript = ownerMode === 'unavailable'
|
|
4001
|
+
? 'echo owner-query-failed >&2; exit 2'
|
|
4002
|
+
: `printf '%s\\n' '${ownerMode === 'team' ? 'team:debt-pane-authority' : 'team:another-team'}'`;
|
|
4003
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
4004
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
4005
|
+
case "$1" in
|
|
4006
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%13' '0' '${livePid}' ;;
|
|
4007
|
+
show-option) ${ownerScript} ;;
|
|
4008
|
+
kill-pane) echo unexpected-kill >&2; exit 99 ;;
|
|
4009
|
+
esac
|
|
4010
|
+
`);
|
|
4011
|
+
await chmod(tmuxStubPath, 0o755);
|
|
4012
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
4013
|
+
await initTeamState('debt-pane-authority', 'task', 'executor', 2, cwd);
|
|
4014
|
+
const config = await readTeamConfig('debt-pane-authority', cwd);
|
|
4015
|
+
assert.ok(config);
|
|
4016
|
+
if (!config)
|
|
4017
|
+
return;
|
|
4018
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
4019
|
+
config.worker_count = config.workers.length;
|
|
4020
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
4021
|
+
await saveTeamConfig(config, cwd);
|
|
4022
|
+
await writeFile(join(cwd, '.omx', 'state', 'team', 'debt-pane-authority', 'workers', 'worker-2', 'identity.json'), JSON.stringify({ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }));
|
|
4023
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-pane-authority', '.scale-down-cleanup-debt.json');
|
|
4024
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4025
|
+
schema_version: 1, operation: 'scale_down', status: 'pending_teardown',
|
|
4026
|
+
removed_worker_names: ['worker-2'],
|
|
4027
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
4028
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4029
|
+
}));
|
|
4030
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-pane-authority', cwd, config), {
|
|
4031
|
+
ok: false, error: 'scale_down_cleanup_debt_unresolved:%13',
|
|
4032
|
+
});
|
|
4033
|
+
const debt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
4034
|
+
assert.ok(debt.reasons.includes(`%13:${expectedReason}`));
|
|
4035
|
+
const commands = existsSync(tmuxLogPath) ? await readFile(tmuxLogPath, 'utf8') : '';
|
|
4036
|
+
assert.doesNotMatch(commands, /kill-pane/);
|
|
4037
|
+
}
|
|
4038
|
+
finally {
|
|
4039
|
+
if (typeof previousPath === 'string')
|
|
4040
|
+
process.env.PATH = previousPath;
|
|
4041
|
+
else
|
|
4042
|
+
delete process.env.PATH;
|
|
4043
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4044
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
4045
|
+
}
|
|
4046
|
+
});
|
|
4047
|
+
}
|
|
4048
|
+
it('rejects mismatched worker pane bindings before any recovery mutation', async () => {
|
|
4049
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-mismatched-binding-'));
|
|
4050
|
+
try {
|
|
4051
|
+
await initTeamState('debt-mismatched-binding', 'task', 'executor', 2, cwd);
|
|
4052
|
+
const config = await readTeamConfig('debt-mismatched-binding', cwd);
|
|
4053
|
+
assert.ok(config);
|
|
4054
|
+
if (!config)
|
|
4055
|
+
return;
|
|
4056
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
4057
|
+
config.worker_count = config.workers.length;
|
|
4058
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
4059
|
+
await saveTeamConfig(config, cwd);
|
|
4060
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'debt-mismatched-binding', 'workers', 'worker-2');
|
|
4061
|
+
const sentinel = join(workerDir, 'must-remain');
|
|
4062
|
+
await writeFile(sentinel, 'preserve');
|
|
4063
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-mismatched-binding', '.scale-down-cleanup-debt.json');
|
|
4064
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4065
|
+
schema_version: 1, operation: 'scale_down', status: 'unresolved',
|
|
4066
|
+
removed_worker_names: ['worker-2'],
|
|
4067
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
4068
|
+
unresolved_panes: [{ name: 'worker-2', index: 2, pane_id: '%14', pid: 42413 }],
|
|
4069
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4070
|
+
}));
|
|
4071
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-mismatched-binding', cwd, config), {
|
|
4072
|
+
ok: false, error: 'scale_down_cleanup_debt_malformed',
|
|
4073
|
+
});
|
|
4074
|
+
assert.equal(await readFile(sentinel, 'utf8'), 'preserve');
|
|
4075
|
+
assert.equal(existsSync(debtPath), true);
|
|
4076
|
+
}
|
|
4077
|
+
finally {
|
|
4078
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4079
|
+
}
|
|
4080
|
+
});
|
|
4081
|
+
it('converges a PID-less legacy pane record only after a fresh global gone proof', async () => {
|
|
4082
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-legacy-gone-'));
|
|
4083
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-legacy-gone-bin-'));
|
|
4084
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
4085
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
4086
|
+
const previousPath = process.env.PATH;
|
|
4087
|
+
try {
|
|
4088
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
4089
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
4090
|
+
case "$1" in list-panes) exit 0 ;; kill-pane) exit 99 ;; esac
|
|
4091
|
+
`);
|
|
4092
|
+
await chmod(tmuxStubPath, 0o755);
|
|
4093
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
4094
|
+
await initTeamState('debt-legacy-gone', 'task', 'executor', 2, cwd);
|
|
4095
|
+
const config = await readTeamConfig('debt-legacy-gone', cwd);
|
|
4096
|
+
assert.ok(config);
|
|
4097
|
+
if (!config)
|
|
4098
|
+
return;
|
|
4099
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
4100
|
+
config.worker_count = config.workers.length;
|
|
4101
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
4102
|
+
await saveTeamConfig(config, cwd);
|
|
4103
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-legacy-gone', '.scale-down-cleanup-debt.json');
|
|
4104
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4105
|
+
schema_version: 1, operation: 'scale_down', status: 'unresolved',
|
|
4106
|
+
removed_worker_names: ['worker-2'],
|
|
4107
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: null }],
|
|
4108
|
+
unresolved_panes: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: null }],
|
|
4109
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4110
|
+
}));
|
|
4111
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-legacy-gone', cwd, config), { ok: true });
|
|
4112
|
+
assert.equal(existsSync(debtPath), false);
|
|
4113
|
+
assert.doesNotMatch(await readFile(tmuxLogPath, 'utf8'), /kill-pane|show-option/);
|
|
1716
4114
|
}
|
|
1717
4115
|
finally {
|
|
1718
4116
|
if (typeof previousPath === 'string')
|