oh-my-codex 0.20.2 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +7 -6
- package/Cargo.toml +1 -1
- package/README.md +26 -0
- package/crates/omx-runtime/Cargo.toml +1 -0
- package/crates/omx-runtime/src/main.rs +25 -2
- package/crates/omx-runtime/tests/execution.rs +219 -0
- package/crates/omx-runtime-core/src/dispatch.rs +96 -10
- package/crates/omx-runtime-core/src/engine.rs +839 -29
- package/crates/omx-runtime-core/src/lib.rs +8 -0
- package/dist/agents/__tests__/definitions.test.js +81 -0
- package/dist/agents/__tests__/definitions.test.js.map +1 -1
- package/dist/agents/__tests__/native-config.test.js +25 -0
- package/dist/agents/__tests__/native-config.test.js.map +1 -1
- package/dist/agents/native-config.d.ts +2 -1
- package/dist/agents/native-config.d.ts.map +1 -1
- package/dist/agents/native-config.js.map +1 -1
- package/dist/auth/hotswap.d.ts +6 -0
- package/dist/auth/hotswap.d.ts.map +1 -1
- package/dist/auth/hotswap.js +85 -15
- package/dist/auth/hotswap.js.map +1 -1
- package/dist/autopilot/__tests__/ralplan-gate.test.js +1 -1
- package/dist/autopilot/__tests__/ralplan-gate.test.js.map +1 -1
- package/dist/cli/__tests__/auth.test.js +60 -6
- package/dist/cli/__tests__/auth.test.js.map +1 -1
- package/dist/cli/__tests__/codex-plugin-layout.test.js +80 -0
- package/dist/cli/__tests__/codex-plugin-layout.test.js.map +1 -1
- package/dist/cli/__tests__/doctor-warning-copy.test.js +103 -1
- package/dist/cli/__tests__/doctor-warning-copy.test.js.map +1 -1
- package/dist/cli/__tests__/exec.test.js +136 -0
- package/dist/cli/__tests__/exec.test.js.map +1 -1
- package/dist/cli/__tests__/index.test.js +258 -4
- package/dist/cli/__tests__/index.test.js.map +1 -1
- package/dist/cli/__tests__/launch-fallback.test.js +287 -1
- package/dist/cli/__tests__/launch-fallback.test.js.map +1 -1
- package/dist/cli/__tests__/mcp-parity.test.js +4 -1
- package/dist/cli/__tests__/mcp-parity.test.js.map +1 -1
- package/dist/cli/__tests__/native-hook-claim-journal.test.js +84 -6
- package/dist/cli/__tests__/native-hook-claim-journal.test.js.map +1 -1
- package/dist/cli/__tests__/ralph-goal-mode-contract.test.js +3 -3
- package/dist/cli/__tests__/ralph-goal-mode-contract.test.js.map +1 -1
- package/dist/cli/__tests__/ralph.test.js +3 -3
- package/dist/cli/__tests__/ralph.test.js.map +1 -1
- package/dist/cli/__tests__/ralplan-bootstrap-3181.test.js +56 -171
- package/dist/cli/__tests__/ralplan-bootstrap-3181.test.js.map +1 -1
- package/dist/cli/__tests__/ralplan.test.js +37 -111
- package/dist/cli/__tests__/ralplan.test.js.map +1 -1
- package/dist/cli/__tests__/resume.test.js +11 -10
- package/dist/cli/__tests__/resume.test.js.map +1 -1
- package/dist/cli/__tests__/session-search-help.test.js +25 -1
- package/dist/cli/__tests__/session-search-help.test.js.map +1 -1
- package/dist/cli/__tests__/setup-install-mode.test.js +42 -4
- package/dist/cli/__tests__/setup-install-mode.test.js.map +1 -1
- package/dist/cli/__tests__/team.test.js +105 -7
- package/dist/cli/__tests__/team.test.js.map +1 -1
- package/dist/cli/__tests__/uninstall.test.js +46 -0
- package/dist/cli/__tests__/uninstall.test.js.map +1 -1
- package/dist/cli/__tests__/windows-popup-loop-contract.test.js +2 -1
- package/dist/cli/__tests__/windows-popup-loop-contract.test.js.map +1 -1
- package/dist/cli/doctor.d.ts +3 -0
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +15 -2
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/index.d.ts +4 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +138 -38
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/mcp-parity.js +2 -2
- package/dist/cli/mcp-parity.js.map +1 -1
- package/dist/cli/native-hook-claim-journal.d.ts +16 -5
- package/dist/cli/native-hook-claim-journal.d.ts.map +1 -1
- package/dist/cli/native-hook-claim-journal.js +42 -29
- package/dist/cli/native-hook-claim-journal.js.map +1 -1
- package/dist/cli/ralph.js +2 -2
- package/dist/cli/ralph.js.map +1 -1
- package/dist/cli/ralplan.d.ts +4 -8
- package/dist/cli/ralplan.d.ts.map +1 -1
- package/dist/cli/ralplan.js +19 -112
- package/dist/cli/ralplan.js.map +1 -1
- package/dist/cli/session-search.d.ts +5 -0
- package/dist/cli/session-search.d.ts.map +1 -1
- package/dist/cli/session-search.js +87 -0
- package/dist/cli/session-search.js.map +1 -1
- package/dist/cli/setup.d.ts +5 -0
- package/dist/cli/setup.d.ts.map +1 -1
- package/dist/cli/setup.js +80 -31
- package/dist/cli/setup.js.map +1 -1
- package/dist/cli/uninstall.d.ts +9 -0
- package/dist/cli/uninstall.d.ts.map +1 -1
- package/dist/cli/uninstall.js +61 -19
- package/dist/cli/uninstall.js.map +1 -1
- package/dist/compat/__tests__/rust-runtime-compat.test.js +1 -1
- package/dist/config/__tests__/generator-idempotent.test.js +330 -1
- package/dist/config/__tests__/generator-idempotent.test.js.map +1 -1
- package/dist/config/__tests__/generator-notify.test.js +2 -2
- package/dist/config/__tests__/generator-notify.test.js.map +1 -1
- package/dist/config/__tests__/models.test.js +67 -3
- package/dist/config/__tests__/models.test.js.map +1 -1
- package/dist/config/generator.d.ts +4 -20
- package/dist/config/generator.d.ts.map +1 -1
- package/dist/config/generator.js +499 -119
- package/dist/config/generator.js.map +1 -1
- package/dist/config/models.d.ts +12 -2
- package/dist/config/models.d.ts.map +1 -1
- package/dist/config/models.js +15 -1
- package/dist/config/models.js.map +1 -1
- package/dist/hooks/__tests__/agents-overlay.test.js +2 -2
- package/dist/hooks/__tests__/agents-overlay.test.js.map +1 -1
- package/dist/hooks/__tests__/keyword-detector.test.js +4 -4
- package/dist/hooks/__tests__/keyword-detector.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-fallback-watcher.test.js +238 -543
- package/dist/hooks/__tests__/notify-fallback-watcher.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-all-workers-idle.test.js +38 -4
- package/dist/hooks/__tests__/notify-hook-all-workers-idle.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-auto-nudge.test.js +96 -9
- package/dist/hooks/__tests__/notify-hook-auto-nudge.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-team-dispatch.test.js +531 -12
- package/dist/hooks/__tests__/notify-hook-team-dispatch.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-team-leader-nudge.test.js +155 -23
- package/dist/hooks/__tests__/notify-hook-team-leader-nudge.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-team-tmux-guard.test.js +482 -32
- package/dist/hooks/__tests__/notify-hook-team-tmux-guard.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-tmux-heal.test.js +68 -1
- package/dist/hooks/__tests__/notify-hook-tmux-heal.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-tmux-scrollback.test.js +7 -1
- package/dist/hooks/__tests__/notify-hook-tmux-scrollback.test.js.map +1 -1
- package/dist/hooks/__tests__/notify-hook-worker-idle.test.js +43 -4
- package/dist/hooks/__tests__/notify-hook-worker-idle.test.js.map +1 -1
- package/dist/hooks/__tests__/session.test.js +348 -2
- package/dist/hooks/__tests__/session.test.js.map +1 -1
- package/dist/hooks/agents-overlay.js +1 -1
- package/dist/hooks/session.d.ts +26 -2
- package/dist/hooks/session.d.ts.map +1 -1
- package/dist/hooks/session.js +169 -13
- package/dist/hooks/session.js.map +1 -1
- package/dist/leader/__tests__/contract.test.js +57 -7
- package/dist/leader/__tests__/contract.test.js.map +1 -1
- package/dist/leader/contract.d.ts +16 -0
- package/dist/leader/contract.d.ts.map +1 -1
- package/dist/leader/contract.js +79 -4
- package/dist/leader/contract.js.map +1 -1
- package/dist/mcp/__tests__/code-intel-server.test.js +31 -3
- package/dist/mcp/__tests__/code-intel-server.test.js.map +1 -1
- package/dist/mcp/__tests__/wiki-server.test.js +18 -0
- package/dist/mcp/__tests__/wiki-server.test.js.map +1 -1
- package/dist/mcp/code-intel-server.d.ts.map +1 -1
- package/dist/mcp/code-intel-server.js +13 -3
- package/dist/mcp/code-intel-server.js.map +1 -1
- package/dist/mcp/wiki-server.d.ts +11 -0
- package/dist/mcp/wiki-server.d.ts.map +1 -1
- package/dist/mcp/wiki-server.js +9 -2
- package/dist/mcp/wiki-server.js.map +1 -1
- package/dist/modes/__tests__/base-tmux-pane.test.js +4 -4
- package/dist/modes/__tests__/base-tmux-pane.test.js.map +1 -1
- package/dist/notifications/temp-contract.d.ts.map +1 -1
- package/dist/notifications/temp-contract.js +4 -0
- package/dist/notifications/temp-contract.js.map +1 -1
- package/dist/pipeline/__tests__/orchestrator.test.js +4 -4
- package/dist/pipeline/__tests__/orchestrator.test.js.map +1 -1
- package/dist/pipeline/__tests__/stages.test.js +8 -8
- package/dist/pipeline/__tests__/stages.test.js.map +1 -1
- package/dist/ralplan/__tests__/consensus-gate.test.js +134 -10
- package/dist/ralplan/__tests__/consensus-gate.test.js.map +1 -1
- package/dist/ralplan/__tests__/documented-leader-preflight.test.d.ts +2 -0
- package/dist/ralplan/__tests__/documented-leader-preflight.test.d.ts.map +1 -0
- package/dist/ralplan/__tests__/documented-leader-preflight.test.js +46 -0
- package/dist/ralplan/__tests__/documented-leader-preflight.test.js.map +1 -0
- package/dist/ralplan/__tests__/runtime.test.js +8 -4
- package/dist/ralplan/__tests__/runtime.test.js.map +1 -1
- package/dist/ralplan/consensus-gate.d.ts.map +1 -1
- package/dist/ralplan/consensus-gate.js +58 -50
- package/dist/ralplan/consensus-gate.js.map +1 -1
- package/dist/ralplan/documented-leader-preflight.d.ts +34 -0
- package/dist/ralplan/documented-leader-preflight.d.ts.map +1 -0
- package/dist/ralplan/documented-leader-preflight.js +58 -0
- package/dist/ralplan/documented-leader-preflight.js.map +1 -0
- package/dist/ralplan/runtime.d.ts +1 -0
- package/dist/ralplan/runtime.d.ts.map +1 -1
- package/dist/ralplan/runtime.js +11 -7
- package/dist/ralplan/runtime.js.map +1 -1
- package/dist/runtime/__tests__/bridge.test.js +42 -1
- package/dist/runtime/__tests__/bridge.test.js.map +1 -1
- package/dist/runtime/bridge.d.ts +12 -0
- package/dist/runtime/bridge.d.ts.map +1 -1
- package/dist/runtime/bridge.js +52 -1
- package/dist/runtime/bridge.js.map +1 -1
- package/dist/scripts/__tests__/codex-native-hook.test.js +2620 -487
- package/dist/scripts/__tests__/codex-native-hook.test.js.map +1 -1
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.d.ts +2 -0
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.d.ts.map +1 -0
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.js +108 -0
- package/dist/scripts/__tests__/reasoning-artifact-contract.test.js.map +1 -0
- package/dist/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.js +46 -173
- package/dist/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.js.map +1 -1
- package/dist/scripts/__tests__/role-intent-durable-recovery-3181.test.js +39 -159
- package/dist/scripts/__tests__/role-intent-durable-recovery-3181.test.js.map +1 -1
- package/dist/scripts/__tests__/smoke-packed-install.test.js +254 -2
- package/dist/scripts/__tests__/smoke-packed-install.test.js.map +1 -1
- package/dist/scripts/codex-native-hook.d.ts +4 -1
- package/dist/scripts/codex-native-hook.d.ts.map +1 -1
- package/dist/scripts/codex-native-hook.js +11013 -1153
- package/dist/scripts/codex-native-hook.js.map +1 -1
- package/dist/scripts/notify-fallback-watcher.js +82 -108
- package/dist/scripts/notify-fallback-watcher.js.map +1 -1
- package/dist/scripts/notify-hook/auto-nudge.d.ts.map +1 -1
- package/dist/scripts/notify-hook/auto-nudge.js +74 -3
- package/dist/scripts/notify-hook/auto-nudge.js.map +1 -1
- package/dist/scripts/notify-hook/team-dispatch.d.ts +9 -0
- package/dist/scripts/notify-hook/team-dispatch.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-dispatch.js +234 -79
- package/dist/scripts/notify-hook/team-dispatch.js.map +1 -1
- package/dist/scripts/notify-hook/team-leader-nudge.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-leader-nudge.js +39 -13
- package/dist/scripts/notify-hook/team-leader-nudge.js.map +1 -1
- package/dist/scripts/notify-hook/team-tmux-guard.d.ts +12 -4
- package/dist/scripts/notify-hook/team-tmux-guard.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-tmux-guard.js +365 -44
- package/dist/scripts/notify-hook/team-tmux-guard.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker-stop.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker-stop.js +19 -18
- package/dist/scripts/notify-hook/team-worker-stop.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker.d.ts +3 -0
- package/dist/scripts/notify-hook/team-worker.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker.js +37 -28
- package/dist/scripts/notify-hook/team-worker.js.map +1 -1
- package/dist/scripts/notify-hook/tmux-injection.d.ts.map +1 -1
- package/dist/scripts/notify-hook/tmux-injection.js +28 -2
- package/dist/scripts/notify-hook/tmux-injection.js.map +1 -1
- package/dist/scripts/notify-hook.js +1 -0
- package/dist/scripts/notify-hook.js.map +1 -1
- package/dist/scripts/smoke-packed-install.d.ts +48 -3
- package/dist/scripts/smoke-packed-install.d.ts.map +1 -1
- package/dist/scripts/smoke-packed-install.js +2796 -13
- package/dist/scripts/smoke-packed-install.js.map +1 -1
- package/dist/state/__tests__/mode-state-context.test.js +54 -0
- package/dist/state/__tests__/mode-state-context.test.js.map +1 -1
- package/dist/state/__tests__/operations.test.js +2 -0
- package/dist/state/__tests__/operations.test.js.map +1 -1
- package/dist/state/mode-state-context.d.ts +5 -0
- package/dist/state/mode-state-context.d.ts.map +1 -1
- package/dist/state/mode-state-context.js +59 -0
- package/dist/state/mode-state-context.js.map +1 -1
- package/dist/subagents/__tests__/leader-bootstrap-3181.test.js +24 -245
- package/dist/subagents/__tests__/leader-bootstrap-3181.test.js.map +1 -1
- package/dist/subagents/tracker.d.ts +1 -82
- package/dist/subagents/tracker.d.ts.map +1 -1
- package/dist/subagents/tracker.js +3 -275
- package/dist/subagents/tracker.js.map +1 -1
- package/dist/team/__tests__/api-interop.test.js +91 -1
- package/dist/team/__tests__/api-interop.test.js.map +1 -1
- package/dist/team/__tests__/delivery-e2e-smoke.test.js +52 -10
- package/dist/team/__tests__/delivery-e2e-smoke.test.js.map +1 -1
- package/dist/team/__tests__/idle-nudge.test.js +45 -15
- package/dist/team/__tests__/idle-nudge.test.js.map +1 -1
- package/dist/team/__tests__/mcp-comm.test.js +93 -27
- package/dist/team/__tests__/mcp-comm.test.js.map +1 -1
- package/dist/team/__tests__/model-contract.test.js +110 -3
- package/dist/team/__tests__/model-contract.test.js.map +1 -1
- package/dist/team/__tests__/runtime-cli.test.js +9 -0
- package/dist/team/__tests__/runtime-cli.test.js.map +1 -1
- package/dist/team/__tests__/runtime.test.js +3204 -335
- package/dist/team/__tests__/runtime.test.js.map +1 -1
- package/dist/team/__tests__/scaling.test.js +2359 -142
- package/dist/team/__tests__/scaling.test.js.map +1 -1
- package/dist/team/__tests__/state.test.js +804 -36
- package/dist/team/__tests__/state.test.js.map +1 -1
- package/dist/team/__tests__/team-ops-contract.test.js +6 -0
- package/dist/team/__tests__/team-ops-contract.test.js.map +1 -1
- package/dist/team/__tests__/tmux-session.test.js +3049 -784
- package/dist/team/__tests__/tmux-session.test.js.map +1 -1
- package/dist/team/__tests__/worker-runtime-identity.test.js +38 -2
- package/dist/team/__tests__/worker-runtime-identity.test.js.map +1 -1
- package/dist/team/__tests__/worktree.test.js +6 -0
- package/dist/team/__tests__/worktree.test.js.map +1 -1
- package/dist/team/api-interop.d.ts.map +1 -1
- package/dist/team/api-interop.js +15 -9
- package/dist/team/api-interop.js.map +1 -1
- package/dist/team/exact-pane.d.ts +23 -0
- package/dist/team/exact-pane.d.ts.map +1 -0
- package/dist/team/exact-pane.js +133 -0
- package/dist/team/exact-pane.js.map +1 -0
- package/dist/team/idle-nudge.d.ts +12 -1
- package/dist/team/idle-nudge.d.ts.map +1 -1
- package/dist/team/idle-nudge.js +41 -14
- package/dist/team/idle-nudge.js.map +1 -1
- package/dist/team/model-contract.d.ts +2 -5
- package/dist/team/model-contract.d.ts.map +1 -1
- package/dist/team/model-contract.js +59 -17
- package/dist/team/model-contract.js.map +1 -1
- package/dist/team/runtime.d.ts +1 -0
- package/dist/team/runtime.d.ts.map +1 -1
- package/dist/team/runtime.js +1244 -180
- package/dist/team/runtime.js.map +1 -1
- package/dist/team/scaling.d.ts +13 -1
- package/dist/team/scaling.d.ts.map +1 -1
- package/dist/team/scaling.js +1430 -384
- package/dist/team/scaling.js.map +1 -1
- package/dist/team/state/dispatch.d.ts.map +1 -1
- package/dist/team/state/dispatch.js +26 -8
- package/dist/team/state/dispatch.js.map +1 -1
- package/dist/team/state/locks.d.ts.map +1 -1
- package/dist/team/state/locks.js +61 -13
- package/dist/team/state/locks.js.map +1 -1
- package/dist/team/state/tasks.d.ts.map +1 -1
- package/dist/team/state/tasks.js +4 -3
- package/dist/team/state/tasks.js.map +1 -1
- package/dist/team/state.d.ts +67 -1
- package/dist/team/state.d.ts.map +1 -1
- package/dist/team/state.js +636 -129
- package/dist/team/state.js.map +1 -1
- package/dist/team/team-ops.d.ts +5 -0
- package/dist/team/team-ops.d.ts.map +1 -1
- package/dist/team/team-ops.js +4 -0
- package/dist/team/team-ops.js.map +1 -1
- package/dist/team/tmux-session.d.ts +123 -23
- package/dist/team/tmux-session.d.ts.map +1 -1
- package/dist/team/tmux-session.js +1250 -291
- package/dist/team/tmux-session.js.map +1 -1
- package/dist/team/worktree.d.ts.map +1 -1
- package/dist/team/worktree.js +4 -0
- package/dist/team/worktree.js.map +1 -1
- package/dist/utils/__tests__/file-durability.test.d.ts +2 -0
- package/dist/utils/__tests__/file-durability.test.d.ts.map +1 -0
- package/dist/utils/__tests__/file-durability.test.js +57 -0
- package/dist/utils/__tests__/file-durability.test.js.map +1 -0
- package/dist/utils/file-durability.d.ts +15 -0
- package/dist/utils/file-durability.d.ts.map +1 -0
- package/dist/utils/file-durability.js +37 -0
- package/dist/utils/file-durability.js.map +1 -0
- package/dist/verification/__tests__/dev-merge-issue-close-workflow.test.js +3 -0
- package/dist/verification/__tests__/dev-merge-issue-close-workflow.test.js.map +1 -1
- package/dist/wiki/lint.d.ts +4 -1
- package/dist/wiki/lint.d.ts.map +1 -1
- package/dist/wiki/lint.js +3 -2
- package/dist/wiki/lint.js.map +1 -1
- package/package.json +1 -1
- package/plugins/oh-my-codex/.codex-plugin/plugin.json +1 -1
- package/plugins/oh-my-codex/hooks/codex-native-hook.mjs +24 -0
- package/plugins/oh-my-codex/skills/ralph/SKILL.md +6 -4
- package/plugins/oh-my-codex/skills/ralplan/SKILL.md +3 -2
- package/plugins/oh-my-codex/skills/team/SKILL.md +13 -9
- package/skills/ralph/SKILL.md +6 -4
- package/skills/ralplan/SKILL.md +3 -2
- package/skills/team/SKILL.md +13 -9
- package/src/scripts/__tests__/codex-native-hook.test.ts +2826 -497
- package/src/scripts/__tests__/reasoning-artifact-contract.test.ts +129 -0
- package/src/scripts/__tests__/role-intent-bootstrap-e2e-3181.test.ts +48 -185
- package/src/scripts/__tests__/role-intent-durable-recovery-3181.test.ts +44 -177
- package/src/scripts/__tests__/smoke-packed-install.test.ts +320 -2
- package/src/scripts/codex-native-hook.ts +10715 -1069
- package/src/scripts/notify-fallback-watcher.ts +84 -115
- package/src/scripts/notify-hook/auto-nudge.ts +71 -3
- package/src/scripts/notify-hook/team-dispatch.ts +242 -75
- package/src/scripts/notify-hook/team-leader-nudge.ts +40 -18
- package/src/scripts/notify-hook/team-tmux-guard.ts +366 -39
- package/src/scripts/notify-hook/team-worker-stop.ts +20 -16
- package/src/scripts/notify-hook/team-worker.ts +39 -26
- package/src/scripts/notify-hook/tmux-injection.ts +28 -2
- package/src/scripts/notify-hook.ts +1 -0
- package/src/scripts/smoke-packed-install.ts +3196 -14
- package/dist/subagents/__tests__/adapted-role-binding.test.d.ts +0 -2
- package/dist/subagents/__tests__/adapted-role-binding.test.d.ts.map +0 -1
- package/dist/subagents/__tests__/adapted-role-binding.test.js +0 -988
- package/dist/subagents/__tests__/adapted-role-binding.test.js.map +0 -1
- package/dist/subagents/adapted-role-binding.d.ts +0 -17
- package/dist/subagents/adapted-role-binding.d.ts.map +0 -1
- package/dist/subagents/adapted-role-binding.js +0 -75
- package/dist/subagents/adapted-role-binding.js.map +0 -1
|
@@ -2,17 +2,18 @@ import { describe, it } from 'node:test';
|
|
|
2
2
|
import assert from 'node:assert/strict';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { execFileSync } from 'node:child_process';
|
|
5
|
-
import { mkdtemp, rm, readFile, writeFile, mkdir, chmod, readdir } from 'fs/promises';
|
|
6
|
-
import { join, relative } from 'path';
|
|
5
|
+
import { mkdtemp, rm, readFile, writeFile, mkdir, chmod, readdir, symlink } from 'fs/promises';
|
|
6
|
+
import { join, posix, relative, win32 } from 'path';
|
|
7
7
|
import { tmpdir } from 'os';
|
|
8
8
|
import { existsSync, readFileSync } from 'fs';
|
|
9
|
-
import { initTeamState, createTask, readTask, readTeamConfig, saveTeamConfig, readWorkerStatus, writeWorkerStatus, withScalingLock, DEFAULT_MAX_WORKERS, } from '../state.js';
|
|
10
|
-
import { isScalingEnabled, scaleUp, scaleDown } from '../scaling.js';
|
|
9
|
+
import { initTeamState, createTask, claimTask, readTask, readTeamConfig, saveTeamConfig, readWorkerStatus, writeWorkerStatus, withScalingLock, recoverTeamMembershipTaskTransaction, commitTeamMembershipTaskTransaction, withTeamTaskBarrier, DEFAULT_MAX_WORKERS, listDispatchRequests, readTeamManifestV2, } from '../state.js';
|
|
10
|
+
import { isSameOrInsidePath, isScalingEnabled, reconcileScaleDownCleanupDebt, scaleUp, scaleDown } from '../scaling.js';
|
|
11
11
|
import { resolveCanonicalTeamStateRoot } from '../state-root.js';
|
|
12
12
|
import { resolvePersistedApprovedTeamExecutionContinuityState, writePersistedApprovedTeamExecutionBinding, } from '../approved-execution.js';
|
|
13
13
|
import { TEAM_WORKER_INHERITED_MODEL_ENV } from '../model-contract.js';
|
|
14
14
|
import { buildWorkerProcessLaunchSpec } from '../tmux-session.js';
|
|
15
15
|
delete process.env.OMX_TEAM_STATE_ROOT;
|
|
16
|
+
process.env.OMX_RUNTIME_BRIDGE = '0';
|
|
16
17
|
async function initCommittedGitRepo(cwd) {
|
|
17
18
|
execFileSync('git', ['init'], { cwd, stdio: 'pipe' });
|
|
18
19
|
execFileSync('git', ['config', 'user.name', 'OMX Test'], { cwd, stdio: 'pipe' });
|
|
@@ -35,6 +36,11 @@ function computeGitBlobSha1(content) {
|
|
|
35
36
|
const header = Buffer.from(`blob ${buffer.length}\0`, 'utf-8');
|
|
36
37
|
return createHash('sha1').update(header).update(buffer).digest('hex');
|
|
37
38
|
}
|
|
39
|
+
function withoutConfigGeneration(bytes) {
|
|
40
|
+
const value = JSON.parse(bytes);
|
|
41
|
+
delete value.config_generation;
|
|
42
|
+
return value;
|
|
43
|
+
}
|
|
38
44
|
function canonicalContextPackRelativePath(slug) {
|
|
39
45
|
return `.omx/context/context-20260507T120000Z-${slug}.json`;
|
|
40
46
|
}
|
|
@@ -147,6 +153,7 @@ async function writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath) {
|
|
|
147
153
|
'set -eu',
|
|
148
154
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
149
155
|
'case "${1:-}" in',
|
|
156
|
+
' show-option) echo "team:scale-up" ;;',
|
|
150
157
|
' -V)',
|
|
151
158
|
' echo "tmux 3.2a"',
|
|
152
159
|
' ;;',
|
|
@@ -154,10 +161,28 @@ async function writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath) {
|
|
|
154
161
|
' echo "%31"',
|
|
155
162
|
' ;;',
|
|
156
163
|
' list-panes)',
|
|
157
|
-
'
|
|
164
|
+
' case "$*" in',
|
|
165
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
166
|
+
" printf '%s\t%s\t%s\n' '%11' '0' '42411'",
|
|
167
|
+
" printf '%s\t%s\t%s\n' '%21' '0' '42421'",
|
|
168
|
+
` if [ ! -f "${join(fakeBinDir, 'killed-%31')}" ]; then printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'; fi`,
|
|
169
|
+
' ;;',
|
|
170
|
+
' *)',
|
|
171
|
+
' echo "42424"',
|
|
172
|
+
' ;;',
|
|
173
|
+
' esac',
|
|
174
|
+
' ;;',
|
|
175
|
+
' show-option)',
|
|
176
|
+
' case "$*" in',
|
|
177
|
+
' *"@omx_team_pane_owner_id"*) echo "team:scale-up" ;;',
|
|
178
|
+
' *) exit 1 ;;',
|
|
179
|
+
' esac',
|
|
158
180
|
' ;;',
|
|
159
181
|
' send-keys)',
|
|
160
182
|
' ;;',
|
|
183
|
+
' kill-pane)',
|
|
184
|
+
` : > "${join(fakeBinDir, 'killed-')}$3"`,
|
|
185
|
+
' ;;',
|
|
161
186
|
' capture-pane)',
|
|
162
187
|
' echo ""',
|
|
163
188
|
' ;;',
|
|
@@ -175,8 +200,12 @@ async function configureScaleUpTeamForDirectDispatch(teamName, cwd) {
|
|
|
175
200
|
throw new Error(`missing team config for ${teamName}`);
|
|
176
201
|
}
|
|
177
202
|
config.tmux_session = `omx-team-${teamName}`;
|
|
203
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
178
204
|
config.leader_pane_id = '%11';
|
|
205
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
206
|
+
config.leader_pane_pid = 42411;
|
|
179
207
|
config.workers[0].pane_id = '%21';
|
|
208
|
+
config.workers[0].pid = 42421;
|
|
180
209
|
await saveTeamConfig(config, cwd);
|
|
181
210
|
const manifestPath = join(cwd, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
182
211
|
if (!existsSync(manifestPath)) {
|
|
@@ -317,6 +346,60 @@ describe('isScalingEnabled', () => {
|
|
|
317
346
|
assert.equal(isScalingEnabled({ OMX_TEAM_SCALING_ENABLED: ' true ' }), true);
|
|
318
347
|
});
|
|
319
348
|
});
|
|
349
|
+
// ── scale-down cleanup debt path containment ──────────────────────────────────
|
|
350
|
+
describe('scale-down cleanup debt path containment', () => {
|
|
351
|
+
const windowsPathSemantics = {
|
|
352
|
+
relative: win32.relative,
|
|
353
|
+
isAbsolute: win32.isAbsolute,
|
|
354
|
+
sep: win32.sep,
|
|
355
|
+
};
|
|
356
|
+
it('accepts same-root and descendant native Windows canonical paths', () => {
|
|
357
|
+
assert.equal(isSameOrInsidePath('C:\\repo', 'C:\\repo', windowsPathSemantics), true);
|
|
358
|
+
assert.equal(isSameOrInsidePath('C:\\repo\\.omx\\team\\alpha', 'C:\\repo', windowsPathSemantics), true);
|
|
359
|
+
});
|
|
360
|
+
it('accepts mixed-separator, drive-letter, and case-equivalent Windows descendants', () => {
|
|
361
|
+
assert.equal(isSameOrInsidePath('c:/REPO/.omx\\team/alpha', 'C:\\repo', windowsPathSemantics), true);
|
|
362
|
+
});
|
|
363
|
+
it('accepts same-share UNC descendants across host and share casing', () => {
|
|
364
|
+
assert.equal(isSameOrInsidePath('\\\\SERVER\\Share\\repo/.omx\\team\\alpha', '\\\\server\\share\\REPO', windowsPathSemantics), true);
|
|
365
|
+
});
|
|
366
|
+
for (const [name, candidate] of [
|
|
367
|
+
['a sibling prefix collision', 'C:\\repo-other\\worker'],
|
|
368
|
+
['a parent traversal escape', 'C:\\repo\\..\\foreign\\worker'],
|
|
369
|
+
['a foreign same-volume root', 'C:\\foreign\\worker'],
|
|
370
|
+
['an absolute relative result from a foreign drive', 'D:\\foreign\\worker'],
|
|
371
|
+
['a foreign UNC host', '\\\\foreign-server\\share\\repo\\worker'],
|
|
372
|
+
['a foreign UNC share', '\\\\server\\foreign-share\\repo\\worker'],
|
|
373
|
+
['a UNC sibling prefix collision', '\\\\server\\share\\repo-other\\worker'],
|
|
374
|
+
]) {
|
|
375
|
+
it(`rejects ${name} under native Windows path semantics`, () => {
|
|
376
|
+
assert.equal(isSameOrInsidePath(candidate, 'C:\\repo', windowsPathSemantics), false);
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
for (const [name, candidate] of [
|
|
380
|
+
['a foreign UNC host', '\\\\foreign-server\\share\\repo\\worker'],
|
|
381
|
+
['a foreign UNC share', '\\\\server\\foreign-share\\repo\\worker'],
|
|
382
|
+
['a UNC sibling prefix collision', '\\\\server\\share\\repo-other\\worker'],
|
|
383
|
+
]) {
|
|
384
|
+
it(`rejects ${name} under native Windows UNC path semantics`, () => {
|
|
385
|
+
assert.equal(isSameOrInsidePath(candidate, '\\\\server\\share\\repo', windowsPathSemantics), false);
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
const posixPathSemantics = {
|
|
389
|
+
relative: posix.relative,
|
|
390
|
+
isAbsolute: posix.isAbsolute,
|
|
391
|
+
sep: posix.sep,
|
|
392
|
+
};
|
|
393
|
+
it('preserves POSIX same-root and descendant containment', () => {
|
|
394
|
+
assert.equal(isSameOrInsidePath('/repo', '/repo', posixPathSemantics), true);
|
|
395
|
+
assert.equal(isSameOrInsidePath('/repo/.omx/team/alpha', '/repo', posixPathSemantics), true);
|
|
396
|
+
});
|
|
397
|
+
it('rejects POSIX siblings, traversal, and foreign roots', () => {
|
|
398
|
+
assert.equal(isSameOrInsidePath('/repo-other/worker', '/repo', posixPathSemantics), false);
|
|
399
|
+
assert.equal(isSameOrInsidePath('/repo/../foreign/worker', '/repo', posixPathSemantics), false);
|
|
400
|
+
assert.equal(isSameOrInsidePath('/foreign/worker', '/repo', posixPathSemantics), false);
|
|
401
|
+
});
|
|
402
|
+
});
|
|
320
403
|
// ── WorkerStatus draining state ───────────────────────────────────────────────
|
|
321
404
|
describe('WorkerStatus draining state', () => {
|
|
322
405
|
it('writeWorkerStatus writes draining status and readWorkerStatus reads it back', async () => {
|
|
@@ -394,7 +477,7 @@ describe('withScalingLock', () => {
|
|
|
394
477
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-lock-'));
|
|
395
478
|
try {
|
|
396
479
|
await initTeamState('lock-test', 'task', 'executor', 1, cwd);
|
|
397
|
-
const lockDir = join(cwd, '.omx', 'state', 'team', 'lock-test
|
|
480
|
+
const lockDir = join(cwd, '.omx', 'state', '.team-locks', 'lock-test.scaling');
|
|
398
481
|
const result = await withScalingLock('lock-test', cwd, async () => {
|
|
399
482
|
// Lock should exist during execution
|
|
400
483
|
assert.equal(existsSync(lockDir), true);
|
|
@@ -412,7 +495,7 @@ describe('withScalingLock', () => {
|
|
|
412
495
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-lock-err-'));
|
|
413
496
|
try {
|
|
414
497
|
await initTeamState('lock-err', 'task', 'executor', 1, cwd);
|
|
415
|
-
const lockDir = join(cwd, '.omx', 'state', 'team', 'lock-err
|
|
498
|
+
const lockDir = join(cwd, '.omx', 'state', '.team-locks', 'lock-err.scaling');
|
|
416
499
|
await assert.rejects(withScalingLock('lock-err', cwd, async () => {
|
|
417
500
|
throw new Error('test error');
|
|
418
501
|
}), { message: 'test error' });
|
|
@@ -589,6 +672,7 @@ exit 0
|
|
|
589
672
|
'set -eu',
|
|
590
673
|
`printf '\%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
591
674
|
'case "${1:-}" in',
|
|
675
|
+
' show-option) echo "team:scale-up-role" ;;',
|
|
592
676
|
' -V)',
|
|
593
677
|
' echo "tmux 3.2a"',
|
|
594
678
|
' ;;',
|
|
@@ -596,10 +680,22 @@ exit 0
|
|
|
596
680
|
' echo "%31"',
|
|
597
681
|
' ;;',
|
|
598
682
|
' list-panes)',
|
|
599
|
-
'
|
|
683
|
+
' case "$*" in',
|
|
684
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
685
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
686
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
687
|
+
` if [ ! -f "${join(fakeBinDir, 'scale-up-role-killed-%31')}" ]; then printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'; fi`,
|
|
688
|
+
' ;;',
|
|
689
|
+
' *)',
|
|
690
|
+
' echo "42424"',
|
|
691
|
+
' ;;',
|
|
692
|
+
' esac',
|
|
600
693
|
' ;;',
|
|
601
694
|
' send-keys)',
|
|
602
695
|
' ;;',
|
|
696
|
+
' kill-pane)',
|
|
697
|
+
` : > "${join(fakeBinDir, 'scale-up-role-killed-')}$3"`,
|
|
698
|
+
' ;;',
|
|
603
699
|
' capture-pane)',
|
|
604
700
|
' echo ""',
|
|
605
701
|
' ;;',
|
|
@@ -627,7 +723,9 @@ exit 0
|
|
|
627
723
|
return;
|
|
628
724
|
config.tmux_session = 'omx-team-scale-up-role';
|
|
629
725
|
config.leader_pane_id = '%11';
|
|
726
|
+
config.tmux_pane_owner_id = 'team:scale-up-role';
|
|
630
727
|
config.workers[0].pane_id = '%21';
|
|
728
|
+
config.workers[0].pid = 42421;
|
|
631
729
|
await saveTeamConfig(config, cwd);
|
|
632
730
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'scale-up-role', 'manifest.v2.json');
|
|
633
731
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -751,18 +849,20 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
751
849
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
752
850
|
}
|
|
753
851
|
});
|
|
754
|
-
it('
|
|
852
|
+
it('persists a newly created worker when rollback pane proof is unavailable', async () => {
|
|
755
853
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-rollback-'));
|
|
756
854
|
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-rollback-bin-'));
|
|
757
855
|
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
758
856
|
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
759
857
|
const previousPath = process.env.PATH;
|
|
858
|
+
const previousBridge = process.env.OMX_RUNTIME_BRIDGE;
|
|
760
859
|
try {
|
|
761
860
|
await writeFile(tmuxStubPath, [
|
|
762
861
|
'#!/bin/sh',
|
|
763
862
|
'set -eu',
|
|
764
863
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
765
864
|
'case "${1:-}" in',
|
|
865
|
+
' show-option) echo "team:scale-up-owner-tag-rollback" ;;',
|
|
766
866
|
' -V)',
|
|
767
867
|
' echo "tmux 3.2a"',
|
|
768
868
|
' ;;',
|
|
@@ -778,7 +878,18 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
778
878
|
' esac',
|
|
779
879
|
' ;;',
|
|
780
880
|
' list-panes)',
|
|
781
|
-
|
|
881
|
+
` count_file="${join(fakeBinDir, 'proof-count')}"`,
|
|
882
|
+
' count=0',
|
|
883
|
+
' if [ -f "$count_file" ]; then count=$(cat "$count_file"); fi',
|
|
884
|
+
' count=$((count + 1))',
|
|
885
|
+
' printf "%s" "$count" > "$count_file"',
|
|
886
|
+
' if [ "$count" -le 5 ]; then',
|
|
887
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
888
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
889
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'",
|
|
890
|
+
' else',
|
|
891
|
+
" printf 'malformed pane snapshot\\n'",
|
|
892
|
+
' fi',
|
|
782
893
|
' ;;',
|
|
783
894
|
' kill-pane|send-keys|capture-pane)',
|
|
784
895
|
' ;;',
|
|
@@ -789,19 +900,155 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
789
900
|
await chmod(tmuxStubPath, 0o755);
|
|
790
901
|
await writeFile(tmuxLogPath, '');
|
|
791
902
|
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
903
|
+
process.env.OMX_RUNTIME_BRIDGE = '0';
|
|
792
904
|
await initTeamState('scale-up-owner-tag-rollback', 'task', 'executor', 1, cwd);
|
|
793
905
|
await configureScaleUpTeamForDirectDispatch('scale-up-owner-tag-rollback', cwd);
|
|
906
|
+
const ownerConfig = await readTeamConfig('scale-up-owner-tag-rollback', cwd);
|
|
907
|
+
assert.ok(ownerConfig);
|
|
908
|
+
if (!ownerConfig)
|
|
909
|
+
return;
|
|
910
|
+
ownerConfig.tmux_pane_owner_id = 'team:scale-up-owner-tag-rollback';
|
|
911
|
+
await saveTeamConfig(ownerConfig, cwd);
|
|
794
912
|
const result = await scaleUp('scale-up-owner-tag-rollback', 1, 'executor', [{ subject: 'new work', description: 'new work', owner: 'worker-2' }], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
795
913
|
assert.equal(result.ok, false);
|
|
796
914
|
if (result.ok)
|
|
797
915
|
return;
|
|
798
|
-
assert.match(result.error, /
|
|
916
|
+
assert.match(result.error, /scale_up_rollback_cleanup_debt:pane_owner_unverified:%31/);
|
|
799
917
|
const config = await readTeamConfig('scale-up-owner-tag-rollback', cwd);
|
|
800
|
-
assert.
|
|
801
|
-
assert.equal(
|
|
918
|
+
assert.deepEqual(config?.workers.map((worker) => worker.name), ['worker-1', 'worker-2']);
|
|
919
|
+
assert.equal(config?.workers[1]?.pane_id, '%31');
|
|
920
|
+
assert.equal(config?.next_worker_index, 3);
|
|
921
|
+
assert.equal((await readTask('scale-up-owner-tag-rollback', '1', cwd))?.owner, 'worker-2');
|
|
922
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-up-owner-tag-rollback', 'workers', 'worker-2', 'identity.json')), false);
|
|
923
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-up-owner-tag-rollback', 'workers', 'worker-2', 'inbox.md')), false);
|
|
924
|
+
const dispatch = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'scale-up-owner-tag-rollback', 'dispatch', 'requests.json'), 'utf8'));
|
|
925
|
+
assert.equal(dispatch.some((request) => request.to_worker === 'worker-2'), false);
|
|
926
|
+
assert.equal((await listDispatchRequests('scale-up-owner-tag-rollback', cwd, { to_worker: 'worker-2' })).length, 0);
|
|
802
927
|
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
803
928
|
assert.ok(tmuxCommands.some((command) => (command === 'set-option -p -t %31 @omx_team_pane_owner_id team:scale-up-owner-tag-rollback')));
|
|
804
|
-
assert.
|
|
929
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %31'), false);
|
|
930
|
+
}
|
|
931
|
+
finally {
|
|
932
|
+
if (typeof previousPath === 'string')
|
|
933
|
+
process.env.PATH = previousPath;
|
|
934
|
+
else
|
|
935
|
+
delete process.env.PATH;
|
|
936
|
+
if (typeof previousBridge === 'string')
|
|
937
|
+
process.env.OMX_RUNTIME_BRIDGE = previousBridge;
|
|
938
|
+
else
|
|
939
|
+
delete process.env.OMX_RUNTIME_BRIDGE;
|
|
940
|
+
await rm(cwd, { recursive: true, force: true });
|
|
941
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
942
|
+
}
|
|
943
|
+
});
|
|
944
|
+
it('retains unpinned rollback debt without killing a pane when its first post-split proof is unavailable', async () => {
|
|
945
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-unpinned-rollback-'));
|
|
946
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-unpinned-rollback-bin-'));
|
|
947
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
948
|
+
const splitPath = join(fakeBinDir, 'split-created');
|
|
949
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
950
|
+
const previousPath = process.env.PATH;
|
|
951
|
+
try {
|
|
952
|
+
await writeFile(tmuxStubPath, [
|
|
953
|
+
'#!/bin/sh',
|
|
954
|
+
'set -eu',
|
|
955
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
956
|
+
'case "${1:-}" in',
|
|
957
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
958
|
+
' show-option) echo "team:scale-up" ;;',
|
|
959
|
+
' split-window) : > "' + splitPath + '"; echo "%31" ;;',
|
|
960
|
+
' list-panes)',
|
|
961
|
+
' if [ -f "' + splitPath + '" ]; then printf "malformed pane snapshot\\n"; else',
|
|
962
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
963
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
964
|
+
' fi',
|
|
965
|
+
' ;;',
|
|
966
|
+
'esac',
|
|
967
|
+
'exit 0',
|
|
968
|
+
'',
|
|
969
|
+
].join('\n'));
|
|
970
|
+
await chmod(tmuxStubPath, 0o755);
|
|
971
|
+
await writeFile(tmuxLogPath, '');
|
|
972
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
973
|
+
const teamName = 'scale-up-unpinned-rollback';
|
|
974
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
975
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
976
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
977
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
978
|
+
});
|
|
979
|
+
assert.equal(result.ok, false);
|
|
980
|
+
if (result.ok)
|
|
981
|
+
return;
|
|
982
|
+
assert.match(result.error, /scale_up_rollback_cleanup_debt:pane_pid_unpinned:%31/);
|
|
983
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
984
|
+
assert.equal(config?.workers[1]?.pane_id, '%31');
|
|
985
|
+
assert.equal(config?.workers[1]?.pid, undefined);
|
|
986
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
987
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %31'), false);
|
|
988
|
+
}
|
|
989
|
+
finally {
|
|
990
|
+
if (typeof previousPath === 'string')
|
|
991
|
+
process.env.PATH = previousPath;
|
|
992
|
+
else
|
|
993
|
+
delete process.env.PATH;
|
|
994
|
+
await rm(cwd, { recursive: true, force: true });
|
|
995
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
996
|
+
}
|
|
997
|
+
});
|
|
998
|
+
it('retains the original pane/PID rollback debt when the new pane ID is reused before owner tagging', async () => {
|
|
999
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-reuse-'));
|
|
1000
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-owner-tag-reuse-bin-'));
|
|
1001
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
1002
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1003
|
+
const previousPath = process.env.PATH;
|
|
1004
|
+
try {
|
|
1005
|
+
await writeFile(tmuxStubPath, [
|
|
1006
|
+
'#!/bin/sh',
|
|
1007
|
+
'set -eu',
|
|
1008
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
1009
|
+
'case "${1:-}" in',
|
|
1010
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1011
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
1012
|
+
' split-window) echo "%31" ;;',
|
|
1013
|
+
' list-panes)',
|
|
1014
|
+
` count_file="${join(fakeBinDir, 'proof-count')}"`,
|
|
1015
|
+
' count=0; [ ! -f "$count_file" ] || count=$(cat "$count_file")',
|
|
1016
|
+
' count=$((count + 1)); printf "%s" "$count" > "$count_file"',
|
|
1017
|
+
' if [ "$count" -le 4 ]; then',
|
|
1018
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
1019
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'",
|
|
1020
|
+
' elif [ "$count" -eq 5 ]; then',
|
|
1021
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
1022
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '52424'",
|
|
1023
|
+
' else',
|
|
1024
|
+
" printf 'malformed pane snapshot\\n'",
|
|
1025
|
+
' fi',
|
|
1026
|
+
' ;;',
|
|
1027
|
+
' kill-pane|send-keys|capture-pane) ;;',
|
|
1028
|
+
'esac',
|
|
1029
|
+
'exit 0',
|
|
1030
|
+
'',
|
|
1031
|
+
].join('\n'));
|
|
1032
|
+
await chmod(tmuxStubPath, 0o755);
|
|
1033
|
+
await writeFile(tmuxLogPath, '');
|
|
1034
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1035
|
+
const teamName = 'scale-up-owner-tag-reuse';
|
|
1036
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
1037
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
1038
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
1039
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
1040
|
+
});
|
|
1041
|
+
assert.equal(result.ok, false);
|
|
1042
|
+
if (result.ok)
|
|
1043
|
+
return;
|
|
1044
|
+
assert.match(result.error, /scale_up_rollback_cleanup_debt:pane_owner_unverified:%31/);
|
|
1045
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
1046
|
+
assert.equal(config?.workers[1]?.pane_id, '%31');
|
|
1047
|
+
assert.equal(config?.workers[1]?.pid, 42424);
|
|
1048
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
1049
|
+
assert.equal(tmuxCommands.some((command) => command.includes('set-option -p -t %31 @omx_team_pane_owner_id')), false);
|
|
1050
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), true);
|
|
1051
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %31'), false);
|
|
805
1052
|
}
|
|
806
1053
|
finally {
|
|
807
1054
|
if (typeof previousPath === 'string')
|
|
@@ -1126,6 +1373,7 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
1126
1373
|
'set -eu',
|
|
1127
1374
|
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
1128
1375
|
'case "${1:-}" in',
|
|
1376
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1129
1377
|
' -V)',
|
|
1130
1378
|
' echo "tmux 3.2a"',
|
|
1131
1379
|
' ;;',
|
|
@@ -1133,7 +1381,16 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
1133
1381
|
' echo "%31"',
|
|
1134
1382
|
' ;;',
|
|
1135
1383
|
' list-panes)',
|
|
1136
|
-
'
|
|
1384
|
+
' case "$*" in',
|
|
1385
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
1386
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '42411'",
|
|
1387
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
1388
|
+
" printf '%s\\t%s\\t%s\\n' '%31' '0' '42424'",
|
|
1389
|
+
' ;;',
|
|
1390
|
+
' *)',
|
|
1391
|
+
' echo "42424"',
|
|
1392
|
+
' ;;',
|
|
1393
|
+
' esac',
|
|
1137
1394
|
' ;;',
|
|
1138
1395
|
' send-keys)',
|
|
1139
1396
|
' ;;',
|
|
@@ -1178,7 +1435,9 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
1178
1435
|
return;
|
|
1179
1436
|
config.tmux_session = 'omx-team-scale-up-project-reasoning';
|
|
1180
1437
|
config.leader_pane_id = '%11';
|
|
1438
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1181
1439
|
config.workers[0].pane_id = '%21';
|
|
1440
|
+
config.workers[0].pid = 42421;
|
|
1182
1441
|
await saveTeamConfig(config, cwd);
|
|
1183
1442
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'scale-up-project-reasoning', 'manifest.v2.json');
|
|
1184
1443
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1232,6 +1491,7 @@ printf '%s\\n' "$@" > '${capturePath}'
|
|
|
1232
1491
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1233
1492
|
set -eu
|
|
1234
1493
|
case "\${1:-}" in
|
|
1494
|
+
show-option) echo 'team:scale-up' ;;
|
|
1235
1495
|
-V)
|
|
1236
1496
|
echo "tmux 3.2a"
|
|
1237
1497
|
;;
|
|
@@ -1239,11 +1499,23 @@ set -eu
|
|
|
1239
1499
|
echo "%31"
|
|
1240
1500
|
;;
|
|
1241
1501
|
list-panes)
|
|
1242
|
-
|
|
1502
|
+
case "$*" in
|
|
1503
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1504
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1505
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1506
|
+
if [ ! -f "${fakeBinDir}/killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1507
|
+
;;
|
|
1508
|
+
*)
|
|
1509
|
+
echo "42424"
|
|
1510
|
+
;;
|
|
1511
|
+
esac
|
|
1243
1512
|
;;
|
|
1244
1513
|
send-keys)
|
|
1245
1514
|
exit 1
|
|
1246
1515
|
;;
|
|
1516
|
+
kill-pane)
|
|
1517
|
+
: > "${fakeBinDir}/killed-$3"
|
|
1518
|
+
;;
|
|
1247
1519
|
capture-pane)
|
|
1248
1520
|
echo ""
|
|
1249
1521
|
;;
|
|
@@ -1267,7 +1539,9 @@ exit 0
|
|
|
1267
1539
|
return;
|
|
1268
1540
|
config.tmux_session = 'omx-team-rollback-worktree';
|
|
1269
1541
|
config.leader_pane_id = '%11';
|
|
1542
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1270
1543
|
config.workers[0].pane_id = '%21';
|
|
1544
|
+
config.workers[0].pid = 42421;
|
|
1271
1545
|
await saveTeamConfig(config, cwd);
|
|
1272
1546
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'rollback-worktree', 'manifest.v2.json');
|
|
1273
1547
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1282,7 +1556,7 @@ exit 0
|
|
|
1282
1556
|
return;
|
|
1283
1557
|
assert.match(result.error, /scale_up_dispatch_failed:worker-2/);
|
|
1284
1558
|
const workerRootAgents = join(cwd, '.omx', 'team', 'rollback-worktree', 'worktrees', 'worker-2', 'AGENTS.md');
|
|
1285
|
-
assert.
|
|
1559
|
+
await assert.rejects(readFile(workerRootAgents, 'utf-8'), { code: 'ENOENT' });
|
|
1286
1560
|
const backupPath = join(cwd, '.git', 'worktrees', 'worker-2', 'omx', 'root-agents-backup.json');
|
|
1287
1561
|
assert.equal(existsSync(backupPath), false);
|
|
1288
1562
|
}
|
|
@@ -1304,6 +1578,7 @@ exit 0
|
|
|
1304
1578
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1305
1579
|
set -eu
|
|
1306
1580
|
case "\${1:-}" in
|
|
1581
|
+
show-option) echo 'team:scale-up' ;;
|
|
1307
1582
|
-V)
|
|
1308
1583
|
echo "tmux 3.2a"
|
|
1309
1584
|
;;
|
|
@@ -1311,11 +1586,23 @@ set -eu
|
|
|
1311
1586
|
echo "%31"
|
|
1312
1587
|
;;
|
|
1313
1588
|
list-panes)
|
|
1314
|
-
|
|
1589
|
+
case "$*" in
|
|
1590
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1591
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1592
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1593
|
+
if [ ! -f "${fakeBinDir}/canonical-root-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1594
|
+
;;
|
|
1595
|
+
*)
|
|
1596
|
+
echo "42424"
|
|
1597
|
+
;;
|
|
1598
|
+
esac
|
|
1315
1599
|
;;
|
|
1316
1600
|
capture-pane)
|
|
1317
1601
|
echo ""
|
|
1318
1602
|
;;
|
|
1603
|
+
kill-pane)
|
|
1604
|
+
: > "${fakeBinDir}/canonical-root-killed-$3"
|
|
1605
|
+
;;
|
|
1319
1606
|
esac
|
|
1320
1607
|
exit 0
|
|
1321
1608
|
`);
|
|
@@ -1336,7 +1623,9 @@ exit 0
|
|
|
1336
1623
|
return;
|
|
1337
1624
|
config.tmux_session = 'omx-team-canonical-root';
|
|
1338
1625
|
config.leader_pane_id = '%11';
|
|
1626
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1339
1627
|
config.workers[0].pane_id = '%21';
|
|
1628
|
+
config.workers[0].pid = 42421;
|
|
1340
1629
|
await saveTeamConfig(config, cwd);
|
|
1341
1630
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'canonical-root', 'manifest.v2.json');
|
|
1342
1631
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1380,6 +1669,7 @@ exit 0
|
|
|
1380
1669
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1381
1670
|
set -eu
|
|
1382
1671
|
case "\${1:-}" in
|
|
1672
|
+
show-option) echo 'team:scale-up' ;;
|
|
1383
1673
|
-V)
|
|
1384
1674
|
echo "tmux 3.2a"
|
|
1385
1675
|
;;
|
|
@@ -1387,11 +1677,23 @@ case "\${1:-}" in
|
|
|
1387
1677
|
echo "%31"
|
|
1388
1678
|
;;
|
|
1389
1679
|
list-panes)
|
|
1390
|
-
|
|
1680
|
+
case "$*" in
|
|
1681
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1682
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1683
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1684
|
+
if [ ! -f "${fakeBinDir}/frontier-role-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1685
|
+
;;
|
|
1686
|
+
*)
|
|
1687
|
+
echo "42424"
|
|
1688
|
+
;;
|
|
1689
|
+
esac
|
|
1391
1690
|
;;
|
|
1392
1691
|
capture-pane)
|
|
1393
1692
|
echo ""
|
|
1394
1693
|
;;
|
|
1694
|
+
kill-pane)
|
|
1695
|
+
: > "${fakeBinDir}/frontier-role-killed-$3"
|
|
1696
|
+
;;
|
|
1395
1697
|
esac
|
|
1396
1698
|
exit 0
|
|
1397
1699
|
`);
|
|
@@ -1414,7 +1716,9 @@ exit 0
|
|
|
1414
1716
|
return;
|
|
1415
1717
|
config.tmux_session = 'omx-team-frontier-role';
|
|
1416
1718
|
config.leader_pane_id = '%11';
|
|
1719
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1417
1720
|
config.workers[0].pane_id = '%21';
|
|
1721
|
+
config.workers[0].pid = 42421;
|
|
1418
1722
|
await saveTeamConfig(config, cwd);
|
|
1419
1723
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'frontier-role', 'manifest.v2.json');
|
|
1420
1724
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1450,6 +1754,7 @@ exit 0
|
|
|
1450
1754
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1451
1755
|
set -eu
|
|
1452
1756
|
case "\${1:-}" in
|
|
1757
|
+
show-option) echo 'team:scale-up' ;;
|
|
1453
1758
|
-V)
|
|
1454
1759
|
echo "tmux 3.2a"
|
|
1455
1760
|
;;
|
|
@@ -1457,11 +1762,23 @@ case "\${1:-}" in
|
|
|
1457
1762
|
echo "%31"
|
|
1458
1763
|
;;
|
|
1459
1764
|
list-panes)
|
|
1460
|
-
|
|
1765
|
+
case "$*" in
|
|
1766
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1767
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1768
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1769
|
+
if [ ! -f "${fakeBinDir}/mini-tuned-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1770
|
+
;;
|
|
1771
|
+
*)
|
|
1772
|
+
echo "42424"
|
|
1773
|
+
;;
|
|
1774
|
+
esac
|
|
1461
1775
|
;;
|
|
1462
1776
|
capture-pane)
|
|
1463
1777
|
echo ""
|
|
1464
1778
|
;;
|
|
1779
|
+
kill-pane)
|
|
1780
|
+
: > "${fakeBinDir}/mini-tuned-killed-$3"
|
|
1781
|
+
;;
|
|
1465
1782
|
esac
|
|
1466
1783
|
exit 0
|
|
1467
1784
|
`);
|
|
@@ -1482,7 +1799,9 @@ exit 0
|
|
|
1482
1799
|
return;
|
|
1483
1800
|
config.tmux_session = 'omx-team-mini-tuned-root';
|
|
1484
1801
|
config.leader_pane_id = '%11';
|
|
1802
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1485
1803
|
config.workers[0].pane_id = '%21';
|
|
1804
|
+
config.workers[0].pid = 42421;
|
|
1486
1805
|
await saveTeamConfig(config, cwd);
|
|
1487
1806
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'mini-tuned-root', 'manifest.v2.json');
|
|
1488
1807
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1524,6 +1843,7 @@ exit 0
|
|
|
1524
1843
|
set -eu
|
|
1525
1844
|
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
1526
1845
|
case "\${1:-}" in
|
|
1846
|
+
show-option) echo 'team:scale-up' ;;
|
|
1527
1847
|
-V)
|
|
1528
1848
|
echo "tmux 3.2a"
|
|
1529
1849
|
;;
|
|
@@ -1531,11 +1851,23 @@ case "\${1:-}" in
|
|
|
1531
1851
|
echo "%31"
|
|
1532
1852
|
;;
|
|
1533
1853
|
list-panes)
|
|
1534
|
-
|
|
1854
|
+
case "$*" in
|
|
1855
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
1856
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
1857
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
1858
|
+
if [ ! -f "${fakeBinDir}/layout-killed-%31" ]; then printf '%s\t%s\t%s\n' '%31' '0' '42424'; fi
|
|
1859
|
+
;;
|
|
1860
|
+
*)
|
|
1861
|
+
echo "42424"
|
|
1862
|
+
;;
|
|
1863
|
+
esac
|
|
1535
1864
|
;;
|
|
1536
1865
|
capture-pane)
|
|
1537
1866
|
echo ""
|
|
1538
1867
|
;;
|
|
1868
|
+
kill-pane)
|
|
1869
|
+
: > "${fakeBinDir}/layout-killed-$3"
|
|
1870
|
+
;;
|
|
1539
1871
|
esac
|
|
1540
1872
|
exit 0
|
|
1541
1873
|
`);
|
|
@@ -1549,7 +1881,9 @@ exit 0
|
|
|
1549
1881
|
return;
|
|
1550
1882
|
config.tmux_session = 'omx-team-scale-up-layout';
|
|
1551
1883
|
config.leader_pane_id = '%11';
|
|
1884
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1552
1885
|
config.workers[0].pane_id = '%21';
|
|
1886
|
+
config.workers[0].pid = 42421;
|
|
1553
1887
|
await saveTeamConfig(config, cwd);
|
|
1554
1888
|
const manifestPath = join(cwd, '.omx', 'state', 'team', 'scale-up-layout', 'manifest.v2.json');
|
|
1555
1889
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1562,9 +1896,14 @@ exit 0
|
|
|
1562
1896
|
assert.equal(result.ok, true);
|
|
1563
1897
|
if (!result.ok)
|
|
1564
1898
|
return;
|
|
1565
|
-
const
|
|
1566
|
-
|
|
1567
|
-
assert.
|
|
1899
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
1900
|
+
const splitWindowIndex = tmuxCommands.findIndex((command) => command.startsWith('split-window '));
|
|
1901
|
+
assert.ok(splitWindowIndex > 2);
|
|
1902
|
+
assert.match(tmuxCommands[splitWindowIndex - 3], /^list-panes -a -F #\{pane_id\}\t#\{pane_dead\}\t#\{pane_pid\}$/);
|
|
1903
|
+
assert.equal(tmuxCommands[splitWindowIndex - 2], 'show-option -qv -p -t %21 @omx_team_pane_owner_id');
|
|
1904
|
+
assert.match(tmuxCommands[splitWindowIndex - 1], /^list-panes -a -F #\{pane_id\}\t#\{pane_dead\}\t#\{pane_pid\}$/);
|
|
1905
|
+
assert.match(tmuxCommands[splitWindowIndex], /split-window -v -t %21/);
|
|
1906
|
+
assert.doesNotMatch(tmuxCommands.join('\n'), /select-layout .*tiled/);
|
|
1568
1907
|
}
|
|
1569
1908
|
finally {
|
|
1570
1909
|
if (typeof previousPath === 'string')
|
|
@@ -1575,29 +1914,216 @@ exit 0
|
|
|
1575
1914
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1576
1915
|
}
|
|
1577
1916
|
});
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1917
|
+
for (const [name, proofOutput, proofExitStatus, expectedReason] of [
|
|
1918
|
+
['dead persisted worker pane', '%21\t1\t42421\n', 0, 'dead'],
|
|
1919
|
+
['similar persisted worker pane ID', '%210\t0\t424210\n', 0, 'absent'],
|
|
1920
|
+
['malformed global pane PID', '%21\t0\tnot-a-pid\n', 0, 'malformed_snapshot'],
|
|
1921
|
+
['duplicate global pane proof rows', '%21\t0\t42421\n%21\t0\t42422\n', 0, 'malformed_snapshot'],
|
|
1922
|
+
['unavailable global pane query', '', 1, 'query_failed'],
|
|
1923
|
+
]) {
|
|
1924
|
+
it(`fails closed before split-window when exact proof has ${name}`, async () => {
|
|
1925
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-proof-'));
|
|
1926
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-proof-bin-'));
|
|
1927
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
1928
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1929
|
+
const previousPath = process.env.PATH;
|
|
1930
|
+
try {
|
|
1931
|
+
await writeFile(tmuxStubPath, [
|
|
1932
|
+
'#!/bin/sh',
|
|
1933
|
+
'set -eu',
|
|
1934
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
1935
|
+
'case "${1:-}" in',
|
|
1936
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1937
|
+
' -V)',
|
|
1938
|
+
' echo "tmux 3.2a"',
|
|
1939
|
+
' ;;',
|
|
1940
|
+
' list-panes)',
|
|
1941
|
+
` printf '%b' ${JSON.stringify(proofOutput)}`,
|
|
1942
|
+
proofExitStatus === 0 ? ' ;;' : ' exit 1',
|
|
1943
|
+
proofExitStatus === 0 ? '' : ' ;;',
|
|
1944
|
+
' split-window)',
|
|
1945
|
+
' echo "%31"',
|
|
1946
|
+
' ;;',
|
|
1947
|
+
'esac',
|
|
1948
|
+
'exit 0',
|
|
1949
|
+
'',
|
|
1950
|
+
].join('\n'));
|
|
1951
|
+
await chmod(tmuxStubPath, 0o755);
|
|
1952
|
+
await writeFile(tmuxLogPath, '');
|
|
1953
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1954
|
+
const teamName = `split-proof-${expectedReason === 'malformed_snapshot' ? name.includes('duplicate') ? 'duplicate' : 'malformed' : expectedReason.replace('_', '-')}`;
|
|
1955
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
1956
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
1957
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1958
|
+
assert.equal(result.ok, false);
|
|
1959
|
+
if (!result.ok) {
|
|
1960
|
+
assert.equal(result.error, `scale_up_split_target_proof_unavailable:%21:${expectedReason}`);
|
|
1961
|
+
}
|
|
1962
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
1963
|
+
assert.equal(config?.workers.length, 1);
|
|
1964
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
1965
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
1966
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
1967
|
+
}
|
|
1968
|
+
finally {
|
|
1969
|
+
if (typeof previousPath === 'string')
|
|
1970
|
+
process.env.PATH = previousPath;
|
|
1971
|
+
else
|
|
1972
|
+
delete process.env.PATH;
|
|
1973
|
+
await rm(cwd, { recursive: true, force: true });
|
|
1974
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1975
|
+
}
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
1978
|
+
it('migrates a legacy worker PID from an exact owner-bound proof before split-window', async () => {
|
|
1979
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-target-pid-'));
|
|
1980
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-target-pid-bin-'));
|
|
1981
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
1982
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1983
|
+
const previousPath = process.env.PATH;
|
|
1984
|
+
try {
|
|
1985
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
1986
|
+
const stub = await readFile(tmuxStubPath, 'utf-8');
|
|
1987
|
+
const configPath = join(cwd, '.omx', 'state', 'team', 'legacy-target-pid', 'config.json');
|
|
1988
|
+
await writeFile(tmuxStubPath, stub.replace(' split-window)', ` split-window)\n grep -q '"pid": 42421' "${configPath}" || exit 23`));
|
|
1989
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1990
|
+
const teamName = 'legacy-target-pid';
|
|
1991
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
1992
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
1993
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
1994
|
+
assert.ok(config);
|
|
1995
|
+
if (!config)
|
|
1996
|
+
return;
|
|
1997
|
+
delete config.workers[0].pid;
|
|
1998
|
+
await saveTeamConfig(config, cwd);
|
|
1999
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
2000
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2001
|
+
});
|
|
2002
|
+
assert.equal(result.ok, true);
|
|
2003
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers[0]?.pid, 42421);
|
|
2004
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2005
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window ')));
|
|
2006
|
+
}
|
|
2007
|
+
finally {
|
|
2008
|
+
if (typeof previousPath === 'string')
|
|
2009
|
+
process.env.PATH = previousPath;
|
|
2010
|
+
else
|
|
2011
|
+
delete process.env.PATH;
|
|
2012
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2013
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2014
|
+
}
|
|
2015
|
+
});
|
|
2016
|
+
for (const [name, proofOutput, owner, expectedError] of [
|
|
2017
|
+
['malformed proof', '%21\\t0\\tnot-a-pid\\n', 'team:scale-up', 'scale_up_split_target_proof_unavailable:%21:malformed_snapshot'],
|
|
2018
|
+
['foreign pane', '%210\\t0\\t424210\\n', 'team:scale-up', 'scale_up_split_target_proof_unavailable:%21:absent'],
|
|
2019
|
+
['foreign owner tag', '%21\\t0\\t42421\\n', 'team:foreign', 'scale_up_split_target_owner_changed:%21'],
|
|
2020
|
+
]) {
|
|
2021
|
+
it(`rejects legacy PID migration with ${name} before a scale effect`, async () => {
|
|
2022
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-proof-reject-'));
|
|
2023
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-legacy-proof-reject-bin-'));
|
|
2024
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2025
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2026
|
+
const previousPath = process.env.PATH;
|
|
2027
|
+
try {
|
|
2028
|
+
await writeFile(tmuxStubPath, [
|
|
2029
|
+
'#!/bin/sh', 'set -eu', `printf '%s\\n' "$*" >> "${tmuxLogPath}"`, 'case "${1:-}" in',
|
|
2030
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
2031
|
+
` list-panes) printf '%b' ${JSON.stringify(proofOutput)} ;;`,
|
|
2032
|
+
` show-option) echo ${JSON.stringify(owner)} ;;`,
|
|
2033
|
+
'esac', 'exit 0', '',
|
|
2034
|
+
].join('\n'));
|
|
2035
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2036
|
+
await writeFile(tmuxLogPath, '');
|
|
2037
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2038
|
+
const teamName = `legacy-proof-${name.replaceAll(' ', '-')}`;
|
|
2039
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2040
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2041
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
2042
|
+
assert.ok(config);
|
|
2043
|
+
if (!config)
|
|
2044
|
+
return;
|
|
2045
|
+
delete config.workers[0].pid;
|
|
2046
|
+
await saveTeamConfig(config, cwd);
|
|
2047
|
+
assert.deepEqual(await scaleUp(teamName, 1, 'executor', [], cwd, {
|
|
2048
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2049
|
+
}), { ok: false, error: expectedError });
|
|
2050
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers[0]?.pid, undefined);
|
|
2051
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2052
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ') || command.startsWith('set-option ')), false);
|
|
2053
|
+
}
|
|
2054
|
+
finally {
|
|
2055
|
+
if (typeof previousPath === 'string')
|
|
2056
|
+
process.env.PATH = previousPath;
|
|
2057
|
+
else
|
|
2058
|
+
delete process.env.PATH;
|
|
2059
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2060
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2061
|
+
}
|
|
2062
|
+
});
|
|
2063
|
+
}
|
|
2064
|
+
it('rejects a PID-reused explicit split target at the first proof before any split or owner tag', async () => {
|
|
2065
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-first-proof-reuse-'));
|
|
2066
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-first-proof-reuse-bin-'));
|
|
2067
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2068
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2069
|
+
const previousPath = process.env.PATH;
|
|
2070
|
+
try {
|
|
2071
|
+
await writeFile(tmuxStubPath, [
|
|
2072
|
+
'#!/bin/sh', 'set -eu', `printf '%s\\n' "$*" >> "${tmuxLogPath}"`, 'case "${1:-}" in', ' show-option) echo "team:scale-up" ;;',
|
|
2073
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2074
|
+
' -V) echo "tmux 3.2a" ;;',
|
|
2075
|
+
" list-panes) printf '%s\\t%s\\t%s\\n' '%21' '0' '52421' ;;",
|
|
2076
|
+
'esac', 'exit 0', '',
|
|
2077
|
+
].join('\n'));
|
|
2078
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2079
|
+
await writeFile(tmuxLogPath, '');
|
|
2080
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2081
|
+
const teamName = 'first-proof-reuse';
|
|
2082
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2083
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2084
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2085
|
+
assert.deepEqual(result, { ok: false, error: 'scale_up_split_target_pid_changed:%21:42421:52421' });
|
|
2086
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2087
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ') || command.startsWith('set-option ')), false);
|
|
2088
|
+
}
|
|
2089
|
+
finally {
|
|
2090
|
+
if (typeof previousPath === 'string')
|
|
2091
|
+
process.env.PATH = previousPath;
|
|
2092
|
+
else
|
|
2093
|
+
delete process.env.PATH;
|
|
2094
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2095
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2096
|
+
}
|
|
2097
|
+
});
|
|
2098
|
+
it('rolls back all preparation when the split target proof is lost immediately before split-window', async () => {
|
|
2099
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-second-split-proof-'));
|
|
2100
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-second-split-proof-bin-'));
|
|
1581
2101
|
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2102
|
+
const proofCountPath = join(fakeBinDir, 'proof-count');
|
|
1582
2103
|
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1583
2104
|
const previousPath = process.env.PATH;
|
|
1584
2105
|
try {
|
|
1585
2106
|
await writeFile(tmuxStubPath, [
|
|
1586
2107
|
'#!/bin/sh',
|
|
1587
2108
|
'set -eu',
|
|
1588
|
-
`printf '%s
|
|
2109
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
1589
2110
|
'case "${1:-}" in',
|
|
2111
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1590
2112
|
' -V)',
|
|
1591
2113
|
' echo "tmux 3.2a"',
|
|
1592
2114
|
' ;;',
|
|
1593
|
-
' split-window)',
|
|
1594
|
-
' echo "%41"',
|
|
1595
|
-
' ;;',
|
|
1596
2115
|
' list-panes)',
|
|
1597
|
-
|
|
2116
|
+
` count=0; [ ! -f "${proofCountPath}" ] || count=$(cat "${proofCountPath}")`,
|
|
2117
|
+
' count=$((count + 1))',
|
|
2118
|
+
` printf '%s' "$count" > "${proofCountPath}"`,
|
|
2119
|
+
' if [ "$count" -eq 1 ]; then',
|
|
2120
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
2121
|
+
' else',
|
|
2122
|
+
' exit 1',
|
|
2123
|
+
' fi',
|
|
1598
2124
|
' ;;',
|
|
1599
|
-
'
|
|
1600
|
-
' echo ""',
|
|
2125
|
+
' split-window)',
|
|
2126
|
+
' echo "%31"',
|
|
1601
2127
|
' ;;',
|
|
1602
2128
|
'esac',
|
|
1603
2129
|
'exit 0',
|
|
@@ -1606,65 +2132,228 @@ exit 0
|
|
|
1606
2132
|
await chmod(tmuxStubPath, 0o755);
|
|
1607
2133
|
await writeFile(tmuxLogPath, '');
|
|
1608
2134
|
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1609
|
-
const teamName = '
|
|
1610
|
-
await
|
|
1611
|
-
await
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2135
|
+
const teamName = 'second-split-proof';
|
|
2136
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2137
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2138
|
+
const teamStateDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
2139
|
+
const configPath = join(teamStateDir, 'config.json');
|
|
2140
|
+
const tasksDir = join(teamStateDir, 'tasks');
|
|
2141
|
+
const workersDir = join(teamStateDir, 'workers');
|
|
2142
|
+
const runtimeDir = join(teamStateDir, 'runtime');
|
|
2143
|
+
const configBefore = await readFile(configPath, 'utf-8');
|
|
2144
|
+
const taskEntriesBefore = (await readdir(tasksDir)).sort();
|
|
2145
|
+
const workerEntriesBefore = (await readdir(workersDir)).sort();
|
|
2146
|
+
const runtimeDirExistedBefore = existsSync(runtimeDir);
|
|
2147
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'must not persist', description: 'proof loss rollback', owner: 'worker-2' }], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2148
|
+
assert.deepEqual(result, {
|
|
2149
|
+
ok: false,
|
|
2150
|
+
error: 'scale_up_split_target_proof_unavailable:%21:query_failed',
|
|
1617
2151
|
});
|
|
1618
|
-
const
|
|
1619
|
-
assert.
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
await
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
};
|
|
1632
|
-
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
1633
|
-
const result = await scaleUp(teamName, 1, 'executor', [], repo, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1634
|
-
assert.equal(result.ok, true);
|
|
1635
|
-
if (!result.ok)
|
|
1636
|
-
return;
|
|
1637
|
-
const updated = await readTeamConfig(teamName, repo);
|
|
1638
|
-
const worker = updated?.workers.find((entry) => entry.name === 'worker-2');
|
|
1639
|
-
assert.deepEqual(updated?.worktree_mode, { enabled: true, detached: true, name: null });
|
|
1640
|
-
assert.ok(worker?.worktree_path, 'scaled worker should have detached worktree path');
|
|
1641
|
-
assert.equal(worker?.working_dir, worker?.worktree_path);
|
|
1642
|
-
assert.equal(worker?.worktree_detached, true);
|
|
1643
|
-
assert.equal(worker?.worktree_created, true);
|
|
1644
|
-
assert.equal(existsSync(worker?.worktree_path), true);
|
|
1645
|
-
assert.throws(() => execFileSync('git', ['symbolic-ref', '--quiet', '--short', 'HEAD'], { cwd: worker?.worktree_path, stdio: 'pipe' }));
|
|
2152
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2153
|
+
assert.deepEqual(tmuxCommands.filter((command) => command.startsWith('list-panes ')), [
|
|
2154
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2155
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2156
|
+
]);
|
|
2157
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
2158
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
2159
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(configPath, 'utf-8')), withoutConfigGeneration(configBefore));
|
|
2160
|
+
assert.deepEqual((await readdir(tasksDir)).sort(), taskEntriesBefore);
|
|
2161
|
+
assert.deepEqual((await readdir(workersDir)).sort(), workerEntriesBefore);
|
|
2162
|
+
assert.equal(existsSync(join(workersDir, 'worker-2')), false);
|
|
2163
|
+
assert.equal(existsSync(join(cwd, '.omx', 'team', teamName, 'worktrees', 'worker-2')), false);
|
|
2164
|
+
assert.equal(existsSync(runtimeDir), runtimeDirExistedBefore);
|
|
1646
2165
|
}
|
|
1647
2166
|
finally {
|
|
1648
2167
|
if (typeof previousPath === 'string')
|
|
1649
2168
|
process.env.PATH = previousPath;
|
|
1650
2169
|
else
|
|
1651
2170
|
delete process.env.PATH;
|
|
1652
|
-
await rm(
|
|
2171
|
+
await rm(cwd, { recursive: true, force: true });
|
|
1653
2172
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1654
2173
|
}
|
|
1655
2174
|
});
|
|
1656
|
-
it('
|
|
1657
|
-
const
|
|
1658
|
-
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-
|
|
2175
|
+
it('rejects a split target ID replacement that occurs during owner authorization', async () => {
|
|
2176
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-target-pid-reuse-'));
|
|
2177
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-split-target-pid-reuse-bin-'));
|
|
1659
2178
|
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2179
|
+
const proofCountPath = join(fakeBinDir, 'proof-count');
|
|
1660
2180
|
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
1661
2181
|
const previousPath = process.env.PATH;
|
|
1662
2182
|
try {
|
|
1663
2183
|
await writeFile(tmuxStubPath, [
|
|
1664
2184
|
'#!/bin/sh',
|
|
1665
2185
|
'set -eu',
|
|
1666
|
-
`printf '%s
|
|
2186
|
+
`printf '%s\\n' "$*" >> "${tmuxLogPath}"`,
|
|
1667
2187
|
'case "${1:-}" in',
|
|
2188
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2189
|
+
' -V)',
|
|
2190
|
+
' echo "tmux 3.2a"',
|
|
2191
|
+
' ;;',
|
|
2192
|
+
' list-panes)',
|
|
2193
|
+
` count=0; [ ! -f "${proofCountPath}" ] || count=$(cat "${proofCountPath}")`,
|
|
2194
|
+
' count=$((count + 1))',
|
|
2195
|
+
` printf '%s' "$count" > "${proofCountPath}"`,
|
|
2196
|
+
' if [ "$count" -eq 1 ]; then',
|
|
2197
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '42421'",
|
|
2198
|
+
' else',
|
|
2199
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '52421'",
|
|
2200
|
+
' fi',
|
|
2201
|
+
' ;;',
|
|
2202
|
+
' split-window)',
|
|
2203
|
+
' echo "%31"',
|
|
2204
|
+
' ;;',
|
|
2205
|
+
'esac',
|
|
2206
|
+
'exit 0',
|
|
2207
|
+
'',
|
|
2208
|
+
].join('\n'));
|
|
2209
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2210
|
+
await writeFile(tmuxLogPath, '');
|
|
2211
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2212
|
+
const teamName = 'split-target-pid-reuse';
|
|
2213
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2214
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2215
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
2216
|
+
assert.ok(config);
|
|
2217
|
+
if (!config)
|
|
2218
|
+
return;
|
|
2219
|
+
config.workers[0].pid = 42421;
|
|
2220
|
+
await saveTeamConfig(config, cwd);
|
|
2221
|
+
const teamStateDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
2222
|
+
const configPath = join(teamStateDir, 'config.json');
|
|
2223
|
+
const tasksDir = join(teamStateDir, 'tasks');
|
|
2224
|
+
const workersDir = join(teamStateDir, 'workers');
|
|
2225
|
+
const runtimeDir = join(teamStateDir, 'runtime');
|
|
2226
|
+
const configBefore = await readFile(configPath, 'utf-8');
|
|
2227
|
+
const taskEntriesBefore = (await readdir(tasksDir)).sort();
|
|
2228
|
+
const workerEntriesBefore = (await readdir(workersDir)).sort();
|
|
2229
|
+
const runtimeDirExistedBefore = existsSync(runtimeDir);
|
|
2230
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'must not persist', description: 'PID reuse rollback', owner: 'worker-2' }], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2231
|
+
assert.deepEqual(result, {
|
|
2232
|
+
ok: false,
|
|
2233
|
+
error: 'scale_up_split_target_pid_changed:%21:42421:52421',
|
|
2234
|
+
});
|
|
2235
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2236
|
+
assert.equal(tmuxCommands.filter((command) => command.startsWith('list-panes ')).length, 2);
|
|
2237
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('split-window ')), false);
|
|
2238
|
+
assert.equal(tmuxCommands.some((command) => command.startsWith('send-keys ')), false);
|
|
2239
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(configPath, 'utf-8')), withoutConfigGeneration(configBefore));
|
|
2240
|
+
assert.deepEqual((await readdir(tasksDir)).sort(), taskEntriesBefore);
|
|
2241
|
+
assert.deepEqual((await readdir(workersDir)).sort(), workerEntriesBefore);
|
|
2242
|
+
assert.equal(existsSync(join(workersDir, 'worker-2')), false);
|
|
2243
|
+
assert.equal(existsSync(runtimeDir), runtimeDirExistedBefore);
|
|
2244
|
+
}
|
|
2245
|
+
finally {
|
|
2246
|
+
if (typeof previousPath === 'string')
|
|
2247
|
+
process.env.PATH = previousPath;
|
|
2248
|
+
else
|
|
2249
|
+
delete process.env.PATH;
|
|
2250
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2251
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2252
|
+
}
|
|
2253
|
+
});
|
|
2254
|
+
it('provisions detached worktrees for scaled-up workers from persisted team worktree mode', async () => {
|
|
2255
|
+
const repo = await initRepo();
|
|
2256
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-detached-bin-'));
|
|
2257
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2258
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2259
|
+
const previousPath = process.env.PATH;
|
|
2260
|
+
try {
|
|
2261
|
+
await writeFile(tmuxStubPath, [
|
|
2262
|
+
'#!/bin/sh',
|
|
2263
|
+
'set -eu',
|
|
2264
|
+
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
2265
|
+
'case "${1:-}" in',
|
|
2266
|
+
' show-option) echo "team:scale-up" ;;',
|
|
2267
|
+
' -V)',
|
|
2268
|
+
' echo "tmux 3.2a"',
|
|
2269
|
+
' ;;',
|
|
2270
|
+
' split-window)',
|
|
2271
|
+
' echo "%41"',
|
|
2272
|
+
' ;;',
|
|
2273
|
+
' list-panes)',
|
|
2274
|
+
' case "$*" in',
|
|
2275
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
2276
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '45451'",
|
|
2277
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '45452'",
|
|
2278
|
+
" printf '%s\\t%s\\t%s\\n' '%41' '0' '45454'",
|
|
2279
|
+
' ;;',
|
|
2280
|
+
' *)',
|
|
2281
|
+
' echo "45454"',
|
|
2282
|
+
' ;;',
|
|
2283
|
+
' esac',
|
|
2284
|
+
' ;;',
|
|
2285
|
+
' capture-pane)',
|
|
2286
|
+
' echo ""',
|
|
2287
|
+
' ;;',
|
|
2288
|
+
'esac',
|
|
2289
|
+
'exit 0',
|
|
2290
|
+
'',
|
|
2291
|
+
].join('\n'));
|
|
2292
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2293
|
+
await writeFile(tmuxLogPath, '');
|
|
2294
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2295
|
+
const teamName = 'scale-up-detached-worktree';
|
|
2296
|
+
await mkdir(join(repo, '.omx', 'state', 'team', teamName), { recursive: true });
|
|
2297
|
+
await writeFile(join(repo, '.omx', 'state', 'team', teamName, 'worker-agents.md'), '# Base worker instructions\n');
|
|
2298
|
+
await initTeamState(teamName, 'task', 'executor', 1, repo, DEFAULT_MAX_WORKERS, process.env, {
|
|
2299
|
+
leader_cwd: repo,
|
|
2300
|
+
team_state_root: join(repo, '.omx', 'state'),
|
|
2301
|
+
workspace_mode: 'worktree',
|
|
2302
|
+
worktree_mode: { enabled: true, detached: true, name: null },
|
|
2303
|
+
});
|
|
2304
|
+
const config = await readTeamConfig(teamName, repo);
|
|
2305
|
+
assert.ok(config);
|
|
2306
|
+
if (!config)
|
|
2307
|
+
return;
|
|
2308
|
+
config.tmux_session = `omx-team-${teamName}`;
|
|
2309
|
+
config.leader_pane_id = '%11';
|
|
2310
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
2311
|
+
config.workers[0].pane_id = '%21';
|
|
2312
|
+
config.workers[0].pid = 45452;
|
|
2313
|
+
await saveTeamConfig(config, repo);
|
|
2314
|
+
const manifestPath = join(repo, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
2315
|
+
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
2316
|
+
manifest.policy = {
|
|
2317
|
+
...(manifest.policy ?? {}),
|
|
2318
|
+
dispatch_mode: 'transport_direct',
|
|
2319
|
+
};
|
|
2320
|
+
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
2321
|
+
const result = await scaleUp(teamName, 1, 'executor', [], repo, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
2322
|
+
assert.equal(result.ok, true);
|
|
2323
|
+
if (!result.ok)
|
|
2324
|
+
return;
|
|
2325
|
+
const updated = await readTeamConfig(teamName, repo);
|
|
2326
|
+
const worker = updated?.workers.find((entry) => entry.name === 'worker-2');
|
|
2327
|
+
assert.deepEqual(updated?.worktree_mode, { enabled: true, detached: true, name: null });
|
|
2328
|
+
assert.ok(worker?.worktree_path, 'scaled worker should have detached worktree path');
|
|
2329
|
+
assert.equal(worker?.working_dir, worker?.worktree_path);
|
|
2330
|
+
assert.equal(worker?.worktree_detached, true);
|
|
2331
|
+
assert.equal(worker?.worktree_created, true);
|
|
2332
|
+
assert.equal(existsSync(worker?.worktree_path), true);
|
|
2333
|
+
assert.throws(() => execFileSync('git', ['symbolic-ref', '--quiet', '--short', 'HEAD'], { cwd: worker?.worktree_path, stdio: 'pipe' }));
|
|
2334
|
+
}
|
|
2335
|
+
finally {
|
|
2336
|
+
if (typeof previousPath === 'string')
|
|
2337
|
+
process.env.PATH = previousPath;
|
|
2338
|
+
else
|
|
2339
|
+
delete process.env.PATH;
|
|
2340
|
+
await rm(repo, { recursive: true, force: true });
|
|
2341
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2342
|
+
}
|
|
2343
|
+
});
|
|
2344
|
+
it('provisions named worktrees for scaled-up workers from persisted team worktree mode', async () => {
|
|
2345
|
+
const repo = await initRepo();
|
|
2346
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-named-bin-'));
|
|
2347
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2348
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2349
|
+
const previousPath = process.env.PATH;
|
|
2350
|
+
try {
|
|
2351
|
+
await writeFile(tmuxStubPath, [
|
|
2352
|
+
'#!/bin/sh',
|
|
2353
|
+
'set -eu',
|
|
2354
|
+
`printf '%s\n' "$*" >> "${tmuxLogPath}"`,
|
|
2355
|
+
'case "${1:-}" in',
|
|
2356
|
+
' show-option) echo "team:scale-up" ;;',
|
|
1668
2357
|
' -V)',
|
|
1669
2358
|
' echo "tmux 3.2a"',
|
|
1670
2359
|
' ;;',
|
|
@@ -1672,7 +2361,16 @@ exit 0
|
|
|
1672
2361
|
' echo "%42"',
|
|
1673
2362
|
' ;;',
|
|
1674
2363
|
' list-panes)',
|
|
1675
|
-
'
|
|
2364
|
+
' case "$*" in',
|
|
2365
|
+
' *"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)',
|
|
2366
|
+
" printf '%s\\t%s\\t%s\\n' '%11' '0' '46461'",
|
|
2367
|
+
" printf '%s\\t%s\\t%s\\n' '%21' '0' '46462'",
|
|
2368
|
+
" printf '%s\\t%s\\t%s\\n' '%42' '0' '46464'",
|
|
2369
|
+
' ;;',
|
|
2370
|
+
' *)',
|
|
2371
|
+
' echo "46464"',
|
|
2372
|
+
' ;;',
|
|
2373
|
+
' esac',
|
|
1676
2374
|
' ;;',
|
|
1677
2375
|
' capture-pane)',
|
|
1678
2376
|
' echo ""',
|
|
@@ -1700,7 +2398,9 @@ exit 0
|
|
|
1700
2398
|
return;
|
|
1701
2399
|
config.tmux_session = `omx-team-${teamName}`;
|
|
1702
2400
|
config.leader_pane_id = '%11';
|
|
2401
|
+
config.tmux_pane_owner_id = 'team:scale-up';
|
|
1703
2402
|
config.workers[0].pane_id = '%21';
|
|
2403
|
+
config.workers[0].pid = 46462;
|
|
1704
2404
|
await saveTeamConfig(config, repo);
|
|
1705
2405
|
const manifestPath = join(repo, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
1706
2406
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
@@ -1732,82 +2432,454 @@ exit 0
|
|
|
1732
2432
|
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1733
2433
|
}
|
|
1734
2434
|
});
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
const
|
|
2435
|
+
it('rolls back pre-split worker preparation when worktree-root instructions cannot materialize', async () => {
|
|
2436
|
+
const cwd = await initRepo();
|
|
2437
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-pre-split-root-failure-bin-'));
|
|
2438
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2439
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2440
|
+
const previousPath = process.env.PATH;
|
|
2441
|
+
const teamName = 'pre-split-root-failure';
|
|
2442
|
+
const worktreePath = join(cwd, '.omx', 'team', teamName, 'worktrees', 'worker-2');
|
|
1742
2443
|
try {
|
|
1743
|
-
|
|
2444
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2445
|
+
set -eu
|
|
2446
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
2447
|
+
case "\${1:-}" in
|
|
2448
|
+
show-option) echo 'team:scale-up' ;;
|
|
2449
|
+
-V) echo 'tmux 3.2a' ;;
|
|
2450
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%21' '0' '42421' ;;
|
|
2451
|
+
esac
|
|
2452
|
+
`);
|
|
2453
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2454
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2455
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2456
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2457
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
2458
|
+
assert.ok(config);
|
|
2459
|
+
if (!config)
|
|
2460
|
+
return;
|
|
2461
|
+
config.tmux_session = `omx-team-${teamName}`;
|
|
2462
|
+
config.workers[0].pane_id = '%21';
|
|
2463
|
+
config.workspace_mode = 'worktree';
|
|
2464
|
+
config.worktree_mode = { enabled: true, detached: true, name: null };
|
|
2465
|
+
await saveTeamConfig(config, cwd);
|
|
2466
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktreePath], { cwd, stdio: 'ignore' });
|
|
2467
|
+
await mkdir(join(worktreePath, 'AGENTS.md'));
|
|
2468
|
+
const result = await scaleUp(teamName, 1, 'executor', [], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1744
2469
|
assert.equal(result.ok, false);
|
|
1745
|
-
if (
|
|
1746
|
-
|
|
1747
|
-
|
|
2470
|
+
if (result.ok)
|
|
2471
|
+
return;
|
|
2472
|
+
assert.match(result.error, /^scale_up_worker_preparation_failed:worker-2:/);
|
|
2473
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
2474
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), false);
|
|
2475
|
+
assert.equal(existsSync(join(worktreePath, 'AGENTS.md')), true);
|
|
2476
|
+
assert.deepEqual(await readScaleUpTmuxLogCommands(tmuxLogPath), [
|
|
2477
|
+
'-V',
|
|
2478
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2479
|
+
]);
|
|
1748
2480
|
}
|
|
1749
2481
|
finally {
|
|
2482
|
+
if (typeof previousPath === 'string')
|
|
2483
|
+
process.env.PATH = previousPath;
|
|
2484
|
+
else
|
|
2485
|
+
delete process.env.PATH;
|
|
1750
2486
|
await rm(cwd, { recursive: true, force: true });
|
|
2487
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1751
2488
|
}
|
|
1752
2489
|
});
|
|
1753
|
-
it('
|
|
1754
|
-
const cwd = await
|
|
2490
|
+
it('retains only unresolved rollback workers and deletes never-created tasks after an explicit proof-loss phase', async () => {
|
|
2491
|
+
const cwd = await initRepo();
|
|
2492
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-rollback-kill-fail-bin-'));
|
|
2493
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2494
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2495
|
+
const proofLossUsedPath = join(fakeBinDir, 'proof-loss-used');
|
|
2496
|
+
const splitCountPath = join(fakeBinDir, 'split-count');
|
|
2497
|
+
const previousPath = process.env.PATH;
|
|
1755
2498
|
try {
|
|
1756
|
-
await
|
|
1757
|
-
|
|
2499
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2500
|
+
set -eu
|
|
2501
|
+
printf '%s\n' "$*" >> "${tmuxLogPath}"
|
|
2502
|
+
case "\${1:-}" in
|
|
2503
|
+
show-option) echo 'team:scale-up' ;;
|
|
2504
|
+
-V) echo 'tmux 3.2a' ;;
|
|
2505
|
+
list-panes)
|
|
2506
|
+
split_count=0; [ ! -f "${splitCountPath}" ] || split_count=$(cat "${splitCountPath}")
|
|
2507
|
+
if [ "$split_count" -ge 2 ] && [ ! -f "${proofLossUsedPath}" ]; then
|
|
2508
|
+
: > "${proofLossUsedPath}"
|
|
2509
|
+
exit 1
|
|
2510
|
+
fi
|
|
2511
|
+
if [ "$split_count" -ge 2 ]; then
|
|
2512
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
2513
|
+
printf '%s\t%s\t%s\n' '%31' '0' '42431'
|
|
2514
|
+
printf '%s\t%s\t%s\n' '%32' '0' '42432'
|
|
2515
|
+
elif [ "$split_count" -eq 1 ]; then
|
|
2516
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
2517
|
+
printf '%s\t%s\t%s\n' '%31' '0' '42431'
|
|
2518
|
+
else
|
|
2519
|
+
printf '%s\t%s\t%s\n' '%21' '0' '42421'
|
|
2520
|
+
fi
|
|
2521
|
+
;;
|
|
2522
|
+
split-window)
|
|
2523
|
+
split_count=0; [ ! -f "${splitCountPath}" ] || split_count=$(cat "${splitCountPath}")
|
|
2524
|
+
split_count=$((split_count + 1)); printf '%s' "$split_count" > "${splitCountPath}"
|
|
2525
|
+
if [ "$split_count" -eq 1 ]; then
|
|
2526
|
+
echo '%31'
|
|
2527
|
+
else
|
|
2528
|
+
echo '%32'
|
|
2529
|
+
fi
|
|
2530
|
+
;;
|
|
2531
|
+
kill-pane)
|
|
2532
|
+
case "$3" in
|
|
2533
|
+
%31|%32) exit 0 ;;
|
|
2534
|
+
*) exit 1 ;;
|
|
2535
|
+
esac
|
|
2536
|
+
;;
|
|
2537
|
+
esac
|
|
2538
|
+
`);
|
|
2539
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2540
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2541
|
+
await initTeamState('rollback-kill-fail', 'task', 'executor', 1, cwd);
|
|
2542
|
+
await configureScaleUpTeamForDirectDispatch('rollback-kill-fail', cwd);
|
|
2543
|
+
const rollbackConfig = await readTeamConfig('rollback-kill-fail', cwd);
|
|
2544
|
+
assert.ok(rollbackConfig);
|
|
2545
|
+
if (!rollbackConfig)
|
|
2546
|
+
return;
|
|
2547
|
+
rollbackConfig.workspace_mode = 'worktree';
|
|
2548
|
+
rollbackConfig.worktree_mode = { enabled: true, detached: true, name: null };
|
|
2549
|
+
await saveTeamConfig(rollbackConfig, cwd);
|
|
2550
|
+
const result = await scaleUp('rollback-kill-fail', 3, 'executor', [
|
|
2551
|
+
{ subject: 'first', description: 'created before mixed rollback', owner: 'worker-2' },
|
|
2552
|
+
{ subject: 'second', description: 'created before mixed rollback', owner: 'worker-3' },
|
|
2553
|
+
{ subject: 'third', description: 'never created after proof loss', owner: 'worker-4' },
|
|
2554
|
+
], cwd, { OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SKIP_READY_WAIT: '1' });
|
|
1758
2555
|
assert.equal(result.ok, false);
|
|
1759
|
-
if (!result.ok)
|
|
1760
|
-
assert.match(result.error, /
|
|
2556
|
+
if (!result.ok)
|
|
2557
|
+
assert.match(result.error, /^scale_up_rollback_cleanup_debt:.*pane_teardown_failed:%31.*pane_teardown_unresolved:%32/);
|
|
2558
|
+
assert.equal(await readFile(splitCountPath, 'utf-8'), '2');
|
|
2559
|
+
const config = await readTeamConfig('rollback-kill-fail', cwd);
|
|
2560
|
+
assert.deepEqual(config?.workers.map((worker) => worker.name), ['worker-1', 'worker-2', 'worker-3']);
|
|
2561
|
+
assert.equal(config?.worker_count, 3);
|
|
2562
|
+
assert.equal(config?.next_worker_index, 4);
|
|
2563
|
+
assert.equal((await readTask('rollback-kill-fail', '1', cwd))?.owner, 'worker-2');
|
|
2564
|
+
assert.equal((await readTask('rollback-kill-fail', '2', cwd))?.owner, 'worker-3');
|
|
2565
|
+
assert.equal(await readTask('rollback-kill-fail', '3', cwd), null);
|
|
2566
|
+
for (const workerName of ['worker-2', 'worker-3']) {
|
|
2567
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'rollback-kill-fail', 'workers', workerName);
|
|
2568
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, 'rollback-kill-fail', workerName)), true, `${workerName} startup script must remain`);
|
|
2569
|
+
assert.equal(existsSync(join(cwd, '.omx', 'team', 'rollback-kill-fail', 'worktrees', workerName)), true, `${workerName} worktree must remain`);
|
|
2570
|
+
if (workerName === 'worker-2') {
|
|
2571
|
+
assert.equal(existsSync(workerDir), true, 'worker-2 materialized state must remain retryable');
|
|
2572
|
+
assert.equal(existsSync(join(workerDir, 'identity.json')), true, 'worker-2 identity must remain');
|
|
2573
|
+
assert.equal(existsSync(join(workerDir, 'inbox.md')), true, 'worker-2 inbox must remain');
|
|
2574
|
+
}
|
|
2575
|
+
else {
|
|
2576
|
+
assert.equal(existsSync(join(workerDir, 'identity.json')), false, 'worker-3 proof failed before identity materialization');
|
|
2577
|
+
assert.equal(existsSync(join(workerDir, 'inbox.md')), false, 'worker-3 proof failed before inbox materialization');
|
|
2578
|
+
}
|
|
1761
2579
|
}
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
2580
|
+
// P2: retry execution of recorded cleanup debt is intentionally a follow-up surface.
|
|
2581
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2582
|
+
const mutationCommands = tmuxCommands
|
|
2583
|
+
.filter((command) => command.startsWith('split-window ') || command.startsWith('kill-pane '))
|
|
2584
|
+
.map((command) => command.startsWith('split-window ') ? command.split(' -c ')[0] : command);
|
|
2585
|
+
assert.deepEqual(mutationCommands, [
|
|
2586
|
+
'split-window -v -t %21 -d -P -F #{pane_id}',
|
|
2587
|
+
'split-window -v -t %31 -d -P -F #{pane_id}',
|
|
2588
|
+
'kill-pane -t %31',
|
|
2589
|
+
]);
|
|
2590
|
+
for (const mutationCommand of mutationCommands) {
|
|
2591
|
+
const mutationIndex = tmuxCommands.findIndex((command) => (command === mutationCommand || command.startsWith(`${mutationCommand} -c `)));
|
|
2592
|
+
if (mutationCommand.startsWith('split-window ')) {
|
|
2593
|
+
assert.ok(mutationIndex > 2);
|
|
2594
|
+
assert.equal(tmuxCommands[mutationIndex - 3], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2595
|
+
assert.match(tmuxCommands[mutationIndex - 2], /^show-option -qv -p -t %(?:21|31) @omx_team_pane_owner_id$/);
|
|
2596
|
+
assert.equal(tmuxCommands[mutationIndex - 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2597
|
+
}
|
|
2598
|
+
else {
|
|
2599
|
+
assert.ok(mutationIndex > 2);
|
|
2600
|
+
assert.equal(tmuxCommands[mutationIndex - 3], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2601
|
+
assert.equal(tmuxCommands[mutationIndex - 2], 'show-option -qv -p -t %31 @omx_team_pane_owner_id');
|
|
2602
|
+
assert.equal(tmuxCommands[mutationIndex - 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
2603
|
+
}
|
|
1775
2604
|
}
|
|
2605
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window -v -t %21 ')));
|
|
2606
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window -v -t %31 ')));
|
|
2607
|
+
assert.ok(tmuxCommands.some((command) => command === 'kill-pane -t %31'));
|
|
2608
|
+
assert.equal(tmuxCommands.some((command) => command === 'kill-pane -t %32'), false);
|
|
1776
2609
|
}
|
|
1777
2610
|
finally {
|
|
2611
|
+
if (typeof previousPath === 'string')
|
|
2612
|
+
process.env.PATH = previousPath;
|
|
2613
|
+
else
|
|
2614
|
+
delete process.env.PATH;
|
|
1778
2615
|
await rm(cwd, { recursive: true, force: true });
|
|
2616
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1779
2617
|
}
|
|
1780
2618
|
});
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
2619
|
+
for (const rollbackFailurePhase of ['rollback-membership-config-persistence', 'rollback-membership-manifest-persistence']) {
|
|
2620
|
+
it(`recovers and raw-verifies original membership after public scaleUp ${rollbackFailurePhase}`, async () => {
|
|
2621
|
+
const cwd = await mkdtemp(join(tmpdir(), `omx-scale-up-${rollbackFailurePhase}-`));
|
|
2622
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), `omx-scale-up-${rollbackFailurePhase}-bin-`));
|
|
2623
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2624
|
+
const previousPath = process.env.PATH;
|
|
2625
|
+
const teamName = `scale-up-${rollbackFailurePhase.replace('rollback-membership-', '').replace('-persistence', '')}`;
|
|
2626
|
+
try {
|
|
2627
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
2628
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2629
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2630
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2631
|
+
const teamDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
2632
|
+
const configPath = join(teamDir, 'config.json');
|
|
2633
|
+
const manifestPath = join(teamDir, 'manifest.v2.json');
|
|
2634
|
+
const originalConfigBytes = await readFile(configPath, 'utf8');
|
|
2635
|
+
const originalManifestBytes = await readFile(manifestPath, 'utf8');
|
|
2636
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: 'public rollback recovery', description: 'must be removed', owner: 'worker-2' }], cwd, {
|
|
2637
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2638
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2639
|
+
OMX_TEAM_SCALE_UP_INJECT_FAILURE: `finalization,${rollbackFailurePhase}`,
|
|
2640
|
+
});
|
|
2641
|
+
assert.equal(result.ok, false);
|
|
2642
|
+
if (!result.ok)
|
|
2643
|
+
assert.match(result.error, /scale_up_rollback_membership_persistence_failed:.*rollback-persistence-failure/);
|
|
2644
|
+
await readTeamConfig(teamName, cwd);
|
|
2645
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(configPath, 'utf8')), withoutConfigGeneration(originalConfigBytes));
|
|
2646
|
+
assert.deepEqual(withoutConfigGeneration(await readFile(manifestPath, 'utf8')), withoutConfigGeneration(originalManifestBytes));
|
|
2647
|
+
assert.equal(existsSync(join(teamDir, '.membership-task-transaction.json')), false);
|
|
2648
|
+
assert.equal(await readTask(teamName, '1', cwd), null);
|
|
2649
|
+
assert.deepEqual(await listDispatchRequests(teamName, cwd), []);
|
|
2650
|
+
assert.equal(existsSync(join(teamDir, 'workers', 'worker-2')), true);
|
|
2651
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), true);
|
|
1800
2652
|
}
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
});
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
2653
|
+
finally {
|
|
2654
|
+
if (typeof previousPath === 'string')
|
|
2655
|
+
process.env.PATH = previousPath;
|
|
2656
|
+
else
|
|
2657
|
+
delete process.env.PATH;
|
|
2658
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2659
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2660
|
+
}
|
|
2661
|
+
});
|
|
2662
|
+
}
|
|
2663
|
+
for (const phase of ['identity', 'inbox', 'config']) {
|
|
2664
|
+
it(`rolls back live pane and materialized state when ${phase} materialization fails`, async () => {
|
|
2665
|
+
const cwd = await mkdtemp(join(tmpdir(), `omx-scale-up-${phase}-failure-`));
|
|
2666
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), `omx-scale-up-${phase}-failure-bin-`));
|
|
2667
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2668
|
+
const previousPath = process.env.PATH;
|
|
2669
|
+
try {
|
|
2670
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
2671
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2672
|
+
const teamName = `scale-up-${phase}-failure`;
|
|
2673
|
+
await initTeamState(teamName, 'task', 'executor', 1, cwd);
|
|
2674
|
+
await configureScaleUpTeamForDirectDispatch(teamName, cwd);
|
|
2675
|
+
const result = await scaleUp(teamName, 1, 'executor', [{ subject: `${phase} rollback`, description: 'must be removed', owner: 'worker-2' }], cwd, {
|
|
2676
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2677
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2678
|
+
OMX_TEAM_SCALE_UP_INJECT_FAILURE: phase,
|
|
2679
|
+
});
|
|
2680
|
+
assert.equal(result.ok, false);
|
|
2681
|
+
if (!result.ok)
|
|
2682
|
+
assert.match(result.error, new RegExp(`scale_up_worker_materialization_failed:worker-2:.*${phase}`));
|
|
2683
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers.length, 1);
|
|
2684
|
+
assert.equal(await readTask(teamName, '1', cwd), null);
|
|
2685
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
2686
|
+
assert.equal(existsSync(workerStartupScriptPath(cwd, teamName, 'worker-2')), false);
|
|
2687
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2688
|
+
assert.ok(tmuxCommands.some((command) => command.startsWith('split-window ')));
|
|
2689
|
+
assert.ok(tmuxCommands.includes('kill-pane -t %31'));
|
|
2690
|
+
}
|
|
2691
|
+
finally {
|
|
2692
|
+
if (typeof previousPath === 'string')
|
|
2693
|
+
process.env.PATH = previousPath;
|
|
2694
|
+
else
|
|
2695
|
+
delete process.env.PATH;
|
|
2696
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2697
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2698
|
+
}
|
|
2699
|
+
});
|
|
2700
|
+
}
|
|
2701
|
+
it('rolls back a named worktree created before injected ensure bookkeeping fails', async () => {
|
|
2702
|
+
const repo = await initRepo();
|
|
2703
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-up-named-post-create-bin-'));
|
|
2704
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2705
|
+
const previousPath = process.env.PATH;
|
|
2706
|
+
const teamName = 'named-post-create-failure';
|
|
2707
|
+
const branchName = 'feature/named-post-create/worker-2';
|
|
2708
|
+
const worktreePath = join(repo, '.omx', 'team', teamName, 'worktrees', 'worker-2');
|
|
2709
|
+
try {
|
|
2710
|
+
await writeSuccessfulScaleUpTmuxStub(fakeBinDir, tmuxLogPath);
|
|
2711
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2712
|
+
await initTeamState(teamName, 'task', 'executor', 1, repo, undefined, process.env, {
|
|
2713
|
+
workspace_mode: 'worktree',
|
|
2714
|
+
leader_cwd: repo,
|
|
2715
|
+
team_state_root: join(repo, '.omx', 'state'),
|
|
2716
|
+
worktree_mode: { enabled: true, detached: false, name: 'feature/named-post-create' },
|
|
2717
|
+
});
|
|
2718
|
+
await configureScaleUpTeamForDirectDispatch(teamName, repo);
|
|
2719
|
+
const result = await scaleUp(teamName, 1, 'executor', [], repo, {
|
|
2720
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2721
|
+
OMX_TEAM_SKIP_READY_WAIT: '1',
|
|
2722
|
+
OMX_TEAM_SCALE_UP_INJECT_FAILURE: 'worktree-ensure-post-create',
|
|
2723
|
+
});
|
|
2724
|
+
assert.equal(result.ok, false);
|
|
2725
|
+
if (!result.ok)
|
|
2726
|
+
assert.match(result.error, /scale_up_worker_preparation_failed:worker-2:.*worktree-ensure-post-create/);
|
|
2727
|
+
assert.equal(existsSync(worktreePath), false);
|
|
2728
|
+
assert.throws(() => execFileSync('git', ['show-ref', '--verify', '--quiet', `refs/heads/${branchName}`], { cwd: repo }));
|
|
2729
|
+
assert.equal((await readTeamConfig(teamName, repo))?.workers.length, 1);
|
|
2730
|
+
assert.equal(existsSync(join(repo, '.omx', 'state', 'team', teamName, 'workers', 'worker-2')), false);
|
|
2731
|
+
assert.deepEqual((await readScaleUpTmuxLogCommands(tmuxLogPath)).filter((command) => command.startsWith('split-window ')), []);
|
|
2732
|
+
}
|
|
2733
|
+
finally {
|
|
2734
|
+
if (typeof previousPath === 'string')
|
|
2735
|
+
process.env.PATH = previousPath;
|
|
2736
|
+
else
|
|
2737
|
+
delete process.env.PATH;
|
|
2738
|
+
await rm(repo, { recursive: true, force: true });
|
|
2739
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2740
|
+
}
|
|
2741
|
+
});
|
|
2742
|
+
});
|
|
2743
|
+
describe('scaleDown', () => {
|
|
2744
|
+
it('rejects when scaling is disabled', async () => {
|
|
2745
|
+
await assert.rejects(scaleDown('test', '/tmp', {}, {}), /Dynamic scaling is disabled/);
|
|
2746
|
+
});
|
|
2747
|
+
it('returns error when team not found', async () => {
|
|
2748
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-nf-'));
|
|
2749
|
+
try {
|
|
2750
|
+
const result = await scaleDown('nonexistent', cwd, {}, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2751
|
+
assert.equal(result.ok, false);
|
|
2752
|
+
if (!result.ok) {
|
|
2753
|
+
assert.match(result.error, /not found/);
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2756
|
+
finally {
|
|
2757
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2758
|
+
}
|
|
2759
|
+
});
|
|
2760
|
+
it('returns error when trying to remove all workers', async () => {
|
|
2761
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-all-'));
|
|
2762
|
+
try {
|
|
2763
|
+
await initTeamState('all-test', 'task', 'executor', 1, cwd);
|
|
2764
|
+
const result = await scaleDown('all-test', cwd, { workerNames: ['worker-1'] }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2765
|
+
assert.equal(result.ok, false);
|
|
2766
|
+
if (!result.ok) {
|
|
2767
|
+
assert.match(result.error, /at least 1 must remain/);
|
|
2768
|
+
}
|
|
2769
|
+
}
|
|
2770
|
+
finally {
|
|
2771
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2772
|
+
}
|
|
2773
|
+
});
|
|
2774
|
+
it('returns error for worker not in team', async () => {
|
|
2775
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-miss-'));
|
|
2776
|
+
try {
|
|
2777
|
+
await initTeamState('miss-test', 'task', 'executor', 2, cwd);
|
|
2778
|
+
const result = await scaleDown('miss-test', cwd, { workerNames: ['worker-99'] }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2779
|
+
assert.equal(result.ok, false);
|
|
2780
|
+
if (!result.ok) {
|
|
2781
|
+
assert.match(result.error, /Worker worker-99 not found/);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
finally {
|
|
2785
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2786
|
+
}
|
|
2787
|
+
});
|
|
2788
|
+
it('returns error when not enough idle workers and force=false', async () => {
|
|
2789
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-busy-'));
|
|
2790
|
+
try {
|
|
2791
|
+
await initTeamState('busy-test', 'task', 'executor', 2, cwd);
|
|
2792
|
+
// Write working status for both workers
|
|
2793
|
+
await writeWorkerStatus('busy-test', 'worker-1', {
|
|
2794
|
+
state: 'working',
|
|
2795
|
+
current_task_id: 't-1',
|
|
2796
|
+
updated_at: new Date().toISOString(),
|
|
2797
|
+
}, cwd);
|
|
2798
|
+
await writeWorkerStatus('busy-test', 'worker-2', {
|
|
2799
|
+
state: 'working',
|
|
2800
|
+
current_task_id: 't-2',
|
|
2801
|
+
updated_at: new Date().toISOString(),
|
|
2802
|
+
}, cwd);
|
|
2803
|
+
const result = await scaleDown('busy-test', cwd, { count: 1 }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2804
|
+
assert.equal(result.ok, false);
|
|
2805
|
+
if (!result.ok) {
|
|
2806
|
+
assert.match(result.error, /Not enough idle workers/);
|
|
2807
|
+
}
|
|
2808
|
+
}
|
|
2809
|
+
finally {
|
|
2810
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2811
|
+
}
|
|
2812
|
+
});
|
|
2813
|
+
it('serializes a concurrent claim across the canonical scale-down boundary', async () => {
|
|
2814
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-claim-boundary-'));
|
|
2815
|
+
try {
|
|
2816
|
+
await initTeamState('claim-boundary', 'task', 'executor', 2, cwd);
|
|
2817
|
+
const task = await createTask('claim-boundary', {
|
|
2818
|
+
subject: 'boundary task', description: 'must not be claimed by a removed worker', status: 'pending', owner: 'worker-2',
|
|
2819
|
+
}, cwd);
|
|
2820
|
+
const down = scaleDown('claim-boundary', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
2821
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2822
|
+
OMX_TEAM_SCALE_DOWN_BOUNDARY_HOLD_MS: '100',
|
|
2823
|
+
});
|
|
2824
|
+
const lockPath = join(cwd, '.omx', 'state', 'team', 'claim-boundary', 'claims', `task-${task.id}.lock`);
|
|
2825
|
+
for (let attempt = 0; attempt < 50 && !existsSync(lockPath); attempt += 1) {
|
|
2826
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
2827
|
+
}
|
|
2828
|
+
assert.equal(existsSync(lockPath), true);
|
|
2829
|
+
const claim = claimTask('claim-boundary', task.id, 'worker-2', task.version ?? 1, cwd);
|
|
2830
|
+
assert.deepEqual(await down, { ok: true, removedWorkers: ['worker-2'], newWorkerCount: 1 });
|
|
2831
|
+
assert.deepEqual(await claim, { ok: false, error: 'claim_conflict' });
|
|
2832
|
+
const reconciled = await readTask('claim-boundary', task.id, cwd);
|
|
2833
|
+
assert.equal(reconciled?.owner, undefined);
|
|
2834
|
+
assert.equal(reconciled?.claim, undefined);
|
|
2835
|
+
}
|
|
2836
|
+
finally {
|
|
2837
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2838
|
+
}
|
|
2839
|
+
});
|
|
2840
|
+
it('blocks post-snapshot task creation and null-version claims until removed membership is committed', async () => {
|
|
2841
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-create-claim-boundary-'));
|
|
2842
|
+
try {
|
|
2843
|
+
await initTeamState('create-claim-boundary', 'task', 'executor', 2, cwd);
|
|
2844
|
+
const seed = await createTask('create-claim-boundary', {
|
|
2845
|
+
subject: 'snapshot marker', description: 'holds the canonical task lock', status: 'pending', owner: 'worker-2',
|
|
2846
|
+
}, cwd);
|
|
2847
|
+
const postSnapshotMarker = join(cwd, 'post-snapshot-held');
|
|
2848
|
+
const down = scaleDown('create-claim-boundary', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
2849
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
2850
|
+
OMX_TEAM_SCALE_DOWN_BOUNDARY_HOLD_MS: '250',
|
|
2851
|
+
OMX_TEAM_SCALE_DOWN_POST_SNAPSHOT_MARKER: postSnapshotMarker,
|
|
2852
|
+
});
|
|
2853
|
+
for (let attempt = 0; attempt < 50 && !existsSync(postSnapshotMarker); attempt += 1) {
|
|
2854
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
2855
|
+
}
|
|
2856
|
+
assert.equal(existsSync(postSnapshotMarker), true);
|
|
2857
|
+
let createSettled = false;
|
|
2858
|
+
let claimSettled = false;
|
|
2859
|
+
const created = createTask('create-claim-boundary', {
|
|
2860
|
+
subject: 'created after snapshot', description: 'must not restore removed membership', status: 'pending', owner: 'worker-2',
|
|
2861
|
+
}, cwd).finally(() => { createSettled = true; });
|
|
2862
|
+
const claim = claimTask('create-claim-boundary', seed.id, 'worker-2', null, cwd)
|
|
2863
|
+
.finally(() => { claimSettled = true; });
|
|
2864
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
2865
|
+
assert.equal(createSettled, false);
|
|
2866
|
+
assert.equal(claimSettled, false);
|
|
2867
|
+
assert.deepEqual(await down, { ok: true, removedWorkers: ['worker-2'], newWorkerCount: 1 });
|
|
2868
|
+
assert.deepEqual(await claim, { ok: false, error: 'worker_not_found' });
|
|
2869
|
+
const task = await created;
|
|
2870
|
+
assert.equal(task.owner, 'worker-2');
|
|
2871
|
+
assert.deepEqual((await readTeamConfig('create-claim-boundary', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
2872
|
+
assert.equal((await readTask('create-claim-boundary', task.id, cwd))?.owner, 'worker-2');
|
|
2873
|
+
}
|
|
2874
|
+
finally {
|
|
2875
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2876
|
+
}
|
|
2877
|
+
});
|
|
2878
|
+
});
|
|
2879
|
+
describe('scaleDown worktree AGENTS cleanup', () => {
|
|
2880
|
+
it('removes generated worktree-root AGENTS during scale-down', async () => {
|
|
2881
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-worktree-agents-'));
|
|
2882
|
+
try {
|
|
1811
2883
|
await initTeamState('scale-down-worktree', 'task', 'executor', 2, cwd, undefined, process.env, {
|
|
1812
2884
|
workspace_mode: 'worktree',
|
|
1813
2885
|
leader_cwd: cwd,
|
|
@@ -1838,15 +2910,47 @@ describe('scaleDown worktree AGENTS cleanup', () => {
|
|
|
1838
2910
|
});
|
|
1839
2911
|
});
|
|
1840
2912
|
describe('scaleDown teardown hardening', () => {
|
|
1841
|
-
it('scaleDown removes
|
|
2913
|
+
it('scaleDown removes a worker with a pane proven dead without targeting it', async () => {
|
|
1842
2914
|
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-dead-'));
|
|
2915
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-dead-bin-'));
|
|
2916
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2917
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2918
|
+
const previousPath = process.env.PATH;
|
|
1843
2919
|
try {
|
|
2920
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2921
|
+
set -eu
|
|
2922
|
+
printf '%s\n' "$*" >> "${tmuxLogPath}"
|
|
2923
|
+
case "\${1:-}" in
|
|
2924
|
+
show-option) echo 'team:scale-down' ;;
|
|
2925
|
+
list-panes)
|
|
2926
|
+
case "$*" in
|
|
2927
|
+
*"-a -F #{pane_id}"*)
|
|
2928
|
+
printf '%%405\t1\t4050\n'
|
|
2929
|
+
;;
|
|
2930
|
+
*)
|
|
2931
|
+
exit 1
|
|
2932
|
+
;;
|
|
2933
|
+
esac
|
|
2934
|
+
;;
|
|
2935
|
+
kill-pane)
|
|
2936
|
+
echo "a proven-dead pane must not be targeted" >&2
|
|
2937
|
+
exit 1
|
|
2938
|
+
;;
|
|
2939
|
+
*)
|
|
2940
|
+
exit 0
|
|
2941
|
+
;;
|
|
2942
|
+
esac
|
|
2943
|
+
`);
|
|
2944
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2945
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
1844
2946
|
await initTeamState('dead-pane', 'task', 'executor', 2, cwd);
|
|
1845
2947
|
const config = await readTeamConfig('dead-pane', cwd);
|
|
1846
2948
|
assert.ok(config);
|
|
1847
2949
|
if (!config)
|
|
1848
2950
|
return;
|
|
1849
|
-
config.workers[1].pane_id = '%
|
|
2951
|
+
config.workers[1].pane_id = '%405';
|
|
2952
|
+
config.workers[1].pid = 424405;
|
|
2953
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
1850
2954
|
await saveTeamConfig(config, cwd);
|
|
1851
2955
|
const result = await scaleDown('dead-pane', cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
1852
2956
|
assert.equal(result.ok, true);
|
|
@@ -1856,9 +2960,81 @@ describe('scaleDown teardown hardening', () => {
|
|
|
1856
2960
|
const updated = await readTeamConfig('dead-pane', cwd);
|
|
1857
2961
|
assert.ok(updated);
|
|
1858
2962
|
assert.equal(updated?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
2963
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
2964
|
+
assert.deepEqual(tmuxCommands, [
|
|
2965
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2966
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
2967
|
+
]);
|
|
2968
|
+
}
|
|
2969
|
+
finally {
|
|
2970
|
+
if (typeof previousPath === 'string')
|
|
2971
|
+
process.env.PATH = previousPath;
|
|
2972
|
+
else
|
|
2973
|
+
delete process.env.PATH;
|
|
2974
|
+
await rm(cwd, { recursive: true, force: true });
|
|
2975
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
2976
|
+
}
|
|
2977
|
+
});
|
|
2978
|
+
it('retains PID-less rollback debt when a recycled same-Team pane remains live', async () => {
|
|
2979
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-unpinned-recycled-'));
|
|
2980
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-unpinned-recycled-bin-'));
|
|
2981
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
2982
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
2983
|
+
const previousPath = process.env.PATH;
|
|
2984
|
+
try {
|
|
2985
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
2986
|
+
set -eu
|
|
2987
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
2988
|
+
case "\${1:-}" in
|
|
2989
|
+
list-panes) printf '%%405\\t0\\t999405\\n' ;;
|
|
2990
|
+
show-option|kill-pane) echo "PID-less debt must not authorize an effect" >&2; exit 99 ;;
|
|
2991
|
+
esac
|
|
2992
|
+
`);
|
|
2993
|
+
await chmod(tmuxStubPath, 0o755);
|
|
2994
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
2995
|
+
await initTeamState('unpinned-recycled', 'task', 'executor', 2, cwd);
|
|
2996
|
+
const config = await readTeamConfig('unpinned-recycled', cwd);
|
|
2997
|
+
assert.ok(config);
|
|
2998
|
+
if (!config)
|
|
2999
|
+
return;
|
|
3000
|
+
config.workers[1].pane_id = '%405';
|
|
3001
|
+
config.workers[1].pid = undefined;
|
|
3002
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3003
|
+
await saveTeamConfig(config, cwd);
|
|
3004
|
+
const initial = await scaleDown('unpinned-recycled', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3005
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
3006
|
+
});
|
|
3007
|
+
assert.deepEqual(initial, {
|
|
3008
|
+
ok: false,
|
|
3009
|
+
error: 'scale_down_cleanup_debt:pane_teardown_unresolved:%405',
|
|
3010
|
+
});
|
|
3011
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'unpinned-recycled', '.scale-down-cleanup-debt.json');
|
|
3012
|
+
const initialDebt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
3013
|
+
assert.deepEqual(initialDebt.unresolved_panes, [{ name: 'worker-2', index: 2, pane_id: '%405', pid: null }]);
|
|
3014
|
+
assert.ok(initialDebt.reasons.includes('%405:legacy_pid_missing_live'));
|
|
3015
|
+
const canonical = await readTeamConfig('unpinned-recycled', cwd);
|
|
3016
|
+
assert.ok(canonical);
|
|
3017
|
+
if (!canonical)
|
|
3018
|
+
return;
|
|
3019
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('unpinned-recycled', cwd, canonical), {
|
|
3020
|
+
ok: false,
|
|
3021
|
+
error: 'scale_down_cleanup_debt_unresolved:%405',
|
|
3022
|
+
});
|
|
3023
|
+
const reconciledDebt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
3024
|
+
assert.ok(reconciledDebt.reasons.includes('%405:legacy_pid_missing_live'));
|
|
3025
|
+
const commands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3026
|
+
assert.deepEqual(commands, [
|
|
3027
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3028
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3029
|
+
]);
|
|
1859
3030
|
}
|
|
1860
3031
|
finally {
|
|
3032
|
+
if (typeof previousPath === 'string')
|
|
3033
|
+
process.env.PATH = previousPath;
|
|
3034
|
+
else
|
|
3035
|
+
delete process.env.PATH;
|
|
1861
3036
|
await rm(cwd, { recursive: true, force: true });
|
|
3037
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
1862
3038
|
}
|
|
1863
3039
|
});
|
|
1864
3040
|
it('scaleDown never targets leader or hud panes during teardown', async () => {
|
|
@@ -1871,6 +3047,22 @@ describe('scaleDown teardown hardening', () => {
|
|
|
1871
3047
|
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
1872
3048
|
set -eu
|
|
1873
3049
|
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3050
|
+
case "\${1:-}" in
|
|
3051
|
+
show-option) echo 'team:scale-down' ;;
|
|
3052
|
+
list-panes)
|
|
3053
|
+
case "$*" in
|
|
3054
|
+
*"#{pane_id}\t#{pane_dead}\t#{pane_pid}"*)
|
|
3055
|
+
printf '%s\t%s\t%s\n' '%11' '0' '42411'
|
|
3056
|
+
printf '%s\t%s\t%s\n' '%12' '0' '42412'
|
|
3057
|
+
if [ ! -f "${tmuxLogPath}.killed-%13" ]; then printf '%s\t%s\t%s\n' '%13' '0' '42413'; fi
|
|
3058
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3059
|
+
;;
|
|
3060
|
+
esac
|
|
3061
|
+
;;
|
|
3062
|
+
kill-pane)
|
|
3063
|
+
: > "${tmuxLogPath}.killed-$3"
|
|
3064
|
+
;;
|
|
3065
|
+
esac
|
|
1874
3066
|
exit 0
|
|
1875
3067
|
`);
|
|
1876
3068
|
await writeFile(tmuxLogPath, '');
|
|
@@ -1884,16 +3076,1041 @@ exit 0
|
|
|
1884
3076
|
config.leader_pane_id = '%11';
|
|
1885
3077
|
config.hud_pane_id = '%12';
|
|
1886
3078
|
config.workers[0].pane_id = '%11';
|
|
3079
|
+
config.workers[0].pid = 42411;
|
|
1887
3080
|
config.workers[1].pane_id = '%12';
|
|
3081
|
+
config.workers[1].pid = 42412;
|
|
1888
3082
|
config.workers[2].pane_id = '%13';
|
|
3083
|
+
config.workers[2].pid = 42413;
|
|
1889
3084
|
config.workers[3].pane_id = '%14';
|
|
3085
|
+
config.workers[3].pid = 42414;
|
|
3086
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
1890
3087
|
await saveTeamConfig(config, cwd);
|
|
1891
3088
|
const result = await scaleDown('exclusions', cwd, { workerNames: ['worker-1', 'worker-2', 'worker-3'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
1892
|
-
assert.equal(result.ok,
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
assert.
|
|
1896
|
-
|
|
3089
|
+
assert.equal(result.ok, false);
|
|
3090
|
+
if (!result.ok)
|
|
3091
|
+
assert.match(result.error, /pane_teardown_unresolved:%11.*pane_teardown_unresolved:%12/);
|
|
3092
|
+
assert.deepEqual((await readTeamConfig('exclusions', cwd))?.workers.map((worker) => worker.name), ['worker-4']);
|
|
3093
|
+
const tmuxCommands = (await readFile(tmuxLogPath, 'utf-8')).trim().split('\n');
|
|
3094
|
+
assert.deepEqual(tmuxCommands, [
|
|
3095
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3096
|
+
'show-option -qv -p -t %13 @omx_team_pane_owner_id',
|
|
3097
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3098
|
+
'kill-pane -t %13',
|
|
3099
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3100
|
+
]);
|
|
3101
|
+
}
|
|
3102
|
+
finally {
|
|
3103
|
+
if (typeof previousPath === 'string')
|
|
3104
|
+
process.env.PATH = previousPath;
|
|
3105
|
+
else
|
|
3106
|
+
delete process.env.PATH;
|
|
3107
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3108
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3109
|
+
}
|
|
3110
|
+
});
|
|
3111
|
+
it('scaleDown commits forward membership and retains artifacts/debt when pane ID is reused', async () => {
|
|
3112
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-kill-fail-'));
|
|
3113
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-kill-fail-bin-'));
|
|
3114
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3115
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3116
|
+
const previousPath = process.env.PATH;
|
|
3117
|
+
try {
|
|
3118
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3119
|
+
set -eu
|
|
3120
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3121
|
+
case "\${1:-}" in
|
|
3122
|
+
show-option) echo 'team:scale-down' ;;
|
|
3123
|
+
list-panes)
|
|
3124
|
+
if [ -f "${cwd}/pane-reused" ]; then
|
|
3125
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42414'
|
|
3126
|
+
else
|
|
3127
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3128
|
+
fi
|
|
3129
|
+
;;
|
|
3130
|
+
kill-pane)
|
|
3131
|
+
: > "${cwd}/pane-reused"
|
|
3132
|
+
exit 0
|
|
3133
|
+
;;
|
|
3134
|
+
esac
|
|
3135
|
+
exit 0
|
|
3136
|
+
`);
|
|
3137
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3138
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3139
|
+
await initTeamState('kill-fail', 'task', 'executor', 2, cwd);
|
|
3140
|
+
const config = await readTeamConfig('kill-fail', cwd);
|
|
3141
|
+
assert.ok(config);
|
|
3142
|
+
if (!config)
|
|
3143
|
+
return;
|
|
3144
|
+
config.workers[1].pane_id = '%13';
|
|
3145
|
+
config.workers[1].pid = 42413;
|
|
3146
|
+
const worktreePath = join(cwd, '.omx', 'team', 'kill-fail', 'worktrees', 'worker-2');
|
|
3147
|
+
await mkdir(worktreePath, { recursive: true });
|
|
3148
|
+
await writeFile(join(worktreePath, 'keep.txt'), 'retryable worktree');
|
|
3149
|
+
config.workers[1].worktree_path = worktreePath;
|
|
3150
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'kill-fail', 'workers', 'worker-2');
|
|
3151
|
+
await writeFile(join(workerDir, 'identity.json'), '{"worker":"worker-2"}');
|
|
3152
|
+
await writeFile(join(workerDir, 'inbox.md'), 'retryable inbox');
|
|
3153
|
+
await mkdir(join(cwd, '.omx', 'state', 'team', 'kill-fail', 'runtime'), { recursive: true });
|
|
3154
|
+
const startupScriptPath = workerStartupScriptPath(cwd, 'kill-fail', 'worker-2');
|
|
3155
|
+
await writeFile(startupScriptPath, '#!/bin/sh\n');
|
|
3156
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3157
|
+
await saveTeamConfig(config, cwd);
|
|
3158
|
+
const priorStatus = {
|
|
3159
|
+
state: 'idle',
|
|
3160
|
+
reason: 'waiting for assignment',
|
|
3161
|
+
updated_at: '2026-07-14T00:00:00.000Z',
|
|
3162
|
+
};
|
|
3163
|
+
await writeWorkerStatus('kill-fail', 'worker-2', priorStatus, cwd);
|
|
3164
|
+
const result = await scaleDown('kill-fail', cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3165
|
+
assert.equal(result.ok, false);
|
|
3166
|
+
if (!result.ok)
|
|
3167
|
+
assert.match(result.error, /^scale_down_cleanup_debt:pane_teardown_failed:%13/);
|
|
3168
|
+
const committed = await readTeamConfig('kill-fail', cwd);
|
|
3169
|
+
assert.deepEqual(committed?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3170
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'kill-fail', 'workers', 'worker-2')), true);
|
|
3171
|
+
assert.equal(await readFile(join(workerDir, 'identity.json'), 'utf8'), '{"worker":"worker-2"}');
|
|
3172
|
+
assert.equal(await readFile(join(workerDir, 'inbox.md'), 'utf8'), 'retryable inbox');
|
|
3173
|
+
assert.equal(await readFile(join(worktreePath, 'keep.txt'), 'utf8'), 'retryable worktree');
|
|
3174
|
+
assert.equal(await readFile(startupScriptPath, 'utf8'), '#!/bin/sh\n');
|
|
3175
|
+
assert.deepEqual(await readWorkerStatus('kill-fail', 'worker-2', cwd), priorStatus);
|
|
3176
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'kill-fail', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3177
|
+
assert.equal(debt.status, 'unresolved');
|
|
3178
|
+
assert.equal(debt.unresolved_panes[0]?.pane_id, '%13');
|
|
3179
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3180
|
+
assert.deepEqual(tmuxCommands, [
|
|
3181
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3182
|
+
'show-option -qv -p -t %13 @omx_team_pane_owner_id',
|
|
3183
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3184
|
+
'kill-pane -t %13',
|
|
3185
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3186
|
+
]);
|
|
3187
|
+
}
|
|
3188
|
+
finally {
|
|
3189
|
+
if (typeof previousPath === 'string')
|
|
3190
|
+
process.env.PATH = previousPath;
|
|
3191
|
+
else
|
|
3192
|
+
delete process.env.PATH;
|
|
3193
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3194
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3195
|
+
}
|
|
3196
|
+
});
|
|
3197
|
+
it('aborts before pane teardown and preserves exact state when task reconciliation cannot acquire a lock', async () => {
|
|
3198
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-task-reconcile-fail-'));
|
|
3199
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-task-reconcile-fail-bin-'));
|
|
3200
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3201
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3202
|
+
const previousPath = process.env.PATH;
|
|
3203
|
+
try {
|
|
3204
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3205
|
+
set -eu
|
|
3206
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3207
|
+
case "\${1:-}" in
|
|
3208
|
+
show-option) echo 'team:scale-down' ;;
|
|
3209
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%11' '0' '42411' ;;
|
|
3210
|
+
esac
|
|
3211
|
+
`);
|
|
3212
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3213
|
+
await writeFile(tmuxLogPath, '');
|
|
3214
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3215
|
+
await initTeamState('task-reconcile-fail', 'task', 'executor', 2, cwd);
|
|
3216
|
+
const config = await readTeamConfig('task-reconcile-fail', cwd);
|
|
3217
|
+
assert.ok(config);
|
|
3218
|
+
if (!config)
|
|
3219
|
+
return;
|
|
3220
|
+
config.workers[1].pane_id = '%13';
|
|
3221
|
+
config.workers[1].pid = 42413;
|
|
3222
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3223
|
+
await saveTeamConfig(config, cwd);
|
|
3224
|
+
const firstTask = await createTask('task-reconcile-fail', {
|
|
3225
|
+
subject: 'first reconciliation', description: 'must be restored after later failure', status: 'pending', owner: 'worker-2',
|
|
3226
|
+
}, cwd);
|
|
3227
|
+
const firstTaskPath = join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'tasks', `task-${firstTask.id}.json`);
|
|
3228
|
+
const firstTaskRaw = await readFile(firstTaskPath, 'utf-8');
|
|
3229
|
+
const task = await createTask('task-reconcile-fail', {
|
|
3230
|
+
subject: 'locked reconciliation', description: 'must remain exact', status: 'pending', owner: 'worker-2',
|
|
3231
|
+
}, cwd);
|
|
3232
|
+
const taskPath = join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'tasks', `task-${task.id}.json`);
|
|
3233
|
+
const taskRaw = await readFile(taskPath, 'utf-8');
|
|
3234
|
+
const priorConfig = structuredClone(await readTeamConfig('task-reconcile-fail', cwd));
|
|
3235
|
+
await mkdir(join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'claims', `task-${task.id}.lock`));
|
|
3236
|
+
const result = await scaleDown('task-reconcile-fail', cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3237
|
+
assert.deepEqual(result, { ok: false, error: 'scale_down_task_reconciliation_failed:Error: Timed out acquiring task claim lock for task-reconcile-fail/2' });
|
|
3238
|
+
assert.deepEqual(await readTeamConfig('task-reconcile-fail', cwd), priorConfig);
|
|
3239
|
+
assert.equal(await readFile(taskPath, 'utf-8'), taskRaw);
|
|
3240
|
+
assert.equal(await readFile(firstTaskPath, 'utf-8'), firstTaskRaw);
|
|
3241
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'task-reconcile-fail', 'workers', 'worker-2')), true);
|
|
3242
|
+
assert.deepEqual(await readScaleUpTmuxLogCommands(tmuxLogPath), []);
|
|
3243
|
+
}
|
|
3244
|
+
finally {
|
|
3245
|
+
if (typeof previousPath === 'string')
|
|
3246
|
+
process.env.PATH = previousPath;
|
|
3247
|
+
else
|
|
3248
|
+
delete process.env.PATH;
|
|
3249
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3250
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3251
|
+
}
|
|
3252
|
+
});
|
|
3253
|
+
it('commits forward membership and durable debt when scale-down pane proof is unavailable', async () => {
|
|
3254
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-proof-unavailable-'));
|
|
3255
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-proof-unavailable-bin-'));
|
|
3256
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3257
|
+
const previousPath = process.env.PATH;
|
|
3258
|
+
try {
|
|
3259
|
+
await writeFile(tmuxStubPath, '#!/bin/sh\n[ "$1" = list-panes ] && exit 1\nexit 0\n');
|
|
3260
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3261
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3262
|
+
await initTeamState('proof-unavailable', 'task', 'executor', 2, cwd);
|
|
3263
|
+
const config = await readTeamConfig('proof-unavailable', cwd);
|
|
3264
|
+
assert.ok(config);
|
|
3265
|
+
if (!config)
|
|
3266
|
+
return;
|
|
3267
|
+
config.workers[1].pane_id = '%13';
|
|
3268
|
+
config.workers[1].pid = 42413;
|
|
3269
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3270
|
+
await saveTeamConfig(config, cwd);
|
|
3271
|
+
const task = await createTask('proof-unavailable', {
|
|
3272
|
+
subject: 'preserved', description: 'exact proof is unavailable', status: 'pending', owner: 'worker-2',
|
|
3273
|
+
}, cwd);
|
|
3274
|
+
const statusPath = join(cwd, '.omx', 'state', 'team', 'proof-unavailable', 'workers', 'worker-2', 'status.json');
|
|
3275
|
+
const statusRaw = '{"state":"idle", "reason":"preserve bytes", "updated_at":"2026-07-14T00:00:00.000Z"}\n';
|
|
3276
|
+
await writeFile(statusPath, statusRaw);
|
|
3277
|
+
const result = await scaleDown('proof-unavailable', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3278
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
3279
|
+
});
|
|
3280
|
+
assert.equal(result.ok, false);
|
|
3281
|
+
assert.deepEqual((await readTeamConfig('proof-unavailable', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3282
|
+
assert.equal((await readTask('proof-unavailable', task.id, cwd))?.owner, undefined);
|
|
3283
|
+
assert.equal(await readFile(statusPath, 'utf8'), statusRaw);
|
|
3284
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'proof-unavailable', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3285
|
+
assert.equal(debt.status, 'unresolved');
|
|
3286
|
+
assert.equal(debt.unresolved_panes[0]?.pane_id, '%13');
|
|
3287
|
+
}
|
|
3288
|
+
finally {
|
|
3289
|
+
if (typeof previousPath === 'string')
|
|
3290
|
+
process.env.PATH = previousPath;
|
|
3291
|
+
else
|
|
3292
|
+
delete process.env.PATH;
|
|
3293
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3294
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3295
|
+
}
|
|
3296
|
+
});
|
|
3297
|
+
it('converges entirely old or entirely new after a mid-commit scale-down interruption', async () => {
|
|
3298
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-atomic-'));
|
|
3299
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-atomic-bin-'));
|
|
3300
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3301
|
+
const previousPath = process.env.PATH;
|
|
3302
|
+
try {
|
|
3303
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3304
|
+
marker="${tmuxStubPath}.killed"
|
|
3305
|
+
if [ "$1" = list-panes ]; then
|
|
3306
|
+
[ -f "$marker" ] || printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3307
|
+
exit 0
|
|
3308
|
+
fi
|
|
3309
|
+
if [ "$1" = kill-pane ]; then : > "$marker"; exit 0; fi
|
|
3310
|
+
exit 0
|
|
3311
|
+
`);
|
|
3312
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3313
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3314
|
+
await initTeamState('atomic-down', 'task', 'executor', 2, cwd);
|
|
3315
|
+
const config = await readTeamConfig('atomic-down', cwd);
|
|
3316
|
+
assert.ok(config);
|
|
3317
|
+
if (!config)
|
|
3318
|
+
return;
|
|
3319
|
+
config.workers[1].pane_id = '%13';
|
|
3320
|
+
config.workers[1].pid = 42413;
|
|
3321
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3322
|
+
await saveTeamConfig(config, cwd);
|
|
3323
|
+
const first = await createTask('atomic-down', { subject: 'first', description: 'first', status: 'pending', owner: 'worker-2' }, cwd);
|
|
3324
|
+
const second = await createTask('atomic-down', { subject: 'second', description: 'second', status: 'pending', owner: 'worker-2' }, cwd);
|
|
3325
|
+
const result = await scaleDown('atomic-down', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3326
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SCALE_DOWN_INJECT_FAILURE: 'after-first-task-write',
|
|
3327
|
+
});
|
|
3328
|
+
assert.match(result.ok ? '' : result.error, /injected_scale_down_interruption:after-first-task-write/);
|
|
3329
|
+
await recoverTeamMembershipTaskTransaction('atomic-down', cwd);
|
|
3330
|
+
const recoveredConfig = await readTeamConfig('atomic-down', cwd);
|
|
3331
|
+
const recoveredFirst = await readTask('atomic-down', first.id, cwd);
|
|
3332
|
+
const recoveredSecond = await readTask('atomic-down', second.id, cwd);
|
|
3333
|
+
const convergedOld = recoveredConfig?.workers.some((worker) => worker.name === 'worker-2') === true
|
|
3334
|
+
&& recoveredFirst?.owner === 'worker-2'
|
|
3335
|
+
&& recoveredSecond?.owner === 'worker-2';
|
|
3336
|
+
const convergedNew = recoveredConfig?.workers.some((worker) => worker.name === 'worker-2') === false
|
|
3337
|
+
&& recoveredFirst?.owner === undefined
|
|
3338
|
+
&& recoveredSecond?.owner === undefined;
|
|
3339
|
+
assert.equal(convergedOld || convergedNew, true);
|
|
3340
|
+
}
|
|
3341
|
+
finally {
|
|
3342
|
+
if (typeof previousPath === 'string')
|
|
3343
|
+
process.env.PATH = previousPath;
|
|
3344
|
+
else
|
|
3345
|
+
delete process.env.PATH;
|
|
3346
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3347
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3348
|
+
}
|
|
3349
|
+
});
|
|
3350
|
+
it('recovers rollback-persistence failure through public config reads with config, manifest, and tasks converged', async () => {
|
|
3351
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-rollback-recovery-'));
|
|
3352
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-rollback-recovery-bin-'));
|
|
3353
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3354
|
+
const previousPath = process.env.PATH;
|
|
3355
|
+
try {
|
|
3356
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3357
|
+
marker="${tmuxStubPath}.killed"
|
|
3358
|
+
if [ "$1" = list-panes ]; then
|
|
3359
|
+
[ -f "$marker" ] || printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3360
|
+
exit 0
|
|
3361
|
+
fi
|
|
3362
|
+
if [ "$1" = kill-pane ]; then : > "$marker"; exit 0; fi
|
|
3363
|
+
exit 0
|
|
3364
|
+
`);
|
|
3365
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3366
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3367
|
+
await initTeamState('rollback-recovery', 'task', 'executor', 2, cwd);
|
|
3368
|
+
const config = await readTeamConfig('rollback-recovery', cwd);
|
|
3369
|
+
assert.ok(config);
|
|
3370
|
+
if (!config)
|
|
3371
|
+
return;
|
|
3372
|
+
config.workers[1].pane_id = '%13';
|
|
3373
|
+
config.workers[1].pid = 42413;
|
|
3374
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3375
|
+
await saveTeamConfig(config, cwd);
|
|
3376
|
+
const task = await createTask('rollback-recovery', { subject: 'owned', description: 'owned', status: 'pending', owner: 'worker-2' }, cwd);
|
|
3377
|
+
const failed = await scaleDown('rollback-recovery', cwd, { workerNames: ['worker-2'], force: true }, {
|
|
3378
|
+
OMX_TEAM_SCALING_ENABLED: '1', OMX_TEAM_SCALE_DOWN_INJECT_FAILURE: 'rollback-persistence-failure',
|
|
3379
|
+
});
|
|
3380
|
+
assert.match(failed.ok ? '' : failed.error, /rollback-persistence-failure/);
|
|
3381
|
+
const recoveredConfig = await readTeamConfig('rollback-recovery', cwd);
|
|
3382
|
+
const recoveredManifest = await readTeamManifestV2('rollback-recovery', cwd);
|
|
3383
|
+
const recoveredTask = await readTask('rollback-recovery', task.id, cwd);
|
|
3384
|
+
assert.equal(recoveredConfig?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3385
|
+
assert.equal(recoveredManifest?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3386
|
+
assert.equal(recoveredTask?.owner, undefined);
|
|
3387
|
+
}
|
|
3388
|
+
finally {
|
|
3389
|
+
if (typeof previousPath === 'string')
|
|
3390
|
+
process.env.PATH = previousPath;
|
|
3391
|
+
else
|
|
3392
|
+
delete process.env.PATH;
|
|
3393
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3394
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3395
|
+
}
|
|
3396
|
+
});
|
|
3397
|
+
for (const target of ['config', 'manifest']) {
|
|
3398
|
+
it(`publicly recovers a committed scale-up membership rollback after partial ${target} old-generation persistence`, async () => {
|
|
3399
|
+
const cwd = await mkdtemp(join(tmpdir(), `omx-scale-up-rollback-${target}-`));
|
|
3400
|
+
try {
|
|
3401
|
+
const teamName = `rollback-${target}`;
|
|
3402
|
+
await initTeamState(teamName, 'task', 'executor', 2, cwd);
|
|
3403
|
+
const task = await createTask(teamName, {
|
|
3404
|
+
subject: 'rollback owner', description: 'must converge with membership', status: 'pending', owner: 'worker-2',
|
|
3405
|
+
}, cwd);
|
|
3406
|
+
const teamDir = join(cwd, '.omx', 'state', 'team', teamName);
|
|
3407
|
+
const configPath = join(teamDir, 'config.json');
|
|
3408
|
+
const manifestPath = join(teamDir, 'manifest.v2.json');
|
|
3409
|
+
const taskPath = join(teamDir, 'tasks', `task-${task.id}.json`);
|
|
3410
|
+
const oldConfig = await readFile(configPath, 'utf8');
|
|
3411
|
+
const oldManifest = await readFile(manifestPath, 'utf8');
|
|
3412
|
+
const oldTask = await readFile(taskPath, 'utf8');
|
|
3413
|
+
const oldConfigValue = JSON.parse(oldConfig);
|
|
3414
|
+
const nextConfig = { ...oldConfigValue, workers: oldConfigValue.workers.filter((worker) => worker.name !== 'worker-2') };
|
|
3415
|
+
const nextManifest = { ...JSON.parse(oldManifest), workers: nextConfig.workers, worker_count: nextConfig.workers.length };
|
|
3416
|
+
const nextTask = { ...JSON.parse(oldTask), owner: undefined, claim: undefined };
|
|
3417
|
+
await assert.rejects(withTeamTaskBarrier(teamName, cwd, () => commitTeamMembershipTaskTransaction(teamName, cwd, {
|
|
3418
|
+
baseGeneration: 0,
|
|
3419
|
+
tasks: [{ taskId: task.id, oldBytes: oldTask, newBytes: JSON.stringify(nextTask, null, 2) }],
|
|
3420
|
+
config: { oldBytes: oldConfig, newBytes: JSON.stringify(nextConfig, null, 2) },
|
|
3421
|
+
manifest: { oldBytes: oldManifest, newBytes: JSON.stringify(nextManifest, null, 2) },
|
|
3422
|
+
recoverToNewOnFailure: true,
|
|
3423
|
+
failRollbackPersistenceAfter: target,
|
|
3424
|
+
})), /rollback-persistence-failure/);
|
|
3425
|
+
assert.equal(existsSync(join(teamDir, '.membership-task-transaction.json')), true);
|
|
3426
|
+
await recoverTeamMembershipTaskTransaction(teamName, cwd);
|
|
3427
|
+
assert.equal((await readTeamConfig(teamName, cwd))?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3428
|
+
assert.equal((await readTeamManifestV2(teamName, cwd))?.workers.some((worker) => worker.name === 'worker-2'), false);
|
|
3429
|
+
assert.equal((await readTask(teamName, task.id, cwd))?.owner, undefined);
|
|
3430
|
+
}
|
|
3431
|
+
finally {
|
|
3432
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3433
|
+
}
|
|
3434
|
+
});
|
|
3435
|
+
}
|
|
3436
|
+
it('discards precommit cleanup debt while removed workers remain canonical', async () => {
|
|
3437
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-precommit-debt-'));
|
|
3438
|
+
try {
|
|
3439
|
+
await initTeamState('precommit-debt', 'task', 'executor', 2, cwd);
|
|
3440
|
+
const config = await readTeamConfig('precommit-debt', cwd);
|
|
3441
|
+
assert.ok(config);
|
|
3442
|
+
if (!config)
|
|
3443
|
+
return;
|
|
3444
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'precommit-debt', '.scale-down-cleanup-debt.json');
|
|
3445
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3446
|
+
schema_version: 1,
|
|
3447
|
+
operation: 'scale_down',
|
|
3448
|
+
status: 'pending_teardown',
|
|
3449
|
+
removed_worker_names: ['worker-2'],
|
|
3450
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
3451
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
3452
|
+
}));
|
|
3453
|
+
const recovered = await reconcileScaleDownCleanupDebt('precommit-debt', cwd, config);
|
|
3454
|
+
assert.deepEqual(recovered, { ok: true });
|
|
3455
|
+
assert.equal(existsSync(debtPath), false);
|
|
3456
|
+
assert.equal((await readTeamConfig('precommit-debt', cwd))?.workers.some((worker) => worker.name === 'worker-2'), true);
|
|
3457
|
+
}
|
|
3458
|
+
finally {
|
|
3459
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3460
|
+
}
|
|
3461
|
+
});
|
|
3462
|
+
it('commits forward state and records unresolved debt when proof is lost after a pane kill', async () => {
|
|
3463
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-success-proof-loss-'));
|
|
3464
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-success-proof-loss-bin-'));
|
|
3465
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3466
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3467
|
+
const proofLossMarkerPath = join(fakeBinDir, 'proof-loss');
|
|
3468
|
+
const previousPath = process.env.PATH;
|
|
3469
|
+
try {
|
|
3470
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3471
|
+
set -eu
|
|
3472
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3473
|
+
case "\${1:-}" in
|
|
3474
|
+
show-option) echo 'team:scale-down' ;;
|
|
3475
|
+
list-panes)
|
|
3476
|
+
if [ -f "${proofLossMarkerPath}.recovery" ]; then
|
|
3477
|
+
if [ ! -f "${proofLossMarkerPath}.killed-13" ]; then printf '%s\t%s\t%s\n' '%13' '0' '42413'; fi
|
|
3478
|
+
if [ ! -f "${proofLossMarkerPath}.killed-14" ]; then printf '%s\t%s\t%s\n' '%14' '0' '42414'; fi
|
|
3479
|
+
else
|
|
3480
|
+
if [ -f "${proofLossMarkerPath}.killed" ]; then exit 1; fi
|
|
3481
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3482
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3483
|
+
fi
|
|
3484
|
+
;;
|
|
3485
|
+
show-option) printf '%s\n' 'team:success-proof-loss' ;;
|
|
3486
|
+
kill-pane)
|
|
3487
|
+
if [ -f "${proofLossMarkerPath}.recovery" ]; then
|
|
3488
|
+
if [ "$3" = '%13' ]; then : > "${proofLossMarkerPath}.killed-13"; fi
|
|
3489
|
+
if [ "$3" = '%14' ]; then : > "${proofLossMarkerPath}.killed-14"; fi
|
|
3490
|
+
elif [ "$3" = '%13' ]; then
|
|
3491
|
+
: > "${proofLossMarkerPath}.killed"
|
|
3492
|
+
fi
|
|
3493
|
+
;;
|
|
3494
|
+
esac
|
|
3495
|
+
exit 0
|
|
3496
|
+
`);
|
|
3497
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3498
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3499
|
+
await initTeamState('success-proof-loss', 'task', 'executor', 3, cwd);
|
|
3500
|
+
const config = await readTeamConfig('success-proof-loss', cwd);
|
|
3501
|
+
assert.ok(config);
|
|
3502
|
+
if (!config)
|
|
3503
|
+
return;
|
|
3504
|
+
config.workers[1].pane_id = '%13';
|
|
3505
|
+
config.workers[1].pid = 42413;
|
|
3506
|
+
config.workers[2].pane_id = '%14';
|
|
3507
|
+
config.workers[2].pid = 42414;
|
|
3508
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3509
|
+
await saveTeamConfig(config, cwd);
|
|
3510
|
+
await writeFile(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', 'workers', 'worker-2', 'identity.json'), JSON.stringify({ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }));
|
|
3511
|
+
await writeFile(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', 'workers', 'worker-3', 'identity.json'), JSON.stringify({ name: 'worker-3', index: 3, pane_id: '%14', pid: 42414 }));
|
|
3512
|
+
const resolvedTask = await createTask('success-proof-loss', {
|
|
3513
|
+
subject: 'resolved owner task', description: 'must be reclaimed', status: 'pending', owner: 'worker-2',
|
|
3514
|
+
}, cwd);
|
|
3515
|
+
const unresolvedTask = await createTask('success-proof-loss', {
|
|
3516
|
+
subject: 'unresolved owner task', description: 'must be preserved', status: 'pending', owner: 'worker-3',
|
|
3517
|
+
}, cwd);
|
|
3518
|
+
const resolvedClaim = await claimTask('success-proof-loss', resolvedTask.id, 'worker-2', resolvedTask.version ?? 1, cwd);
|
|
3519
|
+
assert.equal(resolvedClaim.ok, true);
|
|
3520
|
+
const worker3StatusPath = join(cwd, '.omx', 'state', 'team', 'success-proof-loss', 'workers', 'worker-3', 'status.json');
|
|
3521
|
+
const worker3Raw = '{"reason":"three",\n"updated_at":"2026-07-14T00:00:01.000Z", "state":"idle"}\n';
|
|
3522
|
+
await writeFile(worker3StatusPath, worker3Raw);
|
|
3523
|
+
const result = await scaleDown('success-proof-loss', cwd, { workerNames: ['worker-2', 'worker-3'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3524
|
+
assert.equal(result.ok, false);
|
|
3525
|
+
if (!result.ok)
|
|
3526
|
+
assert.match(result.error, /^scale_down_pane_proof_unavailable:%13:query_failed/);
|
|
3527
|
+
assert.deepEqual((await readTeamConfig('success-proof-loss', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3528
|
+
assert.equal(existsSync(worker3StatusPath), true);
|
|
3529
|
+
assert.equal(await readFile(worker3StatusPath, 'utf8'), worker3Raw);
|
|
3530
|
+
assert.equal((await readTask('success-proof-loss', resolvedTask.id, cwd))?.owner, undefined);
|
|
3531
|
+
assert.equal((await readTask('success-proof-loss', unresolvedTask.id, cwd))?.owner, undefined);
|
|
3532
|
+
const retainedTask = await readTask('success-proof-loss', resolvedTask.id, cwd);
|
|
3533
|
+
assert.equal(retainedTask?.status, 'pending');
|
|
3534
|
+
assert.equal(retainedTask?.claim, undefined);
|
|
3535
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3536
|
+
assert.deepEqual(debt.unresolved_panes.map((pane) => pane.pane_id), ['%13', '%14']);
|
|
3537
|
+
const tmuxCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3538
|
+
assert.deepEqual(tmuxCommands.filter((command) => command.startsWith('kill-pane ')), ['kill-pane -t %13']);
|
|
3539
|
+
const killIndex = tmuxCommands.indexOf('kill-pane -t %13');
|
|
3540
|
+
assert.ok(killIndex > 2);
|
|
3541
|
+
assert.equal(tmuxCommands[killIndex - 3], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
3542
|
+
assert.equal(tmuxCommands[killIndex - 2], 'show-option -qv -p -t %13 @omx_team_pane_owner_id');
|
|
3543
|
+
assert.equal(tmuxCommands[killIndex - 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
3544
|
+
assert.equal(tmuxCommands[killIndex + 1], 'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}');
|
|
3545
|
+
await writeFile(`${proofLossMarkerPath}.recovery`, '1');
|
|
3546
|
+
await rm(`${proofLossMarkerPath}.killed`, { force: true });
|
|
3547
|
+
const committedConfig = await readTeamConfig('success-proof-loss', cwd);
|
|
3548
|
+
assert.ok(committedConfig);
|
|
3549
|
+
if (!committedConfig)
|
|
3550
|
+
return;
|
|
3551
|
+
const recovered = await reconcileScaleDownCleanupDebt('success-proof-loss', cwd, committedConfig);
|
|
3552
|
+
assert.deepEqual(recovered, { ok: true });
|
|
3553
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'success-proof-loss', '.scale-down-cleanup-debt.json')), false);
|
|
3554
|
+
const recoveredCommands = await readScaleUpTmuxLogCommands(tmuxLogPath);
|
|
3555
|
+
assert.deepEqual(recoveredCommands.filter((command) => command.startsWith('kill-pane ')), [
|
|
3556
|
+
'kill-pane -t %13',
|
|
3557
|
+
'kill-pane -t %13',
|
|
3558
|
+
'kill-pane -t %14',
|
|
3559
|
+
]);
|
|
3560
|
+
}
|
|
3561
|
+
finally {
|
|
3562
|
+
if (typeof previousPath === 'string')
|
|
3563
|
+
process.env.PATH = previousPath;
|
|
3564
|
+
else
|
|
3565
|
+
delete process.env.PATH;
|
|
3566
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3567
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3568
|
+
}
|
|
3569
|
+
});
|
|
3570
|
+
it('commits forward membership and retains exact debt when a later pane teardown fails', async () => {
|
|
3571
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-gone-kill-fail-'));
|
|
3572
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-gone-kill-fail-bin-'));
|
|
3573
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3574
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3575
|
+
const previousPath = process.env.PATH;
|
|
3576
|
+
try {
|
|
3577
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3578
|
+
set -eu
|
|
3579
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3580
|
+
case "\${1:-}" in
|
|
3581
|
+
show-option) echo 'team:scale-down' ;;
|
|
3582
|
+
list-panes)
|
|
3583
|
+
if [ -f "${tmuxLogPath}.killed-%13" ]; then
|
|
3584
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3585
|
+
else
|
|
3586
|
+
printf '%s\t%s\t%s\n' '%13' '0' '42413'
|
|
3587
|
+
printf '%s\t%s\t%s\n' '%14' '0' '42414'
|
|
3588
|
+
fi
|
|
3589
|
+
;;
|
|
3590
|
+
kill-pane)
|
|
3591
|
+
if [ "$3" = '%13' ]; then : > "${tmuxLogPath}.killed-%13"; exit 0; fi
|
|
3592
|
+
exit 1
|
|
3593
|
+
;;
|
|
3594
|
+
esac
|
|
3595
|
+
`);
|
|
3596
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3597
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3598
|
+
await initTeamState('gone-kill-fail', 'task', 'executor', 3, cwd);
|
|
3599
|
+
const config = await readTeamConfig('gone-kill-fail', cwd);
|
|
3600
|
+
assert.ok(config);
|
|
3601
|
+
if (!config)
|
|
3602
|
+
return;
|
|
3603
|
+
config.workers[1].pane_id = '%13';
|
|
3604
|
+
config.workers[1].pid = 42413;
|
|
3605
|
+
config.workers[2].pane_id = '%14';
|
|
3606
|
+
config.workers[2].pid = 42414;
|
|
3607
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3608
|
+
await saveTeamConfig(config, cwd);
|
|
3609
|
+
const resolvedTask = await createTask('gone-kill-fail', {
|
|
3610
|
+
subject: 'gone owner task', description: 'must be reclaimed', status: 'pending', owner: 'worker-2',
|
|
3611
|
+
}, cwd);
|
|
3612
|
+
const unresolvedTask = await createTask('gone-kill-fail', {
|
|
3613
|
+
subject: 'failed owner task', description: 'must be preserved', status: 'pending', owner: 'worker-3',
|
|
3614
|
+
}, cwd);
|
|
3615
|
+
const resolvedClaim = await claimTask('gone-kill-fail', resolvedTask.id, 'worker-2', resolvedTask.version ?? 1, cwd);
|
|
3616
|
+
assert.equal(resolvedClaim.ok, true);
|
|
3617
|
+
const worker3StatusPath = join(cwd, '.omx', 'state', 'team', 'gone-kill-fail', 'workers', 'worker-3', 'status.json');
|
|
3618
|
+
await writeFile(worker3StatusPath, '{\n "state" : "idle", "reason":"three", "updated_at":"2026-07-14T00:00:01.000Z"\n}\n');
|
|
3619
|
+
const result = await scaleDown('gone-kill-fail', cwd, { workerNames: ['worker-2', 'worker-3'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3620
|
+
assert.equal(result.ok, false);
|
|
3621
|
+
if (!result.ok)
|
|
3622
|
+
assert.match(result.error, /^scale_down_cleanup_debt:pane_teardown_failed:%14/);
|
|
3623
|
+
assert.deepEqual((await readTeamConfig('gone-kill-fail', cwd))?.workers.map((worker) => worker.name), ['worker-1']);
|
|
3624
|
+
assert.equal(existsSync(worker3StatusPath), true);
|
|
3625
|
+
assert.equal(await readFile(worker3StatusPath, 'utf8'), '{\n "state" : "idle", "reason":"three", "updated_at":"2026-07-14T00:00:01.000Z"\n}\n');
|
|
3626
|
+
assert.equal((await readTask('gone-kill-fail', resolvedTask.id, cwd))?.owner, undefined);
|
|
3627
|
+
assert.equal((await readTask('gone-kill-fail', unresolvedTask.id, cwd))?.owner, undefined);
|
|
3628
|
+
const retainedTask = await readTask('gone-kill-fail', resolvedTask.id, cwd);
|
|
3629
|
+
assert.equal(retainedTask?.status, 'pending');
|
|
3630
|
+
assert.equal(retainedTask?.claim, undefined);
|
|
3631
|
+
const debt = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'gone-kill-fail', '.scale-down-cleanup-debt.json'), 'utf8'));
|
|
3632
|
+
assert.deepEqual(debt.unresolved_panes.map((pane) => pane.pane_id), ['%14']);
|
|
3633
|
+
assert.deepEqual(await readScaleUpTmuxLogCommands(tmuxLogPath), [
|
|
3634
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3635
|
+
'show-option -qv -p -t %13 @omx_team_pane_owner_id',
|
|
3636
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3637
|
+
'kill-pane -t %13',
|
|
3638
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3639
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3640
|
+
'show-option -qv -p -t %14 @omx_team_pane_owner_id',
|
|
3641
|
+
'list-panes -a -F #{pane_id}\t#{pane_dead}\t#{pane_pid}',
|
|
3642
|
+
'kill-pane -t %14',
|
|
3643
|
+
]);
|
|
3644
|
+
}
|
|
3645
|
+
finally {
|
|
3646
|
+
if (typeof previousPath === 'string')
|
|
3647
|
+
process.env.PATH = previousPath;
|
|
3648
|
+
else
|
|
3649
|
+
delete process.env.PATH;
|
|
3650
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3651
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3652
|
+
}
|
|
3653
|
+
});
|
|
3654
|
+
for (const [name, priorRaw] of [
|
|
3655
|
+
['absent status artifact', undefined],
|
|
3656
|
+
['malformed status artifact', '{"state":\n'],
|
|
3657
|
+
['noncanonical valid status artifact', '{\n "updated_at" : "2026-07-14T00:00:00.000Z",\n "reason":"waiting for assignment", "state" : "idle"\n}\n'],
|
|
3658
|
+
]) {
|
|
3659
|
+
it(`scaleDown restores the exact ${name} after kill-pane failure`, async () => {
|
|
3660
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-status-artifact-'));
|
|
3661
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-status-artifact-bin-'));
|
|
3662
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3663
|
+
const previousPath = process.env.PATH;
|
|
3664
|
+
try {
|
|
3665
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3666
|
+
case "\${1:-}" in
|
|
3667
|
+
show-option) echo 'team:scale-down' ;;
|
|
3668
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%13' '0' '42413' ;;
|
|
3669
|
+
kill-pane) exit 1 ;;
|
|
3670
|
+
esac
|
|
3671
|
+
`);
|
|
3672
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3673
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3674
|
+
await initTeamState(`status-${name.startsWith('absent') ? 'absent' : 'malformed'}`, 'task', 'executor', 2, cwd);
|
|
3675
|
+
const teamName = `status-${name.startsWith('absent') ? 'absent' : 'malformed'}`;
|
|
3676
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
3677
|
+
assert.ok(config);
|
|
3678
|
+
if (!config)
|
|
3679
|
+
return;
|
|
3680
|
+
config.workers[1].pane_id = '%13';
|
|
3681
|
+
config.workers[1].pid = 42413;
|
|
3682
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3683
|
+
await saveTeamConfig(config, cwd);
|
|
3684
|
+
const statusPath = join(cwd, '.omx', 'state', 'team', teamName, 'workers', 'worker-2', 'status.json');
|
|
3685
|
+
if (priorRaw === undefined) {
|
|
3686
|
+
await rm(statusPath, { force: true });
|
|
3687
|
+
}
|
|
3688
|
+
else {
|
|
3689
|
+
await writeFile(statusPath, priorRaw);
|
|
3690
|
+
}
|
|
3691
|
+
const result = await scaleDown(teamName, cwd, { workerNames: ['worker-2'], force: true }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
3692
|
+
assert.equal(result.ok, false);
|
|
3693
|
+
if (!result.ok)
|
|
3694
|
+
assert.match(result.error, /^scale_down_cleanup_debt:pane_teardown_failed:%13/);
|
|
3695
|
+
assert.equal(existsSync(statusPath), priorRaw !== undefined);
|
|
3696
|
+
if (priorRaw !== undefined)
|
|
3697
|
+
assert.equal(await readFile(statusPath, 'utf8'), priorRaw);
|
|
3698
|
+
}
|
|
3699
|
+
finally {
|
|
3700
|
+
if (typeof previousPath === 'string')
|
|
3701
|
+
process.env.PATH = previousPath;
|
|
3702
|
+
else
|
|
3703
|
+
delete process.env.PATH;
|
|
3704
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3705
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3706
|
+
}
|
|
3707
|
+
});
|
|
3708
|
+
}
|
|
3709
|
+
it('uses canonical membership rather than a stale caller config before consuming precommit debt', async () => {
|
|
3710
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-canonical-debt-'));
|
|
3711
|
+
try {
|
|
3712
|
+
await initTeamState('canonical-debt', 'task', 'executor', 2, cwd);
|
|
3713
|
+
const canonical = await readTeamConfig('canonical-debt', cwd);
|
|
3714
|
+
assert.ok(canonical);
|
|
3715
|
+
if (!canonical)
|
|
3716
|
+
return;
|
|
3717
|
+
const staleCaller = structuredClone(canonical);
|
|
3718
|
+
staleCaller.workers = staleCaller.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3719
|
+
staleCaller.worker_count = staleCaller.workers.length;
|
|
3720
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'canonical-debt', 'workers', 'worker-2');
|
|
3721
|
+
const sentinelPath = join(workerDir, 'caller-must-not-authorize');
|
|
3722
|
+
await writeFile(sentinelPath, 'preserve');
|
|
3723
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'canonical-debt', '.scale-down-cleanup-debt.json');
|
|
3724
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3725
|
+
schema_version: 1,
|
|
3726
|
+
operation: 'scale_down',
|
|
3727
|
+
status: 'pending_teardown',
|
|
3728
|
+
removed_worker_names: ['worker-2'],
|
|
3729
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
3730
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
3731
|
+
}));
|
|
3732
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('canonical-debt', cwd, staleCaller), { ok: true });
|
|
3733
|
+
assert.equal(existsSync(debtPath), false);
|
|
3734
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'preserve');
|
|
3735
|
+
assert.equal((await readTeamConfig('canonical-debt', cwd))?.workers.some((worker) => worker.name === 'worker-2'), true);
|
|
3736
|
+
}
|
|
3737
|
+
finally {
|
|
3738
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3739
|
+
}
|
|
3740
|
+
});
|
|
3741
|
+
it('converges pane-less crash debt through worker status and directory cleanup', async () => {
|
|
3742
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-zero-pane-debt-'));
|
|
3743
|
+
try {
|
|
3744
|
+
await initTeamState('zero-pane-debt', 'task', 'executor', 2, cwd);
|
|
3745
|
+
const config = await readTeamConfig('zero-pane-debt', cwd);
|
|
3746
|
+
assert.ok(config);
|
|
3747
|
+
if (!config)
|
|
3748
|
+
return;
|
|
3749
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3750
|
+
config.worker_count = config.workers.length;
|
|
3751
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3752
|
+
await saveTeamConfig(config, cwd);
|
|
3753
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'zero-pane-debt', 'workers', 'worker-2');
|
|
3754
|
+
await mkdir(workerDir, { recursive: true });
|
|
3755
|
+
await writeFile(join(workerDir, 'status.json'), '{"state":"draining"}');
|
|
3756
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'zero-pane-debt', '.scale-down-cleanup-debt.json');
|
|
3757
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3758
|
+
schema_version: 1,
|
|
3759
|
+
operation: 'scale_down',
|
|
3760
|
+
status: 'resource_cleanup_pending',
|
|
3761
|
+
removed_worker_names: ['worker-2'],
|
|
3762
|
+
workers: [],
|
|
3763
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
3764
|
+
}));
|
|
3765
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('zero-pane-debt', cwd, config), { ok: true });
|
|
3766
|
+
assert.equal(existsSync(workerDir), false);
|
|
3767
|
+
assert.equal(existsSync(debtPath), false);
|
|
3768
|
+
}
|
|
3769
|
+
finally {
|
|
3770
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3771
|
+
}
|
|
3772
|
+
});
|
|
3773
|
+
it('converges resource debt after a crash following successful worktree cleanup', async () => {
|
|
3774
|
+
const repo = await initRepo();
|
|
3775
|
+
try {
|
|
3776
|
+
await initTeamState('resource-retry', 'task', 'executor', 2, repo);
|
|
3777
|
+
const config = await readTeamConfig('resource-retry', repo);
|
|
3778
|
+
assert.ok(config);
|
|
3779
|
+
if (!config)
|
|
3780
|
+
return;
|
|
3781
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3782
|
+
config.worker_count = config.workers.length;
|
|
3783
|
+
await saveTeamConfig(config, repo);
|
|
3784
|
+
const worktreePath = join(repo, '.omx', 'team', 'resource-retry', 'worktrees', 'worker-2');
|
|
3785
|
+
const workerDir = join(repo, '.omx', 'state', 'team', 'resource-retry', 'workers', 'worker-2');
|
|
3786
|
+
await mkdir(workerDir, { recursive: true });
|
|
3787
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktreePath], { cwd: repo, stdio: 'ignore' });
|
|
3788
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'resource-retry', '.scale-down-cleanup-debt.json');
|
|
3789
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3790
|
+
schema_version: 1,
|
|
3791
|
+
operation: 'scale_down',
|
|
3792
|
+
status: 'resource_cleanup_pending',
|
|
3793
|
+
removed_worker_names: ['worker-2'],
|
|
3794
|
+
workers: [],
|
|
3795
|
+
resource_workers: [{
|
|
3796
|
+
name: 'worker-2', worktree_path: worktreePath, worktree_repo_root: repo,
|
|
3797
|
+
worktree_detached: true, worktree_created: true,
|
|
3798
|
+
}],
|
|
3799
|
+
}));
|
|
3800
|
+
// Simulate a crash after git worktree removal but before worker-state
|
|
3801
|
+
// cleanup and debt deletion. The retry must treat the exact absent target
|
|
3802
|
+
// as converged rather than attempting to remove a replacement.
|
|
3803
|
+
execFileSync('git', ['worktree', 'remove', '--force', worktreePath], { cwd: repo, stdio: 'ignore' });
|
|
3804
|
+
assert.equal(existsSync(worktreePath), false);
|
|
3805
|
+
assert.equal(existsSync(workerDir), true);
|
|
3806
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('resource-retry', repo, config), { ok: true });
|
|
3807
|
+
assert.equal(existsSync(worktreePath), false);
|
|
3808
|
+
assert.equal(existsSync(workerDir), false);
|
|
3809
|
+
assert.equal(existsSync(debtPath), false);
|
|
3810
|
+
}
|
|
3811
|
+
finally {
|
|
3812
|
+
await rm(repo, { recursive: true, force: true });
|
|
3813
|
+
}
|
|
3814
|
+
});
|
|
3815
|
+
it('rejects malformed worktree debt without mutating worker resources', async () => {
|
|
3816
|
+
const repo = await initRepo();
|
|
3817
|
+
try {
|
|
3818
|
+
await initTeamState('malformed-resource-debt', 'task', 'executor', 2, repo);
|
|
3819
|
+
const config = await readTeamConfig('malformed-resource-debt', repo);
|
|
3820
|
+
assert.ok(config);
|
|
3821
|
+
if (!config)
|
|
3822
|
+
return;
|
|
3823
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3824
|
+
config.worker_count = config.workers.length;
|
|
3825
|
+
await saveTeamConfig(config, repo);
|
|
3826
|
+
const workerDir = join(repo, '.omx', 'state', 'team', 'malformed-resource-debt', 'workers', 'worker-2');
|
|
3827
|
+
const sentinelPath = join(workerDir, 'sentinel');
|
|
3828
|
+
await mkdir(workerDir, { recursive: true });
|
|
3829
|
+
await writeFile(sentinelPath, 'preserve');
|
|
3830
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'malformed-resource-debt', '.scale-down-cleanup-debt.json');
|
|
3831
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3832
|
+
schema_version: 1,
|
|
3833
|
+
operation: 'scale_down',
|
|
3834
|
+
status: 'resource_cleanup_pending',
|
|
3835
|
+
removed_worker_names: ['worker-2'],
|
|
3836
|
+
workers: [],
|
|
3837
|
+
resource_workers: [{
|
|
3838
|
+
name: 'worker-2',
|
|
3839
|
+
worktree_path: join(repo, '.omx', 'team', 'malformed-resource-debt', 'worktrees', '..', 'worker-2'),
|
|
3840
|
+
worktree_repo_root: repo,
|
|
3841
|
+
worktree_detached: true,
|
|
3842
|
+
worktree_created: true,
|
|
3843
|
+
}],
|
|
3844
|
+
}));
|
|
3845
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('malformed-resource-debt', repo, config), {
|
|
3846
|
+
ok: false,
|
|
3847
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3848
|
+
});
|
|
3849
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'preserve');
|
|
3850
|
+
assert.equal(existsSync(debtPath), true);
|
|
3851
|
+
}
|
|
3852
|
+
finally {
|
|
3853
|
+
await rm(repo, { recursive: true, force: true });
|
|
3854
|
+
}
|
|
3855
|
+
});
|
|
3856
|
+
it('rejects foreign worktree debt without mutating the foreign location', async () => {
|
|
3857
|
+
const repo = await initRepo();
|
|
3858
|
+
const foreign = await mkdtemp(join(tmpdir(), 'omx-scale-down-foreign-'));
|
|
3859
|
+
try {
|
|
3860
|
+
await initTeamState('foreign-resource-debt', 'task', 'executor', 2, repo);
|
|
3861
|
+
const config = await readTeamConfig('foreign-resource-debt', repo);
|
|
3862
|
+
assert.ok(config);
|
|
3863
|
+
if (!config)
|
|
3864
|
+
return;
|
|
3865
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3866
|
+
config.worker_count = config.workers.length;
|
|
3867
|
+
await saveTeamConfig(config, repo);
|
|
3868
|
+
const sentinelPath = join(foreign, 'AGENTS.md');
|
|
3869
|
+
await writeFile(sentinelPath, 'foreign');
|
|
3870
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'foreign-resource-debt', '.scale-down-cleanup-debt.json');
|
|
3871
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3872
|
+
schema_version: 1,
|
|
3873
|
+
operation: 'scale_down',
|
|
3874
|
+
status: 'resource_cleanup_pending',
|
|
3875
|
+
removed_worker_names: ['worker-2'],
|
|
3876
|
+
workers: [],
|
|
3877
|
+
resource_workers: [{
|
|
3878
|
+
name: 'worker-2', worktree_path: foreign, worktree_repo_root: repo,
|
|
3879
|
+
worktree_detached: true, worktree_created: true,
|
|
3880
|
+
}],
|
|
3881
|
+
}));
|
|
3882
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('foreign-resource-debt', repo, config), {
|
|
3883
|
+
ok: false,
|
|
3884
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3885
|
+
});
|
|
3886
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'foreign');
|
|
3887
|
+
assert.equal(existsSync(debtPath), true);
|
|
3888
|
+
}
|
|
3889
|
+
finally {
|
|
3890
|
+
await rm(repo, { recursive: true, force: true });
|
|
3891
|
+
await rm(foreign, { recursive: true, force: true });
|
|
3892
|
+
}
|
|
3893
|
+
});
|
|
3894
|
+
it('rejects symlink-parent worktree debt without mutating the linked location', async () => {
|
|
3895
|
+
const repo = await initRepo();
|
|
3896
|
+
const foreign = await mkdtemp(join(tmpdir(), 'omx-scale-down-symlink-'));
|
|
3897
|
+
try {
|
|
3898
|
+
await initTeamState('symlink-resource-debt', 'task', 'executor', 2, repo);
|
|
3899
|
+
const config = await readTeamConfig('symlink-resource-debt', repo);
|
|
3900
|
+
assert.ok(config);
|
|
3901
|
+
if (!config)
|
|
3902
|
+
return;
|
|
3903
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
3904
|
+
config.worker_count = config.workers.length;
|
|
3905
|
+
await saveTeamConfig(config, repo);
|
|
3906
|
+
const expectedWorktreeRoot = join(repo, '.omx', 'team', 'symlink-resource-debt', 'worktrees');
|
|
3907
|
+
await mkdir(join(repo, '.omx', 'team', 'symlink-resource-debt'), { recursive: true });
|
|
3908
|
+
await symlink(foreign, expectedWorktreeRoot);
|
|
3909
|
+
const sentinelPath = join(foreign, 'AGENTS.md');
|
|
3910
|
+
await writeFile(sentinelPath, 'foreign');
|
|
3911
|
+
const debtPath = join(repo, '.omx', 'state', 'team', 'symlink-resource-debt', '.scale-down-cleanup-debt.json');
|
|
3912
|
+
await writeFile(debtPath, JSON.stringify({
|
|
3913
|
+
schema_version: 1,
|
|
3914
|
+
operation: 'scale_down',
|
|
3915
|
+
status: 'resource_cleanup_pending',
|
|
3916
|
+
removed_worker_names: ['worker-2'],
|
|
3917
|
+
workers: [],
|
|
3918
|
+
resource_workers: [{
|
|
3919
|
+
name: 'worker-2', worktree_path: join(expectedWorktreeRoot, 'worker-2'), worktree_repo_root: repo,
|
|
3920
|
+
worktree_detached: true, worktree_created: true,
|
|
3921
|
+
}],
|
|
3922
|
+
}));
|
|
3923
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('symlink-resource-debt', repo, config), {
|
|
3924
|
+
ok: false,
|
|
3925
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3926
|
+
});
|
|
3927
|
+
assert.equal(await readFile(sentinelPath, 'utf8'), 'foreign');
|
|
3928
|
+
assert.equal(existsSync(debtPath), true);
|
|
3929
|
+
}
|
|
3930
|
+
finally {
|
|
3931
|
+
await rm(repo, { recursive: true, force: true });
|
|
3932
|
+
await rm(foreign, { recursive: true, force: true });
|
|
3933
|
+
}
|
|
3934
|
+
});
|
|
3935
|
+
it('rejects cleanup debt that collides with a surviving canonical worker before any tmux effect', async () => {
|
|
3936
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-survivor-collision-'));
|
|
3937
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-survivor-collision-bin-'));
|
|
3938
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3939
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3940
|
+
const previousPath = process.env.PATH;
|
|
3941
|
+
try {
|
|
3942
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
3943
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
3944
|
+
exit 99
|
|
3945
|
+
`);
|
|
3946
|
+
await chmod(tmuxStubPath, 0o755);
|
|
3947
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
3948
|
+
await initTeamState('debt-survivor-collision', 'task', 'executor', 3, cwd);
|
|
3949
|
+
const config = await readTeamConfig('debt-survivor-collision', cwd);
|
|
3950
|
+
assert.ok(config);
|
|
3951
|
+
if (!config)
|
|
3952
|
+
return;
|
|
3953
|
+
config.workers[0].pane_id = '%13';
|
|
3954
|
+
config.workers[0].pid = 42413;
|
|
3955
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2' && worker.name !== 'worker-3');
|
|
3956
|
+
config.worker_count = config.workers.length;
|
|
3957
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
3958
|
+
await saveTeamConfig(config, cwd);
|
|
3959
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-survivor-collision', '.scale-down-cleanup-debt.json');
|
|
3960
|
+
const debt = {
|
|
3961
|
+
schema_version: 1,
|
|
3962
|
+
operation: 'scale_down',
|
|
3963
|
+
status: 'pending_teardown',
|
|
3964
|
+
removed_worker_names: ['worker-2', 'worker-3'],
|
|
3965
|
+
workers: [
|
|
3966
|
+
{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42414 },
|
|
3967
|
+
{ name: 'worker-3', index: 3, pane_id: '%14', pid: 42413 },
|
|
3968
|
+
],
|
|
3969
|
+
resource_workers: [{ name: 'worker-2' }, { name: 'worker-3' }],
|
|
3970
|
+
};
|
|
3971
|
+
await writeFile(debtPath, JSON.stringify(debt));
|
|
3972
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-survivor-collision', cwd, config), {
|
|
3973
|
+
ok: false,
|
|
3974
|
+
error: 'scale_down_cleanup_debt_malformed',
|
|
3975
|
+
});
|
|
3976
|
+
assert.deepEqual(JSON.parse(await readFile(debtPath, 'utf8')), debt);
|
|
3977
|
+
assert.equal(existsSync(tmuxLogPath), false);
|
|
3978
|
+
}
|
|
3979
|
+
finally {
|
|
3980
|
+
if (typeof previousPath === 'string')
|
|
3981
|
+
process.env.PATH = previousPath;
|
|
3982
|
+
else
|
|
3983
|
+
delete process.env.PATH;
|
|
3984
|
+
await rm(cwd, { recursive: true, force: true });
|
|
3985
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
3986
|
+
}
|
|
3987
|
+
});
|
|
3988
|
+
for (const [name, livePid, ownerMode, expectedReason] of [
|
|
3989
|
+
['refuses an unrelated current pane occupant', 42413, 'foreign', 'team_owner_mismatch'],
|
|
3990
|
+
['refuses a PID-reused pane even with the Team owner tag', 52413, 'team', 'pane_pid_changed'],
|
|
3991
|
+
['fails closed when Team owner authorization is unavailable', 42413, 'unavailable', 'team_owner_unavailable'],
|
|
3992
|
+
]) {
|
|
3993
|
+
it(name, async () => {
|
|
3994
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-pane-authority-'));
|
|
3995
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-pane-authority-bin-'));
|
|
3996
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
3997
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
3998
|
+
const previousPath = process.env.PATH;
|
|
3999
|
+
try {
|
|
4000
|
+
const ownerScript = ownerMode === 'unavailable'
|
|
4001
|
+
? 'echo owner-query-failed >&2; exit 2'
|
|
4002
|
+
: `printf '%s\\n' '${ownerMode === 'team' ? 'team:debt-pane-authority' : 'team:another-team'}'`;
|
|
4003
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
4004
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
4005
|
+
case "$1" in
|
|
4006
|
+
list-panes) printf '%s\\t%s\\t%s\\n' '%13' '0' '${livePid}' ;;
|
|
4007
|
+
show-option) ${ownerScript} ;;
|
|
4008
|
+
kill-pane) echo unexpected-kill >&2; exit 99 ;;
|
|
4009
|
+
esac
|
|
4010
|
+
`);
|
|
4011
|
+
await chmod(tmuxStubPath, 0o755);
|
|
4012
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
4013
|
+
await initTeamState('debt-pane-authority', 'task', 'executor', 2, cwd);
|
|
4014
|
+
const config = await readTeamConfig('debt-pane-authority', cwd);
|
|
4015
|
+
assert.ok(config);
|
|
4016
|
+
if (!config)
|
|
4017
|
+
return;
|
|
4018
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
4019
|
+
config.worker_count = config.workers.length;
|
|
4020
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
4021
|
+
await saveTeamConfig(config, cwd);
|
|
4022
|
+
await writeFile(join(cwd, '.omx', 'state', 'team', 'debt-pane-authority', 'workers', 'worker-2', 'identity.json'), JSON.stringify({ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }));
|
|
4023
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-pane-authority', '.scale-down-cleanup-debt.json');
|
|
4024
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4025
|
+
schema_version: 1, operation: 'scale_down', status: 'pending_teardown',
|
|
4026
|
+
removed_worker_names: ['worker-2'],
|
|
4027
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
4028
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4029
|
+
}));
|
|
4030
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-pane-authority', cwd, config), {
|
|
4031
|
+
ok: false, error: 'scale_down_cleanup_debt_unresolved:%13',
|
|
4032
|
+
});
|
|
4033
|
+
const debt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
4034
|
+
assert.ok(debt.reasons.includes(`%13:${expectedReason}`));
|
|
4035
|
+
const commands = existsSync(tmuxLogPath) ? await readFile(tmuxLogPath, 'utf8') : '';
|
|
4036
|
+
assert.doesNotMatch(commands, /kill-pane/);
|
|
4037
|
+
}
|
|
4038
|
+
finally {
|
|
4039
|
+
if (typeof previousPath === 'string')
|
|
4040
|
+
process.env.PATH = previousPath;
|
|
4041
|
+
else
|
|
4042
|
+
delete process.env.PATH;
|
|
4043
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4044
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
4045
|
+
}
|
|
4046
|
+
});
|
|
4047
|
+
}
|
|
4048
|
+
it('rejects mismatched worker pane bindings before any recovery mutation', async () => {
|
|
4049
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-mismatched-binding-'));
|
|
4050
|
+
try {
|
|
4051
|
+
await initTeamState('debt-mismatched-binding', 'task', 'executor', 2, cwd);
|
|
4052
|
+
const config = await readTeamConfig('debt-mismatched-binding', cwd);
|
|
4053
|
+
assert.ok(config);
|
|
4054
|
+
if (!config)
|
|
4055
|
+
return;
|
|
4056
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
4057
|
+
config.worker_count = config.workers.length;
|
|
4058
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
4059
|
+
await saveTeamConfig(config, cwd);
|
|
4060
|
+
const workerDir = join(cwd, '.omx', 'state', 'team', 'debt-mismatched-binding', 'workers', 'worker-2');
|
|
4061
|
+
const sentinel = join(workerDir, 'must-remain');
|
|
4062
|
+
await writeFile(sentinel, 'preserve');
|
|
4063
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-mismatched-binding', '.scale-down-cleanup-debt.json');
|
|
4064
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4065
|
+
schema_version: 1, operation: 'scale_down', status: 'unresolved',
|
|
4066
|
+
removed_worker_names: ['worker-2'],
|
|
4067
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: 42413 }],
|
|
4068
|
+
unresolved_panes: [{ name: 'worker-2', index: 2, pane_id: '%14', pid: 42413 }],
|
|
4069
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4070
|
+
}));
|
|
4071
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-mismatched-binding', cwd, config), {
|
|
4072
|
+
ok: false, error: 'scale_down_cleanup_debt_malformed',
|
|
4073
|
+
});
|
|
4074
|
+
assert.equal(await readFile(sentinel, 'utf8'), 'preserve');
|
|
4075
|
+
assert.equal(existsSync(debtPath), true);
|
|
4076
|
+
}
|
|
4077
|
+
finally {
|
|
4078
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4079
|
+
}
|
|
4080
|
+
});
|
|
4081
|
+
it('converges a PID-less legacy pane record only after a fresh global gone proof', async () => {
|
|
4082
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-legacy-gone-'));
|
|
4083
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-down-debt-legacy-gone-bin-'));
|
|
4084
|
+
const tmuxStubPath = join(fakeBinDir, 'tmux');
|
|
4085
|
+
const tmuxLogPath = join(fakeBinDir, 'tmux.log');
|
|
4086
|
+
const previousPath = process.env.PATH;
|
|
4087
|
+
try {
|
|
4088
|
+
await writeFile(tmuxStubPath, `#!/bin/sh
|
|
4089
|
+
printf '%s\\n' "$*" >> "${tmuxLogPath}"
|
|
4090
|
+
case "$1" in list-panes) exit 0 ;; kill-pane) exit 99 ;; esac
|
|
4091
|
+
`);
|
|
4092
|
+
await chmod(tmuxStubPath, 0o755);
|
|
4093
|
+
process.env.PATH = `${fakeBinDir}:${previousPath ?? ''}`;
|
|
4094
|
+
await initTeamState('debt-legacy-gone', 'task', 'executor', 2, cwd);
|
|
4095
|
+
const config = await readTeamConfig('debt-legacy-gone', cwd);
|
|
4096
|
+
assert.ok(config);
|
|
4097
|
+
if (!config)
|
|
4098
|
+
return;
|
|
4099
|
+
config.workers = config.workers.filter((worker) => worker.name !== 'worker-2');
|
|
4100
|
+
config.worker_count = config.workers.length;
|
|
4101
|
+
config.tmux_pane_owner_id = 'team:scale-down';
|
|
4102
|
+
await saveTeamConfig(config, cwd);
|
|
4103
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'debt-legacy-gone', '.scale-down-cleanup-debt.json');
|
|
4104
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4105
|
+
schema_version: 1, operation: 'scale_down', status: 'unresolved',
|
|
4106
|
+
removed_worker_names: ['worker-2'],
|
|
4107
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: null }],
|
|
4108
|
+
unresolved_panes: [{ name: 'worker-2', index: 2, pane_id: '%13', pid: null }],
|
|
4109
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4110
|
+
}));
|
|
4111
|
+
assert.deepEqual(await reconcileScaleDownCleanupDebt('debt-legacy-gone', cwd, config), { ok: true });
|
|
4112
|
+
assert.equal(existsSync(debtPath), false);
|
|
4113
|
+
assert.doesNotMatch(await readFile(tmuxLogPath, 'utf8'), /kill-pane|show-option/);
|
|
1897
4114
|
}
|
|
1898
4115
|
finally {
|
|
1899
4116
|
if (typeof previousPath === 'string')
|