oh-my-codex 0.21.48 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +6 -6
- package/Cargo.toml +1 -1
- package/README.md +15 -0
- package/crates/omx-mux/src/herdr.rs +481 -0
- package/crates/omx-mux/src/lib.rs +5 -0
- package/dist/cli/__tests__/plugin-cache-live-pin-3790.test.js +104 -3
- package/dist/cli/__tests__/plugin-cache-live-pin-3790.test.js.map +1 -1
- package/dist/cli/__tests__/question.test.js +33 -0
- package/dist/cli/__tests__/question.test.js.map +1 -1
- package/dist/cli/__tests__/session-compare.test.d.ts +2 -0
- package/dist/cli/__tests__/session-compare.test.d.ts.map +1 -0
- package/dist/cli/__tests__/session-compare.test.js +183 -0
- package/dist/cli/__tests__/session-compare.test.js.map +1 -0
- package/dist/cli/__tests__/team.test.js +8 -0
- package/dist/cli/__tests__/team.test.js.map +1 -1
- package/dist/cli/plugin-marketplace.d.ts.map +1 -1
- package/dist/cli/plugin-marketplace.js +11 -4
- package/dist/cli/plugin-marketplace.js.map +1 -1
- package/dist/cli/question.d.ts.map +1 -1
- package/dist/cli/question.js +8 -1
- package/dist/cli/question.js.map +1 -1
- package/dist/cli/session-compare.d.ts +11 -0
- package/dist/cli/session-compare.d.ts.map +1 -0
- package/dist/cli/session-compare.js +172 -0
- package/dist/cli/session-compare.js.map +1 -0
- package/dist/cli/session-search.d.ts.map +1 -1
- package/dist/cli/session-search.js +7 -0
- package/dist/cli/session-search.js.map +1 -1
- package/dist/cli/team.d.ts.map +1 -1
- package/dist/cli/team.js +2 -0
- package/dist/cli/team.js.map +1 -1
- package/dist/config/__tests__/multiplexer.test.d.ts +2 -0
- package/dist/config/__tests__/multiplexer.test.d.ts.map +1 -0
- package/dist/config/__tests__/multiplexer.test.js +118 -0
- package/dist/config/__tests__/multiplexer.test.js.map +1 -0
- package/dist/config/multiplexer.d.ts +29 -0
- package/dist/config/multiplexer.d.ts.map +1 -0
- package/dist/config/multiplexer.js +91 -0
- package/dist/config/multiplexer.js.map +1 -0
- package/dist/question/__tests__/ui.test.js +40 -1
- package/dist/question/__tests__/ui.test.js.map +1 -1
- package/dist/question/ui.d.ts.map +1 -1
- package/dist/question/ui.js +37 -7
- package/dist/question/ui.js.map +1 -1
- package/dist/scripts/__tests__/codex-native-hook.test.js +281 -1
- package/dist/scripts/__tests__/codex-native-hook.test.js.map +1 -1
- package/dist/scripts/codex-native-hook.d.ts.map +1 -1
- package/dist/scripts/codex-native-hook.js +8 -2
- package/dist/scripts/codex-native-hook.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker-lease.d.ts +15 -0
- package/dist/scripts/notify-hook/team-worker-lease.d.ts.map +1 -0
- package/dist/scripts/notify-hook/team-worker-lease.js +63 -0
- package/dist/scripts/notify-hook/team-worker-lease.js.map +1 -0
- package/dist/scripts/notify-hook/team-worker-posttooluse.d.ts +1 -0
- package/dist/scripts/notify-hook/team-worker-posttooluse.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker-posttooluse.js +16 -5
- package/dist/scripts/notify-hook/team-worker-posttooluse.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker.d.ts +12 -1
- package/dist/scripts/notify-hook/team-worker.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker.js +16 -4
- package/dist/scripts/notify-hook/team-worker.js.map +1 -1
- package/dist/session-history/__tests__/compare.test.d.ts +2 -0
- package/dist/session-history/__tests__/compare.test.d.ts.map +1 -0
- package/dist/session-history/__tests__/compare.test.js +257 -0
- package/dist/session-history/__tests__/compare.test.js.map +1 -0
- package/dist/session-history/compare.d.ts +74 -0
- package/dist/session-history/compare.d.ts.map +1 -0
- package/dist/session-history/compare.js +165 -0
- package/dist/session-history/compare.js.map +1 -0
- package/dist/team/__tests__/api-interop.test.js +3 -3
- package/dist/team/__tests__/completion-evidence.test.js +31 -1
- package/dist/team/__tests__/completion-evidence.test.js.map +1 -1
- package/dist/team/__tests__/cross-rebase-smoke.test.js +263 -31
- package/dist/team/__tests__/cross-rebase-smoke.test.js.map +1 -1
- package/dist/team/__tests__/herdr-session-realherdr.test.d.ts +2 -0
- package/dist/team/__tests__/herdr-session-realherdr.test.d.ts.map +1 -0
- package/dist/team/__tests__/herdr-session-realherdr.test.js +68 -0
- package/dist/team/__tests__/herdr-session-realherdr.test.js.map +1 -0
- package/dist/team/__tests__/herdr-session.test.d.ts +2 -0
- package/dist/team/__tests__/herdr-session.test.d.ts.map +1 -0
- package/dist/team/__tests__/herdr-session.test.js +228 -0
- package/dist/team/__tests__/herdr-session.test.js.map +1 -0
- package/dist/team/__tests__/hook-primary-e2e-contract.test.js +9 -3
- package/dist/team/__tests__/hook-primary-e2e-contract.test.js.map +1 -1
- package/dist/team/__tests__/idle-nudge.test.js +84 -0
- package/dist/team/__tests__/idle-nudge.test.js.map +1 -1
- package/dist/team/__tests__/mcp-comm.test.js +25 -1
- package/dist/team/__tests__/mcp-comm.test.js.map +1 -1
- package/dist/team/__tests__/rebalance-policy.test.js +27 -9
- package/dist/team/__tests__/rebalance-policy.test.js.map +1 -1
- package/dist/team/__tests__/runtime-delivery.test.d.ts +2 -0
- package/dist/team/__tests__/runtime-delivery.test.d.ts.map +1 -0
- package/dist/team/__tests__/runtime-delivery.test.js +570 -0
- package/dist/team/__tests__/runtime-delivery.test.js.map +1 -0
- package/dist/team/__tests__/runtime.test.js +938 -70
- package/dist/team/__tests__/runtime.test.js.map +1 -1
- package/dist/team/__tests__/scaling.test.js +645 -1
- package/dist/team/__tests__/scaling.test.js.map +1 -1
- package/dist/team/__tests__/state-renewal.test.d.ts +2 -0
- package/dist/team/__tests__/state-renewal.test.d.ts.map +1 -0
- package/dist/team/__tests__/state-renewal.test.js +555 -0
- package/dist/team/__tests__/state-renewal.test.js.map +1 -0
- package/dist/team/__tests__/state.test.js +121 -2
- package/dist/team/__tests__/state.test.js.map +1 -1
- package/dist/team/__tests__/task-lease.test.d.ts +2 -0
- package/dist/team/__tests__/task-lease.test.d.ts.map +1 -0
- package/dist/team/__tests__/task-lease.test.js +179 -0
- package/dist/team/__tests__/task-lease.test.js.map +1 -0
- package/dist/team/__tests__/team-ops-contract.test.js +1 -0
- package/dist/team/__tests__/team-ops-contract.test.js.map +1 -1
- package/dist/team/__tests__/worker-bootstrap.test.js +2 -0
- package/dist/team/__tests__/worker-bootstrap.test.js.map +1 -1
- package/dist/team/api-interop.d.ts +1 -1
- package/dist/team/api-interop.d.ts.map +1 -1
- package/dist/team/api-interop.js +13 -1
- package/dist/team/api-interop.js.map +1 -1
- package/dist/team/completion-evidence.d.ts +1 -1
- package/dist/team/completion-evidence.d.ts.map +1 -1
- package/dist/team/completion-evidence.js +9 -0
- package/dist/team/completion-evidence.js.map +1 -1
- package/dist/team/goal-workflow.js +1 -1
- package/dist/team/herdr-session.d.ts +234 -0
- package/dist/team/herdr-session.d.ts.map +1 -0
- package/dist/team/herdr-session.js +489 -0
- package/dist/team/herdr-session.js.map +1 -0
- package/dist/team/idle-nudge.d.ts +16 -4
- package/dist/team/idle-nudge.d.ts.map +1 -1
- package/dist/team/idle-nudge.js +51 -19
- package/dist/team/idle-nudge.js.map +1 -1
- package/dist/team/mcp-comm.d.ts +6 -4
- package/dist/team/mcp-comm.d.ts.map +1 -1
- package/dist/team/mcp-comm.js +4 -0
- package/dist/team/mcp-comm.js.map +1 -1
- package/dist/team/rebalance-policy.d.ts.map +1 -1
- package/dist/team/rebalance-policy.js +5 -2
- package/dist/team/rebalance-policy.js.map +1 -1
- package/dist/team/runtime.d.ts.map +1 -1
- package/dist/team/runtime.js +463 -166
- package/dist/team/runtime.js.map +1 -1
- package/dist/team/scaling.d.ts +3 -3
- package/dist/team/scaling.d.ts.map +1 -1
- package/dist/team/scaling.js +394 -200
- package/dist/team/scaling.js.map +1 -1
- package/dist/team/state/monitor.d.ts +3 -0
- package/dist/team/state/monitor.d.ts.map +1 -1
- package/dist/team/state/monitor.js +3 -0
- package/dist/team/state/monitor.js.map +1 -1
- package/dist/team/state/tasks.d.ts +4 -1
- package/dist/team/state/tasks.d.ts.map +1 -1
- package/dist/team/state/tasks.js +36 -5
- package/dist/team/state/tasks.js.map +1 -1
- package/dist/team/state/types.d.ts +9 -0
- package/dist/team/state/types.d.ts.map +1 -1
- package/dist/team/state/types.js.map +1 -1
- package/dist/team/state.d.ts +15 -2
- package/dist/team/state.d.ts.map +1 -1
- package/dist/team/state.js +73 -5
- package/dist/team/state.js.map +1 -1
- package/dist/team/task-lease.d.ts +4 -0
- package/dist/team/task-lease.d.ts.map +1 -0
- package/dist/team/task-lease.js +74 -0
- package/dist/team/task-lease.js.map +1 -0
- package/dist/team/team-ops.d.ts +2 -1
- package/dist/team/team-ops.d.ts.map +1 -1
- package/dist/team/team-ops.js +1 -0
- package/dist/team/team-ops.js.map +1 -1
- package/dist/team/tmux-session.d.ts +1 -1
- package/dist/team/tmux-session.d.ts.map +1 -1
- package/dist/team/tmux-session.js +10 -2
- package/dist/team/tmux-session.js.map +1 -1
- package/dist/team/worker-bootstrap.d.ts.map +1 -1
- package/dist/team/worker-bootstrap.js +16 -3
- package/dist/team/worker-bootstrap.js.map +1 -1
- package/package.json +1 -1
- package/plugins/oh-my-codex/.codex-plugin/plugin.json +1 -1
- package/plugins/oh-my-codex/hooks/codex-native-hook.mjs +4 -1
- package/plugins/oh-my-codex/skills/worker/SKILL.md +7 -1
- package/skills/worker/SKILL.md +7 -1
- package/src/scripts/__tests__/codex-native-hook.test.ts +279 -0
- package/src/scripts/codex-native-hook.ts +8 -2
- package/src/scripts/notify-hook/team-worker-lease.ts +69 -0
- package/src/scripts/notify-hook/team-worker-posttooluse.ts +16 -4
- package/src/scripts/notify-hook/team-worker.ts +15 -4
|
@@ -6,7 +6,7 @@ import { mkdtemp, rm, readFile, writeFile, mkdir, chmod, readdir, symlink } from
|
|
|
6
6
|
import { join, posix, relative, win32 } from 'path';
|
|
7
7
|
import { tmpdir as osTmpdir } from 'os';
|
|
8
8
|
import { existsSync, readFileSync, realpathSync } from 'fs';
|
|
9
|
-
import { initTeamState, createTask, claimTask, readTask, readTeamConfig, saveTeamConfig, readWorkerStatus, writeWorkerStatus, withScalingLock, recoverTeamMembershipTaskTransaction, commitTeamMembershipTaskTransaction, withTeamTaskBarrier, DEFAULT_MAX_WORKERS, listDispatchRequests, readTeamManifestV2, } from '../state.js';
|
|
9
|
+
import { initTeamState, createTask, transitionTaskStatus, claimTask, readTask, readTeamConfig, saveTeamConfig, readWorkerStatus, writeWorkerStatus, writeWorkerIdentity, withScalingLock, recoverTeamMembershipTaskTransaction, commitTeamMembershipTaskTransaction, withTeamTaskBarrier, DEFAULT_MAX_WORKERS, listDispatchRequests, readTeamManifestV2, } from '../state.js';
|
|
10
10
|
import { isSameOrInsidePath, isScalingEnabled, reconcileScaleDownCleanupDebt, scaleUp, scaleDown } from '../scaling.js';
|
|
11
11
|
import { resolveCanonicalTeamStateRoot } from '../state-root.js';
|
|
12
12
|
import { resolvePersistedApprovedTeamExecutionContinuityState, writePersistedApprovedTeamExecutionBinding, } from '../approved-execution.js';
|
|
@@ -2826,11 +2826,16 @@ describe('scaleDown', () => {
|
|
|
2826
2826
|
current_task_id: 't-2',
|
|
2827
2827
|
updated_at: new Date().toISOString(),
|
|
2828
2828
|
}, cwd);
|
|
2829
|
+
const teamRoot = join(cwd, '.omx/state/team/busy-test');
|
|
2830
|
+
const artifactPaths = ['config.json', 'manifest.v2.json', 'workers/worker-1/status.json', 'workers/worker-2/status.json'];
|
|
2831
|
+
const before = await Promise.all(artifactPaths.map((path) => readFile(join(teamRoot, path), 'utf8')));
|
|
2829
2832
|
const result = await scaleDown('busy-test', cwd, { count: 1 }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
2830
2833
|
assert.equal(result.ok, false);
|
|
2831
2834
|
if (!result.ok) {
|
|
2832
2835
|
assert.match(result.error, /Not enough idle workers/);
|
|
2833
2836
|
}
|
|
2837
|
+
assert.deepEqual(await Promise.all(artifactPaths.map((path) => readFile(join(teamRoot, path), 'utf8'))), before);
|
|
2838
|
+
assert.equal(existsSync(join(teamRoot, '.scale-down-cleanup-debt.json')), false);
|
|
2834
2839
|
}
|
|
2835
2840
|
finally {
|
|
2836
2841
|
await rm(cwd, { recursive: true, force: true });
|
|
@@ -4175,6 +4180,326 @@ case "$1" in list-panes) exit 0 ;; kill-pane) exit 99 ;; esac
|
|
|
4175
4180
|
}
|
|
4176
4181
|
});
|
|
4177
4182
|
});
|
|
4183
|
+
describe('nonforce scale-down worktree preservation', () => {
|
|
4184
|
+
it('vetoes stale idle with a live claim using the locked task snapshot', async () => {
|
|
4185
|
+
const repo = await initRepo();
|
|
4186
|
+
try {
|
|
4187
|
+
const team = 'stale-idle';
|
|
4188
|
+
await initTeamState(team, 'task', 'executor', 2, repo);
|
|
4189
|
+
const config = await readTeamConfig(team, repo);
|
|
4190
|
+
assert.ok(config);
|
|
4191
|
+
const worktree = join(repo, '.omx/team', team, 'worktrees/worker-2');
|
|
4192
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktree], { cwd: repo, stdio: 'ignore' });
|
|
4193
|
+
Object.assign(config.workers[1], { worktree_path: worktree, worktree_repo_root: repo, worktree_created: true, pane_id: undefined, pid: undefined });
|
|
4194
|
+
await saveTeamConfig(config, repo);
|
|
4195
|
+
await writeWorkerStatus(team, 'worker-2', { state: 'idle', updated_at: new Date(0).toISOString() }, repo);
|
|
4196
|
+
const task = await createTask(team, { subject: 'new work', description: 'claimed after idle', status: 'pending' }, repo);
|
|
4197
|
+
const claim = await claimTask(team, task.id, 'worker-2', null, repo);
|
|
4198
|
+
assert.ok(claim.ok);
|
|
4199
|
+
const stateDir = join(repo, '.omx/state/team', team);
|
|
4200
|
+
const beforeConfig = await readFile(join(stateDir, 'config.json'), 'utf8');
|
|
4201
|
+
const beforeTask = await readTask(team, task.id, repo);
|
|
4202
|
+
const result = await scaleDown(team, repo, { workerNames: ['worker-2'], drainTimeoutMs: 100 }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
4203
|
+
assert.ok(!result.ok && /scale_down_worker_has_active_tasks/.test(result.error));
|
|
4204
|
+
assert.equal(await readFile(join(stateDir, 'config.json'), 'utf8'), beforeConfig);
|
|
4205
|
+
assert.deepEqual(await readTask(team, task.id, repo), beforeTask);
|
|
4206
|
+
assert.equal(existsSync(worktree), true);
|
|
4207
|
+
assert.equal(existsSync(join(stateDir, '.scale-down-cleanup-debt.json')), false);
|
|
4208
|
+
}
|
|
4209
|
+
finally {
|
|
4210
|
+
await rm(repo, { recursive: true, force: true });
|
|
4211
|
+
}
|
|
4212
|
+
});
|
|
4213
|
+
it('never writes worker status while waiting, including acknowledgment during another worker timeout', async () => {
|
|
4214
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-drain-partial-timeout-'));
|
|
4215
|
+
try {
|
|
4216
|
+
const team = 'partial-timeout';
|
|
4217
|
+
await initTeamState(team, 'task', 'executor', 3, cwd);
|
|
4218
|
+
const task = await createTask(team, { subject: 'finishing', description: 'worker 2 finishes', status: 'pending' }, cwd);
|
|
4219
|
+
const claim = await claimTask(team, task.id, 'worker-2', null, cwd);
|
|
4220
|
+
assert.ok(claim.ok);
|
|
4221
|
+
const before = { state: 'working', current_task_id: task.id, updated_at: new Date(0).toISOString() };
|
|
4222
|
+
for (const name of ['worker-2', 'worker-3'])
|
|
4223
|
+
await writeWorkerStatus(team, name, before, cwd);
|
|
4224
|
+
const stateDir = join(cwd, '.omx/state/team', team);
|
|
4225
|
+
const configBytes = await readFile(join(stateDir, 'config.json'), 'utf8');
|
|
4226
|
+
const manifestBytes = await readFile(join(stateDir, 'manifest.v2.json'), 'utf8');
|
|
4227
|
+
const inboxPath = join(stateDir, 'workers/worker-2/inbox.md');
|
|
4228
|
+
await writeFile(inboxPath, 'Keep working on assigned task A.\n');
|
|
4229
|
+
const removal = scaleDown(team, cwd, { workerNames: ['worker-2', 'worker-3'], drainTimeoutMs: 500 }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
4230
|
+
// scaleDown is only a reader: the worker's working status stays authoritative.
|
|
4231
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
4232
|
+
for (const name of ['worker-2', 'worker-3'])
|
|
4233
|
+
assert.deepEqual(await readWorkerStatus(team, name, cwd), before);
|
|
4234
|
+
assert.ok((await transitionTaskStatus(team, task.id, 'in_progress', 'completed', claim.claimToken, cwd)).ok);
|
|
4235
|
+
const acknowledgment = { state: 'done', updated_at: new Date().toISOString() };
|
|
4236
|
+
await writeWorkerStatus(team, 'worker-2', acknowledgment, cwd);
|
|
4237
|
+
assert.deepEqual(await removal, { ok: false, error: 'scale_down_drain_timeout' });
|
|
4238
|
+
assert.deepEqual(await readWorkerStatus(team, 'worker-2', cwd), acknowledgment);
|
|
4239
|
+
assert.deepEqual(await readWorkerStatus(team, 'worker-3', cwd), before);
|
|
4240
|
+
assert.equal((await readTask(team, task.id, cwd))?.status, 'completed');
|
|
4241
|
+
assert.equal((await readTeamConfig(team, cwd))?.workers.length, 3);
|
|
4242
|
+
assert.equal(await readFile(join(stateDir, 'config.json'), 'utf8'), configBytes);
|
|
4243
|
+
assert.equal(await readFile(join(stateDir, 'manifest.v2.json'), 'utf8'), manifestBytes);
|
|
4244
|
+
assert.equal(await readFile(inboxPath, 'utf8'), 'Keep working on assigned task A.\n');
|
|
4245
|
+
assert.equal(existsSync(join(stateDir, '.scale-down-cleanup-debt.json')), false);
|
|
4246
|
+
}
|
|
4247
|
+
finally {
|
|
4248
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4249
|
+
}
|
|
4250
|
+
});
|
|
4251
|
+
for (const scenario of ['busy-timeout', 'dirty-ack', 'unintegrated-ack', 'clean-ack']) {
|
|
4252
|
+
it(`requires a real drain acknowledgment and preserves unsafe work (${scenario})`, async () => {
|
|
4253
|
+
const repo = await initRepo();
|
|
4254
|
+
try {
|
|
4255
|
+
const team = 'safe-drain';
|
|
4256
|
+
await initTeamState(team, 'task', 'executor', 2, repo);
|
|
4257
|
+
const config = await readTeamConfig(team, repo);
|
|
4258
|
+
assert.ok(config);
|
|
4259
|
+
const worktree = join(repo, '.omx', 'team', team, 'worktrees', 'worker-2');
|
|
4260
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktree], { cwd: repo, stdio: 'ignore' });
|
|
4261
|
+
Object.assign(config.workers[1], { worktree_path: worktree, worktree_repo_root: repo, worktree_created: true, worktree_detached: true, pane_id: undefined, pid: undefined });
|
|
4262
|
+
await saveTeamConfig(config, repo);
|
|
4263
|
+
const task = await createTask(team, {
|
|
4264
|
+
subject: 'ongoing work', description: 'must survive failed removal', status: 'pending', owner: 'worker-2',
|
|
4265
|
+
}, repo);
|
|
4266
|
+
const claim = await claimTask(team, task.id, 'worker-2', null, repo);
|
|
4267
|
+
assert.ok(claim.ok);
|
|
4268
|
+
const originalStatus = { state: 'working', current_task_id: task.id, updated_at: new Date().toISOString() };
|
|
4269
|
+
await writeWorkerStatus(team, 'worker-2', originalStatus, repo);
|
|
4270
|
+
if (scenario !== 'clean-ack')
|
|
4271
|
+
await writeFile(join(worktree, 'unsaved.txt'), 'worker output');
|
|
4272
|
+
if (scenario === 'unintegrated-ack') {
|
|
4273
|
+
execFileSync('git', ['add', 'unsaved.txt'], { cwd: worktree });
|
|
4274
|
+
execFileSync('git', ['commit', '-m', 'not integrated'], { cwd: worktree, stdio: 'ignore' });
|
|
4275
|
+
}
|
|
4276
|
+
const stateDir = join(repo, '.omx', 'state', 'team', team);
|
|
4277
|
+
const beforeConfig = await readFile(join(stateDir, 'config.json'), 'utf8');
|
|
4278
|
+
const beforeManifest = await readFile(join(stateDir, 'manifest.v2.json'), 'utf8');
|
|
4279
|
+
const beforeTask = await readTask(team, task.id, repo);
|
|
4280
|
+
const startedAt = Date.now();
|
|
4281
|
+
const removal = scaleDown(team, repo, { workerNames: ['worker-2'], force: false, drainTimeoutMs: scenario === 'busy-timeout' ? 250 : 5_000 }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
4282
|
+
// The leader waits without replacing worker-owned status or task inboxes.
|
|
4283
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
4284
|
+
assert.deepEqual(await readWorkerStatus(team, 'worker-2', repo), originalStatus);
|
|
4285
|
+
assert.equal((await readWorkerStatus(team, 'worker-2', repo)).current_task_id, task.id);
|
|
4286
|
+
assert.equal(existsSync(worktree), true);
|
|
4287
|
+
if (scenario !== 'busy-timeout') {
|
|
4288
|
+
assert.ok((await transitionTaskStatus(team, task.id, 'in_progress', 'completed', claim.claimToken, repo)).ok);
|
|
4289
|
+
await writeWorkerStatus(team, 'worker-2', { state: 'idle', updated_at: new Date().toISOString() }, repo);
|
|
4290
|
+
}
|
|
4291
|
+
const result = await removal;
|
|
4292
|
+
if (scenario === 'clean-ack') {
|
|
4293
|
+
assert.deepEqual(result, { ok: true, removedWorkers: ['worker-2'], newWorkerCount: 1 });
|
|
4294
|
+
assert.equal(existsSync(worktree), false);
|
|
4295
|
+
}
|
|
4296
|
+
else {
|
|
4297
|
+
assert.equal(result.ok, false);
|
|
4298
|
+
if (!result.ok)
|
|
4299
|
+
assert.match(result.error, scenario === 'busy-timeout' ? /scale_down_drain_timeout/ : /scale_down_worktree_not_preserved/);
|
|
4300
|
+
if (scenario === 'busy-timeout')
|
|
4301
|
+
assert.ok(Date.now() - startedAt >= 250, 'busy status is not a drain acknowledgment');
|
|
4302
|
+
assert.equal(await readFile(join(worktree, 'unsaved.txt'), 'utf8'), 'worker output');
|
|
4303
|
+
assert.equal(await readFile(join(stateDir, 'config.json'), 'utf8'), beforeConfig);
|
|
4304
|
+
assert.equal(await readFile(join(stateDir, 'manifest.v2.json'), 'utf8'), beforeManifest);
|
|
4305
|
+
if (scenario === 'busy-timeout') {
|
|
4306
|
+
assert.deepEqual(await readTask(team, task.id, repo), beforeTask);
|
|
4307
|
+
assert.deepEqual(await readWorkerStatus(team, 'worker-2', repo), originalStatus);
|
|
4308
|
+
}
|
|
4309
|
+
else {
|
|
4310
|
+
assert.equal((await readTask(team, task.id, repo))?.status, 'completed');
|
|
4311
|
+
assert.equal((await readWorkerStatus(team, 'worker-2', repo)).state, 'idle');
|
|
4312
|
+
}
|
|
4313
|
+
assert.equal(existsSync(join(stateDir, '.scale-down-cleanup-debt.json')), false);
|
|
4314
|
+
}
|
|
4315
|
+
}
|
|
4316
|
+
finally {
|
|
4317
|
+
await rm(repo, { recursive: true, force: true });
|
|
4318
|
+
}
|
|
4319
|
+
});
|
|
4320
|
+
}
|
|
4321
|
+
});
|
|
4322
|
+
describe('scale-down unfinished Git operation preservation', () => {
|
|
4323
|
+
for (const [operation, recovery] of [
|
|
4324
|
+
['initial-break rebase', false], ['initial-break rebase', true],
|
|
4325
|
+
['clean no-commit merge', false], ['clean no-commit merge', true],
|
|
4326
|
+
]) {
|
|
4327
|
+
it(`preserves an actual ${operation} ${recovery ? 'during cleanup debt recovery' : 'before membership and pane effects'}`, async () => {
|
|
4328
|
+
const repo = await initRepo();
|
|
4329
|
+
const previousPath = process.env.PATH;
|
|
4330
|
+
try {
|
|
4331
|
+
const isRebase = operation === 'initial-break rebase';
|
|
4332
|
+
const team = isRebase ? 'initial-break' : 'no-commit-merge';
|
|
4333
|
+
await initTeamState(team, 'task', 'executor', 2, repo);
|
|
4334
|
+
await createTask(team, { subject: 'finished work', description: 'no active claim', status: 'completed', owner: 'worker-2' }, repo);
|
|
4335
|
+
const config = await readTeamConfig(team, repo);
|
|
4336
|
+
assert.ok(config);
|
|
4337
|
+
const worktree = join(repo, '.omx', 'team', team, 'worktrees', 'worker-2');
|
|
4338
|
+
const git = (args, cwd = worktree, env = process.env) => execFileSync('git', args, { cwd, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
|
|
4339
|
+
git(['worktree', 'add', '--detach', worktree], repo);
|
|
4340
|
+
if (isRebase) {
|
|
4341
|
+
await writeFile(join(worktree, 'unpublished.txt'), 'original worker commit\n');
|
|
4342
|
+
git(['add', 'unpublished.txt']);
|
|
4343
|
+
}
|
|
4344
|
+
git(['commit', '--allow-empty', '-m', 'unpublished worker commit']);
|
|
4345
|
+
const originalHead = git(['rev-parse', 'HEAD']);
|
|
4346
|
+
if (isRebase) {
|
|
4347
|
+
await writeFile(join(repo, 'leader.txt'), 'independent leader commit\n');
|
|
4348
|
+
git(['add', 'leader.txt'], repo);
|
|
4349
|
+
}
|
|
4350
|
+
git(['commit', '--allow-empty', '-m', 'independent leader commit'], repo);
|
|
4351
|
+
const leaderHead = git(['rev-parse', 'HEAD'], repo);
|
|
4352
|
+
Object.assign(config.workers[1], {
|
|
4353
|
+
worktree_path: worktree, worktree_repo_root: repo, worktree_created: true,
|
|
4354
|
+
worktree_detached: true, pane_id: '%13', pid: 42413,
|
|
4355
|
+
});
|
|
4356
|
+
config.tmux_pane_owner_id = `team:${team}`;
|
|
4357
|
+
await writeWorkerIdentity(team, 'worker-2', config.workers[1], repo);
|
|
4358
|
+
if (recovery) {
|
|
4359
|
+
config.workers = config.workers.slice(0, 1);
|
|
4360
|
+
config.worker_count = 1;
|
|
4361
|
+
}
|
|
4362
|
+
await saveTeamConfig(config, repo);
|
|
4363
|
+
await writeWorkerStatus(team, 'worker-2', { state: 'done', updated_at: new Date().toISOString() }, repo);
|
|
4364
|
+
const stateDir = join(repo, '.omx/state/team', team);
|
|
4365
|
+
const debtPath = join(stateDir, '.scale-down-cleanup-debt.json');
|
|
4366
|
+
if (recovery)
|
|
4367
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4368
|
+
schema_version: 1, operation: 'scale_down', status: 'resource_cleanup_pending', force: false,
|
|
4369
|
+
removed_worker_names: ['worker-2'], workers: [], unresolved_panes: [],
|
|
4370
|
+
resource_workers: [{ name: 'worker-2', worktree_path: worktree, worktree_repo_root: repo, worktree_created: true, worktree_detached: true }],
|
|
4371
|
+
}));
|
|
4372
|
+
const beforeDebt = recovery ? JSON.parse(await readFile(debtPath, 'utf8')) : null;
|
|
4373
|
+
const gitDir = git(['rev-parse', '--absolute-git-dir']);
|
|
4374
|
+
const operationPath = join(gitDir, isRebase ? 'rebase-merge' : 'MERGE_HEAD');
|
|
4375
|
+
const mergeMessage = 'Operator-edited merge message: retain this unfinished integration.\n';
|
|
4376
|
+
if (isRebase) {
|
|
4377
|
+
const editor = join(repo, '.git', 'initial-break-editor.sh');
|
|
4378
|
+
await writeFile(editor, '#!/bin/sh\nset -eu\n{ printf "break\\n"; cat "$1"; } > "$1.new"\nmv "$1.new" "$1"\n');
|
|
4379
|
+
await chmod(editor, 0o755);
|
|
4380
|
+
git(['rebase', '-i', leaderHead], worktree, { ...process.env, GIT_SEQUENCE_EDITOR: editor });
|
|
4381
|
+
assert.equal((await readFile(join(operationPath, 'orig-head'), 'utf8')).trim(), originalHead);
|
|
4382
|
+
assert.match(await readFile(join(operationPath, 'git-rebase-todo'), 'utf8'), new RegExp(originalHead.slice(0, 7)));
|
|
4383
|
+
}
|
|
4384
|
+
else {
|
|
4385
|
+
assert.equal(git(['rev-parse', `${originalHead}^{tree}`]), git(['rev-parse', `${leaderHead}^{tree}`]));
|
|
4386
|
+
assert.notEqual(git(['merge-base', originalHead, leaderHead]), originalHead);
|
|
4387
|
+
git(['reset', '--hard', leaderHead]);
|
|
4388
|
+
git(['merge', '--no-commit', '--no-ff', originalHead]);
|
|
4389
|
+
assert.equal((await readFile(operationPath, 'utf8')).trim(), originalHead);
|
|
4390
|
+
await writeFile(join(gitDir, 'MERGE_MSG'), mergeMessage);
|
|
4391
|
+
}
|
|
4392
|
+
assert.equal(git(['rev-parse', 'HEAD']), leaderHead);
|
|
4393
|
+
git(['merge-base', '--is-ancestor', 'HEAD', leaderHead]);
|
|
4394
|
+
assert.equal(git(['status', '--porcelain', '--untracked-files=all', '--ignored', '--ignore-submodules=none']), '');
|
|
4395
|
+
const bin = join(repo, '.git', 'fake-bin');
|
|
4396
|
+
await mkdir(bin);
|
|
4397
|
+
const tmuxLog = join(bin, 'tmux.log');
|
|
4398
|
+
await writeFile(join(bin, 'tmux'), `#!/bin/sh\nprintf '%s\\n' "$*" >> '${tmuxLog}'\nexit 1\n`);
|
|
4399
|
+
await chmod(join(bin, 'tmux'), 0o755);
|
|
4400
|
+
process.env.PATH = `${bin}:${previousPath ?? ''}`;
|
|
4401
|
+
const snapshot = async (root) => {
|
|
4402
|
+
const files = {};
|
|
4403
|
+
const visit = async (directory) => {
|
|
4404
|
+
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
4405
|
+
const path = join(directory, entry.name);
|
|
4406
|
+
if (entry.isDirectory())
|
|
4407
|
+
await visit(path);
|
|
4408
|
+
else
|
|
4409
|
+
files[relative(root, path)] = await readFile(path);
|
|
4410
|
+
}
|
|
4411
|
+
};
|
|
4412
|
+
await visit(root);
|
|
4413
|
+
return files;
|
|
4414
|
+
};
|
|
4415
|
+
const beforeGit = await snapshot(gitDir);
|
|
4416
|
+
const beforeWorktree = await snapshot(worktree);
|
|
4417
|
+
const beforeConfig = await readFile(join(stateDir, 'config.json'));
|
|
4418
|
+
const beforeManifest = await readFile(join(stateDir, 'manifest.v2.json'));
|
|
4419
|
+
const beforeIdentity = await readFile(join(stateDir, 'workers/worker-2/identity.json'));
|
|
4420
|
+
const result = recovery
|
|
4421
|
+
? await reconcileScaleDownCleanupDebt(team, repo, config)
|
|
4422
|
+
: await scaleDown(team, repo, { workerNames: ['worker-2'], force: false, drainTimeoutMs: 100 }, { OMX_TEAM_SCALING_ENABLED: '1' });
|
|
4423
|
+
const observed = JSON.stringify({
|
|
4424
|
+
result, members: (await readTeamConfig(team, repo))?.workers.map((worker) => worker.name),
|
|
4425
|
+
worktreeExists: existsSync(worktree), operationExists: existsSync(operationPath),
|
|
4426
|
+
tmuxCommands: existsSync(tmuxLog) ? await readFile(tmuxLog, 'utf8') : '',
|
|
4427
|
+
});
|
|
4428
|
+
assert.equal(result.ok, false, `unfinished ${operation} must veto removal: ${observed}`);
|
|
4429
|
+
if (!result.ok)
|
|
4430
|
+
assert.match(result.error, /scale_down_worktree_not_preserved:worker-2/, observed);
|
|
4431
|
+
assert.equal(existsSync(worktree), true);
|
|
4432
|
+
assert.deepEqual(await snapshot(gitDir), beforeGit, 'all per-worktree Git metadata must remain byte-identical');
|
|
4433
|
+
assert.deepEqual(await snapshot(worktree), beforeWorktree);
|
|
4434
|
+
assert.deepEqual(await readFile(join(stateDir, 'config.json')), beforeConfig);
|
|
4435
|
+
assert.deepEqual(await readFile(join(stateDir, 'manifest.v2.json')), beforeManifest);
|
|
4436
|
+
assert.deepEqual(await readFile(join(stateDir, 'workers/worker-2/identity.json')), beforeIdentity);
|
|
4437
|
+
assert.equal(git(['rev-parse', 'HEAD']), leaderHead);
|
|
4438
|
+
if (!isRebase)
|
|
4439
|
+
assert.equal(await readFile(join(gitDir, 'MERGE_MSG'), 'utf8'), mergeMessage);
|
|
4440
|
+
assert.equal(existsSync(tmuxLog), false, 'no pane probe or teardown is authorized');
|
|
4441
|
+
assert.equal(existsSync(debtPath), recovery);
|
|
4442
|
+
if (recovery) {
|
|
4443
|
+
const debt = JSON.parse(await readFile(debtPath, 'utf8'));
|
|
4444
|
+
assert.equal(debt.status, 'resource_cleanup_pending');
|
|
4445
|
+
assert.match(debt.reasons[0], /scale_down_worktree_not_preserved:worker-2/);
|
|
4446
|
+
delete debt.updated_at;
|
|
4447
|
+
delete debt.reasons;
|
|
4448
|
+
assert.deepEqual(debt, beforeDebt, 'cleanup authority and resource debt must be retained');
|
|
4449
|
+
}
|
|
4450
|
+
}
|
|
4451
|
+
finally {
|
|
4452
|
+
if (previousPath === undefined)
|
|
4453
|
+
delete process.env.PATH;
|
|
4454
|
+
else
|
|
4455
|
+
process.env.PATH = previousPath;
|
|
4456
|
+
await rm(repo, { recursive: true, force: true });
|
|
4457
|
+
}
|
|
4458
|
+
});
|
|
4459
|
+
}
|
|
4460
|
+
});
|
|
4461
|
+
describe('scale-down cleanup debt force policy', () => {
|
|
4462
|
+
for (const force of [undefined, false, true]) {
|
|
4463
|
+
it(`retains nonforce work after recovery unless force was explicitly recorded (${String(force)})`, async () => {
|
|
4464
|
+
const repo = await initRepo();
|
|
4465
|
+
try {
|
|
4466
|
+
const team = 'debt-force-policy';
|
|
4467
|
+
await initTeamState(team, 'task', 'executor', 2, repo);
|
|
4468
|
+
const config = await readTeamConfig(team, repo);
|
|
4469
|
+
assert.ok(config);
|
|
4470
|
+
const worktree = join(repo, '.omx', 'team', team, 'worktrees', 'worker-2');
|
|
4471
|
+
execFileSync('git', ['worktree', 'add', '--detach', worktree], { cwd: repo, stdio: 'ignore' });
|
|
4472
|
+
await writeFile(join(worktree, 'unsaved.txt'), 'recovery must preserve this');
|
|
4473
|
+
config.workers = config.workers.slice(0, 1);
|
|
4474
|
+
config.worker_count = 1;
|
|
4475
|
+
await saveTeamConfig(config, repo);
|
|
4476
|
+
const debtPath = join(repo, '.omx', 'state', 'team', team, '.scale-down-cleanup-debt.json');
|
|
4477
|
+
await writeFile(debtPath, JSON.stringify({
|
|
4478
|
+
schema_version: 1, operation: 'scale_down', status: 'resource_cleanup_pending',
|
|
4479
|
+
...(force === undefined ? {} : { force }),
|
|
4480
|
+
removed_worker_names: ['worker-2'], workers: [], unresolved_panes: [],
|
|
4481
|
+
resource_workers: [{ name: 'worker-2', worktree_path: worktree, worktree_repo_root: repo, worktree_created: true, worktree_detached: true }],
|
|
4482
|
+
}));
|
|
4483
|
+
const result = await reconcileScaleDownCleanupDebt(team, repo, config);
|
|
4484
|
+
if (force === true) {
|
|
4485
|
+
assert.deepEqual(result, { ok: true });
|
|
4486
|
+
assert.equal(existsSync(worktree), false);
|
|
4487
|
+
assert.equal(existsSync(debtPath), false);
|
|
4488
|
+
}
|
|
4489
|
+
else {
|
|
4490
|
+
assert.equal(result.ok, false);
|
|
4491
|
+
if (!result.ok)
|
|
4492
|
+
assert.match(result.error, /scale_down_worktree_not_preserved/);
|
|
4493
|
+
assert.equal(await readFile(join(worktree, 'unsaved.txt'), 'utf8'), 'recovery must preserve this');
|
|
4494
|
+
assert.equal(existsSync(debtPath), true);
|
|
4495
|
+
}
|
|
4496
|
+
}
|
|
4497
|
+
finally {
|
|
4498
|
+
await rm(repo, { recursive: true, force: true });
|
|
4499
|
+
}
|
|
4500
|
+
});
|
|
4501
|
+
}
|
|
4502
|
+
});
|
|
4178
4503
|
describe('forced scale-down worktree rollback', () => {
|
|
4179
4504
|
it('removes a busy worker with dirty files after explicit force authorization', async () => {
|
|
4180
4505
|
const repo = await initRepo();
|
|
@@ -4198,4 +4523,323 @@ describe('forced scale-down worktree rollback', () => {
|
|
|
4198
4523
|
}
|
|
4199
4524
|
});
|
|
4200
4525
|
});
|
|
4526
|
+
/**
|
|
4527
|
+
* Fake herdr binary answering the JSON-envelope CLI contract (success on
|
|
4528
|
+
* stdout, error envelope on stderr with exit 1). Every invocation is logged.
|
|
4529
|
+
* Tests pin it via HERDR_BIN_PATH (the trust-pin resolveHerdrBinary honors).
|
|
4530
|
+
*/
|
|
4531
|
+
async function writeFakeHerdrStub(fakeBinDir, herdrLogPath, options = {}) {
|
|
4532
|
+
const herdrStubPath = join(fakeBinDir, 'herdr');
|
|
4533
|
+
const agentStartBranch = options.failAgentStartCode
|
|
4534
|
+
? [
|
|
4535
|
+
`printf '%s' '{"error":{"code":"${options.failAgentStartCode}","message":"stub injected failure"}}' >&2`,
|
|
4536
|
+
'exit 1',
|
|
4537
|
+
].join('\n')
|
|
4538
|
+
: [
|
|
4539
|
+
`printf '%s\\n' '{"id":"cli:agent:start","result":{"agent":{"agent_status":"idle"}}}'`,
|
|
4540
|
+
'exit 0',
|
|
4541
|
+
].join('\n');
|
|
4542
|
+
const paneCloseBranch = options.paneCloseNotFound
|
|
4543
|
+
? [
|
|
4544
|
+
`printf '%s' '{"error":{"code":"pane_not_found","message":"pane already closed"}}' >&2`,
|
|
4545
|
+
'exit 1',
|
|
4546
|
+
].join('\n')
|
|
4547
|
+
: [
|
|
4548
|
+
`printf '%s\\n' '{"id":"cli:pane:close","result":{"type":"ok"}}'`,
|
|
4549
|
+
'exit 0',
|
|
4550
|
+
].join('\n');
|
|
4551
|
+
await writeFile(herdrStubPath, [
|
|
4552
|
+
'#!/bin/sh',
|
|
4553
|
+
`printf '%s\\n' "$*" >> "${herdrLogPath}"`,
|
|
4554
|
+
'cmd="${1:-}"',
|
|
4555
|
+
'sub="${2:-}"',
|
|
4556
|
+
'case "$cmd:$sub" in',
|
|
4557
|
+
' --version:)',
|
|
4558
|
+
' printf \'%s\\n\' "herdr 0.9.5-stub"',
|
|
4559
|
+
' exit 0',
|
|
4560
|
+
' ;;',
|
|
4561
|
+
' workspace:list)',
|
|
4562
|
+
` printf '%s\\n' '{"id":"cli:workspace:list","result":{"workspaces":[],"type":"workspace_list"}}'`,
|
|
4563
|
+
' exit 0',
|
|
4564
|
+
' ;;',
|
|
4565
|
+
' tab:create)',
|
|
4566
|
+
` printf '%s\\n' '{"id":"cli:tab:create","result":{"tab":{"tab_id":"w5:t9"},"root_pane":{"pane_id":"w5:p9"}}}'`,
|
|
4567
|
+
' exit 0',
|
|
4568
|
+
' ;;',
|
|
4569
|
+
' agent:start)',
|
|
4570
|
+
` ${agentStartBranch}`,
|
|
4571
|
+
' ;;',
|
|
4572
|
+
' agent:prompt|agent:get|agent:send-keys|agent:wait)',
|
|
4573
|
+
` printf '%s\\n' '{"id":"cli:agent:result","result":{"agent":{}}}'`,
|
|
4574
|
+
' exit 0',
|
|
4575
|
+
' ;;',
|
|
4576
|
+
' agent:list)',
|
|
4577
|
+
` printf '%s\\n' '{"id":"cli:agent:list","result":{"agents":${JSON.stringify((options.livePaneIds ?? []).map((pane_id) => ({ pane_id })))},"type":"agent_list"}}'`,
|
|
4578
|
+
' exit 0',
|
|
4579
|
+
' ;;',
|
|
4580
|
+
' pane:close)',
|
|
4581
|
+
` ${paneCloseBranch}`,
|
|
4582
|
+
' ;;',
|
|
4583
|
+
' pane:read)',
|
|
4584
|
+
' printf \'%s\\n\' "› "',
|
|
4585
|
+
' exit 0',
|
|
4586
|
+
' ;;',
|
|
4587
|
+
' *)',
|
|
4588
|
+
` printf '%s\\n' '{"id":"cli:unknown","result":{}}'`,
|
|
4589
|
+
' exit 0',
|
|
4590
|
+
' ;;',
|
|
4591
|
+
'esac',
|
|
4592
|
+
'',
|
|
4593
|
+
].join('\n'));
|
|
4594
|
+
execFileSync('sh', ['-n', herdrStubPath], { stdio: 'pipe' });
|
|
4595
|
+
await chmod(herdrStubPath, 0o755);
|
|
4596
|
+
await writeFile(herdrLogPath, '');
|
|
4597
|
+
return herdrStubPath;
|
|
4598
|
+
}
|
|
4599
|
+
async function readHerdrStubLog(herdrLogPath) {
|
|
4600
|
+
const content = await readFile(herdrLogPath, 'utf-8');
|
|
4601
|
+
const trimmed = content.trim();
|
|
4602
|
+
return trimmed === '' ? [] : trimmed.split('\n');
|
|
4603
|
+
}
|
|
4604
|
+
async function configureHerdrScaleTeam(teamName, cwd, workerPaneIds, dispatchMode = 'transport_direct') {
|
|
4605
|
+
const config = await readTeamConfig(teamName, cwd);
|
|
4606
|
+
assert.ok(config);
|
|
4607
|
+
if (!config)
|
|
4608
|
+
throw new Error(`missing team config for ${teamName}`);
|
|
4609
|
+
config.worker_launch_mode = 'herdr';
|
|
4610
|
+
config.tmux_session = `herdr-${teamName}`;
|
|
4611
|
+
config.leader_pane_id = null;
|
|
4612
|
+
config.hud_pane_id = null;
|
|
4613
|
+
config.tmux_pane_owner_id = undefined;
|
|
4614
|
+
config.workers.forEach((worker, index) => {
|
|
4615
|
+
worker.pane_id = workerPaneIds[index];
|
|
4616
|
+
worker.pid = undefined;
|
|
4617
|
+
});
|
|
4618
|
+
await saveTeamConfig(config, cwd);
|
|
4619
|
+
const manifestPath = join(cwd, '.omx', 'state', 'team', teamName, 'manifest.v2.json');
|
|
4620
|
+
if (!existsSync(manifestPath)) {
|
|
4621
|
+
await mkdir(join(cwd, '.omx', 'state', 'team', teamName), { recursive: true });
|
|
4622
|
+
await writeFile(manifestPath, `${JSON.stringify({ version: 2, policy: {} }, null, 2)}\n`);
|
|
4623
|
+
}
|
|
4624
|
+
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
4625
|
+
manifest.policy = {
|
|
4626
|
+
...(manifest.policy ?? {}),
|
|
4627
|
+
dispatch_mode: dispatchMode,
|
|
4628
|
+
};
|
|
4629
|
+
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
4630
|
+
}
|
|
4631
|
+
describe('scaling on herdr', () => {
|
|
4632
|
+
it('scaleUp spawns a herdr tab+agent in the team workspace and dispatches via agent prompt', async () => {
|
|
4633
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-up-'));
|
|
4634
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-up-bin-'));
|
|
4635
|
+
const herdrLogPath = join(fakeBinDir, 'herdr.log');
|
|
4636
|
+
try {
|
|
4637
|
+
const herdrBin = await writeFakeHerdrStub(fakeBinDir, herdrLogPath);
|
|
4638
|
+
await initTeamState('scale-herdr-up', 'task', 'executor', 1, cwd, DEFAULT_MAX_WORKERS, {
|
|
4639
|
+
OMX_TEAM_WORKER_LAUNCH_MODE: 'herdr',
|
|
4640
|
+
});
|
|
4641
|
+
await configureHerdrScaleTeam('scale-herdr-up', cwd, ['w5:p2']);
|
|
4642
|
+
const result = await scaleUp('scale-herdr-up', 1, 'executor', [{ subject: 'new task', description: 'new task', owner: 'worker-2' }], cwd, {
|
|
4643
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
4644
|
+
OMX_MULTIPLEXER: 'herdr',
|
|
4645
|
+
HERDR_BIN_PATH: herdrBin,
|
|
4646
|
+
});
|
|
4647
|
+
assert.equal(result.ok, true, JSON.stringify(result));
|
|
4648
|
+
if (!result.ok)
|
|
4649
|
+
return;
|
|
4650
|
+
assert.equal(result.newWorkerCount, 2);
|
|
4651
|
+
assert.equal(result.nextWorkerIndex, 3);
|
|
4652
|
+
assert.equal(result.addedWorkers[0]?.pane_id, 'w5:p9');
|
|
4653
|
+
assert.equal(result.addedWorkers[0]?.pid, undefined);
|
|
4654
|
+
const config = await readTeamConfig('scale-herdr-up', cwd);
|
|
4655
|
+
assert.ok(config);
|
|
4656
|
+
assert.equal(config?.workers.length, 2);
|
|
4657
|
+
assert.equal(config?.workers[1]?.pane_id, 'w5:p9');
|
|
4658
|
+
assert.equal(config?.workers[1]?.pid, undefined);
|
|
4659
|
+
assert.equal(config?.next_worker_index, 3);
|
|
4660
|
+
const identity = JSON.parse(await readFile(join(cwd, '.omx', 'state', 'team', 'scale-herdr-up', 'workers', 'worker-2', 'identity.json'), 'utf-8'));
|
|
4661
|
+
assert.equal(identity.pane_id, 'w5:p9');
|
|
4662
|
+
const commands = await readHerdrStubLog(herdrLogPath);
|
|
4663
|
+
const tabCreate = commands.find((command) => command.startsWith('tab create '));
|
|
4664
|
+
assert.ok(tabCreate, 'worker spawns through herdr tab create');
|
|
4665
|
+
assert.ok(tabCreate.includes('--workspace w5'), 'tab create pins the derived team workspace');
|
|
4666
|
+
assert.ok(tabCreate.includes('--cwd '), 'tab create pins the worker cwd');
|
|
4667
|
+
assert.ok(tabCreate.includes('--env OMX_TEAM_STATE_ROOT='), 'worker env injected via tab create --env');
|
|
4668
|
+
const agentStart = commands.find((command) => command.startsWith('agent start '));
|
|
4669
|
+
assert.ok(agentStart, 'worker agent started');
|
|
4670
|
+
assert.ok(agentStart.includes('--kind codex'), 'agent kind derived from the launch plan');
|
|
4671
|
+
assert.ok(agentStart.includes('--pane w5:p9'), 'agent pinned to the created pane');
|
|
4672
|
+
const agentPrompt = commands.find((command) => command.startsWith('agent prompt w5:p9 '));
|
|
4673
|
+
assert.ok(agentPrompt, 'bootstrap trigger dispatched through herdr agent prompt');
|
|
4674
|
+
// No tmux authority was consulted anywhere in the herdr path.
|
|
4675
|
+
assert.equal(commands.some((command) => command.startsWith('agent list')), true, 'server liveness checked via agent list');
|
|
4676
|
+
}
|
|
4677
|
+
finally {
|
|
4678
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4679
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
4680
|
+
}
|
|
4681
|
+
});
|
|
4682
|
+
it('scaleUp on herdr rolls back the created pane, task, and worker state when the agent start fails', async () => {
|
|
4683
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-fail-'));
|
|
4684
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-fail-bin-'));
|
|
4685
|
+
const herdrLogPath = join(fakeBinDir, 'herdr.log');
|
|
4686
|
+
try {
|
|
4687
|
+
const herdrBin = await writeFakeHerdrStub(fakeBinDir, herdrLogPath, { failAgentStartCode: 'agent_not_found' });
|
|
4688
|
+
await initTeamState('scale-herdr-fail', 'task', 'executor', 1, cwd, DEFAULT_MAX_WORKERS, {
|
|
4689
|
+
OMX_TEAM_WORKER_LAUNCH_MODE: 'herdr',
|
|
4690
|
+
});
|
|
4691
|
+
await configureHerdrScaleTeam('scale-herdr-fail', cwd, ['w5:p2']);
|
|
4692
|
+
const result = await scaleUp('scale-herdr-fail', 1, 'executor', [{ subject: 'new task', description: 'new task', owner: 'worker-2' }], cwd, {
|
|
4693
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
4694
|
+
OMX_MULTIPLEXER: 'herdr',
|
|
4695
|
+
HERDR_BIN_PATH: herdrBin,
|
|
4696
|
+
});
|
|
4697
|
+
assert.equal(result.ok, false);
|
|
4698
|
+
if (result.ok)
|
|
4699
|
+
return;
|
|
4700
|
+
assert.match(result.error, /scale_up_herdr_spawn_failed:worker-2:herdr_agent_not_found/);
|
|
4701
|
+
const commands = await readHerdrStubLog(herdrLogPath);
|
|
4702
|
+
assert.ok(commands.some((command) => command === 'pane close w5:p9'), 'the spawn helper closes its own pane on failure');
|
|
4703
|
+
const config = await readTeamConfig('scale-herdr-fail', cwd);
|
|
4704
|
+
assert.ok(config);
|
|
4705
|
+
assert.equal(config?.workers.length, 1);
|
|
4706
|
+
assert.equal(config?.workers[0]?.pane_id, 'w5:p2');
|
|
4707
|
+
assert.equal(config?.next_worker_index, 2);
|
|
4708
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-herdr-fail', 'workers', 'worker-2')), false);
|
|
4709
|
+
assert.deepEqual(await readScaleUpTaskPayloads('scale-herdr-fail', cwd), []);
|
|
4710
|
+
}
|
|
4711
|
+
finally {
|
|
4712
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4713
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
4714
|
+
}
|
|
4715
|
+
});
|
|
4716
|
+
it('scaleUp returns a clear error when the herdr backend is selected but the binary is missing', async () => {
|
|
4717
|
+
const result = await scaleUp('scale-herdr-missing', 1, 'executor', [], '/tmp', {
|
|
4718
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
4719
|
+
OMX_MULTIPLEXER: 'herdr',
|
|
4720
|
+
PATH: '',
|
|
4721
|
+
});
|
|
4722
|
+
assert.equal(result.ok, false);
|
|
4723
|
+
if (result.ok)
|
|
4724
|
+
return;
|
|
4725
|
+
assert.match(result.error, /herdr binary is not callable/);
|
|
4726
|
+
});
|
|
4727
|
+
it('scaleDown preserves non-idle herdr workers and retires an idle worker', async () => {
|
|
4728
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-down-'));
|
|
4729
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-down-bin-'));
|
|
4730
|
+
const herdrLogPath = join(fakeBinDir, 'herdr.log');
|
|
4731
|
+
const previousHerdrBinPath = process.env.HERDR_BIN_PATH;
|
|
4732
|
+
try {
|
|
4733
|
+
const herdrBin = await writeFakeHerdrStub(fakeBinDir, herdrLogPath);
|
|
4734
|
+
// The final cleanup-debt reconcile resolves the binary from the ambient
|
|
4735
|
+
// env, so pin the stub there too (not only in the scaleDown env param).
|
|
4736
|
+
process.env.HERDR_BIN_PATH = herdrBin;
|
|
4737
|
+
await initTeamState('scale-herdr-down', 'task', 'executor', 2, cwd, DEFAULT_MAX_WORKERS, {
|
|
4738
|
+
OMX_TEAM_WORKER_LAUNCH_MODE: 'herdr',
|
|
4739
|
+
});
|
|
4740
|
+
await configureHerdrScaleTeam('scale-herdr-down', cwd, ['w5:p2', 'w5:p3']);
|
|
4741
|
+
const statusPath = join(cwd, '.omx/state/team/scale-herdr-down/workers/worker-2/status.json');
|
|
4742
|
+
const configBefore = await readTeamConfig('scale-herdr-down', cwd);
|
|
4743
|
+
for (const state of ['working', 'draining', 'unknown']) {
|
|
4744
|
+
await writeWorkerStatus('scale-herdr-down', 'worker-2', { state, updated_at: new Date().toISOString() }, cwd);
|
|
4745
|
+
const statusBefore = await readFile(statusPath, 'utf8');
|
|
4746
|
+
const blocked = await scaleDown('scale-herdr-down', cwd, { workerNames: ['worker-2'], drainTimeoutMs: 0 }, { OMX_TEAM_SCALING_ENABLED: '1', OMX_MULTIPLEXER: 'herdr', HERDR_BIN_PATH: herdrBin });
|
|
4747
|
+
assert.deepEqual(blocked, { ok: false, error: 'scale_down_drain_timeout' });
|
|
4748
|
+
assert.deepEqual(await readTeamConfig('scale-herdr-down', cwd), configBefore);
|
|
4749
|
+
assert.equal(await readFile(statusPath, 'utf8'), statusBefore);
|
|
4750
|
+
const commands = await readHerdrStubLog(herdrLogPath);
|
|
4751
|
+
assert.equal(commands.some((command) => command.startsWith('pane close ')), false);
|
|
4752
|
+
assert.equal(commands.some((command) => command.startsWith('agent list')), true, 'drain checks use herdr liveness');
|
|
4753
|
+
}
|
|
4754
|
+
await writeFakeHerdrStub(fakeBinDir, herdrLogPath, { livePaneIds: ['w5:p3'] });
|
|
4755
|
+
await writeWorkerStatus('scale-herdr-down', 'worker-2', { state: 'working', updated_at: new Date().toISOString() }, cwd);
|
|
4756
|
+
const removal = scaleDown('scale-herdr-down', cwd, { workerNames: ['worker-2'], drainTimeoutMs: 5_000 }, {
|
|
4757
|
+
OMX_TEAM_SCALING_ENABLED: '1',
|
|
4758
|
+
OMX_MULTIPLEXER: 'herdr',
|
|
4759
|
+
HERDR_BIN_PATH: herdrBin,
|
|
4760
|
+
});
|
|
4761
|
+
for (let attempt = 0; attempt < 100; attempt++) {
|
|
4762
|
+
if ((await readHerdrStubLog(herdrLogPath)).some((command) => command.startsWith('agent list')))
|
|
4763
|
+
break;
|
|
4764
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
4765
|
+
}
|
|
4766
|
+
const waitingCommands = await readHerdrStubLog(herdrLogPath);
|
|
4767
|
+
assert.equal(waitingCommands.some((command) => command.startsWith('agent list')), true);
|
|
4768
|
+
assert.equal(waitingCommands.some((command) => command.startsWith('pane close ')), false);
|
|
4769
|
+
assert.equal((await readWorkerStatus('scale-herdr-down', 'worker-2', cwd)).state, 'working');
|
|
4770
|
+
await writeWorkerStatus('scale-herdr-down', 'worker-2', { state: 'idle', updated_at: new Date().toISOString() }, cwd);
|
|
4771
|
+
const result = await removal;
|
|
4772
|
+
assert.deepEqual(result, { ok: true, removedWorkers: ['worker-2'], newWorkerCount: 1 });
|
|
4773
|
+
const commands = await readHerdrStubLog(herdrLogPath);
|
|
4774
|
+
assert.ok(commands.some((command) => command === 'pane close w5:p3'), 'the removed worker pane is closed through herdr');
|
|
4775
|
+
const config = await readTeamConfig('scale-herdr-down', cwd);
|
|
4776
|
+
assert.ok(config);
|
|
4777
|
+
assert.equal(config?.workers.length, 1);
|
|
4778
|
+
assert.equal(config?.workers[0]?.pane_id, 'w5:p2');
|
|
4779
|
+
assert.equal(existsSync(join(cwd, '.omx', 'state', 'team', 'scale-herdr-down', '.scale-down-cleanup-debt.json')), false);
|
|
4780
|
+
}
|
|
4781
|
+
finally {
|
|
4782
|
+
if (typeof previousHerdrBinPath === 'string')
|
|
4783
|
+
process.env.HERDR_BIN_PATH = previousHerdrBinPath;
|
|
4784
|
+
else
|
|
4785
|
+
delete process.env.HERDR_BIN_PATH;
|
|
4786
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4787
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
4788
|
+
}
|
|
4789
|
+
});
|
|
4790
|
+
it('reconcileScaleDownCleanupDebt resolves herdr pane debt, treating pane_not_found as converged', async () => {
|
|
4791
|
+
const cwd = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-debt-'));
|
|
4792
|
+
const fakeBinDir = await mkdtemp(join(tmpdir(), 'omx-scale-herdr-debt-bin-'));
|
|
4793
|
+
const previousHerdrBinPath = process.env.HERDR_BIN_PATH;
|
|
4794
|
+
try {
|
|
4795
|
+
await initTeamState('scale-herdr-debt', 'task', 'executor', 1, cwd, DEFAULT_MAX_WORKERS, {
|
|
4796
|
+
OMX_TEAM_WORKER_LAUNCH_MODE: 'herdr',
|
|
4797
|
+
});
|
|
4798
|
+
await configureHerdrScaleTeam('scale-herdr-debt', cwd, ['w5:p2']);
|
|
4799
|
+
const config = await readTeamConfig('scale-herdr-debt', cwd);
|
|
4800
|
+
assert.ok(config);
|
|
4801
|
+
const debtPath = join(cwd, '.omx', 'state', 'team', 'scale-herdr-debt', '.scale-down-cleanup-debt.json');
|
|
4802
|
+
const debt = {
|
|
4803
|
+
schema_version: 1,
|
|
4804
|
+
operation: 'scale_down',
|
|
4805
|
+
status: 'unresolved',
|
|
4806
|
+
created_at: new Date().toISOString(),
|
|
4807
|
+
removed_worker_names: ['worker-2'],
|
|
4808
|
+
workers: [{ name: 'worker-2', index: 2, pane_id: 'w5:p9', pid: null }],
|
|
4809
|
+
resource_workers: [{ name: 'worker-2' }],
|
|
4810
|
+
};
|
|
4811
|
+
await writeFile(debtPath, JSON.stringify(debt, null, 2));
|
|
4812
|
+
// A herdr pane recorded with a PID is malformed: herdr panes carry none.
|
|
4813
|
+
const malformedBin = await writeFakeHerdrStub(fakeBinDir, join(fakeBinDir, 'malformed.log'));
|
|
4814
|
+
process.env.HERDR_BIN_PATH = malformedBin;
|
|
4815
|
+
await writeFile(debtPath, JSON.stringify({ ...debt, workers: [{ name: 'worker-2', index: 2, pane_id: 'w5:p9', pid: 4242 }] }, null, 2));
|
|
4816
|
+
const malformed = await reconcileScaleDownCleanupDebt('scale-herdr-debt', cwd, config);
|
|
4817
|
+
assert.equal(malformed.ok, false);
|
|
4818
|
+
if (!malformed.ok)
|
|
4819
|
+
assert.match(malformed.error, /scale_down_cleanup_debt_malformed/);
|
|
4820
|
+
// Still-open pane: close it and converge.
|
|
4821
|
+
const openBin = await writeFakeHerdrStub(fakeBinDir, join(fakeBinDir, 'open.log'), { paneCloseNotFound: false });
|
|
4822
|
+
process.env.HERDR_BIN_PATH = openBin;
|
|
4823
|
+
await writeFile(debtPath, JSON.stringify(debt, null, 2));
|
|
4824
|
+
const openOutcome = await reconcileScaleDownCleanupDebt('scale-herdr-debt', cwd, config);
|
|
4825
|
+
assert.deepEqual(openOutcome, { ok: true });
|
|
4826
|
+
assert.equal(existsSync(debtPath), false, 'converged debt is removed');
|
|
4827
|
+
// Already-closed pane: pane_not_found answers prove convergence.
|
|
4828
|
+
const goneBin = await writeFakeHerdrStub(fakeBinDir, join(fakeBinDir, 'gone.log'), { paneCloseNotFound: true });
|
|
4829
|
+
process.env.HERDR_BIN_PATH = goneBin;
|
|
4830
|
+
await writeFile(debtPath, JSON.stringify(debt, null, 2));
|
|
4831
|
+
const goneOutcome = await reconcileScaleDownCleanupDebt('scale-herdr-debt', cwd, config);
|
|
4832
|
+
assert.deepEqual(goneOutcome, { ok: true });
|
|
4833
|
+
assert.equal(existsSync(debtPath), false);
|
|
4834
|
+
}
|
|
4835
|
+
finally {
|
|
4836
|
+
if (typeof previousHerdrBinPath === 'string')
|
|
4837
|
+
process.env.HERDR_BIN_PATH = previousHerdrBinPath;
|
|
4838
|
+
else
|
|
4839
|
+
delete process.env.HERDR_BIN_PATH;
|
|
4840
|
+
await rm(cwd, { recursive: true, force: true });
|
|
4841
|
+
await rm(fakeBinDir, { recursive: true, force: true });
|
|
4842
|
+
}
|
|
4843
|
+
});
|
|
4844
|
+
});
|
|
4201
4845
|
//# sourceMappingURL=scaling.test.js.map
|