oh-my-codex 0.21.48 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +6 -6
- package/Cargo.toml +1 -1
- package/README.md +15 -0
- package/crates/omx-mux/src/herdr.rs +481 -0
- package/crates/omx-mux/src/lib.rs +5 -0
- package/dist/cli/__tests__/plugin-cache-live-pin-3790.test.js +104 -3
- package/dist/cli/__tests__/plugin-cache-live-pin-3790.test.js.map +1 -1
- package/dist/cli/__tests__/question.test.js +33 -0
- package/dist/cli/__tests__/question.test.js.map +1 -1
- package/dist/cli/__tests__/session-compare.test.d.ts +2 -0
- package/dist/cli/__tests__/session-compare.test.d.ts.map +1 -0
- package/dist/cli/__tests__/session-compare.test.js +183 -0
- package/dist/cli/__tests__/session-compare.test.js.map +1 -0
- package/dist/cli/__tests__/team.test.js +8 -0
- package/dist/cli/__tests__/team.test.js.map +1 -1
- package/dist/cli/plugin-marketplace.d.ts.map +1 -1
- package/dist/cli/plugin-marketplace.js +11 -4
- package/dist/cli/plugin-marketplace.js.map +1 -1
- package/dist/cli/question.d.ts.map +1 -1
- package/dist/cli/question.js +8 -1
- package/dist/cli/question.js.map +1 -1
- package/dist/cli/session-compare.d.ts +11 -0
- package/dist/cli/session-compare.d.ts.map +1 -0
- package/dist/cli/session-compare.js +172 -0
- package/dist/cli/session-compare.js.map +1 -0
- package/dist/cli/session-search.d.ts.map +1 -1
- package/dist/cli/session-search.js +7 -0
- package/dist/cli/session-search.js.map +1 -1
- package/dist/cli/team.d.ts.map +1 -1
- package/dist/cli/team.js +2 -0
- package/dist/cli/team.js.map +1 -1
- package/dist/config/__tests__/multiplexer.test.d.ts +2 -0
- package/dist/config/__tests__/multiplexer.test.d.ts.map +1 -0
- package/dist/config/__tests__/multiplexer.test.js +118 -0
- package/dist/config/__tests__/multiplexer.test.js.map +1 -0
- package/dist/config/multiplexer.d.ts +29 -0
- package/dist/config/multiplexer.d.ts.map +1 -0
- package/dist/config/multiplexer.js +91 -0
- package/dist/config/multiplexer.js.map +1 -0
- package/dist/question/__tests__/ui.test.js +40 -1
- package/dist/question/__tests__/ui.test.js.map +1 -1
- package/dist/question/ui.d.ts.map +1 -1
- package/dist/question/ui.js +37 -7
- package/dist/question/ui.js.map +1 -1
- package/dist/scripts/__tests__/codex-native-hook.test.js +281 -1
- package/dist/scripts/__tests__/codex-native-hook.test.js.map +1 -1
- package/dist/scripts/codex-native-hook.d.ts.map +1 -1
- package/dist/scripts/codex-native-hook.js +8 -2
- package/dist/scripts/codex-native-hook.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker-lease.d.ts +15 -0
- package/dist/scripts/notify-hook/team-worker-lease.d.ts.map +1 -0
- package/dist/scripts/notify-hook/team-worker-lease.js +63 -0
- package/dist/scripts/notify-hook/team-worker-lease.js.map +1 -0
- package/dist/scripts/notify-hook/team-worker-posttooluse.d.ts +1 -0
- package/dist/scripts/notify-hook/team-worker-posttooluse.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker-posttooluse.js +16 -5
- package/dist/scripts/notify-hook/team-worker-posttooluse.js.map +1 -1
- package/dist/scripts/notify-hook/team-worker.d.ts +12 -1
- package/dist/scripts/notify-hook/team-worker.d.ts.map +1 -1
- package/dist/scripts/notify-hook/team-worker.js +16 -4
- package/dist/scripts/notify-hook/team-worker.js.map +1 -1
- package/dist/session-history/__tests__/compare.test.d.ts +2 -0
- package/dist/session-history/__tests__/compare.test.d.ts.map +1 -0
- package/dist/session-history/__tests__/compare.test.js +257 -0
- package/dist/session-history/__tests__/compare.test.js.map +1 -0
- package/dist/session-history/compare.d.ts +74 -0
- package/dist/session-history/compare.d.ts.map +1 -0
- package/dist/session-history/compare.js +165 -0
- package/dist/session-history/compare.js.map +1 -0
- package/dist/team/__tests__/api-interop.test.js +3 -3
- package/dist/team/__tests__/completion-evidence.test.js +31 -1
- package/dist/team/__tests__/completion-evidence.test.js.map +1 -1
- package/dist/team/__tests__/cross-rebase-smoke.test.js +263 -31
- package/dist/team/__tests__/cross-rebase-smoke.test.js.map +1 -1
- package/dist/team/__tests__/herdr-session-realherdr.test.d.ts +2 -0
- package/dist/team/__tests__/herdr-session-realherdr.test.d.ts.map +1 -0
- package/dist/team/__tests__/herdr-session-realherdr.test.js +68 -0
- package/dist/team/__tests__/herdr-session-realherdr.test.js.map +1 -0
- package/dist/team/__tests__/herdr-session.test.d.ts +2 -0
- package/dist/team/__tests__/herdr-session.test.d.ts.map +1 -0
- package/dist/team/__tests__/herdr-session.test.js +228 -0
- package/dist/team/__tests__/herdr-session.test.js.map +1 -0
- package/dist/team/__tests__/hook-primary-e2e-contract.test.js +9 -3
- package/dist/team/__tests__/hook-primary-e2e-contract.test.js.map +1 -1
- package/dist/team/__tests__/idle-nudge.test.js +84 -0
- package/dist/team/__tests__/idle-nudge.test.js.map +1 -1
- package/dist/team/__tests__/mcp-comm.test.js +25 -1
- package/dist/team/__tests__/mcp-comm.test.js.map +1 -1
- package/dist/team/__tests__/rebalance-policy.test.js +27 -9
- package/dist/team/__tests__/rebalance-policy.test.js.map +1 -1
- package/dist/team/__tests__/runtime-delivery.test.d.ts +2 -0
- package/dist/team/__tests__/runtime-delivery.test.d.ts.map +1 -0
- package/dist/team/__tests__/runtime-delivery.test.js +570 -0
- package/dist/team/__tests__/runtime-delivery.test.js.map +1 -0
- package/dist/team/__tests__/runtime.test.js +938 -70
- package/dist/team/__tests__/runtime.test.js.map +1 -1
- package/dist/team/__tests__/scaling.test.js +645 -1
- package/dist/team/__tests__/scaling.test.js.map +1 -1
- package/dist/team/__tests__/state-renewal.test.d.ts +2 -0
- package/dist/team/__tests__/state-renewal.test.d.ts.map +1 -0
- package/dist/team/__tests__/state-renewal.test.js +555 -0
- package/dist/team/__tests__/state-renewal.test.js.map +1 -0
- package/dist/team/__tests__/state.test.js +121 -2
- package/dist/team/__tests__/state.test.js.map +1 -1
- package/dist/team/__tests__/task-lease.test.d.ts +2 -0
- package/dist/team/__tests__/task-lease.test.d.ts.map +1 -0
- package/dist/team/__tests__/task-lease.test.js +179 -0
- package/dist/team/__tests__/task-lease.test.js.map +1 -0
- package/dist/team/__tests__/team-ops-contract.test.js +1 -0
- package/dist/team/__tests__/team-ops-contract.test.js.map +1 -1
- package/dist/team/__tests__/worker-bootstrap.test.js +2 -0
- package/dist/team/__tests__/worker-bootstrap.test.js.map +1 -1
- package/dist/team/api-interop.d.ts +1 -1
- package/dist/team/api-interop.d.ts.map +1 -1
- package/dist/team/api-interop.js +13 -1
- package/dist/team/api-interop.js.map +1 -1
- package/dist/team/completion-evidence.d.ts +1 -1
- package/dist/team/completion-evidence.d.ts.map +1 -1
- package/dist/team/completion-evidence.js +9 -0
- package/dist/team/completion-evidence.js.map +1 -1
- package/dist/team/goal-workflow.js +1 -1
- package/dist/team/herdr-session.d.ts +234 -0
- package/dist/team/herdr-session.d.ts.map +1 -0
- package/dist/team/herdr-session.js +489 -0
- package/dist/team/herdr-session.js.map +1 -0
- package/dist/team/idle-nudge.d.ts +16 -4
- package/dist/team/idle-nudge.d.ts.map +1 -1
- package/dist/team/idle-nudge.js +51 -19
- package/dist/team/idle-nudge.js.map +1 -1
- package/dist/team/mcp-comm.d.ts +6 -4
- package/dist/team/mcp-comm.d.ts.map +1 -1
- package/dist/team/mcp-comm.js +4 -0
- package/dist/team/mcp-comm.js.map +1 -1
- package/dist/team/rebalance-policy.d.ts.map +1 -1
- package/dist/team/rebalance-policy.js +5 -2
- package/dist/team/rebalance-policy.js.map +1 -1
- package/dist/team/runtime.d.ts.map +1 -1
- package/dist/team/runtime.js +463 -166
- package/dist/team/runtime.js.map +1 -1
- package/dist/team/scaling.d.ts +3 -3
- package/dist/team/scaling.d.ts.map +1 -1
- package/dist/team/scaling.js +394 -200
- package/dist/team/scaling.js.map +1 -1
- package/dist/team/state/monitor.d.ts +3 -0
- package/dist/team/state/monitor.d.ts.map +1 -1
- package/dist/team/state/monitor.js +3 -0
- package/dist/team/state/monitor.js.map +1 -1
- package/dist/team/state/tasks.d.ts +4 -1
- package/dist/team/state/tasks.d.ts.map +1 -1
- package/dist/team/state/tasks.js +36 -5
- package/dist/team/state/tasks.js.map +1 -1
- package/dist/team/state/types.d.ts +9 -0
- package/dist/team/state/types.d.ts.map +1 -1
- package/dist/team/state/types.js.map +1 -1
- package/dist/team/state.d.ts +15 -2
- package/dist/team/state.d.ts.map +1 -1
- package/dist/team/state.js +73 -5
- package/dist/team/state.js.map +1 -1
- package/dist/team/task-lease.d.ts +4 -0
- package/dist/team/task-lease.d.ts.map +1 -0
- package/dist/team/task-lease.js +74 -0
- package/dist/team/task-lease.js.map +1 -0
- package/dist/team/team-ops.d.ts +2 -1
- package/dist/team/team-ops.d.ts.map +1 -1
- package/dist/team/team-ops.js +1 -0
- package/dist/team/team-ops.js.map +1 -1
- package/dist/team/tmux-session.d.ts +1 -1
- package/dist/team/tmux-session.d.ts.map +1 -1
- package/dist/team/tmux-session.js +10 -2
- package/dist/team/tmux-session.js.map +1 -1
- package/dist/team/worker-bootstrap.d.ts.map +1 -1
- package/dist/team/worker-bootstrap.js +16 -3
- package/dist/team/worker-bootstrap.js.map +1 -1
- package/package.json +1 -1
- package/plugins/oh-my-codex/.codex-plugin/plugin.json +1 -1
- package/plugins/oh-my-codex/hooks/codex-native-hook.mjs +4 -1
- package/plugins/oh-my-codex/skills/worker/SKILL.md +7 -1
- package/skills/worker/SKILL.md +7 -1
- package/src/scripts/__tests__/codex-native-hook.test.ts +279 -0
- package/src/scripts/codex-native-hook.ts +8 -2
- package/src/scripts/notify-hook/team-worker-lease.ts +69 -0
- package/src/scripts/notify-hook/team-worker-posttooluse.ts +16 -4
- package/src/scripts/notify-hook/team-worker.ts +15 -4
package/dist/team/scaling.js
CHANGED
|
@@ -7,16 +7,18 @@
|
|
|
7
7
|
* Key design decisions:
|
|
8
8
|
* - Monotonic worker index counter (next_worker_index in config) ensures unique names
|
|
9
9
|
* - File-based scaling lock prevents concurrent scale operations
|
|
10
|
-
* -
|
|
10
|
+
* - Worker-published idle/done status for graceful scale_down
|
|
11
11
|
*/
|
|
12
12
|
import { randomUUID } from 'crypto';
|
|
13
13
|
import { dirname, isAbsolute, join, relative, resolve, sep } from 'path';
|
|
14
14
|
import { mkdir, readFile, realpath, rm, writeFile } from 'fs/promises';
|
|
15
15
|
import { existsSync } from 'fs';
|
|
16
|
-
import {
|
|
16
|
+
import { readMultiplexerConfig } from '../config/multiplexer.js';
|
|
17
|
+
import { HerdrError, ensureHerdrServer, herdrWorkspaceIdFromPaneId, isHerdrBinaryCallable, isHerdrPaneId, isHerdrWorkerAlive, resolveHerdrBinary, spawnOneHerdrWorker, submitToHerdrWorker, teardownHerdrWorker, } from './herdr-session.js';
|
|
18
|
+
import { sanitizeTeamName, isTmuxAvailable, waitForWorkerReady, dismissTrustPromptIfPresent, isWorkerAlive, buildWorkerProcessLaunchSpec, buildWorkerStartupCommand, trustWorkerMiseConfigIfAvailable, writeWorkerStartupScriptCommand, resolveTeamWorkerCliForResolvedLaunchArgs, assertTeamWorkerCliPolicyCompatibility, readPaneTeamOwnerTagResult, teardownWorkerPanes, } from './tmux-session.js';
|
|
17
19
|
import { readExactPaneProofSync } from './exact-pane.js';
|
|
18
20
|
import { spawnSync } from 'child_process';
|
|
19
|
-
import { teamReadConfig as readTeamConfig, teamSaveConfig as saveTeamConfig, teamWriteWorkerIdentity as writeWorkerIdentity, teamReadManifest as readTeamManifestV2, teamNormalizePolicy as normalizeTeamPolicy, teamReadWorkerStatus as readWorkerStatus,
|
|
21
|
+
import { teamReadConfig as readTeamConfig, teamSaveConfig as saveTeamConfig, teamWriteWorkerIdentity as writeWorkerIdentity, teamReadManifest as readTeamManifestV2, teamNormalizePolicy as normalizeTeamPolicy, teamReadWorkerStatus as readWorkerStatus, teamWithScalingLock as withScalingLock, teamWithTaskMembershipBarrier as withTaskMembershipBarrier, recoverTeamMembershipTaskTransaction, commitTeamMembershipTaskTransaction, finalizeTeamMembershipTaskTransaction, writeAtomic, teamRemoveDurableFile as removeDurableFile, teamAppendEvent as appendTeamEvent, teamCreateTask as createStateTask, teamListTasks as listTasks, teamReadTask as readTask, teamMarkDispatchRequestNotified as markDispatchRequestNotified, teamReadDispatchRequest as readDispatchRequest, teamTransitionDispatchRequest as transitionDispatchRequest, teamRemoveDispatchRequestsForWorkers as removeDispatchRequestsForWorkers, } from './team-ops.js';
|
|
20
22
|
import { queueInboxInstruction, waitForDispatchReceipt, } from './mcp-comm.js';
|
|
21
23
|
import { generateInitialInbox, buildTriggerDirective, writeWorkerRoleInstructionsFile, writeWorkerWorktreeRootAgentsFile, removeWorkerWorktreeRootAgentsFile, } from './worker-bootstrap.js';
|
|
22
24
|
import { buildTeamWorkerGoalInstruction } from './goal-workflow.js';
|
|
@@ -357,11 +359,22 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
357
359
|
if (!Number.isInteger(count) || count < 1) {
|
|
358
360
|
return { ok: false, error: `count must be a positive integer (got ${count})` };
|
|
359
361
|
}
|
|
360
|
-
if (!isTmuxAvailable()) {
|
|
361
|
-
return { ok: false, error: 'tmux is not available' };
|
|
362
|
-
}
|
|
363
362
|
const sanitized = sanitizeTeamName(teamName);
|
|
364
363
|
const leaderCwd = resolve(cwd);
|
|
364
|
+
// Try to read team config early to use its persisted backend instead of env backend.
|
|
365
|
+
// If team doesn't exist yet, we'll validate backends based on env and fail later with
|
|
366
|
+
// a clearer "Team not found" message in the locking phase.
|
|
367
|
+
const existingConfig = await readTeamConfig(sanitized, leaderCwd).catch(() => null);
|
|
368
|
+
const backend = existingConfig?.worker_launch_mode === 'herdr' ? 'herdr' : readMultiplexerConfig({ env }).backend;
|
|
369
|
+
if (backend === 'herdr' && !(await isHerdrBinaryCallable(resolveHerdrBinary(env), env))) {
|
|
370
|
+
return {
|
|
371
|
+
ok: false,
|
|
372
|
+
error: 'herdr backend is selected but the herdr binary is not callable. Install from https://herdr.dev and retry.',
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
if (backend !== 'herdr' && !isTmuxAvailable()) {
|
|
376
|
+
return { ok: false, error: 'tmux is not available' };
|
|
377
|
+
}
|
|
365
378
|
return await withScalingLock(sanitized, leaderCwd, async () => {
|
|
366
379
|
return await withTaskMembershipBarrier(sanitized, leaderCwd, async () => {
|
|
367
380
|
const config = await readTeamConfig(sanitized, leaderCwd);
|
|
@@ -428,87 +441,116 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
428
441
|
}
|
|
429
442
|
const persistedUltragoalContext = await readPersistedTeamUltragoalContext(sanitized, config.leader_cwd ?? leaderCwd, config.team_state_root ?? teamStateRoot);
|
|
430
443
|
const approvedContextSection = joinContextSections(approvedExecutionGate.approvedContextSection, renderLeaderOwnedUltragoalContextSection(persistedUltragoalContext));
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
444
|
+
// Herdr teams scale inside the existing workspace: no tmux split target,
|
|
445
|
+
// owner tag, or pane-PID proof applies (herdr pane ids are never reused,
|
|
446
|
+
// so the recorded pane id is itself the stable address).
|
|
447
|
+
const herdrBackend = config.worker_launch_mode === 'herdr';
|
|
448
|
+
const herdrWorkspacePane = herdrBackend
|
|
449
|
+
? config.workers.find((worker) => typeof worker.pane_id === 'string' && isHerdrPaneId(worker.pane_id))
|
|
434
450
|
: undefined;
|
|
435
|
-
const
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
if (!initialSplitTargetOwnerId) {
|
|
451
|
+
const herdrWorkspaceId = herdrWorkspacePane?.pane_id
|
|
452
|
+
? herdrWorkspaceIdFromPaneId(herdrWorkspacePane.pane_id)
|
|
453
|
+
: null;
|
|
454
|
+
if (herdrBackend && !herdrWorkspaceId) {
|
|
440
455
|
return {
|
|
441
456
|
ok: false,
|
|
442
|
-
error: `
|
|
457
|
+
error: `scale_up_herdr_workspace_unavailable:${sanitized}:no worker pane id to derive the herdr workspace from`,
|
|
443
458
|
};
|
|
444
459
|
}
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
if (typeof initialExpectedSplitTargetPid !== 'number'
|
|
449
|
-
|| !Number.isSafeInteger(initialExpectedSplitTargetPid)
|
|
450
|
-
|| initialExpectedSplitTargetPid <= 0) {
|
|
451
|
-
// Legacy configs may lack a persisted PID, but the persisted pane ID and
|
|
452
|
-
// canonical owner remain mandatory. Pin only the exact live pane proven
|
|
453
|
-
// from one global snapshot, then re-prove it after reading the owner tag
|
|
454
|
-
// so a recycled pane ID cannot be backfilled.
|
|
455
|
-
const migrationProof = readExactPaneProofSync(initialSplitTarget);
|
|
456
|
-
if (migrationProof.status !== 'live') {
|
|
457
|
-
return {
|
|
458
|
-
ok: false,
|
|
459
|
-
error: `scale_up_split_target_proof_unavailable:${initialSplitTarget}:${migrationProof.reason}`,
|
|
460
|
-
};
|
|
460
|
+
if (herdrBackend) {
|
|
461
|
+
try {
|
|
462
|
+
await ensureHerdrServer({ env: launchEnv });
|
|
461
463
|
}
|
|
462
|
-
|
|
463
|
-
if (migrationOwner.status !== 'value' || migrationOwner.value !== initialSplitTargetOwnerId) {
|
|
464
|
+
catch (error) {
|
|
464
465
|
return {
|
|
465
466
|
ok: false,
|
|
466
|
-
error: `
|
|
467
|
+
error: `scale_up_herdr_server_unavailable:${sanitized}:${error instanceof Error ? error.message : String(error)}`,
|
|
467
468
|
};
|
|
468
469
|
}
|
|
469
|
-
|
|
470
|
-
|
|
470
|
+
}
|
|
471
|
+
const initialSplitTargetCandidate = config.workers[config.workers.length - 1];
|
|
472
|
+
const initialSplitTargetWorker = initialSplitTargetCandidate?.pane_id?.trim()
|
|
473
|
+
? initialSplitTargetCandidate
|
|
474
|
+
: undefined;
|
|
475
|
+
const initialSplitTarget = initialSplitTargetWorker?.pane_id ?? config.leader_pane_id ?? '';
|
|
476
|
+
if (!herdrBackend) {
|
|
477
|
+
const initialSplitTargetOwnerId = typeof config.tmux_pane_owner_id === 'string'
|
|
478
|
+
? config.tmux_pane_owner_id.trim()
|
|
479
|
+
: '';
|
|
480
|
+
if (!initialSplitTargetOwnerId) {
|
|
471
481
|
return {
|
|
472
482
|
ok: false,
|
|
473
|
-
error: `
|
|
483
|
+
error: `scale_up_split_target_owner_unavailable:${initialSplitTarget}`,
|
|
474
484
|
};
|
|
475
485
|
}
|
|
476
|
-
|
|
486
|
+
let initialExpectedSplitTargetPid = initialSplitTargetWorker
|
|
487
|
+
? initialSplitTargetWorker.pid
|
|
488
|
+
: config.leader_pane_pid;
|
|
489
|
+
if (typeof initialExpectedSplitTargetPid !== 'number'
|
|
490
|
+
|| !Number.isSafeInteger(initialExpectedSplitTargetPid)
|
|
491
|
+
|| initialExpectedSplitTargetPid <= 0) {
|
|
492
|
+
// Legacy configs may lack a persisted PID, but the persisted pane ID and
|
|
493
|
+
// canonical owner remain mandatory. Pin only the exact live pane proven
|
|
494
|
+
// from one global snapshot, then re-prove it after reading the owner tag
|
|
495
|
+
// so a recycled pane ID cannot be backfilled.
|
|
496
|
+
const migrationProof = readExactPaneProofSync(initialSplitTarget);
|
|
497
|
+
if (migrationProof.status !== 'live') {
|
|
498
|
+
return {
|
|
499
|
+
ok: false,
|
|
500
|
+
error: `scale_up_split_target_proof_unavailable:${initialSplitTarget}:${migrationProof.reason}`,
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
const migrationOwner = readPaneTeamOwnerTagResult(migrationProof.paneId);
|
|
504
|
+
if (migrationOwner.status !== 'value' || migrationOwner.value !== initialSplitTargetOwnerId) {
|
|
505
|
+
return {
|
|
506
|
+
ok: false,
|
|
507
|
+
error: `scale_up_split_target_owner_changed:${migrationProof.paneId}`,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
const finalMigrationProof = readExactPaneProofSync(migrationProof.paneId);
|
|
511
|
+
if (finalMigrationProof.status !== 'live') {
|
|
512
|
+
return {
|
|
513
|
+
ok: false,
|
|
514
|
+
error: `scale_up_split_target_proof_unavailable:${migrationProof.paneId}:${finalMigrationProof.reason}`,
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
if (finalMigrationProof.pid !== migrationProof.pid) {
|
|
518
|
+
return {
|
|
519
|
+
ok: false,
|
|
520
|
+
error: `scale_up_split_target_pid_changed:${migrationProof.paneId}:${migrationProof.pid}:${finalMigrationProof.pid}`,
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
if (initialSplitTargetWorker) {
|
|
524
|
+
initialSplitTargetWorker.pid = finalMigrationProof.pid;
|
|
525
|
+
}
|
|
526
|
+
else {
|
|
527
|
+
config.leader_pane_pid = finalMigrationProof.pid;
|
|
528
|
+
}
|
|
529
|
+
try {
|
|
530
|
+
await saveTeamConfig(config, leaderCwd);
|
|
531
|
+
}
|
|
532
|
+
catch (error) {
|
|
533
|
+
return {
|
|
534
|
+
ok: false,
|
|
535
|
+
error: `scale_up_split_target_pid_persistence_failed:${initialSplitTarget}:${error instanceof Error ? error.message : String(error)}`,
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
Object.assign(originalConfig, JSON.parse(JSON.stringify(config)));
|
|
539
|
+
initialExpectedSplitTargetPid = finalMigrationProof.pid;
|
|
540
|
+
}
|
|
541
|
+
const initialSplitTargetProof = readExactPaneProofSync(initialSplitTarget);
|
|
542
|
+
if (initialSplitTargetProof.status !== 'live') {
|
|
477
543
|
return {
|
|
478
544
|
ok: false,
|
|
479
|
-
error: `
|
|
545
|
+
error: `scale_up_split_target_proof_unavailable:${initialSplitTarget}:${initialSplitTargetProof.reason}`,
|
|
480
546
|
};
|
|
481
547
|
}
|
|
482
|
-
if (
|
|
483
|
-
initialSplitTargetWorker.pid = finalMigrationProof.pid;
|
|
484
|
-
}
|
|
485
|
-
else {
|
|
486
|
-
config.leader_pane_pid = finalMigrationProof.pid;
|
|
487
|
-
}
|
|
488
|
-
try {
|
|
489
|
-
await saveTeamConfig(config, leaderCwd);
|
|
490
|
-
}
|
|
491
|
-
catch (error) {
|
|
548
|
+
if (initialSplitTargetProof.pid !== initialExpectedSplitTargetPid) {
|
|
492
549
|
return {
|
|
493
550
|
ok: false,
|
|
494
|
-
error: `
|
|
551
|
+
error: `scale_up_split_target_pid_changed:${initialSplitTarget}:${initialExpectedSplitTargetPid}:${initialSplitTargetProof.pid}`,
|
|
495
552
|
};
|
|
496
553
|
}
|
|
497
|
-
Object.assign(originalConfig, JSON.parse(JSON.stringify(config)));
|
|
498
|
-
initialExpectedSplitTargetPid = finalMigrationProof.pid;
|
|
499
|
-
}
|
|
500
|
-
const initialSplitTargetProof = readExactPaneProofSync(initialSplitTarget);
|
|
501
|
-
if (initialSplitTargetProof.status !== 'live') {
|
|
502
|
-
return {
|
|
503
|
-
ok: false,
|
|
504
|
-
error: `scale_up_split_target_proof_unavailable:${initialSplitTarget}:${initialSplitTargetProof.reason}`,
|
|
505
|
-
};
|
|
506
|
-
}
|
|
507
|
-
if (initialSplitTargetProof.pid !== initialExpectedSplitTargetPid) {
|
|
508
|
-
return {
|
|
509
|
-
ok: false,
|
|
510
|
-
error: `scale_up_split_target_pid_changed:${initialSplitTarget}:${initialExpectedSplitTargetPid}:${initialSplitTargetProof.pid}`,
|
|
511
|
-
};
|
|
512
554
|
}
|
|
513
555
|
const effectiveWorktreeMode = config.worktree_mode ?? resolveScaleUpWorktreeMode(config);
|
|
514
556
|
if (!config.worktree_mode && effectiveWorktreeMode.enabled) {
|
|
@@ -584,13 +626,32 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
584
626
|
const rollbackPaneIds = [...new Set([
|
|
585
627
|
...rollbackWorkers.map((worker) => worker.pane_id),
|
|
586
628
|
context.paneId,
|
|
587
|
-
].filter((paneId) => typeof paneId === 'string'
|
|
629
|
+
].filter((paneId) => typeof paneId === 'string'
|
|
630
|
+
&& (paneId.trim().startsWith('%') || isHerdrPaneId(paneId.trim()))))];
|
|
588
631
|
const resolvedPaneIds = new Set();
|
|
589
632
|
const unresolvedPaneIds = new Set();
|
|
633
|
+
// Herdr pane ids are unique for the pane's whole lifetime (never
|
|
634
|
+
// recycled), so the recorded id is itself teardown authority — close the
|
|
635
|
+
// pane directly; a pane_not_found answer proves it is already gone.
|
|
636
|
+
for (const paneId of rollbackPaneIds.filter((paneId) => isHerdrPaneId(paneId))) {
|
|
637
|
+
try {
|
|
638
|
+
await teardownHerdrWorker(paneId, { env });
|
|
639
|
+
resolvedPaneIds.add(paneId);
|
|
640
|
+
}
|
|
641
|
+
catch (error) {
|
|
642
|
+
if (error instanceof HerdrError && error.code === 'pane_not_found') {
|
|
643
|
+
resolvedPaneIds.add(paneId);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
unresolvedPaneIds.add(paneId);
|
|
647
|
+
cleanupDebt.push(`herdr_pane_teardown_failed:${paneId}:${error instanceof Error ? error.message : String(error)}`);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
const tmuxRollbackPaneIds = rollbackPaneIds.filter((paneId) => paneId.startsWith('%'));
|
|
590
651
|
const requiredTeamOwnerId = config.tmux_pane_owner_id?.trim();
|
|
591
652
|
const authorizedRollbackPaneIds = new Set();
|
|
592
653
|
const rollbackInstances = new Map();
|
|
593
|
-
for (const paneId of
|
|
654
|
+
for (const paneId of tmuxRollbackPaneIds) {
|
|
594
655
|
const matchingWorkers = rollbackWorkers.filter((worker) => worker.pane_id === paneId);
|
|
595
656
|
const worker = matchingWorkers.length === 1 ? matchingWorkers[0] : undefined;
|
|
596
657
|
const expectedPanePid = worker?.pid;
|
|
@@ -801,25 +862,33 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
801
862
|
const { workerIndex, workerName, runtimeRole, workerLaunchArgs, workerCli, } = workerLaunchPlan;
|
|
802
863
|
// Freeze the split target and its identity before preparing any artifacts.
|
|
803
864
|
// Later proof must establish that this exact pane process still owns the
|
|
804
|
-
// target; a recycled pane ID must never become split authority.
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
:
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
865
|
+
// target; a recycled pane ID must never become split authority. Herdr
|
|
866
|
+
// teams skip this: workers spawn as new tabs pinned to the workspace id,
|
|
867
|
+
// not splits of an existing tmux pane.
|
|
868
|
+
let splitTarget = '';
|
|
869
|
+
let expectedSplitTargetPid;
|
|
870
|
+
let expectedSplitTargetOwnerId = '';
|
|
871
|
+
let splitDirection = '-v';
|
|
872
|
+
if (!herdrBackend) {
|
|
873
|
+
const splitTargetCandidate = config.workers[config.workers.length - 1];
|
|
874
|
+
const splitTargetWorker = splitTargetCandidate?.pane_id?.trim() ? splitTargetCandidate : undefined;
|
|
875
|
+
splitTarget = splitTargetWorker?.pane_id ?? config.leader_pane_id ?? '';
|
|
876
|
+
expectedSplitTargetPid = (splitTargetWorker
|
|
877
|
+
? splitTargetWorker.pid
|
|
878
|
+
: config.leader_pane_pid) ?? undefined;
|
|
879
|
+
expectedSplitTargetOwnerId = typeof config.tmux_pane_owner_id === 'string'
|
|
880
|
+
? config.tmux_pane_owner_id.trim()
|
|
881
|
+
: '';
|
|
882
|
+
if (typeof expectedSplitTargetPid !== 'number'
|
|
883
|
+
|| !Number.isSafeInteger(expectedSplitTargetPid)
|
|
884
|
+
|| expectedSplitTargetPid <= 0) {
|
|
885
|
+
return await rollbackScaleUp(`scale_up_split_target_pid_missing:${splitTarget}`);
|
|
886
|
+
}
|
|
887
|
+
if (!expectedSplitTargetOwnerId) {
|
|
888
|
+
return await rollbackScaleUp(`scale_up_split_target_owner_unavailable:${splitTarget}`);
|
|
889
|
+
}
|
|
890
|
+
splitDirection = splitTarget === (config.leader_pane_id ?? '') ? '-h' : '-v';
|
|
821
891
|
}
|
|
822
|
-
const splitDirection = splitTarget === (config.leader_pane_id ?? '') ? '-h' : '-v';
|
|
823
892
|
nextIndex = workerIndex + 1;
|
|
824
893
|
if (workerLaunchPlan.mixedTaskRoles.length > 1) {
|
|
825
894
|
console.log(`[omx:scaling] ${workerName}: mixed task roles [${workerLaunchPlan.mixedTaskRoles.join(', ')}], falling back to ${agentType}`);
|
|
@@ -832,6 +901,7 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
832
901
|
let workerWorkspace = null;
|
|
833
902
|
let workerCwd = leaderCwd;
|
|
834
903
|
let cmd;
|
|
904
|
+
let extraEnv = {};
|
|
835
905
|
let rawRolePromptContent = null;
|
|
836
906
|
try {
|
|
837
907
|
preparedWorkerDirectoryOwner.set(workerDirPath, workerName);
|
|
@@ -895,7 +965,7 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
895
965
|
: rolePromptContent
|
|
896
966
|
? await writeWorkerRoleInstructionsFile(sanitized, workerName, leaderCwd, teamInstructionsPath, runtimeRole, rolePromptContent)
|
|
897
967
|
: teamInstructionsPath;
|
|
898
|
-
|
|
968
|
+
extraEnv = {
|
|
899
969
|
OMX_TEAM_STATE_ROOT: teamStateRoot,
|
|
900
970
|
OMX_TEAM_LEADER_CWD: leaderCwd,
|
|
901
971
|
OMX_MODEL_INSTRUCTIONS_FILE: instructionsFilePath,
|
|
@@ -909,75 +979,142 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
909
979
|
extraEnv.OMX_TEAM_WORKTREE_DETACHED = workerWorkspace.detached ? '1' : '0';
|
|
910
980
|
}
|
|
911
981
|
trustWorkerMiseConfigIfAvailable(workerCwd);
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
982
|
+
if (herdrBackend) {
|
|
983
|
+
// Herdr workers launch through `agent start` in a fresh tab — the
|
|
984
|
+
// tmux startup script never runs there, so skip writing it.
|
|
985
|
+
cmd = '';
|
|
986
|
+
}
|
|
987
|
+
else {
|
|
988
|
+
preparedStartupScriptOwner.set(startupScriptPath, workerName);
|
|
989
|
+
const startupCommand = writeWorkerStartupScriptCommand(sanitized, workerIndex, workerLaunchArgs, workerCwd, extraEnv, workerCli, undefined, runtimeRole);
|
|
990
|
+
cmd = startupCommand ?? buildWorkerStartupCommand(sanitized, workerIndex, workerLaunchArgs, workerCwd, extraEnv, workerCli, undefined, runtimeRole);
|
|
991
|
+
}
|
|
915
992
|
}
|
|
916
993
|
catch (error) {
|
|
917
994
|
return await rollbackScaleUp(`scale_up_worker_preparation_failed:${workerName}:${error instanceof Error ? error.message : String(error)}`, { workerName, worktreePath: workerWorkspace?.worktreePath });
|
|
918
995
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
996
|
+
let paneId;
|
|
997
|
+
let workerInfo;
|
|
998
|
+
if (herdrBackend) {
|
|
999
|
+
// Spawn one tab + agent in the live team workspace. The spawn helper
|
|
1000
|
+
// proves the agent promptable (first-run trust dialogs dismissed) and
|
|
1001
|
+
// closes its own pane on failure, so a failed spawn is pure rollback.
|
|
1002
|
+
let spawned;
|
|
1003
|
+
try {
|
|
1004
|
+
const spec = buildWorkerProcessLaunchSpec(sanitized, workerIndex, workerLaunchArgs, workerCwd, extraEnv, workerCli);
|
|
1005
|
+
const herdrSpawnPlan = {
|
|
1006
|
+
kind: spec.workerCli,
|
|
1007
|
+
agentArgs: spec.args,
|
|
1008
|
+
env: spec.env,
|
|
1009
|
+
cwd: workerCwd,
|
|
1010
|
+
};
|
|
1011
|
+
spawned = await spawnOneHerdrWorker(sanitized, herdrWorkspaceId, workerIndex, herdrSpawnPlan, { env: launchEnv });
|
|
1012
|
+
}
|
|
1013
|
+
catch (error) {
|
|
1014
|
+
return await rollbackScaleUp(`scale_up_herdr_spawn_failed:${workerName}:${error instanceof Error ? error.message : String(error)}`, { workerName, worktreePath: workerWorkspace?.worktreePath });
|
|
1015
|
+
}
|
|
1016
|
+
paneId = spawned.paneId;
|
|
1017
|
+
workerInfo = {
|
|
1018
|
+
name: workerName,
|
|
1019
|
+
index: workerIndex,
|
|
1020
|
+
role: runtimeRole,
|
|
1021
|
+
worker_cli: workerCli,
|
|
1022
|
+
assigned_tasks: [],
|
|
1023
|
+
pid: undefined,
|
|
1024
|
+
pane_id: paneId,
|
|
1025
|
+
working_dir: workerCwd,
|
|
1026
|
+
worktree_repo_root: workerWorkspace ? workerWorkspace.repoRoot : undefined,
|
|
1027
|
+
worktree_path: workerWorkspace ? workerWorkspace.worktreePath : undefined,
|
|
1028
|
+
worktree_branch: workerWorkspace ? (workerWorkspace.branchName ?? undefined) : undefined,
|
|
1029
|
+
worktree_detached: workerWorkspace ? workerWorkspace.detached : undefined,
|
|
1030
|
+
worktree_created: workerWorkspace ? workerWorkspace.created : undefined,
|
|
1031
|
+
team_state_root: teamStateRoot,
|
|
1032
|
+
};
|
|
940
1033
|
}
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
1034
|
+
else {
|
|
1035
|
+
// Freeze and validate the full source-pane incarnation. The server owns
|
|
1036
|
+
// the final comparison and split, so a recycled pane ID cannot become a
|
|
1037
|
+
// split target between client-side proof and the topology mutation.
|
|
1038
|
+
const splitTargetProof = readExactPaneProofSync(splitTarget);
|
|
1039
|
+
const splitTargetInstance = splitTargetProof.status === 'live'
|
|
1040
|
+
? readScalePaneInstance(splitTargetProof.paneId)
|
|
1041
|
+
: null;
|
|
1042
|
+
if (!splitTargetInstance) {
|
|
1043
|
+
return await rollbackScaleUp(`scale_up_split_target_proof_unavailable:${splitTarget}`, { workerName, worktreePath: workerWorkspace?.worktreePath });
|
|
1044
|
+
}
|
|
1045
|
+
if (splitTargetInstance.pid !== expectedSplitTargetPid) {
|
|
1046
|
+
return await rollbackScaleUp(`scale_up_split_target_pid_changed:${splitTarget}:${expectedSplitTargetPid}:${splitTargetInstance.pid}`, { workerName, worktreePath: workerWorkspace?.worktreePath });
|
|
1047
|
+
}
|
|
1048
|
+
if (splitTargetInstance.sessionName !== sessionName
|
|
1049
|
+
|| (config.tmux_session_id && splitTargetInstance.sessionId !== config.tmux_session_id)
|
|
1050
|
+
|| splitTargetInstance.teamOwnerId !== expectedSplitTargetOwnerId) {
|
|
1051
|
+
return await rollbackScaleUp(`scale_up_split_target_authority_changed:${splitTargetInstance.paneId}`, { workerName, worktreePath: workerWorkspace?.worktreePath });
|
|
1052
|
+
}
|
|
1053
|
+
const createdPane = splitWorkerPaneWithScaleAuthority(splitTargetInstance, splitDirection, workerCwd, cmd);
|
|
1054
|
+
if (!createdPane) {
|
|
1055
|
+
return await rollbackScaleUp(`scale_up_split_target_authority_lost:${splitTargetInstance.paneId}`, { workerName, worktreePath: workerWorkspace?.worktreePath });
|
|
1056
|
+
}
|
|
1057
|
+
paneId = createdPane.paneId;
|
|
1058
|
+
workerInfo = {
|
|
1059
|
+
name: workerName,
|
|
1060
|
+
index: workerIndex,
|
|
1061
|
+
role: runtimeRole,
|
|
1062
|
+
worker_cli: workerCli,
|
|
1063
|
+
assigned_tasks: [],
|
|
1064
|
+
pid: createdPane.pid,
|
|
1065
|
+
pane_id: paneId,
|
|
1066
|
+
session_id: createdPane.sessionId,
|
|
1067
|
+
window_id: createdPane.windowId,
|
|
1068
|
+
working_dir: workerCwd,
|
|
1069
|
+
worktree_repo_root: workerWorkspace ? workerWorkspace.repoRoot : undefined,
|
|
1070
|
+
worktree_path: workerWorkspace ? workerWorkspace.worktreePath : undefined,
|
|
1071
|
+
worktree_branch: workerWorkspace ? (workerWorkspace.branchName ?? undefined) : undefined,
|
|
1072
|
+
worktree_detached: workerWorkspace ? workerWorkspace.detached : undefined,
|
|
1073
|
+
worktree_created: workerWorkspace ? workerWorkspace.created : undefined,
|
|
1074
|
+
team_state_root: teamStateRoot,
|
|
1075
|
+
};
|
|
1076
|
+
const taggedPane = config.tmux_pane_owner_id
|
|
1077
|
+
? tagScalePaneWithAuthority(createdPane, config.tmux_pane_owner_id.trim())
|
|
1078
|
+
: null;
|
|
1079
|
+
if (!taggedPane) {
|
|
1080
|
+
return await rollbackScaleUp(`scale_up_created_pane_owner_tag_authority_lost:${paneId}`, { paneId, worker: workerInfo });
|
|
1081
|
+
}
|
|
1082
|
+
rollbackTaggedPaneOwnerIds.set(paneId, taggedPane.teamOwnerId);
|
|
965
1083
|
}
|
|
966
|
-
rollbackTaggedPaneOwnerIds.set(paneId, taggedPane.teamOwnerId);
|
|
967
1084
|
// Intentionally avoid forcing `select-layout tiled` here.
|
|
968
1085
|
// Tiled relayout reflows leader/HUD panes and breaks team window layout.
|
|
969
1086
|
try {
|
|
970
1087
|
await writeWorkerIdentity(sanitized, workerName, workerInfo, leaderCwd);
|
|
971
1088
|
throwIfScaleUpFailureInjected(env, 'identity');
|
|
972
|
-
// Wait for worker readiness
|
|
1089
|
+
// Wait for worker readiness. Herdr spawn already proves the agent
|
|
1090
|
+
// promptable (agent-start readiness + settle polling), so the tmux
|
|
1091
|
+
// ready-wait does not apply.
|
|
973
1092
|
const readyTimeoutMs = resolveWorkerReadyTimeoutMs(env);
|
|
974
|
-
const skipReadyWait = env.OMX_TEAM_SKIP_READY_WAIT === '1';
|
|
1093
|
+
const skipReadyWait = env.OMX_TEAM_SKIP_READY_WAIT === '1' || herdrBackend;
|
|
975
1094
|
if (!skipReadyWait) {
|
|
976
1095
|
const ready = waitForWorkerReady(sessionName, workerIndex, readyTimeoutMs, paneId, workerInfo.pid, config.tmux_pane_owner_id ?? undefined, config.hud_pane_id ?? undefined);
|
|
977
1096
|
if (!ready) {
|
|
978
1097
|
console.log(`[omx:scaling] Warning: worker ${workerName} did not become ready within timeout`);
|
|
979
1098
|
}
|
|
980
1099
|
}
|
|
1100
|
+
// Backend-aware pane notification: herdr prompts the pane's agent,
|
|
1101
|
+
// tmux proves session/window authority and sends keys.
|
|
1102
|
+
const notifyPane = async (message) => {
|
|
1103
|
+
if (herdrBackend) {
|
|
1104
|
+
try {
|
|
1105
|
+
await submitToHerdrWorker(paneId, message, { env: launchEnv });
|
|
1106
|
+
return { ok: true, transport: 'herdr_prompt', reason: 'herdr_prompt_sent' };
|
|
1107
|
+
}
|
|
1108
|
+
catch (error) {
|
|
1109
|
+
return {
|
|
1110
|
+
ok: false,
|
|
1111
|
+
transport: 'herdr_prompt',
|
|
1112
|
+
reason: `herdr_prompt_failed:${error instanceof Error ? error.message : String(error)}`,
|
|
1113
|
+
};
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
return await notifyWorkerPaneOutcome(sessionName, message, paneId, workerInfo.pid, config.tmux_pane_owner_id ?? undefined, workerInfo.session_id, workerInfo.window_id);
|
|
1117
|
+
};
|
|
981
1118
|
// Get assigned tasks for this worker
|
|
982
1119
|
const workerTasks = materializedTasks.filter(t => t.owner === workerName);
|
|
983
1120
|
const inbox = generateInitialInbox(workerName, sanitized, agentType, workerTasks, {
|
|
@@ -1007,7 +1144,7 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
1007
1144
|
if (dispatchPolicy.dispatch_mode === 'hook_preferred_with_fallback') {
|
|
1008
1145
|
return { ok: true, transport: 'hook', reason: 'queued_for_hook_dispatch' };
|
|
1009
1146
|
}
|
|
1010
|
-
return await
|
|
1147
|
+
return await notifyPane(message);
|
|
1011
1148
|
},
|
|
1012
1149
|
});
|
|
1013
1150
|
let outcome = queued;
|
|
@@ -1020,7 +1157,7 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
1020
1157
|
outcome = { ok: true, transport: 'hook', reason: `hook_receipt_${receipt.status}`, request_id: queued.request_id };
|
|
1021
1158
|
}
|
|
1022
1159
|
else {
|
|
1023
|
-
const fallback = await
|
|
1160
|
+
const fallback = await notifyPane(triggerDirective.text);
|
|
1024
1161
|
if (receipt?.status === 'failed') {
|
|
1025
1162
|
if (fallback.ok) {
|
|
1026
1163
|
await transitionDispatchRequest(sanitized, queued.request_id, 'failed', 'failed', { last_reason: `fallback_confirmed_after_failed_receipt:${fallback.reason}` }, leaderCwd).catch(() => { });
|
|
@@ -1068,9 +1205,10 @@ export async function scaleUp(teamName, count, agentType, tasks, cwd, env = proc
|
|
|
1068
1205
|
}
|
|
1069
1206
|
}
|
|
1070
1207
|
// Retry dispatch once if a trust prompt is blocking the worker pane (fixes #393).
|
|
1071
|
-
|
|
1208
|
+
// Herdr spawns dismiss trust dialogs during agent start, so the retry is tmux-only.
|
|
1209
|
+
if (!outcome.ok && !herdrBackend && dismissTrustPromptIfPresent(sessionName, workerIndex, paneId, workerInfo.pid, config.tmux_pane_owner_id ?? undefined, config.hud_pane_id ?? undefined)) {
|
|
1072
1210
|
waitForWorkerReady(sessionName, workerIndex, readyTimeoutMs, paneId, workerInfo.pid, config.tmux_pane_owner_id ?? undefined, config.hud_pane_id ?? undefined);
|
|
1073
|
-
const retry = await
|
|
1211
|
+
const retry = await notifyPane(triggerDirective.text);
|
|
1074
1212
|
if (retry.ok) {
|
|
1075
1213
|
outcome = retry;
|
|
1076
1214
|
}
|
|
@@ -1180,7 +1318,32 @@ async function validateScaleDownCleanupResources(teamName, leaderCwd, teamStateR
|
|
|
1180
1318
|
}
|
|
1181
1319
|
}
|
|
1182
1320
|
}
|
|
1183
|
-
|
|
1321
|
+
function assertScaleDownWorktreesPreserved(workers, leaderCwd) {
|
|
1322
|
+
for (const worker of workers) {
|
|
1323
|
+
if (!worker.worktree_path || !existsSync(worker.worktree_path))
|
|
1324
|
+
continue;
|
|
1325
|
+
const git = (args, cwd = worker.worktree_path) => spawnSync('git', args, {
|
|
1326
|
+
cwd, encoding: 'utf8', windowsHide: true,
|
|
1327
|
+
});
|
|
1328
|
+
// ORIG_HEAD and COMMIT_EDITMSG survive completed operations; only active state vetoes removal.
|
|
1329
|
+
if (['rebase-merge', 'rebase-apply', 'MERGE_HEAD', 'CHERRY_PICK_HEAD', 'REVERT_HEAD', 'sequencer', 'BISECT_START'].some((marker) => {
|
|
1330
|
+
const path = git(['rev-parse', '--git-path', marker]);
|
|
1331
|
+
return path.status !== 0 || !path.stdout.trim()
|
|
1332
|
+
|| existsSync(resolve(worker.worktree_path, path.stdout.trim()));
|
|
1333
|
+
})) {
|
|
1334
|
+
throw new Error(`scale_down_worktree_not_preserved:${worker.name}`);
|
|
1335
|
+
}
|
|
1336
|
+
const head = git(['rev-parse', 'HEAD']);
|
|
1337
|
+
const status = git(['status', '--porcelain', '--untracked-files=all', '--ignored', '--ignore-submodules=none']);
|
|
1338
|
+
if (head.status !== 0 || status.status !== 0 || status.stdout.trim()
|
|
1339
|
+
|| git(['merge-base', '--is-ancestor', head.stdout.trim(), 'HEAD'], leaderCwd).status !== 0) {
|
|
1340
|
+
throw new Error(`scale_down_worktree_not_preserved:${worker.name}`);
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
async function cleanupScaleDownResources(teamName, teamStateRoot, workers, leaderCwd, force) {
|
|
1345
|
+
if (!force)
|
|
1346
|
+
assertScaleDownWorktreesPreserved(workers, leaderCwd);
|
|
1184
1347
|
for (const worker of workers) {
|
|
1185
1348
|
// An absent exact worktree means a prior cleanup completed before crashing.
|
|
1186
1349
|
// Do not replay a restoration or git removal against a replacement path.
|
|
@@ -1207,7 +1370,7 @@ async function cleanupScaleDownResources(teamName, teamStateRoot, workers) {
|
|
|
1207
1370
|
createdBranch: false,
|
|
1208
1371
|
}));
|
|
1209
1372
|
if (worktrees.length > 0)
|
|
1210
|
-
await rollbackProvisionedWorktrees(worktrees, { force
|
|
1373
|
+
await rollbackProvisionedWorktrees(worktrees, { force });
|
|
1211
1374
|
await Promise.all(workers.map(async (worker) => {
|
|
1212
1375
|
await rm(join(teamStateRoot, 'team', teamName, 'workers', worker.name), { recursive: true, force: true });
|
|
1213
1376
|
}));
|
|
@@ -1246,9 +1409,14 @@ function asScaleDownDebtPanes(debt, removedWorkerNames, resources, leaderPaneId,
|
|
|
1246
1409
|
if (!pane || typeof pane.name !== 'string' || !expectedNames.has(pane.name) || !resourceNames.has(pane.name)
|
|
1247
1410
|
|| typeof pane.index !== 'number' || !Number.isInteger(pane.index) || pane.index <= 0
|
|
1248
1411
|
|| pane.index !== Number(pane.name.slice('worker-'.length))
|
|
1249
|
-
|| typeof pane.pane_id !== 'string'
|
|
1412
|
+
|| typeof pane.pane_id !== 'string'
|
|
1413
|
+
|| !(/^%\d+$/.test(pane.pane_id) || isHerdrPaneId(pane.pane_id))
|
|
1250
1414
|
|| pane.pane_id === normalizedLeaderPaneId || pane.pane_id === normalizedHudPaneId
|
|
1251
|
-
|
|
1415
|
+
// Herdr records carry no pane PID (ids are never reused, so there is
|
|
1416
|
+
// nothing to pin); tmux records keep the positive-PID invariant.
|
|
1417
|
+
|| (isHerdrPaneId(pane.pane_id)
|
|
1418
|
+
? pane.pid !== null
|
|
1419
|
+
: (pane.pid !== null && (!Number.isSafeInteger(pane.pid) || pane.pid <= 0))))
|
|
1252
1420
|
return false;
|
|
1253
1421
|
const canonical = canonicalByName.get(pane.name);
|
|
1254
1422
|
if (requireCanonicalMatch) {
|
|
@@ -1311,7 +1479,8 @@ export async function reconcileScaleDownCleanupDebt(teamName, cwd, _callerConfig
|
|
|
1311
1479
|
catch (error) {
|
|
1312
1480
|
return { ok: false, error: `scale_down_cleanup_debt_unreadable:${String(error)}` };
|
|
1313
1481
|
}
|
|
1314
|
-
if (debt.schema_version !== 1 || debt.operation !== 'scale_down'
|
|
1482
|
+
if (debt.schema_version !== 1 || debt.operation !== 'scale_down'
|
|
1483
|
+
|| (debt.force !== undefined && typeof debt.force !== 'boolean')) {
|
|
1315
1484
|
return { ok: false, error: 'scale_down_cleanup_debt_malformed' };
|
|
1316
1485
|
}
|
|
1317
1486
|
const removedWorkerNames = Array.isArray(debt.removed_worker_names)
|
|
@@ -1358,6 +1527,24 @@ export async function reconcileScaleDownCleanupDebt(teamName, cwd, _callerConfig
|
|
|
1358
1527
|
? config.tmux_pane_owner_id.trim()
|
|
1359
1528
|
: '';
|
|
1360
1529
|
for (const pane of panes) {
|
|
1530
|
+
// Herdr pane ids are unique for the pane's whole lifetime: the committed
|
|
1531
|
+
// debt record is itself teardown authority, and a pane_not_found answer
|
|
1532
|
+
// proves convergence. No tmux PID/owner proof applies.
|
|
1533
|
+
if (isHerdrPaneId(pane.pane_id)) {
|
|
1534
|
+
try {
|
|
1535
|
+
await teardownHerdrWorker(pane.pane_id);
|
|
1536
|
+
resolvedPaneIds.add(pane.pane_id);
|
|
1537
|
+
}
|
|
1538
|
+
catch (error) {
|
|
1539
|
+
if (error instanceof HerdrError && error.code === 'pane_not_found') {
|
|
1540
|
+
resolvedPaneIds.add(pane.pane_id);
|
|
1541
|
+
}
|
|
1542
|
+
else {
|
|
1543
|
+
reasons.push(`${pane.pane_id}:herdr_teardown_failed:${error instanceof Error ? error.message : String(error)}`);
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1361
1548
|
// Legacy PID-less records can establish convergence from an authoritative
|
|
1362
1549
|
// global absence/dead proof, but can never authorize a live-pane effect.
|
|
1363
1550
|
const freshProof = readExactPaneProofSync(pane.pane_id);
|
|
@@ -1431,7 +1618,7 @@ export async function reconcileScaleDownCleanupDebt(teamName, cwd, _callerConfig
|
|
|
1431
1618
|
return { ok: false, error: `scale_down_cleanup_debt_unresolved:${unresolvedPanes.map((pane) => pane.pane_id).join(',')}` };
|
|
1432
1619
|
}
|
|
1433
1620
|
try {
|
|
1434
|
-
await cleanupScaleDownResources(sanitized, teamStateRoot, resources);
|
|
1621
|
+
await cleanupScaleDownResources(sanitized, teamStateRoot, resources, leaderCwd, debt.force === true);
|
|
1435
1622
|
}
|
|
1436
1623
|
catch (error) {
|
|
1437
1624
|
await writeAtomic(cleanupDebtPath, JSON.stringify({
|
|
@@ -1450,8 +1637,8 @@ export async function reconcileScaleDownCleanupDebt(teamName, cwd, _callerConfig
|
|
|
1450
1637
|
/**
|
|
1451
1638
|
* Remove workers from a running team.
|
|
1452
1639
|
*
|
|
1453
|
-
*
|
|
1454
|
-
*
|
|
1640
|
+
* Waits for targeted workers to publish idle/done (unless forced), then removes
|
|
1641
|
+
* tmux panes and updates config without mutating worker-owned status files.
|
|
1455
1642
|
*/
|
|
1456
1643
|
export async function scaleDown(teamName, cwd, options = {}, env = process.env) {
|
|
1457
1644
|
assertScalingEnabled(env);
|
|
@@ -1459,6 +1646,9 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1459
1646
|
const leaderCwd = resolve(cwd);
|
|
1460
1647
|
const force = options.force === true;
|
|
1461
1648
|
const drainTimeoutMs = options.drainTimeoutMs ?? 30_000;
|
|
1649
|
+
if (!Number.isFinite(drainTimeoutMs) || drainTimeoutMs < 0) {
|
|
1650
|
+
return { ok: false, error: 'drainTimeoutMs must be a non-negative finite number' };
|
|
1651
|
+
}
|
|
1462
1652
|
return await withScalingLock(sanitized, leaderCwd, async () => {
|
|
1463
1653
|
const config = await withTaskMembershipBarrier(sanitized, leaderCwd, async () => {
|
|
1464
1654
|
await recoverTeamMembershipTaskTransaction(sanitized, leaderCwd);
|
|
@@ -1492,7 +1682,7 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1492
1682
|
const idleWorkers = [];
|
|
1493
1683
|
for (const w of config.workers) {
|
|
1494
1684
|
const status = await readWorkerStatus(sanitized, w.name, leaderCwd);
|
|
1495
|
-
if (status.state === 'idle' || status.state === 'done'
|
|
1685
|
+
if (status.state === 'idle' || status.state === 'done') {
|
|
1496
1686
|
idleWorkers.push(w);
|
|
1497
1687
|
}
|
|
1498
1688
|
}
|
|
@@ -1518,56 +1708,25 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1518
1708
|
if (config.workers.length - targetWorkers.length < 1) {
|
|
1519
1709
|
return { ok: false, error: 'Cannot remove all workers — at least 1 must remain' };
|
|
1520
1710
|
}
|
|
1521
|
-
const sessionName = config.tmux_session;
|
|
1522
1711
|
const removedNames = [];
|
|
1523
|
-
const priorWorkerStatusArtifacts = new Map();
|
|
1524
|
-
for (const worker of targetWorkers) {
|
|
1525
|
-
const statusPath = join(teamStateRoot, 'team', sanitized, 'workers', worker.name, 'status.json');
|
|
1526
|
-
try {
|
|
1527
|
-
priorWorkerStatusArtifacts.set(worker.name, { exists: true, raw: await readFile(statusPath) });
|
|
1528
|
-
}
|
|
1529
|
-
catch (error) {
|
|
1530
|
-
if (error.code === 'ENOENT') {
|
|
1531
|
-
priorWorkerStatusArtifacts.set(worker.name, { exists: false });
|
|
1532
|
-
}
|
|
1533
|
-
else {
|
|
1534
|
-
throw error;
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
|
-
}
|
|
1538
|
-
const restorePriorWorkerStatuses = async (workers) => {
|
|
1539
|
-
await Promise.all(workers.map(async (worker) => {
|
|
1540
|
-
const statusPath = join(teamStateRoot, 'team', sanitized, 'workers', worker.name, 'status.json');
|
|
1541
|
-
const priorArtifact = priorWorkerStatusArtifacts.get(worker.name);
|
|
1542
|
-
if (priorArtifact?.exists) {
|
|
1543
|
-
await writeFile(statusPath, priorArtifact.raw);
|
|
1544
|
-
}
|
|
1545
|
-
else {
|
|
1546
|
-
await rm(statusPath, { force: true });
|
|
1547
|
-
}
|
|
1548
|
-
}));
|
|
1549
|
-
};
|
|
1550
|
-
// Phase 1: Set workers to 'draining' status
|
|
1551
|
-
for (const w of targetWorkers) {
|
|
1552
|
-
const drainingStatus = {
|
|
1553
|
-
state: 'draining',
|
|
1554
|
-
reason: 'scale_down requested by leader',
|
|
1555
|
-
updated_at: new Date().toISOString(),
|
|
1556
|
-
};
|
|
1557
|
-
await writeWorkerStatus(sanitized, w.name, drainingStatus, leaderCwd);
|
|
1558
|
-
}
|
|
1559
|
-
// Phase 2: Wait for draining workers to finish or timeout
|
|
1560
1712
|
if (!force) {
|
|
1561
1713
|
const deadline = Date.now() + drainTimeoutMs;
|
|
1562
|
-
while (
|
|
1563
|
-
const
|
|
1564
|
-
|
|
1565
|
-
return status.state === 'idle' || status.state === 'done' ||
|
|
1566
|
-
status.state === 'draining' || !isWorkerAlive(sessionName, w.index, w.pane_id, w.pid, config.tmux_pane_owner_id ?? undefined, config.hud_pane_id ?? undefined);
|
|
1567
|
-
}));
|
|
1568
|
-
if (allDrained.every(Boolean))
|
|
1714
|
+
while (true) {
|
|
1715
|
+
const statuses = await Promise.all(targetWorkers.map((w) => readWorkerStatus(sanitized, w.name, leaderCwd)));
|
|
1716
|
+
if (statuses.every((status) => status.state === 'idle' || status.state === 'done'))
|
|
1569
1717
|
break;
|
|
1570
|
-
|
|
1718
|
+
// A dead process cannot acknowledge a drain; liveness never authorizes removal.
|
|
1719
|
+
const alive = await Promise.all(targetWorkers.map((worker, index) => {
|
|
1720
|
+
if (statuses[index].state === 'idle' || statuses[index].state === 'done' || !worker.pane_id)
|
|
1721
|
+
return true;
|
|
1722
|
+
return isHerdrPaneId(worker.pane_id)
|
|
1723
|
+
? isHerdrWorkerAlive(worker.pane_id, { env })
|
|
1724
|
+
: isWorkerAlive(config.tmux_session, worker.index, worker.pane_id, worker.pid, config.tmux_pane_owner_id ?? undefined, config.hud_pane_id ?? undefined);
|
|
1725
|
+
}));
|
|
1726
|
+
if (alive.some((value) => !value) || Date.now() >= deadline) {
|
|
1727
|
+
return { ok: false, error: 'scale_down_drain_timeout' };
|
|
1728
|
+
}
|
|
1729
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(100, Math.max(0, deadline - Date.now()))));
|
|
1571
1730
|
}
|
|
1572
1731
|
}
|
|
1573
1732
|
// Phase 3: acquire the membership barrier and task claim locks before any
|
|
@@ -1597,6 +1756,17 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1597
1756
|
.map((task) => task.id);
|
|
1598
1757
|
await withTaskClaimLocks(sanitized, candidateTaskIds, leaderCwd, async () => {
|
|
1599
1758
|
const lockedTasks = await listTasks(sanitized, leaderCwd);
|
|
1759
|
+
if (!force) {
|
|
1760
|
+
const statuses = await Promise.all(removableWorkers.map((w) => readWorkerStatus(sanitized, w.name, leaderCwd)));
|
|
1761
|
+
if (!statuses.every((status) => status.state === 'idle' || status.state === 'done')) {
|
|
1762
|
+
throw new Error('scale_down_worker_not_drained');
|
|
1763
|
+
}
|
|
1764
|
+
if (lockedTasks.some((task) => task.status !== 'completed' && task.status !== 'failed'
|
|
1765
|
+
&& (removableWorkerNames.has(task.owner ?? '') || removableWorkerNames.has(task.claim?.owner ?? '')))) {
|
|
1766
|
+
throw new Error('scale_down_worker_has_active_tasks');
|
|
1767
|
+
}
|
|
1768
|
+
assertScaleDownWorktreesPreserved(removableWorkers, leaderCwd);
|
|
1769
|
+
}
|
|
1600
1770
|
const configPath = join(teamStateRoot, 'team', sanitized, 'config.json');
|
|
1601
1771
|
const configSnapshot = await readFile(configPath);
|
|
1602
1772
|
const manifestPath = join(teamStateRoot, 'team', sanitized, 'manifest.v2.json');
|
|
@@ -1655,11 +1825,13 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1655
1825
|
const cleanupDebtBase = {
|
|
1656
1826
|
schema_version: 1,
|
|
1657
1827
|
operation: 'scale_down',
|
|
1828
|
+
force,
|
|
1658
1829
|
status: 'pending_teardown',
|
|
1659
1830
|
created_at: new Date().toISOString(),
|
|
1660
1831
|
removed_worker_names: removableWorkers.map((worker) => worker.name),
|
|
1661
1832
|
workers: removableWorkers
|
|
1662
|
-
.filter((worker) => (typeof worker.pane_id === 'string'
|
|
1833
|
+
.filter((worker) => (typeof worker.pane_id === 'string'
|
|
1834
|
+
&& (/^%\d+$/.test(worker.pane_id) || isHerdrPaneId(worker.pane_id))))
|
|
1663
1835
|
.map((worker) => ({
|
|
1664
1836
|
name: worker.name,
|
|
1665
1837
|
index: worker.index,
|
|
@@ -1690,6 +1862,25 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1690
1862
|
throw new Error(`canonical_scale_down_task_verification_failed:${task.id}`);
|
|
1691
1863
|
}
|
|
1692
1864
|
}
|
|
1865
|
+
// Herdr panes close directly: ids are never reused, so the recorded
|
|
1866
|
+
// pane id is itself teardown authority and a pane_not_found answer
|
|
1867
|
+
// proves the pane is already gone.
|
|
1868
|
+
const herdrPaneWorkers = removableWorkers.filter((worker) => (typeof worker.pane_id === 'string' && isHerdrPaneId(worker.pane_id)));
|
|
1869
|
+
const herdrResolvedPaneIds = new Set();
|
|
1870
|
+
const herdrProofDebt = [];
|
|
1871
|
+
for (const worker of herdrPaneWorkers) {
|
|
1872
|
+
try {
|
|
1873
|
+
await teardownHerdrWorker(worker.pane_id, { env });
|
|
1874
|
+
herdrResolvedPaneIds.add(worker.pane_id);
|
|
1875
|
+
}
|
|
1876
|
+
catch (error) {
|
|
1877
|
+
if (error instanceof HerdrError && error.code === 'pane_not_found') {
|
|
1878
|
+
herdrResolvedPaneIds.add(worker.pane_id);
|
|
1879
|
+
continue;
|
|
1880
|
+
}
|
|
1881
|
+
herdrProofDebt.push(`${worker.pane_id}:herdr_teardown_failed:${error instanceof Error ? error.message : String(error)}`);
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1693
1884
|
// PID-less rollback records are cleanup debt, not effect authority. A
|
|
1694
1885
|
// fresh gone proof may resolve one, but a live or unavailable proof
|
|
1695
1886
|
// must remain debt rather than letting teardown adopt its current PID.
|
|
@@ -1731,14 +1922,18 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1731
1922
|
},
|
|
1732
1923
|
});
|
|
1733
1924
|
const resolvedPaneIds = new Set([
|
|
1925
|
+
...herdrResolvedPaneIds,
|
|
1734
1926
|
...unpinnedResolvedPaneIds,
|
|
1735
1927
|
...paneTeardown.provenGonePaneIds,
|
|
1736
1928
|
...paneTeardown.killedPaneIds,
|
|
1737
1929
|
]);
|
|
1738
1930
|
const unresolvedWorkers = removableWorkers.filter((worker) => (typeof worker.pane_id === 'string' && !resolvedPaneIds.has(worker.pane_id)));
|
|
1739
1931
|
if (unresolvedWorkers.length > 0) {
|
|
1740
|
-
|
|
1741
|
-
|
|
1932
|
+
const proofDebt = [
|
|
1933
|
+
...herdrProofDebt,
|
|
1934
|
+
...unpinnedProofDebt,
|
|
1935
|
+
...paneTeardown.proofUnavailable.map((proof) => `${proof.paneId}:${proof.reason}`),
|
|
1936
|
+
];
|
|
1742
1937
|
const killDebt = paneTeardown.kill.failedPaneIds.map((paneId) => `${paneId}:kill_failed`);
|
|
1743
1938
|
await writeAtomic(cleanupDebtPath, JSON.stringify({
|
|
1744
1939
|
...cleanupDebtBase,
|
|
@@ -1775,7 +1970,6 @@ export async function scaleDown(teamName, cwd, options = {}, env = process.env)
|
|
|
1775
1970
|
});
|
|
1776
1971
|
}
|
|
1777
1972
|
catch (error) {
|
|
1778
|
-
await restorePriorWorkerStatuses(removableWorkers);
|
|
1779
1973
|
return { ok: false, error: `scale_down_task_reconciliation_failed:${String(error)}` };
|
|
1780
1974
|
}
|
|
1781
1975
|
if (teardownFailure)
|