baxian 2.0.7 → 2.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/manager.d.ts +5 -14
- package/dist/agent/manager.d.ts.map +1 -1
- package/dist/agent/manager.js +134 -353
- package/dist/agent/manager.js.map +1 -1
- package/dist/agent/prompt.d.ts +0 -4
- package/dist/agent/prompt.d.ts.map +1 -1
- package/dist/agent/tmux.d.ts +1 -0
- package/dist/agent/tmux.d.ts.map +1 -1
- package/dist/agent/tmux.js +3 -0
- package/dist/agent/tmux.js.map +1 -1
- package/dist/event/handlers.d.ts.map +1 -1
- package/dist/event/handlers.js +1 -14
- package/dist/event/handlers.js.map +1 -1
- package/dist/web/assets/index-BJy8xS0v.js +17 -0
- package/dist/web/assets/index-C-v_n7bS.css +1 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -1
- package/dist/web/assets/index-KH_4rYrI.js +0 -15
- package/dist/web/assets/index-Rmp4wT_5.css +0 -1
package/dist/agent/manager.js
CHANGED
|
@@ -18,7 +18,7 @@ import { AGENT_STORE_NOOP } from '../state/agent-store.js';
|
|
|
18
18
|
import { createRunner, LocalRunner, shellQuote, resolveAgentHost, workdirHostGroupKey, } from './runner.js';
|
|
19
19
|
import { isTransientNetworkFailure } from './net-exec.js';
|
|
20
20
|
import { imageFilename, agentHostPath, writeImageToHost } from './image-input.js';
|
|
21
|
-
import { TmuxManager, ReplNotReadyError, detectStartupDialog, detectRuntimeMenu, runtimeBusyCheck, hasRuntimeReadyView, hasReplProcTitle, hasOscTitleWorking, hasOscTitleIdle, screenAllowsTitleIdle, PaneGoneError, } from './tmux.js';
|
|
21
|
+
import { TmuxManager, ReplNotReadyError, detectStartupDialog, detectRuntimeMenu, runtimeBusyCheck, hasRuntimeReadyView, hasReplProcTitle, isShellProcTitle, hasOscTitleWorking, hasOscTitleIdle, screenAllowsTitleIdle, PaneGoneError, } from './tmux.js';
|
|
22
22
|
import { BranchManager, DirtyWorkdirError, ReviewHeadMismatchError, isAutoDeletableTaskBranch } from './branch.js';
|
|
23
23
|
import { RepoStore, createRepoStoreCache } from './repo-store.js';
|
|
24
24
|
import { PhaseSignalWatcher, } from './phase-signal-watcher.js';
|
|
@@ -195,6 +195,10 @@ function cancelPhaseDowngrades(prev, next) {
|
|
|
195
195
|
return cancelPhaseRank(next) < cancelPhaseRank(prev);
|
|
196
196
|
}
|
|
197
197
|
const REGREET_REQUIRED_HOLD_PHASES = new Set(['greeting_failed']);
|
|
198
|
+
const RECOVERY_FAILED_AGENT_RUNBOOK = 'Fix the agent Workdir (or wait for the transient failure to clear), then Resume the agent.';
|
|
199
|
+
const RECOVERY_FAILED_TASK_RUNBOOK = 'Fix the agent Workdir (or wait for the transient failure to clear), '
|
|
200
|
+
+ 'then Resume the agent and use the recommended action on the task page.';
|
|
201
|
+
const RECOVERY_FAILED_UNBOUND_RUNBOOK = 'Inspect or recreate the agent runtime before dispatching work to it.';
|
|
198
202
|
const CHECKOUT_HOLD_PHASES = new Set([
|
|
199
203
|
'dirty-workdir',
|
|
200
204
|
'checkout-preparation-failed',
|
|
@@ -699,7 +703,7 @@ export class AgentManager {
|
|
|
699
703
|
}
|
|
700
704
|
catch (err) {
|
|
701
705
|
console.warn(`[AgentManager] prompt for ${agentId}/${taskId} was delivered, but its context marker could not be saved; ` +
|
|
702
|
-
`the next phase will
|
|
706
|
+
`the next phase will resend the full task context:`, err);
|
|
703
707
|
}
|
|
704
708
|
}
|
|
705
709
|
async resolveClaimedPane(tmux, agentId, expectedPaneId) {
|
|
@@ -1730,7 +1734,7 @@ export class AgentManager {
|
|
|
1730
1734
|
};
|
|
1731
1735
|
try {
|
|
1732
1736
|
await this.assertTaskGeneration(agentId, expectedTaskId, lockToken, releaseWorkdir);
|
|
1733
|
-
const runtime = await this.inspectReleaseRuntime(tmux, agentId);
|
|
1737
|
+
const runtime = await this.inspectReleaseRuntime(tmux, agentId, agentRuntimeKindFor(cfg));
|
|
1734
1738
|
if (runtime.kind === 'hold')
|
|
1735
1739
|
return hold(runtime.reason);
|
|
1736
1740
|
await this.assertTaskGeneration(agentId, expectedTaskId, lockToken, releaseWorkdir);
|
|
@@ -2689,7 +2693,11 @@ export class AgentManager {
|
|
|
2689
2693
|
return { resumed: false, releasedBinding: false, reason };
|
|
2690
2694
|
}
|
|
2691
2695
|
const now = new Date().toISOString();
|
|
2692
|
-
const
|
|
2696
|
+
const pendingRecoveryBootstrap = state.awaitingPhase === 'recovery-failed'
|
|
2697
|
+
&& state.taskId !== undefined
|
|
2698
|
+
&& state.bootstrappingTaskId === state.taskId
|
|
2699
|
+
&& boundTask?.status === 'pending';
|
|
2700
|
+
const shouldReleaseBinding = pendingRecoveryBootstrap || shouldReleaseHeldBinding(state, boundTask);
|
|
2693
2701
|
if (shouldReleaseBinding && state.taskId) {
|
|
2694
2702
|
return {
|
|
2695
2703
|
resumed: true,
|
|
@@ -2916,7 +2924,7 @@ export class AgentManager {
|
|
|
2916
2924
|
return null;
|
|
2917
2925
|
}
|
|
2918
2926
|
}
|
|
2919
|
-
async inspectReleaseRuntime(tmux, agentId) {
|
|
2927
|
+
async inspectReleaseRuntime(tmux, agentId, runtime) {
|
|
2920
2928
|
let sessionAlive;
|
|
2921
2929
|
try {
|
|
2922
2930
|
sessionAlive = await tmux.hasSession(agentId);
|
|
@@ -2930,8 +2938,9 @@ export class AgentManager {
|
|
|
2930
2938
|
}
|
|
2931
2939
|
if (!sessionAlive)
|
|
2932
2940
|
return { kind: 'absent' };
|
|
2941
|
+
let pane;
|
|
2933
2942
|
try {
|
|
2934
|
-
|
|
2943
|
+
pane = await this.resolveClaimedPane(tmux, agentId);
|
|
2935
2944
|
}
|
|
2936
2945
|
catch (err) {
|
|
2937
2946
|
if (err instanceof PaneGoneError) {
|
|
@@ -2946,6 +2955,22 @@ export class AgentManager {
|
|
|
2946
2955
|
`${err instanceof Error ? err.message : String(err)}`,
|
|
2947
2956
|
};
|
|
2948
2957
|
}
|
|
2958
|
+
try {
|
|
2959
|
+
const proc = await tmux.displayMessage(pane, '#{pane_current_command}');
|
|
2960
|
+
if (!hasReplProcTitle(proc, runtime)) {
|
|
2961
|
+
if (isShellProcTitle(proc))
|
|
2962
|
+
return { kind: 'absent' };
|
|
2963
|
+
return {
|
|
2964
|
+
kind: 'hold',
|
|
2965
|
+
reason: `Runtime pane is running a non-runtime foreground process (${proc}); ` +
|
|
2966
|
+
`refusing checkout cleanup while the Workdir may be in use`,
|
|
2967
|
+
};
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
catch {
|
|
2971
|
+
// proc probe unavailable: fall through to the pane path; the idle wait decides
|
|
2972
|
+
}
|
|
2973
|
+
return { kind: 'pane', pane };
|
|
2949
2974
|
}
|
|
2950
2975
|
async interruptPaneAndWaitReady(state, cfg) {
|
|
2951
2976
|
const pane = await this.resolveClaimedPaneOrNull(state, cfg);
|
|
@@ -3150,7 +3175,7 @@ export class AgentManager {
|
|
|
3150
3175
|
replDrained: err.replDrained,
|
|
3151
3176
|
});
|
|
3152
3177
|
}
|
|
3153
|
-
async failTasksForAgent(agentId, reason
|
|
3178
|
+
async failTasksForAgent(agentId, reason) {
|
|
3154
3179
|
const failed = await this.withTaskLock(async () => {
|
|
3155
3180
|
const tasks = await this.taskStore.list({});
|
|
3156
3181
|
const out = [];
|
|
@@ -3180,9 +3205,7 @@ export class AgentManager {
|
|
|
3180
3205
|
}
|
|
3181
3206
|
const failedTaskIds = failed.map(t => t.id);
|
|
3182
3207
|
const projectIds = [...new Set(failed.map(t => t.projectId))];
|
|
3183
|
-
|
|
3184
|
-
await this.releasePartnersAndDrain(agentId, failedTaskIds, projectIds);
|
|
3185
|
-
}
|
|
3208
|
+
await this.releasePartnersAndDrain(agentId, failedTaskIds, projectIds);
|
|
3186
3209
|
return { failedTaskIds, projectIds };
|
|
3187
3210
|
}
|
|
3188
3211
|
async releasePartnersAndDrain(excludeAgentId, failedTaskIds, _projectIds) {
|
|
@@ -3849,6 +3872,8 @@ export class AgentManager {
|
|
|
3849
3872
|
? event.data.reason
|
|
3850
3873
|
: 'human-intervention';
|
|
3851
3874
|
const reason = reasonValue.trim() || 'human-intervention';
|
|
3875
|
+
if (reason === 'recovery-failed' && TERMINAL_STATUSES.includes(task.status))
|
|
3876
|
+
return;
|
|
3852
3877
|
const previousPhase = typeof event.data.previousPhase === 'string'
|
|
3853
3878
|
? event.data.previousPhase.trim()
|
|
3854
3879
|
: '';
|
|
@@ -3863,6 +3888,15 @@ export class AgentManager {
|
|
|
3863
3888
|
if (task.attention?.reason !== clearedReason
|
|
3864
3889
|
|| Date.parse(event.timestamp) < Date.parse(task.attention.occurredAt))
|
|
3865
3890
|
return;
|
|
3891
|
+
for (const participantId of [task.agentId, task.qaAgentId]) {
|
|
3892
|
+
if (!participantId || participantId === event.agentId)
|
|
3893
|
+
continue;
|
|
3894
|
+
const participant = await this.agentStore.get(participantId);
|
|
3895
|
+
if (participant?.taskId === task.id
|
|
3896
|
+
&& participant.status === 'awaiting_human'
|
|
3897
|
+
&& participant.awaitingPhase === clearedReason)
|
|
3898
|
+
return;
|
|
3899
|
+
}
|
|
3866
3900
|
const next = { ...task, updatedAt: new Date().toISOString() };
|
|
3867
3901
|
delete next.attention;
|
|
3868
3902
|
await this.taskStore.set(next);
|
|
@@ -3898,9 +3932,14 @@ export class AgentManager {
|
|
|
3898
3932
|
: ['retry'];
|
|
3899
3933
|
}
|
|
3900
3934
|
else if (task.status === 'review') {
|
|
3935
|
+
const qa = task.qaAgentId ? await this.agentStore.get(task.qaAgentId) : null;
|
|
3936
|
+
const qaBound = qa?.taskId === task.id;
|
|
3937
|
+
const qaAdvanceable = !qaBound
|
|
3938
|
+
|| (qa.status === 'awaiting_human'
|
|
3939
|
+
&& (qa.awaitingPhase === 'recovery-failed' || isRecoverableQaDispatchHold(qa)));
|
|
3901
3940
|
recommendedActions = isSpecStagePhase(task.phase)
|
|
3902
|
-
? ['advance', 'cancel']
|
|
3903
|
-
: ['advance', 'verdict', 'cancel'];
|
|
3941
|
+
? (qaAdvanceable ? ['advance', 'cancel'] : ['cancel'])
|
|
3942
|
+
: (qaAdvanceable ? ['advance', 'verdict', 'cancel'] : ['verdict', 'cancel']);
|
|
3904
3943
|
}
|
|
3905
3944
|
else if (task.status === 'spec-ready' || task.status === 'merge-ready') {
|
|
3906
3945
|
recommendedActions = ['verdict', 'cancel'];
|
|
@@ -3914,6 +3953,8 @@ export class AgentManager {
|
|
|
3914
3953
|
if (task.attention
|
|
3915
3954
|
&& task.attention.reason === reason
|
|
3916
3955
|
&& task.attention.runbook === runbook
|
|
3956
|
+
&& task.attention.recommendedActions.length === recommendedActions.length
|
|
3957
|
+
&& task.attention.recommendedActions.every((action, index) => action === recommendedActions[index])
|
|
3917
3958
|
&& Date.parse(task.attention.occurredAt) >= Date.parse(occurredAt))
|
|
3918
3959
|
return;
|
|
3919
3960
|
await this.taskStore.set({
|
|
@@ -5808,6 +5849,7 @@ export class AgentManager {
|
|
|
5808
5849
|
projectId,
|
|
5809
5850
|
taskId: undefined,
|
|
5810
5851
|
lockToken: undefined,
|
|
5852
|
+
bootstrappingTaskId: undefined,
|
|
5811
5853
|
updatedAt: now,
|
|
5812
5854
|
};
|
|
5813
5855
|
});
|
|
@@ -6306,34 +6348,6 @@ export class AgentManager {
|
|
|
6306
6348
|
`${phase} queued for redispatch when idle (${(err.message.split('\n')[0])})`);
|
|
6307
6349
|
return new EnsureSessionError({ createdSession, agentId, handled: true, busyPending: true }, `QA REPL busy; ${phase} dispatch for task ${taskId} queued for redispatch when idle`);
|
|
6308
6350
|
}
|
|
6309
|
-
async clearRuntimeForDispatchBoundary(tmux, pane, agentId, runtime, sessionRef, revalidate) {
|
|
6310
|
-
await this.acquireCompactGuard(agentId);
|
|
6311
|
-
try {
|
|
6312
|
-
await this.waitForReplPromptReady(tmux, pane, runtime, this.cleanComposerWaitMs, { stableIdle: true });
|
|
6313
|
-
await revalidate();
|
|
6314
|
-
await tmux.clearComposerDraft(pane);
|
|
6315
|
-
const baseline = await tmux.captureSettledSnapshot(pane, {
|
|
6316
|
-
timeoutMs: this.dispatchSettleTimeoutMs,
|
|
6317
|
-
});
|
|
6318
|
-
const baselineTitle = await tmux.readPaneTitle(pane);
|
|
6319
|
-
await this.setSessionOptions(tmux, agentId, sessionRef, [[TASK_CONTEXT_SESSION_OPTION, '']]);
|
|
6320
|
-
await tmux.sendKeysLiteral(pane, '/clear');
|
|
6321
|
-
await tmux.sendEnter(pane);
|
|
6322
|
-
await tmux.waitSubmitAck(pane, baseline, runtime, {
|
|
6323
|
-
timeoutMs: this.dispatchAckTimeoutMs,
|
|
6324
|
-
baselineTitle,
|
|
6325
|
-
acceptComposerChange: true,
|
|
6326
|
-
});
|
|
6327
|
-
await this.waitForReplPromptReady(tmux, pane, runtime, this.dispatchAckTimeoutMs, { stableIdle: true });
|
|
6328
|
-
if (await this.hasRuntimeSlashCommandRejection(tmux, pane, '/clear')) {
|
|
6329
|
-
throw new Error('Runtime rejected /clear at the dispatch boundary; refusing to reuse prior context');
|
|
6330
|
-
}
|
|
6331
|
-
await revalidate();
|
|
6332
|
-
}
|
|
6333
|
-
finally {
|
|
6334
|
-
this.compactInFlight.delete(agentId);
|
|
6335
|
-
}
|
|
6336
|
-
}
|
|
6337
6351
|
async persistTaskImages(taskId, images) {
|
|
6338
6352
|
const dir = join(this.imageStagingRoot, taskId);
|
|
6339
6353
|
await mkdir(dir, { recursive: true });
|
|
@@ -6742,9 +6756,6 @@ export class AgentManager {
|
|
|
6742
6756
|
await this.clearBranchLocalCleaned(taskId);
|
|
6743
6757
|
}
|
|
6744
6758
|
await assertOwner();
|
|
6745
|
-
if (!ensure.freshRuntime && !reuseTaskContext) {
|
|
6746
|
-
await this.clearRuntimeForDispatchBoundary(tmux, pane, agentId, agent.runtime, ensure.sessionRef, assertOwner);
|
|
6747
|
-
}
|
|
6748
6759
|
}
|
|
6749
6760
|
catch (err) {
|
|
6750
6761
|
const busyPend = await this.queueQaBusyPendingRetry(taskId, agentId, phase, ensure.createdSession, err, { passToken: opts.dispatchPassToken, pendingBudget: opts.dispatchPendingBudget });
|
|
@@ -7401,31 +7412,6 @@ export class AgentManager {
|
|
|
7401
7412
|
return false;
|
|
7402
7413
|
}
|
|
7403
7414
|
const guardBeforeInject = opts.guardBeforeInject ?? (phase === 'post-approve' ? postApprovePassStillLive : undefined);
|
|
7404
|
-
if (!ensure.freshRuntime && !reuseTaskContext) {
|
|
7405
|
-
const revalidateContextReset = async () => {
|
|
7406
|
-
const [currentTask, currentAgent] = await Promise.all([
|
|
7407
|
-
this.taskStore.get(taskId),
|
|
7408
|
-
this.agentStore.get(agentId),
|
|
7409
|
-
]);
|
|
7410
|
-
const taskStillDispatchable = currentTask !== null
|
|
7411
|
-
&& !TERMINAL_STATUSES.includes(currentTask.status)
|
|
7412
|
-
&& (opts.bypassTaskStatusGate || expectedStatuses.includes(currentTask.status));
|
|
7413
|
-
const bindingStillDispatchable = PHASE_REQUIRES_AGENT_BOUND_TO_TASK[phase]
|
|
7414
|
-
? currentAgent?.taskId === taskId
|
|
7415
|
-
: currentAgent?.taskId === undefined || currentAgent.taskId === taskId;
|
|
7416
|
-
if (!taskStillDispatchable
|
|
7417
|
-
|| !bindingStillDispatchable
|
|
7418
|
-
|| currentAgent?.lockToken !== lockToken
|
|
7419
|
-
|| currentAgent.workdir !== verifiedWorkdir) {
|
|
7420
|
-
throw new Error(`Task or agent ownership changed while resetting context for ${phase} dispatch`);
|
|
7421
|
-
}
|
|
7422
|
-
await this.assertTaskLockOwner(agentId, taskId, lockToken);
|
|
7423
|
-
if (guardBeforeInject && !(await guardBeforeInject())) {
|
|
7424
|
-
throw new Error(`Task generation changed while resetting context for ${phase} dispatch`);
|
|
7425
|
-
}
|
|
7426
|
-
};
|
|
7427
|
-
await this.clearRuntimeForDispatchBoundary(tmux, pane, agentId, agent.runtime, ensure.sessionRef, revalidateContextReset);
|
|
7428
|
-
}
|
|
7429
7415
|
if (opts.armBeforeInject && !(await opts.armBeforeInject())) {
|
|
7430
7416
|
return false;
|
|
7431
7417
|
}
|
|
@@ -7472,6 +7458,37 @@ export class AgentManager {
|
|
|
7472
7458
|
}
|
|
7473
7459
|
return true;
|
|
7474
7460
|
}
|
|
7461
|
+
async clearRecoveredBootstrapHold(state) {
|
|
7462
|
+
if (!state.taskId || !state.awaitingPhase)
|
|
7463
|
+
return;
|
|
7464
|
+
let cleared = false;
|
|
7465
|
+
await this.agentStore.update(state.id, (latest) => {
|
|
7466
|
+
if (!latest || latest.taskId !== undefined)
|
|
7467
|
+
return AGENT_STORE_NOOP;
|
|
7468
|
+
if (latest.awaitingPhase !== state.awaitingPhase
|
|
7469
|
+
|| latest.awaitingSince !== state.awaitingSince
|
|
7470
|
+
|| latest.awaitingNonce !== state.awaitingNonce)
|
|
7471
|
+
return AGENT_STORE_NOOP;
|
|
7472
|
+
const { status: _status, awaitingPhase: _phase, awaitingReason: _reason, awaitingSince: _since, awaitingNonce: _nonce, ...ready } = latest;
|
|
7473
|
+
cleared = true;
|
|
7474
|
+
return { ...ready, updatedAt: new Date().toISOString() };
|
|
7475
|
+
});
|
|
7476
|
+
if (!cleared)
|
|
7477
|
+
return;
|
|
7478
|
+
const intervention = {
|
|
7479
|
+
id: '',
|
|
7480
|
+
type: 'human.intervention',
|
|
7481
|
+
timestamp: new Date().toISOString(),
|
|
7482
|
+
projectId: state.projectId,
|
|
7483
|
+
agentId: state.id,
|
|
7484
|
+
taskId: state.taskId,
|
|
7485
|
+
data: { phase: 'resumed', previousPhase: state.awaitingPhase, releasedBinding: true },
|
|
7486
|
+
};
|
|
7487
|
+
await this.safeEmit(intervention);
|
|
7488
|
+
await this.recordTaskAttention(intervention).catch(err => {
|
|
7489
|
+
console.warn(`[recover] could not clear ${state.awaitingPhase} attention for ${state.id}/${state.taskId}:`, err);
|
|
7490
|
+
});
|
|
7491
|
+
}
|
|
7475
7492
|
async rollbackUndeliveredBootstrap(state) {
|
|
7476
7493
|
if (!state.taskId
|
|
7477
7494
|
|| state.bootstrappingTaskId !== state.taskId
|
|
@@ -7481,6 +7498,14 @@ export class AgentManager {
|
|
|
7481
7498
|
const boundTask = await this.taskStore.get(state.taskId);
|
|
7482
7499
|
if (!boundTask)
|
|
7483
7500
|
return false;
|
|
7501
|
+
if (boundTask.status === 'pending') {
|
|
7502
|
+
console.warn(`[recover] agent ${state.id} retained a pending bootstrap binding for ${state.taskId}; releasing it`);
|
|
7503
|
+
await this.rollbackFailedDispatch(state.taskId, state.id, undefined, state.lockToken);
|
|
7504
|
+
const released = (await this.agentStore.get(state.id))?.taskId !== state.taskId;
|
|
7505
|
+
if (released)
|
|
7506
|
+
await this.clearRecoveredBootstrapHold(state);
|
|
7507
|
+
return released;
|
|
7508
|
+
}
|
|
7484
7509
|
if (boundTask.status === 'spec-ready' && isSpecStagePhase(boundTask.phase)) {
|
|
7485
7510
|
if (state.id === boundTask.devAgentId && state.id !== boundTask.agentId) {
|
|
7486
7511
|
const lockToken = state.lockToken;
|
|
@@ -7521,12 +7546,7 @@ export class AgentManager {
|
|
|
7521
7546
|
console.warn(`[recover] agent ${state.id} was mid-bootstrap for in_progress task ${state.taskId} ` +
|
|
7522
7547
|
`(prompt never ack'd); rolling the task back to pending`);
|
|
7523
7548
|
await this.rollbackFailedDispatch(state.taskId, state.id, undefined, state.lockToken);
|
|
7524
|
-
await this.
|
|
7525
|
-
if (!latest || latest.status !== 'awaiting_human')
|
|
7526
|
-
return AGENT_STORE_NOOP;
|
|
7527
|
-
const { status: _s, awaitingPhase: _p, awaitingReason: _r, awaitingSince: _a, ...rest } = latest;
|
|
7528
|
-
return { ...rest, updatedAt: new Date().toISOString() };
|
|
7529
|
-
});
|
|
7549
|
+
await this.clearRecoveredBootstrapHold(state);
|
|
7530
7550
|
return true;
|
|
7531
7551
|
}
|
|
7532
7552
|
async bootstrapPromptWasDelivered(taskId, createdAtIso, agentId, phase) {
|
|
@@ -7617,7 +7637,6 @@ export class AgentManager {
|
|
|
7617
7637
|
});
|
|
7618
7638
|
const states = await this.agentStore.list();
|
|
7619
7639
|
await this.releaseOrphanedLocks(states);
|
|
7620
|
-
const deferredCleanups = [];
|
|
7621
7640
|
for (const state of states) {
|
|
7622
7641
|
const agentConfig = this.getAgentConfig(state.id);
|
|
7623
7642
|
if (!agentConfig)
|
|
@@ -7692,18 +7711,11 @@ export class AgentManager {
|
|
|
7692
7711
|
|| recovered.status === 'awaiting_human') {
|
|
7693
7712
|
continue;
|
|
7694
7713
|
}
|
|
7695
|
-
|
|
7696
|
-
|
|
7697
|
-
taskId: boundTask.id,
|
|
7698
|
-
branch: boundTask.branch,
|
|
7699
|
-
});
|
|
7700
|
-
continue;
|
|
7701
|
-
}
|
|
7702
|
-
if (this.startCompactionThenRelease(recovered.id, recovered, boundTask.id))
|
|
7703
|
-
continue;
|
|
7714
|
+
await this.releaseAgentForTask(state.id, boundTask.id, 'idle');
|
|
7715
|
+
continue;
|
|
7704
7716
|
}
|
|
7705
7717
|
catch (cleanupErr) {
|
|
7706
|
-
console.warn(`[recover]
|
|
7718
|
+
console.warn(`[recover] releaseAgentForTask(${state.id}, ${boundTask.id}) failed:`, cleanupErr);
|
|
7707
7719
|
}
|
|
7708
7720
|
}
|
|
7709
7721
|
const cancelHold = isCancelCleanupHold(state);
|
|
@@ -7774,6 +7786,17 @@ export class AgentManager {
|
|
|
7774
7786
|
}
|
|
7775
7787
|
const message = err instanceof Error ? err.message : String(err);
|
|
7776
7788
|
console.warn(`[recover] ensureSession failed for agent=${state.id}: ${message}`);
|
|
7789
|
+
const boundTask = state.taskId ? await this.taskStore.get(state.taskId) : null;
|
|
7790
|
+
const attentionTaskId = state.taskId
|
|
7791
|
+
&& boundTask
|
|
7792
|
+
&& !TERMINAL_STATUSES.includes(boundTask.status)
|
|
7793
|
+
? state.taskId
|
|
7794
|
+
: undefined;
|
|
7795
|
+
const runbook = attentionTaskId
|
|
7796
|
+
? RECOVERY_FAILED_TASK_RUNBOOK
|
|
7797
|
+
: state.taskId
|
|
7798
|
+
? RECOVERY_FAILED_AGENT_RUNBOOK
|
|
7799
|
+
: RECOVERY_FAILED_UNBOUND_RUNBOOK;
|
|
7777
7800
|
await this.agentStore.update(state.id, (latest) => {
|
|
7778
7801
|
if (!latest)
|
|
7779
7802
|
return AGENT_STORE_NOOP;
|
|
@@ -7790,12 +7813,6 @@ export class AgentManager {
|
|
|
7790
7813
|
updatedAt: new Date().toISOString(),
|
|
7791
7814
|
};
|
|
7792
7815
|
});
|
|
7793
|
-
const cleanup = await this.failTasksForAgent(state.id, `recovery: ${message}`, { deferPartnerCleanup: true });
|
|
7794
|
-
deferredCleanups.push({
|
|
7795
|
-
failingAgentId: state.id,
|
|
7796
|
-
failedTaskIds: cleanup.failedTaskIds,
|
|
7797
|
-
projectIds: cleanup.projectIds,
|
|
7798
|
-
});
|
|
7799
7816
|
await this.recordError({
|
|
7800
7817
|
agentId: state.id,
|
|
7801
7818
|
projectId: state.projectId,
|
|
@@ -7804,14 +7821,25 @@ export class AgentManager {
|
|
|
7804
7821
|
reason: 'RECOVERY_ENSURE_SESSION_FAILED',
|
|
7805
7822
|
message,
|
|
7806
7823
|
observation: { phase: 'recovery-failed' },
|
|
7807
|
-
recommendation:
|
|
7824
|
+
recommendation: runbook,
|
|
7808
7825
|
});
|
|
7809
|
-
|
|
7826
|
+
const intervention = {
|
|
7827
|
+
id: '',
|
|
7828
|
+
type: 'human.intervention',
|
|
7829
|
+
timestamp: new Date().toISOString(),
|
|
7830
|
+
projectId: state.projectId,
|
|
7831
|
+
agentId: state.id,
|
|
7832
|
+
...(attentionTaskId ? { taskId: attentionTaskId } : {}),
|
|
7833
|
+
data: { phase: 'recovery-failed', error: message, note: runbook },
|
|
7834
|
+
};
|
|
7835
|
+
await this.safeEmit(intervention);
|
|
7836
|
+
if (attentionTaskId) {
|
|
7837
|
+
await this.recordTaskAttention(intervention).catch(attentionErr => {
|
|
7838
|
+
console.warn(`[recover] recordTaskAttention for ${state.id}/${attentionTaskId} failed:`, attentionErr);
|
|
7839
|
+
});
|
|
7840
|
+
}
|
|
7810
7841
|
}
|
|
7811
7842
|
}
|
|
7812
|
-
for (const c of deferredCleanups) {
|
|
7813
|
-
await this.releasePartnersAndDrain(c.failingAgentId, c.failedTaskIds, c.projectIds);
|
|
7814
|
-
}
|
|
7815
7843
|
await this.recoverClaimedGitReviewDispatches().catch(err => {
|
|
7816
7844
|
console.warn('[AgentManager] recover: claimed git review recovery failed:', err);
|
|
7817
7845
|
});
|
|
@@ -8889,108 +8917,18 @@ export class AgentManager {
|
|
|
8889
8917
|
if (!dev)
|
|
8890
8918
|
return;
|
|
8891
8919
|
this.phaseSignalWatcher?.stop(taskId);
|
|
8892
|
-
|
|
8893
|
-
const ctx = {
|
|
8894
|
-
taskId: task.id,
|
|
8895
|
-
branch: task.branch,
|
|
8896
|
-
};
|
|
8897
|
-
await this.dispatchPostMergeCleanup(task.agentId, ctx).catch(err => console.warn(`[AgentManager] cleanupAfterMerge: dispatchPostMergeCleanup(${task.agentId}) failed:`, err));
|
|
8898
|
-
}
|
|
8899
|
-
else {
|
|
8900
|
-
await this.releaseAgentForTask(task.agentId, taskId, 'idle').catch(err => console.warn(`[AgentManager] cleanupAfterMerge: releaseAgentForTask(${task.agentId}, ${taskId}) failed:`, err));
|
|
8901
|
-
}
|
|
8920
|
+
this.startTaskAgentRelease(task.agentId, taskId);
|
|
8902
8921
|
}
|
|
8903
|
-
|
|
8904
|
-
|
|
8905
|
-
|
|
8906
|
-
|
|
8907
|
-
|
|
8908
|
-
const state = await this.agentStore.get(agentId);
|
|
8909
|
-
if (!state)
|
|
8910
|
-
return;
|
|
8911
|
-
if (state.taskId && state.taskId !== ctx.taskId)
|
|
8912
|
-
return;
|
|
8913
|
-
if (state.creationToken)
|
|
8914
|
-
return;
|
|
8915
|
-
if (state.status === 'awaiting_human')
|
|
8916
|
-
return;
|
|
8917
|
-
if (!state.paneId) {
|
|
8918
|
-
await this.releasePostMergeAgent(agentId, ctx.taskId);
|
|
8919
|
-
return;
|
|
8920
|
-
}
|
|
8921
|
-
if (!state.taskId) {
|
|
8922
|
-
if (this.isDeletionInFlight(agentId))
|
|
8923
|
-
return;
|
|
8924
|
-
const lockToken = await this.lockManager.acquire(agentId, ctx.taskId);
|
|
8925
|
-
if (!lockToken)
|
|
8926
|
-
return;
|
|
8927
|
-
const commit = await this.agentStore.update(agentId, (existing) => {
|
|
8928
|
-
if (!existing)
|
|
8929
|
-
return AGENT_STORE_NOOP;
|
|
8930
|
-
if (!this.deletionGateOpen(agentId, genAtEntry))
|
|
8931
|
-
return AGENT_STORE_NOOP;
|
|
8932
|
-
if (existing.taskId && existing.taskId !== ctx.taskId)
|
|
8933
|
-
return AGENT_STORE_NOOP;
|
|
8934
|
-
if (existing.creationToken)
|
|
8935
|
-
return AGENT_STORE_NOOP;
|
|
8936
|
-
if (existing.status === 'awaiting_human')
|
|
8937
|
-
return AGENT_STORE_NOOP;
|
|
8938
|
-
if (existing.paneId !== state.paneId)
|
|
8939
|
-
return AGENT_STORE_NOOP;
|
|
8940
|
-
return {
|
|
8941
|
-
...existing,
|
|
8942
|
-
taskId: ctx.taskId,
|
|
8943
|
-
lockToken,
|
|
8944
|
-
updatedAt: new Date().toISOString(),
|
|
8945
|
-
};
|
|
8946
|
-
}).catch(async (err) => {
|
|
8947
|
-
await this.lockManager.releaseIfOwner(agentId, ctx.taskId, lockToken).catch((releaseErr) => {
|
|
8948
|
-
console.warn(`[AgentManager] postMergeCleanup(${agentId}): lock release after commit failure also failed:`, releaseErr);
|
|
8949
|
-
});
|
|
8950
|
-
throw err;
|
|
8951
|
-
});
|
|
8952
|
-
if (commit !== 'committed') {
|
|
8953
|
-
await this.lockManager.releaseIfOwner(agentId, ctx.taskId, lockToken);
|
|
8954
|
-
return;
|
|
8922
|
+
startTaskAgentRelease(agentId, taskId, opts = {}) {
|
|
8923
|
+
void (async () => {
|
|
8924
|
+
await this.acquireCompactGuard(agentId);
|
|
8925
|
+
try {
|
|
8926
|
+
await this.releaseAgentForTask(agentId, taskId, 'idle', opts);
|
|
8955
8927
|
}
|
|
8956
|
-
|
|
8957
|
-
|
|
8958
|
-
await this.lockManager.releaseIfOwner(agentId, ctx.taskId, lockToken);
|
|
8959
|
-
return;
|
|
8928
|
+
finally {
|
|
8929
|
+
this.compactInFlight.delete(agentId);
|
|
8960
8930
|
}
|
|
8961
|
-
}
|
|
8962
|
-
if (!this.deletionGateOpen(agentId, genAtEntry))
|
|
8963
|
-
return;
|
|
8964
|
-
const runner = this.createRunnerFor(agent);
|
|
8965
|
-
const tmux = new TmuxManager(runner);
|
|
8966
|
-
const runtime = agentRuntimeKindFor(agent);
|
|
8967
|
-
const expectedPaneId = state.paneId;
|
|
8968
|
-
void (async () => {
|
|
8969
|
-
if (!this.deletionGateOpen(agentId, genAtEntry))
|
|
8970
|
-
return;
|
|
8971
|
-
const pane = await this.resolveClaimedPane(tmux, agentId, expectedPaneId);
|
|
8972
|
-
await this.runPostMergeCompaction(tmux, pane, agentId, ctx.taskId, runtime);
|
|
8973
|
-
})().catch(err => console.warn(`[AgentManager] runPostMergeCompaction(${agentId}) failed:`, err));
|
|
8974
|
-
}
|
|
8975
|
-
async releasePostMergeAgent(agentId, taskId) {
|
|
8976
|
-
const state = await this.agentStore.get(agentId);
|
|
8977
|
-
if (state?.taskId !== taskId)
|
|
8978
|
-
return;
|
|
8979
|
-
try {
|
|
8980
|
-
await this.releaseAgentForTask(agentId, taskId, 'idle');
|
|
8981
|
-
}
|
|
8982
|
-
catch (err) {
|
|
8983
|
-
console.warn(`[AgentManager] releasePostMergeAgent: releaseAgentForTask(${agentId}, ${taskId}) failed:`, err);
|
|
8984
|
-
}
|
|
8985
|
-
}
|
|
8986
|
-
async runPostMergeCompaction(tmux, pane, agentId, originalTaskId, runtime) {
|
|
8987
|
-
await this.acquireCompactGuard(agentId);
|
|
8988
|
-
try {
|
|
8989
|
-
await this.runPostMergeCompactionSteps(tmux, pane, agentId, originalTaskId, runtime);
|
|
8990
|
-
}
|
|
8991
|
-
finally {
|
|
8992
|
-
this.compactInFlight.delete(agentId);
|
|
8993
|
-
}
|
|
8931
|
+
})().catch(err => console.warn(`[AgentManager] background release(${agentId}, ${taskId}) failed:`, err));
|
|
8994
8932
|
}
|
|
8995
8933
|
async acquireCompactGuard(agentId) {
|
|
8996
8934
|
while (!this.tryAcquireCompactGuard(agentId)) {
|
|
@@ -9012,144 +8950,6 @@ export class AgentManager {
|
|
|
9012
8950
|
this.compactInFlight.add(agentId);
|
|
9013
8951
|
return true;
|
|
9014
8952
|
}
|
|
9015
|
-
async runPostMergeCompactionSteps(tmux, pane, agentId, originalTaskId, runtime) {
|
|
9016
|
-
const paneId = pane.paneId;
|
|
9017
|
-
const initial = await this.agentStore.get(agentId);
|
|
9018
|
-
if (!initial || initial.taskId !== originalTaskId || initial.paneId !== paneId)
|
|
9019
|
-
return;
|
|
9020
|
-
const lockToken = await this.resolveTaskLockToken(initial, originalTaskId);
|
|
9021
|
-
if (!lockToken)
|
|
9022
|
-
return;
|
|
9023
|
-
const bindingStillOurs = async () => {
|
|
9024
|
-
const s = await this.agentStore.get(agentId);
|
|
9025
|
-
return !!s
|
|
9026
|
-
&& s.taskId === originalTaskId
|
|
9027
|
-
&& s.lockToken === lockToken
|
|
9028
|
-
&& s.paneId === paneId
|
|
9029
|
-
&& await this.lockManager.isOwner(agentId, originalTaskId, lockToken);
|
|
9030
|
-
};
|
|
9031
|
-
if (!await bindingStillOurs())
|
|
9032
|
-
return;
|
|
9033
|
-
let cleared = false;
|
|
9034
|
-
let clearError;
|
|
9035
|
-
try {
|
|
9036
|
-
cleared = await this.sendPostMergeSlashCommand(tmux, pane, agentId, runtime, bindingStillOurs);
|
|
9037
|
-
}
|
|
9038
|
-
catch (err) {
|
|
9039
|
-
clearError = err;
|
|
9040
|
-
}
|
|
9041
|
-
if (!cleared) {
|
|
9042
|
-
if (await this.recoverPostMergeExitedRuntime(tmux, pane, agentId, originalTaskId, lockToken, runtime)) {
|
|
9043
|
-
cleared = true;
|
|
9044
|
-
}
|
|
9045
|
-
else if (!await bindingStillOurs()) {
|
|
9046
|
-
return;
|
|
9047
|
-
}
|
|
9048
|
-
else if (clearError instanceof Error
|
|
9049
|
-
&& clearError.message.includes('runtime rejected /clear')) {
|
|
9050
|
-
try {
|
|
9051
|
-
await this.waitForReplPromptReady(tmux, pane, runtime, this.compactIdleWaitMs);
|
|
9052
|
-
}
|
|
9053
|
-
catch (err) {
|
|
9054
|
-
clearError = err;
|
|
9055
|
-
}
|
|
9056
|
-
if (clearError instanceof Error && !clearError.message.includes('runtime rejected /clear')) {
|
|
9057
|
-
await this.markAwaitingHuman(agentId, 'post-merge-cleanup-not-idle', `Task finished, but the runtime could not be proven idle after /clear: ${clearError.message}`, { expectedTaskId: originalTaskId });
|
|
9058
|
-
return;
|
|
9059
|
-
}
|
|
9060
|
-
}
|
|
9061
|
-
else {
|
|
9062
|
-
const reason = clearError instanceof Error ? clearError.message : String(clearError ?? 'unknown error');
|
|
9063
|
-
await this.markAwaitingHuman(agentId, 'post-merge-cleanup-not-idle', `Task finished, but /clear and the runtime idle check did not complete safely: ${reason}`, { expectedTaskId: originalTaskId });
|
|
9064
|
-
return;
|
|
9065
|
-
}
|
|
9066
|
-
}
|
|
9067
|
-
await this.releasePostMergeAgent(agentId, originalTaskId);
|
|
9068
|
-
}
|
|
9069
|
-
async recoverPostMergeExitedRuntime(tmux, pane, agentId, taskId, lockToken, runtime) {
|
|
9070
|
-
const paneId = pane.paneId;
|
|
9071
|
-
let paneExists = true;
|
|
9072
|
-
try {
|
|
9073
|
-
if (hasReplProcTitle(await tmux.displayMessage(pane, '#{pane_current_command}'), runtime))
|
|
9074
|
-
return false;
|
|
9075
|
-
}
|
|
9076
|
-
catch (err) {
|
|
9077
|
-
const detail = err instanceof Error ? err.message : String(err);
|
|
9078
|
-
if (!/no server running|session not found|can't find (?:pane|session)|no such (?:pane|session)/i.test(detail)) {
|
|
9079
|
-
return false;
|
|
9080
|
-
}
|
|
9081
|
-
paneExists = false;
|
|
9082
|
-
}
|
|
9083
|
-
const state = await this.agentStore.get(agentId);
|
|
9084
|
-
if (!state
|
|
9085
|
-
|| state.taskId !== taskId
|
|
9086
|
-
|| state.lockToken !== lockToken
|
|
9087
|
-
|| state.paneId !== paneId
|
|
9088
|
-
|| !(await this.lockManager.isOwner(agentId, taskId, lockToken)))
|
|
9089
|
-
return false;
|
|
9090
|
-
try {
|
|
9091
|
-
if (paneExists)
|
|
9092
|
-
await tmux.sendKeysToPane(pane, 'C-c');
|
|
9093
|
-
const ensure = await this.ensureSession(agentId, 'runtime');
|
|
9094
|
-
if (!ensure.freshRuntime)
|
|
9095
|
-
return false;
|
|
9096
|
-
await this.agentStore.update(agentId, (existing) => {
|
|
9097
|
-
if (!existing || existing.taskId !== taskId || existing.lockToken !== lockToken) {
|
|
9098
|
-
return AGENT_STORE_NOOP;
|
|
9099
|
-
}
|
|
9100
|
-
if (existing.paneId === ensure.paneId && existing.workdir === ensure.workdir) {
|
|
9101
|
-
return AGENT_STORE_NOOP;
|
|
9102
|
-
}
|
|
9103
|
-
return {
|
|
9104
|
-
...existing,
|
|
9105
|
-
paneId: ensure.paneId,
|
|
9106
|
-
workdir: ensure.workdir,
|
|
9107
|
-
updatedAt: new Date().toISOString(),
|
|
9108
|
-
};
|
|
9109
|
-
});
|
|
9110
|
-
const current = await this.agentStore.get(agentId);
|
|
9111
|
-
return current?.taskId === taskId
|
|
9112
|
-
&& current.lockToken === lockToken
|
|
9113
|
-
&& current.paneId === ensure.paneId
|
|
9114
|
-
&& await this.lockManager.isOwner(agentId, taskId, lockToken);
|
|
9115
|
-
}
|
|
9116
|
-
catch (err) {
|
|
9117
|
-
console.warn(`[AgentManager] recoverPostMergeExitedRuntime(${agentId}, ${taskId}) failed:`, err);
|
|
9118
|
-
return false;
|
|
9119
|
-
}
|
|
9120
|
-
}
|
|
9121
|
-
async sendPostMergeSlashCommand(tmux, pane, agentId, runtime, bindingStillOurs) {
|
|
9122
|
-
let rejection;
|
|
9123
|
-
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
9124
|
-
if (!await bindingStillOurs())
|
|
9125
|
-
return false;
|
|
9126
|
-
await tmux.clearComposerDraft(pane);
|
|
9127
|
-
await this.waitForReplPromptReady(tmux, pane, runtime, this.compactIdleWaitMs);
|
|
9128
|
-
if (!await bindingStillOurs())
|
|
9129
|
-
return false;
|
|
9130
|
-
await tmux.sendKeysLiteral(pane, '/clear');
|
|
9131
|
-
await tmux.sendEnter(pane);
|
|
9132
|
-
await new Promise(r => setTimeout(r, this.compactIdlePollMs));
|
|
9133
|
-
await this.waitForReplPromptReady(tmux, pane, runtime, this.compactIdleWaitMs);
|
|
9134
|
-
if (!await bindingStillOurs())
|
|
9135
|
-
return false;
|
|
9136
|
-
if (!await this.hasRuntimeSlashCommandRejection(tmux, pane, '/clear'))
|
|
9137
|
-
return true;
|
|
9138
|
-
rejection = new Error('runtime rejected /clear because a task is still in progress');
|
|
9139
|
-
if (attempt < 2) {
|
|
9140
|
-
console.warn(`[AgentManager] runPostMergeCompaction(${agentId}) /clear rejected; retrying`);
|
|
9141
|
-
await new Promise(r => setTimeout(r, this.compactIdlePollMs));
|
|
9142
|
-
}
|
|
9143
|
-
}
|
|
9144
|
-
throw rejection ?? new Error('runtime rejected /clear');
|
|
9145
|
-
}
|
|
9146
|
-
async hasRuntimeSlashCommandRejection(tmux, pane, command) {
|
|
9147
|
-
const cap = await tmux.capturePaneById(pane, { ansi: false, scrollback: 0 });
|
|
9148
|
-
return this.runtimeSlashCommandRejectedPattern(command).test(cap);
|
|
9149
|
-
}
|
|
9150
|
-
runtimeSlashCommandRejectedPattern(command) {
|
|
9151
|
-
return new RegExp(`["'“”‘’]?${command}["'“”‘’]?\\s+is disabled while a task is in progress\\.`, 'gi');
|
|
9152
|
-
}
|
|
9153
8953
|
async waitForReplPromptReady(tmux, pane, runtime, timeoutMs, opts = {}) {
|
|
9154
8954
|
if (opts.stableIdle) {
|
|
9155
8955
|
return this.waitForReplPromptStableIdle(tmux, pane, runtime, timeoutMs);
|
|
@@ -10224,28 +10024,9 @@ export class AgentManager {
|
|
|
10224
10024
|
const state = await this.agentStore.get(id);
|
|
10225
10025
|
if (!state || state.taskId !== taskId)
|
|
10226
10026
|
continue;
|
|
10227
|
-
|
|
10228
|
-
continue;
|
|
10229
|
-
await this.releaseAgentForTask(id, taskId, 'idle', { allowAwaitingHuman: true })
|
|
10230
|
-
.catch(err => {
|
|
10231
|
-
console.warn(`[AgentManager] confirm release ${id} failed:`, err);
|
|
10232
|
-
});
|
|
10027
|
+
this.startTaskAgentRelease(id, taskId, { allowAwaitingHuman: true });
|
|
10233
10028
|
}
|
|
10234
10029
|
}
|
|
10235
|
-
startCompactionThenRelease(agentId, state, taskId) {
|
|
10236
|
-
const expectedPaneId = state.paneId;
|
|
10237
|
-
if (!expectedPaneId || state.status === 'awaiting_human' || state.creationToken)
|
|
10238
|
-
return false;
|
|
10239
|
-
const cfg = this.getAgentConfig(agentId);
|
|
10240
|
-
if (!cfg)
|
|
10241
|
-
return false;
|
|
10242
|
-
const tmux = new TmuxManager(this.createRunnerFor(cfg));
|
|
10243
|
-
void (async () => {
|
|
10244
|
-
const pane = await this.resolveClaimedPane(tmux, agentId, expectedPaneId);
|
|
10245
|
-
await this.runPostMergeCompaction(tmux, pane, agentId, taskId, agentRuntimeKindFor(cfg));
|
|
10246
|
-
})().catch(err => console.warn(`[AgentManager] releaseTaskAgents compaction(${agentId}) failed:`, err));
|
|
10247
|
-
return true;
|
|
10248
|
-
}
|
|
10249
10030
|
}
|
|
10250
10031
|
function buildAgentIndex(config) {
|
|
10251
10032
|
const index = new Map();
|