@keyou007/agent-bot 0.1.24-alpha.4 → 0.1.24-alpha.5

package/dist/proxy/ProxySessionController.js

@@ -48,6 +48,7 @@ const REMOTE_TURN_HISTORY_REFRESH_INTERVAL_MS = 30_000;
 const MAX_LLM_TURN_RETRIES = 3;
 const QUEUED_PROMPT_RETRY_DELAY_MS = 5_000;
 const RECOVERY_ACTIVITY_WINDOW_MS = 10 * 60 * 1_000;
+const RECOVERY_RETRY_WINDOW_MS = 60_000;
 const RECOVERY_HEARTBEAT_INTERVAL_MS = 60 * 1_000;
 const SHELL_COMMAND_JOB_POLL_INTERVAL_MS = 1_000;
 const SHELL_COMMAND_JOB_RELAUNCH_DELAY_MS = 5_000;
@@ -241,6 +242,9 @@ export class ProxySessionController {
     retriedFailureTurnIds = new Set();
     queuedPromptRetryTimers = new Map();
     recoveryRetryTimers = new Map();
+    recoveryRetryStartedAt = new Map();
+    recoveredLiveAttempts = new Set();
+    recoveryClosed = false;
     pendingMergedForwards = new Map();
     pendingResourceForwards = new Map();
     relatedReactionMessageIds = new Map();
@@ -812,7 +816,7 @@ export class ProxySessionController {
                     if (scopedAction.value.cardView === "status")
                         await this.refreshStatusCardFromAction(scopedAction, sessionId);
                     else if (scopedAction.value.cardView !== "safe_restart") {
-                        await this.refreshSessionsCardFromAction(scopedAction, { forceSwitchTaskId: sessionId });
+                        await this.refreshSessionsCardFromAction(scopedAction);
                     }
                 }
                 else if (kind === "session_turns" || kind === "session_turns_page") {
@@ -953,6 +957,7 @@ export class ProxySessionController {
         }
     }
     close() {
+        this.recoveryClosed = true;
         for (const unsubscribe of this.unsubscribe)
             unsubscribe();
         this.unsubscribe.length = 0;
@@ -977,6 +982,8 @@ export class ProxySessionController {
         for (const timer of this.recoveryRetryTimers.values())
             clearTimeout(timer);
         this.recoveryRetryTimers.clear();
+        this.recoveryRetryStartedAt.clear();
+        this.recoveredLiveAttempts.clear();
         for (const schedule of this.appServerReleaseSchedules.values()) {
             if (schedule.timer)
                 clearTimeout(schedule.timer);
@@ -1727,12 +1734,16 @@ export class ProxySessionController {
             }
             if (!this.outbound.canRoute(attempt.contextKey))
                 continue;
+            this.recoveryRetryStartedAt.set(attempt.attemptId, Date.now());
             try {
                 await this.recoverTurnAttempt(attempt);
             }
             catch (error) {
-                this.logger.error({ error, attemptId: attempt.attemptId, sessionId: attempt.localSessionId }, "Failed to recover an unfinished task; it will be retried while the server is running.");
-                this.scheduleRecoveryRetry(attempt.attemptId);
+                await this.handleRecoveryError(attempt.attemptId, error);
+            }
+            finally {
+                if (!this.recoveryRetryTimers.has(attempt.attemptId))
+                    this.recoveryRetryStartedAt.delete(attempt.attemptId);
             }
         }
         this.restorePersistedQueuedPrompts();
@@ -1815,22 +1826,36 @@ export class ProxySessionController {
             });
         }
     }
-    async expireStaleTurnAttempt(attempt) {
-        const session = this.store.getSession(attempt.localSessionId);
+    async expireStaleTurnAttempt(attempt, message = "执行中断已超过 10 分钟，未自动恢复。") {
+        clearTimeout(this.recoveryRetryTimers.get(attempt.attemptId));
+        this.recoveryRetryTimers.delete(attempt.attemptId);
+        this.recoveryRetryStartedAt.delete(attempt.attemptId);
+        this.recoveredLiveAttempts.delete(attempt.attemptId);
+        let session = this.store.getSession(attempt.localSessionId);
+        this.store.updateTurnAttempt(attempt.attemptId, { status: "interrupted" });
         const oldCardTurnId = attempt.pendingTurnId ?? attempt.turnId;
         if (session && oldCardTurnId && this.outbound.canRoute(attempt.contextKey)) {
-            await this.outbound.interruptTurnForRecovery(session.localSessionId, attempt.contextKey, oldCardTurnId, "执行中断已超过 10 分钟，未自动恢复。").catch((error) => {
+            await this.outbound.interruptTurnForRecovery(session.localSessionId, attempt.contextKey, oldCardTurnId, message).catch((error) => {
                 this.logger.warn({ error, attemptId: attempt.attemptId, turnId: oldCardTurnId }, "Failed to close an expired interrupted thinking card.");
             });
         }
-        this.store.updateTurnAttempt(attempt.attemptId, { status: "interrupted" });
+        session = this.store.getSession(attempt.localSessionId);
         if (!session || session.status === "closed")
             return;
-        if (attempt.turnId && session.lastTurnId === attempt.turnId) {
+        const runtime = this.runtimes.forAgent(session.agentName);
+        const activeTurnId = runtime.getSession(session.localSessionId)?.activeTurnId;
+        const sameTurn = session.lastTurnId === attempt.turnId
+            || session.lastTurnId === attempt.recoveredFromTurnId
+            || !session.lastTurnId;
+        if (sameTurn && session.lastTurnStatus !== "completed" && session.lastTurnStatus !== "failed"
+            && (!activeTurnId || activeTurnId === attempt.turnId)) {
+            // Forget monitoring, not the remote task. Never cancel an unknown writer.
+            if (activeTurnId)
+                runtime.forgetSession?.(session.localSessionId);
             this.store.updateRuntimeSession(session.localSessionId, { lastTurnStatus: "cancelled" });
-        }
-        if (session.status === "starting" || session.status === "running") {
-            this.store.updateSession(session.localSessionId, { status: "ready" });
+            if (session.status === "starting" || session.status === "running") {
+                this.store.updateSession(session.localSessionId, { status: "ready" });
+            }
         }
         if (attempt.turnId && this.outbound.canRoute(attempt.contextKey)) {
             await this.finalizeTurnMessageReactions(attempt.turnId, "cancelled").catch((error) => {
@@ -1838,6 +1863,47 @@ export class ProxySessionController {
             });
         }
     }
+    async handleRecoveryError(attemptId, error, activeTurnId) {
+        if (this.recoveryClosed)
+            return;
+        const attempt = this.store.getTurnAttempt(attemptId);
+        if (!attempt || !isIncompleteTurnAttemptStatus(attempt.status))
+            return;
+        this.logger.warn({ error, attemptId, sessionId: attempt.localSessionId }, "Task recovery failed.");
+        const session = this.store.getSession(attempt.localSessionId);
+        const writerConflict = session?.remoteSessionId === activeWriterThreadId(error) && Boolean(session?.remoteSessionId);
+        const startedAt = this.recoveryRetryStartedAt.get(attemptId) ?? Date.now();
+        this.recoveryRetryStartedAt.set(attemptId, startedAt);
+        const localActiveTurnId = session ? this.runtimes.forAgent(session.agentName).getSession(session.localSessionId)?.activeTurnId : undefined;
+        const monitoredTurnId = activeTurnId ?? (localActiveTurnId === attempt.turnId ? localActiveTurnId : undefined);
+        // Once a live turn was confirmed, a monitoring outage must not make safe restart
+        // consider it idle. Keep reconciling without cancelling or replaying the prompt.
+        if (monitoredTurnId && this.recoveredLiveAttempts.has(attemptId) && !writerConflict) {
+            this.scheduleRecoveryRetry(attemptId, monitoredTurnId);
+            return;
+        }
+        if (writerConflict || error instanceof CodexVersionError || Date.now() - startedAt >= RECOVERY_RETRY_WINDOW_MS) {
+            const message = writerConflict
+                ? "自动恢复未完成：任务仍被另一个 App Server 占用。已停止自动重试，请确认占用进程后释放任务再继续。"
+                : `自动恢复未完成，已停止自动重试。请检查 Agent 状态后重新发送消息。\n${error instanceof Error ? error.message : String(error)}`;
+            await this.expireStaleTurnAttempt(attempt, message);
+            try {
+                const replyTarget = attempt.replyMessageId
+                    ? { messageId: attempt.replyMessageId, replyInThread: true } : undefined;
+                await this.outbound.withReplyTarget(attempt.contextKey, replyTarget, async () => {
+                    await this.outbound.sendText(attempt.contextKey, message);
+                    if (writerConflict && session)
+                        await this.presentThreadWriterConflict(session, attempt.contextKey, error, false);
+                });
+            }
+            catch (notificationError) {
+                this.logger.warn({ error: notificationError, attemptId }, "Failed to notify about stopped task recovery.");
+            }
+            return;
+        }
+        this.store.updateTurnAttempt(attemptId, { status: "recovering" });
+        this.scheduleRecoveryRetry(attemptId, monitoredTurnId);
+    }
     persistActiveTurnHeartbeats() {
         for (const attempt of this.store.listIncompleteTurnAttempts()) {
             if (!attempt.turnId)
@@ -1851,6 +1917,8 @@ export class ProxySessionController {
         }
     }
     async recoverTurnAttempt(attempt, announce = true) {
+        if (this.recoveryClosed)
+            return;
         const session = this.store.getSession(attempt.localSessionId);
         if (!session || session.status === "closed") {
             this.store.updateTurnAttempt(attempt.attemptId, { status: "interrupted" });
@@ -1879,9 +1947,9 @@ export class ProxySessionController {
                 remote = await runtime.readRemoteSession(session.remoteSessionId);
             }
             catch (error) {
-                if (error instanceof CodexVersionError)
+                // A failed activity check is not proof of interruption. Do not replay a prompt blindly.
+                if (sourceTurnId || !isUnmaterializedCodexThreadError(error))
                     throw error;
-                this.logger.warn({ error, attemptId: attempt.attemptId, remoteSessionId: session.remoteSessionId }, "Failed to read the remote task during startup recovery; treating the old local execution as interrupted.");
             }
         }
         const remoteTurnBelongsToAttempt = remoteTurnMatchesAttempt(remote, attempt);
@@ -1927,14 +1995,22 @@ export class ProxySessionController {
             this.scheduleRecoveryRetry(attempt.attemptId, synchronized.activeTurnId);
     }
     scheduleRecoveryRetry(attemptId, activeTurnId) {
-        if (this.recoveryRetryTimers.has(attemptId))
+        if (this.recoveryClosed || this.recoveryRetryTimers.has(attemptId))
             return;
+        if (!this.recoveryRetryStartedAt.has(attemptId))
+            this.recoveryRetryStartedAt.set(attemptId, Date.now());
         const timer = setTimeout(() => {
             this.recoveryRetryTimers.delete(attemptId);
             const attempt = this.store.getTurnAttempt(attemptId);
-            if (!attempt || !isIncompleteTurnAttemptStatus(attempt.status))
+            if (!attempt || !isIncompleteTurnAttemptStatus(attempt.status)) {
+                this.recoveryRetryStartedAt.delete(attemptId);
+                this.recoveredLiveAttempts.delete(attemptId);
                 return;
+            }
             const retry = async () => {
+                if (!activeTurnId && Date.now() - this.recoveryRetryStartedAt.get(attemptId) >= RECOVERY_RETRY_WINDOW_MS) {
+                    throw new Error("等待恢复超过 1 分钟，尚未确认执行已恢复。");
+                }
                 let recoveryAttempt = attempt;
                 if (activeTurnId) {
                     const session = this.store.getSession(attempt.localSessionId);
@@ -1942,6 +2018,8 @@ export class ProxySessionController {
                         const runtime = this.runtimes.forAgent(session.agentName);
                         const synchronized = await runtime.synchronizeSession(session.localSessionId);
                         if (synchronized.activeTurnId === activeTurnId) {
+                            this.recoveredLiveAttempts.add(attemptId);
+                            this.recoveryRetryStartedAt.delete(attemptId);
                             this.scheduleRecoveryRetry(attemptId, activeTurnId);
                             return;
                         }
@@ -1952,16 +2030,27 @@ export class ProxySessionController {
                     }
                 }
                 await this.recoverTurnAttempt(recoveryAttempt, false);
+                if (!this.recoveryRetryTimers.has(attemptId))
+                    this.recoveryRetryStartedAt.delete(attemptId);
             };
             void retry().catch((error) => {
-                this.logger.warn({ error, attemptId, sessionId: attempt.localSessionId }, "Failed to retry an unfinished task recovery.");
-                this.scheduleRecoveryRetry(attemptId, activeTurnId);
+                return this.handleRecoveryError(attemptId, error, activeTurnId);
+            }).catch((error) => {
+                this.logger.error({ error, attemptId }, "Failed to finalize task recovery.");
             });
         }, 5_000);
         timer.unref?.();
         this.recoveryRetryTimers.set(attemptId, timer);
     }
     async reattachActiveRecoveredTurn(attempt, session, turnId, replyTarget) {
+        // Confirm resume before replacing the old card or announcing a running turn.
+        const current = this.store.getSession(session.localSessionId) ?? session;
+        const loaded = await this.loadSession({
+            ...current, contextKey: attempt.contextKey, status: "running", lastTurnId: turnId, lastTurnStatus: "running",
+        });
+        const pendingAttempt = this.store.getTurnAttempt(attempt.attemptId);
+        if (!pendingAttempt || !isIncompleteTurnAttemptStatus(pendingAttempt.status))
+            return;
         const oldCardTurnId = attempt.pendingTurnId ?? attempt.turnId;
         if (oldCardTurnId) {
             await this.outbound.interruptTurnForRecovery(session.localSessionId, attempt.contextKey, oldCardTurnId, "Agent Bot 已重启，执行仍在继续；进度已转移到新的思考卡片。");
@@ -1979,17 +2068,12 @@ export class ProxySessionController {
             startedAt: Date.now(),
         });
         this.store.updateTurnAttempt(attempt.attemptId, { pendingTurnId: null });
-        const current = this.store.getSession(session.localSessionId) ?? session;
-        const loaded = await this.loadSession({
-            ...current,
-            contextKey: attempt.contextKey,
-            status: "running",
-            lastTurnId: turnId,
-            lastTurnStatus: "running",
-        });
         const synchronized = await loaded.runtime.synchronizeSession(session.localSessionId);
-        if (synchronized.activeTurnId)
+        this.recoveryRetryStartedAt.delete(attempt.attemptId);
+        if (synchronized.activeTurnId) {
+            this.recoveredLiveAttempts.add(attempt.attemptId);
             this.scheduleRecoveryRetry(attempt.attemptId, synchronized.activeTurnId);
+        }
     }
     async continueInterruptedAttempt(attempt, session, sourceTurnId, replyTarget) {
         const oldCardTurnId = attempt.pendingTurnId ?? sourceTurnId;
@@ -3614,6 +3698,11 @@ export class ProxySessionController {
     }
     async handleRuntimeEvent(event) {
         if ("turnId" in event) {
+            const recoveryAttempt = this.recoveryRetryStartedAt.size > 0
+                ? this.store.findIncompleteTurnAttemptByTurnId(event.turnId) : undefined;
+            if (recoveryAttempt && this.recoveryRetryStartedAt.has(recoveryAttempt.attemptId)) {
+                this.recoveredLiveAttempts.add(recoveryAttempt.attemptId);
+            }
             this.store.touchTurnAttempt(event.turnId);
             const source = this.store.getSession(event.sessionId);
             if (source?.remoteSessionId && this.isCodexSession(source)) {
@@ -3652,6 +3741,13 @@ export class ProxySessionController {
             this.store.updateRuntimeSession(event.sessionId, { lastTurnId: event.turnId, lastTurnStatus: "running" });
         }
         else if (event.type === "turn_completed" || event.type === "turn_cancelled" || event.type === "turn_failed") {
+            const attempt = this.store.findIncompleteTurnAttemptByTurnId(event.turnId);
+            if (attempt) {
+                clearTimeout(this.recoveryRetryTimers.get(attempt.attemptId));
+                this.recoveryRetryTimers.delete(attempt.attemptId);
+                this.recoveryRetryStartedAt.delete(attempt.attemptId);
+                this.recoveredLiveAttempts.delete(attempt.attemptId);
+            }
             this.store.markTurnAttemptTerminal(event.turnId, event.type === "turn_completed" ? "completed" : event.type === "turn_cancelled" ? "cancelled" : "failed");
             this.store.updateRuntimeSession(event.sessionId, {
                 lastTurnId: event.turnId,
@@ -5489,20 +5585,29 @@ export class ProxySessionController {
             const project = taskProjectInfo(entry);
             const projectActionSource = projectActionSources.get(project.key) ?? entry;
             const marker = entry.current ? "✅ " : entry.active ? "🟢 " : "";
-            const showStop = entry.status === "外部执行中"
-                && entry.reference !== options.forceSwitchTaskId
-                && entry.id !== options.forceSwitchTaskId;
-            const actions = entry.current ? [] : [bindCardAction({
-                    text: showStop ? "Stop" : "Switch",
-                    type: showStop ? "danger" : "default",
+            const stopAction = entry.active ? bindCardAction({
+                text: "Stop",
+                type: "danger",
+                value: {
+                    action: "session_stop",
+                    sessionId: entry.reference,
+                    ...(searchTerm ? { searchTerm } : {}),
+                    page: String(page),
+                    contextKey,
+                },
+            }) : undefined;
+            const actions = [];
+            if (!entry.current && entry.status !== "外部执行中")
+                actions.push(bindCardAction({
+                    text: "Switch",
                     value: {
-                        action: showStop ? "session_stop" : "session_switch",
+                        action: "session_switch",
                         sessionId: entry.reference,
                         ...(searchTerm ? { searchTerm } : {}),
                         page: String(page),
                         contextKey,
                     },
-                })];
+                }));
             actions.push(bindCardAction({
                 text: "SwitchGroup",
                 value: {
@@ -5568,16 +5673,17 @@ export class ProxySessionController {
                     contextKey,
                 },
             }));
-            actions.push(bindCardAction({
-                text: "Archive",
-                value: {
-                    action: "session_archive",
-                    sessionId: entry.reference,
-                    ...(searchTerm ? { searchTerm } : {}),
-                    page: String(page),
-                    contextKey,
-                },
-            }));
+            if (!entry.active)
+                actions.push(bindCardAction({
+                    text: "Archive",
+                    value: {
+                        action: "session_archive",
+                        sessionId: entry.reference,
+                        ...(searchTerm ? { searchTerm } : {}),
+                        page: String(page),
+                        contextKey,
+                    },
+                }));
             const title = truncateText(entry.title.replace(/\s+/gu, " ").trim() || "未命名任务", 56);
             const lastUserPrompt = sessionPromptPreview(entry.lastUserPrompt);
             return {
@@ -5590,6 +5696,7 @@ export class ProxySessionController {
                         `**最新 Prompt**：${cardText(lastUserPrompt)}`,
                         `<font color='grey'>最近更新：${cardText(entry.updatedLabel)}</font>`,
                     ],
+                    stopAction,
                     actions,
                     current: entry.current,
                 },
@@ -5616,7 +5723,7 @@ export class ProxySessionController {
             `第 ${page + 1} 页 · 每页 ${SESSION_PAGE_SIZE} 个任务${hasNext ? "" : ` · 当前共 ${entries.length} 个任务`}`,
             "",
             "> 项目菜单：**New** 新建任务，**NewGroup** 新建群。",
-            "> 任务详情：**Switch** 切换，**Stop** 停止，**Fork** / **ForkGroup** 创建分支，**Status** 查看状态，**Archive** 归档。",
+            "> 任务详情：**Switch** 切换，运行中任务可用 **Stop** 发送停止信号，**Fork** / **ForkGroup** 创建分支，**Status** 查看状态，**Archive** 归档。",
         ], [
             ...(hasPrevious ? [bindCardAction({
                     text: "Previous",
@@ -5690,7 +5797,6 @@ export class ProxySessionController {
             : undefined;
         await this.listSessions(action.contextKey, searchTerm, {
             updateMessageId: action.messageId,
-            forceSwitchTaskId: options.forceSwitchTaskId,
             page: options.page ?? parseSessionPage(action.value.page),
         });
     }