@keyou007/agent-bot 0.1.24-alpha.4 → 0.1.24-alpha.5

package/src/proxy/ProxySessionController.ts

@@ -193,6 +193,7 @@ const REMOTE_TURN_HISTORY_REFRESH_INTERVAL_MS = 30_000;
 const MAX_LLM_TURN_RETRIES = 3;
 const QUEUED_PROMPT_RETRY_DELAY_MS = 5_000;
 const RECOVERY_ACTIVITY_WINDOW_MS = 10 * 60 * 1_000;
+const RECOVERY_RETRY_WINDOW_MS = 60_000;
 const RECOVERY_HEARTBEAT_INTERVAL_MS = 60 * 1_000;
 const SHELL_COMMAND_JOB_POLL_INTERVAL_MS = 1_000;
 const SHELL_COMMAND_JOB_RELAUNCH_DELAY_MS = 5_000;
@@ -383,7 +384,6 @@ interface AgentRemoteSessionSummary {
 
 interface SessionsCardOptions {
   updateMessageId?: string;
-  forceSwitchTaskId?: string;
   page?: number;
 }
 
@@ -554,6 +554,9 @@ export class ProxySessionController {
   private readonly retriedFailureTurnIds = new Set<string>();
   private readonly queuedPromptRetryTimers = new Map<string, NodeJS.Timeout>();
   private readonly recoveryRetryTimers = new Map<string, NodeJS.Timeout>();
+  private readonly recoveryRetryStartedAt = new Map<string, number>();
+  private readonly recoveredLiveAttempts = new Set<string>();
+  private recoveryClosed = false;
   private readonly pendingMergedForwards = new Map<string, PendingForwardAttachment>();
   private readonly pendingResourceForwards = new Map<string, PendingForwardAttachment>();
   private readonly relatedReactionMessageIds = new Map<string, string[]>();
@@ -1208,7 +1211,7 @@ export class ProxySessionController {
           await this.stopSessionReference(contextKey, sessionId);
           if (scopedAction.value.cardView === "status") await this.refreshStatusCardFromAction(scopedAction, sessionId);
           else if (scopedAction.value.cardView !== "safe_restart") {
-            await this.refreshSessionsCardFromAction(scopedAction, { forceSwitchTaskId: sessionId });
+            await this.refreshSessionsCardFromAction(scopedAction);
           }
         } else if (kind === "session_turns" || kind === "session_turns_page") {
           const sessionReference = String(scopedAction.value.sessionId ?? "").trim();
@@ -1353,6 +1356,7 @@ export class ProxySessionController {
   }
 
   close(): void {
+    this.recoveryClosed = true;
     for (const unsubscribe of this.unsubscribe) unsubscribe();
     this.unsubscribe.length = 0;
     this.lastSessionListings.clear();
@@ -1372,6 +1376,8 @@ export class ProxySessionController {
     this.queuedPromptRetryTimers.clear();
     for (const timer of this.recoveryRetryTimers.values()) clearTimeout(timer);
     this.recoveryRetryTimers.clear();
+    this.recoveryRetryStartedAt.clear();
+    this.recoveredLiveAttempts.clear();
     for (const schedule of this.appServerReleaseSchedules.values()) {
       if (schedule.timer) clearTimeout(schedule.timer);
     }
@@ -2294,14 +2300,13 @@ export class ProxySessionController {
         continue;
       }
       if (!this.outbound.canRoute(attempt.contextKey)) continue;
+      this.recoveryRetryStartedAt.set(attempt.attemptId, Date.now());
       try {
         await this.recoverTurnAttempt(attempt);
       } catch (error) {
-        this.logger.error(
-          { error, attemptId: attempt.attemptId, sessionId: attempt.localSessionId },
-          "Failed to recover an unfinished task; it will be retried while the server is running.",
-        );
-        this.scheduleRecoveryRetry(attempt.attemptId);
+        await this.handleRecoveryError(attempt.attemptId, error);
+      } finally {
+        if (!this.recoveryRetryTimers.has(attempt.attemptId)) this.recoveryRetryStartedAt.delete(attempt.attemptId);
       }
     }
     this.restorePersistedQueuedPrompts();
@@ -2394,15 +2399,23 @@ export class ProxySessionController {
     }
   }
 
-  private async expireStaleTurnAttempt(attempt: TurnAttemptRecord): Promise<void> {
-    const session = this.store.getSession(attempt.localSessionId);
+  private async expireStaleTurnAttempt(
+    attempt: TurnAttemptRecord,
+    message = "执行中断已超过 10 分钟，未自动恢复。",
+  ): Promise<void> {
+    clearTimeout(this.recoveryRetryTimers.get(attempt.attemptId));
+    this.recoveryRetryTimers.delete(attempt.attemptId);
+    this.recoveryRetryStartedAt.delete(attempt.attemptId);
+    this.recoveredLiveAttempts.delete(attempt.attemptId);
+    let session = this.store.getSession(attempt.localSessionId);
+    this.store.updateTurnAttempt(attempt.attemptId, { status: "interrupted" });
     const oldCardTurnId = attempt.pendingTurnId ?? attempt.turnId;
     if (session && oldCardTurnId && this.outbound.canRoute(attempt.contextKey)) {
       await this.outbound.interruptTurnForRecovery(
         session.localSessionId,
         attempt.contextKey,
         oldCardTurnId,
-        "执行中断已超过 10 分钟，未自动恢复。",
+        message,
       ).catch((error: unknown) => {
         this.logger.warn(
           { error, attemptId: attempt.attemptId, turnId: oldCardTurnId },
@@ -2410,13 +2423,21 @@ export class ProxySessionController {
         );
       });
     }
-    this.store.updateTurnAttempt(attempt.attemptId, { status: "interrupted" });
+    session = this.store.getSession(attempt.localSessionId);
     if (!session || session.status === "closed") return;
-    if (attempt.turnId && session.lastTurnId === attempt.turnId) {
+    const runtime = this.runtimes.forAgent(session.agentName);
+    const activeTurnId = runtime.getSession(session.localSessionId)?.activeTurnId;
+    const sameTurn = session.lastTurnId === attempt.turnId
+      || session.lastTurnId === attempt.recoveredFromTurnId
+      || !session.lastTurnId;
+    if (sameTurn && session.lastTurnStatus !== "completed" && session.lastTurnStatus !== "failed"
+      && (!activeTurnId || activeTurnId === attempt.turnId)) {
+      // Forget monitoring, not the remote task. Never cancel an unknown writer.
+      if (activeTurnId) runtime.forgetSession?.(session.localSessionId);
       this.store.updateRuntimeSession(session.localSessionId, { lastTurnStatus: "cancelled" });
-    }
-    if (session.status === "starting" || session.status === "running") {
-      this.store.updateSession(session.localSessionId, { status: "ready" });
+      if (session.status === "starting" || session.status === "running") {
+        this.store.updateSession(session.localSessionId, { status: "ready" });
+      }
     }
     if (attempt.turnId && this.outbound.canRoute(attempt.contextKey)) {
       await this.finalizeTurnMessageReactions(attempt.turnId, "cancelled").catch((error: unknown) => {
@@ -2428,6 +2449,44 @@ export class ProxySessionController {
     }
   }
 
+  private async handleRecoveryError(attemptId: string, error: unknown, activeTurnId?: string): Promise<void> {
+    if (this.recoveryClosed) return;
+    const attempt = this.store.getTurnAttempt(attemptId);
+    if (!attempt || !isIncompleteTurnAttemptStatus(attempt.status)) return;
+    this.logger.warn({ error, attemptId, sessionId: attempt.localSessionId }, "Task recovery failed.");
+    const session = this.store.getSession(attempt.localSessionId);
+    const writerConflict = session?.remoteSessionId === activeWriterThreadId(error) && Boolean(session?.remoteSessionId);
+    const startedAt = this.recoveryRetryStartedAt.get(attemptId) ?? Date.now();
+    this.recoveryRetryStartedAt.set(attemptId, startedAt);
+    const localActiveTurnId = session ? this.runtimes.forAgent(session.agentName).getSession(session.localSessionId)?.activeTurnId : undefined;
+    const monitoredTurnId = activeTurnId ?? (localActiveTurnId === attempt.turnId ? localActiveTurnId : undefined);
+    // Once a live turn was confirmed, a monitoring outage must not make safe restart
+    // consider it idle. Keep reconciling without cancelling or replaying the prompt.
+    if (monitoredTurnId && this.recoveredLiveAttempts.has(attemptId) && !writerConflict) {
+      this.scheduleRecoveryRetry(attemptId, monitoredTurnId);
+      return;
+    }
+    if (writerConflict || error instanceof CodexVersionError || Date.now() - startedAt >= RECOVERY_RETRY_WINDOW_MS) {
+      const message = writerConflict
+        ? "自动恢复未完成：任务仍被另一个 App Server 占用。已停止自动重试，请确认占用进程后释放任务再继续。"
+        : `自动恢复未完成，已停止自动重试。请检查 Agent 状态后重新发送消息。\n${error instanceof Error ? error.message : String(error)}`;
+      await this.expireStaleTurnAttempt(attempt, message);
+      try {
+        const replyTarget = attempt.replyMessageId
+          ? { messageId: attempt.replyMessageId, replyInThread: true as const } : undefined;
+        await this.outbound.withReplyTarget(attempt.contextKey, replyTarget, async () => {
+          await this.outbound.sendText(attempt.contextKey, message);
+          if (writerConflict && session) await this.presentThreadWriterConflict(session, attempt.contextKey, error, false);
+        });
+      } catch (notificationError) {
+        this.logger.warn({ error: notificationError, attemptId }, "Failed to notify about stopped task recovery.");
+      }
+      return;
+    }
+    this.store.updateTurnAttempt(attemptId, { status: "recovering" });
+    this.scheduleRecoveryRetry(attemptId, monitoredTurnId);
+  }
+
   private persistActiveTurnHeartbeats(): void {
     for (const attempt of this.store.listIncompleteTurnAttempts()) {
       if (!attempt.turnId) continue;
@@ -2439,6 +2498,7 @@ export class ProxySessionController {
   }
 
   private async recoverTurnAttempt(attempt: TurnAttemptRecord, announce = true): Promise<void> {
+    if (this.recoveryClosed) return;
     const session = this.store.getSession(attempt.localSessionId);
     if (!session || session.status === "closed") {
       this.store.updateTurnAttempt(attempt.attemptId, { status: "interrupted" });
@@ -2481,11 +2541,8 @@ export class ProxySessionController {
       try {
         remote = await runtime.readRemoteSession(session.remoteSessionId);
       } catch (error) {
-        if (error instanceof CodexVersionError) throw error;
-        this.logger.warn(
-          { error, attemptId: attempt.attemptId, remoteSessionId: session.remoteSessionId },
-          "Failed to read the remote task during startup recovery; treating the old local execution as interrupted.",
-        );
+        // A failed activity check is not proof of interruption. Do not replay a prompt blindly.
+        if (sourceTurnId || !isUnmaterializedCodexThreadError(error)) throw error;
       }
     }
     const remoteTurnBelongsToAttempt = remoteTurnMatchesAttempt(remote, attempt);
@@ -2538,12 +2595,20 @@ export class ProxySessionController {
   }
 
   private scheduleRecoveryRetry(attemptId: string, activeTurnId?: string): void {
-    if (this.recoveryRetryTimers.has(attemptId)) return;
+    if (this.recoveryClosed || this.recoveryRetryTimers.has(attemptId)) return;
+    if (!this.recoveryRetryStartedAt.has(attemptId)) this.recoveryRetryStartedAt.set(attemptId, Date.now());
     const timer = setTimeout(() => {
       this.recoveryRetryTimers.delete(attemptId);
       const attempt = this.store.getTurnAttempt(attemptId);
-      if (!attempt || !isIncompleteTurnAttemptStatus(attempt.status)) return;
+      if (!attempt || !isIncompleteTurnAttemptStatus(attempt.status)) {
+        this.recoveryRetryStartedAt.delete(attemptId);
+        this.recoveredLiveAttempts.delete(attemptId);
+        return;
+      }
       const retry = async (): Promise<void> => {
+        if (!activeTurnId && Date.now() - this.recoveryRetryStartedAt.get(attemptId)! >= RECOVERY_RETRY_WINDOW_MS) {
+          throw new Error("等待恢复超过 1 分钟，尚未确认执行已恢复。");
+        }
         let recoveryAttempt = attempt;
         if (activeTurnId) {
           const session = this.store.getSession(attempt.localSessionId);
@@ -2551,6 +2616,8 @@ export class ProxySessionController {
             const runtime = this.runtimes.forAgent(session.agentName);
             const synchronized = await runtime.synchronizeSession(session.localSessionId);
             if (synchronized.activeTurnId === activeTurnId) {
+              this.recoveredLiveAttempts.add(attemptId);
+              this.recoveryRetryStartedAt.delete(attemptId);
               this.scheduleRecoveryRetry(attemptId, activeTurnId);
               return;
             }
@@ -2560,13 +2627,12 @@ export class ProxySessionController {
           }
         }
         await this.recoverTurnAttempt(recoveryAttempt, false);
+        if (!this.recoveryRetryTimers.has(attemptId)) this.recoveryRetryStartedAt.delete(attemptId);
       };
       void retry().catch((error: unknown) => {
-        this.logger.warn(
-          { error, attemptId, sessionId: attempt.localSessionId },
-          "Failed to retry an unfinished task recovery.",
-        );
-        this.scheduleRecoveryRetry(attemptId, activeTurnId);
+        return this.handleRecoveryError(attemptId, error, activeTurnId);
+      }).catch((error: unknown) => {
+        this.logger.error({ error, attemptId }, "Failed to finalize task recovery.");
       });
     }, 5_000);
     timer.unref?.();
@@ -2579,6 +2645,13 @@ export class ProxySessionController {
     turnId: string,
     replyTarget?: MessageReplyTarget,
   ): Promise<void> {
+    // Confirm resume before replacing the old card or announcing a running turn.
+    const current = this.store.getSession(session.localSessionId) ?? session;
+    const loaded = await this.loadSession({
+      ...current, contextKey: attempt.contextKey, status: "running", lastTurnId: turnId, lastTurnStatus: "running",
+    });
+    const pendingAttempt = this.store.getTurnAttempt(attempt.attemptId);
+    if (!pendingAttempt || !isIncompleteTurnAttemptStatus(pendingAttempt.status)) return;
     const oldCardTurnId = attempt.pendingTurnId ?? attempt.turnId;
     if (oldCardTurnId) {
       await this.outbound.interruptTurnForRecovery(
@@ -2607,16 +2680,12 @@ export class ProxySessionController {
       startedAt: Date.now(),
     });
     this.store.updateTurnAttempt(attempt.attemptId, { pendingTurnId: null });
-    const current = this.store.getSession(session.localSessionId) ?? session;
-    const loaded = await this.loadSession({
-      ...current,
-      contextKey: attempt.contextKey,
-      status: "running",
-      lastTurnId: turnId,
-      lastTurnStatus: "running",
-    });
     const synchronized = await loaded.runtime.synchronizeSession(session.localSessionId);
-    if (synchronized.activeTurnId) this.scheduleRecoveryRetry(attempt.attemptId, synchronized.activeTurnId);
+    this.recoveryRetryStartedAt.delete(attempt.attemptId);
+    if (synchronized.activeTurnId) {
+      this.recoveredLiveAttempts.add(attempt.attemptId);
+      this.scheduleRecoveryRetry(attempt.attemptId, synchronized.activeTurnId);
+    }
   }
 
   private async continueInterruptedAttempt(
@@ -4638,6 +4707,11 @@ export class ProxySessionController {
 
   private async handleRuntimeEvent(event: RuntimeEvent): Promise<void> {
     if ("turnId" in event) {
+      const recoveryAttempt = this.recoveryRetryStartedAt.size > 0
+        ? this.store.findIncompleteTurnAttemptByTurnId(event.turnId) : undefined;
+      if (recoveryAttempt && this.recoveryRetryStartedAt.has(recoveryAttempt.attemptId)) {
+        this.recoveredLiveAttempts.add(recoveryAttempt.attemptId);
+      }
       this.store.touchTurnAttempt(event.turnId);
       const source = this.store.getSession(event.sessionId);
       if (source?.remoteSessionId && this.isCodexSession(source)) {
@@ -4683,6 +4757,13 @@ export class ProxySessionController {
       this.store.updateSession(event.sessionId, { status: "running" });
       this.store.updateRuntimeSession(event.sessionId, { lastTurnId: event.turnId, lastTurnStatus: "running" });
     } else if (event.type === "turn_completed" || event.type === "turn_cancelled" || event.type === "turn_failed") {
+      const attempt = this.store.findIncompleteTurnAttemptByTurnId(event.turnId);
+      if (attempt) {
+        clearTimeout(this.recoveryRetryTimers.get(attempt.attemptId));
+        this.recoveryRetryTimers.delete(attempt.attemptId);
+        this.recoveryRetryStartedAt.delete(attempt.attemptId);
+        this.recoveredLiveAttempts.delete(attempt.attemptId);
+      }
       this.store.markTurnAttemptTerminal(
         event.turnId,
         event.type === "turn_completed" ? "completed" : event.type === "turn_cancelled" ? "cancelled" : "failed",
@@ -6754,20 +6835,28 @@ export class ProxySessionController {
       const project = taskProjectInfo(entry);
       const projectActionSource = projectActionSources.get(project.key) ?? entry;
       const marker = entry.current ? "✅ " : entry.active ? "🟢 " : "";
-      const showStop = entry.status === "外部执行中"
-        && entry.reference !== options.forceSwitchTaskId
-        && entry.id !== options.forceSwitchTaskId;
-      const actions: TaskListCardAction[] = entry.current ? [] : [bindCardAction({
-        text: showStop ? "Stop" : "Switch",
-        type: showStop ? "danger" as const : "default" as const,
+      const stopAction = entry.active ? bindCardAction({
+        text: "Stop",
+        type: "danger",
         value: {
-          action: showStop ? "session_stop" : "session_switch",
+          action: "session_stop",
           sessionId: entry.reference,
           ...(searchTerm ? { searchTerm } : {}),
           page: String(page),
           contextKey,
         },
-      })];
+      }) : undefined;
+      const actions: TaskListCardAction[] = [];
+      if (!entry.current && entry.status !== "外部执行中") actions.push(bindCardAction({
+        text: "Switch",
+        value: {
+          action: "session_switch",
+          sessionId: entry.reference,
+          ...(searchTerm ? { searchTerm } : {}),
+          page: String(page),
+          contextKey,
+        },
+      }));
       actions.push(bindCardAction({
         text: "SwitchGroup",
         value: {
@@ -6833,7 +6922,7 @@ export class ProxySessionController {
           contextKey,
         },
       }));
-      actions.push(bindCardAction({
+      if (!entry.active) actions.push(bindCardAction({
         text: "Archive",
         value: {
           action: "session_archive",
@@ -6855,6 +6944,7 @@ export class ProxySessionController {
             `**最新 Prompt**：${cardText(lastUserPrompt)}`,
             `<font color='grey'>最近更新：${cardText(entry.updatedLabel)}</font>`,
           ],
+          stopAction,
           actions,
           current: entry.current,
         },
@@ -6884,7 +6974,7 @@ export class ProxySessionController {
         `第 ${page + 1} 页 · 每页 ${SESSION_PAGE_SIZE} 个任务${hasNext ? "" : ` · 当前共 ${entries.length} 个任务`}`,
         "",
         "> 项目菜单：**New** 新建任务，**NewGroup** 新建群。",
-        "> 任务详情：**Switch** 切换，**Stop** 停止，**Fork** / **ForkGroup** 创建分支，**Status** 查看状态，**Archive** 归档。",
+        "> 任务详情：**Switch** 切换，运行中任务可用 **Stop** 发送停止信号，**Fork** / **ForkGroup** 创建分支，**Status** 查看状态，**Archive** 归档。",
       ],
       [
         ...(hasPrevious ? [bindCardAction({
@@ -6959,7 +7049,7 @@ export class ProxySessionController {
 
   private async refreshSessionsCardFromAction(
     action: CardAction,
-    options: { forceSwitchTaskId?: string; page?: number } = {},
+    options: { page?: number } = {},
   ): Promise<void> {
     if (!action.messageId) return;
     const searchTerm = typeof action.value.searchTerm === "string" && action.value.searchTerm.trim()
@@ -6967,7 +7057,6 @@ export class ProxySessionController {
       : undefined;
     await this.listSessions(action.contextKey, searchTerm, {
       updateMessageId: action.messageId,
-      forceSwitchTaskId: options.forceSwitchTaskId,
       page: options.page ?? parseSessionPage(action.value.page),
     });
   }

package/src/runtime/types.ts

@@ -286,6 +286,8 @@ export interface AgentRuntime {
   resumeSession(input: ResumeRuntimeSessionInput): Promise<RuntimeSession>;
   forkSession?(input: ForkRuntimeSessionInput): Promise<RuntimeSession>;
   getSession(localSessionId: string): RuntimeSession | undefined;
+  // Drop local monitoring only; do not interrupt, unsubscribe, or archive remote work.
+  forgetSession?(localSessionId: string): void;
   readSessionMetadata(remoteSessionId: string): Promise<RuntimeSessionMetadata>;
   listRemoteSessions?(input?: { searchTerm?: string; cursor?: string; limit?: number }): Promise<RemoteSessionPage>;
   readRemoteSession?(remoteSessionId: string, view?: "metadata" | "latest" | "latest-full"): Promise<RemoteSessionSummary>;