@keyou007/agent-bot 0.1.16-alpha.2 → 0.1.16-alpha.4

package/src/proxy/ProxySessionController.ts

@@ -119,7 +119,7 @@ const SESSION_PROMPT_PREVIEW_LENGTH = 50;
 const DIRECTORY_PAGE_SIZE = 15;
 const RESET_HISTORY_PAGE_SIZE = 10;
 const MAX_LLM_TURN_RETRIES = 3;
-const RECOVERY_ACTIVITY_WINDOW_MS = 5 * 60 * 1_000;
+const RECOVERY_ACTIVITY_WINDOW_MS = 10 * 60 * 1_000;
 const RECOVERY_HEARTBEAT_INTERVAL_MS = 60 * 1_000;
 const SHELL_COMMAND_JOB_POLL_INTERVAL_MS = 1_000;
 const SHELL_COMMAND_JOB_RELAUNCH_DELAY_MS = 5_000;
@@ -1912,6 +1912,7 @@ export class ProxySessionController {
   }
 
   private async runStartupRecovery(): Promise<void> {
+    await this.finishPersistedCancellationRequests();
     await this.recoverShellCommandJobs();
     this.backfillTurnAttemptsForUpgrade();
     const attempts = this.store.listIncompleteTurnAttempts();
@@ -1942,6 +1943,40 @@ export class ProxySessionController {
     this.restorePersistedQueuedPrompts();
   }
 
+  private async finishPersistedCancellationRequests(): Promise<void> {
+    for (const attempt of this.store.listCancellationRequestedTurnAttempts()) {
+      const session = this.store.getSession(attempt.localSessionId);
+      if (!attempt.turnId) {
+        this.store.updateTurnAttempt(attempt.attemptId, { status: "cancelled" });
+        if (session?.status === "starting" || session?.status === "running") {
+          this.store.updateSession(session.localSessionId, { status: "ready" });
+        }
+        continue;
+      }
+      if (session) {
+        this.outbound.registerSession(
+          session.localSessionId,
+          attempt.contextKey,
+          session.title,
+          session.cwd,
+          this.agentLabel(session.agentName),
+        );
+      }
+      try {
+        await this.handleRuntimeEvent({
+          type: "turn_cancelled",
+          sessionId: attempt.localSessionId,
+          turnId: attempt.turnId,
+        });
+      } catch (error) {
+        this.logger.warn(
+          { error, attemptId: attempt.attemptId, turnId: attempt.turnId },
+          "Failed to finish a persisted user-requested turn cancellation.",
+        );
+      }
+    }
+  }
+
   private backfillTurnAttemptsForUpgrade(): void {
     for (const session of this.store.listAllSessions()) {
       if (session.status !== "starting" && session.status !== "running" && session.status !== "ready") continue;
@@ -2001,7 +2036,7 @@ export class ProxySessionController {
         session.localSessionId,
         attempt.contextKey,
         oldCardTurnId,
-        "执行中断已超过 5 分钟，未自动恢复。",
+        "执行中断已超过 10 分钟，未自动恢复。",
       ).catch((error: unknown) => {
         this.logger.warn(
           { error, attemptId: attempt.attemptId, turnId: oldCardTurnId },
@@ -3082,28 +3117,80 @@ export class ProxySessionController {
       });
     }
     let turnId: string;
+    let runtimeText = text;
+    let cardPrompt = options.displayPrompt ?? text;
+    let retriedStart = false;
     try {
-      const pendingTurnId = await this.outbound.startPendingTurn(
-        loaded.record.localSessionId,
-        loaded.record.contextKey,
-        title,
-        replyTarget,
-        options.displayPrompt ?? text,
-      );
-      this.store.updateTurnAttempt(attemptId, { pendingTurnId: pendingTurnId ?? null });
-      turnId = await loaded.runtime.startTurn(
-        loaded.record.localSessionId,
-        runtimePrompt(text, localImagePaths),
-      );
-    } catch (error) {
-      this.store.updateTurnAttempt(attemptId, {
-        status: options.preserveAttemptOnFailure ? "recovering" : "failed",
-      });
-      await this.outbound.failPendingTurn(
-        loaded.record.localSessionId,
-        error instanceof Error ? error.message : String(error),
-      );
-      throw error;
+      while (true) {
+        let pendingTurnId: string | undefined;
+        try {
+          pendingTurnId = await this.outbound.startPendingTurn(
+            loaded.record.localSessionId,
+            loaded.record.contextKey,
+            title,
+            replyTarget,
+            cardPrompt,
+          );
+        } catch (error) {
+          this.store.updateTurnAttempt(attemptId, {
+            status: options.preserveAttemptOnFailure ? "recovering" : "failed",
+          });
+          throw error;
+        }
+        this.store.updateTurnAttempt(attemptId, { pendingTurnId: pendingTurnId ?? null });
+
+        try {
+          turnId = await loaded.runtime.startTurn(
+            loaded.record.localSessionId,
+            runtimePrompt(runtimeText, localImagePaths),
+          );
+          break;
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          const attempt = this.store.getTurnAttempt(attemptId);
+          const retryAttempt = attempt
+            && isRetryableLlmTurnFailure(message)
+            && attempt.retryCount < MAX_LLM_TURN_RETRIES
+            ? this.store.prepareUnstartedTurnAttemptRetry(attemptId)
+            : undefined;
+          if (!retryAttempt) {
+            const exhausted = attempt
+              && isRetryableLlmTurnFailure(message)
+              && attempt.retryCount >= MAX_LLM_TURN_RETRIES;
+            const finalMessage = exhausted
+              ? `${message}\n\n已自动重试 ${MAX_LLM_TURN_RETRIES} 次，仍未成功。`
+              : message;
+            this.store.updateTurnAttempt(attemptId, {
+              status: options.preserveAttemptOnFailure ? "recovering" : "failed",
+            });
+            await this.outbound.failPendingTurn(loaded.record.localSessionId, finalMessage);
+            if (finalMessage === message) throw error;
+            throw new Error(finalMessage, { cause: error });
+          }
+
+          retriedStart = true;
+          this.llmRetryingSessions.add(loaded.record.localSessionId);
+          this.store.audit(loaded.record.contextKey, "llm_turn_start_retry_started", {
+            attemptId,
+            retryNumber: retryAttempt.retryCount,
+            maxRetries: MAX_LLM_TURN_RETRIES,
+            error: message,
+          });
+          await this.outbound.failPendingTurn(
+            loaded.record.localSessionId,
+            `${message}\n\n检测到临时模型服务错误，正在自动重试（${retryAttempt.retryCount}/${MAX_LLM_TURN_RETRIES}）。`,
+          ).catch((presentationError: unknown) => {
+            this.logger.warn(
+              { error: presentationError, sessionId: loaded.record.localSessionId },
+              "Failed to finalize the pending thinking card before an LLM start retry.",
+            );
+          });
+          runtimeText = llmRetryRuntimePrompt(retryAttempt.promptText, retryAttempt.retryCount);
+          cardPrompt = llmRetryCardPrompt(options.displayPrompt ?? retryAttempt.promptText, retryAttempt.retryCount);
+        }
+      }
+    } finally {
+      if (retriedStart) this.llmRetryingSessions.delete(loaded.record.localSessionId);
     }
     const attempt = this.store.getTurnAttempt(attemptId);
     this.store.updateTurnAttempt(attemptId, {
@@ -3956,7 +4043,13 @@ export class ProxySessionController {
         await this.outbound.sendText(record.contextKey, "当前没有正在执行的任务。");
         return;
       }
-      await runtime.interruptRemoteTurn(record.remoteSessionId, turnId);
+      const cancellationAttempt = this.store.requestTurnAttemptCancellation(turnId);
+      try {
+        await runtime.interruptRemoteTurn(record.remoteSessionId, turnId);
+      } catch (error) {
+        if (cancellationAttempt) this.store.restoreTurnAttemptAfterCancellationFailure(cancellationAttempt);
+        throw error;
+      }
       this.store.audit(record.contextKey, "turn_interrupt_sent", {
         localSessionId: record.localSessionId,
         remoteSessionId: record.remoteSessionId,
@@ -3972,7 +4065,13 @@ export class ProxySessionController {
       await this.outbound.sendText(record.contextKey, "当前没有正在执行的任务。");
       return;
     }
-    await loaded.runtime.cancelTurn(record.localSessionId, turnId);
+    const cancellationAttempt = this.store.requestTurnAttemptCancellation(turnId);
+    try {
+      await loaded.runtime.cancelTurn(record.localSessionId, turnId);
+    } catch (error) {
+      if (cancellationAttempt) this.store.restoreTurnAttemptAfterCancellationFailure(cancellationAttempt);
+      throw error;
+    }
   }
 
   private async resetCurrentSessionToTurn(
@@ -4851,7 +4950,43 @@ export class ProxySessionController {
     if (!current.remoteSessionId || !this.isCodexSession(current)) {
       throw new Error("当前任务不是可 Reset 的 App Server 任务。");
     }
-    const allTurns = this.store.listTaskTurnGraph(current.localSessionId);
+    const completedTurns = this.store.listTaskTurnGraph(current.localSessionId);
+    const completedTurnIds = new Set(completedTurns.map((turn) => turn.turnId));
+    const runningTurnId = current.lastTurnStatus === "running" && current.lastTurnId
+      && !completedTurnIds.has(current.lastTurnId)
+      ? current.lastTurnId
+      : undefined;
+    const persistedRunningSnapshot = runningTurnId
+      ? turnViewSnapshot(this.store.getTurnSnapshot(runningTurnId))
+      : undefined;
+    const runningAttempt = runningTurnId
+      ? this.store.findIncompleteTurnAttemptByTurnId(runningTurnId)
+      : undefined;
+    const attemptStartedAt = Date.parse(runningAttempt?.createdAt ?? "");
+    const runningSnapshot: TurnViewState | undefined = runningTurnId
+      ? persistedRunningSnapshot ?? {
+          sessionId: current.localSessionId,
+          turnId: runningTurnId,
+          prompt: runningAttempt?.promptText ?? current.title,
+          status: "running",
+          startedAt: Number.isFinite(attemptStartedAt) ? attemptStartedAt : Date.now(),
+          assistantText: "",
+          plan: [],
+          activities: [],
+          completedTools: [],
+          failedTools: [],
+          fileSummary: [],
+        }
+      : undefined;
+    const runningTurn = runningTurnId && runningSnapshot
+      ? {
+          turnId: runningTurnId,
+          parentTurnId: this.store.getTurnParent(runningTurnId, current.localSessionId),
+          snapshot: runningSnapshot,
+          updatedAt: runningAttempt?.updatedAt ?? new Date(runningSnapshot.startedAt).toISOString(),
+        }
+      : undefined;
+    const allTurns = runningTurn ? [runningTurn, ...completedTurns] : completedTurns;
     const graphRows = buildTurnGraphRows(allTurns.map((turn) => ({
       turnId: turn.turnId,
       parentTurnId: turn.parentTurnId,
@@ -4870,16 +5005,18 @@ export class ProxySessionController {
         100,
       );
       const isCurrent = current.lastTurnId === turn.turnId && current.lastTurnStatus === "completed";
+      const isRunning = runningTurnId === turn.turnId;
       return [{
         sequence: graph.sequence,
         graphNodeLine: graph.nodeLine,
         graphConnectorLine: graph.connectorLine,
         lines: [
-          `**${cardText(summary || "未记录对话内容")}**`,
-          `${formatResetTurnTime(snapshot.completedAt ?? snapshot.startedAt)} · ${cardCode(turn.turnId)}`,
+          cardText(summary || "未记录对话内容"),
+          `${formatResetTurnTime(snapshot.completedAt ?? snapshot.startedAt)} · ${cardText(turn.turnId)}`,
         ],
         current: isCurrent,
-        actions: isCurrent ? undefined : [{
+        running: isRunning,
+        actions: isCurrent || isRunning ? undefined : [{
           text: "Reset",
           value: {
             action: "turn_reset",
@@ -4915,7 +5052,9 @@ export class ProxySessionController {
     const card = this.cardRenderer.renderResetHistoryCard({
       entries,
       footerLines: [
-        `第 ${page + 1}/${totalPages} 页 · 共 ${total} 个已完成 turn`,
+        runningTurn
+          ? `第 ${page + 1}/${totalPages} 页 · 共 ${total} 个 turn（${completedTurns.length} 个已完成，1 个运行中）`
+          : `第 ${page + 1}/${totalPages} 页 · 共 ${total} 个已完成 turn`,
       ],
       pageActions,
     });
@@ -5282,8 +5421,8 @@ export class ProxySessionController {
           reference: entry.reference,
           summary: `${offset + index + 1}. ${marker}${title} · ${entry.agentName}`,
           detailLines: [
-            cardText(lastUserPrompt),
-            `**更新时间**：${cardText(entry.updatedLabel)}`,
+            `**最新 Prompt**：${cardText(lastUserPrompt)}`,
+            `<font color='grey'>最近更新：${cardText(entry.updatedLabel)}</font>`,
           ],
           actions,
           current: entry.current,
@@ -6996,7 +7135,8 @@ function isRetryableLlmTurnFailure(message: string): boolean {
     /service unavailable|internal server error|bad gateway|gateway timeout/u,
     /(?:^|\D)(?:500|502|503|504)(?:\D|$)|upstream (?:error|failure|timeout)/u,
     /deadline exceeded|request timed out|request timeout|response timeout/u,
-    /connection (?:reset|closed|aborted)|network error|socket hang up/u,
+    /connection (?:reset|closed|aborted|disconnected)|network error|socket hang up/u,
+    /app server .*(?:closed|disconnected|exited|reset|aborted)/u,
     /stream (?:closed|disconnected|interrupted|error)/u,
     /\b(?:llm|model|provider|inference)\b.*\b(?:error|failed|failure|unavailable|timeout|timed out)\b/u,
   ].some((pattern) => pattern.test(normalized));

package/src/startup/SessionMetadataHydrator.ts

@@ -9,33 +9,6 @@ export class SessionMetadataHydrator {
   ) {}
 
   async hydrate(session: SessionRecord): Promise<SessionRecord> {
-    if (
-      session.runtimeKind === "codex"
-      && session.remoteSessionId
-      && session.lastTurnId
-      && session.status === "running"
-      && session.lastTurnStatus === "running"
-    ) {
-      const runtime = this.runtimes.forAgent(session.agentName);
-      const loaded = runtime.getSession(session.localSessionId) ?? await runtime.resumeSession({
-        localSessionId: session.localSessionId,
-        remoteSessionId: session.remoteSessionId,
-        activeTurnId: session.lastTurnId,
-        lastTurnId: session.lastTurnId,
-        lastTurnStatus: session.lastTurnStatus,
-        agentName: session.agentName,
-        cwd: session.cwd,
-        title: session.title,
-        model: session.model,
-        reasoningEffort: session.reasoningEffort,
-        permissionMode: session.permissionMode ?? "auto",
-      });
-      const synchronized = await runtime.synchronizeSession(loaded.localSessionId);
-      if (synchronized.title) {
-        this.store.updateRuntimeSession(session.localSessionId, { title: synchronized.title });
-      }
-      return this.store.getSession(session.localSessionId) ?? session;
-    }
     if (session.title || !session.runtimeKind || !session.remoteSessionId) return session;
     if (session.runtimeKind === "codex" && !session.lastTurnId) return session;
     const metadata = await this.runtimes.forAgent(session.agentName).readSessionMetadata(session.remoteSessionId);

package/src/state/StateStore.ts

@@ -111,6 +111,7 @@ export type TurnAttemptStatus =
   | "accepted"
   | "running"
   | "recovering"
+  | "cancelling"
   | "completed"
   | "failed"
   | "cancelled"
@@ -633,6 +634,15 @@ export class StateStore {
     return rows.map(mapTurnAttempt);
   }
 
+  listCancellationRequestedTurnAttempts(): TurnAttemptRecord[] {
+    const rows = this.db.prepare(`
+      SELECT * FROM turn_attempts
+      WHERE status = 'cancelling'
+      ORDER BY created_at ASC, rowid ASC
+    `).all() as TurnAttemptRow[];
+    return rows.map(mapTurnAttempt);
+  }
+
   updateTurnAttempt(
     attemptId: string,
     patch: Partial<{
@@ -697,6 +707,19 @@ export class StateStore {
     return row ? mapTurnAttempt(row) : undefined;
   }
 
+  requestTurnAttemptCancellation(turnId: string): TurnAttemptRecord | undefined {
+    const existing = this.findIncompleteTurnAttemptByTurnId(turnId);
+    if (!existing) return undefined;
+    this.updateTurnAttempt(existing.attemptId, { status: "cancelling" });
+    return existing;
+  }
+
+  restoreTurnAttemptAfterCancellationFailure(attempt: TurnAttemptRecord): void {
+    const current = this.getTurnAttempt(attempt.attemptId);
+    if (current?.status !== "cancelling") return;
+    this.updateTurnAttempt(attempt.attemptId, { status: attempt.status });
+  }
+
   touchTurnAttempt(turnId: string, activityAt = new Date()): void {
     const timestamp = activityAt.toISOString();
     const throttleCutoff = new Date(activityAt.getTime() - 15_000).toISOString();
@@ -737,6 +760,17 @@ export class StateStore {
     return this.getTurnAttempt(attemptId);
   }
 
+  prepareUnstartedTurnAttemptRetry(attemptId: string): TurnAttemptRecord | undefined {
+    const existing = this.getTurnAttempt(attemptId);
+    if (!existing || existing.turnId || !isIncompleteTurnAttemptStatus(existing.status)) return undefined;
+    this.updateTurnAttempt(attemptId, {
+      pendingTurnId: null,
+      retryCount: existing.retryCount + 1,
+      status: "recovering",
+    });
+    return this.getTurnAttempt(attemptId);
+  }
+
   rebindPendingTurnMessages(localSessionId: string, sourceTurnId: string, targetTurnId: string): void {
     const now = new Date().toISOString();
     const rebind = this.db.transaction(() => {
@@ -1057,6 +1091,15 @@ export class StateStore {
     `).run(turnId, localSessionId, parentTurnId ?? null, new Date().toISOString());
   }
 
+  getTurnParent(turnId: string, localSessionId: string): string | undefined {
+    const row = this.db.prepare(`
+      SELECT parent_turn_id
+      FROM turn_parent_links
+      WHERE turn_id = ? AND local_session_id = ?
+    `).get(turnId, localSessionId) as { parent_turn_id: string | null } | undefined;
+    return row?.parent_turn_id ?? undefined;
+  }
+
   findLatestCompletedTurnId(localSessionId: string, contextKey: string): string | undefined {
     const row = this.db
       .prepare(`