gsd-pi 2.38.0-dev.eeb3520 → 2.39.0-dev.20aba06

package/src/resources/extensions/gsd/auto.ts

@@ -111,6 +111,7 @@ import {
   recordHealthSnapshot,
   checkHealEscalation,
   resetProactiveHealing,
+  setLevelChangeCallback,
   formatHealthSummary,
   getConsecutiveErrorUnits,
 } from "./doctor-proactive.js";
@@ -195,7 +196,7 @@ import {
   postUnitPostVerification,
 } from "./auto-post-unit.js";
 import { bootstrapAutoSession, type BootstrapDeps } from "./auto-start.js";
-import { autoLoop, resolveAgentEnd, type LoopDeps } from "./auto-loop.js";
+import { autoLoop, resolveAgentEnd, isSessionSwitchInFlight, type LoopDeps } from "./auto-loop.js";
 import {
   WorktreeResolver,
   type WorktreeResolverDeps,
@@ -536,129 +537,168 @@ export async function stopAuto(
   if (!s.active && !s.paused) return;
   const loadedPreferences = loadEffectiveGSDPreferences()?.preferences;
   const reasonSuffix = reason ? ` — ${reason}` : "";
-  clearUnitTimeout();
-  if (lockBase()) clearLock(lockBase());
-  if (lockBase()) releaseSessionLock(lockBase());
-  clearSkillSnapshot();
-  resetSkillTelemetry();
 
-  // Remove SIGTERM handler registered at auto-mode start
-  deregisterSigtermHandler();
+  try {
+    // ── Step 1: Timers and locks ──
+    try {
+      clearUnitTimeout();
+      if (lockBase()) clearLock(lockBase());
+      if (lockBase()) releaseSessionLock(lockBase());
+    } catch (e) {
+      debugLog("stop-cleanup-locks", { error: e instanceof Error ? e.message : String(e) });
+    }
 
-  // ── Auto-worktree: exit worktree and reset s.basePath on stop ──
-  if (s.currentMilestoneId) {
-    const notifyCtx = ctx
-      ? { notify: ctx.ui.notify.bind(ctx.ui) }
-      : { notify: () => {} };
-    buildResolver().exitMilestone(s.currentMilestoneId, notifyCtx, {
-      preserveBranch: true,
-    });
-  }
+    // ── Step 2: Skill state ──
+    try {
+      clearSkillSnapshot();
+      resetSkillTelemetry();
+    } catch (e) {
+      debugLog("stop-cleanup-skills", { error: e instanceof Error ? e.message : String(e) });
+    }
 
-  // ── DB cleanup: close the SQLite connection ──
-  if (isDbAvailable()) {
+    // ── Step 3: SIGTERM handler ──
     try {
-      const { closeDatabase } = await import("./gsd-db.js");
-      closeDatabase();
+      deregisterSigtermHandler();
     } catch (e) {
-      debugLog("db-close-failed", {
-        error: e instanceof Error ? e.message : String(e),
-      });
+      debugLog("stop-cleanup-sigterm", { error: e instanceof Error ? e.message : String(e) });
     }
-  }
 
-  if (s.originalBasePath) {
-    s.basePath = s.originalBasePath;
+    // ── Step 4: Auto-worktree exit ──
     try {
-      process.chdir(s.basePath);
-    } catch {
-      /* best-effort */
+      if (s.currentMilestoneId) {
+        const notifyCtx = ctx
+          ? { notify: ctx.ui.notify.bind(ctx.ui) }
+          : { notify: () => {} };
+        buildResolver().exitMilestone(s.currentMilestoneId, notifyCtx, {
+          preserveBranch: true,
+        });
+      }
+    } catch (e) {
+      debugLog("stop-cleanup-worktree", { error: e instanceof Error ? e.message : String(e) });
     }
-  }
 
-  const ledger = getLedger();
-  if (ledger && ledger.units.length > 0) {
-    const totals = getProjectTotals(ledger.units);
-    ctx?.ui.notify(
-      `Auto-mode stopped${reasonSuffix}. Session: ${formatCost(totals.cost)} · ${formatTokenCount(totals.tokens.total)} tokens · ${ledger.units.length} units`,
-      "info",
-    );
-  } else {
-    ctx?.ui.notify(`Auto-mode stopped${reasonSuffix}.`, "info");
-  }
+    // ── Step 5: DB cleanup ──
+    if (isDbAvailable()) {
+      try {
+        const { closeDatabase } = await import("./gsd-db.js");
+        closeDatabase();
+      } catch (e) {
+        debugLog("db-close-failed", {
+          error: e instanceof Error ? e.message : String(e),
+        });
+      }
+    }
 
-  if (s.basePath) {
+    // ── Step 6: Restore basePath and chdir ──
     try {
-      await rebuildState(s.basePath);
+      if (s.originalBasePath) {
+        s.basePath = s.originalBasePath;
+        try {
+          process.chdir(s.basePath);
+        } catch {
+          /* best-effort */
+        }
+      }
     } catch (e) {
-      debugLog("stop-rebuild-state-failed", {
-        error: e instanceof Error ? e.message : String(e),
-      });
+      debugLog("stop-cleanup-basepath", { error: e instanceof Error ? e.message : String(e) });
     }
-  }
 
-  clearCmuxSidebar(loadedPreferences);
-  logCmuxEvent(
-    loadedPreferences,
-    `Auto-mode stopped${reasonSuffix || ""}.`,
-    reason?.startsWith("Blocked:") ? "warning" : "info",
-  );
+    // ── Step 7: Ledger notification ──
+    try {
+      const ledger = getLedger();
+      if (ledger && ledger.units.length > 0) {
+        const totals = getProjectTotals(ledger.units);
+        ctx?.ui.notify(
+          `Auto-mode stopped${reasonSuffix}. Session: ${formatCost(totals.cost)} · ${formatTokenCount(totals.tokens.total)} tokens · ${ledger.units.length} units`,
+          "info",
+        );
+      } else {
+        ctx?.ui.notify(`Auto-mode stopped${reasonSuffix}.`, "info");
+      }
+    } catch (e) {
+      debugLog("stop-cleanup-ledger", { error: e instanceof Error ? e.message : String(e) });
+    }
 
-  if (isDebugEnabled()) {
-    const logPath = writeDebugSummary();
-    if (logPath) {
-      ctx?.ui.notify(`Debug log written → ${logPath}`, "info");
+    // ── Step 8: Rebuild state ──
+    if (s.basePath) {
+      try {
+        await rebuildState(s.basePath);
+      } catch (e) {
+        debugLog("stop-rebuild-state-failed", {
+          error: e instanceof Error ? e.message : String(e),
+        });
+      }
     }
-  }
 
-  resetMetrics();
-  resetRoutingHistory();
-  resetHookState();
-  if (s.basePath) clearPersistedHookState(s.basePath);
+    // ── Step 9: Cmux sidebar / event log ──
+    try {
+      clearCmuxSidebar(loadedPreferences);
+      logCmuxEvent(
+        loadedPreferences,
+        `Auto-mode stopped${reasonSuffix || ""}.`,
+        reason?.startsWith("Blocked:") ? "warning" : "info",
+      );
+    } catch (e) {
+      debugLog("stop-cleanup-cmux", { error: e instanceof Error ? e.message : String(e) });
+    }
 
-  // Remove paused-session metadata if present (#1383)
-  try {
-    const pausedPath = join(gsdRoot(s.originalBasePath || s.basePath), "runtime", "paused-session.json");
-    if (existsSync(pausedPath)) unlinkSync(pausedPath);
-  } catch { /* non-fatal */ }
+    // ── Step 10: Debug summary ──
+    try {
+      if (isDebugEnabled()) {
+        const logPath = writeDebugSummary();
+        if (logPath) {
+          ctx?.ui.notify(`Debug log written → ${logPath}`, "info");
+        }
+      }
+    } catch (e) {
+      debugLog("stop-cleanup-debug", { error: e instanceof Error ? e.message : String(e) });
+    }
 
-  s.active = false;
-  s.paused = false;
-  s.stepMode = false;
-  s.unitDispatchCount.clear();
-  s.unitRecoveryCount.clear();
-  clearInFlightTools();
-  s.lastBudgetAlertLevel = 0;
-  s.lastStateRebuildAt = 0;
-  s.unitLifetimeDispatches.clear();
-  s.currentUnit = null;
-  s.autoModeStartModel = null;
-  s.currentMilestoneId = null;
-  s.originalBasePath = "";
-  s.completedUnits = [];
-  s.pendingQuickTasks = [];
-  clearSliceProgressCache();
-  clearActivityLogState();
-  resetProactiveHealing();
-  s.pendingCrashRecovery = null;
-  s.pendingVerificationRetry = null;
-  s.verificationRetryCount.clear();
-  s.pausedSessionFile = null;
-  ctx?.ui.setStatus("gsd-auto", undefined);
-  ctx?.ui.setWidget("gsd-progress", undefined);
-  ctx?.ui.setFooter(undefined);
+    // ── Step 11: Reset metrics, routing, hooks ──
+    try {
+      resetMetrics();
+      resetRoutingHistory();
+      resetHookState();
+      if (s.basePath) clearPersistedHookState(s.basePath);
+    } catch (e) {
+      debugLog("stop-cleanup-metrics", { error: e instanceof Error ? e.message : String(e) });
+    }
 
-  if (pi && ctx && s.originalModelId && s.originalModelProvider) {
-    const original = ctx.modelRegistry.find(
-      s.originalModelProvider,
-      s.originalModelId,
-    );
-    if (original) await pi.setModel(original);
-    s.originalModelId = null;
-    s.originalModelProvider = null;
-  }
+    // ── Step 12: Remove paused-session metadata (#1383) ──
+    try {
+      const pausedPath = join(gsdRoot(s.originalBasePath || s.basePath), "runtime", "paused-session.json");
+      if (existsSync(pausedPath)) unlinkSync(pausedPath);
+    } catch { /* non-fatal */ }
 
-  s.cmdCtx = null;
+    // ── Step 13: Restore original model (before reset clears IDs) ──
+    try {
+      if (pi && ctx && s.originalModelId && s.originalModelProvider) {
+        const original = ctx.modelRegistry.find(
+          s.originalModelProvider,
+          s.originalModelId,
+        );
+        if (original) await pi.setModel(original);
+      }
+    } catch (e) {
+      debugLog("stop-cleanup-model", { error: e instanceof Error ? e.message : String(e) });
+    }
+  } finally {
+    // ── Critical invariants: these MUST execute regardless of errors ──
+    // External cleanup (not covered by session reset)
+    clearInFlightTools();
+    clearSliceProgressCache();
+    clearActivityLogState();
+    setLevelChangeCallback(null);
+    resetProactiveHealing();
+
+    // UI cleanup
+    ctx?.ui.setStatus("gsd-auto", undefined);
+    ctx?.ui.setWidget("gsd-progress", undefined);
+    ctx?.ui.setFooter(undefined);
+
+    // Reset all session state in one call
+    s.reset();
+  }
 }
 
 /**
@@ -1091,6 +1131,7 @@ const widgetStateAccessors: WidgetStateAccessors = {
   getCmdCtx: () => s.cmdCtx,
   getBasePath: () => s.basePath,
   isVerbose: () => s.verbose,
+  isSessionSwitching: isSessionSwitchInFlight,
 };
 
 // ─── Preconditions ────────────────────────────────────────────────────────────
@@ -1145,15 +1186,6 @@ function buildRecoveryContext(): import("./auto-timeout-recovery.js").RecoveryCo
   };
 }
 
-// Re-export recovery functions for external consumers
-export {
-  resolveExpectedArtifactPath,
-  verifyExpectedArtifact,
-  writeBlockerPlaceholder,
-  skipExecuteTask,
-  buildLoopRemediationSteps,
-} from "./auto-recovery.js";
-
 /**
  * Test-only: expose skip-loop state for unit tests.
  * Not part of the public API.
@@ -1289,3 +1321,12 @@ export async function dispatchHookUnit(
 
 // Direct phase dispatch → auto-direct-dispatch.ts
 export { dispatchDirectPhase } from "./auto-direct-dispatch.js";
+
+// Re-export recovery functions for external consumers
+export {
+  resolveExpectedArtifactPath,
+  verifyExpectedArtifact,
+  writeBlockerPlaceholder,
+  skipExecuteTask,
+  buildLoopRemediationSteps,
+} from "./auto-recovery.js";

package/src/resources/extensions/gsd/bootstrap/agent-end-recovery.ts

@@ -0,0 +1,142 @@
+import type { ExtensionAPI, ExtensionContext } from "@gsd/pi-coding-agent";
+
+import { checkAutoStartAfterDiscuss } from "../guided-flow.js";
+import { getAutoDashboardData, getAutoModeStartModel, isAutoActive, pauseAuto } from "../auto.js";
+import { getNextFallbackModel, isTransientNetworkError, resolveModelWithFallbacksForUnit } from "../preferences.js";
+import { classifyProviderError, pauseAutoForProviderError } from "../provider-error-pause.js";
+import { isSessionSwitchInFlight, resolveAgentEnd } from "../auto-loop.js";
+import { clearDiscussionFlowState } from "./write-gate.js";
+
+const networkRetryCounters = new Map<string, number>();
+const MAX_TRANSIENT_AUTO_RESUMES = 3;
+let consecutiveTransientErrors = 0;
+
+export async function handleAgentEnd(
+  pi: ExtensionAPI,
+  event: { messages: any[] },
+  ctx: ExtensionContext,
+): Promise<void> {
+  if (checkAutoStartAfterDiscuss()) {
+    clearDiscussionFlowState();
+    return;
+  }
+  if (!isAutoActive()) return;
+  if (isSessionSwitchInFlight()) return;
+
+  const lastMsg = event.messages[event.messages.length - 1];
+  if (lastMsg && "stopReason" in lastMsg && lastMsg.stopReason === "aborted") {
+    await pauseAuto(ctx, pi);
+    return;
+  }
+  if (lastMsg && "stopReason" in lastMsg && lastMsg.stopReason === "error") {
+    const errorDetail = "errorMessage" in lastMsg && lastMsg.errorMessage ? `: ${lastMsg.errorMessage}` : "";
+    const errorMsg = ("errorMessage" in lastMsg && lastMsg.errorMessage) ? String(lastMsg.errorMessage) : "";
+
+    if (isTransientNetworkError(errorMsg)) {
+      const currentModelId = ctx.model?.id ?? "unknown";
+      const retryKey = `network-retry:${currentModelId}`;
+      const currentRetries = networkRetryCounters.get(retryKey) ?? 0;
+      const maxRetries = 2;
+      if (currentRetries < maxRetries) {
+        networkRetryCounters.set(retryKey, currentRetries + 1);
+        const attempt = currentRetries + 1;
+        const delayMs = attempt * 3000;
+        ctx.ui.notify(`Network error on ${currentModelId}${errorDetail}. Retry ${attempt}/${maxRetries} in ${delayMs / 1000}s...`, "warning");
+        setTimeout(() => {
+          pi.sendMessage(
+            { customType: "gsd-auto-timeout-recovery", content: "Continue execution — retrying after transient network error.", display: false },
+            { triggerTurn: true },
+          );
+        }, delayMs);
+        return;
+      }
+      networkRetryCounters.delete(retryKey);
+      ctx.ui.notify(`Network retries exhausted for ${currentModelId}. Attempting model fallback.`, "warning");
+    }
+
+    const dash = getAutoDashboardData();
+    if (dash.currentUnit) {
+      const modelConfig = resolveModelWithFallbacksForUnit(dash.currentUnit.type);
+      if (modelConfig && modelConfig.fallbacks.length > 0) {
+        const availableModels = ctx.modelRegistry.getAvailable();
+        const nextModelId = getNextFallbackModel(ctx.model?.id, modelConfig);
+        if (nextModelId) {
+          networkRetryCounters.clear();
+          const slashIdx = nextModelId.indexOf("/");
+          const modelToSet = slashIdx !== -1
+            ? availableModels.find((m) => m.provider.toLowerCase() === nextModelId.substring(0, slashIdx).toLowerCase() && m.id.toLowerCase() === nextModelId.substring(slashIdx + 1).toLowerCase())
+            : (availableModels.find((m) => m.id === nextModelId && m.provider === ctx.model?.provider) ?? availableModels.find((m) => m.id === nextModelId));
+          if (modelToSet) {
+            const ok = await pi.setModel(modelToSet, { persist: false });
+            if (ok) {
+              ctx.ui.notify(`Model error${errorDetail}. Switched to fallback: ${nextModelId} and resuming.`, "warning");
+              pi.sendMessage({ customType: "gsd-auto-timeout-recovery", content: "Continue execution.", display: false }, { triggerTurn: true });
+              return;
+            }
+          }
+        }
+      }
+    }
+
+    const sessionModel = getAutoModeStartModel();
+    if (sessionModel) {
+      if (ctx.model?.id !== sessionModel.id || ctx.model?.provider !== sessionModel.provider) {
+        const startModel = ctx.modelRegistry.getAvailable().find((m) => m.provider === sessionModel.provider && m.id === sessionModel.id);
+        if (startModel) {
+          const ok = await pi.setModel(startModel, { persist: false });
+          if (ok) {
+            networkRetryCounters.clear();
+            ctx.ui.notify(`Model error${errorDetail}. Restored session model: ${sessionModel.provider}/${sessionModel.id} and resuming.`, "warning");
+            pi.sendMessage({ customType: "gsd-auto-timeout-recovery", content: "Continue execution.", display: false }, { triggerTurn: true });
+            return;
+          }
+        }
+      }
+    }
+
+    const classification = classifyProviderError(errorMsg);
+    const explicitRetryAfterMs = ("retryAfterMs" in lastMsg && typeof lastMsg.retryAfterMs === "number") ? lastMsg.retryAfterMs : undefined;
+    if (classification.isTransient) {
+      consecutiveTransientErrors += 1;
+    } else {
+      consecutiveTransientErrors = 0;
+    }
+    const baseRetryAfterMs = explicitRetryAfterMs ?? classification.suggestedDelayMs;
+    const retryAfterMs = classification.isTransient
+      ? baseRetryAfterMs * 2 ** Math.max(0, consecutiveTransientErrors - 1)
+      : baseRetryAfterMs;
+    const allowAutoResume = classification.isTransient && consecutiveTransientErrors <= MAX_TRANSIENT_AUTO_RESUMES;
+    if (classification.isTransient && !allowAutoResume) {
+      ctx.ui.notify(`Transient provider errors persisted after ${MAX_TRANSIENT_AUTO_RESUMES} auto-resume attempts. Pausing for manual review.`, "warning");
+    }
+    await pauseAutoForProviderError(ctx.ui, errorDetail, () => pauseAuto(ctx, pi), {
+      isRateLimit: classification.isRateLimit,
+      isTransient: allowAutoResume,
+      retryAfterMs,
+      resume: allowAutoResume
+        ? () => {
+          pi.sendMessage(
+            { customType: "gsd-auto-timeout-recovery", content: "Continue execution — provider error recovery delay elapsed.", display: false },
+            { triggerTurn: true },
+          );
+        }
+        : undefined,
+    });
+    return;
+  }
+
+  try {
+    consecutiveTransientErrors = 0;
+    networkRetryCounters.clear();
+    resolveAgentEnd(event);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    ctx.ui.notify(`Auto-mode error in agent_end handler: ${message}. Stopping auto-mode.`, "error");
+    try {
+      await pauseAuto(ctx, pi);
+    } catch {
+      // best-effort
+    }
+  }
+}
+