gsd-pi 2.26.1-next.1 → 2.28.0-dev.346ee62

package/src/resources/extensions/gsd/auto-stuck-detection.ts

@@ -0,0 +1,220 @@
+/**
+ * Stuck detection and loop recovery for auto-mode unit dispatch.
+ *
+ * Tracks dispatch counts per unit, enforces lifetime caps, and attempts
+ * stub/artifact recovery before stopping.
+ *
+ * Extracted from dispatchNextUnit() in auto.ts. Returns action values
+ * instead of calling stopAuto/dispatchNextUnit — the caller handles
+ * control flow.
+ */
+
+import type { ExtensionContext } from "@gsd/pi-coding-agent";
+import {
+  inspectExecuteTaskDurability,
+} from "./unit-runtime.js";
+import {
+  verifyExpectedArtifact,
+  diagnoseExpectedArtifact,
+  skipExecuteTask,
+  persistCompletedKey,
+  buildLoopRemediationSteps,
+} from "./auto-recovery.js";
+import { closeoutUnit, type CloseoutOptions } from "./auto-unit-closeout.js";
+import { saveActivityLog } from "./activity-log.js";
+import { invalidateAllCaches } from "./cache.js";
+import { sendDesktopNotification } from "./notifications.js";
+import { debugLog } from "./debug-logger.js";
+import {
+  resolveMilestonePath,
+  resolveSlicePath,
+  resolveTasksDir,
+  buildTaskFileName,
+} from "./paths.js";
+import {
+  MAX_UNIT_DISPATCHES,
+  STUB_RECOVERY_THRESHOLD,
+  MAX_LIFETIME_DISPATCHES,
+} from "./auto/session.js";
+import type { AutoSession } from "./auto/session.js";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+export interface StuckContext {
+  s: AutoSession;
+  ctx: ExtensionContext;
+  unitType: string;
+  unitId: string;
+  basePath: string;
+  buildSnapshotOpts: () => CloseoutOptions & Record<string, unknown>;
+}
+
+export type StuckResult =
+  | { action: "proceed" }
+  | { action: "recovered"; dispatchAgain: true }
+  | { action: "stop"; reason: string; notifyMessage?: string };
+
+/**
+ * Check dispatch counts, enforce lifetime cap and MAX_UNIT_DISPATCHES,
+ * attempt stub/artifact recovery. Returns an action for the caller.
+ */
+export async function checkStuckAndRecover(sctx: StuckContext): Promise<StuckResult> {
+  const { s, ctx, unitType, unitId, basePath, buildSnapshotOpts } = sctx;
+  const dispatchKey = `${unitType}/${unitId}`;
+  const prevCount = s.unitDispatchCount.get(dispatchKey) ?? 0;
+
+  // Real dispatch reached — clear the consecutive-skip counter for this unit.
+  s.unitConsecutiveSkips.delete(dispatchKey);
+
+  debugLog("dispatch-unit", {
+    type: unitType,
+    id: unitId,
+    cycle: prevCount + 1,
+    lifetime: (s.unitLifetimeDispatches.get(dispatchKey) ?? 0) + 1,
+  });
+
+  // Hard lifetime cap — survives counter resets from loop-recovery/self-repair.
+  const lifetimeCount = (s.unitLifetimeDispatches.get(dispatchKey) ?? 0) + 1;
+  s.unitLifetimeDispatches.set(dispatchKey, lifetimeCount);
+  if (lifetimeCount > MAX_LIFETIME_DISPATCHES) {
+    if (s.currentUnit) {
+      await closeoutUnit(ctx, s.basePath, s.currentUnit.type, s.currentUnit.id, s.currentUnit.startedAt, buildSnapshotOpts());
+    } else {
+      saveActivityLog(ctx, s.basePath, unitType, unitId);
+    }
+    const expected = diagnoseExpectedArtifact(unitType, unitId, basePath);
+    return {
+      action: "stop",
+      reason: `Hard loop: ${unitType} ${unitId}`,
+      notifyMessage: `Hard loop detected: ${unitType} ${unitId} dispatched ${lifetimeCount} times total (across reconciliation cycles).${expected ? `\n   Expected artifact: ${expected}` : ""}\n   This may indicate deriveState() keeps returning the same unit despite artifacts existing.\n   Check .gsd/completed-units.json and the slice plan checkbox state.`,
+    };
+  }
+
+  if (prevCount >= MAX_UNIT_DISPATCHES) {
+    if (s.currentUnit) {
+      await closeoutUnit(ctx, s.basePath, s.currentUnit.type, s.currentUnit.id, s.currentUnit.startedAt, buildSnapshotOpts());
+    } else {
+      saveActivityLog(ctx, s.basePath, unitType, unitId);
+    }
+
+    // Final reconciliation pass for execute-task
+    if (unitType === "execute-task") {
+      const [mid, sid, tid] = unitId.split("/");
+      if (mid && sid && tid) {
+        const status = await inspectExecuteTaskDurability(basePath, unitId);
+        if (status) {
+          const reconciled = skipExecuteTask(basePath, mid, sid, tid, status, "loop-recovery", prevCount);
+          if (reconciled && verifyExpectedArtifact(unitType, unitId, basePath)) {
+            ctx.ui.notify(
+              `Loop recovery: ${unitId} reconciled after ${prevCount + 1} dispatches — blocker artifacts written, pipeline advancing.\n   Review ${status.summaryPath} and replace the placeholder with real work.`,
+              "warning",
+            );
+            const reconciledKey = `${unitType}/${unitId}`;
+            persistCompletedKey(basePath, reconciledKey);
+            s.completedKeySet.add(reconciledKey);
+            s.unitDispatchCount.delete(dispatchKey);
+            invalidateAllCaches();
+            return { action: "recovered", dispatchAgain: true };
+          }
+        }
+      }
+    }
+
+    // General reconciliation: artifact appeared on last attempt
+    if (verifyExpectedArtifact(unitType, unitId, basePath)) {
+      ctx.ui.notify(
+        `Loop recovery: ${unitType} ${unitId} — artifact verified after ${prevCount + 1} dispatches. Advancing.`,
+        "info",
+      );
+      persistCompletedKey(basePath, dispatchKey);
+      s.completedKeySet.add(dispatchKey);
+      s.unitDispatchCount.delete(dispatchKey);
+      invalidateAllCaches();
+      return { action: "recovered", dispatchAgain: true };
+    }
+
+    // Last resort for complete-milestone: generate stub summary
+    if (unitType === "complete-milestone") {
+      try {
+        const mPath = resolveMilestonePath(basePath, unitId);
+        if (mPath) {
+          const stubPath = join(mPath, `${unitId}-SUMMARY.md`);
+          if (!existsSync(stubPath)) {
+            writeFileSync(stubPath, `# ${unitId} Summary\n\nAuto-generated stub — milestone tasks completed but summary generation failed after ${prevCount + 1} attempts.\nReview and replace this stub with a proper summary.\n`);
+            ctx.ui.notify(`Generated stub summary for ${unitId} to unblock pipeline. Review later.`, "warning");
+            persistCompletedKey(basePath, dispatchKey);
+            s.completedKeySet.add(dispatchKey);
+            s.unitDispatchCount.delete(dispatchKey);
+            invalidateAllCaches();
+            return { action: "recovered", dispatchAgain: true };
+          }
+        }
+      } catch { /* non-fatal — fall through to normal stop */ }
+    }
+
+    const expected = diagnoseExpectedArtifact(unitType, unitId, basePath);
+    const remediation = buildLoopRemediationSteps(unitType, unitId, basePath);
+    sendDesktopNotification("GSD", `Loop detected: ${unitType} ${unitId}`, "error", "error");
+    return {
+      action: "stop",
+      reason: `Loop: ${unitType} ${unitId}`,
+      notifyMessage: `Loop detected: ${unitType} ${unitId} dispatched ${prevCount + 1} times total. Expected artifact not found.${expected ? `\n   Expected: ${expected}` : ""}${remediation ? `\n\n   Remediation steps:\n${remediation}` : "\n   Check branch state and .gsd/ artifacts."}`,
+    };
+  }
+
+  s.unitDispatchCount.set(dispatchKey, prevCount + 1);
+
+  if (prevCount > 0) {
+    // Adaptive self-repair: each retry attempts a different remediation step.
+    if (unitType === "execute-task") {
+      const status = await inspectExecuteTaskDurability(basePath, unitId);
+      const [mid, sid, tid] = unitId.split("/");
+      if (status && mid && sid && tid) {
+        if (status.summaryExists && !status.taskChecked) {
+          const repaired = skipExecuteTask(basePath, mid, sid, tid, status, "self-repair", 0);
+          if (repaired && verifyExpectedArtifact(unitType, unitId, basePath)) {
+            ctx.ui.notify(
+              `Self-repaired ${unitId}: summary existed but checkbox was unmarked. Marked [x] and advancing.`,
+              "warning",
+            );
+            const repairedKey = `${unitType}/${unitId}`;
+            persistCompletedKey(basePath, repairedKey);
+            s.completedKeySet.add(repairedKey);
+            s.unitDispatchCount.delete(dispatchKey);
+            invalidateAllCaches();
+            return { action: "recovered", dispatchAgain: true };
+          }
+        } else if (prevCount >= STUB_RECOVERY_THRESHOLD && !status.summaryExists) {
+          const tasksDir = resolveTasksDir(basePath, mid, sid);
+          const sDir = resolveSlicePath(basePath, mid, sid);
+          const targetDir = tasksDir ?? (sDir ? join(sDir, "tasks") : null);
+          if (targetDir) {
+            if (!existsSync(targetDir)) mkdirSync(targetDir, { recursive: true });
+            const summaryPath = join(targetDir, buildTaskFileName(tid, "SUMMARY"));
+            if (!existsSync(summaryPath)) {
+              const stubContent = [
+                `# PARTIAL RECOVERY — attempt ${prevCount + 1} of ${MAX_UNIT_DISPATCHES}`,
+                ``,
+                `Task \`${tid}\` in slice \`${sid}\` (milestone \`${mid}\`) has not yet produced a real summary.`,
+                `This placeholder was written by auto-mode after ${prevCount} dispatch attempts.`,
+                ``,
+                `The next agent session will retry this task. Replace this file with real work when done.`,
+              ].join("\n");
+              writeFileSync(summaryPath, stubContent, "utf-8");
+              ctx.ui.notify(
+                `Stub recovery (attempt ${prevCount + 1}/${MAX_UNIT_DISPATCHES}): ${unitId} stub summary placeholder written. Retrying with recovery context.`,
+                "warning",
+              );
+            }
+          }
+        }
+      }
+    }
+    ctx.ui.notify(
+      `${unitType} ${unitId} didn't produce expected artifact. Retrying (${prevCount + 1}/${MAX_UNIT_DISPATCHES}).`,
+      "warning",
+    );
+  }
+
+  return { action: "proceed" };
+}

package/src/resources/extensions/gsd/auto-timeout-recovery.ts

@@ -0,0 +1,262 @@
+/**
+ * Timeout recovery logic for auto-mode units.
+ * Handles idle and hard timeout recovery with escalation, steering messages,
+ * and blocker placeholder generation.
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@gsd/pi-coding-agent";
+import {
+  readUnitRuntimeRecord,
+  writeUnitRuntimeRecord,
+  formatExecuteTaskRecoveryStatus,
+  inspectExecuteTaskDurability,
+} from "./unit-runtime.js";
+import {
+  resolveExpectedArtifactPath,
+  diagnoseExpectedArtifact,
+  skipExecuteTask,
+  writeBlockerPlaceholder,
+} from "./auto-recovery.js";
+import { existsSync } from "node:fs";
+
+export interface RecoveryContext {
+  basePath: string;
+  verbose: boolean;
+  currentUnitStartedAt: number;
+  unitRecoveryCount: Map<string, number>;
+  dispatchNextUnit: (ctx: ExtensionContext, pi: ExtensionAPI) => Promise<void>;
+}
+
+export async function recoverTimedOutUnit(
+  ctx: ExtensionContext,
+  pi: ExtensionAPI,
+  unitType: string,
+  unitId: string,
+  reason: "idle" | "hard",
+  rctx: RecoveryContext,
+): Promise<"recovered" | "paused"> {
+  const { basePath, verbose, currentUnitStartedAt, unitRecoveryCount, dispatchNextUnit } = rctx;
+
+  const runtime = readUnitRuntimeRecord(basePath, unitType, unitId);
+  const recoveryAttempts = runtime?.recoveryAttempts ?? 0;
+  const maxRecoveryAttempts = reason === "idle" ? 2 : 1;
+
+  const recoveryKey = `${unitType}/${unitId}`;
+  const attemptNumber = (unitRecoveryCount.get(recoveryKey) ?? 0) + 1;
+  unitRecoveryCount.set(recoveryKey, attemptNumber);
+
+  if (attemptNumber > 1) {
+    // Exponential backoff: 2^(n-1) seconds, capped at 30s
+    const backoffMs = Math.min(1000 * Math.pow(2, attemptNumber - 2), 30000);
+    ctx.ui.notify(
+      `Recovery attempt ${attemptNumber} for ${unitType} ${unitId}. Waiting ${backoffMs / 1000}s before retry.`,
+      "info",
+    );
+    await new Promise(r => setTimeout(r, backoffMs));
+  }
+
+  if (unitType === "execute-task") {
+    const status = await inspectExecuteTaskDurability(basePath, unitId);
+    if (!status) return "paused";
+
+    writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+      recovery: status,
+    });
+
+    const durableComplete = status.summaryExists && status.taskChecked && status.nextActionAdvanced;
+    if (durableComplete) {
+      writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+        phase: "finalized",
+        recovery: status,
+      });
+      ctx.ui.notify(
+        `${reason === "idle" ? "Idle" : "Timeout"} recovery: ${unitType} ${unitId} already completed on disk. Continuing auto-mode. (attempt ${attemptNumber})`,
+        "info",
+      );
+      unitRecoveryCount.delete(recoveryKey);
+      await dispatchNextUnit(ctx, pi);
+      return "recovered";
+    }
+
+    if (recoveryAttempts < maxRecoveryAttempts) {
+      const isEscalation = recoveryAttempts > 0;
+      writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+        phase: "recovered",
+        recovery: status,
+        recoveryAttempts: recoveryAttempts + 1,
+        lastRecoveryReason: reason,
+        lastProgressAt: Date.now(),
+        progressCount: (runtime?.progressCount ?? 0) + 1,
+        lastProgressKind: reason === "idle" ? "idle-recovery-retry" : "hard-recovery-retry",
+      });
+
+      const steeringLines = isEscalation
+        ? [
+            `**FINAL ${reason === "idle" ? "IDLE" : "HARD TIMEOUT"} RECOVERY — last chance before this task is skipped.**`,
+            `You are still executing ${unitType} ${unitId}.`,
+            `Recovery attempt ${recoveryAttempts + 1} of ${maxRecoveryAttempts}.`,
+            `Current durability status: ${formatExecuteTaskRecoveryStatus(status)}.`,
+            "You MUST finish the durable output NOW, even if incomplete.",
+            "Write the task summary with whatever you have accomplished so far.",
+            "Mark the task [x] in the plan. Commit your work.",
+            "A partial summary is infinitely better than no summary.",
+          ]
+        : [
+            `**${reason === "idle" ? "IDLE" : "HARD TIMEOUT"} RECOVERY — do not stop.**`,
+            `You are still executing ${unitType} ${unitId}.`,
+            `Recovery attempt ${recoveryAttempts + 1} of ${maxRecoveryAttempts}.`,
+            `Current durability status: ${formatExecuteTaskRecoveryStatus(status)}.`,
+            "Do not keep exploring.",
+            "Immediately finish the required durable output for this unit.",
+            "If full completion is impossible, write the partial artifact/state needed for recovery and make the blocker explicit.",
+          ];
+
+      pi.sendMessage(
+        {
+          customType: "gsd-auto-timeout-recovery",
+          display: verbose,
+          content: steeringLines.join("\n"),
+        },
+        { triggerTurn: true, deliverAs: "steer" },
+      );
+      ctx.ui.notify(
+        `${reason === "idle" ? "Idle" : "Timeout"} recovery: steering ${unitType} ${unitId} to finish durable output (attempt ${attemptNumber}, session ${recoveryAttempts + 1}/${maxRecoveryAttempts}).`,
+        "warning",
+      );
+      return "recovered";
+    }
+
+    // Retries exhausted — write missing durable artifacts and advance.
+    const diagnostic = formatExecuteTaskRecoveryStatus(status);
+    const [mid, sid, tid] = unitId.split("/");
+    const skipped = mid && sid && tid
+      ? skipExecuteTask(basePath, mid, sid, tid, status, reason, maxRecoveryAttempts)
+      : false;
+
+    if (skipped) {
+      writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+        phase: "skipped",
+        recovery: status,
+        recoveryAttempts: recoveryAttempts + 1,
+        lastRecoveryReason: reason,
+      });
+      ctx.ui.notify(
+        `${unitType} ${unitId} skipped after ${maxRecoveryAttempts} recovery attempts (${diagnostic}). Blocker artifacts written. Advancing pipeline. (attempt ${attemptNumber})`,
+        "warning",
+      );
+      unitRecoveryCount.delete(recoveryKey);
+      await dispatchNextUnit(ctx, pi);
+      return "recovered";
+    }
+
+    // Fallback: couldn't write skip artifacts — pause as before.
+    writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+      phase: "paused",
+      recovery: status,
+      recoveryAttempts: recoveryAttempts + 1,
+      lastRecoveryReason: reason,
+    });
+    ctx.ui.notify(
+      `${reason === "idle" ? "Idle" : "Timeout"} recovery check for ${unitType} ${unitId}: ${diagnostic}`,
+      "warning",
+    );
+    return "paused";
+  }
+
+  const expected = diagnoseExpectedArtifact(unitType, unitId, basePath) ?? "required durable artifact";
+
+  // Check if the artifact already exists on disk — agent may have written it
+  // without signaling completion.
+  const artifactPath = resolveExpectedArtifactPath(unitType, unitId, basePath);
+  if (artifactPath && existsSync(artifactPath)) {
+    writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+      phase: "finalized",
+      recoveryAttempts: recoveryAttempts + 1,
+      lastRecoveryReason: reason,
+    });
+    ctx.ui.notify(
+      `${reason === "idle" ? "Idle" : "Timeout"} recovery: ${unitType} ${unitId} artifact already exists on disk. Advancing. (attempt ${attemptNumber})`,
+      "info",
+    );
+    unitRecoveryCount.delete(recoveryKey);
+    await dispatchNextUnit(ctx, pi);
+    return "recovered";
+  }
+
+  if (recoveryAttempts < maxRecoveryAttempts) {
+    const isEscalation = recoveryAttempts > 0;
+    writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+      phase: "recovered",
+      recoveryAttempts: recoveryAttempts + 1,
+      lastRecoveryReason: reason,
+      lastProgressAt: Date.now(),
+      progressCount: (runtime?.progressCount ?? 0) + 1,
+      lastProgressKind: reason === "idle" ? "idle-recovery-retry" : "hard-recovery-retry",
+    });
+
+    const steeringLines = isEscalation
+      ? [
+          `**FINAL ${reason === "idle" ? "IDLE" : "HARD TIMEOUT"} RECOVERY — last chance before skip.**`,
+          `You are still executing ${unitType} ${unitId}.`,
+          `Recovery attempt ${recoveryAttempts + 1} of ${maxRecoveryAttempts} — next failure skips this unit.`,
+          `Expected durable output: ${expected}.`,
+          "You MUST write the artifact file NOW, even if incomplete.",
+          "Write whatever you have — partial research, preliminary findings, best-effort analysis.",
+          "A partial artifact is infinitely better than no artifact.",
+          "If you are truly blocked, write the file with a BLOCKER section explaining why.",
+        ]
+      : [
+          `**${reason === "idle" ? "IDLE" : "HARD TIMEOUT"} RECOVERY — stay in auto-mode.**`,
+          `You are still executing ${unitType} ${unitId}.`,
+          `Recovery attempt ${recoveryAttempts + 1} of ${maxRecoveryAttempts}.`,
+          `Expected durable output: ${expected}.`,
+          "Stop broad exploration.",
+          "Write the required artifact now.",
+          "If blocked, write the partial artifact and explicitly record the blocker instead of going silent.",
+        ];
+
+    pi.sendMessage(
+      {
+        customType: "gsd-auto-timeout-recovery",
+        display: verbose,
+        content: steeringLines.join("\n"),
+      },
+      { triggerTurn: true, deliverAs: "steer" },
+    );
+    ctx.ui.notify(
+      `${reason === "idle" ? "Idle" : "Timeout"} recovery: steering ${unitType} ${unitId} to produce ${expected} (attempt ${attemptNumber}, session ${recoveryAttempts + 1}/${maxRecoveryAttempts}).`,
+      "warning",
+    );
+    return "recovered";
+  }
+
+  // Retries exhausted — write a blocker placeholder and advance the pipeline
+  // instead of silently stalling.
+  const placeholder = writeBlockerPlaceholder(
+    unitType, unitId, basePath,
+    `${reason} recovery exhausted ${maxRecoveryAttempts} attempts without producing the artifact.`,
+  );
+
+  if (placeholder) {
+    writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+      phase: "skipped",
+      recoveryAttempts: recoveryAttempts + 1,
+      lastRecoveryReason: reason,
+    });
+    ctx.ui.notify(
+      `${unitType} ${unitId} skipped after ${maxRecoveryAttempts} recovery attempts. Blocker placeholder written to ${placeholder}. Advancing pipeline. (attempt ${attemptNumber})`,
+      "warning",
+    );
+    unitRecoveryCount.delete(recoveryKey);
+    await dispatchNextUnit(ctx, pi);
+    return "recovered";
+  }
+
+  // Fallback: couldn't resolve artifact path — pause as before.
+  writeUnitRuntimeRecord(basePath, unitType, unitId, currentUnitStartedAt, {
+    phase: "paused",
+    recoveryAttempts: recoveryAttempts + 1,
+    lastRecoveryReason: reason,
+  });
+  return "paused";
+}