gsd-pi 2.81.0 → 2.82.0-dev.ed17d078d

package/src/resources/extensions/gsd/auto/phases.ts

@@ -58,6 +58,7 @@ import { withTimeout, FINALIZE_PRE_TIMEOUT_MS, FINALIZE_POST_TIMEOUT_MS } from "
 import { getEligibleSlices } from "../slice-parallel-eligibility.js";
 import { startSliceParallel } from "../slice-parallel-orchestrator.js";
 import { isDbAvailable, getMilestoneSlices } from "../gsd-db.js";
+import { reconcileBeforeSpawn } from "../state-reconciliation.js";
 import type { MinimalModelRegistry } from "../context-budget.js";
 import type { PostflightResult, PreflightResult } from "../clean-root-preflight.js";
 import { ensurePlanV2Graph, isEmptyPlanV2GraphResult, isMissingFinalizedContextResult } from "../uok/plan-v2.js";
@@ -74,6 +75,8 @@ import {
 } from "../workflow-mcp.js";
 import { resolveManifest } from "../unit-context-manifest.js";
 import { createWorktreeSafetyModule, type WorktreeSafetyResult } from "../worktree-safety.js";
+import { isSuspiciousGhostCompletion } from "../auto-unit-closeout.js";
+import { decideVerificationRetry, verificationRetryKey } from "./verification-retry-policy.js";
 
 // ─── Path Comparison Helper ───────────────────────────────────────────────
 /** Compare two paths for physical identity, tolerating trailing slashes and symlinks. */
@@ -81,6 +84,56 @@ function isSamePathLocal(a: string, b: string): boolean {
   return normalizeWorktreePathForCompare(a) === normalizeWorktreePathForCompare(b);
 }
 
+async function applyVerificationRetryPolicy(
+  ic: IterationContext,
+  unitType: string | undefined,
+  phase: "artifact-verification-retry" | "verification-retry",
+): Promise<PhaseResult | null> {
+  const { ctx, pi, s, deps } = ic;
+  const retryInfo = s.pendingVerificationRetry;
+  const key = unitType && retryInfo
+    ? verificationRetryKey(unitType, retryInfo.unitId)
+    : undefined;
+  const decision = decideVerificationRetry({
+    unitType,
+    retryInfo,
+    previousFailureHash: key ? s.verificationRetryFailureHashes.get(key) : undefined,
+  });
+
+  if (decision.action === "pause") {
+    s.pendingVerificationRetry = null;
+    debugLog("autoLoop", {
+      phase: `${phase}-paused`,
+      reason: decision.reason,
+      unitType,
+      unitId: retryInfo?.unitId,
+      failureHash: decision.failureHash,
+    });
+    ctx.ui.notify(
+      decision.reason === "duplicate-failure-context"
+        ? `Verification retry for ${unitType ?? "unit"} ${retryInfo?.unitId ?? "unknown"} produced the same failure context. Pausing auto-mode instead of re-dispatching.`
+        : "Verification retry requested without retry context. Pausing auto-mode instead of re-dispatching.",
+      "warning",
+    );
+    await deps.pauseAuto(ctx, pi);
+    return { action: "break", reason: decision.reason };
+  }
+
+  s.verificationRetryFailureHashes.set(decision.key, decision.failureHash);
+  debugLog("autoLoop", {
+    phase: `${phase}-backoff`,
+    iteration: ic.iteration,
+    unitType,
+    unitId: retryInfo?.unitId,
+    attempt: retryInfo?.attempt,
+    delayMs: decision.delayMs,
+    baseDelayMs: decision.baseDelayMs,
+    failureHash: decision.failureHash,
+  });
+  await new Promise<void>((resolve) => setTimeout(resolve, decision.delayMs));
+  return null;
+}
+
 export function shouldDegradeEmptyWorktreeToProjectRoot(
   worktreeClassification: ReturnType<typeof classifyProject>,
   projectRootClassification: ReturnType<typeof classifyProject>,
@@ -826,6 +879,16 @@ export async function runPreDispatch(
             `Slice-parallel: dispatching ${eligible.length} eligible slices for ${mid}.`,
             "info",
           );
+          // ADR-017 #5707: reconcile before spawning so each worker doesn't
+          // independently race on the same drift. Failure aborts the spawn.
+          const spawnGate = await reconcileBeforeSpawn(s.basePath);
+          if (!spawnGate.ok) {
+            ctx.ui.notify(
+              `Slice-parallel: aborting spawn — ${spawnGate.reason}`,
+              "error",
+            );
+            return { action: "break", reason: `slice-parallel-reconciliation-failed: ${spawnGate.reason}` };
+          }
           const result = await startSliceParallel(
             s.basePath,
             mid,
@@ -1008,7 +1071,13 @@ export async function runPreDispatch(
         "All milestones complete.",
         "success",
       );
-      await deps.stopAuto(ctx, pi, "All milestones complete");
+      await deps.stopAuto(ctx, pi, "All milestones complete", {
+        completionWidget: {
+          milestoneId: s.currentMilestoneId,
+          milestoneTitle: midTitle,
+          allMilestonesComplete: true,
+        },
+      });
     } else if (incomplete.length === 0 && state.registry.length === 0) {
       // Empty registry — no milestones visible, likely a path resolution bug
       const diag = `basePath=${s.basePath}, phase=${state.phase}`;
@@ -1103,7 +1172,23 @@ export async function runPreDispatch(
       `Milestone ${mid} complete.`,
       "success",
     );
-    await closeoutAndStop(ctx, pi, s, deps, `Milestone ${mid} complete`);
+    if (s.currentUnit) {
+      await deps.closeoutUnit(
+        ctx,
+        s.basePath,
+        s.currentUnit.type,
+        s.currentUnit.id,
+        s.currentUnit.startedAt,
+        deps.buildSnapshotOpts(s.currentUnit.type, s.currentUnit.id),
+      );
+      s.currentUnit = null;
+    }
+    await deps.stopAuto(ctx, pi, `Milestone ${mid} complete`, {
+      completionWidget: {
+        milestoneId: mid,
+        milestoneTitle: midTitle,
+      },
+    });
     debugLog("autoLoop", { phase: "exit", reason: "milestone-complete" });
     deps.emitJournalEvent({ ts: new Date().toISOString(), flowId: ic.flowId, seq: ic.nextSeq(), eventType: "terminal", data: { reason: "milestone-complete", milestoneId: mid } });
     return { action: "break", reason: "milestone-complete" };
@@ -1266,16 +1351,14 @@ export async function runDispatch(
   // ── Sliding-window stuck detection with graduated recovery ──
   const derivedKey = `${unitType}/${unitId}`;
 
-  // Always record this dispatch in the sliding window so detectStuck() has
-  // accurate history. Skipping the push when pendingVerificationRetry is set
-  // caused infinite artifact-retry loops to be invisible to stuck detection
-  // (#2007). Only the *response* to a stuck signal is suppressed during retries.
+  // Always record this dispatch in the sliding window and run detection so
+  // Rules 1/3/4 can catch retry loops with repeated failure content (#5719).
+  // Rules 2/2b suppress legitimate retry backoff through the dispatch ledger.
   loopState.recentUnits.push({ key: derivedKey });
   if (loopState.recentUnits.length > STUCK_WINDOW_SIZE) loopState.recentUnits.shift();
 
-  if (!s.pendingVerificationRetry) {
-    const stuckSignal = detectStuck(loopState.recentUnits);
-    if (stuckSignal) {
+  const stuckSignal = detectStuck(loopState.recentUnits);
+  if (stuckSignal) {
       debugLog("autoLoop", {
         phase: "stuck-check",
         unitType,
@@ -1391,16 +1474,15 @@ export async function runDispatch(
         );
         return { action: "break", reason: "stuck-detected" };
       }
-    } else {
-      // Progress detected — reset recovery counter
-      if (loopState.stuckRecoveryAttempts > 0) {
-        debugLog("autoLoop", {
-          phase: "stuck-counter-reset",
-          from: loopState.recentUnits[loopState.recentUnits.length - 2]?.key ?? "",
-          to: derivedKey,
-        });
-        loopState.stuckRecoveryAttempts = 0;
-      }
+  } else {
+    // Progress detected — reset recovery counter
+    if (loopState.stuckRecoveryAttempts > 0) {
+      debugLog("autoLoop", {
+        phase: "stuck-counter-reset",
+        from: loopState.recentUnits[loopState.recentUnits.length - 2]?.key ?? "",
+        to: derivedKey,
+      });
+      loopState.stuckRecoveryAttempts = 0;
     }
   }
 
@@ -1984,6 +2066,33 @@ export async function runUnitPhase(
     status: unitResult.status,
   });
 
+  if (
+    unitResult.status === "completed" &&
+    s.currentUnit &&
+    (unitResult.event?.messages?.length ?? 0) === 0 &&
+    isSuspiciousGhostCompletion(ctx, unitResult.requestDispatchedAt ?? s.currentUnit.startedAt)
+  ) {
+    const message =
+      `${unitType} ${unitId} completed without assistant output or tool calls; treating as a stale ghost completion.`;
+    debugLog("autoLoop", {
+      phase: "ghost-completion",
+      iteration: ic.iteration,
+      unitType,
+      unitId,
+      elapsedMs: Date.now() - (unitResult.requestDispatchedAt ?? s.currentUnit.startedAt),
+    });
+    logWarning("engine", message);
+    ctx.ui.notify(`${message} Pausing auto-mode before closeout side effects.`, "warning");
+    await emitCancelledUnitEnd(ic, unitType, unitId, unitStartSeq, {
+      message,
+      category: "unknown",
+      isTransient: true,
+    });
+    s.currentUnit = null;
+    await deps.pauseAuto(ctx, pi);
+    return { action: "break", reason: "ghost-completion" };
+  }
+
   // Now that runUnit has called newSession(), the session file path is correct.
   const sessionFile = deps.getSessionFile(ctx);
   deps.updateSessionLock(
@@ -2124,6 +2233,20 @@ export async function runUnitPhase(
       await emitCancelledUnitEnd(ic, unitType, unitId, unitStartSeq, unitResult.errorContext);
       return { action: "break", reason: "session-timeout" };
     }
+    if (
+      unitResult.errorContext?.isTransient &&
+      errorCategory === "aborted"
+    ) {
+      ctx.ui.notify(
+        `Unit ${unitType} ${unitId} was aborted by the user. Pausing auto-mode (recoverable).`,
+        "warning",
+      );
+      debugLog("autoLoop", { phase: "unit-aborted-transient-pause", unitType, unitId, category: errorCategory });
+      await deps.pauseAuto(ctx, pi);
+      await deps.autoCommitUnit?.(s.basePath, unitType, unitId, ctx);
+      await emitCancelledUnitEnd(ic, unitType, unitId, unitStartSeq, unitResult.errorContext);
+      return { action: "break", reason: "unit-aborted-pause" };
+    }
     // All other cancelled states (structural errors, non-transient failures): hard stop
     if (s.currentUnit) {
       await deps.closeoutUnit(
@@ -2364,6 +2487,14 @@ export async function runFinalize(
           attempt: retryInfo?.attempt,
         },
       });
+      const retryPolicyResult = await applyVerificationRetryPolicy(
+        ic,
+        preUnitSnapshot?.type,
+        "artifact-verification-retry",
+      );
+      if (retryPolicyResult) {
+        return retryPolicyResult;
+      }
       // Continue the loop — next iteration will inject the retry context into the prompt.
       debugLog("autoLoop", { phase: "artifact-verification-retry", iteration: ic.iteration });
       return { action: "continue" };
@@ -2401,6 +2532,14 @@ export async function runFinalize(
         debugLog("autoLoop", { phase: "sidecar-verification-retry-skipped", iteration: ic.iteration });
       } else {
         // s.pendingVerificationRetry was set by runPostUnitVerification.
+        const retryPolicyResult = await applyVerificationRetryPolicy(
+          ic,
+          iterData.unitType,
+          "verification-retry",
+        );
+        if (retryPolicyResult) {
+          return retryPolicyResult;
+        }
         // Continue the loop — next iteration will inject the retry context into the prompt.
         debugLog("autoLoop", { phase: "verification-retry", iteration: ic.iteration });
         return { action: "continue" };

package/src/resources/extensions/gsd/auto/run-unit.ts

@@ -26,6 +26,31 @@ import { debugLog } from "../debug-logger.js";
 import { logWarning, logError } from "../workflow-logger.js";
 import { resolveAutoSupervisorConfig } from "../preferences.js";
 import { formatAutoUnitWorkingMessage } from "../working-output-messages.js";
+import { readUnitRuntimeRecord, type AutoUnitRuntimeRecord } from "../unit-runtime.js";
+
+const UNIT_FAILSAFE_BUFFER_MS = 30_000;
+const UNIT_FAILSAFE_RECHECK_MS = 30_000;
+
+export function shouldDeferUnitFailsafeTimeout(
+  runtime: AutoUnitRuntimeRecord | null,
+  opts: {
+    nowMs: number;
+    currentUnitStartedAt?: number;
+    freshProgressMs: number;
+  },
+): boolean {
+  if (!runtime) return false;
+  if (opts.currentUnitStartedAt === undefined) return false;
+  if (runtime.startedAt !== opts.currentUnitStartedAt) return false;
+  if (runtime.lastProgressAt <= 0) return false;
+  const progressAgeMs = opts.nowMs - runtime.lastProgressAt;
+  if (progressAgeMs < 0) return false;
+  if (progressAgeMs > opts.freshProgressMs) return false;
+  if (runtime.phase === "recovered") return true;
+  if (runtime.lastProgressKind.includes("recovery")) return true;
+  if (runtime.recoveryAttempts && runtime.recoveryAttempts > 0) return true;
+  return progressAgeMs >= 0 && progressAgeMs <= opts.freshProgressMs;
+}
 
 // Tracks the latest session-switch attempt so a late timeout settlement from an
 // older runUnit() call cannot clear the guard for a newer one.
@@ -192,8 +217,12 @@ export async function runUnit(
   // Without this, a crashed agent that never emits agent_end hangs the loop (#3161).
   const supervisor = resolveAutoSupervisorConfig();
   const UNIT_HARD_TIMEOUT_MS = Math.max(
-    30_000,
-    ((supervisor.hard_timeout_minutes ?? 30) * 60 * 1000) + 30_000,
+    UNIT_FAILSAFE_BUFFER_MS,
+    ((supervisor.hard_timeout_minutes ?? 30) * 60 * 1000) + UNIT_FAILSAFE_BUFFER_MS,
+  );
+  const freshProgressMs = Math.max(
+    UNIT_FAILSAFE_BUFFER_MS,
+    ((supervisor.idle_timeout_minutes ?? 10) * 60 * 1000) + UNIT_FAILSAFE_BUFFER_MS,
   );
   let unitTimeoutHandle: ReturnType<typeof setTimeout> | undefined;
   let result: UnitResult;
@@ -205,9 +234,45 @@ export async function runUnit(
 
     debugLog("runUnit", { phase: "awaiting-agent-end", unitType, unitId });
     const timeoutResult = new Promise<UnitResult>((resolve) => {
-      unitTimeoutHandle = setTimeout(() => {
+      const settleOrDefer = () => {
+        let runtime: AutoUnitRuntimeRecord | null;
+        try {
+          runtime = readUnitRuntimeRecord(s.basePath, unitType, unitId);
+        } catch (error) {
+          debugLog("runUnit", {
+            phase: "unit-failsafe-runtime-read-failed",
+            unitType,
+            unitId,
+            error: error instanceof Error ? error.message : String(error),
+          });
+          resolve({
+            status: "cancelled",
+            errorContext: {
+              message: "Unit hard timeout — supervision may have failed; runtime progress could not be read",
+              category: "timeout",
+              isTransient: true,
+            },
+          });
+          return;
+        }
+        if (shouldDeferUnitFailsafeTimeout(runtime, {
+          nowMs: Date.now(),
+          currentUnitStartedAt: s.currentUnit?.startedAt,
+          freshProgressMs,
+        })) {
+          debugLog("runUnit", {
+            phase: "unit-failsafe-deferred",
+            unitType,
+            unitId,
+            runtimePhase: runtime?.phase,
+            lastProgressKind: runtime?.lastProgressKind,
+          });
+          unitTimeoutHandle = setTimeout(settleOrDefer, UNIT_FAILSAFE_RECHECK_MS);
+          return;
+        }
         resolve({ status: "cancelled", errorContext: { message: "Unit hard timeout — supervision may have failed", category: "timeout", isTransient: true } });
-      }, UNIT_HARD_TIMEOUT_MS);
+      };
+      unitTimeoutHandle = setTimeout(settleOrDefer, UNIT_HARD_TIMEOUT_MS);
     });
     result = await runWithTurnGeneration(capturedTurnGen, () =>
       Promise.race([unitPromise, timeoutResult]),

package/src/resources/extensions/gsd/auto/session.ts

@@ -88,6 +88,7 @@ export class AutoSession {
   // ── Lifecycle ────────────────────────────────────────────────────────────
   active = false;
   paused = false;
+  completionStopInProgress = false;
   stepMode = false;
   verbose = false;
   activeEngineId: string | null = null;
@@ -158,6 +159,7 @@ export class AutoSession {
   pendingCrashRecovery: string | null = null;
   pendingVerificationRetry: PendingVerificationRetry | null = null;
   readonly verificationRetryCount = new Map<string, number>();
+  readonly verificationRetryFailureHashes = new Map<string, string>();
   pausedSessionFile: string | null = null;
   pausedUnitType: string | null = null;
   pausedUnitId: string | null = null;
@@ -286,6 +288,7 @@ export class AutoSession {
     // Lifecycle
     this.active = false;
     this.paused = false;
+    this.completionStopInProgress = false;
     this.stepMode = false;
     this.verbose = false;
     this.activeEngineId = null;
@@ -334,6 +337,7 @@ export class AutoSession {
     this.pendingCrashRecovery = null;
     this.pendingVerificationRetry = null;
     this.verificationRetryCount.clear();
+    this.verificationRetryFailureHashes.clear();
     this.pausedSessionFile = null;
     this.pausedUnitType = null;
     this.pausedUnitId = null;
@@ -372,6 +376,12 @@ export class AutoSession {
     // Loop promise state lives in auto-loop.ts module scope
   }
 
+  resetAfterStop(options: { preserveCompletionSurface?: boolean } = {}): void {
+    const completionStopInProgress = options.preserveCompletionSurface ? this.completionStopInProgress : false;
+    this.reset();
+    this.completionStopInProgress = completionStopInProgress;
+  }
+
   toJSON(): Record<string, unknown> {
     const orchestrationStatus = this.orchestration?.getStatus();
     return {

package/src/resources/extensions/gsd/auto/verification-retry-policy.ts

@@ -0,0 +1,82 @@
+// Project/App: GSD-2
+// File Purpose: Central retry policy for auto-mode verification redispatches.
+
+import { createHash } from "node:crypto";
+
+import type { PendingVerificationRetry } from "./session.js";
+
+export const VERIFICATION_RETRY_BASE_DELAY_MS = 2_000;
+export const VERIFICATION_RETRY_MAX_DELAY_MS = 30_000;
+export const VERIFICATION_RETRY_JITTER_RATIO = 0.1;
+
+export type VerificationRetryDecision =
+  | {
+      action: "delay";
+      key: string;
+      failureHash: string;
+      delayMs: number;
+      baseDelayMs: number;
+    }
+  | {
+      action: "pause";
+      reason: "missing-retry-context" | "duplicate-failure-context";
+      key?: string;
+      failureHash?: string;
+    };
+
+export function verificationRetryKey(unitType: string, unitId: string): string {
+  return `${unitType}:${unitId}`;
+}
+
+export function hashVerificationFailureContext(failureContext: string): string {
+  const normalized = failureContext.replace(/\r\n/g, "\n").trim();
+  return createHash("sha256").update(normalized).digest("hex");
+}
+
+export function verificationRetryDelayMs(
+  attempt: number,
+  random: () => number = Math.random,
+): { delayMs: number; baseDelayMs: number } {
+  const safeAttempt = Math.max(1, Math.floor(attempt));
+  const baseDelayMs = Math.min(
+    VERIFICATION_RETRY_MAX_DELAY_MS,
+    VERIFICATION_RETRY_BASE_DELAY_MS * 2 ** (safeAttempt - 1),
+  );
+  const jitterSpanMs = Math.round(baseDelayMs * VERIFICATION_RETRY_JITTER_RATIO);
+  const jitterMs = Math.round((random() - 0.5) * 2 * jitterSpanMs);
+  const delayMs = Math.min(
+    VERIFICATION_RETRY_MAX_DELAY_MS,
+    Math.max(0, baseDelayMs + jitterMs),
+  );
+  return { delayMs, baseDelayMs };
+}
+
+export function decideVerificationRetry(input: {
+  unitType: string | undefined;
+  retryInfo: PendingVerificationRetry | null | undefined;
+  previousFailureHash: string | undefined;
+  random?: () => number;
+}): VerificationRetryDecision {
+  const { retryInfo, unitType } = input;
+  if (!retryInfo || !unitType) {
+    return { action: "pause", reason: "missing-retry-context" };
+  }
+
+  const key = verificationRetryKey(unitType, retryInfo.unitId);
+  const failureHash = hashVerificationFailureContext(retryInfo.failureContext);
+  if (input.previousFailureHash === failureHash) {
+    return {
+      action: "pause",
+      reason: "duplicate-failure-context",
+      key,
+      failureHash,
+    };
+  }
+
+  return {
+    action: "delay",
+    key,
+    failureHash,
+    ...verificationRetryDelayMs(retryInfo.attempt, input.random),
+  };
+}