@opengsd/gsd-pi 1.2.0-dev.fb12b103 → 1.2.0-dev.fbdca60b

package/src/resources/extensions/gsd/auto/finalize.ts

@@ -0,0 +1,445 @@
+// Project/App: gsd-pi
+// File Purpose: Auto-loop finalize phase — post-unit verification and UAT pause.
+
+import type { SidecarItem } from "./session.js";
+import {
+  type PostUnitContext,
+  type PreVerificationOpts,
+} from "../auto-post-unit.js";
+import { clearCurrentPhase } from "../../shared/gsd-phase-state.js";
+import { withTimeout, FINALIZE_PRE_TIMEOUT_MS, FINALIZE_POST_TIMEOUT_MS } from "./finalize-timeout.js";
+import { writeUnitRuntimeRecord } from "../unit-runtime.js";
+import { buildManualValidationGuidance } from "../worktree-manager.js";
+import { relSliceFile } from "../paths.js";
+import {
+  detectRootWriteLeak,
+  formatRootWriteLeakMessage,
+} from "../root-write-leak-guard.js";
+import {
+  logWarning,
+  drainLogs,
+  drainAndSummarize,
+  formatForNotification,
+  hasAnyIssues,
+} from "../workflow-logger.js";
+import { debugLog } from "../debug-logger.js";
+import { buildPhaseHandoffOutcome, setAutoOutcomeWidget } from "../auto-dashboard.js";
+import {
+  applyVerificationRetryPolicy,
+  rememberRetryDispatch,
+  _resolveCurrentUnitStartedAtForTest,
+  isIsolatedWorktreeSession,
+} from "./phase-helpers.js";
+import { _runMilestoneMergeOnceWithStashRestore } from "./closeout.js";
+import type { IterationContext, IterationData, LoopState, PhaseResult } from "./types.js";
+import { MAX_FINALIZE_TIMEOUTS } from "./types.js";
+
+export async function failClosedOnFinalizeTimeout(
+  ic: IterationContext,
+  iterData: IterationData,
+  loopState: LoopState,
+  stage: "pre" | "post",
+  startedAt: number,
+): Promise<PhaseResult> {
+  const { ctx, pi, s, deps } = ic;
+  const now = Date.now();
+  const unitType = iterData.unitType;
+  const unitId = iterData.unitId;
+  const timeoutMs = stage === "pre" ? FINALIZE_PRE_TIMEOUT_MS : FINALIZE_POST_TIMEOUT_MS;
+  const progressKind = stage === "pre" ? "finalize-pre-timeout" : "finalize-post-timeout";
+
+  writeUnitRuntimeRecord(s.basePath, unitType, unitId, startedAt, {
+    phase: "finalize-timeout",
+    timeoutAt: now,
+    lastProgressAt: now,
+    lastProgressKind: progressKind,
+  });
+
+  deps.emitJournalEvent({
+    ts: new Date(now).toISOString(),
+    flowId: ic.flowId,
+    seq: ic.nextSeq(),
+    eventType: "unit-end",
+    data: {
+      unitType,
+      unitId,
+      status: "timed-out-finalize",
+      artifactVerified: false,
+      finalizeStage: stage,
+    },
+  });
+
+  loopState.consecutiveFinalizeTimeouts++;
+  debugLog("autoLoop", {
+    phase: progressKind,
+    iteration: ic.iteration,
+    unitType,
+    unitId,
+    consecutiveTimeouts: loopState.consecutiveFinalizeTimeouts,
+  });
+
+  ctx.ui.notify(
+    `${stage === "pre" ? "postUnitPreVerification" : "postUnitPostVerification"} timed out after ${timeoutMs / 1000}s for ${unitType} ${unitId} (${loopState.consecutiveFinalizeTimeouts}/${MAX_FINALIZE_TIMEOUTS}) — pausing auto-mode for recovery.`,
+    "warning",
+  );
+
+  await deps.pauseAuto(ctx, pi);
+  s.clearCurrentUnit();
+  clearCurrentPhase();
+  drainLogs();
+  return { action: "break", reason: progressKind };
+}
+
+/**
+ * Phase 5: Post-unit finalize — pre/post verification, UAT pause, step-wizard.
+ * Returns break/continue/next to control the outer loop.
+ */
+export async function runFinalize(
+  ic: IterationContext,
+  iterData: IterationData,
+  loopState: LoopState,
+  sidecarItem?: SidecarItem,
+): Promise<PhaseResult> {
+  const { ctx, pi, s, deps } = ic;
+  const { pauseAfterUatDispatch } = iterData;
+
+  debugLog("autoLoop", { phase: "finalize", iteration: ic.iteration });
+
+  // Clear unit timeout (unit completed)
+  deps.clearUnitTimeout();
+
+  // Post-unit context for pre/post verification
+  const postUnitCtx: PostUnitContext = {
+    s,
+    ctx,
+    pi,
+    buildSnapshotOpts: deps.buildSnapshotOpts,
+    lockBase: deps.lockBase,
+    stopAuto: deps.stopAuto,
+    pauseAuto: deps.pauseAuto,
+    updateProgressWidget: deps.updateProgressWidget,
+  };
+
+  // Pre-verification processing (commit, doctor, state rebuild, etc.)
+  // Timeout guard: if postUnitPreVerification hangs (e.g., safety harness
+  // deadlock, browser teardown hang, worktree sync stall), force-continue
+  // after timeout so the auto-loop is not permanently frozen (#3757).
+  //
+  // On timeout, null out s.currentUnit so the timed-out task's late async
+  // mutations are harmless — postUnitPreVerification guards all side effects
+  // behind `if (s.currentUnit)`. The next iteration sets a fresh currentUnit.
+  // Sidecar items use lightweight pre-verification opts
+  const preVerificationOpts: PreVerificationOpts = sidecarItem
+    ? sidecarItem.kind === "hook"
+      ? { skipSettleDelay: true, skipWorktreeSync: true, agentEndMessages: s.lastUnitAgentEndMessages ?? undefined }
+      : { skipSettleDelay: true, agentEndMessages: s.lastUnitAgentEndMessages ?? undefined }
+    : { agentEndMessages: s.lastUnitAgentEndMessages ?? undefined };
+  const preUnitSnapshot = s.currentUnit
+    ? { type: s.currentUnit.type, id: s.currentUnit.id, startedAt: s.currentUnit.startedAt }
+    : null;
+  const clearFinalizingUnit = () => {
+    if (
+      preUnitSnapshot &&
+      s.currentUnit?.type === preUnitSnapshot.type &&
+      s.currentUnit?.id === preUnitSnapshot.id &&
+      s.currentUnit?.startedAt === preUnitSnapshot.startedAt
+    ) {
+      s.clearCurrentUnit();
+    }
+    s.rootWriteBaseline = null;
+  };
+  clearCurrentPhase();
+  const preResultGuard = await withTimeout(
+    deps.postUnitPreVerification(postUnitCtx, preVerificationOpts),
+    FINALIZE_PRE_TIMEOUT_MS,
+    "postUnitPreVerification",
+  );
+
+  if (preResultGuard.timedOut) {
+    return failClosedOnFinalizeTimeout(
+      ic,
+      iterData,
+      loopState,
+      "pre",
+      preUnitSnapshot?.startedAt ?? Date.now(),
+    );
+  }
+
+  const preResult = preResultGuard.value;
+  if (preResult === "dispatched") {
+    const dispatchedReason = s.lastGitActionFailure
+      ? "git-closeout-failure"
+      : "pre-verification-dispatched";
+    debugLog("autoLoop", {
+      phase: "exit",
+      reason: dispatchedReason,
+      gitError: s.lastGitActionFailure ?? undefined,
+    });
+    clearFinalizingUnit();
+    return { action: "break", reason: dispatchedReason };
+  }
+  if (preResult === "retry") {
+    if (sidecarItem) {
+      // Sidecar artifact retries are skipped — just continue
+      debugLog("autoLoop", { phase: "sidecar-artifact-retry-skipped", iteration: ic.iteration });
+    } else {
+      // s.pendingVerificationRetry was set by postUnitPreVerification.
+      // Emit a dedicated journal event so forensics can distinguish bounded
+      // verification retries from genuine stuck-loop dispatch repetitions (#4540).
+      const retryInfo = s.pendingVerificationRetry;
+      deps.emitJournalEvent({
+        ts: new Date().toISOString(),
+        flowId: ic.flowId,
+        seq: ic.nextSeq(),
+        eventType: "artifact-verification-retry",
+        data: {
+          unitType: preUnitSnapshot?.type,
+          unitId: retryInfo?.unitId,
+          attempt: retryInfo?.attempt,
+        },
+      });
+      const retryPolicyResult = await applyVerificationRetryPolicy(
+        ic,
+        preUnitSnapshot?.type,
+        "artifact-verification-retry",
+      );
+      if (retryPolicyResult) {
+        clearFinalizingUnit();
+        return retryPolicyResult;
+      }
+      // Continue the loop — next iteration will inject the retry context into the prompt.
+      rememberRetryDispatch(s, preUnitSnapshot, iterData);
+      debugLog("autoLoop", { phase: "artifact-verification-retry", iteration: ic.iteration });
+      clearFinalizingUnit();
+      return { action: "continue" };
+    }
+  }
+
+  if (pauseAfterUatDispatch) {
+    const pauseMid = iterData.mid;
+    const pauseSliceId = pauseMid && iterData.unitId.startsWith(`${pauseMid}/`)
+      ? iterData.unitId.slice(pauseMid.length + 1)
+      : undefined;
+    const guidance = pauseMid
+      ? buildManualValidationGuidance(s.basePath, pauseMid, {
+          uatPath: pauseSliceId
+            ? relSliceFile(s.basePath, pauseMid, pauseSliceId, "UAT")
+            : undefined,
+        })
+      : null;
+    const pauseMessage = guidance
+      ? `UAT requires human execution. Auto-mode will pause after this unit writes the result file.\n\n${guidance}`
+      : "UAT requires human execution. Auto-mode will pause after this unit writes the result file.";
+    ctx.ui.notify(pauseMessage, "info");
+    await deps.pauseAuto(ctx, pi);
+    debugLog("autoLoop", { phase: "exit", reason: "uat-pause" });
+    clearFinalizingUnit();
+    return { action: "break", reason: "uat-pause" };
+  }
+
+  // Verification gate
+  // Hook sidecar items skip verification entirely.
+  // Non-hook sidecar items run verification but skip retries (just continue).
+  const skipVerification = sidecarItem?.kind === "hook";
+  if (!skipVerification) {
+    const verificationResult = await deps.runPostUnitVerification(
+      { s, ctx, pi },
+      deps.pauseAuto,
+    );
+
+    if (verificationResult === "pause") {
+      debugLog("autoLoop", { phase: "exit", reason: "verification-pause" });
+      clearFinalizingUnit();
+      return { action: "break", reason: "verification-pause" };
+    }
+
+    if (verificationResult === "retry") {
+      if (sidecarItem) {
+        // Sidecar verification retries are skipped — just continue
+        debugLog("autoLoop", { phase: "sidecar-verification-retry-skipped", iteration: ic.iteration });
+      } else {
+        // s.pendingVerificationRetry was set by runPostUnitVerification.
+        const retryPolicyResult = await applyVerificationRetryPolicy(
+          ic,
+          iterData.unitType,
+          "verification-retry",
+        );
+        if (retryPolicyResult) {
+          clearFinalizingUnit();
+          return retryPolicyResult;
+        }
+        // Continue the loop — next iteration will inject the retry context into the prompt.
+        rememberRetryDispatch(s, preUnitSnapshot, iterData);
+        debugLog("autoLoop", { phase: "verification-retry", iteration: ic.iteration });
+        clearFinalizingUnit();
+        return { action: "continue" };
+      }
+    }
+  }
+
+  // Post-verification processing (DB dual-write, hooks, triage, quick-tasks)
+  // Timeout guard: if postUnitPostVerification hangs (e.g., module import
+  // deadlock, SQLite transaction hang), force-continue after timeout so the
+  // auto-loop is not permanently frozen (#2344).
+  const postResultGuard = await withTimeout(
+    deps.postUnitPostVerification(postUnitCtx),
+    FINALIZE_POST_TIMEOUT_MS,
+    "postUnitPostVerification",
+  );
+
+  if (postResultGuard.timedOut) {
+    return failClosedOnFinalizeTimeout(
+      ic,
+      iterData,
+      loopState,
+      "post",
+      preUnitSnapshot?.startedAt ?? Date.now(),
+    );
+  }
+
+  const postResult = postResultGuard.value;
+
+  if (postResult === "retry") {
+    if (sidecarItem) {
+      debugLog("autoLoop", { phase: "sidecar-pre-execution-retry-skipped", iteration: ic.iteration });
+    } else {
+      const retryInfo = s.pendingVerificationRetry;
+      deps.emitJournalEvent({
+        ts: new Date().toISOString(),
+        flowId: ic.flowId,
+        seq: ic.nextSeq(),
+        eventType: "pre-execution-retry",
+        data: {
+          unitType: preUnitSnapshot?.type,
+          unitId: retryInfo?.unitId,
+          attempt: retryInfo?.attempt,
+        },
+      });
+      const retryPolicyResult = await applyVerificationRetryPolicy(
+        ic,
+        preUnitSnapshot?.type,
+        "pre-execution-retry",
+      );
+      if (retryPolicyResult) {
+        clearFinalizingUnit();
+        return retryPolicyResult;
+      }
+      rememberRetryDispatch(s, preUnitSnapshot, iterData);
+      debugLog("autoLoop", {
+        phase: "pre-execution-retry",
+        iteration: ic.iteration,
+        unitType: preUnitSnapshot?.type,
+        unitId: retryInfo?.unitId,
+        attempt: retryInfo?.attempt,
+      });
+      clearFinalizingUnit();
+      return { action: "continue" };
+    }
+  }
+
+  if (postResult === "stopped") {
+    debugLog("autoLoop", {
+      phase: "exit",
+      reason: "post-verification-stopped",
+    });
+    clearFinalizingUnit();
+    return { action: "break", reason: "post-verification-stopped" };
+  }
+
+  if (postResult === "step-wizard") {
+    // Step mode — exit the loop (caller handles wizard)
+    debugLog("autoLoop", { phase: "exit", reason: "step-wizard" });
+    clearFinalizingUnit();
+    return { action: "break", reason: "step-wizard" };
+  }
+
+  if (preUnitSnapshot && isIsolatedWorktreeSession(s)) {
+    const leak = detectRootWriteLeak({
+      rootPath: s.originalBasePath,
+      worktreePath: s.basePath,
+      unitType: preUnitSnapshot.type,
+      unitId: preUnitSnapshot.id,
+      before: s.rootWriteBaseline,
+    });
+    s.rootWriteBaseline = null;
+    if (leak) {
+      const message = formatRootWriteLeakMessage(leak);
+      debugLog("autoLoop", {
+        phase: "root-write-leak",
+        unitType: preUnitSnapshot.type,
+        unitId: preUnitSnapshot.id,
+        rootPath: leak.rootPath,
+        worktreePath: leak.worktreePath,
+        files: leak.files.map((file) => ({ path: file.path, status: file.status })),
+      });
+      ctx.ui.notify(message, "error");
+      await deps.stopAuto(ctx, pi, "Root-write leak during isolated auto-mode", {
+        preserveCompletedMilestoneBranch: true,
+      });
+      clearFinalizingUnit();
+      return { action: "break", reason: "root-write-leak" };
+    }
+  } else {
+    s.rootWriteBaseline = null;
+  }
+
+  if (preUnitSnapshot?.type === "complete-milestone" && s.currentMilestoneId) {
+    const stop = await _runMilestoneMergeOnceWithStashRestore(ic, s.currentMilestoneId, {
+      preserveCloseoutTranscript: true,
+    });
+    if (stop) {
+      clearFinalizingUnit();
+      return stop;
+    }
+  }
+
+  // Both pre and post verification completed without timeout — reset counter
+  loopState.consecutiveFinalizeTimeouts = 0;
+  if (preUnitSnapshot) {
+    writeUnitRuntimeRecord(s.basePath, preUnitSnapshot.type, preUnitSnapshot.id, preUnitSnapshot.startedAt, {
+      phase: "finalized",
+      lastProgressAt: Date.now(),
+      lastProgressKind: "finalize-success",
+    });
+    if (
+      !preUnitSnapshot.type.startsWith("hook/") &&
+      preUnitSnapshot.type !== "custom-step" &&
+      preUnitSnapshot.type !== "complete-milestone"
+    ) {
+      setAutoOutcomeWidget(ctx, {
+        ...buildPhaseHandoffOutcome({
+          unitType: preUnitSnapshot.type,
+          unitId: preUnitSnapshot.id,
+          agentEndMessages: s.lastUnitAgentEndMessages,
+        }),
+        startedAt: s.autoStartTime,
+      });
+    }
+  }
+  clearFinalizingUnit();
+  // Surface accumulated workflow-logger issues for this unit to the user.
+  // Warnings/errors logged during the unit are buffered in the logger and
+  // drained here so the user sees a single consolidated post-unit alert.
+  if (hasAnyIssues()) {
+    const { logs } = drainAndSummarize();
+    if (logs.length > 0) {
+      const severity = logs.some((e) => e.severity === "error") ? "error" : "warning";
+      ctx.ui.notify(formatForNotification(logs), severity);
+    }
+  }
+
+  if (preUnitSnapshot?.type === "complete-milestone" && s.currentMilestoneId) {
+    // cleanupAfterLoopExit skips gsd-progress when preserveCompletionSurface is true, so clear stale controls here.
+    ctx.ui.setStatus?.("gsd-step", undefined);
+    ctx.ui.setWidget?.("gsd-progress", undefined);
+    await deps.stopAuto(ctx, pi, `Milestone ${s.currentMilestoneId} complete`, {
+      completionWidget: {
+        milestoneId: s.currentMilestoneId,
+        milestoneTitle: iterData.midTitle,
+      },
+    });
+    return { action: "break", reason: "milestone-complete" };
+  }
+
+  return { action: "next", data: undefined as void };
+}

package/src/resources/extensions/gsd/auto/loop.ts

@@ -25,7 +25,10 @@ import {
   type IterationData,
 } from "./types.js";
 import { _clearCurrentResolve } from "./resolve.js";
-import { runGuards, runFinalize, STUCK_WINDOW_SIZE } from "./phases.js";
+import { runGuards } from "./phases.js";
+import { runFinalize } from "./finalize.js";
+import { resetSessionTimeoutState } from "./unit-phase.js";
+import { STUCK_WINDOW_SIZE } from "./dispatch-history.js";
 import { debugLog } from "../debug-logger.js";
 import { isInfrastructureError, isTransientCooldownError, getCooldownRetryAfterMs, COOLDOWN_FALLBACK_WAIT_MS, MAX_COOLDOWN_RETRIES } from "./infra-errors.js";
 import { ModelPolicyDispatchBlockedError } from "../auto-model-selection.js";
@@ -384,11 +387,14 @@ export async function autoLoop(
   options?: AutoLoopOptions,
 ): Promise<void> {
   debugLog("autoLoop", { phase: "enter" });
+  resetSessionTimeoutState();
   let iteration = 0;
   const dispatchContract = options?.dispatchContract ?? "legacy-direct";
   const unitDispatchDeps = createExecutionGraphUnitDispatchDeps();
   // Load persisted stuck state so counters survive session restarts (#3704)
   const persisted = loadStuckState(s);
+  // Load persisted verification retry state so the exhausted-unit guard fires on restart (#651)
+  hydrateCustomVerifyRetryCounts(s, { logFailure: logCustomVerifyRetryLoadFailure });
   const loopState: LoopState = {
     recentUnits: persisted.recentUnits,
     stuckRecoveryAttempts: persisted.stuckRecoveryAttempts,

package/src/resources/extensions/gsd/auto/milestone-lease-reclaim.ts

@@ -0,0 +1,74 @@
+// Project/App: gsd-pi
+// File Purpose: Recover missing milestone lease state for resumed isolated workers.
+
+import { debugLog } from "../debug-logger.js";
+import { claimMilestoneLease } from "../db/milestone-leases.js";
+import type { getIsolationMode } from "../preferences.js";
+
+type IsolationMode = ReturnType<typeof getIsolationMode>;
+
+export interface MilestoneLeaseSession {
+  workerId: string | null;
+  currentMilestoneId: string | null;
+  milestoneLeaseToken: number | null;
+}
+
+export function hasHeldMilestoneLease(
+  session: MilestoneLeaseSession,
+  milestoneId: string | null | undefined,
+): boolean {
+  return (
+    Boolean(milestoneId) &&
+    session.currentMilestoneId === milestoneId &&
+    typeof session.milestoneLeaseToken === "number"
+  );
+}
+
+export function reclaimMissingMilestoneLease(
+  session: MilestoneLeaseSession,
+  milestoneId: string | null | undefined,
+  isolationMode: IsolationMode,
+  phase: string,
+): void {
+  if (isolationMode === "none") return;
+  if (!session.workerId || !milestoneId) return;
+  if (hasHeldMilestoneLease(session, milestoneId)) return;
+  // Note: we intentionally do NOT bail just because the session already holds a
+  // lease for a *different* milestone. When dispatch advances to a new
+  // milestone the session's stale `currentMilestoneId`/token are for the prior
+  // one, and the active milestone's lease must still be claimed — otherwise
+  // worktree safety sees `held: false` and fails dispatch (#760). Claiming is
+  // safe: claimMilestoneLease refuses to steal a lease another worker holds.
+
+  try {
+    const claim = claimMilestoneLease(session.workerId, milestoneId);
+    if (claim.ok) {
+      session.currentMilestoneId = milestoneId;
+      session.milestoneLeaseToken = claim.token;
+      debugLog("worktreeSafety", {
+        phase: "lease-reclaimed",
+        source: phase,
+        milestoneId,
+        workerId: session.workerId,
+        token: claim.token,
+      });
+    } else {
+      debugLog("worktreeSafety", {
+        phase: "lease-reclaim-blocked",
+        source: phase,
+        milestoneId,
+        workerId: session.workerId,
+        holderWorkerId: claim.byWorker,
+        expiresAt: claim.expiresAt,
+      });
+    }
+  } catch (err) {
+    debugLog("worktreeSafety", {
+      phase: "lease-reclaim-failed",
+      source: phase,
+      milestoneId,
+      workerId: session.workerId,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+}