@opengsd/gsd-pi 1.2.0 → 1.3.0-dev.65546769

package/src/resources/extensions/gsd/auto/loop.ts

@@ -3,7 +3,7 @@
 /**
  * auto/loop.ts — Main auto-mode execution loop.
  *
- * Iterates: derive → dispatch → guards → runUnit → finalize → repeat.
+ * Iterates: orchestration.advance → guards → runUnit → finalize → repeat.
  * Exits when s.active becomes false or a terminal condition is reached.
  *
  * Imports from: auto/types, auto/resolve, auto/phases
@@ -15,6 +15,7 @@ import { randomUUID } from "node:crypto";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import type { AutoSession } from "./session.js";
+import type { AutoTerminalOutcome } from "./contracts.js";
 import type { LoopDeps, StopAutoOptions } from "./loop-deps.js";
 import type { GSDState } from "../types.js";
 import {
@@ -24,13 +25,10 @@ import {
   type IterationData,
 } from "./types.js";
 import { _clearCurrentResolve } from "./resolve.js";
-import {
-  runPreDispatch,
-  runDispatch,
-  runGuards,
-  runFinalize,
-  STUCK_WINDOW_SIZE,
-} from "./phases.js";
+import { runGuards } from "./phases.js";
+import { runFinalize } from "./finalize.js";
+import { resetSessionTimeoutState } from "./unit-phase.js";
+import { STUCK_WINDOW_SIZE } from "./dispatch-history.js";
 import { debugLog } from "../debug-logger.js";
 import { isInfrastructureError, isTransientCooldownError, getCooldownRetryAfterMs, COOLDOWN_FALLBACK_WAIT_MS, MAX_COOLDOWN_RETRIES } from "./infra-errors.js";
 import { ModelPolicyDispatchBlockedError } from "../auto-model-selection.js";
@@ -146,6 +144,25 @@ function resolveCompletionStopFromState(
   };
 }
 
+function resolveCompletionStopFromOutcome(
+  outcome: AutoTerminalOutcome | undefined,
+  stateSnapshot: GSDState | undefined,
+): { reason: string; options: StopAutoOptions } | null {
+  if (outcome?.code !== "all-complete") return null;
+  const completedMilestone = stateSnapshot?.lastCompletedMilestone ?? stateSnapshot?.activeMilestone;
+  return {
+    reason: outcome.displayReason,
+    options: {
+      completionWidget: {
+        milestoneId: completedMilestone?.id ?? null,
+        milestoneTitle: completedMilestone?.title ?? null,
+        allMilestonesComplete: true,
+      },
+      terminalOutcome: outcome,
+    },
+  };
+}
+
 // ── Stuck detection persistence (#3704) ──────────────────────────────────
 // Phase C migration: stuck-state.json deleted in favor of DB-backed
 // equivalents. recentUnits is rebuilt from unit_dispatches (Phase B
@@ -158,6 +175,9 @@ function resolveCompletionStopFromState(
 // tolerates — same behavior as a fresh session.
 const STUCK_RECOVERY_ATTEMPTS_KEY = "stuck_recovery_attempts";
 const MAX_CONSECUTIVE_ALREADY_ACTIVE_SKIPS = 3;
+const MAX_CONSECUTIVE_ORCHESTRATION_SKIPS = 3;
+const ORCHESTRATION_MISSING_REASON =
+  "Auto Orchestration Module is not wired; cannot dispatch built-in GSD Unit.";
 
 function stableStuckStateScopeId(s: AutoSession): string {
   return normalizeRealPath(s.scope?.workspace.projectRoot ?? (s.originalBasePath || s.basePath));
@@ -167,7 +187,20 @@ function loadStuckState(s: AutoSession): { recentUnits: Array<{ key: string }>;
   const scopeId = stableStuckStateScopeId(s);
   if (!scopeId) return { recentUnits: [], stuckRecoveryAttempts: 0 };
   try {
-    const recentUnits = getRecentUnitKeysForProjectRoot(scopeId, STUCK_WINDOW_SIZE);
+    // Scope the stuck window to the CURRENT session (trace_id) so stale
+    // finalize-retry entries from previous sessions don't fire detectStuck
+    // Rule 1 on the first iteration — killing auto-mode before any new
+    // dispatch runs (#852). A prior session's two consecutive finalize-retry
+    // failures would otherwise permanently block every new session.
+    //
+    // s.currentTraceId is only set inside the first loop iteration (line ~443),
+    // so at this call-site it is always null for a fresh session. Return []
+    // directly in that case — no prior dispatch has occurred in this session,
+    // so the stuck window is trivially empty and the DB query is skipped.
+    const recentUnits =
+      s.currentTraceId != null
+        ? getRecentUnitKeysForProjectRoot(scopeId, STUCK_WINDOW_SIZE, s.currentTraceId)
+        : [];
     const stuckRecoveryAttempts =
       getRuntimeKv<number>("global", scopeId, STUCK_RECOVERY_ATTEMPTS_KEY) ?? 0;
     return { recentUnits, stuckRecoveryAttempts };
@@ -353,9 +386,9 @@ function closeOutCrashedUnit(s: AutoSession, iterData: IterationData, err: unkno
 }
 
 /**
- * Main auto-mode execution loop. Iterates: derive → dispatch → guards →
- * runUnit → finalize → repeat. Exits when s.active becomes false or a
- * terminal condition is reached.
+ * Main auto-mode execution loop. Iterates: orchestration.advance → guards →
+ * runUnit → finalize → repeat. Exits when s.active becomes false or a terminal
+ * condition is reached.
  *
  * This is the linear replacement for the recursive
  * dispatchNextUnit → resolveAgentEnd → dispatchNextUnit chain.
@@ -368,11 +401,14 @@ export async function autoLoop(
   options?: AutoLoopOptions,
 ): Promise<void> {
   debugLog("autoLoop", { phase: "enter" });
+  resetSessionTimeoutState();
   let iteration = 0;
   const dispatchContract = options?.dispatchContract ?? "legacy-direct";
   const unitDispatchDeps = createExecutionGraphUnitDispatchDeps();
   // Load persisted stuck state so counters survive session restarts (#3704)
   const persisted = loadStuckState(s);
+  // Load persisted verification retry state so the exhausted-unit guard fires on restart (#651)
+  hydrateCustomVerifyRetryCounts(s, { logFailure: logCustomVerifyRetryLoadFailure });
   const loopState: LoopState = {
     recentUnits: persisted.recentUnits,
     stuckRecoveryAttempts: persisted.stuckRecoveryAttempts,
@@ -384,6 +420,8 @@ export async function autoLoop(
   let consecutiveErrors = 0;
   let consecutiveCooldowns = 0;
   let consecutiveAlreadyActiveSkips = 0;
+  let consecutiveOrchestrationSkips = 0;
+  let lastOrchestrationSkipKey: string | null = null;
   const recentErrorMessages: string[] = [];
 
   while (s.active) {
@@ -572,10 +610,9 @@ export async function autoLoop(
       let iterData: IterationData;
 
       // ── Custom engine path ──────────────────────────────────────────────
-      // When activeEngineId is a non-dev value, bypass runPreDispatch and
-      // runDispatch entirely — the custom engine drives its own state via
-      // GRAPH.yaml. Shares runGuards and runUnitPhase with the dev path.
-      // After unit execution, verifies then reconciles via the engine layer.
+      // When activeEngineId is a non-dev value, the custom engine drives its own
+      // state via GRAPH.yaml. It shares guards and Unit execution with the dev
+      // path, then verifies and reconciles via the engine layer.
       //
       // GSD_ENGINE_BYPASS=1 skips the engine layer entirely — falls through
       // to the dev path below.
@@ -656,7 +693,7 @@ export async function autoLoop(
         observedUnitType = iterData.unitType;
         observedUnitId = iterData.unitId;
 
-        // ── Progress widget (mirrors dev path in runDispatch) ──
+        // ── Progress widget (mirrors the dev path) ──
         deps.updateProgressWidget(ctx, iterData.unitType, iterData.unitId, iterData.state);
 
         // ── Guards (shared with dev path) ──
@@ -876,14 +913,22 @@ export async function autoLoop(
 
           if (orchestrationResult.kind === "blocked") {
             s.pendingOrchestrationDispatch = null;
+            const blockMessage = orchestrationResult.terminalOutcome?.code === "settlement-blocked"
+              ? [
+                  orchestrationResult.terminalOutcome.displayReason,
+                  `Next: ${orchestrationResult.terminalOutcome.nextAction}`,
+                ].join("\n")
+              : orchestrationResult.reason;
             if (orchestrationResult.action === "pause") {
               await deps.pauseAuto(ctx, pi, {
-                message: orchestrationResult.reason,
+                message: blockMessage,
                 category: "unknown",
+              }, {
+                expectedCurrentUnit: null,
               });
               finishTurn("paused", "manual-attention", "orchestration-blocked");
             } else {
-              await deps.stopAuto(ctx, pi, orchestrationResult.reason);
+              await deps.stopAuto(ctx, pi, blockMessage);
               finishTurn("stopped", "manual-attention", "orchestration-blocked");
             }
             finishIncompleteIteration({
@@ -894,8 +939,61 @@ export async function autoLoop(
             break;
           }
 
+          if (orchestrationResult.kind === "skipped") {
+            s.pendingOrchestrationDispatch = null;
+            // Idempotent re-poll skips are benign (an active unit is still running).
+            // The orchestrator's own stuck-window detection handles the truly-stuck case.
+            // Do not count these toward the consecutive-skip streak.
+            if (orchestrationResult.reason === "idempotent advance: unit already active") {
+              emitIterationEnd({ skipped: true });
+              completeIteration();
+              finishTurn("skipped");
+              continue;
+            }
+            const skipState = orchestrationResult.stateSnapshot;
+            const skipKey = [
+              orchestrationResult.reason,
+              skipState?.phase,
+              skipState?.activeMilestone?.id,
+              skipState?.activeSlice?.id,
+              skipState?.activeTask?.id,
+            ].join("|");
+            consecutiveOrchestrationSkips = skipKey === lastOrchestrationSkipKey
+              ? consecutiveOrchestrationSkips + 1
+              : 1;
+            lastOrchestrationSkipKey = skipKey;
+            if (consecutiveOrchestrationSkips >= MAX_CONSECUTIVE_ORCHESTRATION_SKIPS) {
+              const msg = `Orchestration skipped ${consecutiveOrchestrationSkips} consecutive attempts without progress. Pausing auto-mode for manual recovery.`;
+              ctx.ui.notify(msg, "error");
+              await deps.pauseAuto(ctx, pi, {
+                message: msg,
+                category: "unknown",
+              }, {
+                expectedCurrentUnit: null,
+              });
+              finishTurn("paused", "manual-attention", orchestrationResult.reason ?? "orchestration-skipped");
+              finishIncompleteIteration({
+                status: "paused",
+                reason: orchestrationResult.reason ?? "orchestration-skipped",
+                failureClass: "manual-attention",
+              });
+              break;
+            }
+            emitIterationEnd({ skipped: true });
+            completeIteration();
+            finishTurn("skipped");
+            continue;
+          }
+
+          consecutiveOrchestrationSkips = 0;
+          lastOrchestrationSkipKey = null;
+
           if (orchestrationResult.kind === "paused") {
             s.pendingOrchestrationDispatch = null;
+            finishIncompleteIteration({
+              status: "paused",
+              reason: orchestrationResult.reason,
+            });
             finishTurn("skipped");
             continue;
           }
@@ -917,7 +1015,11 @@ export async function autoLoop(
 
           if (orchestrationResult.kind === "stopped") {
             s.pendingOrchestrationDispatch = null;
-            const completionStop = resolveCompletionStopFromState(orchestrationResult.stateSnapshot);
+            let completionStop = resolveCompletionStopFromOutcome(
+              orchestrationResult.terminalOutcome,
+              orchestrationResult.stateSnapshot,
+            );
+            completionStop ??= resolveCompletionStopFromState(orchestrationResult.stateSnapshot);
             if (completionStop) {
               await deps.stopAuto(ctx, pi, completionStop.reason, completionStop.options);
             } else {
@@ -998,54 +1100,37 @@ export async function autoLoop(
           });
           observedUnitType = iterData.unitType;
           observedUnitId = iterData.unitId;
-        } else {
-          const preDispatchResult = await runPreDispatch(ic, loopState);
-          phaseReporter.report("pre-dispatch", preDispatchResult.action);
-          if (preDispatchResult.action === "break") {
-            finishTurn("stopped", "manual-attention", "pre-dispatch-break");
+
+          const guardMilestoneId = iterData.mid ?? s.currentMilestoneId ?? "workflow";
+          const guardsResult = await runGuards(ic, guardMilestoneId);
+          phaseReporter.report("guard", guardsResult.action, {
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+          });
+          if (guardsResult.action === "break") {
+            finishTurn("stopped", "manual-attention", "guard-break");
             finishIncompleteIteration({
               status: "stopped",
-              reason: "pre-dispatch-break",
+              reason: "guard-break",
+              unitType: iterData.unitType,
+              unitId: iterData.unitId,
               failureClass: "manual-attention",
             });
             break;
           }
-          if (preDispatchResult.action === "continue") {
-            emitIterationEnd({ skipped: true });
-            completeIteration();
-            finishTurn("skipped");
-            continue;
-          }
-          if (preDispatchResult.action === "retry") {
-            finishTurn("retry", "execution", preDispatchResult.reason);
-            continue;
-          }
-          const preData = preDispatchResult.data;
-          const guardsResult = await runGuards(ic, preData.mid);
-          phaseReporter.report("guard", guardsResult.action);
-          if (guardsResult.action === "break") {
-            finishTurn("stopped", "manual-attention", "guard-break");
-            break;
-          }
-          const dispatchResult = await runDispatch(ic, preData, loopState);
-          phaseReporter.report("dispatch", dispatchResult.action);
-          if (dispatchResult.action === "break") {
-            finishTurn("stopped", "manual-attention", "dispatch-break");
-            break;
-          }
-          if (dispatchResult.action === "continue") {
-            emitIterationEnd({ skipped: true });
-            completeIteration();
-            finishTurn("skipped");
-            continue;
-          }
-          if (dispatchResult.action === "retry") {
-            finishTurn("retry", "execution", dispatchResult.reason);
-            continue;
-          }
-          iterData = dispatchResult.data;
-          observedUnitType = iterData.unitType;
-          observedUnitId = iterData.unitId;
+        } else {
+          s.pendingOrchestrationDispatch = null;
+          await deps.pauseAuto(ctx, pi, {
+            message: ORCHESTRATION_MISSING_REASON,
+            category: "unknown",
+          });
+          finishTurn("paused", "manual-attention", "orchestration-missing");
+          finishIncompleteIteration({
+            status: "paused",
+            reason: ORCHESTRATION_MISSING_REASON,
+            failureClass: "manual-attention",
+          });
+          break;
         }
       } else {
         iterData = await buildSidecarIterationData({
@@ -1303,7 +1388,17 @@ export async function autoLoop(
         finalizeResult.action === "break"
           ? { action: "break", reason: finalizeResult.reason }
           : finalizeResult.action === "continue"
-            ? { action: "continue" }
+            ? {
+                action: "continue",
+                // Surface the specific verification failure (e.g. "roadmap has
+                // zero slices") in the ledger/stuck-loop reason instead of the
+                // opaque bare "finalize-retry" token. The retry paths in
+                // runFinalize set s.pendingVerificationRetry.failureContext
+                // from describeArtifactVerificationFailure before returning
+                // continue; it is cleared on the next dispatch, so reading it
+                // here captures the reason for THIS finalize (#852 follow-up).
+                failureDetail: s.pendingVerificationRetry?.failureContext,
+              }
             : { action: "next" },
       );
       if (finalizeDecision.action === "stop") {

package/src/resources/extensions/gsd/auto/milestone-lease-reclaim.ts

@@ -0,0 +1,74 @@
+// Project/App: gsd-pi
+// File Purpose: Recover missing milestone lease state for resumed isolated workers.
+
+import { debugLog } from "../debug-logger.js";
+import { claimMilestoneLease } from "../db/milestone-leases.js";
+import type { getIsolationMode } from "../preferences.js";
+
+type IsolationMode = ReturnType<typeof getIsolationMode>;
+
+export interface MilestoneLeaseSession {
+  workerId: string | null;
+  currentMilestoneId: string | null;
+  milestoneLeaseToken: number | null;
+}
+
+export function hasHeldMilestoneLease(
+  session: MilestoneLeaseSession,
+  milestoneId: string | null | undefined,
+): boolean {
+  return (
+    Boolean(milestoneId) &&
+    session.currentMilestoneId === milestoneId &&
+    typeof session.milestoneLeaseToken === "number"
+  );
+}
+
+export function reclaimMissingMilestoneLease(
+  session: MilestoneLeaseSession,
+  milestoneId: string | null | undefined,
+  isolationMode: IsolationMode,
+  phase: string,
+): void {
+  if (isolationMode === "none") return;
+  if (!session.workerId || !milestoneId) return;
+  if (hasHeldMilestoneLease(session, milestoneId)) return;
+  // Note: we intentionally do NOT bail just because the session already holds a
+  // lease for a *different* milestone. When dispatch advances to a new
+  // milestone the session's stale `currentMilestoneId`/token are for the prior
+  // one, and the active milestone's lease must still be claimed — otherwise
+  // worktree safety sees `held: false` and fails dispatch (#760). Claiming is
+  // safe: claimMilestoneLease refuses to steal a lease another worker holds.
+
+  try {
+    const claim = claimMilestoneLease(session.workerId, milestoneId);
+    if (claim.ok) {
+      session.currentMilestoneId = milestoneId;
+      session.milestoneLeaseToken = claim.token;
+      debugLog("worktreeSafety", {
+        phase: "lease-reclaimed",
+        source: phase,
+        milestoneId,
+        workerId: session.workerId,
+        token: claim.token,
+      });
+    } else {
+      debugLog("worktreeSafety", {
+        phase: "lease-reclaim-blocked",
+        source: phase,
+        milestoneId,
+        workerId: session.workerId,
+        holderWorkerId: claim.byWorker,
+        expiresAt: claim.expiresAt,
+      });
+    }
+  } catch (err) {
+    debugLog("worktreeSafety", {
+      phase: "lease-reclaim-failed",
+      source: phase,
+      milestoneId,
+      workerId: session.workerId,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+}