gsd-pi 2.80.0-dev.fbe7c8c6f → 2.81.0-dev.3cddbbba2

package/src/resources/extensions/gsd/auto/loop.ts

@@ -38,7 +38,7 @@ import {
   getRecentForUnit as getRecentDispatchesForUnit,
   getRecentUnitKeysForProjectRoot,
 } from "../db/unit-dispatches.js";
-import { refreshMilestoneLease } from "../db/milestone-leases.js";
+import { claimMilestoneLease, refreshMilestoneLease } from "../db/milestone-leases.js";
 import { heartbeatAutoWorker } from "../db/auto-workers.js";
 import { getRuntimeKv, setRuntimeKv } from "../db/runtime-kv.js";
 import { resolveUokFlags } from "../uok/flags.js";
@@ -68,7 +68,9 @@ import {
   settleDispatchCompleted,
   settleDispatchFailed,
 } from "./workflow-dispatch-ledger.js";
-import { openDispatchClaim } from "./workflow-dispatch-claim.js";
+import { emitOpenUnitEndForUnit } from "../crash-recovery.js";
+import { writeUnitRuntimeRecord } from "../unit-runtime.js";
+import { ensureDispatchLease, openDispatchClaim } from "./workflow-dispatch-claim.js";
 import { completeWorkflowIteration } from "./workflow-iteration-completion.js";
 import { createWorkflowJournalReporter } from "./workflow-journal-reporter.js";
 import { createWorkflowPhaseReporter } from "./workflow-phase-reporter.js";
@@ -163,6 +165,29 @@ function logDispatchClaimFailed(err: unknown): void {
   });
 }
 
+function logDispatchLeaseRecovered(details: {
+  milestoneId: string;
+  workerId: string;
+  token: number;
+  recovered: boolean;
+}): void {
+  debugLog("autoLoop", {
+    phase: details.recovered ? "dispatch-lease-recovered" : "dispatch-lease-acquired",
+    ...details,
+  });
+}
+
+function logDispatchLeaseRecoveryFailed(details: {
+  milestoneId?: string;
+  workerId?: string;
+  reason: string;
+}): void {
+  debugLog("autoLoop", {
+    phase: "dispatch-lease-recovery-failed",
+    ...details,
+  });
+}
+
 function logCustomVerifyRetryLoadFailure(err: unknown): void {
   debugLog("autoLoop", {
     phase: "load-custom-verify-retries-failed",
@@ -201,6 +226,36 @@ async function enforceMinRequestInterval(s: AutoSession, prefs: IterationContext
   }
 }
 
+function closeOutCrashedUnit(s: AutoSession, iterData: IterationData, err: unknown): void {
+  const summary = formatDispatchExceptionSummary({ error: err });
+  try {
+    emitOpenUnitEndForUnit(
+      s.basePath,
+      iterData.unitType,
+      iterData.unitId,
+      "cancelled",
+      {
+        message: summary,
+        category: "unit-exception",
+        isTransient: false,
+      },
+    );
+    writeUnitRuntimeRecord(
+      s.basePath,
+      iterData.unitType,
+      iterData.unitId,
+      s.currentUnit?.startedAt ?? Date.now(),
+      {
+        phase: "crashed",
+        lastProgressAt: Date.now(),
+        lastProgressKind: "unit-exception",
+      },
+    );
+  } catch (closeoutErr) {
+    logWarning("dispatch", `unit crash closeout failed: ${closeoutErr instanceof Error ? closeoutErr.message : String(closeoutErr)}`);
+  }
+}
+
 /**
  * Main auto-mode execution loop. Iterates: derive → dispatch → guards →
  * runUnit → finalize → repeat. Exits when s.active becomes false or a
@@ -242,6 +297,10 @@ export async function autoLoop(
         phase: "heartbeat-failed",
         error: err instanceof Error ? err.message : String(err),
       }),
+      logLeaseRefreshMiss: details => debugLog("autoLoop", {
+        phase: "lease-refresh-missed",
+        ...details,
+      }),
     });
 
     // ── Journal: per-iteration flow grouping ──
@@ -340,6 +399,12 @@ export async function autoLoop(
 
     let dispatchId: number | null = null;
     let dispatchSettled = false;
+    let iterationEndEmitted = false;
+    const emitIterationEnd = (details: Record<string, unknown> = {}): void => {
+      if (iterationEndEmitted) return;
+      iterationEndEmitted = true;
+      journalReporter.emit("iteration-end", { iteration, ...details });
+    };
     const completeIteration = (): void => {
       completeWorkflowIteration({
         get consecutiveErrors() { return consecutiveErrors; },
@@ -348,11 +413,15 @@ export async function autoLoop(
         set consecutiveCooldowns(value) { consecutiveCooldowns = value; },
         recentErrorMessages,
       }, {
-        emitIterationEnd: () => journalReporter.emit("iteration-end", { iteration }),
+        emitIterationEnd: () => emitIterationEnd(),
         saveStuckState: () => saveStuckState(s, loopState),
         logIterationComplete: () => debugLog("autoLoop", { phase: "iteration-complete", iteration }),
       });
     };
+    const finishIncompleteIteration = (details: Record<string, unknown>): void => {
+      emitIterationEnd(details);
+      saveStuckState(s, loopState);
+    };
 
     try {
       // ── Blanket try/catch: one bad iteration must not kill the session
@@ -433,6 +502,7 @@ export async function autoLoop(
         });
         if (engineState.isComplete) {
           finishTurn("completed");
+          emitIterationEnd({ status: "completed", reason: "custom-engine-complete" });
           await deps.stopAuto(ctx, pi, "Workflow complete");
           break;
         }
@@ -450,16 +520,23 @@ export async function autoLoop(
         });
         if (dispatchFlow.action === "break") {
           finishTurn("stopped", "manual-attention", "custom-engine-dispatch-stop");
+          finishIncompleteIteration({
+            status: "stopped",
+            reason: "custom-engine-dispatch-stop",
+            failureClass: "manual-attention",
+          });
           break;
         }
         if (dispatchFlow.action === "continue") {
           finishTurn("skipped");
+          emitIterationEnd({ status: "skipped", reason: "custom-engine-dispatch-skip" });
           continue;
         }
 
         // dispatch.action === "dispatch"
         if (dispatch.action !== "dispatch") {
           finishTurn("skipped");
+          emitIterationEnd({ status: "skipped", reason: "custom-engine-dispatch-mismatch" });
           continue;
         }
         const step = dispatch.step;
@@ -488,19 +565,35 @@ export async function autoLoop(
         });
         if (guardsResult.action === "break") {
           finishTurn("stopped", "manual-attention", "guard-break");
+          finishIncompleteIteration({
+            status: "stopped",
+            reason: "guard-break",
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+            failureClass: "manual-attention",
+          });
           break;
         }
 
         // ── Unit execution (shared with dev path) ──
         await enforceMinRequestInterval(s, prefs);
-        const unitPhaseResult = await runUnitPhaseViaContract(
-          dispatchContract,
-          ic,
-          iterData,
-          loopState,
-          undefined,
-          unitDispatchDeps,
-        );
+        let unitPhaseResult: Awaited<ReturnType<typeof runUnitPhaseViaContract>>;
+        try {
+          unitPhaseResult = await runUnitPhaseViaContract(
+            dispatchContract,
+            ic,
+            iterData,
+            loopState,
+            undefined,
+            unitDispatchDeps,
+          );
+        } catch (err) {
+          if (err instanceof ModelPolicyDispatchBlockedError) {
+            throw err;
+          }
+          closeOutCrashedUnit(s, iterData, err);
+          throw err;
+        }
         if (unitPhaseResult.action === "next") {
           const requestTimestamp = resolveUnitRequestTimestamp(unitPhaseResult.data);
           if (requestTimestamp !== undefined) s.lastRequestTimestamp = requestTimestamp;
@@ -510,6 +603,13 @@ export async function autoLoop(
           unitId: iterData.unitId,
         });
         if (unitPhaseResult.action === "break") {
+          finishIncompleteIteration({
+            status: "stopped",
+            reason: unitPhaseResult.reason ?? "unit-break",
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+            failureClass: "execution",
+          });
           finishTurn("stopped", "execution", "unit-break");
           break;
         }
@@ -528,7 +628,16 @@ export async function autoLoop(
               finishTurn,
             },
           });
-          if (verifyFlow.action === "break") break;
+          if (verifyFlow.action === "break") {
+            finishIncompleteIteration({
+              status: "paused",
+              reason: "custom-engine-verify-pause",
+              unitType: iterData.unitType,
+              unitId: iterData.unitId,
+              failureClass: "manual-attention",
+            });
+            break;
+          }
         }
         if (verifyResult === "retry") {
           const retryOutcome = await handleCustomEngineVerifyRetry({
@@ -562,7 +671,22 @@ export async function autoLoop(
               finishTurn,
             },
           });
-          if (retryFlow.action === "break") break;
+          if (retryFlow.action === "break") {
+            finishIncompleteIteration({
+              status: retryOutcome.action === "stop" ? "stopped" : "paused",
+              reason: retryOutcome.action === "retry" ? "custom-engine-verify-retry" : retryOutcome.turnError,
+              unitType: iterData.unitType,
+              unitId: iterData.unitId,
+              failureClass: "manual-attention",
+            });
+            break;
+          }
+          finishIncompleteIteration({
+            status: "retry",
+            reason: "custom-engine-verify-retry",
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+          });
           continue;
         }
 
@@ -662,25 +786,74 @@ export async function autoLoop(
 
       // Phase B: claim a unit_dispatches row before invoking the unit. The
       // partial unique index idx_unit_dispatches_active_per_unit prevents
-      // a second worker from claiming the same unit concurrently. Returns
-      // null when DB unavailable, no worker registered, or no active lease
-      // — those degraded paths fall through to the existing single-worker
-      // semantics with no ledger entry, preserving back-compat.
-      const dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
+      // a second worker from claiming the same unit concurrently. When this
+      // process has a worker identity, make the milestone lease explicit before
+      // claiming so a step-mode handoff cannot leave us running with a stale
+      // in-memory token and no backing lease row.
+      const leaseBeforeClaim = ensureDispatchLease(s, iterData.mid, {
+        claimMilestoneLease,
+        logLeaseRecovered: logDispatchLeaseRecovered,
+        logLeaseRecoveryFailed: logDispatchLeaseRecoveryFailed,
+      });
+      if (leaseBeforeClaim.kind === "blocked" || leaseBeforeClaim.kind === "failed") {
+        const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} before dispatching ${iterData.unitType} ${iterData.unitId}: ${leaseBeforeClaim.reason}`;
+        ctx.ui.notify(msg, "error");
+        finishTurn("stopped", "execution", msg);
+        await deps.stopAuto(ctx, pi, msg);
+        break;
+      }
+
+      let dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
         getRecentDispatchesForUnit,
         recordDispatchClaim,
         markDispatchRunning,
         logClaimRejected: logDispatchClaimRejected,
         logClaimFailed: logDispatchClaimFailed,
       });
-      const dispatchDecision = decideDispatchClaim(
+      let dispatchDecision = decideDispatchClaim(
         dispatchClaim.kind === "opened"
           ? { kind: "opened", dispatchId: dispatchClaim.dispatchId }
           : dispatchClaim.kind === "skip"
             ? { kind: "skip", reason: dispatchClaim.reason }
             : { kind: "degraded" },
       );
+      if (dispatchDecision.action === "skip" && dispatchDecision.reason === "stale-lease") {
+        const leaseRecovery = ensureDispatchLease(s, iterData.mid, {
+          claimMilestoneLease,
+          logLeaseRecovered: logDispatchLeaseRecovered,
+          logLeaseRecoveryFailed: logDispatchLeaseRecoveryFailed,
+        }, { forceReclaim: true });
+        if (leaseRecovery.kind === "ready") {
+          dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
+            getRecentDispatchesForUnit,
+            recordDispatchClaim,
+            markDispatchRunning,
+            logClaimRejected: logDispatchClaimRejected,
+            logClaimFailed: logDispatchClaimFailed,
+          });
+          dispatchDecision = decideDispatchClaim(
+            dispatchClaim.kind === "opened"
+              ? { kind: "opened", dispatchId: dispatchClaim.dispatchId }
+              : dispatchClaim.kind === "skip"
+                ? { kind: "skip", reason: dispatchClaim.reason }
+                : { kind: "degraded" },
+          );
+        } else {
+          const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} while claiming ${iterData.unitType} ${iterData.unitId}: ${leaseRecovery.reason}`;
+          ctx.ui.notify(msg, "error");
+          finishTurn("stopped", "execution", msg);
+          await deps.stopAuto(ctx, pi, msg);
+          break;
+        }
+      }
       if (dispatchDecision.action === "skip") {
+        if (dispatchDecision.reason === "stale-lease") {
+          const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} while claiming ${iterData.unitType} ${iterData.unitId}; dispatch claim still failed after recovery.`;
+          ctx.ui.notify(msg, "error");
+          finishTurn("stopped", "execution", msg);
+          await deps.stopAuto(ctx, pi, msg);
+          break;
+        }
         finishTurn("skipped", "execution", dispatchDecision.reason);
         continue;
       }
@@ -700,6 +873,7 @@ export async function autoLoop(
         if (err instanceof ModelPolicyDispatchBlockedError) {
           throw err;
         }
+        closeOutCrashedUnit(s, iterData, err);
         dispatchSettled = settleDispatchFailed(
           dispatchId,
           formatDispatchExceptionSummary({ error: err }),
@@ -723,6 +897,13 @@ export async function autoLoop(
           markFailed: markDispatchFailed,
           logWriteFailure: logDispatchLedgerWriteFailure,
         }) || dispatchSettled;
+        finishIncompleteIteration({
+          status: "stopped",
+          reason: unitPhaseResult.reason ?? "unit-break",
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+          failureClass: "execution",
+        });
         finishTurn("stopped", "execution", "unit-break");
         break;
       }
@@ -730,12 +911,25 @@ export async function autoLoop(
       // ── Phase 5: Finalize ───────────────────────────────────────────────
 
       let finalizeResult: Awaited<ReturnType<typeof runFinalize>>;
+      journalReporter.emit("post-unit-finalize-start", {
+        iteration,
+        unitType: iterData.unitType,
+        unitId: iterData.unitId,
+      });
       try {
         finalizeResult = await runFinalize(ic, iterData, loopState, sidecarItem);
       } catch (err) {
+        const error = formatDispatchExceptionSummary({ error: err });
+        journalReporter.emit("post-unit-finalize-end", {
+          iteration,
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+          status: "failed",
+          error,
+        });
         dispatchSettled = settleDispatchFailed(
           dispatchId,
-          formatDispatchExceptionSummary({ error: err }),
+          error,
           {
             markFailed: markDispatchFailed,
             logWriteFailure: logDispatchLedgerWriteFailure,
@@ -747,6 +941,15 @@ export async function autoLoop(
         unitType: iterData.unitType,
         unitId: iterData.unitId,
       });
+      const finalizeReason = finalizeResult.action === "break" ? finalizeResult.reason : undefined;
+      journalReporter.emit("post-unit-finalize-end", {
+        iteration,
+        unitType: iterData.unitType,
+        unitId: iterData.unitId,
+        status: finalizeResult.action === "next" ? "completed" : finalizeResult.action === "continue" ? "retry" : "stopped",
+        action: finalizeResult.action,
+        ...(finalizeReason ? { reason: finalizeReason } : {}),
+      });
       const finalizeDecision = decideFinalizeResult(
         finalizeResult.action === "break"
           ? { action: "break", reason: finalizeResult.reason }
@@ -759,6 +962,13 @@ export async function autoLoop(
           markFailed: markDispatchFailed,
           logWriteFailure: logDispatchLedgerWriteFailure,
         }) || dispatchSettled;
+        finishIncompleteIteration({
+          status: "stopped",
+          reason: finalizeReason ?? "finalize-break",
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+          failureClass: finalizeDecision.failureClass,
+        });
         finishTurn("stopped", finalizeDecision.failureClass, finalizeDecision.turnError);
         break;
       }
@@ -767,6 +977,12 @@ export async function autoLoop(
           markFailed: markDispatchFailed,
           logWriteFailure: logDispatchLedgerWriteFailure,
         }) || dispatchSettled;
+        finishIncompleteIteration({
+          status: "retry",
+          reason: "finalize-retry",
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+        });
         finishTurn("retry");
         continue;
       }
@@ -791,11 +1007,6 @@ export async function autoLoop(
         ) || dispatchSettled;
       }
 
-      // Always emit iteration-end on error so the journal records iteration
-      // completion even on failure (#2344). Without this, errors in
-      // runFinalize leave the journal incomplete, making diagnosis harder.
-      journalReporter.emit("iteration-end", { iteration, error: msg });
-
       // ── Pre-send model-policy block: not a retryable error (#4959 / #4850) ──
       // The model-policy gate runs before the prompt is sent.  When every
       // candidate model is denied (cross-provider disabled + flat-rate
@@ -820,6 +1031,12 @@ export async function autoLoop(
         });
         ctx.ui.notify(policyDecision.notifyMessage, "error");
         journalReporter.emit("unit-end", policyDecision.journalData);
+        finishIncompleteIteration({
+          status: "blocked",
+          reason: "model-policy-dispatch-blocked",
+          unitType: loopErr.unitType,
+          unitId: loopErr.unitId,
+        });
         // Carry the blocked unit identity into the turn-result observer:
         // the throw originated inside dispatch, so observedUnitType/Id were
         // not assigned by the success path at lines 453/631/647 — but the
@@ -833,6 +1050,11 @@ export async function autoLoop(
         break;
       }
 
+      // Always emit iteration-end on error so the journal records iteration
+      // completion even on failure (#2344). Without this, errors in
+      // runFinalize leave the journal incomplete, making diagnosis harder.
+      finishIncompleteIteration({ status: "failed", error: msg });
+
       // ── Infrastructure errors: immediate stop, no retry ──
       // These are unrecoverable (disk full, OOM, etc.). Retrying just burns
       // LLM budget on guaranteed failures.

package/src/resources/extensions/gsd/auto/orchestrator.ts

@@ -1,3 +1,6 @@
+// Project/App: GSD-2
+// File Purpose: Auto Orchestration module implementation and ADR-015 invariant pipeline owner.
+
 import type { AutoAdvanceResult, AutoOrchestrationModule, AutoOrchestratorDeps, AutoSessionContext, AutoStatus } from "./contracts.js";
 
 function now(): number {
@@ -36,9 +39,21 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
         return blocked;
       }
 
-      const decision = await this.deps.dispatch.decideNextUnit();
+      const reconciliation = await this.deps.stateReconciliation.reconcileBeforeDispatch();
+      if (!reconciliation.ok || !reconciliation.stateSnapshot) {
+        const blocked: AutoAdvanceResult = {
+          kind: "blocked",
+          reason: reconciliation.reason,
+          stateSnapshot: reconciliation.stateSnapshot,
+        };
+        await this.deps.runtime.journalTransition({ name: "advance-blocked", reason: blocked.reason });
+        await this.deps.health.postAdvanceRecord(blocked);
+        return blocked;
+      }
+
+      const decision = await this.deps.dispatch.decideNextUnit({ stateSnapshot: reconciliation.stateSnapshot });
       if (!decision) {
-        const stopped: AutoAdvanceResult = { kind: "stopped", reason: "no remaining units" };
+        const stopped: AutoAdvanceResult = { kind: "stopped", reason: "no remaining units", stateSnapshot: reconciliation.stateSnapshot };
         this.status.phase = "stopped";
         this.status.activeUnit = undefined;
         this.lastAdvanceKey = null;
@@ -61,6 +76,40 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
         return blocked;
       }
 
+      const contract = await this.deps.toolContract.compileUnitToolContract(decision.unitType, decision.unitId);
+      if (!contract.ok) {
+        const blocked: AutoAdvanceResult = {
+          kind: "blocked",
+          reason: contract.reason,
+          stateSnapshot: reconciliation.stateSnapshot,
+        };
+        await this.deps.runtime.journalTransition({
+          name: "advance-blocked",
+          reason: blocked.reason,
+          unitType: decision.unitType,
+          unitId: decision.unitId,
+        });
+        await this.deps.health.postAdvanceRecord(blocked);
+        return blocked;
+      }
+
+      const worktree = await this.deps.worktree.prepareForUnit(decision.unitType, decision.unitId);
+      if (!worktree.ok) {
+        const blocked: AutoAdvanceResult = {
+          kind: "blocked",
+          reason: worktree.reason,
+          stateSnapshot: reconciliation.stateSnapshot,
+        };
+        await this.deps.runtime.journalTransition({
+          name: "advance-blocked",
+          reason: blocked.reason,
+          unitType: decision.unitType,
+          unitId: decision.unitId,
+        });
+        await this.deps.health.postAdvanceRecord(blocked);
+        return blocked;
+      }
+
       this.status.activeUnit = { unitType: decision.unitType, unitId: decision.unitId };
       this.status.phase = "running";
       this.lastAdvanceKey = nextKey;
@@ -72,10 +121,9 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
         unitType: decision.unitType,
         unitId: decision.unitId,
       });
-      await this.deps.worktree.prepareForUnit(decision.unitType, decision.unitId);
       await this.deps.worktree.syncAfterUnit(decision.unitType, decision.unitId);
 
-      const advanced: AutoAdvanceResult = { kind: "advanced" };
+      const advanced: AutoAdvanceResult = { kind: "advanced", stateSnapshot: reconciliation.stateSnapshot };
       await this.deps.health.postAdvanceRecord(advanced);
       return advanced;
     } catch (error) {