gsd-pi 2.80.0-dev.4ea7d80e7 → 2.80.0-dev.7582bf37a

package/src/resources/extensions/gsd/auto/loop-deps.ts

@@ -23,7 +23,7 @@ import type { CmuxLogLevel } from "../../shared/cmux-events.js";
 import type { JournalEntry } from "../journal.js";
 import type { MergeReconcileResult } from "../auto-recovery.js";
 import type { UokTurnObserver } from "../uok/contracts.js";
-import type { PreflightResult } from "../clean-root-preflight.js";
+import type { PostflightResult, PreflightResult } from "../clean-root-preflight.js";
 
 type PauseAutoFn = (
   ctx?: ExtensionContext,
@@ -139,8 +139,9 @@ export interface LoopDeps {
   postflightPopStash: (
     basePath: string,
     milestoneId: string,
+    stashMarker: string | undefined,
     notify: (message: string, level: "info" | "warning" | "error") => void,
-  ) => void;
+  ) => PostflightResult;
 
   // Budget/context/secrets
   getLedger: () => unknown;

package/src/resources/extensions/gsd/auto/loop.ts

@@ -38,7 +38,7 @@ import {
   getRecentForUnit as getRecentDispatchesForUnit,
   getRecentUnitKeysForProjectRoot,
 } from "../db/unit-dispatches.js";
-import { refreshMilestoneLease } from "../db/milestone-leases.js";
+import { claimMilestoneLease, refreshMilestoneLease } from "../db/milestone-leases.js";
 import { heartbeatAutoWorker } from "../db/auto-workers.js";
 import { getRuntimeKv, setRuntimeKv } from "../db/runtime-kv.js";
 import { resolveUokFlags } from "../uok/flags.js";
@@ -68,7 +68,9 @@ import {
   settleDispatchCompleted,
   settleDispatchFailed,
 } from "./workflow-dispatch-ledger.js";
-import { openDispatchClaim } from "./workflow-dispatch-claim.js";
+import { emitOpenUnitEndForUnit } from "../crash-recovery.js";
+import { writeUnitRuntimeRecord } from "../unit-runtime.js";
+import { ensureDispatchLease, openDispatchClaim } from "./workflow-dispatch-claim.js";
 import { completeWorkflowIteration } from "./workflow-iteration-completion.js";
 import { createWorkflowJournalReporter } from "./workflow-journal-reporter.js";
 import { createWorkflowPhaseReporter } from "./workflow-phase-reporter.js";
@@ -163,6 +165,29 @@ function logDispatchClaimFailed(err: unknown): void {
   });
 }
 
+function logDispatchLeaseRecovered(details: {
+  milestoneId: string;
+  workerId: string;
+  token: number;
+  recovered: boolean;
+}): void {
+  debugLog("autoLoop", {
+    phase: details.recovered ? "dispatch-lease-recovered" : "dispatch-lease-acquired",
+    ...details,
+  });
+}
+
+function logDispatchLeaseRecoveryFailed(details: {
+  milestoneId?: string;
+  workerId?: string;
+  reason: string;
+}): void {
+  debugLog("autoLoop", {
+    phase: "dispatch-lease-recovery-failed",
+    ...details,
+  });
+}
+
 function logCustomVerifyRetryLoadFailure(err: unknown): void {
   debugLog("autoLoop", {
     phase: "load-custom-verify-retries-failed",
@@ -201,6 +226,36 @@ async function enforceMinRequestInterval(s: AutoSession, prefs: IterationContext
   }
 }
 
+function closeOutCrashedUnit(s: AutoSession, iterData: IterationData, err: unknown): void {
+  const summary = formatDispatchExceptionSummary({ error: err });
+  try {
+    emitOpenUnitEndForUnit(
+      s.basePath,
+      iterData.unitType,
+      iterData.unitId,
+      "cancelled",
+      {
+        message: summary,
+        category: "unit-exception",
+        isTransient: false,
+      },
+    );
+    writeUnitRuntimeRecord(
+      s.basePath,
+      iterData.unitType,
+      iterData.unitId,
+      s.currentUnit?.startedAt ?? Date.now(),
+      {
+        phase: "crashed",
+        lastProgressAt: Date.now(),
+        lastProgressKind: "unit-exception",
+      },
+    );
+  } catch (closeoutErr) {
+    logWarning("dispatch", `unit crash closeout failed: ${closeoutErr instanceof Error ? closeoutErr.message : String(closeoutErr)}`);
+  }
+}
+
 /**
  * Main auto-mode execution loop. Iterates: derive → dispatch → guards →
  * runUnit → finalize → repeat. Exits when s.active becomes false or a
@@ -242,6 +297,10 @@ export async function autoLoop(
         phase: "heartbeat-failed",
         error: err instanceof Error ? err.message : String(err),
       }),
+      logLeaseRefreshMiss: details => debugLog("autoLoop", {
+        phase: "lease-refresh-missed",
+        ...details,
+      }),
     });
 
     // ── Journal: per-iteration flow grouping ──
@@ -493,14 +552,23 @@ export async function autoLoop(
 
         // ── Unit execution (shared with dev path) ──
         await enforceMinRequestInterval(s, prefs);
-        const unitPhaseResult = await runUnitPhaseViaContract(
-          dispatchContract,
-          ic,
-          iterData,
-          loopState,
-          undefined,
-          unitDispatchDeps,
-        );
+        let unitPhaseResult: Awaited<ReturnType<typeof runUnitPhaseViaContract>>;
+        try {
+          unitPhaseResult = await runUnitPhaseViaContract(
+            dispatchContract,
+            ic,
+            iterData,
+            loopState,
+            undefined,
+            unitDispatchDeps,
+          );
+        } catch (err) {
+          if (err instanceof ModelPolicyDispatchBlockedError) {
+            throw err;
+          }
+          closeOutCrashedUnit(s, iterData, err);
+          throw err;
+        }
         if (unitPhaseResult.action === "next") {
           const requestTimestamp = resolveUnitRequestTimestamp(unitPhaseResult.data);
           if (requestTimestamp !== undefined) s.lastRequestTimestamp = requestTimestamp;
@@ -662,25 +730,74 @@ export async function autoLoop(
 
       // Phase B: claim a unit_dispatches row before invoking the unit. The
       // partial unique index idx_unit_dispatches_active_per_unit prevents
-      // a second worker from claiming the same unit concurrently. Returns
-      // null when DB unavailable, no worker registered, or no active lease
-      // — those degraded paths fall through to the existing single-worker
-      // semantics with no ledger entry, preserving back-compat.
-      const dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
+      // a second worker from claiming the same unit concurrently. When this
+      // process has a worker identity, make the milestone lease explicit before
+      // claiming so a step-mode handoff cannot leave us running with a stale
+      // in-memory token and no backing lease row.
+      const leaseBeforeClaim = ensureDispatchLease(s, iterData.mid, {
+        claimMilestoneLease,
+        logLeaseRecovered: logDispatchLeaseRecovered,
+        logLeaseRecoveryFailed: logDispatchLeaseRecoveryFailed,
+      });
+      if (leaseBeforeClaim.kind === "blocked" || leaseBeforeClaim.kind === "failed") {
+        const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} before dispatching ${iterData.unitType} ${iterData.unitId}: ${leaseBeforeClaim.reason}`;
+        ctx.ui.notify(msg, "error");
+        finishTurn("stopped", "execution", msg);
+        await deps.stopAuto(ctx, pi, msg);
+        break;
+      }
+
+      let dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
         getRecentDispatchesForUnit,
         recordDispatchClaim,
         markDispatchRunning,
         logClaimRejected: logDispatchClaimRejected,
         logClaimFailed: logDispatchClaimFailed,
       });
-      const dispatchDecision = decideDispatchClaim(
+      let dispatchDecision = decideDispatchClaim(
         dispatchClaim.kind === "opened"
           ? { kind: "opened", dispatchId: dispatchClaim.dispatchId }
           : dispatchClaim.kind === "skip"
             ? { kind: "skip", reason: dispatchClaim.reason }
             : { kind: "degraded" },
       );
+      if (dispatchDecision.action === "skip" && dispatchDecision.reason === "stale-lease") {
+        const leaseRecovery = ensureDispatchLease(s, iterData.mid, {
+          claimMilestoneLease,
+          logLeaseRecovered: logDispatchLeaseRecovered,
+          logLeaseRecoveryFailed: logDispatchLeaseRecoveryFailed,
+        }, { forceReclaim: true });
+        if (leaseRecovery.kind === "ready") {
+          dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
+            getRecentDispatchesForUnit,
+            recordDispatchClaim,
+            markDispatchRunning,
+            logClaimRejected: logDispatchClaimRejected,
+            logClaimFailed: logDispatchClaimFailed,
+          });
+          dispatchDecision = decideDispatchClaim(
+            dispatchClaim.kind === "opened"
+              ? { kind: "opened", dispatchId: dispatchClaim.dispatchId }
+              : dispatchClaim.kind === "skip"
+                ? { kind: "skip", reason: dispatchClaim.reason }
+                : { kind: "degraded" },
+          );
+        } else {
+          const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} while claiming ${iterData.unitType} ${iterData.unitId}: ${leaseRecovery.reason}`;
+          ctx.ui.notify(msg, "error");
+          finishTurn("stopped", "execution", msg);
+          await deps.stopAuto(ctx, pi, msg);
+          break;
+        }
+      }
       if (dispatchDecision.action === "skip") {
+        if (dispatchDecision.reason === "stale-lease") {
+          const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} while claiming ${iterData.unitType} ${iterData.unitId}; dispatch claim still failed after recovery.`;
+          ctx.ui.notify(msg, "error");
+          finishTurn("stopped", "execution", msg);
+          await deps.stopAuto(ctx, pi, msg);
+          break;
+        }
         finishTurn("skipped", "execution", dispatchDecision.reason);
         continue;
       }
@@ -700,6 +817,7 @@ export async function autoLoop(
         if (err instanceof ModelPolicyDispatchBlockedError) {
           throw err;
         }
+        closeOutCrashedUnit(s, iterData, err);
         dispatchSettled = settleDispatchFailed(
           dispatchId,
           formatDispatchExceptionSummary({ error: err }),