gsd-pi 2.80.0-dev.fbe7c8c6f → 2.81.0

package/dist/resources/extensions/gsd/auto/loop.js

@@ -16,7 +16,7 @@ import { ModelPolicyDispatchBlockedError } from "../auto-model-selection.js";
 import { resolveEngine } from "../engine-resolver.js";
 import { logWarning } from "../workflow-logger.js";
 import { recordDispatchClaim, markRunning as markDispatchRunning, markCompleted as markDispatchCompleted, markFailed as markDispatchFailed, getRecentForUnit as getRecentDispatchesForUnit, getRecentUnitKeysForProjectRoot, } from "../db/unit-dispatches.js";
-import { refreshMilestoneLease } from "../db/milestone-leases.js";
+import { claimMilestoneLease, refreshMilestoneLease } from "../db/milestone-leases.js";
 import { heartbeatAutoWorker } from "../db/auto-workers.js";
 import { getRuntimeKv, setRuntimeKv } from "../db/runtime-kv.js";
 import { resolveUokFlags } from "../uok/flags.js";
@@ -25,7 +25,9 @@ import { normalizeRealPath } from "../paths.js";
 import { decideCooldownRecovery, decideDispatchClaim, decideEngineDispatch, decideFinalizeResult, decideInfrastructureError, decideIterationErrorRecovery, decideMemoryPressure, decideModelPolicyBlocked, decideMinRequestInterval, decideWorkflowLoop, formatDispatchExceptionSummary, formatUnhandledDispatchErrorSummary, resolveUnitRequestTimestamp, shouldUseCustomEnginePath, } from "./workflow-kernel.js";
 import { hydrateCustomVerifyRetryCounts, saveCustomVerifyRetryCounts, } from "./custom-verify-retry-store.js";
 import { settleDispatchCompleted, settleDispatchFailed, } from "./workflow-dispatch-ledger.js";
-import { openDispatchClaim } from "./workflow-dispatch-claim.js";
+import { emitOpenUnitEndForUnit } from "../crash-recovery.js";
+import { writeUnitRuntimeRecord } from "../unit-runtime.js";
+import { ensureDispatchLease, openDispatchClaim } from "./workflow-dispatch-claim.js";
 import { completeWorkflowIteration } from "./workflow-iteration-completion.js";
 import { createWorkflowJournalReporter } from "./workflow-journal-reporter.js";
 import { createWorkflowPhaseReporter } from "./workflow-phase-reporter.js";
@@ -103,6 +105,18 @@ function logDispatchClaimFailed(err) {
         error: err instanceof Error ? err.message : String(err),
     });
 }
+function logDispatchLeaseRecovered(details) {
+    debugLog("autoLoop", {
+        phase: details.recovered ? "dispatch-lease-recovered" : "dispatch-lease-acquired",
+        ...details,
+    });
+}
+function logDispatchLeaseRecoveryFailed(details) {
+    debugLog("autoLoop", {
+        phase: "dispatch-lease-recovery-failed",
+        ...details,
+    });
+}
 function logCustomVerifyRetryLoadFailure(err) {
     debugLog("autoLoop", {
         phase: "load-custom-verify-retries-failed",
@@ -133,6 +147,24 @@ async function enforceMinRequestInterval(s, prefs) {
         await new Promise(r => setTimeout(r, decision.waitMs));
     }
 }
+function closeOutCrashedUnit(s, iterData, err) {
+    const summary = formatDispatchExceptionSummary({ error: err });
+    try {
+        emitOpenUnitEndForUnit(s.basePath, iterData.unitType, iterData.unitId, "cancelled", {
+            message: summary,
+            category: "unit-exception",
+            isTransient: false,
+        });
+        writeUnitRuntimeRecord(s.basePath, iterData.unitType, iterData.unitId, s.currentUnit?.startedAt ?? Date.now(), {
+            phase: "crashed",
+            lastProgressAt: Date.now(),
+            lastProgressKind: "unit-exception",
+        });
+    }
+    catch (closeoutErr) {
+        logWarning("dispatch", `unit crash closeout failed: ${closeoutErr instanceof Error ? closeoutErr.message : String(closeoutErr)}`);
+    }
+}
 /**
  * Main auto-mode execution loop. Iterates: derive → dispatch → guards →
  * runUnit → finalize → repeat. Exits when s.active becomes false or a
@@ -166,6 +198,10 @@ export async function autoLoop(ctx, pi, s, deps, options) {
                 phase: "heartbeat-failed",
                 error: err instanceof Error ? err.message : String(err),
             }),
+            logLeaseRefreshMiss: details => debugLog("autoLoop", {
+                phase: "lease-refresh-missed",
+                ...details,
+            }),
         });
         // ── Journal: per-iteration flow grouping ──
         const flowId = randomUUID();
@@ -392,7 +428,17 @@ export async function autoLoop(ctx, pi, s, deps, options) {
                 }
                 // ── Unit execution (shared with dev path) ──
                 await enforceMinRequestInterval(s, prefs);
-                const unitPhaseResult = await runUnitPhaseViaContract(dispatchContract, ic, iterData, loopState, undefined, unitDispatchDeps);
+                let unitPhaseResult;
+                try {
+                    unitPhaseResult = await runUnitPhaseViaContract(dispatchContract, ic, iterData, loopState, undefined, unitDispatchDeps);
+                }
+                catch (err) {
+                    if (err instanceof ModelPolicyDispatchBlockedError) {
+                        throw err;
+                    }
+                    closeOutCrashedUnit(s, iterData, err);
+                    throw err;
+                }
                 if (unitPhaseResult.action === "next") {
                     const requestTimestamp = resolveUnitRequestTimestamp(unitPhaseResult.data);
                     if (requestTimestamp !== undefined)
@@ -551,23 +597,70 @@ export async function autoLoop(ctx, pi, s, deps, options) {
             await enforceMinRequestInterval(s, prefs);
             // Phase B: claim a unit_dispatches row before invoking the unit. The
             // partial unique index idx_unit_dispatches_active_per_unit prevents
-            // a second worker from claiming the same unit concurrently. Returns
-            // null when DB unavailable, no worker registered, or no active lease
-            // — those degraded paths fall through to the existing single-worker
-            // semantics with no ledger entry, preserving back-compat.
-            const dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
+            // a second worker from claiming the same unit concurrently. When this
+            // process has a worker identity, make the milestone lease explicit before
+            // claiming so a step-mode handoff cannot leave us running with a stale
+            // in-memory token and no backing lease row.
+            const leaseBeforeClaim = ensureDispatchLease(s, iterData.mid, {
+                claimMilestoneLease,
+                logLeaseRecovered: logDispatchLeaseRecovered,
+                logLeaseRecoveryFailed: logDispatchLeaseRecoveryFailed,
+            });
+            if (leaseBeforeClaim.kind === "blocked" || leaseBeforeClaim.kind === "failed") {
+                const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} before dispatching ${iterData.unitType} ${iterData.unitId}: ${leaseBeforeClaim.reason}`;
+                ctx.ui.notify(msg, "error");
+                finishTurn("stopped", "execution", msg);
+                await deps.stopAuto(ctx, pi, msg);
+                break;
+            }
+            let dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
                 getRecentDispatchesForUnit,
                 recordDispatchClaim,
                 markDispatchRunning,
                 logClaimRejected: logDispatchClaimRejected,
                 logClaimFailed: logDispatchClaimFailed,
             });
-            const dispatchDecision = decideDispatchClaim(dispatchClaim.kind === "opened"
+            let dispatchDecision = decideDispatchClaim(dispatchClaim.kind === "opened"
                 ? { kind: "opened", dispatchId: dispatchClaim.dispatchId }
                 : dispatchClaim.kind === "skip"
                     ? { kind: "skip", reason: dispatchClaim.reason }
                     : { kind: "degraded" });
+            if (dispatchDecision.action === "skip" && dispatchDecision.reason === "stale-lease") {
+                const leaseRecovery = ensureDispatchLease(s, iterData.mid, {
+                    claimMilestoneLease,
+                    logLeaseRecovered: logDispatchLeaseRecovered,
+                    logLeaseRecoveryFailed: logDispatchLeaseRecoveryFailed,
+                }, { forceReclaim: true });
+                if (leaseRecovery.kind === "ready") {
+                    dispatchClaim = openDispatchClaim(s, flowId, turnId, iterData, {
+                        getRecentDispatchesForUnit,
+                        recordDispatchClaim,
+                        markDispatchRunning,
+                        logClaimRejected: logDispatchClaimRejected,
+                        logClaimFailed: logDispatchClaimFailed,
+                    });
+                    dispatchDecision = decideDispatchClaim(dispatchClaim.kind === "opened"
+                        ? { kind: "opened", dispatchId: dispatchClaim.dispatchId }
+                        : dispatchClaim.kind === "skip"
+                            ? { kind: "skip", reason: dispatchClaim.reason }
+                            : { kind: "degraded" });
+                }
+                else {
+                    const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} while claiming ${iterData.unitType} ${iterData.unitId}: ${leaseRecovery.reason}`;
+                    ctx.ui.notify(msg, "error");
+                    finishTurn("stopped", "execution", msg);
+                    await deps.stopAuto(ctx, pi, msg);
+                    break;
+                }
+            }
             if (dispatchDecision.action === "skip") {
+                if (dispatchDecision.reason === "stale-lease") {
+                    const msg = `Lost milestone lease for ${iterData.mid ?? "unknown"} while claiming ${iterData.unitType} ${iterData.unitId}; dispatch claim still failed after recovery.`;
+                    ctx.ui.notify(msg, "error");
+                    finishTurn("stopped", "execution", msg);
+                    await deps.stopAuto(ctx, pi, msg);
+                    break;
+                }
                 finishTurn("skipped", "execution", dispatchDecision.reason);
                 continue;
             }
@@ -580,6 +673,7 @@ export async function autoLoop(ctx, pi, s, deps, options) {
                 if (err instanceof ModelPolicyDispatchBlockedError) {
                     throw err;
                 }
+                closeOutCrashedUnit(s, iterData, err);
                 dispatchSettled = settleDispatchFailed(dispatchId, formatDispatchExceptionSummary({ error: err }), {
                     markFailed: markDispatchFailed,
                     logWriteFailure: logDispatchLedgerWriteFailure,

package/dist/resources/extensions/gsd/auto/orchestrator.js

@@ -1,3 +1,5 @@
+// Project/App: GSD-2
+// File Purpose: Auto Orchestration module implementation and ADR-015 invariant pipeline owner.
 function now() {
     return Date.now();
 }
@@ -29,9 +31,20 @@ export class AutoOrchestrator {
                 await this.deps.health.postAdvanceRecord(blocked);
                 return blocked;
             }
-            const decision = await this.deps.dispatch.decideNextUnit();
+            const reconciliation = await this.deps.stateReconciliation.reconcileBeforeDispatch();
+            if (!reconciliation.ok || !reconciliation.stateSnapshot) {
+                const blocked = {
+                    kind: "blocked",
+                    reason: reconciliation.reason,
+                    stateSnapshot: reconciliation.stateSnapshot,
+                };
+                await this.deps.runtime.journalTransition({ name: "advance-blocked", reason: blocked.reason });
+                await this.deps.health.postAdvanceRecord(blocked);
+                return blocked;
+            }
+            const decision = await this.deps.dispatch.decideNextUnit({ stateSnapshot: reconciliation.stateSnapshot });
             if (!decision) {
-                const stopped = { kind: "stopped", reason: "no remaining units" };
+                const stopped = { kind: "stopped", reason: "no remaining units", stateSnapshot: reconciliation.stateSnapshot };
                 this.status.phase = "stopped";
                 this.status.activeUnit = undefined;
                 this.lastAdvanceKey = null;
@@ -52,6 +65,38 @@ export class AutoOrchestrator {
                 await this.deps.health.postAdvanceRecord(blocked);
                 return blocked;
             }
+            const contract = await this.deps.toolContract.compileUnitToolContract(decision.unitType, decision.unitId);
+            if (!contract.ok) {
+                const blocked = {
+                    kind: "blocked",
+                    reason: contract.reason,
+                    stateSnapshot: reconciliation.stateSnapshot,
+                };
+                await this.deps.runtime.journalTransition({
+                    name: "advance-blocked",
+                    reason: blocked.reason,
+                    unitType: decision.unitType,
+                    unitId: decision.unitId,
+                });
+                await this.deps.health.postAdvanceRecord(blocked);
+                return blocked;
+            }
+            const worktree = await this.deps.worktree.prepareForUnit(decision.unitType, decision.unitId);
+            if (!worktree.ok) {
+                const blocked = {
+                    kind: "blocked",
+                    reason: worktree.reason,
+                    stateSnapshot: reconciliation.stateSnapshot,
+                };
+                await this.deps.runtime.journalTransition({
+                    name: "advance-blocked",
+                    reason: blocked.reason,
+                    unitType: decision.unitType,
+                    unitId: decision.unitId,
+                });
+                await this.deps.health.postAdvanceRecord(blocked);
+                return blocked;
+            }
             this.status.activeUnit = { unitType: decision.unitType, unitId: decision.unitId };
             this.status.phase = "running";
             this.lastAdvanceKey = nextKey;
@@ -62,9 +107,8 @@ export class AutoOrchestrator {
                 unitType: decision.unitType,
                 unitId: decision.unitId,
             });
-            await this.deps.worktree.prepareForUnit(decision.unitType, decision.unitId);
             await this.deps.worktree.syncAfterUnit(decision.unitType, decision.unitId);
-            const advanced = { kind: "advanced" };
+            const advanced = { kind: "advanced", stateSnapshot: reconciliation.stateSnapshot };
             await this.deps.health.postAdvanceRecord(advanced);
             return advanced;
         }