gsd-pi 2.81.0 → 2.82.0-dev.20138ae42

package/src/resources/extensions/gsd/auto/loop.ts

@@ -78,7 +78,10 @@ import { createWorkflowTurnReporter } from "./workflow-turn-reporter.js";
 import { validateWorkflowSessionLock } from "./workflow-session-lock.js";
 import { dequeueSidecarItem } from "./workflow-sidecar-queue.js";
 import { maintainWorkerHeartbeat } from "./workflow-worker-heartbeat.js";
-import { measureMemoryPressure } from "./workflow-memory-pressure.js";
+import {
+  measureMemoryPressure,
+  shouldCheckMemoryPressure,
+} from "./workflow-memory-pressure.js";
 import { buildSidecarIterationData } from "./workflow-sidecar-iteration.js";
 import {
   createExecutionGraphUnitDispatchDeps,
@@ -203,9 +206,9 @@ function logCustomVerifyRetrySaveFailure(err: unknown): void {
 }
 
 // ── Memory pressure monitoring (#3331) ──────────────────────────────────
-// Check heap usage every N iterations and trigger graceful shutdown before
-// the OS OOM killer sends SIGKILL. The threshold is 90% of the V8 heap
-// limit (--max-old-space-size or default ~1.5-4GB depending on platform).
+// Check heap usage on session startup, then every N iterations, and trigger
+// graceful shutdown before the OS OOM killer sends SIGKILL. The threshold is
+// 90% of the V8 heap limit (--max-old-space-size or default ~1.5-4GB depending on platform).
 const MEMORY_CHECK_INTERVAL = 5; // check every 5 iterations
 const MAX_CUSTOM_ENGINE_VERIFY_RETRIES = 3;
 
@@ -372,7 +375,7 @@ export async function autoLoop(
 
     // ── Memory pressure check (#3331) ──
     // Graceful shutdown before OOM killer sends SIGKILL.
-    if (iteration % MEMORY_CHECK_INTERVAL === 0) {
+    if (shouldCheckMemoryPressure(iteration, MEMORY_CHECK_INTERVAL)) {
       const mem = measureMemoryPressure();
       debugLog("autoLoop", { phase: "memory-check", ...mem });
       const memoryDecision = decideMemoryPressure({ ...mem, iteration });
@@ -399,6 +402,12 @@ export async function autoLoop(
 
     let dispatchId: number | null = null;
     let dispatchSettled = false;
+    let iterationEndEmitted = false;
+    const emitIterationEnd = (details: Record<string, unknown> = {}): void => {
+      if (iterationEndEmitted) return;
+      iterationEndEmitted = true;
+      journalReporter.emit("iteration-end", { iteration, ...details });
+    };
     const completeIteration = (): void => {
       completeWorkflowIteration({
         get consecutiveErrors() { return consecutiveErrors; },
@@ -407,11 +416,15 @@ export async function autoLoop(
         set consecutiveCooldowns(value) { consecutiveCooldowns = value; },
         recentErrorMessages,
       }, {
-        emitIterationEnd: () => journalReporter.emit("iteration-end", { iteration }),
+        emitIterationEnd: () => emitIterationEnd(),
         saveStuckState: () => saveStuckState(s, loopState),
         logIterationComplete: () => debugLog("autoLoop", { phase: "iteration-complete", iteration }),
       });
     };
+    const finishIncompleteIteration = (details: Record<string, unknown>): void => {
+      emitIterationEnd(details);
+      saveStuckState(s, loopState);
+    };
 
     try {
       // ── Blanket try/catch: one bad iteration must not kill the session
@@ -492,6 +505,7 @@ export async function autoLoop(
         });
         if (engineState.isComplete) {
           finishTurn("completed");
+          emitIterationEnd({ status: "completed", reason: "custom-engine-complete" });
           await deps.stopAuto(ctx, pi, "Workflow complete");
           break;
         }
@@ -509,16 +523,23 @@ export async function autoLoop(
         });
         if (dispatchFlow.action === "break") {
           finishTurn("stopped", "manual-attention", "custom-engine-dispatch-stop");
+          finishIncompleteIteration({
+            status: "stopped",
+            reason: "custom-engine-dispatch-stop",
+            failureClass: "manual-attention",
+          });
           break;
         }
         if (dispatchFlow.action === "continue") {
           finishTurn("skipped");
+          emitIterationEnd({ status: "skipped", reason: "custom-engine-dispatch-skip" });
           continue;
         }
 
         // dispatch.action === "dispatch"
         if (dispatch.action !== "dispatch") {
           finishTurn("skipped");
+          emitIterationEnd({ status: "skipped", reason: "custom-engine-dispatch-mismatch" });
           continue;
         }
         const step = dispatch.step;
@@ -547,6 +568,13 @@ export async function autoLoop(
         });
         if (guardsResult.action === "break") {
           finishTurn("stopped", "manual-attention", "guard-break");
+          finishIncompleteIteration({
+            status: "stopped",
+            reason: "guard-break",
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+            failureClass: "manual-attention",
+          });
           break;
         }
 
@@ -578,6 +606,13 @@ export async function autoLoop(
           unitId: iterData.unitId,
         });
         if (unitPhaseResult.action === "break") {
+          finishIncompleteIteration({
+            status: "stopped",
+            reason: unitPhaseResult.reason ?? "unit-break",
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+            failureClass: "execution",
+          });
           finishTurn("stopped", "execution", "unit-break");
           break;
         }
@@ -596,7 +631,16 @@ export async function autoLoop(
               finishTurn,
             },
           });
-          if (verifyFlow.action === "break") break;
+          if (verifyFlow.action === "break") {
+            finishIncompleteIteration({
+              status: "paused",
+              reason: "custom-engine-verify-pause",
+              unitType: iterData.unitType,
+              unitId: iterData.unitId,
+              failureClass: "manual-attention",
+            });
+            break;
+          }
         }
         if (verifyResult === "retry") {
           const retryOutcome = await handleCustomEngineVerifyRetry({
@@ -630,7 +674,22 @@ export async function autoLoop(
               finishTurn,
             },
           });
-          if (retryFlow.action === "break") break;
+          if (retryFlow.action === "break") {
+            finishIncompleteIteration({
+              status: retryOutcome.action === "stop" ? "stopped" : "paused",
+              reason: retryOutcome.action === "retry" ? "custom-engine-verify-retry" : retryOutcome.turnError,
+              unitType: iterData.unitType,
+              unitId: iterData.unitId,
+              failureClass: "manual-attention",
+            });
+            break;
+          }
+          finishIncompleteIteration({
+            status: "retry",
+            reason: "custom-engine-verify-retry",
+            unitType: iterData.unitType,
+            unitId: iterData.unitId,
+          });
           continue;
         }
 
@@ -841,6 +900,13 @@ export async function autoLoop(
           markFailed: markDispatchFailed,
           logWriteFailure: logDispatchLedgerWriteFailure,
         }) || dispatchSettled;
+        finishIncompleteIteration({
+          status: "stopped",
+          reason: unitPhaseResult.reason ?? "unit-break",
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+          failureClass: "execution",
+        });
         finishTurn("stopped", "execution", "unit-break");
         break;
       }
@@ -848,12 +914,25 @@ export async function autoLoop(
       // ── Phase 5: Finalize ───────────────────────────────────────────────
 
       let finalizeResult: Awaited<ReturnType<typeof runFinalize>>;
+      journalReporter.emit("post-unit-finalize-start", {
+        iteration,
+        unitType: iterData.unitType,
+        unitId: iterData.unitId,
+      });
       try {
         finalizeResult = await runFinalize(ic, iterData, loopState, sidecarItem);
       } catch (err) {
+        const error = formatDispatchExceptionSummary({ error: err });
+        journalReporter.emit("post-unit-finalize-end", {
+          iteration,
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+          status: "failed",
+          error,
+        });
         dispatchSettled = settleDispatchFailed(
           dispatchId,
-          formatDispatchExceptionSummary({ error: err }),
+          error,
           {
             markFailed: markDispatchFailed,
             logWriteFailure: logDispatchLedgerWriteFailure,
@@ -865,6 +944,15 @@ export async function autoLoop(
         unitType: iterData.unitType,
         unitId: iterData.unitId,
       });
+      const finalizeReason = finalizeResult.action === "break" ? finalizeResult.reason : undefined;
+      journalReporter.emit("post-unit-finalize-end", {
+        iteration,
+        unitType: iterData.unitType,
+        unitId: iterData.unitId,
+        status: finalizeResult.action === "next" ? "completed" : finalizeResult.action === "continue" ? "retry" : "stopped",
+        action: finalizeResult.action,
+        ...(finalizeReason ? { reason: finalizeReason } : {}),
+      });
       const finalizeDecision = decideFinalizeResult(
         finalizeResult.action === "break"
           ? { action: "break", reason: finalizeResult.reason }
@@ -877,6 +965,13 @@ export async function autoLoop(
           markFailed: markDispatchFailed,
           logWriteFailure: logDispatchLedgerWriteFailure,
         }) || dispatchSettled;
+        finishIncompleteIteration({
+          status: "stopped",
+          reason: finalizeReason ?? "finalize-break",
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+          failureClass: finalizeDecision.failureClass,
+        });
         finishTurn("stopped", finalizeDecision.failureClass, finalizeDecision.turnError);
         break;
       }
@@ -885,6 +980,12 @@ export async function autoLoop(
           markFailed: markDispatchFailed,
           logWriteFailure: logDispatchLedgerWriteFailure,
         }) || dispatchSettled;
+        finishIncompleteIteration({
+          status: "retry",
+          reason: "finalize-retry",
+          unitType: iterData.unitType,
+          unitId: iterData.unitId,
+        });
         finishTurn("retry");
         continue;
       }
@@ -909,11 +1010,6 @@ export async function autoLoop(
         ) || dispatchSettled;
       }
 
-      // Always emit iteration-end on error so the journal records iteration
-      // completion even on failure (#2344). Without this, errors in
-      // runFinalize leave the journal incomplete, making diagnosis harder.
-      journalReporter.emit("iteration-end", { iteration, error: msg });
-
       // ── Pre-send model-policy block: not a retryable error (#4959 / #4850) ──
       // The model-policy gate runs before the prompt is sent.  When every
       // candidate model is denied (cross-provider disabled + flat-rate
@@ -938,6 +1034,12 @@ export async function autoLoop(
         });
         ctx.ui.notify(policyDecision.notifyMessage, "error");
         journalReporter.emit("unit-end", policyDecision.journalData);
+        finishIncompleteIteration({
+          status: "blocked",
+          reason: "model-policy-dispatch-blocked",
+          unitType: loopErr.unitType,
+          unitId: loopErr.unitId,
+        });
         // Carry the blocked unit identity into the turn-result observer:
         // the throw originated inside dispatch, so observedUnitType/Id were
         // not assigned by the success path at lines 453/631/647 — but the
@@ -951,6 +1053,11 @@ export async function autoLoop(
         break;
       }
 
+      // Always emit iteration-end on error so the journal records iteration
+      // completion even on failure (#2344). Without this, errors in
+      // runFinalize leave the journal incomplete, making diagnosis harder.
+      finishIncompleteIteration({ status: "failed", error: msg });
+
       // ── Infrastructure errors: immediate stop, no retry ──
       // These are unrecoverable (disk full, OOM, etc.). Retrying just burns
       // LLM budget on guaranteed failures.

package/src/resources/extensions/gsd/auto/orchestrator.ts

@@ -7,6 +7,16 @@ function now(): number {
   return Date.now();
 }
 
+/**
+ * Size of the dispatch-decision ring buffer used by the Auto Orchestration
+ * module's stuck-loop detector. When the same `${unitType}:${unitId}` key
+ * fills the window, advance() blocks with `action: "stop"`.
+ *
+ * Mirrors the legacy `STUCK_WINDOW_SIZE` in auto/phases.ts so behaviour is
+ * preserved across the eventual cutover (issue #5791).
+ */
+export const STUCK_WINDOW_SIZE = 6;
+
 export class AutoOrchestrator implements AutoOrchestrationModule {
   private status: AutoStatus = {
     phase: "idle",
@@ -14,6 +24,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
   };
   private readonly deps: AutoOrchestratorDeps;
   private lastAdvanceKey: string | null = null;
+  private dispatchKeyWindow: string[] = [];
 
   public constructor(deps: AutoOrchestratorDeps) {
     this.deps = deps;
@@ -21,6 +32,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
 
   public async start(_sessionContext: AutoSessionContext): Promise<AutoAdvanceResult> {
     this.lastAdvanceKey = null;
+    this.dispatchKeyWindow = [];
     this.status.phase = "running";
     this.bumpTransition();
     await this.deps.runtime.journalTransition({ name: "start" });
@@ -31,19 +43,72 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
   public async advance(): Promise<AutoAdvanceResult> {
     try {
       await this.deps.runtime.ensureLockOwnership();
+
+      const staleMsg = this.deps.health.checkResourcesStale();
+      if (staleMsg) {
+        await this.deps.uokGate.emit({
+          gateId: "resource-version-guard",
+          gateType: "policy",
+          outcome: "fail",
+          failureClass: "policy",
+          rationale: "resource version guard blocked dispatch",
+          findings: staleMsg,
+        });
+        const blocked: AutoAdvanceResult = { kind: "blocked", reason: staleMsg, action: "stop" };
+        await this.deps.runtime.journalTransition({ name: "advance-blocked", reason: blocked.reason });
+        await this.deps.health.postAdvanceRecord(blocked);
+        return blocked;
+      }
+      await this.deps.uokGate.emit({
+        gateId: "resource-version-guard",
+        gateType: "policy",
+        outcome: "pass",
+        failureClass: "none",
+        rationale: "resource version guard passed",
+      });
+
       const gate = await this.deps.health.preAdvanceGate();
-      if (!gate.allow) {
-        const blocked: AutoAdvanceResult = { kind: "blocked", reason: gate.reason ?? "health gate blocked" };
+      if (gate.kind === "fail") {
+        await this.deps.uokGate.emit({
+          gateId: "pre-dispatch-health-gate",
+          gateType: "execution",
+          outcome: "manual-attention",
+          failureClass: "manual-attention",
+          rationale: "pre-dispatch health gate blocked dispatch",
+          findings: gate.reason,
+        });
+        const blocked: AutoAdvanceResult = { kind: "blocked", reason: gate.reason, action: "pause" };
         await this.deps.runtime.journalTransition({ name: "advance-blocked", reason: blocked.reason });
         await this.deps.health.postAdvanceRecord(blocked);
         return blocked;
       }
+      if (gate.kind === "threw") {
+        await this.deps.uokGate.emit({
+          gateId: "pre-dispatch-health-gate",
+          gateType: "execution",
+          outcome: "manual-attention",
+          failureClass: "manual-attention",
+          rationale: "pre-dispatch health gate threw unexpectedly",
+          findings: String(gate.error),
+        });
+        // intentional fall-through: matches runPreDispatch behaviour
+      } else {
+        await this.deps.uokGate.emit({
+          gateId: "pre-dispatch-health-gate",
+          gateType: "execution",
+          outcome: "pass",
+          failureClass: "none",
+          rationale: "pre-dispatch health gate passed",
+          findings: gate.fixesApplied?.join(", ") ?? "",
+        });
+      }
 
       const reconciliation = await this.deps.stateReconciliation.reconcileBeforeDispatch();
       if (!reconciliation.ok || !reconciliation.stateSnapshot) {
         const blocked: AutoAdvanceResult = {
           kind: "blocked",
-          reason: reconciliation.reason,
+          reason: reconciliation.reason ?? "state reconciliation produced no snapshot",
+          action: "pause",
           stateSnapshot: reconciliation.stateSnapshot,
         };
         await this.deps.runtime.journalTransition({ name: "advance-blocked", reason: blocked.reason });
@@ -57,15 +122,64 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
         this.status.phase = "stopped";
         this.status.activeUnit = undefined;
         this.lastAdvanceKey = null;
+        this.dispatchKeyWindow = [];
         this.bumpTransition();
         await this.deps.runtime.journalTransition({ name: "advance-stopped", reason: stopped.reason });
         await this.deps.health.postAdvanceRecord(stopped);
         return stopped;
       }
+      if (!("unitType" in decision)) {
+        const blocked: AutoAdvanceResult = {
+          kind: "blocked",
+          reason: decision.reason,
+          action: decision.action,
+          stateSnapshot: reconciliation.stateSnapshot,
+        };
+        await this.deps.runtime.journalTransition({ name: "advance-blocked", reason: blocked.reason });
+        await this.deps.health.postAdvanceRecord(blocked);
+        return blocked;
+      }
 
       const nextKey = `${decision.unitType}:${decision.unitId}`;
-      if (this.lastAdvanceKey === nextKey) {
-        const blocked: AutoAdvanceResult = { kind: "blocked", reason: "idempotent advance: unit already active" };
+
+      // Record every dispatch decision in the ring buffer before pre-flight
+      // checks so the stuck-loop detector observes the full decision history
+      // (including decisions that idempotency would otherwise short-circuit).
+      // The ring is capped at STUCK_WINDOW_SIZE and evicts oldest-first.
+      this.dispatchKeyWindow.push(nextKey);
+      if (this.dispatchKeyWindow.length > STUCK_WINDOW_SIZE) {
+        this.dispatchKeyWindow.shift();
+      }
+
+      // Idempotency: same key as immediately previous successful advance.
+      // This is the soft, fast-path block kept from #5786. It only fires when
+      // the ring is NOT yet saturated for this key — once the ring is full of
+      // `nextKey`, the stuck-loop verdict takes precedence (see below). Both
+      // checks coexist: idempotency for the common immediate-repeat case,
+      // stuck-loop for the saturated-window case.
+      const matchingCount = this.dispatchKeyWindow.filter((k) => k === nextKey).length;
+      if (this.lastAdvanceKey === nextKey && matchingCount < STUCK_WINDOW_SIZE) {
+        const blocked: AutoAdvanceResult = { kind: "blocked", reason: "idempotent advance: unit already active", action: "stop" };
+        await this.deps.runtime.journalTransition({
+          name: "advance-blocked",
+          reason: blocked.reason,
+          unitType: decision.unitType,
+          unitId: decision.unitId,
+        });
+        await this.deps.health.postAdvanceRecord(blocked);
+        return blocked;
+      }
+
+      // Stuck-loop detection: when the ring is saturated with copies of
+      // `nextKey` (count >= STUCK_WINDOW_SIZE), the orchestrator has been
+      // picking the same unit across the whole window and must hard-stop with
+      // a diagnosable reason.
+      if (matchingCount >= STUCK_WINDOW_SIZE) {
+        const blocked: AutoAdvanceResult = {
+          kind: "blocked",
+          reason: `stuck-loop: ${nextKey} picked ${matchingCount} times`,
+          action: "stop",
+        };
         await this.deps.runtime.journalTransition({
           name: "advance-blocked",
           reason: blocked.reason,
@@ -81,6 +195,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
         const blocked: AutoAdvanceResult = {
           kind: "blocked",
           reason: contract.reason,
+          action: "pause",
           stateSnapshot: reconciliation.stateSnapshot,
         };
         await this.deps.runtime.journalTransition({
@@ -98,6 +213,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
         const blocked: AutoAdvanceResult = {
           kind: "blocked",
           reason: worktree.reason,
+          action: "pause",
           stateSnapshot: reconciliation.stateSnapshot,
         };
         await this.deps.runtime.journalTransition({
@@ -123,7 +239,11 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
       });
       await this.deps.worktree.syncAfterUnit(decision.unitType, decision.unitId);
 
-      const advanced: AutoAdvanceResult = { kind: "advanced", stateSnapshot: reconciliation.stateSnapshot };
+      const advanced: AutoAdvanceResult = {
+        kind: "advanced",
+        unit: { unitType: decision.unitType, unitId: decision.unitId },
+        stateSnapshot: reconciliation.stateSnapshot,
+      };
       await this.deps.health.postAdvanceRecord(advanced);
       return advanced;
     } catch (error) {
@@ -148,6 +268,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
 
       if (result.kind === "stopped") {
         this.lastAdvanceKey = null;
+        this.dispatchKeyWindow = [];
         this.status.activeUnit = undefined;
       }
       this.bumpTransition();
@@ -173,6 +294,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
 
   public async resume(): Promise<AutoAdvanceResult> {
     this.lastAdvanceKey = null;
+    this.dispatchKeyWindow = [];
     this.status.phase = "running";
     this.bumpTransition();
     await this.deps.runtime.journalTransition({ name: "resume" });
@@ -188,6 +310,7 @@ export class AutoOrchestrator implements AutoOrchestrationModule {
     this.status.phase = "stopped";
     this.status.activeUnit = undefined;
     this.lastAdvanceKey = null;
+    this.dispatchKeyWindow = [];
     this.bumpTransition();
     await this.deps.runtime.journalTransition({ name: "stop", reason });
     await this.deps.notifications.notifyLifecycle({ name: "stop", detail: reason });