claude-code-session-manager 0.85.0 → 0.86.0

package/src/main/scheduler.cjs

@@ -95,6 +95,7 @@ const {
   PIDLESS_SPAWN_GRACE_MS,
   INVESTIGATION_MAX_MS,
   STARVATION_ESCALATE_MS,
+  STARVE_ESCALATION_MS,
 } = require('./lib/schedulerConfig.cjs');
 const QUARANTINE_ESCALATE_MS = process.env.SM_QUARANTINE_ESCALATE_HOURS
   ? Number(process.env.SM_QUARANTINE_ESCALATE_HOURS) * 60 * 60_000
@@ -1484,14 +1485,15 @@ function computeBlockedChains(jobs) {
  * findStaleQuarantinedJobs(jobs, now, thresholdMs) → [{ slug, cwd, ageMs }]
  *
  * Pure, no IO. A 'quarantined' row (no createdVia provenance) can otherwise
- * sit forever with nothing looking at it — quarantine only ever clears via a
- * human adopting or archiving it. This is the escalation half of that gate:
- * any quarantined row whose recorded quarantine timestamp (statusHistory's
- * `to === 'quarantined'` entry — stamped at creation, or backfilled from the
- * PRD file's mtime by reconcile() for rows quarantined before that stamp
- * existed) is older than `thresholdMs` is reported so the caller can
- * warn-log and surface it distinctly. A row with no recoverable timestamp is
- * skipped rather than guessed at.
+ * sit forever with nothing looking at it — quarantine used to clear only via
+ * a human adopting or archiving it; autoResolveQuarantine below now gives it
+ * a bounded automatic exit too. This function stays the escalation/warn half
+ * of that gate: any quarantined row whose recorded quarantine timestamp
+ * (statusHistory's `to === 'quarantined'` entry — stamped at creation, or
+ * backfilled from the PRD file's mtime by reconcile() for rows quarantined
+ * before that stamp existed) is older than `thresholdMs` is reported so the
+ * caller can warn-log and surface it distinctly. A row with no recoverable
+ * timestamp is skipped rather than guessed at.
  */
 function findStaleQuarantinedJobs(jobs, now, thresholdMs) {
   const stale = [];
@@ -1507,6 +1509,106 @@ function findStaleQuarantinedJobs(jobs, now, thresholdMs) {
   return stale;
 }
 
+// Bounded automatic exit for a quarantined row (this PRD): up to
+// QUARANTINE_RESOLVE_CAP auto-resolve attempts, each gated on having sat
+// `quarantined` for QUARANTINE_ESCALATE_MS, before autoResolveQuarantine
+// below settles the row to 'skipped' rather than leaving it as a dead end
+// only a human `scheduler_reset_job`/adopt action could ever clear. A single
+// attempt is enough in practice — the outcome is terminal — but the counter
+// still guards against two overlapping ticks both trying to resolve the
+// same row.
+const QUARANTINE_RESOLVE_CAP = 1;
+
+/**
+ * Kill-switch gate for the quarantine auto-resolve pass below
+ * (SM_QUARANTINE_AUTORESOLVE_DISABLE=1), same shape as
+ * failedAutoResetDisabled/needsReviewAutoResolveDisabled.
+ */
+function quarantineAutoResolveDisabled() {
+  return process.env.SM_QUARANTINE_AUTORESOLVE_DISABLE === '1';
+}
+
+/**
+ * selectQuarantineAutoResolveTargets(jobs, now, thresholdMs) →
+ *   [{ slug, cwd, ageMs }]
+ *
+ * Pure selector — no IO. Same age computation as findStaleQuarantinedJobs
+ * above, bounded additionally by quarantineResolveAttempts so a row already
+ * auto-resolved (or mid-resolve on a race) is never re-selected. Deliberately
+ * does NOT check createdVia here — that requires a disk read of the PRD
+ * file, and doing it at selection time would let this pass act on a
+ * snapshot that's gone stale by the time the mutate() pass actually runs.
+ * autoResolveQuarantine below re-reads createdVia fresh, immediately before
+ * transitioning, inside the same mutate() callback that applies this
+ * selector's targets — see that function's own header for why.
+ */
+function selectQuarantineAutoResolveTargets(jobs, now, thresholdMs) {
+  const targets = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'quarantined') continue;
+    if ((j.quarantineResolveAttempts ?? 0) >= QUARANTINE_RESOLVE_CAP) continue;
+    const entry = (j.statusHistory || []).find((h) => h.to === 'quarantined');
+    if (!entry) continue;
+    const since = Date.parse(entry.at);
+    if (Number.isNaN(since)) continue;
+    const ageMs = now - since;
+    if (ageMs < thresholdMs) continue;
+    targets.push({ slug: j.slug, cwd: j.cwd ?? null, ageMs });
+  }
+  return targets;
+}
+
+/**
+ * autoResolveQuarantine(job, ageMs) → Promise<'skipped'|null>
+ *
+ * Applies the bounded automatic exit to a single quarantined row (mutates in
+ * place; calls transitionJob + appendAuditEvent) — extracted so it's
+ * unit-testable without going through mutate()/queue.json IO, same shape as
+ * applyNeedsReviewAutoResolve above.
+ *
+ * Re-validates status + the attempts cap itself (race guard, mirrors the
+ * other auto-resolve loops in the 10-minute interval body), THEN re-reads the
+ * PRD file's createdVia frontmatter fresh from disk before doing anything
+ * else. That ordering is load-bearing: reconcile()'s adopt path (the only
+ * OTHER route off 'quarantined') promotes a row to 'pending' the instant it
+ * observes a createdVia stamp, on its own independent pass — if this
+ * function trusted a snapshot taken before its own turn to run, it could
+ * transition a row to 'skipped' the same tick reconcile() already adopted it
+ * to 'pending', silently discarding a PRD a human just fixed. Checking here,
+ * immediately before the transition, inside the caller's mutate() callback,
+ * closes that window.
+ *
+ * A PRD file that cannot be found or parsed at all is treated as still
+ * lacking provenance — there is no proof it has one, and stalling forever on
+ * an unreadable file would defeat the point of a bounded exit (same
+ * can't-prove-it/don't-guess-but-don't-stall posture as the rest of this
+ * file's stale-row detectors).
+ */
+async function autoResolveQuarantine(job, ageMs) {
+  if (!job || job.status !== 'quarantined') return null;
+  if ((job.quarantineResolveAttempts ?? 0) >= QUARANTINE_RESOLVE_CAP) return null;
+
+  let createdVia = null;
+  try {
+    const resolvedDir = await findPrdDir(job.slug);
+    const prdPath = resolvedDir ? path.join(resolvedDir, `${job.slug}.md`) : prdPathForJob(job);
+    const parsed = await parsePrd(prdPath);
+    createdVia = parsed.createdVia ?? null;
+  } catch { /* unreadable/gone — no provenance found, so it stays "lacking" */ }
+  if (createdVia) return null; // reconcile()'s own adopt path owns this row now
+
+  const attempt = (job.quarantineResolveAttempts ?? 0) + 1;
+  job.quarantineResolveAttempts = attempt;
+  job.error = `quarantined without createdVia provenance past the ${Math.round(QUARANTINE_ESCALATE_MS / 3_600_000)}h `
+    + 'escalation window — auto-resolved to skipped';
+  transitionJob(job, 'skipped', {
+    reason: 'quarantined without createdVia provenance past escalation window',
+    source: 'autoResolveQuarantine',
+  });
+  appendAuditEvent('quarantine_auto_resolved', { slug: job.slug, cwd: job.cwd ?? null, ageMs: ageMs ?? null, attempt });
+  return 'skipped';
+}
+
 /**
  * findOverrunningJobs(jobs, now, { factor, floorMs }) → [{ slug, cwd, estimateMinutes, ranMs, ratio }]
  *
@@ -2559,6 +2661,19 @@ function applyPauseCleared(wasPaused, token) {
   return token;
 }
 
+/**
+ * Human-readable explanation for a `reason: 'load-deferred'` tick, surfaced
+ * to the renderer via lastTick.detail. Names the gate, the measured ratio,
+ * the threshold and how long the stretch has been held — the box could sit
+ * gated for 80+ minutes with nothing in the UI naming why (PRD: load gate
+ * hysteresis). Pure so it's unit-testable without driving tickQueue's full
+ * fs/worktree machinery.
+ */
+function formatLoadGateDetail(load) {
+  const heldMinutes = Math.round(load.gatedSinceMs / 60_000);
+  return `CPU load gate: loadavg1 ${load.loadavg1} / ${load.cores} cores = ${load.ratio} > threshold ${load.threshold}, held for ${heldMinutes}m`;
+}
+
 function attachWindow(w) { mainWindow = w; }
 
 /**
@@ -6714,7 +6829,7 @@ function tickQueue({ bypassLoadGate = false } = {}) {
       }
       return recordTick(
         { fired: false, reason: 'load-deferred', deferredCount: gatedBatch.length, ratio: load.ratio, threshold: load.threshold },
-        { detail: `load gate: ${load.loadavg1} / ${load.cores} cores = ${load.ratio} > ${load.threshold}`, holds },
+        { detail: formatLoadGateDetail(load), holds },
       );
     }
 
@@ -6860,43 +6975,107 @@ function classifyQueueStarvation({ jobs, paused, runningCount, lastRunAtMs, now,
 }
 
 /**
- * The watchdog half: acts on classifyQueueStarvation. Called from the
- * heartbeat, which already runs on its own timer independent of the billing
- * poll loop — so a wedged or never-succeeding poll (the /api/oauth/usage
- * endpoint was itself 429ing all of 2026-09-05) can no longer leave a queue
- * with ready work idle indefinitely.
+ * classifyQueueStarvationByProject({ jobs, paused, runningSet, lastRunAtMs, now, thresholdMs })
+ *   → [{ cwd, kind: 'starved' | 'blocked', pending, dispatchable, blockedChains, idleMs }]
+ *
+ * Per-project driver around classifyQueueStarvation's pure single-project
+ * core. `runningCount > 0` inside that core used to be fed the MACHINE-WIDE
+ * `runningSet.size`, which meant one long-lived job in ANY project disarmed
+ * the watchdog for EVERY other project on the box — observed live
+ * 2026-09-12: a job in starry-night-ships ran 80+ minutes while two other
+ * projects sat starved/blocked for hours, and the watchdog never fired once
+ * because "work is flowing" was true somewhere else. Partitioning by cwd
+ * (the same grouping computeBlockedChains already does) fixes DETECTION only
+ * — the idle clock (`lastRunAtMs`) stays machine-wide, since
+ * `lastDispatchAttemptAt` is machine-level state, and only one tick is ever
+ * forced per watchdog pass regardless of how many cwds are starved.
+ *
+ * Pure, no IO. Returns [] when paused (a DECISION, not a stall) or when no
+ * project has a verdict.
+ */
+function classifyQueueStarvationByProject({ jobs, paused, runningSet: runningSlugs, lastRunAtMs, now, thresholdMs = QUEUE_STARVATION_MS } = {}) {
+  if (paused) return [];
+  const rows = (Array.isArray(jobs) ? jobs : []).filter(Boolean);
+  const byCwd = new Map();
+  for (const j of rows) {
+    const key = j.cwd || '(unknown)';
+    if (!byCwd.has(key)) byCwd.set(key, []);
+    byCwd.get(key).push(j);
+  }
+
+  const verdicts = [];
+  for (const [cwd, projectJobs] of byCwd) {
+    // Same source of truth tickQueue itself uses for "is anything running":
+    // the in-process runningSet OR a row already stamped status:'running'.
+    const projRunningCount = projectJobs.filter(
+      (j) => j.status === 'running' || runningSlugs?.has?.(j.slug),
+    ).length;
+    const verdict = classifyQueueStarvation({
+      jobs: projectJobs,
+      paused: false,
+      runningCount: projRunningCount,
+      lastRunAtMs,
+      now,
+      thresholdMs,
+    });
+    if (verdict) verdicts.push({ cwd, ...verdict });
+  }
+  return verdicts;
+}
+
+/**
+ * The watchdog half: acts on classifyQueueStarvationByProject. Called from
+ * the heartbeat, which already runs on its own timer independent of the
+ * billing poll loop — so a wedged or never-succeeding poll (the
+ * /api/oauth/usage endpoint was itself 429ing all of 2026-09-05) can no
+ * longer leave a queue with ready work idle indefinitely.
+ *
+ * Logs and audits one event PER starved/blocked cwd (each carrying that
+ * cwd), but still forces at most one machine-wide tickQueue() per pass —
+ * the tick itself is machine-wide (it drives whatever the picker finds
+ * across every project), only the DETECTION is per-project.
  */
 async function runQueueStarvationWatchdog(state, { now = Date.now(), thresholdMs = QUEUE_STARVATION_MS } = {}) {
   // lastDispatchAttemptAt, not lastRunAt: the latter only advances when a
   // batch actually launches, so a poll that keeps succeeding while dispatch
   // itself never gets invoked would otherwise mask a stall behind a fresh-
   // looking timestamp that was never actually tracking dispatch liveness.
-  const verdict = classifyQueueStarvation({
+  const verdicts = classifyQueueStarvationByProject({
     jobs: state?.jobs,
     paused: state?.paused,
-    runningCount: runningSet.size,
+    runningSet,
     lastRunAtMs: Date.parse(state?.lastDispatchAttemptAt ?? ''),
     now,
     thresholdMs,
   });
-  if (!verdict) return null;
+  if (verdicts.length === 0) return null;
+
+  let anyStarved = false;
+  let primary = null;
+  for (const verdict of verdicts) {
+    const mins = Math.round(verdict.idleMs / 60_000);
+    if (verdict.kind === 'blocked') {
+      console.warn(
+        `[scheduler] QUEUE BLOCKED (${verdict.cwd}): ${verdict.pending} pending job(s), 0 running, idle ${mins}m — every ready row is behind a `
+        + `terminal or parked dependency, so ticking cannot help. Blockers: `
+        + verdict.blockedChains.map((c) => `${c.cwd} [${c.blockedBy.join(', ')}]`).join(' · '),
+      );
+      appendAuditEvent('queue_blocked_stall', { cwd: verdict.cwd, pending: verdict.pending, idleMs: verdict.idleMs, chains: verdict.blockedChains });
+      if (!primary) primary = verdict;
+      continue;
+    }
 
-  const mins = Math.round(verdict.idleMs / 60_000);
-  if (verdict.kind === 'blocked') {
     console.warn(
-      `[scheduler] QUEUE BLOCKED: ${verdict.pending} pending job(s), 0 running, idle ${mins}m — every ready row is behind a `
-      + `terminal or parked dependency, so ticking cannot help. Blockers: `
-      + verdict.blockedChains.map((c) => `${c.cwd} [${c.blockedBy.join(', ')}]`).join(' · '),
+      `[scheduler] QUEUE STARVED (${verdict.cwd}): ${verdict.dispatchable} dispatchable job(s) of ${verdict.pending} pending, 0 running, `
+      + `idle ${mins}m (>= ${Math.round(thresholdMs / 60_000)}m) — forcing a tick`,
     );
-    appendAuditEvent('queue_blocked_stall', { pending: verdict.pending, idleMs: verdict.idleMs, chains: verdict.blockedChains });
-    return verdict;
+    appendAuditEvent('queue_starvation_forced_tick', { cwd: verdict.cwd, pending: verdict.pending, dispatchable: verdict.dispatchable, idleMs: verdict.idleMs });
+    anyStarved = true;
+    primary = verdict;
   }
 
-  console.warn(
-    `[scheduler] QUEUE STARVED: ${verdict.dispatchable} dispatchable job(s) of ${verdict.pending} pending, 0 running, `
-    + `idle ${mins}m (>= ${Math.round(thresholdMs / 60_000)}m) — forcing a tick`,
-  );
-  appendAuditEvent('queue_starvation_forced_tick', { pending: verdict.pending, dispatchable: verdict.dispatchable, idleMs: verdict.idleMs });
+  if (!anyStarved) return primary;
+
   // A never-populated utilization reading is itself one of the ways the
   // when-available path silently never fires (maybeLaunchWhenAvailable
   // returns early on null). Treat unknown as safe here, exactly as the
@@ -6911,8 +7090,76 @@ async function runQueueStarvationWatchdog(state, { now = Date.now(), thresholdMs
   // actually ticked. The watchdog is the last line of defence against a
   // wedged dispatcher, so it must be able to un-wedge this too.
   cancelToken.cancelled = false;
+  // A forced tick is machine-wide by construction (the picker considers
+  // every project's rows) — one call here services every starved cwd found
+  // this pass, not one call per cwd.
   await tickQueue({ bypassLoadGate: false }).catch((e) => console.error('[scheduler] starvation tick error', e));
-  return verdict;
+  return primary;
+}
+
+// One-shot latch, keyed per cwd, for the starve-escalation consequence below —
+// never escalate the same starve stretch twice. Cleared the moment that cwd
+// stops appearing in findStarvedProjects at all (dispatched, or the machine
+// went idle/paused), mirroring the heartbeat's stallSince/stallToasted pair.
+const starveEscalated = new Set();
+
+/**
+ * selectStarveEscalations(starvedProjects, escalatedCwds, thresholdMs)
+ *   → { toEscalate: [...sp], toClear: [cwd, ...] }
+ *
+ * Pure. `starvedProjects` is this sweep's findStarvedProjects() output (the
+ * per-cwd STARVED verdict — cwd, pendingCount, oldestPendingSlug, ageMs);
+ * `escalatedCwds` is the Set already latched from a prior sweep.
+ *
+ * toEscalate: rows crossing thresholdMs for the FIRST time this stretch —
+ * i.e. old enough AND not already latched.
+ * toClear: previously-latched cwds no longer reported as starved at all this
+ * sweep, so a LATER starve on that project escalates again instead of being
+ * silently suppressed forever by a stale latch.
+ */
+function selectStarveEscalations(starvedProjects, escalatedCwds, thresholdMs = STARVE_ESCALATION_MS) {
+  const stillStarved = new Set(starvedProjects.map((sp) => sp.cwd));
+  const toClear = [...escalatedCwds].filter((cwd) => !stillStarved.has(cwd));
+  const toEscalate = starvedProjects.filter((sp) => sp.ageMs >= thresholdMs && !escalatedCwds.has(sp.cwd));
+  return { toEscalate, toClear };
+}
+
+/**
+ * runStarveEscalationSweep(starvedProjects) — acts on selectStarveEscalations'
+ * verdict: audits a DISTINCT 'project_starve_escalated' event (once per starve
+ * stretch, per cwd) and pushes the same toast-channel error the heartbeat's
+ * stall detector already uses ('schedule:stall' → renderer toast.error), so a
+ * starve that has gone on long enough to matter is visible without grepping
+ * the audit log. The hold reason is read from `lastTick` (recordTick's own
+ * last-computed outcome) — never re-evaluated here, so this can never
+ * disagree with what actually happened on the last tick.
+ *
+ * Escalation only: never mutates a job, never dispatches, never bypasses a
+ * gate. Exported for direct unit testing (attach a fake window via
+ * attachWindow() first to assert the toast send).
+ */
+function runStarveEscalationSweep(starvedProjects) {
+  const { toEscalate, toClear } = selectStarveEscalations(starvedProjects, starveEscalated, STARVE_ESCALATION_MS);
+  for (const cwd of toClear) starveEscalated.delete(cwd);
+  for (const sp of toEscalate) {
+    starveEscalated.add(sp.cwd);
+    const holdReason = lastTick?.reason ?? 'unknown';
+    const mins = Math.round(sp.ageMs / 60_000);
+    console.error(
+      `[scheduler] PROJECT STARVE ESCALATED: project=${sp.cwd} pending=${sp.pendingCount} oldest=${sp.oldestPendingSlug} `
+      + `waiting=${mins}m (>= ${Math.round(STARVE_ESCALATION_MS / 60_000)}m escalation threshold), hold reason=${holdReason} — `
+      + 'a bounded escalation only; nothing was auto-reset, cancelled, or dispatched',
+    );
+    appendAuditEvent('project_starve_escalated', {
+      cwd: sp.cwd, pendingCount: sp.pendingCount, oldestPendingSlug: sp.oldestPendingSlug, ageMs: sp.ageMs, holdReason,
+    });
+    sendIfAlive(mainWindow, 'schedule:stall', {
+      message: `Project starved: ${sp.cwd} has ${sp.pendingCount} pending PRD(s), oldest waiting ~${mins}m `
+        + `(hold reason: ${holdReason}) — check the Scheduler tab.`,
+      total: sp.pendingCount,
+      byProject: { [sp.cwd]: { starved: sp.pendingCount } },
+    });
+  }
 }
 
 // ---------- dead-process reaper ----------
@@ -7781,24 +8028,90 @@ function stuckFailedEscalationDisabled() {
   return process.env.SM_STUCK_FAILED_ESCALATE_DISABLE === '1';
 }
 
+// Bounded automatic failed -> pending recovery (PRD 1151): a failed row gets
+// up to FAILED_AUTORESET_CAP auto-reset attempts, each gated on having sat
+// `failed` for FAILED_AUTORESET_MS, before the stuck-failed escalation below
+// is allowed to page a human. Same env-override shape as
+// STUCK_FAILED_ESCALATE_MS/QUARANTINE_ESCALATE_MS above.
+const FAILED_AUTORESET_CAP = 3;
+const FAILED_AUTORESET_MS = process.env.SM_FAILED_AUTORESET_MINUTES
+  ? Number(process.env.SM_FAILED_AUTORESET_MINUTES) * 60_000
+  : 10 * 60_000;
+
+/**
+ * Kill-switch gate for the failed-autoreset pass below
+ * (SM_FAILED_AUTORESET_DISABLE=1), same shape as stuckFailedEscalationDisabled
+ * above.
+ */
+function failedAutoResetDisabled() {
+  return process.env.SM_FAILED_AUTORESET_DISABLE === '1';
+}
+
+/**
+ * selectFailedAutoResetTargets(jobs, now, thresholdMs) →
+ *   [{ slug, cwd, ageMs, attempts }]
+ *
+ * Pure selector — no IO, no `require` inside the function. Selects `failed`
+ * rows whose newest statusHistory entry with `to === 'failed'` is older than
+ * `thresholdMs` and whose failedAutoResetAttempts counter hasn't yet spent
+ * FAILED_AUTORESET_CAP attempts. "Newest" (not first) matters because a row
+ * can have failed more than once across its lifetime (an earlier auto-reset
+ * attempt that itself failed again) — only the most recent failed-since
+ * timestamp should gate the next attempt.
+ *
+ * Excludes a row whose newest failed-entry came from spawnJob:fail-dirty —
+ * that source means a transient failure left genuinely uncommitted work in
+ * the job's worktree and the system already decided once, deliberately, not
+ * to auto-requeue it (see that call site's own comment: "could discard
+ * uncommitted work left by the failed run"). This bounded auto-reset is a
+ * different, slower mechanism and must not quietly override that decision
+ * 10 minutes later — a human should look at a dirty worktree before it gets
+ * re-driven.
+ */
+function selectFailedAutoResetTargets(jobs, now, thresholdMs) {
+  const targets = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'failed') continue;
+    const attempts = j.failedAutoResetAttempts ?? 0;
+    if (attempts >= FAILED_AUTORESET_CAP) continue;
+    const history = j.statusHistory || [];
+    let entry = null;
+    for (let i = history.length - 1; i >= 0; i--) {
+      if (history[i].to === 'failed') { entry = history[i]; break; }
+    }
+    if (!entry) continue;
+    if (entry.source === 'spawnJob:fail-dirty') continue;
+    const since = Date.parse(entry.at);
+    if (Number.isNaN(since)) continue;
+    const ageMs = now - since;
+    if (ageMs < thresholdMs) continue;
+    targets.push({ slug: j.slug, cwd: j.cwd ?? null, ageMs, attempts });
+  }
+  return targets;
+}
+
 /**
  * findStuckFailedJobs(jobs, now, thresholdMs) → [{ slug, cwd, ageMs }]
  *
  * Pure predicate (isRescanCandidate's own resolveRunId/classifyRunOutcome log
  * read is the only IO, gated per-job exactly like shouldRunPeriodicReverify
- * above). `failed` is a fully terminal state for every automated recovery
- * path — selectResumeRecoveryTarget/selectAutoFixTargets both require
- * needs_review, reapDeadRunningJobs only ever writes running → failed, and
- * reconcile-repair's to-pending is for structurally invalid rows. Only a
- * human's scheduler_reset_job ever takes failed → pending (LEGAL_TRANSITIONS).
- * A rescan candidate (isRescanCandidate) that has sat failed longer than
- * `thresholdMs` can therefore go silently stuck forever — job
- * 4056-outcome-stats sat `failed` for five days with no operator signal
- * (reported 2026-09-10, social-signals-trader) even though the periodic
- * reverify pass (once shouldRunPeriodicReverify's guard was fixed) WAS firing
- * on it — reverifyNeedsReview's failed branch can annotate looksDone but can
- * never resolve a failed row itself (see its own header). This is the
- * visibility half that guard fix was missing: escalate once, never requeue.
+ * above). `failed` used to be a fully terminal state for every automated
+ * recovery path — selectResumeRecoveryTarget/selectAutoFixTargets both
+ * require needs_review, reapDeadRunningJobs only ever writes running →
+ * failed, and reconcile-repair's to-pending is for structurally invalid rows.
+ * That is no longer true: selectFailedAutoResetTargets above now drives a
+ * bounded failed → pending auto-reset (LEGAL_TRANSITIONS already allowed the
+ * edge). This escalation now only fires once that auto-reset budget is
+ * genuinely spent (see the interval body's filter on failedAutoResetAttempts)
+ * — a rescan candidate (isRescanCandidate) that has sat failed longer than
+ * `thresholdMs` AND exhausted its auto-reset attempts can therefore go
+ * silently stuck forever — job 4056-outcome-stats sat `failed` for five days
+ * with no operator signal (reported 2026-09-10, social-signals-trader) even
+ * though the periodic reverify pass (once shouldRunPeriodicReverify's guard
+ * was fixed) WAS firing on it — reverifyNeedsReview's failed branch can
+ * annotate looksDone but can never resolve a failed row itself (see its own
+ * header). This is the visibility half that guard fix was missing: escalate
+ * once per exhausted row, never requeue from here.
  *
  * `stuckFailedNotified` gates this to exactly once per row — once the caller
  * stamps it, this always excludes that row so a human is never re-paged on
@@ -7812,7 +8125,18 @@ function findStuckFailedJobs(jobs, now, thresholdMs) {
     if (j.status !== 'failed') continue;
     if (j.stuckFailedNotified === true) continue;
     if (!isRescanCandidate(j)) continue;
-    const entry = (j.statusHistory || []).find((h) => h.to === 'failed');
+    // Newest (not first) to === 'failed' entry — same rationale as
+    // selectFailedAutoResetTargets above: a row can have failed more than
+    // once across its lifetime (an earlier auto-reset attempt that itself
+    // failed again), and only the CURRENT failure episode's age should gate
+    // escalation. Using the first/oldest entry would report a stale age
+    // (and become instantly escalation-eligible) for a row that failed
+    // months ago, recovered, and has only just failed again.
+    const history = j.statusHistory || [];
+    let entry = null;
+    for (let i = history.length - 1; i >= 0; i--) {
+      if (history[i].to === 'failed') { entry = history[i]; break; }
+    }
     if (!entry) continue;
     const since = Date.parse(entry.at);
     if (Number.isNaN(since)) continue;
@@ -7822,6 +8146,123 @@ function findStuckFailedJobs(jobs, now, thresholdMs) {
   return stuck;
 }
 
+// Bounded automatic terminal decision for an EXHAUSTED needs_review row
+// (isExhaustedAutoFix === true — auto-fix attempted, no plan produced,
+// retries spent): up to NEEDS_REVIEW_RESOLVE_CAP requeue attempts (a
+// needs_review -> pending -> ... -> needs_review round trip counts as one
+// spent attempt), each gated on having sat exhausted-needs_review for
+// NEEDS_REVIEW_RESOLVE_MS, before the row is auto-skipped so a `dependsOn`
+// chain behind it always drains without an operator. Same env-override
+// shape as FAILED_AUTORESET_MS above.
+const NEEDS_REVIEW_RESOLVE_CAP = 2;
+const NEEDS_REVIEW_RESOLVE_MS = process.env.SM_NEEDS_REVIEW_RESOLVE_MINUTES
+  ? Number(process.env.SM_NEEDS_REVIEW_RESOLVE_MINUTES) * 60_000
+  : 30 * 60_000;
+
+/**
+ * Kill-switch gate for the needs_review auto-resolve pass below
+ * (SM_NEEDS_REVIEW_AUTORESOLVE_DISABLE=1), same shape as
+ * failedAutoResetDisabled/stuckFailedEscalationDisabled above.
+ */
+function needsReviewAutoResolveDisabled() {
+  return process.env.SM_NEEDS_REVIEW_AUTORESOLVE_DISABLE === '1';
+}
+
+/**
+ * selectExhaustedNeedsReviewTargets(jobs, now, thresholdMs) →
+ *   [{ slug, cwd, ageMs, attempts }]
+ *
+ * Pure selector — no IO. Selects `needs_review` rows whose auto-fix path is
+ * genuinely spent (isExhaustedAutoFix), whose newest statusHistory entry
+ * with `to === 'needs_review'` is older than `thresholdMs`, and whose
+ * exhaustedResolveAttempts counter has not yet spent its cap.
+ *
+ * The inclusion bound is inclusive of the cap itself (`<= CAP`, not `<
+ * CAP`): NEEDS_REVIEW_RESOLVE_CAP counts REQUEUE attempts already spent, and
+ * the pass that observes attempts === CAP is exactly the one that must fire
+ * the terminal skip (see the interval body's branch below) — excluding that
+ * row here would mean the cap-exhausted row is never selected again and the
+ * dependsOn chain behind it never drains, defeating this PRD's own purpose.
+ */
+function selectExhaustedNeedsReviewTargets(jobs, now, thresholdMs) {
+  const targets = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'needs_review') continue;
+    if (!isExhaustedAutoFix(j)) continue;
+    if ((j.exhaustedResolveAttempts ?? 0) > NEEDS_REVIEW_RESOLVE_CAP) continue;
+    const history = j.statusHistory || [];
+    let entry = null;
+    for (let i = history.length - 1; i >= 0; i--) {
+      if (history[i].to === 'needs_review') { entry = history[i]; break; }
+    }
+    if (!entry) continue;
+    const since = Date.parse(entry.at);
+    if (Number.isNaN(since)) continue;
+    const ageMs = now - since;
+    if (ageMs < thresholdMs) continue;
+    targets.push({ slug: j.slug, cwd: j.cwd ?? null, ageMs, attempts: j.exhaustedResolveAttempts ?? 0 });
+  }
+  return targets;
+}
+
+/**
+ * Applies the needs_review auto-resolve decision to a single job (mutates in
+ * place; calls transitionJob + appendAuditEvent). Extracted from the
+ * interval body so the three branches are unit-testable without going
+ * through mutate()/queue.json IO. Order of decision:
+ *   1. job.looksDone (the annotation reverifyNeedsReview writes when a
+ *      later commit touches the PRD's declared paths) -> 'completed'.
+ *   2. otherwise, one more bounded requeue -> 'pending', incrementing
+ *      exhaustedResolveAttempts.
+ *   3. once NEEDS_REVIEW_RESOLVE_CAP requeue attempts are spent -> 'skipped',
+ *      with job.error naming the exhausted path so the Queue UI still shows
+ *      why, and a marker (needsReviewAutoResolvedSkip) that findBlockingDep
+ *      reads to stop treating this SPECIFIC skip as a permanent dependsOn
+ *      block — unlike a generic "PRD source vanished" skip, this row was
+ *      given every bounded chance to resolve itself.
+ * Re-validates status/exhaustion/cap itself (same race-guard shape as the
+ * failed-autoreset loop above) so a stale target computed before this
+ * mutate() pass can never double-apply. Returns the outcome, or null if the
+ * race guard rejected it.
+ */
+function applyNeedsReviewAutoResolve(j) {
+  if (!j || j.status !== 'needs_review' || !isExhaustedAutoFix(j)) return null;
+  if ((j.exhaustedResolveAttempts ?? 0) > NEEDS_REVIEW_RESOLVE_CAP) return null;
+
+  if (j.looksDone) {
+    const attempt = j.exhaustedResolveAttempts ?? 0;
+    transitionJob(j, 'completed', {
+      reason: `needs_review auto-resolve: verifier annotation shows work landed (${j.looksDone.commits.length} commit(s) since this run touch the PRD's declared paths)`,
+      source: 'needsReviewAutoResolve',
+    });
+    appendAuditEvent('needs_review_auto_resolved', { slug: j.slug, cwd: j.cwd ?? null, outcome: 'completed', attempt });
+    return 'completed';
+  }
+
+  const attemptsSoFar = j.exhaustedResolveAttempts ?? 0;
+  if (attemptsSoFar < NEEDS_REVIEW_RESOLVE_CAP) {
+    const attempt = attemptsSoFar + 1;
+    j.exhaustedResolveAttempts = attempt;
+    transitionJob(j, 'pending', {
+      reason: `needs_review auto-resolve: exhausted auto-fix, no completion evidence — requeued for one more run (attempt ${attempt}/${NEEDS_REVIEW_RESOLVE_CAP})`,
+      source: 'needsReviewAutoResolve',
+    });
+    appendAuditEvent('needs_review_auto_resolved', { slug: j.slug, cwd: j.cwd ?? null, outcome: 'requeued', attempt });
+    return 'requeued';
+  }
+
+  j.needsReviewAutoResolvedSkip = true;
+  j.error = `needs_review auto-resolve: exhausted auto-fix path (autoFixOutcome=${j.autoFixOutcome ?? 'none'}, `
+    + `autoFixRetries=${j.autoFixRetries ?? 0}) with no completion evidence after ${NEEDS_REVIEW_RESOLVE_CAP} `
+    + `requeue attempt(s) — auto-skipped to unblock downstream dependsOn rows`;
+  transitionJob(j, 'skipped', {
+    reason: `needs_review auto-resolve: cap exhausted (${NEEDS_REVIEW_RESOLVE_CAP}/${NEEDS_REVIEW_RESOLVE_CAP} requeue attempts) — auto-skipped`,
+    source: 'needsReviewAutoResolve',
+  });
+  appendAuditEvent('needs_review_auto_resolved', { slug: j.slug, cwd: j.cwd ?? null, outcome: 'skipped', attempt: attemptsSoFar });
+  return 'skipped';
+}
+
 /**
  * Self-healing pass over needs_review jobs. The verifier runs in-process, so a
  * fix to runVerify.cjs only takes effect for jobs verified AFTER an app
@@ -8879,7 +9320,8 @@ async function init() {
     // else distinguishes "no pending work" from "pending work, never
     // started" — the 2026-09-01 NN-ordering starvation ran 3.5 h unnoticed.
     // Escalation only, same shape as the quarantine/overrun warnings above.
-    for (const sp of findStarvedProjects(s.jobs, Date.now(), STARVATION_ESCALATE_MS)) {
+    const starvedProjects = findStarvedProjects(s.jobs, Date.now(), STARVATION_ESCALATE_MS);
+    for (const sp of starvedProjects) {
       console.warn(
         `[scheduler] PROJECT STARVED: project=${sp.cwd} pending=${sp.pendingCount} oldest=${sp.oldestPendingSlug} `
         + `waiting=${Math.round(sp.ageMs / 60_000)}m (>= ${Math.round(STARVATION_ESCALATE_MS / 60_000)}m threshold) `
@@ -8887,30 +9329,104 @@ async function init() {
       );
       appendAuditEvent('project_starved', { cwd: sp.cwd, pendingCount: sp.pendingCount, oldestPendingSlug: sp.oldestPendingSlug, ageMs: sp.ageMs });
     }
-
-    // Stuck-failed escalation (2026-09-10, social-signals-trader): see
-    // findStuckFailedJobs' header for why `failed` has no automated way
-    // back to pending. Escalation only, same shape as the three warnings
-    // above — never an automatic failed → pending requeue (that could
-    // discard uncommitted work left by the failed run; see
-    // spawnJob:fail-dirty). Kill-switch: SM_STUCK_FAILED_ESCALATE_DISABLE=1.
-    if (!stuckFailedEscalationDisabled()) {
-      const stuckFailed = findStuckFailedJobs(s.jobs, Date.now(), STUCK_FAILED_ESCALATE_MS);
-      if (stuckFailed.length > 0) {
-        mutate((ms) => {
-          for (const stuck of stuckFailed) {
-            const j = ms.jobs.find((x) => x.slug === stuck.slug);
-            if (!j || j.status !== 'failed' || j.stuckFailedNotified === true) continue; // race guard
-            j.stuckFailedNotified = true;
+    // Bounded, automated consequence for a starve that outlives the WARN
+    // above (PRD: the 2026-09-12 19h Bilko starve had ~115 identical
+    // project_starved rows and zero consequence). STARVE_ESCALATION_MS is
+    // strictly later than STARVATION_ESCALATE_MS, so this only ever fires on
+    // a subset of the rows already reported above — same verdict, no
+    // re-derivation.
+    runStarveEscalationSweep(starvedProjects);
+
+    // Bounded failed -> pending auto-reset (PRD 1151), plus the stuck-failed
+    // escalation now narrowed to only the rows that auto-reset gave up on.
+    // See selectFailedAutoResetTargets' + findStuckFailedJobs' headers.
+    // Computed together, acted on in the SAME mutate(...) pass, so the
+    // stuckFailedNotified race guard below and the auto-reset race guard
+    // above it can never observe two different snapshots of the same row.
+    // Kill-switches: SM_FAILED_AUTORESET_DISABLE=1 / SM_STUCK_FAILED_ESCALATE_DISABLE=1.
+    const autoResetTargets = failedAutoResetDisabled()
+      ? []
+      : selectFailedAutoResetTargets(s.jobs, Date.now(), FAILED_AUTORESET_MS);
+    const stuckFailed = stuckFailedEscalationDisabled()
+      ? []
+      : findStuckFailedJobs(s.jobs, Date.now(), STUCK_FAILED_ESCALATE_MS);
+    // Bounded automatic terminal decision for exhausted needs_review rows
+    // (this PRD): computed alongside the failed-row passes above and acted
+    // on in the SAME mutate(...) pass below, for the same race-guard reason
+    // — a row's exhaustedResolveAttempts counter must never be read from one
+    // snapshot and written from another. Kill-switch: SM_NEEDS_REVIEW_AUTORESOLVE_DISABLE=1.
+    const exhaustedNeedsReviewTargets = needsReviewAutoResolveDisabled()
+      ? []
+      : selectExhaustedNeedsReviewTargets(s.jobs, Date.now(), NEEDS_REVIEW_RESOLVE_MS);
+    // Bounded automatic exit for quarantined rows (this PRD): computed
+    // alongside the passes above and acted on in the SAME mutate(...) pass
+    // below, for the same race-guard reason — quarantineResolveAttempts must
+    // never be read from one snapshot and written from another, and the
+    // createdVia re-check inside autoResolveQuarantine must happen in the
+    // same turn as the transition it gates. Kill-switch:
+    // SM_QUARANTINE_AUTORESOLVE_DISABLE=1.
+    const quarantineTargets = quarantineAutoResolveDisabled()
+      ? []
+      : selectQuarantineAutoResolveTargets(s.jobs, Date.now(), QUARANTINE_ESCALATE_MS);
+    if (autoResetTargets.length > 0 || stuckFailed.length > 0 || exhaustedNeedsReviewTargets.length > 0 || quarantineTargets.length > 0) {
+      mutate(async (ms) => {
+        for (const target of autoResetTargets) {
+          const j = ms.jobs.find((x) => x.slug === target.slug);
+          if (!j || j.status !== 'failed' || (j.failedAutoResetAttempts ?? 0) >= FAILED_AUTORESET_CAP) continue; // race guard
+          const attempt = (j.failedAutoResetAttempts ?? 0) + 1;
+          j.failedAutoResetAttempts = attempt;
+          const reason = `auto-reset after ${Math.round(FAILED_AUTORESET_MS / 60_000)}m failed (attempt ${attempt}/${FAILED_AUTORESET_CAP})`;
+          // resetJobFields is the same field-clearing list the admin
+          // scheduler_reset_job handler uses (ipc:schedule:reset-job) — reuse
+          // it rather than inventing a second list. It also sets job.error to
+          // the reason text passed in; we clear that back to null right
+          // after since this is a clean auto-reset, not a recorded error.
+          if (!resetJobFields(j, reason, { source: 'autoResetFailed' })) continue;
+          j.error = null;
+          delete j.stuckFailedNotified;
+          console.warn(
+            `[scheduler] FAILED PRD AUTO-RESET: project=${j.cwd ?? '(unknown)'} slug=${j.slug} `
+            + `failed=${Math.round(target.ageMs / 60_000)}m (>= ${Math.round(FAILED_AUTORESET_MS / 60_000)}m threshold) — ${reason}`,
+          );
+          appendAuditEvent('job_auto_reset_failed', { slug: j.slug, cwd: j.cwd, ageMs: target.ageMs, attempt });
+        }
+        for (const stuck of stuckFailed) {
+          const j = ms.jobs.find((x) => x.slug === stuck.slug);
+          if (!j || j.status !== 'failed' || j.stuckFailedNotified === true) continue; // race guard
+          // Still has auto-reset attempts left — it will be (or already was,
+          // earlier this same pass) picked up by the loop above instead.
+          // Never log "reset it by hand" for a row that isn't actually stuck.
+          if ((j.failedAutoResetAttempts ?? 0) < FAILED_AUTORESET_CAP) continue;
+          j.stuckFailedNotified = true;
+          console.warn(
+            `[scheduler] FAILED PRD STUCK: project=${stuck.cwd ?? '(unknown)'} slug=${stuck.slug} `
+            + `failed=${Math.round(stuck.ageMs / 3_600_000)}h (>= ${Math.round(STUCK_FAILED_ESCALATE_MS / 3_600_000)}h threshold) — `
+            + `auto-reset cap exhausted (${FAILED_AUTORESET_CAP}/${FAILED_AUTORESET_CAP} attempts); reset it by hand via scheduler_reset_job`,
+          );
+          appendAuditEvent('job_stuck_failed', { slug: stuck.slug, cwd: stuck.cwd, ageMs: stuck.ageMs });
+        }
+        for (const target of exhaustedNeedsReviewTargets) {
+          const j = ms.jobs.find((x) => x.slug === target.slug);
+          const outcome = applyNeedsReviewAutoResolve(j);
+          if (outcome) {
             console.warn(
-              `[scheduler] FAILED PRD STUCK: project=${stuck.cwd ?? '(unknown)'} slug=${stuck.slug} `
-              + `failed=${Math.round(stuck.ageMs / 3_600_000)}h (>= ${Math.round(STUCK_FAILED_ESCALATE_MS / 3_600_000)}h threshold) — `
-              + `no automated recovery reaches a failed row; reset it by hand via scheduler_reset_job`,
+              `[scheduler] NEEDS_REVIEW AUTO-RESOLVE: project=${j.cwd ?? '(unknown)'} slug=${j.slug} `
+              + `exhausted=${Math.round(target.ageMs / 60_000)}m (>= ${Math.round(NEEDS_REVIEW_RESOLVE_MS / 60_000)}m threshold) — outcome=${outcome}`,
             );
-            appendAuditEvent('job_stuck_failed', { slug: stuck.slug, cwd: stuck.cwd, ageMs: stuck.ageMs });
           }
-        }).catch(() => {});
-      }
+        }
+        for (const target of quarantineTargets) {
+          const j = ms.jobs.find((x) => x.slug === target.slug);
+          if (!j || j.status !== 'quarantined' || (j.quarantineResolveAttempts ?? 0) >= QUARANTINE_RESOLVE_CAP) continue; // race guard
+          const outcome = await autoResolveQuarantine(j, target.ageMs);
+          if (outcome) {
+            console.warn(
+              `[scheduler] QUARANTINED PRD AUTO-RESOLVE: project=${j.cwd ?? '(unknown)'} slug=${j.slug} `
+              + `age=${Math.round(target.ageMs / 3_600_000)}h (>= ${Math.round(QUARANTINE_ESCALATE_MS / 3_600_000)}h threshold) — outcome=${outcome}`,
+            );
+          }
+        }
+      }).catch(() => {});
     }
   }, 10 * 60_000);
 
@@ -9575,8 +10091,12 @@ function registerAdminRoutes(adminHttp, remoteObj = remote) {
 
 module.exports = {
   classifyQueueStarvation,
+  classifyQueueStarvationByProject,
   runQueueStarvationWatchdog,
   QUEUE_STARVATION_MS,
+  selectStarveEscalations,
+  runStarveEscalationSweep,
+  STARVE_ESCALATION_MS,
   computeBlockedChains,
   stripAppOwnedChurn,
   findOverrunningJobs,
@@ -9616,6 +10136,15 @@ module.exports = {
   findStuckFailedJobs,
   STUCK_FAILED_ESCALATE_MS,
   stuckFailedEscalationDisabled,
+  selectFailedAutoResetTargets,
+  FAILED_AUTORESET_CAP,
+  FAILED_AUTORESET_MS,
+  failedAutoResetDisabled,
+  selectExhaustedNeedsReviewTargets,
+  applyNeedsReviewAutoResolve,
+  NEEDS_REVIEW_RESOLVE_CAP,
+  NEEDS_REVIEW_RESOLVE_MS,
+  needsReviewAutoResolveDisabled,
   isRescanCandidate,
   isFailedUnverifiedShaped,
   computeLooksDone,
@@ -9646,6 +10175,7 @@ module.exports = {
   MAX_INVESTIGATION_DEPTH,
   forceTickOutcome,
   applyPauseCleared,
+  formatLoadGateDetail,
   detectNetworkErrorInLog,
   detectRateLimitInLog,
   classifyFailureOutcome,
@@ -9695,6 +10225,10 @@ module.exports = {
   computeStallSummary,
   findStaleQuarantinedJobs,
   QUARANTINE_ESCALATE_MS,
+  selectQuarantineAutoResolveTargets,
+  autoResolveQuarantine,
+  QUARANTINE_RESOLVE_CAP,
+  quarantineAutoResolveDisabled,
   applyClearQueueVictims,
   PIDLESS_SPAWN_GRACE_MS,
   findStrandedInvestigations,