claude-code-session-manager 0.83.0 → 0.84.0

package/src/main/scheduler.cjs

@@ -143,6 +143,9 @@ function resolveOriginSessionId(cwd, epicId) {
 const sessionSlots = require('./lib/sessionSlots.cjs');
 const quietMachineLease = require('./lib/quietMachineLease.cjs');
 const jobWorktree = require('./lib/jobWorktree.cjs');
+const gitWorktree = require('./lib/gitWorktree.cjs');
+const { buildJobWorktreeIsLive } = require('./lib/jobWorktreeBootLive.cjs');
+const { buildTerminalOrphanIsLive } = require('./lib/jobWorktreeTerminalOrphanLive.cjs');
 const { reconcileEpicWorktreesOnBoot } = require('./lib/epicWorktreeBoot.cjs');
 const queueStore = require('./lib/queueStore.cjs');
 const { splitFrontmatter, parsePrdFile, serializePrdFile } = require('./lib/prdFrontmatter.cjs');
@@ -1294,6 +1297,7 @@ function loadSchedulerState() {
     if (typeof s.backoffMs === 'number') backoffMs = s.backoffMs;
     if (typeof s.pauseClearedManuallyAt === 'number') pauseClearedManuallyAt = s.pauseClearedManuallyAt;
     if (typeof s.lastPollAt === 'number') lastPollAt = s.lastPollAt;
+    if (typeof s.failureStreakWarned === 'boolean') failureStreakWarned = s.failureStreakWarned;
   } catch { /* first boot or corrupt — start fresh */ }
 }
 
@@ -1313,6 +1317,7 @@ function persistSchedulerState() {
       pausedReason: null,
       pausedSince: null,
       pauseClearedManuallyAt,
+      failureStreakWarned,
     });
   } catch (e) {
     console.warn('[scheduler] failed to persist scheduler state', e?.message);
@@ -1629,14 +1634,16 @@ function findStrandedInvestigations(jobs, now, maxMs, isAlive = claudePidAlive)
 // (The single-file EMPTY_QUEUE/shapeQueue readers were retired with the
 // global queue.json — queueStore.cjs's merged readers own the shape now.)
 
-// Quarantine a corrupt shard alongside itself (once per process — the first
-// copy is the one that matters; later ticks would just overwrite it with the
-// same bytes) so a human can diff it against the .bak-* snapshots.
-let quarantined = false;
+// Quarantine a corrupt shard alongside itself (once per PATH, not once per
+// process — a module-level boolean let one file's tear consume the latch and
+// silently swallow every other file's `.corrupt-*` copy for the rest of the
+// process's life, observed live 2026-09-11) so a human can diff it against
+// the .bak-* snapshots.
+const quarantinedPaths = new Set();
 function flagUnreadable(state) {
   if (!state.unreadable) return state;
-  if (!quarantined && state.unreadablePath) {
-    quarantined = true;
+  if (state.unreadablePath && !quarantinedPaths.has(state.unreadablePath)) {
+    quarantinedPaths.add(state.unreadablePath);
     try {
       fs.copyFileSync(state.unreadablePath, `${state.unreadablePath}.corrupt-${Date.now()}`);
     } catch { /* best-effort: the read already failed, the copy may too */ }
@@ -2411,6 +2418,48 @@ let firstFailureAt = null;
 let firstNon429FailureAt = null; // tracks only transient/config failures; 429s don't count toward network-pause threshold
 let lastFailureKind = null; // 'transient' | 'meter_rate_limited' | 'auth' | null
 let pauseClearedManuallyAt = null;
+// PRD: the usage-poller silent-failure-streak WARN is emitted once per streak,
+// not once per failure (57 failures must produce ONE opsErrorLog line, not 57).
+// Reset alongside consecutiveFailures everywhere that resets to 0.
+let failureStreakWarned = false;
+// Ceiling on pollLoop's exponential poll backoff (both the 'transient'/'config'
+// branch and the 'meter_rate_limited' branch below share this cap — a single
+// constant so the two never drift to different ceilings).
+const BACKOFF_MAX_MS = 480_000; // 8 minutes
+// Threshold past which a growing consecutiveFailures streak stops being normal
+// jitter and becomes worth a human's attention. health.cjs imports this so the
+// WARN and the `npm run health` non-GREEN trip at the exact same count.
+const FAILURE_STREAK_WARN_THRESHOLD = 5;
+
+/** Pure: exponential backoff with a cap, shared by every pollLoop failure branch. Exported for unit testing. */
+function nextBackoffMs(prevBackoffMs) {
+  return prevBackoffMs ? Math.min(prevBackoffMs * 2, BACKOFF_MAX_MS) : 30_000;
+}
+
+/**
+ * Pure: should this failure count trigger the one-time streak WARN? Exported
+ * for unit testing. `alreadyWarned` is the current streak's warned flag —
+ * true for every failure after the threshold-crossing one, so this returns
+ * true exactly once per streak.
+ */
+function shouldWarnFailureStreak(consecutiveFailures, alreadyWarned, threshold = FAILURE_STREAK_WARN_THRESHOLD) {
+  return consecutiveFailures >= threshold && !alreadyWarned;
+}
+
+/** Emits the one-time opsErrorLog WARN for a failure streak crossing the threshold, if not already warned this streak. */
+function warnFailureStreakIfNeeded() {
+  if (!shouldWarnFailureStreak(consecutiveFailures, failureStreakWarned)) return;
+  failureStreakWarned = true;
+  try {
+    appendError({
+      cwd: DEFAULT_PROJECT_CWD,
+      scope: 'scheduler',
+      level: 'warn',
+      message: `usage/rate-limit poller has failed ${consecutiveFailures} consecutive times (kind=${lastFailureKind}, backoffMs=${backoffMs}) — see ${SCHEDULER_STATE_PATH}`,
+      meta: { consecutiveFailures, backoffMs, lastFailureKind },
+    });
+  } catch { /* durable logging must never break the poll loop */ }
+}
 // PRD 1119: consecutive-rapid-rate-limit hard-pause tracking, keyed per slug.
 // See isCooldownSuppressed/nextRapidRateLimitCount below for the pure rules.
 const consecutiveRapidRateLimitsBySlug = new Map();
@@ -2783,6 +2832,7 @@ async function clearPause(source) {
     firstFailureAt = null;
     firstNon429FailureAt = null;
     lastFailureKind = null;
+    failureStreakWarned = false;
     persistSchedulerState();
   }
   if (wasPaused) await broadcast({ flush: true });
@@ -2836,6 +2886,9 @@ function resetJobFields(job, errorMsg, opts = {}) {
   // human-driven reset must genuinely start the auto-fix budget over,
   // including the one-time dead-fix-plan-child reopen (PRD 1129).
   delete job.autoFixReopened;
+  // Same category — an overrun badge is this run's outcome (see
+  // findOverrunningJobs's header), never durable across a reset.
+  delete job.overrun;
   // Like exitCode: this run's outcome, not durable across a reset — a stale
   // leak badge from a prior attempt must not linger once the job re-fires.
   delete job.leakedDescendants;
@@ -3140,6 +3193,7 @@ async function notifyOriginatingTab(job, {
         await appendTranscriptTurn(job.cwd, epicIdForTranscript, {
           role: 'assistant',
           text: resultText || message,
+          eventId: `prd-result:${job.slug}:${job.runId || ''}`,
         });
       } catch (e) {
         console.error('[scheduler] notifyOriginatingTab transcript append error', job?.slug, e);
@@ -3997,13 +4051,20 @@ function pickRunDir() {
 /**
  * Execute a single PRD job. Writes stdout/stderr to a log file and a meta
  * JSON sidecar. Accepts an optional onPid(pid) callback called synchronously
- * after spawn so callers can persist the pid before the job finishes.
+ * after spawn so callers can persist the pid before the job finishes, and an
+ * optional onPhase(phase) callback invoked at the top of this function (see
+ * spawnJob's dispatchPhase breadcrumb) — this function has no access to
+ * `mutate`, so the caller injects the stamping side effect instead.
  *
  * Uses withChildAndLog for the child lifecycle (fd open/close, watchdog timers).
  * Watchdogs are declared as an array; the result-tailer's exit-code mapping
  * (success+killedBySignal → 0) is scheduler-specific and lives in onExit.
  */
-async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget = null, foreignWip = null, launchEnv = null) {
+async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget = null, foreignWip = null, launchEnv = null, onPhase = null) {
+  // First statement — stamps the 'exec-entered' dispatch-phase breadcrumb
+  // before openLog below, so a hang inside openLog/spawn itself still shows
+  // execution reached this function (see spawnJob's dispatchPhase comment).
+  if (onPhase) await onPhase('exec-entered');
   const logPath = path.join(runDir, `${job.slug}.log`);
   const metaPath = path.join(runDir, `${job.slug}.meta.json`);
   // `cwd` stays the MAIN tree throughout — PRD lookup (findPrdDir/prdPathForJob)
@@ -4049,6 +4110,12 @@ async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget
 
   let prompt;
   let prdPath = null;
+  // Captured on a successful parsePrd() so the raw, untruncated PRD body can
+  // be recorded as a 'user' turn in the durable per-Epic transcript below —
+  // the durable store must see the actual prompt the executor ran, not just
+  // the scheduler's own short status chip (that's notifyOriginatingTab's job
+  // for the assistant side).
+  let parsedPrdMeta = null;
   if (resumeTarget) {
     // Resume mode (PRD 1111): a short deterministic preamble naming the
     // recorded dirty paths, NEVER the original PRD body — the resumed
@@ -4071,6 +4138,7 @@ async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget
     // resolved — never concatenated here, so it can't end up ahead of the
     // digest in the composed prompt (PRD 992).
     prompt = parsed.body;
+    parsedPrdMeta = parsed;
   } catch (e) {
     // The project-scoped dir isn't the only place a PRD source can live — a
     // writer that hasn't migrated to prdLocations.cjs yet (or a not-yet-run
@@ -4084,6 +4152,7 @@ async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget
         const parsed = await parsePrd(fallbackPath);
         prompt = parsed.body;
         prdPath = fallbackPath;
+        parsedPrdMeta = parsed;
       } catch (e2) {
         // Found the dir a moment ago but the read still failed. Case A: the
         // source has since been archived — stale-skip as usual. Case B: it
@@ -4121,6 +4190,26 @@ async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget
   }
   } // end resumeTarget ? preamble : normal-PRD-read
 
+  // Record the raw, untruncated PRD body as a 'user' turn before any digest/
+  // finish-protocol boilerplate is concatenated onto `prompt` below — mirrors
+  // notifyOriginatingTab's epicId resolution order (sourcePromptId, then
+  // sourceTabId, then job.epicId) so the same Epic's user/assistant turns
+  // land in the same transcript file. Best-effort: never blocks a spawn.
+  if (!resumeTarget && parsedPrdMeta) {
+    const epicIdForPromptTurn = parsedPrdMeta.sourcePromptId || parsedPrdMeta.sourceTabId || job.epicId || null;
+    if (epicIdForPromptTurn && cwd) {
+      try {
+        await promptSessionTranscript.appendTurn(cwd, epicIdForPromptTurn, {
+          role: 'user',
+          text: prompt,
+          eventId: `prd:${job.slug}`,
+        });
+      } catch (e) {
+        safeLog(`[scheduler] transcript append (PRD prompt) failed: ${e?.message ?? e}\n`);
+      }
+    }
+  }
+
   let contextDigestApplied = false;
   let originSessionId = null;
   if (!resumeTarget) {
@@ -5165,10 +5254,30 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
       : await jobWorktree.createJobWorktree({ cwd: job.cwd || defaultCwd, slug: job.slug });
     if (!preflightWorktree.ok && /^worktree cap reached\b/.test(preflightWorktree.reason || '')) {
       console.log(`[scheduler] ${job.slug}: deferring — ${preflightWorktree.reason}`);
-      await mutate((s) => {
+      // A leaked worktree cap count used to strand a job silently — nothing
+      // but this console.log + a heldReason field the human had to go
+      // looking for (see gitWorktree.cjs's reserveWorktreeSlot: the cap
+      // itself now self-heals, but a GENUINE block, while it lasts, must
+      // still reach a durable, visible channel). Fired only the first time
+      // THIS slug gets held on this exact reason — a job stuck for hours
+      // must not flood opsErrorLog with one line per dispatch attempt.
+      const wasAlreadyFlaggedForThisReason = await mutate((s) => {
         const idx = s.jobs.findIndex((x) => x.slug === job.slug);
+        const already = idx >= 0 && s.jobs[idx].heldReason === preflightWorktree.reason;
         if (idx >= 0) s.jobs[idx].heldReason = preflightWorktree.reason;
+        return already;
       });
+      if (!wasAlreadyFlaggedForThisReason) {
+        try {
+          appendError({
+            cwd: job.cwd || defaultCwd,
+            scope: 'scheduler',
+            level: 'error',
+            message: `${job.slug}: dispatch blocked — ${preflightWorktree.reason}`,
+            meta: { slug: job.slug, reason: preflightWorktree.reason },
+          });
+        } catch { /* durable logging must never break the queue */ }
+      }
       await broadcast({ flush: true });
       return;
     }
@@ -5290,6 +5399,14 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
         delete s.jobs[idx].heldReason;
         s.jobs[idx].runId = runId;
         s.jobs[idx].startedAt = new Date().toISOString();
+        // Dispatch-phase breadcrumb (PRD: dispatch-region diagnostic
+        // breadcrumb) — a transient marker of how far THIS dispatch got,
+        // exactly like heldReason: stamped at each step of the
+        // running-transition → executeJob → onPid region and deleted at
+        // finalize (see the two finalize mutates below) so a
+        // completed/failed row never carries a stale one.
+        s.jobs[idx].dispatchPhase = 'running-stamped';
+        s.jobs[idx].dispatchPhaseAt = s.jobs[idx].startedAt;
         dispatchStartedAtMs = Date.parse(s.jobs[idx].startedAt);
         if (job.quietMachine === true) {
           s.jobs[idx].quietMachine = true;
@@ -5308,6 +5425,14 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
     await broadcast({ flush: true });
     if (dispatchSkippedAlreadyCompleted) return;
 
+    await mutate((s) => {
+      const idx = s.jobs.findIndex((x) => x.slug === job.slug);
+      if (idx >= 0) {
+        s.jobs[idx].dispatchPhase = 'pre-run-git-snapshot';
+        s.jobs[idx].dispatchPhaseAt = new Date().toISOString();
+      }
+    });
+
     // Commit-guard baseline: snapshot the working tree BEFORE the run so the
     // post-run check flags only paths THIS job left dirty, not pre-existing WIP.
     const guardCwd = job.cwd || defaultCwd;
@@ -5340,6 +5465,8 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
         s.jobs[idx].guardHeadBefore = guardHeadBefore || null;
         if (preRunDirtyPaths.length) s.jobs[idx].preRunDirtyPaths = preRunDirtyPaths;
         else delete s.jobs[idx].preRunDirtyPaths;
+        s.jobs[idx].dispatchPhase = 'baseline-persisted';
+        s.jobs[idx].dispatchPhaseAt = new Date().toISOString();
       }
     });
 
@@ -5361,17 +5488,21 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
       console.log(`[scheduler] ${job.slug}: isolated in worktree ${worktree.dir} (branch ${worktree.branch})`);
     } else {
       console.log(`[scheduler] ${job.slug}: running in main tree (worktree not used: ${worktree.reason})`);
-      // Surface any degraded-isolation fallback on the job row itself so it's
-      // queryable from the queue instead of console-only — except the
-      // deliberate env-disable flag, which is an intentional opt-out, not a
-      // degradation worth flagging.
-      if (!jobWorktree.isWorktreeDisabled()) {
-        await mutate((s) => {
-          const idx = s.jobs.findIndex((x) => x.slug === job.slug);
-          if (idx >= 0) s.jobs[idx].worktreeFallbackReason = worktree.reason;
-        });
-      }
     }
+    // dispatchPhase stamp folded into a single unconditional mutate covering
+    // both branches above — the degraded-isolation fallback flag (skipped
+    // for the deliberate env-disable opt-out) rides along in the SAME
+    // mutate rather than a second one.
+    await mutate((s) => {
+      const idx = s.jobs.findIndex((x) => x.slug === job.slug);
+      if (idx >= 0) {
+        s.jobs[idx].dispatchPhase = 'worktree-resolved';
+        s.jobs[idx].dispatchPhaseAt = new Date().toISOString();
+        if (!worktree.ok && !jobWorktree.isWorktreeDisabled()) {
+          s.jobs[idx].worktreeFallbackReason = worktree.reason;
+        }
+      }
+    });
     // Base-tree WIP carried into the worktree (createWorktree, PRD 1094) —
     // recorded on the job row so integration can exclude these paths from
     // the branch diff below, and so it's queryable from the queue.
@@ -5422,10 +5553,20 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
           if (idx >= 0) {
             s.jobs[idx].sessionId = sessionId;
             s.jobs[idx].runtime = { pid, runId, startedAt: s.jobs[idx].startedAt, sessionId, cwd };
+            s.jobs[idx].dispatchPhase = 'spawned';
+            s.jobs[idx].dispatchPhaseAt = new Date().toISOString();
           }
         });
         await broadcast({ flush: true });
-      }, worktree.ok ? worktree.dir : undefined, resumeTarget, foreignWip, launchEnv);
+      }, worktree.ok ? worktree.dir : undefined, resumeTarget, foreignWip, launchEnv, async (phase) => {
+        await mutate((s) => {
+          const idx = s.jobs.findIndex((x) => x.slug === job.slug);
+          if (idx >= 0) {
+            s.jobs[idx].dispatchPhase = phase;
+            s.jobs[idx].dispatchPhaseAt = new Date().toISOString();
+          }
+        });
+      });
     } finally {
       if (worktree.ok) {
         worktreeLeftoverDirty = (await uncommittedChanges(worktree.dir)) || [];
@@ -6033,6 +6174,17 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
             delete s.jobs[i2].sharedTreeGuard;
           }
           delete s.jobs[i2].runtime;
+          // Dispatch-phase breadcrumb was only ever meant to cover THIS
+          // dispatch's pre-spawn/spawn region — a terminal row must not
+          // carry a stale one into its next dispatch (unlike guardBaseline's
+          // preRunDirtyPaths sibling, which deliberately survives to
+          // history.jsonl — this has no such carry-forward use).
+          delete s.jobs[i2].dispatchPhase;
+          delete s.jobs[i2].dispatchPhaseAt;
+          // A completed/failed/needs_review row is no longer running — its
+          // overrun badge (if any) was this run's outcome and must not
+          // linger onto whatever the next dispatch of this slug does.
+          delete s.jobs[i2].overrun;
           // Pre-run baseline no longer needed once this run has finalized —
           // its whole purpose (letting THIS finalize compute a truthful
           // delta) is done; a fresh one is captured at the next dispatch.
@@ -6326,6 +6478,78 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
   }
 }
 
+// Throttle for reclaimTerminalJobOrphansThrottled below — a `git worktree
+// list` + per-candidate `status`/`merge-base` call per known project cwd is
+// cheap once, but tickQueue can fire every few seconds; there is no value in
+// re-running this on every single tick.
+const TERMINAL_ORPHAN_RECLAIM_INTERVAL_MS = 10 * 60_000; // 10 minutes
+let lastTerminalOrphanReclaimAt = 0;
+
+/**
+ * Fire-and-forget, throttled sweep for job-kind worktrees whose owning queue
+ * row has already resolved (completed/failed/skipped) but whose checkout is
+ * still on disk — see gitWorktree.cjs's reclaimTerminalJobOrphans header for
+ * the two independent safety proofs it requires before deleting anything,
+ * and jobWorktreeTerminalOrphanLive.cjs for the liveness gate checked BEFORE
+ * those proofs (PRD 1163: neither proof is itself a liveness check, so a
+ * still-running executor behind a prematurely-terminal row could otherwise
+ * satisfy both).
+ *
+ * Runs once per TERMINAL_ORPHAN_RECLAIM_INTERVAL_MS across every project cwd
+ * this tick's queue state knows about. The `/proc` holder scan
+ * (`listCwdHolders`) is computed ONCE per sweep here and reused for every
+ * candidate across every cwd, rather than re-scanning `/proc` per candidate.
+ * Never throws (each per-cwd call is itself never-throws; this wrapper is
+ * just the throttle + fan-out + queue-row stamping).
+ */
+async function reclaimTerminalJobOrphansThrottled(state) {
+  const now = Date.now();
+  if (now - lastTerminalOrphanReclaimAt < TERMINAL_ORPHAN_RECLAIM_INTERVAL_MS) return;
+  lastTerminalOrphanReclaimAt = now;
+
+  const terminalJobsByCwd = new Map();
+  for (const j of state.jobs || []) {
+    if (j.status !== 'completed' && j.status !== 'failed' && j.status !== 'skipped') continue;
+    const cwd = j.cwd || DEFAULT_PROJECT_CWD;
+    if (!terminalJobsByCwd.has(cwd)) terminalJobsByCwd.set(cwd, []);
+    terminalJobsByCwd.get(cwd).push(j);
+  }
+
+  const cwdHolders = gitWorktree.listCwdHolders();
+  // { slug -> reason }, collected across every cwd this sweep touches, so a
+  // single mutate() at the end can stamp every blocked-live row at once.
+  const blockedLive = new Map();
+
+  for (const [cwd, terminalJobs] of terminalJobsByCwd) {
+    const terminalSlugs = new Set(terminalJobs.map((j) => j.slug));
+    const isLive = buildTerminalOrphanIsLive({
+      terminalJobs,
+      claudePidAlive,
+      hasLiveHolder: gitWorktree.hasLiveHolder,
+      cwdHolders,
+      onLive: (slug, reason) => blockedLive.set(slug, reason),
+    });
+    const reclaimed = await jobWorktree.reclaimTerminalJobOrphans({ cwd, terminalSlugs, isLive });
+    if (reclaimed.length) {
+      console.log(`[scheduler] reclaimed ${reclaimed.length} terminal job-worktree orphan(s) in ${cwd}: ${reclaimed.join(', ')}`);
+    }
+  }
+
+  if (blockedLive.size > 0) {
+    const stampedAt = new Date().toISOString();
+    await mutate((s) => {
+      for (const j of s.jobs || []) {
+        if (!blockedLive.has(j.slug)) continue;
+        j.worktreeReclaimBlockedLive = true;
+        j.worktreeReclaimBlockedLiveAt = stampedAt;
+        j.worktreeReclaimBlockedLiveReason = blockedLive.get(j.slug);
+      }
+    }).catch((e) => {
+      console.warn('[scheduler] failed to stamp worktreeReclaimBlockedLive', e?.message);
+    });
+  }
+}
+
 /**
  * Dispatch a resume-recovery attempt (PRD 1111) for a job already found
  * eligible by selectResumeRecoveryTarget. Thin wrapper around spawnJob —
@@ -6365,10 +6589,29 @@ function tickQueue({ bypassLoadGate = false } = {}) {
     }
     if (cancelToken.cancelled) return { fired: false, reason: 'cancelled' };
 
+    // Stamped here — the moment tickQueue actually reaches the picker,
+    // regardless of whether this pass ends in a launch — so
+    // classifyQueueStarvation can tell "the engine keeps evaluating the
+    // queue" apart from "nothing has invoked tickQueue in a long time".
+    // Distinct from `lastRunAt` below, which stays true to its existing
+    // meaning (a batch actually launched) since other readers depend on that.
+    await mutate((s) => { s.lastDispatchAttemptAt = new Date().toISOString(); });
+
     // The retired-flat-dir sweep now lives inside reconcile() itself (see its
     // own comment) so every caller of reconcile — not just this tick — gets
     // the guarantee.
     await reconcile(state);
+    // Reclaim any job-kind worktree whose owning row already resolved
+    // (completed/failed/skipped) without the run ever reaching
+    // cleanupWorktree — a leaked checkout that would otherwise sit until the
+    // stale-age sweep (default 24h) or the next app restart
+    // (reconcileWorktreesOnBoot only runs once, at boot). Throttled to once
+    // per interval (not every tick) since it's a `git worktree list` +
+    // per-candidate git call per known project cwd. Best-effort and
+    // fire-and-forget: must never hold up dispatch.
+    reclaimTerminalJobOrphansThrottled(state).catch((e) => {
+      console.warn('[scheduler] terminal job-worktree orphan reclaim failed', e?.message);
+    });
     // Session-Manager's machine-wide slot pool is the ONLY concurrency limit
     // the picker answers to (plus the memory gate below). The scheduler used
     // to also carry a private `concurrencyCap` of 3 — the exact per-consumer
@@ -6624,11 +6867,15 @@ function classifyQueueStarvation({ jobs, paused, runningCount, lastRunAtMs, now,
  * with ready work idle indefinitely.
  */
 async function runQueueStarvationWatchdog(state, { now = Date.now(), thresholdMs = QUEUE_STARVATION_MS } = {}) {
+  // lastDispatchAttemptAt, not lastRunAt: the latter only advances when a
+  // batch actually launches, so a poll that keeps succeeding while dispatch
+  // itself never gets invoked would otherwise mask a stall behind a fresh-
+  // looking timestamp that was never actually tracking dispatch liveness.
   const verdict = classifyQueueStarvation({
     jobs: state?.jobs,
     paused: state?.paused,
     runningCount: runningSet.size,
-    lastRunAtMs: Date.parse(state?.lastRunAt ?? ''),
+    lastRunAtMs: Date.parse(state?.lastDispatchAttemptAt ?? ''),
     now,
     thresholdMs,
   });
@@ -6655,6 +6902,15 @@ async function runQueueStarvationWatchdog(state, { now = Date.now(), thresholdMs
   // returns early on null). Treat unknown as safe here, exactly as the
   // billing meter's own 429 fallback already does.
   if (cachedUtilization === null || cachedUtilization === undefined) cachedUtilization = 0;
+  // The in-process cancelToken is only ever reset by runDueJobs() (force-tick
+  // / run-now / resume-timer) — every other path that clears a pause
+  // (clearPause(), the poll loop's own auto-recovery) leaves it untouched
+  // (see applyPauseCleared's header for the 2026-07-14 incident this caused).
+  // A watchdog that fires but doesn't clear a stuck cancelToken would forever
+  // hit tickQueue's very first guard and report a forced tick that never
+  // actually ticked. The watchdog is the last line of defence against a
+  // wedged dispatcher, so it must be able to un-wedge this too.
+  cancelToken.cancelled = false;
   await tickQueue({ bypassLoadGate: false }).catch((e) => console.error('[scheduler] starvation tick error', e));
   return verdict;
 }
@@ -6794,8 +7050,17 @@ async function reapDeadRunningJobs() {
       // a true claim when the run dir produced no log output at all; a log
       // with real content proves the job DID run (see
       // resolvePidlessGateOutcome's header).
-      const gateOutcome = pidless ? resolvePidlessGateOutcome(outcome, logHasOutput(logPath)) : mapOutcomeToGateOutcome(outcome);
-      dead.push({ slug, pid, outcome, gateOutcome, pidless, reason, logPath });
+      // A pidless row was stamped with the runId of whatever batch dir
+      // tickQueue handed its dispatch (pickRunDir's header: "tickQueue hands
+      // ONE shared batch dir to every spawnJob in the batch") BEFORE the
+      // spawn that never completed — so that dir may hold nothing of this
+      // slug's own, or only a sibling's `<other-slug>.log` from the same
+      // batch. logHasOutput's own existsSync+size check is exactly "does
+      // THIS slug have any artifact in there" — reused rather than
+      // re-derived so a phantom link never survives the reap.
+      const hasOwnArtifact = pidless ? logHasOutput(logPath) : true;
+      const gateOutcome = pidless ? resolvePidlessGateOutcome(outcome, hasOwnArtifact) : mapOutcomeToGateOutcome(outcome);
+      dead.push({ slug, pid, outcome, gateOutcome, pidless, reason, logPath, noOwnArtifact: pidless && !hasOwnArtifact });
     }
 
     queueHealthSweepCycle += 1;
@@ -6894,7 +7159,7 @@ async function reapDeadRunningJobs() {
     }
 
     await mutate(async (s) => {
-      for (const { slug, pid, outcome, gateOutcome, pidless, reason } of dead) {
+      for (const { slug, pid, outcome, gateOutcome, pidless, reason, noOwnArtifact } of dead) {
         const idx = s.jobs.findIndex((x) => x.slug === slug);
         if (idx < 0 || s.jobs[idx].status !== 'running') continue; // race guard
         const rateLimited = outcome === 'rate_limited';
@@ -6984,7 +7249,22 @@ async function reapDeadRunningJobs() {
           }
           if (landedCommit) s.jobs[idx].landedCommit = landedCommit;
         }
+        // A pidless spawn that never wrote its own '<slug>.log' into the
+        // batch runId dir it was stamped with must not keep that runId —
+        // it points at a directory with no artifacts for THIS slug (at best
+        // empty, at worst only a sibling's log from the same batch — see the
+        // 'dead' push above). resolveRunId's own backfill scan
+        // (existsSync-per-dir on '<slug>.log') would find nothing here
+        // either, so clearing this makes job.runId and resolveRunId(job)
+        // agree: both null, rather than one falsely pointing at a run this
+        // slug never produced.
+        if (!rateLimited && noOwnArtifact) {
+          s.jobs[idx].runId = null;
+        }
         delete s.jobs[idx].runtime;
+        delete s.jobs[idx].dispatchPhase;
+        delete s.jobs[idx].dispatchPhaseAt;
+        delete s.jobs[idx].overrun;
         delete s.jobs[idx].guardBaseline;
         delete s.jobs[idx].guardHeadBefore;
         applyLeftoverFields(s.jobs[idx], deltaPaths);
@@ -7046,6 +7326,7 @@ async function pollLoop() {
       firstFailureAt = null;
       firstNon429FailureAt = null;
       lastFailureKind = null;
+      failureStreakWarned = false;
       lastPollAt = Date.now();
       lastPollOk = true;
       persistSchedulerState();
@@ -7072,6 +7353,7 @@ async function pollLoop() {
       firstFailureAt = null;
       firstNon429FailureAt = null;
       lastFailureKind = null;
+      failureStreakWarned = false;
       lastPollAt = Date.now();
       lastPollOk = true;
       persistSchedulerState();
@@ -7089,13 +7371,23 @@ async function pollLoop() {
     } else if (r.kind === 'meter_rate_limited') {
       // Billing meter is itself being rate-limited. Treat as "utilization unknown but safe":
       // fire available jobs anyway at utilization=0 rather than pausing the queue.
+      // Still back off the POLL cadence itself (same curve/cap as the transient
+      // branch) and persist state every cycle — without this, a sustained 429
+      // streak hammered the already-rate-limited endpoint every POLL_INTERVAL_MS
+      // forever AND never wrote lastPollAt/consecutiveFailures back to
+      // scheduler-state.json, so the sidecar froze stale while the loop kept
+      // failing silently underneath it (the 57-consecutive-failure incident).
       lastPollAt = Date.now();
       lastPollOk = false;
       consecutiveFailures++;
       lastFailureKind = 'meter_rate_limited';
       // Don't update firstNon429FailureAt — 429s don't count toward the 30-min network-pause threshold.
+      backoffMs = nextBackoffMs(backoffMs);
+      backoffNextAt = Date.now() + backoffMs;
       cachedUtilization = 0; // assume safe; fire any pending work
-      console.log(`[scheduler] billing meter rate-limited (HTTP 429) — firing on heuristic (failure #${consecutiveFailures})`);
+      console.log(`[scheduler] billing meter rate-limited (HTTP 429) — firing on heuristic (failure #${consecutiveFailures}); retry in ${backoffMs / 1000}s`);
+      warnFailureStreakIfNeeded();
+      persistSchedulerState();
       const cur = await readQueue();
       await maybeLaunchWhenAvailable(cur);
       await broadcast();
@@ -7113,7 +7405,7 @@ async function pollLoop() {
         // transient or config — apply exponential backoff and count toward 30-min threshold.
         lastFailureKind = 'transient';
         if (!firstNon429FailureAt) firstNon429FailureAt = Date.now();
-        backoffMs = backoffMs ? Math.min(backoffMs * 2, 480_000) : 30_000;
+        backoffMs = nextBackoffMs(backoffMs);
         const totalNon429FailureMs = Date.now() - firstNon429FailureAt;
         console.log(`[scheduler] transient failure #${consecutiveFailures}: ${r.kind} ${r.message ?? ''}; retry in ${backoffMs / 1000}s`);
 
@@ -7127,7 +7419,21 @@ async function pollLoop() {
       }
 
       backoffNextAt = Date.now() + backoffMs;
+      warnFailureStreakIfNeeded();
       persistSchedulerState();
+      // A failed billing poll must not silently stop dispatch — only the
+      // 'ok' and 'meter_rate_limited' branches used to reach
+      // maybeLaunchWhenAvailable, so auth/transient failures left ready
+      // pending work untouched until either the queue-starvation watchdog's
+      // 10-minute safety net fired or the poll itself recovered. Utilization
+      // is unknown during a failed poll, not unsafe — treated the same way
+      // the meter_rate_limited branch above already treats a 429 as safe to
+      // fire through. maybeLaunchWhenAvailable itself still honors an
+      // 'auth'/'network' pause (state.paused), so this is a no-op whenever
+      // setPaused() above actually engaged one.
+      if (cachedUtilization === null || cachedUtilization === undefined) cachedUtilization = 0;
+      await maybeLaunchWhenAvailable(await readQueue());
+      await broadcast();
     }
   } catch (e) {
     // Unexpected error (e.g., IPC transport failure)
@@ -7137,9 +7443,17 @@ async function pollLoop() {
     lastFailureKind = 'transient';
     if (!firstFailureAt) firstFailureAt = Date.now();
     if (!firstNon429FailureAt) firstNon429FailureAt = Date.now();
-    backoffMs = backoffMs ? Math.min(backoffMs * 2, 480_000) : 30_000;
+    backoffMs = nextBackoffMs(backoffMs);
     backoffNextAt = Date.now() + backoffMs;
+    warnFailureStreakIfNeeded();
     persistSchedulerState();
+    // Same rationale as the auth/transient branch above: the outer catch
+    // must not be a silent dispatch dead-end either.
+    try {
+      if (cachedUtilization === null || cachedUtilization === undefined) cachedUtilization = 0;
+      await maybeLaunchWhenAvailable(await readQueue());
+      await broadcast();
+    } catch { /* best-effort — the poll loop must still re-arm below */ }
   } finally {
     const delay = backoffMs || POLL_INTERVAL_MS;
     pollLoopTimer = setTimeout(() => { pollLoop().catch(() => {}); }, delay);
@@ -7422,13 +7736,90 @@ function isRescanCandidate(job) {
  * the unguarded boot call, could reach it). Reported 2026-09-10 from
  * social-signals-trader.
  *
- * Cost: one extra classifyRunOutcome() log stat per `failed` row, and only
- * for rows that reach isFailedUnverifiedShaped's log check — bounded by the
- * failed-row count, and strictly cheaper than firing the whole pass (which
- * does git fetches via computeLooksDone) on every tick.
+ * Reopened through a different door 2026-09-12 (starry-night-ships
+ * 231-saturn-record-and-docs / 243-neptune-kurama-mode): the guard above only
+ * covered isRescanCandidate (re-verification), but reverifyNeedsReview's auto-
+ * fix loop ALSO lives behind this same guard, and a `needs_review` row whose
+ * mechanical recovery already ran and failed (mechanicalRecoveryAttempted:
+ * true, verdict still 'worktree_integration_failed', not itself a
+ * RESCANNABLE_VERDICTS member) is invisible to isRescanCandidate — so the
+ * next rung (a fix-plan investigation via selectAutoFixTargets) never got a
+ * chance to fire either. Widened to OR in every live target of the recovery
+ * ladder the periodic pass actually drives (selectMechanicalRecoveryTarget /
+ * selectResumeRecoveryTarget / selectAutoFixTargets) so the guard can never
+ * again be narrower than the work reverifyNeedsReview performs.
+ *
+ * Cost: selectMechanicalRecoveryTarget/selectResumeRecoveryTarget are pure
+ * (no I/O). selectAutoFixTargets is called with an injected fixSlugExists
+ * that always returns false — cheap and deliberately over-inclusive (a false
+ * positive here just means one extra periodic pass, never a missed one) so
+ * this guard never pays selectAutoFixTargets's production fs.existsSync scan
+ * per tick. resolveRunId's IO only fires for rows missing job.runId, same as
+ * isRescanCandidate already incurs above.
  */
 function shouldRunPeriodicReverify(jobs) {
-  return Array.isArray(jobs) && jobs.some((j) => isRescanCandidate(j));
+  if (!Array.isArray(jobs)) return false;
+  if (jobs.some((j) => isRescanCandidate(j))) return true;
+  if (jobs.some((j) => selectMechanicalRecoveryTarget(j) || selectResumeRecoveryTarget(j))) return true;
+  return selectAutoFixTargets(jobs, { fixSlugExists: () => false }).length > 0;
+}
+
+// Default 24h, overridable via SM_STUCK_FAILED_ESCALATE_HOURS — same
+// env-override shape as QUARANTINE_ESCALATE_MS above.
+const STUCK_FAILED_ESCALATE_MS = process.env.SM_STUCK_FAILED_ESCALATE_HOURS
+  ? Number(process.env.SM_STUCK_FAILED_ESCALATE_HOURS) * 60 * 60_000
+  : 24 * 60 * 60_000;
+
+/**
+ * Kill-switch gate for the stuck-failed escalation below
+ * (SM_STUCK_FAILED_ESCALATE_DISABLE=1), mirroring the
+ * SM_REVERIFY_PERIODIC_DISABLE / SM_RCA_DISABLE convention. A tiny wrapper
+ * so the disable path is unit-testable without invoking the 10-minute
+ * setInterval body directly.
+ */
+function stuckFailedEscalationDisabled() {
+  return process.env.SM_STUCK_FAILED_ESCALATE_DISABLE === '1';
+}
+
+/**
+ * findStuckFailedJobs(jobs, now, thresholdMs) → [{ slug, cwd, ageMs }]
+ *
+ * Pure predicate (isRescanCandidate's own resolveRunId/classifyRunOutcome log
+ * read is the only IO, gated per-job exactly like shouldRunPeriodicReverify
+ * above). `failed` is a fully terminal state for every automated recovery
+ * path — selectResumeRecoveryTarget/selectAutoFixTargets both require
+ * needs_review, reapDeadRunningJobs only ever writes running → failed, and
+ * reconcile-repair's to-pending is for structurally invalid rows. Only a
+ * human's scheduler_reset_job ever takes failed → pending (LEGAL_TRANSITIONS).
+ * A rescan candidate (isRescanCandidate) that has sat failed longer than
+ * `thresholdMs` can therefore go silently stuck forever — job
+ * 4056-outcome-stats sat `failed` for five days with no operator signal
+ * (reported 2026-09-10, social-signals-trader) even though the periodic
+ * reverify pass (once shouldRunPeriodicReverify's guard was fixed) WAS firing
+ * on it — reverifyNeedsReview's failed branch can annotate looksDone but can
+ * never resolve a failed row itself (see its own header). This is the
+ * visibility half that guard fix was missing: escalate once, never requeue.
+ *
+ * `stuckFailedNotified` gates this to exactly once per row — once the caller
+ * stamps it, this always excludes that row so a human is never re-paged on
+ * every 10-minute tick for the same stuck job. A row with no recoverable
+ * 'failed' timestamp is skipped rather than guessed at (mirrors
+ * findStaleQuarantinedJobs above).
+ */
+function findStuckFailedJobs(jobs, now, thresholdMs) {
+  const stuck = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'failed') continue;
+    if (j.stuckFailedNotified === true) continue;
+    if (!isRescanCandidate(j)) continue;
+    const entry = (j.statusHistory || []).find((h) => h.to === 'failed');
+    if (!entry) continue;
+    const since = Date.parse(entry.at);
+    if (Number.isNaN(since)) continue;
+    const ageMs = now - since;
+    if (ageMs >= thresholdMs) stuck.push({ slug: j.slug, cwd: j.cwd ?? null, ageMs });
+  }
+  return stuck;
 }
 
 /**
@@ -8264,7 +8655,18 @@ async function init() {
     try {
       const worktreeCwds = new Set(bootSnap.jobs.map((j) => j.cwd).filter(Boolean));
       worktreeCwds.add(DEFAULT_PROJECT_CWD);
-      await jobWorktree.reconcileWorktreesOnBoot([...worktreeCwds]);
+      // A job is spawned `detached: true`, so its `claude -p` executor can
+      // survive this very app restart — a worktree found at boot is NOT, by
+      // itself, proof its run already died. isLive checks the already-read
+      // bootSnap (no extra queue read) for a live running-row pid, OR a live
+      // /proc cwd holder under the checkout itself. See jobWorktreeBootLive.cjs.
+      const isLive = buildJobWorktreeIsLive({
+        bootJobs: bootSnap.jobs,
+        claudePidAlive,
+        hasLiveHolder: gitWorktree.hasLiveHolder,
+        cwdHolders: gitWorktree.listCwdHolders(),
+      });
+      await jobWorktree.reconcileWorktreesOnBoot([...worktreeCwds], { isLive });
     } catch (e) {
       console.error('[scheduler] boot worktree reconciliation failed', e?.message);
     }
@@ -8430,6 +8832,17 @@ async function init() {
       appendAuditEvent('job_overrunning_estimate', {
         slug: over.slug, cwd: over.cwd, estimateMinutes: over.estimateMinutes, ranMs: over.ranMs, ratio: over.ratio,
       });
+      // Durable stamp so schedule:state (and therefore the renderer) can see
+      // this without re-deriving it — the console.warn/audit event above are
+      // visible only in the log, never on the row itself. Display-only
+      // advisory field; re-stamped in place every sweep, never appended.
+      mutate((state) => {
+        const j = state.jobs.find((x) => x.slug === over.slug);
+        if (!j) return;
+        j.overrun = {
+          ratio: over.ratio, ranMs: over.ranMs, estimateMinutes: over.estimateMinutes, at: new Date().toISOString(),
+        };
+      }).catch((e) => console.warn('[scheduler] overrun stamp failed', e?.message));
     }
 
     // Stranded-investigation restore. Unlike the two escalations above, this
@@ -8474,6 +8887,31 @@ async function init() {
       );
       appendAuditEvent('project_starved', { cwd: sp.cwd, pendingCount: sp.pendingCount, oldestPendingSlug: sp.oldestPendingSlug, ageMs: sp.ageMs });
     }
+
+    // Stuck-failed escalation (2026-09-10, social-signals-trader): see
+    // findStuckFailedJobs' header for why `failed` has no automated way
+    // back to pending. Escalation only, same shape as the three warnings
+    // above — never an automatic failed → pending requeue (that could
+    // discard uncommitted work left by the failed run; see
+    // spawnJob:fail-dirty). Kill-switch: SM_STUCK_FAILED_ESCALATE_DISABLE=1.
+    if (!stuckFailedEscalationDisabled()) {
+      const stuckFailed = findStuckFailedJobs(s.jobs, Date.now(), STUCK_FAILED_ESCALATE_MS);
+      if (stuckFailed.length > 0) {
+        mutate((ms) => {
+          for (const stuck of stuckFailed) {
+            const j = ms.jobs.find((x) => x.slug === stuck.slug);
+            if (!j || j.status !== 'failed' || j.stuckFailedNotified === true) continue; // race guard
+            j.stuckFailedNotified = true;
+            console.warn(
+              `[scheduler] FAILED PRD STUCK: project=${stuck.cwd ?? '(unknown)'} slug=${stuck.slug} `
+              + `failed=${Math.round(stuck.ageMs / 3_600_000)}h (>= ${Math.round(STUCK_FAILED_ESCALATE_MS / 3_600_000)}h threshold) — `
+              + `no automated recovery reaches a failed row; reset it by hand via scheduler_reset_job`,
+            );
+            appendAuditEvent('job_stuck_failed', { slug: stuck.slug, cwd: stuck.cwd, ageMs: stuck.ageMs });
+          }
+        }).catch(() => {});
+      }
+    }
   }, 10 * 60_000);
 
   // Self-rescheduling poll loop with exponential backoff. Replaces the
@@ -9149,6 +9587,11 @@ module.exports = {
   init,
   ROOT,
   PRDS_DIR,
+  SCHEDULER_STATE_PATH,
+  BACKOFF_MAX_MS,
+  FAILURE_STREAK_WARN_THRESHOLD,
+  nextBackoffMs,
+  shouldWarnFailureStreak,
   healRefusalReason,
   writeQueue,
   reconcile,
@@ -9170,6 +9613,9 @@ module.exports = {
   availableForJobs,
   reverifyNeedsReview,
   shouldRunPeriodicReverify,
+  findStuckFailedJobs,
+  STUCK_FAILED_ESCALATE_MS,
+  stuckFailedEscalationDisabled,
   isRescanCandidate,
   isFailedUnverifiedShaped,
   computeLooksDone,
@@ -9281,6 +9727,8 @@ module.exports = {
   clearPause,
   tickQueue,
   runDueJobs,
+  pollLoop,
+  maybeLaunchWhenAvailable,
   isCooldownSuppressed,
   nextRapidRateLimitCount,
   CONSECUTIVE_RAPID_RATE_LIMIT_THRESHOLD,