claude-code-session-manager 0.77.0 → 0.79.0

package/src/main/scheduler.cjs

@@ -58,6 +58,8 @@ const launchFailure = require('./lib/launchFailure.cjs');
 const { appendError } = require('./lib/opsErrorLog.cjs');
 const { readTail } = require('./lib/fileTail.cjs');
 const { claudePidAlive, classifyRunOutcome, mapOutcomeToGateOutcome, ORPHAN_REQUEUE_CAP, selectReapableJobs } = require('./lib/reaperHelpers.cjs');
+const { detectRateLimitInLog } = require('./lib/rateLimitDetect.cjs');
+const { stripAppOwnedChurn } = require('./lib/jobDirtFilter.cjs');
 const { computeQueueHealth } = require('./lib/queueHealth.cjs');
 const { createLoadGate, topCpuConsumers } = require('./lib/loadGate.cjs');
 const { openLog, withChildAndLog } = require('./lib/childWithLog.cjs');
@@ -1252,6 +1254,89 @@ function computeStallSummary(state) {
   return { stalled, total, running, pending, byProject };
 }
 
+/**
+ * computeBlockedChains(jobs) → [{ cwd, blockedBy, blocked }]
+ *
+ * Pure, no IO. The gap computeStallSummary above cannot see.
+ *
+ * `stalled` is defined as `running === 0 && pending === 0`, which encodes an
+ * assumption that a PENDING row is healthy in-progress work. It is not: a
+ * pending row whose `dependsOn` chain terminates in a TERMINAL non-completed
+ * status (`failed`/`skipped`) can never be dispatched by pickForProject, and
+ * never will be, but it still counts toward `pending` and so reads as a
+ * healthy queue to every monitor in the app.
+ *
+ * On 2026-09-05 starry-night-ships held 42 such rows behind one `failed`
+ * job for three hours. Machine-wide `stalled` was false (42 pending),
+ * per-project `stalled` was false (42 pending), the queue-health sweep
+ * doesn't count `failed` at all, and the supervisor only probes `running` —
+ * so nothing anywhere reported a problem while nothing could ever run.
+ *
+ * Reported per project as { blockedBy: [terminal slugs], blocked: count }.
+ * Transitive by construction: a row blocked by a row that is itself blocked
+ * resolves through the same walk.
+ */
+function computeBlockedChains(jobs) {
+  const rows = (Array.isArray(jobs) ? jobs : []).filter(Boolean);
+  const byCwd = new Map();
+  for (const j of rows) {
+    const key = j.cwd || '(unknown)';
+    if (!byCwd.has(key)) byCwd.set(key, []);
+    byCwd.get(key).push(j);
+  }
+
+  const out = [];
+  for (const [cwd, projectJobs] of byCwd) {
+    // Reuse the picker's OWN dep resolution so this can never disagree with
+    // what the scheduler will actually dispatch (bare-name fallback included).
+    const rowBySlug = new Map(projectJobs.map((j) => [j.slug, j]));
+    const rowsByBareSlug = new Map();
+    for (const j of projectJobs) {
+      const bare = String(j.slug ?? '').replace(/^\d+-/, '');
+      if (!rowsByBareSlug.has(bare)) rowsByBareSlug.set(bare, []);
+      rowsByBareSlug.get(bare).push(j);
+    }
+    const rowsForDep = (slug) => {
+      const exact = rowBySlug.get(slug);
+      if (exact) return [exact];
+      return rowsByBareSlug.get(String(slug ?? '').replace(/^\d+-/, '')) ?? [];
+    };
+
+    // Memoised walk: does this row's dep closure hit a terminally-stuck row?
+    const TERMINAL_STUCK = new Set(['failed', 'skipped']);
+    const verdicts = new Map(); // slug -> Set of terminal blocker slugs
+    const visiting = new Set();
+    const blockersFor = (job) => {
+      if (!job) return new Set();
+      if (verdicts.has(job.slug)) return verdicts.get(job.slug);
+      if (visiting.has(job.slug)) return new Set(); // dependsOn cycle — not our problem here
+      visiting.add(job.slug);
+      const found = new Set();
+      for (const depSlug of job.dependsOn ?? []) {
+        for (const dep of rowsForDep(depSlug)) {
+          if (TERMINAL_STUCK.has(dep.status)) found.add(dep.slug);
+          else if (dep.status !== 'completed') for (const b of blockersFor(dep)) found.add(b);
+        }
+      }
+      visiting.delete(job.slug);
+      verdicts.set(job.slug, found);
+      return found;
+    };
+
+    const blockedBy = new Set();
+    let blocked = 0;
+    for (const j of projectJobs) {
+      if (j.status !== 'pending') continue;
+      const bs = blockersFor(j);
+      if (bs.size === 0) continue;
+      blocked += 1;
+      for (const b of bs) blockedBy.add(b);
+    }
+    if (blocked > 0) out.push({ cwd, blockedBy: [...blockedBy].sort(), blocked });
+  }
+  return out;
+}
+
 /**
  * findStaleQuarantinedJobs(jobs, now, thresholdMs) → [{ slug, cwd, ageMs }]
  *
@@ -2119,6 +2204,9 @@ let firstFailureAt = null;
 let firstNon429FailureAt = null; // tracks only transient/config failures; 429s don't count toward network-pause threshold
 let lastFailureKind = null; // 'transient' | 'meter_rate_limited' | 'auth' | null
 let pauseClearedManuallyAt = null;
+// PRD 1119: consecutive-rapid-rate-limit hard-pause tracking, keyed per slug.
+// See isCooldownSuppressed/nextRapidRateLimitCount below for the pure rules.
+const consecutiveRapidRateLimitsBySlug = new Map();
 
 // ---------- timer ----------
 
@@ -2337,13 +2425,71 @@ async function rescheduleTimer() {
 
 // ---------- pause / resume ----------
 
-async function setPaused(reason, resumeAtIso) {
+const MANUAL_PAUSE_COOLDOWN_MS = 300_000;
+// PRD 1119: after this many consecutive rate-limited dispatches of the SAME
+// slug that EACH also finished in under RAPID_RATE_LIMIT_WINDOW_MS, the rate
+// limit is not a stale/flaky auto-detection any more — it's real and
+// persistent for this job. Engage a hard pause the manual-clear cooldown
+// cannot suppress at all. This exists because the freshness check alone
+// (isCooldownSuppressed) is not sufficient: if the computed resumeAt is
+// itself wrong or stale (e.g. a failed usage-API fetch), the resume timer
+// can keep re-clearing the pause every ~30s, and every SUBSEQUENT dispatch
+// is genuinely "fresh" (it started after that re-clear) — so freshness alone
+// would let the spin continue indefinitely within the same 5-minute cooldown
+// window. The rapid-repeat count is an independent circuit breaker of last
+// resort for exactly that case.
+const CONSECUTIVE_RAPID_RATE_LIMIT_THRESHOLD = 3;
+const RAPID_RATE_LIMIT_WINDOW_MS = 30_000;
+
+/**
+ * Pure: should setPaused()'s manual-override cooldown suppress WRITING this
+ * pause? `force` (the rapid-repeat hard pause) always answers no — that path
+ * exists precisely to bypass the cooldown. Otherwise, suppress only while
+ * inside the cooldown window AND the triggering observation is stale, i.e.
+ * it was NOT produced by a run that started after the human's manual clear.
+ * A run that started after the clear is fresh evidence the human's fix (if
+ * any) did not hold, and must be allowed to re-engage the pause regardless
+ * of the cooldown — the cooldown's job is to ignore STALE auto-detections,
+ * never to ignore new evidence.
+ */
+function isCooldownSuppressed({ pauseClearedManuallyAt: clearedAt, now, observedAt, force }) {
+  if (force) return false;
+  if (!clearedAt) return false;
+  if (now - clearedAt >= MANUAL_PAUSE_COOLDOWN_MS) return false;
+  const isFresh = typeof observedAt === 'number' && observedAt > clearedAt;
+  return !isFresh;
+}
+
+/**
+ * Pure: the next consecutive-rapid-rate-limit count for a slug, given its
+ * previous count and this run's outcome. Increments only on a rate-limited
+ * run that ALSO ran under RAPID_RATE_LIMIT_WINDOW_MS (a genuine "dispatch,
+ * 429, die" cycle — not a job that ran for a while before hitting the
+ * limit). Resets to 0 on any non-rate-limited outcome. A rate-limited-but-
+ * slow run leaves the count unchanged: still a rate limit, just not the
+ * rapid-spin shape this cap exists to catch.
+ */
+function nextRapidRateLimitCount(prevCount, { rateLimited, durationMs }) {
+  if (!rateLimited) return 0;
+  if (durationMs < RAPID_RATE_LIMIT_WINDOW_MS) return (prevCount || 0) + 1;
+  return prevCount || 0;
+}
+
+async function setPaused(reason, resumeAtIso, opts = {}) {
+  const { observedAt = null, force = false } = opts;
   // Honor manual-override cooldown: if the user cleared a pause within the
-  // last 5 minutes, suppress auto-pause re-engagement on the same condition.
-  if (pauseClearedManuallyAt && Date.now() - pauseClearedManuallyAt < 300_000) {
+  // last 5 minutes, suppress auto-pause re-engagement UNLESS this pause is
+  // backed by a fresh observation (a run that started after the clear) or is
+  // forced (the rapid-repeat hard pause, which the cooldown cannot suppress).
+  if (isCooldownSuppressed({ pauseClearedManuallyAt, now: Date.now(), observedAt, force })) {
     console.log(`[scheduler] setPaused(${reason}) suppressed by manual override cooldown`);
     return;
   }
+  if (force) {
+    console.log(`[scheduler] setPaused(${reason}) forced past manual override cooldown — rapid-repeat rate-limit cap engaged`);
+  } else if (pauseClearedManuallyAt && Date.now() - pauseClearedManuallyAt < MANUAL_PAUSE_COOLDOWN_MS) {
+    console.log(`[scheduler] setPaused(${reason}) engaging despite manual override cooldown — triggering run started after the manual clear`);
+  }
 
   // For 'network' with no explicit resumeAt, auto-resume after 30 minutes.
   let effectiveResumeAt = resumeAtIso;
@@ -2862,21 +3008,6 @@ async function notifyNeedsReview(job, report, {
   }
 }
 
-/** Scan the tail of a job's log for the canonical rate-limit signal. We look
- *  at the last 16 KB — final result event always lands at the end.
- *  Uses readTail() so no raw fd lifecycle is needed here. */
-function detectRateLimitInLog(logPath) {
-  try {
-    const text = readTail(logPath, 16384);
-    if (!text) return false;
-    return /"rateLimitType":"five_hour"/.test(text)
-      || /"api_error_status":429/.test(text)
-      || /You'?ve hit your limit/.test(text);
-  } catch {
-    return false;
-  }
-}
-
 /** Scan the tail of a job's log for a network-outage signal: the structured
  *  `terminal_reason":"api_error"` field alongside a network-class error
  *  string. This is NOT a real code defect — spawning an auto-fix
@@ -3197,10 +3328,17 @@ function buildClaudeSpawnArgs({ prompt, model, sessionId, resume, systemPrompt }
 
 // ---------- execution ----------
 
+// Allocates the runId/dir pair at dispatch time WITHOUT creating the
+// directory — a dispatch that aborts inside spawnJob before executeJob's
+// openLog() call (slot-acquire miss, worktree-cap deferral, launch-gate
+// block, ...) must leave no trace on disk. The directory is materialised
+// lazily, the first time something actually needs to write into it (see
+// openLog's mkdirSync in executeJob below). Because tickQueue hands ONE
+// shared batch dir to every spawnJob in the batch, several jobs may race to
+// create it — `recursive: true` makes that race safe.
 function pickRunDir() {
   const ts = new Date().toISOString().replace(/[:.]/g, '-');
   const dir = path.join(RUNS_DIR, ts);
-  fs.mkdirSync(dir, { recursive: true });
   return { runId: ts, dir };
 }
 
@@ -3228,6 +3366,12 @@ async function executeJob(job, runDir, defaultCwd, onPid, execCwd, resumeTarget
   // fresh one via `--session-id` is the entire point of the recovery.
   const sessionId = resumeTarget ? resumeTarget.sessionId : randomUUID();
 
+  // Materialise the (possibly shared-batch) run dir lazily, right before the
+  // first write into it — see pickRunDir's comment for why this is deferred
+  // this far. recursive:true makes it safe if a sibling job in the same
+  // batch dir already created it.
+  fs.mkdirSync(runDir, { recursive: true });
+
   // Phase 1: open log fd so we can emit pre-spawn diagnostics (early-exit
   // error paths) before the child is created. withChildAndLog takes ownership
   // of fd/safeLog/closeFd from the point it is called.
@@ -4317,6 +4461,13 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
       console.log(`[scheduler] ${job.slug}: dispatching as launch probe for '${launchKey}'${launchEnv ? ` with mitigation ${JSON.stringify(launchEnv)}` : ''}`);
     }
 
+    // Captured here (not read back off `job`, a pre-dispatch snapshot that
+    // mutate()'s fresh-from-disk read never touches) so the rate-limited
+    // branch below has this run's OWN start time — the freshness check
+    // (isCooldownSuppressed) needs to know whether this specific dispatch
+    // started after the manual clear, not whatever startedAt this row
+    // carried from a prior run.
+    let dispatchStartedAtMs = null;
     await mutate((s) => {
       const idx = s.jobs.findIndex((x) => x.slug === job.slug);
       if (idx >= 0) {
@@ -4327,6 +4478,7 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
         delete s.jobs[idx].heldReason;
         s.jobs[idx].runId = runId;
         s.jobs[idx].startedAt = new Date().toISOString();
+        dispatchStartedAtMs = Date.parse(s.jobs[idx].startedAt);
         if (job.quietMachine === true) {
           s.jobs[idx].quietMachine = true;
           s.jobs[idx].quietLeaseDegraded = job.quietLeaseDegraded === true;
@@ -4531,12 +4683,18 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
     // (non-git cwd / git errored) — NEVER treated as "left nothing", exactly
     // like every other best-effort git-state check in this function.
     const afterGuardCwd = await uncommittedChanges(guardCwd);
+    // stripAppOwnedChurn: the app writes session-manager-operations/ (queue.json,
+    // history.jsonl, active-index.json, transcripts) DURING this job's own guard
+    // window, so those land in the delta and get blamed on the job. A job can
+    // never be responsible for them — see jobDirtFilter.cjs. Applied here, at
+    // the single place the delta is computed, so the commit guard, the
+    // transient-retry dirty check and leftoverPaths all agree.
     const newlyDirtyAll = afterGuardCwd === null
       ? null
-      : [...new Set([
+      : stripAppOwnedChurn([...new Set([
           ...afterGuardCwd.filter((p) => !new Set(guardBaseline || []).has(p)),
           ...worktreeLeftoverDirty,
-        ])];
+        ])]);
 
     if (res.launchFailure) {
       await handleLaunchFailure({ job, res, runId, runDir, launchKey, launchEnv, claudeVersion: claudeVersionNow });
@@ -4570,7 +4728,17 @@ async function spawnJob(job, runId, runDir, defaultCwd, resumeTarget = null) {
 
     if (res.rateLimited) {
       const resetIso = await refreshNextReset().catch(() => cachedNextReset);
-      await setPaused('rate_limit', resetIso);
+      const observedAt = dispatchStartedAtMs;
+      const prevCount = consecutiveRapidRateLimitsBySlug.get(job.slug) || 0;
+      const nextCount = nextRapidRateLimitCount(prevCount, { rateLimited: true, durationMs: res.durationMs });
+      consecutiveRapidRateLimitsBySlug.set(job.slug, nextCount);
+      const forceHardPause = nextCount >= CONSECUTIVE_RAPID_RATE_LIMIT_THRESHOLD;
+      if (forceHardPause) {
+        console.log(`[scheduler] ${job.slug}: ${nextCount} consecutive rate-limited dispatches under ${RAPID_RATE_LIMIT_WINDOW_MS / 1000}s each — engaging hard pause`);
+      }
+      await setPaused('rate_limit', resetIso, { observedAt, force: forceHardPause });
+    } else {
+      consecutiveRapidRateLimitsBySlug.delete(job.slug);
     }
 
     // Stale queue entry: the PRD was archived (already shipped) or is gone
@@ -5366,6 +5534,109 @@ async function maybeLaunchWhenAvailable(state) {
   tickQueue().catch((e) => console.error('[scheduler] tickQueue error', e));
 }
 
+/** How long a queue may hold ready work with nothing running before the
+ *  starvation watchdog forces a tick. Deliberately longer than the poll
+ *  loop's own cadence + backoff, so this only ever fires when the normal
+ *  path has genuinely stopped driving the queue — it is a safety net, not a
+ *  second scheduler. */
+const QUEUE_STARVATION_MS = 10 * 60_000;
+
+/**
+ * classifyQueueStarvation({ jobs, paused, runningCount, lastRunAtMs, now, thresholdMs })
+ *   → null | { kind: 'starved' | 'blocked', pending, dispatchable, blockedChains, idleMs }
+ *
+ * Pure, no IO. Answers the one question the user's invariant reduces to:
+ * "there are PRDs in a queue — is anything actually going to run them?"
+ *
+ * Every stall this codebase has seen was a DIFFERENT cause with the SAME
+ * shape: ready rows, nothing running, nobody ticking. A rate-limited exit
+ * stamped terminal `failed` (2026-09-05, 42 rows); a spin loop past the
+ * manual-clear cooldown; a worktree merge-back that left the project on a
+ * job branch; a job parked `needs_review` with no fix plan; app churn
+ * counted as unfinished work. Guarding each cause individually will always
+ * lag the next one, so this guards the SHAPE instead.
+ *
+ * Two outcomes, deliberately distinguished — they need opposite responses:
+ *   'starved' — at least one pending row is dispatchable RIGHT NOW and
+ *               nothing is running. Whatever should have ticked, didn't.
+ *               Forcing a tick is safe and fixes it.
+ *   'blocked' — every pending row is behind a terminal/parked dependency.
+ *               A tick cannot help; this needs a human (or a heal pass) to
+ *               resolve the blocker, and must be reported as such rather
+ *               than silently re-ticking forever.
+ *
+ * Returns null when the queue is healthy (work running, nothing pending,
+ * paused on purpose, or simply not idle long enough yet).
+ */
+function classifyQueueStarvation({ jobs, paused, runningCount, lastRunAtMs, now, thresholdMs = QUEUE_STARVATION_MS } = {}) {
+  if (paused) return null;                      // paused is a DECISION, not a stall
+  if (runningCount > 0) return null;            // work is flowing
+  const rows = Array.isArray(jobs) ? jobs : [];
+  const pending = rows.filter((j) => j && j.status === 'pending');
+  if (pending.length === 0) return null;        // nothing to run — not a stall
+
+  const idleMs = Number.isFinite(lastRunAtMs) ? now - lastRunAtMs : Infinity;
+  if (idleMs < thresholdMs) return null;        // give the normal path its chance first
+
+  // Which pending rows could actually dispatch? Anything NOT named by a
+  // blocked chain. computeBlockedChains already walks dependsOn with the
+  // picker's own resolution, so the two can never disagree.
+  const blockedChains = computeBlockedChains(rows);
+  const blockedTotal = blockedChains.reduce((n, c) => n + c.blocked, 0);
+  const dispatchable = pending.length - blockedTotal;
+
+  return {
+    kind: dispatchable > 0 ? 'starved' : 'blocked',
+    pending: pending.length,
+    dispatchable,
+    blockedChains,
+    idleMs,
+  };
+}
+
+/**
+ * The watchdog half: acts on classifyQueueStarvation. Called from the
+ * heartbeat, which already runs on its own timer independent of the billing
+ * poll loop — so a wedged or never-succeeding poll (the /api/oauth/usage
+ * endpoint was itself 429ing all of 2026-09-05) can no longer leave a queue
+ * with ready work idle indefinitely.
+ */
+async function runQueueStarvationWatchdog(state, { now = Date.now(), thresholdMs = QUEUE_STARVATION_MS } = {}) {
+  const verdict = classifyQueueStarvation({
+    jobs: state?.jobs,
+    paused: state?.paused,
+    runningCount: runningSet.size,
+    lastRunAtMs: Date.parse(state?.lastRunAt ?? ''),
+    now,
+    thresholdMs,
+  });
+  if (!verdict) return null;
+
+  const mins = Math.round(verdict.idleMs / 60_000);
+  if (verdict.kind === 'blocked') {
+    console.warn(
+      `[scheduler] QUEUE BLOCKED: ${verdict.pending} pending job(s), 0 running, idle ${mins}m — every ready row is behind a `
+      + `terminal or parked dependency, so ticking cannot help. Blockers: `
+      + verdict.blockedChains.map((c) => `${c.cwd} [${c.blockedBy.join(', ')}]`).join(' · '),
+    );
+    appendAuditEvent('queue_blocked_stall', { pending: verdict.pending, idleMs: verdict.idleMs, chains: verdict.blockedChains });
+    return verdict;
+  }
+
+  console.warn(
+    `[scheduler] QUEUE STARVED: ${verdict.dispatchable} dispatchable job(s) of ${verdict.pending} pending, 0 running, `
+    + `idle ${mins}m (>= ${Math.round(thresholdMs / 60_000)}m) — forcing a tick`,
+  );
+  appendAuditEvent('queue_starvation_forced_tick', { pending: verdict.pending, dispatchable: verdict.dispatchable, idleMs: verdict.idleMs });
+  // A never-populated utilization reading is itself one of the ways the
+  // when-available path silently never fires (maybeLaunchWhenAvailable
+  // returns early on null). Treat unknown as safe here, exactly as the
+  // billing meter's own 429 fallback already does.
+  if (cachedUtilization === null || cachedUtilization === undefined) cachedUtilization = 0;
+  await tickQueue({ bypassLoadGate: false }).catch((e) => console.error('[scheduler] starvation tick error', e));
+  return verdict;
+}
+
 // ---------- dead-process reaper ----------
 
 // Queue-health sweep cadence: hangs off reapDeadRunningJobs's own cycle
@@ -5440,11 +5711,41 @@ async function reapDeadRunningJobs() {
 
     if (dead.length === 0) return;
 
+    // A rate-limited death is retryable, not terminal — mirror spawnJob's own
+    // live-process handling (PRD 1117) exactly: engage the SAME setPaused
+    // pause here too. Skipping this would reset the row to 'pending' but
+    // leave dispatch unpaused, so the next tick immediately re-fires it into
+    // the same still-active rate limit — the spin loop this PRD exists to
+    // stop. Done once, outside mutate(), before finalizing any row below.
+    if (dead.some((d) => d.outcome === 'rate_limited')) {
+      const resetIso = await refreshNextReset().catch(() => cachedNextReset);
+      const triggering = dead.find((d) => d.outcome === 'rate_limited');
+      const triggeringRow = triggering ? state.jobs.find((x) => x.slug === triggering.slug) : null;
+      const observedAtMs = triggeringRow?.startedAt ? Date.parse(triggeringRow.startedAt) : null;
+      // Same rapid-repeat circuit breaker spawnJob's own res.rateLimited
+      // branch drives (see consecutiveRapidRateLimitsBySlug above) — a
+      // process that gets rate-limited and then dies without spawnJob's own
+      // branch ever running is reconciled HERE instead, and must feed the
+      // same counter or a stale/wrong resumeAt could keep re-clearing this
+      // path's "fresh" pause every reap cycle with no hard cap ever engaging.
+      const durationMs = Number.isFinite(observedAtMs) ? Date.now() - observedAtMs : Infinity;
+      const prevCount = consecutiveRapidRateLimitsBySlug.get(triggering.slug) || 0;
+      const nextCount = nextRapidRateLimitCount(prevCount, { rateLimited: true, durationMs });
+      consecutiveRapidRateLimitsBySlug.set(triggering.slug, nextCount);
+      const forceHardPause = nextCount >= CONSECUTIVE_RAPID_RATE_LIMIT_THRESHOLD;
+      if (forceHardPause) {
+        console.log(`[scheduler] ${triggering.slug}: ${nextCount} consecutive rate-limited dispatches under ${RAPID_RATE_LIMIT_WINDOW_MS / 1000}s each (reaped) — engaging hard pause`);
+      }
+      await setPaused('rate_limit', resetIso, { observedAt: observedAtMs, force: forceHardPause });
+    }
+
     await mutate(async (s) => {
       for (const { slug, pid, outcome, gateOutcome, pidless, reason } of dead) {
         const idx = s.jobs.findIndex((x) => x.slug === slug);
         if (idx < 0 || s.jobs[idx].status !== 'running') continue; // race guard
+        const rateLimited = outcome === 'rate_limited';
         const success = outcome === 'success';
+        if (!rateLimited) consecutiveRapidRateLimitsBySlug.delete(slug);
 
         // Best-effort in-place leftover computation: a job whose owning
         // process vanished without spawnJob()'s own finally block ever
@@ -5479,13 +5780,23 @@ async function reapDeadRunningJobs() {
         const leftoverSuffix = deltaPaths && deltaPaths.length
           ? ` — left ${deltaPaths.length} files uncommitted`
           : '';
-        const transitionReason = (pidless ? reason : `reaped: process gone (outcome=${outcome})`) + leftoverSuffix;
-
-        transitionJob(s.jobs[idx], success ? 'completed' : 'failed', { reason: transitionReason, source: 'reapDeadRunningJobs' });
-        s.jobs[idx].exitCode = success ? 0 : (s.jobs[idx].exitCode ?? 1);
-        s.jobs[idx].finishedAt = new Date().toISOString();
-        s.jobs[idx].error = success ? null : `${transitionReason} (outcome=${outcome})`;
-        s.jobs[idx].gateOutcome = gateOutcome;
+        const transitionReason = rateLimited
+          ? `reaped: rate limit detected — reset to pending, not failed (outcome=${outcome})${leftoverSuffix}`
+          : (pidless ? reason : `reaped: process gone (outcome=${outcome})`) + leftoverSuffix;
+
+        if (rateLimited) {
+          // Retryable, never terminal (PRD 1117) — same resetJobFields path
+          // spawnJob's own rateLimited branch uses (see ~4797's
+          // treatAsPending), so the row comes back exactly like any other
+          // paused-for-rate-limit reset: fresh runId/startedAt/exitCode.
+          resetJobFields(s.jobs[idx], transitionReason, { source: 'reapDeadRunningJobs:rate-limit' });
+        } else {
+          transitionJob(s.jobs[idx], success ? 'completed' : 'failed', { reason: transitionReason, source: 'reapDeadRunningJobs' });
+          s.jobs[idx].exitCode = success ? 0 : (s.jobs[idx].exitCode ?? 1);
+          s.jobs[idx].finishedAt = new Date().toISOString();
+          s.jobs[idx].error = success ? null : `${transitionReason} (outcome=${outcome})`;
+          s.jobs[idx].gateOutcome = gateOutcome;
+        }
         delete s.jobs[idx].runtime;
         delete s.jobs[idx].guardBaseline;
         delete s.jobs[idx].guardHeadBefore;
@@ -5499,7 +5810,10 @@ async function reapDeadRunningJobs() {
         // with no exit event) wedges the lease held forever and stalls
         // dispatch for every project until the app restarts.
         if (s.jobs[idx].quietMachine === true) quietMachineLease.release(slug);
-        if (pidless) {
+        if (rateLimited) {
+          console.log(`[scheduler] reaped rate-limited job slug=${slug} — reset to pending, pause engaged`);
+          appendAuditEvent('job_reaped_rate_limited', { slug, cwd: s.jobs[idx].cwd ?? null });
+        } else if (pidless) {
           console.log(`[scheduler] reaped pidless zombie job slug=${slug} outcome=${outcome}`);
           appendAuditEvent('job_reaped_pidless', { slug, cwd: s.jobs[idx].cwd ?? null, outcome, graceMs: PIDLESS_SPAWN_GRACE_MS });
         } else {
@@ -6865,6 +7179,14 @@ async function init() {
   if (heartbeatInterval) clearInterval(heartbeatInterval);
   heartbeatInterval = setInterval(() => {
     const s = readQueueSync();
+    // NEVER-STOP INVARIANT: if a queue holds ready PRDs and nothing is
+    // running, something must drive it. This is the only driver that does
+    // not depend on the billing poll loop, a pause timer, or a completing
+    // job to schedule the next tick — every one of which has failed at
+    // least once. See classifyQueueStarvation.
+    if (!s.unreadable) {
+      runQueueStarvationWatchdog(s).catch((e) => console.error('[scheduler] starvation watchdog error', e));
+    }
     // Initialise from the real status union (scheduleJobSchema.cjs) rather
     // than a hand-maintained subset — the old `{ pending, running, completed,
     // failed }` literal silently minted a NEW key for any other value
@@ -7465,4 +7787,4 @@ function registerAdminRoutes(adminHttp, remoteObj = remote) {
   });
 }
 
-module.exports = { findOverrunningJobs, JOB_OVERRUN_FACTOR, JOB_OVERRUN_FLOOR_MS, registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, healRefusalReason, writeQueue, reconcile, reconcileSourcePromptId, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, IDLE_OUTPUT_KILL_MS, BASH_DEFAULT_TIMEOUT_MS, BASH_MAX_TIMEOUT_MS, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isFailedUnverifiedShaped, computeLooksDone, isPromotableOriginal, selectAutoFixTargets, applyRcaClassification, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, isExhaustedAutoFix, isPlanUnqueued, fixSlugFor, healTargetForFix, buildInvestigationPrompt, isGitRepoSync, committedInWindow, computeCommittedDuringRun, classifySigtermWithCommit, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, commitGuardVerdict, leftoverFieldsFrom, applyLeftoverFields, LEFTOVER_PATHS_CAP, capDirtyPaths, buildForeignWipSection, PRE_RUN_DIRTY_PATHS_CAP, FOREIGN_WIP_DELIMITER, FOREIGN_WIP_END_DELIMITER, TRANSIENT_RETRY_CAP, buildScheduleStatePayload, partitionBootOrphans, applyOrphanOutcome, BOOT_ORPHAN_KILL_GRACE_MS, registerAdminRoutes, notifyOriginatingTab, notifyNeedsReview, isNotifiableTerminalStatus, extractResultTextFromLog, candidatePrdsDirs, candidateArchivedPrdsDirs, resolveArchivedPrdStatus, prdDirForCwd, prdPathForJob, archivedPrdPathForJob, archivedTwinExists, findPrdDir, resolveVerifyPrdPath, resolveFixPlanPath, resolveNotifyPrd, runPrdMigration, consolidateAllFlatPrds, shouldSkipInvestigationForCleanRun, archiveCompletedPrd, retireCompletedSlugs, SCHEDULER_BOOTED_AT, SCHEDULER_CODE_SHA, resetJobFields, executeJob, prdArchivedSkipResult, spawnJob, listPrdsInternal, computeStallSummary, findStaleQuarantinedJobs, QUARANTINE_ESCALATE_MS, applyClearQueueVictims, PIDLESS_SPAWN_GRACE_MS, findStrandedInvestigations, INVESTIGATION_MAX_MS, stashList, parseStashLine, pathsChangedSince, restoreSpecificStash, evaluateSharedTreeGuard, checkSharedTreeGuard, uncommittedChanges, gitHead, selectResumeRecoveryTarget, buildResumeRecoveryPreamble, buildClaudeSpawnArgs, spawnResumeRecovery, spawnInvestigation, computeLaunchHolds, handleLaunchFailure, applyLaunchFailure };
+module.exports = { classifyQueueStarvation, runQueueStarvationWatchdog, QUEUE_STARVATION_MS, computeBlockedChains, stripAppOwnedChurn, findOverrunningJobs, JOB_OVERRUN_FACTOR, JOB_OVERRUN_FLOOR_MS, registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, healRefusalReason, writeQueue, reconcile, reconcileSourcePromptId, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, IDLE_OUTPUT_KILL_MS, BASH_DEFAULT_TIMEOUT_MS, BASH_MAX_TIMEOUT_MS, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isFailedUnverifiedShaped, computeLooksDone, isPromotableOriginal, selectAutoFixTargets, applyRcaClassification, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, isExhaustedAutoFix, isPlanUnqueued, fixSlugFor, healTargetForFix, buildInvestigationPrompt, isGitRepoSync, committedInWindow, computeCommittedDuringRun, classifySigtermWithCommit, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, commitGuardVerdict, leftoverFieldsFrom, applyLeftoverFields, LEFTOVER_PATHS_CAP, capDirtyPaths, buildForeignWipSection, PRE_RUN_DIRTY_PATHS_CAP, FOREIGN_WIP_DELIMITER, FOREIGN_WIP_END_DELIMITER, TRANSIENT_RETRY_CAP, buildScheduleStatePayload, partitionBootOrphans, applyOrphanOutcome, BOOT_ORPHAN_KILL_GRACE_MS, registerAdminRoutes, notifyOriginatingTab, notifyNeedsReview, isNotifiableTerminalStatus, extractResultTextFromLog, candidatePrdsDirs, candidateArchivedPrdsDirs, resolveArchivedPrdStatus, prdDirForCwd, prdPathForJob, archivedPrdPathForJob, archivedTwinExists, findPrdDir, resolveVerifyPrdPath, resolveFixPlanPath, resolveNotifyPrd, runPrdMigration, consolidateAllFlatPrds, shouldSkipInvestigationForCleanRun, archiveCompletedPrd, retireCompletedSlugs, SCHEDULER_BOOTED_AT, SCHEDULER_CODE_SHA, resetJobFields, executeJob, prdArchivedSkipResult, spawnJob, listPrdsInternal, computeStallSummary, findStaleQuarantinedJobs, QUARANTINE_ESCALATE_MS, applyClearQueueVictims, PIDLESS_SPAWN_GRACE_MS, findStrandedInvestigations, INVESTIGATION_MAX_MS, stashList, parseStashLine, pathsChangedSince, restoreSpecificStash, evaluateSharedTreeGuard, checkSharedTreeGuard, uncommittedChanges, gitHead, selectResumeRecoveryTarget, buildResumeRecoveryPreamble, buildClaudeSpawnArgs, spawnResumeRecovery, spawnInvestigation, computeLaunchHolds, handleLaunchFailure, applyLaunchFailure, setPaused, clearPause, tickQueue, runDueJobs, isCooldownSuppressed, nextRapidRateLimitCount, CONSECUTIVE_RAPID_RATE_LIMIT_THRESHOLD, RAPID_RATE_LIMIT_WINDOW_MS, MANUAL_PAUSE_COOLDOWN_MS, RUNS_DIR, pickRunDir };

package/src/preload/api.d.ts

@@ -354,20 +354,23 @@ export interface DelegationReadinessCheck {
     | 'scheduler-mcp-project-duplicate'
     | 'dev-plugin'
     | 'agent-personas'
-    | 'prd-write-guard';
+    | 'prd-write-guard'
+    | 'destructive-git-guard';
   label: string;
   ok: boolean;
   detail: string;
   fix: string | null;
   /** Non-null when Session Manager can install this fix itself, one press. */
-  fixAction: 'install-prd-write-guard' | null;
+  fixAction: 'install-prd-write-guard' | 'install-destructive-git-guard' | null;
   /** True when ok:true is a WARNING (still passing, but worth a human's attention) — today only scheduler-mcp-project-duplicate. */
   warn?: boolean;
   /** True when this check didn't run because a precondition (another check) already failed — reported ok:true, not a failure. */
   skipped?: boolean;
 }
 
-export interface InstallPrdWriteGuardResult {
+/** Result of installPrdWriteGuard and installDestructiveGitGuard alike
+ *  (delegationReadiness.cjs) — the two installers share one contract. */
+export interface InstallGuardResult {
   ok: boolean;
   action: 'installed' | 'repaired' | 'already-installed' | 'error';
   settingsPath?: string;
@@ -1472,7 +1475,8 @@ export interface SessionManagerAPI {
     /** "Can this project actually delegate?" — the 4 preconditions for
      *  scheduler_create_prd being in an agent's tool list at all. */
     delegationReadiness: (cwd: string) => Promise<DelegationReadiness>;
-    installPrdWriteGuard: (cwd: string) => Promise<InstallPrdWriteGuardResult>;
+    installPrdWriteGuard: (cwd: string) => Promise<InstallGuardResult>;
+    installDestructiveGitGuard: (cwd: string) => Promise<InstallGuardResult>;
     onNewSession: (handler: () => void) => () => void;
     onRebootSession: (handler: () => void) => () => void;
     archiveProject: (encoded: string) => Promise<{ ok: boolean; error?: string }>;

package/src/preload/index.cjs

@@ -24,6 +24,7 @@ contextBridge.exposeInMainWorld('api', {
     seedStatus: () => ipcRenderer.invoke('app:seed-status'),
     delegationReadiness: (cwd) => ipcRenderer.invoke('app:delegation-readiness', { cwd }),
     installPrdWriteGuard: (cwd) => ipcRenderer.invoke('app:install-prd-write-guard', { cwd }),
+    installDestructiveGitGuard: (cwd) => ipcRenderer.invoke('app:install-destructive-git-guard', { cwd }),
     onNewSession: (handler) => {
       const listener = () => handler();
       ipcRenderer.on('app:new-session', listener);