claude-code-session-manager 0.35.18 → 0.37.0

package/src/main/queueOps.cjs

@@ -38,6 +38,7 @@ const { SCHEDULE_SLUG_RE: SLUG_RE, schemas } = require('./ipcSchemas.cjs');
 const logs = require('./logs.cjs');
 const config = require('./config.cjs');
 const { expandHome } = require('./lib/expandHome.cjs');
+const { HISTORY_RETENTION_MS } = require('./lib/schedulerConfig.cjs');
 
 const ROOT = path.join(os.homedir(), '.claude', 'session-manager', 'scheduled-plans');
 const PRDS_DIR = path.join(ROOT, 'prds');
@@ -243,6 +244,95 @@ async function archiveMany(slugs) {
   return { ok: true, archived, archivedTo: archiveDir, results };
 }
 
+// ────────────────────────────────────────────── auto-archive completed PRDs
+//
+// PRDS_DIR grows monotonically (738+ .md files, ~100% long-completed) because
+// nothing ever moves a finished PRD's .md out of the live directory — every
+// allocateParallelGroup scan, reconcile listing, and lint pass walks all of
+// them. Once a job is durably 'completed' (queueHistory.cjs's same retention
+// window has already judged it safe to leave jobs[]) and nothing still needs
+// its file in place, move the .md to prds-archived/ alongside it.
+//
+// 'needs_review' and 'failed' PRDs are NEVER selected here — humans and the
+// auto-fix loop (scheduler.cjs's healTargetForFix) still need those files on
+// disk. Same fix-plan-protection regex as queueHistory.cjs's partitionJobs,
+// duplicated (not shared) because the two modules intentionally stay
+// decoupled — kept in sync by comment, same convention queueHistory.cjs uses
+// against scheduler.cjs's healTargetForFix.
+
+const FIX_PLAN_RE = /^\d+-fix-/;
+
+function protectedByPendingFix(jobs) {
+  const protectedSlugs = new Set();
+  for (const j of jobs) {
+    if (!j || !FIX_PLAN_RE.test(j.slug || '')) continue;
+    if (j.status !== 'pending' && j.status !== 'running') continue;
+    protectedSlugs.add(j.slug.replace(/^(\d+)-fix-/, '$1-'));
+  }
+  return protectedSlugs;
+}
+
+/**
+ * Pure predicate: which slugs are safe to move to prds-archived/. O(n) over
+ * `jobs`. A slug qualifies only when ALL of:
+ *   - its job status is 'completed' (never 'needs_review'/'failed'/
+ *     'pending'/'running' — the completed check alone also covers "not
+ *     itself a pending/running job")
+ *   - finishedAt parses and is older than retentionMs (default
+ *     HISTORY_RETENTION_MS, the same constant queueHistory.cjs's
+ *     partitionJobs uses — by the time a file is archive-eligible its queue
+ *     row has already left jobs[] into history.jsonl)
+ *   - no pending/running `NN-fix-<slug>` sibling still needs it in place
+ */
+function selectAutoArchivable(jobs, { nowMs, retentionMs } = {}) {
+  const list = Array.isArray(jobs) ? jobs : [];
+  const retention = retentionMs ?? HISTORY_RETENTION_MS;
+  const now = Number.isFinite(nowMs) ? nowMs : Date.now();
+  const protectedSlugs = protectedByPendingFix(list);
+
+  const slugs = [];
+  for (const j of list) {
+    if (!j || !j.slug) continue;
+    if (j.status !== 'completed') continue;
+    const finishedMs = j.finishedAt ? Date.parse(j.finishedAt) : NaN;
+    if (!Number.isFinite(finishedMs)) continue;
+    if ((now - finishedMs) <= retention) continue;
+    if (protectedSlugs.has(j.slug)) continue;
+    slugs.push(j.slug);
+  }
+  return slugs;
+}
+
+/**
+ * Selects and moves eligible completed PRDs' .md files to prds-archived/.
+ * Reuses archiveMany (no second mover) so moves stay atomic fsp.rename with
+ * the same path-containment. No-op (without touching disk) when
+ * SM_PRD_AUTOARCHIVE_DISABLE=1. Each successful move is logged one line
+ * (slug -> dest).
+ */
+async function autoArchiveCompleted(state, { nowMs } = {}) {
+  if (process.env.SM_PRD_AUTOARCHIVE_DISABLE === '1') {
+    return { ok: true, archived: 0, archivedTo: null, results: [], skipped: 'disabled' };
+  }
+  const jobs = Array.isArray(state?.jobs) ? state.jobs : [];
+  const slugs = selectAutoArchivable(jobs, { nowMs });
+  if (slugs.length === 0) {
+    return { ok: true, archived: 0, archivedTo: null, results: [] };
+  }
+  const result = await archiveMany(slugs);
+  for (const r of result.results) {
+    if (r.ok) {
+      logs.writeLine({
+        level: 'info',
+        scope: 'queueOps',
+        message: `auto-archived completed PRD: ${r.slug} -> ${r.archivedTo}`,
+        meta: { slug: r.slug, dest: r.archivedTo },
+      });
+    }
+  }
+  return result;
+}
+
 // ────────────────────────────────────────────── retag
 
 /**
@@ -398,6 +488,8 @@ module.exports = {
   lintAll,
   lintOneAsync,
   archiveMany,
+  selectAutoArchivable,
+  autoArchiveCompleted,
   retagMany,
   PRDS_DIR,
   PRDS_ARCHIVE_DIR,

package/src/main/scheduler/prdParser.cjs

@@ -172,6 +172,48 @@ const RESERVATION_RE = /^\.reserved-(\d+)$/;
 const GROUP_PREFIX_RE = /^(\d+)-/;
 const MAX_RESERVE_ATTEMPTS = 1000;
 
+// Auto-archiving completed PRDs (see queueOps.cjs's autoArchiveCompleted)
+// moves old `NN-*.md` files out of prdsDir into prds-archived/, which would
+// let maxParallelGroupInUse's directory scan regress once the highest-NN
+// files age out — a later allocation could then reissue an already-used NN
+// and collide with an archived group. A high-water-mark sidecar closes that:
+// once a scan or allocation has seen NN, the allocator never returns
+// anything ≤ NN again, even if every file at or above NN is later archived.
+// Sidecar (not queue.json) so this module stays self-contained and testable
+// against a bare tmp dir, matching the `.reserved-NN` marker convention.
+const MAX_GROUP_FILE = '.max-allocated-group';
+
+async function readHighWaterMark(prdsDir) {
+  try {
+    const raw = await fsp.readFile(path.join(prdsDir, MAX_GROUP_FILE), 'utf8');
+    const n = Number(raw.trim());
+    return Number.isFinite(n) && n >= 0 ? n : 0;
+  } catch {
+    return 0;
+  }
+}
+
+async function writeHighWaterMark(prdsDir, n) {
+  // Re-read immediately before writing so two concurrent allocations that
+  // both raced past the caller's earlier read can't have the smaller `n`
+  // land last and regress the persisted mark below the larger one. Narrows
+  // (does not fully eliminate) the race; the reservation markers remain the
+  // authoritative concurrency guard, this file is a best-effort backstop
+  // for after files are archived away.
+  const current = await readHighWaterMark(prdsDir);
+  if (n <= current) return;
+
+  const dest = path.join(prdsDir, MAX_GROUP_FILE);
+  // `n` itself is unique per concurrent caller (the reservation marker each
+  // caller holds guarantees no two callers ever share a candidate), so it
+  // doubles as a collision-free tmp-file suffix — process.pid + Date.now()
+  // is NOT safe here since concurrent allocateParallelGroup calls in the
+  // same process can land in the same millisecond.
+  const tmp = `${dest}.tmp-${n}-${process.pid}`;
+  await fsp.writeFile(tmp, String(n), 'utf8');
+  await fsp.rename(tmp, dest);
+}
+
 /**
  * Highest `NN` currently in use under prdsDir, across real `NN-slug.md`
  * PRD files (any digit count, unpadded or not — matches groupFromName
@@ -208,7 +250,10 @@ async function maxParallelGroupInUse(prdsDir) {
  */
 async function allocateParallelGroup(prdsDir) {
   await fsp.mkdir(prdsDir, { recursive: true });
-  let candidate = (await maxParallelGroupInUse(prdsDir)) + 1;
+  const scanMax = await maxParallelGroupInUse(prdsDir);
+  const highWater = await readHighWaterMark(prdsDir);
+  const floor = Math.max(scanMax, highWater);
+  let candidate = floor + 1;
   for (let attempt = 0; attempt < MAX_RESERVE_ATTEMPTS; attempt += 1) {
     const markerPath = path.join(prdsDir, `.reserved-${candidate}`);
     let fh;
@@ -222,6 +267,12 @@ async function allocateParallelGroup(prdsDir) {
       throw e;
     }
     await fh.close();
+    // Persist the new floor before returning so a later archive of every
+    // NN-*.md file (including this one) can never regress the allocator
+    // below it.
+    if (candidate > highWater) {
+      await writeHighWaterMark(prdsDir, candidate);
+    }
     return candidate;
   }
   throw new Error(`allocateParallelGroup: exhausted ${MAX_RESERVE_ATTEMPTS} reservation attempts starting at ${candidate}`);

package/src/main/scheduler.cjs

@@ -36,8 +36,11 @@
  *     Orphaned entries (.md gone) are pruned.
  *
  * Renderer events:
- *   - 'schedule:state' broadcasts the full state on any change. Keeps the
- *     panel UI dead simple — no diff machinery.
+ *   - 'schedule:state' broadcasts the full state on mutation. Keeps the
+ *     panel UI dead simple — no diff machinery. Sends are trailing-edge
+ *     debounced (~BROADCAST_COALESCE_MS) by default so a burst of mutations
+ *     collapses into one push; state-machine transitions (pause/resume, job
+ *     start/finish) call broadcast({ flush: true }) to bypass the window.
  */
 
 const fs = require('node:fs');
@@ -55,6 +58,7 @@ const { readTail } = require('./lib/fileTail.cjs');
 const { claudePidAlive, classifyRunOutcome, ORPHAN_REQUEUE_CAP } = require('./lib/reaperHelpers.cjs');
 const { openLog, withChildAndLog } = require('./lib/childWithLog.cjs');
 const { sendIfAlive } = require('./lib/sendToRenderer.cjs');
+const { createBroadcastCoalescer } = require('./lib/broadcastCoalescer.cjs');
 const prdParser = require('./scheduler/prdParser.cjs');
 const { verifyRun } = require('./runVerify.cjs');
 const logs = require('./logs.cjs');
@@ -63,9 +67,13 @@ const {
   POLL_INTERVAL_MS,
   USAGE_REFRESH_INTERVAL_MS,
   MAX_JOB_DURATION_MS,
+  BROADCAST_COALESCE_MS,
 } = require('./lib/schedulerConfig.cjs');
 const { pickForProject, pickNextBatch, DEFAULT_PROJECT_CWD } = require('./lib/schedulerBatch.cjs');
 const { runDefinitionOfDoneOnDrain } = require('./lib/dodDrainHook.cjs');
+const { fileRcaFeedback, extractRcaBlock } = require('./lib/rcaFeedbackHook.cjs');
+const queueHistory = require('./lib/queueHistory.cjs');
+const queueOps = require('./queueOps.cjs');
 
 const MAX_INVESTIGATION_DURATION_MS = 30 * 60_000;
 
@@ -224,9 +232,14 @@ const HEARTBEAT_PATH = path.join(os.homedir(), '.claude', 'session-manager', 'sc
 const HEARTBEAT_MAX_BYTES = 1024 * 1024;
 // DEFAULT_PROJECT_CWD imported from lib/schedulerBatch.cjs (single source of truth).
 
-const ENV_CAP = process.env.SM_SCHEDULER_MAX_CONCURRENCY
-  ? Math.max(1, Math.min(20, parseInt(process.env.SM_SCHEDULER_MAX_CONCURRENCY, 10) || 3))
-  : null;
+// Read lazily (not captured at module-load time) so tests can stub the env
+// per-case without vi.resetModules(); production behavior is unaffected
+// since the env var never changes mid-process.
+function getEnvCap() {
+  return process.env.SM_SCHEDULER_MAX_CONCURRENCY
+    ? Math.max(1, Math.min(20, parseInt(process.env.SM_SCHEDULER_MAX_CONCURRENCY, 10) || 3))
+    : null;
+}
 
 // Each headless claude -p job can shell out to tsc/vite/pytest and grow well
 // past 1 GB at peak; reserve 2.5 GB per running+pending slot. Raised from 1.5 GB
@@ -252,7 +265,7 @@ const OOM_SCORE_ADJ_JOB = 500;
 
 const DEFAULT_CONFIG = {
   offsetMinutes: 15,
-  concurrencyCap: ENV_CAP ?? 3,
+  concurrencyCap: getEnvCap() ?? 3,
   defaultCwd: DEFAULT_PROJECT_CWD,
   // 'when-available' = poll usage and fire whenever utilization < threshold.
   // 'on-reset'        = fire offsetMinutes after the next 5h reset (legacy).
@@ -361,6 +374,46 @@ function ensureDirs() {
   } catch { /* non-fatal: the guide is a convenience, not load-bearing for a run */ }
 }
 
+// Matches only numbered timestamp backups (queue.json.bak-<epoch>), not the
+// bare `queue.json.bak` single-shot copy some older code paths left behind.
+const QUEUE_BAK_RE = /^queue\.json\.bak-\d+$/;
+const QUEUE_BAK_KEEP = 5;
+
+/**
+ * Sweeps accumulated queue.json.bak-<epoch> files down to the newest
+ * QUEUE_BAK_KEEP, deleting the rest. Runs at boot. `entries` comes from
+ * fs.readdir(ROOT), so every candidate path is already contained in ROOT.
+ */
+async function sweepQueueBackups() {
+  let entries;
+  try {
+    entries = await fsp.readdir(ROOT);
+  } catch {
+    return;
+  }
+  const baks = entries.filter((f) => QUEUE_BAK_RE.test(f));
+  if (baks.length <= QUEUE_BAK_KEEP) return;
+
+  baks.sort((a, b) => {
+    const ta = Number(a.slice('queue.json.bak-'.length));
+    const tb = Number(b.slice('queue.json.bak-'.length));
+    return tb - ta; // newest (largest epoch) first
+  });
+  const toDelete = baks.slice(QUEUE_BAK_KEEP);
+  let removed = 0;
+  for (const f of toDelete) {
+    try {
+      await fsp.unlink(path.join(ROOT, f));
+      removed++;
+    } catch (e) {
+      console.warn('[scheduler] backup sweep: unlink failed', f, e?.message);
+    }
+  }
+  if (removed > 0) {
+    console.log(`[scheduler] backup sweep: removed ${removed} old queue.json.bak-* file(s), kept ${QUEUE_BAK_KEEP}`);
+  }
+}
+
 // Atomic JSON write helpers delegate to config.cjs's shared implementation.
 // Sync variant is required for the executeJob exit handler (Promise resolver
 // callback that must flush meta.json before resolving) — replacing with async
@@ -625,8 +678,41 @@ async function reconcile(state) {
       bodyPreview: p.body.split('\n').slice(0, 6).join('\n'),
     });
   }
+  // Slugs on disk with no matching state.jobs row are normally brand-new
+  // PRDs — but once queueHistory.partitionJobs (above, later this same
+  // function) has been dropping terminal jobs into history.jsonl across
+  // prior reconcile passes, an old completed/failed job's row can ALSO be
+  // "unmatched" simply because it already left jobs[]. Without this check
+  // it would get resurrected below as a fresh 'pending' entry and the
+  // scheduler would genuinely re-execute an already-completed PRD — the
+  // exact backlog this auto-archive feature exists to clean up. Only pay
+  // the history lookup when there's actually an unmatched slug to resolve.
+  const unmatchedSlugs = [];
+  for (const [slug] of onDisk) {
+    if (!seen.has(slug)) unmatchedSlugs.push(slug);
+  }
+  const historyBySlug = unmatchedSlugs.length > 0
+    ? await queueHistory.historyTerminalBySlug()
+    : new Map();
+
+  // Terminal-in-history slugs whose .md file is still on disk: fed into the
+  // auto-archive selection pass below (as synthetic completed entries) so
+  // their file can still be swept, without ever creating a live job row
+  // for them (that row already exists, durably, in history.jsonl).
+  const historyArchiveCandidates = [];
+
   for (const [slug, p] of onDisk) {
     if (seen.has(slug)) continue;
+    const hist = historyBySlug.get(slug);
+    if (hist) {
+      if (hist.status === 'completed') {
+        historyArchiveCandidates.push({ slug, status: hist.status, finishedAt: hist.finishedAt });
+      }
+      // 'failed' (or any other terminal status found in history) is left
+      // alone entirely: not resurrected, not archived — matches "needs_
+      // review and failed PRD files are NEVER auto-archived."
+      continue;
+    }
     const entry = {
       slug,
       title: p.title,
@@ -651,7 +737,44 @@ async function reconcile(state) {
     }
     next.push(entry);
   }
-  state.jobs = next.sort((a, b) => b.slug.localeCompare(a.slug));
+  const sorted = next.sort((a, b) => b.slug.localeCompare(a.slug));
+
+  // Move terminal jobs past the retention window out to history.jsonl so
+  // queue.json (mutation cost, broadcast payload, pickNextBatch scan) stays
+  // small. Append BEFORE dropping so a crash between the two can't lose a
+  // record — appendHistory dedupes by slug+runId, so a replay of the same
+  // batch on next boot is a safe no-op.
+  const nowMs = Date.now();
+  const { hot, toArchive } = queueHistory.partitionJobs(sorted, nowMs);
+  if (toArchive.length > 0) {
+    await queueHistory.appendHistory(toArchive);
+    console.log(`[scheduler] queue history: archived ${toArchive.length} job(s), jobs[] ${sorted.length} -> ${hot.length}`);
+    state.jobs = hot;
+  } else {
+    state.jobs = sorted;
+  }
+
+  // Auto-archive completed PRDs' .md files out of the live prds/ dir. Runs
+  // AFTER the history append above (which is awaited) so a job's queue row
+  // is always durably in history.jsonl before its file can be moved — a
+  // file is never removed ahead of its queue row. `sorted` (not `hot`) is
+  // passed so a job that just crossed retention THIS pass is eligible for
+  // file-archiving in the same pass, per its already-persisted history row.
+  // `historyArchiveCandidates` adds slugs whose row left jobs[] on an
+  // EARLIER pass (or before this feature shipped) — same predicate applies
+  // via selectAutoArchivable, they just don't get a live job row.
+  try {
+    const archiveInput = historyArchiveCandidates.length > 0
+      ? sorted.concat(historyArchiveCandidates)
+      : sorted;
+    const archiveResult = await queueOps.autoArchiveCompleted({ jobs: archiveInput }, { nowMs });
+    if (archiveResult.archived > 0) {
+      console.log(`[scheduler] auto-archived ${archiveResult.archived} completed PRD file(s) -> ${archiveResult.archivedTo}`);
+    }
+  } catch (e) {
+    console.warn('[scheduler] autoArchiveCompleted failed', e?.message);
+  }
+
   return state;
 }
 
@@ -775,6 +898,10 @@ function buildScheduleStatePayload(state, { withPaths = false } = {}) {
       lastFailureKind,
     },
     memGate: lastMemGate,
+    effectiveConcurrency: {
+      cap: getEnvCap() ?? state.config.concurrencyCap,
+      source: getEnvCap() != null ? 'env' : 'config',
+    },
   };
   if (withPaths) {
     payload.paths = { root: ROOT, prds: PRDS_DIR, runs: RUNS_DIR, queue: QUEUE_PATH };
@@ -782,12 +909,32 @@ function buildScheduleStatePayload(state, { withPaths = false } = {}) {
   return payload;
 }
 
-async function broadcast() {
+// Trailing-edge debounce for the `schedule:state` IPC push: a burst of
+// broadcast() calls (boot reverify healing several rows, poll-loop refreshes,
+// queue-linter fixups) arms one BROADCAST_COALESCE_MS timer and sends a
+// single payload built fresh at fire time, instead of one full-payload push
+// per mutation. Callers where latency matters (pause/resume, job
+// start/finish/reap/reset) pass `{ flush: true }` to bypass the window and
+// send immediately.
+const broadcastCoalescer = createBroadcastCoalescer({
+  delayMs: BROADCAST_COALESCE_MS,
+  send: (payload) => {
+    if (!mainWindow || mainWindow.isDestroyed()) return;
+    sendIfAlive(mainWindow, 'schedule:state', payload);
+  },
+  getPayload: async () => buildScheduleStatePayload(await readQueue()),
+});
+
+async function broadcast(opts = {}) {
   if (!mainWindow || mainWindow.isDestroyed()) return;
   const state = await readQueue();
   await reconcile(state);
   await writeQueue(state);
-  sendIfAlive(mainWindow, 'schedule:state', buildScheduleStatePayload(state));
+  if (opts.flush) {
+    await broadcastCoalescer.flush();
+  } else {
+    broadcastCoalescer.schedule();
+  }
 }
 
 function clearFireTimer() {
@@ -855,7 +1002,7 @@ async function setPaused(reason, resumeAtIso) {
       s.paused = { reason, since: new Date().toISOString(), resumeAt: effectiveResumeAt || null };
     }
   });
-  await broadcast();
+  await broadcast({ flush: true });
   cancelToken.cancelled = true;
   if (resumeTimer) { clearTimeout(resumeTimer); resumeTimer = null; }
   if (!effectiveResumeAt) return;
@@ -897,7 +1044,7 @@ async function clearPause(source) {
     lastFailureKind = null;
     persistSchedulerState();
   }
-  if (wasPaused) await broadcast();
+  if (wasPaused) await broadcast({ flush: true });
 }
 
 /** Mutate a job in place to "pending" with cleared run metadata. */
@@ -1312,6 +1459,17 @@ ${logTail}
 1. Read the full failure log at ${failedLogPath} if the tail above isn't sufficient.
 2. Read source files in ${cwd} as needed to understand the context.
 3. Identify the root cause of the failure.
+3.5. Before writing the fix-plan PRD file, print a block summarizing the root
+   cause in 15 lines or fewer, in exactly this form (plain text between the
+   tags, no markdown fences):
+
+   <RCA>
+   <your root-cause summary here>
+   </RCA>
+
+   This is parsed out of your transcript and folded into the RCA feedback item
+   already filed for this job, so it must stand alone (the reader won't have
+   your reasoning, only this block).
 4. Write a NEW fix-plan PRD file at exactly this path:
 
    ${fixPath}
@@ -1482,6 +1640,21 @@ async function spawnInvestigation(failedJob, runDir) {
         return;
       }
       sl(`\n[scheduler] investigation exit code=${exitCode}\n`);
+      // Fold the investigation's <RCA> summary into the RCA feedback item already
+      // filed for this job (needs_review jobs only — fileRcaFeedback no-ops when
+      // failedJob has no verifierVerdict, e.g. plain 'failed' jobs never got one).
+      const investigationText = extractRcaBlock(readTail(investigationLogPath, 64 * 1024));
+      if (investigationText) {
+        fileRcaFeedback({
+          job: failedJob,
+          runDir,
+          verdict: failedJob.verifierVerdict,
+          annotations: failedJob.verifierAnnotations,
+          investigationText,
+        }).catch((e) => {
+          console.error('[scheduler] fileRcaFeedback (investigation enrichment) error', failedJob.slug, e);
+        });
+      }
       if (fs.existsSync(fixPath)) {
         console.log(`[scheduler] investigation produced fix plan: ${fixSlug}`);
       } else {
@@ -1522,7 +1695,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
         s.jobs[idx].startedAt = new Date().toISOString();
       }
     });
-    await broadcast();
+    await broadcast({ flush: true });
 
     // Commit-guard baseline: snapshot the working tree BEFORE the run so the
     // post-run check flags only paths THIS job left dirty, not pre-existing WIP.
@@ -1538,7 +1711,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
           s.jobs[idx].runtime = { pid, runId, startedAt: s.jobs[idx].startedAt, sessionId, cwd };
         }
       });
-      await broadcast();
+      await broadcast({ flush: true });
     });
 
     if (res.rateLimited) {
@@ -1635,6 +1808,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
     let failedJobSnapshot = null;
     let needsInvestigationNow = false;
     let investigationJobSnapshot = null;
+    let needsReviewRcaSnapshot = null;
     await mutate((s) => {
       const i2 = s.jobs.findIndex((x) => x.slug === job.slug);
       if (i2 >= 0) {
@@ -1692,6 +1866,12 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
             actuallyFailed = true;
             failedJobSnapshot = { ...s.jobs[i2] };
           } else if (effectiveStatus === 'needs_review') {
+            // Snapshot for the RCA feedback hook (fired outside mutate(), below).
+            // Every needs_review transition gets an RCA — including uncommitted_changes,
+            // the least self-healing verdict — but never a rate-limit pause (that
+            // takes the treatAsPending branch above and never reaches here).
+            needsReviewRcaSnapshot = { ...s.jobs[i2] };
+
             // Same-tick auto-fix (feedback 2026-07-12): rather than waiting up to
             // 10 min for reverifyNeedsReview()'s periodic pass, check right here
             // whether this job qualifies for auto-fix (same eligibility rule
@@ -1738,7 +1918,20 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
         }
       }
     });
-    await broadcast();
+    await broadcast({ flush: true });
+
+    if (needsReviewRcaSnapshot) {
+      // Fire-and-forget, mirroring the DoD drain hook: never blocks the status
+      // transition, never throws to this caller.
+      fileRcaFeedback({
+        job: needsReviewRcaSnapshot,
+        runDir,
+        verdict: needsReviewRcaSnapshot.verifierVerdict,
+        annotations: needsReviewRcaSnapshot.verifierAnnotations,
+      }).catch((e) => {
+        console.error('[scheduler] fileRcaFeedback error', job.slug, e);
+      });
+    }
 
     if (actuallyFailed && failedJobSnapshot) {
       // Transient-failure detector. A 143/137 exit is ALWAYS a signal kill — the
@@ -1786,7 +1979,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
             s.jobs[i].transientRetries = decision.retries + 1;
           }
         });
-        await broadcast();
+        await broadcast({ flush: true });
       } else if (decision.action === 'fail-dirty') {
         console.log(`[scheduler] transient failure (${decision.transientKind}) for ${job.slug} left ${newlyDirtyCount} uncommitted file(s) (e.g. ${dirtySample}) — not auto-requeuing`);
         await mutate((s) => {
@@ -1796,7 +1989,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
             s.jobs[i].error = `transient failure (${decision.transientKind}) left ${newlyDirtyCount} uncommitted file(s) in working tree (e.g. ${dirtySample}) — not auto-requeued to avoid overwriting partial work; review and commit or discard manually`;
           }
         });
-        await broadcast();
+        await broadcast({ flush: true });
         // No auto-fix investigation: this isn't a code defect, and the
         // dirty tree needs a human, not a diagnosis run.
       } else if (decision.action === 'fail-cap') {
@@ -1810,7 +2003,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
             s.jobs[i].error = `transient failure (${decision.transientKind}) exhausted retry cap (${decision.retries}/${TRANSIENT_RETRY_CAP}) — marking failed without auto-fix investigation`;
           }
         });
-        await broadcast();
+        await broadcast({ flush: true });
       } else {
         spawnInvestigation(failedJobSnapshot, runDir).catch((e) => {
           console.error('[scheduler] spawnInvestigation error', job.slug, e);
@@ -1846,7 +2039,7 @@ function tickQueue() {
     if (cancelToken.cancelled) return { fired: false, reason: 'cancelled' };
 
     await reconcile(state);
-    const cap = ENV_CAP ?? state.config.concurrencyCap;
+    const cap = getEnvCap() ?? state.config.concurrencyCap;
     const { batch, reason: holdReason } = pickNextBatch(state.jobs, runningSet, cap);
     if (batch.length === 0) {
       // Queue drained — run the definition-of-done gate fire-and-forget.
@@ -2009,7 +2202,7 @@ async function reapDeadRunningJobs() {
       }
     });
 
-    await broadcast();
+    await broadcast({ flush: true });
     tickQueue().catch(() => {});
   } catch (e) {
     console.warn('[scheduler] reapDeadRunningJobs error', e?.message);
@@ -2149,13 +2342,26 @@ async function pollLoop() {
 
 // ---------- IPC ----------
 
-// Pure helper: filter to completed/failed, sort newest-finished first, cap to
-// limit clamped to [1, 500] (default 50). O(n log n) on queue size (small).
-// Exported for unit testing.
-function selectHistoryJobs(jobs, limit) {
+// Pure helper: filter to completed/failed, merge with archived history
+// entries (now that terminal jobs age out of jobs[] into history.jsonl —
+// see queueHistory.cjs), dedupe by slug+runId (jobs[] wins on overlap —
+// it's the fresher copy in the append-before-drop crash window), sort
+// newest-finished first, cap to limit clamped to [1, 500] (default 50).
+// O(n log n) on queue+history size (small at any realistic scale).
+// Exported for unit testing. `historyEntries` defaults to [] so existing
+// callers/tests that only pass `jobs` are unaffected.
+function selectHistoryJobs(jobs, limit, historyEntries = []) {
   const cap = Math.max(1, Math.min(500, Number.isFinite(limit) ? Math.floor(limit) : 50));
-  return (Array.isArray(jobs) ? jobs : [])
-    .filter((j) => j && (j.status === 'completed' || j.status === 'failed'))
+  const hot = (Array.isArray(jobs) ? jobs : []).filter((j) => j && (j.status === 'completed' || j.status === 'failed'));
+  const seen = new Set(hot.map((j) => `${j.slug}|${j.runId ?? ''}`));
+  const archived = (Array.isArray(historyEntries) ? historyEntries : []).filter((j) => {
+    if (!j) return false;
+    const key = `${j.slug}|${j.runId ?? ''}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+  return [...hot, ...archived]
     .sort((a, b) => {
       const at = a.finishedAt ? Date.parse(a.finishedAt) : 0;
       const bt = b.finishedAt ? Date.parse(b.finishedAt) : 0;
@@ -2550,7 +2756,7 @@ function registerScheduleHandlers() {
       return true;
     });
     if (!found) return { ok: false, error: 'not found' };
-    await broadcast();
+    await broadcast({ flush: true });
     return { ok: true };
   }));
 
@@ -2713,7 +2919,8 @@ function registerScheduleHandlers() {
     const limit = (payload && typeof payload.limit === 'number') ? payload.limit : 50;
     try {
       const state = await readQueue();
-      return { ok: true, jobs: selectHistoryJobs(state.jobs, limit) };
+      const historyEntries = await queueHistory.readHistory({ limit });
+      return { ok: true, jobs: selectHistoryJobs(state.jobs, limit, historyEntries) };
     } catch (e) {
       return { ok: false, jobs: [], error: e?.message ?? 'read failed' };
     }
@@ -2722,6 +2929,7 @@ function registerScheduleHandlers() {
 
 async function init() {
   ensureDirs();
+  sweepQueueBackups().catch((e) => console.warn('[scheduler] backup sweep failed', e?.message));
 
   // Hydrate cached state from the sidecar before any scheduling decisions.
   loadSchedulerState();
@@ -2970,7 +3178,7 @@ const remote = {
       return true;
     });
     if (!found) return { ok: false, error: 'not found' };
-    await broadcast();
+    await broadcast({ flush: true });
     return { ok: true, slug, status: 'pending' };
   },
 
@@ -3008,4 +3216,4 @@ const remote = {
   },
 };
 
-module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, healTargetForFix, buildInvestigationPrompt, committedInWindow, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, TRANSIENT_RETRY_CAP };
+module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, healTargetForFix, buildInvestigationPrompt, committedInWindow, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, TRANSIENT_RETRY_CAP, buildScheduleStatePayload };