claude-code-session-manager 0.66.0 → 0.68.0

package/src/main/__tests__/scheduler-stall-per-project.test.cjs

@@ -0,0 +1,112 @@
+/**
+ * scheduler-stall-per-project.test.cjs — unit tests for computeStallSummary's
+ * per-project `stalled` verdict and findStaleQuarantinedJobs' age-based
+ * escalation.
+ *
+ * Covers the exact burrow-vs-others shape observed live: one project (burrow)
+ * holds only quarantined rows (0 running, 0 pending) while another project
+ * has running + pending work — the machine-wide `stalled` boolean reads
+ * false (masking burrow), but burrow's own byProject entry must read true.
+ *
+ * Run: timeout 120 npx vitest run src/main/__tests__/scheduler-stall-per-project.test.cjs
+ */
+
+'use strict';
+
+// vitest, NOT node:test — this repo's suite is vitest-only (see CLAUDE.md:
+// "This repo does not use `node --test`"). A node:test file loads under
+// vitest as "No test suite found" and contributes zero tests, so the guard
+// silently doesn't exist. node:assert works fine inside a vitest run.
+import { test } from 'vitest';
+const assert = require('node:assert/strict');
+const { computeStallSummary, findStaleQuarantinedJobs, QUARANTINE_ESCALATE_MS } = require('../scheduler.cjs');
+
+test('computeStallSummary: burrow fully stalled (quarantined only) while another project is busy — machine-wide false, burrow true', () => {
+  const state = {
+    paused: null,
+    jobs: [
+      { slug: '821-a', cwd: '/home/bilko/Projects/burrow', status: 'quarantined' },
+      { slug: '822-b', cwd: '/home/bilko/Projects/burrow', status: 'quarantined' },
+      { slug: '823-c', cwd: '/home/bilko/Projects/burrow', status: 'quarantined' },
+      { slug: '824-d', cwd: '/home/bilko/Projects/burrow', status: 'quarantined' },
+      { slug: '900-e', cwd: '/home/bilko/Projects/other', status: 'running' },
+      { slug: '901-f', cwd: '/home/bilko/Projects/other', status: 'pending' },
+    ],
+    invalidJobs: [],
+  };
+  const summary = computeStallSummary(state);
+  assert.strictEqual(summary.stalled, false, 'machine-wide must NOT read stalled — other project is busy');
+  assert.strictEqual(summary.byProject['/home/bilko/Projects/burrow'].stalled, true, 'burrow must read stalled on its own');
+  assert.strictEqual(summary.byProject['/home/bilko/Projects/other'].stalled, false);
+});
+
+test('computeStallSummary: both projects busy — neither stalled', () => {
+  const state = {
+    paused: null,
+    jobs: [
+      { slug: 'a', cwd: '/p1', status: 'running' },
+      { slug: 'b', cwd: '/p2', status: 'pending' },
+    ],
+    invalidJobs: [],
+  };
+  const summary = computeStallSummary(state);
+  assert.strictEqual(summary.stalled, false);
+  assert.strictEqual(summary.byProject['/p1'].stalled, false);
+  assert.strictEqual(summary.byProject['/p2'].stalled, false);
+});
+
+test('computeStallSummary: both projects stalled — machine-wide AND both per-project read true', () => {
+  const state = {
+    paused: null,
+    jobs: [
+      { slug: 'a', cwd: '/p1', status: 'failed' },
+      { slug: 'b', cwd: '/p2', status: 'needs_review' },
+    ],
+    invalidJobs: [],
+  };
+  const summary = computeStallSummary(state);
+  assert.strictEqual(summary.stalled, true);
+  assert.strictEqual(summary.byProject['/p1'].stalled, true);
+  assert.strictEqual(summary.byProject['/p2'].stalled, true);
+});
+
+test('computeStallSummary: paused scheduler never reads stalled, machine-wide or per-project', () => {
+  const state = {
+    paused: { reason: 'rate_limit' },
+    jobs: [{ slug: 'a', cwd: '/p1', status: 'quarantined' }],
+    invalidJobs: [],
+  };
+  const summary = computeStallSummary(state);
+  assert.strictEqual(summary.stalled, false);
+  assert.strictEqual(summary.byProject['/p1'].stalled, false);
+});
+
+test('findStaleQuarantinedJobs: fires past threshold, not before', () => {
+  const now = Date.parse('2026-08-08T12:00:00.000Z');
+  const jobs = [
+    {
+      slug: 'fresh',
+      cwd: '/p1',
+      status: 'quarantined',
+      statusHistory: [{ to: 'quarantined', at: new Date(now - 1 * 60 * 60_000).toISOString() }],
+    },
+    {
+      slug: 'stale',
+      cwd: '/p1',
+      status: 'quarantined',
+      statusHistory: [{ to: 'quarantined', at: new Date(now - 25 * 60 * 60_000).toISOString() }],
+    },
+    {
+      slug: 'exactly-at-threshold',
+      cwd: '/p1',
+      status: 'quarantined',
+      statusHistory: [{ to: 'quarantined', at: new Date(now - QUARANTINE_ESCALATE_MS).toISOString() }],
+    },
+    { slug: 'no-history', cwd: '/p1', status: 'quarantined' },
+    { slug: 'not-quarantined', cwd: '/p1', status: 'pending' },
+  ];
+  const stale = findStaleQuarantinedJobs(jobs, now, QUARANTINE_ESCALATE_MS);
+  const slugs = stale.map((s) => s.slug).sort();
+  assert.deepStrictEqual(slugs, ['exactly-at-threshold', 'stale']);
+  assert.strictEqual(stale.find((s) => s.slug === 'stale').cwd, '/p1');
+});

package/src/main/health.cjs

@@ -14,6 +14,7 @@ const { checkPersonaImports } = require('./lib/personaImportHealth.cjs');
 const { resolvePrdsDirs } = require('./lib/prdLocations.cjs');
 const { migratePrds } = require('./lib/prdMigration.cjs');
 const queueStore = require('./lib/queueStore.cjs');
+const { computeStallSummary } = require('./scheduler.cjs');
 const { DEFAULT_RUNS_DIR, computeReport, isRetentionEnabled, liveKeysFromJobs } = require('./lib/runLogRetention.cjs');
 
 const MAX_LOG_AGE_MS = 5 * 60_000; // 5 min — warn if no logs this old
@@ -131,6 +132,55 @@ function evaluateTickLiveness(queueState, heartbeat, now, runningCount) {
   };
 }
 
+// computeProjectProblemCounts(jobs) → { [cwd]: { failed, needs_review, quarantined } }
+//
+// health.cjs's machine-wide `failed` count answered "is the machine stuck",
+// never "is any ONE project stuck" — a single project with 4 quarantined
+// PRDs and nothing else running was invisible in a rollup dominated by other
+// projects' healthy jobs. Breaks down every non-terminal-problem status
+// (failed/needs_review/quarantined — deliberately NOT 'completed'/'running'/
+// 'pending'/'investigating', which are not problems) by project cwd.
+function computeProjectProblemCounts(jobs) {
+  const byProject = {};
+  for (const j of jobs || []) {
+    if (j.status !== 'failed' && j.status !== 'needs_review' && j.status !== 'quarantined') continue;
+    const cwd = j.cwd || '(unknown)';
+    byProject[cwd] = byProject[cwd] || { failed: 0, needs_review: 0, quarantined: 0 };
+    byProject[cwd][j.status] += 1;
+  }
+  return byProject;
+}
+
+// evaluatePerProjectStall(stallSummary, lastRunAtIso, now, thresholdMs) →
+// { [cwd]: { stalled, pastThreshold?, ageMs?, caveat? } }
+//
+// computeStallSummary's per-project `stalled` flag (scheduler.cjs) is a
+// point-in-time verdict with no duration attached — a project can flip
+// stalled/unstalled within a single tick as work completes, so flagging RED
+// the instant it's true would false-trip on ordinary queue churn. There is
+// no per-project lastRunAt persisted (only a machine-wide one), so this
+// reuses that machine-wide timestamp as the best available "has the
+// scheduler ticked recently at all" signal, gated per-project by whether
+// THAT project currently holds stalled work.
+function evaluatePerProjectStall(stallSummary, lastRunAtIso, now, thresholdMs) {
+  const lastRunAt = lastRunAtIso ? Date.parse(lastRunAtIso) : null;
+  const results = {};
+  for (const cwd of Object.keys(stallSummary?.byProject || {})) {
+    const counts = stallSummary.byProject[cwd];
+    if (!counts.stalled) {
+      results[cwd] = { stalled: false };
+      continue;
+    }
+    if (lastRunAt == null || Number.isNaN(lastRunAt)) {
+      results[cwd] = { stalled: true, pastThreshold: false, caveat: 'no-lastRunAt' };
+      continue;
+    }
+    const ageMs = now - lastRunAt;
+    results[cwd] = { stalled: true, pastThreshold: ageMs >= thresholdMs, ageMs };
+  }
+  return results;
+}
+
 // Pure evaluator over migratePrds()'s { moved, skipped, unresolved } result —
 // kept separate from the fs-touching check() call site so it's directly
 // unit-testable, matching evaluateTickLiveness's pattern.
@@ -222,18 +272,42 @@ async function check() {
     const failedCount = Object.values(queueState.jobs || {}).filter(
       (j) => j.status === 'failed'
     ).length;
+    const needsReviewCount = Object.values(queueState.jobs || {}).filter(
+      (j) => j.status === 'needs_review'
+    ).length;
+    const quarantinedCount = Object.values(queueState.jobs || {}).filter(
+      (j) => j.status === 'quarantined'
+    ).length;
     const heartbeatPath = path.join(
       os.homedir(),
       '.claude/session-manager/scheduler-heartbeat.log'
     );
     const heartbeat = readFreshHeartbeat(heartbeatPath);
     const liveness = evaluateTickLiveness(queueState, heartbeat, Date.now(), runningCount);
+
+    // Per-project rollup (PRD: monitoring must not collapse per-project
+    // reality into one machine-wide boolean — see computeStallSummary /
+    // computeProjectProblemCounts headers). A project holding ONLY
+    // failed/needs_review/quarantined rows (0 running, 0 pending) never
+    // trips evaluateTickLiveness above, since that check requires actual
+    // pending work — this is what let the burrow project go dark.
+    const stallSummary = computeStallSummary(queueState);
+    const now = Date.now();
+    const perProjectStall = evaluatePerProjectStall(stallSummary, queueState.lastRunAt, now, TICK_STALL_THRESHOLD_MS);
+    const projectsPastThreshold = Object.entries(perProjectStall)
+      .filter(([, v]) => v.pastThreshold)
+      .map(([cwd]) => cwd);
+
     status.components.scheduler_queue = {
-      ok: !liveness.stalled,
+      ok: !liveness.stalled && projectsPastThreshold.length === 0,
       path: queuePath,
       jobs: Object.keys(queueState.jobs || {}).length,
       running: runningCount,
       failed: failedCount,
+      needsReview: needsReviewCount,
+      quarantined: quarantinedCount,
+      byProject: computeProjectProblemCounts(queueState.jobs),
+      perProjectStall,
       tickLiveness: liveness.reason,
     };
     if (liveness.stalled) {
@@ -249,6 +323,18 @@ async function check() {
           ? `Tick hasn't advanced in a while but scheduler-heartbeat.log is missing/stale, so current billing utilization can't be checked — cannot rule out a legitimate when-available hold`
           : 'No lastRunAt recorded yet — cannot assess tick liveness';
     }
+    if (projectsPastThreshold.length > 0) {
+      status.components.scheduler_queue.stalledProjects = projectsPastThreshold;
+      for (const cwd of projectsPastThreshold) {
+        const ageMin = Math.round(perProjectStall[cwd].ageMs / 60_000);
+        const counts = status.components.scheduler_queue.byProject[cwd] || {};
+        status.issues.push(
+          `Project fully stalled: ${cwd} — 0 running, 0 pending, only problem jobs `
+          + `(failed=${counts.failed ?? 0} needs_review=${counts.needs_review ?? 0} quarantined=${counts.quarantined ?? 0}), `
+          + `no scheduler tick in ~${ageMin}m (threshold ${Math.round(TICK_STALL_THRESHOLD_MS / 60_000)}m)`
+        );
+      }
+    }
   } catch (e) {
     if (e.code !== 'ENOENT') {
       status.issues.push(`Scheduler queue unreadable: ${e.message}`);
@@ -422,4 +508,13 @@ if (require.main === module) {
   })();
 }
 
-module.exports = { check, evaluateTickLiveness, readFreshHeartbeat, evaluatePrdMigrationHealth, TICK_STALL_THRESHOLD_MS, HEARTBEAT_STALE_MS };
+module.exports = {
+  check,
+  evaluateTickLiveness,
+  readFreshHeartbeat,
+  evaluatePrdMigrationHealth,
+  computeProjectProblemCounts,
+  evaluatePerProjectStall,
+  TICK_STALL_THRESHOLD_MS,
+  HEARTBEAT_STALE_MS,
+};

package/src/main/lib/schedulerConfig.cjs

@@ -36,4 +36,26 @@ module.exports = {
   // rollup line current (see historyAggregator.cjs's refreshIntradayToday).
   // Cheap: LRU-warm live parse, no full transcript re-read.
   HISTORY_INTRADAY_REFRESH_MS: 5 * 60_000,
+  // A 'quarantined' PRD (no createdVia provenance) sitting un-adopted past
+  // this age is escalated: warn-logged naming project + slug + age, and
+  // surfaced distinctly on Home (see homeNeedsYou.ts's matching constant)
+  // so it cannot be stranded indefinitely with nothing looking at it — see
+  // findStaleQuarantinedJobs in scheduler.cjs. Overridable via
+  // SM_QUARANTINE_ESCALATE_HOURS for testing/tuning.
+  QUARANTINE_ESCALATE_MS: 24 * 60 * 60_000,
+
+  // A RUNNING job that has overrun its own PRD's `estimateMinutes` by this
+  // factor is escalated. Distinct from MAX_JOB_DURATION_MS (4h), which is a
+  // deadman kill: a 20-minute PRD still running at 3h is 9x over estimate but
+  // comfortably under the deadman, and if it keeps writing to its log the
+  // 20-minute IDLE_OUTPUT_KILL_MS watchdog never fires either — so before
+  // this, the ONLY signal was a human happening to notice. This escalates,
+  // it does NOT kill: overrunning is evidence of trouble, not proof of it,
+  // and killing on an estimate would murder legitimately-slow work.
+  // Override with SM_JOB_OVERRUN_FACTOR.
+  JOB_OVERRUN_FACTOR: 3,
+  // Floor so a tiny estimate can't escalate almost immediately — a 5-minute
+  // PRD at 3x is 15 minutes, which is noise. Override with
+  // SM_JOB_OVERRUN_FLOOR_MINUTES.
+  JOB_OVERRUN_FLOOR_MS: 45 * 60_000,
 };

package/src/main/scheduler.cjs

@@ -75,7 +75,19 @@ const {
   USAGE_REFRESH_INTERVAL_MS,
   MAX_JOB_DURATION_MS,
   BROADCAST_COALESCE_MS,
+  QUARANTINE_ESCALATE_MS: QUARANTINE_ESCALATE_MS_DEFAULT,
+  JOB_OVERRUN_FACTOR: JOB_OVERRUN_FACTOR_DEFAULT,
+  JOB_OVERRUN_FLOOR_MS: JOB_OVERRUN_FLOOR_MS_DEFAULT,
 } = require('./lib/schedulerConfig.cjs');
+const QUARANTINE_ESCALATE_MS = process.env.SM_QUARANTINE_ESCALATE_HOURS
+  ? Number(process.env.SM_QUARANTINE_ESCALATE_HOURS) * 60 * 60_000
+  : QUARANTINE_ESCALATE_MS_DEFAULT;
+const JOB_OVERRUN_FACTOR = process.env.SM_JOB_OVERRUN_FACTOR
+  ? Number(process.env.SM_JOB_OVERRUN_FACTOR)
+  : JOB_OVERRUN_FACTOR_DEFAULT;
+const JOB_OVERRUN_FLOOR_MS = process.env.SM_JOB_OVERRUN_FLOOR_MINUTES
+  ? Number(process.env.SM_JOB_OVERRUN_FLOOR_MINUTES) * 60_000
+  : JOB_OVERRUN_FLOOR_MS_DEFAULT;
 const { pickForProject, pickNextBatch, DEFAULT_PROJECT_CWD } = require('./lib/schedulerBatch.cjs');
 const { runDefinitionOfDoneOnDrain } = require('./lib/dodDrainHook.cjs');
 const { writeRcaReport, extractRcaBlock } = require('./lib/rcaReport.cjs');
@@ -961,13 +973,24 @@ function appendHeartbeat(entry) {
  * computeStallSummary(state) → { stalled, total, running, pending, byProject }
  *
  * Pure, no IO. `state` is a merged queue-store read ({ jobs, invalidJobs,
- * paused }). "Stalled" = the queue holds work — valid rows OR rows
- * quarantined for an invalid status — but nothing is running or pending and
- * the scheduler isn't paused. The 2026-08-07 incident sat exactly in this
- * state for 4+ hours: 2 jobs, 0 running, 0 pending, and the only visible
- * symptom was a heartbeat `counts` object that had silently minted a
- * `queued` bucket instead of reporting anything actionable. `byProject`
- * breaks the stalled rows down by cwd for the log line / toast.
+ * paused }). The engine (reconcile/reaper/auto-fix/reverify) already
+ * operates machine-wide via queueStore's stateCwds() — this function is
+ * MONITORING, and monitoring must not collapse per-project reality into one
+ * boolean. `stalled` (top-level) is the pre-existing machine-wide roll-up:
+ * the queue holds work — valid rows OR rows quarantined for an invalid
+ * status — but nothing anywhere is running or pending and the scheduler
+ * isn't paused. The 2026-08-07 incident sat exactly in this state for 4+
+ * hours: 2 jobs, 0 running, 0 pending, and the only visible symptom was a
+ * heartbeat `counts` object that had silently minted a `queued` bucket
+ * instead of reporting anything actionable.
+ *
+ * `byProject[cwd].stalled` is the PER-PROJECT verdict added for the
+ * "burrow went dark while other projects were busy" gap: a project can hold
+ * jobs (including ones parked `quarantined`) with 0 running and 0 pending
+ * while the machine-wide `stalled` above reads false because a different
+ * project has running/pending work. Each project's own status counts
+ * (`byProject[cwd][status]`) already summed to a total before this — the
+ * fix is only the boolean, not the counting.
  */
 function computeStallSummary(state) {
   const jobs = Array.isArray(state?.jobs) ? state.jobs : [];
@@ -989,9 +1012,91 @@ function computeStallSummary(state) {
   }
   const total = jobs.length + invalidJobs.length;
   const stalled = total > 0 && running === 0 && pending === 0 && !state?.paused;
+  for (const key of Object.keys(byProject)) {
+    const counts = byProject[key];
+    const projRunning = counts.running || 0;
+    const projPending = counts.pending || 0;
+    const projTotal = Object.keys(counts)
+      .filter((k) => k !== 'stalled')
+      .reduce((sum, k) => sum + counts[k], 0);
+    counts.stalled = projTotal > 0 && projRunning === 0 && projPending === 0 && !state?.paused;
+  }
   return { stalled, total, running, pending, byProject };
 }
 
+/**
+ * findStaleQuarantinedJobs(jobs, now, thresholdMs) → [{ slug, cwd, ageMs }]
+ *
+ * Pure, no IO. A 'quarantined' row (no createdVia provenance) can otherwise
+ * sit forever with nothing looking at it — quarantine only ever clears via a
+ * human adopting or archiving it. This is the escalation half of that gate:
+ * any quarantined row whose recorded quarantine timestamp (statusHistory's
+ * `to === 'quarantined'` entry — stamped at creation, or backfilled from the
+ * PRD file's mtime by reconcile() for rows quarantined before that stamp
+ * existed) is older than `thresholdMs` is reported so the caller can
+ * warn-log and surface it distinctly. A row with no recoverable timestamp is
+ * skipped rather than guessed at.
+ */
+function findStaleQuarantinedJobs(jobs, now, thresholdMs) {
+  const stale = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'quarantined') continue;
+    const entry = (j.statusHistory || []).find((h) => h.to === 'quarantined');
+    if (!entry) continue;
+    const since = Date.parse(entry.at);
+    if (Number.isNaN(since)) continue;
+    const ageMs = now - since;
+    if (ageMs >= thresholdMs) stale.push({ slug: j.slug, cwd: j.cwd ?? null, ageMs });
+  }
+  return stale;
+}
+
+/**
+ * findOverrunningJobs(jobs, now, { factor, floorMs }) → [{ slug, cwd, estimateMinutes, ranMs, ratio }]
+ *
+ * Pure, no IO. A RUNNING job whose elapsed time exceeds
+ * `max(estimateMinutes * factor, floorMs)` is overrunning its own PRD's
+ * declared estimate.
+ *
+ * This closes the gap between the two existing kill paths, which a
+ * long-running-but-chatty job slips straight through:
+ *  - MAX_JOB_DURATION_MS (4h) is a deadman — a 20-minute PRD at 3h is 9x
+ *    over estimate and still an hour away from it.
+ *  - IDLE_OUTPUT_KILL_MS (20m) only fires when the log mtime STALLS; an
+ *    agent stuck in a productive-looking loop keeps writing and never trips it.
+ * `estimateMinutes` was parsed, stored and displayed but never once compared
+ * against actual runtime, so the only thing standing between a runaway job
+ * and the 4h ceiling was a human noticing. (2026-08-08: a PRD ran 3h+ and was
+ * caught only because the operator opened a second session to look.)
+ *
+ * Escalation only — see JOB_OVERRUN_FACTOR's note on why this must not kill.
+ * A job with no usable estimate is skipped rather than guessed at.
+ */
+function findOverrunningJobs(jobs, now, { factor, floorMs } = {}) {
+  const f = typeof factor === 'number' && factor > 0 ? factor : JOB_OVERRUN_FACTOR;
+  const floor = typeof floorMs === 'number' && floorMs >= 0 ? floorMs : JOB_OVERRUN_FLOOR_MS;
+  const out = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'running') continue;
+    const est = Number(j.estimateMinutes);
+    if (!Number.isFinite(est) || est <= 0) continue; // no estimate to overrun
+    const startedAt = Date.parse(j.startedAt ?? '');
+    if (Number.isNaN(startedAt)) continue;
+    const ranMs = now - startedAt;
+    if (ranMs <= 0) continue;
+    const thresholdMs = Math.max(est * 60_000 * f, floor);
+    if (ranMs < thresholdMs) continue;
+    out.push({
+      slug: j.slug,
+      cwd: j.cwd ?? null,
+      estimateMinutes: est,
+      ranMs,
+      ratio: ranMs / (est * 60_000),
+    });
+  }
+  return out;
+}
+
 // An empty queue and an unreadable queue are NOT the same thing, and
 // conflating them is destructive: reconcile() treats every PRD .md with no
 // matching jobs[] row as a brand-new goal and re-mints it as 'pending', so a
@@ -1356,6 +1461,21 @@ async function reconcile(state) {
       console.log(`[scheduler] reconcile: adopted quarantined PRD ${job.slug} — createdVia=${p.createdVia}`);
       appendAuditEvent('scheduler_prd_adopted', { slug: job.slug, cwd: p.cwd, createdVia: p.createdVia, source: 'reconcile' });
     }
+    // Backfill a quarantine timestamp for rows quarantined before the
+    // statusHistory stamp below existed (e.g. the burrow-project rows
+    // quarantined under the PRD-authoring lockdown) — findStaleQuarantinedJobs
+    // needs SOME timestamp to escalate an un-adopted row past its age
+    // threshold, and the PRD file's own mtime is the best available proxy
+    // for "when this file first showed up unstamped" for a row that has
+    // never been touched since.
+    if (updatedJob.status === 'quarantined' && !(updatedJob.statusHistory || []).some((h) => h.to === 'quarantined')) {
+      try {
+        const at = new Date(fs.statSync(p.path).mtimeMs).toISOString();
+        const history = Array.isArray(updatedJob.statusHistory) ? [...updatedJob.statusHistory] : [];
+        history.push({ from: null, to: 'quarantined', reason: 'backfilled from PRD file mtime', source: 'reconcile-backfill', at });
+        updatedJob.statusHistory = history;
+      } catch { /* best-effort only — a missing/unreadable file just skips the backfill */ }
+    }
     next.push(updatedJob);
   }
   // Slugs on disk with no matching state.jobs row are normally brand-new
@@ -1590,6 +1710,18 @@ async function reconcile(state) {
     // to 'pending' on the very next pass, within one tick of being stamped.
     if (!p.createdVia && !isFixPlanSlug(slug)) {
       entry.status = 'quarantined';
+      // Stamped at creation (not via transitionJob, since this is a
+      // brand-new row minted directly at 'quarantined' rather than
+      // transitioning through 'pending') so findStaleQuarantinedJobs has a
+      // real quarantine timestamp to escalate against, instead of only the
+      // reconcile-backfill fallback above.
+      entry.statusHistory = [{
+        from: null,
+        to: 'quarantined',
+        reason: 'missing createdVia provenance frontmatter',
+        source: 'reconcile',
+        at: new Date().toISOString(),
+      }];
       console.warn(`[scheduler] reconcile: quarantining unstamped PRD ${slug} (${p.path}) — no createdVia provenance; adopt it from the Scheduler tab's Quarantined filter or via scheduler_update_prd to make it runnable`);
       appendAuditEvent('prd_quarantined', { slug, cwd: p.cwd, path: p.path, reason: 'missing createdVia provenance frontmatter' });
     }
@@ -1692,12 +1824,15 @@ let pollLoopTimer = null;
 let rescheduleInterval = null;
 let heartbeatInterval = null;
 // Stall-detector state (computeStallSummary), read/written only inside the
-// heartbeat interval below. stallSince: wall-clock ms the stalled condition
-// was first observed, null when clear. stallToasted: rate-limits the
-// error-log + toast to once per stall episode (cleared the moment the queue
-// stops being stalled) rather than every 60s heartbeat tick.
-let stallSince = null;
-let stallToasted = false;
+// heartbeat interval below. Keyed per-project cwd (never a single value) —
+// a single module-level flag would let one busy project's activity clear or
+// suppress another stalled project's alert. stallSince.get(cwd): wall-clock
+// ms that project's stalled condition was first observed, absent when clear.
+// stallToasted.get(cwd): rate-limits that project's error-log + toast to
+// once per stall episode (cleared the moment that project stops being
+// stalled) rather than every 60s heartbeat tick.
+let stallSince = new Map();
+let stallToasted = new Map();
 // (The 5-minute feedback sweep that used to piggyback on this heartbeat is
 // gone: it scanned each active project's session-manager-operations/feedback/
 // and auto-queued a /process-feedback PRD. Both the folder and that skill are
@@ -4924,6 +5059,7 @@ async function init() {
   if (rescheduleInterval) clearInterval(rescheduleInterval);
   rescheduleInterval = setInterval(() => {
     rescheduleTimer().catch(() => {});
+    const s = readQueueSync();
     // Periodic self-heal: re-run the verifier over stale needs_review jobs so a
     // job whose work actually landed (committed in-window, no FAIL sentinel)
     // auto-clears WITHOUT waiting for the next app restart. Cheap-guarded — the
@@ -4933,10 +5069,50 @@ async function init() {
     // MAX_CONCURRENT_INVESTIGATIONS (spawnInvestigation queues/early-returns
     // past it), so this interval firing cannot fan out investigations.
     if (process.env.SM_REVERIFY_PERIODIC_DISABLE !== '1') {
-      const s = readQueueSync();
       if (s.jobs.some((j) => j.status === 'needs_review')) {
         reverifyNeedsReview().catch(() => {});
       }
+      // A quarantined row only ever promotes to 'pending' through
+      // reconcile()'s adopt path (see reconcile()'s "Adopt path" comment) —
+      // it re-checks the PRD file's createdVia stamp every pass. broadcast()
+      // already runs reconcile+writeQueue on every normal poll tick, but an
+      // idle queue (nothing pending/running to fire) can back off that
+      // cadence for a long time; this guarantees an adopted-but-still-
+      // quarantined row is re-checked within 10 minutes regardless.
+      if (s.jobs.some((j) => j.status === 'quarantined')) {
+        broadcast().catch(() => {});
+      }
+    }
+    // Age-based escalation (independent of the self-heal kill-switch above —
+    // this is a monitoring signal, not an auto-fix action): a quarantined
+    // row nobody has adopted or archived past QUARANTINE_ESCALATE_MS is
+    // warn-logged by project + slug + age so it cannot sit stranded and
+    // silent (the four burrow-project rows this PRD was written against).
+    for (const stale of findStaleQuarantinedJobs(s.jobs, Date.now(), QUARANTINE_ESCALATE_MS)) {
+      console.warn(
+        `[scheduler] QUARANTINED PRD STALE: project=${stale.cwd ?? '(unknown)'} slug=${stale.slug} `
+        + `age=${Math.round(stale.ageMs / 3_600_000)}h (>= ${Math.round(QUARANTINE_ESCALATE_MS / 3_600_000)}h threshold) — `
+        + `adopt it from the Scheduler tab's Quarantined filter, or archive it; nothing else will clear this`,
+      );
+      appendAuditEvent('prd_quarantine_stale', { slug: stale.slug, cwd: stale.cwd, ageMs: stale.ageMs });
+    }
+
+    // Estimate-relative overrun escalation. Sits in the blind spot between
+    // the 4h deadman and the 20-minute idle-output watchdog: a job that keeps
+    // producing output while looping trips neither, so nothing noticed a PRD
+    // running 9x its own estimate until a human went looking. Escalate loudly;
+    // never kill on an estimate (see JOB_OVERRUN_FACTOR).
+    for (const over of findOverrunningJobs(s.jobs, Date.now())) {
+      console.warn(
+        `[scheduler] JOB OVERRUNNING ESTIMATE: project=${over.cwd ?? '(unknown)'} slug=${over.slug} `
+        + `ran=${Math.round(over.ranMs / 60_000)}m vs estimate=${over.estimateMinutes}m `
+        + `(${over.ratio.toFixed(1)}x, threshold ${JOB_OVERRUN_FACTOR}x floor ${Math.round(JOB_OVERRUN_FLOOR_MS / 60_000)}m) — `
+        + `still running; the ${Math.round(MAX_JOB_DURATION_MS / 3_600_000)}h deadman has NOT fired yet. `
+        + `Check the run log, then let it finish or cancel it via scheduler_cancel_job`,
+      );
+      appendAuditEvent('job_overrunning_estimate', {
+        slug: over.slug, cwd: over.cwd, estimateMinutes: over.estimateMinutes, ranMs: over.ranMs, ratio: over.ratio,
+      });
     }
   }, 10 * 60_000);
 
@@ -4981,27 +5157,42 @@ async function init() {
     }
 
     const stall = computeStallSummary(s);
-    if (stall.stalled) {
-      if (stallSince === null) stallSince = Date.now();
-      if (!stallToasted && Date.now() - stallSince >= POLL_INTERVAL_MS) {
-        stallToasted = true;
-        console.error(
-          `[scheduler] STALL DETECTED: ${stall.total} job(s) queued, 0 running, 0 pending, not paused, `
-          + `for >= ${Math.round(POLL_INTERVAL_MS / 1000)}s`,
-          stall.byProject,
-        );
-        appendAuditEvent('scheduler_stall_detected', { total: stall.total, byProject: stall.byProject });
-        if (mainWindow && !mainWindow.isDestroyed()) {
-          sendIfAlive(mainWindow, 'schedule:stall', {
-            message: `Scheduler stall: ${stall.total} job(s) queued but none running or pending. Check the Scheduler tab.`,
-            total: stall.total,
-            byProject: stall.byProject,
-          });
-        }
+    // Per-project alerting (see computeStallSummary's header): a project
+    // stalled while others are busy must still fire, and one project
+    // recovering must not clear or suppress another's still-open episode —
+    // that is exactly what a single module-level stallSince/stallToasted
+    // flag masked before (the burrow-vs-others incident this PRD fixes).
+    const now = Date.now();
+    const stalledCwds = Object.keys(stall.byProject).filter((cwd) => stall.byProject[cwd].stalled);
+    for (const cwd of [...stallSince.keys()]) {
+      if (!stalledCwds.includes(cwd)) {
+        stallSince.delete(cwd);
+        stallToasted.delete(cwd);
+      }
+    }
+    const toAlert = [];
+    for (const cwd of stalledCwds) {
+      if (!stallSince.has(cwd)) stallSince.set(cwd, now);
+      if (!stallToasted.get(cwd) && now - stallSince.get(cwd) >= POLL_INTERVAL_MS) {
+        stallToasted.set(cwd, true);
+        toAlert.push(cwd);
+      }
+    }
+    if (toAlert.length > 0) {
+      console.error(
+        `[scheduler] STALL DETECTED in project(s): ${toAlert.join(', ')} — 0 running, 0 pending, not paused, `
+        + `for >= ${Math.round(POLL_INTERVAL_MS / 1000)}s`,
+        stall.byProject,
+      );
+      appendAuditEvent('scheduler_stall_detected', { projects: toAlert, total: stall.total, byProject: stall.byProject });
+      if (mainWindow && !mainWindow.isDestroyed()) {
+        sendIfAlive(mainWindow, 'schedule:stall', {
+          message: `Scheduler stall in ${toAlert.length} project(s): ${toAlert.join(', ')}. Check the Scheduler tab.`,
+          projects: toAlert,
+          total: stall.total,
+          byProject: stall.byProject,
+        });
       }
-    } else {
-      stallSince = null;
-      stallToasted = false;
     }
 
     appendHeartbeat({
@@ -5507,4 +5698,4 @@ function registerAdminRoutes(adminHttp, remoteObj = remote) {
   });
 }
 
-module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, healRefusalReason, writeQueue, reconcile, reconcileSourcePromptId, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, healTargetForFix, buildInvestigationPrompt, isGitRepoSync, committedInWindow, computeCommittedDuringRun, classifySigtermWithCommit, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, commitGuardVerdict, TRANSIENT_RETRY_CAP, buildScheduleStatePayload, partitionBootOrphans, applyOrphanOutcome, BOOT_ORPHAN_KILL_GRACE_MS, registerAdminRoutes, notifyOriginatingTab, notifyNeedsReview, isNotifiableTerminalStatus, extractResultTextFromLog, candidatePrdsDirs, candidateArchivedPrdsDirs, resolveArchivedPrdStatus, prdDirForCwd, prdPathForJob, archivedPrdPathForJob, archivedTwinExists, findPrdDir, resolveVerifyPrdPath, resolveNotifyPrd, runPrdMigration, consolidateAllFlatPrds, shouldSkipInvestigationForCleanRun, archiveCompletedPrd, retireCompletedSlugs, SCHEDULER_BOOTED_AT, SCHEDULER_CODE_SHA, resetJobFields, executeJob, prdArchivedSkipResult, spawnJob, listPrdsInternal, computeStallSummary };
+module.exports = { findOverrunningJobs, JOB_OVERRUN_FACTOR, JOB_OVERRUN_FLOOR_MS, registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, healRefusalReason, writeQueue, reconcile, reconcileSourcePromptId, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, healTargetForFix, buildInvestigationPrompt, isGitRepoSync, committedInWindow, computeCommittedDuringRun, classifySigtermWithCommit, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, commitGuardVerdict, TRANSIENT_RETRY_CAP, buildScheduleStatePayload, partitionBootOrphans, applyOrphanOutcome, BOOT_ORPHAN_KILL_GRACE_MS, registerAdminRoutes, notifyOriginatingTab, notifyNeedsReview, isNotifiableTerminalStatus, extractResultTextFromLog, candidatePrdsDirs, candidateArchivedPrdsDirs, resolveArchivedPrdStatus, prdDirForCwd, prdPathForJob, archivedPrdPathForJob, archivedTwinExists, findPrdDir, resolveVerifyPrdPath, resolveNotifyPrd, runPrdMigration, consolidateAllFlatPrds, shouldSkipInvestigationForCleanRun, archiveCompletedPrd, retireCompletedSlugs, SCHEDULER_BOOTED_AT, SCHEDULER_CODE_SHA, resetJobFields, executeJob, prdArchivedSkipResult, spawnJob, listPrdsInternal, computeStallSummary, findStaleQuarantinedJobs, QUARANTINE_ESCALATE_MS };

package/dist/assets/ScopeSwitcher-DZ_3gEus.js

@@ -1 +0,0 @@
-import{K as e,G as o,S as x}from"./index-TejhHSzN.js";function f({dirty:i,busy:a,parseError:t,lastSavedAt:c,onSave:s,onRevert:l,leading:n}){const d=i&&!a&&!t;return e.jsxs("div",{className:"px-3 py-1.5 flex items-center gap-2 text-xs",children:[e.jsx("div",{className:"flex-1 min-w-0 truncate text-fg-faint",children:n}),t&&e.jsx("span",{className:"text-red-400 truncate max-w-xs",title:t,children:t}),!i&&!t&&c&&e.jsx("span",{className:"text-fg-faint",children:o(c)}),i&&!t&&e.jsx("span",{className:"text-accent",children:"unsaved"}),e.jsx("button",{onClick:l,disabled:!i||a,className:"px-2 py-0.5 border border-line rounded text-fg-dim hover:text-fg hover:bg-bg-hi disabled:opacity-40 disabled:cursor-not-allowed",children:"Revert"}),e.jsx("button",{onClick:s,disabled:!d,className:`px-2.5 py-0.5 rounded border ${d?"border-accent bg-accent/15 text-accent hover:bg-accent/25":"border-line text-fg-faint opacity-50 cursor-not-allowed"}`,children:a?"Saving…":"Save"})]})}function b({scopes:i,active:a,onChange:t,annotate:c}){return e.jsx("div",{className:"inline-flex rounded border border-line overflow-hidden",children:i.map(s=>{const l=x[s],n=c==null?void 0:c(s),d=s===a;return e.jsxs("button",{title:l.hint,onClick:()=>t(s),className:`px-2.5 py-1 text-xs flex items-center gap-1.5 transition-colors ${d?"bg-bg-hi text-fg":"bg-bg-elev text-fg-dim hover:text-fg hover:bg-bg-hi"}`,children:[e.jsx("span",{children:l.label}),(n==null?void 0:n.dirty)&&e.jsx("span",{className:"w-1 h-1 rounded-full bg-accent",title:"unsaved edits"}),n&&n.exists===!1&&!n.dirty&&e.jsx("span",{className:"w-1 h-1 rounded-full bg-fg-faint",title:"not yet created"})]},s)})})}export{f as S,b as a};