claude-code-session-manager 0.75.3 → 0.76.0

package/src/main/scheduler.cjs

@@ -55,7 +55,8 @@ const { cleanChildEnv, pathWithUserBins } = require('./lib/cleanEnv.cjs');
 const supervisor = require('./supervisor.cjs');
 const { resolveClaudeBin } = require('./lib/claudeBin.cjs');
 const { readTail } = require('./lib/fileTail.cjs');
-const { claudePidAlive, classifyRunOutcome, ORPHAN_REQUEUE_CAP } = require('./lib/reaperHelpers.cjs');
+const { claudePidAlive, classifyRunOutcome, ORPHAN_REQUEUE_CAP, selectReapableJobs } = require('./lib/reaperHelpers.cjs');
+const { createLoadGate, topCpuConsumers } = require('./lib/loadGate.cjs');
 const { openLog, withChildAndLog } = require('./lib/childWithLog.cjs');
 const { sendIfAlive } = require('./lib/sendToRenderer.cjs');
 const { createBroadcastCoalescer } = require('./lib/broadcastCoalescer.cjs');
@@ -68,7 +69,7 @@ const promptSessionTranscript = require('./promptSessionTranscript.cjs');
 const { verifyRun } = require('./runVerify.cjs');
 const { latestTerminalOutcomeForSlug, COMPLETED_EQUIVALENT_VERDICTS } = require('./lib/terminalRunOutcome.cjs');
 const logs = require('./logs.cjs');
-const { schemas, validated } = require('./ipcSchemas.cjs');
+const { schemas, validated, SCHEDULE_SLUG_RE } = require('./ipcSchemas.cjs');
 const { readBody, sendJson } = require('./lib/localAdminHttp.cjs');
 const {
   POLL_INTERVAL_MS,
@@ -78,6 +79,9 @@ const {
   QUARANTINE_ESCALATE_MS: QUARANTINE_ESCALATE_MS_DEFAULT,
   JOB_OVERRUN_FACTOR: JOB_OVERRUN_FACTOR_DEFAULT,
   JOB_OVERRUN_FLOOR_MS: JOB_OVERRUN_FLOOR_MS_DEFAULT,
+  PIDLESS_SPAWN_GRACE_MS,
+  INVESTIGATION_MAX_MS,
+  STARVATION_ESCALATE_MS,
 } = require('./lib/schedulerConfig.cjs');
 const QUARANTINE_ESCALATE_MS = process.env.SM_QUARANTINE_ESCALATE_HOURS
   ? Number(process.env.SM_QUARANTINE_ESCALATE_HOURS) * 60 * 60_000
@@ -88,7 +92,7 @@ const JOB_OVERRUN_FACTOR = process.env.SM_JOB_OVERRUN_FACTOR
 const JOB_OVERRUN_FLOOR_MS = process.env.SM_JOB_OVERRUN_FLOOR_MINUTES
   ? Number(process.env.SM_JOB_OVERRUN_FLOOR_MINUTES) * 60_000
   : JOB_OVERRUN_FLOOR_MS_DEFAULT;
-const { pickForProject, pickNextBatch, DEFAULT_PROJECT_CWD } = require('./lib/schedulerBatch.cjs');
+const { pickForProject, pickNextBatch, findStarvedProjects, DEFAULT_PROJECT_CWD } = require('./lib/schedulerBatch.cjs');
 const { runDefinitionOfDoneOnDrain } = require('./lib/dodDrainHook.cjs');
 const { writeRcaReport, extractRcaBlock } = require('./lib/rcaReport.cjs');
 const queueHistory = require('./lib/queueHistory.cjs');
@@ -100,7 +104,7 @@ const queueOps = require('./queueOps.cjs');
 // home-dir layout.
 const { resolvePrdsDirs, resolveArchivedPrdsDirs, resolvePrdWriteDir, listEpicPrdDirs, listArchivedPrdDirs } = require('./lib/prdLocations.cjs');
 const { ensureEpic, appendPrdCreatedEvent, readActiveIndex } = require('./lib/epicMint.cjs');
-const { transitionJob, STATUS_HISTORY_CAP } = require('./lib/scheduleJobTransitions.cjs');
+const { transitionJob, STATUS_HISTORY_CAP, LEGAL_TRANSITIONS } = require('./lib/scheduleJobTransitions.cjs');
 const { buildContextDigest, composeExecutorPrompt } = require('./lib/epicContextDigest.cjs');
 const { JOB_STATUSES } = require('./lib/scheduleJobSchema.cjs');
 const { appendAuditEvent } = require('./lib/auditLog.cjs');
@@ -205,6 +209,18 @@ const FINISH_PROTOCOL = `
 Once every acceptance-criteria line above is satisfied, finish in this EXACT
 sequence. Do not stop before the commit lands; committing is part of the job.
 
+RUN VERIFICATION IN THE FOREGROUND — this applies to the whole run, not just
+step 3 below: every test/typecheck/lint/build command you run, whether while
+implementing the AC or during VERIFY, must run SYNCHRONOUSLY and you must wait
+for it to return. Never start a verification command as a background task
+(no background Bash) and then call Monitor, TaskOutput, or ScheduleWakeup to
+pick up its result later — a headless \`claude -p\` run has no later turn, so
+nothing ever delivers that notification and the run dies mid-verification with
+no commit and no verdict. For a long-running command, bound it yourself with
+the shell (e.g. \`timeout 300 npm test\`) and a matching foreground tool
+timeout; if it still cannot finish inside budget, stop and emit
+SCHEDULER_VERDICT: FAIL with the reason instead of deferring it.
+
 1. CODE REVIEW — run \`/code-review --fix\` on your changes and apply the fixes it
    surfaces (correctness first). For any finding you judge a false positive, say
    why in your result; do not silently skip it. If \`/code-review\` is not
@@ -690,6 +706,43 @@ async function safeSlugPath(slug) {
   return safeSlugPathIn(dir, slug);
 }
 
+/**
+ * The two distinct failure modes safeSlugPath collapses into one nullable
+ * return (the defect this fixes — see the PRD that added this helper's
+ * Goal): a slug that fails SCHEDULE_SLUG_RE is a caller mistake ("invalid
+ * slug"), while a well-formed slug that exists in no candidate PRD dir is a
+ * lookup miss ("unknown slug") — an agent retrying the first as if it were
+ * the second (or vice versa) burns a turn on the wrong fix. Returns
+ * `{ ok: true, path }` or `{ ok: false, reason: 'invalid-slug' | 'not-found' }`.
+ * `cwd`, if given, narrows the search to that one project's own PRD dirs
+ * (prdDirForCwd + its Epic-scoped dirs — same pattern as getPrdParsed);
+ * omitted, it searches every candidate dir machine-wide via findPrdDir.
+ */
+async function resolveSlugOrReason(slug, cwd) {
+  if (!SCHEDULE_SLUG_RE.test(slug)) return { ok: false, reason: 'invalid-slug' };
+  if (cwd) {
+    for (const dir of [prdDirForCwd(cwd), ...listEpicPrdDirs(cwd)]) {
+      const p = safeSlugPathIn(dir, slug);
+      if (!p) continue;
+      try {
+        await fsp.access(p);
+        return { ok: true, path: p };
+      } catch { /* not in this dir — try the next candidate */ }
+    }
+    return { ok: false, reason: 'not-found' };
+  }
+  const dir = await findPrdDir(slug);
+  if (!dir) return { ok: false, reason: 'not-found' };
+  const p = safeSlugPathIn(dir, slug);
+  if (!p) return { ok: false, reason: 'not-found' };
+  return { ok: true, path: p };
+}
+
+/** Actionable message for `resolveSlugOrReason`'s 'not-found' reason. */
+function unknownSlugMessage(slug) {
+  return `unknown slug "${slug}": no PRD file with that name in any known project — call scheduler_list_prds (optionally with cwd) to see what exists`;
+}
+
 /**
  * Move a completed job's `<slug>.md` out of its PRD dir into that dir's
  * sibling `prds-archived/`, so a finished slug can't be re-fired by the
@@ -1098,6 +1151,73 @@ function findOverrunningJobs(jobs, now, { factor, floorMs } = {}) {
   return out;
 }
 
+/**
+ * findStrandedInvestigations(jobs, now, maxMs, isAlive = claudePidAlive)
+ *   → [{ slug, cwd, ageMs, restoreStatus }]
+ *
+ * Pure (besides the warn-log side effect on the two unprovable-age cases
+ * below), no other IO. spawnInvestigation's own restore of a job's
+ * pre-investigation status runs entirely inside the process that spawned the
+ * probe (its withChildAndLog onExit handler, or the synchronous-throw catch
+ * path) — so a job left 'investigating' when the app itself dies or restarts
+ * has NOTHING left to restore it. The comment at spawnInvestigation's onExit
+ * asserts "'investigating' must never be the job's resting state"; this is
+ * the sweep that makes that true across a restart, not just within one.
+ *
+ * A row qualifies only when ALL of:
+ *  - status is 'investigating'
+ *  - its most recent transition INTO 'investigating' (statusHistory's last
+ *    `to === 'investigating'` entry — a job can be investigated more than
+ *    once across its life, e.g. a retried auto-fix) is older than `maxMs`
+ *  - it has no live probe process behind it (checked via runtime.pid, set by
+ *    spawnInvestigation once its child spawns and cleared on every restore
+ *    path, the same shape reapDeadRunningJobs already uses for 'running' rows)
+ *
+ * `restoreStatus` is that transition entry's `from` — the exact value
+ * spawnInvestigation itself would have restored to (`failedJob.status ||
+ * 'failed'`), which for a row that already finished and recorded
+ * finishedAt+exitCode (the burrow-834 shape) is whatever terminal status was
+ * computed for that outcome BEFORE the probe was spawned — this sweep never
+ * re-derives it from exitCode, only replays the already-recorded decision.
+ *
+ * A row with no recoverable transition timestamp cannot have its age proven,
+ * so it is warn-logged and left alone rather than guessed at — same posture
+ * as findStaleQuarantinedJobs/findOverrunningJobs.
+ *
+ * `restoreStatus` is validated against LEGAL_TRANSITIONS['investigating']
+ * before being returned — `statusHistory`'s `from` should only ever be
+ * 'failed' or 'needs_review' (the only two states LEGAL_TRANSITIONS allows
+ * into 'investigating'), but a corrupted/unexpected value must not be handed
+ * straight to transitionJob: an illegal target is refused outright (row stays
+ * stuck at 'investigating', re-detected as stranded every sweep with no path
+ * out), so an out-of-set `from` falls back to 'failed' here instead.
+ */
+const INVESTIGATING_RESTORE_TARGETS = new Set(LEGAL_TRANSITIONS.investigating);
+function findStrandedInvestigations(jobs, now, maxMs, isAlive = claudePidAlive) {
+  const out = [];
+  for (const j of jobs ?? []) {
+    if (j.status !== 'investigating') continue;
+    const entries = (j.statusHistory || []).filter((h) => h.to === 'investigating');
+    const entry = entries[entries.length - 1];
+    if (!entry) {
+      console.warn(`[scheduler] findStrandedInvestigations: ${j.slug} is 'investigating' with no statusHistory entry recording the transition — cannot prove age, leaving alone`);
+      continue;
+    }
+    const since = Date.parse(entry.at ?? '');
+    if (Number.isNaN(since)) {
+      console.warn(`[scheduler] findStrandedInvestigations: ${j.slug} has an unparseable investigating-transition timestamp (${entry.at}) — cannot prove age, leaving alone`);
+      continue;
+    }
+    const ageMs = now - since;
+    if (ageMs < maxMs) continue; // a live probe must not be yanked out from under itself
+    const pid = j.runtime?.pid;
+    if (pid && isAlive(pid)) continue; // probe genuinely still running — not stranded
+    const restoreStatus = INVESTIGATING_RESTORE_TARGETS.has(entry.from) ? entry.from : 'failed';
+    out.push({ slug: j.slug, cwd: j.cwd ?? null, ageMs, restoreStatus });
+  }
+  return out;
+}
+
 // An empty queue and an unreadable queue are NOT the same thing, and
 // conflating them is destructive: reconcile() treats every PRD .md with no
 // matching jobs[] row as a brand-new goal and re-mints it as 'pending', so a
@@ -1677,6 +1797,10 @@ async function reconcile(state) {
       originSessionId: resolveOriginSessionId(p.cwd, p.epicId ?? p.sourcePromptId),
       bodyPreview: p.body.split('\n').slice(0, 6).join('\n'),
       status: 'pending',
+      // Enqueue time (PRD 1086/1087): the cross-project fairness tiebreak and
+      // the starvation escalation both need a provable age for a pending row;
+      // before this stamp a freshly minted row carried no timestamp at all.
+      queuedAt: new Date().toISOString(),
       runId: null,
       startedAt: null,
       finishedAt: null,
@@ -1864,6 +1988,9 @@ function drainDeferredInvestigation() {
 let cancelToken = { cancelled: false };
 // Last memory-gate observation; included in snapshot for renderer visibility.
 let lastMemGate = null;
+// CPU-load launch gate (PRD 1085, lib/loadGate.cjs) — innermost launch
+// predicate after pool → project cap → memory. Withholds launches only.
+const loadGate = createLoadGate();
 
 // Last tickQueue outcome, kept for the UI. tickQueue already computes a precise
 // reason for every way a batch can come back empty (dependency holds, slot
@@ -1932,6 +2059,8 @@ function buildScheduleStatePayload(state, { withPaths = false } = {}) {
       lastFailureKind,
     },
     memGate: lastMemGate,
+    // Why nothing is launching when the box is CPU-saturated (PRD 1085).
+    loadGate: loadGate.snapshot(),
     lastTick,
     // The machine-wide slot pool IS the concurrency limit — there is no
     // separate scheduler cap any more. `source` distinguishes the
@@ -2609,9 +2738,15 @@ function classifyFailureOutcome({ exitCode, networkError, durationMs, transientR
  * tree was dirty — closed by widening that call site's condition, not by
  * changing this function's four defenses below, which still apply to both
  * shapes identically:
- *   - siblingRunning: a concurrent job in the same cwd makes working-tree
- *     evidence unreliable in both directions (extra dirt OR a clean tree
- *     that isn't this job's doing).
+ *   - siblingRunning: on a SHARED tree only — a concurrent job in the same
+ *     cwd makes working-tree evidence unreliable in both directions (extra
+ *     dirt OR a clean tree that isn't this job's doing). Suppressed by
+ *     ranInWorktree: when this job ran in its own git worktree, the
+ *     newly-dirty set and the integrated HEAD are attributable to this job
+ *     alone regardless of what siblings were doing concurrently in their own
+ *     worktrees, so the excuse does not apply (PRD 109 shipped 'completed'
+ *     with nothing committed specifically because this carve-out fired
+ *     unconditionally during a high-concurrency run).
  *   - jobSelfCommitted: HEAD moved during the run, so the job's deliverable
  *     landed even if dirt (from a concurrent actor) remains.
  *   - legitimateNoOp (COMPLETED_EQUIVALENT_VERDICTS): runVerify.cjs's own
@@ -2631,8 +2766,8 @@ function classifyFailureOutcome({ exitCode, networkError, durationMs, transientR
  *     still a genuine finish-protocol violation (incident:
  *     523-fix-bounded-fix-plan-retry, 2026-07-12).
  */
-function commitGuardVerdict({ newlyDirty, siblingRunning, jobSelfCommitted, legitimateNoOp, isFixPlanJob, verifyResult }) {
-  if (siblingRunning || jobSelfCommitted || legitimateNoOp) return null;
+function commitGuardVerdict({ newlyDirty, siblingRunning, ranInWorktree, jobSelfCommitted, legitimateNoOp, isFixPlanJob, verifyResult, salvagePatch }) {
+  if ((siblingRunning && !ranInWorktree) || jobSelfCommitted || legitimateNoOp) return null;
   const dirty = newlyDirty || [];
   if (dirty.length === 0 && isFixPlanJob) return null;
 
@@ -2651,9 +2786,10 @@ function commitGuardVerdict({ newlyDirty, siblingRunning, jobSelfCommitted, legi
   }
 
   const sample = dirty.slice(0, 3).join(', ');
+  const salvageNote = salvagePatch ? ` — recoverable from salvage patch ${salvagePatch}` : '';
   return {
     verdict: 'uncommitted_changes',
-    reason: `finish protocol incomplete: ${dirty.length} uncommitted file(s) left in working tree (e.g. ${sample})`,
+    reason: `finish protocol incomplete: ${dirty.length} uncommitted file(s) left in working tree (e.g. ${sample})${salvageNote}`,
     downgradeTo: 'needs_review',
     annotations: carried.length ? carried : undefined,
   };
@@ -2813,7 +2949,12 @@ async function executeJob(job, runDir, defaultCwd, onPid, execCwd) {
     // overrides `--model sonnet`, so scheduled jobs burn Opus credits silently.
     // PATH must include Homebrew/user bins or the job's node/git children ENOENT
     // when Electron was launched from Finder/Dock on macOS (stripped PATH).
-    const childEnv = cleanChildEnv({ PATH: pathWithUserBins() });
+    // SM_PROJECT_ROOT is the main-tree cwd (never spawnCwd, which may be a
+    // job/epic worktree) — forwarded by scheduler-mcp-server.cjs as
+    // originProjectRoot so a job running inside its own worktree can still
+    // resolve the real project for create-prd/open-session/readiness. See
+    // projectRootResolve.cjs.
+    const childEnv = cleanChildEnv({ PATH: pathWithUserBins(), SM_PROJECT_ROOT: cwd });
 
     // Track whether the agent has emitted a `result` event in its JSONL stream.
     // null until seen; then one of "success" | "error_max_turns" | … per the
@@ -3317,7 +3458,10 @@ async function spawnInvestigation(failedJob, runDir) {
       // 'investigating' must never be the job's resting state.
       mutate((s) => {
         const j = s.jobs.find((x) => x.slug === failedJob.slug);
-        if (j && j.status === 'investigating') transitionJob(j, failedJob.status || 'failed', { reason: 'investigation probe exited — restoring prior status', source: 'spawnInvestigation:onExit' });
+        if (j && j.status === 'investigating') {
+          transitionJob(j, failedJob.status || 'failed', { reason: 'investigation probe exited — restoring prior status', source: 'spawnInvestigation:onExit' });
+          delete j.runtime;
+        }
       })
         .then(() => broadcast({ flush: true }))
         .catch(() => {});
@@ -3374,6 +3518,15 @@ async function spawnInvestigation(failedJob, runDir) {
 
   if (child) {
     safeLog(`[scheduler] investigation pid=${child.pid}\n\n`);
+    // Recorded so findStrandedInvestigations (a post-restart maintenance
+    // sweep — the live process has no other way to know a probe is still
+    // running) can tell a live probe apart from one whose owning process is
+    // long gone, the same way reapDeadRunningJobs checks a running job's
+    // runtime.pid.
+    mutate((s) => {
+      const j = s.jobs.find((x) => x.slug === failedJob.slug);
+      if (j && j.status === 'investigating') j.runtime = { pid: child.pid };
+    }).catch(() => {});
   }
   return { deferred: false };
   } catch (e) {
@@ -3383,7 +3536,10 @@ async function spawnInvestigation(failedJob, runDir) {
     releaseSlot();
     mutate((s) => {
       const j = s.jobs.find((x) => x.slug === failedJob.slug);
-      if (j && j.status === 'investigating') transitionJob(j, failedJob.status || 'failed', { reason: 'investigation spawn threw before exiting — restoring prior status', source: 'spawnInvestigation:catch' });
+      if (j && j.status === 'investigating') {
+        transitionJob(j, failedJob.status || 'failed', { reason: 'investigation spawn threw before exiting — restoring prior status', source: 'spawnInvestigation:catch' });
+        delete j.runtime;
+      }
     })
       .then(() => broadcast({ flush: true }))
       .catch(() => {});
@@ -3430,6 +3586,16 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
       console.log(`[scheduler] ${job.slug}: isolated in worktree ${worktree.dir} (branch ${worktree.branch})`);
     } else {
       console.log(`[scheduler] ${job.slug}: running in main tree (worktree not used: ${worktree.reason})`);
+      // Surface any degraded-isolation fallback on the job row itself so it's
+      // queryable from the queue instead of console-only — except the
+      // deliberate env-disable flag, which is an intentional opt-out, not a
+      // degradation worth flagging.
+      if (!jobWorktree.isWorktreeDisabled()) {
+        await mutate((s) => {
+          const idx = s.jobs.findIndex((x) => x.slug === job.slug);
+          if (idx >= 0) s.jobs[idx].worktreeFallbackReason = worktree.reason;
+        });
+      }
     }
 
     // Integrate the job's branch back into guardCwd's own HEAD, THEN tear the
@@ -3448,6 +3614,7 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
     let res;
     let worktreeLeftoverDirty = [];
     let worktreeIntegrationFailure = null;
+    let worktreeSalvagePatch = null;
     try {
       res = await executeJob(job, runDir, defaultCwd, async (pid, sessionId, cwd) => {
         await mutate((s) => {
@@ -3462,6 +3629,19 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
     } finally {
       if (worktree.ok) {
         worktreeLeftoverDirty = (await uncommittedChanges(worktree.dir)) || [];
+        // Salvage the worktree's full diff (tracked + untracked) to the run
+        // dir BEFORE the checkout is removed below — otherwise a job killed
+        // before its finish-protocol commit loses that work outright, with
+        // no branch, no stash, no patch anywhere. Best-effort: never blocks
+        // integration/cleanup and never changes the job's verdict.
+        if (worktreeLeftoverDirty.length) {
+          const salvagePath = path.join(runDir, `${job.slug}.uncommitted.patch`);
+          const salvage = await jobWorktree.salvageJobWorktreeDiff({ dir: worktree.dir, outFile: salvagePath });
+          if (salvage && salvage.ok) {
+            worktreeSalvagePatch = salvagePath;
+            console.log(`[scheduler] ${job.slug}: salvaged ${salvage.bytes} byte(s) of uncommitted worktree diff to ${salvagePath}`);
+          }
+        }
         const integration = await jobWorktree.integrateJobBranch({ cwd: guardCwd, branch: worktree.branch, slug: job.slug });
         if (!integration.ok) {
           worktreeIntegrationFailure = integration.reason;
@@ -3614,10 +3794,12 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
         const guardVerdict = commitGuardVerdict({
           newlyDirty,
           siblingRunning,
+          ranInWorktree: worktree.ok,
           jobSelfCommitted,
           legitimateNoOp: guardIsLegitimateNoOp,
           isFixPlanJob: isFixPlanSlug(job.slug),
           verifyResult,
+          salvagePatch: worktreeSalvagePatch,
         });
         if (guardVerdict) {
           verifyResult = guardVerdict;
@@ -3709,6 +3891,11 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
           transitionJob(s.jobs[i2], effectiveStatus, { reason: sigtermOverrideReason ?? `run finished with exit ${res.exitCode}`, source: 'spawnJob:finalize' });
           s.jobs[i2].finishedAt = new Date().toISOString();
           s.jobs[i2].exitCode = res.exitCode;
+          if (worktreeSalvagePatch) {
+            s.jobs[i2].worktreeSalvagePatch = worktreeSalvagePatch;
+          } else {
+            delete s.jobs[i2].worktreeSalvagePatch;
+          }
           s.jobs[i2].error = effectiveStatus === 'needs_review'
             ? (verifyResult?.reason ?? sigtermOverrideReason ?? null)
             // A failed job (non-zero exit) never consults verifyResult above,
@@ -3887,12 +4074,13 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
         });
         await broadcast({ flush: true });
       } else if (decision.action === 'fail-dirty') {
-        console.log(`[scheduler] transient failure (${decision.transientKind}) for ${job.slug} left ${newlyDirtyCount} uncommitted file(s) (e.g. ${dirtySample}) — not auto-requeuing`);
+        const salvageNote = worktreeSalvagePatch ? ` — recoverable from salvage patch ${worktreeSalvagePatch}` : '';
+        console.log(`[scheduler] transient failure (${decision.transientKind}) for ${job.slug} left ${newlyDirtyCount} uncommitted file(s) (e.g. ${dirtySample})${salvageNote} — not auto-requeuing`);
         await mutate((s) => {
           const i = s.jobs.findIndex((x) => x.slug === job.slug);
           if (i >= 0) {
             transitionJob(s.jobs[i], 'failed', { reason: `transient failure (${decision.transientKind}) left uncommitted work — not auto-requeued`, source: 'spawnJob:fail-dirty' });
-            s.jobs[i].error = `transient failure (${decision.transientKind}) left ${newlyDirtyCount} uncommitted file(s) in working tree (e.g. ${dirtySample}) — not auto-requeued to avoid overwriting partial work; review and commit or discard manually`;
+            s.jobs[i].error = `transient failure (${decision.transientKind}) left ${newlyDirtyCount} uncommitted file(s) in working tree (e.g. ${dirtySample})${salvageNote} — not auto-requeued to avoid overwriting partial work; review and commit or discard manually`;
           }
         });
         await broadcast({ flush: true });
@@ -3937,7 +4125,10 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
 // is synchronous and spawnJob is fire-and-forget.
 let tickTail = Promise.resolve();
 
-function tickQueue() {
+// `bypassLoadGate` is set only by the explicit human run-now / force-tick
+// paths (via runDueJobs): the human is asking, so the CPU-load gate yields
+// and logs that it did. Every automatic caller leaves it false.
+function tickQueue({ bypassLoadGate = false } = {}) {
   const next = tickTail.then(async () => {
     const state = await readQueue();
     // Never reconcile against an unreadable queue: reconcile() would see zero
@@ -4025,6 +4216,35 @@ function tickQueue() {
       lastMemGate = null;
     }
 
+    // Load gate (PRD 1085) — the INNERMOST launch predicate, evaluated only
+    // once every outer gate (sessionSlots pool → per-project cap inside
+    // pickNextBatch → memory above) has already admitted `gatedBatch`. It
+    // never touches running jobs and never becomes a second pool: it only
+    // withholds this tick's launches while the 1-minute loadavg per core is
+    // over LOAD_GATE_PER_CORE. An explicit human Run now bypasses it.
+    const load = loadGate.evaluate({ bypass: bypassLoadGate });
+    if (load.bypassed) {
+      console.log(`[scheduler] load gate: BYPASSED by run-now (loadavg1=${load.loadavg1} cores=${load.cores} ratio=${load.ratio} > ${load.threshold})`);
+    } else if (load.gated) {
+      const line = `[scheduler] load gate: loadavg1=${load.loadavg1} cores=${load.cores} ratio=${load.ratio} > ${load.threshold} — holding ${gatedBatch.length} eligible job(s)`;
+      if (load.escalate) {
+        const top = topCpuConsumers(3);
+        console.warn(`${line} for ${Math.round(load.gatedSinceMs / 60_000)}m; top CPU: ${top.length ? top.join(' | ') : 'n/a'}`);
+      } else {
+        console.log(line);
+      }
+      if (load.shouldAudit) {
+        appendAuditEvent('launch_load_gated', {
+          loadavg1: load.loadavg1, cores: load.cores, ratio: load.ratio, threshold: load.threshold,
+          held: gatedBatch.map((j) => j.slug), gatedSinceMs: load.gatedSinceMs,
+        });
+      }
+      return recordTick(
+        { fired: false, reason: 'load-deferred', deferredCount: gatedBatch.length, ratio: load.ratio, threshold: load.threshold },
+        { detail: `load gate: ${load.loadavg1} / ${load.cores} cores = ${load.ratio} > ${load.threshold}`, holds },
+      );
+    }
+
     await mutate((s) => { s.lastRunAt = new Date().toISOString(); });
     await broadcast();
 
@@ -4072,7 +4292,7 @@ function forceTickOutcome(result) {
   }
 }
 
-async function runDueJobs() {
+async function runDueJobs({ bypassLoadGate = false } = {}) {
   const state = await readQueue();
   if (state.unreadable) {
     console.error('[scheduler] runDueJobs skipped: queue.json unreadable');
@@ -4083,7 +4303,7 @@ async function runDueJobs() {
     return { fired: false, reason: 'paused' };
   }
   cancelToken = { cancelled: false };
-  const result = await tickQueue();
+  const result = await tickQueue({ bypassLoadGate });
   // Clear the one-shot scheduledFor without waiting for jobs to settle.
   await mutate((s) => { s.scheduledFor = null; });
   await broadcast();
@@ -4109,11 +4329,13 @@ async function maybeLaunchWhenAvailable(state) {
 // ---------- dead-process reaper ----------
 
 /**
- * Scan running jobs, identify those whose claude process is provably dead, and
- * finalize them to completed/failed by reading the run log. Called once per
- * poll cycle. Conservative: a job with no runtime.pid yet (spawn mid-flight)
- * is always skipped. A job whose pid is alive (claudePidAlive) is always skipped.
- * Exported so unit tests can invoke it directly.
+ * Scan running jobs, identify those whose claude process is provably dead OR
+ * whose spawn never got far enough to record a runtime.pid in the first
+ * place, and finalize them to completed/failed by reading the run log.
+ * Called once per poll cycle. A job whose pid is alive (claudePidAlive) is
+ * always skipped. A pidless job younger than PIDLESS_SPAWN_GRACE_MS is
+ * skipped too (spawn may still be mid-flight) — see selectReapableJobs for
+ * the full predicate. Exported so unit tests can invoke it directly.
  */
 async function reapDeadRunningJobs() {
   try {
@@ -4123,32 +4345,45 @@ async function reapDeadRunningJobs() {
     // status:"running" with no slug left in runningSet to trigger reconciliation.
     // queue.json is the source of truth for which jobs are actually running.
     const state = await readQueue();
+    const { reapable, warnings } = selectReapableJobs(state.jobs, Date.now(), {
+      pidAlive: claudePidAlive,
+      grace: PIDLESS_SPAWN_GRACE_MS,
+    });
+    for (const w of warnings) {
+      console.warn(`[scheduler] reapDeadRunningJobs: ${w.reason} slug=${w.slug} — leaving row alone`);
+    }
+
     const dead = [];
-    for (const j of state.jobs) {
-      if (j.status !== 'running') continue;
-      const pid = j.runtime?.pid;
-      if (!pid) continue; // spawn may be mid-flight; give it a cycle
-      if (claudePidAlive(pid)) continue;
-      const logPath = j.runId
+    for (const { slug, pid, pidless, reason } of reapable) {
+      const j = state.jobs.find((x) => x.slug === slug);
+      const logPath = j?.runId
         ? path.join(RUNS_DIR, j.runId, `${j.slug}.log`)
         : null;
+      // Absent/empty run dir → classifyRunOutcome finds no result event →
+      // 'no_result' → non-success below → filed as failed, never completed.
       const outcome = logPath ? classifyRunOutcome(logPath) : 'unknown';
-      dead.push({ slug: j.slug, pid, outcome });
+      dead.push({ slug, pid, outcome, pidless, reason });
     }
     if (dead.length === 0) return;
 
     await mutate((s) => {
-      for (const { slug, pid, outcome } of dead) {
+      for (const { slug, pid, outcome, pidless, reason } of dead) {
         const idx = s.jobs.findIndex((x) => x.slug === slug);
         if (idx < 0 || s.jobs[idx].status !== 'running') continue; // race guard
         const success = outcome === 'success';
-        transitionJob(s.jobs[idx], success ? 'completed' : 'failed', { reason: `reaped: process gone (outcome=${outcome})`, source: 'reapDeadRunningJobs' });
+        const transitionReason = pidless ? reason : `reaped: process gone (outcome=${outcome})`;
+        transitionJob(s.jobs[idx], success ? 'completed' : 'failed', { reason: transitionReason, source: 'reapDeadRunningJobs' });
         s.jobs[idx].exitCode = success ? 0 : (s.jobs[idx].exitCode ?? 1);
         s.jobs[idx].finishedAt = new Date().toISOString();
-        s.jobs[idx].error = success ? null : `reaped: process gone, no success result in log (${outcome})`;
+        s.jobs[idx].error = success ? null : `${transitionReason} (outcome=${outcome})`;
         delete s.jobs[idx].runtime;
         runningSet.delete(slug);
-        console.log(`[scheduler] reaped dead job slug=${slug} pid=${pid} outcome=${outcome}`);
+        if (pidless) {
+          console.log(`[scheduler] reaped pidless zombie job slug=${slug} outcome=${outcome}`);
+          appendAuditEvent('job_reaped_pidless', { slug, cwd: s.jobs[idx].cwd ?? null, outcome, graceMs: PIDLESS_SPAWN_GRACE_MS });
+        } else {
+          console.log(`[scheduler] reaped dead job slug=${slug} pid=${pid} outcome=${outcome}`);
+        }
       }
     });
 
@@ -4337,12 +4572,17 @@ function selectHistoryJobs(jobs, limit, historyEntries = []) {
 // their pass_no_commit verdict is a harmless no-op (same facts, same verdict).
 const RESCANNABLE_VERDICTS = new Set(['transcript_errors', 'verify_unavailable', 'no_verdict_sentinel', 'pass_no_commit', 'pass_no_commit_already_shipped']);
 
-// Bounds fix-plan recursion: depth 1 = the original job, depth 2 = its fix
-// (gets exactly one follow-up investigation if it also lands in
-// needs_review), depth 3+ (a fix-of-a-fix-of-a-fix) is excluded. Shared by
-// selectAutoFixTargets and spawnInvestigation so both call sites agree on
-// one threshold.
-const MAX_INVESTIGATION_DEPTH = 2;
+// Bounds fix-plan recursion: cap N permits at most N+1 fix jobs per original
+// slug (depth 1 = the original job, depth 2 = its `-fix`, depth 3+ is
+// excluded). With N=1 that's `<slug>-fix` and `<slug>-fix-fix`, never a third
+// `-fix-fix-fix`. Lowered from 2 to 1 on 2026-08-31 (starry-night-ships):
+// three concurrent chains (115-fix-fix, 113-fix-fix, 111-fix-fix-fix) were
+// riding the old cap, and 115-fix-fix's own root-cause section read "The
+// code was already CORRECT. Only verification and commit failed." — a third
+// auto-retry re-runs an entire PRD and test battery to redo a `git commit`,
+// at near-zero marginal success probability. Shared by selectAutoFixTargets
+// and spawnInvestigation so both call sites agree on one threshold.
+const MAX_INVESTIGATION_DEPTH = 1;
 
 /**
  * True when a fix-plan job's investigationDepth is at or past the recursion
@@ -4841,7 +5081,7 @@ function registerScheduleHandlers() {
     // Clears any existing pause first (same semantics as run-now).
     await clearPause('run-now');
     try {
-      const result = await runDueJobs();
+      const result = await runDueJobs({ bypassLoadGate: true });
       return forceTickOutcome(result);
     } catch (e) {
       logs.writeLine({ level: 'error', scope: 'scheduler', message: 'runDueJobs error (force-tick)', meta: { error: e?.message } });
@@ -4917,7 +5157,7 @@ function registerScheduleHandlers() {
   ipcMain.handle('schedule:run-now', async () => {
     // Manual run-now overrides any auto-pause. Clear it first.
     await clearPause('run-now');
-    runDueJobs().catch((e) => logs.writeLine({ level: 'error', scope: 'scheduler', message: 'runDueJobs error (run-now)', meta: { error: e?.message } }));
+    runDueJobs({ bypassLoadGate: true }).catch((e) => logs.writeLine({ level: 'error', scope: 'scheduler', message: 'runDueJobs error (run-now)', meta: { error: e?.message } }));
     return { ok: true };
   });
 
@@ -5286,6 +5526,49 @@ async function init() {
         slug: over.slug, cwd: over.cwd, estimateMinutes: over.estimateMinutes, ranMs: over.ranMs, ratio: over.ratio,
       });
     }
+
+    // Stranded-investigation restore. Unlike the two escalations above, this
+    // one ACTS: 'investigating' is a transient status whose restore
+    // (spawnInvestigation's onExit/catch) only runs inside the process that
+    // spawned the probe, so an app restart mid-probe leaves the row frozen
+    // there forever (see findStrandedInvestigations' header, and the
+    // "'investigating' must never be the job's resting state" comment at
+    // spawnInvestigation's onExit). This restores each stranded row to the
+    // exact terminal status it already carried before the probe was
+    // spawned — it never re-runs or re-investigates anything.
+    const stranded = findStrandedInvestigations(s.jobs, Date.now(), INVESTIGATION_MAX_MS);
+    if (stranded.length > 0) {
+      mutate((ms) => {
+        for (const st of stranded) {
+          const j = ms.jobs.find((x) => x.slug === st.slug);
+          if (!j || j.status !== 'investigating') continue; // race guard — may have resolved since the scan above
+          transitionJob(j, st.restoreStatus, { reason: `stranded investigation restored after ${Math.round(st.ageMs / 60_000)}m with no live probe behind it`, source: 'findStrandedInvestigations' });
+          delete j.runtime;
+          console.warn(
+            `[scheduler] STRANDED INVESTIGATION RESTORED: project=${st.cwd ?? '(unknown)'} slug=${st.slug} `
+            + `age=${Math.round(st.ageMs / 3_600_000)}h (>= ${Math.round(INVESTIGATION_MAX_MS / 3_600_000)}h threshold), no live probe — `
+            + `restored to '${st.restoreStatus}'`,
+          );
+          appendAuditEvent('investigation_stranded_restored', { slug: st.slug, cwd: st.cwd, ageMs: st.ageMs, restoreStatus: st.restoreStatus });
+        }
+      })
+        .then(() => broadcast({ flush: true }))
+        .catch(() => {});
+    }
+
+    // Per-project starvation (PRD 1087): a project with pending work that has
+    // been passed over on every tick while OTHER projects dispatch. Nothing
+    // else distinguishes "no pending work" from "pending work, never
+    // started" — the 2026-09-01 NN-ordering starvation ran 3.5 h unnoticed.
+    // Escalation only, same shape as the quarantine/overrun warnings above.
+    for (const sp of findStarvedProjects(s.jobs, Date.now(), STARVATION_ESCALATE_MS)) {
+      console.warn(
+        `[scheduler] PROJECT STARVED: project=${sp.cwd} pending=${sp.pendingCount} oldest=${sp.oldestPendingSlug} `
+        + `waiting=${Math.round(sp.ageMs / 60_000)}m (>= ${Math.round(STARVATION_ESCALATE_MS / 60_000)}m threshold) `
+        + `while other projects are running — check the cross-project fairness rule in pickNextBatch`,
+      );
+      appendAuditEvent('project_starved', { cwd: sp.cwd, pendingCount: sp.pendingCount, oldestPendingSlug: sp.oldestPendingSlug, ageMs: sp.ageMs });
+    }
   }, 10 * 60_000);
 
   // Self-rescheduling poll loop with exponential backoff. Replaces the
@@ -5638,9 +5921,16 @@ const remote = {
   },
 
   async resetJob(slug, opts = {}) {
-    if (!(await safeSlugPath(slug))) return { ok: false, error: 'invalid slug' };
+    const resolved = await resolveSlugOrReason(slug, opts.cwd);
+    if (!resolved.ok) {
+      return { ok: false, error: resolved.reason === 'invalid-slug' ? 'invalid slug' : unknownSlugMessage(slug) };
+    }
     const outcome = await mutate((state) => {
-      const idx = state.jobs.findIndex((j) => j.slug === slug);
+      // Same cwd filter as resolveSlugOrReason's file lookup above — slugs are
+      // derived from title text with no cwd salt, so two different projects
+      // can independently produce the identical slug; an opts.cwd caller must
+      // reset THAT project's job, not just any queue row matching the string.
+      const idx = state.jobs.findIndex((j) => j.slug === slug && (!opts.cwd || j.cwd === opts.cwd));
       if (idx < 0) return { kind: 'not-found' };
       // Terminal-status guard lives in resetJobFields itself; force:true
       // threads through to override it.
@@ -5828,10 +6118,16 @@ const remote = {
   // cancelled job lands in 'failed' with an error naming the cause,
   // consistent with every other non-success terminal outcome. Refuses a
   // slug that's already terminal — nothing left to cancel.
-  async cancelJob(slug) {
+  async cancelJob(slug, opts = {}) {
+    if (!SCHEDULE_SLUG_RE.test(slug)) return { ok: false, error: 'invalid slug' };
     const state = await readQueue();
-    const job = state.jobs.find((j) => j.slug === slug);
-    if (!job) return { ok: false, error: 'not found' };
+    const job = state.jobs.find((j) => j.slug === slug && (!opts.cwd || j.cwd === opts.cwd));
+    if (!job) {
+      return {
+        ok: false,
+        error: `unknown slug "${slug}": no queued job with that name${opts.cwd ? ` in cwd ${opts.cwd}` : ''} — call scheduler_list_jobs to see what exists`,
+      };
+    }
     if (job.status === 'completed' || job.status === 'failed' || job.status === 'needs_review' || job.status === 'skipped') {
       return { ok: false, error: `job already terminal (status: "${job.status}") — nothing to cancel` };
     }
@@ -5889,9 +6185,10 @@ function registerAdminRoutes(adminHttp, remoteObj = remote) {
       return;
     }
     const force = parsed.force === true;
-    const result = await remoteObj.resetJob(slug, { force });
+    const cwd = typeof parsed.cwd === 'string' ? parsed.cwd : undefined;
+    const result = await remoteObj.resetJob(slug, { force, cwd });
     sendJson(res, 200, result);
   });
 }
 
-module.exports = { findOverrunningJobs, JOB_OVERRUN_FACTOR, JOB_OVERRUN_FLOOR_MS, registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, healRefusalReason, writeQueue, reconcile, reconcileSourcePromptId, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, isExhaustedAutoFix, isPlanUnqueued, fixSlugFor, healTargetForFix, buildInvestigationPrompt, isGitRepoSync, committedInWindow, computeCommittedDuringRun, classifySigtermWithCommit, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, commitGuardVerdict, TRANSIENT_RETRY_CAP, buildScheduleStatePayload, partitionBootOrphans, applyOrphanOutcome, BOOT_ORPHAN_KILL_GRACE_MS, registerAdminRoutes, notifyOriginatingTab, notifyNeedsReview, isNotifiableTerminalStatus, extractResultTextFromLog, candidatePrdsDirs, candidateArchivedPrdsDirs, resolveArchivedPrdStatus, prdDirForCwd, prdPathForJob, archivedPrdPathForJob, archivedTwinExists, findPrdDir, resolveVerifyPrdPath, resolveFixPlanPath, resolveNotifyPrd, runPrdMigration, consolidateAllFlatPrds, shouldSkipInvestigationForCleanRun, archiveCompletedPrd, retireCompletedSlugs, SCHEDULER_BOOTED_AT, SCHEDULER_CODE_SHA, resetJobFields, executeJob, prdArchivedSkipResult, spawnJob, listPrdsInternal, computeStallSummary, findStaleQuarantinedJobs, QUARANTINE_ESCALATE_MS, applyClearQueueVictims };
+module.exports = { findOverrunningJobs, JOB_OVERRUN_FACTOR, JOB_OVERRUN_FLOOR_MS, registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, healRefusalReason, writeQueue, reconcile, reconcileSourcePromptId, allocateParallelGroup, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets, isEligibleForImmediateAutoFix, resolveRunId, isUnresolvableNeedsReview, isExhaustedAutoFix, isPlanUnqueued, fixSlugFor, healTargetForFix, buildInvestigationPrompt, isGitRepoSync, committedInWindow, computeCommittedDuringRun, classifySigtermWithCommit, isFixPlanSlug, isFixPlanBeyondDepthCap, MAX_INVESTIGATION_DEPTH, forceTickOutcome, applyPauseCleared, detectNetworkErrorInLog, detectRateLimitInLog, classifyFailureOutcome, commitGuardVerdict, TRANSIENT_RETRY_CAP, buildScheduleStatePayload, partitionBootOrphans, applyOrphanOutcome, BOOT_ORPHAN_KILL_GRACE_MS, registerAdminRoutes, notifyOriginatingTab, notifyNeedsReview, isNotifiableTerminalStatus, extractResultTextFromLog, candidatePrdsDirs, candidateArchivedPrdsDirs, resolveArchivedPrdStatus, prdDirForCwd, prdPathForJob, archivedPrdPathForJob, archivedTwinExists, findPrdDir, resolveVerifyPrdPath, resolveFixPlanPath, resolveNotifyPrd, runPrdMigration, consolidateAllFlatPrds, shouldSkipInvestigationForCleanRun, archiveCompletedPrd, retireCompletedSlugs, SCHEDULER_BOOTED_AT, SCHEDULER_CODE_SHA, resetJobFields, executeJob, prdArchivedSkipResult, spawnJob, listPrdsInternal, computeStallSummary, findStaleQuarantinedJobs, QUARANTINE_ESCALATE_MS, applyClearQueueVictims, PIDLESS_SPAWN_GRACE_MS, findStrandedInvestigations, INVESTIGATION_MAX_MS };