claude-code-session-manager 0.20.0 → 0.20.1

package/src/main/scheduler.cjs

@@ -52,6 +52,7 @@ const { cleanChildEnv } = require('./lib/cleanEnv.cjs');
 const supervisor = require('./supervisor.cjs');
 const { resolveClaudeBin } = require('./lib/claudeBin.cjs');
 const { readTail } = require('./lib/fileTail.cjs');
+const { claudePidAlive, classifyRunOutcome } = require('./lib/reaperHelpers.cjs');
 const { openLog, withChildAndLog } = require('./lib/childWithLog.cjs');
 const { sendIfAlive } = require('./lib/sendToRenderer.cjs');
 const prdParser = require('./scheduler/prdParser.cjs');
@@ -63,6 +64,7 @@ const {
   USAGE_REFRESH_INTERVAL_MS,
   MAX_JOB_DURATION_MS,
 } = require('./lib/schedulerConfig.cjs');
+const { pickForProject, pickNextBatch, DEFAULT_PROJECT_CWD } = require('./lib/schedulerBatch.cjs');
 
 const MAX_INVESTIGATION_DURATION_MS = 30 * 60_000;
 
@@ -159,7 +161,7 @@ const QUEUE_PATH = path.join(ROOT, 'queue.json');
 const SCHEDULER_STATE_PATH = path.join(os.homedir(), '.claude', 'session-manager', 'scheduler-state.json');
 const HEARTBEAT_PATH = path.join(os.homedir(), '.claude', 'session-manager', 'scheduler-heartbeat.log');
 const HEARTBEAT_MAX_BYTES = 1024 * 1024;
-const DEFAULT_PROJECT_CWD = path.join(os.homedir(), 'Projects', 'session-manager');
+// DEFAULT_PROJECT_CWD imported from lib/schedulerBatch.cjs (single source of truth).
 
 const ENV_CAP = process.env.SM_SCHEDULER_MAX_CONCURRENCY
   ? Math.max(1, Math.min(20, parseInt(process.env.SM_SCHEDULER_MAX_CONCURRENCY, 10) || 4))
@@ -942,122 +944,10 @@ async function executeJob(job, runDir, defaultCwd, onPid) {
   });
 }
 
-/**
- * Pick the next batch of jobs to spawn this tick.
- *
- * Rules:
- *   1. Find the lowest parallelGroup that has pending jobs not already in
- *      runningSet.
- *   2. If that group has jobs in runningSet (i.e., we're mid-group), backfill
- *      up to (cap - runningSet.size) more from the SAME group.
- *   3. If the current group has NO jobs in runningSet (new group), and there
- *      are still jobs from an earlier group in runningSet, do nothing — wait
- *      for the earlier group to drain before advancing.
- *   4. **Late-arrival**: if a lower-numbered (higher-priority) PRD reconciles
- *      AFTER a higher-numbered group was already picked, fire the late-arrival
- *      immediately in parallel with the active group rather than starving it
- *      until the active group drains. This handles the reconcile-race where
- *      a PRD file lands on disk between two pickNextBatch invocations.
- *   5. A singleton group (unique NN, no other jobs share it) runs alone;
- *      no bleed into adjacent groups.
- *
- * Returns array of job objects to spawn. O(N) where N = pending.length.
- */
-function pickNextBatch(allJobs, running, cap) {
-  const pending = allJobs.filter((j) => j.status === 'pending' && !running.has(j.slug));
-  if (pending.length === 0) return [];
-
-  // Lowest pending group (computed up-front so the failure gate can compare).
-  const lowestPendingGroup = pending.reduce(
-    (min, j) => Math.min(min, j.parallelGroup ?? 99),
-    Infinity,
-  );
-
-  // Cross-group failure gate: refuse to advance past a group with failed jobs.
-  // Without this, a failed foundation PRD (e.g. 03-doc-editor-foundation
-  // crashed with a NUL-byte spawn error on 2026-05-21) doesn't stop later
-  // groups (04, 05, 06...) from running and silently corrupting the project
-  // state. The user can re-queue the failed job (pending) or archive it to
-  // unblock the gate, but the default is to halt until the failure is
-  // acknowledged. (needs_review is NOT a blocker — it just means the job ran
-  // but the verifier flagged something for human review.)
-  const blockingFailures = allJobs.filter((j) =>
-    j.status === 'failed' &&
-    (j.parallelGroup ?? 99) < lowestPendingGroup,
-  );
-  if (blockingFailures.length > 0) {
-    const slugs = blockingFailures.map((j) => j.slug).join(', ');
-    console.log(`[scheduler] failure-gate: holding g${lowestPendingGroup} — ${blockingFailures.length} failed job(s) in earlier groups [${slugs}]. Reset to pending or archive to unblock.`);
-    return [];
-  }
-
-  // Groups with at least one job in flight: either tracked in runningSet
-  // (this process spawned it) or still marked 'running' in queue.json
-  // (persisted from a previous session that hasn't been orphan-reset yet).
-  const activeGroups = new Set();
-  for (const slug of running) {
-    const job = allJobs.find((j) => j.slug === slug);
-    if (job) activeGroups.add(job.parallelGroup ?? 99);
-  }
-  for (const j of allJobs) {
-    if (j.status === 'running' && !running.has(j.slug)) {
-      activeGroups.add(j.parallelGroup ?? 99);
-    }
-  }
-  // Total slots consumed: in-process spawns + queue.json running count.
-  const queueRunningCount = allJobs.filter((j) => j.status === 'running').length;
-  const effectiveRunning = Math.max(running.size, queueRunningCount);
-
-  // (lowestPendingGroup was computed up-front for the failure-gate check.)
-
-  if (activeGroups.size > 0) {
-    const lowestActive = Math.min(...activeGroups);
-    if (lowestPendingGroup > lowestActive) {
-      // Earlier group still running — wait for it to drain before advancing.
-      console.log(`[scheduler] concurrency: g${lowestActive} in flight, holding g${lowestPendingGroup}`);
-      return [];
-    }
-    if (lowestPendingGroup < lowestActive) {
-      // Late-arrival: a lower-numbered (higher-priority) PRD reconciled AFTER
-      // a higher-numbered group was already picked. Without this branch the
-      // pending PRD starves until the active group drains — the bug observed
-      // on 2026-05-10 where 118-studio-add-wave2-games (g118) was held while
-      // the g130 hardening trio ran. Honor priority: fire the late-arrival
-      // now, in parallel with the active group. (Strict serial group
-      // ordering still applies between groups that were both present at the
-      // time of picking; this only handles the reconcile-race edge case.)
-      const slots = cap - effectiveRunning;
-      if (slots <= 0) {
-        console.log(`[scheduler] concurrency: cap ${cap} reached (${effectiveRunning} running), no slots for late-arrival g${lowestPendingGroup}`);
-        return [];
-      }
-      const batch = pending.filter((j) => (j.parallelGroup ?? 99) === lowestPendingGroup).slice(0, slots);
-      console.log(`[scheduler] concurrency: firing late-arrival g${lowestPendingGroup} (${batch.length} job(s)) alongside active g${lowestActive}`);
-      return batch;
-    }
-    // Backfill slots remaining in the current group.
-    const slots = cap - effectiveRunning;
-    if (slots <= 0) {
-      console.log(`[scheduler] concurrency: cap ${cap} reached (${effectiveRunning} running), no slots`);
-      return [];
-    }
-    const batch = pending.filter((j) => (j.parallelGroup ?? 99) === lowestActive).slice(0, slots);
-    if (batch.length > 0) {
-      console.log(`[scheduler] concurrency: backfilling ${batch.length} into g${lowestActive} (${effectiveRunning}/${cap} running)`);
-    }
-    return batch;
-  }
-
-  // No active group — start the next group fresh.
-  const slots = cap - effectiveRunning;
-  if (slots <= 0) {
-    console.log(`[scheduler] concurrency: cap ${cap} reached (${effectiveRunning} running), no slots`);
-    return [];
-  }
-  const batch = pending.filter((j) => (j.parallelGroup ?? 99) === lowestPendingGroup).slice(0, slots);
-  console.log(`[scheduler] concurrency: starting g${lowestPendingGroup} with ${batch.length} job(s) (cap ${cap})`);
-  return batch;
-}
+// pickNextBatch and pickForProject are defined in lib/schedulerBatch.cjs and
+// required at the top of this file. Group-ordering gates are evaluated per
+// project (keyed by cwd) so jobs in different repos run concurrently up to
+// the cap; within one project, sequential-group semantics are preserved.
 
 /**
  * Recognize fix-plan slugs (NN-fix-...) so we don't recurse on a fix-plan that
@@ -1486,10 +1376,60 @@ async function maybeLaunchWhenAvailable(state) {
   tickQueue().catch((e) => console.error('[scheduler] tickQueue error', e));
 }
 
+// ---------- dead-process reaper ----------
+
+/**
+ * Scan running jobs, identify those whose claude process is provably dead, and
+ * finalize them to completed/failed by reading the run log. Called once per
+ * poll cycle. Conservative: a job with no runtime.pid yet (spawn mid-flight)
+ * is always skipped. A job whose pid is alive (claudePidAlive) is always skipped.
+ * Exported so unit tests can invoke it directly.
+ */
+async function reapDeadRunningJobs() {
+  try {
+    if (runningSet.size === 0) return; // fast path: no in-flight jobs
+    const state = await readQueue();
+    const dead = [];
+    for (const j of state.jobs) {
+      if (j.status !== 'running') continue;
+      const pid = j.runtime?.pid;
+      if (!pid) continue; // spawn may be mid-flight; give it a cycle
+      if (claudePidAlive(pid)) continue;
+      const logPath = j.runId
+        ? path.join(RUNS_DIR, j.runId, `${j.slug}.log`)
+        : null;
+      const outcome = logPath ? classifyRunOutcome(logPath) : 'unknown';
+      dead.push({ slug: j.slug, pid, outcome });
+    }
+    if (dead.length === 0) return;
+
+    await mutate((s) => {
+      for (const { slug, pid, outcome } of dead) {
+        const idx = s.jobs.findIndex((x) => x.slug === slug);
+        if (idx < 0 || s.jobs[idx].status !== 'running') continue; // race guard
+        const success = outcome === 'success';
+        s.jobs[idx].status = success ? 'completed' : 'failed';
+        s.jobs[idx].exitCode = success ? 0 : (s.jobs[idx].exitCode ?? 1);
+        s.jobs[idx].finishedAt = new Date().toISOString();
+        s.jobs[idx].error = success ? null : `reaped: process gone, no success result in log (${outcome})`;
+        delete s.jobs[idx].runtime;
+        runningSet.delete(slug);
+        console.log(`[scheduler] reaped dead job slug=${slug} pid=${pid} outcome=${outcome}`);
+      }
+    });
+
+    await broadcast();
+    tickQueue().catch(() => {});
+  } catch (e) {
+    console.warn('[scheduler] reapDeadRunningJobs error', e?.message);
+  }
+}
+
 // ---------- poll loop with exponential backoff ----------
 
 async function pollLoop() {
   try {
+    await reapDeadRunningJobs().catch(() => {});
     const r = await billing.fetchUsage();
 
     if (r.kind === 'ok') {
@@ -1837,11 +1777,22 @@ async function init() {
   loadSchedulerState();
   bootedAt = Date.now();
 
-  // Boot reconciliation: mark any job that was 'running' when the app died as
-  // 'failed', AND kill its detached claude child if still alive. Without the
-  // kill step the child keeps running as a zombie writing to the project on
-  // its own schedule, which is exactly what happened on 2026-05-21 (PID 78230
-  // writing PRD 05's output while the scheduler thought the job was orphaned).
+  // Boot reconciliation: finalize any job that was 'running' when the app died.
+  // Check the run log first — a job that emitted result/success before the crash
+  // should be marked 'completed', not 'failed', so it doesn't wedge the queue
+  // via the failure-gate. Also kill any still-live orphan claude child to prevent
+  // it from continuing to write to the project unsupervised (2026-05-21 incident).
+  //
+  // classifyRunOutcome calls readTail → fs.readFileSync (up to 64 KB per job).
+  // Pre-compute all outcomes BEFORE entering the mutate lock so the blocking I/O
+  // does not stall the event loop or hold the mutateTail chain during startup.
+  const bootSnap = readQueueSync();
+  const bootOutcomes = new Map();
+  for (const j of bootSnap.jobs) {
+    if (j.status !== 'running') continue;
+    const logPath = j.runId ? path.join(RUNS_DIR, j.runId, `${j.slug}.log`) : null;
+    bootOutcomes.set(j.slug, logPath ? classifyRunOutcome(logPath) : 'unknown');
+  }
   await mutate((state) => {
     for (const j of state.jobs) {
       if (j.status === 'running') {
@@ -1854,10 +1805,14 @@ async function init() {
             console.log(`[scheduler] boot: SIGTERM'd orphan claude pid=${pid} for ${j.slug}`);
           }
         }
-        j.status = 'failed';
-        j.error = `orphaned: app restarted while running${killNote}`;
+        const outcome = bootOutcomes.get(j.slug) ?? 'unknown';
+        const success = outcome === 'success';
+        j.status = success ? 'completed' : 'failed';
+        j.exitCode = success ? 0 : (j.exitCode ?? 1);
+        j.error = success ? null : `orphaned: app restarted while running${killNote}`;
         j.finishedAt = new Date().toISOString();
         delete j.runtime;
+        console.log(`[scheduler] boot reconcile: slug=${j.slug} outcome=${outcome} → ${j.status}`);
       }
     }
   });
@@ -1934,4 +1889,97 @@ async function init() {
   }
 }
 
-module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL };
+// remote — callable from webRemote.cjs without going through IPC.
+const remote = {
+  async getState() {
+    const state = await readQueue();
+    await reconcile(state);
+    await writeQueue(state);
+    return buildScheduleStatePayload(state, { withPaths: true });
+  },
+
+  async readPrd(slug) {
+    const filePath = safeSlugPath(slug);
+    if (!filePath) return { ok: false, error: 'invalid slug' };
+    try {
+      // realpath resolves symlinks; re-check boundary to block a rogue agent job
+      // that places a symlink inside PRDS_DIR pointing outside the safe root.
+      const real = await fsp.realpath(filePath);
+      if (!real.startsWith(PRDS_DIR + path.sep)) {
+        return { ok: false, error: 'invalid slug' };
+      }
+      const text = await fsp.readFile(real, 'utf8');
+      return { ok: true, text };
+    } catch (e) {
+      return { ok: false, error: e?.message };
+    }
+  },
+
+  async readLog(slug, runId) {
+    const logPath = path.resolve(path.join(RUNS_DIR, runId, `${slug}.log`));
+    if (!logPath.startsWith(RUNS_DIR + path.sep)) {
+      return { ok: false, error: 'invalid slug or runId' };
+    }
+    try {
+      // realpath resolves symlinks; re-check boundary to block a rogue agent job
+      // that places a symlink inside RUNS_DIR pointing outside the safe root.
+      const real = await fsp.realpath(logPath);
+      if (!real.startsWith(RUNS_DIR + path.sep)) {
+        return { ok: false, error: 'invalid slug or runId' };
+      }
+      const text = await fsp.readFile(real, 'utf8');
+      return { ok: true, text };
+    } catch (e) {
+      return { ok: false, error: e?.message };
+    }
+  },
+
+  async writePrd(slug, body) {
+    const resolved = safeSlugPath(slug);
+    if (!resolved) return { ok: false, error: 'invalid slug' };
+    try {
+      await config.writeTextAtomic(resolved, body);
+      const stat = await fsp.stat(resolved);
+      return { ok: true, bytesWritten: stat.size };
+    } catch (e) {
+      return { ok: false, error: e?.message ?? 'write failed' };
+    }
+  },
+
+  async resetJob(slug) {
+    if (!safeSlugPath(slug)) return { ok: false, error: 'invalid slug' };
+    const found = await mutate((state) => {
+      const idx = state.jobs.findIndex((j) => j.slug === slug);
+      if (idx < 0) return false;
+      resetJobFields(state.jobs[idx]);
+      return true;
+    });
+    if (!found) return { ok: false, error: 'not found' };
+    await broadcast();
+    return { ok: true };
+  },
+
+  async runNow() {
+    await clearPause('run-now');
+    runDueJobs().catch((e) => logs.writeLine({
+      level: 'error', scope: 'scheduler',
+      message: 'runDueJobs error (remote:run-now)', meta: { error: e?.message },
+    }));
+    return { ok: true };
+  },
+
+  async setConfig(partial) {
+    const cfg = await mutate((state) => {
+      const { supervisor: supPartial, ...rest } = partial;
+      state.config = { ...state.config, ...rest };
+      if (supPartial !== undefined) {
+        state.config.supervisor = { ...(state.config.supervisor ?? {}), ...supPartial };
+      }
+      return state.config;
+    });
+    await rescheduleTimer();
+    return { ok: true, config: cfg };
+  },
+};
+
+module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs };