claude-code-session-manager 0.97.0 → 0.98.0

package/src/main/lib/watchdogHelpers.cjs

@@ -96,19 +96,28 @@ function heartbeatFresh(heartbeatPath = schedulerPaths.heartbeatPath(), maxAgeMs
 }
 
 const DEFAULT_DISPATCH_DEAD_MS = 30 * 60_000;
+const DEFAULT_TICK_WEDGED_DEAD_MS = 10 * 60_000;
 
 /**
- * evaluateDispatchLiveness(entry, now, { deadMs }?) → { dead, reason }
+ * evaluateDispatchLiveness(entry, now, { deadMs, wedgedDeadMs }?) → { dead, reason }
  *
  * Pure. Observability only — a logically-dead scheduler (wedged tick chain,
  * leaked slots) still writes fresh heartbeats, so freshness alone can't see
- * it. `dead` only when ALL hold: heartbeat fresh (not degraded, within
- * DEFAULT_MAX_AGE_MS), dispatch.lastRunAt older than deadMs (absent/unparseable
- * counts as older), pendingDispatchable > 0, runningCount === 0, not paused.
+ * it. `dead` when EITHER:
+ *   - dispatch.lastTickReason === 'wedged' and lastDispatchAttemptAt is
+ *     older than wedgedDeadMs (absent/unparseable counts as older) — a
+ *     wedged reconcile never derives rows, so pendingDispatchable stays 0
+ *     and the rule below can never fire (2026-09-25: watchdog logged
+ *     'alive'/'nothing-dispatchable' every 3 min through a 30+ min mutate
+ *     deadlock because of exactly this gap); or
+ *   - ALL of: dispatch.lastRunAt older than deadMs (absent/unparseable
+ *     counts as older), pendingDispatchable > 0, runningCount === 0.
+ * Both rules require: heartbeat fresh (not degraded, within
+ * DEFAULT_MAX_AGE_MS) and not paused/draining.
  * Callers must NOT relaunch on this: the process is alive, and a relaunch
  * would spawn a second app on top of it.
  */
-function evaluateDispatchLiveness(entry, now = Date.now(), { deadMs = DEFAULT_DISPATCH_DEAD_MS } = {}) {
+function evaluateDispatchLiveness(entry, now = Date.now(), { deadMs = DEFAULT_DISPATCH_DEAD_MS, wedgedDeadMs = DEFAULT_TICK_WEDGED_DEAD_MS } = {}) {
   if (!entry || entry.degraded === true || typeof entry.ts !== 'number') {
     return { dead: false, reason: 'no-fresh-heartbeat' };
   }
@@ -117,6 +126,12 @@ function evaluateDispatchLiveness(entry, now = Date.now(), { deadMs = DEFAULT_DI
   if (!d || typeof d !== 'object') return { dead: false, reason: 'no-dispatch-field' };
   if (d.paused) return { dead: false, reason: 'paused' };
   if (d.drain) return { dead: false, reason: 'draining' };
+  if (d.lastTickReason === 'wedged') {
+    const lastAttemptMs = Date.parse(d.lastDispatchAttemptAt ?? '');
+    if (!Number.isFinite(lastAttemptMs) || now - lastAttemptMs > wedgedDeadMs) {
+      return { dead: true, reason: 'tick-wedged' };
+    }
+  }
   if (!(d.pendingDispatchable > 0)) return { dead: false, reason: 'nothing-dispatchable' };
   if (d.runningCount !== 0) return { dead: false, reason: 'jobs-running' };
   const lastRunMs = Date.parse(d.lastRunAt ?? '');
@@ -461,6 +476,7 @@ module.exports = {
   maybeRelaunchApp,
   DEFAULT_MAX_AGE_MS,
   DEFAULT_DISPATCH_DEAD_MS,
+  DEFAULT_TICK_WEDGED_DEAD_MS,
   DEFAULT_LOCK_STALE_MS,
   DEFAULT_FINALIZE_BUDGET_MS,
   DEFAULT_RELAUNCH_DEBOUNCE_MS,

package/src/main/queueOps.cjs

@@ -341,13 +341,17 @@ async function archiveOne(slug, ts, cwd) {
  * Injected by index.cjs at registration time (see registerQueueOpsHandlers)
  * so a manual archive can retire any still-runnable queue job for the same
  * slug without queueOps.cjs importing scheduler.cjs (circular — scheduler.cjs
- * already requires queueOps.cjs). No-op until set; the auto-archive path
- * never needs it (selectAutoArchivable only ever selects already-completed
- * jobs).
+ * already requires queueOps.cjs). No-op until set. Only ever invoked when
+ * archiveMany is called with { retire: true } (the default, used by the
+ * manual schedule:archive-prd path) — autoArchiveCompleted passes
+ * { retire: false } because it calls retireCompletedSlugsFn (== scheduler's
+ * retireCompletedSlugs) from inside reconcile(), itself always inside a
+ * mutate() body; retireCompletedSlugs's own mutate() call would then queue
+ * behind the mutate awaiting it and deadlock every tick (2026-09-25 incident).
  */
 let retireCompletedSlugsFn = async () => {};
 
-async function archiveMany(slugs, cwd) {
+async function archiveMany(slugs, cwd, { retire = true } = {}) {
   if (!Array.isArray(slugs) || slugs.length === 0) {
     return { ok: true, archived: 0, archivedTo: null, results: [] };
   }
@@ -362,7 +366,7 @@ async function archiveMany(slugs, cwd) {
   }
   const archived = results.filter((r) => r.ok).length;
   const archivedSlugs = results.filter((r) => r.ok).map((r) => r.slug);
-  if (archivedSlugs.length > 0) {
+  if (retire && archivedSlugs.length > 0) {
     await retireCompletedSlugsFn(archivedSlugs).catch((e) => {
       logs.writeLine({ level: 'warn', scope: 'queueOps', message: 'archiveMany: retireCompletedSlugs failed', meta: { error: e?.message } });
     });
@@ -440,6 +444,13 @@ function selectAutoArchivable(jobs, { nowMs, retentionMs } = {}) {
  * the same path-containment. No-op (without touching disk) when
  * SM_PRD_AUTOARCHIVE_DISABLE=1. Each successful move is logged one line
  * (slug -> dest).
+ *
+ * Passes { retire: false } to archiveMany: this runs inside reconcile(),
+ * which always runs inside a mutate() body or tickBody, so invoking
+ * retireCompletedSlugs (which itself calls mutate()) here would queue that
+ * mutate behind the one already awaiting it and deadlock every tick — proven
+ * live 2026-09-25. Also unneeded on the merits: selectAutoArchivable only
+ * ever selects already-completed jobs, so there is nothing left to retire.
  */
 async function autoArchiveCompleted(state, { nowMs } = {}) {
   if (process.env.SM_PRD_AUTOARCHIVE_DISABLE === '1') {
@@ -450,7 +461,7 @@ async function autoArchiveCompleted(state, { nowMs } = {}) {
   if (slugs.length === 0) {
     return { ok: true, archived: 0, archivedTo: null, results: [] };
   }
-  const result = await archiveMany(slugs);
+  const result = await archiveMany(slugs, undefined, { retire: false });
   for (const r of result.results) {
     if (r.ok) {
       logs.writeLine({

package/src/main/scheduler.cjs

@@ -47,6 +47,7 @@ const fs = require('node:fs');
 const fsp = require('node:fs/promises');
 const path = require('node:path');
 const os = require('node:os');
+const { AsyncLocalStorage } = require('node:async_hooks');
 const { startDispatchLoop } = require('./lib/dispatchLoop.cjs');
 const schedulerPaths = require('./lib/schedulerPaths.cjs');
 const { randomUUID } = require('node:crypto');
@@ -1328,9 +1329,17 @@ async function archiveCompletedPrd(slug, cwd) {
  * `schedule:archive-prd`) so a stale queue entry can never survive to fire
  * against a PRD that no longer exists in the live prds/ dir — the same
  * ENOENT-avoidance archivedTwinExists provides in executeJob, applied at the
- * archiving source instead of at fire-time. auto-archived slugs never need
- * this (selectAutoArchivable in queueOps.cjs only selects already-completed
- * jobs), so this is exercised only by the manual archive path.
+ * archiving source instead of at fire-time.
+ *
+ * MUST NEVER be called from inside a mutate() body (directly, or transitively
+ * via reconcile()/tickBody, which always run inside one) — this function
+ * itself calls mutate() below, and a mutate queued from inside a mutate body
+ * queues behind the very mutate awaiting it, so mutateTail never settles:
+ * every tick wedges machine-wide until the app restarts (proven live
+ * 2026-09-25). That is exactly why queueOps.cjs's autoArchiveCompleted (which
+ * DOES run inside reconcile()) calls archiveMany with { retire: false } and
+ * skips this function entirely — it is exercised only by the manual archive
+ * path (queueOps.cjs's `schedule:archive-prd`, called from outside mutate()).
  */
 async function retireCompletedSlugs(slugs) {
   const list = Array.isArray(slugs) ? slugs.filter(Boolean) : [];
@@ -2469,6 +2478,15 @@ async function writeQueue(state) {
 // preceding mutate threw, so the chain never deadlocks.
 let mutateTail = Promise.resolve();
 
+// Tracks whether the current async continuation is running inside a live
+// mutate() body. A mutate() called (and awaited) from inside another mutate's
+// `fn` would otherwise queue behind its own caller on mutateTail and deadlock
+// forever — this turns that class of bug into an immediate, audited rejection.
+// A callback (setTimeout/promise) scheduled from inside a mutate body but
+// firing AFTER that body has finished sees ctx.active === false and proceeds
+// normally, since AsyncLocalStorage propagates the store into it.
+const mutateCtx = new AsyncLocalStorage();
+
 // Observe-only watchdog: a mutate body over MUTATE_WATCHDOG_MS is logged and
 // audited once per episode (latched until a mutate completes). mutateTail is
 // deliberately NEVER reset — it is what enforces the single-writer law, and
@@ -2477,6 +2495,12 @@ const MUTATE_WATCHDOG_MS = 60_000;
 let mutateWedgeLatched = false;
 
 function mutate(fn) {
+  if (mutateCtx.getStore()?.active === true) {
+    const err = new Error('mutate() re-entered from inside a running mutate body — would deadlock');
+    console.error('[scheduler] MUTATE RE-ENTRANT', err.stack);
+    appendAuditEvent('mutate_reentrant', { stack: String(err.stack).split('\n').slice(0, 8).join('\n') });
+    return Promise.reject(err);
+  }
   const next = mutateTail.then(async () => {
     const wedgeTimer = setTimeout(() => {
       if (mutateWedgeLatched) return;
@@ -2485,9 +2509,11 @@ function mutate(fn) {
       appendAuditEvent('mutate_wedged', { budgetMs: MUTATE_WATCHDOG_MS });
     }, MUTATE_WATCHDOG_MS);
     if (typeof wedgeTimer.unref === 'function') wedgeTimer.unref();
+    const ctx = { active: true };
     try {
-      return await mutateBody(fn);
+      return await mutateCtx.run(ctx, () => mutateBody(fn));
     } finally {
+      ctx.active = false;
       clearTimeout(wedgeTimer);
       mutateWedgeLatched = false;
     }
@@ -3775,13 +3801,31 @@ function computeFireAt(state, nextResetIso) {
   return reset + (state.config.offsetMinutes * 60_000);
 }
 
+// PRD 1446: refreshNextReset() awaits billing.fetchUsage(), which can hang
+// or back off for minutes while the usage meter is rate-limited. Left
+// unbounded, that stalled THIS function's own mutate(reconcile) call for as
+// long as the fetch took — the 10-minute rescheduleTimer was meant to be a
+// second, independent path into reconcile() alongside the 60s pollLoop, but
+// a hung fetch silently took it out too, leaving newly-created PRDs
+// unadopted (PRDs 1420-1441, 2026-09-25). Racing against this timeout lets
+// the reconcile still run on the cached reset value; the abandoned
+// refreshNextReset() call is left to resolve on its own and update the
+// cache for next time.
+const RESCHEDULE_TIMER_BILLING_RACE_MS = 10_000;
+
 async function rescheduleTimer() {
   clearFireTimer();
   // Wrap in try/catch — on failure use the cached value so the on-reset
   // timer can still be armed from the last known reset.
   let nextResetIso;
   try {
-    nextResetIso = await refreshNextReset();
+    nextResetIso = await Promise.race([
+      refreshNextReset(),
+      new Promise((resolve) => {
+        const t = setTimeout(() => resolve(cachedNextReset), RESCHEDULE_TIMER_BILLING_RACE_MS);
+        if (typeof t.unref === 'function') t.unref();
+      }),
+    ]);
   } catch {
     nextResetIso = cachedNextReset;
   }
@@ -12453,6 +12497,19 @@ const remote = {
     }
   },
 
+  // Triggers an immediate reconcile pass through the SAME seam resetJob
+  // already uses below (broadcast's coalescer, whose getPayload runs
+  // module.exports.reconcile) — no second reconcile implementation. PRD
+  // 1446: prdCreate.cjs's createPrd() calls this right after a successful
+  // write so a fresh PRD becomes a pending queue row without waiting for
+  // the next scheduled pass — the 60s pollLoop only reaches reconcile() via
+  // maybeLaunchWhenAvailable, which returns early while zero rows are
+  // pending, and the 10-minute rescheduleTimer can itself stall behind a
+  // slow/hung billing fetch (see rescheduleTimer's own bounded race below).
+  async requestReconcile() {
+    await broadcast({ flush: true });
+  },
+
   // User-initiated pause/resume — the admin-route/MCP twins of the
   // schedule:pause / schedule:resume IPC handlers, through the same setPaused /
   // clearPause. Pause stops NEW dispatch only; running jobs are never touched.
@@ -12876,8 +12933,11 @@ module.exports = {
   computeDegradedBudget,
   healRefusalReason,
   writeQueue,
+  _mutateForTests: mutate, // mutate() has no other exported call site; test-only seam.
   reconcile,
   broadcast,
+  rescheduleTimer,
+  RESCHEDULE_TIMER_BILLING_RACE_MS,
   reconcileSourcePromptId,
   allocateParallelGroup,
   selectHistoryJobs,