@phnx-labs/agents-cli 1.22.66 → 1.22.68

package/dist/lib/daemon/daemon.js

@@ -34,7 +34,8 @@ import { ServiceSupervisor } from './supervisor.js';
 import { SessionIndexService } from './session-index-service.js';
 import { SecretsBrokerService } from './secrets-broker-service.js';
 import { MonitorEngineService } from './monitor-engine-service.js';
-import { AccountStateDaemonService } from './account-state-daemon-service.js';
+import { AccountUsageService, AccountAuthService } from './account-state-daemon-service.js';
+import { CatchupService } from './catchup-service.js';
 import { BrowserIPCService } from './browser-ipc-service.js';
 import { WatchdogService } from './watchdog-service.js';
 import { DeviceProbeService } from './device-probe-service.js';
@@ -50,6 +51,7 @@ import { TmuxReapService } from './tmux-reap-service.js';
 import { BrowserTaskReapService } from './browser-task-reap-service.js';
 import { emit, emitRoutineEnd } from '../feed/events.js';
 import { readDaemonServicesConfig, drainDaemonServiceRestartQueue } from '../daemon-services.js';
+import { sleepSync } from '../fs-atomic.js';
 /**
  * The live `ServiceSupervisor` for the current `runDaemon()` invocation, or
  * `null` before boot / after shutdown. In-process only — a separate `agents
@@ -114,14 +116,9 @@ export function daemonSystemdUnitName() {
     const suffix = isolatedHomeSuffix();
     return suffix ? `agents-daemon-sandbox-${suffix}.service` : SYSTEMD_UNIT;
 }
-/**
- * How often to re-scan for missed fires. Deliberately slower than the monitor
- * tick: detection walks a week of cron occurrences per routine
- * (`previousExpectedFire`), and a fire that was already missed is not urgent to
- * the second — five minutes bounds the cost while still recovering from a
- * wedge or an OS suspend the process survived.
- */
-const CATCHUP_TICK_MS = 5 * 60_000;
+// Catch-up cadence + the supervised CatchupService live in catchup-service.ts
+// (PHNX-3608): the pass now runs under the ServiceSupervisor with a deadline +
+// AbortSignal + circuit breaker instead of a bare setInterval.
 /**
  * Cadences for the in-process background ticks, named here beside the other
  * tick constants rather than left as inline literals at their `setInterval`
@@ -265,6 +262,25 @@ function acquireStartLock() {
         throw err;
     }
 }
+/**
+ * Stop is a lifecycle mutation just like start/claim, so it must cross the same
+ * lock. A claim can legitimately hold the lock through the incumbent's 5s
+ * graceful window plus the 2s hard-kill backstop; wait beyond that complete
+ * takeover window before failing loud instead of running teardown unlocked.
+ */
+const STOP_LOCK_WAIT_MS = 10_000;
+const STOP_LOCK_POLL_MS = 50;
+function acquireStopLock() {
+    const deadline = Date.now() + STOP_LOCK_WAIT_MS;
+    for (;;) {
+        const release = acquireStartLock();
+        if (release)
+            return release;
+        if (Date.now() >= deadline)
+            return null;
+        sleepSync(Math.min(STOP_LOCK_POLL_MS, deadline - Date.now()));
+    }
+}
 /**
  * Absolute path to the daemon's structured log.
  *
@@ -306,6 +322,16 @@ export function removeDaemonPid() {
         fs.unlinkSync(pidPath);
     }
 }
+/** Remove the pid registration only while it still names the owner we observed. */
+function removeDaemonPidIfOwned(pid) {
+    if (readDaemonPid() !== pid)
+        return false;
+    try {
+        fs.unlinkSync(getPidPath());
+    }
+    catch { /* already removed */ }
+    return readDaemonPid() !== pid;
+}
 function getHeartbeatPath() {
     return path.join(ensureDaemonDir(), HEARTBEAT_FILE);
 }
@@ -348,7 +374,7 @@ export function isDaemonWedged() {
     const pid = readDaemonPid();
     if (!pid)
         return false;
-    if (!isAlive(pid))
+    if (!isLiveDaemon(pid))
         return false;
     const hb = readHeartbeat();
     if (!hb)
@@ -372,32 +398,66 @@ const STOP_KILL_GRACE_MS = 2000;
  * the pid file then reports "stopped" for a running scheduler, and (worse) lets
  * claimDaemonInstance() start a SECOND daemon that double-fires every routine.
  *
- * So: trust the pid file when its pid is alive; otherwise trust a FRESH
- * heartbeat whose pid is alive, and re-adopt the pid file so the desync heals.
- * Returns null only when neither points at a live process (clearing a stale pid
- * file on the way out).
+ * So: trust the pid file only when its pid is a live `__daemon-run`; otherwise
+ * trust a FRESH heartbeat whose pid passes the same identity check. Callers that
+ * already own daemon.lock may request repair, re-adopting the heartbeat pid or
+ * removing the exact stale pid they observed. Read-only liveness probes never
+ * mutate shared state outside that lock.
  */
-function resolveLiveDaemonPid() {
+function resolveLiveDaemonPid(repair = false) {
     const pid = readDaemonPid();
-    if (pid !== null && isAlive(pid))
+    const pidIdentity = pid !== null ? daemonProcessIdentity(pid) : 'dead';
+    if (pid !== null && pidIdentity === 'daemon')
         return pid;
     const hb = readHeartbeat();
-    if (hb && isAlive(hb.pid) && isHeartbeatFresh(hb)) {
-        if (pid !== hb.pid)
-            writeDaemonPid(hb.pid); // heal the pid-file/heartbeat desync
+    if (hb && isHeartbeatFresh(hb) && daemonProcessIdentity(hb.pid) === 'daemon') {
+        if (repair && pid !== hb.pid)
+            writeDaemonPid(hb.pid); // lock owner heals the desync
         return hb.pid;
     }
-    if (pid !== null)
-        removeDaemonPid();
+    // A failed command-line inspection is not proof the pid is stale. Leave the
+    // registration intact so a sandbox/permission failure cannot erase the only
+    // owner record and trigger a duplicate daemon.
+    if (repair && pid !== null && pidIdentity !== 'unknown')
+        removeDaemonPidIfOwned(pid);
+    return null;
+}
+/** A live recorded pid whose command identity could not be inspected. */
+function unverifiedLiveDaemonPid() {
+    const pid = readDaemonPid();
+    if (pid !== null && daemonProcessIdentity(pid) === 'unknown')
+        return pid;
+    const hb = readHeartbeat();
+    if (hb && isHeartbeatFresh(hb) && daemonProcessIdentity(hb.pid) === 'unknown')
+        return hb.pid;
     return null;
 }
 /**
  * Check whether a daemon is alive — via the pid file, or a fresh heartbeat when
- * the pid file has been lost (see resolveLiveDaemonPid). Heals the pid file as a
- * side effect so a subsequent read is consistent.
+ * the pid file has been lost (see resolveLiveDaemonPid). The observation itself
+ * is read-only; when it finds desync it opportunistically acquires daemon.lock
+ * and repeats the observation there before repairing. A contended probe still
+ * returns the observed liveness without mutating another lifecycle operation's
+ * state.
  */
 export function isDaemonRunning() {
-    return resolveLiveDaemonPid() !== null;
+    // Fail safe for reporting/start suppression: inability to inspect a live pid
+    // is never permission to declare it dead and launch a duplicate.
+    if (unverifiedLiveDaemonPid() !== null)
+        return true;
+    const recordedPid = readDaemonPid();
+    const livePid = resolveLiveDaemonPid();
+    if (recordedPid === livePid)
+        return livePid !== null;
+    const release = acquireStartLock();
+    if (!release)
+        return livePid !== null;
+    try {
+        return resolveLiveDaemonPid(true) !== null;
+    }
+    finally {
+        release();
+    }
 }
 /**
  * Single-instance claim for the daemon foreground entrypoint.
@@ -412,9 +472,9 @@ export function isDaemonRunning() {
  * pid file, this does NOT defer to it — it evicts the incumbent and becomes the
  * survivor, so a second install can never leave two daemons running. Returns true
  * and records our PID once the incumbent is provably dead (its resources
- * released). Returns false ONLY when another `__daemon-run` currently holds the
- * O_EXCL start lock — i.e. a concurrent claimer is mid-takeover and will be the
- * singleton — in which case the caller must exit without touching further state.
+ * released). Returns false when another `__daemon-run` currently holds the
+ * O_EXCL start lock, or when the incumbent cannot be safely identified/evicted,
+ * in which case the caller must exit without touching further state.
  * The read-evict-write is serialized behind the same start lock startDaemon()
  * uses, so two `_run` processes can't both claim in the window between the
  * liveness check and the write.
@@ -431,11 +491,15 @@ export function claimDaemonInstance() {
     if (!release)
         return false;
     try {
+        // Do not overwrite a live-but-uninspectable owner. This is the non-
+        // destructive side of the same fail-closed rule stopDaemon applies.
+        if (unverifiedLiveDaemonPid() !== null)
+            return false;
         // resolveLiveDaemonPid() also consults a fresh heartbeat, so a live daemon
         // whose pid file was lost is still found and evicted — otherwise a missing
         // pid file would let both this instance AND the orphaned incumbent run a
         // JobScheduler at once and double-fire every routine.
-        const existing = resolveLiveDaemonPid();
+        const existing = resolveLiveDaemonPid(true);
         if (existing !== null && existing !== process.pid) {
             // Evict, and WAIT for the incumbent to be provably dead — its graceful
             // handleShutdown releasing the browser IPC binding and the secrets broker
@@ -443,7 +507,8 @@ export function claimDaemonInstance() {
             // Binding before the release recreates the two-brokers-on-one-socket orphan
             // documented at stopDaemon below, so the pid file is not written until the
             // prior owner is gone.
-            evictIncumbentDaemon(existing);
+            if (!evictIncumbentDaemon(existing))
+                return false;
         }
         writeDaemonPid(process.pid);
         return true;
@@ -465,24 +530,40 @@ export function claimDaemonInstance() {
  * proof-of-release requirement applies.
  */
 function evictIncumbentDaemon(pid) {
+    const beforeSignal = daemonProcessIdentity(pid);
+    if (beforeSignal === 'dead' || beforeSignal === 'other')
+        return true;
+    if (beforeSignal === 'unknown')
+        return false;
     if (process.platform === 'win32') {
         // No graceful termination signal on Windows — take the incumbent down and
         // still wait for the kill to land before the caller binds anything (mirrors
         // stopDaemon's win32 branch).
         killTree(pid);
-        waitForExit(pid, STOP_KILL_GRACE_MS);
-        return;
+        if (waitForExit(pid, STOP_KILL_GRACE_MS))
+            return true;
+        const afterKill = daemonProcessIdentity(pid);
+        return afterKill === 'dead' || afterKill === 'other';
     }
     try {
         process.kill(pid, 'SIGTERM');
     }
     catch {
-        return; // already gone between resolveLiveDaemonPid() and here
+        const afterSignal = daemonProcessIdentity(pid);
+        return afterSignal === 'dead' || afterSignal === 'other';
     }
     if (waitForExit(pid, STOP_GRACE_MS))
-        return; // graceful release complete
+        return true; // graceful release complete
+    const beforeKill = daemonProcessIdentity(pid);
+    if (beforeKill === 'dead' || beforeKill === 'other')
+        return true;
+    if (beforeKill === 'unknown')
+        return false;
     killTree(pid); // positive pid: SIGKILL reaches the daemon, not its job children
-    waitForExit(pid, STOP_KILL_GRACE_MS);
+    if (waitForExit(pid, STOP_KILL_GRACE_MS))
+        return true;
+    const afterKill = daemonProcessIdentity(pid);
+    return afterKill === 'dead' || afterKill === 'other';
 }
 /** Directory that registers every live daemon of THIS device (one per state dir). */
 function getDaemonInstancesDir() {
@@ -546,7 +627,7 @@ export function reapStrayDaemons(keepPid = process.pid) {
     catch {
         return { reaped, details }; // no registry yet — nothing to reap
     }
-    const ownerPid = readDaemonPid();
+    const ownerPid = resolveLiveDaemonPid();
     const dropMarker = (name) => {
         try {
             fs.rmSync(path.join(dir, name), { force: true });
@@ -566,40 +647,73 @@ export function reapStrayDaemons(keepPid = process.pid) {
         }
         // Live pid, but guard against pid reuse: only a real `__daemon-run` is ours.
         // Process ARGS are readable cross-platform (unlike ENV on hardened macOS).
-        if (!isDaemonRunProcess(pid)) {
+        const identity = daemonProcessIdentity(pid);
+        if (identity === 'unknown') {
+            details.push(`could not verify stray daemon pid ${pid}; marker retained`);
+            continue;
+        }
+        if (identity !== 'daemon') {
             dropMarker(name);
             continue;
         }
         try {
             process.kill(pid, 'SIGTERM');
+        }
+        catch { /* already gone between the alive check and the signal */ }
+        if (waitForExit(pid, STOP_GRACE_MS)) {
             reaped++;
             details.push(`reaped stray daemon pid ${pid}`);
+            dropMarker(name);
+            continue;
+        }
+        // The pid may have been recycled during the grace window. Never hard-kill
+        // it unless it still identifies as a daemon; a stale registry marker is all
+        // we own when the command identity changed.
+        const beforeKill = daemonProcessIdentity(pid);
+        if (beforeKill === 'unknown') {
+            details.push(`could not reverify stray daemon pid ${pid}; marker retained`);
+            continue;
+        }
+        if (beforeKill !== 'daemon') {
+            dropMarker(name);
+            continue;
+        }
+        killTree(pid);
+        if (waitForExit(pid, STOP_KILL_GRACE_MS)) {
+            reaped++;
+            details.push(`reaped stray daemon pid ${pid} (escalated)`);
+            dropMarker(name);
+        }
+        else {
+            // Keep the only marker for a process that survived both signals so the
+            // next reaper/doctor can still see it.
+            details.push(`stray daemon pid ${pid} survived SIGKILL`);
         }
-        catch { /* already gone between the alive check and the signal */ }
-        dropMarker(name);
     }
     return { reaped, details };
 }
-/**
- * Whether `pid` is a live `agents __daemon-run` process. Reads the process's
- * command line (`ps -p <pid> -o command=`), which — unlike its environment — is
- * visible cross-platform, including on hardened macOS. Guards the reaper against
- * killing an unrelated process that reused a dead registrant's pid.
- */
-function isDaemonRunProcess(pid) {
-    if (process.platform === 'win32')
-        return false;
+function daemonProcessIdentity(pid) {
+    if (!isAlive(pid))
+        return 'dead';
     try {
-        const out = execFileSync('ps', ['-p', String(pid), '-o', 'command='], {
-            encoding: 'utf-8',
-            stdio: ['ignore', 'pipe', 'ignore'],
-        });
-        return /\b__daemon-run\b/.test(out);
+        const out = process.platform === 'win32'
+            ? execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', `(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true, timeout: 5000 })
+            : execFileSync('ps', ['-ww', '-p', String(pid), '-o', 'command='], {
+                encoding: 'utf-8',
+                stdio: ['ignore', 'pipe', 'ignore'],
+            });
+        // getDaemonLaunch always places __daemon-run last. Matching it anywhere in
+        // the command line mistakes an agent prompt or shell script that merely
+        // mentions the token for the shared daemon.
+        return /(?:^|\s)["']?__daemon-run["']?\s*$/.test(out.trim()) ? 'daemon' : 'other';
     }
     catch {
-        return false;
+        return 'unknown';
     }
 }
+export function isLiveDaemon(pid) {
+    return daemonProcessIdentity(pid) === 'daemon';
+}
 function rotateLogsIfNeeded(logPath) {
     try {
         const stat = fs.statSync(logPath);
@@ -643,6 +757,17 @@ export function log(level, message) {
     }
     catch { /* never crash the daemon on event-log failure */ }
 }
+/** Keep synchronous signal-handler failures inside the daemon's crash barrier. */
+export function guardSignalHandler(handler, onError) {
+    return () => {
+        try {
+            handler();
+        }
+        catch (err) {
+            onError(err);
+        }
+    };
+}
 /** Main daemon loop: load jobs, schedule crons, monitor runs, and handle signals. */
 /**
  * Anchor the daemon's working directory to a stable, always-present path.
@@ -759,17 +884,47 @@ export async function runDaemon() {
     // HOME override must refuse to run against the operator's real state rather than
     // schedule against the real host. No-op in production (the marker is never set).
     assertTestDaemonHome();
+    // Install the shared-daemon reload signal boundary BEFORE publishing our PID
+    // in claimDaemonInstance(). Browser/routines clients use that PID to decide a
+    // daemon exists and may request a service reload immediately. POSIX otherwise
+    // applies its default SIGHUP action during the rest of startup and terminates
+    // the whole process — exactly the client-caused eviction PHNX-3605 forbids.
+    // Requests received before services are ready coalesce into one reload and are
+    // applied through the normal guarded handler once startup completes.
+    let reloadRequestedDuringStartup = false;
+    let liveReloadHandler = null;
+    const dispatchReloadSignal = () => {
+        if (liveReloadHandler)
+            liveReloadHandler();
+        else
+            reloadRequestedDuringStartup = true;
+    };
+    if (process.platform !== 'win32')
+        process.on('SIGHUP', dispatchReloadSignal);
     // Single-instance guard (last-wins, SING-11): a direct `agents __daemon-run`
     // (manual, or a service-manager restart racing a live predecessor) EVICTS the
     // incumbent and becomes the survivor. claimDaemonInstance returns false only
     // when a concurrent `__daemon-run` currently holds the start lock — that peer
     // is mid-takeover and will be the singleton, so this instance stands down.
     if (!claimDaemonInstance()) {
-        log('WARN', `Another daemon is mid-takeover (holds the start lock); this instance (PID ${process.pid}) is exiting`);
+        if (process.platform !== 'win32')
+            process.removeListener('SIGHUP', dispatchReloadSignal);
+        log('WARN', `Another daemon owns lifecycle state or is mid-takeover; this instance (PID ${process.pid}) is exiting`);
         // Exit cleanly (0) so a service manager treats it as an orderly no-op
         // rather than a failure to restart-flap on.
         process.exit(0);
     }
+    // Deterministic integration-test seam for the PID-published/startup-complete
+    // signal window above. It is honored only inside an explicitly isolated test
+    // HOME; production daemons never pause here.
+    const startupDelayRaw = process.env.AGENTS_DAEMON_TEST_STARTUP_DELAY_MS;
+    if (process.env.AGENTS_DAEMON_TEST_HOME && startupDelayRaw) {
+        const startupDelayMs = Number(startupDelayRaw);
+        if (!Number.isInteger(startupDelayMs) || startupDelayMs < 1 || startupDelayMs > 10_000) {
+            throw new Error('AGENTS_DAEMON_TEST_STARTUP_DELAY_MS must be an integer from 1 to 10000');
+        }
+        await new Promise((resolve) => setTimeout(resolve, startupDelayMs));
+    }
     // Unlike the pid and heartbeat files, this marker is written exactly once
     // for this daemon lifetime. Status probes deliberately repair those other
     // files, so they cannot prove that the original state dir still exists.
@@ -857,18 +1012,46 @@ export async function runDaemon() {
         supervisor.register(monitorEngineSvc);
     else
         log('INFO', 'Monitor engine disabled');
+    // Usage and auth refresh are two INDEPENDENT supervised services (PHNX-3608)
+    // so a run of usage-refresh failures parks only usage and never starves the
+    // slower auth refresh — each carries its own circuit breaker.
     if (isEnabled('account-state'))
-        supervisor.register(new AccountStateDaemonService());
+        supervisor.register(new AccountUsageService());
     else
         log('INFO', 'Account-state service disabled');
-    // BrowserIPCService and BrowserTaskReapService share one long-lived
-    // BrowserService when both are enabled. Either service can run independently.
-    const browserService = isEnabled('browser-ipc') || isEnabled('browser-task-reap')
-        ? new BrowserService()
-        : null;
-    if (isEnabled('browser-ipc') && browserService)
-        supervisor.register(new BrowserIPCService(browserService));
+    if (isEnabled('account-auth'))
+        supervisor.register(new AccountAuthService());
     else
+        log('INFO', 'Account-auth service disabled');
+    // The routine scheduler handle. Declared HERE — before the CatchupService
+    // registration — because `supervisor.startAll()` below fires each service's
+    // first tick synchronously, so the `catchup` tick reads `scheduler` during
+    // startAll, BEFORE this `let` would initialise if it lived at its old textual
+    // position further down. That was a real TDZ `ReferenceError` on every boot
+    // ("Cannot access 'scheduler' before initialization"), not a race (PHNX-3608).
+    // `bootScheduler`/`stopScheduler` (hoisted below) assign this same binding.
+    let scheduler = null;
+    // Catch-up recovery under the supervisor (PHNX-3608). The closures reference
+    // `scheduler` (declared just above) and `catchupPass` (a hoisted function
+    // declaration). The tick self-gates on the scheduler being booted, so it is a
+    // cheap no-op — including on its immediate first tick during startAll, when
+    // `scheduler` is still null — on a device whose scheduler.enabled gate is off.
+    if (isEnabled('catchup')) {
+        supervisor.register(new CatchupService({
+            isSchedulerBooted: () => scheduler !== null,
+            runPass: (signal) => catchupPass(signal),
+        }));
+    }
+    else {
+        log('INFO', 'Catch-up recovery service disabled');
+    }
+    // BrowserIPCService and BrowserTaskReapService share one long-lived
+    // BrowserService. Browser IPC is registered even when disabled at boot so an
+    // explicit later `agents browser start` can enable it live over SIGHUP — the
+    // client owns a service transition, never a whole-daemon restart (PHNX-3605).
+    const browserService = new BrowserService();
+    supervisor.register(new BrowserIPCService(browserService), { enabled: isEnabled('browser-ipc') });
+    if (!isEnabled('browser-ipc'))
         log('INFO', 'Browser IPC service disabled');
     if (isEnabled('session-index'))
         supervisor.register(new SessionIndexService());
@@ -914,7 +1097,7 @@ export async function runDaemon() {
         supervisor.register(new TmuxReapService());
     else
         log('INFO', 'Tmux reap service disabled');
-    if (isEnabled('browser-task-reap') && browserService) {
+    if (isEnabled('browser-task-reap')) {
         supervisor.register(new BrowserTaskReapService(browserService));
     }
     else {
@@ -1014,15 +1197,13 @@ export async function runDaemon() {
                 .catch(() => { });
         }
     };
-    let scheduler = null;
-    let catchupInterval;
-    // Catchup overlap guard. Declared up here (not beside catchupPass) because
-    // bootScheduler() runs before catchupPass's textual position — a `let` down
-    // there would still be in its TDZ at the first call and crash the daemon.
-    let catchingUp = false;
-    // Boot the scheduler + catchup recovery. Called at daemon start when the gate
-    // allows, and again from handleReload when the gate flips on (function
-    // declarations hoist — catchupPass below is in scope).
+    // `scheduler` is declared earlier (before the CatchupService registration) to
+    // avoid a TDZ read during supervisor.startAll — see the comment there.
+    // Boot the scheduler. Called at daemon start when the gate allows, and again
+    // from handleReload when the gate flips on. Catch-up recovery is a separate
+    // supervised service (CatchupService, registered above) that self-gates on
+    // `scheduler !== null`; here we just kick an immediate supervised pass so a
+    // fresh boot catches up missed fires without waiting a full CATCHUP_TICK_MS.
     function bootScheduler() {
         scheduler = new JobScheduler(triggerJob);
         scheduler.loadAll();
@@ -1031,17 +1212,14 @@ export async function runDaemon() {
         for (const job of scheduled) {
             log('INFO', `  ${job.name} -> next: ${job.nextRun?.toISOString() || 'unknown'}`);
         }
-        void catchupPass();
-        catchupInterval = setInterval(() => { void catchupPass(); }, CATCHUP_TICK_MS);
+        if (supervisor.isRegistered('catchup'))
+            supervisor.runNow('catchup');
     }
-    // Stop the scheduler + catchup recovery (gate flipped off on reload).
+    // Stop the scheduler (gate flipped off on reload). The CatchupService keeps its
+    // supervised timer but its tick no-ops once `scheduler` is null.
     function stopScheduler() {
         scheduler?.stopAll();
         scheduler = null;
-        if (catchupInterval !== undefined) {
-            clearInterval(catchupInterval);
-            catchupInterval = undefined;
-        }
     }
     // Materialise opted-in project routines into the user layer on every start
     // so a fresh daemon picks up project YAML without a separate sync step.
@@ -1078,19 +1256,15 @@ export async function runDaemon() {
     // `catchup: false`, RUN late. Runs on a timer as well as at startup: a startup
     // pass alone misses a fire lost while the daemon stayed up but its event loop
     // was wedged, or one lost across an OS suspend that the process survived.
-    // Overlap guard, same shape as SelfHealService's supervisor-owned inFlight guard. A pass
-    // awaits executeJobDetached per job and an off-box (host/cloud) dispatch can
-    // block for a while, so a slow pass could still be working when the next tick
-    // fires. Both passes would then see a job the first has not yet reached as
-    // overdue — the miss is recorded before the await, but only for jobs already
-    // processed — and spawn it twice. The idempotency of the `missed` record
-    // guards across passes, not within one that is mid-flight.
-    // Function declaration (hoisted) so bootScheduler() can schedule it. Its
-    // guard (`catchingUp`) is declared beside `scheduler` above for TDZ safety.
-    async function catchupPass() {
-        if (catchingUp)
-            return;
-        catchingUp = true;
+    // A pass awaits executeJobDetached per job and an off-box (host/cloud)
+    // dispatch can block for a while. Overlap is now guarded by the supervisor's
+    // per-service inFlight guard (CatchupService) — a slow pass never overlaps the
+    // next supervised tick — rather than a local `catchingUp` flag; and the
+    // idempotency of the `missed` record still guards across passes and daemon
+    // restarts. `signal` aborts at the CatchupService deadline, so a wedged pass is
+    // abandoned + restarted instead of latching (PHNX-3608). Function declaration
+    // (hoisted) so the CatchupService registration above can reference it.
+    async function catchupPass(signal) {
         try {
             const overdue = detectOverdueJobs();
             if (overdue.length === 0)
@@ -1100,6 +1274,8 @@ export async function runDaemon() {
                 const last = job.lastRanAt ? job.lastRanAt.toISOString() : 'never';
                 log('WARN', `  ${job.name} -- expected ${job.expectedAt.toISOString()}, last ran ${last}`);
             }
+            if (signal?.aborted)
+                return;
             notifyOverdue(overdue);
             const outcomes = await runCatchup({ overdue });
             for (const o of outcomes) {
@@ -1130,13 +1306,12 @@ export async function runDaemon() {
             }
         }
         catch (err) {
+            // Ordinary pass errors are logged, not re-thrown: a transient catchup
+            // failure should not trip the circuit breaker. A HANG is still caught — the
+            // CatchupService deadline aborts the tick and the supervisor parks +
+            // restarts it regardless of this swallow (PHNX-3608).
             log('ERROR', `Catchup pass failed: ${err.message}`);
         }
-        finally {
-            // finally, not a tail assignment: the no-overdue path returns early, and a
-            // throw must not leave the guard latched shut for the daemon's lifetime.
-            catchingUp = false;
-        }
     }
     // Browser orphan reap and IPC server start are now managed by BrowserIPCService
     // on the supervisor (RUSH-3193 P2). The orphan reap runs inside onStart().
@@ -1181,22 +1356,22 @@ export async function runDaemon() {
             const was = servicesConfig.services[id] !== false;
             const now = reloadedEnabled(id);
             if (was !== now) {
-                // scheduler and monitors' off-transition are re-evaluated live later in
-                // this handler, so don't tell the user they need a restart for those.
-                if (id === 'scheduler' || (id === 'monitors' && !now)) {
-                    continue;
-                }
-                if (id === 'monitors') {
-                    // monitors' on-transition still needs a restart — starting the
-                    // engine live is not wired up (see the liveMonitorEngine block below).
-                    log('INFO', `Service '${id}' toggled on — restart daemon to apply`);
+                // The scheduler is not a supervised service — its gate is re-evaluated
+                // live later in this handler, so don't tell the user to restart for it.
+                if (id === 'scheduler') {
                     continue;
                 }
-                // RUSH-3193 P4: a service the supervisor already owns (it was enabled
-                // at daemon boot, so it was registered) takes the toggle live via
-                // supervisor.start/stop — no restart needed. One disabled at boot was
-                // never registered, so it falls through to the same "restart to
-                // apply" advice as before.
+                // RUSH-3193 P4: a service the supervisor already owns takes the toggle
+                // live via supervisor.start/stop — no restart needed. monitors is a
+                // supervised service (PHNX-3608): its enable/disable takes effect through
+                // that generic supervisor.start/stop path, so a disabled monitors service
+                // actually stops dispatching (its supervised tick is torn down) instead of
+                // being ticked with the last-loaded set. browser-ipc is deliberately
+                // registered in a stopped state too (PHNX-3605), so a later browser client
+                // can enable that service without restarting the shared daemon. Most other
+                // services are only registered when enabled at boot; one disabled at boot
+                // was never registered, so it falls through to the same "restart to apply"
+                // advice as before.
                 if (supervisor.isRegistered(id)) {
                     // A periodic service may be inside a real tick when SIGHUP arrives.
                     // Queue the desired transition behind that exact promise: deadlines
@@ -1258,24 +1433,19 @@ export async function runDaemon() {
             const reloaded = scheduler.listScheduled();
             log('INFO', `Reloaded ${reloaded.length} jobs`);
         }
-        // Monitor engine can be stopped on reload if disabled; starting it requires
-        // a restart, so we only handle the off transition here.
+        // Refresh monitor CONFIGS when the engine is live and monitors stays enabled
+        // (the common `monitors add/edit` + SIGHUP case). The enable/disable
+        // TRANSITION itself is handled by the generic supervisor.start/stop loop
+        // above (PHNX-3608) — a disabled monitors service is supervisor.stop()'d
+        // there, which tears down its supervised tick so nothing dispatches; an
+        // off-transition leaves getEngine() null, so this reload is correctly skipped.
         const liveMonitorEngine = monitorEngineSvc.getEngine();
-        if (liveMonitorEngine) {
-            if (!reloadedEnabled('monitors')) {
-                log('WARN', 'monitors service is now disabled — stopping monitor engine; restart daemon to re-enable');
-                try {
-                    liveMonitorEngine.stop();
-                }
-                catch { /* best-effort */ }
+        if (liveMonitorEngine && reloadedEnabled('monitors')) {
+            try {
+                liveMonitorEngine.reload();
             }
-            else {
-                try {
-                    liveMonitorEngine.reload();
-                }
-                catch (err) {
-                    log('ERROR', `Monitor engine reload failed: ${err.message}`);
-                }
+            catch (err) {
+                log('ERROR', `Monitor engine reload failed: ${err.message}`);
             }
         }
     };
@@ -1301,8 +1471,9 @@ export async function runDaemon() {
         catch {
             // Already removed with the state dir, or replaced by a newer owner.
         }
-        removeDaemonPid();
-        removeHeartbeat();
+        removeDaemonPidIfOwned(process.pid);
+        if (readHeartbeat()?.pid === process.pid)
+            removeHeartbeat();
         unregisterDaemonInstance();
         process.exit(0);
     });
@@ -1325,7 +1496,20 @@ export async function runDaemon() {
     else {
         log('INFO', 'State-dir self-check disabled');
     }
-    process.on('SIGHUP', handleReload);
+    liveReloadHandler = guardSignalHandler(handleReload, (err) => {
+        // Signal callbacks sit outside the supervisor's service barriers. A
+        // reload failure must be observable without reaching the process-wide
+        // uncaughtException handler and taking down every daemon service.
+        try {
+            log('ERROR', `SIGHUP reload failed: ${err.message}`);
+        }
+        catch { /* logging must not crash the daemon */ }
+    });
+    if (reloadRequestedDuringStartup) {
+        reloadRequestedDuringStartup = false;
+        log('INFO', 'Applying service reload requested while the daemon was starting');
+        liveReloadHandler();
+    }
     process.on('SIGTERM', () => handleShutdown());
     process.on('SIGINT', () => handleShutdown());
     await new Promise(() => { });
@@ -1903,7 +2087,8 @@ export function stopResidueArtifacts(stoppedPid, survivors = []) {
     artifacts.push({
         label: 'daemon lifetime marker',
         present: fs.existsSync(lifetimePath),
-        ownedByLiveOther: lifetimeOwner !== null && lifetimeOwner !== stoppedPid && isAlive(lifetimeOwner),
+        ownedByLiveOther: lifetimeOwner !== null && (survivors.includes(lifetimeOwner)
+            || (lifetimeOwner !== stoppedPid && isAlive(lifetimeOwner))),
         reclaim: () => { try {
             fs.unlinkSync(lifetimePath);
         }
@@ -1918,7 +2103,8 @@ export function stopResidueArtifacts(stoppedPid, survivors = []) {
         // A stale heartbeat is not cosmetic: resolveLiveDaemonPid() trusts a FRESH
         // one to re-adopt a daemon whose pid file was lost, so leaving one behind
         // can make a dead daemon read as running.
-        ownedByLiveOther: hb !== null && hb.pid !== stoppedPid && isAlive(hb.pid),
+        ownedByLiveOther: hb !== null && (survivors.includes(hb.pid)
+            || (hb.pid !== stoppedPid && isAlive(hb.pid))),
         reclaim: () => removeHeartbeat(),
         stillPresent: () => fs.existsSync(heartbeatPath),
     });
@@ -1964,31 +2150,35 @@ export function stopResidueArtifacts(stoppedPid, survivors = []) {
  * and the stop postcondition (`stopDaemon`) already use, so the display and the
  * reaper agree on what a duplicate is.
  */
-export function findSurvivingStateDirDaemons(exclude) {
+function findStateDirDaemonProcesses(exclude) {
+    const live = [];
+    const unverified = [];
     if (process.platform === 'win32')
-        return [];
+        return { live, unverified };
     const dir = getDaemonInstancesDir();
     let entries;
     try {
         entries = fs.readdirSync(dir);
     }
     catch {
-        return [];
+        return { live, unverified };
     }
-    const found = [];
     for (const name of entries) {
         const pid = parseInt(name, 10);
         if (isNaN(pid) || String(pid) !== name)
             continue; // not a pid marker
         if (exclude.has(pid))
             continue;
-        if (!isAlive(pid))
-            continue; // dead marker — reaper self-heals it
-        if (!isDaemonRunProcess(pid))
-            continue; // pid reused by an unrelated process
-        found.push(pid);
+        const identity = daemonProcessIdentity(pid);
+        if (identity === 'daemon')
+            live.push(pid);
+        else if (identity === 'unknown')
+            unverified.push(pid);
     }
-    return found;
+    return { live, unverified };
+}
+export function findSurvivingStateDirDaemons(exclude) {
+    return findStateDirDaemonProcesses(exclude).live;
 }
 /**
  * Stop the daemon and ASSERT its postcondition (SING-12, RUSH-2355), unloading it
@@ -2005,11 +2195,61 @@ export function findSurvivingStateDirDaemons(exclude) {
  * never reports success on an unverified stop.
  */
 export function stopDaemon() {
+    const releaseLock = acquireStopLock();
+    if (!releaseLock) {
+        return {
+            ok: false,
+            stoppedPid: null,
+            escalated: false,
+            released: [],
+            surviving: [`daemon lifecycle lock remained held for ${STOP_LOCK_WAIT_MS}ms`],
+            detachedChildren: listLiveRoutineChildren(),
+        };
+    }
+    try {
+        return stopDaemonLocked();
+    }
+    finally {
+        releaseLock();
+    }
+}
+function readPathIdentity(filePath) {
+    try {
+        const stat = fs.lstatSync(filePath);
+        return { dev: stat.dev, ino: stat.ino };
+    }
+    catch {
+        return null;
+    }
+}
+function pathIdentityMatches(filePath, expected) {
+    const current = readPathIdentity(filePath);
+    return current !== null && current.dev === expected.dev && current.ino === expected.ino;
+}
+/** daemon.lock is held for this entire read-signal-verify-cleanup transaction. */
+function stopDaemonLocked() {
     const platform = os.platform();
     const released = [];
     const surviving = [];
     let escalated = false;
     const reg = serviceManagerRegistrationAllowed();
+    const unverifiedPid = unverifiedLiveDaemonPid();
+    if (unverifiedPid !== null) {
+        return {
+            ok: false,
+            stoppedPid: null,
+            escalated: false,
+            released,
+            surviving: [`daemon pid ${unverifiedPid} is live but its __daemon-run identity could not be verified`],
+            detachedChildren: listLiveRoutineChildren(),
+        };
+    }
+    // Capture the target and its path-bound resources while lifecycle writers are
+    // excluded. resolveLiveDaemonPid(true) rejects a reused/non-daemon pid before
+    // any service-manager teardown or direct signal and repairs only under lock.
+    const pid = resolveLiveDaemonPid(true);
+    const browserSock = process.platform === 'win32' ? null : getBrowserIpcSocketPath();
+    const browserSockOwner = pid !== null && browserSock ? readPathIdentity(browserSock) : null;
     if (platform === 'darwin') {
         const plistPath = getLaunchdPlistPath();
         if (fs.existsSync(plistPath)) {
@@ -2055,65 +2295,112 @@ export function stopDaemon() {
             catch { /* unit file already removed */ }
         }
     }
-    const pid = readDaemonPid();
     if (pid) {
         if (process.platform === 'win32') {
             // Windows has no graceful termination signal — terminate the daemon and
             // its job/browser child tree in one shot (taskkill /T), so stop doesn't
             // report success while children keep running.
-            killTree(pid);
-            escalated = true;
-        }
-        else {
-            try {
-                process.kill(pid, 'SIGTERM');
-            }
-            catch { /* process already exited */ }
-            // Wait for it to actually go. This used to be a setTimeout escalation plus
-            // an immediate removeDaemonPid(), which had two failure modes: in a
-            // short-lived process (the npm postinstall) the timer never fired at all,
-            // and clearing the pid file while the old daemon still ran made
-            // isDaemonRunning() report false, so startDaemon() launched a SECOND
-            // daemon. Its hosted broker then unlinked the live socket and rebound,
-            // orphaning the first broker with every unlocked bundle still in its RAM
-            // and unreachable — two brokers on one socket path, seen on a real machine
-            // after an install into a second prefix.
-            if (!waitForExit(pid, STOP_GRACE_MS)) {
+            if (isLiveDaemon(pid)) {
                 killTree(pid);
                 escalated = true;
                 waitForExit(pid, STOP_KILL_GRACE_MS);
             }
         }
+        else {
+            // Revalidate immediately before the signal: the pid may have exited and
+            // been reused since the ownership snapshot above.
+            if (isLiveDaemon(pid)) {
+                try {
+                    process.kill(pid, 'SIGTERM');
+                }
+                catch { /* process already exited */ }
+                // Wait for it to actually go. This used to be a setTimeout escalation plus
+                // an immediate removeDaemonPid(), which had two failure modes: in a
+                // short-lived process (the npm postinstall) the timer never fired at all,
+                // and clearing the pid file while the old daemon still ran made
+                // isDaemonRunning() report false, so startDaemon() launched a SECOND
+                // daemon. Its hosted broker then unlinked the live socket and rebound,
+                // orphaning the first broker with every unlocked bundle still in its RAM
+                // and unreachable — two brokers on one socket path, seen on a real machine
+                // after an install into a second prefix.
+                if (!waitForExit(pid, STOP_GRACE_MS) && isLiveDaemon(pid)) {
+                    killTree(pid);
+                    escalated = true;
+                    waitForExit(pid, STOP_KILL_GRACE_MS);
+                }
+            }
+        }
     }
-    removeDaemonPid();
     // ── Assert the postcondition (SING-12) ────────────────────────────────────
     // No `__daemon-run` for this state dir may survive the stop.
-    const survivors = findSurvivingStateDirDaemons(new Set([process.pid]));
-    if (pid && process.platform === 'win32' && isAlive(pid) && !survivors.includes(pid)) {
-        survivors.push(pid); // registry is POSIX-only; check the killed pid directly
+    const stateDirProcesses = findStateDirDaemonProcesses(new Set([process.pid]));
+    const survivors = stateDirProcesses.live;
+    const unverifiedSurvivors = stateDirProcesses.unverified;
+    // The registry is best-effort and may be missing, so always re-check the
+    // direct target as well. This is the only check available on Windows.
+    if (pid && !survivors.includes(pid) && !unverifiedSurvivors.includes(pid)) {
+        const identity = daemonProcessIdentity(pid);
+        if (identity === 'daemon')
+            survivors.push(pid);
+        else if (identity === 'unknown')
+            unverifiedSurvivors.push(pid);
     }
     if (survivors.length > 0) {
         for (const s of survivors)
             surviving.push(`__daemon-run pid ${s} still alive`);
     }
-    else if (pid) {
+    for (const s of unverifiedSurvivors) {
+        surviving.push(`daemon pid ${s} is live but its __daemon-run identity could not be verified`);
+    }
+    const targetStillOwnsResources = pid !== null
+        && (survivors.includes(pid) || unverifiedSurvivors.includes(pid));
+    if (pid && !targetStillOwnsResources) {
         released.push('daemon process');
     }
+    // Delete the registration only after the target is provably gone, and only
+    // if the file still names that target. A replacement value belongs to a
+    // successor (or another writer) and is both preserved and reported.
+    const currentPid = readDaemonPid();
+    if (pid !== null && !targetStillOwnsResources) {
+        if (removeDaemonPidIfOwned(pid))
+            released.push('daemon pid registration');
+        else if (currentPid === null)
+            released.push('daemon pid registration');
+        else
+            surviving.push(`daemon pid registration changed to ${currentPid} during stop`);
+    }
+    else if (pid === null && currentPid === null) {
+        released.push('daemon pid registration');
+    }
+    else if (pid === null && currentPid !== null) {
+        surviving.push(`daemon pid registration changed to ${currentPid} during stop`);
+    }
     // Browser IPC binding: on POSIX the listening socket is a filesystem object.
     // A graceful handleShutdown unlinks it; if it survives, the daemon exited
     // ungracefully (killTree) and left a stale binding — the owner is provably
     // dead, so reclaim it and report.
     if (process.platform !== 'win32') {
-        const browserSock = getBrowserIpcSocketPath();
-        if (fs.existsSync(browserSock)) {
-            try {
-                fs.unlinkSync(browserSock);
+        if (browserSock && fs.existsSync(browserSock)) {
+            if (browserSockOwner && pathIdentityMatches(browserSock, browserSockOwner) && !targetStillOwnsResources) {
+                try {
+                    fs.unlinkSync(browserSock);
+                }
+                catch { /* failed to reclaim the captured binding */ }
+                if (pathIdentityMatches(browserSock, browserSockOwner))
+                    surviving.push('browser IPC socket not released');
+                else if (fs.existsSync(browserSock))
+                    surviving.push('browser IPC socket ownership changed during stop');
+                else
+                    released.push('browser IPC socket (reclaimed)');
+            }
+            else {
+                // The path was absent when the target was captured, there was no proven
+                // daemon target, or a successor replaced the inode. In every case this
+                // invocation does not own the current binding and must leave it alone.
+                surviving.push(targetStillOwnsResources
+                    ? 'browser IPC socket still owned by a surviving daemon'
+                    : 'browser IPC socket ownership could not be verified');
             }
-            catch { /* raced with a fresh start */ }
-            if (fs.existsSync(browserSock))
-                surviving.push('browser IPC socket not released');
-            else
-                released.push('browser IPC socket (reclaimed)');
         }
         else {
             released.push('browser IPC socket');
@@ -2149,7 +2436,7 @@ export function stopDaemon() {
     // consults to re-adopt a "live" daemon, and a leftover registry entry is what
     // `reapStrayDaemons` enumerates. Same shape as the sockets above — reclaim
     // what a provably dead owner left, never touch what a live one owns.
-    for (const artifact of stopResidueArtifacts(pid, survivors)) {
+    for (const artifact of stopResidueArtifacts(pid, [...survivors, ...unverifiedSurvivors])) {
         if (!artifact.present) {
             released.push(artifact.label);
             continue;