c8ctl-plugin-nano 1.72.0 → 1.73.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +76 -12
  2. package/c8ctl-plugin.js +1460 -36
  3. package/package.json +8 -8
package/c8ctl-plugin.js CHANGED
@@ -49,6 +49,7 @@ import {
49
49
  utimesSync,
50
50
  mkdtempSync,
51
51
  closeSync,
52
+ readlinkSync,
52
53
  watchFile,
53
54
  unwatchFile,
54
55
  } from 'node:fs';
@@ -295,6 +296,40 @@ const SUPERVISOR_RELOAD_READY_TIMEOUT_MS = Math.max(
295
296
  0,
296
297
  Number.parseInt(process.env.NANO_SUPERVISOR_RELOAD_READY_TIMEOUT_MS ?? '', 10) || 30_000,
297
298
  );
299
+ // Rolling daemon hot-upgrade (issue #309). A `supervisor upgrade` hands the
300
+ // fleet to a SUCCESSOR daemon process running the on-disk (new) plugin code:
301
+ // the old daemon quiesces (stops accepting control connections + stops
302
+ // restarting workers), asks each worker to re-arm its parent-death watchdog at
303
+ // the successor's pid (SIGUSR2 + NANO_SUPERVISOR_UPGRADE_PID — a retarget, NOT
304
+ // a drain), persists a final state snapshot, and exits WITHOUT touching the
305
+ // workers. The successor binds the canonical control socket and adopts the
306
+ // still-running worker pids. Workers never die mid-job; only the parent swaps.
307
+ //
308
+ // How long the old daemon waits for workers to acknowledge the watchdog
309
+ // retarget (by stamping `watchdogPid` on their activity marker) before it exits
310
+ // anyway. Bounded so a wedged/older worker can't stall the upgrade forever —
311
+ // the successor adopts every live pid regardless, so an un-acked worker is
312
+ // still parented (its watchdog self-reaps only if the successor ALSO dies).
313
+ const SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS = Math.max(
314
+ 0,
315
+ Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS ?? '', 10) || 15_000,
316
+ );
317
+ // How long a freshly spawned successor waits for the old daemon to quiesce
318
+ // before giving up (it must never bind the socket while the old daemon still
319
+ // accepts on it). Generous: quiesce is prompt (server.close + a latch), but a
320
+ // loaded machine can delay the old daemon's socket loop.
321
+ const SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS = Math.max(
322
+ 0,
323
+ Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS ?? '', 10) || 30_000,
324
+ );
325
+ // How often the successor polls an adopted worker's pid for liveness. An
326
+ // adopted worker has no ChildProcess handle (the successor never spawned it),
327
+ // so there is no 'exit' event — this poll is the death detector that feeds the
328
+ // same restart-backoff path a spawned child's exit would.
329
+ const SUPERVISOR_ADOPT_POLL_MS = Math.max(
330
+ 100,
331
+ Number.parseInt(process.env.NANO_SUPERVISOR_ADOPT_POLL_MS ?? '', 10) || 2_000,
332
+ );
298
333
  // Upper bound on one `--auto` engine-read reconcile (enumerate deployed
299
334
  // definitions + fetch each BPMN). A read that stalls past this is treated as a
300
335
  // transient failure so the running poller set is KEPT and, crucially, shutdown
@@ -474,6 +509,19 @@ function activityMarkerReadyFor(act, pid) {
474
509
  return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
475
510
  }
476
511
 
512
+ /**
513
+ * Whether a worker's SIGUSR2 (drain/retarget) handler is provably installed
514
+ * (#309 review): its marker belongs to `pid` AND carries a finite
515
+ * `signalReadyAt` (stamped right after the handler is armed, independent of
516
+ * leasing). Falls back to `readyAt` for a worker that predates the field (its
517
+ * handler is armed before its first activation). Signalling earlier would hit
518
+ * Node's DEFAULT SIGUSR2 action and terminate the worker. Pure for unit tests.
519
+ */
520
+ function drainSignalReadyFor(act, pid) {
521
+ if (!act || act.pid !== pid) return false;
522
+ return Number.isFinite(act.signalReadyAt) || Number.isFinite(act.readyAt);
523
+ }
524
+
477
525
  /**
478
526
  * Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
479
527
  * All three must hold (#293): the worker exists, its PID is alive, AND its
@@ -2658,6 +2706,170 @@ function sampleSubtreeCpu(rootPid) {
2658
2706
  return sampleSubtreeCpuPs(rootPid);
2659
2707
  }
2660
2708
 
2709
+ // ---- Workspace-CPU liveness + in-flight tool holds (#313) ------------------
2710
+ // The #130 subtree probe only sees processes INSIDE the agent's own process
2711
+ // tree. Real work often happens OUTSIDE that tree: `cargo` builds through a
2712
+ // shared `sccache` server (a daemon parented to launchd/init, not the agent),
2713
+ // and a long foreground bash tool call can compile/test silently for minutes
2714
+ // (nanobpm/nano-bpm#1664 attempt 1 was idle-killed at 216 turns / 156 min with
2715
+ // a build genuinely running). Two extra liveness signals close that gap:
2716
+ // 1. `sampleWorkspaceCpu` — aggregate CPU of ANY process whose cwd is inside
2717
+ // the agent's workspace (the sccache/rustc case: the daemon's compile
2718
+ // workers chdir into the worktree they compile for).
2719
+ // 2. an in-flight-tool-call hold — while the agent has an ACP tool call still
2720
+ // open (a bash call it started has not reported a terminal status) it is
2721
+ // treated as active; the call's own timeout / the absolute hard cap still
2722
+ // bound a genuinely hung call.
2723
+ // Both ride the monitor below as extra samplers consulted only during an idle
2724
+ // window, so a truly quiescent agent still ages out exactly as before.
2725
+
2726
+ /** Default upper bound for the workspace-cwd scan (pid count). */
2727
+ const WORKSPACE_CPU_SCAN_LIMIT = 4096;
2728
+
2729
+ /**
2730
+ * Canonicalize a workspace root for cwd prefix checks. The kernel reports a
2731
+ * process cwd as the PHYSICAL path (`/proc/<pid>/cwd`, lsof), so a root reached
2732
+ * through a symlink (e.g. `C8CTL_NANO_HOME` under a symlinked dir, or macOS
2733
+ * `/tmp` → `/private/tmp`) must be realpath'd or every in-workspace process
2734
+ * fails the prefix check. Returns the list of distinct roots to accept (the
2735
+ * physical one plus the lexical one), or `[]` when the input is unusable.
2736
+ */
2737
+ function workspaceRoots(workspaceDir, realpath = realpathSync) {
2738
+ if (typeof workspaceDir !== 'string' || !workspaceDir) return [];
2739
+ const lexical = resolvePath(workspaceDir);
2740
+ let physical = lexical;
2741
+ try { physical = realpath(lexical); } catch { /* missing dir → lexical only */ }
2742
+ return physical === lexical ? [lexical] : [physical, lexical];
2743
+ }
2744
+
2745
+ /** True when `cwd` is one of `roots` or lies beneath one. */
2746
+ function cwdInRoots(cwd, roots) {
2747
+ if (typeof cwd !== 'string' || !cwd) return false;
2748
+ return roots.some((root) => cwd === root || cwd.startsWith(root.endsWith(sep) ? root : root + sep));
2749
+ }
2750
+
2751
+ /**
2752
+ * Linux: aggregate CPU (utime+stime ticks) of every process whose cwd resolves
2753
+ * inside `workspaceDir` — regardless of parentage — via `/proc/<pid>/cwd`
2754
+ * symlinks. Returns `{ processes, cpu }` (cpu in clock ticks, matching the
2755
+ * `/proc` subtree sampler), or `null` when the workspace is unusable (so the
2756
+ * caller degrades to subtree-only liveness rather than mis-killing). A process
2757
+ * counted here is NOT a descendant of the agent, so it is a pure CPU-progress
2758
+ * signal, never a presence signal. Bounded by `scanLimit` pids so a wedged
2759
+ * /proc read can't stall the watchdog. `readCwd`/`listPids` are injectable for
2760
+ * tests.
2761
+ */
2762
+ function sampleWorkspaceCpuProc(workspaceDir, { readCwd, listPids, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
2763
+ const roots = workspaceRoots(workspaceDir, realpath);
2764
+ if (roots.length === 0) return null;
2765
+ const read = typeof readCwd === 'function' ? readCwd : (pid) => {
2766
+ try { return readlinkSync(`/proc/${pid}/cwd`); } catch { return null; }
2767
+ };
2768
+ let entries;
2769
+ if (typeof listPids === 'function') entries = listPids();
2770
+ else {
2771
+ try { entries = readdirSync('/proc'); } catch { return null; }
2772
+ }
2773
+ let processes = 0;
2774
+ let totalCpu = 0;
2775
+ let scanned = 0;
2776
+ for (const name of entries) {
2777
+ if (scanned >= scanLimit) break;
2778
+ if (!/^\d+$/.test(name)) continue;
2779
+ scanned += 1;
2780
+ const pid = Number(name);
2781
+ if (pid === process.pid || excludePids.includes(pid)) continue;
2782
+ const cwd = read(pid);
2783
+ if (typeof cwd !== 'string' || !cwd) continue;
2784
+ if (!cwdInRoots(cwd, roots)) continue;
2785
+ let stat;
2786
+ try { stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { continue; }
2787
+ const rparen = stat.lastIndexOf(')');
2788
+ if (rparen < 0) continue;
2789
+ const tail = stat.slice(rparen + 1).trim().split(/\s+/);
2790
+ const utime = Number(tail[11]);
2791
+ const stime = Number(tail[12]);
2792
+ totalCpu += (Number.isFinite(utime) ? utime : 0) + (Number.isFinite(stime) ? stime : 0);
2793
+ processes += 1;
2794
+ }
2795
+ return { processes, cpu: totalCpu };
2796
+ }
2797
+
2798
+ /**
2799
+ * Non-Linux fallback (macOS): aggregate CPU of processes whose cwd is inside
2800
+ * `workspaceDir`, via a bounded `lsof` + `ps`. Heavier (two short-lived
2801
+ * spawns) but runs only during an idle window. Returns
2802
+ * `{ processes, cpu, resolutionMs: 1000 }` (cpu in whole seconds, so the
2803
+ * coarse-resolution flat-CPU rule applies) or `null` when unavailable.
2804
+ */
2805
+ function sampleWorkspaceCpuLsof(workspaceDir, { runLsof, runPs, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
2806
+ const roots = workspaceRoots(workspaceDir, realpath);
2807
+ if (roots.length === 0) return null;
2808
+ const lsof = typeof runLsof === 'function' ? runLsof : () => {
2809
+ // `lsof -d cwd -Fpn` lists the cwd of EVERY process as machine-readable
2810
+ // `p<pid>` / `n<path>` records. A path filter (`-- <root>`) would only match
2811
+ // processes whose cwd is EXACTLY the root, missing a compiler working in a
2812
+ // nested crate dir, so we collect all cwd records and apply the
2813
+ // root-or-descendant check ourselves. Bounded by timeout + maxBuffer; lsof
2814
+ // exits 1 on partial permission errors, which still yields usable stdout.
2815
+ const r = spawnSync('lsof', ['-n', '-P', '-w', '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
2816
+ if (r.error && r.error.code === 'ENOENT') return null; // lsof missing → degrade
2817
+ if (r.error && !r.stdout) return null; // timeout / overflow with nothing usable
2818
+ return typeof r.stdout === 'string' ? r.stdout : '';
2819
+ };
2820
+ let lsofOut;
2821
+ try { lsofOut = lsof(); } catch { return null; }
2822
+ if (typeof lsofOut !== 'string') return null;
2823
+ const pids = [];
2824
+ let curPid = null;
2825
+ for (const line of lsofOut.split('\n')) {
2826
+ const tag = line.charCodeAt(0);
2827
+ if (tag === 112 /* 'p' */) {
2828
+ const pid = Number(line.slice(1));
2829
+ curPid = Number.isFinite(pid) && pid > 0 && pid !== process.pid && !excludePids.includes(pid) ? pid : null;
2830
+ } else if (tag === 110 /* 'n' */ && curPid != null) {
2831
+ if (cwdInRoots(line.slice(1), roots) && !pids.includes(curPid)) pids.push(curPid);
2832
+ curPid = null;
2833
+ if (pids.length >= scanLimit) break;
2834
+ }
2835
+ }
2836
+ if (pids.length === 0) return { processes: 0, cpu: 0, resolutionMs: 1000 };
2837
+ const ps = typeof runPs === 'function' ? runPs : (list) => {
2838
+ const r = spawnSync('ps', ['-p', list.join(','), '-o', 'pid=,time='], { encoding: 'utf8', timeout: 5_000 });
2839
+ // ps exits 1 when some pids vanished between scans; partial stdout is valid.
2840
+ return typeof r.stdout === 'string' ? r.stdout : null;
2841
+ };
2842
+ let psOut;
2843
+ try { psOut = ps(pids); } catch { return null; }
2844
+ if (typeof psOut !== 'string') return null;
2845
+ let processes = 0;
2846
+ let totalCpu = 0;
2847
+ for (const line of psOut.split('\n')) {
2848
+ const m = line.trim().match(/^(\d+)\s+(\S+)$/);
2849
+ if (!m) continue;
2850
+ totalCpu += parsePsTime(m[2]);
2851
+ processes += 1;
2852
+ }
2853
+ return { processes, cpu: totalCpu, resolutionMs: 1000 };
2854
+ }
2855
+
2856
+ /**
2857
+ * Sample aggregate CPU of processes working inside `workspaceDir` regardless of
2858
+ * parentage (the sccache/shared-daemon case, #313). Linux reads `/proc`; macOS
2859
+ * falls back to `lsof`+`ps`; other platforms return `null` (unsupported → the
2860
+ * monitor degrades to subtree-only liveness). The result shape mirrors the
2861
+ * subtree sampler (`cpu` in ticks on Linux, whole seconds under `ps`) with
2862
+ * `processes` instead of `descendants`.
2863
+ */
2864
+ function sampleWorkspaceCpu(workspaceDir, { excludePids } = {}) {
2865
+ // The agent itself runs with cwd == workspace; its own CPU (an idle harness
2866
+ // ticking timers) is NOT out-of-tree work and must never defer its idle-kill.
2867
+ const exclude = Array.isArray(excludePids) ? excludePids.filter((p) => Number.isFinite(p) && p > 0) : [];
2868
+ if (process.platform === 'linux') return sampleWorkspaceCpuProc(workspaceDir, { excludePids: exclude });
2869
+ if (process.platform === 'darwin') return sampleWorkspaceCpuLsof(workspaceDir, { excludePids: exclude });
2870
+ return null; // win32 et al: unsupported → degrade, never mis-kill
2871
+ }
2872
+
2661
2873
  /**
2662
2874
  * Build a progress-aware idle-liveness monitor shared by the pipe/PTY/ACP
2663
2875
  * capture paths. `arm()` is called on every output chunk (and once at start) to
@@ -2683,37 +2895,155 @@ function sampleSubtreeCpu(rootPid) {
2683
2895
  * read under a coarse sampler must NOT kill; the fine-grained `/proc` path
2684
2896
  * carries no resolution and so ages out on the first flat window as before →
2685
2897
  * `onIdleKill()`.
2898
+ * #313 extensions:
2899
+ * - `extraSamplers`: additional activity probes (e.g. the workspace-cwd CPU
2900
+ * scan covering out-of-tree sccache/rustc daemons). Each returns
2901
+ * `{ processes, cpu, resolutionMs? } | null`; advancing CPU on ANY sampler
2902
+ * defers the kill. Extra samplers are pure progress signals — they never
2903
+ * satisfy the presence gate (a childless agent with a busy workspace daemon
2904
+ * still needs its own live descendant to count as present... EXCEPT that a
2905
+ * workspace daemon burning CPU IS evidence the agent's work is in flight, so
2906
+ * when an extra sampler shows advancing CPU the kill is deferred even with
2907
+ * no live descendant: the agent may be blocked reading a pipe its daemon
2908
+ * fills. A flat/zero extra sample never kills on its own — presence is
2909
+ * judged from the subtree as before).
2910
+ * - `isActivityHold`: an in-flight-tool-call latch (ACP). While it returns
2911
+ * true the agent has an open tool call, so silence is expected; the kill is
2912
+ * deferred window-by-window. The absolute `--job-timeout` hard cap (and the
2913
+ * tool call's own timeout, enforced by the agent) remain the bound for a
2914
+ * genuinely hung call.
2915
+ * - `logger`: when given, the monitor logs WHY it considered the agent active
2916
+ * or idle at each probe (diagnosability, #313).
2686
2917
  * The first elapse with live descendants takes a baseline (a single sample can't
2687
2918
  * show movement) and re-probes after the recovery window. The absolute
2688
2919
  * `--job-timeout` hard cap remains the ultimate backstop, untouched by this.
2689
2920
  */
2690
- function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu }) {
2921
+ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu, extraSamplers = null, isActivityHold = null, maxActivityHoldMs, logger = null }) {
2691
2922
  const enabled = idleTimeoutMs && idleTimeoutMs > 0;
2692
2923
  const probeWindow = recoveryWindowMs && recoveryWindowMs > 0 ? recoveryWindowMs : idleTimeoutMs;
2693
2924
  let timer = null;
2694
2925
  let prev = null; // last subtree CPU probe taken during the current silent stretch
2926
+ let prevExtra = null; // last per-extra-sampler CPU readings (array parallel to extraSamplers)
2927
+ let extraBaselined = false; // true once the extra samplers have a baseline to compare against
2928
+ let extraResolutionMs = 0; // coarsest resolution among live extra samples at the last probe
2695
2929
  let flatMs = 0; // cumulative time observed with flat CPU since the last movement/baseline
2696
2930
  let lastWindow = idleTimeoutMs; // duration of the currently pending probe window
2931
+ let extraFlatMs = 0; // cumulative flat time across the extra (workspace) samplers
2932
+ let holdMs = 0; // cumulative silent time deferred by the activity hold this stretch
2933
+ // #313: the activity hold must be FINITE — `nano work` defaults to no hard cap,
2934
+ // so an announced-then-hung tool call would otherwise defer forever. Default:
2935
+ // 4 idle windows (configurable). Past it, the hold is ignored and the normal
2936
+ // CPU probes decide (real silent work still defers via CPU progress).
2937
+ const holdCap = Number.isFinite(maxActivityHoldMs) && maxActivityHoldMs >= 0 ? maxActivityHoldMs : (idleTimeoutMs > 0 ? idleTimeoutMs * 4 : 0);
2697
2938
  const settled = () => (typeof isSettled === 'function' ? isSettled() : false);
2698
2939
  const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
2699
2940
  const schedule = (ms) => { lastWindow = ms; timer = setTimeout(onElapse, ms); };
2941
+ const log = (msg) => { try { logger?.info?.(`[idle-watchdog] ${msg}`); } catch { /* best effort */ } };
2942
+ const extras = Array.isArray(extraSamplers) ? extraSamplers.filter((s) => typeof s === 'function') : [];
2943
+ const held = () => { try { return typeof isActivityHold === 'function' && isActivityHold(); } catch { return false; } };
2700
2944
  function onElapse() {
2701
2945
  timer = null;
2702
2946
  if (settled()) return;
2703
2947
  const elapsed = lastWindow;
2948
+ // #313: an in-flight tool call (the agent started a bash call that has not
2949
+ // reported a terminal status) means the silence is expected — defer. The
2950
+ // call's own timeout / the absolute hard cap still bound a hung call.
2951
+ if (held()) {
2952
+ if (holdMs < holdCap) {
2953
+ holdMs += elapsed;
2954
+ log(`silent for >= ${elapsed}ms but a tool call is still in flight; deferring the idle-kill (hold ${holdMs}/${holdCap}ms)`);
2955
+ prev = null; prevExtra = null; extraBaselined = false; flatMs = 0; extraFlatMs = 0;
2956
+ schedule(probeWindow);
2957
+ return;
2958
+ }
2959
+ log(`a tool call has been in flight for >= ${holdMs}ms (hold cap ${holdCap}ms); falling back to CPU-progress probes`);
2960
+ }
2704
2961
  const pid = typeof getPid === 'function' ? getPid() : undefined;
2705
2962
  const sample = pid != null ? sampleCpu(pid) : null;
2706
- // Childless / unsampleable / dead tree → silence really is a hang: kill.
2707
- if (!sample || sample.descendants <= 0) { onIdleKill(); return; }
2963
+ // #313: consult the extra (workspace-cwd) samplers. Advancing CPU on any of
2964
+ // them is work attributable to this agent even when it lives outside the
2965
+ // agent's process tree (a shared sccache daemon), so it defers the kill —
2966
+ // INCLUDING when the subtree itself looks childless/quiescent.
2967
+ let extraActive = false;
2968
+ if (extras.length) {
2969
+ const next = [];
2970
+ extraResolutionMs = 0;
2971
+ for (let i = 0; i < extras.length; i += 1) {
2972
+ let s = null;
2973
+ try { s = extras[i](); } catch { s = null; }
2974
+ next.push(s);
2975
+ if (!s || !Number.isFinite(s.cpu)) continue;
2976
+ if (s.processes > 0 && Number.isFinite(s.resolutionMs)) extraResolutionMs = Math.max(extraResolutionMs, s.resolutionMs);
2977
+ const before = prevExtra && prevExtra[i];
2978
+ if (before != null && Number.isFinite(before.cpu) && s.cpu !== before.cpu) extraActive = true;
2979
+ }
2980
+ const hadBaseline = prevExtra != null;
2981
+ const prevReadings = prevExtra || [];
2982
+ prevExtra = next;
2983
+ if (extraActive) {
2984
+ log(`silent for >= ${elapsed}ms but a workspace process outside the agent tree is burning CPU (e.g. sccache/rustc); deferring the idle-kill`);
2985
+ prev = sample; flatMs = 0; extraFlatMs = 0;
2986
+ schedule(probeWindow);
2987
+ return;
2988
+ }
2989
+ // Flat time only counts while a workspace sampler stays LIVE (a reading
2990
+ // with processes > 0) across consecutive probes. Empty / unreadable
2991
+ // readings must not pre-pay a sampler's resolution allowance: when any
2992
+ // sampler transitions absent/unreadable → live, restart the accumulator
2993
+ // so a fresh process gets its full resolution window before a kill.
2994
+ const isLive = (r) => r != null && Number.isFinite(r.cpu) && r.processes > 0;
2995
+ let newlyLive = false; let continuedLive = false;
2996
+ for (let i = 0; i < next.length; i += 1) {
2997
+ if (!isLive(next[i])) continue;
2998
+ if (hadBaseline && isLive(prevReadings[i])) continuedLive = true; else newlyLive = true;
2999
+ }
3000
+ if (newlyLive || !continuedLive) extraFlatMs = 0;
3001
+ else extraFlatMs += elapsed;
3002
+ }
3003
+ // A live workspace process read flat, but a coarse sampler (whole-second
3004
+ // `ps`) can read flat across a sub-resolution window while still working:
3005
+ // only treat workspace CPU as idle once the flat time covers its resolution.
3006
+ const extraIdleConfirmed = extraFlatMs >= extraResolutionMs;
3007
+ // Childless / unsampleable / dead tree → silence is very likely a hang.
3008
+ // But with extra (workspace) samplers wired we cannot yet distinguish "no
3009
+ // workspace work" from "a workspace daemon whose baseline we have not
3010
+ // taken" on a single reading — so take ONE baseline probe window before
3011
+ // killing, exactly like the subtree first-elapse baseline below. Without
3012
+ // extra samplers (or once a baseline shows no workspace activity) the
3013
+ // legacy immediate kill applies.
3014
+ if (!sample || sample.descendants <= 0) {
3015
+ if (extras.length && !extraBaselined) {
3016
+ // Consume the baseline: the NEXT elapse compares against prevExtra and
3017
+ // either defers (workspace CPU moved) or kills (flat).
3018
+ extraBaselined = true;
3019
+ log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; baselining the workspace CPU probe and re-checking in ${probeWindow}ms`);
3020
+ schedule(probeWindow);
3021
+ return;
3022
+ }
3023
+ if (!extraIdleConfirmed) {
3024
+ log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; workspace CPU flat for ${extraFlatMs}ms < sampler resolution ${extraResolutionMs}ms, re-checking in ${probeWindow}ms`);
3025
+ schedule(probeWindow);
3026
+ return;
3027
+ }
3028
+ log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'} and no workspace CPU; idle-killing`);
3029
+ onIdleKill();
3030
+ return;
3031
+ }
2708
3032
  if (prev != null) {
2709
3033
  // Aggregate CPU advanced across the window → the tree is doing work; defer.
2710
- if (sample.cpu > prev.cpu) { prev = sample; flatMs = 0; schedule(probeWindow); return; }
3034
+ if (sample.cpu > prev.cpu) {
3035
+ log(`silent for >= ${elapsed}ms but subtree CPU advanced (${prev.cpu} → ${sample.cpu}); deferring the idle-kill`);
3036
+ prev = sample; flatMs = 0; schedule(probeWindow); return;
3037
+ }
2711
3038
  // Aggregate CPU REGRESSED → `sampleSubtreeCpu*` only sums CPU for live
2712
3039
  // PIDs, so the total can legitimately drop when a busy descendant exits
2713
3040
  // (its ticks stop counting) even though the subtree just made progress.
2714
3041
  // A regression is activity, not a stall: re-baseline and defer rather
2715
3042
  // than mis-killing a silent-but-busy job.
2716
- if (sample.cpu < prev.cpu) { prev = sample; flatMs = 0; schedule(probeWindow); return; }
3043
+ if (sample.cpu < prev.cpu) {
3044
+ log(`silent for >= ${elapsed}ms but subtree CPU regressed (${prev.cpu} → ${sample.cpu}; a busy descendant exited) — progress, not a stall; deferring the idle-kill`);
3045
+ prev = sample; flatMs = 0; schedule(probeWindow); return;
3046
+ }
2717
3047
  // Live descendants but CPU exactly unchanged across a full window → a
2718
3048
  // candidate stall. Only confirm it once we have observed flat CPU for at
2719
3049
  // least the sampler's CPU-time resolution: a probe window shorter than the
@@ -2721,12 +3051,17 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
2721
3051
  // low-churn subtree is still progressing but hasn't crossed the next tick.
2722
3052
  flatMs += elapsed;
2723
3053
  const resolutionMs = Number.isFinite(sample.resolutionMs) ? sample.resolutionMs : 0;
2724
- if (flatMs >= resolutionMs) { onIdleKill(); return; }
3054
+ if (flatMs >= resolutionMs && extraIdleConfirmed) {
3055
+ log(`silent for >= ${elapsed}ms with a live but CPU-flat subtree (${sample.descendants} descendant(s), cpu ${sample.cpu}) and no workspace CPU; idle-killing`);
3056
+ onIdleKill();
3057
+ return;
3058
+ }
2725
3059
  schedule(probeWindow);
2726
3060
  return;
2727
3061
  }
2728
3062
  // First elapse with live descendants: baseline, then re-probe (can't judge
2729
3063
  // progress from one sample).
3064
+ log(`silent for >= ${elapsed}ms with ${sample.descendants} live descendant(s); baselining CPU (${sample.cpu}) and re-probing in ${probeWindow}ms`);
2730
3065
  prev = sample;
2731
3066
  flatMs = 0;
2732
3067
  schedule(probeWindow);
@@ -2736,7 +3071,12 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
2736
3071
  arm() {
2737
3072
  if (!enabled || settled()) return;
2738
3073
  prev = null; // output resumed → a fresh silent stretch starts
3074
+ prevExtra = null;
3075
+ extraBaselined = false;
2739
3076
  flatMs = 0;
3077
+ extraFlatMs = 0;
3078
+ extraResolutionMs = 0;
3079
+ holdMs = 0;
2740
3080
  clear();
2741
3081
  schedule(idleTimeoutMs);
2742
3082
  },
@@ -8749,6 +9089,11 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
8749
9089
  idleTimeoutMs,
8750
9090
  recoveryWindowMs,
8751
9091
  isSettled: () => settled,
9092
+ logger: getLogger(),
9093
+ // #313: count CPU burned by any process whose cwd is inside the agent's
9094
+ // workspace (a shared sccache daemon lives outside the agent's tree), so
9095
+ // a silent compile holds off the idle-kill.
9096
+ extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
8752
9097
  onIdleKill: () => {
8753
9098
  try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
8754
9099
  finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
@@ -8921,6 +9266,11 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
8921
9266
  idleTimeoutMs,
8922
9267
  recoveryWindowMs,
8923
9268
  isSettled: () => settled,
9269
+ logger: getLogger(),
9270
+ // #313: count CPU burned by any process whose cwd is inside the agent's
9271
+ // workspace (a shared sccache daemon lives outside the agent's tree), so
9272
+ // a silent compile holds off the idle-kill.
9273
+ extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [term?.pid] })] : null,
8924
9274
  onIdleKill: () => {
8925
9275
  killTerm();
8926
9276
  finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
@@ -9295,6 +9645,28 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9295
9645
  // flips this; the floor never does.
9296
9646
  let sawRealTranscript = false;
9297
9647
 
9648
+ // #313: ACP tool calls currently in flight (`toolCallId` → true). A
9649
+ // `tool_call` notification with a non-terminal status opens the call; a
9650
+ // `tool_call`/`tool_call_update` with a terminal status closes it. While
9651
+ // any call is open the idle watchdog treats the agent as ACTIVE (a long
9652
+ // silent bash call is work, not a hang) — the call's own timeout and the
9653
+ // absolute job hard cap still bound a genuinely hung call.
9654
+ const inFlightToolCalls = new Map();
9655
+ const ACP_TOOL_TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled', 'canceled']);
9656
+ const noteToolCallUpdate = (update) => {
9657
+ if (!update || typeof update !== 'object') return;
9658
+ const kind = update.sessionUpdate || update.type;
9659
+ if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
9660
+ const id = update.toolCallId;
9661
+ if (id == null) return;
9662
+ const status = typeof update.status === 'string' ? update.status.toLowerCase() : '';
9663
+ if (ACP_TOOL_TERMINAL_STATUSES.has(status)) inFlightToolCalls.delete(id);
9664
+ else if (!inFlightToolCalls.has(id)) {
9665
+ inFlightToolCalls.set(id, true);
9666
+ try { logger.info?.(`[idle-watchdog] tool call in flight (${update.title || id}${status ? `, status ${status}` : ''}) — the idle-kill is held while it runs`); } catch { /* best effort */ }
9667
+ }
9668
+ };
9669
+
9298
9670
  // Live "spy" tee (--stream), line-buffered, mirroring the other paths.
9299
9671
  // Separate line buffers per lane (stdout-human vs stderr) so a partial line
9300
9672
  // on one lane never interleaves mid-line with the other — matching pipe/PTY.
@@ -9623,6 +9995,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9623
9995
  // A request or notification FROM the agent.
9624
9996
  if (typeof msg.method === 'string') {
9625
9997
  if (msg.method === 'session/update') {
9998
+ noteToolCallUpdate(msg.params?.update);
9626
9999
  emitTranscript(msg.params?.update);
9627
10000
  // #194: durable engine-native producer seam. In parallel with the
9628
10001
  // transcript-chunk relay overlay, hand the SAME raw `session/update` to
@@ -9673,6 +10046,16 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9673
10046
  idleTimeoutMs,
9674
10047
  recoveryWindowMs,
9675
10048
  isSettled: () => settled,
10049
+ logger,
10050
+ // #313: while an ACP tool call is in flight the agent is active even
10051
+ // when totally silent (a long foreground bash compile/test). Bounded by
10052
+ // the call's own timeout and the absolute job hard cap.
10053
+ isActivityHold: () => inFlightToolCalls.size > 0,
10054
+ // #313: also count CPU burned by processes whose cwd is inside the
10055
+ // agent's workspace, wherever they sit in the process tree — a shared
10056
+ // sccache daemon (parented to launchd/init, not the agent) compiling
10057
+ // this worktree is real work and must hold off the idle-kill.
10058
+ extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
9676
10059
  onIdleKill: () => {
9677
10060
  try { killTree(child); } catch { /* best effort */ }
9678
10061
  finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
@@ -11281,9 +11664,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
11281
11664
  * `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
11282
11665
  * longer running here, so it rides the marker as a flagged, observational entry
11283
11666
  * without making an idle worker look busy or inflating the drain's in-flight count.
11284
- * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
11667
+ * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null), watchdogPid:(number|null|undefined), signalReadyAt?:(number|null) }} fields
11285
11668
  */
11286
- function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
11669
+ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth, watchdogPid, signalReadyAt }) {
11287
11670
  const jobList = Array.isArray(jobs) ? jobs : [];
11288
11671
  return {
11289
11672
  pid,
@@ -11296,11 +11679,23 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt,
11296
11679
  // SDK client and begun leasing), the producer stamps this; null until then.
11297
11680
  // The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
11298
11681
  readyAt: readyAt ?? null,
11682
+ // #309 review: stamped once this worker's SIGUSR2 handler is INSTALLED —
11683
+ // independent of leasing readiness (`readyAt` needs a nonempty poll set, so
11684
+ // a healthy `--auto` worker with no deployed types never stamps it). The
11685
+ // daemon-upgrade handoff gates its retarget SIGUSR2 on THIS, not readyAt.
11686
+ signalReadyAt: signalReadyAt ?? null,
11299
11687
  // #293: the cached gh auth token's observable identity — { present,
11300
11688
  // fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
11301
11689
  // `supervisor status` can show the credential's age/scopes per worker. Null
11302
11690
  // on a worker that has not primed the cache yet (or predates the field).
11303
11691
  ghAuth: ghAuth ?? null,
11692
+ // #309: the daemon pid this worker's parent-death watchdog currently guards.
11693
+ // A rolling daemon upgrade SIGUSR2s each worker to RETARGET that watchdog at
11694
+ // the successor daemon before the old one exits; the old daemon waits
11695
+ // (bounded) for this stamp to name the successor — the ack that the worker
11696
+ // survives the handoff — so it never exits while a worker would still read
11697
+ // the (expected) reparent-to-init as an orphan event and self-reap.
11698
+ watchdogPid: Number.isInteger(watchdogPid) ? watchdogPid : null,
11304
11699
  };
11305
11700
  }
11306
11701
 
@@ -12118,9 +12513,13 @@ async function workAgent(req, flags, ctx) {
12118
12513
  // this worker would otherwise idle forever as an orphan reparented to init.
12119
12514
  // The watchdog makes the child self-exit the moment it is reparented. A
12120
12515
  // standalone `nano work` (no activity file) keeps classic nohup semantics.
12516
+ // The handle is retained so a rolling daemon upgrade (#309) can RETARGET the
12517
+ // same installation at the successor daemon's pid instead of self-reaping
12518
+ // when the old daemon hands over and exits (see the SIGUSR2 handler below).
12519
+ let parentWatchdog = null;
12121
12520
  if (activityFile) {
12122
12521
  const daemonPid = Number.parseInt(process.env.NANO_SUPERVISOR_DAEMON_PID ?? '', 10);
12123
- installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
12522
+ parentWatchdog = installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
12124
12523
  }
12125
12524
  const activeJobs = new Map(); // jobKey -> { type, since (ms epoch) }
12126
12525
  // #254: settlement-pending ghosts — jobKey -> { type, since, phase, leaseToken }.
@@ -12267,6 +12666,9 @@ async function workAgent(req, flags, ctx) {
12267
12666
  // waits for this stamp before draining the next worker so a bare spawn/PID is
12268
12667
  // never mistaken for a serving replacement.
12269
12668
  let readyAt = null;
12669
+ // #309 review: set once the SIGUSR2 (drain/retarget) handler is installed.
12670
+ let signalReadyAt = null;
12671
+ const markSignalReady = () => { signalReadyAt = Date.now(); };
12270
12672
  const writeActivity = () => {
12271
12673
  if (!activityFile) return;
12272
12674
  const jobs = [...activeJobs.entries()].map(([key, v]) => ({ key, type: v.type, since: v.since }));
@@ -12291,7 +12693,7 @@ async function workAgent(req, flags, ctx) {
12291
12693
  // #293: publish the cached gh token's identity (fingerprint/scopes/age —
12292
12694
  // never the value) so `supervisor status` can answer "is this worker holding
12293
12695
  // a stale token?" in one line.
12294
- const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
12696
+ const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, signalReadyAt, ghAuth: ghAuthTokenInfo(), watchdogPid: parentWatchdog && typeof parentWatchdog.watchedPid === 'function' ? parentWatchdog.watchedPid() : null });
12295
12697
  const tmp = `${activityFile}.${process.pid}.tmp`;
12296
12698
  try {
12297
12699
  mkdirSync(dirname(activityFile), { recursive: true });
@@ -14330,7 +14732,62 @@ async function workAgent(req, flags, ctx) {
14330
14732
  setTimeout(tick, 200);
14331
14733
  };
14332
14734
 
14333
- process.once(SUPERVISOR_DRAIN_SIGNAL, () => { gracefulDrain(SUPERVISOR_DRAIN_SIGNAL); });
14735
+ // #309: the SIGUSR2 handler is a named, re-armable function. The legacy path
14736
+ // (graceful drain) fires once and the worker exits, so `once` semantics
14737
+ // suffice there; but the daemon-upgrade path (below) does NOT exit the
14738
+ // worker, so it re-arms the handler for a later drain/upgrade.
14739
+ const onDrainSignal = () => {
14740
+ // #309 rolling daemon upgrade: the daemon reuses SIGUSR2 as its
14741
+ // per-worker signal, but when it names a successor daemon pid this is a
14742
+ // watchdog RETARGET, not a drain — the old daemon is handing the fleet to
14743
+ // a successor running the new plugin code and this worker must KEEP
14744
+ // SERVING (jobs in flight are never touched; only the parent process is
14745
+ // swapped). The successor pid arrives two ways, in priority order:
14746
+ // 1. `<activityFile>.upgrade` — written by the old daemon just before
14747
+ // it SIGUSR2s (authoritative: the daemon learns the successor pid
14748
+ // only at handoff time, after this worker was spawned, so it cannot
14749
+ // have ridden this worker's env);
14750
+ // 2. NANO_SUPERVISOR_UPGRADE_PID — covers a worker spawned mid-upgrade.
14751
+ // Re-arm the parent-death watchdog at the successor so the old daemon's
14752
+ // (expected) exit — which reparents this worker to init — is not read as
14753
+ // an orphan event, then republish the activity marker: the old daemon
14754
+ // waits (bounded) for the marker's `watchdogPid` to name the successor
14755
+ // before it exits, so the handoff never strands a worker that would
14756
+ // self-reap mid-upgrade.
14757
+ let upgradePid = Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
14758
+ if (activityFile) {
14759
+ try {
14760
+ const raw = readFileSync(`${activityFile}.upgrade`, 'utf8').trim();
14761
+ const fromFile = Number.parseInt(raw, 10);
14762
+ if (Number.isInteger(fromFile) && fromFile > 0) upgradePid = fromFile;
14763
+ } catch { /* no upgrade file — fall back to the env var */ }
14764
+ }
14765
+ if (Number.isInteger(upgradePid) && upgradePid > 0 && parentWatchdog && typeof parentWatchdog.retarget === 'function') {
14766
+ const ok = parentWatchdog.retarget(upgradePid);
14767
+ logger.info(`Received ${SUPERVISOR_DRAIN_SIGNAL} — daemon upgrade in progress: parent-death watchdog ${ok ? `retargeted to successor daemon pid ${upgradePid}` : `could NOT be retargeted (invalid pid)`}; this worker keeps serving.`);
14768
+ // Consume the retarget request: delete the `.upgrade` sidecar so a LATER
14769
+ // SIGUSR2 (a genuine drain, or a later upgrade that has not written a
14770
+ // fresh sidecar yet) is not misread as another retarget to this same
14771
+ // successor. Without this the sidecar is sticky — every later SIGUSR2
14772
+ // would retarget to the same (possibly long-gone) pid instead of
14773
+ // draining, breaking graceful stop / drain-remove / worker reload after
14774
+ // an upgrade (#309 review). Best-effort: a leftover only matters if it
14775
+ // names a live pid, and the successor also clears it on adopt.
14776
+ if (activityFile) { try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ } }
14777
+ try { writeActivityRef?.(); } catch { /* best effort — the ack stamp is advisory */ }
14778
+ // Re-arm: an upgrade that aborts (successor died before binding) must
14779
+ // leave this worker drainable by the surviving old daemon, and a LATER
14780
+ // upgrade/drain needs the handler again. (The drain path never re-arms —
14781
+ // the worker is on its way out.)
14782
+ process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
14783
+ return;
14784
+ }
14785
+ gracefulDrain(SUPERVISOR_DRAIN_SIGNAL);
14786
+ };
14787
+ process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
14788
+ // #309 review: publish that the handler is installed, so an upgrade handoff
14789
+ // can retarget this worker even before (or without ever) leasing.
14790
+ try { markSignalReady(); writeActivityRef?.(); } catch { /* best effort */ }
14334
14791
  process.once('SIGINT', () => { forceAbort('SIGINT'); });
14335
14792
  process.once('SIGTERM', () => { forceAbort('SIGTERM'); });
14336
14793
  });
@@ -15128,7 +15585,18 @@ function formatSupervisorStatus(status) {
15128
15585
  const onDiskVersion = status.pluginVersion
15129
15586
  ?? (() => { try { return pluginPackage().version; } catch { return null; } })();
15130
15587
  if (onDiskVersion && d.version && onDiskVersion !== d.version) {
15131
- lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code; restart the daemon for new supervisor code)`);
15588
+ lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code, and \`c8ctl nano supervisor upgrade\` to roll the daemon itself onto the new code)`);
15589
+ }
15590
+ // #309: while a rolling daemon upgrade is in flight the status frame (and the
15591
+ // persisted-state fallback) carries an `upgrade` block — render it as the
15592
+ // daemon-roll line, mirroring the worker-roll visibility (`rolling <from> →
15593
+ // <to>`). Absent on a quiet daemon.
15594
+ const up = status.upgrade;
15595
+ if (up && typeof up === 'object' && typeof up.state === 'string') {
15596
+ const from = up.fromVersion ?? '?';
15597
+ const to = up.toVersion ?? '?';
15598
+ const age = Number.isFinite(up.sinceEpochMs) ? ` (${formatDuration(Math.max(0, Date.now() - up.sinceEpochMs))})` : '';
15599
+ lines.push(` upgrade: ${up.state} — daemon ${from} → ${to}${age}`);
15132
15600
  }
15133
15601
  // A rolling reload can run for a long time (each worker drains in-flight
15134
15602
  // work). Surface it on every `status` so a detached operator — or someone
@@ -15263,6 +15731,10 @@ function stateFromStatus(res, socketPath) {
15263
15731
  version: res.daemon?.version,
15264
15732
  socket: res.daemon?.socket || socketPath,
15265
15733
  logFile: res.daemon?.logFile,
15734
+ // #309 review: carry the daemon's in-flight upgrade block so a stop client
15735
+ // can see the handoff (and the successor's pid) even when it connected to
15736
+ // the PREDECESSOR just before the socket flipped.
15737
+ ...(res.upgrade && typeof res.upgrade === 'object' ? { upgrade: res.upgrade } : {}),
15266
15738
  workers: res.workers || [],
15267
15739
  };
15268
15740
  }
@@ -15277,6 +15749,10 @@ function stateFromStatus(res, socketPath) {
15277
15749
  function statusFromState(running) {
15278
15750
  return {
15279
15751
  daemon: supervisorDaemonDescriptor(running),
15752
+ // #309: the persisted `upgrade` block rides the fallback render too — a
15753
+ // mid-handoff daemon whose socket is briefly unreachable must still show
15754
+ // the roll (same visibility guarantee as `version`/`logFile` above).
15755
+ ...(running && running.upgrade && typeof running.upgrade === 'object' ? { upgrade: running.upgrade } : {}),
15280
15756
  workers: (running.workers || []).map((w) => summarizeSupervisorWorker(w)),
15281
15757
  };
15282
15758
  }
@@ -15367,6 +15843,20 @@ function waitForChildExit(child, timeoutMs) {
15367
15843
  * Unix-only: reparent-to-init is a POSIX semantic; Windows job objects are the
15368
15844
  * equivalent lifecycle tie and are out of scope here (matching the daemon's
15369
15845
  * other `win32` guards). Returns a canceller so callers/tests can stop it.
15846
+ *
15847
+ * Rolling daemon hot-upgrade (issue #309): `retargetToPid` re-arms the SAME
15848
+ * watchdog installation at a NEW daemon pid — the successor daemon the old one
15849
+ * is handing the fleet to. Without it the swap is impossible: the old daemon
15850
+ * exits once the successor owns the socket, the worker reparents to init, and
15851
+ * an un-retargeted watchdog would read exactly that as "daemon died
15852
+ * ungracefully" and self-reap — killing every in-flight job mid-upgrade, the
15853
+ * precise outcome the rolling replace exists to avoid. The retarget is safe
15854
+ * against a stale signal because the watched pid is only ever advanced to the
15855
+ * successor the CURRENT daemon named (via NANO_SUPERVISOR_UPGRADE_PID, set by
15856
+ * the CLI that spawned it); a SIGUSR2 with no upgrade in flight keeps the
15857
+ * legacy graceful-drain semantics (the caller branches on the env var, not
15858
+ * here). No-op on win32 / when the pid is invalid; returns whether the
15859
+ * retarget landed.
15370
15860
  */
15371
15861
  function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, readPpid } = {}) {
15372
15862
  if (osPlatform() === 'win32') return () => {};
@@ -15380,14 +15870,40 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
15380
15870
  // NOT treat that as "orphaned". A *fallback* ppid of 1, by contrast, means we
15381
15871
  // were already reparented to init with no daemon left to watch.
15382
15872
  const explicit = Number.isInteger(parentPid) && parentPid > 0;
15383
- const watched = explicit ? parentPid : ppid();
15873
+ // `watched` is deliberately MUTABLE: a rolling daemon upgrade (#309) retargets
15874
+ // it at the successor daemon's pid the moment the old daemon hands over, so
15875
+ // the watchdog keeps guarding the worker across the swap instead of reading
15876
+ // the (expected) reparent-to-init as an orphan event and self-reaping.
15877
+ let watched = explicit ? parentPid : ppid();
15878
+ // #309: after a retarget, the ppid leg changes meaning. The old daemon exits
15879
+ // on purpose once the successor owns the socket, so this worker reparents to
15880
+ // init — and that is the EXPECTED end state, not an orphan event. From the
15881
+ // retarget on, only the SUCCESSOR's death orphans the worker (the ppid leg
15882
+ // would fire immediately and forever on the already-reparented worker).
15883
+ let retargeted = false;
15884
+ // #309 review (rollback): the guard state BEFORE the latest retarget. If the
15885
+ // successor dies mid-handoff while the predecessor is still alive (it rolls
15886
+ // back and keeps the fleet), the worker must fall back to guarding the
15887
+ // predecessor instead of self-reaping an in-flight job. Once the predecessor
15888
+ // has exited too, both are gone and the worker is genuinely orphaned.
15889
+ let fallback = null;
15384
15890
  const orphaned = typeof onOrphan === 'function' ? onOrphan : () => process.exit(0);
15385
15891
  const isOrphan = () => {
15386
15892
  // Reparented away from the daemon (typically to init, pid 1) → orphaned.
15387
- if (ppid() !== watched) return true;
15388
- // Defensive: the daemon pid is gone even though ppid still names it (a
15389
- // zombie/racey read). ESRCH ⇒ gone; EPERM ⇒ alive but not ours to signal.
15390
- try { process.kill(watched, 0); return false; } catch (err) { return err.code !== 'EPERM'; }
15893
+ // Skipped after an upgrade retarget: the reparent to init IS the handoff.
15894
+ if (!retargeted && ppid() !== watched) return true;
15895
+ // The watched daemon is gone (ESRCH ⇒ gone; EPERM ⇒ alive but not ours to
15896
+ // signal). After an upgrade retarget this is the ONLY orphan signal — the
15897
+ // ppid leg above is skipped (the reparent to init is expected), so the
15898
+ // worker self-reaps exactly when the SUCCESSOR daemon dies.
15899
+ const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; } };
15900
+ if (alive(watched)) return false;
15901
+ if (fallback && alive(fallback.watched) && (fallback.retargeted || ppid() === fallback.watched)) {
15902
+ ({ watched, retargeted } = fallback);
15903
+ fallback = null;
15904
+ return false;
15905
+ }
15906
+ return true;
15391
15907
  };
15392
15908
  // Orphaned already — self-reap now rather than idle forever. Either we fell
15393
15909
  // back to ppid and it is already init (no known parent), or the recorded
@@ -15395,7 +15911,22 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
15395
15911
  if ((!explicit && watched === 1) || isOrphan()) { orphaned(); return () => {}; }
15396
15912
  const timer = setInterval(() => { if (isOrphan()) { clearInterval(timer); orphaned(); } }, intervalMs);
15397
15913
  if (typeof timer.unref === 'function') timer.unref();
15398
- return () => { try { clearInterval(timer); } catch { /* ignore */ } };
15914
+ const cancel = () => { try { clearInterval(timer); } catch { /* ignore */ } };
15915
+ // #309: re-arm at the successor daemon's pid. The ppid leg of isOrphan() then
15916
+ // reads "reparented to init" as expected (the old daemon exited on purpose),
15917
+ // and only the SUCCESSOR's death orphans the worker from here on.
15918
+ cancel.retarget = (newPid) => {
15919
+ if (!Number.isInteger(newPid) || newPid <= 0) return false;
15920
+ if (newPid !== watched) fallback = { watched, retargeted };
15921
+ watched = newPid;
15922
+ retargeted = true;
15923
+ return true;
15924
+ };
15925
+ // The pid this installation currently guards — the daemon stamps it on the
15926
+ // worker's activity marker as the upgrade ack (the old daemon waits, bounded,
15927
+ // for it to name the successor before exiting).
15928
+ cancel.watchedPid = () => watched;
15929
+ return cancel;
15399
15930
  }
15400
15931
 
15401
15932
  // --- Daemon ----------------------------------------------------------------
@@ -15404,6 +15935,18 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
15404
15935
  * The supervisor daemon body. Runs under `c8ctl nano supervisor __daemon`,
15405
15936
  * spawned detached by `startSupervisorDaemon`. Never returns — it runs until a
15406
15937
  * stop request or SIGTERM, then drains children and exits.
15938
+ *
15939
+ * Issue #309 (rolling daemon hot-upgrade): when spawned with
15940
+ * NANO_SUPERVISOR_UPGRADE=1 this process is a SUCCESSOR — the CLI launched it
15941
+ * to take over the fleet from an old daemon running stale plugin code. Before
15942
+ * binding the control socket it waits for the old daemon to quiesce (its
15943
+ * `prepare-upgrade` op stops the accept loop), then adopts the old daemon's
15944
+ * workers from the persisted state (pid-polled synthetic child handles — the
15945
+ * workers themselves are never signaled, so in-flight jobs run undisturbed)
15946
+ * and finally asks the old daemon to hand over (its `upgrade` op retargets
15947
+ * every worker's parent-death watchdog here and exits). A successor that
15948
+ * cannot reach a quiesced old daemon in time exits 1 and leaves the old fleet
15949
+ * untouched.
15407
15950
  */
15408
15951
  async function runSupervisorDaemon() {
15409
15952
  const startedAt = new Date().toISOString();
@@ -15417,6 +15960,68 @@ async function runSupervisorDaemon() {
15417
15960
  const daemonLogFile = supervisorDaemonLogFile();
15418
15961
  mkdirSync(getSupervisorLogDir(), { recursive: true });
15419
15962
 
15963
+ // Boot-time log line, available before the closure-scoped `dlog` exists (the
15964
+ // upgrade-mode quiesce handshake below runs before the workers map and its
15965
+ // helpers are initialized). Identical format.
15966
+ const dlogBoot = (msg) => {
15967
+ try { appendFileSync(daemonLogFile, `[${new Date().toISOString()}] ${msg}\n`); } catch { /* best effort */ }
15968
+ };
15969
+
15970
+ // #309: successor boot. `upgradeMode` latches once the old daemon has been
15971
+ // asked to quiesce; `predecessorPid`/`upgradeFromVersion` feed the status
15972
+ // frame's `upgrade` block so `supervisor status` shows the roll in flight
15973
+ // (`rolling <from> → <to>`), mirroring the worker-roll visibility.
15974
+ const upgradeMode = process.env.NANO_SUPERVISOR_UPGRADE === '1';
15975
+ let predecessorPid = null;
15976
+ let upgradeFromVersion = null;
15977
+ let upgradeStartedMs = null;
15978
+ if (upgradeMode) {
15979
+ const t0 = Date.now();
15980
+ dlogBoot(`upgrade mode: waiting for the current daemon to quiesce (pid from ${getSupervisorStateFile()})`);
15981
+ for (;;) {
15982
+ const st = readSupervisorState();
15983
+ if (!st || !isPidAlive(st.pid)) {
15984
+ // No live old daemon — nothing to hand over. Serve the fleet normally
15985
+ // (there is nothing to adopt; the state file, if any, is stale).
15986
+ dlogBoot('upgrade mode: no live predecessor daemon — starting as a plain daemon');
15987
+ break;
15988
+ }
15989
+ predecessorPid = st.pid;
15990
+ upgradeFromVersion = typeof st.version === 'string' ? st.version : null;
15991
+ // Ask the old daemon to stop accepting new control connections. An old
15992
+ // daemon that predates #309 answers `unknown op` — refuse to upgrade from
15993
+ // it rather than racing its accept loop for the socket.
15994
+ let prep = null;
15995
+ try {
15996
+ // #309 review: quiescence awaits admitted opQueue mutations (a restart
15997
+ // can take ~28s to stop its worker), and the predecessor closes its
15998
+ // listener as soon as it latches — so a retry could never reach it.
15999
+ // Keep THIS connection open for the remaining quiesce budget.
16000
+ const remainingMs = SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS - (Date.now() - t0);
16001
+ prep = await supervisorRequest({ op: 'prepare-upgrade', pid: process.pid }, { socketPath, timeoutMs: 750, responseTimeoutMs: Math.max(SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS, remainingMs) });
16002
+ } catch { /* socket briefly unreachable — retry until the deadline */ }
16003
+ if (prep && prep.ok) break; // quiesced — the accept loop is closed
16004
+ if (prep && prep.ok === false && /unknown op/.test(String(prep.error || ''))) {
16005
+ dlogBoot(`upgrade mode: predecessor pid ${st.pid} does not understand prepare-upgrade (too old) — refusing to upgrade`);
16006
+ process.exit(1);
16007
+ }
16008
+ // #309 review: the predecessor bound its handoff to a DIFFERENT successor
16009
+ // (a concurrent upgrade won the race). Never keep retrying — the binding
16010
+ // will never accept this pid, and two successors must not both wait to
16011
+ // adopt the same fleet. Exit and leave the bound successor to finish.
16012
+ if (prep && prep.ok === false && /already bound to successor pid/.test(String(prep.error || ''))) {
16013
+ dlogBoot(`upgrade mode: predecessor pid ${st.pid} is bound to a different successor — exiting so the bound successor completes the handoff`);
16014
+ process.exit(1);
16015
+ }
16016
+ if (Date.now() - t0 > SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS) {
16017
+ dlogBoot(`upgrade mode: predecessor pid ${st.pid} did not quiesce within ${SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS}ms — leaving it in place`);
16018
+ process.exit(1);
16019
+ }
16020
+ await new Promise((r) => setTimeout(r, 150));
16021
+ }
16022
+ if (predecessorPid != null) upgradeStartedMs = Date.now();
16023
+ }
16024
+
15420
16025
  const workers = new Map();
15421
16026
  const attachClients = new Set();
15422
16027
  // #202: sockets that issued a `stop`/drain and are waiting for the daemon to
@@ -15434,6 +16039,23 @@ async function runSupervisorDaemon() {
15434
16039
  // restart-on-exit path and let a second `stop --force` escalate a live drain.
15435
16040
  let draining = false;
15436
16041
  let forcing = false;
16042
+ // #309 review: `shuttingDown` is latched EARLY (in the `stop` op, before the
16043
+ // deferred shutdown() runs) to reserve shutdown ownership against a racing
16044
+ // prepare-upgrade. `shutdownStarted` is the separate "shutdown()'s body has
16045
+ // begun" guard: the deferred shutdown() must still run its body even though
16046
+ // `shuttingDown` is already true, while a SECOND stop/signal must escalate
16047
+ // (force) or return instead of re-running the body.
16048
+ let shutdownStarted = false;
16049
+ // #309: a rolling daemon upgrade owns this daemon. `upgrading` is the
16050
+ // OLD-daemon latch: set by `prepare-upgrade` (accept loop closed; mutations
16051
+ // refused) and held through the `upgrade` handoff until this process exits.
16052
+ // `quiesced` flips when the accept loop has actually closed — the successor
16053
+ // polls for it before binding the socket. `upgradeInfo` carries the roll's
16054
+ // identity ({ state, fromVersion, toVersion, successorPid, sinceEpochMs })
16055
+ // for the status frame's `upgrade` block.
16056
+ let upgrading = false;
16057
+ let quiesced = false;
16058
+ let upgradeInfo = null;
15437
16059
  // Hot code reload (rolling drain+respawn). A `reload` op adopts new on-disk
15438
16060
  // plugin code into the worker children by gracefully draining and respawning
15439
16061
  // them one at a time (so the fleet keeps serving). This flag rejects a second
@@ -15448,6 +16070,19 @@ async function runSupervisorDaemon() {
15448
16070
  // refreshed status to attached consoles only on real change (see below).
15449
16071
  let monitorTimer = null;
15450
16072
  let lastMonitorSig = null;
16073
+ // #309: while quiesced for an upgrade the predecessor's accept loop is closed
16074
+ // and — on a second upgrade — every worker is an ADOPTED handle whose liveness
16075
+ // poll (`adoptTimer`) is unref'd, as is `monitorTimer`. With no referenced
16076
+ // handle left, Node would let the predecessor EXIT before the successor's
16077
+ // watchdog-retarget acks arrive / before SIGUSR1 lands — and the never-
16078
+ // resolving handoff Promise does NOT keep the event loop alive. This
16079
+ // referenced keepalive holds the predecessor up for exactly the quiesce→
16080
+ // handover window; it is cleared on the SIGUSR1 handover (and on shutdown).
16081
+ let quiesceKeepalive = null;
16082
+ // #309: polls the successor pid while quiesced so a successor that dies
16083
+ // mid-handoff (before SIGUSR1) triggers a rollback instead of leaving this
16084
+ // daemon permanently quiesced + unreachable. Cleared on handover/rollback.
16085
+ let successorWatchTimer = null;
15451
16086
 
15452
16087
  // Daemon-wide mutation serialization: `add`/`remove`/`restart` must not
15453
16088
  // interleave, or two clients racing the same worker could each spawn an
@@ -15465,16 +16100,46 @@ async function runSupervisorDaemon() {
15465
16100
 
15466
16101
  const workerPublic = (w) => summarizeSupervisorWorker(w);
15467
16102
 
15468
- const persist = () => {
16103
+ // #309: the daemon descriptor gains an `upgrade` block while a rolling daemon
16104
+ // upgrade is in flight — on the OLD daemon (state 'rolling', naming the
16105
+ // successor it is handing to) and on the SUCCESSOR (state 'adopting' until
16106
+ // the predecessor exits). `supervisor status` renders it as
16107
+ // `upgrade: rolling <from> → <to>` — the same visibility the worker roll has.
16108
+ const currentUpgradeBlock = () => {
16109
+ if (upgradeInfo) return upgradeInfo;
16110
+ if (upgradeMode && predecessorPid != null) {
16111
+ return { state: 'adopting', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, sinceEpochMs: upgradeStartedMs };
16112
+ }
16113
+ return null;
16114
+ };
16115
+
16116
+ // #309: once the successor has confirmed ownership (SIGUSR1 → handedOver) the
16117
+ // predecessor must make its final snapshot its LAST canonical write. More
16118
+ // broadly, once QUIESCED (`upgrading` latched) the successor has already read
16119
+ // the quiesce snapshot and adopted the inventory, so any later automatic
16120
+ // persist (a straggler worker-death handler firing between quiesce and exit)
16121
+ // would overwrite the successor's pid + inventory with the dead/dying daemon's
16122
+ // — corrupting discovery/recovery until the successor happens to persist
16123
+ // again. So persist() is suppressed while quiesced UNLESS forced (the quiesce
16124
+ // snapshot and the final handed-over snapshot are the two deliberate writes).
16125
+ let handedOver = false;
16126
+ const persist = (force = false) => {
16127
+ if (handedOver) return; // the successor owns the state file now
16128
+ if (upgrading && !force) return; // quiesced — the successor owns the inventory
15469
16129
  try {
15470
16130
  writeSupervisorState({
15471
16131
  ...supervisorDaemonDescriptor({
15472
16132
  pid: process.pid, startedAt, version: daemonVersion, socket: socketPath, logFile: daemonLogFile,
15473
16133
  }),
16134
+ ...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
15474
16135
  workers: [...workers.values()].map((w) => ({
15475
16136
  id: w.id, profile: w.profile, args: w.args, pid: isPidAlive(w.pid) ? w.pid : null,
15476
16137
  startedAt: w.startedAt || null, restarts: w.restarts, lastExit: w.lastExit ?? null,
15477
16138
  stopping: !!w.stopping, logFile: w.logFile, configFailed: w.configFailed === true,
16139
+ // #309: persist drain-remove intent so an adopted worker that exits
16140
+ // mid-drain is still REMOVED by the successor (not retained forever,
16141
+ // which would block name reuse and hang `workforce stop`).
16142
+ removeOnExit: w.removeOnExit === true,
15478
16143
  })),
15479
16144
  });
15480
16145
  } catch { /* best effort */ }
@@ -15511,6 +16176,11 @@ async function runSupervisorDaemon() {
15511
16176
  const activityFile = supervisorWorkerActivityFile(w.id);
15512
16177
  w.activityFile = activityFile;
15513
16178
  try { rmSync(activityFile, { force: true }); } catch { /* best effort */ }
16179
+ // #309 review: also clear a stale `.upgrade` retarget sidecar. A worker
16180
+ // force-restarted mid-handoff before consuming it would otherwise inherit
16181
+ // the same path and treat its next genuine drain SIGUSR2 as a retarget.
16182
+ // (A handoff in progress re-writes the sidecar before it SIGUSR2s.)
16183
+ try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
15514
16184
  // `--name w.id` makes the child's broker workerName match this worker's
15515
16185
  // supervisor id, so the same profile launched twice is distinct end-to-end.
15516
16186
  // NANO_SUPERVISOR_ACTIVITY_FILE tells the child where to report per-job
@@ -15519,8 +16189,20 @@ async function runSupervisorDaemon() {
15519
16189
  // watchdog can self-reap if we die ungracefully (SIGKILL/crash) and can't
15520
16190
  // run the child-draining shutdown — even if we die mid-startup, before the
15521
16191
  // child reparents to init.
16192
+ // NANO_SUPERVISOR_UPGRADE_PID (#309): when a daemon upgrade is in flight,
16193
+ // the successor's pid rides the child's env so the worker's SIGUSR2 handler
16194
+ // can tell a watchdog RETARGET (upgrade — keep serving) apart from a
16195
+ // graceful DRAIN (no upgrade pid — quiesce and exit). The daemon learns the
16196
+ // successor pid only at handoff time (the `upgrade` op), AFTER most workers
16197
+ // were spawned — so env alone can't reach them. The daemon therefore also
16198
+ // writes `<activityFile>.upgrade` (the successor pid) before it SIGUSR2s,
16199
+ // and the worker reads THAT as the authoritative retarget target; this env
16200
+ // forwarding covers only a worker spawned mid-upgrade.
16201
+ const childEnv = { ...process.env, NANO_SUPERVISOR_ACTIVITY_FILE: activityFile, NANO_SUPERVISOR_DAEMON_PID: String(process.pid) };
16202
+ const upgradeSuccessorPid = Number(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
16203
+ if (Number.isInteger(upgradeSuccessorPid) && upgradeSuccessorPid > 0) childEnv.NANO_SUPERVISOR_UPGRADE_PID = String(upgradeSuccessorPid);
15522
16204
  const child = spawn(exec, [entry, 'nano', 'work', w.profile, '--name', w.id, ...w.args], {
15523
- env: { ...process.env, NANO_SUPERVISOR_ACTIVITY_FILE: activityFile, NANO_SUPERVISOR_DAEMON_PID: String(process.pid) },
16205
+ env: childEnv,
15524
16206
  stdio: ['ignore', ringed ? 'pipe' : fd, ringed ? 'pipe' : fd],
15525
16207
  });
15526
16208
  if (typeof fd === 'number') { try { closeSync(fd); } catch { /* dup'd into child */ } }
@@ -15613,7 +16295,9 @@ async function runSupervisorDaemon() {
15613
16295
  broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
15614
16296
  w.restartTimer = setTimeout(() => {
15615
16297
  w.restartTimer = null;
15616
- if (!w.stopping && !shuttingDown && workers.has(w.id)) startWorker(w);
16298
+ // #309: a quiesced old daemon (mid-upgrade) never respawns — the
16299
+ // successor owns the fleet and would double-spawn the worker.
16300
+ if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
15617
16301
  }, delay);
15618
16302
  if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
15619
16303
  persist();
@@ -15623,6 +16307,123 @@ async function runSupervisorDaemon() {
15623
16307
  persist();
15624
16308
  };
15625
16309
 
16310
+ // #309: a synthetic ChildProcess-shaped handle for an ADOPTED worker — one
16311
+ // the old daemon spawned and the successor inherits by pid. The successor
16312
+ // never spawned it, so there is no real 'exit' event: a poll timer is the
16313
+ // death detector, feeding the same `handleDeath` path a real child's exit
16314
+ // would (restart backoff included). The handle provides the surface the
16315
+ // daemon touches: `pid`, `exitCode`/`signalCode` (null while alive — the
16316
+ // readiness gate and reload checks read them), `kill` (force-stop),
16317
+ // `unref`/`ref`, null stdio (the log ring stays with the OLD daemon process),
16318
+ // and — critically — REAL `exit` listener semantics (`once`/`on`/
16319
+ // `removeListener` + `markExited`), because `waitForChildExit()` resolves only
16320
+ // when `exit` fires.
16321
+ const makeAdoptedChild = (pid) => {
16322
+ const exitWaiters = new Set();
16323
+ const handle = {
16324
+ pid,
16325
+ exitCode: null,
16326
+ signalCode: null,
16327
+ killed: false,
16328
+ stdout: null,
16329
+ stderr: null,
16330
+ stdin: null,
16331
+ kill(signal) { try { return process.kill(pid, signal); } catch { return false; } },
16332
+ once(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
16333
+ on(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
16334
+ removeListener(event, fn) { if (event === 'exit') exitWaiters.delete(fn); return handle; },
16335
+ unref() { return handle; },
16336
+ ref() { return handle; },
16337
+ // Invoked by the adopt liveness poll when the pid is gone: record the exit
16338
+ // and flush every waiter so graceful-stop/reload promises resolve.
16339
+ markExited(code, signal) {
16340
+ if (handle.exitCode !== null || handle.signalCode !== null) return;
16341
+ handle.exitCode = code;
16342
+ handle.signalCode = signal ?? null;
16343
+ for (const fn of [...exitWaiters]) { exitWaiters.delete(fn); try { fn(code, signal ?? null); } catch { /* ignore */ } }
16344
+ },
16345
+ };
16346
+ return handle;
16347
+ };
16348
+
16349
+ // Adopt one worker record from the predecessor's persisted state. Returns the
16350
+ // live worker entry, or null when the pid is already gone (nothing to adopt —
16351
+ // the fleet is exactly what survives the swap). The poll timer is unref'd so
16352
+ // it never keeps the daemon alive on its own.
16353
+ const adoptWorker = (rec) => {
16354
+ if (!rec || typeof rec.id !== 'string' || !rec.id) return null;
16355
+ if (!isPidAlive(rec.pid)) return null;
16356
+ const w = {
16357
+ id: rec.id,
16358
+ profile: typeof rec.profile === 'string' ? rec.profile : '',
16359
+ args: Array.isArray(rec.args) ? rec.args.map(String) : [],
16360
+ restarts: Number(rec.restarts) || 0,
16361
+ stopping: rec.stopping === true,
16362
+ // #309: restore drain-remove intent persisted by the predecessor, so a
16363
+ // worker draining for removal that exits after the swap is REMOVED here
16364
+ // (its record deleted) rather than retained — which would block name reuse
16365
+ // and hang a `workforce stop` waiting for the record to disappear.
16366
+ removeOnExit: rec.removeOnExit === true,
16367
+ lastExit: rec.lastExit ?? null,
16368
+ logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
16369
+ configFailed: rec.configFailed === true,
16370
+ adopted: true,
16371
+ };
16372
+ w.activityFile = supervisorWorkerActivityFile(w.id);
16373
+ // Clear any stale `.upgrade` sidecar the handoff left: the worker consumed
16374
+ // it to retarget its watchdog here, and a leftover would make a LATER
16375
+ // SIGUSR2 (a genuine drain) misread as another upgrade retarget.
16376
+ try { rmSync(`${w.activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
16377
+ const child = makeAdoptedChild(rec.pid);
16378
+ w.child = child;
16379
+ w.pid = rec.pid;
16380
+ w.startedAt = rec.startedAt || null;
16381
+ w.spawnedAt = rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now();
16382
+ let settled = false;
16383
+ const handleDeath = (reason) => {
16384
+ if (w.child !== child || settled) return;
16385
+ settled = true;
16386
+ if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
16387
+ // Mark the synthetic handle exited BEFORE running the death path so any
16388
+ // `waitForChildExit` waiter (graceful stop / reload) resolves — the poll
16389
+ // below is the only death detector for an adopted worker.
16390
+ if (typeof child.markExited === 'function') child.markExited(1, null);
16391
+ w.pid = null;
16392
+ w.lastExit = reason;
16393
+ try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
16394
+ const ranMs = Date.now() - (w.spawnedAt || Date.now());
16395
+ if (ranMs >= SUPERVISOR_HEALTHY_UPTIME_MS) w.restarts = 0;
16396
+ if (w.stopping || shuttingDown || !workers.has(w.id)) {
16397
+ if (w.removeOnExit && workers.get(w.id) === w) {
16398
+ workers.delete(w.id);
16399
+ try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
16400
+ dlog(`worker '${w.id}' removed (drained)`);
16401
+ broadcast({ type: 'event', event: 'worker-remove', id: w.id });
16402
+ }
16403
+ persist();
16404
+ if (draining) { try { broadcast(statusFrame(false)); } catch { /* best effort */ } }
16405
+ return;
16406
+ }
16407
+ if (upgrading) { persist(); return; } // quiesced for the handoff — the successor owns respawns
16408
+ const delay = supervisorBackoffMs(w.restarts);
16409
+ w.restarts += 1;
16410
+ dlog(`adopted worker '${w.id}' down (${reason}); restarting in ${delay}ms (restart #${w.restarts})`);
16411
+ broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
16412
+ w.restartTimer = setTimeout(() => {
16413
+ w.restartTimer = null;
16414
+ if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
16415
+ }, delay);
16416
+ if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
16417
+ persist();
16418
+ };
16419
+ w.adoptTimer = setInterval(() => { if (!isPidAlive(rec.pid)) handleDeath('code 1 (adopted worker exited)'); }, SUPERVISOR_ADOPT_POLL_MS);
16420
+ if (typeof w.adoptTimer.unref === 'function') w.adoptTimer.unref();
16421
+ workers.set(w.id, w);
16422
+ dlog(`worker '${w.id}' (profile ${w.profile}) adopted from predecessor daemon (pid ${w.pid})`);
16423
+ broadcast({ type: 'event', event: 'worker-adopted', worker: workerPublic(w) });
16424
+ return w;
16425
+ };
16426
+
15626
16427
  const addWorker = (profile, args, name) => {
15627
16428
  const taken = new Set(workers.keys());
15628
16429
  let id;
@@ -15645,6 +16446,33 @@ async function runSupervisorDaemon() {
15645
16446
  return w;
15646
16447
  };
15647
16448
 
16449
+ // #309 review: workers whose upgrade watchdog-retarget is still outstanding
16450
+ // (id -> { pid, signalled, drainOnly?, drainDeferred? }), owned by the
16451
+ // successor's handoff loop. A graceful drain (stop/remove/reload) also uses
16452
+ // SIGUSR2, so it must not race a pending retarget: a drain signal consumed as
16453
+ // the retarget leaves the drain unfulfilled, and a second SIGUSR2 after the
16454
+ // one-shot handler fired would kill in-flight work. sendDrainSignal
16455
+ // coordinates the two so both are delivered, in order.
16456
+ const retargetPending = new Map();
16457
+ const sendDrainSignal = (w, pid) => {
16458
+ const p = retargetPending.get(w.id);
16459
+ if (p && p.pid === pid) {
16460
+ if (!p.signalled) {
16461
+ // Retarget not sent yet: cancel it (drop the sidecar) and drain
16462
+ // instead — the handoff loop delivers the drain once the handler is
16463
+ // installed and keeps the predecessor alive until the worker exits.
16464
+ try { rmSync(`${w.activityFile || supervisorWorkerActivityFile(w.id)}.upgrade`, { force: true }); } catch { /* best effort */ }
16465
+ p.signalled = true; p.drainOnly = true; p.drainDeferred = true;
16466
+ } else if (!p.drainOnly) {
16467
+ // Retarget in flight: deliver the drain right after its ack.
16468
+ p.drainDeferred = true;
16469
+ }
16470
+ // drainOnly without drainDeferred: already draining — nothing to send.
16471
+ return;
16472
+ }
16473
+ try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
16474
+ };
16475
+
15648
16476
  const forceStopWorker = async (w) => {
15649
16477
  const pid = w.pid;
15650
16478
  if (w.child && pid) {
@@ -15671,7 +16499,7 @@ async function runSupervisorDaemon() {
15671
16499
  // — a `stop --force` escalation (forceStopWorker) is the only way to cut a
15672
16500
  // stuck drain short, and it resolves this same wait when the child dies.
15673
16501
  if (w.child && pid) {
15674
- try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
16502
+ sendDrainSignal(w, pid);
15675
16503
  await waitForChildExit(w.child, null);
15676
16504
  }
15677
16505
  return true;
@@ -15693,7 +16521,7 @@ async function runSupervisorDaemon() {
15693
16521
  w.removeOnExit = true;
15694
16522
  if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
15695
16523
  const pid = w.pid;
15696
- if (w.child && pid) { try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
16524
+ if (w.child && pid) { sendDrainSignal(w, pid); }
15697
16525
  else { workers.delete(id); try { rmSync(supervisorWorkerActivityFile(id), { force: true }); } catch { /* best effort */ } broadcast({ type: 'event', event: 'worker-remove', id }); persist(); }
15698
16526
  dlog(`worker '${id}' draining for removal`);
15699
16527
  return true;
@@ -15790,7 +16618,7 @@ async function runSupervisorDaemon() {
15790
16618
  if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
15791
16619
  const pid = w.pid;
15792
16620
  if (child && pid) {
15793
- try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
16621
+ sendDrainSignal(w, pid);
15794
16622
  await waitForChildExit(child, null);
15795
16623
  }
15796
16624
  // Only respawn if nobody else acted on this worker while we drained: it must
@@ -15987,12 +16815,28 @@ async function runSupervisorDaemon() {
15987
16815
  reloaded: [...reloadDone],
15988
16816
  total: reloadTargets.length,
15989
16817
  },
16818
+ // #309: present only while a rolling daemon upgrade is in flight
16819
+ // (`{ state, fromVersion, toVersion, … }`) — the successor polls the old
16820
+ // daemon's `upgrade.state === 'quiesced'` through this field before binding
16821
+ // the socket, and `supervisor status` renders it as the daemon-roll line.
16822
+ ...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
15990
16823
  workers: pub || [...workers.values()].map(workerPublic),
15991
16824
  ...(final ? { final: true } : {}),
15992
16825
  });
15993
16826
 
15994
16827
  const shutdown = async (signal, { force = true } = {}) => {
15995
- if (shuttingDown) {
16828
+ // #309 review: a quiesced (mid-upgrade) daemon must NEVER run the shutdown
16829
+ // path — it would SIGTERM/SIGKILL the very workers the successor has
16830
+ // adopted, and its cleanup would delete the successor's socket and state
16831
+ // file. The upgrade's SIGUSR1 handover is this daemon's only exit while
16832
+ // quiesced; a signal (SIGTERM/SIGINT/SIGHUP) is logged and ignored so the
16833
+ // handoff can finish. The socket `stop` op refuses earlier (the `upgrading`
16834
+ // check in handleRequest) and never reaches here.
16835
+ if (upgrading) {
16836
+ dlog(`ignoring ${signal || 'shutdown request'} — a daemon upgrade owns this daemon (successor pid ${upgradeInfo?.successorPid ?? 'unknown'}); the successor owns the fleet now`);
16837
+ return;
16838
+ }
16839
+ if (shutdownStarted) {
15996
16840
  // A shutdown is already running. A `stop --force` arriving mid-DRAIN
15997
16841
  // escalates it: hard-stop every worker still finishing its jobs so the
15998
16842
  // operator isn't stuck waiting. Idempotent — only the first force escalates.
@@ -16005,6 +16849,7 @@ async function runSupervisorDaemon() {
16005
16849
  return;
16006
16850
  }
16007
16851
  shuttingDown = true;
16852
+ shutdownStarted = true;
16008
16853
  draining = !force;
16009
16854
  // Let any in-flight mutation finish before we snapshot the worker set, so
16010
16855
  // an add/restart racing the shutdown can't leave an orphaned child behind.
@@ -16013,6 +16858,7 @@ async function runSupervisorDaemon() {
16013
16858
  if (!force) broadcast({ type: 'event', event: 'draining', workers: [...workers.values()].map(workerPublic) });
16014
16859
  await Promise.all([...workers.keys()].map((id) => stopWorker(id, { force })));
16015
16860
  if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
16861
+ if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
16016
16862
  broadcast({ type: 'event', event: 'daemon-stop' });
16017
16863
  // Terminal frame for every waiting `stop` client (streaming or one-shot).
16018
16864
  // We must FLUSH these before exiting: `process.exit()` does NOT drain pending
@@ -16036,6 +16882,179 @@ async function runSupervisorDaemon() {
16036
16882
  process.exit(0);
16037
16883
  };
16038
16884
 
16885
+ // #309: quiesce this daemon for an upgrade WITHOUT exiting: stop accepting
16886
+ // new control connections (the successor binds the socket the moment our
16887
+ // accept loop closes) and latch `upgrading` so mutations and worker respawns
16888
+ // are refused. Idempotent — the successor polls until it observes the
16889
+ // quiesced state. Attached consoles are destroyed (their daemon is going
16890
+ // away). The handoff is then driven by the SUCCESSOR (it retargets the
16891
+ // workers' watchdogs and SIGUSR1s this daemon to exit) — this daemon's
16892
+ // accept loop is already closed, so it cannot be asked over the socket.
16893
+ //
16894
+ // Snapshot AFTER the mutation queue drains (#309 review): latching
16895
+ // `upgrading` rejects NEW mutations, but one may already be admitted to
16896
+ // `opQueue` (e.g. a restart that is mid-respawn). Persisting before that
16897
+ // settles would snapshot a half-swapped worker set the successor then adopts
16898
+ // — missing the replacement the in-flight restart is about to spawn. So we
16899
+ // latch admission synchronously, then AWAIT `opQueue` (exactly as
16900
+ // `shutdown()` does) before persisting and reporting `quiesced`.
16901
+ let quiescePromise = null;
16902
+ // #309 review: the worker-id set captured in the quiesce snapshot. Rollback
16903
+ // reconciles the live map against it — the successor owns add/remove while
16904
+ // this daemon is quiesced, so a worker the successor removed mid-handoff must
16905
+ // stay removed if the handoff rolls back (never resurrected).
16906
+ let quiesceSnapshotIds = null;
16907
+ const quiesceForUpgrade = (successorPid, successorVersion) => {
16908
+ // #309 review: bind the handoff to ONE successor. `doQuiesce` latches
16909
+ // `upgrading` only inside its async body, so two prepare-upgrade requests
16910
+ // that arrive before the first body's first await both observe
16911
+ // `upgrading === false` and would share one quiesce promise — two
16912
+ // successors, each believing it owns the bind+adopt, racing for the same
16913
+ // socket and fleet. Validate and bind preparation to the FIRST successor
16914
+ // pid synchronously (before any await), and reject every other pid until
16915
+ // this handoff finishes (handover) or rolls back. The bound successor's
16916
+ // own retries (same pid) stay idempotent — they share the promise.
16917
+ if (!Number.isInteger(successorPid) || successorPid <= 0) {
16918
+ // An unidentifiable caller never earns the binding — without a pid the
16919
+ // successor watch cannot run and no retry could ever match the binding.
16920
+ const err = new Error('prepare-upgrade requires a valid successor pid');
16921
+ err.code = 'UPGRADE_BOUND';
16922
+ throw err;
16923
+ }
16924
+ if (upgrading) {
16925
+ const boundTo = upgradeInfo && upgradeInfo.successorPid;
16926
+ if (boundTo !== successorPid) {
16927
+ const err = new Error(`a daemon upgrade is already bound to successor pid ${boundTo} — refusing a concurrent handoff`);
16928
+ err.code = 'UPGRADE_BOUND';
16929
+ throw err;
16930
+ }
16931
+ }
16932
+ if (!upgrading || !quiescePromise) quiescePromise = doQuiesce(successorPid, successorVersion);
16933
+ return quiescePromise;
16934
+ };
16935
+ const doQuiesce = async (successorPid, successorVersion) => {
16936
+ if (!upgrading) {
16937
+ upgrading = true; // latch admission FIRST — rejects new mutations/respawns
16938
+ upgradeInfo = { state: 'quiescing', fromVersion: daemonVersion, toVersion: successorVersion ?? null, successorPid, sinceEpochMs: Date.now() };
16939
+ dlog(`daemon upgrade: quiescing for successor pid ${successorPid} — control socket accept loop closing, mutations refused`);
16940
+ try { server.close(); } catch { /* ignore */ }
16941
+ for (const s of [...attachClients]) { try { s.destroy(); } catch { /* ignore */ } }
16942
+ // Hold the event loop open until the successor's SIGUSR1 handover (see the
16943
+ // quiesceKeepalive declaration) — a no-op tick on a REFERENCED timer.
16944
+ if (!quiesceKeepalive) quiesceKeepalive = setInterval(() => {}, SUPERVISOR_ADOPT_POLL_MS);
16945
+ // #309 review (no-rollback): watch the successor. If it dies AFTER we
16946
+ // quiesced but BEFORE handing over (SIGUSR1), this daemon is left quiesced
16947
+ // and unreachable — respawns disabled, accept loop closed, no way for even
16948
+ // a later upgrade to reach it. Roll back: re-open the accept loop and
16949
+ // unlatch `upgrading` so normal operation (and a retry) resumes.
16950
+ armSuccessorWatch(successorPid);
16951
+ // Wait for any ALREADY-ADMITTED mutation (e.g. a restart mid-respawn) to
16952
+ // settle before snapshotting, so the successor adopts its replacement.
16953
+ try { await opQueue; } catch { /* mutation already logged */ }
16954
+ if (!upgrading) return; // rolled back while draining the queue
16955
+ upgradeInfo = { ...upgradeInfo, state: 'quiesced' };
16956
+ quiesceSnapshotIds = [...workers.keys()];
16957
+ persist(true); // the deliberate quiesce snapshot the successor adopts from
16958
+ }
16959
+ quiesced = true;
16960
+ };
16961
+
16962
+ // #309 review (no-rollback): while quiesced, poll the successor pid. If it
16963
+ // dies before handing over (SIGUSR1), this daemon would otherwise stay
16964
+ // quiesced + unreachable forever (accept loop closed, respawns disabled, no
16965
+ // listener for even a later upgrade). On detecting a dead successor, roll
16966
+ // back: re-open the accept loop and unlatch `upgrading`.
16967
+ const armSuccessorWatch = (successorPid) => {
16968
+ if (!Number.isInteger(successorPid) || successorPid <= 0) return;
16969
+ if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
16970
+ successorWatchTimer = setInterval(() => {
16971
+ if (!upgrading || handedOver) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; return; }
16972
+ if (!isPidAlive(successorPid)) {
16973
+ try { clearInterval(successorWatchTimer); } catch { /* ignore */ }
16974
+ successorWatchTimer = null;
16975
+ rollbackFromUpgrade(successorPid);
16976
+ }
16977
+ }, SUPERVISOR_ADOPT_POLL_MS);
16978
+ if (typeof successorWatchTimer.unref === 'function') successorWatchTimer.unref();
16979
+ };
16980
+
16981
+ // Restore normal operation after a successor died mid-handoff: re-listen on
16982
+ // the control socket (the successor never bound it, or died holding it) and
16983
+ // unlatch `upgrading` so mutations/respawns resume. Best-effort — if the
16984
+ // re-listen fails the daemon stays quiesced and logs, but the common case
16985
+ // (successor crashed before binding) recovers cleanly.
16986
+ const rollbackFromUpgrade = (deadSuccessorPid) => {
16987
+ if (!upgrading || handedOver) return;
16988
+ quiescePromise = null;
16989
+ dlog(`daemon upgrade: successor pid ${deadSuccessorPid} died before handover — rolling back: re-opening the control socket and resuming normal operation`);
16990
+ upgrading = false;
16991
+ quiesced = false;
16992
+ upgradeInfo = null;
16993
+ // Capture the snapshot BEFORE clearing it — the membership reconcile below
16994
+ // reads this copy (clearing first made every rollback drop the whole fleet).
16995
+ // A null snapshot (quiesce never reached the snapshot point) means there is
16996
+ // nothing to reconcile against, so membership is left untouched.
16997
+ const rollbackSnapshotIds = quiesceSnapshotIds;
16998
+ quiesceSnapshotIds = null;
16999
+ if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
17000
+ // Re-open the accept loop. The socket path may still be held by the dead
17001
+ // successor's listen socket; remove it first (same takeover the successor
17002
+ // itself performs), then re-bind.
17003
+ if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
17004
+ // Same owner-only umask as the initial bind, so the socket is private from
17005
+ // creation (no listen()→chmod window for another local user to connect).
17006
+ const rbWin = osPlatform() === 'win32';
17007
+ const rbPrevUmask = rbWin ? null : process.umask(0o177);
17008
+ try {
17009
+ server.listen(socketPath);
17010
+ if (!rbWin) { try { chmodSync(socketPath, 0o600); } catch { /* ignore */ } }
17011
+ dlog('daemon upgrade rollback: control socket re-opened — fleet serving normally');
17012
+ } catch (err) {
17013
+ dlog(`daemon upgrade rollback: failed to re-open the control socket: ${err?.message || err}`);
17014
+ } finally {
17015
+ if (!rbWin) { try { process.umask(rbPrevUmask); } catch { /* ignore */ } }
17016
+ }
17017
+ // #309 review: reconcile the inventory. Respawns were suppressed while
17018
+ // quiesced (expired restart timers and death handlers bailed), so reschedule
17019
+ // every eligible down worker now — preserving stop/config-failure/removal
17020
+ // intent and skipping any with a live process or a pending restart timer.
17021
+ // Membership is reconciled against the quiesce snapshot: a worker ABSENT
17022
+ // from it was removed through the SUCCESSOR while this daemon was quiesced
17023
+ // (the successor owns add/remove during the handoff), so re-adding it here
17024
+ // would resurrect a deliberately removed worker.
17025
+ // The authoritative handoff inventory is what the SUCCESSOR persisted (it
17026
+ // owns add/remove during the handoff): if the state file was last written
17027
+ // by the dead successor, its worker list wins; otherwise fall back to our
17028
+ // quiesce snapshot (the successor never wrote — nothing was removed).
17029
+ let snapshotIds = rollbackSnapshotIds ? new Set(rollbackSnapshotIds) : null;
17030
+ try {
17031
+ const succState = readSupervisorState();
17032
+ if (succState && succState.pid === deadSuccessorPid && Array.isArray(succState.workers)) {
17033
+ snapshotIds = new Set(succState.workers.map((x) => x && x.id).filter(Boolean));
17034
+ }
17035
+ } catch { /* unreadable — keep the snapshot */ }
17036
+ for (const id of snapshotIds ? [...workers.keys()] : []) {
17037
+ if (!snapshotIds.has(id)) {
17038
+ dlog(`daemon upgrade rollback: worker '${id}' is not in the quiesce snapshot (removed through the successor mid-handoff?) — leaving it removed`);
17039
+ workers.delete(id);
17040
+ }
17041
+ }
17042
+ for (const w of [...workers.values()]) {
17043
+ if (w.restartTimer || (w.pid && isPidAlive(w.pid))) continue;
17044
+ if (w.removeOnExit) {
17045
+ workers.delete(w.id);
17046
+ try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
17047
+ dlog(`worker '${w.id}' removed (drained during the aborted upgrade)`);
17048
+ continue;
17049
+ }
17050
+ if (w.stopping || w.configFailed) continue;
17051
+ if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
17052
+ dlog(`daemon upgrade rollback: worker '${w.id}' was down while quiesced — rescheduling`);
17053
+ startWorker(w);
17054
+ }
17055
+ persist();
17056
+ };
17057
+
16039
17058
  const handleRequest = async (req, sock) => {
16040
17059
  const op = req && req.op;
16041
17060
  try {
@@ -16043,8 +17062,30 @@ async function runSupervisorDaemon() {
16043
17062
  case 'status':
16044
17063
  sock.write(encodeFrame(statusFrame(true)));
16045
17064
  break;
17065
+ case 'prepare-upgrade': {
17066
+ // #309: a successor daemon (spawned with NANO_SUPERVISOR_UPGRADE=1)
17067
+ // asks us to quiesce so it can bind the control socket. Refuse while
17068
+ // a worker reload roll or a shutdown owns the fleet — an upgrade on
17069
+ // top of either would strand its bookkeeping. Idempotent: the
17070
+ // successor polls until it observes the quiesced state. Bound to ONE
17071
+ // successor pid (quiesceForUpgrade throws UPGRADE_BOUND for any
17072
+ // other) so two racing successors can never both win the fleet.
17073
+ if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
17074
+ if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a worker reload is in progress — retry the daemon upgrade once it finishes', final: true })); break; }
17075
+ try {
17076
+ await quiesceForUpgrade(Number(req.pid) || null, typeof req.version === 'string' ? req.version : null);
17077
+ } catch (err) {
17078
+ if (err && err.code === 'UPGRADE_BOUND') { sock.write(encodeFrame({ ok: false, error: err.message, final: true })); break; }
17079
+ throw err;
17080
+ }
17081
+ // Report quiesced only AFTER the mutation queue has drained and the
17082
+ // snapshot persisted — the successor binds+adopts off this ack.
17083
+ sock.write(encodeFrame({ ok: true, type: 'prepare-upgrade', quiesced: true, final: true }));
17084
+ break;
17085
+ }
16046
17086
  case 'add': {
16047
17087
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
17088
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16048
17089
  if (!req.profile) { sock.write(encodeFrame({ ok: false, error: 'add requires a profile', final: true })); break; }
16049
17090
  // A supervised worker's name is supplied out-of-band as `req.name`
16050
17091
  // (which the daemon forwards to the child as `nano work … --name`).
@@ -16067,6 +17108,7 @@ async function runSupervisorDaemon() {
16067
17108
  }
16068
17109
  case 'remove': {
16069
17110
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
17111
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16070
17112
  // #202: `req.force === false` drains each worker (finish in-flight jobs
16071
17113
  // then exit); the default remains a fast force-stop (used by reconcile
16072
17114
  // and interactive remove). A drain-remove returns immediately and the
@@ -16082,6 +17124,7 @@ async function runSupervisorDaemon() {
16082
17124
  }
16083
17125
  case 'restart': {
16084
17126
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
17127
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16085
17128
  const restarted = await serializeOp(async () => {
16086
17129
  const ids = resolveTargets(req.target);
16087
17130
  for (const id of ids) await restartWorker(id);
@@ -16098,6 +17141,7 @@ async function runSupervisorDaemon() {
16098
17141
  // and ends with a terminal `reloaded` frame — draining can take a long
16099
17142
  // time, so this MUST be a streaming op, not a one-shot request.
16100
17143
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
17144
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16101
17145
  if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a reload is already in progress', final: true })); break; }
16102
17146
  // Reject reload explicitly on Windows BEFORE draining anything. The
16103
17147
  // rolling reload's graceful drain relies on SIGUSR2 (SUPERVISOR_DRAIN_SIGNAL)
@@ -16179,10 +17223,30 @@ async function runSupervisorDaemon() {
16179
17223
  // #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
16180
17224
  // jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
16181
17225
  // yield jobs) and can also ESCALATE a drain already in progress.
17226
+ // #309: a quiesced (mid-upgrade) daemon must never run the stop path —
17227
+ // it would SIGTERM/SIGUSR2 the very workers the successor has adopted.
17228
+ // The upgrade's own exit path replaces this op for the old daemon.
17229
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — the successor owns the fleet now; stop it instead (its pid is in `supervisor status`)', final: true })); break; }
16182
17230
  const force = !!req.force;
16183
- // Register this client as an attach consumer so it streams drain
16184
- // progress (shrinking in-flight counts), and as a stop client so it
16185
- // gets a terminal `stopped` frame when the daemon has finished.
17231
+ // #309 review: reserve shutdown ownership HERE, synchronously with the
17232
+ // acceptance — not 20ms later when the deferred shutdown() runs. An
17233
+ // unreserved accept lets a prepare-upgrade squeeze into the defer
17234
+ // window and latch `upgrading` first; the shutdown then overtakes the
17235
+ // quiesce and SIGTERMs the very workers the successor is adopting.
17236
+ // Latching now makes the race impossible in either order: a stop
17237
+ // accepted first refuses the upgrade (prepare-upgrade checks
17238
+ // `shuttingDown`), and an upgrade quiesced first refuses the stop
17239
+ // (the `upgrading` check above).
17240
+ //
17241
+ // A SECOND stop while one is already reserved is not a fresh shutdown:
17242
+ // it only ESCALATES a drain (force) or is a no-op. Never re-latch
17243
+ // `draining` here — a force stop arriving during a graceful drain must
17244
+ // leave `draining === true` so shutdown()'s escalation branch fires,
17245
+ // and a graceful stop during a force stop must not flip it back.
17246
+ if (!shutdownStarted && !shuttingDown) {
17247
+ shuttingDown = true;
17248
+ draining = !force;
17249
+ }
16186
17250
  attachClients.add(sock);
16187
17251
  stopClients.add(sock);
16188
17252
  sock.write(encodeFrame(force
@@ -16191,7 +17255,12 @@ async function runSupervisorDaemon() {
16191
17255
  sock.write(encodeFrame(statusFrame(false)));
16192
17256
  // Kick the shutdown asynchronously; don't await it here so the control
16193
17257
  // loop keeps serving (streaming status, accepting a force escalation).
16194
- setTimeout(() => { shutdown('stop', { force }); }, 20);
17258
+ // The 250ms defer lets a `stop --force` escalation (sent right behind
17259
+ // a graceful `stop`, as `stopStream`'s sendAfter does) land and run
17260
+ // shutdown's escalation branch BEFORE the drain's body force-stops on
17261
+ // its own — preserving the pre-#309 escalation contract now that
17262
+ // `shuttingDown` is latched early (see below).
17263
+ setTimeout(() => { shutdown('stop', { force }); }, 250);
16195
17264
  break;
16196
17265
  }
16197
17266
  default:
@@ -16204,7 +17273,11 @@ async function runSupervisorDaemon() {
16204
17273
 
16205
17274
  // Bind the control socket. A stale unix socket file from a crashed daemon
16206
17275
  // would make listen() fail with EADDRINUSE even though nobody is listening;
16207
- // remove it first (we already know no live daemon owns our state).
17276
+ // remove it first. In upgrade mode the old daemon has QUIESCED (its accept
17277
+ // loop is closed) but its listen socket still holds the path until it exits,
17278
+ // so we must unlink + rebind to take over — the same operation, deliberately
17279
+ // ordered AFTER the quiesce handshake so we can never steal the socket from a
17280
+ // daemon that is still accepting (#309).
16208
17281
  if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
16209
17282
 
16210
17283
  const server = createServer((sock) => {
@@ -16270,7 +17343,208 @@ async function runSupervisorDaemon() {
16270
17343
  // and hard crashes still can't run this — the per-worker parent-death
16271
17344
  // watchdog is the backstop for those.
16272
17345
  process.once('SIGHUP', () => shutdown('SIGHUP'));
17346
+ // #309: SIGUSR1 is the upgrade handoff's "the successor owns the fleet — exit
17347
+ // WITHOUT touching the workers" signal. It is ONLY meaningful after a
17348
+ // `prepare-upgrade` quiesce (the successor sends it once it has adopted the
17349
+ // workers and retargeted their watchdogs). A SIGUSR1 outside an upgrade must
17350
+ // NOT exit (that would strand the fleet with no daemon), so it is ignored
17351
+ // unless `upgrading` is latched. Unlike `shutdown`, this path never signals
17352
+ // the workers and never clears the state file — the successor owns both now.
17353
+ process.on('SIGUSR1', () => {
17354
+ if (!upgrading) return;
17355
+ dlog(`daemon upgrade: successor confirmed ownership — old daemon (pid ${process.pid}) exiting; workers keep serving`);
17356
+ upgradeInfo = upgradeInfo ? { ...upgradeInfo, state: 'handed-over' } : upgradeInfo;
17357
+ // #309 review: NO persist here. The quiesce snapshot was this daemon's last
17358
+ // canonical write — the successor has since written its own pid + inventory,
17359
+ // and any write now would clobber it with the exiting predecessor's state.
17360
+ handedOver = true;
17361
+ quiesceSnapshotIds = null;
17362
+ if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
17363
+ if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
17364
+ if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
17365
+ process.exit(0);
17366
+ });
16273
17367
  dlog(`supervisor daemon up (pid ${process.pid}) — control ${socketPath}`);
17368
+
17369
+ // #309: adopt the predecessor's workers. The old daemon quiesced (above) and
17370
+ // its last persist() — run when it latched the upgrade — names every worker
17371
+ // it owned. Each live pid becomes an adopted worker (polled for liveness;
17372
+ // never signaled, so in-flight jobs run undisturbed). Dead pids are dropped
17373
+ // with a log line — the fleet after the swap is exactly what survived it.
17374
+ if (upgradeMode && predecessorPid != null) {
17375
+ const prior = readSupervisorState();
17376
+ const adopted = [];
17377
+ for (const rec of (prior && Array.isArray(prior.workers) ? prior.workers : [])) {
17378
+ const w = adoptWorker(rec);
17379
+ if (w) { adopted.push(w.id); continue; }
17380
+ // #309 review: transfer the FULL inventory, not only live pids. A worker
17381
+ // the predecessor recorded but that is not alive right now falls into one
17382
+ // of three kept categories — dropping it would permanently shrink the
17383
+ // fleet (the predecessor refuses respawns once quiesced):
17384
+ if (!rec || typeof rec.id !== 'string' || !rec.id) { dlog('upgrade: skipping a worker record with no id'); continue; }
17385
+ if (rec.configFailed === true) {
17386
+ // Configuration-failed: retained (NOT restarted) so the operator's
17387
+ // `configFailed` signal survives the swap and a later fix+restart still
17388
+ // targets the same record.
17389
+ workers.set(rec.id, {
17390
+ id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
17391
+ args: Array.isArray(rec.args) ? rec.args.map(String) : [],
17392
+ restarts: Number(rec.restarts) || 0, stopping: rec.stopping === true,
17393
+ removeOnExit: rec.removeOnExit === true, lastExit: rec.lastExit ?? null,
17394
+ logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
17395
+ configFailed: true, adopted: true, pid: null, child: null,
17396
+ startedAt: rec.startedAt || null, spawnedAt: rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now(),
17397
+ });
17398
+ dlog(`upgrade: retained configuration-failed worker '${rec.id}' (not restarted)`);
17399
+ continue;
17400
+ }
17401
+ if (rec.stopping === true || rec.removeOnExit === true) {
17402
+ // Draining for stop/remove: the process is already gone, so the removal
17403
+ // completes here — delete the record rather than resurrect it.
17404
+ dlog(`upgrade: worker '${rec.id}' was draining for removal and is gone — record dropped`);
17405
+ try { rmSync(supervisorWorkerActivityFile(rec.id), { force: true }); } catch { /* best effort */ }
17406
+ continue;
17407
+ }
17408
+ // Down but eligible (a crash/backoff the predecessor had not yet
17409
+ // respawned when it quiesced): reschedule it here so fleet capacity is
17410
+ // preserved across the swap.
17411
+ dlog(`upgrade: worker '${rec.id}' was down at handoff — rescheduling to preserve fleet capacity`);
17412
+ const rw = {
17413
+ id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
17414
+ args: Array.isArray(rec.args) ? rec.args.map(String) : [],
17415
+ restarts: Number(rec.restarts) || 0, stopping: false, removeOnExit: false,
17416
+ lastExit: rec.lastExit ?? null,
17417
+ logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
17418
+ configFailed: false, adopted: true,
17419
+ };
17420
+ workers.set(rw.id, rw);
17421
+ startWorker(rw);
17422
+ rw.spawnedBySuccessor = true; // its watchdog already guards this daemon
17423
+ adopted.push(rw.id);
17424
+ }
17425
+ dlog(`upgrade: adopted ${adopted.length} worker(s) from predecessor pid ${predecessorPid}: ${adopted.join(', ') || '(none)'}`);
17426
+ broadcast({ type: 'event', event: 'daemon-adopted', predecessorPid, workers: adopted });
17427
+
17428
+ // Drive the handoff to completion. The old daemon's accept loop is closed
17429
+ // (quiesce), so it can no longer be asked over the socket — instead the
17430
+ // SUCCESSOR finishes the swap: write each live worker's `.upgrade` sidecar
17431
+ // (the successor pid the worker's SIGUSR2 handler reads), SIGUSR2 the
17432
+ // workers to retarget their watchdogs here, wait (bounded) for their
17433
+ // `watchdogPid` acks, then SIGUSR1 the old daemon so it persists and exits
17434
+ // WITHOUT touching the workers. Runs async so the control loop serves
17435
+ // immediately; the `upgrade` status block stays 'adopting' until the old
17436
+ // daemon is gone.
17437
+ (async () => {
17438
+ const myPid = process.pid;
17439
+ // Every live PREDECESSOR-owned worker stays pending until it has either
17440
+ // acked the retarget (`watchdogPid === myPid`) or exited (#309 review).
17441
+ // A worker still booting has Node's DEFAULT SIGUSR2 action (terminate)
17442
+ // until its handler is up, so it is signalled only once its activity
17443
+ // marker proves readiness (finite `readyAt` + matching pid) — but it is
17444
+ // NEVER dropped from the wait set, so the predecessor (its watchdog
17445
+ // target) is not released under it. Workers this successor spawned
17446
+ // itself already guard it and need no retarget.
17447
+ const pending = retargetPending;
17448
+ for (const id of adopted) {
17449
+ const w = workers.get(id);
17450
+ if (!w || w.spawnedBySuccessor || !w.pid || !isPidAlive(w.pid)) continue;
17451
+ if (w.stopping) {
17452
+ // #309 review: already draining (stop/remove) under the predecessor.
17453
+ // Its one-shot SIGUSR2 handler was consumed by that drain, so another
17454
+ // SIGUSR2 would hit Node's default action and kill its in-flight work.
17455
+ // Never write a sidecar or signal it — just keep it pending (its
17456
+ // watchdog still guards the predecessor) until it exits.
17457
+ pending.set(id, { pid: w.pid, signalled: true, drainOnly: true });
17458
+ continue;
17459
+ }
17460
+ const actFile = w.activityFile || supervisorWorkerActivityFile(w.id);
17461
+ try { writeFileSync(`${actFile}.upgrade`, String(myPid), { mode: 0o600 }); } catch { /* best effort */ }
17462
+ pending.set(id, { pid: w.pid, signalled: false });
17463
+ }
17464
+ // Bounded wait first; past the deadline report INCOMPLETE but KEEP
17465
+ // monitoring (#309 review) so a late ack / drain still releases the
17466
+ // predecessor and its keepalive — a retried upgrade would target this
17467
+ // successor, never the retained predecessor.
17468
+ const ackDeadline = Date.now() + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS;
17469
+ let reportedIncomplete = false;
17470
+ while (pending.size > 0) {
17471
+ if (!isPidAlive(predecessorPid)) break; // predecessor gone — nothing left to release
17472
+ for (const [id, p] of [...pending]) {
17473
+ const w = workers.get(id);
17474
+ if (!w || w.pid !== p.pid || !isPidAlive(p.pid)) { pending.delete(id); continue; }
17475
+ const act = readWorkerActivity(id);
17476
+ if (p.drainOnly) {
17477
+ // Draining, not retargeting: a deferred drain request (below) is
17478
+ // delivered once the handler is provably installed; then just wait
17479
+ // for the exit.
17480
+ if (p.drainDeferred && drainSignalReadyFor(act, p.pid)) {
17481
+ p.drainDeferred = false;
17482
+ try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
17483
+ }
17484
+ continue;
17485
+ }
17486
+ if (act && act.pid === p.pid && act.watchdogPid === myPid) {
17487
+ pending.delete(id);
17488
+ // Retarget landed and the handler re-armed — now deliver any drain
17489
+ // that was requested while the retarget was outstanding.
17490
+ if (p.drainDeferred) { try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
17491
+ continue;
17492
+ }
17493
+ if (!p.signalled && drainSignalReadyFor(act, p.pid)) {
17494
+ p.signalled = true;
17495
+ try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
17496
+ }
17497
+ }
17498
+ if (pending.size === 0) break;
17499
+ if (!reportedIncomplete && Date.now() >= ackDeadline) {
17500
+ reportedIncomplete = true;
17501
+ dlog(`upgrade: ${pending.size} worker(s) have not acked the watchdog retarget within ${SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS}ms (${[...pending.keys()].join(', ')}) — NOT releasing the predecessor yet; handoff INCOMPLETE, still monitoring until they ack or exit`);
17502
+ upgradeInfo = { state: 'incomplete', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, unacked: [...pending.keys()], sinceEpochMs: upgradeStartedMs };
17503
+ persist();
17504
+ }
17505
+ await new Promise((r) => setTimeout(r, reportedIncomplete ? SUPERVISOR_ADOPT_POLL_MS : 100));
17506
+ }
17507
+ // Predecessor gone with entries still pending: deliver deferred drains
17508
+ // that are safe now (handler installed and no retarget signal in flight
17509
+ // that the drain could be mistaken for). A drain deferred behind an
17510
+ // un-acked retarget is logged — signalling now could hit a consumed
17511
+ // one-shot handler and kill in-flight work.
17512
+ for (const [id, p] of pending) {
17513
+ if (!p.drainDeferred || !isPidAlive(p.pid)) continue;
17514
+ if (p.drainOnly && drainSignalReadyFor(readWorkerActivity(id), p.pid)) {
17515
+ try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
17516
+ } else {
17517
+ dlog(`upgrade: deferred drain for worker '${id}' not delivered (retarget unacknowledged or handler not ready) — re-issue stop/remove once it settles`);
17518
+ }
17519
+ }
17520
+ pending.clear();
17521
+ // Release the old daemon: it persists a final snapshot and exits 0 without
17522
+ // touching the workers. If it is already gone (crashed mid-handoff), the
17523
+ // workers are still adopted here — nothing more to do.
17524
+ if (isPidAlive(predecessorPid)) {
17525
+ try { process.kill(predecessorPid, 'SIGUSR1'); } catch { /* already gone */ }
17526
+ }
17527
+ dlog(`upgrade: handoff complete — successor pid ${myPid} owns the fleet (${daemonVersion})`);
17528
+ })().catch((err) => dlog(`upgrade handoff error: ${err?.message || err}`));
17529
+
17530
+ // Clear the status `upgrade` block once the predecessor has actually exited
17531
+ // — the roll is complete from then on, and a quiet fleet's status should
17532
+ // not carry a stale 'adopting' line forever. Unref'd: it never keeps the
17533
+ // daemon alive.
17534
+ const clearTimer = setInterval(() => {
17535
+ if (!isPidAlive(predecessorPid)) {
17536
+ try { clearInterval(clearTimer); } catch { /* ignore */ }
17537
+ predecessorPid = null;
17538
+ // #309 review: a handoff reported INCOMPLETE that later completed (late
17539
+ // acks/drains released the predecessor) must not keep reporting
17540
+ // 'incomplete' forever — currentUpgradeBlock() returns upgradeInfo first.
17541
+ if (upgradeInfo && upgradeInfo.state === 'incomplete') upgradeInfo = null;
17542
+ persist();
17543
+ }
17544
+ }, SUPERVISOR_ADOPT_POLL_MS);
17545
+ if (typeof clearTimer.unref === 'function') clearTimer.unref();
17546
+ }
17547
+
16274
17548
  persist();
16275
17549
 
16276
17550
  // Live-view refresh: periodically re-sample worker activity and push a fresh
@@ -16663,9 +17937,9 @@ async function streamSupervisorReload(socketPath, req, logger, { label = 'fleet'
16663
17937
  * Hot-adopt new plugin code into the running fleet with zero downtime: roll
16664
17938
  * through the target workers, gracefully draining (finish in-flight jobs) and
16665
17939
  * respawning each so the new child re-reads the updated `c8ctl-plugin.js` from
16666
- * disk. Defaults to the whole fleet. Note this adopts new WORKER code only; the
16667
- * supervisor daemon itself keeps running its startup code until a full restart
16668
- * (`supervisor stop && supervisor start`).
17940
+ * disk. Defaults to the whole fleet. Note this adopts new WORKER code only;
17941
+ * the supervisor daemon itself is rolled separately, without stopping the
17942
+ * fleet, by `supervisor upgrade` (issue #309).
16669
17943
  */
16670
17944
  async function supervisorReloadCmd(req) {
16671
17945
  const logger = getLogger();
@@ -16680,6 +17954,116 @@ async function supervisorReloadCmd(req) {
16680
17954
  if (outcome === 'error') process.exit(1);
16681
17955
  }
16682
17956
 
17957
+ /**
17958
+ * Rolling hot-upgrade of the supervisor DAEMON process (issue #309) — the
17959
+ * daemon-level counterpart of `supervisor reload` (which rolls only workers).
17960
+ * The daemon keeps the module graph it imported at start, so adopting new
17961
+ * SUPERVISOR-side code used to require `supervisor stop && start` — a fleet
17962
+ * outage (or, with --force, killed in-flight jobs). This command swaps the
17963
+ * parent process instead:
17964
+ *
17965
+ * 1. spawn a SUCCESSOR daemon (detached, NANO_SUPERVISOR_UPGRADE=1) running
17966
+ * the on-disk — post-`nano update` — plugin code;
17967
+ * 2. the successor asks the old daemon to QUIESCE (`prepare-upgrade`: accept
17968
+ * loop closed, mutations refused), then binds the canonical control
17969
+ * socket and ADOPTS the old daemon's workers from the persisted state
17970
+ * (pid-polled; the workers are never signaled, so in-flight jobs run
17971
+ * undisturbed);
17972
+ * 3. the successor then finishes the swap itself: it writes each worker's
17973
+ * `<activityFile>.upgrade` sidecar, SIGUSR2s the workers to retarget their
17974
+ * parent-death watchdogs at it, waits (bounded) for the `watchdogPid`
17975
+ * acks, and SIGUSR1s the old daemon — which persists a final snapshot and
17976
+ * exits 0 WITHOUT touching the workers.
17977
+ *
17978
+ * Workers never die mid-job; only the parent is swapped. A successor that
17979
+ * cannot reach a quiesced old daemon exits 1 and leaves the fleet untouched.
17980
+ * This client just spawns the successor and waits for the swap to land; Ctrl-C
17981
+ * detaches — the handoff completes in the background.
17982
+ */
17983
+ async function supervisorUpgradeCmd() {
17984
+ const logger = getLogger();
17985
+ if (osPlatform() === 'win32') {
17986
+ logger.error('supervisor upgrade is not supported on Windows (its worker watchdog retarget relies on SIGUSR2, which Windows cannot deliver) — use `supervisor stop` + `start` to adopt new supervisor code.');
17987
+ process.exit(1);
17988
+ }
17989
+ const running = await liveSupervisor();
17990
+ if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
17991
+ const socketPath = running.socket || getSupervisorSocketPath();
17992
+ const oldPid = running.pid;
17993
+ const fromVersion = running.version || null;
17994
+ const toVersion = (() => { try { return pluginPackage().version; } catch { return null; } })();
17995
+
17996
+ // 1. Spawn the successor. NANO_SUPERVISOR_UPGRADE=1 puts its boot into
17997
+ // upgrade mode (quiesce handshake → bind → adopt → drive the handoff). We
17998
+ // deliberately do NOT clear the state file first — the successor reads the
17999
+ // old daemon's worker list from it. Ctrl-C detaches the waiter below; the
18000
+ // successor is detached and completes the swap regardless.
18001
+ const { exec, entry } = c8ctlInvocation();
18002
+ mkdirSync(getSupervisorLogDir(), { recursive: true });
18003
+ const logFile = supervisorDaemonLogFile();
18004
+ let fd;
18005
+ try { fd = openSync(logFile, 'a'); } catch { fd = 'ignore'; }
18006
+ const child = spawn(exec, [entry, 'nano', 'supervisor', '__daemon'], {
18007
+ env: { ...process.env, NANO_SUPERVISOR_UPGRADE: '1' },
18008
+ detached: true,
18009
+ stdio: ['ignore', fd, fd],
18010
+ });
18011
+ child.unref();
18012
+ if (typeof fd === 'number') { try { closeSync(fd); } catch { /* ignore */ } }
18013
+ if (typeof child.pid !== 'number') { logger.error('Failed to spawn the successor supervisor daemon.'); process.exit(1); }
18014
+ const successorPid = child.pid;
18015
+ logger.info(`Upgrading the supervisor daemon (pid ${oldPid}${fromVersion ? `, v${fromVersion}` : ''}) → successor pid ${successorPid}${toVersion ? ` (v${toVersion})` : ''}. Workers keep serving; only the parent process is swapped. Press Ctrl-C to detach.`);
18016
+ // #309 review (detached-breaks-supervision): the successor is spawned DETACHED
18017
+ // in this session, not under launchd/systemd. If a service unit is installed,
18018
+ // warn the operator that the new daemon is no longer service-supervised (and
18019
+ // that the old unit's clean exit 0 won't restart it — the generated units use
18020
+ // crash-only policies), and tell them how to re-establish supervision.
18021
+ try {
18022
+ if (supervisorServiceInstalled()) {
18023
+ logger.warn('A supervisor service is installed, but the upgraded daemon runs DETACHED in this session — it is no longer under launchd/systemd supervision (and the old unit\'s clean exit will not auto-restart it). After the upgrade, run `c8ctl nano supervisor install` (or `stop` + `start`) to bring the new-code daemon back under the service.');
18024
+ }
18025
+ } catch { /* best effort — never block the upgrade on a detection hiccup */ }
18026
+
18027
+ // 2. Wait for the swap to land: the successor takes the socket (status names
18028
+ // its pid) and the old daemon exits (released by the successor's SIGUSR1
18029
+ // once the workers' watchdogs are retargeted). Ctrl-C detaches — the
18030
+ // handoff completes in the background.
18031
+ let detached = false;
18032
+ const onSigint = () => { detached = true; };
18033
+ process.on('SIGINT', onSigint);
18034
+ const deadline = Date.now() + SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS + 10_000;
18035
+ try {
18036
+ for (;;) {
18037
+ if (detached) {
18038
+ logger.info('Detached — the daemon upgrade completes in the background. Rerun `nano supervisor status` to confirm.');
18039
+ return;
18040
+ }
18041
+ if (!isPidAlive(successorPid)) {
18042
+ logger.error(`The successor daemon (pid ${successorPid}) died during the upgrade — the old daemon (pid ${oldPid}) may still be in place. See ${logFile}.`);
18043
+ process.exit(1);
18044
+ }
18045
+ let successorServing = false;
18046
+ try {
18047
+ const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
18048
+ successorServing = !!(res && res.ok && res.daemon && res.daemon.pid === successorPid);
18049
+ } catch { /* socket mid-handoff — retry */ }
18050
+ const oldGone = !isPidAlive(oldPid);
18051
+ if (successorServing && oldGone) {
18052
+ logger.info(`Supervisor daemon upgraded${fromVersion || toVersion ? ` (${fromVersion ?? '?'} → ${toVersion ?? '?'})` : ''} — successor pid ${successorPid} owns the fleet; no worker was stopped.`);
18053
+ return;
18054
+ }
18055
+ if (Date.now() >= deadline) {
18056
+ if (!successorServing) logger.warn(`The successor daemon (pid ${successorPid}) did not take the control socket in time — the old daemon (pid ${oldPid}) may still own the fleet. Check \`nano supervisor status\` and ${logFile}.`);
18057
+ else logger.warn(`The old daemon (pid ${oldPid}) has not exited yet — the watchdog handoff is still completing. Check \`nano supervisor status\` in a moment.`);
18058
+ process.exit(1);
18059
+ }
18060
+ await new Promise((r) => setTimeout(r, 150));
18061
+ }
18062
+ } finally {
18063
+ try { process.removeListener('SIGINT', onSigint); } catch { /* ignore */ }
18064
+ }
18065
+ }
18066
+
16683
18067
  /**
16684
18068
  * Re-read every supervised worker's cached gh auth token WITHOUT draining
16685
18069
  * in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
@@ -16787,7 +18171,15 @@ async function supervisorStopCmd(force = false) {
16787
18171
  const { frames, rest } = decodeFrames(buf);
16788
18172
  buf = rest;
16789
18173
  for (const frame of frames) {
16790
- if (frame && frame.type === 'stopping') {
18174
+ // #309 review: an ok:false terminal frame is a REFUSAL, not a stop —
18175
+ // the daemon rejected the op (a daemon upgrade owns the fleet; the
18176
+ // successor holds the socket now). Treating it as a successful stop
18177
+ // would wait indefinitely for the wrong pid and, with --force,
18178
+ // SIGKILL the quiesced predecessor's process group — including the
18179
+ // workers the successor adopted — then erase the successor's state.
18180
+ if (frame && frame.ok === false) {
18181
+ finish({ error: String(frame.error || 'the supervisor refused the stop request') });
18182
+ } else if (frame && frame.type === 'stopping') {
16791
18183
  logger.info('Aborting in-flight work — killing harnesses and yielding jobs for retry...');
16792
18184
  } else if (frame && frame.event === 'draining-escalated') {
16793
18185
  logger.info('Escalating to --force — aborting in-flight work...');
@@ -16813,6 +18205,15 @@ async function supervisorStopCmd(force = false) {
16813
18205
 
16814
18206
  if (outcome === 'detached') return;
16815
18207
 
18208
+ // A refused stop (ok:false) is an ERROR outcome: surface it and exit before
18209
+ // any pid wait, force escalation, or state cleanup — the daemon that refused
18210
+ // (e.g. a quiesced predecessor mid-upgrade) is not ours to reap, and the
18211
+ // state file now names the successor that owns the fleet.
18212
+ if (outcome && typeof outcome === 'object' && outcome.error) {
18213
+ logger.error(`Supervisor stop refused: ${outcome.error}`);
18214
+ process.exit(1);
18215
+ }
18216
+
16816
18217
  if (outcome === 'unreachable') {
16817
18218
  if (force) {
16818
18219
  // Socket unreachable — fall back to signalling the daemon pid directly.
@@ -16828,6 +18229,15 @@ async function supervisorStopCmd(force = false) {
16828
18229
  // backstop so a wedged daemon/worker group can't linger. The window must cover
16829
18230
  // the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
16830
18231
  // group while a worker is still promoting its durable recovery ref (#295).
18232
+ // #309 review: if the daemon we asked is mid-upgrade (quiesced), the pid we
18233
+ // captured is the PREDECESSOR's — it exits on the successor's handover, not on
18234
+ // our refused/naive stop. Say so, and never SIGKILL its process group (the
18235
+ // successor's adopted workers share it) or clear the successor's state.
18236
+ if (running.upgrade && typeof running.upgrade === 'object') {
18237
+ const successorPid = Number(running.upgrade.successorPid);
18238
+ logger.warn(`The supervisor is mid-upgrade (handing the fleet to successor pid ${Number.isInteger(successorPid) && successorPid > 0 ? successorPid : 'unknown'}) — stop the successor instead; this daemon exits when the handoff completes.`);
18239
+ return;
18240
+ }
16831
18241
  if (force) {
16832
18242
  const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
16833
18243
  while (Date.now() < deadline) {
@@ -17534,8 +18944,14 @@ async function supervisorCommand(req, flags, ctx) {
17534
18944
  await supervisorRestartCmd(req);
17535
18945
  return;
17536
18946
  case 'reload':
18947
+ // `reload --daemon` is the daemon-upgrade alias (issue #309): the plain
18948
+ // op rolls WORKER code; this rolls the daemon process itself.
18949
+ if (flags?.daemon) { await supervisorUpgradeCmd(); return; }
17537
18950
  await supervisorReloadCmd(req);
17538
18951
  return;
18952
+ case 'upgrade':
18953
+ await supervisorUpgradeCmd();
18954
+ return;
17539
18955
  case 'reload-credentials':
17540
18956
  await supervisorReloadCredentialsCmd(req);
17541
18957
  return;
@@ -17547,7 +18963,7 @@ async function supervisorCommand(req, flags, ctx) {
17547
18963
  supervisorLogsCmd(req);
17548
18964
  return;
17549
18965
  default:
17550
- getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
18966
+ getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|upgrade|reload-credentials|stop|logs|attach`);
17551
18967
  process.exit(1);
17552
18968
  }
17553
18969
  }
@@ -20318,6 +21734,9 @@ export {
20318
21734
  spawnCapturePty,
20319
21735
  spawnCaptureAcp,
20320
21736
  sampleSubtreeCpu,
21737
+ sampleWorkspaceCpu,
21738
+ sampleWorkspaceCpuProc,
21739
+ sampleWorkspaceCpuLsof,
20321
21740
  createIdleLivenessMonitor,
20322
21741
  resolveLivenessOverrides,
20323
21742
  parsePsTime,
@@ -20446,6 +21865,7 @@ export {
20446
21865
  summarizeSupervisorWorker,
20447
21866
  formatSupervisorStatus,
20448
21867
  statusFromState,
21868
+ stateFromStatus,
20449
21869
  supervisorDaemonDescriptor,
20450
21870
  formatSupervisorLogsLines,
20451
21871
  reageSupervisorStatus,
@@ -20462,6 +21882,7 @@ export {
20462
21882
  normalizeAgenticMessage,
20463
21883
  buildActivityPayload,
20464
21884
  activityMarkerReadyFor,
21885
+ drainSignalReadyFor,
20465
21886
  reloadCredentialsReady,
20466
21887
  waitForChildExit,
20467
21888
  supervisorWorkerActivityFile,
@@ -20474,6 +21895,7 @@ export {
20474
21895
  supervisorStartCmd,
20475
21896
  supervisorAddCmd,
20476
21897
  supervisorReloadCmd,
21898
+ supervisorUpgradeCmd,
20477
21899
  supervisorReloadCredentialsCmd,
20478
21900
  workforceReloadCmd,
20479
21901
  runningSupervisor,
@@ -20582,7 +22004,8 @@ export const metadata = {
20582
22004
  { command: 'c8ctl nano supervisor add decider', description: 'Add a supervised worker (forwarding work flags) to the running supervisor' },
20583
22005
  { command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
20584
22006
  { command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
20585
- { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
22007
+ { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only — use `supervisor upgrade` for the daemon)' },
22008
+ { command: 'c8ctl nano supervisor upgrade', description: 'Rolling hot-upgrade of the supervisor DAEMON itself (alias: `reload --daemon`): a successor daemon on the new code takes the control socket and ADOPTS the running workers (no drain — in-flight jobs never stop), then the old daemon exits. Status shows `upgrade: rolling <from> → <to>` while it runs' },
20586
22009
  { command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
20587
22010
  { command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
20588
22011
  { command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
@@ -20664,6 +22087,7 @@ export const commands = {
20664
22087
  worker: { type: 'string', multiple: true, description: 'supervisor start: profile to launch as a supervised worker (repeatable)' },
20665
22088
  instances: { type: 'string', description: `supervisor add / workforce add: spawn/compose N distinct instances of the profile in one call (default 1, max ${MAX_ADD_INSTANCES}; for supervisor add cannot combine with --name)` },
20666
22089
  attach: { type: 'boolean', description: 'supervisor start: attach the interactive console after starting the daemon' },
22090
+ daemon: { type: 'boolean', description: 'supervisor reload: roll the DAEMON process itself onto the on-disk code (alias of `supervisor upgrade`) instead of rolling worker code' },
20667
22091
  manifest: { type: 'string', description: `workforce: manifest name to operate on (default ${DEFAULT_WORKFORCE_MANIFEST}); each subcommand reads/writes <stateHome>/workforce/<name>.json. Renamed from --profile (which now collides with c8ctl's global connection-profile flag).` },
20668
22092
  roles: { type: 'string', description: 'workforce add: comma-separated role list for the entry (→ --job-type <rank>:<role> at start); mutually exclusive with --auto' },
20669
22093
  },