c8ctl-plugin-nano 1.71.0 → 1.73.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +76 -9
  2. package/c8ctl-plugin.js +1179 -31
  3. package/package.json +8 -8
package/c8ctl-plugin.js CHANGED
@@ -295,6 +295,40 @@ const SUPERVISOR_RELOAD_READY_TIMEOUT_MS = Math.max(
295
295
  0,
296
296
  Number.parseInt(process.env.NANO_SUPERVISOR_RELOAD_READY_TIMEOUT_MS ?? '', 10) || 30_000,
297
297
  );
298
+ // Rolling daemon hot-upgrade (issue #309). A `supervisor upgrade` hands the
299
+ // fleet to a SUCCESSOR daemon process running the on-disk (new) plugin code:
300
+ // the old daemon quiesces (stops accepting control connections + stops
301
+ // restarting workers), asks each worker to re-arm its parent-death watchdog at
302
+ // the successor's pid (SIGUSR2 + NANO_SUPERVISOR_UPGRADE_PID — a retarget, NOT
303
+ // a drain), persists a final state snapshot, and exits WITHOUT touching the
304
+ // workers. The successor binds the canonical control socket and adopts the
305
+ // still-running worker pids. Workers never die mid-job; only the parent swaps.
306
+ //
307
+ // How long the old daemon waits for workers to acknowledge the watchdog
308
+ // retarget (by stamping `watchdogPid` on their activity marker) before it exits
309
+ // anyway. Bounded so a wedged/older worker can't stall the upgrade forever —
310
+ // the successor adopts every live pid regardless, so an un-acked worker is
311
+ // still parented (its watchdog self-reaps only if the successor ALSO dies).
312
+ const SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS = Math.max(
313
+ 0,
314
+ Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS ?? '', 10) || 15_000,
315
+ );
316
+ // How long a freshly spawned successor waits for the old daemon to quiesce
317
+ // before giving up (it must never bind the socket while the old daemon still
318
+ // accepts on it). Generous: quiesce is prompt (server.close + a latch), but a
319
+ // loaded machine can delay the old daemon's socket loop.
320
+ const SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS = Math.max(
321
+ 0,
322
+ Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS ?? '', 10) || 30_000,
323
+ );
324
+ // How often the successor polls an adopted worker's pid for liveness. An
325
+ // adopted worker has no ChildProcess handle (the successor never spawned it),
326
+ // so there is no 'exit' event — this poll is the death detector that feeds the
327
+ // same restart-backoff path a spawned child's exit would.
328
+ const SUPERVISOR_ADOPT_POLL_MS = Math.max(
329
+ 100,
330
+ Number.parseInt(process.env.NANO_SUPERVISOR_ADOPT_POLL_MS ?? '', 10) || 2_000,
331
+ );
298
332
  // Upper bound on one `--auto` engine-read reconcile (enumerate deployed
299
333
  // definitions + fetch each BPMN). A read that stalls past this is treated as a
300
334
  // transient failure so the running poller set is KEPT and, crucially, shutdown
@@ -474,6 +508,19 @@ function activityMarkerReadyFor(act, pid) {
474
508
  return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
475
509
  }
476
510
 
511
+ /**
512
+ * Whether a worker's SIGUSR2 (drain/retarget) handler is provably installed
513
+ * (#309 review): its marker belongs to `pid` AND carries a finite
514
+ * `signalReadyAt` (stamped right after the handler is armed, independent of
515
+ * leasing). Falls back to `readyAt` for a worker that predates the field (its
516
+ * handler is armed before its first activation). Signalling earlier would hit
517
+ * Node's DEFAULT SIGUSR2 action and terminate the worker. Pure for unit tests.
518
+ */
519
+ function drainSignalReadyFor(act, pid) {
520
+ if (!act || act.pid !== pid) return false;
521
+ return Number.isFinite(act.signalReadyAt) || Number.isFinite(act.readyAt);
522
+ }
523
+
477
524
  /**
478
525
  * Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
479
526
  * All three must hold (#293): the worker exists, its PID is alive, AND its
@@ -11281,9 +11328,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
11281
11328
  * `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
11282
11329
  * longer running here, so it rides the marker as a flagged, observational entry
11283
11330
  * without making an idle worker look busy or inflating the drain's in-flight count.
11284
- * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
11331
+ * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null), watchdogPid:(number|null|undefined), signalReadyAt?:(number|null) }} fields
11285
11332
  */
11286
- function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
11333
+ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth, watchdogPid, signalReadyAt }) {
11287
11334
  const jobList = Array.isArray(jobs) ? jobs : [];
11288
11335
  return {
11289
11336
  pid,
@@ -11296,11 +11343,23 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt,
11296
11343
  // SDK client and begun leasing), the producer stamps this; null until then.
11297
11344
  // The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
11298
11345
  readyAt: readyAt ?? null,
11346
+ // #309 review: stamped once this worker's SIGUSR2 handler is INSTALLED —
11347
+ // independent of leasing readiness (`readyAt` needs a nonempty poll set, so
11348
+ // a healthy `--auto` worker with no deployed types never stamps it). The
11349
+ // daemon-upgrade handoff gates its retarget SIGUSR2 on THIS, not readyAt.
11350
+ signalReadyAt: signalReadyAt ?? null,
11299
11351
  // #293: the cached gh auth token's observable identity — { present,
11300
11352
  // fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
11301
11353
  // `supervisor status` can show the credential's age/scopes per worker. Null
11302
11354
  // on a worker that has not primed the cache yet (or predates the field).
11303
11355
  ghAuth: ghAuth ?? null,
11356
+ // #309: the daemon pid this worker's parent-death watchdog currently guards.
11357
+ // A rolling daemon upgrade SIGUSR2s each worker to RETARGET that watchdog at
11358
+ // the successor daemon before the old one exits; the old daemon waits
11359
+ // (bounded) for this stamp to name the successor — the ack that the worker
11360
+ // survives the handoff — so it never exits while a worker would still read
11361
+ // the (expected) reparent-to-init as an orphan event and self-reap.
11362
+ watchdogPid: Number.isInteger(watchdogPid) ? watchdogPid : null,
11304
11363
  };
11305
11364
  }
11306
11365
 
@@ -12118,9 +12177,13 @@ async function workAgent(req, flags, ctx) {
12118
12177
  // this worker would otherwise idle forever as an orphan reparented to init.
12119
12178
  // The watchdog makes the child self-exit the moment it is reparented. A
12120
12179
  // standalone `nano work` (no activity file) keeps classic nohup semantics.
12180
+ // The handle is retained so a rolling daemon upgrade (#309) can RETARGET the
12181
+ // same installation at the successor daemon's pid instead of self-reaping
12182
+ // when the old daemon hands over and exits (see the SIGUSR2 handler below).
12183
+ let parentWatchdog = null;
12121
12184
  if (activityFile) {
12122
12185
  const daemonPid = Number.parseInt(process.env.NANO_SUPERVISOR_DAEMON_PID ?? '', 10);
12123
- installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
12186
+ parentWatchdog = installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
12124
12187
  }
12125
12188
  const activeJobs = new Map(); // jobKey -> { type, since (ms epoch) }
12126
12189
  // #254: settlement-pending ghosts — jobKey -> { type, since, phase, leaseToken }.
@@ -12267,6 +12330,9 @@ async function workAgent(req, flags, ctx) {
12267
12330
  // waits for this stamp before draining the next worker so a bare spawn/PID is
12268
12331
  // never mistaken for a serving replacement.
12269
12332
  let readyAt = null;
12333
+ // #309 review: set once the SIGUSR2 (drain/retarget) handler is installed.
12334
+ let signalReadyAt = null;
12335
+ const markSignalReady = () => { signalReadyAt = Date.now(); };
12270
12336
  const writeActivity = () => {
12271
12337
  if (!activityFile) return;
12272
12338
  const jobs = [...activeJobs.entries()].map(([key, v]) => ({ key, type: v.type, since: v.since }));
@@ -12291,7 +12357,7 @@ async function workAgent(req, flags, ctx) {
12291
12357
  // #293: publish the cached gh token's identity (fingerprint/scopes/age —
12292
12358
  // never the value) so `supervisor status` can answer "is this worker holding
12293
12359
  // a stale token?" in one line.
12294
- const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
12360
+ const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, signalReadyAt, ghAuth: ghAuthTokenInfo(), watchdogPid: parentWatchdog && typeof parentWatchdog.watchedPid === 'function' ? parentWatchdog.watchedPid() : null });
12295
12361
  const tmp = `${activityFile}.${process.pid}.tmp`;
12296
12362
  try {
12297
12363
  mkdirSync(dirname(activityFile), { recursive: true });
@@ -14330,7 +14396,62 @@ async function workAgent(req, flags, ctx) {
14330
14396
  setTimeout(tick, 200);
14331
14397
  };
14332
14398
 
14333
- process.once(SUPERVISOR_DRAIN_SIGNAL, () => { gracefulDrain(SUPERVISOR_DRAIN_SIGNAL); });
14399
+ // #309: the SIGUSR2 handler is a named, re-armable function. The legacy path
14400
+ // (graceful drain) fires once and the worker exits, so `once` semantics
14401
+ // suffice there; but the daemon-upgrade path (below) does NOT exit the
14402
+ // worker, so it re-arms the handler for a later drain/upgrade.
14403
+ const onDrainSignal = () => {
14404
+ // #309 rolling daemon upgrade: the daemon reuses SIGUSR2 as its
14405
+ // per-worker signal, but when it names a successor daemon pid this is a
14406
+ // watchdog RETARGET, not a drain — the old daemon is handing the fleet to
14407
+ // a successor running the new plugin code and this worker must KEEP
14408
+ // SERVING (jobs in flight are never touched; only the parent process is
14409
+ // swapped). The successor pid arrives two ways, in priority order:
14410
+ // 1. `<activityFile>.upgrade` — written by the old daemon just before
14411
+ // it SIGUSR2s (authoritative: the daemon learns the successor pid
14412
+ // only at handoff time, after this worker was spawned, so it cannot
14413
+ // have ridden this worker's env);
14414
+ // 2. NANO_SUPERVISOR_UPGRADE_PID — covers a worker spawned mid-upgrade.
14415
+ // Re-arm the parent-death watchdog at the successor so the old daemon's
14416
+ // (expected) exit — which reparents this worker to init — is not read as
14417
+ // an orphan event, then republish the activity marker: the old daemon
14418
+ // waits (bounded) for the marker's `watchdogPid` to name the successor
14419
+ // before it exits, so the handoff never strands a worker that would
14420
+ // self-reap mid-upgrade.
14421
+ let upgradePid = Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
14422
+ if (activityFile) {
14423
+ try {
14424
+ const raw = readFileSync(`${activityFile}.upgrade`, 'utf8').trim();
14425
+ const fromFile = Number.parseInt(raw, 10);
14426
+ if (Number.isInteger(fromFile) && fromFile > 0) upgradePid = fromFile;
14427
+ } catch { /* no upgrade file — fall back to the env var */ }
14428
+ }
14429
+ if (Number.isInteger(upgradePid) && upgradePid > 0 && parentWatchdog && typeof parentWatchdog.retarget === 'function') {
14430
+ const ok = parentWatchdog.retarget(upgradePid);
14431
+ logger.info(`Received ${SUPERVISOR_DRAIN_SIGNAL} — daemon upgrade in progress: parent-death watchdog ${ok ? `retargeted to successor daemon pid ${upgradePid}` : `could NOT be retargeted (invalid pid)`}; this worker keeps serving.`);
14432
+ // Consume the retarget request: delete the `.upgrade` sidecar so a LATER
14433
+ // SIGUSR2 (a genuine drain, or a later upgrade that has not written a
14434
+ // fresh sidecar yet) is not misread as another retarget to this same
14435
+ // successor. Without this the sidecar is sticky — every later SIGUSR2
14436
+ // would retarget to the same (possibly long-gone) pid instead of
14437
+ // draining, breaking graceful stop / drain-remove / worker reload after
14438
+ // an upgrade (#309 review). Best-effort: a leftover only matters if it
14439
+ // names a live pid, and the successor also clears it on adopt.
14440
+ if (activityFile) { try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ } }
14441
+ try { writeActivityRef?.(); } catch { /* best effort — the ack stamp is advisory */ }
14442
+ // Re-arm: an upgrade that aborts (successor died before binding) must
14443
+ // leave this worker drainable by the surviving old daemon, and a LATER
14444
+ // upgrade/drain needs the handler again. (The drain path never re-arms —
14445
+ // the worker is on its way out.)
14446
+ process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
14447
+ return;
14448
+ }
14449
+ gracefulDrain(SUPERVISOR_DRAIN_SIGNAL);
14450
+ };
14451
+ process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
14452
+ // #309 review: publish that the handler is installed, so an upgrade handoff
14453
+ // can retarget this worker even before (or without ever) leasing.
14454
+ try { markSignalReady(); writeActivityRef?.(); } catch { /* best effort */ }
14334
14455
  process.once('SIGINT', () => { forceAbort('SIGINT'); });
14335
14456
  process.once('SIGTERM', () => { forceAbort('SIGTERM'); });
14336
14457
  });
@@ -15128,7 +15249,36 @@ function formatSupervisorStatus(status) {
15128
15249
  const onDiskVersion = status.pluginVersion
15129
15250
  ?? (() => { try { return pluginPackage().version; } catch { return null; } })();
15130
15251
  if (onDiskVersion && d.version && onDiskVersion !== d.version) {
15131
- lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code; restart the daemon for new supervisor code)`);
15252
+ lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code, and \`c8ctl nano supervisor upgrade\` to roll the daemon itself onto the new code)`);
15253
+ }
15254
+ // #309: while a rolling daemon upgrade is in flight the status frame (and the
15255
+ // persisted-state fallback) carries an `upgrade` block — render it as the
15256
+ // daemon-roll line, mirroring the worker-roll visibility (`rolling <from> →
15257
+ // <to>`). Absent on a quiet daemon.
15258
+ const up = status.upgrade;
15259
+ if (up && typeof up === 'object' && typeof up.state === 'string') {
15260
+ const from = up.fromVersion ?? '?';
15261
+ const to = up.toVersion ?? '?';
15262
+ const age = Number.isFinite(up.sinceEpochMs) ? ` (${formatDuration(Math.max(0, Date.now() - up.sinceEpochMs))})` : '';
15263
+ lines.push(` upgrade: ${up.state} — daemon ${from} → ${to}${age}`);
15264
+ }
15265
+ // A rolling reload can run for a long time (each worker drains in-flight
15266
+ // work). Surface it on every `status` so a detached operator — or someone
15267
+ // who only ever polls status — can see the roll is live and must not
15268
+ // `restart` the daemon. `reload` is the progress object from the live
15269
+ // socket frame; absent on the socket-unreachable fallback.
15270
+ const roll = status.reload;
15271
+ if (roll && roll.inProgress) {
15272
+ const done = Array.isArray(roll.reloaded) ? roll.reloaded.length : 0;
15273
+ const total = Number(roll.total) || 0;
15274
+ const current = roll.current ? String(roll.current) : null;
15275
+ const parts = [`rolling ${done}/${total} adopted`];
15276
+ if (current) parts.push(`draining ${current}`);
15277
+ if (onDiskVersion && d.version && onDiskVersion !== d.version) {
15278
+ parts.push(`${d.version} → ${onDiskVersion}`);
15279
+ }
15280
+ parts.push('do not restart the daemon');
15281
+ lines.push(` reload: ${parts.join(' — ')}`);
15132
15282
  }
15133
15283
  if (d.startedAt) lines.push(` started: ${d.startedAt}`);
15134
15284
  if (d.socket) lines.push(` control: ${d.socket}`);
@@ -15245,6 +15395,10 @@ function stateFromStatus(res, socketPath) {
15245
15395
  version: res.daemon?.version,
15246
15396
  socket: res.daemon?.socket || socketPath,
15247
15397
  logFile: res.daemon?.logFile,
15398
+ // #309 review: carry the daemon's in-flight upgrade block so a stop client
15399
+ // can see the handoff (and the successor's pid) even when it connected to
15400
+ // the PREDECESSOR just before the socket flipped.
15401
+ ...(res.upgrade && typeof res.upgrade === 'object' ? { upgrade: res.upgrade } : {}),
15248
15402
  workers: res.workers || [],
15249
15403
  };
15250
15404
  }
@@ -15259,6 +15413,10 @@ function stateFromStatus(res, socketPath) {
15259
15413
  function statusFromState(running) {
15260
15414
  return {
15261
15415
  daemon: supervisorDaemonDescriptor(running),
15416
+ // #309: the persisted `upgrade` block rides the fallback render too — a
15417
+ // mid-handoff daemon whose socket is briefly unreachable must still show
15418
+ // the roll (same visibility guarantee as `version`/`logFile` above).
15419
+ ...(running && running.upgrade && typeof running.upgrade === 'object' ? { upgrade: running.upgrade } : {}),
15262
15420
  workers: (running.workers || []).map((w) => summarizeSupervisorWorker(w)),
15263
15421
  };
15264
15422
  }
@@ -15349,6 +15507,20 @@ function waitForChildExit(child, timeoutMs) {
15349
15507
  * Unix-only: reparent-to-init is a POSIX semantic; Windows job objects are the
15350
15508
  * equivalent lifecycle tie and are out of scope here (matching the daemon's
15351
15509
  * other `win32` guards). Returns a canceller so callers/tests can stop it.
15510
+ *
15511
+ * Rolling daemon hot-upgrade (issue #309): `retargetToPid` re-arms the SAME
15512
+ * watchdog installation at a NEW daemon pid — the successor daemon the old one
15513
+ * is handing the fleet to. Without it the swap is impossible: the old daemon
15514
+ * exits once the successor owns the socket, the worker reparents to init, and
15515
+ * an un-retargeted watchdog would read exactly that as "daemon died
15516
+ * ungracefully" and self-reap — killing every in-flight job mid-upgrade, the
15517
+ * precise outcome the rolling replace exists to avoid. The retarget is safe
15518
+ * against a stale signal because the watched pid is only ever advanced to the
15519
+ * successor the CURRENT daemon named (via NANO_SUPERVISOR_UPGRADE_PID, set by
15520
+ * the CLI that spawned it); a SIGUSR2 with no upgrade in flight keeps the
15521
+ * legacy graceful-drain semantics (the caller branches on the env var, not
15522
+ * here). No-op on win32 / when the pid is invalid; returns whether the
15523
+ * retarget landed.
15352
15524
  */
15353
15525
  function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, readPpid } = {}) {
15354
15526
  if (osPlatform() === 'win32') return () => {};
@@ -15362,14 +15534,40 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
15362
15534
  // NOT treat that as "orphaned". A *fallback* ppid of 1, by contrast, means we
15363
15535
  // were already reparented to init with no daemon left to watch.
15364
15536
  const explicit = Number.isInteger(parentPid) && parentPid > 0;
15365
- const watched = explicit ? parentPid : ppid();
15537
+ // `watched` is deliberately MUTABLE: a rolling daemon upgrade (#309) retargets
15538
+ // it at the successor daemon's pid the moment the old daemon hands over, so
15539
+ // the watchdog keeps guarding the worker across the swap instead of reading
15540
+ // the (expected) reparent-to-init as an orphan event and self-reaping.
15541
+ let watched = explicit ? parentPid : ppid();
15542
+ // #309: after a retarget, the ppid leg changes meaning. The old daemon exits
15543
+ // on purpose once the successor owns the socket, so this worker reparents to
15544
+ // init — and that is the EXPECTED end state, not an orphan event. From the
15545
+ // retarget on, only the SUCCESSOR's death orphans the worker (the ppid leg
15546
+ // would fire immediately and forever on the already-reparented worker).
15547
+ let retargeted = false;
15548
+ // #309 review (rollback): the guard state BEFORE the latest retarget. If the
15549
+ // successor dies mid-handoff while the predecessor is still alive (it rolls
15550
+ // back and keeps the fleet), the worker must fall back to guarding the
15551
+ // predecessor instead of self-reaping an in-flight job. Once the predecessor
15552
+ // has exited too, both are gone and the worker is genuinely orphaned.
15553
+ let fallback = null;
15366
15554
  const orphaned = typeof onOrphan === 'function' ? onOrphan : () => process.exit(0);
15367
15555
  const isOrphan = () => {
15368
15556
  // Reparented away from the daemon (typically to init, pid 1) → orphaned.
15369
- if (ppid() !== watched) return true;
15370
- // Defensive: the daemon pid is gone even though ppid still names it (a
15371
- // zombie/racey read). ESRCH ⇒ gone; EPERM ⇒ alive but not ours to signal.
15372
- try { process.kill(watched, 0); return false; } catch (err) { return err.code !== 'EPERM'; }
15557
+ // Skipped after an upgrade retarget: the reparent to init IS the handoff.
15558
+ if (!retargeted && ppid() !== watched) return true;
15559
+ // The watched daemon is gone (ESRCH ⇒ gone; EPERM ⇒ alive but not ours to
15560
+ // signal). After an upgrade retarget this is the ONLY orphan signal — the
15561
+ // ppid leg above is skipped (the reparent to init is expected), so the
15562
+ // worker self-reaps exactly when the SUCCESSOR daemon dies.
15563
+ const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; } };
15564
+ if (alive(watched)) return false;
15565
+ if (fallback && alive(fallback.watched) && (fallback.retargeted || ppid() === fallback.watched)) {
15566
+ ({ watched, retargeted } = fallback);
15567
+ fallback = null;
15568
+ return false;
15569
+ }
15570
+ return true;
15373
15571
  };
15374
15572
  // Orphaned already — self-reap now rather than idle forever. Either we fell
15375
15573
  // back to ppid and it is already init (no known parent), or the recorded
@@ -15377,7 +15575,22 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
15377
15575
  if ((!explicit && watched === 1) || isOrphan()) { orphaned(); return () => {}; }
15378
15576
  const timer = setInterval(() => { if (isOrphan()) { clearInterval(timer); orphaned(); } }, intervalMs);
15379
15577
  if (typeof timer.unref === 'function') timer.unref();
15380
- return () => { try { clearInterval(timer); } catch { /* ignore */ } };
15578
+ const cancel = () => { try { clearInterval(timer); } catch { /* ignore */ } };
15579
+ // #309: re-arm at the successor daemon's pid. The ppid leg of isOrphan() then
15580
+ // reads "reparented to init" as expected (the old daemon exited on purpose),
15581
+ // and only the SUCCESSOR's death orphans the worker from here on.
15582
+ cancel.retarget = (newPid) => {
15583
+ if (!Number.isInteger(newPid) || newPid <= 0) return false;
15584
+ if (newPid !== watched) fallback = { watched, retargeted };
15585
+ watched = newPid;
15586
+ retargeted = true;
15587
+ return true;
15588
+ };
15589
+ // The pid this installation currently guards — the daemon stamps it on the
15590
+ // worker's activity marker as the upgrade ack (the old daemon waits, bounded,
15591
+ // for it to name the successor before exiting).
15592
+ cancel.watchedPid = () => watched;
15593
+ return cancel;
15381
15594
  }
15382
15595
 
15383
15596
  // --- Daemon ----------------------------------------------------------------
@@ -15386,6 +15599,18 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
15386
15599
  * The supervisor daemon body. Runs under `c8ctl nano supervisor __daemon`,
15387
15600
  * spawned detached by `startSupervisorDaemon`. Never returns — it runs until a
15388
15601
  * stop request or SIGTERM, then drains children and exits.
15602
+ *
15603
+ * Issue #309 (rolling daemon hot-upgrade): when spawned with
15604
+ * NANO_SUPERVISOR_UPGRADE=1 this process is a SUCCESSOR — the CLI launched it
15605
+ * to take over the fleet from an old daemon running stale plugin code. Before
15606
+ * binding the control socket it waits for the old daemon to quiesce (its
15607
+ * `prepare-upgrade` op stops the accept loop), then adopts the old daemon's
15608
+ * workers from the persisted state (pid-polled synthetic child handles — the
15609
+ * workers themselves are never signaled, so in-flight jobs run undisturbed)
15610
+ * and finally asks the old daemon to hand over (its `upgrade` op retargets
15611
+ * every worker's parent-death watchdog here and exits). A successor that
15612
+ * cannot reach a quiesced old daemon in time exits 1 and leaves the old fleet
15613
+ * untouched.
15389
15614
  */
15390
15615
  async function runSupervisorDaemon() {
15391
15616
  const startedAt = new Date().toISOString();
@@ -15399,6 +15624,68 @@ async function runSupervisorDaemon() {
15399
15624
  const daemonLogFile = supervisorDaemonLogFile();
15400
15625
  mkdirSync(getSupervisorLogDir(), { recursive: true });
15401
15626
 
15627
+ // Boot-time log line, available before the closure-scoped `dlog` exists (the
15628
+ // upgrade-mode quiesce handshake below runs before the workers map and its
15629
+ // helpers are initialized). Identical format.
15630
+ const dlogBoot = (msg) => {
15631
+ try { appendFileSync(daemonLogFile, `[${new Date().toISOString()}] ${msg}\n`); } catch { /* best effort */ }
15632
+ };
15633
+
15634
+ // #309: successor boot. `upgradeMode` latches once the old daemon has been
15635
+ // asked to quiesce; `predecessorPid`/`upgradeFromVersion` feed the status
15636
+ // frame's `upgrade` block so `supervisor status` shows the roll in flight
15637
+ // (`rolling <from> → <to>`), mirroring the worker-roll visibility.
15638
+ const upgradeMode = process.env.NANO_SUPERVISOR_UPGRADE === '1';
15639
+ let predecessorPid = null;
15640
+ let upgradeFromVersion = null;
15641
+ let upgradeStartedMs = null;
15642
+ if (upgradeMode) {
15643
+ const t0 = Date.now();
15644
+ dlogBoot(`upgrade mode: waiting for the current daemon to quiesce (pid from ${getSupervisorStateFile()})`);
15645
+ for (;;) {
15646
+ const st = readSupervisorState();
15647
+ if (!st || !isPidAlive(st.pid)) {
15648
+ // No live old daemon — nothing to hand over. Serve the fleet normally
15649
+ // (there is nothing to adopt; the state file, if any, is stale).
15650
+ dlogBoot('upgrade mode: no live predecessor daemon — starting as a plain daemon');
15651
+ break;
15652
+ }
15653
+ predecessorPid = st.pid;
15654
+ upgradeFromVersion = typeof st.version === 'string' ? st.version : null;
15655
+ // Ask the old daemon to stop accepting new control connections. An old
15656
+ // daemon that predates #309 answers `unknown op` — refuse to upgrade from
15657
+ // it rather than racing its accept loop for the socket.
15658
+ let prep = null;
15659
+ try {
15660
+ // #309 review: quiescence awaits admitted opQueue mutations (a restart
15661
+ // can take ~28s to stop its worker), and the predecessor closes its
15662
+ // listener as soon as it latches — so a retry could never reach it.
15663
+ // Keep THIS connection open for the remaining quiesce budget.
15664
+ const remainingMs = SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS - (Date.now() - t0);
15665
+ prep = await supervisorRequest({ op: 'prepare-upgrade', pid: process.pid }, { socketPath, timeoutMs: 750, responseTimeoutMs: Math.max(SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS, remainingMs) });
15666
+ } catch { /* socket briefly unreachable — retry until the deadline */ }
15667
+ if (prep && prep.ok) break; // quiesced — the accept loop is closed
15668
+ if (prep && prep.ok === false && /unknown op/.test(String(prep.error || ''))) {
15669
+ dlogBoot(`upgrade mode: predecessor pid ${st.pid} does not understand prepare-upgrade (too old) — refusing to upgrade`);
15670
+ process.exit(1);
15671
+ }
15672
+ // #309 review: the predecessor bound its handoff to a DIFFERENT successor
15673
+ // (a concurrent upgrade won the race). Never keep retrying — the binding
15674
+ // will never accept this pid, and two successors must not both wait to
15675
+ // adopt the same fleet. Exit and leave the bound successor to finish.
15676
+ if (prep && prep.ok === false && /already bound to successor pid/.test(String(prep.error || ''))) {
15677
+ dlogBoot(`upgrade mode: predecessor pid ${st.pid} is bound to a different successor — exiting so the bound successor completes the handoff`);
15678
+ process.exit(1);
15679
+ }
15680
+ if (Date.now() - t0 > SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS) {
15681
+ dlogBoot(`upgrade mode: predecessor pid ${st.pid} did not quiesce within ${SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS}ms — leaving it in place`);
15682
+ process.exit(1);
15683
+ }
15684
+ await new Promise((r) => setTimeout(r, 150));
15685
+ }
15686
+ if (predecessorPid != null) upgradeStartedMs = Date.now();
15687
+ }
15688
+
15402
15689
  const workers = new Map();
15403
15690
  const attachClients = new Set();
15404
15691
  // #202: sockets that issued a `stop`/drain and are waiting for the daemon to
@@ -15416,15 +15703,50 @@ async function runSupervisorDaemon() {
15416
15703
  // restart-on-exit path and let a second `stop --force` escalate a live drain.
15417
15704
  let draining = false;
15418
15705
  let forcing = false;
15706
+ // #309 review: `shuttingDown` is latched EARLY (in the `stop` op, before the
15707
+ // deferred shutdown() runs) to reserve shutdown ownership against a racing
15708
+ // prepare-upgrade. `shutdownStarted` is the separate "shutdown()'s body has
15709
+ // begun" guard: the deferred shutdown() must still run its body even though
15710
+ // `shuttingDown` is already true, while a SECOND stop/signal must escalate
15711
+ // (force) or return instead of re-running the body.
15712
+ let shutdownStarted = false;
15713
+ // #309: a rolling daemon upgrade owns this daemon. `upgrading` is the
15714
+ // OLD-daemon latch: set by `prepare-upgrade` (accept loop closed; mutations
15715
+ // refused) and held through the `upgrade` handoff until this process exits.
15716
+ // `quiesced` flips when the accept loop has actually closed — the successor
15717
+ // polls for it before binding the socket. `upgradeInfo` carries the roll's
15718
+ // identity ({ state, fromVersion, toVersion, successorPid, sinceEpochMs })
15719
+ // for the status frame's `upgrade` block.
15720
+ let upgrading = false;
15721
+ let quiesced = false;
15722
+ let upgradeInfo = null;
15419
15723
  // Hot code reload (rolling drain+respawn). A `reload` op adopts new on-disk
15420
15724
  // plugin code into the worker children by gracefully draining and respawning
15421
15725
  // them one at a time (so the fleet keeps serving). This flag rejects a second
15422
15726
  // concurrent reload — a single rolling pass owns the fleet until it finishes.
15423
15727
  let reloading = false;
15728
+ // Progress for `status.reload` while `reloading` is true. Reset when the
15729
+ // roll starts; `reloadCurrent` is the id in `reloadWorker` right now.
15730
+ let reloadTargets = [];
15731
+ let reloadDone = [];
15732
+ let reloadCurrent = null;
15424
15733
  // Live-view monitor: tracks the last-broadcast fleet signature so we push a
15425
15734
  // refreshed status to attached consoles only on real change (see below).
15426
15735
  let monitorTimer = null;
15427
15736
  let lastMonitorSig = null;
15737
+ // #309: while quiesced for an upgrade the predecessor's accept loop is closed
15738
+ // and — on a second upgrade — every worker is an ADOPTED handle whose liveness
15739
+ // poll (`adoptTimer`) is unref'd, as is `monitorTimer`. With no referenced
15740
+ // handle left, Node would let the predecessor EXIT before the successor's
15741
+ // watchdog-retarget acks arrive / before SIGUSR1 lands — and the never-
15742
+ // resolving handoff Promise does NOT keep the event loop alive. This
15743
+ // referenced keepalive holds the predecessor up for exactly the quiesce→
15744
+ // handover window; it is cleared on the SIGUSR1 handover (and on shutdown).
15745
+ let quiesceKeepalive = null;
15746
+ // #309: polls the successor pid while quiesced so a successor that dies
15747
+ // mid-handoff (before SIGUSR1) triggers a rollback instead of leaving this
15748
+ // daemon permanently quiesced + unreachable. Cleared on handover/rollback.
15749
+ let successorWatchTimer = null;
15428
15750
 
15429
15751
  // Daemon-wide mutation serialization: `add`/`remove`/`restart` must not
15430
15752
  // interleave, or two clients racing the same worker could each spawn an
@@ -15442,16 +15764,46 @@ async function runSupervisorDaemon() {
15442
15764
 
15443
15765
  const workerPublic = (w) => summarizeSupervisorWorker(w);
15444
15766
 
15445
- const persist = () => {
15767
+ // #309: the daemon descriptor gains an `upgrade` block while a rolling daemon
15768
+ // upgrade is in flight — on the OLD daemon (state 'rolling', naming the
15769
+ // successor it is handing to) and on the SUCCESSOR (state 'adopting' until
15770
+ // the predecessor exits). `supervisor status` renders it as
15771
+ // `upgrade: rolling <from> → <to>` — the same visibility the worker roll has.
15772
+ const currentUpgradeBlock = () => {
15773
+ if (upgradeInfo) return upgradeInfo;
15774
+ if (upgradeMode && predecessorPid != null) {
15775
+ return { state: 'adopting', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, sinceEpochMs: upgradeStartedMs };
15776
+ }
15777
+ return null;
15778
+ };
15779
+
15780
+ // #309: once the successor has confirmed ownership (SIGUSR1 → handedOver) the
15781
+ // predecessor must make its final snapshot its LAST canonical write. More
15782
+ // broadly, once QUIESCED (`upgrading` latched) the successor has already read
15783
+ // the quiesce snapshot and adopted the inventory, so any later automatic
15784
+ // persist (a straggler worker-death handler firing between quiesce and exit)
15785
+ // would overwrite the successor's pid + inventory with the dead/dying daemon's
15786
+ // — corrupting discovery/recovery until the successor happens to persist
15787
+ // again. So persist() is suppressed while quiesced UNLESS forced (the quiesce
15788
+ // snapshot and the final handed-over snapshot are the two deliberate writes).
15789
+ let handedOver = false;
15790
+ const persist = (force = false) => {
15791
+ if (handedOver) return; // the successor owns the state file now
15792
+ if (upgrading && !force) return; // quiesced — the successor owns the inventory
15446
15793
  try {
15447
15794
  writeSupervisorState({
15448
15795
  ...supervisorDaemonDescriptor({
15449
15796
  pid: process.pid, startedAt, version: daemonVersion, socket: socketPath, logFile: daemonLogFile,
15450
15797
  }),
15798
+ ...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
15451
15799
  workers: [...workers.values()].map((w) => ({
15452
15800
  id: w.id, profile: w.profile, args: w.args, pid: isPidAlive(w.pid) ? w.pid : null,
15453
15801
  startedAt: w.startedAt || null, restarts: w.restarts, lastExit: w.lastExit ?? null,
15454
15802
  stopping: !!w.stopping, logFile: w.logFile, configFailed: w.configFailed === true,
15803
+ // #309: persist drain-remove intent so an adopted worker that exits
15804
+ // mid-drain is still REMOVED by the successor (not retained forever,
15805
+ // which would block name reuse and hang `workforce stop`).
15806
+ removeOnExit: w.removeOnExit === true,
15455
15807
  })),
15456
15808
  });
15457
15809
  } catch { /* best effort */ }
@@ -15488,6 +15840,11 @@ async function runSupervisorDaemon() {
15488
15840
  const activityFile = supervisorWorkerActivityFile(w.id);
15489
15841
  w.activityFile = activityFile;
15490
15842
  try { rmSync(activityFile, { force: true }); } catch { /* best effort */ }
15843
+ // #309 review: also clear a stale `.upgrade` retarget sidecar. A worker
15844
+ // force-restarted mid-handoff before consuming it would otherwise inherit
15845
+ // the same path and treat its next genuine drain SIGUSR2 as a retarget.
15846
+ // (A handoff in progress re-writes the sidecar before it SIGUSR2s.)
15847
+ try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
15491
15848
  // `--name w.id` makes the child's broker workerName match this worker's
15492
15849
  // supervisor id, so the same profile launched twice is distinct end-to-end.
15493
15850
  // NANO_SUPERVISOR_ACTIVITY_FILE tells the child where to report per-job
@@ -15496,8 +15853,20 @@ async function runSupervisorDaemon() {
15496
15853
  // watchdog can self-reap if we die ungracefully (SIGKILL/crash) and can't
15497
15854
  // run the child-draining shutdown — even if we die mid-startup, before the
15498
15855
  // child reparents to init.
15856
+ // NANO_SUPERVISOR_UPGRADE_PID (#309): when a daemon upgrade is in flight,
15857
+ // the successor's pid rides the child's env so the worker's SIGUSR2 handler
15858
+ // can tell a watchdog RETARGET (upgrade — keep serving) apart from a
15859
+ // graceful DRAIN (no upgrade pid — quiesce and exit). The daemon learns the
15860
+ // successor pid only at handoff time (the `upgrade` op), AFTER most workers
15861
+ // were spawned — so env alone can't reach them. The daemon therefore also
15862
+ // writes `<activityFile>.upgrade` (the successor pid) before it SIGUSR2s,
15863
+ // and the worker reads THAT as the authoritative retarget target; this env
15864
+ // forwarding covers only a worker spawned mid-upgrade.
15865
+ const childEnv = { ...process.env, NANO_SUPERVISOR_ACTIVITY_FILE: activityFile, NANO_SUPERVISOR_DAEMON_PID: String(process.pid) };
15866
+ const upgradeSuccessorPid = Number(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
15867
+ if (Number.isInteger(upgradeSuccessorPid) && upgradeSuccessorPid > 0) childEnv.NANO_SUPERVISOR_UPGRADE_PID = String(upgradeSuccessorPid);
15499
15868
  const child = spawn(exec, [entry, 'nano', 'work', w.profile, '--name', w.id, ...w.args], {
15500
- env: { ...process.env, NANO_SUPERVISOR_ACTIVITY_FILE: activityFile, NANO_SUPERVISOR_DAEMON_PID: String(process.pid) },
15869
+ env: childEnv,
15501
15870
  stdio: ['ignore', ringed ? 'pipe' : fd, ringed ? 'pipe' : fd],
15502
15871
  });
15503
15872
  if (typeof fd === 'number') { try { closeSync(fd); } catch { /* dup'd into child */ } }
@@ -15590,7 +15959,9 @@ async function runSupervisorDaemon() {
15590
15959
  broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
15591
15960
  w.restartTimer = setTimeout(() => {
15592
15961
  w.restartTimer = null;
15593
- if (!w.stopping && !shuttingDown && workers.has(w.id)) startWorker(w);
15962
+ // #309: a quiesced old daemon (mid-upgrade) never respawns — the
15963
+ // successor owns the fleet and would double-spawn the worker.
15964
+ if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
15594
15965
  }, delay);
15595
15966
  if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
15596
15967
  persist();
@@ -15600,6 +15971,123 @@ async function runSupervisorDaemon() {
15600
15971
  persist();
15601
15972
  };
15602
15973
 
15974
+ // #309: a synthetic ChildProcess-shaped handle for an ADOPTED worker — one
15975
+ // the old daemon spawned and the successor inherits by pid. The successor
15976
+ // never spawned it, so there is no real 'exit' event: a poll timer is the
15977
+ // death detector, feeding the same `handleDeath` path a real child's exit
15978
+ // would (restart backoff included). The handle provides the surface the
15979
+ // daemon touches: `pid`, `exitCode`/`signalCode` (null while alive — the
15980
+ // readiness gate and reload checks read them), `kill` (force-stop),
15981
+ // `unref`/`ref`, null stdio (the log ring stays with the OLD daemon process),
15982
+ // and — critically — REAL `exit` listener semantics (`once`/`on`/
15983
+ // `removeListener` + `markExited`), because `waitForChildExit()` resolves only
15984
+ // when `exit` fires.
15985
+ const makeAdoptedChild = (pid) => {
15986
+ const exitWaiters = new Set();
15987
+ const handle = {
15988
+ pid,
15989
+ exitCode: null,
15990
+ signalCode: null,
15991
+ killed: false,
15992
+ stdout: null,
15993
+ stderr: null,
15994
+ stdin: null,
15995
+ kill(signal) { try { return process.kill(pid, signal); } catch { return false; } },
15996
+ once(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
15997
+ on(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
15998
+ removeListener(event, fn) { if (event === 'exit') exitWaiters.delete(fn); return handle; },
15999
+ unref() { return handle; },
16000
+ ref() { return handle; },
16001
+ // Invoked by the adopt liveness poll when the pid is gone: record the exit
16002
+ // and flush every waiter so graceful-stop/reload promises resolve.
16003
+ markExited(code, signal) {
16004
+ if (handle.exitCode !== null || handle.signalCode !== null) return;
16005
+ handle.exitCode = code;
16006
+ handle.signalCode = signal ?? null;
16007
+ for (const fn of [...exitWaiters]) { exitWaiters.delete(fn); try { fn(code, signal ?? null); } catch { /* ignore */ } }
16008
+ },
16009
+ };
16010
+ return handle;
16011
+ };
16012
+
16013
+ // Adopt one worker record from the predecessor's persisted state. Returns the
16014
+ // live worker entry, or null when the pid is already gone (nothing to adopt —
16015
+ // the fleet is exactly what survives the swap). The poll timer is unref'd so
16016
+ // it never keeps the daemon alive on its own.
16017
+ const adoptWorker = (rec) => {
16018
+ if (!rec || typeof rec.id !== 'string' || !rec.id) return null;
16019
+ if (!isPidAlive(rec.pid)) return null;
16020
+ const w = {
16021
+ id: rec.id,
16022
+ profile: typeof rec.profile === 'string' ? rec.profile : '',
16023
+ args: Array.isArray(rec.args) ? rec.args.map(String) : [],
16024
+ restarts: Number(rec.restarts) || 0,
16025
+ stopping: rec.stopping === true,
16026
+ // #309: restore drain-remove intent persisted by the predecessor, so a
16027
+ // worker draining for removal that exits after the swap is REMOVED here
16028
+ // (its record deleted) rather than retained — which would block name reuse
16029
+ // and hang a `workforce stop` waiting for the record to disappear.
16030
+ removeOnExit: rec.removeOnExit === true,
16031
+ lastExit: rec.lastExit ?? null,
16032
+ logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
16033
+ configFailed: rec.configFailed === true,
16034
+ adopted: true,
16035
+ };
16036
+ w.activityFile = supervisorWorkerActivityFile(w.id);
16037
+ // Clear any stale `.upgrade` sidecar the handoff left: the worker consumed
16038
+ // it to retarget its watchdog here, and a leftover would make a LATER
16039
+ // SIGUSR2 (a genuine drain) misread as another upgrade retarget.
16040
+ try { rmSync(`${w.activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
16041
+ const child = makeAdoptedChild(rec.pid);
16042
+ w.child = child;
16043
+ w.pid = rec.pid;
16044
+ w.startedAt = rec.startedAt || null;
16045
+ w.spawnedAt = rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now();
16046
+ let settled = false;
16047
+ const handleDeath = (reason) => {
16048
+ if (w.child !== child || settled) return;
16049
+ settled = true;
16050
+ if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
16051
+ // Mark the synthetic handle exited BEFORE running the death path so any
16052
+ // `waitForChildExit` waiter (graceful stop / reload) resolves — the poll
16053
+ // below is the only death detector for an adopted worker.
16054
+ if (typeof child.markExited === 'function') child.markExited(1, null);
16055
+ w.pid = null;
16056
+ w.lastExit = reason;
16057
+ try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
16058
+ const ranMs = Date.now() - (w.spawnedAt || Date.now());
16059
+ if (ranMs >= SUPERVISOR_HEALTHY_UPTIME_MS) w.restarts = 0;
16060
+ if (w.stopping || shuttingDown || !workers.has(w.id)) {
16061
+ if (w.removeOnExit && workers.get(w.id) === w) {
16062
+ workers.delete(w.id);
16063
+ try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
16064
+ dlog(`worker '${w.id}' removed (drained)`);
16065
+ broadcast({ type: 'event', event: 'worker-remove', id: w.id });
16066
+ }
16067
+ persist();
16068
+ if (draining) { try { broadcast(statusFrame(false)); } catch { /* best effort */ } }
16069
+ return;
16070
+ }
16071
+ if (upgrading) { persist(); return; } // quiesced for the handoff — the successor owns respawns
16072
+ const delay = supervisorBackoffMs(w.restarts);
16073
+ w.restarts += 1;
16074
+ dlog(`adopted worker '${w.id}' down (${reason}); restarting in ${delay}ms (restart #${w.restarts})`);
16075
+ broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
16076
+ w.restartTimer = setTimeout(() => {
16077
+ w.restartTimer = null;
16078
+ if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
16079
+ }, delay);
16080
+ if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
16081
+ persist();
16082
+ };
16083
+ w.adoptTimer = setInterval(() => { if (!isPidAlive(rec.pid)) handleDeath('code 1 (adopted worker exited)'); }, SUPERVISOR_ADOPT_POLL_MS);
16084
+ if (typeof w.adoptTimer.unref === 'function') w.adoptTimer.unref();
16085
+ workers.set(w.id, w);
16086
+ dlog(`worker '${w.id}' (profile ${w.profile}) adopted from predecessor daemon (pid ${w.pid})`);
16087
+ broadcast({ type: 'event', event: 'worker-adopted', worker: workerPublic(w) });
16088
+ return w;
16089
+ };
16090
+
15603
16091
  const addWorker = (profile, args, name) => {
15604
16092
  const taken = new Set(workers.keys());
15605
16093
  let id;
@@ -15622,6 +16110,33 @@ async function runSupervisorDaemon() {
15622
16110
  return w;
15623
16111
  };
15624
16112
 
16113
+ // #309 review: workers whose upgrade watchdog-retarget is still outstanding
16114
+ // (id -> { pid, signalled, drainOnly?, drainDeferred? }), owned by the
16115
+ // successor's handoff loop. A graceful drain (stop/remove/reload) also uses
16116
+ // SIGUSR2, so it must not race a pending retarget: a drain signal consumed as
16117
+ // the retarget leaves the drain unfulfilled, and a second SIGUSR2 after the
16118
+ // one-shot handler fired would kill in-flight work. sendDrainSignal
16119
+ // coordinates the two so both are delivered, in order.
16120
+ const retargetPending = new Map();
16121
+ const sendDrainSignal = (w, pid) => {
16122
+ const p = retargetPending.get(w.id);
16123
+ if (p && p.pid === pid) {
16124
+ if (!p.signalled) {
16125
+ // Retarget not sent yet: cancel it (drop the sidecar) and drain
16126
+ // instead — the handoff loop delivers the drain once the handler is
16127
+ // installed and keeps the predecessor alive until the worker exits.
16128
+ try { rmSync(`${w.activityFile || supervisorWorkerActivityFile(w.id)}.upgrade`, { force: true }); } catch { /* best effort */ }
16129
+ p.signalled = true; p.drainOnly = true; p.drainDeferred = true;
16130
+ } else if (!p.drainOnly) {
16131
+ // Retarget in flight: deliver the drain right after its ack.
16132
+ p.drainDeferred = true;
16133
+ }
16134
+ // drainOnly without drainDeferred: already draining — nothing to send.
16135
+ return;
16136
+ }
16137
+ try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
16138
+ };
16139
+
15625
16140
  const forceStopWorker = async (w) => {
15626
16141
  const pid = w.pid;
15627
16142
  if (w.child && pid) {
@@ -15648,7 +16163,7 @@ async function runSupervisorDaemon() {
15648
16163
  // — a `stop --force` escalation (forceStopWorker) is the only way to cut a
15649
16164
  // stuck drain short, and it resolves this same wait when the child dies.
15650
16165
  if (w.child && pid) {
15651
- try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
16166
+ sendDrainSignal(w, pid);
15652
16167
  await waitForChildExit(w.child, null);
15653
16168
  }
15654
16169
  return true;
@@ -15670,7 +16185,7 @@ async function runSupervisorDaemon() {
15670
16185
  w.removeOnExit = true;
15671
16186
  if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
15672
16187
  const pid = w.pid;
15673
- if (w.child && pid) { try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
16188
+ if (w.child && pid) { sendDrainSignal(w, pid); }
15674
16189
  else { workers.delete(id); try { rmSync(supervisorWorkerActivityFile(id), { force: true }); } catch { /* best effort */ } broadcast({ type: 'event', event: 'worker-remove', id }); persist(); }
15675
16190
  dlog(`worker '${id}' draining for removal`);
15676
16191
  return true;
@@ -15767,7 +16282,7 @@ async function runSupervisorDaemon() {
15767
16282
  if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
15768
16283
  const pid = w.pid;
15769
16284
  if (child && pid) {
15770
- try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
16285
+ sendDrainSignal(w, pid);
15771
16286
  await waitForChildExit(child, null);
15772
16287
  }
15773
16288
  // Only respawn if nobody else acted on this worker while we drained: it must
@@ -15839,12 +16354,24 @@ async function runSupervisorDaemon() {
15839
16354
  // shutting down. Never clears `supervisor.json` — a reload is not a stop.
15840
16355
  const runReload = async (ids, sock) => {
15841
16356
  reloading = true;
16357
+ reloadTargets = [...ids];
16358
+ reloadDone = [];
16359
+ reloadCurrent = null;
15842
16360
  const reloaded = [];
15843
16361
  const skipped = [];
15844
16362
  let interrupted = false;
15845
16363
  try {
15846
16364
  for (let i = 0; i < ids.length; i++) {
15847
16365
  const id = ids[i];
16366
+ reloadCurrent = id;
16367
+ // Broadcast (not just to the initiating socket) on every target
16368
+ // transition so every attached console sees the drain progress and the
16369
+ // "do not restart the daemon" warning even with periodic monitoring
16370
+ // disabled (NANO_SUPERVISOR_MONITOR_MS=0). The initiator is itself an
16371
+ // attach client, so it is covered too.
16372
+ if (!shuttingDown) {
16373
+ try { broadcast(statusFrame(false)); } catch { /* best effort */ }
16374
+ }
15848
16375
  // If the daemon starts shutting down mid-roll, the remaining workers
15849
16376
  // never adopt the new code — record them as skipped (and flag the roll
15850
16377
  // interrupted) so the terminal frame can't report a clean success for a
@@ -15868,6 +16395,8 @@ async function runSupervisorDaemon() {
15868
16395
  const ok = await reloadWorker(id);
15869
16396
  if (ok) {
15870
16397
  reloaded.push(id);
16398
+ reloadDone.push(id);
16399
+ reloadCurrent = null;
15871
16400
  // Emit the per-worker "reloaded" progress signal ONLY after the final
15872
16401
  // success gate above confirmed a live, still-current replacement — not
15873
16402
  // at spawn time. A crash/spawn-fail makes `reloadWorker` return false
@@ -15888,13 +16417,24 @@ async function runSupervisorDaemon() {
15888
16417
  for (let j = i; j < ids.length; j++) skipped.push(ids[j]);
15889
16418
  break;
15890
16419
  }
15891
- try { sock.write(encodeFrame(statusFrame(false))); } catch { /* client gone */ }
16420
+ try { broadcast(statusFrame(false)); } catch { /* best effort */ }
15892
16421
  }
15893
16422
  } catch (err) {
15894
16423
  interrupted = true;
15895
16424
  dlog(`reload error: ${err?.message || err}`);
15896
16425
  } finally {
15897
16426
  reloading = false;
16427
+ reloadCurrent = null;
16428
+ reloadTargets = [];
16429
+ reloadDone = [];
16430
+ // Clearing the progress flags may not change the monitor signature (it may
16431
+ // already have sampled the final replacement mid-readiness-wait), and the
16432
+ // terminal `reloaded` frame goes only to the initiating socket — so push a
16433
+ // fresh status to every attached console now, or they could keep showing
16434
+ // the "rolling … do not restart the daemon" line after completion.
16435
+ if (!shuttingDown && attachClients.size > 0) {
16436
+ try { broadcast(statusFrame(false)); } catch { /* best effort */ }
16437
+ }
15898
16438
  // Terminal success requires BOTH a clean pass (not interrupted) AND nothing
15899
16439
  // skipped: a skipped worker (failed reload, or removed/absent mid-roll) is a
15900
16440
  // partial reload, so `ok: true` would let the streaming client exit zero and
@@ -15927,12 +16467,40 @@ async function runSupervisorDaemon() {
15927
16467
  // i.e. a `supervisor reload` would adopt new worker code (and a daemon
15928
16468
  // restart new supervisor code). Best-effort; falls back to the daemon's own.
15929
16469
  pluginVersion: (() => { try { return pluginPackage().version; } catch { return daemonVersion; } })(),
16470
+ // Live rolling-reload progress so `supervisor status` (and the attach
16471
+ // view) show a roll that was detached or started by someone else.
16472
+ // `inProgress` is the latch; `current` is the worker being drained
16473
+ // right now (null between respawn and the next drain); `reloaded` /
16474
+ // `targets` / `total` let the renderer print N/M.
16475
+ reload: {
16476
+ inProgress: reloading,
16477
+ current: reloadCurrent,
16478
+ targets: reloadTargets,
16479
+ reloaded: [...reloadDone],
16480
+ total: reloadTargets.length,
16481
+ },
16482
+ // #309: present only while a rolling daemon upgrade is in flight
16483
+ // (`{ state, fromVersion, toVersion, … }`) — the successor polls the old
16484
+ // daemon's `upgrade.state === 'quiesced'` through this field before binding
16485
+ // the socket, and `supervisor status` renders it as the daemon-roll line.
16486
+ ...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
15930
16487
  workers: pub || [...workers.values()].map(workerPublic),
15931
16488
  ...(final ? { final: true } : {}),
15932
16489
  });
15933
16490
 
15934
16491
  const shutdown = async (signal, { force = true } = {}) => {
15935
- if (shuttingDown) {
16492
+ // #309 review: a quiesced (mid-upgrade) daemon must NEVER run the shutdown
16493
+ // path — it would SIGTERM/SIGKILL the very workers the successor has
16494
+ // adopted, and its cleanup would delete the successor's socket and state
16495
+ // file. The upgrade's SIGUSR1 handover is this daemon's only exit while
16496
+ // quiesced; a signal (SIGTERM/SIGINT/SIGHUP) is logged and ignored so the
16497
+ // handoff can finish. The socket `stop` op refuses earlier (the `upgrading`
16498
+ // check in handleRequest) and never reaches here.
16499
+ if (upgrading) {
16500
+ dlog(`ignoring ${signal || 'shutdown request'} — a daemon upgrade owns this daemon (successor pid ${upgradeInfo?.successorPid ?? 'unknown'}); the successor owns the fleet now`);
16501
+ return;
16502
+ }
16503
+ if (shutdownStarted) {
15936
16504
  // A shutdown is already running. A `stop --force` arriving mid-DRAIN
15937
16505
  // escalates it: hard-stop every worker still finishing its jobs so the
15938
16506
  // operator isn't stuck waiting. Idempotent — only the first force escalates.
@@ -15945,6 +16513,7 @@ async function runSupervisorDaemon() {
15945
16513
  return;
15946
16514
  }
15947
16515
  shuttingDown = true;
16516
+ shutdownStarted = true;
15948
16517
  draining = !force;
15949
16518
  // Let any in-flight mutation finish before we snapshot the worker set, so
15950
16519
  // an add/restart racing the shutdown can't leave an orphaned child behind.
@@ -15953,6 +16522,7 @@ async function runSupervisorDaemon() {
15953
16522
  if (!force) broadcast({ type: 'event', event: 'draining', workers: [...workers.values()].map(workerPublic) });
15954
16523
  await Promise.all([...workers.keys()].map((id) => stopWorker(id, { force })));
15955
16524
  if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
16525
+ if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
15956
16526
  broadcast({ type: 'event', event: 'daemon-stop' });
15957
16527
  // Terminal frame for every waiting `stop` client (streaming or one-shot).
15958
16528
  // We must FLUSH these before exiting: `process.exit()` does NOT drain pending
@@ -15976,6 +16546,179 @@ async function runSupervisorDaemon() {
15976
16546
  process.exit(0);
15977
16547
  };
15978
16548
 
16549
+ // #309: quiesce this daemon for an upgrade WITHOUT exiting: stop accepting
16550
+ // new control connections (the successor binds the socket the moment our
16551
+ // accept loop closes) and latch `upgrading` so mutations and worker respawns
16552
+ // are refused. Idempotent — the successor polls until it observes the
16553
+ // quiesced state. Attached consoles are destroyed (their daemon is going
16554
+ // away). The handoff is then driven by the SUCCESSOR (it retargets the
16555
+ // workers' watchdogs and SIGUSR1s this daemon to exit) — this daemon's
16556
+ // accept loop is already closed, so it cannot be asked over the socket.
16557
+ //
16558
+ // Snapshot AFTER the mutation queue drains (#309 review): latching
16559
+ // `upgrading` rejects NEW mutations, but one may already be admitted to
16560
+ // `opQueue` (e.g. a restart that is mid-respawn). Persisting before that
16561
+ // settles would snapshot a half-swapped worker set the successor then adopts
16562
+ // — missing the replacement the in-flight restart is about to spawn. So we
16563
+ // latch admission synchronously, then AWAIT `opQueue` (exactly as
16564
+ // `shutdown()` does) before persisting and reporting `quiesced`.
16565
+ let quiescePromise = null;
16566
+ // #309 review: the worker-id set captured in the quiesce snapshot. Rollback
16567
+ // reconciles the live map against it — the successor owns add/remove while
16568
+ // this daemon is quiesced, so a worker the successor removed mid-handoff must
16569
+ // stay removed if the handoff rolls back (never resurrected).
16570
+ let quiesceSnapshotIds = null;
16571
+ const quiesceForUpgrade = (successorPid, successorVersion) => {
16572
+ // #309 review: bind the handoff to ONE successor. `doQuiesce` latches
16573
+ // `upgrading` only inside its async body, so two prepare-upgrade requests
16574
+ // that arrive before the first body's first await both observe
16575
+ // `upgrading === false` and would share one quiesce promise — two
16576
+ // successors, each believing it owns the bind+adopt, racing for the same
16577
+ // socket and fleet. Validate and bind preparation to the FIRST successor
16578
+ // pid synchronously (before any await), and reject every other pid until
16579
+ // this handoff finishes (handover) or rolls back. The bound successor's
16580
+ // own retries (same pid) stay idempotent — they share the promise.
16581
+ if (!Number.isInteger(successorPid) || successorPid <= 0) {
16582
+ // An unidentifiable caller never earns the binding — without a pid the
16583
+ // successor watch cannot run and no retry could ever match the binding.
16584
+ const err = new Error('prepare-upgrade requires a valid successor pid');
16585
+ err.code = 'UPGRADE_BOUND';
16586
+ throw err;
16587
+ }
16588
+ if (upgrading) {
16589
+ const boundTo = upgradeInfo && upgradeInfo.successorPid;
16590
+ if (boundTo !== successorPid) {
16591
+ const err = new Error(`a daemon upgrade is already bound to successor pid ${boundTo} — refusing a concurrent handoff`);
16592
+ err.code = 'UPGRADE_BOUND';
16593
+ throw err;
16594
+ }
16595
+ }
16596
+ if (!upgrading || !quiescePromise) quiescePromise = doQuiesce(successorPid, successorVersion);
16597
+ return quiescePromise;
16598
+ };
16599
+ const doQuiesce = async (successorPid, successorVersion) => {
16600
+ if (!upgrading) {
16601
+ upgrading = true; // latch admission FIRST — rejects new mutations/respawns
16602
+ upgradeInfo = { state: 'quiescing', fromVersion: daemonVersion, toVersion: successorVersion ?? null, successorPid, sinceEpochMs: Date.now() };
16603
+ dlog(`daemon upgrade: quiescing for successor pid ${successorPid} — control socket accept loop closing, mutations refused`);
16604
+ try { server.close(); } catch { /* ignore */ }
16605
+ for (const s of [...attachClients]) { try { s.destroy(); } catch { /* ignore */ } }
16606
+ // Hold the event loop open until the successor's SIGUSR1 handover (see the
16607
+ // quiesceKeepalive declaration) — a no-op tick on a REFERENCED timer.
16608
+ if (!quiesceKeepalive) quiesceKeepalive = setInterval(() => {}, SUPERVISOR_ADOPT_POLL_MS);
16609
+ // #309 review (no-rollback): watch the successor. If it dies AFTER we
16610
+ // quiesced but BEFORE handing over (SIGUSR1), this daemon is left quiesced
16611
+ // and unreachable — respawns disabled, accept loop closed, no way for even
16612
+ // a later upgrade to reach it. Roll back: re-open the accept loop and
16613
+ // unlatch `upgrading` so normal operation (and a retry) resumes.
16614
+ armSuccessorWatch(successorPid);
16615
+ // Wait for any ALREADY-ADMITTED mutation (e.g. a restart mid-respawn) to
16616
+ // settle before snapshotting, so the successor adopts its replacement.
16617
+ try { await opQueue; } catch { /* mutation already logged */ }
16618
+ if (!upgrading) return; // rolled back while draining the queue
16619
+ upgradeInfo = { ...upgradeInfo, state: 'quiesced' };
16620
+ quiesceSnapshotIds = [...workers.keys()];
16621
+ persist(true); // the deliberate quiesce snapshot the successor adopts from
16622
+ }
16623
+ quiesced = true;
16624
+ };
16625
+
16626
+ // #309 review (no-rollback): while quiesced, poll the successor pid. If it
16627
+ // dies before handing over (SIGUSR1), this daemon would otherwise stay
16628
+ // quiesced + unreachable forever (accept loop closed, respawns disabled, no
16629
+ // listener for even a later upgrade). On detecting a dead successor, roll
16630
+ // back: re-open the accept loop and unlatch `upgrading`.
16631
+ const armSuccessorWatch = (successorPid) => {
16632
+ if (!Number.isInteger(successorPid) || successorPid <= 0) return;
16633
+ if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
16634
+ successorWatchTimer = setInterval(() => {
16635
+ if (!upgrading || handedOver) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; return; }
16636
+ if (!isPidAlive(successorPid)) {
16637
+ try { clearInterval(successorWatchTimer); } catch { /* ignore */ }
16638
+ successorWatchTimer = null;
16639
+ rollbackFromUpgrade(successorPid);
16640
+ }
16641
+ }, SUPERVISOR_ADOPT_POLL_MS);
16642
+ if (typeof successorWatchTimer.unref === 'function') successorWatchTimer.unref();
16643
+ };
16644
+
16645
+ // Restore normal operation after a successor died mid-handoff: re-listen on
16646
+ // the control socket (the successor never bound it, or died holding it) and
16647
+ // unlatch `upgrading` so mutations/respawns resume. Best-effort — if the
16648
+ // re-listen fails the daemon stays quiesced and logs, but the common case
16649
+ // (successor crashed before binding) recovers cleanly.
16650
+ const rollbackFromUpgrade = (deadSuccessorPid) => {
16651
+ if (!upgrading || handedOver) return;
16652
+ quiescePromise = null;
16653
+ dlog(`daemon upgrade: successor pid ${deadSuccessorPid} died before handover — rolling back: re-opening the control socket and resuming normal operation`);
16654
+ upgrading = false;
16655
+ quiesced = false;
16656
+ upgradeInfo = null;
16657
+ // Capture the snapshot BEFORE clearing it — the membership reconcile below
16658
+ // reads this copy (clearing first made every rollback drop the whole fleet).
16659
+ // A null snapshot (quiesce never reached the snapshot point) means there is
16660
+ // nothing to reconcile against, so membership is left untouched.
16661
+ const rollbackSnapshotIds = quiesceSnapshotIds;
16662
+ quiesceSnapshotIds = null;
16663
+ if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
16664
+ // Re-open the accept loop. The socket path may still be held by the dead
16665
+ // successor's listen socket; remove it first (same takeover the successor
16666
+ // itself performs), then re-bind.
16667
+ if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
16668
+ // Same owner-only umask as the initial bind, so the socket is private from
16669
+ // creation (no listen()→chmod window for another local user to connect).
16670
+ const rbWin = osPlatform() === 'win32';
16671
+ const rbPrevUmask = rbWin ? null : process.umask(0o177);
16672
+ try {
16673
+ server.listen(socketPath);
16674
+ if (!rbWin) { try { chmodSync(socketPath, 0o600); } catch { /* ignore */ } }
16675
+ dlog('daemon upgrade rollback: control socket re-opened — fleet serving normally');
16676
+ } catch (err) {
16677
+ dlog(`daemon upgrade rollback: failed to re-open the control socket: ${err?.message || err}`);
16678
+ } finally {
16679
+ if (!rbWin) { try { process.umask(rbPrevUmask); } catch { /* ignore */ } }
16680
+ }
16681
+ // #309 review: reconcile the inventory. Respawns were suppressed while
16682
+ // quiesced (expired restart timers and death handlers bailed), so reschedule
16683
+ // every eligible down worker now — preserving stop/config-failure/removal
16684
+ // intent and skipping any with a live process or a pending restart timer.
16685
+ // Membership is reconciled against the quiesce snapshot: a worker ABSENT
16686
+ // from it was removed through the SUCCESSOR while this daemon was quiesced
16687
+ // (the successor owns add/remove during the handoff), so re-adding it here
16688
+ // would resurrect a deliberately removed worker.
16689
+ // The authoritative handoff inventory is what the SUCCESSOR persisted (it
16690
+ // owns add/remove during the handoff): if the state file was last written
16691
+ // by the dead successor, its worker list wins; otherwise fall back to our
16692
+ // quiesce snapshot (the successor never wrote — nothing was removed).
16693
+ let snapshotIds = rollbackSnapshotIds ? new Set(rollbackSnapshotIds) : null;
16694
+ try {
16695
+ const succState = readSupervisorState();
16696
+ if (succState && succState.pid === deadSuccessorPid && Array.isArray(succState.workers)) {
16697
+ snapshotIds = new Set(succState.workers.map((x) => x && x.id).filter(Boolean));
16698
+ }
16699
+ } catch { /* unreadable — keep the snapshot */ }
16700
+ for (const id of snapshotIds ? [...workers.keys()] : []) {
16701
+ if (!snapshotIds.has(id)) {
16702
+ dlog(`daemon upgrade rollback: worker '${id}' is not in the quiesce snapshot (removed through the successor mid-handoff?) — leaving it removed`);
16703
+ workers.delete(id);
16704
+ }
16705
+ }
16706
+ for (const w of [...workers.values()]) {
16707
+ if (w.restartTimer || (w.pid && isPidAlive(w.pid))) continue;
16708
+ if (w.removeOnExit) {
16709
+ workers.delete(w.id);
16710
+ try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
16711
+ dlog(`worker '${w.id}' removed (drained during the aborted upgrade)`);
16712
+ continue;
16713
+ }
16714
+ if (w.stopping || w.configFailed) continue;
16715
+ if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
16716
+ dlog(`daemon upgrade rollback: worker '${w.id}' was down while quiesced — rescheduling`);
16717
+ startWorker(w);
16718
+ }
16719
+ persist();
16720
+ };
16721
+
15979
16722
  const handleRequest = async (req, sock) => {
15980
16723
  const op = req && req.op;
15981
16724
  try {
@@ -15983,8 +16726,30 @@ async function runSupervisorDaemon() {
15983
16726
  case 'status':
15984
16727
  sock.write(encodeFrame(statusFrame(true)));
15985
16728
  break;
16729
+ case 'prepare-upgrade': {
16730
+ // #309: a successor daemon (spawned with NANO_SUPERVISOR_UPGRADE=1)
16731
+ // asks us to quiesce so it can bind the control socket. Refuse while
16732
+ // a worker reload roll or a shutdown owns the fleet — an upgrade on
16733
+ // top of either would strand its bookkeeping. Idempotent: the
16734
+ // successor polls until it observes the quiesced state. Bound to ONE
16735
+ // successor pid (quiesceForUpgrade throws UPGRADE_BOUND for any
16736
+ // other) so two racing successors can never both win the fleet.
16737
+ if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
16738
+ if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a worker reload is in progress — retry the daemon upgrade once it finishes', final: true })); break; }
16739
+ try {
16740
+ await quiesceForUpgrade(Number(req.pid) || null, typeof req.version === 'string' ? req.version : null);
16741
+ } catch (err) {
16742
+ if (err && err.code === 'UPGRADE_BOUND') { sock.write(encodeFrame({ ok: false, error: err.message, final: true })); break; }
16743
+ throw err;
16744
+ }
16745
+ // Report quiesced only AFTER the mutation queue has drained and the
16746
+ // snapshot persisted — the successor binds+adopts off this ack.
16747
+ sock.write(encodeFrame({ ok: true, type: 'prepare-upgrade', quiesced: true, final: true }));
16748
+ break;
16749
+ }
15986
16750
  case 'add': {
15987
16751
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
16752
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
15988
16753
  if (!req.profile) { sock.write(encodeFrame({ ok: false, error: 'add requires a profile', final: true })); break; }
15989
16754
  // A supervised worker's name is supplied out-of-band as `req.name`
15990
16755
  // (which the daemon forwards to the child as `nano work … --name`).
@@ -16007,6 +16772,7 @@ async function runSupervisorDaemon() {
16007
16772
  }
16008
16773
  case 'remove': {
16009
16774
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
16775
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16010
16776
  // #202: `req.force === false` drains each worker (finish in-flight jobs
16011
16777
  // then exit); the default remains a fast force-stop (used by reconcile
16012
16778
  // and interactive remove). A drain-remove returns immediately and the
@@ -16022,6 +16788,7 @@ async function runSupervisorDaemon() {
16022
16788
  }
16023
16789
  case 'restart': {
16024
16790
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
16791
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16025
16792
  const restarted = await serializeOp(async () => {
16026
16793
  const ids = resolveTargets(req.target);
16027
16794
  for (const id of ids) await restartWorker(id);
@@ -16038,6 +16805,7 @@ async function runSupervisorDaemon() {
16038
16805
  // and ends with a terminal `reloaded` frame — draining can take a long
16039
16806
  // time, so this MUST be a streaming op, not a one-shot request.
16040
16807
  if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
16808
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
16041
16809
  if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a reload is already in progress', final: true })); break; }
16042
16810
  // Reject reload explicitly on Windows BEFORE draining anything. The
16043
16811
  // rolling reload's graceful drain relies on SIGUSR2 (SUPERVISOR_DRAIN_SIGNAL)
@@ -16068,6 +16836,9 @@ async function runSupervisorDaemon() {
16068
16836
  // rolling pass over the same workers. The flag is reset in runReload's
16069
16837
  // `finally`.
16070
16838
  reloading = true;
16839
+ reloadTargets = [...ids];
16840
+ reloadDone = [];
16841
+ reloadCurrent = null;
16071
16842
  attachClients.add(sock);
16072
16843
  sock.write(encodeFrame({ ok: true, type: 'reloading', targets: ids }));
16073
16844
  sock.write(encodeFrame(statusFrame(false)));
@@ -16116,10 +16887,30 @@ async function runSupervisorDaemon() {
16116
16887
  // #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
16117
16888
  // jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
16118
16889
  // yield jobs) and can also ESCALATE a drain already in progress.
16890
+ // #309: a quiesced (mid-upgrade) daemon must never run the stop path —
16891
+ // it would SIGTERM/SIGUSR2 the very workers the successor has adopted.
16892
+ // The upgrade's own exit path replaces this op for the old daemon.
16893
+ if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — the successor owns the fleet now; stop it instead (its pid is in `supervisor status`)', final: true })); break; }
16119
16894
  const force = !!req.force;
16120
- // Register this client as an attach consumer so it streams drain
16121
- // progress (shrinking in-flight counts), and as a stop client so it
16122
- // gets a terminal `stopped` frame when the daemon has finished.
16895
+ // #309 review: reserve shutdown ownership HERE, synchronously with the
16896
+ // acceptance — not 20ms later when the deferred shutdown() runs. An
16897
+ // unreserved accept lets a prepare-upgrade squeeze into the defer
16898
+ // window and latch `upgrading` first; the shutdown then overtakes the
16899
+ // quiesce and SIGTERMs the very workers the successor is adopting.
16900
+ // Latching now makes the race impossible in either order: a stop
16901
+ // accepted first refuses the upgrade (prepare-upgrade checks
16902
+ // `shuttingDown`), and an upgrade quiesced first refuses the stop
16903
+ // (the `upgrading` check above).
16904
+ //
16905
+ // A SECOND stop while one is already reserved is not a fresh shutdown:
16906
+ // it only ESCALATES a drain (force) or is a no-op. Never re-latch
16907
+ // `draining` here — a force stop arriving during a graceful drain must
16908
+ // leave `draining === true` so shutdown()'s escalation branch fires,
16909
+ // and a graceful stop during a force stop must not flip it back.
16910
+ if (!shutdownStarted && !shuttingDown) {
16911
+ shuttingDown = true;
16912
+ draining = !force;
16913
+ }
16123
16914
  attachClients.add(sock);
16124
16915
  stopClients.add(sock);
16125
16916
  sock.write(encodeFrame(force
@@ -16128,7 +16919,12 @@ async function runSupervisorDaemon() {
16128
16919
  sock.write(encodeFrame(statusFrame(false)));
16129
16920
  // Kick the shutdown asynchronously; don't await it here so the control
16130
16921
  // loop keeps serving (streaming status, accepting a force escalation).
16131
- setTimeout(() => { shutdown('stop', { force }); }, 20);
16922
+ // The 250ms defer lets a `stop --force` escalation (sent right behind
16923
+ // a graceful `stop`, as `stopStream`'s sendAfter does) land and run
16924
+ // shutdown's escalation branch BEFORE the drain's body force-stops on
16925
+ // its own — preserving the pre-#309 escalation contract now that
16926
+ // `shuttingDown` is latched early (see below).
16927
+ setTimeout(() => { shutdown('stop', { force }); }, 250);
16132
16928
  break;
16133
16929
  }
16134
16930
  default:
@@ -16141,7 +16937,11 @@ async function runSupervisorDaemon() {
16141
16937
 
16142
16938
  // Bind the control socket. A stale unix socket file from a crashed daemon
16143
16939
  // would make listen() fail with EADDRINUSE even though nobody is listening;
16144
- // remove it first (we already know no live daemon owns our state).
16940
+ // remove it first. In upgrade mode the old daemon has QUIESCED (its accept
16941
+ // loop is closed) but its listen socket still holds the path until it exits,
16942
+ // so we must unlink + rebind to take over — the same operation, deliberately
16943
+ // ordered AFTER the quiesce handshake so we can never steal the socket from a
16944
+ // daemon that is still accepting (#309).
16145
16945
  if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
16146
16946
 
16147
16947
  const server = createServer((sock) => {
@@ -16207,7 +17007,208 @@ async function runSupervisorDaemon() {
16207
17007
  // and hard crashes still can't run this — the per-worker parent-death
16208
17008
  // watchdog is the backstop for those.
16209
17009
  process.once('SIGHUP', () => shutdown('SIGHUP'));
17010
+ // #309: SIGUSR1 is the upgrade handoff's "the successor owns the fleet — exit
17011
+ // WITHOUT touching the workers" signal. It is ONLY meaningful after a
17012
+ // `prepare-upgrade` quiesce (the successor sends it once it has adopted the
17013
+ // workers and retargeted their watchdogs). A SIGUSR1 outside an upgrade must
17014
+ // NOT exit (that would strand the fleet with no daemon), so it is ignored
17015
+ // unless `upgrading` is latched. Unlike `shutdown`, this path never signals
17016
+ // the workers and never clears the state file — the successor owns both now.
17017
+ process.on('SIGUSR1', () => {
17018
+ if (!upgrading) return;
17019
+ dlog(`daemon upgrade: successor confirmed ownership — old daemon (pid ${process.pid}) exiting; workers keep serving`);
17020
+ upgradeInfo = upgradeInfo ? { ...upgradeInfo, state: 'handed-over' } : upgradeInfo;
17021
+ // #309 review: NO persist here. The quiesce snapshot was this daemon's last
17022
+ // canonical write — the successor has since written its own pid + inventory,
17023
+ // and any write now would clobber it with the exiting predecessor's state.
17024
+ handedOver = true;
17025
+ quiesceSnapshotIds = null;
17026
+ if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
17027
+ if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
17028
+ if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
17029
+ process.exit(0);
17030
+ });
16210
17031
  dlog(`supervisor daemon up (pid ${process.pid}) — control ${socketPath}`);
17032
+
17033
+ // #309: adopt the predecessor's workers. The old daemon quiesced (above) and
17034
+ // its last persist() — run when it latched the upgrade — names every worker
17035
+ // it owned. Each live pid becomes an adopted worker (polled for liveness;
17036
+ // never signaled, so in-flight jobs run undisturbed). Dead pids are dropped
17037
+ // with a log line — the fleet after the swap is exactly what survived it.
17038
+ if (upgradeMode && predecessorPid != null) {
17039
+ const prior = readSupervisorState();
17040
+ const adopted = [];
17041
+ for (const rec of (prior && Array.isArray(prior.workers) ? prior.workers : [])) {
17042
+ const w = adoptWorker(rec);
17043
+ if (w) { adopted.push(w.id); continue; }
17044
+ // #309 review: transfer the FULL inventory, not only live pids. A worker
17045
+ // the predecessor recorded but that is not alive right now falls into one
17046
+ // of three kept categories — dropping it would permanently shrink the
17047
+ // fleet (the predecessor refuses respawns once quiesced):
17048
+ if (!rec || typeof rec.id !== 'string' || !rec.id) { dlog('upgrade: skipping a worker record with no id'); continue; }
17049
+ if (rec.configFailed === true) {
17050
+ // Configuration-failed: retained (NOT restarted) so the operator's
17051
+ // `configFailed` signal survives the swap and a later fix+restart still
17052
+ // targets the same record.
17053
+ workers.set(rec.id, {
17054
+ id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
17055
+ args: Array.isArray(rec.args) ? rec.args.map(String) : [],
17056
+ restarts: Number(rec.restarts) || 0, stopping: rec.stopping === true,
17057
+ removeOnExit: rec.removeOnExit === true, lastExit: rec.lastExit ?? null,
17058
+ logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
17059
+ configFailed: true, adopted: true, pid: null, child: null,
17060
+ startedAt: rec.startedAt || null, spawnedAt: rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now(),
17061
+ });
17062
+ dlog(`upgrade: retained configuration-failed worker '${rec.id}' (not restarted)`);
17063
+ continue;
17064
+ }
17065
+ if (rec.stopping === true || rec.removeOnExit === true) {
17066
+ // Draining for stop/remove: the process is already gone, so the removal
17067
+ // completes here — delete the record rather than resurrect it.
17068
+ dlog(`upgrade: worker '${rec.id}' was draining for removal and is gone — record dropped`);
17069
+ try { rmSync(supervisorWorkerActivityFile(rec.id), { force: true }); } catch { /* best effort */ }
17070
+ continue;
17071
+ }
17072
+ // Down but eligible (a crash/backoff the predecessor had not yet
17073
+ // respawned when it quiesced): reschedule it here so fleet capacity is
17074
+ // preserved across the swap.
17075
+ dlog(`upgrade: worker '${rec.id}' was down at handoff — rescheduling to preserve fleet capacity`);
17076
+ const rw = {
17077
+ id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
17078
+ args: Array.isArray(rec.args) ? rec.args.map(String) : [],
17079
+ restarts: Number(rec.restarts) || 0, stopping: false, removeOnExit: false,
17080
+ lastExit: rec.lastExit ?? null,
17081
+ logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
17082
+ configFailed: false, adopted: true,
17083
+ };
17084
+ workers.set(rw.id, rw);
17085
+ startWorker(rw);
17086
+ rw.spawnedBySuccessor = true; // its watchdog already guards this daemon
17087
+ adopted.push(rw.id);
17088
+ }
17089
+ dlog(`upgrade: adopted ${adopted.length} worker(s) from predecessor pid ${predecessorPid}: ${adopted.join(', ') || '(none)'}`);
17090
+ broadcast({ type: 'event', event: 'daemon-adopted', predecessorPid, workers: adopted });
17091
+
17092
+ // Drive the handoff to completion. The old daemon's accept loop is closed
17093
+ // (quiesce), so it can no longer be asked over the socket — instead the
17094
+ // SUCCESSOR finishes the swap: write each live worker's `.upgrade` sidecar
17095
+ // (the successor pid the worker's SIGUSR2 handler reads), SIGUSR2 the
17096
+ // workers to retarget their watchdogs here, wait (bounded) for their
17097
+ // `watchdogPid` acks, then SIGUSR1 the old daemon so it persists and exits
17098
+ // WITHOUT touching the workers. Runs async so the control loop serves
17099
+ // immediately; the `upgrade` status block stays 'adopting' until the old
17100
+ // daemon is gone.
17101
+ (async () => {
17102
+ const myPid = process.pid;
17103
+ // Every live PREDECESSOR-owned worker stays pending until it has either
17104
+ // acked the retarget (`watchdogPid === myPid`) or exited (#309 review).
17105
+ // A worker still booting has Node's DEFAULT SIGUSR2 action (terminate)
17106
+ // until its handler is up, so it is signalled only once its activity
17107
+ // marker proves readiness (finite `readyAt` + matching pid) — but it is
17108
+ // NEVER dropped from the wait set, so the predecessor (its watchdog
17109
+ // target) is not released under it. Workers this successor spawned
17110
+ // itself already guard it and need no retarget.
17111
+ const pending = retargetPending;
17112
+ for (const id of adopted) {
17113
+ const w = workers.get(id);
17114
+ if (!w || w.spawnedBySuccessor || !w.pid || !isPidAlive(w.pid)) continue;
17115
+ if (w.stopping) {
17116
+ // #309 review: already draining (stop/remove) under the predecessor.
17117
+ // Its one-shot SIGUSR2 handler was consumed by that drain, so another
17118
+ // SIGUSR2 would hit Node's default action and kill its in-flight work.
17119
+ // Never write a sidecar or signal it — just keep it pending (its
17120
+ // watchdog still guards the predecessor) until it exits.
17121
+ pending.set(id, { pid: w.pid, signalled: true, drainOnly: true });
17122
+ continue;
17123
+ }
17124
+ const actFile = w.activityFile || supervisorWorkerActivityFile(w.id);
17125
+ try { writeFileSync(`${actFile}.upgrade`, String(myPid), { mode: 0o600 }); } catch { /* best effort */ }
17126
+ pending.set(id, { pid: w.pid, signalled: false });
17127
+ }
17128
+ // Bounded wait first; past the deadline report INCOMPLETE but KEEP
17129
+ // monitoring (#309 review) so a late ack / drain still releases the
17130
+ // predecessor and its keepalive — a retried upgrade would target this
17131
+ // successor, never the retained predecessor.
17132
+ const ackDeadline = Date.now() + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS;
17133
+ let reportedIncomplete = false;
17134
+ while (pending.size > 0) {
17135
+ if (!isPidAlive(predecessorPid)) break; // predecessor gone — nothing left to release
17136
+ for (const [id, p] of [...pending]) {
17137
+ const w = workers.get(id);
17138
+ if (!w || w.pid !== p.pid || !isPidAlive(p.pid)) { pending.delete(id); continue; }
17139
+ const act = readWorkerActivity(id);
17140
+ if (p.drainOnly) {
17141
+ // Draining, not retargeting: a deferred drain request (below) is
17142
+ // delivered once the handler is provably installed; then just wait
17143
+ // for the exit.
17144
+ if (p.drainDeferred && drainSignalReadyFor(act, p.pid)) {
17145
+ p.drainDeferred = false;
17146
+ try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
17147
+ }
17148
+ continue;
17149
+ }
17150
+ if (act && act.pid === p.pid && act.watchdogPid === myPid) {
17151
+ pending.delete(id);
17152
+ // Retarget landed and the handler re-armed — now deliver any drain
17153
+ // that was requested while the retarget was outstanding.
17154
+ if (p.drainDeferred) { try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
17155
+ continue;
17156
+ }
17157
+ if (!p.signalled && drainSignalReadyFor(act, p.pid)) {
17158
+ p.signalled = true;
17159
+ try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
17160
+ }
17161
+ }
17162
+ if (pending.size === 0) break;
17163
+ if (!reportedIncomplete && Date.now() >= ackDeadline) {
17164
+ reportedIncomplete = true;
17165
+ dlog(`upgrade: ${pending.size} worker(s) have not acked the watchdog retarget within ${SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS}ms (${[...pending.keys()].join(', ')}) — NOT releasing the predecessor yet; handoff INCOMPLETE, still monitoring until they ack or exit`);
17166
+ upgradeInfo = { state: 'incomplete', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, unacked: [...pending.keys()], sinceEpochMs: upgradeStartedMs };
17167
+ persist();
17168
+ }
17169
+ await new Promise((r) => setTimeout(r, reportedIncomplete ? SUPERVISOR_ADOPT_POLL_MS : 100));
17170
+ }
17171
+ // Predecessor gone with entries still pending: deliver deferred drains
17172
+ // that are safe now (handler installed and no retarget signal in flight
17173
+ // that the drain could be mistaken for). A drain deferred behind an
17174
+ // un-acked retarget is logged — signalling now could hit a consumed
17175
+ // one-shot handler and kill in-flight work.
17176
+ for (const [id, p] of pending) {
17177
+ if (!p.drainDeferred || !isPidAlive(p.pid)) continue;
17178
+ if (p.drainOnly && drainSignalReadyFor(readWorkerActivity(id), p.pid)) {
17179
+ try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
17180
+ } else {
17181
+ dlog(`upgrade: deferred drain for worker '${id}' not delivered (retarget unacknowledged or handler not ready) — re-issue stop/remove once it settles`);
17182
+ }
17183
+ }
17184
+ pending.clear();
17185
+ // Release the old daemon: it persists a final snapshot and exits 0 without
17186
+ // touching the workers. If it is already gone (crashed mid-handoff), the
17187
+ // workers are still adopted here — nothing more to do.
17188
+ if (isPidAlive(predecessorPid)) {
17189
+ try { process.kill(predecessorPid, 'SIGUSR1'); } catch { /* already gone */ }
17190
+ }
17191
+ dlog(`upgrade: handoff complete — successor pid ${myPid} owns the fleet (${daemonVersion})`);
17192
+ })().catch((err) => dlog(`upgrade handoff error: ${err?.message || err}`));
17193
+
17194
+ // Clear the status `upgrade` block once the predecessor has actually exited
17195
+ // — the roll is complete from then on, and a quiet fleet's status should
17196
+ // not carry a stale 'adopting' line forever. Unref'd: it never keeps the
17197
+ // daemon alive.
17198
+ const clearTimer = setInterval(() => {
17199
+ if (!isPidAlive(predecessorPid)) {
17200
+ try { clearInterval(clearTimer); } catch { /* ignore */ }
17201
+ predecessorPid = null;
17202
+ // #309 review: a handoff reported INCOMPLETE that later completed (late
17203
+ // acks/drains released the predecessor) must not keep reporting
17204
+ // 'incomplete' forever — currentUpgradeBlock() returns upgradeInfo first.
17205
+ if (upgradeInfo && upgradeInfo.state === 'incomplete') upgradeInfo = null;
17206
+ persist();
17207
+ }
17208
+ }, SUPERVISOR_ADOPT_POLL_MS);
17209
+ if (typeof clearTimer.unref === 'function') clearTimer.unref();
17210
+ }
17211
+
16211
17212
  persist();
16212
17213
 
16213
17214
  // Live-view refresh: periodically re-sample worker activity and push a fresh
@@ -16600,9 +17601,9 @@ async function streamSupervisorReload(socketPath, req, logger, { label = 'fleet'
16600
17601
  * Hot-adopt new plugin code into the running fleet with zero downtime: roll
16601
17602
  * through the target workers, gracefully draining (finish in-flight jobs) and
16602
17603
  * respawning each so the new child re-reads the updated `c8ctl-plugin.js` from
16603
- * disk. Defaults to the whole fleet. Note this adopts new WORKER code only; the
16604
- * supervisor daemon itself keeps running its startup code until a full restart
16605
- * (`supervisor stop && supervisor start`).
17604
+ * disk. Defaults to the whole fleet. Note this adopts new WORKER code only;
17605
+ * the supervisor daemon itself is rolled separately, without stopping the
17606
+ * fleet, by `supervisor upgrade` (issue #309).
16606
17607
  */
16607
17608
  async function supervisorReloadCmd(req) {
16608
17609
  const logger = getLogger();
@@ -16617,6 +17618,116 @@ async function supervisorReloadCmd(req) {
16617
17618
  if (outcome === 'error') process.exit(1);
16618
17619
  }
16619
17620
 
17621
+ /**
17622
+ * Rolling hot-upgrade of the supervisor DAEMON process (issue #309) — the
17623
+ * daemon-level counterpart of `supervisor reload` (which rolls only workers).
17624
+ * The daemon keeps the module graph it imported at start, so adopting new
17625
+ * SUPERVISOR-side code used to require `supervisor stop && start` — a fleet
17626
+ * outage (or, with --force, killed in-flight jobs). This command swaps the
17627
+ * parent process instead:
17628
+ *
17629
+ * 1. spawn a SUCCESSOR daemon (detached, NANO_SUPERVISOR_UPGRADE=1) running
17630
+ * the on-disk — post-`nano update` — plugin code;
17631
+ * 2. the successor asks the old daemon to QUIESCE (`prepare-upgrade`: accept
17632
+ * loop closed, mutations refused), then binds the canonical control
17633
+ * socket and ADOPTS the old daemon's workers from the persisted state
17634
+ * (pid-polled; the workers are never signaled, so in-flight jobs run
17635
+ * undisturbed);
17636
+ * 3. the successor then finishes the swap itself: it writes each worker's
17637
+ * `<activityFile>.upgrade` sidecar, SIGUSR2s the workers to retarget their
17638
+ * parent-death watchdogs at it, waits (bounded) for the `watchdogPid`
17639
+ * acks, and SIGUSR1s the old daemon — which persists a final snapshot and
17640
+ * exits 0 WITHOUT touching the workers.
17641
+ *
17642
+ * Workers never die mid-job; only the parent is swapped. A successor that
17643
+ * cannot reach a quiesced old daemon exits 1 and leaves the fleet untouched.
17644
+ * This client just spawns the successor and waits for the swap to land; Ctrl-C
17645
+ * detaches — the handoff completes in the background.
17646
+ */
17647
+ async function supervisorUpgradeCmd() {
17648
+ const logger = getLogger();
17649
+ if (osPlatform() === 'win32') {
17650
+ logger.error('supervisor upgrade is not supported on Windows (its worker watchdog retarget relies on SIGUSR2, which Windows cannot deliver) — use `supervisor stop` + `start` to adopt new supervisor code.');
17651
+ process.exit(1);
17652
+ }
17653
+ const running = await liveSupervisor();
17654
+ if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
17655
+ const socketPath = running.socket || getSupervisorSocketPath();
17656
+ const oldPid = running.pid;
17657
+ const fromVersion = running.version || null;
17658
+ const toVersion = (() => { try { return pluginPackage().version; } catch { return null; } })();
17659
+
17660
+ // 1. Spawn the successor. NANO_SUPERVISOR_UPGRADE=1 puts its boot into
17661
+ // upgrade mode (quiesce handshake → bind → adopt → drive the handoff). We
17662
+ // deliberately do NOT clear the state file first — the successor reads the
17663
+ // old daemon's worker list from it. Ctrl-C detaches the waiter below; the
17664
+ // successor is detached and completes the swap regardless.
17665
+ const { exec, entry } = c8ctlInvocation();
17666
+ mkdirSync(getSupervisorLogDir(), { recursive: true });
17667
+ const logFile = supervisorDaemonLogFile();
17668
+ let fd;
17669
+ try { fd = openSync(logFile, 'a'); } catch { fd = 'ignore'; }
17670
+ const child = spawn(exec, [entry, 'nano', 'supervisor', '__daemon'], {
17671
+ env: { ...process.env, NANO_SUPERVISOR_UPGRADE: '1' },
17672
+ detached: true,
17673
+ stdio: ['ignore', fd, fd],
17674
+ });
17675
+ child.unref();
17676
+ if (typeof fd === 'number') { try { closeSync(fd); } catch { /* ignore */ } }
17677
+ if (typeof child.pid !== 'number') { logger.error('Failed to spawn the successor supervisor daemon.'); process.exit(1); }
17678
+ const successorPid = child.pid;
17679
+ logger.info(`Upgrading the supervisor daemon (pid ${oldPid}${fromVersion ? `, v${fromVersion}` : ''}) → successor pid ${successorPid}${toVersion ? ` (v${toVersion})` : ''}. Workers keep serving; only the parent process is swapped. Press Ctrl-C to detach.`);
17680
+ // #309 review (detached-breaks-supervision): the successor is spawned DETACHED
17681
+ // in this session, not under launchd/systemd. If a service unit is installed,
17682
+ // warn the operator that the new daemon is no longer service-supervised (and
17683
+ // that the old unit's clean exit 0 won't restart it — the generated units use
17684
+ // crash-only policies), and tell them how to re-establish supervision.
17685
+ try {
17686
+ if (supervisorServiceInstalled()) {
17687
+ logger.warn('A supervisor service is installed, but the upgraded daemon runs DETACHED in this session — it is no longer under launchd/systemd supervision (and the old unit\'s clean exit will not auto-restart it). After the upgrade, run `c8ctl nano supervisor install` (or `stop` + `start`) to bring the new-code daemon back under the service.');
17688
+ }
17689
+ } catch { /* best effort — never block the upgrade on a detection hiccup */ }
17690
+
17691
+ // 2. Wait for the swap to land: the successor takes the socket (status names
17692
+ // its pid) and the old daemon exits (released by the successor's SIGUSR1
17693
+ // once the workers' watchdogs are retargeted). Ctrl-C detaches — the
17694
+ // handoff completes in the background.
17695
+ let detached = false;
17696
+ const onSigint = () => { detached = true; };
17697
+ process.on('SIGINT', onSigint);
17698
+ const deadline = Date.now() + SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS + 10_000;
17699
+ try {
17700
+ for (;;) {
17701
+ if (detached) {
17702
+ logger.info('Detached — the daemon upgrade completes in the background. Rerun `nano supervisor status` to confirm.');
17703
+ return;
17704
+ }
17705
+ if (!isPidAlive(successorPid)) {
17706
+ logger.error(`The successor daemon (pid ${successorPid}) died during the upgrade — the old daemon (pid ${oldPid}) may still be in place. See ${logFile}.`);
17707
+ process.exit(1);
17708
+ }
17709
+ let successorServing = false;
17710
+ try {
17711
+ const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
17712
+ successorServing = !!(res && res.ok && res.daemon && res.daemon.pid === successorPid);
17713
+ } catch { /* socket mid-handoff — retry */ }
17714
+ const oldGone = !isPidAlive(oldPid);
17715
+ if (successorServing && oldGone) {
17716
+ logger.info(`Supervisor daemon upgraded${fromVersion || toVersion ? ` (${fromVersion ?? '?'} → ${toVersion ?? '?'})` : ''} — successor pid ${successorPid} owns the fleet; no worker was stopped.`);
17717
+ return;
17718
+ }
17719
+ if (Date.now() >= deadline) {
17720
+ if (!successorServing) logger.warn(`The successor daemon (pid ${successorPid}) did not take the control socket in time — the old daemon (pid ${oldPid}) may still own the fleet. Check \`nano supervisor status\` and ${logFile}.`);
17721
+ else logger.warn(`The old daemon (pid ${oldPid}) has not exited yet — the watchdog handoff is still completing. Check \`nano supervisor status\` in a moment.`);
17722
+ process.exit(1);
17723
+ }
17724
+ await new Promise((r) => setTimeout(r, 150));
17725
+ }
17726
+ } finally {
17727
+ try { process.removeListener('SIGINT', onSigint); } catch { /* ignore */ }
17728
+ }
17729
+ }
17730
+
16620
17731
  /**
16621
17732
  * Re-read every supervised worker's cached gh auth token WITHOUT draining
16622
17733
  * in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
@@ -16724,7 +17835,15 @@ async function supervisorStopCmd(force = false) {
16724
17835
  const { frames, rest } = decodeFrames(buf);
16725
17836
  buf = rest;
16726
17837
  for (const frame of frames) {
16727
- if (frame && frame.type === 'stopping') {
17838
+ // #309 review: an ok:false terminal frame is a REFUSAL, not a stop —
17839
+ // the daemon rejected the op (a daemon upgrade owns the fleet; the
17840
+ // successor holds the socket now). Treating it as a successful stop
17841
+ // would wait indefinitely for the wrong pid and, with --force,
17842
+ // SIGKILL the quiesced predecessor's process group — including the
17843
+ // workers the successor adopted — then erase the successor's state.
17844
+ if (frame && frame.ok === false) {
17845
+ finish({ error: String(frame.error || 'the supervisor refused the stop request') });
17846
+ } else if (frame && frame.type === 'stopping') {
16728
17847
  logger.info('Aborting in-flight work — killing harnesses and yielding jobs for retry...');
16729
17848
  } else if (frame && frame.event === 'draining-escalated') {
16730
17849
  logger.info('Escalating to --force — aborting in-flight work...');
@@ -16750,6 +17869,15 @@ async function supervisorStopCmd(force = false) {
16750
17869
 
16751
17870
  if (outcome === 'detached') return;
16752
17871
 
17872
+ // A refused stop (ok:false) is an ERROR outcome: surface it and exit before
17873
+ // any pid wait, force escalation, or state cleanup — the daemon that refused
17874
+ // (e.g. a quiesced predecessor mid-upgrade) is not ours to reap, and the
17875
+ // state file now names the successor that owns the fleet.
17876
+ if (outcome && typeof outcome === 'object' && outcome.error) {
17877
+ logger.error(`Supervisor stop refused: ${outcome.error}`);
17878
+ process.exit(1);
17879
+ }
17880
+
16753
17881
  if (outcome === 'unreachable') {
16754
17882
  if (force) {
16755
17883
  // Socket unreachable — fall back to signalling the daemon pid directly.
@@ -16765,6 +17893,15 @@ async function supervisorStopCmd(force = false) {
16765
17893
  // backstop so a wedged daemon/worker group can't linger. The window must cover
16766
17894
  // the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
16767
17895
  // group while a worker is still promoting its durable recovery ref (#295).
17896
+ // #309 review: if the daemon we asked is mid-upgrade (quiesced), the pid we
17897
+ // captured is the PREDECESSOR's — it exits on the successor's handover, not on
17898
+ // our refused/naive stop. Say so, and never SIGKILL its process group (the
17899
+ // successor's adopted workers share it) or clear the successor's state.
17900
+ if (running.upgrade && typeof running.upgrade === 'object') {
17901
+ const successorPid = Number(running.upgrade.successorPid);
17902
+ logger.warn(`The supervisor is mid-upgrade (handing the fleet to successor pid ${Number.isInteger(successorPid) && successorPid > 0 ? successorPid : 'unknown'}) — stop the successor instead; this daemon exits when the handoff completes.`);
17903
+ return;
17904
+ }
16768
17905
  if (force) {
16769
17906
  const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
16770
17907
  while (Date.now() < deadline) {
@@ -17471,8 +18608,14 @@ async function supervisorCommand(req, flags, ctx) {
17471
18608
  await supervisorRestartCmd(req);
17472
18609
  return;
17473
18610
  case 'reload':
18611
+ // `reload --daemon` is the daemon-upgrade alias (issue #309): the plain
18612
+ // op rolls WORKER code; this rolls the daemon process itself.
18613
+ if (flags?.daemon) { await supervisorUpgradeCmd(); return; }
17474
18614
  await supervisorReloadCmd(req);
17475
18615
  return;
18616
+ case 'upgrade':
18617
+ await supervisorUpgradeCmd();
18618
+ return;
17476
18619
  case 'reload-credentials':
17477
18620
  await supervisorReloadCredentialsCmd(req);
17478
18621
  return;
@@ -17484,7 +18627,7 @@ async function supervisorCommand(req, flags, ctx) {
17484
18627
  supervisorLogsCmd(req);
17485
18628
  return;
17486
18629
  default:
17487
- getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
18630
+ getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|upgrade|reload-credentials|stop|logs|attach`);
17488
18631
  process.exit(1);
17489
18632
  }
17490
18633
  }
@@ -20383,6 +21526,7 @@ export {
20383
21526
  summarizeSupervisorWorker,
20384
21527
  formatSupervisorStatus,
20385
21528
  statusFromState,
21529
+ stateFromStatus,
20386
21530
  supervisorDaemonDescriptor,
20387
21531
  formatSupervisorLogsLines,
20388
21532
  reageSupervisorStatus,
@@ -20399,6 +21543,7 @@ export {
20399
21543
  normalizeAgenticMessage,
20400
21544
  buildActivityPayload,
20401
21545
  activityMarkerReadyFor,
21546
+ drainSignalReadyFor,
20402
21547
  reloadCredentialsReady,
20403
21548
  waitForChildExit,
20404
21549
  supervisorWorkerActivityFile,
@@ -20411,6 +21556,7 @@ export {
20411
21556
  supervisorStartCmd,
20412
21557
  supervisorAddCmd,
20413
21558
  supervisorReloadCmd,
21559
+ supervisorUpgradeCmd,
20414
21560
  supervisorReloadCredentialsCmd,
20415
21561
  workforceReloadCmd,
20416
21562
  runningSupervisor,
@@ -20519,7 +21665,8 @@ export const metadata = {
20519
21665
  { command: 'c8ctl nano supervisor add decider', description: 'Add a supervised worker (forwarding work flags) to the running supervisor' },
20520
21666
  { command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
20521
21667
  { command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
20522
- { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
21668
+ { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only — use `supervisor upgrade` for the daemon)' },
21669
+ { command: 'c8ctl nano supervisor upgrade', description: 'Rolling hot-upgrade of the supervisor DAEMON itself (alias: `reload --daemon`): a successor daemon on the new code takes the control socket and ADOPTS the running workers (no drain — in-flight jobs never stop), then the old daemon exits. Status shows `upgrade: rolling <from> → <to>` while it runs' },
20523
21670
  { command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
20524
21671
  { command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
20525
21672
  { command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
@@ -20601,6 +21748,7 @@ export const commands = {
20601
21748
  worker: { type: 'string', multiple: true, description: 'supervisor start: profile to launch as a supervised worker (repeatable)' },
20602
21749
  instances: { type: 'string', description: `supervisor add / workforce add: spawn/compose N distinct instances of the profile in one call (default 1, max ${MAX_ADD_INSTANCES}; for supervisor add cannot combine with --name)` },
20603
21750
  attach: { type: 'boolean', description: 'supervisor start: attach the interactive console after starting the daemon' },
21751
+ daemon: { type: 'boolean', description: 'supervisor reload: roll the DAEMON process itself onto the on-disk code (alias of `supervisor upgrade`) instead of rolling worker code' },
20604
21752
  manifest: { type: 'string', description: `workforce: manifest name to operate on (default ${DEFAULT_WORKFORCE_MANIFEST}); each subcommand reads/writes <stateHome>/workforce/<name>.json. Renamed from --profile (which now collides with c8ctl's global connection-profile flag).` },
20605
21753
  roles: { type: 'string', description: 'workforce add: comma-separated role list for the entry (→ --job-type <rank>:<role> at start); mutually exclusive with --auto' },
20606
21754
  },