c8ctl-plugin-nano 1.71.0 → 1.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -9
- package/c8ctl-plugin.js +1179 -31
- package/package.json +8 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -295,6 +295,40 @@ const SUPERVISOR_RELOAD_READY_TIMEOUT_MS = Math.max(
|
|
|
295
295
|
0,
|
|
296
296
|
Number.parseInt(process.env.NANO_SUPERVISOR_RELOAD_READY_TIMEOUT_MS ?? '', 10) || 30_000,
|
|
297
297
|
);
|
|
298
|
+
// Rolling daemon hot-upgrade (issue #309). A `supervisor upgrade` hands the
|
|
299
|
+
// fleet to a SUCCESSOR daemon process running the on-disk (new) plugin code:
|
|
300
|
+
// the old daemon quiesces (stops accepting control connections + stops
|
|
301
|
+
// restarting workers), asks each worker to re-arm its parent-death watchdog at
|
|
302
|
+
// the successor's pid (SIGUSR2 + NANO_SUPERVISOR_UPGRADE_PID — a retarget, NOT
|
|
303
|
+
// a drain), persists a final state snapshot, and exits WITHOUT touching the
|
|
304
|
+
// workers. The successor binds the canonical control socket and adopts the
|
|
305
|
+
// still-running worker pids. Workers never die mid-job; only the parent swaps.
|
|
306
|
+
//
|
|
307
|
+
// How long the old daemon waits for workers to acknowledge the watchdog
|
|
308
|
+
// retarget (by stamping `watchdogPid` on their activity marker) before it exits
|
|
309
|
+
// anyway. Bounded so a wedged/older worker can't stall the upgrade forever —
|
|
310
|
+
// the successor adopts every live pid regardless, so an un-acked worker is
|
|
311
|
+
// still parented (its watchdog self-reaps only if the successor ALSO dies).
|
|
312
|
+
const SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS = Math.max(
|
|
313
|
+
0,
|
|
314
|
+
Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS ?? '', 10) || 15_000,
|
|
315
|
+
);
|
|
316
|
+
// How long a freshly spawned successor waits for the old daemon to quiesce
|
|
317
|
+
// before giving up (it must never bind the socket while the old daemon still
|
|
318
|
+
// accepts on it). Generous: quiesce is prompt (server.close + a latch), but a
|
|
319
|
+
// loaded machine can delay the old daemon's socket loop.
|
|
320
|
+
const SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS = Math.max(
|
|
321
|
+
0,
|
|
322
|
+
Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS ?? '', 10) || 30_000,
|
|
323
|
+
);
|
|
324
|
+
// How often the successor polls an adopted worker's pid for liveness. An
|
|
325
|
+
// adopted worker has no ChildProcess handle (the successor never spawned it),
|
|
326
|
+
// so there is no 'exit' event — this poll is the death detector that feeds the
|
|
327
|
+
// same restart-backoff path a spawned child's exit would.
|
|
328
|
+
const SUPERVISOR_ADOPT_POLL_MS = Math.max(
|
|
329
|
+
100,
|
|
330
|
+
Number.parseInt(process.env.NANO_SUPERVISOR_ADOPT_POLL_MS ?? '', 10) || 2_000,
|
|
331
|
+
);
|
|
298
332
|
// Upper bound on one `--auto` engine-read reconcile (enumerate deployed
|
|
299
333
|
// definitions + fetch each BPMN). A read that stalls past this is treated as a
|
|
300
334
|
// transient failure so the running poller set is KEPT and, crucially, shutdown
|
|
@@ -474,6 +508,19 @@ function activityMarkerReadyFor(act, pid) {
|
|
|
474
508
|
return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
|
|
475
509
|
}
|
|
476
510
|
|
|
511
|
+
/**
|
|
512
|
+
* Whether a worker's SIGUSR2 (drain/retarget) handler is provably installed
|
|
513
|
+
* (#309 review): its marker belongs to `pid` AND carries a finite
|
|
514
|
+
* `signalReadyAt` (stamped right after the handler is armed, independent of
|
|
515
|
+
* leasing). Falls back to `readyAt` for a worker that predates the field (its
|
|
516
|
+
* handler is armed before its first activation). Signalling earlier would hit
|
|
517
|
+
* Node's DEFAULT SIGUSR2 action and terminate the worker. Pure for unit tests.
|
|
518
|
+
*/
|
|
519
|
+
function drainSignalReadyFor(act, pid) {
|
|
520
|
+
if (!act || act.pid !== pid) return false;
|
|
521
|
+
return Number.isFinite(act.signalReadyAt) || Number.isFinite(act.readyAt);
|
|
522
|
+
}
|
|
523
|
+
|
|
477
524
|
/**
|
|
478
525
|
* Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
|
|
479
526
|
* All three must hold (#293): the worker exists, its PID is alive, AND its
|
|
@@ -11281,9 +11328,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
|
|
|
11281
11328
|
* `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
|
|
11282
11329
|
* longer running here, so it rides the marker as a flagged, observational entry
|
|
11283
11330
|
* without making an idle worker look busy or inflating the drain's in-flight count.
|
|
11284
|
-
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
|
|
11331
|
+
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null), watchdogPid:(number|null|undefined), signalReadyAt?:(number|null) }} fields
|
|
11285
11332
|
*/
|
|
11286
|
-
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
|
|
11333
|
+
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth, watchdogPid, signalReadyAt }) {
|
|
11287
11334
|
const jobList = Array.isArray(jobs) ? jobs : [];
|
|
11288
11335
|
return {
|
|
11289
11336
|
pid,
|
|
@@ -11296,11 +11343,23 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt,
|
|
|
11296
11343
|
// SDK client and begun leasing), the producer stamps this; null until then.
|
|
11297
11344
|
// The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
|
|
11298
11345
|
readyAt: readyAt ?? null,
|
|
11346
|
+
// #309 review: stamped once this worker's SIGUSR2 handler is INSTALLED —
|
|
11347
|
+
// independent of leasing readiness (`readyAt` needs a nonempty poll set, so
|
|
11348
|
+
// a healthy `--auto` worker with no deployed types never stamps it). The
|
|
11349
|
+
// daemon-upgrade handoff gates its retarget SIGUSR2 on THIS, not readyAt.
|
|
11350
|
+
signalReadyAt: signalReadyAt ?? null,
|
|
11299
11351
|
// #293: the cached gh auth token's observable identity — { present,
|
|
11300
11352
|
// fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
|
|
11301
11353
|
// `supervisor status` can show the credential's age/scopes per worker. Null
|
|
11302
11354
|
// on a worker that has not primed the cache yet (or predates the field).
|
|
11303
11355
|
ghAuth: ghAuth ?? null,
|
|
11356
|
+
// #309: the daemon pid this worker's parent-death watchdog currently guards.
|
|
11357
|
+
// A rolling daemon upgrade SIGUSR2s each worker to RETARGET that watchdog at
|
|
11358
|
+
// the successor daemon before the old one exits; the old daemon waits
|
|
11359
|
+
// (bounded) for this stamp to name the successor — the ack that the worker
|
|
11360
|
+
// survives the handoff — so it never exits while a worker would still read
|
|
11361
|
+
// the (expected) reparent-to-init as an orphan event and self-reap.
|
|
11362
|
+
watchdogPid: Number.isInteger(watchdogPid) ? watchdogPid : null,
|
|
11304
11363
|
};
|
|
11305
11364
|
}
|
|
11306
11365
|
|
|
@@ -12118,9 +12177,13 @@ async function workAgent(req, flags, ctx) {
|
|
|
12118
12177
|
// this worker would otherwise idle forever as an orphan reparented to init.
|
|
12119
12178
|
// The watchdog makes the child self-exit the moment it is reparented. A
|
|
12120
12179
|
// standalone `nano work` (no activity file) keeps classic nohup semantics.
|
|
12180
|
+
// The handle is retained so a rolling daemon upgrade (#309) can RETARGET the
|
|
12181
|
+
// same installation at the successor daemon's pid instead of self-reaping
|
|
12182
|
+
// when the old daemon hands over and exits (see the SIGUSR2 handler below).
|
|
12183
|
+
let parentWatchdog = null;
|
|
12121
12184
|
if (activityFile) {
|
|
12122
12185
|
const daemonPid = Number.parseInt(process.env.NANO_SUPERVISOR_DAEMON_PID ?? '', 10);
|
|
12123
|
-
installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
|
|
12186
|
+
parentWatchdog = installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
|
|
12124
12187
|
}
|
|
12125
12188
|
const activeJobs = new Map(); // jobKey -> { type, since (ms epoch) }
|
|
12126
12189
|
// #254: settlement-pending ghosts — jobKey -> { type, since, phase, leaseToken }.
|
|
@@ -12267,6 +12330,9 @@ async function workAgent(req, flags, ctx) {
|
|
|
12267
12330
|
// waits for this stamp before draining the next worker so a bare spawn/PID is
|
|
12268
12331
|
// never mistaken for a serving replacement.
|
|
12269
12332
|
let readyAt = null;
|
|
12333
|
+
// #309 review: set once the SIGUSR2 (drain/retarget) handler is installed.
|
|
12334
|
+
let signalReadyAt = null;
|
|
12335
|
+
const markSignalReady = () => { signalReadyAt = Date.now(); };
|
|
12270
12336
|
const writeActivity = () => {
|
|
12271
12337
|
if (!activityFile) return;
|
|
12272
12338
|
const jobs = [...activeJobs.entries()].map(([key, v]) => ({ key, type: v.type, since: v.since }));
|
|
@@ -12291,7 +12357,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
12291
12357
|
// #293: publish the cached gh token's identity (fingerprint/scopes/age —
|
|
12292
12358
|
// never the value) so `supervisor status` can answer "is this worker holding
|
|
12293
12359
|
// a stale token?" in one line.
|
|
12294
|
-
const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
|
|
12360
|
+
const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, signalReadyAt, ghAuth: ghAuthTokenInfo(), watchdogPid: parentWatchdog && typeof parentWatchdog.watchedPid === 'function' ? parentWatchdog.watchedPid() : null });
|
|
12295
12361
|
const tmp = `${activityFile}.${process.pid}.tmp`;
|
|
12296
12362
|
try {
|
|
12297
12363
|
mkdirSync(dirname(activityFile), { recursive: true });
|
|
@@ -14330,7 +14396,62 @@ async function workAgent(req, flags, ctx) {
|
|
|
14330
14396
|
setTimeout(tick, 200);
|
|
14331
14397
|
};
|
|
14332
14398
|
|
|
14333
|
-
|
|
14399
|
+
// #309: the SIGUSR2 handler is a named, re-armable function. The legacy path
|
|
14400
|
+
// (graceful drain) fires once and the worker exits, so `once` semantics
|
|
14401
|
+
// suffice there; but the daemon-upgrade path (below) does NOT exit the
|
|
14402
|
+
// worker, so it re-arms the handler for a later drain/upgrade.
|
|
14403
|
+
const onDrainSignal = () => {
|
|
14404
|
+
// #309 rolling daemon upgrade: the daemon reuses SIGUSR2 as its
|
|
14405
|
+
// per-worker signal, but when it names a successor daemon pid this is a
|
|
14406
|
+
// watchdog RETARGET, not a drain — the old daemon is handing the fleet to
|
|
14407
|
+
// a successor running the new plugin code and this worker must KEEP
|
|
14408
|
+
// SERVING (jobs in flight are never touched; only the parent process is
|
|
14409
|
+
// swapped). The successor pid arrives two ways, in priority order:
|
|
14410
|
+
// 1. `<activityFile>.upgrade` — written by the old daemon just before
|
|
14411
|
+
// it SIGUSR2s (authoritative: the daemon learns the successor pid
|
|
14412
|
+
// only at handoff time, after this worker was spawned, so it cannot
|
|
14413
|
+
// have ridden this worker's env);
|
|
14414
|
+
// 2. NANO_SUPERVISOR_UPGRADE_PID — covers a worker spawned mid-upgrade.
|
|
14415
|
+
// Re-arm the parent-death watchdog at the successor so the old daemon's
|
|
14416
|
+
// (expected) exit — which reparents this worker to init — is not read as
|
|
14417
|
+
// an orphan event, then republish the activity marker: the old daemon
|
|
14418
|
+
// waits (bounded) for the marker's `watchdogPid` to name the successor
|
|
14419
|
+
// before it exits, so the handoff never strands a worker that would
|
|
14420
|
+
// self-reap mid-upgrade.
|
|
14421
|
+
let upgradePid = Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
|
|
14422
|
+
if (activityFile) {
|
|
14423
|
+
try {
|
|
14424
|
+
const raw = readFileSync(`${activityFile}.upgrade`, 'utf8').trim();
|
|
14425
|
+
const fromFile = Number.parseInt(raw, 10);
|
|
14426
|
+
if (Number.isInteger(fromFile) && fromFile > 0) upgradePid = fromFile;
|
|
14427
|
+
} catch { /* no upgrade file — fall back to the env var */ }
|
|
14428
|
+
}
|
|
14429
|
+
if (Number.isInteger(upgradePid) && upgradePid > 0 && parentWatchdog && typeof parentWatchdog.retarget === 'function') {
|
|
14430
|
+
const ok = parentWatchdog.retarget(upgradePid);
|
|
14431
|
+
logger.info(`Received ${SUPERVISOR_DRAIN_SIGNAL} — daemon upgrade in progress: parent-death watchdog ${ok ? `retargeted to successor daemon pid ${upgradePid}` : `could NOT be retargeted (invalid pid)`}; this worker keeps serving.`);
|
|
14432
|
+
// Consume the retarget request: delete the `.upgrade` sidecar so a LATER
|
|
14433
|
+
// SIGUSR2 (a genuine drain, or a later upgrade that has not written a
|
|
14434
|
+
// fresh sidecar yet) is not misread as another retarget to this same
|
|
14435
|
+
// successor. Without this the sidecar is sticky — every later SIGUSR2
|
|
14436
|
+
// would retarget to the same (possibly long-gone) pid instead of
|
|
14437
|
+
// draining, breaking graceful stop / drain-remove / worker reload after
|
|
14438
|
+
// an upgrade (#309 review). Best-effort: a leftover only matters if it
|
|
14439
|
+
// names a live pid, and the successor also clears it on adopt.
|
|
14440
|
+
if (activityFile) { try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ } }
|
|
14441
|
+
try { writeActivityRef?.(); } catch { /* best effort — the ack stamp is advisory */ }
|
|
14442
|
+
// Re-arm: an upgrade that aborts (successor died before binding) must
|
|
14443
|
+
// leave this worker drainable by the surviving old daemon, and a LATER
|
|
14444
|
+
// upgrade/drain needs the handler again. (The drain path never re-arms —
|
|
14445
|
+
// the worker is on its way out.)
|
|
14446
|
+
process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
|
|
14447
|
+
return;
|
|
14448
|
+
}
|
|
14449
|
+
gracefulDrain(SUPERVISOR_DRAIN_SIGNAL);
|
|
14450
|
+
};
|
|
14451
|
+
process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
|
|
14452
|
+
// #309 review: publish that the handler is installed, so an upgrade handoff
|
|
14453
|
+
// can retarget this worker even before (or without ever) leasing.
|
|
14454
|
+
try { markSignalReady(); writeActivityRef?.(); } catch { /* best effort */ }
|
|
14334
14455
|
process.once('SIGINT', () => { forceAbort('SIGINT'); });
|
|
14335
14456
|
process.once('SIGTERM', () => { forceAbort('SIGTERM'); });
|
|
14336
14457
|
});
|
|
@@ -15128,7 +15249,36 @@ function formatSupervisorStatus(status) {
|
|
|
15128
15249
|
const onDiskVersion = status.pluginVersion
|
|
15129
15250
|
?? (() => { try { return pluginPackage().version; } catch { return null; } })();
|
|
15130
15251
|
if (onDiskVersion && d.version && onDiskVersion !== d.version) {
|
|
15131
|
-
lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code
|
|
15252
|
+
lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code, and \`c8ctl nano supervisor upgrade\` to roll the daemon itself onto the new code)`);
|
|
15253
|
+
}
|
|
15254
|
+
// #309: while a rolling daemon upgrade is in flight the status frame (and the
|
|
15255
|
+
// persisted-state fallback) carries an `upgrade` block — render it as the
|
|
15256
|
+
// daemon-roll line, mirroring the worker-roll visibility (`rolling <from> →
|
|
15257
|
+
// <to>`). Absent on a quiet daemon.
|
|
15258
|
+
const up = status.upgrade;
|
|
15259
|
+
if (up && typeof up === 'object' && typeof up.state === 'string') {
|
|
15260
|
+
const from = up.fromVersion ?? '?';
|
|
15261
|
+
const to = up.toVersion ?? '?';
|
|
15262
|
+
const age = Number.isFinite(up.sinceEpochMs) ? ` (${formatDuration(Math.max(0, Date.now() - up.sinceEpochMs))})` : '';
|
|
15263
|
+
lines.push(` upgrade: ${up.state} — daemon ${from} → ${to}${age}`);
|
|
15264
|
+
}
|
|
15265
|
+
// A rolling reload can run for a long time (each worker drains in-flight
|
|
15266
|
+
// work). Surface it on every `status` so a detached operator — or someone
|
|
15267
|
+
// who only ever polls status — can see the roll is live and must not
|
|
15268
|
+
// `restart` the daemon. `reload` is the progress object from the live
|
|
15269
|
+
// socket frame; absent on the socket-unreachable fallback.
|
|
15270
|
+
const roll = status.reload;
|
|
15271
|
+
if (roll && roll.inProgress) {
|
|
15272
|
+
const done = Array.isArray(roll.reloaded) ? roll.reloaded.length : 0;
|
|
15273
|
+
const total = Number(roll.total) || 0;
|
|
15274
|
+
const current = roll.current ? String(roll.current) : null;
|
|
15275
|
+
const parts = [`rolling ${done}/${total} adopted`];
|
|
15276
|
+
if (current) parts.push(`draining ${current}`);
|
|
15277
|
+
if (onDiskVersion && d.version && onDiskVersion !== d.version) {
|
|
15278
|
+
parts.push(`${d.version} → ${onDiskVersion}`);
|
|
15279
|
+
}
|
|
15280
|
+
parts.push('do not restart the daemon');
|
|
15281
|
+
lines.push(` reload: ${parts.join(' — ')}`);
|
|
15132
15282
|
}
|
|
15133
15283
|
if (d.startedAt) lines.push(` started: ${d.startedAt}`);
|
|
15134
15284
|
if (d.socket) lines.push(` control: ${d.socket}`);
|
|
@@ -15245,6 +15395,10 @@ function stateFromStatus(res, socketPath) {
|
|
|
15245
15395
|
version: res.daemon?.version,
|
|
15246
15396
|
socket: res.daemon?.socket || socketPath,
|
|
15247
15397
|
logFile: res.daemon?.logFile,
|
|
15398
|
+
// #309 review: carry the daemon's in-flight upgrade block so a stop client
|
|
15399
|
+
// can see the handoff (and the successor's pid) even when it connected to
|
|
15400
|
+
// the PREDECESSOR just before the socket flipped.
|
|
15401
|
+
...(res.upgrade && typeof res.upgrade === 'object' ? { upgrade: res.upgrade } : {}),
|
|
15248
15402
|
workers: res.workers || [],
|
|
15249
15403
|
};
|
|
15250
15404
|
}
|
|
@@ -15259,6 +15413,10 @@ function stateFromStatus(res, socketPath) {
|
|
|
15259
15413
|
function statusFromState(running) {
|
|
15260
15414
|
return {
|
|
15261
15415
|
daemon: supervisorDaemonDescriptor(running),
|
|
15416
|
+
// #309: the persisted `upgrade` block rides the fallback render too — a
|
|
15417
|
+
// mid-handoff daemon whose socket is briefly unreachable must still show
|
|
15418
|
+
// the roll (same visibility guarantee as `version`/`logFile` above).
|
|
15419
|
+
...(running && running.upgrade && typeof running.upgrade === 'object' ? { upgrade: running.upgrade } : {}),
|
|
15262
15420
|
workers: (running.workers || []).map((w) => summarizeSupervisorWorker(w)),
|
|
15263
15421
|
};
|
|
15264
15422
|
}
|
|
@@ -15349,6 +15507,20 @@ function waitForChildExit(child, timeoutMs) {
|
|
|
15349
15507
|
* Unix-only: reparent-to-init is a POSIX semantic; Windows job objects are the
|
|
15350
15508
|
* equivalent lifecycle tie and are out of scope here (matching the daemon's
|
|
15351
15509
|
* other `win32` guards). Returns a canceller so callers/tests can stop it.
|
|
15510
|
+
*
|
|
15511
|
+
* Rolling daemon hot-upgrade (issue #309): `retargetToPid` re-arms the SAME
|
|
15512
|
+
* watchdog installation at a NEW daemon pid — the successor daemon the old one
|
|
15513
|
+
* is handing the fleet to. Without it the swap is impossible: the old daemon
|
|
15514
|
+
* exits once the successor owns the socket, the worker reparents to init, and
|
|
15515
|
+
* an un-retargeted watchdog would read exactly that as "daemon died
|
|
15516
|
+
* ungracefully" and self-reap — killing every in-flight job mid-upgrade, the
|
|
15517
|
+
* precise outcome the rolling replace exists to avoid. The retarget is safe
|
|
15518
|
+
* against a stale signal because the watched pid is only ever advanced to the
|
|
15519
|
+
* successor the CURRENT daemon named (via NANO_SUPERVISOR_UPGRADE_PID, set by
|
|
15520
|
+
* the CLI that spawned it); a SIGUSR2 with no upgrade in flight keeps the
|
|
15521
|
+
* legacy graceful-drain semantics (the caller branches on the env var, not
|
|
15522
|
+
* here). No-op on win32 / when the pid is invalid; returns whether the
|
|
15523
|
+
* retarget landed.
|
|
15352
15524
|
*/
|
|
15353
15525
|
function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, readPpid } = {}) {
|
|
15354
15526
|
if (osPlatform() === 'win32') return () => {};
|
|
@@ -15362,14 +15534,40 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
|
|
|
15362
15534
|
// NOT treat that as "orphaned". A *fallback* ppid of 1, by contrast, means we
|
|
15363
15535
|
// were already reparented to init with no daemon left to watch.
|
|
15364
15536
|
const explicit = Number.isInteger(parentPid) && parentPid > 0;
|
|
15365
|
-
|
|
15537
|
+
// `watched` is deliberately MUTABLE: a rolling daemon upgrade (#309) retargets
|
|
15538
|
+
// it at the successor daemon's pid the moment the old daemon hands over, so
|
|
15539
|
+
// the watchdog keeps guarding the worker across the swap instead of reading
|
|
15540
|
+
// the (expected) reparent-to-init as an orphan event and self-reaping.
|
|
15541
|
+
let watched = explicit ? parentPid : ppid();
|
|
15542
|
+
// #309: after a retarget, the ppid leg changes meaning. The old daemon exits
|
|
15543
|
+
// on purpose once the successor owns the socket, so this worker reparents to
|
|
15544
|
+
// init — and that is the EXPECTED end state, not an orphan event. From the
|
|
15545
|
+
// retarget on, only the SUCCESSOR's death orphans the worker (the ppid leg
|
|
15546
|
+
// would fire immediately and forever on the already-reparented worker).
|
|
15547
|
+
let retargeted = false;
|
|
15548
|
+
// #309 review (rollback): the guard state BEFORE the latest retarget. If the
|
|
15549
|
+
// successor dies mid-handoff while the predecessor is still alive (it rolls
|
|
15550
|
+
// back and keeps the fleet), the worker must fall back to guarding the
|
|
15551
|
+
// predecessor instead of self-reaping an in-flight job. Once the predecessor
|
|
15552
|
+
// has exited too, both are gone and the worker is genuinely orphaned.
|
|
15553
|
+
let fallback = null;
|
|
15366
15554
|
const orphaned = typeof onOrphan === 'function' ? onOrphan : () => process.exit(0);
|
|
15367
15555
|
const isOrphan = () => {
|
|
15368
15556
|
// Reparented away from the daemon (typically to init, pid 1) → orphaned.
|
|
15369
|
-
|
|
15370
|
-
|
|
15371
|
-
//
|
|
15372
|
-
|
|
15557
|
+
// Skipped after an upgrade retarget: the reparent to init IS the handoff.
|
|
15558
|
+
if (!retargeted && ppid() !== watched) return true;
|
|
15559
|
+
// The watched daemon is gone (ESRCH ⇒ gone; EPERM ⇒ alive but not ours to
|
|
15560
|
+
// signal). After an upgrade retarget this is the ONLY orphan signal — the
|
|
15561
|
+
// ppid leg above is skipped (the reparent to init is expected), so the
|
|
15562
|
+
// worker self-reaps exactly when the SUCCESSOR daemon dies.
|
|
15563
|
+
const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; } };
|
|
15564
|
+
if (alive(watched)) return false;
|
|
15565
|
+
if (fallback && alive(fallback.watched) && (fallback.retargeted || ppid() === fallback.watched)) {
|
|
15566
|
+
({ watched, retargeted } = fallback);
|
|
15567
|
+
fallback = null;
|
|
15568
|
+
return false;
|
|
15569
|
+
}
|
|
15570
|
+
return true;
|
|
15373
15571
|
};
|
|
15374
15572
|
// Orphaned already — self-reap now rather than idle forever. Either we fell
|
|
15375
15573
|
// back to ppid and it is already init (no known parent), or the recorded
|
|
@@ -15377,7 +15575,22 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
|
|
|
15377
15575
|
if ((!explicit && watched === 1) || isOrphan()) { orphaned(); return () => {}; }
|
|
15378
15576
|
const timer = setInterval(() => { if (isOrphan()) { clearInterval(timer); orphaned(); } }, intervalMs);
|
|
15379
15577
|
if (typeof timer.unref === 'function') timer.unref();
|
|
15380
|
-
|
|
15578
|
+
const cancel = () => { try { clearInterval(timer); } catch { /* ignore */ } };
|
|
15579
|
+
// #309: re-arm at the successor daemon's pid. The ppid leg of isOrphan() then
|
|
15580
|
+
// reads "reparented to init" as expected (the old daemon exited on purpose),
|
|
15581
|
+
// and only the SUCCESSOR's death orphans the worker from here on.
|
|
15582
|
+
cancel.retarget = (newPid) => {
|
|
15583
|
+
if (!Number.isInteger(newPid) || newPid <= 0) return false;
|
|
15584
|
+
if (newPid !== watched) fallback = { watched, retargeted };
|
|
15585
|
+
watched = newPid;
|
|
15586
|
+
retargeted = true;
|
|
15587
|
+
return true;
|
|
15588
|
+
};
|
|
15589
|
+
// The pid this installation currently guards — the daemon stamps it on the
|
|
15590
|
+
// worker's activity marker as the upgrade ack (the old daemon waits, bounded,
|
|
15591
|
+
// for it to name the successor before exiting).
|
|
15592
|
+
cancel.watchedPid = () => watched;
|
|
15593
|
+
return cancel;
|
|
15381
15594
|
}
|
|
15382
15595
|
|
|
15383
15596
|
// --- Daemon ----------------------------------------------------------------
|
|
@@ -15386,6 +15599,18 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
|
|
|
15386
15599
|
* The supervisor daemon body. Runs under `c8ctl nano supervisor __daemon`,
|
|
15387
15600
|
* spawned detached by `startSupervisorDaemon`. Never returns — it runs until a
|
|
15388
15601
|
* stop request or SIGTERM, then drains children and exits.
|
|
15602
|
+
*
|
|
15603
|
+
* Issue #309 (rolling daemon hot-upgrade): when spawned with
|
|
15604
|
+
* NANO_SUPERVISOR_UPGRADE=1 this process is a SUCCESSOR — the CLI launched it
|
|
15605
|
+
* to take over the fleet from an old daemon running stale plugin code. Before
|
|
15606
|
+
* binding the control socket it waits for the old daemon to quiesce (its
|
|
15607
|
+
* `prepare-upgrade` op stops the accept loop), then adopts the old daemon's
|
|
15608
|
+
* workers from the persisted state (pid-polled synthetic child handles — the
|
|
15609
|
+
* workers themselves are never signaled, so in-flight jobs run undisturbed)
|
|
15610
|
+
* and finally asks the old daemon to hand over (its `upgrade` op retargets
|
|
15611
|
+
* every worker's parent-death watchdog here and exits). A successor that
|
|
15612
|
+
* cannot reach a quiesced old daemon in time exits 1 and leaves the old fleet
|
|
15613
|
+
* untouched.
|
|
15389
15614
|
*/
|
|
15390
15615
|
async function runSupervisorDaemon() {
|
|
15391
15616
|
const startedAt = new Date().toISOString();
|
|
@@ -15399,6 +15624,68 @@ async function runSupervisorDaemon() {
|
|
|
15399
15624
|
const daemonLogFile = supervisorDaemonLogFile();
|
|
15400
15625
|
mkdirSync(getSupervisorLogDir(), { recursive: true });
|
|
15401
15626
|
|
|
15627
|
+
// Boot-time log line, available before the closure-scoped `dlog` exists (the
|
|
15628
|
+
// upgrade-mode quiesce handshake below runs before the workers map and its
|
|
15629
|
+
// helpers are initialized). Identical format.
|
|
15630
|
+
const dlogBoot = (msg) => {
|
|
15631
|
+
try { appendFileSync(daemonLogFile, `[${new Date().toISOString()}] ${msg}\n`); } catch { /* best effort */ }
|
|
15632
|
+
};
|
|
15633
|
+
|
|
15634
|
+
// #309: successor boot. `upgradeMode` latches once the old daemon has been
|
|
15635
|
+
// asked to quiesce; `predecessorPid`/`upgradeFromVersion` feed the status
|
|
15636
|
+
// frame's `upgrade` block so `supervisor status` shows the roll in flight
|
|
15637
|
+
// (`rolling <from> → <to>`), mirroring the worker-roll visibility.
|
|
15638
|
+
const upgradeMode = process.env.NANO_SUPERVISOR_UPGRADE === '1';
|
|
15639
|
+
let predecessorPid = null;
|
|
15640
|
+
let upgradeFromVersion = null;
|
|
15641
|
+
let upgradeStartedMs = null;
|
|
15642
|
+
if (upgradeMode) {
|
|
15643
|
+
const t0 = Date.now();
|
|
15644
|
+
dlogBoot(`upgrade mode: waiting for the current daemon to quiesce (pid from ${getSupervisorStateFile()})`);
|
|
15645
|
+
for (;;) {
|
|
15646
|
+
const st = readSupervisorState();
|
|
15647
|
+
if (!st || !isPidAlive(st.pid)) {
|
|
15648
|
+
// No live old daemon — nothing to hand over. Serve the fleet normally
|
|
15649
|
+
// (there is nothing to adopt; the state file, if any, is stale).
|
|
15650
|
+
dlogBoot('upgrade mode: no live predecessor daemon — starting as a plain daemon');
|
|
15651
|
+
break;
|
|
15652
|
+
}
|
|
15653
|
+
predecessorPid = st.pid;
|
|
15654
|
+
upgradeFromVersion = typeof st.version === 'string' ? st.version : null;
|
|
15655
|
+
// Ask the old daemon to stop accepting new control connections. An old
|
|
15656
|
+
// daemon that predates #309 answers `unknown op` — refuse to upgrade from
|
|
15657
|
+
// it rather than racing its accept loop for the socket.
|
|
15658
|
+
let prep = null;
|
|
15659
|
+
try {
|
|
15660
|
+
// #309 review: quiescence awaits admitted opQueue mutations (a restart
|
|
15661
|
+
// can take ~28s to stop its worker), and the predecessor closes its
|
|
15662
|
+
// listener as soon as it latches — so a retry could never reach it.
|
|
15663
|
+
// Keep THIS connection open for the remaining quiesce budget.
|
|
15664
|
+
const remainingMs = SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS - (Date.now() - t0);
|
|
15665
|
+
prep = await supervisorRequest({ op: 'prepare-upgrade', pid: process.pid }, { socketPath, timeoutMs: 750, responseTimeoutMs: Math.max(SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS, remainingMs) });
|
|
15666
|
+
} catch { /* socket briefly unreachable — retry until the deadline */ }
|
|
15667
|
+
if (prep && prep.ok) break; // quiesced — the accept loop is closed
|
|
15668
|
+
if (prep && prep.ok === false && /unknown op/.test(String(prep.error || ''))) {
|
|
15669
|
+
dlogBoot(`upgrade mode: predecessor pid ${st.pid} does not understand prepare-upgrade (too old) — refusing to upgrade`);
|
|
15670
|
+
process.exit(1);
|
|
15671
|
+
}
|
|
15672
|
+
// #309 review: the predecessor bound its handoff to a DIFFERENT successor
|
|
15673
|
+
// (a concurrent upgrade won the race). Never keep retrying — the binding
|
|
15674
|
+
// will never accept this pid, and two successors must not both wait to
|
|
15675
|
+
// adopt the same fleet. Exit and leave the bound successor to finish.
|
|
15676
|
+
if (prep && prep.ok === false && /already bound to successor pid/.test(String(prep.error || ''))) {
|
|
15677
|
+
dlogBoot(`upgrade mode: predecessor pid ${st.pid} is bound to a different successor — exiting so the bound successor completes the handoff`);
|
|
15678
|
+
process.exit(1);
|
|
15679
|
+
}
|
|
15680
|
+
if (Date.now() - t0 > SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS) {
|
|
15681
|
+
dlogBoot(`upgrade mode: predecessor pid ${st.pid} did not quiesce within ${SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS}ms — leaving it in place`);
|
|
15682
|
+
process.exit(1);
|
|
15683
|
+
}
|
|
15684
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
15685
|
+
}
|
|
15686
|
+
if (predecessorPid != null) upgradeStartedMs = Date.now();
|
|
15687
|
+
}
|
|
15688
|
+
|
|
15402
15689
|
const workers = new Map();
|
|
15403
15690
|
const attachClients = new Set();
|
|
15404
15691
|
// #202: sockets that issued a `stop`/drain and are waiting for the daemon to
|
|
@@ -15416,15 +15703,50 @@ async function runSupervisorDaemon() {
|
|
|
15416
15703
|
// restart-on-exit path and let a second `stop --force` escalate a live drain.
|
|
15417
15704
|
let draining = false;
|
|
15418
15705
|
let forcing = false;
|
|
15706
|
+
// #309 review: `shuttingDown` is latched EARLY (in the `stop` op, before the
|
|
15707
|
+
// deferred shutdown() runs) to reserve shutdown ownership against a racing
|
|
15708
|
+
// prepare-upgrade. `shutdownStarted` is the separate "shutdown()'s body has
|
|
15709
|
+
// begun" guard: the deferred shutdown() must still run its body even though
|
|
15710
|
+
// `shuttingDown` is already true, while a SECOND stop/signal must escalate
|
|
15711
|
+
// (force) or return instead of re-running the body.
|
|
15712
|
+
let shutdownStarted = false;
|
|
15713
|
+
// #309: a rolling daemon upgrade owns this daemon. `upgrading` is the
|
|
15714
|
+
// OLD-daemon latch: set by `prepare-upgrade` (accept loop closed; mutations
|
|
15715
|
+
// refused) and held through the `upgrade` handoff until this process exits.
|
|
15716
|
+
// `quiesced` flips when the accept loop has actually closed — the successor
|
|
15717
|
+
// polls for it before binding the socket. `upgradeInfo` carries the roll's
|
|
15718
|
+
// identity ({ state, fromVersion, toVersion, successorPid, sinceEpochMs })
|
|
15719
|
+
// for the status frame's `upgrade` block.
|
|
15720
|
+
let upgrading = false;
|
|
15721
|
+
let quiesced = false;
|
|
15722
|
+
let upgradeInfo = null;
|
|
15419
15723
|
// Hot code reload (rolling drain+respawn). A `reload` op adopts new on-disk
|
|
15420
15724
|
// plugin code into the worker children by gracefully draining and respawning
|
|
15421
15725
|
// them one at a time (so the fleet keeps serving). This flag rejects a second
|
|
15422
15726
|
// concurrent reload — a single rolling pass owns the fleet until it finishes.
|
|
15423
15727
|
let reloading = false;
|
|
15728
|
+
// Progress for `status.reload` while `reloading` is true. Reset when the
|
|
15729
|
+
// roll starts; `reloadCurrent` is the id in `reloadWorker` right now.
|
|
15730
|
+
let reloadTargets = [];
|
|
15731
|
+
let reloadDone = [];
|
|
15732
|
+
let reloadCurrent = null;
|
|
15424
15733
|
// Live-view monitor: tracks the last-broadcast fleet signature so we push a
|
|
15425
15734
|
// refreshed status to attached consoles only on real change (see below).
|
|
15426
15735
|
let monitorTimer = null;
|
|
15427
15736
|
let lastMonitorSig = null;
|
|
15737
|
+
// #309: while quiesced for an upgrade the predecessor's accept loop is closed
|
|
15738
|
+
// and — on a second upgrade — every worker is an ADOPTED handle whose liveness
|
|
15739
|
+
// poll (`adoptTimer`) is unref'd, as is `monitorTimer`. With no referenced
|
|
15740
|
+
// handle left, Node would let the predecessor EXIT before the successor's
|
|
15741
|
+
// watchdog-retarget acks arrive / before SIGUSR1 lands — and the never-
|
|
15742
|
+
// resolving handoff Promise does NOT keep the event loop alive. This
|
|
15743
|
+
// referenced keepalive holds the predecessor up for exactly the quiesce→
|
|
15744
|
+
// handover window; it is cleared on the SIGUSR1 handover (and on shutdown).
|
|
15745
|
+
let quiesceKeepalive = null;
|
|
15746
|
+
// #309: polls the successor pid while quiesced so a successor that dies
|
|
15747
|
+
// mid-handoff (before SIGUSR1) triggers a rollback instead of leaving this
|
|
15748
|
+
// daemon permanently quiesced + unreachable. Cleared on handover/rollback.
|
|
15749
|
+
let successorWatchTimer = null;
|
|
15428
15750
|
|
|
15429
15751
|
// Daemon-wide mutation serialization: `add`/`remove`/`restart` must not
|
|
15430
15752
|
// interleave, or two clients racing the same worker could each spawn an
|
|
@@ -15442,16 +15764,46 @@ async function runSupervisorDaemon() {
|
|
|
15442
15764
|
|
|
15443
15765
|
const workerPublic = (w) => summarizeSupervisorWorker(w);
|
|
15444
15766
|
|
|
15445
|
-
|
|
15767
|
+
// #309: the daemon descriptor gains an `upgrade` block while a rolling daemon
|
|
15768
|
+
// upgrade is in flight — on the OLD daemon (state 'rolling', naming the
|
|
15769
|
+
// successor it is handing to) and on the SUCCESSOR (state 'adopting' until
|
|
15770
|
+
// the predecessor exits). `supervisor status` renders it as
|
|
15771
|
+
// `upgrade: rolling <from> → <to>` — the same visibility the worker roll has.
|
|
15772
|
+
const currentUpgradeBlock = () => {
|
|
15773
|
+
if (upgradeInfo) return upgradeInfo;
|
|
15774
|
+
if (upgradeMode && predecessorPid != null) {
|
|
15775
|
+
return { state: 'adopting', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, sinceEpochMs: upgradeStartedMs };
|
|
15776
|
+
}
|
|
15777
|
+
return null;
|
|
15778
|
+
};
|
|
15779
|
+
|
|
15780
|
+
// #309: once the successor has confirmed ownership (SIGUSR1 → handedOver) the
|
|
15781
|
+
// predecessor must make its final snapshot its LAST canonical write. More
|
|
15782
|
+
// broadly, once QUIESCED (`upgrading` latched) the successor has already read
|
|
15783
|
+
// the quiesce snapshot and adopted the inventory, so any later automatic
|
|
15784
|
+
// persist (a straggler worker-death handler firing between quiesce and exit)
|
|
15785
|
+
// would overwrite the successor's pid + inventory with the dead/dying daemon's
|
|
15786
|
+
// — corrupting discovery/recovery until the successor happens to persist
|
|
15787
|
+
// again. So persist() is suppressed while quiesced UNLESS forced (the quiesce
|
|
15788
|
+
// snapshot and the final handed-over snapshot are the two deliberate writes).
|
|
15789
|
+
let handedOver = false;
|
|
15790
|
+
const persist = (force = false) => {
|
|
15791
|
+
if (handedOver) return; // the successor owns the state file now
|
|
15792
|
+
if (upgrading && !force) return; // quiesced — the successor owns the inventory
|
|
15446
15793
|
try {
|
|
15447
15794
|
writeSupervisorState({
|
|
15448
15795
|
...supervisorDaemonDescriptor({
|
|
15449
15796
|
pid: process.pid, startedAt, version: daemonVersion, socket: socketPath, logFile: daemonLogFile,
|
|
15450
15797
|
}),
|
|
15798
|
+
...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
|
|
15451
15799
|
workers: [...workers.values()].map((w) => ({
|
|
15452
15800
|
id: w.id, profile: w.profile, args: w.args, pid: isPidAlive(w.pid) ? w.pid : null,
|
|
15453
15801
|
startedAt: w.startedAt || null, restarts: w.restarts, lastExit: w.lastExit ?? null,
|
|
15454
15802
|
stopping: !!w.stopping, logFile: w.logFile, configFailed: w.configFailed === true,
|
|
15803
|
+
// #309: persist drain-remove intent so an adopted worker that exits
|
|
15804
|
+
// mid-drain is still REMOVED by the successor (not retained forever,
|
|
15805
|
+
// which would block name reuse and hang `workforce stop`).
|
|
15806
|
+
removeOnExit: w.removeOnExit === true,
|
|
15455
15807
|
})),
|
|
15456
15808
|
});
|
|
15457
15809
|
} catch { /* best effort */ }
|
|
@@ -15488,6 +15840,11 @@ async function runSupervisorDaemon() {
|
|
|
15488
15840
|
const activityFile = supervisorWorkerActivityFile(w.id);
|
|
15489
15841
|
w.activityFile = activityFile;
|
|
15490
15842
|
try { rmSync(activityFile, { force: true }); } catch { /* best effort */ }
|
|
15843
|
+
// #309 review: also clear a stale `.upgrade` retarget sidecar. A worker
|
|
15844
|
+
// force-restarted mid-handoff before consuming it would otherwise inherit
|
|
15845
|
+
// the same path and treat its next genuine drain SIGUSR2 as a retarget.
|
|
15846
|
+
// (A handoff in progress re-writes the sidecar before it SIGUSR2s.)
|
|
15847
|
+
try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
|
|
15491
15848
|
// `--name w.id` makes the child's broker workerName match this worker's
|
|
15492
15849
|
// supervisor id, so the same profile launched twice is distinct end-to-end.
|
|
15493
15850
|
// NANO_SUPERVISOR_ACTIVITY_FILE tells the child where to report per-job
|
|
@@ -15496,8 +15853,20 @@ async function runSupervisorDaemon() {
|
|
|
15496
15853
|
// watchdog can self-reap if we die ungracefully (SIGKILL/crash) and can't
|
|
15497
15854
|
// run the child-draining shutdown — even if we die mid-startup, before the
|
|
15498
15855
|
// child reparents to init.
|
|
15856
|
+
// NANO_SUPERVISOR_UPGRADE_PID (#309): when a daemon upgrade is in flight,
|
|
15857
|
+
// the successor's pid rides the child's env so the worker's SIGUSR2 handler
|
|
15858
|
+
// can tell a watchdog RETARGET (upgrade — keep serving) apart from a
|
|
15859
|
+
// graceful DRAIN (no upgrade pid — quiesce and exit). The daemon learns the
|
|
15860
|
+
// successor pid only at handoff time (the `upgrade` op), AFTER most workers
|
|
15861
|
+
// were spawned — so env alone can't reach them. The daemon therefore also
|
|
15862
|
+
// writes `<activityFile>.upgrade` (the successor pid) before it SIGUSR2s,
|
|
15863
|
+
// and the worker reads THAT as the authoritative retarget target; this env
|
|
15864
|
+
// forwarding covers only a worker spawned mid-upgrade.
|
|
15865
|
+
const childEnv = { ...process.env, NANO_SUPERVISOR_ACTIVITY_FILE: activityFile, NANO_SUPERVISOR_DAEMON_PID: String(process.pid) };
|
|
15866
|
+
const upgradeSuccessorPid = Number(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
|
|
15867
|
+
if (Number.isInteger(upgradeSuccessorPid) && upgradeSuccessorPid > 0) childEnv.NANO_SUPERVISOR_UPGRADE_PID = String(upgradeSuccessorPid);
|
|
15499
15868
|
const child = spawn(exec, [entry, 'nano', 'work', w.profile, '--name', w.id, ...w.args], {
|
|
15500
|
-
env:
|
|
15869
|
+
env: childEnv,
|
|
15501
15870
|
stdio: ['ignore', ringed ? 'pipe' : fd, ringed ? 'pipe' : fd],
|
|
15502
15871
|
});
|
|
15503
15872
|
if (typeof fd === 'number') { try { closeSync(fd); } catch { /* dup'd into child */ } }
|
|
@@ -15590,7 +15959,9 @@ async function runSupervisorDaemon() {
|
|
|
15590
15959
|
broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
|
|
15591
15960
|
w.restartTimer = setTimeout(() => {
|
|
15592
15961
|
w.restartTimer = null;
|
|
15593
|
-
|
|
15962
|
+
// #309: a quiesced old daemon (mid-upgrade) never respawns — the
|
|
15963
|
+
// successor owns the fleet and would double-spawn the worker.
|
|
15964
|
+
if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
|
|
15594
15965
|
}, delay);
|
|
15595
15966
|
if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
|
|
15596
15967
|
persist();
|
|
@@ -15600,6 +15971,123 @@ async function runSupervisorDaemon() {
|
|
|
15600
15971
|
persist();
|
|
15601
15972
|
};
|
|
15602
15973
|
|
|
15974
|
+
// #309: a synthetic ChildProcess-shaped handle for an ADOPTED worker — one
|
|
15975
|
+
// the old daemon spawned and the successor inherits by pid. The successor
|
|
15976
|
+
// never spawned it, so there is no real 'exit' event: a poll timer is the
|
|
15977
|
+
// death detector, feeding the same `handleDeath` path a real child's exit
|
|
15978
|
+
// would (restart backoff included). The handle provides the surface the
|
|
15979
|
+
// daemon touches: `pid`, `exitCode`/`signalCode` (null while alive — the
|
|
15980
|
+
// readiness gate and reload checks read them), `kill` (force-stop),
|
|
15981
|
+
// `unref`/`ref`, null stdio (the log ring stays with the OLD daemon process),
|
|
15982
|
+
// and — critically — REAL `exit` listener semantics (`once`/`on`/
|
|
15983
|
+
// `removeListener` + `markExited`), because `waitForChildExit()` resolves only
|
|
15984
|
+
// when `exit` fires.
|
|
15985
|
+
const makeAdoptedChild = (pid) => {
|
|
15986
|
+
const exitWaiters = new Set();
|
|
15987
|
+
const handle = {
|
|
15988
|
+
pid,
|
|
15989
|
+
exitCode: null,
|
|
15990
|
+
signalCode: null,
|
|
15991
|
+
killed: false,
|
|
15992
|
+
stdout: null,
|
|
15993
|
+
stderr: null,
|
|
15994
|
+
stdin: null,
|
|
15995
|
+
kill(signal) { try { return process.kill(pid, signal); } catch { return false; } },
|
|
15996
|
+
once(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
|
|
15997
|
+
on(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
|
|
15998
|
+
removeListener(event, fn) { if (event === 'exit') exitWaiters.delete(fn); return handle; },
|
|
15999
|
+
unref() { return handle; },
|
|
16000
|
+
ref() { return handle; },
|
|
16001
|
+
// Invoked by the adopt liveness poll when the pid is gone: record the exit
|
|
16002
|
+
// and flush every waiter so graceful-stop/reload promises resolve.
|
|
16003
|
+
markExited(code, signal) {
|
|
16004
|
+
if (handle.exitCode !== null || handle.signalCode !== null) return;
|
|
16005
|
+
handle.exitCode = code;
|
|
16006
|
+
handle.signalCode = signal ?? null;
|
|
16007
|
+
for (const fn of [...exitWaiters]) { exitWaiters.delete(fn); try { fn(code, signal ?? null); } catch { /* ignore */ } }
|
|
16008
|
+
},
|
|
16009
|
+
};
|
|
16010
|
+
return handle;
|
|
16011
|
+
};
|
|
16012
|
+
|
|
16013
|
+
// Adopt one worker record from the predecessor's persisted state. Returns the
|
|
16014
|
+
// live worker entry, or null when the pid is already gone (nothing to adopt —
|
|
16015
|
+
// the fleet is exactly what survives the swap). The poll timer is unref'd so
|
|
16016
|
+
// it never keeps the daemon alive on its own.
|
|
16017
|
+
const adoptWorker = (rec) => {
|
|
16018
|
+
if (!rec || typeof rec.id !== 'string' || !rec.id) return null;
|
|
16019
|
+
if (!isPidAlive(rec.pid)) return null;
|
|
16020
|
+
const w = {
|
|
16021
|
+
id: rec.id,
|
|
16022
|
+
profile: typeof rec.profile === 'string' ? rec.profile : '',
|
|
16023
|
+
args: Array.isArray(rec.args) ? rec.args.map(String) : [],
|
|
16024
|
+
restarts: Number(rec.restarts) || 0,
|
|
16025
|
+
stopping: rec.stopping === true,
|
|
16026
|
+
// #309: restore drain-remove intent persisted by the predecessor, so a
|
|
16027
|
+
// worker draining for removal that exits after the swap is REMOVED here
|
|
16028
|
+
// (its record deleted) rather than retained — which would block name reuse
|
|
16029
|
+
// and hang a `workforce stop` waiting for the record to disappear.
|
|
16030
|
+
removeOnExit: rec.removeOnExit === true,
|
|
16031
|
+
lastExit: rec.lastExit ?? null,
|
|
16032
|
+
logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
|
|
16033
|
+
configFailed: rec.configFailed === true,
|
|
16034
|
+
adopted: true,
|
|
16035
|
+
};
|
|
16036
|
+
w.activityFile = supervisorWorkerActivityFile(w.id);
|
|
16037
|
+
// Clear any stale `.upgrade` sidecar the handoff left: the worker consumed
|
|
16038
|
+
// it to retarget its watchdog here, and a leftover would make a LATER
|
|
16039
|
+
// SIGUSR2 (a genuine drain) misread as another upgrade retarget.
|
|
16040
|
+
try { rmSync(`${w.activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
|
|
16041
|
+
const child = makeAdoptedChild(rec.pid);
|
|
16042
|
+
w.child = child;
|
|
16043
|
+
w.pid = rec.pid;
|
|
16044
|
+
w.startedAt = rec.startedAt || null;
|
|
16045
|
+
w.spawnedAt = rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now();
|
|
16046
|
+
let settled = false;
|
|
16047
|
+
const handleDeath = (reason) => {
|
|
16048
|
+
if (w.child !== child || settled) return;
|
|
16049
|
+
settled = true;
|
|
16050
|
+
if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
|
|
16051
|
+
// Mark the synthetic handle exited BEFORE running the death path so any
|
|
16052
|
+
// `waitForChildExit` waiter (graceful stop / reload) resolves — the poll
|
|
16053
|
+
// below is the only death detector for an adopted worker.
|
|
16054
|
+
if (typeof child.markExited === 'function') child.markExited(1, null);
|
|
16055
|
+
w.pid = null;
|
|
16056
|
+
w.lastExit = reason;
|
|
16057
|
+
try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
|
|
16058
|
+
const ranMs = Date.now() - (w.spawnedAt || Date.now());
|
|
16059
|
+
if (ranMs >= SUPERVISOR_HEALTHY_UPTIME_MS) w.restarts = 0;
|
|
16060
|
+
if (w.stopping || shuttingDown || !workers.has(w.id)) {
|
|
16061
|
+
if (w.removeOnExit && workers.get(w.id) === w) {
|
|
16062
|
+
workers.delete(w.id);
|
|
16063
|
+
try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
|
|
16064
|
+
dlog(`worker '${w.id}' removed (drained)`);
|
|
16065
|
+
broadcast({ type: 'event', event: 'worker-remove', id: w.id });
|
|
16066
|
+
}
|
|
16067
|
+
persist();
|
|
16068
|
+
if (draining) { try { broadcast(statusFrame(false)); } catch { /* best effort */ } }
|
|
16069
|
+
return;
|
|
16070
|
+
}
|
|
16071
|
+
if (upgrading) { persist(); return; } // quiesced for the handoff — the successor owns respawns
|
|
16072
|
+
const delay = supervisorBackoffMs(w.restarts);
|
|
16073
|
+
w.restarts += 1;
|
|
16074
|
+
dlog(`adopted worker '${w.id}' down (${reason}); restarting in ${delay}ms (restart #${w.restarts})`);
|
|
16075
|
+
broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
|
|
16076
|
+
w.restartTimer = setTimeout(() => {
|
|
16077
|
+
w.restartTimer = null;
|
|
16078
|
+
if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
|
|
16079
|
+
}, delay);
|
|
16080
|
+
if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
|
|
16081
|
+
persist();
|
|
16082
|
+
};
|
|
16083
|
+
w.adoptTimer = setInterval(() => { if (!isPidAlive(rec.pid)) handleDeath('code 1 (adopted worker exited)'); }, SUPERVISOR_ADOPT_POLL_MS);
|
|
16084
|
+
if (typeof w.adoptTimer.unref === 'function') w.adoptTimer.unref();
|
|
16085
|
+
workers.set(w.id, w);
|
|
16086
|
+
dlog(`worker '${w.id}' (profile ${w.profile}) adopted from predecessor daemon (pid ${w.pid})`);
|
|
16087
|
+
broadcast({ type: 'event', event: 'worker-adopted', worker: workerPublic(w) });
|
|
16088
|
+
return w;
|
|
16089
|
+
};
|
|
16090
|
+
|
|
15603
16091
|
const addWorker = (profile, args, name) => {
|
|
15604
16092
|
const taken = new Set(workers.keys());
|
|
15605
16093
|
let id;
|
|
@@ -15622,6 +16110,33 @@ async function runSupervisorDaemon() {
|
|
|
15622
16110
|
return w;
|
|
15623
16111
|
};
|
|
15624
16112
|
|
|
16113
|
+
// #309 review: workers whose upgrade watchdog-retarget is still outstanding
|
|
16114
|
+
// (id -> { pid, signalled, drainOnly?, drainDeferred? }), owned by the
|
|
16115
|
+
// successor's handoff loop. A graceful drain (stop/remove/reload) also uses
|
|
16116
|
+
// SIGUSR2, so it must not race a pending retarget: a drain signal consumed as
|
|
16117
|
+
// the retarget leaves the drain unfulfilled, and a second SIGUSR2 after the
|
|
16118
|
+
// one-shot handler fired would kill in-flight work. sendDrainSignal
|
|
16119
|
+
// coordinates the two so both are delivered, in order.
|
|
16120
|
+
const retargetPending = new Map();
|
|
16121
|
+
const sendDrainSignal = (w, pid) => {
|
|
16122
|
+
const p = retargetPending.get(w.id);
|
|
16123
|
+
if (p && p.pid === pid) {
|
|
16124
|
+
if (!p.signalled) {
|
|
16125
|
+
// Retarget not sent yet: cancel it (drop the sidecar) and drain
|
|
16126
|
+
// instead — the handoff loop delivers the drain once the handler is
|
|
16127
|
+
// installed and keeps the predecessor alive until the worker exits.
|
|
16128
|
+
try { rmSync(`${w.activityFile || supervisorWorkerActivityFile(w.id)}.upgrade`, { force: true }); } catch { /* best effort */ }
|
|
16129
|
+
p.signalled = true; p.drainOnly = true; p.drainDeferred = true;
|
|
16130
|
+
} else if (!p.drainOnly) {
|
|
16131
|
+
// Retarget in flight: deliver the drain right after its ack.
|
|
16132
|
+
p.drainDeferred = true;
|
|
16133
|
+
}
|
|
16134
|
+
// drainOnly without drainDeferred: already draining — nothing to send.
|
|
16135
|
+
return;
|
|
16136
|
+
}
|
|
16137
|
+
try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
16138
|
+
};
|
|
16139
|
+
|
|
15625
16140
|
const forceStopWorker = async (w) => {
|
|
15626
16141
|
const pid = w.pid;
|
|
15627
16142
|
if (w.child && pid) {
|
|
@@ -15648,7 +16163,7 @@ async function runSupervisorDaemon() {
|
|
|
15648
16163
|
// — a `stop --force` escalation (forceStopWorker) is the only way to cut a
|
|
15649
16164
|
// stuck drain short, and it resolves this same wait when the child dies.
|
|
15650
16165
|
if (w.child && pid) {
|
|
15651
|
-
|
|
16166
|
+
sendDrainSignal(w, pid);
|
|
15652
16167
|
await waitForChildExit(w.child, null);
|
|
15653
16168
|
}
|
|
15654
16169
|
return true;
|
|
@@ -15670,7 +16185,7 @@ async function runSupervisorDaemon() {
|
|
|
15670
16185
|
w.removeOnExit = true;
|
|
15671
16186
|
if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
|
|
15672
16187
|
const pid = w.pid;
|
|
15673
|
-
if (w.child && pid) {
|
|
16188
|
+
if (w.child && pid) { sendDrainSignal(w, pid); }
|
|
15674
16189
|
else { workers.delete(id); try { rmSync(supervisorWorkerActivityFile(id), { force: true }); } catch { /* best effort */ } broadcast({ type: 'event', event: 'worker-remove', id }); persist(); }
|
|
15675
16190
|
dlog(`worker '${id}' draining for removal`);
|
|
15676
16191
|
return true;
|
|
@@ -15767,7 +16282,7 @@ async function runSupervisorDaemon() {
|
|
|
15767
16282
|
if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
|
|
15768
16283
|
const pid = w.pid;
|
|
15769
16284
|
if (child && pid) {
|
|
15770
|
-
|
|
16285
|
+
sendDrainSignal(w, pid);
|
|
15771
16286
|
await waitForChildExit(child, null);
|
|
15772
16287
|
}
|
|
15773
16288
|
// Only respawn if nobody else acted on this worker while we drained: it must
|
|
@@ -15839,12 +16354,24 @@ async function runSupervisorDaemon() {
|
|
|
15839
16354
|
// shutting down. Never clears `supervisor.json` — a reload is not a stop.
|
|
15840
16355
|
const runReload = async (ids, sock) => {
|
|
15841
16356
|
reloading = true;
|
|
16357
|
+
reloadTargets = [...ids];
|
|
16358
|
+
reloadDone = [];
|
|
16359
|
+
reloadCurrent = null;
|
|
15842
16360
|
const reloaded = [];
|
|
15843
16361
|
const skipped = [];
|
|
15844
16362
|
let interrupted = false;
|
|
15845
16363
|
try {
|
|
15846
16364
|
for (let i = 0; i < ids.length; i++) {
|
|
15847
16365
|
const id = ids[i];
|
|
16366
|
+
reloadCurrent = id;
|
|
16367
|
+
// Broadcast (not just to the initiating socket) on every target
|
|
16368
|
+
// transition so every attached console sees the drain progress and the
|
|
16369
|
+
// "do not restart the daemon" warning even with periodic monitoring
|
|
16370
|
+
// disabled (NANO_SUPERVISOR_MONITOR_MS=0). The initiator is itself an
|
|
16371
|
+
// attach client, so it is covered too.
|
|
16372
|
+
if (!shuttingDown) {
|
|
16373
|
+
try { broadcast(statusFrame(false)); } catch { /* best effort */ }
|
|
16374
|
+
}
|
|
15848
16375
|
// If the daemon starts shutting down mid-roll, the remaining workers
|
|
15849
16376
|
// never adopt the new code — record them as skipped (and flag the roll
|
|
15850
16377
|
// interrupted) so the terminal frame can't report a clean success for a
|
|
@@ -15868,6 +16395,8 @@ async function runSupervisorDaemon() {
|
|
|
15868
16395
|
const ok = await reloadWorker(id);
|
|
15869
16396
|
if (ok) {
|
|
15870
16397
|
reloaded.push(id);
|
|
16398
|
+
reloadDone.push(id);
|
|
16399
|
+
reloadCurrent = null;
|
|
15871
16400
|
// Emit the per-worker "reloaded" progress signal ONLY after the final
|
|
15872
16401
|
// success gate above confirmed a live, still-current replacement — not
|
|
15873
16402
|
// at spawn time. A crash/spawn-fail makes `reloadWorker` return false
|
|
@@ -15888,13 +16417,24 @@ async function runSupervisorDaemon() {
|
|
|
15888
16417
|
for (let j = i; j < ids.length; j++) skipped.push(ids[j]);
|
|
15889
16418
|
break;
|
|
15890
16419
|
}
|
|
15891
|
-
try {
|
|
16420
|
+
try { broadcast(statusFrame(false)); } catch { /* best effort */ }
|
|
15892
16421
|
}
|
|
15893
16422
|
} catch (err) {
|
|
15894
16423
|
interrupted = true;
|
|
15895
16424
|
dlog(`reload error: ${err?.message || err}`);
|
|
15896
16425
|
} finally {
|
|
15897
16426
|
reloading = false;
|
|
16427
|
+
reloadCurrent = null;
|
|
16428
|
+
reloadTargets = [];
|
|
16429
|
+
reloadDone = [];
|
|
16430
|
+
// Clearing the progress flags may not change the monitor signature (it may
|
|
16431
|
+
// already have sampled the final replacement mid-readiness-wait), and the
|
|
16432
|
+
// terminal `reloaded` frame goes only to the initiating socket — so push a
|
|
16433
|
+
// fresh status to every attached console now, or they could keep showing
|
|
16434
|
+
// the "rolling … do not restart the daemon" line after completion.
|
|
16435
|
+
if (!shuttingDown && attachClients.size > 0) {
|
|
16436
|
+
try { broadcast(statusFrame(false)); } catch { /* best effort */ }
|
|
16437
|
+
}
|
|
15898
16438
|
// Terminal success requires BOTH a clean pass (not interrupted) AND nothing
|
|
15899
16439
|
// skipped: a skipped worker (failed reload, or removed/absent mid-roll) is a
|
|
15900
16440
|
// partial reload, so `ok: true` would let the streaming client exit zero and
|
|
@@ -15927,12 +16467,40 @@ async function runSupervisorDaemon() {
|
|
|
15927
16467
|
// i.e. a `supervisor reload` would adopt new worker code (and a daemon
|
|
15928
16468
|
// restart new supervisor code). Best-effort; falls back to the daemon's own.
|
|
15929
16469
|
pluginVersion: (() => { try { return pluginPackage().version; } catch { return daemonVersion; } })(),
|
|
16470
|
+
// Live rolling-reload progress so `supervisor status` (and the attach
|
|
16471
|
+
// view) show a roll that was detached or started by someone else.
|
|
16472
|
+
// `inProgress` is the latch; `current` is the worker being drained
|
|
16473
|
+
// right now (null between respawn and the next drain); `reloaded` /
|
|
16474
|
+
// `targets` / `total` let the renderer print N/M.
|
|
16475
|
+
reload: {
|
|
16476
|
+
inProgress: reloading,
|
|
16477
|
+
current: reloadCurrent,
|
|
16478
|
+
targets: reloadTargets,
|
|
16479
|
+
reloaded: [...reloadDone],
|
|
16480
|
+
total: reloadTargets.length,
|
|
16481
|
+
},
|
|
16482
|
+
// #309: present only while a rolling daemon upgrade is in flight
|
|
16483
|
+
// (`{ state, fromVersion, toVersion, … }`) — the successor polls the old
|
|
16484
|
+
// daemon's `upgrade.state === 'quiesced'` through this field before binding
|
|
16485
|
+
// the socket, and `supervisor status` renders it as the daemon-roll line.
|
|
16486
|
+
...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
|
|
15930
16487
|
workers: pub || [...workers.values()].map(workerPublic),
|
|
15931
16488
|
...(final ? { final: true } : {}),
|
|
15932
16489
|
});
|
|
15933
16490
|
|
|
15934
16491
|
const shutdown = async (signal, { force = true } = {}) => {
|
|
15935
|
-
|
|
16492
|
+
// #309 review: a quiesced (mid-upgrade) daemon must NEVER run the shutdown
|
|
16493
|
+
// path — it would SIGTERM/SIGKILL the very workers the successor has
|
|
16494
|
+
// adopted, and its cleanup would delete the successor's socket and state
|
|
16495
|
+
// file. The upgrade's SIGUSR1 handover is this daemon's only exit while
|
|
16496
|
+
// quiesced; a signal (SIGTERM/SIGINT/SIGHUP) is logged and ignored so the
|
|
16497
|
+
// handoff can finish. The socket `stop` op refuses earlier (the `upgrading`
|
|
16498
|
+
// check in handleRequest) and never reaches here.
|
|
16499
|
+
if (upgrading) {
|
|
16500
|
+
dlog(`ignoring ${signal || 'shutdown request'} — a daemon upgrade owns this daemon (successor pid ${upgradeInfo?.successorPid ?? 'unknown'}); the successor owns the fleet now`);
|
|
16501
|
+
return;
|
|
16502
|
+
}
|
|
16503
|
+
if (shutdownStarted) {
|
|
15936
16504
|
// A shutdown is already running. A `stop --force` arriving mid-DRAIN
|
|
15937
16505
|
// escalates it: hard-stop every worker still finishing its jobs so the
|
|
15938
16506
|
// operator isn't stuck waiting. Idempotent — only the first force escalates.
|
|
@@ -15945,6 +16513,7 @@ async function runSupervisorDaemon() {
|
|
|
15945
16513
|
return;
|
|
15946
16514
|
}
|
|
15947
16515
|
shuttingDown = true;
|
|
16516
|
+
shutdownStarted = true;
|
|
15948
16517
|
draining = !force;
|
|
15949
16518
|
// Let any in-flight mutation finish before we snapshot the worker set, so
|
|
15950
16519
|
// an add/restart racing the shutdown can't leave an orphaned child behind.
|
|
@@ -15953,6 +16522,7 @@ async function runSupervisorDaemon() {
|
|
|
15953
16522
|
if (!force) broadcast({ type: 'event', event: 'draining', workers: [...workers.values()].map(workerPublic) });
|
|
15954
16523
|
await Promise.all([...workers.keys()].map((id) => stopWorker(id, { force })));
|
|
15955
16524
|
if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
|
|
16525
|
+
if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
|
|
15956
16526
|
broadcast({ type: 'event', event: 'daemon-stop' });
|
|
15957
16527
|
// Terminal frame for every waiting `stop` client (streaming or one-shot).
|
|
15958
16528
|
// We must FLUSH these before exiting: `process.exit()` does NOT drain pending
|
|
@@ -15976,6 +16546,179 @@ async function runSupervisorDaemon() {
|
|
|
15976
16546
|
process.exit(0);
|
|
15977
16547
|
};
|
|
15978
16548
|
|
|
16549
|
+
// #309: quiesce this daemon for an upgrade WITHOUT exiting: stop accepting
|
|
16550
|
+
// new control connections (the successor binds the socket the moment our
|
|
16551
|
+
// accept loop closes) and latch `upgrading` so mutations and worker respawns
|
|
16552
|
+
// are refused. Idempotent — the successor polls until it observes the
|
|
16553
|
+
// quiesced state. Attached consoles are destroyed (their daemon is going
|
|
16554
|
+
// away). The handoff is then driven by the SUCCESSOR (it retargets the
|
|
16555
|
+
// workers' watchdogs and SIGUSR1s this daemon to exit) — this daemon's
|
|
16556
|
+
// accept loop is already closed, so it cannot be asked over the socket.
|
|
16557
|
+
//
|
|
16558
|
+
// Snapshot AFTER the mutation queue drains (#309 review): latching
|
|
16559
|
+
// `upgrading` rejects NEW mutations, but one may already be admitted to
|
|
16560
|
+
// `opQueue` (e.g. a restart that is mid-respawn). Persisting before that
|
|
16561
|
+
// settles would snapshot a half-swapped worker set the successor then adopts
|
|
16562
|
+
// — missing the replacement the in-flight restart is about to spawn. So we
|
|
16563
|
+
// latch admission synchronously, then AWAIT `opQueue` (exactly as
|
|
16564
|
+
// `shutdown()` does) before persisting and reporting `quiesced`.
|
|
16565
|
+
let quiescePromise = null;
|
|
16566
|
+
// #309 review: the worker-id set captured in the quiesce snapshot. Rollback
|
|
16567
|
+
// reconciles the live map against it — the successor owns add/remove while
|
|
16568
|
+
// this daemon is quiesced, so a worker the successor removed mid-handoff must
|
|
16569
|
+
// stay removed if the handoff rolls back (never resurrected).
|
|
16570
|
+
let quiesceSnapshotIds = null;
|
|
16571
|
+
const quiesceForUpgrade = (successorPid, successorVersion) => {
|
|
16572
|
+
// #309 review: bind the handoff to ONE successor. `doQuiesce` latches
|
|
16573
|
+
// `upgrading` only inside its async body, so two prepare-upgrade requests
|
|
16574
|
+
// that arrive before the first body's first await both observe
|
|
16575
|
+
// `upgrading === false` and would share one quiesce promise — two
|
|
16576
|
+
// successors, each believing it owns the bind+adopt, racing for the same
|
|
16577
|
+
// socket and fleet. Validate and bind preparation to the FIRST successor
|
|
16578
|
+
// pid synchronously (before any await), and reject every other pid until
|
|
16579
|
+
// this handoff finishes (handover) or rolls back. The bound successor's
|
|
16580
|
+
// own retries (same pid) stay idempotent — they share the promise.
|
|
16581
|
+
if (!Number.isInteger(successorPid) || successorPid <= 0) {
|
|
16582
|
+
// An unidentifiable caller never earns the binding — without a pid the
|
|
16583
|
+
// successor watch cannot run and no retry could ever match the binding.
|
|
16584
|
+
const err = new Error('prepare-upgrade requires a valid successor pid');
|
|
16585
|
+
err.code = 'UPGRADE_BOUND';
|
|
16586
|
+
throw err;
|
|
16587
|
+
}
|
|
16588
|
+
if (upgrading) {
|
|
16589
|
+
const boundTo = upgradeInfo && upgradeInfo.successorPid;
|
|
16590
|
+
if (boundTo !== successorPid) {
|
|
16591
|
+
const err = new Error(`a daemon upgrade is already bound to successor pid ${boundTo} — refusing a concurrent handoff`);
|
|
16592
|
+
err.code = 'UPGRADE_BOUND';
|
|
16593
|
+
throw err;
|
|
16594
|
+
}
|
|
16595
|
+
}
|
|
16596
|
+
if (!upgrading || !quiescePromise) quiescePromise = doQuiesce(successorPid, successorVersion);
|
|
16597
|
+
return quiescePromise;
|
|
16598
|
+
};
|
|
16599
|
+
const doQuiesce = async (successorPid, successorVersion) => {
|
|
16600
|
+
if (!upgrading) {
|
|
16601
|
+
upgrading = true; // latch admission FIRST — rejects new mutations/respawns
|
|
16602
|
+
upgradeInfo = { state: 'quiescing', fromVersion: daemonVersion, toVersion: successorVersion ?? null, successorPid, sinceEpochMs: Date.now() };
|
|
16603
|
+
dlog(`daemon upgrade: quiescing for successor pid ${successorPid} — control socket accept loop closing, mutations refused`);
|
|
16604
|
+
try { server.close(); } catch { /* ignore */ }
|
|
16605
|
+
for (const s of [...attachClients]) { try { s.destroy(); } catch { /* ignore */ } }
|
|
16606
|
+
// Hold the event loop open until the successor's SIGUSR1 handover (see the
|
|
16607
|
+
// quiesceKeepalive declaration) — a no-op tick on a REFERENCED timer.
|
|
16608
|
+
if (!quiesceKeepalive) quiesceKeepalive = setInterval(() => {}, SUPERVISOR_ADOPT_POLL_MS);
|
|
16609
|
+
// #309 review (no-rollback): watch the successor. If it dies AFTER we
|
|
16610
|
+
// quiesced but BEFORE handing over (SIGUSR1), this daemon is left quiesced
|
|
16611
|
+
// and unreachable — respawns disabled, accept loop closed, no way for even
|
|
16612
|
+
// a later upgrade to reach it. Roll back: re-open the accept loop and
|
|
16613
|
+
// unlatch `upgrading` so normal operation (and a retry) resumes.
|
|
16614
|
+
armSuccessorWatch(successorPid);
|
|
16615
|
+
// Wait for any ALREADY-ADMITTED mutation (e.g. a restart mid-respawn) to
|
|
16616
|
+
// settle before snapshotting, so the successor adopts its replacement.
|
|
16617
|
+
try { await opQueue; } catch { /* mutation already logged */ }
|
|
16618
|
+
if (!upgrading) return; // rolled back while draining the queue
|
|
16619
|
+
upgradeInfo = { ...upgradeInfo, state: 'quiesced' };
|
|
16620
|
+
quiesceSnapshotIds = [...workers.keys()];
|
|
16621
|
+
persist(true); // the deliberate quiesce snapshot the successor adopts from
|
|
16622
|
+
}
|
|
16623
|
+
quiesced = true;
|
|
16624
|
+
};
|
|
16625
|
+
|
|
16626
|
+
// #309 review (no-rollback): while quiesced, poll the successor pid. If it
|
|
16627
|
+
// dies before handing over (SIGUSR1), this daemon would otherwise stay
|
|
16628
|
+
// quiesced + unreachable forever (accept loop closed, respawns disabled, no
|
|
16629
|
+
// listener for even a later upgrade). On detecting a dead successor, roll
|
|
16630
|
+
// back: re-open the accept loop and unlatch `upgrading`.
|
|
16631
|
+
const armSuccessorWatch = (successorPid) => {
|
|
16632
|
+
if (!Number.isInteger(successorPid) || successorPid <= 0) return;
|
|
16633
|
+
if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
|
|
16634
|
+
successorWatchTimer = setInterval(() => {
|
|
16635
|
+
if (!upgrading || handedOver) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; return; }
|
|
16636
|
+
if (!isPidAlive(successorPid)) {
|
|
16637
|
+
try { clearInterval(successorWatchTimer); } catch { /* ignore */ }
|
|
16638
|
+
successorWatchTimer = null;
|
|
16639
|
+
rollbackFromUpgrade(successorPid);
|
|
16640
|
+
}
|
|
16641
|
+
}, SUPERVISOR_ADOPT_POLL_MS);
|
|
16642
|
+
if (typeof successorWatchTimer.unref === 'function') successorWatchTimer.unref();
|
|
16643
|
+
};
|
|
16644
|
+
|
|
16645
|
+
// Restore normal operation after a successor died mid-handoff: re-listen on
|
|
16646
|
+
// the control socket (the successor never bound it, or died holding it) and
|
|
16647
|
+
// unlatch `upgrading` so mutations/respawns resume. Best-effort — if the
|
|
16648
|
+
// re-listen fails the daemon stays quiesced and logs, but the common case
|
|
16649
|
+
// (successor crashed before binding) recovers cleanly.
|
|
16650
|
+
const rollbackFromUpgrade = (deadSuccessorPid) => {
|
|
16651
|
+
if (!upgrading || handedOver) return;
|
|
16652
|
+
quiescePromise = null;
|
|
16653
|
+
dlog(`daemon upgrade: successor pid ${deadSuccessorPid} died before handover — rolling back: re-opening the control socket and resuming normal operation`);
|
|
16654
|
+
upgrading = false;
|
|
16655
|
+
quiesced = false;
|
|
16656
|
+
upgradeInfo = null;
|
|
16657
|
+
// Capture the snapshot BEFORE clearing it — the membership reconcile below
|
|
16658
|
+
// reads this copy (clearing first made every rollback drop the whole fleet).
|
|
16659
|
+
// A null snapshot (quiesce never reached the snapshot point) means there is
|
|
16660
|
+
// nothing to reconcile against, so membership is left untouched.
|
|
16661
|
+
const rollbackSnapshotIds = quiesceSnapshotIds;
|
|
16662
|
+
quiesceSnapshotIds = null;
|
|
16663
|
+
if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
|
|
16664
|
+
// Re-open the accept loop. The socket path may still be held by the dead
|
|
16665
|
+
// successor's listen socket; remove it first (same takeover the successor
|
|
16666
|
+
// itself performs), then re-bind.
|
|
16667
|
+
if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
|
|
16668
|
+
// Same owner-only umask as the initial bind, so the socket is private from
|
|
16669
|
+
// creation (no listen()→chmod window for another local user to connect).
|
|
16670
|
+
const rbWin = osPlatform() === 'win32';
|
|
16671
|
+
const rbPrevUmask = rbWin ? null : process.umask(0o177);
|
|
16672
|
+
try {
|
|
16673
|
+
server.listen(socketPath);
|
|
16674
|
+
if (!rbWin) { try { chmodSync(socketPath, 0o600); } catch { /* ignore */ } }
|
|
16675
|
+
dlog('daemon upgrade rollback: control socket re-opened — fleet serving normally');
|
|
16676
|
+
} catch (err) {
|
|
16677
|
+
dlog(`daemon upgrade rollback: failed to re-open the control socket: ${err?.message || err}`);
|
|
16678
|
+
} finally {
|
|
16679
|
+
if (!rbWin) { try { process.umask(rbPrevUmask); } catch { /* ignore */ } }
|
|
16680
|
+
}
|
|
16681
|
+
// #309 review: reconcile the inventory. Respawns were suppressed while
|
|
16682
|
+
// quiesced (expired restart timers and death handlers bailed), so reschedule
|
|
16683
|
+
// every eligible down worker now — preserving stop/config-failure/removal
|
|
16684
|
+
// intent and skipping any with a live process or a pending restart timer.
|
|
16685
|
+
// Membership is reconciled against the quiesce snapshot: a worker ABSENT
|
|
16686
|
+
// from it was removed through the SUCCESSOR while this daemon was quiesced
|
|
16687
|
+
// (the successor owns add/remove during the handoff), so re-adding it here
|
|
16688
|
+
// would resurrect a deliberately removed worker.
|
|
16689
|
+
// The authoritative handoff inventory is what the SUCCESSOR persisted (it
|
|
16690
|
+
// owns add/remove during the handoff): if the state file was last written
|
|
16691
|
+
// by the dead successor, its worker list wins; otherwise fall back to our
|
|
16692
|
+
// quiesce snapshot (the successor never wrote — nothing was removed).
|
|
16693
|
+
let snapshotIds = rollbackSnapshotIds ? new Set(rollbackSnapshotIds) : null;
|
|
16694
|
+
try {
|
|
16695
|
+
const succState = readSupervisorState();
|
|
16696
|
+
if (succState && succState.pid === deadSuccessorPid && Array.isArray(succState.workers)) {
|
|
16697
|
+
snapshotIds = new Set(succState.workers.map((x) => x && x.id).filter(Boolean));
|
|
16698
|
+
}
|
|
16699
|
+
} catch { /* unreadable — keep the snapshot */ }
|
|
16700
|
+
for (const id of snapshotIds ? [...workers.keys()] : []) {
|
|
16701
|
+
if (!snapshotIds.has(id)) {
|
|
16702
|
+
dlog(`daemon upgrade rollback: worker '${id}' is not in the quiesce snapshot (removed through the successor mid-handoff?) — leaving it removed`);
|
|
16703
|
+
workers.delete(id);
|
|
16704
|
+
}
|
|
16705
|
+
}
|
|
16706
|
+
for (const w of [...workers.values()]) {
|
|
16707
|
+
if (w.restartTimer || (w.pid && isPidAlive(w.pid))) continue;
|
|
16708
|
+
if (w.removeOnExit) {
|
|
16709
|
+
workers.delete(w.id);
|
|
16710
|
+
try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
|
|
16711
|
+
dlog(`worker '${w.id}' removed (drained during the aborted upgrade)`);
|
|
16712
|
+
continue;
|
|
16713
|
+
}
|
|
16714
|
+
if (w.stopping || w.configFailed) continue;
|
|
16715
|
+
if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
|
|
16716
|
+
dlog(`daemon upgrade rollback: worker '${w.id}' was down while quiesced — rescheduling`);
|
|
16717
|
+
startWorker(w);
|
|
16718
|
+
}
|
|
16719
|
+
persist();
|
|
16720
|
+
};
|
|
16721
|
+
|
|
15979
16722
|
const handleRequest = async (req, sock) => {
|
|
15980
16723
|
const op = req && req.op;
|
|
15981
16724
|
try {
|
|
@@ -15983,8 +16726,30 @@ async function runSupervisorDaemon() {
|
|
|
15983
16726
|
case 'status':
|
|
15984
16727
|
sock.write(encodeFrame(statusFrame(true)));
|
|
15985
16728
|
break;
|
|
16729
|
+
case 'prepare-upgrade': {
|
|
16730
|
+
// #309: a successor daemon (spawned with NANO_SUPERVISOR_UPGRADE=1)
|
|
16731
|
+
// asks us to quiesce so it can bind the control socket. Refuse while
|
|
16732
|
+
// a worker reload roll or a shutdown owns the fleet — an upgrade on
|
|
16733
|
+
// top of either would strand its bookkeeping. Idempotent: the
|
|
16734
|
+
// successor polls until it observes the quiesced state. Bound to ONE
|
|
16735
|
+
// successor pid (quiesceForUpgrade throws UPGRADE_BOUND for any
|
|
16736
|
+
// other) so two racing successors can never both win the fleet.
|
|
16737
|
+
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
16738
|
+
if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a worker reload is in progress — retry the daemon upgrade once it finishes', final: true })); break; }
|
|
16739
|
+
try {
|
|
16740
|
+
await quiesceForUpgrade(Number(req.pid) || null, typeof req.version === 'string' ? req.version : null);
|
|
16741
|
+
} catch (err) {
|
|
16742
|
+
if (err && err.code === 'UPGRADE_BOUND') { sock.write(encodeFrame({ ok: false, error: err.message, final: true })); break; }
|
|
16743
|
+
throw err;
|
|
16744
|
+
}
|
|
16745
|
+
// Report quiesced only AFTER the mutation queue has drained and the
|
|
16746
|
+
// snapshot persisted — the successor binds+adopts off this ack.
|
|
16747
|
+
sock.write(encodeFrame({ ok: true, type: 'prepare-upgrade', quiesced: true, final: true }));
|
|
16748
|
+
break;
|
|
16749
|
+
}
|
|
15986
16750
|
case 'add': {
|
|
15987
16751
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
16752
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
15988
16753
|
if (!req.profile) { sock.write(encodeFrame({ ok: false, error: 'add requires a profile', final: true })); break; }
|
|
15989
16754
|
// A supervised worker's name is supplied out-of-band as `req.name`
|
|
15990
16755
|
// (which the daemon forwards to the child as `nano work … --name`).
|
|
@@ -16007,6 +16772,7 @@ async function runSupervisorDaemon() {
|
|
|
16007
16772
|
}
|
|
16008
16773
|
case 'remove': {
|
|
16009
16774
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
16775
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16010
16776
|
// #202: `req.force === false` drains each worker (finish in-flight jobs
|
|
16011
16777
|
// then exit); the default remains a fast force-stop (used by reconcile
|
|
16012
16778
|
// and interactive remove). A drain-remove returns immediately and the
|
|
@@ -16022,6 +16788,7 @@ async function runSupervisorDaemon() {
|
|
|
16022
16788
|
}
|
|
16023
16789
|
case 'restart': {
|
|
16024
16790
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
16791
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16025
16792
|
const restarted = await serializeOp(async () => {
|
|
16026
16793
|
const ids = resolveTargets(req.target);
|
|
16027
16794
|
for (const id of ids) await restartWorker(id);
|
|
@@ -16038,6 +16805,7 @@ async function runSupervisorDaemon() {
|
|
|
16038
16805
|
// and ends with a terminal `reloaded` frame — draining can take a long
|
|
16039
16806
|
// time, so this MUST be a streaming op, not a one-shot request.
|
|
16040
16807
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
16808
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16041
16809
|
if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a reload is already in progress', final: true })); break; }
|
|
16042
16810
|
// Reject reload explicitly on Windows BEFORE draining anything. The
|
|
16043
16811
|
// rolling reload's graceful drain relies on SIGUSR2 (SUPERVISOR_DRAIN_SIGNAL)
|
|
@@ -16068,6 +16836,9 @@ async function runSupervisorDaemon() {
|
|
|
16068
16836
|
// rolling pass over the same workers. The flag is reset in runReload's
|
|
16069
16837
|
// `finally`.
|
|
16070
16838
|
reloading = true;
|
|
16839
|
+
reloadTargets = [...ids];
|
|
16840
|
+
reloadDone = [];
|
|
16841
|
+
reloadCurrent = null;
|
|
16071
16842
|
attachClients.add(sock);
|
|
16072
16843
|
sock.write(encodeFrame({ ok: true, type: 'reloading', targets: ids }));
|
|
16073
16844
|
sock.write(encodeFrame(statusFrame(false)));
|
|
@@ -16116,10 +16887,30 @@ async function runSupervisorDaemon() {
|
|
|
16116
16887
|
// #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
|
|
16117
16888
|
// jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
|
|
16118
16889
|
// yield jobs) and can also ESCALATE a drain already in progress.
|
|
16890
|
+
// #309: a quiesced (mid-upgrade) daemon must never run the stop path —
|
|
16891
|
+
// it would SIGTERM/SIGUSR2 the very workers the successor has adopted.
|
|
16892
|
+
// The upgrade's own exit path replaces this op for the old daemon.
|
|
16893
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — the successor owns the fleet now; stop it instead (its pid is in `supervisor status`)', final: true })); break; }
|
|
16119
16894
|
const force = !!req.force;
|
|
16120
|
-
//
|
|
16121
|
-
//
|
|
16122
|
-
//
|
|
16895
|
+
// #309 review: reserve shutdown ownership HERE, synchronously with the
|
|
16896
|
+
// acceptance — not 20ms later when the deferred shutdown() runs. An
|
|
16897
|
+
// unreserved accept lets a prepare-upgrade squeeze into the defer
|
|
16898
|
+
// window and latch `upgrading` first; the shutdown then overtakes the
|
|
16899
|
+
// quiesce and SIGTERMs the very workers the successor is adopting.
|
|
16900
|
+
// Latching now makes the race impossible in either order: a stop
|
|
16901
|
+
// accepted first refuses the upgrade (prepare-upgrade checks
|
|
16902
|
+
// `shuttingDown`), and an upgrade quiesced first refuses the stop
|
|
16903
|
+
// (the `upgrading` check above).
|
|
16904
|
+
//
|
|
16905
|
+
// A SECOND stop while one is already reserved is not a fresh shutdown:
|
|
16906
|
+
// it only ESCALATES a drain (force) or is a no-op. Never re-latch
|
|
16907
|
+
// `draining` here — a force stop arriving during a graceful drain must
|
|
16908
|
+
// leave `draining === true` so shutdown()'s escalation branch fires,
|
|
16909
|
+
// and a graceful stop during a force stop must not flip it back.
|
|
16910
|
+
if (!shutdownStarted && !shuttingDown) {
|
|
16911
|
+
shuttingDown = true;
|
|
16912
|
+
draining = !force;
|
|
16913
|
+
}
|
|
16123
16914
|
attachClients.add(sock);
|
|
16124
16915
|
stopClients.add(sock);
|
|
16125
16916
|
sock.write(encodeFrame(force
|
|
@@ -16128,7 +16919,12 @@ async function runSupervisorDaemon() {
|
|
|
16128
16919
|
sock.write(encodeFrame(statusFrame(false)));
|
|
16129
16920
|
// Kick the shutdown asynchronously; don't await it here so the control
|
|
16130
16921
|
// loop keeps serving (streaming status, accepting a force escalation).
|
|
16131
|
-
|
|
16922
|
+
// The 250ms defer lets a `stop --force` escalation (sent right behind
|
|
16923
|
+
// a graceful `stop`, as `stopStream`'s sendAfter does) land and run
|
|
16924
|
+
// shutdown's escalation branch BEFORE the drain's body force-stops on
|
|
16925
|
+
// its own — preserving the pre-#309 escalation contract now that
|
|
16926
|
+
// `shuttingDown` is latched early (see below).
|
|
16927
|
+
setTimeout(() => { shutdown('stop', { force }); }, 250);
|
|
16132
16928
|
break;
|
|
16133
16929
|
}
|
|
16134
16930
|
default:
|
|
@@ -16141,7 +16937,11 @@ async function runSupervisorDaemon() {
|
|
|
16141
16937
|
|
|
16142
16938
|
// Bind the control socket. A stale unix socket file from a crashed daemon
|
|
16143
16939
|
// would make listen() fail with EADDRINUSE even though nobody is listening;
|
|
16144
|
-
// remove it first
|
|
16940
|
+
// remove it first. In upgrade mode the old daemon has QUIESCED (its accept
|
|
16941
|
+
// loop is closed) but its listen socket still holds the path until it exits,
|
|
16942
|
+
// so we must unlink + rebind to take over — the same operation, deliberately
|
|
16943
|
+
// ordered AFTER the quiesce handshake so we can never steal the socket from a
|
|
16944
|
+
// daemon that is still accepting (#309).
|
|
16145
16945
|
if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
|
|
16146
16946
|
|
|
16147
16947
|
const server = createServer((sock) => {
|
|
@@ -16207,7 +17007,208 @@ async function runSupervisorDaemon() {
|
|
|
16207
17007
|
// and hard crashes still can't run this — the per-worker parent-death
|
|
16208
17008
|
// watchdog is the backstop for those.
|
|
16209
17009
|
process.once('SIGHUP', () => shutdown('SIGHUP'));
|
|
17010
|
+
// #309: SIGUSR1 is the upgrade handoff's "the successor owns the fleet — exit
|
|
17011
|
+
// WITHOUT touching the workers" signal. It is ONLY meaningful after a
|
|
17012
|
+
// `prepare-upgrade` quiesce (the successor sends it once it has adopted the
|
|
17013
|
+
// workers and retargeted their watchdogs). A SIGUSR1 outside an upgrade must
|
|
17014
|
+
// NOT exit (that would strand the fleet with no daemon), so it is ignored
|
|
17015
|
+
// unless `upgrading` is latched. Unlike `shutdown`, this path never signals
|
|
17016
|
+
// the workers and never clears the state file — the successor owns both now.
|
|
17017
|
+
process.on('SIGUSR1', () => {
|
|
17018
|
+
if (!upgrading) return;
|
|
17019
|
+
dlog(`daemon upgrade: successor confirmed ownership — old daemon (pid ${process.pid}) exiting; workers keep serving`);
|
|
17020
|
+
upgradeInfo = upgradeInfo ? { ...upgradeInfo, state: 'handed-over' } : upgradeInfo;
|
|
17021
|
+
// #309 review: NO persist here. The quiesce snapshot was this daemon's last
|
|
17022
|
+
// canonical write — the successor has since written its own pid + inventory,
|
|
17023
|
+
// and any write now would clobber it with the exiting predecessor's state.
|
|
17024
|
+
handedOver = true;
|
|
17025
|
+
quiesceSnapshotIds = null;
|
|
17026
|
+
if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
|
|
17027
|
+
if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
|
|
17028
|
+
if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
|
|
17029
|
+
process.exit(0);
|
|
17030
|
+
});
|
|
16210
17031
|
dlog(`supervisor daemon up (pid ${process.pid}) — control ${socketPath}`);
|
|
17032
|
+
|
|
17033
|
+
// #309: adopt the predecessor's workers. The old daemon quiesced (above) and
|
|
17034
|
+
// its last persist() — run when it latched the upgrade — names every worker
|
|
17035
|
+
// it owned. Each live pid becomes an adopted worker (polled for liveness;
|
|
17036
|
+
// never signaled, so in-flight jobs run undisturbed). Dead pids are dropped
|
|
17037
|
+
// with a log line — the fleet after the swap is exactly what survived it.
|
|
17038
|
+
if (upgradeMode && predecessorPid != null) {
|
|
17039
|
+
const prior = readSupervisorState();
|
|
17040
|
+
const adopted = [];
|
|
17041
|
+
for (const rec of (prior && Array.isArray(prior.workers) ? prior.workers : [])) {
|
|
17042
|
+
const w = adoptWorker(rec);
|
|
17043
|
+
if (w) { adopted.push(w.id); continue; }
|
|
17044
|
+
// #309 review: transfer the FULL inventory, not only live pids. A worker
|
|
17045
|
+
// the predecessor recorded but that is not alive right now falls into one
|
|
17046
|
+
// of three kept categories — dropping it would permanently shrink the
|
|
17047
|
+
// fleet (the predecessor refuses respawns once quiesced):
|
|
17048
|
+
if (!rec || typeof rec.id !== 'string' || !rec.id) { dlog('upgrade: skipping a worker record with no id'); continue; }
|
|
17049
|
+
if (rec.configFailed === true) {
|
|
17050
|
+
// Configuration-failed: retained (NOT restarted) so the operator's
|
|
17051
|
+
// `configFailed` signal survives the swap and a later fix+restart still
|
|
17052
|
+
// targets the same record.
|
|
17053
|
+
workers.set(rec.id, {
|
|
17054
|
+
id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
|
|
17055
|
+
args: Array.isArray(rec.args) ? rec.args.map(String) : [],
|
|
17056
|
+
restarts: Number(rec.restarts) || 0, stopping: rec.stopping === true,
|
|
17057
|
+
removeOnExit: rec.removeOnExit === true, lastExit: rec.lastExit ?? null,
|
|
17058
|
+
logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
|
|
17059
|
+
configFailed: true, adopted: true, pid: null, child: null,
|
|
17060
|
+
startedAt: rec.startedAt || null, spawnedAt: rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now(),
|
|
17061
|
+
});
|
|
17062
|
+
dlog(`upgrade: retained configuration-failed worker '${rec.id}' (not restarted)`);
|
|
17063
|
+
continue;
|
|
17064
|
+
}
|
|
17065
|
+
if (rec.stopping === true || rec.removeOnExit === true) {
|
|
17066
|
+
// Draining for stop/remove: the process is already gone, so the removal
|
|
17067
|
+
// completes here — delete the record rather than resurrect it.
|
|
17068
|
+
dlog(`upgrade: worker '${rec.id}' was draining for removal and is gone — record dropped`);
|
|
17069
|
+
try { rmSync(supervisorWorkerActivityFile(rec.id), { force: true }); } catch { /* best effort */ }
|
|
17070
|
+
continue;
|
|
17071
|
+
}
|
|
17072
|
+
// Down but eligible (a crash/backoff the predecessor had not yet
|
|
17073
|
+
// respawned when it quiesced): reschedule it here so fleet capacity is
|
|
17074
|
+
// preserved across the swap.
|
|
17075
|
+
dlog(`upgrade: worker '${rec.id}' was down at handoff — rescheduling to preserve fleet capacity`);
|
|
17076
|
+
const rw = {
|
|
17077
|
+
id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
|
|
17078
|
+
args: Array.isArray(rec.args) ? rec.args.map(String) : [],
|
|
17079
|
+
restarts: Number(rec.restarts) || 0, stopping: false, removeOnExit: false,
|
|
17080
|
+
lastExit: rec.lastExit ?? null,
|
|
17081
|
+
logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
|
|
17082
|
+
configFailed: false, adopted: true,
|
|
17083
|
+
};
|
|
17084
|
+
workers.set(rw.id, rw);
|
|
17085
|
+
startWorker(rw);
|
|
17086
|
+
rw.spawnedBySuccessor = true; // its watchdog already guards this daemon
|
|
17087
|
+
adopted.push(rw.id);
|
|
17088
|
+
}
|
|
17089
|
+
dlog(`upgrade: adopted ${adopted.length} worker(s) from predecessor pid ${predecessorPid}: ${adopted.join(', ') || '(none)'}`);
|
|
17090
|
+
broadcast({ type: 'event', event: 'daemon-adopted', predecessorPid, workers: adopted });
|
|
17091
|
+
|
|
17092
|
+
// Drive the handoff to completion. The old daemon's accept loop is closed
|
|
17093
|
+
// (quiesce), so it can no longer be asked over the socket — instead the
|
|
17094
|
+
// SUCCESSOR finishes the swap: write each live worker's `.upgrade` sidecar
|
|
17095
|
+
// (the successor pid the worker's SIGUSR2 handler reads), SIGUSR2 the
|
|
17096
|
+
// workers to retarget their watchdogs here, wait (bounded) for their
|
|
17097
|
+
// `watchdogPid` acks, then SIGUSR1 the old daemon so it persists and exits
|
|
17098
|
+
// WITHOUT touching the workers. Runs async so the control loop serves
|
|
17099
|
+
// immediately; the `upgrade` status block stays 'adopting' until the old
|
|
17100
|
+
// daemon is gone.
|
|
17101
|
+
(async () => {
|
|
17102
|
+
const myPid = process.pid;
|
|
17103
|
+
// Every live PREDECESSOR-owned worker stays pending until it has either
|
|
17104
|
+
// acked the retarget (`watchdogPid === myPid`) or exited (#309 review).
|
|
17105
|
+
// A worker still booting has Node's DEFAULT SIGUSR2 action (terminate)
|
|
17106
|
+
// until its handler is up, so it is signalled only once its activity
|
|
17107
|
+
// marker proves readiness (finite `readyAt` + matching pid) — but it is
|
|
17108
|
+
// NEVER dropped from the wait set, so the predecessor (its watchdog
|
|
17109
|
+
// target) is not released under it. Workers this successor spawned
|
|
17110
|
+
// itself already guard it and need no retarget.
|
|
17111
|
+
const pending = retargetPending;
|
|
17112
|
+
for (const id of adopted) {
|
|
17113
|
+
const w = workers.get(id);
|
|
17114
|
+
if (!w || w.spawnedBySuccessor || !w.pid || !isPidAlive(w.pid)) continue;
|
|
17115
|
+
if (w.stopping) {
|
|
17116
|
+
// #309 review: already draining (stop/remove) under the predecessor.
|
|
17117
|
+
// Its one-shot SIGUSR2 handler was consumed by that drain, so another
|
|
17118
|
+
// SIGUSR2 would hit Node's default action and kill its in-flight work.
|
|
17119
|
+
// Never write a sidecar or signal it — just keep it pending (its
|
|
17120
|
+
// watchdog still guards the predecessor) until it exits.
|
|
17121
|
+
pending.set(id, { pid: w.pid, signalled: true, drainOnly: true });
|
|
17122
|
+
continue;
|
|
17123
|
+
}
|
|
17124
|
+
const actFile = w.activityFile || supervisorWorkerActivityFile(w.id);
|
|
17125
|
+
try { writeFileSync(`${actFile}.upgrade`, String(myPid), { mode: 0o600 }); } catch { /* best effort */ }
|
|
17126
|
+
pending.set(id, { pid: w.pid, signalled: false });
|
|
17127
|
+
}
|
|
17128
|
+
// Bounded wait first; past the deadline report INCOMPLETE but KEEP
|
|
17129
|
+
// monitoring (#309 review) so a late ack / drain still releases the
|
|
17130
|
+
// predecessor and its keepalive — a retried upgrade would target this
|
|
17131
|
+
// successor, never the retained predecessor.
|
|
17132
|
+
const ackDeadline = Date.now() + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS;
|
|
17133
|
+
let reportedIncomplete = false;
|
|
17134
|
+
while (pending.size > 0) {
|
|
17135
|
+
if (!isPidAlive(predecessorPid)) break; // predecessor gone — nothing left to release
|
|
17136
|
+
for (const [id, p] of [...pending]) {
|
|
17137
|
+
const w = workers.get(id);
|
|
17138
|
+
if (!w || w.pid !== p.pid || !isPidAlive(p.pid)) { pending.delete(id); continue; }
|
|
17139
|
+
const act = readWorkerActivity(id);
|
|
17140
|
+
if (p.drainOnly) {
|
|
17141
|
+
// Draining, not retargeting: a deferred drain request (below) is
|
|
17142
|
+
// delivered once the handler is provably installed; then just wait
|
|
17143
|
+
// for the exit.
|
|
17144
|
+
if (p.drainDeferred && drainSignalReadyFor(act, p.pid)) {
|
|
17145
|
+
p.drainDeferred = false;
|
|
17146
|
+
try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
17147
|
+
}
|
|
17148
|
+
continue;
|
|
17149
|
+
}
|
|
17150
|
+
if (act && act.pid === p.pid && act.watchdogPid === myPid) {
|
|
17151
|
+
pending.delete(id);
|
|
17152
|
+
// Retarget landed and the handler re-armed — now deliver any drain
|
|
17153
|
+
// that was requested while the retarget was outstanding.
|
|
17154
|
+
if (p.drainDeferred) { try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
|
|
17155
|
+
continue;
|
|
17156
|
+
}
|
|
17157
|
+
if (!p.signalled && drainSignalReadyFor(act, p.pid)) {
|
|
17158
|
+
p.signalled = true;
|
|
17159
|
+
try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
17160
|
+
}
|
|
17161
|
+
}
|
|
17162
|
+
if (pending.size === 0) break;
|
|
17163
|
+
if (!reportedIncomplete && Date.now() >= ackDeadline) {
|
|
17164
|
+
reportedIncomplete = true;
|
|
17165
|
+
dlog(`upgrade: ${pending.size} worker(s) have not acked the watchdog retarget within ${SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS}ms (${[...pending.keys()].join(', ')}) — NOT releasing the predecessor yet; handoff INCOMPLETE, still monitoring until they ack or exit`);
|
|
17166
|
+
upgradeInfo = { state: 'incomplete', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, unacked: [...pending.keys()], sinceEpochMs: upgradeStartedMs };
|
|
17167
|
+
persist();
|
|
17168
|
+
}
|
|
17169
|
+
await new Promise((r) => setTimeout(r, reportedIncomplete ? SUPERVISOR_ADOPT_POLL_MS : 100));
|
|
17170
|
+
}
|
|
17171
|
+
// Predecessor gone with entries still pending: deliver deferred drains
|
|
17172
|
+
// that are safe now (handler installed and no retarget signal in flight
|
|
17173
|
+
// that the drain could be mistaken for). A drain deferred behind an
|
|
17174
|
+
// un-acked retarget is logged — signalling now could hit a consumed
|
|
17175
|
+
// one-shot handler and kill in-flight work.
|
|
17176
|
+
for (const [id, p] of pending) {
|
|
17177
|
+
if (!p.drainDeferred || !isPidAlive(p.pid)) continue;
|
|
17178
|
+
if (p.drainOnly && drainSignalReadyFor(readWorkerActivity(id), p.pid)) {
|
|
17179
|
+
try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
17180
|
+
} else {
|
|
17181
|
+
dlog(`upgrade: deferred drain for worker '${id}' not delivered (retarget unacknowledged or handler not ready) — re-issue stop/remove once it settles`);
|
|
17182
|
+
}
|
|
17183
|
+
}
|
|
17184
|
+
pending.clear();
|
|
17185
|
+
// Release the old daemon: it persists a final snapshot and exits 0 without
|
|
17186
|
+
// touching the workers. If it is already gone (crashed mid-handoff), the
|
|
17187
|
+
// workers are still adopted here — nothing more to do.
|
|
17188
|
+
if (isPidAlive(predecessorPid)) {
|
|
17189
|
+
try { process.kill(predecessorPid, 'SIGUSR1'); } catch { /* already gone */ }
|
|
17190
|
+
}
|
|
17191
|
+
dlog(`upgrade: handoff complete — successor pid ${myPid} owns the fleet (${daemonVersion})`);
|
|
17192
|
+
})().catch((err) => dlog(`upgrade handoff error: ${err?.message || err}`));
|
|
17193
|
+
|
|
17194
|
+
// Clear the status `upgrade` block once the predecessor has actually exited
|
|
17195
|
+
// — the roll is complete from then on, and a quiet fleet's status should
|
|
17196
|
+
// not carry a stale 'adopting' line forever. Unref'd: it never keeps the
|
|
17197
|
+
// daemon alive.
|
|
17198
|
+
const clearTimer = setInterval(() => {
|
|
17199
|
+
if (!isPidAlive(predecessorPid)) {
|
|
17200
|
+
try { clearInterval(clearTimer); } catch { /* ignore */ }
|
|
17201
|
+
predecessorPid = null;
|
|
17202
|
+
// #309 review: a handoff reported INCOMPLETE that later completed (late
|
|
17203
|
+
// acks/drains released the predecessor) must not keep reporting
|
|
17204
|
+
// 'incomplete' forever — currentUpgradeBlock() returns upgradeInfo first.
|
|
17205
|
+
if (upgradeInfo && upgradeInfo.state === 'incomplete') upgradeInfo = null;
|
|
17206
|
+
persist();
|
|
17207
|
+
}
|
|
17208
|
+
}, SUPERVISOR_ADOPT_POLL_MS);
|
|
17209
|
+
if (typeof clearTimer.unref === 'function') clearTimer.unref();
|
|
17210
|
+
}
|
|
17211
|
+
|
|
16211
17212
|
persist();
|
|
16212
17213
|
|
|
16213
17214
|
// Live-view refresh: periodically re-sample worker activity and push a fresh
|
|
@@ -16600,9 +17601,9 @@ async function streamSupervisorReload(socketPath, req, logger, { label = 'fleet'
|
|
|
16600
17601
|
* Hot-adopt new plugin code into the running fleet with zero downtime: roll
|
|
16601
17602
|
* through the target workers, gracefully draining (finish in-flight jobs) and
|
|
16602
17603
|
* respawning each so the new child re-reads the updated `c8ctl-plugin.js` from
|
|
16603
|
-
* disk. Defaults to the whole fleet. Note this adopts new WORKER code only;
|
|
16604
|
-
* supervisor daemon itself
|
|
16605
|
-
*
|
|
17604
|
+
* disk. Defaults to the whole fleet. Note this adopts new WORKER code only;
|
|
17605
|
+
* the supervisor daemon itself is rolled separately, without stopping the
|
|
17606
|
+
* fleet, by `supervisor upgrade` (issue #309).
|
|
16606
17607
|
*/
|
|
16607
17608
|
async function supervisorReloadCmd(req) {
|
|
16608
17609
|
const logger = getLogger();
|
|
@@ -16617,6 +17618,116 @@ async function supervisorReloadCmd(req) {
|
|
|
16617
17618
|
if (outcome === 'error') process.exit(1);
|
|
16618
17619
|
}
|
|
16619
17620
|
|
|
17621
|
+
/**
|
|
17622
|
+
* Rolling hot-upgrade of the supervisor DAEMON process (issue #309) — the
|
|
17623
|
+
* daemon-level counterpart of `supervisor reload` (which rolls only workers).
|
|
17624
|
+
* The daemon keeps the module graph it imported at start, so adopting new
|
|
17625
|
+
* SUPERVISOR-side code used to require `supervisor stop && start` — a fleet
|
|
17626
|
+
* outage (or, with --force, killed in-flight jobs). This command swaps the
|
|
17627
|
+
* parent process instead:
|
|
17628
|
+
*
|
|
17629
|
+
* 1. spawn a SUCCESSOR daemon (detached, NANO_SUPERVISOR_UPGRADE=1) running
|
|
17630
|
+
* the on-disk — post-`nano update` — plugin code;
|
|
17631
|
+
* 2. the successor asks the old daemon to QUIESCE (`prepare-upgrade`: accept
|
|
17632
|
+
* loop closed, mutations refused), then binds the canonical control
|
|
17633
|
+
* socket and ADOPTS the old daemon's workers from the persisted state
|
|
17634
|
+
* (pid-polled; the workers are never signaled, so in-flight jobs run
|
|
17635
|
+
* undisturbed);
|
|
17636
|
+
* 3. the successor then finishes the swap itself: it writes each worker's
|
|
17637
|
+
* `<activityFile>.upgrade` sidecar, SIGUSR2s the workers to retarget their
|
|
17638
|
+
* parent-death watchdogs at it, waits (bounded) for the `watchdogPid`
|
|
17639
|
+
* acks, and SIGUSR1s the old daemon — which persists a final snapshot and
|
|
17640
|
+
* exits 0 WITHOUT touching the workers.
|
|
17641
|
+
*
|
|
17642
|
+
* Workers never die mid-job; only the parent is swapped. A successor that
|
|
17643
|
+
* cannot reach a quiesced old daemon exits 1 and leaves the fleet untouched.
|
|
17644
|
+
* This client just spawns the successor and waits for the swap to land; Ctrl-C
|
|
17645
|
+
* detaches — the handoff completes in the background.
|
|
17646
|
+
*/
|
|
17647
|
+
async function supervisorUpgradeCmd() {
|
|
17648
|
+
const logger = getLogger();
|
|
17649
|
+
if (osPlatform() === 'win32') {
|
|
17650
|
+
logger.error('supervisor upgrade is not supported on Windows (its worker watchdog retarget relies on SIGUSR2, which Windows cannot deliver) — use `supervisor stop` + `start` to adopt new supervisor code.');
|
|
17651
|
+
process.exit(1);
|
|
17652
|
+
}
|
|
17653
|
+
const running = await liveSupervisor();
|
|
17654
|
+
if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
|
|
17655
|
+
const socketPath = running.socket || getSupervisorSocketPath();
|
|
17656
|
+
const oldPid = running.pid;
|
|
17657
|
+
const fromVersion = running.version || null;
|
|
17658
|
+
const toVersion = (() => { try { return pluginPackage().version; } catch { return null; } })();
|
|
17659
|
+
|
|
17660
|
+
// 1. Spawn the successor. NANO_SUPERVISOR_UPGRADE=1 puts its boot into
|
|
17661
|
+
// upgrade mode (quiesce handshake → bind → adopt → drive the handoff). We
|
|
17662
|
+
// deliberately do NOT clear the state file first — the successor reads the
|
|
17663
|
+
// old daemon's worker list from it. Ctrl-C detaches the waiter below; the
|
|
17664
|
+
// successor is detached and completes the swap regardless.
|
|
17665
|
+
const { exec, entry } = c8ctlInvocation();
|
|
17666
|
+
mkdirSync(getSupervisorLogDir(), { recursive: true });
|
|
17667
|
+
const logFile = supervisorDaemonLogFile();
|
|
17668
|
+
let fd;
|
|
17669
|
+
try { fd = openSync(logFile, 'a'); } catch { fd = 'ignore'; }
|
|
17670
|
+
const child = spawn(exec, [entry, 'nano', 'supervisor', '__daemon'], {
|
|
17671
|
+
env: { ...process.env, NANO_SUPERVISOR_UPGRADE: '1' },
|
|
17672
|
+
detached: true,
|
|
17673
|
+
stdio: ['ignore', fd, fd],
|
|
17674
|
+
});
|
|
17675
|
+
child.unref();
|
|
17676
|
+
if (typeof fd === 'number') { try { closeSync(fd); } catch { /* ignore */ } }
|
|
17677
|
+
if (typeof child.pid !== 'number') { logger.error('Failed to spawn the successor supervisor daemon.'); process.exit(1); }
|
|
17678
|
+
const successorPid = child.pid;
|
|
17679
|
+
logger.info(`Upgrading the supervisor daemon (pid ${oldPid}${fromVersion ? `, v${fromVersion}` : ''}) → successor pid ${successorPid}${toVersion ? ` (v${toVersion})` : ''}. Workers keep serving; only the parent process is swapped. Press Ctrl-C to detach.`);
|
|
17680
|
+
// #309 review (detached-breaks-supervision): the successor is spawned DETACHED
|
|
17681
|
+
// in this session, not under launchd/systemd. If a service unit is installed,
|
|
17682
|
+
// warn the operator that the new daemon is no longer service-supervised (and
|
|
17683
|
+
// that the old unit's clean exit 0 won't restart it — the generated units use
|
|
17684
|
+
// crash-only policies), and tell them how to re-establish supervision.
|
|
17685
|
+
try {
|
|
17686
|
+
if (supervisorServiceInstalled()) {
|
|
17687
|
+
logger.warn('A supervisor service is installed, but the upgraded daemon runs DETACHED in this session — it is no longer under launchd/systemd supervision (and the old unit\'s clean exit will not auto-restart it). After the upgrade, run `c8ctl nano supervisor install` (or `stop` + `start`) to bring the new-code daemon back under the service.');
|
|
17688
|
+
}
|
|
17689
|
+
} catch { /* best effort — never block the upgrade on a detection hiccup */ }
|
|
17690
|
+
|
|
17691
|
+
// 2. Wait for the swap to land: the successor takes the socket (status names
|
|
17692
|
+
// its pid) and the old daemon exits (released by the successor's SIGUSR1
|
|
17693
|
+
// once the workers' watchdogs are retargeted). Ctrl-C detaches — the
|
|
17694
|
+
// handoff completes in the background.
|
|
17695
|
+
let detached = false;
|
|
17696
|
+
const onSigint = () => { detached = true; };
|
|
17697
|
+
process.on('SIGINT', onSigint);
|
|
17698
|
+
const deadline = Date.now() + SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS + 10_000;
|
|
17699
|
+
try {
|
|
17700
|
+
for (;;) {
|
|
17701
|
+
if (detached) {
|
|
17702
|
+
logger.info('Detached — the daemon upgrade completes in the background. Rerun `nano supervisor status` to confirm.');
|
|
17703
|
+
return;
|
|
17704
|
+
}
|
|
17705
|
+
if (!isPidAlive(successorPid)) {
|
|
17706
|
+
logger.error(`The successor daemon (pid ${successorPid}) died during the upgrade — the old daemon (pid ${oldPid}) may still be in place. See ${logFile}.`);
|
|
17707
|
+
process.exit(1);
|
|
17708
|
+
}
|
|
17709
|
+
let successorServing = false;
|
|
17710
|
+
try {
|
|
17711
|
+
const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
|
|
17712
|
+
successorServing = !!(res && res.ok && res.daemon && res.daemon.pid === successorPid);
|
|
17713
|
+
} catch { /* socket mid-handoff — retry */ }
|
|
17714
|
+
const oldGone = !isPidAlive(oldPid);
|
|
17715
|
+
if (successorServing && oldGone) {
|
|
17716
|
+
logger.info(`Supervisor daemon upgraded${fromVersion || toVersion ? ` (${fromVersion ?? '?'} → ${toVersion ?? '?'})` : ''} — successor pid ${successorPid} owns the fleet; no worker was stopped.`);
|
|
17717
|
+
return;
|
|
17718
|
+
}
|
|
17719
|
+
if (Date.now() >= deadline) {
|
|
17720
|
+
if (!successorServing) logger.warn(`The successor daemon (pid ${successorPid}) did not take the control socket in time — the old daemon (pid ${oldPid}) may still own the fleet. Check \`nano supervisor status\` and ${logFile}.`);
|
|
17721
|
+
else logger.warn(`The old daemon (pid ${oldPid}) has not exited yet — the watchdog handoff is still completing. Check \`nano supervisor status\` in a moment.`);
|
|
17722
|
+
process.exit(1);
|
|
17723
|
+
}
|
|
17724
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
17725
|
+
}
|
|
17726
|
+
} finally {
|
|
17727
|
+
try { process.removeListener('SIGINT', onSigint); } catch { /* ignore */ }
|
|
17728
|
+
}
|
|
17729
|
+
}
|
|
17730
|
+
|
|
16620
17731
|
/**
|
|
16621
17732
|
* Re-read every supervised worker's cached gh auth token WITHOUT draining
|
|
16622
17733
|
* in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
|
|
@@ -16724,7 +17835,15 @@ async function supervisorStopCmd(force = false) {
|
|
|
16724
17835
|
const { frames, rest } = decodeFrames(buf);
|
|
16725
17836
|
buf = rest;
|
|
16726
17837
|
for (const frame of frames) {
|
|
16727
|
-
|
|
17838
|
+
// #309 review: an ok:false terminal frame is a REFUSAL, not a stop —
|
|
17839
|
+
// the daemon rejected the op (a daemon upgrade owns the fleet; the
|
|
17840
|
+
// successor holds the socket now). Treating it as a successful stop
|
|
17841
|
+
// would wait indefinitely for the wrong pid and, with --force,
|
|
17842
|
+
// SIGKILL the quiesced predecessor's process group — including the
|
|
17843
|
+
// workers the successor adopted — then erase the successor's state.
|
|
17844
|
+
if (frame && frame.ok === false) {
|
|
17845
|
+
finish({ error: String(frame.error || 'the supervisor refused the stop request') });
|
|
17846
|
+
} else if (frame && frame.type === 'stopping') {
|
|
16728
17847
|
logger.info('Aborting in-flight work — killing harnesses and yielding jobs for retry...');
|
|
16729
17848
|
} else if (frame && frame.event === 'draining-escalated') {
|
|
16730
17849
|
logger.info('Escalating to --force — aborting in-flight work...');
|
|
@@ -16750,6 +17869,15 @@ async function supervisorStopCmd(force = false) {
|
|
|
16750
17869
|
|
|
16751
17870
|
if (outcome === 'detached') return;
|
|
16752
17871
|
|
|
17872
|
+
// A refused stop (ok:false) is an ERROR outcome: surface it and exit before
|
|
17873
|
+
// any pid wait, force escalation, or state cleanup — the daemon that refused
|
|
17874
|
+
// (e.g. a quiesced predecessor mid-upgrade) is not ours to reap, and the
|
|
17875
|
+
// state file now names the successor that owns the fleet.
|
|
17876
|
+
if (outcome && typeof outcome === 'object' && outcome.error) {
|
|
17877
|
+
logger.error(`Supervisor stop refused: ${outcome.error}`);
|
|
17878
|
+
process.exit(1);
|
|
17879
|
+
}
|
|
17880
|
+
|
|
16753
17881
|
if (outcome === 'unreachable') {
|
|
16754
17882
|
if (force) {
|
|
16755
17883
|
// Socket unreachable — fall back to signalling the daemon pid directly.
|
|
@@ -16765,6 +17893,15 @@ async function supervisorStopCmd(force = false) {
|
|
|
16765
17893
|
// backstop so a wedged daemon/worker group can't linger. The window must cover
|
|
16766
17894
|
// the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
|
|
16767
17895
|
// group while a worker is still promoting its durable recovery ref (#295).
|
|
17896
|
+
// #309 review: if the daemon we asked is mid-upgrade (quiesced), the pid we
|
|
17897
|
+
// captured is the PREDECESSOR's — it exits on the successor's handover, not on
|
|
17898
|
+
// our refused/naive stop. Say so, and never SIGKILL its process group (the
|
|
17899
|
+
// successor's adopted workers share it) or clear the successor's state.
|
|
17900
|
+
if (running.upgrade && typeof running.upgrade === 'object') {
|
|
17901
|
+
const successorPid = Number(running.upgrade.successorPid);
|
|
17902
|
+
logger.warn(`The supervisor is mid-upgrade (handing the fleet to successor pid ${Number.isInteger(successorPid) && successorPid > 0 ? successorPid : 'unknown'}) — stop the successor instead; this daemon exits when the handoff completes.`);
|
|
17903
|
+
return;
|
|
17904
|
+
}
|
|
16768
17905
|
if (force) {
|
|
16769
17906
|
const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
|
|
16770
17907
|
while (Date.now() < deadline) {
|
|
@@ -17471,8 +18608,14 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
17471
18608
|
await supervisorRestartCmd(req);
|
|
17472
18609
|
return;
|
|
17473
18610
|
case 'reload':
|
|
18611
|
+
// `reload --daemon` is the daemon-upgrade alias (issue #309): the plain
|
|
18612
|
+
// op rolls WORKER code; this rolls the daemon process itself.
|
|
18613
|
+
if (flags?.daemon) { await supervisorUpgradeCmd(); return; }
|
|
17474
18614
|
await supervisorReloadCmd(req);
|
|
17475
18615
|
return;
|
|
18616
|
+
case 'upgrade':
|
|
18617
|
+
await supervisorUpgradeCmd();
|
|
18618
|
+
return;
|
|
17476
18619
|
case 'reload-credentials':
|
|
17477
18620
|
await supervisorReloadCredentialsCmd(req);
|
|
17478
18621
|
return;
|
|
@@ -17484,7 +18627,7 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
17484
18627
|
supervisorLogsCmd(req);
|
|
17485
18628
|
return;
|
|
17486
18629
|
default:
|
|
17487
|
-
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
|
|
18630
|
+
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|upgrade|reload-credentials|stop|logs|attach`);
|
|
17488
18631
|
process.exit(1);
|
|
17489
18632
|
}
|
|
17490
18633
|
}
|
|
@@ -20383,6 +21526,7 @@ export {
|
|
|
20383
21526
|
summarizeSupervisorWorker,
|
|
20384
21527
|
formatSupervisorStatus,
|
|
20385
21528
|
statusFromState,
|
|
21529
|
+
stateFromStatus,
|
|
20386
21530
|
supervisorDaemonDescriptor,
|
|
20387
21531
|
formatSupervisorLogsLines,
|
|
20388
21532
|
reageSupervisorStatus,
|
|
@@ -20399,6 +21543,7 @@ export {
|
|
|
20399
21543
|
normalizeAgenticMessage,
|
|
20400
21544
|
buildActivityPayload,
|
|
20401
21545
|
activityMarkerReadyFor,
|
|
21546
|
+
drainSignalReadyFor,
|
|
20402
21547
|
reloadCredentialsReady,
|
|
20403
21548
|
waitForChildExit,
|
|
20404
21549
|
supervisorWorkerActivityFile,
|
|
@@ -20411,6 +21556,7 @@ export {
|
|
|
20411
21556
|
supervisorStartCmd,
|
|
20412
21557
|
supervisorAddCmd,
|
|
20413
21558
|
supervisorReloadCmd,
|
|
21559
|
+
supervisorUpgradeCmd,
|
|
20414
21560
|
supervisorReloadCredentialsCmd,
|
|
20415
21561
|
workforceReloadCmd,
|
|
20416
21562
|
runningSupervisor,
|
|
@@ -20519,7 +21665,8 @@ export const metadata = {
|
|
|
20519
21665
|
{ command: 'c8ctl nano supervisor add decider', description: 'Add a supervised worker (forwarding work flags) to the running supervisor' },
|
|
20520
21666
|
{ command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
|
|
20521
21667
|
{ command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
|
|
20522
|
-
{ command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only
|
|
21668
|
+
{ command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only — use `supervisor upgrade` for the daemon)' },
|
|
21669
|
+
{ command: 'c8ctl nano supervisor upgrade', description: 'Rolling hot-upgrade of the supervisor DAEMON itself (alias: `reload --daemon`): a successor daemon on the new code takes the control socket and ADOPTS the running workers (no drain — in-flight jobs never stop), then the old daemon exits. Status shows `upgrade: rolling <from> → <to>` while it runs' },
|
|
20523
21670
|
{ command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
|
|
20524
21671
|
{ command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
|
|
20525
21672
|
{ command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
|
|
@@ -20601,6 +21748,7 @@ export const commands = {
|
|
|
20601
21748
|
worker: { type: 'string', multiple: true, description: 'supervisor start: profile to launch as a supervised worker (repeatable)' },
|
|
20602
21749
|
instances: { type: 'string', description: `supervisor add / workforce add: spawn/compose N distinct instances of the profile in one call (default 1, max ${MAX_ADD_INSTANCES}; for supervisor add cannot combine with --name)` },
|
|
20603
21750
|
attach: { type: 'boolean', description: 'supervisor start: attach the interactive console after starting the daemon' },
|
|
21751
|
+
daemon: { type: 'boolean', description: 'supervisor reload: roll the DAEMON process itself onto the on-disk code (alias of `supervisor upgrade`) instead of rolling worker code' },
|
|
20604
21752
|
manifest: { type: 'string', description: `workforce: manifest name to operate on (default ${DEFAULT_WORKFORCE_MANIFEST}); each subcommand reads/writes <stateHome>/workforce/<name>.json. Renamed from --profile (which now collides with c8ctl's global connection-profile flag).` },
|
|
20605
21753
|
roles: { type: 'string', description: 'workforce add: comma-separated role list for the entry (→ --job-type <rank>:<role> at start); mutually exclusive with --auto' },
|
|
20606
21754
|
},
|