c8ctl-plugin-nano 1.72.0 → 1.73.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -12
- package/c8ctl-plugin.js +1460 -36
- package/package.json +8 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -49,6 +49,7 @@ import {
|
|
|
49
49
|
utimesSync,
|
|
50
50
|
mkdtempSync,
|
|
51
51
|
closeSync,
|
|
52
|
+
readlinkSync,
|
|
52
53
|
watchFile,
|
|
53
54
|
unwatchFile,
|
|
54
55
|
} from 'node:fs';
|
|
@@ -295,6 +296,40 @@ const SUPERVISOR_RELOAD_READY_TIMEOUT_MS = Math.max(
|
|
|
295
296
|
0,
|
|
296
297
|
Number.parseInt(process.env.NANO_SUPERVISOR_RELOAD_READY_TIMEOUT_MS ?? '', 10) || 30_000,
|
|
297
298
|
);
|
|
299
|
+
// Rolling daemon hot-upgrade (issue #309). A `supervisor upgrade` hands the
|
|
300
|
+
// fleet to a SUCCESSOR daemon process running the on-disk (new) plugin code:
|
|
301
|
+
// the old daemon quiesces (stops accepting control connections + stops
|
|
302
|
+
// restarting workers), asks each worker to re-arm its parent-death watchdog at
|
|
303
|
+
// the successor's pid (SIGUSR2 + NANO_SUPERVISOR_UPGRADE_PID — a retarget, NOT
|
|
304
|
+
// a drain), persists a final state snapshot, and exits WITHOUT touching the
|
|
305
|
+
// workers. The successor binds the canonical control socket and adopts the
|
|
306
|
+
// still-running worker pids. Workers never die mid-job; only the parent swaps.
|
|
307
|
+
//
|
|
308
|
+
// How long the old daemon waits for workers to acknowledge the watchdog
|
|
309
|
+
// retarget (by stamping `watchdogPid` on their activity marker) before it exits
|
|
310
|
+
// anyway. Bounded so a wedged/older worker can't stall the upgrade forever —
|
|
311
|
+
// the successor adopts every live pid regardless, so an un-acked worker is
|
|
312
|
+
// still parented (its watchdog self-reaps only if the successor ALSO dies).
|
|
313
|
+
const SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS = Math.max(
|
|
314
|
+
0,
|
|
315
|
+
Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS ?? '', 10) || 15_000,
|
|
316
|
+
);
|
|
317
|
+
// How long a freshly spawned successor waits for the old daemon to quiesce
|
|
318
|
+
// before giving up (it must never bind the socket while the old daemon still
|
|
319
|
+
// accepts on it). Generous: quiesce is prompt (server.close + a latch), but a
|
|
320
|
+
// loaded machine can delay the old daemon's socket loop.
|
|
321
|
+
const SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS = Math.max(
|
|
322
|
+
0,
|
|
323
|
+
Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS ?? '', 10) || 30_000,
|
|
324
|
+
);
|
|
325
|
+
// How often the successor polls an adopted worker's pid for liveness. An
|
|
326
|
+
// adopted worker has no ChildProcess handle (the successor never spawned it),
|
|
327
|
+
// so there is no 'exit' event — this poll is the death detector that feeds the
|
|
328
|
+
// same restart-backoff path a spawned child's exit would.
|
|
329
|
+
const SUPERVISOR_ADOPT_POLL_MS = Math.max(
|
|
330
|
+
100,
|
|
331
|
+
Number.parseInt(process.env.NANO_SUPERVISOR_ADOPT_POLL_MS ?? '', 10) || 2_000,
|
|
332
|
+
);
|
|
298
333
|
// Upper bound on one `--auto` engine-read reconcile (enumerate deployed
|
|
299
334
|
// definitions + fetch each BPMN). A read that stalls past this is treated as a
|
|
300
335
|
// transient failure so the running poller set is KEPT and, crucially, shutdown
|
|
@@ -474,6 +509,19 @@ function activityMarkerReadyFor(act, pid) {
|
|
|
474
509
|
return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
|
|
475
510
|
}
|
|
476
511
|
|
|
512
|
+
/**
|
|
513
|
+
* Whether a worker's SIGUSR2 (drain/retarget) handler is provably installed
|
|
514
|
+
* (#309 review): its marker belongs to `pid` AND carries a finite
|
|
515
|
+
* `signalReadyAt` (stamped right after the handler is armed, independent of
|
|
516
|
+
* leasing). Falls back to `readyAt` for a worker that predates the field (its
|
|
517
|
+
* handler is armed before its first activation). Signalling earlier would hit
|
|
518
|
+
* Node's DEFAULT SIGUSR2 action and terminate the worker. Pure for unit tests.
|
|
519
|
+
*/
|
|
520
|
+
function drainSignalReadyFor(act, pid) {
|
|
521
|
+
if (!act || act.pid !== pid) return false;
|
|
522
|
+
return Number.isFinite(act.signalReadyAt) || Number.isFinite(act.readyAt);
|
|
523
|
+
}
|
|
524
|
+
|
|
477
525
|
/**
|
|
478
526
|
* Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
|
|
479
527
|
* All three must hold (#293): the worker exists, its PID is alive, AND its
|
|
@@ -2658,6 +2706,170 @@ function sampleSubtreeCpu(rootPid) {
|
|
|
2658
2706
|
return sampleSubtreeCpuPs(rootPid);
|
|
2659
2707
|
}
|
|
2660
2708
|
|
|
2709
|
+
// ---- Workspace-CPU liveness + in-flight tool holds (#313) ------------------
|
|
2710
|
+
// The #130 subtree probe only sees processes INSIDE the agent's own process
|
|
2711
|
+
// tree. Real work often happens OUTSIDE that tree: `cargo` builds through a
|
|
2712
|
+
// shared `sccache` server (a daemon parented to launchd/init, not the agent),
|
|
2713
|
+
// and a long foreground bash tool call can compile/test silently for minutes
|
|
2714
|
+
// (nanobpm/nano-bpm#1664 attempt 1 was idle-killed at 216 turns / 156 min with
|
|
2715
|
+
// a build genuinely running). Two extra liveness signals close that gap:
|
|
2716
|
+
// 1. `sampleWorkspaceCpu` — aggregate CPU of ANY process whose cwd is inside
|
|
2717
|
+
// the agent's workspace (the sccache/rustc case: the daemon's compile
|
|
2718
|
+
// workers chdir into the worktree they compile for).
|
|
2719
|
+
// 2. an in-flight-tool-call hold — while the agent has an ACP tool call still
|
|
2720
|
+
// open (a bash call it started has not reported a terminal status) it is
|
|
2721
|
+
// treated as active; the call's own timeout / the absolute hard cap still
|
|
2722
|
+
// bound a genuinely hung call.
|
|
2723
|
+
// Both ride the monitor below as extra samplers consulted only during an idle
|
|
2724
|
+
// window, so a truly quiescent agent still ages out exactly as before.
|
|
2725
|
+
|
|
2726
|
+
/** Default upper bound for the workspace-cwd scan (pid count). */
|
|
2727
|
+
const WORKSPACE_CPU_SCAN_LIMIT = 4096;
|
|
2728
|
+
|
|
2729
|
+
/**
|
|
2730
|
+
* Canonicalize a workspace root for cwd prefix checks. The kernel reports a
|
|
2731
|
+
* process cwd as the PHYSICAL path (`/proc/<pid>/cwd`, lsof), so a root reached
|
|
2732
|
+
* through a symlink (e.g. `C8CTL_NANO_HOME` under a symlinked dir, or macOS
|
|
2733
|
+
* `/tmp` → `/private/tmp`) must be realpath'd or every in-workspace process
|
|
2734
|
+
* fails the prefix check. Returns the list of distinct roots to accept (the
|
|
2735
|
+
* physical one plus the lexical one), or `[]` when the input is unusable.
|
|
2736
|
+
*/
|
|
2737
|
+
function workspaceRoots(workspaceDir, realpath = realpathSync) {
|
|
2738
|
+
if (typeof workspaceDir !== 'string' || !workspaceDir) return [];
|
|
2739
|
+
const lexical = resolvePath(workspaceDir);
|
|
2740
|
+
let physical = lexical;
|
|
2741
|
+
try { physical = realpath(lexical); } catch { /* missing dir → lexical only */ }
|
|
2742
|
+
return physical === lexical ? [lexical] : [physical, lexical];
|
|
2743
|
+
}
|
|
2744
|
+
|
|
2745
|
+
/** True when `cwd` is one of `roots` or lies beneath one. */
|
|
2746
|
+
function cwdInRoots(cwd, roots) {
|
|
2747
|
+
if (typeof cwd !== 'string' || !cwd) return false;
|
|
2748
|
+
return roots.some((root) => cwd === root || cwd.startsWith(root.endsWith(sep) ? root : root + sep));
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
/**
|
|
2752
|
+
* Linux: aggregate CPU (utime+stime ticks) of every process whose cwd resolves
|
|
2753
|
+
* inside `workspaceDir` — regardless of parentage — via `/proc/<pid>/cwd`
|
|
2754
|
+
* symlinks. Returns `{ processes, cpu }` (cpu in clock ticks, matching the
|
|
2755
|
+
* `/proc` subtree sampler), or `null` when the workspace is unusable (so the
|
|
2756
|
+
* caller degrades to subtree-only liveness rather than mis-killing). A process
|
|
2757
|
+
* counted here is NOT a descendant of the agent, so it is a pure CPU-progress
|
|
2758
|
+
* signal, never a presence signal. Bounded by `scanLimit` pids so a wedged
|
|
2759
|
+
* /proc read can't stall the watchdog. `readCwd`/`listPids` are injectable for
|
|
2760
|
+
* tests.
|
|
2761
|
+
*/
|
|
2762
|
+
function sampleWorkspaceCpuProc(workspaceDir, { readCwd, listPids, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
|
|
2763
|
+
const roots = workspaceRoots(workspaceDir, realpath);
|
|
2764
|
+
if (roots.length === 0) return null;
|
|
2765
|
+
const read = typeof readCwd === 'function' ? readCwd : (pid) => {
|
|
2766
|
+
try { return readlinkSync(`/proc/${pid}/cwd`); } catch { return null; }
|
|
2767
|
+
};
|
|
2768
|
+
let entries;
|
|
2769
|
+
if (typeof listPids === 'function') entries = listPids();
|
|
2770
|
+
else {
|
|
2771
|
+
try { entries = readdirSync('/proc'); } catch { return null; }
|
|
2772
|
+
}
|
|
2773
|
+
let processes = 0;
|
|
2774
|
+
let totalCpu = 0;
|
|
2775
|
+
let scanned = 0;
|
|
2776
|
+
for (const name of entries) {
|
|
2777
|
+
if (scanned >= scanLimit) break;
|
|
2778
|
+
if (!/^\d+$/.test(name)) continue;
|
|
2779
|
+
scanned += 1;
|
|
2780
|
+
const pid = Number(name);
|
|
2781
|
+
if (pid === process.pid || excludePids.includes(pid)) continue;
|
|
2782
|
+
const cwd = read(pid);
|
|
2783
|
+
if (typeof cwd !== 'string' || !cwd) continue;
|
|
2784
|
+
if (!cwdInRoots(cwd, roots)) continue;
|
|
2785
|
+
let stat;
|
|
2786
|
+
try { stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { continue; }
|
|
2787
|
+
const rparen = stat.lastIndexOf(')');
|
|
2788
|
+
if (rparen < 0) continue;
|
|
2789
|
+
const tail = stat.slice(rparen + 1).trim().split(/\s+/);
|
|
2790
|
+
const utime = Number(tail[11]);
|
|
2791
|
+
const stime = Number(tail[12]);
|
|
2792
|
+
totalCpu += (Number.isFinite(utime) ? utime : 0) + (Number.isFinite(stime) ? stime : 0);
|
|
2793
|
+
processes += 1;
|
|
2794
|
+
}
|
|
2795
|
+
return { processes, cpu: totalCpu };
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
/**
|
|
2799
|
+
* Non-Linux fallback (macOS): aggregate CPU of processes whose cwd is inside
|
|
2800
|
+
* `workspaceDir`, via a bounded `lsof` + `ps`. Heavier (two short-lived
|
|
2801
|
+
* spawns) but runs only during an idle window. Returns
|
|
2802
|
+
* `{ processes, cpu, resolutionMs: 1000 }` (cpu in whole seconds, so the
|
|
2803
|
+
* coarse-resolution flat-CPU rule applies) or `null` when unavailable.
|
|
2804
|
+
*/
|
|
2805
|
+
function sampleWorkspaceCpuLsof(workspaceDir, { runLsof, runPs, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
|
|
2806
|
+
const roots = workspaceRoots(workspaceDir, realpath);
|
|
2807
|
+
if (roots.length === 0) return null;
|
|
2808
|
+
const lsof = typeof runLsof === 'function' ? runLsof : () => {
|
|
2809
|
+
// `lsof -d cwd -Fpn` lists the cwd of EVERY process as machine-readable
|
|
2810
|
+
// `p<pid>` / `n<path>` records. A path filter (`-- <root>`) would only match
|
|
2811
|
+
// processes whose cwd is EXACTLY the root, missing a compiler working in a
|
|
2812
|
+
// nested crate dir, so we collect all cwd records and apply the
|
|
2813
|
+
// root-or-descendant check ourselves. Bounded by timeout + maxBuffer; lsof
|
|
2814
|
+
// exits 1 on partial permission errors, which still yields usable stdout.
|
|
2815
|
+
const r = spawnSync('lsof', ['-n', '-P', '-w', '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
|
|
2816
|
+
if (r.error && r.error.code === 'ENOENT') return null; // lsof missing → degrade
|
|
2817
|
+
if (r.error && !r.stdout) return null; // timeout / overflow with nothing usable
|
|
2818
|
+
return typeof r.stdout === 'string' ? r.stdout : '';
|
|
2819
|
+
};
|
|
2820
|
+
let lsofOut;
|
|
2821
|
+
try { lsofOut = lsof(); } catch { return null; }
|
|
2822
|
+
if (typeof lsofOut !== 'string') return null;
|
|
2823
|
+
const pids = [];
|
|
2824
|
+
let curPid = null;
|
|
2825
|
+
for (const line of lsofOut.split('\n')) {
|
|
2826
|
+
const tag = line.charCodeAt(0);
|
|
2827
|
+
if (tag === 112 /* 'p' */) {
|
|
2828
|
+
const pid = Number(line.slice(1));
|
|
2829
|
+
curPid = Number.isFinite(pid) && pid > 0 && pid !== process.pid && !excludePids.includes(pid) ? pid : null;
|
|
2830
|
+
} else if (tag === 110 /* 'n' */ && curPid != null) {
|
|
2831
|
+
if (cwdInRoots(line.slice(1), roots) && !pids.includes(curPid)) pids.push(curPid);
|
|
2832
|
+
curPid = null;
|
|
2833
|
+
if (pids.length >= scanLimit) break;
|
|
2834
|
+
}
|
|
2835
|
+
}
|
|
2836
|
+
if (pids.length === 0) return { processes: 0, cpu: 0, resolutionMs: 1000 };
|
|
2837
|
+
const ps = typeof runPs === 'function' ? runPs : (list) => {
|
|
2838
|
+
const r = spawnSync('ps', ['-p', list.join(','), '-o', 'pid=,time='], { encoding: 'utf8', timeout: 5_000 });
|
|
2839
|
+
// ps exits 1 when some pids vanished between scans; partial stdout is valid.
|
|
2840
|
+
return typeof r.stdout === 'string' ? r.stdout : null;
|
|
2841
|
+
};
|
|
2842
|
+
let psOut;
|
|
2843
|
+
try { psOut = ps(pids); } catch { return null; }
|
|
2844
|
+
if (typeof psOut !== 'string') return null;
|
|
2845
|
+
let processes = 0;
|
|
2846
|
+
let totalCpu = 0;
|
|
2847
|
+
for (const line of psOut.split('\n')) {
|
|
2848
|
+
const m = line.trim().match(/^(\d+)\s+(\S+)$/);
|
|
2849
|
+
if (!m) continue;
|
|
2850
|
+
totalCpu += parsePsTime(m[2]);
|
|
2851
|
+
processes += 1;
|
|
2852
|
+
}
|
|
2853
|
+
return { processes, cpu: totalCpu, resolutionMs: 1000 };
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
/**
|
|
2857
|
+
* Sample aggregate CPU of processes working inside `workspaceDir` regardless of
|
|
2858
|
+
* parentage (the sccache/shared-daemon case, #313). Linux reads `/proc`; macOS
|
|
2859
|
+
* falls back to `lsof`+`ps`; other platforms return `null` (unsupported → the
|
|
2860
|
+
* monitor degrades to subtree-only liveness). The result shape mirrors the
|
|
2861
|
+
* subtree sampler (`cpu` in ticks on Linux, whole seconds under `ps`) with
|
|
2862
|
+
* `processes` instead of `descendants`.
|
|
2863
|
+
*/
|
|
2864
|
+
function sampleWorkspaceCpu(workspaceDir, { excludePids } = {}) {
|
|
2865
|
+
// The agent itself runs with cwd == workspace; its own CPU (an idle harness
|
|
2866
|
+
// ticking timers) is NOT out-of-tree work and must never defer its idle-kill.
|
|
2867
|
+
const exclude = Array.isArray(excludePids) ? excludePids.filter((p) => Number.isFinite(p) && p > 0) : [];
|
|
2868
|
+
if (process.platform === 'linux') return sampleWorkspaceCpuProc(workspaceDir, { excludePids: exclude });
|
|
2869
|
+
if (process.platform === 'darwin') return sampleWorkspaceCpuLsof(workspaceDir, { excludePids: exclude });
|
|
2870
|
+
return null; // win32 et al: unsupported → degrade, never mis-kill
|
|
2871
|
+
}
|
|
2872
|
+
|
|
2661
2873
|
/**
|
|
2662
2874
|
* Build a progress-aware idle-liveness monitor shared by the pipe/PTY/ACP
|
|
2663
2875
|
* capture paths. `arm()` is called on every output chunk (and once at start) to
|
|
@@ -2683,37 +2895,155 @@ function sampleSubtreeCpu(rootPid) {
|
|
|
2683
2895
|
* read under a coarse sampler must NOT kill; the fine-grained `/proc` path
|
|
2684
2896
|
* carries no resolution and so ages out on the first flat window as before →
|
|
2685
2897
|
* `onIdleKill()`.
|
|
2898
|
+
* #313 extensions:
|
|
2899
|
+
* - `extraSamplers`: additional activity probes (e.g. the workspace-cwd CPU
|
|
2900
|
+
* scan covering out-of-tree sccache/rustc daemons). Each returns
|
|
2901
|
+
* `{ processes, cpu, resolutionMs? } | null`; advancing CPU on ANY sampler
|
|
2902
|
+
* defers the kill. Extra samplers are pure progress signals — they never
|
|
2903
|
+
* satisfy the presence gate (a childless agent with a busy workspace daemon
|
|
2904
|
+
* still needs its own live descendant to count as present... EXCEPT that a
|
|
2905
|
+
* workspace daemon burning CPU IS evidence the agent's work is in flight, so
|
|
2906
|
+
* when an extra sampler shows advancing CPU the kill is deferred even with
|
|
2907
|
+
* no live descendant: the agent may be blocked reading a pipe its daemon
|
|
2908
|
+
* fills. A flat/zero extra sample never kills on its own — presence is
|
|
2909
|
+
* judged from the subtree as before).
|
|
2910
|
+
* - `isActivityHold`: an in-flight-tool-call latch (ACP). While it returns
|
|
2911
|
+
* true the agent has an open tool call, so silence is expected; the kill is
|
|
2912
|
+
* deferred window-by-window. The absolute `--job-timeout` hard cap (and the
|
|
2913
|
+
* tool call's own timeout, enforced by the agent) remain the bound for a
|
|
2914
|
+
* genuinely hung call.
|
|
2915
|
+
* - `logger`: when given, the monitor logs WHY it considered the agent active
|
|
2916
|
+
* or idle at each probe (diagnosability, #313).
|
|
2686
2917
|
* The first elapse with live descendants takes a baseline (a single sample can't
|
|
2687
2918
|
* show movement) and re-probes after the recovery window. The absolute
|
|
2688
2919
|
* `--job-timeout` hard cap remains the ultimate backstop, untouched by this.
|
|
2689
2920
|
*/
|
|
2690
|
-
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu }) {
|
|
2921
|
+
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu, extraSamplers = null, isActivityHold = null, maxActivityHoldMs, logger = null }) {
|
|
2691
2922
|
const enabled = idleTimeoutMs && idleTimeoutMs > 0;
|
|
2692
2923
|
const probeWindow = recoveryWindowMs && recoveryWindowMs > 0 ? recoveryWindowMs : idleTimeoutMs;
|
|
2693
2924
|
let timer = null;
|
|
2694
2925
|
let prev = null; // last subtree CPU probe taken during the current silent stretch
|
|
2926
|
+
let prevExtra = null; // last per-extra-sampler CPU readings (array parallel to extraSamplers)
|
|
2927
|
+
let extraBaselined = false; // true once the extra samplers have a baseline to compare against
|
|
2928
|
+
let extraResolutionMs = 0; // coarsest resolution among live extra samples at the last probe
|
|
2695
2929
|
let flatMs = 0; // cumulative time observed with flat CPU since the last movement/baseline
|
|
2696
2930
|
let lastWindow = idleTimeoutMs; // duration of the currently pending probe window
|
|
2931
|
+
let extraFlatMs = 0; // cumulative flat time across the extra (workspace) samplers
|
|
2932
|
+
let holdMs = 0; // cumulative silent time deferred by the activity hold this stretch
|
|
2933
|
+
// #313: the activity hold must be FINITE — `nano work` defaults to no hard cap,
|
|
2934
|
+
// so an announced-then-hung tool call would otherwise defer forever. Default:
|
|
2935
|
+
// 4 idle windows (configurable). Past it, the hold is ignored and the normal
|
|
2936
|
+
// CPU probes decide (real silent work still defers via CPU progress).
|
|
2937
|
+
const holdCap = Number.isFinite(maxActivityHoldMs) && maxActivityHoldMs >= 0 ? maxActivityHoldMs : (idleTimeoutMs > 0 ? idleTimeoutMs * 4 : 0);
|
|
2697
2938
|
const settled = () => (typeof isSettled === 'function' ? isSettled() : false);
|
|
2698
2939
|
const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
|
|
2699
2940
|
const schedule = (ms) => { lastWindow = ms; timer = setTimeout(onElapse, ms); };
|
|
2941
|
+
const log = (msg) => { try { logger?.info?.(`[idle-watchdog] ${msg}`); } catch { /* best effort */ } };
|
|
2942
|
+
const extras = Array.isArray(extraSamplers) ? extraSamplers.filter((s) => typeof s === 'function') : [];
|
|
2943
|
+
const held = () => { try { return typeof isActivityHold === 'function' && isActivityHold(); } catch { return false; } };
|
|
2700
2944
|
function onElapse() {
|
|
2701
2945
|
timer = null;
|
|
2702
2946
|
if (settled()) return;
|
|
2703
2947
|
const elapsed = lastWindow;
|
|
2948
|
+
// #313: an in-flight tool call (the agent started a bash call that has not
|
|
2949
|
+
// reported a terminal status) means the silence is expected — defer. The
|
|
2950
|
+
// call's own timeout / the absolute hard cap still bound a hung call.
|
|
2951
|
+
if (held()) {
|
|
2952
|
+
if (holdMs < holdCap) {
|
|
2953
|
+
holdMs += elapsed;
|
|
2954
|
+
log(`silent for >= ${elapsed}ms but a tool call is still in flight; deferring the idle-kill (hold ${holdMs}/${holdCap}ms)`);
|
|
2955
|
+
prev = null; prevExtra = null; extraBaselined = false; flatMs = 0; extraFlatMs = 0;
|
|
2956
|
+
schedule(probeWindow);
|
|
2957
|
+
return;
|
|
2958
|
+
}
|
|
2959
|
+
log(`a tool call has been in flight for >= ${holdMs}ms (hold cap ${holdCap}ms); falling back to CPU-progress probes`);
|
|
2960
|
+
}
|
|
2704
2961
|
const pid = typeof getPid === 'function' ? getPid() : undefined;
|
|
2705
2962
|
const sample = pid != null ? sampleCpu(pid) : null;
|
|
2706
|
-
//
|
|
2707
|
-
|
|
2963
|
+
// #313: consult the extra (workspace-cwd) samplers. Advancing CPU on any of
|
|
2964
|
+
// them is work attributable to this agent even when it lives outside the
|
|
2965
|
+
// agent's process tree (a shared sccache daemon), so it defers the kill —
|
|
2966
|
+
// INCLUDING when the subtree itself looks childless/quiescent.
|
|
2967
|
+
let extraActive = false;
|
|
2968
|
+
if (extras.length) {
|
|
2969
|
+
const next = [];
|
|
2970
|
+
extraResolutionMs = 0;
|
|
2971
|
+
for (let i = 0; i < extras.length; i += 1) {
|
|
2972
|
+
let s = null;
|
|
2973
|
+
try { s = extras[i](); } catch { s = null; }
|
|
2974
|
+
next.push(s);
|
|
2975
|
+
if (!s || !Number.isFinite(s.cpu)) continue;
|
|
2976
|
+
if (s.processes > 0 && Number.isFinite(s.resolutionMs)) extraResolutionMs = Math.max(extraResolutionMs, s.resolutionMs);
|
|
2977
|
+
const before = prevExtra && prevExtra[i];
|
|
2978
|
+
if (before != null && Number.isFinite(before.cpu) && s.cpu !== before.cpu) extraActive = true;
|
|
2979
|
+
}
|
|
2980
|
+
const hadBaseline = prevExtra != null;
|
|
2981
|
+
const prevReadings = prevExtra || [];
|
|
2982
|
+
prevExtra = next;
|
|
2983
|
+
if (extraActive) {
|
|
2984
|
+
log(`silent for >= ${elapsed}ms but a workspace process outside the agent tree is burning CPU (e.g. sccache/rustc); deferring the idle-kill`);
|
|
2985
|
+
prev = sample; flatMs = 0; extraFlatMs = 0;
|
|
2986
|
+
schedule(probeWindow);
|
|
2987
|
+
return;
|
|
2988
|
+
}
|
|
2989
|
+
// Flat time only counts while a workspace sampler stays LIVE (a reading
|
|
2990
|
+
// with processes > 0) across consecutive probes. Empty / unreadable
|
|
2991
|
+
// readings must not pre-pay a sampler's resolution allowance: when any
|
|
2992
|
+
// sampler transitions absent/unreadable → live, restart the accumulator
|
|
2993
|
+
// so a fresh process gets its full resolution window before a kill.
|
|
2994
|
+
const isLive = (r) => r != null && Number.isFinite(r.cpu) && r.processes > 0;
|
|
2995
|
+
let newlyLive = false; let continuedLive = false;
|
|
2996
|
+
for (let i = 0; i < next.length; i += 1) {
|
|
2997
|
+
if (!isLive(next[i])) continue;
|
|
2998
|
+
if (hadBaseline && isLive(prevReadings[i])) continuedLive = true; else newlyLive = true;
|
|
2999
|
+
}
|
|
3000
|
+
if (newlyLive || !continuedLive) extraFlatMs = 0;
|
|
3001
|
+
else extraFlatMs += elapsed;
|
|
3002
|
+
}
|
|
3003
|
+
// A live workspace process read flat, but a coarse sampler (whole-second
|
|
3004
|
+
// `ps`) can read flat across a sub-resolution window while still working:
|
|
3005
|
+
// only treat workspace CPU as idle once the flat time covers its resolution.
|
|
3006
|
+
const extraIdleConfirmed = extraFlatMs >= extraResolutionMs;
|
|
3007
|
+
// Childless / unsampleable / dead tree → silence is very likely a hang.
|
|
3008
|
+
// But with extra (workspace) samplers wired we cannot yet distinguish "no
|
|
3009
|
+
// workspace work" from "a workspace daemon whose baseline we have not
|
|
3010
|
+
// taken" on a single reading — so take ONE baseline probe window before
|
|
3011
|
+
// killing, exactly like the subtree first-elapse baseline below. Without
|
|
3012
|
+
// extra samplers (or once a baseline shows no workspace activity) the
|
|
3013
|
+
// legacy immediate kill applies.
|
|
3014
|
+
if (!sample || sample.descendants <= 0) {
|
|
3015
|
+
if (extras.length && !extraBaselined) {
|
|
3016
|
+
// Consume the baseline: the NEXT elapse compares against prevExtra and
|
|
3017
|
+
// either defers (workspace CPU moved) or kills (flat).
|
|
3018
|
+
extraBaselined = true;
|
|
3019
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; baselining the workspace CPU probe and re-checking in ${probeWindow}ms`);
|
|
3020
|
+
schedule(probeWindow);
|
|
3021
|
+
return;
|
|
3022
|
+
}
|
|
3023
|
+
if (!extraIdleConfirmed) {
|
|
3024
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; workspace CPU flat for ${extraFlatMs}ms < sampler resolution ${extraResolutionMs}ms, re-checking in ${probeWindow}ms`);
|
|
3025
|
+
schedule(probeWindow);
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'} and no workspace CPU; idle-killing`);
|
|
3029
|
+
onIdleKill();
|
|
3030
|
+
return;
|
|
3031
|
+
}
|
|
2708
3032
|
if (prev != null) {
|
|
2709
3033
|
// Aggregate CPU advanced across the window → the tree is doing work; defer.
|
|
2710
|
-
if (sample.cpu > prev.cpu) {
|
|
3034
|
+
if (sample.cpu > prev.cpu) {
|
|
3035
|
+
log(`silent for >= ${elapsed}ms but subtree CPU advanced (${prev.cpu} → ${sample.cpu}); deferring the idle-kill`);
|
|
3036
|
+
prev = sample; flatMs = 0; schedule(probeWindow); return;
|
|
3037
|
+
}
|
|
2711
3038
|
// Aggregate CPU REGRESSED → `sampleSubtreeCpu*` only sums CPU for live
|
|
2712
3039
|
// PIDs, so the total can legitimately drop when a busy descendant exits
|
|
2713
3040
|
// (its ticks stop counting) even though the subtree just made progress.
|
|
2714
3041
|
// A regression is activity, not a stall: re-baseline and defer rather
|
|
2715
3042
|
// than mis-killing a silent-but-busy job.
|
|
2716
|
-
if (sample.cpu < prev.cpu) {
|
|
3043
|
+
if (sample.cpu < prev.cpu) {
|
|
3044
|
+
log(`silent for >= ${elapsed}ms but subtree CPU regressed (${prev.cpu} → ${sample.cpu}; a busy descendant exited) — progress, not a stall; deferring the idle-kill`);
|
|
3045
|
+
prev = sample; flatMs = 0; schedule(probeWindow); return;
|
|
3046
|
+
}
|
|
2717
3047
|
// Live descendants but CPU exactly unchanged across a full window → a
|
|
2718
3048
|
// candidate stall. Only confirm it once we have observed flat CPU for at
|
|
2719
3049
|
// least the sampler's CPU-time resolution: a probe window shorter than the
|
|
@@ -2721,12 +3051,17 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
|
|
|
2721
3051
|
// low-churn subtree is still progressing but hasn't crossed the next tick.
|
|
2722
3052
|
flatMs += elapsed;
|
|
2723
3053
|
const resolutionMs = Number.isFinite(sample.resolutionMs) ? sample.resolutionMs : 0;
|
|
2724
|
-
if (flatMs >= resolutionMs
|
|
3054
|
+
if (flatMs >= resolutionMs && extraIdleConfirmed) {
|
|
3055
|
+
log(`silent for >= ${elapsed}ms with a live but CPU-flat subtree (${sample.descendants} descendant(s), cpu ${sample.cpu}) and no workspace CPU; idle-killing`);
|
|
3056
|
+
onIdleKill();
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
2725
3059
|
schedule(probeWindow);
|
|
2726
3060
|
return;
|
|
2727
3061
|
}
|
|
2728
3062
|
// First elapse with live descendants: baseline, then re-probe (can't judge
|
|
2729
3063
|
// progress from one sample).
|
|
3064
|
+
log(`silent for >= ${elapsed}ms with ${sample.descendants} live descendant(s); baselining CPU (${sample.cpu}) and re-probing in ${probeWindow}ms`);
|
|
2730
3065
|
prev = sample;
|
|
2731
3066
|
flatMs = 0;
|
|
2732
3067
|
schedule(probeWindow);
|
|
@@ -2736,7 +3071,12 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
|
|
|
2736
3071
|
arm() {
|
|
2737
3072
|
if (!enabled || settled()) return;
|
|
2738
3073
|
prev = null; // output resumed → a fresh silent stretch starts
|
|
3074
|
+
prevExtra = null;
|
|
3075
|
+
extraBaselined = false;
|
|
2739
3076
|
flatMs = 0;
|
|
3077
|
+
extraFlatMs = 0;
|
|
3078
|
+
extraResolutionMs = 0;
|
|
3079
|
+
holdMs = 0;
|
|
2740
3080
|
clear();
|
|
2741
3081
|
schedule(idleTimeoutMs);
|
|
2742
3082
|
},
|
|
@@ -8749,6 +9089,11 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
|
|
|
8749
9089
|
idleTimeoutMs,
|
|
8750
9090
|
recoveryWindowMs,
|
|
8751
9091
|
isSettled: () => settled,
|
|
9092
|
+
logger: getLogger(),
|
|
9093
|
+
// #313: count CPU burned by any process whose cwd is inside the agent's
|
|
9094
|
+
// workspace (a shared sccache daemon lives outside the agent's tree), so
|
|
9095
|
+
// a silent compile holds off the idle-kill.
|
|
9096
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
|
|
8752
9097
|
onIdleKill: () => {
|
|
8753
9098
|
try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
|
|
8754
9099
|
finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
|
|
@@ -8921,6 +9266,11 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
8921
9266
|
idleTimeoutMs,
|
|
8922
9267
|
recoveryWindowMs,
|
|
8923
9268
|
isSettled: () => settled,
|
|
9269
|
+
logger: getLogger(),
|
|
9270
|
+
// #313: count CPU burned by any process whose cwd is inside the agent's
|
|
9271
|
+
// workspace (a shared sccache daemon lives outside the agent's tree), so
|
|
9272
|
+
// a silent compile holds off the idle-kill.
|
|
9273
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [term?.pid] })] : null,
|
|
8924
9274
|
onIdleKill: () => {
|
|
8925
9275
|
killTerm();
|
|
8926
9276
|
finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
|
|
@@ -9295,6 +9645,28 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9295
9645
|
// flips this; the floor never does.
|
|
9296
9646
|
let sawRealTranscript = false;
|
|
9297
9647
|
|
|
9648
|
+
// #313: ACP tool calls currently in flight (`toolCallId` → true). A
|
|
9649
|
+
// `tool_call` notification with a non-terminal status opens the call; a
|
|
9650
|
+
// `tool_call`/`tool_call_update` with a terminal status closes it. While
|
|
9651
|
+
// any call is open the idle watchdog treats the agent as ACTIVE (a long
|
|
9652
|
+
// silent bash call is work, not a hang) — the call's own timeout and the
|
|
9653
|
+
// absolute job hard cap still bound a genuinely hung call.
|
|
9654
|
+
const inFlightToolCalls = new Map();
|
|
9655
|
+
const ACP_TOOL_TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled', 'canceled']);
|
|
9656
|
+
const noteToolCallUpdate = (update) => {
|
|
9657
|
+
if (!update || typeof update !== 'object') return;
|
|
9658
|
+
const kind = update.sessionUpdate || update.type;
|
|
9659
|
+
if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
|
|
9660
|
+
const id = update.toolCallId;
|
|
9661
|
+
if (id == null) return;
|
|
9662
|
+
const status = typeof update.status === 'string' ? update.status.toLowerCase() : '';
|
|
9663
|
+
if (ACP_TOOL_TERMINAL_STATUSES.has(status)) inFlightToolCalls.delete(id);
|
|
9664
|
+
else if (!inFlightToolCalls.has(id)) {
|
|
9665
|
+
inFlightToolCalls.set(id, true);
|
|
9666
|
+
try { logger.info?.(`[idle-watchdog] tool call in flight (${update.title || id}${status ? `, status ${status}` : ''}) — the idle-kill is held while it runs`); } catch { /* best effort */ }
|
|
9667
|
+
}
|
|
9668
|
+
};
|
|
9669
|
+
|
|
9298
9670
|
// Live "spy" tee (--stream), line-buffered, mirroring the other paths.
|
|
9299
9671
|
// Separate line buffers per lane (stdout-human vs stderr) so a partial line
|
|
9300
9672
|
// on one lane never interleaves mid-line with the other — matching pipe/PTY.
|
|
@@ -9623,6 +9995,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9623
9995
|
// A request or notification FROM the agent.
|
|
9624
9996
|
if (typeof msg.method === 'string') {
|
|
9625
9997
|
if (msg.method === 'session/update') {
|
|
9998
|
+
noteToolCallUpdate(msg.params?.update);
|
|
9626
9999
|
emitTranscript(msg.params?.update);
|
|
9627
10000
|
// #194: durable engine-native producer seam. In parallel with the
|
|
9628
10001
|
// transcript-chunk relay overlay, hand the SAME raw `session/update` to
|
|
@@ -9673,6 +10046,16 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9673
10046
|
idleTimeoutMs,
|
|
9674
10047
|
recoveryWindowMs,
|
|
9675
10048
|
isSettled: () => settled,
|
|
10049
|
+
logger,
|
|
10050
|
+
// #313: while an ACP tool call is in flight the agent is active even
|
|
10051
|
+
// when totally silent (a long foreground bash compile/test). Bounded by
|
|
10052
|
+
// the call's own timeout and the absolute job hard cap.
|
|
10053
|
+
isActivityHold: () => inFlightToolCalls.size > 0,
|
|
10054
|
+
// #313: also count CPU burned by processes whose cwd is inside the
|
|
10055
|
+
// agent's workspace, wherever they sit in the process tree — a shared
|
|
10056
|
+
// sccache daemon (parented to launchd/init, not the agent) compiling
|
|
10057
|
+
// this worktree is real work and must hold off the idle-kill.
|
|
10058
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
|
|
9676
10059
|
onIdleKill: () => {
|
|
9677
10060
|
try { killTree(child); } catch { /* best effort */ }
|
|
9678
10061
|
finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
|
|
@@ -11281,9 +11664,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
|
|
|
11281
11664
|
* `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
|
|
11282
11665
|
* longer running here, so it rides the marker as a flagged, observational entry
|
|
11283
11666
|
* without making an idle worker look busy or inflating the drain's in-flight count.
|
|
11284
|
-
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
|
|
11667
|
+
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null), watchdogPid:(number|null|undefined), signalReadyAt?:(number|null) }} fields
|
|
11285
11668
|
*/
|
|
11286
|
-
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
|
|
11669
|
+
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth, watchdogPid, signalReadyAt }) {
|
|
11287
11670
|
const jobList = Array.isArray(jobs) ? jobs : [];
|
|
11288
11671
|
return {
|
|
11289
11672
|
pid,
|
|
@@ -11296,11 +11679,23 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt,
|
|
|
11296
11679
|
// SDK client and begun leasing), the producer stamps this; null until then.
|
|
11297
11680
|
// The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
|
|
11298
11681
|
readyAt: readyAt ?? null,
|
|
11682
|
+
// #309 review: stamped once this worker's SIGUSR2 handler is INSTALLED —
|
|
11683
|
+
// independent of leasing readiness (`readyAt` needs a nonempty poll set, so
|
|
11684
|
+
// a healthy `--auto` worker with no deployed types never stamps it). The
|
|
11685
|
+
// daemon-upgrade handoff gates its retarget SIGUSR2 on THIS, not readyAt.
|
|
11686
|
+
signalReadyAt: signalReadyAt ?? null,
|
|
11299
11687
|
// #293: the cached gh auth token's observable identity — { present,
|
|
11300
11688
|
// fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
|
|
11301
11689
|
// `supervisor status` can show the credential's age/scopes per worker. Null
|
|
11302
11690
|
// on a worker that has not primed the cache yet (or predates the field).
|
|
11303
11691
|
ghAuth: ghAuth ?? null,
|
|
11692
|
+
// #309: the daemon pid this worker's parent-death watchdog currently guards.
|
|
11693
|
+
// A rolling daemon upgrade SIGUSR2s each worker to RETARGET that watchdog at
|
|
11694
|
+
// the successor daemon before the old one exits; the old daemon waits
|
|
11695
|
+
// (bounded) for this stamp to name the successor — the ack that the worker
|
|
11696
|
+
// survives the handoff — so it never exits while a worker would still read
|
|
11697
|
+
// the (expected) reparent-to-init as an orphan event and self-reap.
|
|
11698
|
+
watchdogPid: Number.isInteger(watchdogPid) ? watchdogPid : null,
|
|
11304
11699
|
};
|
|
11305
11700
|
}
|
|
11306
11701
|
|
|
@@ -12118,9 +12513,13 @@ async function workAgent(req, flags, ctx) {
|
|
|
12118
12513
|
// this worker would otherwise idle forever as an orphan reparented to init.
|
|
12119
12514
|
// The watchdog makes the child self-exit the moment it is reparented. A
|
|
12120
12515
|
// standalone `nano work` (no activity file) keeps classic nohup semantics.
|
|
12516
|
+
// The handle is retained so a rolling daemon upgrade (#309) can RETARGET the
|
|
12517
|
+
// same installation at the successor daemon's pid instead of self-reaping
|
|
12518
|
+
// when the old daemon hands over and exits (see the SIGUSR2 handler below).
|
|
12519
|
+
let parentWatchdog = null;
|
|
12121
12520
|
if (activityFile) {
|
|
12122
12521
|
const daemonPid = Number.parseInt(process.env.NANO_SUPERVISOR_DAEMON_PID ?? '', 10);
|
|
12123
|
-
installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
|
|
12522
|
+
parentWatchdog = installParentDeathWatchdog({ parentPid: Number.isInteger(daemonPid) ? daemonPid : undefined });
|
|
12124
12523
|
}
|
|
12125
12524
|
const activeJobs = new Map(); // jobKey -> { type, since (ms epoch) }
|
|
12126
12525
|
// #254: settlement-pending ghosts — jobKey -> { type, since, phase, leaseToken }.
|
|
@@ -12267,6 +12666,9 @@ async function workAgent(req, flags, ctx) {
|
|
|
12267
12666
|
// waits for this stamp before draining the next worker so a bare spawn/PID is
|
|
12268
12667
|
// never mistaken for a serving replacement.
|
|
12269
12668
|
let readyAt = null;
|
|
12669
|
+
// #309 review: set once the SIGUSR2 (drain/retarget) handler is installed.
|
|
12670
|
+
let signalReadyAt = null;
|
|
12671
|
+
const markSignalReady = () => { signalReadyAt = Date.now(); };
|
|
12270
12672
|
const writeActivity = () => {
|
|
12271
12673
|
if (!activityFile) return;
|
|
12272
12674
|
const jobs = [...activeJobs.entries()].map(([key, v]) => ({ key, type: v.type, since: v.since }));
|
|
@@ -12291,7 +12693,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
12291
12693
|
// #293: publish the cached gh token's identity (fingerprint/scopes/age —
|
|
12292
12694
|
// never the value) so `supervisor status` can answer "is this worker holding
|
|
12293
12695
|
// a stale token?" in one line.
|
|
12294
|
-
const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
|
|
12696
|
+
const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, signalReadyAt, ghAuth: ghAuthTokenInfo(), watchdogPid: parentWatchdog && typeof parentWatchdog.watchedPid === 'function' ? parentWatchdog.watchedPid() : null });
|
|
12295
12697
|
const tmp = `${activityFile}.${process.pid}.tmp`;
|
|
12296
12698
|
try {
|
|
12297
12699
|
mkdirSync(dirname(activityFile), { recursive: true });
|
|
@@ -14330,7 +14732,62 @@ async function workAgent(req, flags, ctx) {
|
|
|
14330
14732
|
setTimeout(tick, 200);
|
|
14331
14733
|
};
|
|
14332
14734
|
|
|
14333
|
-
|
|
14735
|
+
// #309: the SIGUSR2 handler is a named, re-armable function. The legacy path
|
|
14736
|
+
// (graceful drain) fires once and the worker exits, so `once` semantics
|
|
14737
|
+
// suffice there; but the daemon-upgrade path (below) does NOT exit the
|
|
14738
|
+
// worker, so it re-arms the handler for a later drain/upgrade.
|
|
14739
|
+
const onDrainSignal = () => {
|
|
14740
|
+
// #309 rolling daemon upgrade: the daemon reuses SIGUSR2 as its
|
|
14741
|
+
// per-worker signal, but when it names a successor daemon pid this is a
|
|
14742
|
+
// watchdog RETARGET, not a drain — the old daemon is handing the fleet to
|
|
14743
|
+
// a successor running the new plugin code and this worker must KEEP
|
|
14744
|
+
// SERVING (jobs in flight are never touched; only the parent process is
|
|
14745
|
+
// swapped). The successor pid arrives two ways, in priority order:
|
|
14746
|
+
// 1. `<activityFile>.upgrade` — written by the old daemon just before
|
|
14747
|
+
// it SIGUSR2s (authoritative: the daemon learns the successor pid
|
|
14748
|
+
// only at handoff time, after this worker was spawned, so it cannot
|
|
14749
|
+
// have ridden this worker's env);
|
|
14750
|
+
// 2. NANO_SUPERVISOR_UPGRADE_PID — covers a worker spawned mid-upgrade.
|
|
14751
|
+
// Re-arm the parent-death watchdog at the successor so the old daemon's
|
|
14752
|
+
// (expected) exit — which reparents this worker to init — is not read as
|
|
14753
|
+
// an orphan event, then republish the activity marker: the old daemon
|
|
14754
|
+
// waits (bounded) for the marker's `watchdogPid` to name the successor
|
|
14755
|
+
// before it exits, so the handoff never strands a worker that would
|
|
14756
|
+
// self-reap mid-upgrade.
|
|
14757
|
+
let upgradePid = Number.parseInt(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
|
|
14758
|
+
if (activityFile) {
|
|
14759
|
+
try {
|
|
14760
|
+
const raw = readFileSync(`${activityFile}.upgrade`, 'utf8').trim();
|
|
14761
|
+
const fromFile = Number.parseInt(raw, 10);
|
|
14762
|
+
if (Number.isInteger(fromFile) && fromFile > 0) upgradePid = fromFile;
|
|
14763
|
+
} catch { /* no upgrade file — fall back to the env var */ }
|
|
14764
|
+
}
|
|
14765
|
+
if (Number.isInteger(upgradePid) && upgradePid > 0 && parentWatchdog && typeof parentWatchdog.retarget === 'function') {
|
|
14766
|
+
const ok = parentWatchdog.retarget(upgradePid);
|
|
14767
|
+
logger.info(`Received ${SUPERVISOR_DRAIN_SIGNAL} — daemon upgrade in progress: parent-death watchdog ${ok ? `retargeted to successor daemon pid ${upgradePid}` : `could NOT be retargeted (invalid pid)`}; this worker keeps serving.`);
|
|
14768
|
+
// Consume the retarget request: delete the `.upgrade` sidecar so a LATER
|
|
14769
|
+
// SIGUSR2 (a genuine drain, or a later upgrade that has not written a
|
|
14770
|
+
// fresh sidecar yet) is not misread as another retarget to this same
|
|
14771
|
+
// successor. Without this the sidecar is sticky — every later SIGUSR2
|
|
14772
|
+
// would retarget to the same (possibly long-gone) pid instead of
|
|
14773
|
+
// draining, breaking graceful stop / drain-remove / worker reload after
|
|
14774
|
+
// an upgrade (#309 review). Best-effort: a leftover only matters if it
|
|
14775
|
+
// names a live pid, and the successor also clears it on adopt.
|
|
14776
|
+
if (activityFile) { try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ } }
|
|
14777
|
+
try { writeActivityRef?.(); } catch { /* best effort — the ack stamp is advisory */ }
|
|
14778
|
+
// Re-arm: an upgrade that aborts (successor died before binding) must
|
|
14779
|
+
// leave this worker drainable by the surviving old daemon, and a LATER
|
|
14780
|
+
// upgrade/drain needs the handler again. (The drain path never re-arms —
|
|
14781
|
+
// the worker is on its way out.)
|
|
14782
|
+
process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
|
|
14783
|
+
return;
|
|
14784
|
+
}
|
|
14785
|
+
gracefulDrain(SUPERVISOR_DRAIN_SIGNAL);
|
|
14786
|
+
};
|
|
14787
|
+
process.once(SUPERVISOR_DRAIN_SIGNAL, onDrainSignal);
|
|
14788
|
+
// #309 review: publish that the handler is installed, so an upgrade handoff
|
|
14789
|
+
// can retarget this worker even before (or without ever) leasing.
|
|
14790
|
+
try { markSignalReady(); writeActivityRef?.(); } catch { /* best effort */ }
|
|
14334
14791
|
process.once('SIGINT', () => { forceAbort('SIGINT'); });
|
|
14335
14792
|
process.once('SIGTERM', () => { forceAbort('SIGTERM'); });
|
|
14336
14793
|
});
|
|
@@ -15128,7 +15585,18 @@ function formatSupervisorStatus(status) {
|
|
|
15128
15585
|
const onDiskVersion = status.pluginVersion
|
|
15129
15586
|
?? (() => { try { return pluginPackage().version; } catch { return null; } })();
|
|
15130
15587
|
if (onDiskVersion && d.version && onDiskVersion !== d.version) {
|
|
15131
|
-
lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code
|
|
15588
|
+
lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code, and \`c8ctl nano supervisor upgrade\` to roll the daemon itself onto the new code)`);
|
|
15589
|
+
}
|
|
15590
|
+
// #309: while a rolling daemon upgrade is in flight the status frame (and the
|
|
15591
|
+
// persisted-state fallback) carries an `upgrade` block — render it as the
|
|
15592
|
+
// daemon-roll line, mirroring the worker-roll visibility (`rolling <from> →
|
|
15593
|
+
// <to>`). Absent on a quiet daemon.
|
|
15594
|
+
const up = status.upgrade;
|
|
15595
|
+
if (up && typeof up === 'object' && typeof up.state === 'string') {
|
|
15596
|
+
const from = up.fromVersion ?? '?';
|
|
15597
|
+
const to = up.toVersion ?? '?';
|
|
15598
|
+
const age = Number.isFinite(up.sinceEpochMs) ? ` (${formatDuration(Math.max(0, Date.now() - up.sinceEpochMs))})` : '';
|
|
15599
|
+
lines.push(` upgrade: ${up.state} — daemon ${from} → ${to}${age}`);
|
|
15132
15600
|
}
|
|
15133
15601
|
// A rolling reload can run for a long time (each worker drains in-flight
|
|
15134
15602
|
// work). Surface it on every `status` so a detached operator — or someone
|
|
@@ -15263,6 +15731,10 @@ function stateFromStatus(res, socketPath) {
|
|
|
15263
15731
|
version: res.daemon?.version,
|
|
15264
15732
|
socket: res.daemon?.socket || socketPath,
|
|
15265
15733
|
logFile: res.daemon?.logFile,
|
|
15734
|
+
// #309 review: carry the daemon's in-flight upgrade block so a stop client
|
|
15735
|
+
// can see the handoff (and the successor's pid) even when it connected to
|
|
15736
|
+
// the PREDECESSOR just before the socket flipped.
|
|
15737
|
+
...(res.upgrade && typeof res.upgrade === 'object' ? { upgrade: res.upgrade } : {}),
|
|
15266
15738
|
workers: res.workers || [],
|
|
15267
15739
|
};
|
|
15268
15740
|
}
|
|
@@ -15277,6 +15749,10 @@ function stateFromStatus(res, socketPath) {
|
|
|
15277
15749
|
function statusFromState(running) {
|
|
15278
15750
|
return {
|
|
15279
15751
|
daemon: supervisorDaemonDescriptor(running),
|
|
15752
|
+
// #309: the persisted `upgrade` block rides the fallback render too — a
|
|
15753
|
+
// mid-handoff daemon whose socket is briefly unreachable must still show
|
|
15754
|
+
// the roll (same visibility guarantee as `version`/`logFile` above).
|
|
15755
|
+
...(running && running.upgrade && typeof running.upgrade === 'object' ? { upgrade: running.upgrade } : {}),
|
|
15280
15756
|
workers: (running.workers || []).map((w) => summarizeSupervisorWorker(w)),
|
|
15281
15757
|
};
|
|
15282
15758
|
}
|
|
@@ -15367,6 +15843,20 @@ function waitForChildExit(child, timeoutMs) {
|
|
|
15367
15843
|
* Unix-only: reparent-to-init is a POSIX semantic; Windows job objects are the
|
|
15368
15844
|
* equivalent lifecycle tie and are out of scope here (matching the daemon's
|
|
15369
15845
|
* other `win32` guards). Returns a canceller so callers/tests can stop it.
|
|
15846
|
+
*
|
|
15847
|
+
* Rolling daemon hot-upgrade (issue #309): `retargetToPid` re-arms the SAME
|
|
15848
|
+
* watchdog installation at a NEW daemon pid — the successor daemon the old one
|
|
15849
|
+
* is handing the fleet to. Without it the swap is impossible: the old daemon
|
|
15850
|
+
* exits once the successor owns the socket, the worker reparents to init, and
|
|
15851
|
+
* an un-retargeted watchdog would read exactly that as "daemon died
|
|
15852
|
+
* ungracefully" and self-reap — killing every in-flight job mid-upgrade, the
|
|
15853
|
+
* precise outcome the rolling replace exists to avoid. The retarget is safe
|
|
15854
|
+
* against a stale signal because the watched pid is only ever advanced to the
|
|
15855
|
+
* successor the CURRENT daemon named (via NANO_SUPERVISOR_UPGRADE_PID, set by
|
|
15856
|
+
* the CLI that spawned it); a SIGUSR2 with no upgrade in flight keeps the
|
|
15857
|
+
* legacy graceful-drain semantics (the caller branches on the env var, not
|
|
15858
|
+
* here). No-op on win32 / when the pid is invalid; returns whether the
|
|
15859
|
+
* retarget landed.
|
|
15370
15860
|
*/
|
|
15371
15861
|
function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, readPpid } = {}) {
|
|
15372
15862
|
if (osPlatform() === 'win32') return () => {};
|
|
@@ -15380,14 +15870,40 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
|
|
|
15380
15870
|
// NOT treat that as "orphaned". A *fallback* ppid of 1, by contrast, means we
|
|
15381
15871
|
// were already reparented to init with no daemon left to watch.
|
|
15382
15872
|
const explicit = Number.isInteger(parentPid) && parentPid > 0;
|
|
15383
|
-
|
|
15873
|
+
// `watched` is deliberately MUTABLE: a rolling daemon upgrade (#309) retargets
|
|
15874
|
+
// it at the successor daemon's pid the moment the old daemon hands over, so
|
|
15875
|
+
// the watchdog keeps guarding the worker across the swap instead of reading
|
|
15876
|
+
// the (expected) reparent-to-init as an orphan event and self-reaping.
|
|
15877
|
+
let watched = explicit ? parentPid : ppid();
|
|
15878
|
+
// #309: after a retarget, the ppid leg changes meaning. The old daemon exits
|
|
15879
|
+
// on purpose once the successor owns the socket, so this worker reparents to
|
|
15880
|
+
// init — and that is the EXPECTED end state, not an orphan event. From the
|
|
15881
|
+
// retarget on, only the SUCCESSOR's death orphans the worker (the ppid leg
|
|
15882
|
+
// would fire immediately and forever on the already-reparented worker).
|
|
15883
|
+
let retargeted = false;
|
|
15884
|
+
// #309 review (rollback): the guard state BEFORE the latest retarget. If the
|
|
15885
|
+
// successor dies mid-handoff while the predecessor is still alive (it rolls
|
|
15886
|
+
// back and keeps the fleet), the worker must fall back to guarding the
|
|
15887
|
+
// predecessor instead of self-reaping an in-flight job. Once the predecessor
|
|
15888
|
+
// has exited too, both are gone and the worker is genuinely orphaned.
|
|
15889
|
+
let fallback = null;
|
|
15384
15890
|
const orphaned = typeof onOrphan === 'function' ? onOrphan : () => process.exit(0);
|
|
15385
15891
|
const isOrphan = () => {
|
|
15386
15892
|
// Reparented away from the daemon (typically to init, pid 1) → orphaned.
|
|
15387
|
-
|
|
15388
|
-
|
|
15389
|
-
//
|
|
15390
|
-
|
|
15893
|
+
// Skipped after an upgrade retarget: the reparent to init IS the handoff.
|
|
15894
|
+
if (!retargeted && ppid() !== watched) return true;
|
|
15895
|
+
// The watched daemon is gone (ESRCH ⇒ gone; EPERM ⇒ alive but not ours to
|
|
15896
|
+
// signal). After an upgrade retarget this is the ONLY orphan signal — the
|
|
15897
|
+
// ppid leg above is skipped (the reparent to init is expected), so the
|
|
15898
|
+
// worker self-reaps exactly when the SUCCESSOR daemon dies.
|
|
15899
|
+
const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; } };
|
|
15900
|
+
if (alive(watched)) return false;
|
|
15901
|
+
if (fallback && alive(fallback.watched) && (fallback.retargeted || ppid() === fallback.watched)) {
|
|
15902
|
+
({ watched, retargeted } = fallback);
|
|
15903
|
+
fallback = null;
|
|
15904
|
+
return false;
|
|
15905
|
+
}
|
|
15906
|
+
return true;
|
|
15391
15907
|
};
|
|
15392
15908
|
// Orphaned already — self-reap now rather than idle forever. Either we fell
|
|
15393
15909
|
// back to ppid and it is already init (no known parent), or the recorded
|
|
@@ -15395,7 +15911,22 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
|
|
|
15395
15911
|
if ((!explicit && watched === 1) || isOrphan()) { orphaned(); return () => {}; }
|
|
15396
15912
|
const timer = setInterval(() => { if (isOrphan()) { clearInterval(timer); orphaned(); } }, intervalMs);
|
|
15397
15913
|
if (typeof timer.unref === 'function') timer.unref();
|
|
15398
|
-
|
|
15914
|
+
const cancel = () => { try { clearInterval(timer); } catch { /* ignore */ } };
|
|
15915
|
+
// #309: re-arm at the successor daemon's pid. The ppid leg of isOrphan() then
|
|
15916
|
+
// reads "reparented to init" as expected (the old daemon exited on purpose),
|
|
15917
|
+
// and only the SUCCESSOR's death orphans the worker from here on.
|
|
15918
|
+
cancel.retarget = (newPid) => {
|
|
15919
|
+
if (!Number.isInteger(newPid) || newPid <= 0) return false;
|
|
15920
|
+
if (newPid !== watched) fallback = { watched, retargeted };
|
|
15921
|
+
watched = newPid;
|
|
15922
|
+
retargeted = true;
|
|
15923
|
+
return true;
|
|
15924
|
+
};
|
|
15925
|
+
// The pid this installation currently guards — the daemon stamps it on the
|
|
15926
|
+
// worker's activity marker as the upgrade ack (the old daemon waits, bounded,
|
|
15927
|
+
// for it to name the successor before exiting).
|
|
15928
|
+
cancel.watchedPid = () => watched;
|
|
15929
|
+
return cancel;
|
|
15399
15930
|
}
|
|
15400
15931
|
|
|
15401
15932
|
// --- Daemon ----------------------------------------------------------------
|
|
@@ -15404,6 +15935,18 @@ function installParentDeathWatchdog({ intervalMs = 2000, parentPid, onOrphan, re
|
|
|
15404
15935
|
* The supervisor daemon body. Runs under `c8ctl nano supervisor __daemon`,
|
|
15405
15936
|
* spawned detached by `startSupervisorDaemon`. Never returns — it runs until a
|
|
15406
15937
|
* stop request or SIGTERM, then drains children and exits.
|
|
15938
|
+
*
|
|
15939
|
+
* Issue #309 (rolling daemon hot-upgrade): when spawned with
|
|
15940
|
+
* NANO_SUPERVISOR_UPGRADE=1 this process is a SUCCESSOR — the CLI launched it
|
|
15941
|
+
* to take over the fleet from an old daemon running stale plugin code. Before
|
|
15942
|
+
* binding the control socket it waits for the old daemon to quiesce (its
|
|
15943
|
+
* `prepare-upgrade` op stops the accept loop), then adopts the old daemon's
|
|
15944
|
+
* workers from the persisted state (pid-polled synthetic child handles — the
|
|
15945
|
+
* workers themselves are never signaled, so in-flight jobs run undisturbed)
|
|
15946
|
+
* and finally asks the old daemon to hand over (its `upgrade` op retargets
|
|
15947
|
+
* every worker's parent-death watchdog here and exits). A successor that
|
|
15948
|
+
* cannot reach a quiesced old daemon in time exits 1 and leaves the old fleet
|
|
15949
|
+
* untouched.
|
|
15407
15950
|
*/
|
|
15408
15951
|
async function runSupervisorDaemon() {
|
|
15409
15952
|
const startedAt = new Date().toISOString();
|
|
@@ -15417,6 +15960,68 @@ async function runSupervisorDaemon() {
|
|
|
15417
15960
|
const daemonLogFile = supervisorDaemonLogFile();
|
|
15418
15961
|
mkdirSync(getSupervisorLogDir(), { recursive: true });
|
|
15419
15962
|
|
|
15963
|
+
// Boot-time log line, available before the closure-scoped `dlog` exists (the
|
|
15964
|
+
// upgrade-mode quiesce handshake below runs before the workers map and its
|
|
15965
|
+
// helpers are initialized). Identical format.
|
|
15966
|
+
const dlogBoot = (msg) => {
|
|
15967
|
+
try { appendFileSync(daemonLogFile, `[${new Date().toISOString()}] ${msg}\n`); } catch { /* best effort */ }
|
|
15968
|
+
};
|
|
15969
|
+
|
|
15970
|
+
// #309: successor boot. `upgradeMode` latches once the old daemon has been
|
|
15971
|
+
// asked to quiesce; `predecessorPid`/`upgradeFromVersion` feed the status
|
|
15972
|
+
// frame's `upgrade` block so `supervisor status` shows the roll in flight
|
|
15973
|
+
// (`rolling <from> → <to>`), mirroring the worker-roll visibility.
|
|
15974
|
+
const upgradeMode = process.env.NANO_SUPERVISOR_UPGRADE === '1';
|
|
15975
|
+
let predecessorPid = null;
|
|
15976
|
+
let upgradeFromVersion = null;
|
|
15977
|
+
let upgradeStartedMs = null;
|
|
15978
|
+
if (upgradeMode) {
|
|
15979
|
+
const t0 = Date.now();
|
|
15980
|
+
dlogBoot(`upgrade mode: waiting for the current daemon to quiesce (pid from ${getSupervisorStateFile()})`);
|
|
15981
|
+
for (;;) {
|
|
15982
|
+
const st = readSupervisorState();
|
|
15983
|
+
if (!st || !isPidAlive(st.pid)) {
|
|
15984
|
+
// No live old daemon — nothing to hand over. Serve the fleet normally
|
|
15985
|
+
// (there is nothing to adopt; the state file, if any, is stale).
|
|
15986
|
+
dlogBoot('upgrade mode: no live predecessor daemon — starting as a plain daemon');
|
|
15987
|
+
break;
|
|
15988
|
+
}
|
|
15989
|
+
predecessorPid = st.pid;
|
|
15990
|
+
upgradeFromVersion = typeof st.version === 'string' ? st.version : null;
|
|
15991
|
+
// Ask the old daemon to stop accepting new control connections. An old
|
|
15992
|
+
// daemon that predates #309 answers `unknown op` — refuse to upgrade from
|
|
15993
|
+
// it rather than racing its accept loop for the socket.
|
|
15994
|
+
let prep = null;
|
|
15995
|
+
try {
|
|
15996
|
+
// #309 review: quiescence awaits admitted opQueue mutations (a restart
|
|
15997
|
+
// can take ~28s to stop its worker), and the predecessor closes its
|
|
15998
|
+
// listener as soon as it latches — so a retry could never reach it.
|
|
15999
|
+
// Keep THIS connection open for the remaining quiesce budget.
|
|
16000
|
+
const remainingMs = SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS - (Date.now() - t0);
|
|
16001
|
+
prep = await supervisorRequest({ op: 'prepare-upgrade', pid: process.pid }, { socketPath, timeoutMs: 750, responseTimeoutMs: Math.max(SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS, remainingMs) });
|
|
16002
|
+
} catch { /* socket briefly unreachable — retry until the deadline */ }
|
|
16003
|
+
if (prep && prep.ok) break; // quiesced — the accept loop is closed
|
|
16004
|
+
if (prep && prep.ok === false && /unknown op/.test(String(prep.error || ''))) {
|
|
16005
|
+
dlogBoot(`upgrade mode: predecessor pid ${st.pid} does not understand prepare-upgrade (too old) — refusing to upgrade`);
|
|
16006
|
+
process.exit(1);
|
|
16007
|
+
}
|
|
16008
|
+
// #309 review: the predecessor bound its handoff to a DIFFERENT successor
|
|
16009
|
+
// (a concurrent upgrade won the race). Never keep retrying — the binding
|
|
16010
|
+
// will never accept this pid, and two successors must not both wait to
|
|
16011
|
+
// adopt the same fleet. Exit and leave the bound successor to finish.
|
|
16012
|
+
if (prep && prep.ok === false && /already bound to successor pid/.test(String(prep.error || ''))) {
|
|
16013
|
+
dlogBoot(`upgrade mode: predecessor pid ${st.pid} is bound to a different successor — exiting so the bound successor completes the handoff`);
|
|
16014
|
+
process.exit(1);
|
|
16015
|
+
}
|
|
16016
|
+
if (Date.now() - t0 > SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS) {
|
|
16017
|
+
dlogBoot(`upgrade mode: predecessor pid ${st.pid} did not quiesce within ${SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS}ms — leaving it in place`);
|
|
16018
|
+
process.exit(1);
|
|
16019
|
+
}
|
|
16020
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
16021
|
+
}
|
|
16022
|
+
if (predecessorPid != null) upgradeStartedMs = Date.now();
|
|
16023
|
+
}
|
|
16024
|
+
|
|
15420
16025
|
const workers = new Map();
|
|
15421
16026
|
const attachClients = new Set();
|
|
15422
16027
|
// #202: sockets that issued a `stop`/drain and are waiting for the daemon to
|
|
@@ -15434,6 +16039,23 @@ async function runSupervisorDaemon() {
|
|
|
15434
16039
|
// restart-on-exit path and let a second `stop --force` escalate a live drain.
|
|
15435
16040
|
let draining = false;
|
|
15436
16041
|
let forcing = false;
|
|
16042
|
+
// #309 review: `shuttingDown` is latched EARLY (in the `stop` op, before the
|
|
16043
|
+
// deferred shutdown() runs) to reserve shutdown ownership against a racing
|
|
16044
|
+
// prepare-upgrade. `shutdownStarted` is the separate "shutdown()'s body has
|
|
16045
|
+
// begun" guard: the deferred shutdown() must still run its body even though
|
|
16046
|
+
// `shuttingDown` is already true, while a SECOND stop/signal must escalate
|
|
16047
|
+
// (force) or return instead of re-running the body.
|
|
16048
|
+
let shutdownStarted = false;
|
|
16049
|
+
// #309: a rolling daemon upgrade owns this daemon. `upgrading` is the
|
|
16050
|
+
// OLD-daemon latch: set by `prepare-upgrade` (accept loop closed; mutations
|
|
16051
|
+
// refused) and held through the `upgrade` handoff until this process exits.
|
|
16052
|
+
// `quiesced` flips when the accept loop has actually closed — the successor
|
|
16053
|
+
// polls for it before binding the socket. `upgradeInfo` carries the roll's
|
|
16054
|
+
// identity ({ state, fromVersion, toVersion, successorPid, sinceEpochMs })
|
|
16055
|
+
// for the status frame's `upgrade` block.
|
|
16056
|
+
let upgrading = false;
|
|
16057
|
+
let quiesced = false;
|
|
16058
|
+
let upgradeInfo = null;
|
|
15437
16059
|
// Hot code reload (rolling drain+respawn). A `reload` op adopts new on-disk
|
|
15438
16060
|
// plugin code into the worker children by gracefully draining and respawning
|
|
15439
16061
|
// them one at a time (so the fleet keeps serving). This flag rejects a second
|
|
@@ -15448,6 +16070,19 @@ async function runSupervisorDaemon() {
|
|
|
15448
16070
|
// refreshed status to attached consoles only on real change (see below).
|
|
15449
16071
|
let monitorTimer = null;
|
|
15450
16072
|
let lastMonitorSig = null;
|
|
16073
|
+
// #309: while quiesced for an upgrade the predecessor's accept loop is closed
|
|
16074
|
+
// and — on a second upgrade — every worker is an ADOPTED handle whose liveness
|
|
16075
|
+
// poll (`adoptTimer`) is unref'd, as is `monitorTimer`. With no referenced
|
|
16076
|
+
// handle left, Node would let the predecessor EXIT before the successor's
|
|
16077
|
+
// watchdog-retarget acks arrive / before SIGUSR1 lands — and the never-
|
|
16078
|
+
// resolving handoff Promise does NOT keep the event loop alive. This
|
|
16079
|
+
// referenced keepalive holds the predecessor up for exactly the quiesce→
|
|
16080
|
+
// handover window; it is cleared on the SIGUSR1 handover (and on shutdown).
|
|
16081
|
+
let quiesceKeepalive = null;
|
|
16082
|
+
// #309: polls the successor pid while quiesced so a successor that dies
|
|
16083
|
+
// mid-handoff (before SIGUSR1) triggers a rollback instead of leaving this
|
|
16084
|
+
// daemon permanently quiesced + unreachable. Cleared on handover/rollback.
|
|
16085
|
+
let successorWatchTimer = null;
|
|
15451
16086
|
|
|
15452
16087
|
// Daemon-wide mutation serialization: `add`/`remove`/`restart` must not
|
|
15453
16088
|
// interleave, or two clients racing the same worker could each spawn an
|
|
@@ -15465,16 +16100,46 @@ async function runSupervisorDaemon() {
|
|
|
15465
16100
|
|
|
15466
16101
|
const workerPublic = (w) => summarizeSupervisorWorker(w);
|
|
15467
16102
|
|
|
15468
|
-
|
|
16103
|
+
// #309: the daemon descriptor gains an `upgrade` block while a rolling daemon
|
|
16104
|
+
// upgrade is in flight — on the OLD daemon (state 'rolling', naming the
|
|
16105
|
+
// successor it is handing to) and on the SUCCESSOR (state 'adopting' until
|
|
16106
|
+
// the predecessor exits). `supervisor status` renders it as
|
|
16107
|
+
// `upgrade: rolling <from> → <to>` — the same visibility the worker roll has.
|
|
16108
|
+
const currentUpgradeBlock = () => {
|
|
16109
|
+
if (upgradeInfo) return upgradeInfo;
|
|
16110
|
+
if (upgradeMode && predecessorPid != null) {
|
|
16111
|
+
return { state: 'adopting', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, sinceEpochMs: upgradeStartedMs };
|
|
16112
|
+
}
|
|
16113
|
+
return null;
|
|
16114
|
+
};
|
|
16115
|
+
|
|
16116
|
+
// #309: once the successor has confirmed ownership (SIGUSR1 → handedOver) the
|
|
16117
|
+
// predecessor must make its final snapshot its LAST canonical write. More
|
|
16118
|
+
// broadly, once QUIESCED (`upgrading` latched) the successor has already read
|
|
16119
|
+
// the quiesce snapshot and adopted the inventory, so any later automatic
|
|
16120
|
+
// persist (a straggler worker-death handler firing between quiesce and exit)
|
|
16121
|
+
// would overwrite the successor's pid + inventory with the dead/dying daemon's
|
|
16122
|
+
// — corrupting discovery/recovery until the successor happens to persist
|
|
16123
|
+
// again. So persist() is suppressed while quiesced UNLESS forced (the quiesce
|
|
16124
|
+
// snapshot and the final handed-over snapshot are the two deliberate writes).
|
|
16125
|
+
let handedOver = false;
|
|
16126
|
+
const persist = (force = false) => {
|
|
16127
|
+
if (handedOver) return; // the successor owns the state file now
|
|
16128
|
+
if (upgrading && !force) return; // quiesced — the successor owns the inventory
|
|
15469
16129
|
try {
|
|
15470
16130
|
writeSupervisorState({
|
|
15471
16131
|
...supervisorDaemonDescriptor({
|
|
15472
16132
|
pid: process.pid, startedAt, version: daemonVersion, socket: socketPath, logFile: daemonLogFile,
|
|
15473
16133
|
}),
|
|
16134
|
+
...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
|
|
15474
16135
|
workers: [...workers.values()].map((w) => ({
|
|
15475
16136
|
id: w.id, profile: w.profile, args: w.args, pid: isPidAlive(w.pid) ? w.pid : null,
|
|
15476
16137
|
startedAt: w.startedAt || null, restarts: w.restarts, lastExit: w.lastExit ?? null,
|
|
15477
16138
|
stopping: !!w.stopping, logFile: w.logFile, configFailed: w.configFailed === true,
|
|
16139
|
+
// #309: persist drain-remove intent so an adopted worker that exits
|
|
16140
|
+
// mid-drain is still REMOVED by the successor (not retained forever,
|
|
16141
|
+
// which would block name reuse and hang `workforce stop`).
|
|
16142
|
+
removeOnExit: w.removeOnExit === true,
|
|
15478
16143
|
})),
|
|
15479
16144
|
});
|
|
15480
16145
|
} catch { /* best effort */ }
|
|
@@ -15511,6 +16176,11 @@ async function runSupervisorDaemon() {
|
|
|
15511
16176
|
const activityFile = supervisorWorkerActivityFile(w.id);
|
|
15512
16177
|
w.activityFile = activityFile;
|
|
15513
16178
|
try { rmSync(activityFile, { force: true }); } catch { /* best effort */ }
|
|
16179
|
+
// #309 review: also clear a stale `.upgrade` retarget sidecar. A worker
|
|
16180
|
+
// force-restarted mid-handoff before consuming it would otherwise inherit
|
|
16181
|
+
// the same path and treat its next genuine drain SIGUSR2 as a retarget.
|
|
16182
|
+
// (A handoff in progress re-writes the sidecar before it SIGUSR2s.)
|
|
16183
|
+
try { rmSync(`${activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
|
|
15514
16184
|
// `--name w.id` makes the child's broker workerName match this worker's
|
|
15515
16185
|
// supervisor id, so the same profile launched twice is distinct end-to-end.
|
|
15516
16186
|
// NANO_SUPERVISOR_ACTIVITY_FILE tells the child where to report per-job
|
|
@@ -15519,8 +16189,20 @@ async function runSupervisorDaemon() {
|
|
|
15519
16189
|
// watchdog can self-reap if we die ungracefully (SIGKILL/crash) and can't
|
|
15520
16190
|
// run the child-draining shutdown — even if we die mid-startup, before the
|
|
15521
16191
|
// child reparents to init.
|
|
16192
|
+
// NANO_SUPERVISOR_UPGRADE_PID (#309): when a daemon upgrade is in flight,
|
|
16193
|
+
// the successor's pid rides the child's env so the worker's SIGUSR2 handler
|
|
16194
|
+
// can tell a watchdog RETARGET (upgrade — keep serving) apart from a
|
|
16195
|
+
// graceful DRAIN (no upgrade pid — quiesce and exit). The daemon learns the
|
|
16196
|
+
// successor pid only at handoff time (the `upgrade` op), AFTER most workers
|
|
16197
|
+
// were spawned — so env alone can't reach them. The daemon therefore also
|
|
16198
|
+
// writes `<activityFile>.upgrade` (the successor pid) before it SIGUSR2s,
|
|
16199
|
+
// and the worker reads THAT as the authoritative retarget target; this env
|
|
16200
|
+
// forwarding covers only a worker spawned mid-upgrade.
|
|
16201
|
+
const childEnv = { ...process.env, NANO_SUPERVISOR_ACTIVITY_FILE: activityFile, NANO_SUPERVISOR_DAEMON_PID: String(process.pid) };
|
|
16202
|
+
const upgradeSuccessorPid = Number(process.env.NANO_SUPERVISOR_UPGRADE_PID ?? '', 10);
|
|
16203
|
+
if (Number.isInteger(upgradeSuccessorPid) && upgradeSuccessorPid > 0) childEnv.NANO_SUPERVISOR_UPGRADE_PID = String(upgradeSuccessorPid);
|
|
15522
16204
|
const child = spawn(exec, [entry, 'nano', 'work', w.profile, '--name', w.id, ...w.args], {
|
|
15523
|
-
env:
|
|
16205
|
+
env: childEnv,
|
|
15524
16206
|
stdio: ['ignore', ringed ? 'pipe' : fd, ringed ? 'pipe' : fd],
|
|
15525
16207
|
});
|
|
15526
16208
|
if (typeof fd === 'number') { try { closeSync(fd); } catch { /* dup'd into child */ } }
|
|
@@ -15613,7 +16295,9 @@ async function runSupervisorDaemon() {
|
|
|
15613
16295
|
broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
|
|
15614
16296
|
w.restartTimer = setTimeout(() => {
|
|
15615
16297
|
w.restartTimer = null;
|
|
15616
|
-
|
|
16298
|
+
// #309: a quiesced old daemon (mid-upgrade) never respawns — the
|
|
16299
|
+
// successor owns the fleet and would double-spawn the worker.
|
|
16300
|
+
if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
|
|
15617
16301
|
}, delay);
|
|
15618
16302
|
if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
|
|
15619
16303
|
persist();
|
|
@@ -15623,6 +16307,123 @@ async function runSupervisorDaemon() {
|
|
|
15623
16307
|
persist();
|
|
15624
16308
|
};
|
|
15625
16309
|
|
|
16310
|
+
// #309: a synthetic ChildProcess-shaped handle for an ADOPTED worker — one
|
|
16311
|
+
// the old daemon spawned and the successor inherits by pid. The successor
|
|
16312
|
+
// never spawned it, so there is no real 'exit' event: a poll timer is the
|
|
16313
|
+
// death detector, feeding the same `handleDeath` path a real child's exit
|
|
16314
|
+
// would (restart backoff included). The handle provides the surface the
|
|
16315
|
+
// daemon touches: `pid`, `exitCode`/`signalCode` (null while alive — the
|
|
16316
|
+
// readiness gate and reload checks read them), `kill` (force-stop),
|
|
16317
|
+
// `unref`/`ref`, null stdio (the log ring stays with the OLD daemon process),
|
|
16318
|
+
// and — critically — REAL `exit` listener semantics (`once`/`on`/
|
|
16319
|
+
// `removeListener` + `markExited`), because `waitForChildExit()` resolves only
|
|
16320
|
+
// when `exit` fires.
|
|
16321
|
+
const makeAdoptedChild = (pid) => {
|
|
16322
|
+
const exitWaiters = new Set();
|
|
16323
|
+
const handle = {
|
|
16324
|
+
pid,
|
|
16325
|
+
exitCode: null,
|
|
16326
|
+
signalCode: null,
|
|
16327
|
+
killed: false,
|
|
16328
|
+
stdout: null,
|
|
16329
|
+
stderr: null,
|
|
16330
|
+
stdin: null,
|
|
16331
|
+
kill(signal) { try { return process.kill(pid, signal); } catch { return false; } },
|
|
16332
|
+
once(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
|
|
16333
|
+
on(event, fn) { if (event === 'exit' && typeof fn === 'function') { if (handle.exitCode !== null || handle.signalCode !== null) fn(handle.exitCode, handle.signalCode); else exitWaiters.add(fn); } return handle; },
|
|
16334
|
+
removeListener(event, fn) { if (event === 'exit') exitWaiters.delete(fn); return handle; },
|
|
16335
|
+
unref() { return handle; },
|
|
16336
|
+
ref() { return handle; },
|
|
16337
|
+
// Invoked by the adopt liveness poll when the pid is gone: record the exit
|
|
16338
|
+
// and flush every waiter so graceful-stop/reload promises resolve.
|
|
16339
|
+
markExited(code, signal) {
|
|
16340
|
+
if (handle.exitCode !== null || handle.signalCode !== null) return;
|
|
16341
|
+
handle.exitCode = code;
|
|
16342
|
+
handle.signalCode = signal ?? null;
|
|
16343
|
+
for (const fn of [...exitWaiters]) { exitWaiters.delete(fn); try { fn(code, signal ?? null); } catch { /* ignore */ } }
|
|
16344
|
+
},
|
|
16345
|
+
};
|
|
16346
|
+
return handle;
|
|
16347
|
+
};
|
|
16348
|
+
|
|
16349
|
+
// Adopt one worker record from the predecessor's persisted state. Returns the
|
|
16350
|
+
// live worker entry, or null when the pid is already gone (nothing to adopt —
|
|
16351
|
+
// the fleet is exactly what survives the swap). The poll timer is unref'd so
|
|
16352
|
+
// it never keeps the daemon alive on its own.
|
|
16353
|
+
const adoptWorker = (rec) => {
|
|
16354
|
+
if (!rec || typeof rec.id !== 'string' || !rec.id) return null;
|
|
16355
|
+
if (!isPidAlive(rec.pid)) return null;
|
|
16356
|
+
const w = {
|
|
16357
|
+
id: rec.id,
|
|
16358
|
+
profile: typeof rec.profile === 'string' ? rec.profile : '',
|
|
16359
|
+
args: Array.isArray(rec.args) ? rec.args.map(String) : [],
|
|
16360
|
+
restarts: Number(rec.restarts) || 0,
|
|
16361
|
+
stopping: rec.stopping === true,
|
|
16362
|
+
// #309: restore drain-remove intent persisted by the predecessor, so a
|
|
16363
|
+
// worker draining for removal that exits after the swap is REMOVED here
|
|
16364
|
+
// (its record deleted) rather than retained — which would block name reuse
|
|
16365
|
+
// and hang a `workforce stop` waiting for the record to disappear.
|
|
16366
|
+
removeOnExit: rec.removeOnExit === true,
|
|
16367
|
+
lastExit: rec.lastExit ?? null,
|
|
16368
|
+
logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
|
|
16369
|
+
configFailed: rec.configFailed === true,
|
|
16370
|
+
adopted: true,
|
|
16371
|
+
};
|
|
16372
|
+
w.activityFile = supervisorWorkerActivityFile(w.id);
|
|
16373
|
+
// Clear any stale `.upgrade` sidecar the handoff left: the worker consumed
|
|
16374
|
+
// it to retarget its watchdog here, and a leftover would make a LATER
|
|
16375
|
+
// SIGUSR2 (a genuine drain) misread as another upgrade retarget.
|
|
16376
|
+
try { rmSync(`${w.activityFile}.upgrade`, { force: true }); } catch { /* best effort */ }
|
|
16377
|
+
const child = makeAdoptedChild(rec.pid);
|
|
16378
|
+
w.child = child;
|
|
16379
|
+
w.pid = rec.pid;
|
|
16380
|
+
w.startedAt = rec.startedAt || null;
|
|
16381
|
+
w.spawnedAt = rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now();
|
|
16382
|
+
let settled = false;
|
|
16383
|
+
const handleDeath = (reason) => {
|
|
16384
|
+
if (w.child !== child || settled) return;
|
|
16385
|
+
settled = true;
|
|
16386
|
+
if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
|
|
16387
|
+
// Mark the synthetic handle exited BEFORE running the death path so any
|
|
16388
|
+
// `waitForChildExit` waiter (graceful stop / reload) resolves — the poll
|
|
16389
|
+
// below is the only death detector for an adopted worker.
|
|
16390
|
+
if (typeof child.markExited === 'function') child.markExited(1, null);
|
|
16391
|
+
w.pid = null;
|
|
16392
|
+
w.lastExit = reason;
|
|
16393
|
+
try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
|
|
16394
|
+
const ranMs = Date.now() - (w.spawnedAt || Date.now());
|
|
16395
|
+
if (ranMs >= SUPERVISOR_HEALTHY_UPTIME_MS) w.restarts = 0;
|
|
16396
|
+
if (w.stopping || shuttingDown || !workers.has(w.id)) {
|
|
16397
|
+
if (w.removeOnExit && workers.get(w.id) === w) {
|
|
16398
|
+
workers.delete(w.id);
|
|
16399
|
+
try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
|
|
16400
|
+
dlog(`worker '${w.id}' removed (drained)`);
|
|
16401
|
+
broadcast({ type: 'event', event: 'worker-remove', id: w.id });
|
|
16402
|
+
}
|
|
16403
|
+
persist();
|
|
16404
|
+
if (draining) { try { broadcast(statusFrame(false)); } catch { /* best effort */ } }
|
|
16405
|
+
return;
|
|
16406
|
+
}
|
|
16407
|
+
if (upgrading) { persist(); return; } // quiesced for the handoff — the successor owns respawns
|
|
16408
|
+
const delay = supervisorBackoffMs(w.restarts);
|
|
16409
|
+
w.restarts += 1;
|
|
16410
|
+
dlog(`adopted worker '${w.id}' down (${reason}); restarting in ${delay}ms (restart #${w.restarts})`);
|
|
16411
|
+
broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
|
|
16412
|
+
w.restartTimer = setTimeout(() => {
|
|
16413
|
+
w.restartTimer = null;
|
|
16414
|
+
if (!w.stopping && !shuttingDown && !upgrading && workers.has(w.id)) startWorker(w);
|
|
16415
|
+
}, delay);
|
|
16416
|
+
if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
|
|
16417
|
+
persist();
|
|
16418
|
+
};
|
|
16419
|
+
w.adoptTimer = setInterval(() => { if (!isPidAlive(rec.pid)) handleDeath('code 1 (adopted worker exited)'); }, SUPERVISOR_ADOPT_POLL_MS);
|
|
16420
|
+
if (typeof w.adoptTimer.unref === 'function') w.adoptTimer.unref();
|
|
16421
|
+
workers.set(w.id, w);
|
|
16422
|
+
dlog(`worker '${w.id}' (profile ${w.profile}) adopted from predecessor daemon (pid ${w.pid})`);
|
|
16423
|
+
broadcast({ type: 'event', event: 'worker-adopted', worker: workerPublic(w) });
|
|
16424
|
+
return w;
|
|
16425
|
+
};
|
|
16426
|
+
|
|
15626
16427
|
const addWorker = (profile, args, name) => {
|
|
15627
16428
|
const taken = new Set(workers.keys());
|
|
15628
16429
|
let id;
|
|
@@ -15645,6 +16446,33 @@ async function runSupervisorDaemon() {
|
|
|
15645
16446
|
return w;
|
|
15646
16447
|
};
|
|
15647
16448
|
|
|
16449
|
+
// #309 review: workers whose upgrade watchdog-retarget is still outstanding
|
|
16450
|
+
// (id -> { pid, signalled, drainOnly?, drainDeferred? }), owned by the
|
|
16451
|
+
// successor's handoff loop. A graceful drain (stop/remove/reload) also uses
|
|
16452
|
+
// SIGUSR2, so it must not race a pending retarget: a drain signal consumed as
|
|
16453
|
+
// the retarget leaves the drain unfulfilled, and a second SIGUSR2 after the
|
|
16454
|
+
// one-shot handler fired would kill in-flight work. sendDrainSignal
|
|
16455
|
+
// coordinates the two so both are delivered, in order.
|
|
16456
|
+
const retargetPending = new Map();
|
|
16457
|
+
const sendDrainSignal = (w, pid) => {
|
|
16458
|
+
const p = retargetPending.get(w.id);
|
|
16459
|
+
if (p && p.pid === pid) {
|
|
16460
|
+
if (!p.signalled) {
|
|
16461
|
+
// Retarget not sent yet: cancel it (drop the sidecar) and drain
|
|
16462
|
+
// instead — the handoff loop delivers the drain once the handler is
|
|
16463
|
+
// installed and keeps the predecessor alive until the worker exits.
|
|
16464
|
+
try { rmSync(`${w.activityFile || supervisorWorkerActivityFile(w.id)}.upgrade`, { force: true }); } catch { /* best effort */ }
|
|
16465
|
+
p.signalled = true; p.drainOnly = true; p.drainDeferred = true;
|
|
16466
|
+
} else if (!p.drainOnly) {
|
|
16467
|
+
// Retarget in flight: deliver the drain right after its ack.
|
|
16468
|
+
p.drainDeferred = true;
|
|
16469
|
+
}
|
|
16470
|
+
// drainOnly without drainDeferred: already draining — nothing to send.
|
|
16471
|
+
return;
|
|
16472
|
+
}
|
|
16473
|
+
try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
16474
|
+
};
|
|
16475
|
+
|
|
15648
16476
|
const forceStopWorker = async (w) => {
|
|
15649
16477
|
const pid = w.pid;
|
|
15650
16478
|
if (w.child && pid) {
|
|
@@ -15671,7 +16499,7 @@ async function runSupervisorDaemon() {
|
|
|
15671
16499
|
// — a `stop --force` escalation (forceStopWorker) is the only way to cut a
|
|
15672
16500
|
// stuck drain short, and it resolves this same wait when the child dies.
|
|
15673
16501
|
if (w.child && pid) {
|
|
15674
|
-
|
|
16502
|
+
sendDrainSignal(w, pid);
|
|
15675
16503
|
await waitForChildExit(w.child, null);
|
|
15676
16504
|
}
|
|
15677
16505
|
return true;
|
|
@@ -15693,7 +16521,7 @@ async function runSupervisorDaemon() {
|
|
|
15693
16521
|
w.removeOnExit = true;
|
|
15694
16522
|
if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
|
|
15695
16523
|
const pid = w.pid;
|
|
15696
|
-
if (w.child && pid) {
|
|
16524
|
+
if (w.child && pid) { sendDrainSignal(w, pid); }
|
|
15697
16525
|
else { workers.delete(id); try { rmSync(supervisorWorkerActivityFile(id), { force: true }); } catch { /* best effort */ } broadcast({ type: 'event', event: 'worker-remove', id }); persist(); }
|
|
15698
16526
|
dlog(`worker '${id}' draining for removal`);
|
|
15699
16527
|
return true;
|
|
@@ -15790,7 +16618,7 @@ async function runSupervisorDaemon() {
|
|
|
15790
16618
|
if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
|
|
15791
16619
|
const pid = w.pid;
|
|
15792
16620
|
if (child && pid) {
|
|
15793
|
-
|
|
16621
|
+
sendDrainSignal(w, pid);
|
|
15794
16622
|
await waitForChildExit(child, null);
|
|
15795
16623
|
}
|
|
15796
16624
|
// Only respawn if nobody else acted on this worker while we drained: it must
|
|
@@ -15987,12 +16815,28 @@ async function runSupervisorDaemon() {
|
|
|
15987
16815
|
reloaded: [...reloadDone],
|
|
15988
16816
|
total: reloadTargets.length,
|
|
15989
16817
|
},
|
|
16818
|
+
// #309: present only while a rolling daemon upgrade is in flight
|
|
16819
|
+
// (`{ state, fromVersion, toVersion, … }`) — the successor polls the old
|
|
16820
|
+
// daemon's `upgrade.state === 'quiesced'` through this field before binding
|
|
16821
|
+
// the socket, and `supervisor status` renders it as the daemon-roll line.
|
|
16822
|
+
...(currentUpgradeBlock() ? { upgrade: currentUpgradeBlock() } : {}),
|
|
15990
16823
|
workers: pub || [...workers.values()].map(workerPublic),
|
|
15991
16824
|
...(final ? { final: true } : {}),
|
|
15992
16825
|
});
|
|
15993
16826
|
|
|
15994
16827
|
const shutdown = async (signal, { force = true } = {}) => {
|
|
15995
|
-
|
|
16828
|
+
// #309 review: a quiesced (mid-upgrade) daemon must NEVER run the shutdown
|
|
16829
|
+
// path — it would SIGTERM/SIGKILL the very workers the successor has
|
|
16830
|
+
// adopted, and its cleanup would delete the successor's socket and state
|
|
16831
|
+
// file. The upgrade's SIGUSR1 handover is this daemon's only exit while
|
|
16832
|
+
// quiesced; a signal (SIGTERM/SIGINT/SIGHUP) is logged and ignored so the
|
|
16833
|
+
// handoff can finish. The socket `stop` op refuses earlier (the `upgrading`
|
|
16834
|
+
// check in handleRequest) and never reaches here.
|
|
16835
|
+
if (upgrading) {
|
|
16836
|
+
dlog(`ignoring ${signal || 'shutdown request'} — a daemon upgrade owns this daemon (successor pid ${upgradeInfo?.successorPid ?? 'unknown'}); the successor owns the fleet now`);
|
|
16837
|
+
return;
|
|
16838
|
+
}
|
|
16839
|
+
if (shutdownStarted) {
|
|
15996
16840
|
// A shutdown is already running. A `stop --force` arriving mid-DRAIN
|
|
15997
16841
|
// escalates it: hard-stop every worker still finishing its jobs so the
|
|
15998
16842
|
// operator isn't stuck waiting. Idempotent — only the first force escalates.
|
|
@@ -16005,6 +16849,7 @@ async function runSupervisorDaemon() {
|
|
|
16005
16849
|
return;
|
|
16006
16850
|
}
|
|
16007
16851
|
shuttingDown = true;
|
|
16852
|
+
shutdownStarted = true;
|
|
16008
16853
|
draining = !force;
|
|
16009
16854
|
// Let any in-flight mutation finish before we snapshot the worker set, so
|
|
16010
16855
|
// an add/restart racing the shutdown can't leave an orphaned child behind.
|
|
@@ -16013,6 +16858,7 @@ async function runSupervisorDaemon() {
|
|
|
16013
16858
|
if (!force) broadcast({ type: 'event', event: 'draining', workers: [...workers.values()].map(workerPublic) });
|
|
16014
16859
|
await Promise.all([...workers.keys()].map((id) => stopWorker(id, { force })));
|
|
16015
16860
|
if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
|
|
16861
|
+
if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
|
|
16016
16862
|
broadcast({ type: 'event', event: 'daemon-stop' });
|
|
16017
16863
|
// Terminal frame for every waiting `stop` client (streaming or one-shot).
|
|
16018
16864
|
// We must FLUSH these before exiting: `process.exit()` does NOT drain pending
|
|
@@ -16036,6 +16882,179 @@ async function runSupervisorDaemon() {
|
|
|
16036
16882
|
process.exit(0);
|
|
16037
16883
|
};
|
|
16038
16884
|
|
|
16885
|
+
// #309: quiesce this daemon for an upgrade WITHOUT exiting: stop accepting
|
|
16886
|
+
// new control connections (the successor binds the socket the moment our
|
|
16887
|
+
// accept loop closes) and latch `upgrading` so mutations and worker respawns
|
|
16888
|
+
// are refused. Idempotent — the successor polls until it observes the
|
|
16889
|
+
// quiesced state. Attached consoles are destroyed (their daemon is going
|
|
16890
|
+
// away). The handoff is then driven by the SUCCESSOR (it retargets the
|
|
16891
|
+
// workers' watchdogs and SIGUSR1s this daemon to exit) — this daemon's
|
|
16892
|
+
// accept loop is already closed, so it cannot be asked over the socket.
|
|
16893
|
+
//
|
|
16894
|
+
// Snapshot AFTER the mutation queue drains (#309 review): latching
|
|
16895
|
+
// `upgrading` rejects NEW mutations, but one may already be admitted to
|
|
16896
|
+
// `opQueue` (e.g. a restart that is mid-respawn). Persisting before that
|
|
16897
|
+
// settles would snapshot a half-swapped worker set the successor then adopts
|
|
16898
|
+
// — missing the replacement the in-flight restart is about to spawn. So we
|
|
16899
|
+
// latch admission synchronously, then AWAIT `opQueue` (exactly as
|
|
16900
|
+
// `shutdown()` does) before persisting and reporting `quiesced`.
|
|
16901
|
+
let quiescePromise = null;
|
|
16902
|
+
// #309 review: the worker-id set captured in the quiesce snapshot. Rollback
|
|
16903
|
+
// reconciles the live map against it — the successor owns add/remove while
|
|
16904
|
+
// this daemon is quiesced, so a worker the successor removed mid-handoff must
|
|
16905
|
+
// stay removed if the handoff rolls back (never resurrected).
|
|
16906
|
+
let quiesceSnapshotIds = null;
|
|
16907
|
+
const quiesceForUpgrade = (successorPid, successorVersion) => {
|
|
16908
|
+
// #309 review: bind the handoff to ONE successor. `doQuiesce` latches
|
|
16909
|
+
// `upgrading` only inside its async body, so two prepare-upgrade requests
|
|
16910
|
+
// that arrive before the first body's first await both observe
|
|
16911
|
+
// `upgrading === false` and would share one quiesce promise — two
|
|
16912
|
+
// successors, each believing it owns the bind+adopt, racing for the same
|
|
16913
|
+
// socket and fleet. Validate and bind preparation to the FIRST successor
|
|
16914
|
+
// pid synchronously (before any await), and reject every other pid until
|
|
16915
|
+
// this handoff finishes (handover) or rolls back. The bound successor's
|
|
16916
|
+
// own retries (same pid) stay idempotent — they share the promise.
|
|
16917
|
+
if (!Number.isInteger(successorPid) || successorPid <= 0) {
|
|
16918
|
+
// An unidentifiable caller never earns the binding — without a pid the
|
|
16919
|
+
// successor watch cannot run and no retry could ever match the binding.
|
|
16920
|
+
const err = new Error('prepare-upgrade requires a valid successor pid');
|
|
16921
|
+
err.code = 'UPGRADE_BOUND';
|
|
16922
|
+
throw err;
|
|
16923
|
+
}
|
|
16924
|
+
if (upgrading) {
|
|
16925
|
+
const boundTo = upgradeInfo && upgradeInfo.successorPid;
|
|
16926
|
+
if (boundTo !== successorPid) {
|
|
16927
|
+
const err = new Error(`a daemon upgrade is already bound to successor pid ${boundTo} — refusing a concurrent handoff`);
|
|
16928
|
+
err.code = 'UPGRADE_BOUND';
|
|
16929
|
+
throw err;
|
|
16930
|
+
}
|
|
16931
|
+
}
|
|
16932
|
+
if (!upgrading || !quiescePromise) quiescePromise = doQuiesce(successorPid, successorVersion);
|
|
16933
|
+
return quiescePromise;
|
|
16934
|
+
};
|
|
16935
|
+
const doQuiesce = async (successorPid, successorVersion) => {
|
|
16936
|
+
if (!upgrading) {
|
|
16937
|
+
upgrading = true; // latch admission FIRST — rejects new mutations/respawns
|
|
16938
|
+
upgradeInfo = { state: 'quiescing', fromVersion: daemonVersion, toVersion: successorVersion ?? null, successorPid, sinceEpochMs: Date.now() };
|
|
16939
|
+
dlog(`daemon upgrade: quiescing for successor pid ${successorPid} — control socket accept loop closing, mutations refused`);
|
|
16940
|
+
try { server.close(); } catch { /* ignore */ }
|
|
16941
|
+
for (const s of [...attachClients]) { try { s.destroy(); } catch { /* ignore */ } }
|
|
16942
|
+
// Hold the event loop open until the successor's SIGUSR1 handover (see the
|
|
16943
|
+
// quiesceKeepalive declaration) — a no-op tick on a REFERENCED timer.
|
|
16944
|
+
if (!quiesceKeepalive) quiesceKeepalive = setInterval(() => {}, SUPERVISOR_ADOPT_POLL_MS);
|
|
16945
|
+
// #309 review (no-rollback): watch the successor. If it dies AFTER we
|
|
16946
|
+
// quiesced but BEFORE handing over (SIGUSR1), this daemon is left quiesced
|
|
16947
|
+
// and unreachable — respawns disabled, accept loop closed, no way for even
|
|
16948
|
+
// a later upgrade to reach it. Roll back: re-open the accept loop and
|
|
16949
|
+
// unlatch `upgrading` so normal operation (and a retry) resumes.
|
|
16950
|
+
armSuccessorWatch(successorPid);
|
|
16951
|
+
// Wait for any ALREADY-ADMITTED mutation (e.g. a restart mid-respawn) to
|
|
16952
|
+
// settle before snapshotting, so the successor adopts its replacement.
|
|
16953
|
+
try { await opQueue; } catch { /* mutation already logged */ }
|
|
16954
|
+
if (!upgrading) return; // rolled back while draining the queue
|
|
16955
|
+
upgradeInfo = { ...upgradeInfo, state: 'quiesced' };
|
|
16956
|
+
quiesceSnapshotIds = [...workers.keys()];
|
|
16957
|
+
persist(true); // the deliberate quiesce snapshot the successor adopts from
|
|
16958
|
+
}
|
|
16959
|
+
quiesced = true;
|
|
16960
|
+
};
|
|
16961
|
+
|
|
16962
|
+
// #309 review (no-rollback): while quiesced, poll the successor pid. If it
|
|
16963
|
+
// dies before handing over (SIGUSR1), this daemon would otherwise stay
|
|
16964
|
+
// quiesced + unreachable forever (accept loop closed, respawns disabled, no
|
|
16965
|
+
// listener for even a later upgrade). On detecting a dead successor, roll
|
|
16966
|
+
// back: re-open the accept loop and unlatch `upgrading`.
|
|
16967
|
+
const armSuccessorWatch = (successorPid) => {
|
|
16968
|
+
if (!Number.isInteger(successorPid) || successorPid <= 0) return;
|
|
16969
|
+
if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
|
|
16970
|
+
successorWatchTimer = setInterval(() => {
|
|
16971
|
+
if (!upgrading || handedOver) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; return; }
|
|
16972
|
+
if (!isPidAlive(successorPid)) {
|
|
16973
|
+
try { clearInterval(successorWatchTimer); } catch { /* ignore */ }
|
|
16974
|
+
successorWatchTimer = null;
|
|
16975
|
+
rollbackFromUpgrade(successorPid);
|
|
16976
|
+
}
|
|
16977
|
+
}, SUPERVISOR_ADOPT_POLL_MS);
|
|
16978
|
+
if (typeof successorWatchTimer.unref === 'function') successorWatchTimer.unref();
|
|
16979
|
+
};
|
|
16980
|
+
|
|
16981
|
+
// Restore normal operation after a successor died mid-handoff: re-listen on
|
|
16982
|
+
// the control socket (the successor never bound it, or died holding it) and
|
|
16983
|
+
// unlatch `upgrading` so mutations/respawns resume. Best-effort — if the
|
|
16984
|
+
// re-listen fails the daemon stays quiesced and logs, but the common case
|
|
16985
|
+
// (successor crashed before binding) recovers cleanly.
|
|
16986
|
+
const rollbackFromUpgrade = (deadSuccessorPid) => {
|
|
16987
|
+
if (!upgrading || handedOver) return;
|
|
16988
|
+
quiescePromise = null;
|
|
16989
|
+
dlog(`daemon upgrade: successor pid ${deadSuccessorPid} died before handover — rolling back: re-opening the control socket and resuming normal operation`);
|
|
16990
|
+
upgrading = false;
|
|
16991
|
+
quiesced = false;
|
|
16992
|
+
upgradeInfo = null;
|
|
16993
|
+
// Capture the snapshot BEFORE clearing it — the membership reconcile below
|
|
16994
|
+
// reads this copy (clearing first made every rollback drop the whole fleet).
|
|
16995
|
+
// A null snapshot (quiesce never reached the snapshot point) means there is
|
|
16996
|
+
// nothing to reconcile against, so membership is left untouched.
|
|
16997
|
+
const rollbackSnapshotIds = quiesceSnapshotIds;
|
|
16998
|
+
quiesceSnapshotIds = null;
|
|
16999
|
+
if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
|
|
17000
|
+
// Re-open the accept loop. The socket path may still be held by the dead
|
|
17001
|
+
// successor's listen socket; remove it first (same takeover the successor
|
|
17002
|
+
// itself performs), then re-bind.
|
|
17003
|
+
if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
|
|
17004
|
+
// Same owner-only umask as the initial bind, so the socket is private from
|
|
17005
|
+
// creation (no listen()→chmod window for another local user to connect).
|
|
17006
|
+
const rbWin = osPlatform() === 'win32';
|
|
17007
|
+
const rbPrevUmask = rbWin ? null : process.umask(0o177);
|
|
17008
|
+
try {
|
|
17009
|
+
server.listen(socketPath);
|
|
17010
|
+
if (!rbWin) { try { chmodSync(socketPath, 0o600); } catch { /* ignore */ } }
|
|
17011
|
+
dlog('daemon upgrade rollback: control socket re-opened — fleet serving normally');
|
|
17012
|
+
} catch (err) {
|
|
17013
|
+
dlog(`daemon upgrade rollback: failed to re-open the control socket: ${err?.message || err}`);
|
|
17014
|
+
} finally {
|
|
17015
|
+
if (!rbWin) { try { process.umask(rbPrevUmask); } catch { /* ignore */ } }
|
|
17016
|
+
}
|
|
17017
|
+
// #309 review: reconcile the inventory. Respawns were suppressed while
|
|
17018
|
+
// quiesced (expired restart timers and death handlers bailed), so reschedule
|
|
17019
|
+
// every eligible down worker now — preserving stop/config-failure/removal
|
|
17020
|
+
// intent and skipping any with a live process or a pending restart timer.
|
|
17021
|
+
// Membership is reconciled against the quiesce snapshot: a worker ABSENT
|
|
17022
|
+
// from it was removed through the SUCCESSOR while this daemon was quiesced
|
|
17023
|
+
// (the successor owns add/remove during the handoff), so re-adding it here
|
|
17024
|
+
// would resurrect a deliberately removed worker.
|
|
17025
|
+
// The authoritative handoff inventory is what the SUCCESSOR persisted (it
|
|
17026
|
+
// owns add/remove during the handoff): if the state file was last written
|
|
17027
|
+
// by the dead successor, its worker list wins; otherwise fall back to our
|
|
17028
|
+
// quiesce snapshot (the successor never wrote — nothing was removed).
|
|
17029
|
+
let snapshotIds = rollbackSnapshotIds ? new Set(rollbackSnapshotIds) : null;
|
|
17030
|
+
try {
|
|
17031
|
+
const succState = readSupervisorState();
|
|
17032
|
+
if (succState && succState.pid === deadSuccessorPid && Array.isArray(succState.workers)) {
|
|
17033
|
+
snapshotIds = new Set(succState.workers.map((x) => x && x.id).filter(Boolean));
|
|
17034
|
+
}
|
|
17035
|
+
} catch { /* unreadable — keep the snapshot */ }
|
|
17036
|
+
for (const id of snapshotIds ? [...workers.keys()] : []) {
|
|
17037
|
+
if (!snapshotIds.has(id)) {
|
|
17038
|
+
dlog(`daemon upgrade rollback: worker '${id}' is not in the quiesce snapshot (removed through the successor mid-handoff?) — leaving it removed`);
|
|
17039
|
+
workers.delete(id);
|
|
17040
|
+
}
|
|
17041
|
+
}
|
|
17042
|
+
for (const w of [...workers.values()]) {
|
|
17043
|
+
if (w.restartTimer || (w.pid && isPidAlive(w.pid))) continue;
|
|
17044
|
+
if (w.removeOnExit) {
|
|
17045
|
+
workers.delete(w.id);
|
|
17046
|
+
try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
|
|
17047
|
+
dlog(`worker '${w.id}' removed (drained during the aborted upgrade)`);
|
|
17048
|
+
continue;
|
|
17049
|
+
}
|
|
17050
|
+
if (w.stopping || w.configFailed) continue;
|
|
17051
|
+
if (w.adoptTimer) { try { clearInterval(w.adoptTimer); } catch { /* ignore */ } w.adoptTimer = null; }
|
|
17052
|
+
dlog(`daemon upgrade rollback: worker '${w.id}' was down while quiesced — rescheduling`);
|
|
17053
|
+
startWorker(w);
|
|
17054
|
+
}
|
|
17055
|
+
persist();
|
|
17056
|
+
};
|
|
17057
|
+
|
|
16039
17058
|
const handleRequest = async (req, sock) => {
|
|
16040
17059
|
const op = req && req.op;
|
|
16041
17060
|
try {
|
|
@@ -16043,8 +17062,30 @@ async function runSupervisorDaemon() {
|
|
|
16043
17062
|
case 'status':
|
|
16044
17063
|
sock.write(encodeFrame(statusFrame(true)));
|
|
16045
17064
|
break;
|
|
17065
|
+
case 'prepare-upgrade': {
|
|
17066
|
+
// #309: a successor daemon (spawned with NANO_SUPERVISOR_UPGRADE=1)
|
|
17067
|
+
// asks us to quiesce so it can bind the control socket. Refuse while
|
|
17068
|
+
// a worker reload roll or a shutdown owns the fleet — an upgrade on
|
|
17069
|
+
// top of either would strand its bookkeeping. Idempotent: the
|
|
17070
|
+
// successor polls until it observes the quiesced state. Bound to ONE
|
|
17071
|
+
// successor pid (quiesceForUpgrade throws UPGRADE_BOUND for any
|
|
17072
|
+
// other) so two racing successors can never both win the fleet.
|
|
17073
|
+
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
17074
|
+
if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a worker reload is in progress — retry the daemon upgrade once it finishes', final: true })); break; }
|
|
17075
|
+
try {
|
|
17076
|
+
await quiesceForUpgrade(Number(req.pid) || null, typeof req.version === 'string' ? req.version : null);
|
|
17077
|
+
} catch (err) {
|
|
17078
|
+
if (err && err.code === 'UPGRADE_BOUND') { sock.write(encodeFrame({ ok: false, error: err.message, final: true })); break; }
|
|
17079
|
+
throw err;
|
|
17080
|
+
}
|
|
17081
|
+
// Report quiesced only AFTER the mutation queue has drained and the
|
|
17082
|
+
// snapshot persisted — the successor binds+adopts off this ack.
|
|
17083
|
+
sock.write(encodeFrame({ ok: true, type: 'prepare-upgrade', quiesced: true, final: true }));
|
|
17084
|
+
break;
|
|
17085
|
+
}
|
|
16046
17086
|
case 'add': {
|
|
16047
17087
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
17088
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16048
17089
|
if (!req.profile) { sock.write(encodeFrame({ ok: false, error: 'add requires a profile', final: true })); break; }
|
|
16049
17090
|
// A supervised worker's name is supplied out-of-band as `req.name`
|
|
16050
17091
|
// (which the daemon forwards to the child as `nano work … --name`).
|
|
@@ -16067,6 +17108,7 @@ async function runSupervisorDaemon() {
|
|
|
16067
17108
|
}
|
|
16068
17109
|
case 'remove': {
|
|
16069
17110
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
17111
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16070
17112
|
// #202: `req.force === false` drains each worker (finish in-flight jobs
|
|
16071
17113
|
// then exit); the default remains a fast force-stop (used by reconcile
|
|
16072
17114
|
// and interactive remove). A drain-remove returns immediately and the
|
|
@@ -16082,6 +17124,7 @@ async function runSupervisorDaemon() {
|
|
|
16082
17124
|
}
|
|
16083
17125
|
case 'restart': {
|
|
16084
17126
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
17127
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16085
17128
|
const restarted = await serializeOp(async () => {
|
|
16086
17129
|
const ids = resolveTargets(req.target);
|
|
16087
17130
|
for (const id of ids) await restartWorker(id);
|
|
@@ -16098,6 +17141,7 @@ async function runSupervisorDaemon() {
|
|
|
16098
17141
|
// and ends with a terminal `reloaded` frame — draining can take a long
|
|
16099
17142
|
// time, so this MUST be a streaming op, not a one-shot request.
|
|
16100
17143
|
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
17144
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — retry against the successor in a moment', final: true })); break; }
|
|
16101
17145
|
if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a reload is already in progress', final: true })); break; }
|
|
16102
17146
|
// Reject reload explicitly on Windows BEFORE draining anything. The
|
|
16103
17147
|
// rolling reload's graceful drain relies on SIGUSR2 (SUPERVISOR_DRAIN_SIGNAL)
|
|
@@ -16179,10 +17223,30 @@ async function runSupervisorDaemon() {
|
|
|
16179
17223
|
// #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
|
|
16180
17224
|
// jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
|
|
16181
17225
|
// yield jobs) and can also ESCALATE a drain already in progress.
|
|
17226
|
+
// #309: a quiesced (mid-upgrade) daemon must never run the stop path —
|
|
17227
|
+
// it would SIGTERM/SIGUSR2 the very workers the successor has adopted.
|
|
17228
|
+
// The upgrade's own exit path replaces this op for the old daemon.
|
|
17229
|
+
if (upgrading) { sock.write(encodeFrame({ ok: false, error: 'supervisor daemon upgrade in progress — the successor owns the fleet now; stop it instead (its pid is in `supervisor status`)', final: true })); break; }
|
|
16182
17230
|
const force = !!req.force;
|
|
16183
|
-
//
|
|
16184
|
-
//
|
|
16185
|
-
//
|
|
17231
|
+
// #309 review: reserve shutdown ownership HERE, synchronously with the
|
|
17232
|
+
// acceptance — not 20ms later when the deferred shutdown() runs. An
|
|
17233
|
+
// unreserved accept lets a prepare-upgrade squeeze into the defer
|
|
17234
|
+
// window and latch `upgrading` first; the shutdown then overtakes the
|
|
17235
|
+
// quiesce and SIGTERMs the very workers the successor is adopting.
|
|
17236
|
+
// Latching now makes the race impossible in either order: a stop
|
|
17237
|
+
// accepted first refuses the upgrade (prepare-upgrade checks
|
|
17238
|
+
// `shuttingDown`), and an upgrade quiesced first refuses the stop
|
|
17239
|
+
// (the `upgrading` check above).
|
|
17240
|
+
//
|
|
17241
|
+
// A SECOND stop while one is already reserved is not a fresh shutdown:
|
|
17242
|
+
// it only ESCALATES a drain (force) or is a no-op. Never re-latch
|
|
17243
|
+
// `draining` here — a force stop arriving during a graceful drain must
|
|
17244
|
+
// leave `draining === true` so shutdown()'s escalation branch fires,
|
|
17245
|
+
// and a graceful stop during a force stop must not flip it back.
|
|
17246
|
+
if (!shutdownStarted && !shuttingDown) {
|
|
17247
|
+
shuttingDown = true;
|
|
17248
|
+
draining = !force;
|
|
17249
|
+
}
|
|
16186
17250
|
attachClients.add(sock);
|
|
16187
17251
|
stopClients.add(sock);
|
|
16188
17252
|
sock.write(encodeFrame(force
|
|
@@ -16191,7 +17255,12 @@ async function runSupervisorDaemon() {
|
|
|
16191
17255
|
sock.write(encodeFrame(statusFrame(false)));
|
|
16192
17256
|
// Kick the shutdown asynchronously; don't await it here so the control
|
|
16193
17257
|
// loop keeps serving (streaming status, accepting a force escalation).
|
|
16194
|
-
|
|
17258
|
+
// The 250ms defer lets a `stop --force` escalation (sent right behind
|
|
17259
|
+
// a graceful `stop`, as `stopStream`'s sendAfter does) land and run
|
|
17260
|
+
// shutdown's escalation branch BEFORE the drain's body force-stops on
|
|
17261
|
+
// its own — preserving the pre-#309 escalation contract now that
|
|
17262
|
+
// `shuttingDown` is latched early (see below).
|
|
17263
|
+
setTimeout(() => { shutdown('stop', { force }); }, 250);
|
|
16195
17264
|
break;
|
|
16196
17265
|
}
|
|
16197
17266
|
default:
|
|
@@ -16204,7 +17273,11 @@ async function runSupervisorDaemon() {
|
|
|
16204
17273
|
|
|
16205
17274
|
// Bind the control socket. A stale unix socket file from a crashed daemon
|
|
16206
17275
|
// would make listen() fail with EADDRINUSE even though nobody is listening;
|
|
16207
|
-
// remove it first
|
|
17276
|
+
// remove it first. In upgrade mode the old daemon has QUIESCED (its accept
|
|
17277
|
+
// loop is closed) but its listen socket still holds the path until it exits,
|
|
17278
|
+
// so we must unlink + rebind to take over — the same operation, deliberately
|
|
17279
|
+
// ordered AFTER the quiesce handshake so we can never steal the socket from a
|
|
17280
|
+
// daemon that is still accepting (#309).
|
|
16208
17281
|
if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
|
|
16209
17282
|
|
|
16210
17283
|
const server = createServer((sock) => {
|
|
@@ -16270,7 +17343,208 @@ async function runSupervisorDaemon() {
|
|
|
16270
17343
|
// and hard crashes still can't run this — the per-worker parent-death
|
|
16271
17344
|
// watchdog is the backstop for those.
|
|
16272
17345
|
process.once('SIGHUP', () => shutdown('SIGHUP'));
|
|
17346
|
+
// #309: SIGUSR1 is the upgrade handoff's "the successor owns the fleet — exit
|
|
17347
|
+
// WITHOUT touching the workers" signal. It is ONLY meaningful after a
|
|
17348
|
+
// `prepare-upgrade` quiesce (the successor sends it once it has adopted the
|
|
17349
|
+
// workers and retargeted their watchdogs). A SIGUSR1 outside an upgrade must
|
|
17350
|
+
// NOT exit (that would strand the fleet with no daemon), so it is ignored
|
|
17351
|
+
// unless `upgrading` is latched. Unlike `shutdown`, this path never signals
|
|
17352
|
+
// the workers and never clears the state file — the successor owns both now.
|
|
17353
|
+
process.on('SIGUSR1', () => {
|
|
17354
|
+
if (!upgrading) return;
|
|
17355
|
+
dlog(`daemon upgrade: successor confirmed ownership — old daemon (pid ${process.pid}) exiting; workers keep serving`);
|
|
17356
|
+
upgradeInfo = upgradeInfo ? { ...upgradeInfo, state: 'handed-over' } : upgradeInfo;
|
|
17357
|
+
// #309 review: NO persist here. The quiesce snapshot was this daemon's last
|
|
17358
|
+
// canonical write — the successor has since written its own pid + inventory,
|
|
17359
|
+
// and any write now would clobber it with the exiting predecessor's state.
|
|
17360
|
+
handedOver = true;
|
|
17361
|
+
quiesceSnapshotIds = null;
|
|
17362
|
+
if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
|
|
17363
|
+
if (quiesceKeepalive) { try { clearInterval(quiesceKeepalive); } catch { /* ignore */ } quiesceKeepalive = null; }
|
|
17364
|
+
if (successorWatchTimer) { try { clearInterval(successorWatchTimer); } catch { /* ignore */ } successorWatchTimer = null; }
|
|
17365
|
+
process.exit(0);
|
|
17366
|
+
});
|
|
16273
17367
|
dlog(`supervisor daemon up (pid ${process.pid}) — control ${socketPath}`);
|
|
17368
|
+
|
|
17369
|
+
// #309: adopt the predecessor's workers. The old daemon quiesced (above) and
|
|
17370
|
+
// its last persist() — run when it latched the upgrade — names every worker
|
|
17371
|
+
// it owned. Each live pid becomes an adopted worker (polled for liveness;
|
|
17372
|
+
// never signaled, so in-flight jobs run undisturbed). Dead pids are dropped
|
|
17373
|
+
// with a log line — the fleet after the swap is exactly what survived it.
|
|
17374
|
+
if (upgradeMode && predecessorPid != null) {
|
|
17375
|
+
const prior = readSupervisorState();
|
|
17376
|
+
const adopted = [];
|
|
17377
|
+
for (const rec of (prior && Array.isArray(prior.workers) ? prior.workers : [])) {
|
|
17378
|
+
const w = adoptWorker(rec);
|
|
17379
|
+
if (w) { adopted.push(w.id); continue; }
|
|
17380
|
+
// #309 review: transfer the FULL inventory, not only live pids. A worker
|
|
17381
|
+
// the predecessor recorded but that is not alive right now falls into one
|
|
17382
|
+
// of three kept categories — dropping it would permanently shrink the
|
|
17383
|
+
// fleet (the predecessor refuses respawns once quiesced):
|
|
17384
|
+
if (!rec || typeof rec.id !== 'string' || !rec.id) { dlog('upgrade: skipping a worker record with no id'); continue; }
|
|
17385
|
+
if (rec.configFailed === true) {
|
|
17386
|
+
// Configuration-failed: retained (NOT restarted) so the operator's
|
|
17387
|
+
// `configFailed` signal survives the swap and a later fix+restart still
|
|
17388
|
+
// targets the same record.
|
|
17389
|
+
workers.set(rec.id, {
|
|
17390
|
+
id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
|
|
17391
|
+
args: Array.isArray(rec.args) ? rec.args.map(String) : [],
|
|
17392
|
+
restarts: Number(rec.restarts) || 0, stopping: rec.stopping === true,
|
|
17393
|
+
removeOnExit: rec.removeOnExit === true, lastExit: rec.lastExit ?? null,
|
|
17394
|
+
logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
|
|
17395
|
+
configFailed: true, adopted: true, pid: null, child: null,
|
|
17396
|
+
startedAt: rec.startedAt || null, spawnedAt: rec.startedAt ? new Date(rec.startedAt).getTime() : Date.now(),
|
|
17397
|
+
});
|
|
17398
|
+
dlog(`upgrade: retained configuration-failed worker '${rec.id}' (not restarted)`);
|
|
17399
|
+
continue;
|
|
17400
|
+
}
|
|
17401
|
+
if (rec.stopping === true || rec.removeOnExit === true) {
|
|
17402
|
+
// Draining for stop/remove: the process is already gone, so the removal
|
|
17403
|
+
// completes here — delete the record rather than resurrect it.
|
|
17404
|
+
dlog(`upgrade: worker '${rec.id}' was draining for removal and is gone — record dropped`);
|
|
17405
|
+
try { rmSync(supervisorWorkerActivityFile(rec.id), { force: true }); } catch { /* best effort */ }
|
|
17406
|
+
continue;
|
|
17407
|
+
}
|
|
17408
|
+
// Down but eligible (a crash/backoff the predecessor had not yet
|
|
17409
|
+
// respawned when it quiesced): reschedule it here so fleet capacity is
|
|
17410
|
+
// preserved across the swap.
|
|
17411
|
+
dlog(`upgrade: worker '${rec.id}' was down at handoff — rescheduling to preserve fleet capacity`);
|
|
17412
|
+
const rw = {
|
|
17413
|
+
id: rec.id, profile: typeof rec.profile === 'string' ? rec.profile : '',
|
|
17414
|
+
args: Array.isArray(rec.args) ? rec.args.map(String) : [],
|
|
17415
|
+
restarts: Number(rec.restarts) || 0, stopping: false, removeOnExit: false,
|
|
17416
|
+
lastExit: rec.lastExit ?? null,
|
|
17417
|
+
logFile: typeof rec.logFile === 'string' ? rec.logFile : supervisorWorkerLogFile(rec.id),
|
|
17418
|
+
configFailed: false, adopted: true,
|
|
17419
|
+
};
|
|
17420
|
+
workers.set(rw.id, rw);
|
|
17421
|
+
startWorker(rw);
|
|
17422
|
+
rw.spawnedBySuccessor = true; // its watchdog already guards this daemon
|
|
17423
|
+
adopted.push(rw.id);
|
|
17424
|
+
}
|
|
17425
|
+
dlog(`upgrade: adopted ${adopted.length} worker(s) from predecessor pid ${predecessorPid}: ${adopted.join(', ') || '(none)'}`);
|
|
17426
|
+
broadcast({ type: 'event', event: 'daemon-adopted', predecessorPid, workers: adopted });
|
|
17427
|
+
|
|
17428
|
+
// Drive the handoff to completion. The old daemon's accept loop is closed
|
|
17429
|
+
// (quiesce), so it can no longer be asked over the socket — instead the
|
|
17430
|
+
// SUCCESSOR finishes the swap: write each live worker's `.upgrade` sidecar
|
|
17431
|
+
// (the successor pid the worker's SIGUSR2 handler reads), SIGUSR2 the
|
|
17432
|
+
// workers to retarget their watchdogs here, wait (bounded) for their
|
|
17433
|
+
// `watchdogPid` acks, then SIGUSR1 the old daemon so it persists and exits
|
|
17434
|
+
// WITHOUT touching the workers. Runs async so the control loop serves
|
|
17435
|
+
// immediately; the `upgrade` status block stays 'adopting' until the old
|
|
17436
|
+
// daemon is gone.
|
|
17437
|
+
(async () => {
|
|
17438
|
+
const myPid = process.pid;
|
|
17439
|
+
// Every live PREDECESSOR-owned worker stays pending until it has either
|
|
17440
|
+
// acked the retarget (`watchdogPid === myPid`) or exited (#309 review).
|
|
17441
|
+
// A worker still booting has Node's DEFAULT SIGUSR2 action (terminate)
|
|
17442
|
+
// until its handler is up, so it is signalled only once its activity
|
|
17443
|
+
// marker proves readiness (finite `readyAt` + matching pid) — but it is
|
|
17444
|
+
// NEVER dropped from the wait set, so the predecessor (its watchdog
|
|
17445
|
+
// target) is not released under it. Workers this successor spawned
|
|
17446
|
+
// itself already guard it and need no retarget.
|
|
17447
|
+
const pending = retargetPending;
|
|
17448
|
+
for (const id of adopted) {
|
|
17449
|
+
const w = workers.get(id);
|
|
17450
|
+
if (!w || w.spawnedBySuccessor || !w.pid || !isPidAlive(w.pid)) continue;
|
|
17451
|
+
if (w.stopping) {
|
|
17452
|
+
// #309 review: already draining (stop/remove) under the predecessor.
|
|
17453
|
+
// Its one-shot SIGUSR2 handler was consumed by that drain, so another
|
|
17454
|
+
// SIGUSR2 would hit Node's default action and kill its in-flight work.
|
|
17455
|
+
// Never write a sidecar or signal it — just keep it pending (its
|
|
17456
|
+
// watchdog still guards the predecessor) until it exits.
|
|
17457
|
+
pending.set(id, { pid: w.pid, signalled: true, drainOnly: true });
|
|
17458
|
+
continue;
|
|
17459
|
+
}
|
|
17460
|
+
const actFile = w.activityFile || supervisorWorkerActivityFile(w.id);
|
|
17461
|
+
try { writeFileSync(`${actFile}.upgrade`, String(myPid), { mode: 0o600 }); } catch { /* best effort */ }
|
|
17462
|
+
pending.set(id, { pid: w.pid, signalled: false });
|
|
17463
|
+
}
|
|
17464
|
+
// Bounded wait first; past the deadline report INCOMPLETE but KEEP
|
|
17465
|
+
// monitoring (#309 review) so a late ack / drain still releases the
|
|
17466
|
+
// predecessor and its keepalive — a retried upgrade would target this
|
|
17467
|
+
// successor, never the retained predecessor.
|
|
17468
|
+
const ackDeadline = Date.now() + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS;
|
|
17469
|
+
let reportedIncomplete = false;
|
|
17470
|
+
while (pending.size > 0) {
|
|
17471
|
+
if (!isPidAlive(predecessorPid)) break; // predecessor gone — nothing left to release
|
|
17472
|
+
for (const [id, p] of [...pending]) {
|
|
17473
|
+
const w = workers.get(id);
|
|
17474
|
+
if (!w || w.pid !== p.pid || !isPidAlive(p.pid)) { pending.delete(id); continue; }
|
|
17475
|
+
const act = readWorkerActivity(id);
|
|
17476
|
+
if (p.drainOnly) {
|
|
17477
|
+
// Draining, not retargeting: a deferred drain request (below) is
|
|
17478
|
+
// delivered once the handler is provably installed; then just wait
|
|
17479
|
+
// for the exit.
|
|
17480
|
+
if (p.drainDeferred && drainSignalReadyFor(act, p.pid)) {
|
|
17481
|
+
p.drainDeferred = false;
|
|
17482
|
+
try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
17483
|
+
}
|
|
17484
|
+
continue;
|
|
17485
|
+
}
|
|
17486
|
+
if (act && act.pid === p.pid && act.watchdogPid === myPid) {
|
|
17487
|
+
pending.delete(id);
|
|
17488
|
+
// Retarget landed and the handler re-armed — now deliver any drain
|
|
17489
|
+
// that was requested while the retarget was outstanding.
|
|
17490
|
+
if (p.drainDeferred) { try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
|
|
17491
|
+
continue;
|
|
17492
|
+
}
|
|
17493
|
+
if (!p.signalled && drainSignalReadyFor(act, p.pid)) {
|
|
17494
|
+
p.signalled = true;
|
|
17495
|
+
try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
17496
|
+
}
|
|
17497
|
+
}
|
|
17498
|
+
if (pending.size === 0) break;
|
|
17499
|
+
if (!reportedIncomplete && Date.now() >= ackDeadline) {
|
|
17500
|
+
reportedIncomplete = true;
|
|
17501
|
+
dlog(`upgrade: ${pending.size} worker(s) have not acked the watchdog retarget within ${SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS}ms (${[...pending.keys()].join(', ')}) — NOT releasing the predecessor yet; handoff INCOMPLETE, still monitoring until they ack or exit`);
|
|
17502
|
+
upgradeInfo = { state: 'incomplete', fromVersion: upgradeFromVersion, toVersion: daemonVersion, predecessorPid, unacked: [...pending.keys()], sinceEpochMs: upgradeStartedMs };
|
|
17503
|
+
persist();
|
|
17504
|
+
}
|
|
17505
|
+
await new Promise((r) => setTimeout(r, reportedIncomplete ? SUPERVISOR_ADOPT_POLL_MS : 100));
|
|
17506
|
+
}
|
|
17507
|
+
// Predecessor gone with entries still pending: deliver deferred drains
|
|
17508
|
+
// that are safe now (handler installed and no retarget signal in flight
|
|
17509
|
+
// that the drain could be mistaken for). A drain deferred behind an
|
|
17510
|
+
// un-acked retarget is logged — signalling now could hit a consumed
|
|
17511
|
+
// one-shot handler and kill in-flight work.
|
|
17512
|
+
for (const [id, p] of pending) {
|
|
17513
|
+
if (!p.drainDeferred || !isPidAlive(p.pid)) continue;
|
|
17514
|
+
if (p.drainOnly && drainSignalReadyFor(readWorkerActivity(id), p.pid)) {
|
|
17515
|
+
try { process.kill(p.pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
|
|
17516
|
+
} else {
|
|
17517
|
+
dlog(`upgrade: deferred drain for worker '${id}' not delivered (retarget unacknowledged or handler not ready) — re-issue stop/remove once it settles`);
|
|
17518
|
+
}
|
|
17519
|
+
}
|
|
17520
|
+
pending.clear();
|
|
17521
|
+
// Release the old daemon: it persists a final snapshot and exits 0 without
|
|
17522
|
+
// touching the workers. If it is already gone (crashed mid-handoff), the
|
|
17523
|
+
// workers are still adopted here — nothing more to do.
|
|
17524
|
+
if (isPidAlive(predecessorPid)) {
|
|
17525
|
+
try { process.kill(predecessorPid, 'SIGUSR1'); } catch { /* already gone */ }
|
|
17526
|
+
}
|
|
17527
|
+
dlog(`upgrade: handoff complete — successor pid ${myPid} owns the fleet (${daemonVersion})`);
|
|
17528
|
+
})().catch((err) => dlog(`upgrade handoff error: ${err?.message || err}`));
|
|
17529
|
+
|
|
17530
|
+
// Clear the status `upgrade` block once the predecessor has actually exited
|
|
17531
|
+
// — the roll is complete from then on, and a quiet fleet's status should
|
|
17532
|
+
// not carry a stale 'adopting' line forever. Unref'd: it never keeps the
|
|
17533
|
+
// daemon alive.
|
|
17534
|
+
const clearTimer = setInterval(() => {
|
|
17535
|
+
if (!isPidAlive(predecessorPid)) {
|
|
17536
|
+
try { clearInterval(clearTimer); } catch { /* ignore */ }
|
|
17537
|
+
predecessorPid = null;
|
|
17538
|
+
// #309 review: a handoff reported INCOMPLETE that later completed (late
|
|
17539
|
+
// acks/drains released the predecessor) must not keep reporting
|
|
17540
|
+
// 'incomplete' forever — currentUpgradeBlock() returns upgradeInfo first.
|
|
17541
|
+
if (upgradeInfo && upgradeInfo.state === 'incomplete') upgradeInfo = null;
|
|
17542
|
+
persist();
|
|
17543
|
+
}
|
|
17544
|
+
}, SUPERVISOR_ADOPT_POLL_MS);
|
|
17545
|
+
if (typeof clearTimer.unref === 'function') clearTimer.unref();
|
|
17546
|
+
}
|
|
17547
|
+
|
|
16274
17548
|
persist();
|
|
16275
17549
|
|
|
16276
17550
|
// Live-view refresh: periodically re-sample worker activity and push a fresh
|
|
@@ -16663,9 +17937,9 @@ async function streamSupervisorReload(socketPath, req, logger, { label = 'fleet'
|
|
|
16663
17937
|
* Hot-adopt new plugin code into the running fleet with zero downtime: roll
|
|
16664
17938
|
* through the target workers, gracefully draining (finish in-flight jobs) and
|
|
16665
17939
|
* respawning each so the new child re-reads the updated `c8ctl-plugin.js` from
|
|
16666
|
-
* disk. Defaults to the whole fleet. Note this adopts new WORKER code only;
|
|
16667
|
-
* supervisor daemon itself
|
|
16668
|
-
*
|
|
17940
|
+
* disk. Defaults to the whole fleet. Note this adopts new WORKER code only;
|
|
17941
|
+
* the supervisor daemon itself is rolled separately, without stopping the
|
|
17942
|
+
* fleet, by `supervisor upgrade` (issue #309).
|
|
16669
17943
|
*/
|
|
16670
17944
|
async function supervisorReloadCmd(req) {
|
|
16671
17945
|
const logger = getLogger();
|
|
@@ -16680,6 +17954,116 @@ async function supervisorReloadCmd(req) {
|
|
|
16680
17954
|
if (outcome === 'error') process.exit(1);
|
|
16681
17955
|
}
|
|
16682
17956
|
|
|
17957
|
+
/**
|
|
17958
|
+
* Rolling hot-upgrade of the supervisor DAEMON process (issue #309) — the
|
|
17959
|
+
* daemon-level counterpart of `supervisor reload` (which rolls only workers).
|
|
17960
|
+
* The daemon keeps the module graph it imported at start, so adopting new
|
|
17961
|
+
* SUPERVISOR-side code used to require `supervisor stop && start` — a fleet
|
|
17962
|
+
* outage (or, with --force, killed in-flight jobs). This command swaps the
|
|
17963
|
+
* parent process instead:
|
|
17964
|
+
*
|
|
17965
|
+
* 1. spawn a SUCCESSOR daemon (detached, NANO_SUPERVISOR_UPGRADE=1) running
|
|
17966
|
+
* the on-disk — post-`nano update` — plugin code;
|
|
17967
|
+
* 2. the successor asks the old daemon to QUIESCE (`prepare-upgrade`: accept
|
|
17968
|
+
* loop closed, mutations refused), then binds the canonical control
|
|
17969
|
+
* socket and ADOPTS the old daemon's workers from the persisted state
|
|
17970
|
+
* (pid-polled; the workers are never signaled, so in-flight jobs run
|
|
17971
|
+
* undisturbed);
|
|
17972
|
+
* 3. the successor then finishes the swap itself: it writes each worker's
|
|
17973
|
+
* `<activityFile>.upgrade` sidecar, SIGUSR2s the workers to retarget their
|
|
17974
|
+
* parent-death watchdogs at it, waits (bounded) for the `watchdogPid`
|
|
17975
|
+
* acks, and SIGUSR1s the old daemon — which persists a final snapshot and
|
|
17976
|
+
* exits 0 WITHOUT touching the workers.
|
|
17977
|
+
*
|
|
17978
|
+
* Workers never die mid-job; only the parent is swapped. A successor that
|
|
17979
|
+
* cannot reach a quiesced old daemon exits 1 and leaves the fleet untouched.
|
|
17980
|
+
* This client just spawns the successor and waits for the swap to land; Ctrl-C
|
|
17981
|
+
* detaches — the handoff completes in the background.
|
|
17982
|
+
*/
|
|
17983
|
+
async function supervisorUpgradeCmd() {
|
|
17984
|
+
const logger = getLogger();
|
|
17985
|
+
if (osPlatform() === 'win32') {
|
|
17986
|
+
logger.error('supervisor upgrade is not supported on Windows (its worker watchdog retarget relies on SIGUSR2, which Windows cannot deliver) — use `supervisor stop` + `start` to adopt new supervisor code.');
|
|
17987
|
+
process.exit(1);
|
|
17988
|
+
}
|
|
17989
|
+
const running = await liveSupervisor();
|
|
17990
|
+
if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
|
|
17991
|
+
const socketPath = running.socket || getSupervisorSocketPath();
|
|
17992
|
+
const oldPid = running.pid;
|
|
17993
|
+
const fromVersion = running.version || null;
|
|
17994
|
+
const toVersion = (() => { try { return pluginPackage().version; } catch { return null; } })();
|
|
17995
|
+
|
|
17996
|
+
// 1. Spawn the successor. NANO_SUPERVISOR_UPGRADE=1 puts its boot into
|
|
17997
|
+
// upgrade mode (quiesce handshake → bind → adopt → drive the handoff). We
|
|
17998
|
+
// deliberately do NOT clear the state file first — the successor reads the
|
|
17999
|
+
// old daemon's worker list from it. Ctrl-C detaches the waiter below; the
|
|
18000
|
+
// successor is detached and completes the swap regardless.
|
|
18001
|
+
const { exec, entry } = c8ctlInvocation();
|
|
18002
|
+
mkdirSync(getSupervisorLogDir(), { recursive: true });
|
|
18003
|
+
const logFile = supervisorDaemonLogFile();
|
|
18004
|
+
let fd;
|
|
18005
|
+
try { fd = openSync(logFile, 'a'); } catch { fd = 'ignore'; }
|
|
18006
|
+
const child = spawn(exec, [entry, 'nano', 'supervisor', '__daemon'], {
|
|
18007
|
+
env: { ...process.env, NANO_SUPERVISOR_UPGRADE: '1' },
|
|
18008
|
+
detached: true,
|
|
18009
|
+
stdio: ['ignore', fd, fd],
|
|
18010
|
+
});
|
|
18011
|
+
child.unref();
|
|
18012
|
+
if (typeof fd === 'number') { try { closeSync(fd); } catch { /* ignore */ } }
|
|
18013
|
+
if (typeof child.pid !== 'number') { logger.error('Failed to spawn the successor supervisor daemon.'); process.exit(1); }
|
|
18014
|
+
const successorPid = child.pid;
|
|
18015
|
+
logger.info(`Upgrading the supervisor daemon (pid ${oldPid}${fromVersion ? `, v${fromVersion}` : ''}) → successor pid ${successorPid}${toVersion ? ` (v${toVersion})` : ''}. Workers keep serving; only the parent process is swapped. Press Ctrl-C to detach.`);
|
|
18016
|
+
// #309 review (detached-breaks-supervision): the successor is spawned DETACHED
|
|
18017
|
+
// in this session, not under launchd/systemd. If a service unit is installed,
|
|
18018
|
+
// warn the operator that the new daemon is no longer service-supervised (and
|
|
18019
|
+
// that the old unit's clean exit 0 won't restart it — the generated units use
|
|
18020
|
+
// crash-only policies), and tell them how to re-establish supervision.
|
|
18021
|
+
try {
|
|
18022
|
+
if (supervisorServiceInstalled()) {
|
|
18023
|
+
logger.warn('A supervisor service is installed, but the upgraded daemon runs DETACHED in this session — it is no longer under launchd/systemd supervision (and the old unit\'s clean exit will not auto-restart it). After the upgrade, run `c8ctl nano supervisor install` (or `stop` + `start`) to bring the new-code daemon back under the service.');
|
|
18024
|
+
}
|
|
18025
|
+
} catch { /* best effort — never block the upgrade on a detection hiccup */ }
|
|
18026
|
+
|
|
18027
|
+
// 2. Wait for the swap to land: the successor takes the socket (status names
|
|
18028
|
+
// its pid) and the old daemon exits (released by the successor's SIGUSR1
|
|
18029
|
+
// once the workers' watchdogs are retargeted). Ctrl-C detaches — the
|
|
18030
|
+
// handoff completes in the background.
|
|
18031
|
+
let detached = false;
|
|
18032
|
+
const onSigint = () => { detached = true; };
|
|
18033
|
+
process.on('SIGINT', onSigint);
|
|
18034
|
+
const deadline = Date.now() + SUPERVISOR_UPGRADE_QUIESCE_TIMEOUT_MS + SUPERVISOR_UPGRADE_ACK_TIMEOUT_MS + 10_000;
|
|
18035
|
+
try {
|
|
18036
|
+
for (;;) {
|
|
18037
|
+
if (detached) {
|
|
18038
|
+
logger.info('Detached — the daemon upgrade completes in the background. Rerun `nano supervisor status` to confirm.');
|
|
18039
|
+
return;
|
|
18040
|
+
}
|
|
18041
|
+
if (!isPidAlive(successorPid)) {
|
|
18042
|
+
logger.error(`The successor daemon (pid ${successorPid}) died during the upgrade — the old daemon (pid ${oldPid}) may still be in place. See ${logFile}.`);
|
|
18043
|
+
process.exit(1);
|
|
18044
|
+
}
|
|
18045
|
+
let successorServing = false;
|
|
18046
|
+
try {
|
|
18047
|
+
const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
|
|
18048
|
+
successorServing = !!(res && res.ok && res.daemon && res.daemon.pid === successorPid);
|
|
18049
|
+
} catch { /* socket mid-handoff — retry */ }
|
|
18050
|
+
const oldGone = !isPidAlive(oldPid);
|
|
18051
|
+
if (successorServing && oldGone) {
|
|
18052
|
+
logger.info(`Supervisor daemon upgraded${fromVersion || toVersion ? ` (${fromVersion ?? '?'} → ${toVersion ?? '?'})` : ''} — successor pid ${successorPid} owns the fleet; no worker was stopped.`);
|
|
18053
|
+
return;
|
|
18054
|
+
}
|
|
18055
|
+
if (Date.now() >= deadline) {
|
|
18056
|
+
if (!successorServing) logger.warn(`The successor daemon (pid ${successorPid}) did not take the control socket in time — the old daemon (pid ${oldPid}) may still own the fleet. Check \`nano supervisor status\` and ${logFile}.`);
|
|
18057
|
+
else logger.warn(`The old daemon (pid ${oldPid}) has not exited yet — the watchdog handoff is still completing. Check \`nano supervisor status\` in a moment.`);
|
|
18058
|
+
process.exit(1);
|
|
18059
|
+
}
|
|
18060
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
18061
|
+
}
|
|
18062
|
+
} finally {
|
|
18063
|
+
try { process.removeListener('SIGINT', onSigint); } catch { /* ignore */ }
|
|
18064
|
+
}
|
|
18065
|
+
}
|
|
18066
|
+
|
|
16683
18067
|
/**
|
|
16684
18068
|
* Re-read every supervised worker's cached gh auth token WITHOUT draining
|
|
16685
18069
|
* in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
|
|
@@ -16787,7 +18171,15 @@ async function supervisorStopCmd(force = false) {
|
|
|
16787
18171
|
const { frames, rest } = decodeFrames(buf);
|
|
16788
18172
|
buf = rest;
|
|
16789
18173
|
for (const frame of frames) {
|
|
16790
|
-
|
|
18174
|
+
// #309 review: an ok:false terminal frame is a REFUSAL, not a stop —
|
|
18175
|
+
// the daemon rejected the op (a daemon upgrade owns the fleet; the
|
|
18176
|
+
// successor holds the socket now). Treating it as a successful stop
|
|
18177
|
+
// would wait indefinitely for the wrong pid and, with --force,
|
|
18178
|
+
// SIGKILL the quiesced predecessor's process group — including the
|
|
18179
|
+
// workers the successor adopted — then erase the successor's state.
|
|
18180
|
+
if (frame && frame.ok === false) {
|
|
18181
|
+
finish({ error: String(frame.error || 'the supervisor refused the stop request') });
|
|
18182
|
+
} else if (frame && frame.type === 'stopping') {
|
|
16791
18183
|
logger.info('Aborting in-flight work — killing harnesses and yielding jobs for retry...');
|
|
16792
18184
|
} else if (frame && frame.event === 'draining-escalated') {
|
|
16793
18185
|
logger.info('Escalating to --force — aborting in-flight work...');
|
|
@@ -16813,6 +18205,15 @@ async function supervisorStopCmd(force = false) {
|
|
|
16813
18205
|
|
|
16814
18206
|
if (outcome === 'detached') return;
|
|
16815
18207
|
|
|
18208
|
+
// A refused stop (ok:false) is an ERROR outcome: surface it and exit before
|
|
18209
|
+
// any pid wait, force escalation, or state cleanup — the daemon that refused
|
|
18210
|
+
// (e.g. a quiesced predecessor mid-upgrade) is not ours to reap, and the
|
|
18211
|
+
// state file now names the successor that owns the fleet.
|
|
18212
|
+
if (outcome && typeof outcome === 'object' && outcome.error) {
|
|
18213
|
+
logger.error(`Supervisor stop refused: ${outcome.error}`);
|
|
18214
|
+
process.exit(1);
|
|
18215
|
+
}
|
|
18216
|
+
|
|
16816
18217
|
if (outcome === 'unreachable') {
|
|
16817
18218
|
if (force) {
|
|
16818
18219
|
// Socket unreachable — fall back to signalling the daemon pid directly.
|
|
@@ -16828,6 +18229,15 @@ async function supervisorStopCmd(force = false) {
|
|
|
16828
18229
|
// backstop so a wedged daemon/worker group can't linger. The window must cover
|
|
16829
18230
|
// the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
|
|
16830
18231
|
// group while a worker is still promoting its durable recovery ref (#295).
|
|
18232
|
+
// #309 review: if the daemon we asked is mid-upgrade (quiesced), the pid we
|
|
18233
|
+
// captured is the PREDECESSOR's — it exits on the successor's handover, not on
|
|
18234
|
+
// our refused/naive stop. Say so, and never SIGKILL its process group (the
|
|
18235
|
+
// successor's adopted workers share it) or clear the successor's state.
|
|
18236
|
+
if (running.upgrade && typeof running.upgrade === 'object') {
|
|
18237
|
+
const successorPid = Number(running.upgrade.successorPid);
|
|
18238
|
+
logger.warn(`The supervisor is mid-upgrade (handing the fleet to successor pid ${Number.isInteger(successorPid) && successorPid > 0 ? successorPid : 'unknown'}) — stop the successor instead; this daemon exits when the handoff completes.`);
|
|
18239
|
+
return;
|
|
18240
|
+
}
|
|
16831
18241
|
if (force) {
|
|
16832
18242
|
const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
|
|
16833
18243
|
while (Date.now() < deadline) {
|
|
@@ -17534,8 +18944,14 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
17534
18944
|
await supervisorRestartCmd(req);
|
|
17535
18945
|
return;
|
|
17536
18946
|
case 'reload':
|
|
18947
|
+
// `reload --daemon` is the daemon-upgrade alias (issue #309): the plain
|
|
18948
|
+
// op rolls WORKER code; this rolls the daemon process itself.
|
|
18949
|
+
if (flags?.daemon) { await supervisorUpgradeCmd(); return; }
|
|
17537
18950
|
await supervisorReloadCmd(req);
|
|
17538
18951
|
return;
|
|
18952
|
+
case 'upgrade':
|
|
18953
|
+
await supervisorUpgradeCmd();
|
|
18954
|
+
return;
|
|
17539
18955
|
case 'reload-credentials':
|
|
17540
18956
|
await supervisorReloadCredentialsCmd(req);
|
|
17541
18957
|
return;
|
|
@@ -17547,7 +18963,7 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
17547
18963
|
supervisorLogsCmd(req);
|
|
17548
18964
|
return;
|
|
17549
18965
|
default:
|
|
17550
|
-
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
|
|
18966
|
+
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|upgrade|reload-credentials|stop|logs|attach`);
|
|
17551
18967
|
process.exit(1);
|
|
17552
18968
|
}
|
|
17553
18969
|
}
|
|
@@ -20318,6 +21734,9 @@ export {
|
|
|
20318
21734
|
spawnCapturePty,
|
|
20319
21735
|
spawnCaptureAcp,
|
|
20320
21736
|
sampleSubtreeCpu,
|
|
21737
|
+
sampleWorkspaceCpu,
|
|
21738
|
+
sampleWorkspaceCpuProc,
|
|
21739
|
+
sampleWorkspaceCpuLsof,
|
|
20321
21740
|
createIdleLivenessMonitor,
|
|
20322
21741
|
resolveLivenessOverrides,
|
|
20323
21742
|
parsePsTime,
|
|
@@ -20446,6 +21865,7 @@ export {
|
|
|
20446
21865
|
summarizeSupervisorWorker,
|
|
20447
21866
|
formatSupervisorStatus,
|
|
20448
21867
|
statusFromState,
|
|
21868
|
+
stateFromStatus,
|
|
20449
21869
|
supervisorDaemonDescriptor,
|
|
20450
21870
|
formatSupervisorLogsLines,
|
|
20451
21871
|
reageSupervisorStatus,
|
|
@@ -20462,6 +21882,7 @@ export {
|
|
|
20462
21882
|
normalizeAgenticMessage,
|
|
20463
21883
|
buildActivityPayload,
|
|
20464
21884
|
activityMarkerReadyFor,
|
|
21885
|
+
drainSignalReadyFor,
|
|
20465
21886
|
reloadCredentialsReady,
|
|
20466
21887
|
waitForChildExit,
|
|
20467
21888
|
supervisorWorkerActivityFile,
|
|
@@ -20474,6 +21895,7 @@ export {
|
|
|
20474
21895
|
supervisorStartCmd,
|
|
20475
21896
|
supervisorAddCmd,
|
|
20476
21897
|
supervisorReloadCmd,
|
|
21898
|
+
supervisorUpgradeCmd,
|
|
20477
21899
|
supervisorReloadCredentialsCmd,
|
|
20478
21900
|
workforceReloadCmd,
|
|
20479
21901
|
runningSupervisor,
|
|
@@ -20582,7 +22004,8 @@ export const metadata = {
|
|
|
20582
22004
|
{ command: 'c8ctl nano supervisor add decider', description: 'Add a supervised worker (forwarding work flags) to the running supervisor' },
|
|
20583
22005
|
{ command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
|
|
20584
22006
|
{ command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
|
|
20585
|
-
{ command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only
|
|
22007
|
+
{ command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only — use `supervisor upgrade` for the daemon)' },
|
|
22008
|
+
{ command: 'c8ctl nano supervisor upgrade', description: 'Rolling hot-upgrade of the supervisor DAEMON itself (alias: `reload --daemon`): a successor daemon on the new code takes the control socket and ADOPTS the running workers (no drain — in-flight jobs never stop), then the old daemon exits. Status shows `upgrade: rolling <from> → <to>` while it runs' },
|
|
20586
22009
|
{ command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
|
|
20587
22010
|
{ command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
|
|
20588
22011
|
{ command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
|
|
@@ -20664,6 +22087,7 @@ export const commands = {
|
|
|
20664
22087
|
worker: { type: 'string', multiple: true, description: 'supervisor start: profile to launch as a supervised worker (repeatable)' },
|
|
20665
22088
|
instances: { type: 'string', description: `supervisor add / workforce add: spawn/compose N distinct instances of the profile in one call (default 1, max ${MAX_ADD_INSTANCES}; for supervisor add cannot combine with --name)` },
|
|
20666
22089
|
attach: { type: 'boolean', description: 'supervisor start: attach the interactive console after starting the daemon' },
|
|
22090
|
+
daemon: { type: 'boolean', description: 'supervisor reload: roll the DAEMON process itself onto the on-disk code (alias of `supervisor upgrade`) instead of rolling worker code' },
|
|
20667
22091
|
manifest: { type: 'string', description: `workforce: manifest name to operate on (default ${DEFAULT_WORKFORCE_MANIFEST}); each subcommand reads/writes <stateHome>/workforce/<name>.json. Renamed from --profile (which now collides with c8ctl's global connection-profile flag).` },
|
|
20668
22092
|
roles: { type: 'string', description: 'workforce add: comma-separated role list for the entry (→ --job-type <rank>:<role> at start); mutually exclusive with --auto' },
|
|
20669
22093
|
},
|