c8ctl-plugin-nano 1.63.2 → 1.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -58,7 +58,7 @@ import { lookup as dnsLookup } from 'node:dns/promises';
58
58
  import * as nodeDns from 'node:dns';
59
59
  import { randomUUID, createHash, randomBytes } from 'node:crypto';
60
60
  import { homedir, platform as osPlatform, devNull, tmpdir, hostname } from 'node:os';
61
- import { join, isAbsolute, resolve as resolvePath, dirname, basename, sep } from 'node:path';
61
+ import { join, isAbsolute, resolve as resolvePath, dirname, basename, sep, delimiter } from 'node:path';
62
62
  import { createRequire } from 'node:module';
63
63
  import { fileURLToPath } from 'node:url';
64
64
  import { createInterface } from 'node:readline/promises';
@@ -159,6 +159,19 @@ const STOP_GRACE_MS = 8_000;
159
159
  // `nano work` child to quiesce it — stop leasing new jobs, finish in-flight work,
160
160
  // then exit. SIGTERM/SIGINT remain the FORCE abort (kill harness, yield jobs).
161
161
  const SUPERVISOR_DRAIN_SIGNAL = 'SIGUSR2';
162
+ // Hot-reload readiness gate: after a rolling `supervisor reload` respawns a
163
+ // worker, the daemon waits (bounded) for the replacement to STAMP `readyAt` on
164
+ // its activity marker — i.e. its activation loop is up and leasing jobs — before
165
+ // draining the NEXT worker, so at most one worker is ever unavailable at a time.
166
+ // A bare spawn/PID is not readiness: the child still has to import the plugin,
167
+ // build its SDK client and start its activation loop. The wait is bounded so a
168
+ // slow/never-ready replacement (e.g. a wedged engine) can't stall the roll — on
169
+ // timeout the daemon advances anyway (degrading to the old spawn-and-advance).
170
+ const SUPERVISOR_RELOAD_READY_POLL_MS = 100;
171
+ const SUPERVISOR_RELOAD_READY_TIMEOUT_MS = Math.max(
172
+ 0,
173
+ Number.parseInt(process.env.NANO_SUPERVISOR_RELOAD_READY_TIMEOUT_MS ?? '', 10) || 30_000,
174
+ );
162
175
  // Upper bound on one `--auto` engine-read reconcile (enumerate deployed
163
176
  // definitions + fetch each BPMN). A read that stalls past this is treated as a
164
177
  // transient failure so the running poller set is KEPT and, crucially, shutdown
@@ -325,6 +338,19 @@ function readWorkerActivity(id) {
325
338
  }
326
339
  }
327
340
 
341
+ /**
342
+ * Whether an activity marker proves a specific child (`pid`) is up and leasing.
343
+ * Both conditions are required (#253): a finite `readyAt` (the runtime's
344
+ * first-activation handshake fired) AND `act.pid === pid` (the marker belongs to
345
+ * THIS child, not a stale one a failed best-effort delete left behind from a
346
+ * previous incarnation — which would otherwise let the rolling reload advance
347
+ * before the fresh replacement has actually reported ready). Pure so it can be
348
+ * unit-tested directly.
349
+ */
350
+ function activityMarkerReadyFor(act, pid) {
351
+ return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
352
+ }
353
+
328
354
  /**
329
355
  * Deterministic control-socket path shared by the daemon and every client.
330
356
  * Derived from a hash of the (possibly overridden) state home so distinct
@@ -1721,6 +1747,104 @@ function buildAgentCommandLine(command, args) {
1721
1747
  return `${command} ${list.map(shQuote).join(' ')}`;
1722
1748
  }
1723
1749
 
1750
+ // #243: extract a plausible version token from an agent CLI's `--version` output for
1751
+ // the durable transcript's provenance block. Prefers a semver-ish token, else the
1752
+ // first non-empty line; length-capped so a chatty (or adversarial) harness can't bloat
1753
+ // the transcript. Returns null when nothing usable is present.
1754
+ export function extractVersionToken(text) {
1755
+ const s = String(text || '');
1756
+ const semver = s.match(/\bv?\d+\.\d+(?:\.\d+)?(?:[-+][0-9A-Za-z.-]+)?\b/);
1757
+ if (semver) return semver[0].slice(0, 64);
1758
+ const firstLine = s.split(/\r?\n/).map((l) => l.trim()).find((l) => l.length > 0);
1759
+ return firstLine ? firstLine.slice(0, 64) : null;
1760
+ }
1761
+
1762
+ // #243: best-effort probe of an agent harness CLI's OWN version (distinct from the nano
1763
+ // plugin/supervisor version), for the durable transcript's provenance block. Runs
1764
+ // `<command> --version` through a shell (PATH resolution, mirroring how the harness is
1765
+ // spawned) with a hard timeout + SIGKILL and stdin closed so an interactive harness
1766
+ // gets EOF and exits rather than hanging. Restricted to a BARE single executable and run
1767
+ // under the caller-supplied `env` (the merged profile env) so it can neither run an
1768
+ // embedded-argument command nor resolve a different PATH binary than the real run
1769
+ // (#257). ENTIRELY best-effort: any failure, timeout, non-zero exit with no version-ish
1770
+ // output, an embedded/compound command, or unrecognizable output yields null (the field
1771
+ // is simply omitted). Meant to be called ONCE per worker process (the command is fixed
1772
+ // per profile), so it never taxes the activation hot path. Disable via
1773
+ // NANO_AGENT_CLI_PROBE=off. The spawner is injected for deterministic testing.
1774
+ function probeAgentCliVersion(command, { timeoutMs = 1500, run = spawnSync, env = undefined } = {}) {
1775
+ if (typeof command !== 'string' || command.trim() === '') return null;
1776
+ if (String(process.env.NANO_AGENT_CLI_PROBE || '').trim().toLowerCase() === 'off') return null;
1777
+ // #257 review: only probe a BARE single executable. `buildAgentCommandLine`
1778
+ // preserves an embedded-argument `command` verbatim when structured args are
1779
+ // empty (e.g. `command: "node agent.js"`), so `${command} --version` would run
1780
+ // that script/compound command — probing the interpreter or a wholly different
1781
+ // program — and persist a false `agentCliVersion`. A token carrying whitespace or
1782
+ // shell metacharacters is not a bare executable: omit the probe rather than
1783
+ // record a wrong (or side-effecting) reading. A plain path token (with / \ : . _ -)
1784
+ // is still allowed so an absolute harness path probes normally.
1785
+ if (!/^[\w./\\:+-]+$/.test(command.trim())) return null;
1786
+ // #257 review: SKIP a RELATIVE path command (a token carrying a path separator
1787
+ // that is not absolute, e.g. `./harness`, `../bin/tool`, `sub/dir/cmd`). Unlike a
1788
+ // bare PATH-resolved name (cwd-independent) or an absolute path (fully determined),
1789
+ // a relative path resolves against the probe's cwd — which is the WORKER's cwd, NOT
1790
+ // the per-job `cwd` (a fresh run dir / cloned repo) the harness is actually launched
1791
+ // from in `runAgentJob`. Probing `./harness` here could read a different file/version
1792
+ // than the command the job runs (or find nothing where the job would), persisting a
1793
+ // false/omitted `agentCliVersion`. Best-effort: omit rather than record a wrong
1794
+ // reading for a command whose resolution is cwd-ambiguous.
1795
+ {
1796
+ const cmd = command.trim();
1797
+ if (/[\\/]/.test(cmd) && !isAbsolute(cmd)) return null;
1798
+ // #257 review: a BARE PATH-resolved name is only cwd-independent if PATH itself
1799
+ // is. If the caller-supplied env's PATH carries a RELATIVE or EMPTY entry (e.g.
1800
+ // `.`, an empty field meaning cwd, or `./node_modules/.bin`), resolution depends
1801
+ // on the working directory — and the probe's cwd (the WORKER's) is NOT the per-job
1802
+ // cwd (the run dir / cloned repo) the harness is launched from — so `--version`
1803
+ // could resolve a different executable/version than the job's. Omit rather than
1804
+ // record a cwd-ambiguous reading. An ABSOLUTE command bypasses PATH entirely, so
1805
+ // it is unaffected; a probe with no explicit `env` inherits `process.env` verbatim
1806
+ // (the caller opted into that resolution) and is left untouched.
1807
+ if (env && !isAbsolute(cmd)) {
1808
+ const pathVar = env.PATH ?? env.Path ?? env.path ?? '';
1809
+ if (String(pathVar).split(delimiter).some((e) => e === '' || !isAbsolute(e))) return null;
1810
+ }
1811
+ }
1812
+ let out;
1813
+ try {
1814
+ out = run(`${command} --version`, {
1815
+ shell: true,
1816
+ timeout: timeoutMs,
1817
+ killSignal: 'SIGKILL',
1818
+ encoding: 'utf8',
1819
+ stdio: ['ignore', 'pipe', 'pipe'],
1820
+ // #257 review: run the probe under the SAME environment the harness will run
1821
+ // under (the profile env merged over the host env), so PATH resolution matches
1822
+ // the real invocation and the probe can't resolve a different binary than the
1823
+ // one the job will spawn. Undefined inherits `process.env` (spawnSync default).
1824
+ ...(env ? { env } : {}),
1825
+ // #257 review: cap the captured output. Without a finite maxBuffer a harness
1826
+ // that streams continuously during the timeout window makes each worker retain
1827
+ // unbounded stdout/stderr (exhausting memory) before the 64-byte transcript cap
1828
+ // is ever applied. On overflow spawnSync sets `out.error` (ENOBUFS), which the
1829
+ // guard below already rejects — so the probe stays best-effort (yields null).
1830
+ maxBuffer: 256 * 1024,
1831
+ windowsHide: true,
1832
+ });
1833
+ } catch {
1834
+ return null;
1835
+ }
1836
+ if (!out) return null;
1837
+ // #257 review: spawnSync does NOT throw for a normal shell failure or timeout —
1838
+ // with `shell: true` it returns a result carrying `error` (spawn failure /
1839
+ // ETIMEDOUT) and/or a non-zero/null exit `status`, alongside diagnostic text on
1840
+ // stderr (e.g. `/bin/sh: <cmd>: not found`) or a partial capture. Feeding that to
1841
+ // extractVersionToken (whose fallback accepts the first non-empty line) would
1842
+ // persist the error banner as a bogus `agentCliVersion`. Trust only a clean exit
1843
+ // (no `error`, status 0); anything else omits the field.
1844
+ if (out.error || out.status !== 0) return null;
1845
+ return extractVersionToken(`${out.stdout || ''}\n${out.stderr || ''}`);
1846
+ }
1847
+
1724
1848
  // A worker job-type token: rank/capability tokens use `:` (rank↔cap) and `+`
1725
1849
  // (combined caps) as delimiters, and code-first `@nanobpm/workflow` job types
1726
1850
  // are `<flowId>:<taskName>` or an explicit override. The first character must be
@@ -3491,6 +3615,11 @@ async function createSupervisorDeps(opts = {}) {
3491
3615
  config,
3492
3616
  fetchImpl,
3493
3617
  env = process.env,
3618
+ // A plain JS thunk (Effect-free, monolith-supplied) fired ONCE when the
3619
+ // activation loop begins leasing — lifted below into the runtime's
3620
+ // `onFirstActivation` Effect. Used as the rolling-reload readiness handshake
3621
+ // (#253): stamping readiness only when the runtime is actually serving.
3622
+ onFirstActivation,
3494
3623
  } = opts;
3495
3624
  if (!runner || typeof runner.run !== 'function') {
3496
3625
  throw new TypeError('createSupervisorDeps: `runner` must be a raw job runner `{ run(job): Promise<void> }`');
@@ -3499,7 +3628,6 @@ async function createSupervisorDeps(opts = {}) {
3499
3628
  const rt = await loadSupervisorRuntime();
3500
3629
  const { demand } = await import('./agentic.mjs');
3501
3630
  const { createRawEngineClient } = await import('./supervisor-engine.mjs');
3502
-
3503
3631
  // Base/auth: the single canonical worker-engine chain (explicit restConfig →
3504
3632
  // profile restAddress → localhost), ALWAYS run through the token same-origin
3505
3633
  // gate — even when a caller pins the base via `restConfig` — so token
@@ -3563,6 +3691,11 @@ async function createSupervisorDeps(opts = {}) {
3563
3691
  agenticEndpoint,
3564
3692
  agenticConfig,
3565
3693
  config: scope ? { ...config, scope } : config,
3694
+ // Lift the plain readiness thunk into an Effect the runtime runs on its own
3695
+ // fiber the instant it starts leasing (#253). Effect-free JS in, Effect out —
3696
+ // the same "monolith supplies plain JS, TS lifts it" seam as the other ports.
3697
+ onFirstActivation:
3698
+ typeof onFirstActivation === 'function' ? rt.Effect.sync(onFirstActivation) : undefined,
3566
3699
  });
3567
3700
 
3568
3701
  // The `settle` seam (issue #156, escalation answer (a)): the runner settles a
@@ -8288,7 +8421,7 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
8288
8421
  * list; `busy` is derived so callers can't desync it from `jobs`.
8289
8422
  * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number}>, engine:(string|null), agentic:object }} fields
8290
8423
  */
8291
- function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic }) {
8424
+ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }) {
8292
8425
  const jobList = Array.isArray(jobs) ? jobs : [];
8293
8426
  return {
8294
8427
  pid,
@@ -8297,6 +8430,10 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic }) {
8297
8430
  jobs: jobList,
8298
8431
  engine: engine ?? null,
8299
8432
  agentic,
8433
+ // When the worker's activation loop has started (it has imported, built its
8434
+ // SDK client and begun leasing), the producer stamps this; null until then.
8435
+ // The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
8436
+ readyAt: readyAt ?? null,
8300
8437
  };
8301
8438
  }
8302
8439
 
@@ -8753,6 +8890,17 @@ async function workAgent(req, flags, ctx) {
8753
8890
  const workerPidStart = pidStartToken(process.pid);
8754
8891
  let pluginVersion = null;
8755
8892
  try { pluginVersion = JSON.parse(readFileSync(join(pluginDir, 'package.json'), 'utf-8')).version ?? null; } catch { /* best effort */ }
8893
+ // #243: the agent harness CLI's own version is probed ONCE per worker process
8894
+ // (bounded, best-effort) so the durable transcript's provenance block can attribute
8895
+ // a run to a specific harness build. The probe is DEFERRED until the FIRST
8896
+ // external-agent activation (its only consumer is the AgentInstance producer), so a
8897
+ // worker that only services ordinary jobs never runs the harness `--version`. The
8898
+ // probe env is WORKER-STATIC — `process.env` merged with the profile `env` only,
8899
+ // NOT the per-job `setup.env` — so the once-per-worker cache can't leak one job's
8900
+ // setup-derived reading to later jobs (#257). Never fatal — a null result just omits
8901
+ // the field. `agentCliProbed` latches the once-only semantics.
8902
+ let agentCliVersion = null;
8903
+ let agentCliProbed = false;
8756
8904
  let workerNsDir;
8757
8905
  try {
8758
8906
  ({ nsDir: workerNsDir } = allocateWorkerNamespace({
@@ -8864,6 +9012,66 @@ async function workAgent(req, flags, ctx) {
8864
9012
  // active profile itself — identical to the old no-arg behaviour.
8865
9013
  const camunda = globalThis.c8ctl.createClient(resolveConnectionProfile(ctx));
8866
9014
 
9015
+ // #243/#257: precompute the job-independent gates for the harness `--version`
9016
+ // probe (the probe itself is DEFERRED to the first external-agent activation
9017
+ // below — see maybeProbeAgentCliVersion). The probe only yields a consumable
9018
+ // reading when:
9019
+ // - HOST execution (a container job runs `sh -c` inside the selected image, so a
9020
+ // same-named host binary would report a plausible-but-wrong version);
9021
+ // - the AgentInstance kill-switch is not set (NANO_AGENT_INSTANCE=off disables the
9022
+ // producer — the probe's only consumer — so probing would needlessly run a
9023
+ // possibly non-idempotent harness with no reader);
9024
+ // - the host SDK actually supports AgentInstance (create/update). The producer
9025
+ // degrades to a disabled facade for older clients, so probing under such a client
9026
+ // is pure waste + an unnecessary side effect (Copilot review, #257);
9027
+ // - the ACP classifier is available. `createAgentInstanceProducer` is ALSO inert
9028
+ // (the `usable` gate requires `classifyUpdate`) when the agentic classifier is
9029
+ // missing, even with a create/update-capable SDK — so probing would run the
9030
+ // harness `--version` for a producer that can never mint a transcript to consume
9031
+ // it. Mirror that precondition here so the probe stays side-effect-free whenever
9032
+ // the producer is disabled (Copilot review, #257);
9033
+ // - the profile command alone is the full invocation (no extra args). An
9034
+ // interpreter-style profile (`command:'node', args:['agent.js']`) would probe the
9035
+ // interpreter, not the harness — omit rather than misattribute.
9036
+ // The probe itself (probeAgentCliVersion) additionally refuses an embedded-argument
9037
+ // command and runs under the WORKER-STATIC profile env (see maybeProbeAgentCliVersion
9038
+ // for why per-job setup.env is deliberately excluded), so PATH resolves a stable
9039
+ // representative harness binary.
9040
+ const agentInstanceProbeOff = String(process.env.NANO_AGENT_INSTANCE || '').trim().toLowerCase() === 'off';
9041
+ const sdkSupportsAgentInstance =
9042
+ !!camunda &&
9043
+ typeof camunda.createAgentInstance === 'function' &&
9044
+ typeof camunda.updateAgentInstance === 'function';
9045
+ // The producer's `usable` gate also requires the ACP classifier (agent-instance.mjs);
9046
+ // without it the producer is inert regardless of SDK support, so exclude the probe too.
9047
+ const acpClassifierAvailable = typeof agenticSessionAcp?.classifyUpdate === 'function';
9048
+ const agentCliProbeEligible =
9049
+ !isContainer && !agentInstanceProbeOff && sdkSupportsAgentInstance && acpClassifierAvailable && effectiveArgs.length === 0;
9050
+ // #257 review: DEFER the probe until an external-agent job is actually being
9051
+ // serviced — so a worker that only ever receives ordinary service jobs never runs
9052
+ // the harness `--version` at all (no wasted startup delay / side effect). Runs at
9053
+ // most once per worker (cached), so the per-job hot path pays nothing after the
9054
+ // first external activation. The probe env is WORKER-STATIC — `process.env` merged
9055
+ // with the profile `env` only, NOT the per-job `setup.env`: `agentCliProbed` is a
9056
+ // worker-wide latch, so folding a single job's `setup.env` (which can change PATH,
9057
+ // and thus which CLI build resolves) into the cache would leak THAT job's reading
9058
+ // to every later job serviced by the same worker (#257). A worker-static basis makes
9059
+ // the cached `agentCliVersion` a correct-by-construction representative reading for
9060
+ // the profile; a job whose `setup.env` genuinely alters the resolved binary is a
9061
+ // per-job divergence the single cached provenance reading deliberately does not chase.
9062
+ const maybeProbeAgentCliVersion = () => {
9063
+ if (agentCliProbed || !agentCliProbeEligible) return;
9064
+ agentCliProbed = true;
9065
+ try {
9066
+ agentCliVersion = probeAgentCliVersion(profile?.command, {
9067
+ env: {
9068
+ ...process.env,
9069
+ ...normalizeEnvMap(profileEnv),
9070
+ },
9071
+ });
9072
+ } catch { /* best effort */ }
9073
+ };
9074
+
8867
9075
  // Broker REST endpoint for live linked-resource prompts (issue #63) and the
8868
9076
  // C8 REST source for `--auto`'s engine-read enrolment. Derived from the SAME
8869
9077
  // client that activates jobs (its profile REST address) when no explicit
@@ -8955,10 +9163,15 @@ async function workAgent(req, flags, ctx) {
8955
9163
  // is updated once the channel target is resolved and again on each
8956
9164
  // connect/disconnect below.
8957
9165
  let agenticState = { status: 'starting' };
9166
+ // Readiness handshake (Copilot review on #253): null until this worker's
9167
+ // activation loop is actually up and leasing; the supervisor's rolling reload
9168
+ // waits for this stamp before draining the next worker so a bare spawn/PID is
9169
+ // never mistaken for a serving replacement.
9170
+ let readyAt = null;
8958
9171
  const writeActivity = () => {
8959
9172
  if (!activityFile) return;
8960
9173
  const jobs = [...activeJobs.entries()].map(([key, v]) => ({ key, type: v.type, since: v.since }));
8961
- const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState });
9174
+ const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt });
8962
9175
  const tmp = `${activityFile}.${process.pid}.tmp`;
8963
9176
  try {
8964
9177
  mkdirSync(dirname(activityFile), { recursive: true });
@@ -9366,7 +9579,19 @@ async function workAgent(req, flags, ctx) {
9366
9579
  // job, never on these lines).
9367
9580
  const aiCorr = `job ${job.jobKey} eik ${job.elementInstanceKey ?? '?'} pik ${job.processInstanceKey ?? '?'}`;
9368
9581
  if (!agentInstanceOff && isExternalAgentJob(job)) {
9369
- agentInstanceProducer = createAgentInstanceProducer({ camunda, job, profile, envelope, logger });
9582
+ maybeProbeAgentCliVersion();
9583
+ // #257 review: the synchronous CLI `--version` probe above can block for up
9584
+ // to its timeout on the FIRST external activation. A force-stop/lease-loss
9585
+ // can win that race while spawnSync is blocked, so re-check the setup-abort
9586
+ // gate before minting the durable AgentInstance — otherwise an already-aborted
9587
+ // run would call activate() and mint an instance/transcript that is then
9588
+ // immediately orphaned (the post-activate gates run too late to prevent the
9589
+ // create). Mirrors the pre-probe agent-instance gate above.
9590
+ if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'agent-instance', logger })) {
9591
+ if (isContainer) liveRunIds.delete(runId);
9592
+ return;
9593
+ }
9594
+ agentInstanceProducer = createAgentInstanceProducer({ camunda, job, profile, envelope, logger, runtimeVersion: pluginVersion, agentCliVersion });
9370
9595
  try {
9371
9596
  // `createAgentInstanceProducer` always returns an object — including a
9372
9597
  // DISABLED facade when the host SDK lacks createAgentInstance/
@@ -10017,6 +10242,16 @@ async function workAgent(req, flags, ctx) {
10017
10242
  // agentic target didn't resolve to a connect — the runtime then runs with no
10018
10243
  // agentic scope and presence/steer degrade to no-ops.
10019
10244
  agenticEndpoint: agenticEndpoint || undefined,
10245
+ // Readiness handshake (#253): the runtime fires this the instant its activation
10246
+ // loop begins leasing (on its OWN fiber, after reconcile/presence are forked;
10247
+ // under agentic the connect cycle is a concurrent child, so the connection may
10248
+ // still be connecting — readiness is LEASING-gated, not connection-gated). Stamp
10249
+ // `readyAt` on the
10250
+ // activity marker here so the supervisor's rolling `reload` waits for THIS
10251
+ // replacement to be genuinely serving before draining the next worker. A bare
10252
+ // `runFork` return (which only schedules the fiber) is NOT readiness.
10253
+ // Best-effort — a marker write never fails the worker.
10254
+ onFirstActivation: () => { readyAt = Date.now(); writeActivity(); },
10020
10255
  config: {
10021
10256
  activation: { requestTimeoutMs: pollTimeoutMs },
10022
10257
  dispatch: { recoveryWindowMs, extendIntervalMs: lockExtendIntervalMs },
@@ -10044,6 +10279,12 @@ async function workAgent(req, flags, ctx) {
10044
10279
  const supervisor = await SupervisorEffect.runPromise(makeSupervisorRuntime(supervisorDeps));
10045
10280
  const supervisorFiber = SupervisorEffect.runFork(supervisor.run);
10046
10281
 
10282
+ // Readiness is stamped by the runtime's `onFirstActivation` handshake (wired
10283
+ // into `createSupervisorDeps` above), NOT here: `runFork` only *schedules* the
10284
+ // fiber and can return before it has executed at all, so stamping `readyAt`
10285
+ // in this continuation could report the worker ready before its activation loop
10286
+ // is leasing — defeating the rolling reload's one-at-a-time guarantee (#253).
10287
+
10047
10288
  // Seed this worker's presence into the runtime's ownership registry (issue
10048
10289
  // #173) and late-bind the per-job relay seam to the running supervisor. The
10049
10290
  // presence-projection fiber announces (register) then heartbeats this instance
@@ -10937,6 +11178,19 @@ function formatSupervisorStatus(status) {
10937
11178
  lines.push('Supervisor:');
10938
11179
  lines.push(` daemon pid: ${d.pid ?? '-'} ${alive ? '(alive)' : '(dead — stale state)'}`);
10939
11180
  if (d.version) lines.push(` version: ${d.version}`);
11181
+ // Flag a code update the running daemon hasn't adopted yet: the plugin on disk
11182
+ // has advanced past the daemon's version (e.g. after `nano update`). A rolling
11183
+ // `supervisor reload` adopts the new WORKER code with zero downtime; a daemon
11184
+ // restart is needed for new SUPERVISOR code. `status.pluginVersion` is only
11185
+ // present on a live socket `status` frame; the socket-unreachable fallback
11186
+ // (`statusFromState()`) has no such field, so read the on-disk package version
11187
+ // locally as a fallback — otherwise the warning silently disappears exactly
11188
+ // when the daemon is alive but its control socket is briefly unreachable.
11189
+ const onDiskVersion = status.pluginVersion
11190
+ ?? (() => { try { return pluginPackage().version; } catch { return null; } })();
11191
+ if (onDiskVersion && d.version && onDiskVersion !== d.version) {
11192
+ lines.push(` on disk: ${onDiskVersion} (update available — run \`c8ctl nano supervisor reload\` to adopt new worker code; restart the daemon for new supervisor code)`);
11193
+ }
10940
11194
  if (d.startedAt) lines.push(` started: ${d.startedAt}`);
10941
11195
  if (d.socket) lines.push(` control: ${d.socket}`);
10942
11196
  const workers = Array.isArray(status.workers) ? status.workers : [];
@@ -11107,7 +11361,19 @@ function waitForChildExit(child, timeoutMs) {
11107
11361
  // #202: a null/undefined timeout means WAIT INDEFINITELY (graceful drain) —
11108
11362
  // no timer is armed, so we only resolve when the child actually exits.
11109
11363
  const t = timeoutMs == null ? null : setTimeout(() => finish(), timeoutMs);
11110
- function finish() { if (done) return; done = true; if (t) clearTimeout(t); resolve(); }
11364
+ // `finish` ALWAYS removes the `exit` listener, including the timeout path:
11365
+ // `child.once` only self-removes when the event fires, so a timed-out wait
11366
+ // would otherwise leave its listener attached. The readiness poll calls this
11367
+ // ~every 100ms for up to 30s, so a leaked listener per poll accumulates
11368
+ // hundreds on a long-lived child — a `MaxListenersExceededWarning` plus
11369
+ // retained closures (#253 review).
11370
+ function finish() {
11371
+ if (done) return;
11372
+ done = true;
11373
+ if (t) clearTimeout(t);
11374
+ child.removeListener('exit', finish);
11375
+ resolve();
11376
+ }
11111
11377
  child.once('exit', finish);
11112
11378
  });
11113
11379
  }
@@ -11205,6 +11471,11 @@ async function runSupervisorDaemon() {
11205
11471
  // restart-on-exit path and let a second `stop --force` escalate a live drain.
11206
11472
  let draining = false;
11207
11473
  let forcing = false;
11474
+ // Hot code reload (rolling drain+respawn). A `reload` op adopts new on-disk
11475
+ // plugin code into the worker children by gracefully draining and respawning
11476
+ // them one at a time (so the fleet keeps serving). This flag rejects a second
11477
+ // concurrent reload — a single rolling pass owns the fleet until it finishes.
11478
+ let reloading = false;
11208
11479
  // Live-view monitor: tracks the last-broadcast fleet signature so we push a
11209
11480
  // refreshed status to attached consoles only on real change (see below).
11210
11481
  let monitorTimer = null;
@@ -11457,6 +11728,216 @@ async function runSupervisorDaemon() {
11457
11728
  return true;
11458
11729
  };
11459
11730
 
11731
+ // Hot code reload of a single worker: GRACEFULLY drain it (SIGUSR2 — finish
11732
+ // in-flight jobs, then exit) and respawn it, so the new child re-reads the
11733
+ // updated plugin from disk. Unlike `restartWorker` (a force SIGTERM/SIGKILL
11734
+ // swap), this waits INDEFINITELY for the drain so no in-flight job is lost —
11735
+ // adopting new code is never worth killing running work.
11736
+ //
11737
+ // CAPACITY CAVEAT: this drains the worker BEFORE spawning its replacement, so
11738
+ // for the drain+boot window that worker serves no jobs. The fleet's "zero
11739
+ // downtime" guarantee is therefore a FLEET-level one — with >1 worker the rest
11740
+ // keep serving while one drains. A single-worker fleet (or a job type served by
11741
+ // only this one worker) does lose that type's serving capacity until the drain
11742
+ // finishes, `startWorker` runs, AND the replacement reports ready. Preserving an
11743
+ // overlapping serving replacement would need a two-child handoff; that is
11744
+ // intentionally out of scope here (documented in README/AGENTS).
11745
+ //
11746
+ // ONE-AT-A-TIME: after respawning, this waits (bounded) for the replacement to
11747
+ // stamp `readyAt` on its activity marker — it is up and leasing — before it
11748
+ // returns, so `runReload` never drains the NEXT worker while this one is still
11749
+ // booting. A bare spawn/PID is not readiness (Copilot review on #253).
11750
+ //
11751
+ // The drain runs OUTSIDE the op lock (it can be arbitrarily long) so a
11752
+ // `stop --force` or a `remove`/`restart` for this same worker isn't blocked
11753
+ // and can escalate/interrupt it. Because of that, the respawn is guarded by
11754
+ // the child-identity check (`w.child === child`): if a concurrent
11755
+ // force-stop/restart/remove already swapped or deleted this worker while we
11756
+ // drained, we must NOT respawn (that would leak a duplicate child or revive a
11757
+ // removed worker). We also skip the respawn when the daemon is shutting down.
11758
+ // Poll a freshly (re)spawned worker's activity marker until it stamps `readyAt`
11759
+ // (its activation loop is up and leasing), the child is swapped/exits, the
11760
+ // daemon starts shutting down, or the bounded deadline passes. Returns whether
11761
+ // it became ready; the caller advances regardless — readiness is a best-effort
11762
+ // gate, never a hard block. Used by the rolling reload so it does not drain the
11763
+ // next worker while this replacement is still booting (Copilot review on #253).
11764
+ const waitForWorkerReady = async (w, child, timeoutMs) => {
11765
+ const deadline = Date.now() + Math.max(0, timeoutMs);
11766
+ for (;;) {
11767
+ // Stop waiting if a concurrent restart/force-stop swapped this child, or it
11768
+ // already exited — it is no longer a booting replacement to gate on.
11769
+ if (w.child !== child) return false;
11770
+ if (child.exitCode !== null || child.signalCode !== null) return false;
11771
+ // Abort at once on a spawn failure (ENOENT/EMFILE/…): it emits only 'error'
11772
+ // with NO 'exit', so `exitCode`/`signalCode` stay null and the two checks
11773
+ // above never fire — without this the loop would poll the full ready-timeout
11774
+ // (~30s) before the final live-PID gate rejects a worker that never started
11775
+ // (#253 review). `handleDeath` nulls `w.pid` on that 'error' (and a failed
11776
+ // spawn has no `child.pid` to begin with), so a null `w.pid` for THIS still
11777
+ // -current child means the replacement is dead — stop waiting immediately.
11778
+ if (w.pid == null) return false;
11779
+ const act = readWorkerActivity(w.id);
11780
+ // Require the marker to be from THIS replacement child (`act.pid === child.pid`)
11781
+ // AND carry a finite `readyAt` (#253): a stale marker left by a previous
11782
+ // incarnation must not pass this gate before the freshly spawned child has
11783
+ // reported ready — see activityMarkerReadyFor.
11784
+ if (activityMarkerReadyFor(act, child.pid)) return true;
11785
+ if (shuttingDown || Date.now() >= deadline) {
11786
+ dlog(`worker '${w.id}' not ready within ${timeoutMs}ms after reload — advancing anyway`);
11787
+ return false;
11788
+ }
11789
+ // waitForChildExit doubles as a poll sleep: it resolves early if the child
11790
+ // exits (the loop-top guard then returns) so we never busy-spin on a dead child.
11791
+ await waitForChildExit(child, SUPERVISOR_RELOAD_READY_POLL_MS);
11792
+ }
11793
+ };
11794
+
11795
+ const reloadWorker = async (id) => {
11796
+ const w = workers.get(id);
11797
+ if (!w) return false;
11798
+ const child = w.child;
11799
+ w.stopping = true;
11800
+ if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
11801
+ const pid = w.pid;
11802
+ if (child && pid) {
11803
+ try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
11804
+ await waitForChildExit(child, null);
11805
+ }
11806
+ // Only respawn if nobody else acted on this worker while we drained: it must
11807
+ // still exist, we must not be shutting down, and its child handle must still
11808
+ // be the one we drained (a concurrent restart/force-stop would have swapped
11809
+ // it). The startWorker+guard runs under the op lock so it can't interleave
11810
+ // with add/remove.
11811
+ const started = await serializeOp(async () => {
11812
+ const cur = workers.get(id);
11813
+ if (!cur || cur !== w || shuttingDown || w.child !== child) return null;
11814
+ w.stopping = false;
11815
+ w.restarts = 0;
11816
+ startWorker(w);
11817
+ dlog(`worker '${id}' respawned (awaiting readiness before adopting new code)`);
11818
+ return w.child;
11819
+ });
11820
+ if (!started) return false;
11821
+ // `startWorker` returns the instant the child is forked — but the replacement
11822
+ // still has to import the plugin, build its SDK client, and start its
11823
+ // activation loop before it serves jobs. Signalling "reloaded" here lets
11824
+ // `runReload` advance to drain the NEXT worker, so returning on the bare spawn
11825
+ // could leave the just-respawned worker AND the next (draining) worker down at
11826
+ // once, breaking the one-at-a-time guarantee (Copilot review on #253). Gate on
11827
+ // the replacement stamping `readyAt` on its activity marker (it is up and
11828
+ // leasing) before we return. Bounded so a slow/never-ready replacement can't
11829
+ // wedge the roll — on timeout we advance anyway (degrading to spawn-and-advance).
11830
+ await waitForWorkerReady(w, started, SUPERVISOR_RELOAD_READY_TIMEOUT_MS);
11831
+ // Report reloaded only when the child we spawned is STILL this worker's live
11832
+ // current child. `waitForWorkerReady` returns even when the replacement exited
11833
+ // (crash/spawn-fail) or was swapped by a concurrent restart/remove — in those
11834
+ // cases no replacement is actually running, so counting it as reloaded would
11835
+ // let the terminal frame claim success and let the roll drain the next worker
11836
+ // with this one down (#253 review). A readiness *timeout* on a still-live
11837
+ // current child still counts as success — readiness is best-effort.
11838
+ //
11839
+ // Gate on the LIVE PID, not just object identity + null exit/signal: a spawn
11840
+ // failure (ENOENT/EMFILE/…) emits only 'error' with NO 'exit', so
11841
+ // `exitCode`/`signalCode` stay null and `w.child` keeps pointing at the failed
11842
+ // ChildProcess until its backoff retry — the identity+exit check alone would
11843
+ // count that as reloaded (#253 review). `handleDeath` nulls `w.pid` on every
11844
+ // death (error OR exit), and a failed spawn has no `child.pid`, so requiring
11845
+ // `w.pid` to be non-null AND still equal to this child's pid rejects both a
11846
+ // failed spawn and a dead/retrying child while accepting a live replacement
11847
+ // (readiness timeout included).
11848
+ //
11849
+ // Also reject `w.stopping`: a concurrent `remove`/`restart`/`stop` sets that
11850
+ // flag (and clears the restart timer) BEFORE its kill signal lands, so for a
11851
+ // brief window `w.child`/`w.pid` still point at the live replacement we just
11852
+ // spawned. Counting that as reloaded would let the roll drain the NEXT worker
11853
+ // while this one is being torn down — a partial-fleet outage. A worker being
11854
+ // stopped has no confirmed serving replacement, so treat it as a failed
11855
+ // reload (the `runReload` else-branch then aborts the roll, #253 review).
11856
+ //
11857
+ // Also reject `shuttingDown`: a `stop` can begin DURING this replacement's
11858
+ // readiness wait, after which `waitForWorkerReady` still returns and (for a
11859
+ // last target) `w.stopping` may not be latched yet — so without this the gate
11860
+ // would report a clean reload while shutdown is already tearing the fleet
11861
+ // down. A daemon that is shutting down has no serving future for this worker,
11862
+ // so treat a shutdown observed before the gate as a failed/interrupted reload
11863
+ // (the `runReload` else-branch aborts the roll, #253 review).
11864
+ return w.child === started && !w.stopping && !shuttingDown && w.pid != null && w.pid === started.pid
11865
+ && started.exitCode === null && started.signalCode === null;
11866
+ };
11867
+
11868
+ // Rolling hot reload across a set of worker ids: drain+respawn each in turn
11869
+ // (one at a time, so the rest of the fleet keeps serving). Streams progress to
11870
+ // `sock` (registered as an attach consumer for the interleaved worker events)
11871
+ // and ends with a terminal `reloaded` frame. Aborts early if the daemon starts
11872
+ // shutting down. Never clears `supervisor.json` — a reload is not a stop.
11873
+ const runReload = async (ids, sock) => {
11874
+ reloading = true;
11875
+ const reloaded = [];
11876
+ const skipped = [];
11877
+ let interrupted = false;
11878
+ try {
11879
+ for (let i = 0; i < ids.length; i++) {
11880
+ const id = ids[i];
11881
+ // If the daemon starts shutting down mid-roll, the remaining workers
11882
+ // never adopt the new code — record them as skipped (and flag the roll
11883
+ // interrupted) so the terminal frame can't report a clean success for a
11884
+ // pass that stopped early.
11885
+ if (shuttingDown) {
11886
+ interrupted = true;
11887
+ for (let j = i; j < ids.length; j++) skipped.push(ids[j]);
11888
+ break;
11889
+ }
11890
+ // A target that has vanished mid-roll (only `remove`/drain-remove deletes
11891
+ // the entry — `restart` keeps it) has NO confirmed serving replacement,
11892
+ // exactly like the reload-failure branch below. Continuing to drain the
11893
+ // NEXT worker on top of that gap is the same partial-fleet risk the canary
11894
+ // exists to prevent, so treat a removed-mid-roll target uniformly: mark it
11895
+ // and every remaining id skipped and abort the roll (#253 review).
11896
+ if (!workers.has(id)) {
11897
+ interrupted = true;
11898
+ for (let j = i; j < ids.length; j++) skipped.push(ids[j]);
11899
+ break;
11900
+ }
11901
+ const ok = await reloadWorker(id);
11902
+ if (ok) {
11903
+ reloaded.push(id);
11904
+ // Emit the per-worker "reloaded" progress signal ONLY after the final
11905
+ // success gate above confirmed a live, still-current replacement — not
11906
+ // at spawn time. A crash/spawn-fail makes `reloadWorker` return false
11907
+ // and the worker is skipped, so broadcasting at spawn time would let the
11908
+ // streaming client print `reloaded "…" (adopted new code)` for a reload
11909
+ // that actually failed (#253 review).
11910
+ broadcast({ type: 'event', event: 'worker-reload', id });
11911
+ }
11912
+ else {
11913
+ // A reload FAILURE (not a mere readiness timeout — that returns true on
11914
+ // a still-live child) means this worker has NO confirmed serving
11915
+ // replacement: it crashed, failed to spawn, or was concurrently swapped.
11916
+ // Draining the NEXT worker on top of that gap breaks the one-at-a-time
11917
+ // guarantee AND could roll a broken replacement across the whole fleet.
11918
+ // Stop the roll here (a canary), marking this + every remaining id
11919
+ // skipped so the terminal frame reports a partial failure (#253 review).
11920
+ interrupted = true;
11921
+ for (let j = i; j < ids.length; j++) skipped.push(ids[j]);
11922
+ break;
11923
+ }
11924
+ try { sock.write(encodeFrame(statusFrame(false))); } catch { /* client gone */ }
11925
+ }
11926
+ } catch (err) {
11927
+ interrupted = true;
11928
+ dlog(`reload error: ${err?.message || err}`);
11929
+ } finally {
11930
+ reloading = false;
11931
+ // Terminal success requires BOTH a clean pass (not interrupted) AND nothing
11932
+ // skipped: a skipped worker (failed reload, or removed/absent mid-roll) is a
11933
+ // partial reload, so `ok: true` would let the streaming client exit zero and
11934
+ // hide it from automation (#253 review). Propagate the partial failure.
11935
+ const ok = !interrupted && skipped.length === 0;
11936
+ try { sock.write(encodeFrame({ ok, type: 'reloaded', reloaded, skipped, interrupted, final: true })); } catch { /* client gone */ }
11937
+ }
11938
+ };
11939
+
11940
+
11460
11941
  // Resolve a target token to worker ids: exact id, else all with that profile.
11461
11942
  const resolveTargets = (target) => {
11462
11943
  const t = String(target || '').trim();
@@ -11474,6 +11955,11 @@ async function runSupervisorDaemon() {
11474
11955
  ok: true,
11475
11956
  type: 'status',
11476
11957
  daemon: supervisorDaemonDescriptor({ pid: process.pid, startedAt, version: daemonVersion, socket: socketPath, logFile: daemonLogFile }),
11958
+ // The plugin version currently ON DISK (re-read each call), so `status` can
11959
+ // flag when a `nano update` has advanced the code past the running daemon —
11960
+ // i.e. a `supervisor reload` would adopt new worker code (and a daemon
11961
+ // restart new supervisor code). Best-effort; falls back to the daemon's own.
11962
+ pluginVersion: (() => { try { return pluginPackage().version; } catch { return daemonVersion; } })(),
11477
11963
  workers: pub || [...workers.values()].map(workerPublic),
11478
11964
  ...(final ? { final: true } : {}),
11479
11965
  });
@@ -11577,6 +12063,50 @@ async function runSupervisorDaemon() {
11577
12063
  sock.write(encodeFrame({ ok: true, type: 'restarted', restarted, final: true }));
11578
12064
  break;
11579
12065
  }
12066
+ case 'reload': {
12067
+ // Hot code adoption: rolling graceful drain+respawn so worker children
12068
+ // re-read the updated plugin from disk with zero fleet downtime.
12069
+ // Accepts a single `target` token (id|profile|all) or a `targets`
12070
+ // array (workforce passes its exact owned id list). Streams progress
12071
+ // and ends with a terminal `reloaded` frame — draining can take a long
12072
+ // time, so this MUST be a streaming op, not a one-shot request.
12073
+ if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
12074
+ if (reloading) { sock.write(encodeFrame({ ok: false, error: 'a reload is already in progress', final: true })); break; }
12075
+ // Reject reload explicitly on Windows BEFORE draining anything. The
12076
+ // rolling reload's graceful drain relies on SIGUSR2 (SUPERVISOR_DRAIN_SIGNAL)
12077
+ // to quiesce each worker child; Windows cannot deliver SIGUSR2 (Node maps
12078
+ // a non-zero signal there to a forceful, SIGKILL-like termination, and the
12079
+ // child's `process.once('SIGUSR2')` drain handler never fires), so a
12080
+ // "graceful" drain either hard-kills in-flight work or leaves `reloadWorker`
12081
+ // waiting forever for a child that was never asked to exit. Fail fast with
12082
+ // an actionable message instead of hanging the roll on the first worker
12083
+ // (#253 review). `restart`/`stop`+`start` remain the Windows path to adopt
12084
+ // new code.
12085
+ if (osPlatform() === 'win32') {
12086
+ sock.write(encodeFrame({ ok: false, error: 'hot reload is not supported on Windows (its graceful drain relies on SIGUSR2, which Windows cannot deliver) — use `supervisor restart <target>`, or `supervisor stop` + `start`, to adopt new code', final: true }));
12087
+ break;
12088
+ }
12089
+ const raw = Array.isArray(req.targets)
12090
+ ? req.targets.flatMap((t) => resolveTargets(t))
12091
+ : resolveTargets(req.target);
12092
+ const ids = [...new Set(raw)];
12093
+ if (ids.length === 0) { sock.write(encodeFrame({ ok: true, type: 'reloaded', reloaded: [], skipped: [], final: true })); break; }
12094
+ // Register as an attach consumer so the client also sees the
12095
+ // interleaved worker-reload/start events, then send an opening frame
12096
+ // and kick the rolling reload asynchronously (don't block the control
12097
+ // loop — a `stop`/`status` must still be serviceable meanwhile). Latch
12098
+ // `reloading` HERE, before scheduling: `runReload` only sets it once it
12099
+ // actually runs on a later tick, so a second `reload` socket arriving
12100
+ // in that window would otherwise still see `false` and start a duplicate
12101
+ // rolling pass over the same workers. The flag is reset in runReload's
12102
+ // `finally`.
12103
+ reloading = true;
12104
+ attachClients.add(sock);
12105
+ sock.write(encodeFrame({ ok: true, type: 'reloading', targets: ids }));
12106
+ sock.write(encodeFrame(statusFrame(false)));
12107
+ setTimeout(() => { runReload(ids, sock); }, 0);
12108
+ break;
12109
+ }
11580
12110
  case 'attach':
11581
12111
  attachClients.add(sock);
11582
12112
  sock.write(encodeFrame(statusFrame(false)));
@@ -11698,11 +12228,19 @@ async function runSupervisorDaemon() {
11698
12228
  return Number.isFinite(n) && n >= 0 ? Math.floor(n) : SUPERVISOR_MONITOR_INTERVAL_MS;
11699
12229
  })();
11700
12230
  if (monitorMs > 0) {
11701
- lastMonitorSig = supervisorStatusSignature([...workers.values()].map(workerPublic));
12231
+ // Fold the on-disk plugin version into the monitor signature so a `nano update`
12232
+ // that changes ONLY the on-disk package (no worker transition) still repaints
12233
+ // attached consoles with the new version + `supervisor reload` hint (#253
12234
+ // review). The worker-field signature alone never changes on a version-only
12235
+ // bump, so an idle fleet would otherwise hide an available reload until an
12236
+ // unrelated worker transition or a manual `status`.
12237
+ const monitorSignature = (pub) =>
12238
+ `${supervisorStatusSignature(pub)}\u0000${(() => { try { return pluginPackage().version; } catch { return daemonVersion; } })()}`;
12239
+ lastMonitorSig = monitorSignature([...workers.values()].map(workerPublic));
11702
12240
  monitorTimer = setInterval(() => {
11703
12241
  if (shuttingDown) return;
11704
12242
  const pub = [...workers.values()].map(workerPublic);
11705
- const sig = supervisorStatusSignature(pub);
12243
+ const sig = monitorSignature(pub);
11706
12244
  const changed = sig !== lastMonitorSig;
11707
12245
  lastMonitorSig = sig;
11708
12246
  if (changed && attachClients.size > 0) broadcast(statusFrame(false, pub));
@@ -11894,7 +12432,7 @@ async function supervisorStartCmd(req, flags, ctx) {
11894
12432
  await supervisorStatusCmd();
11895
12433
  logger.info('');
11896
12434
  logger.info('Attach an interactive console with: c8ctl nano supervisor');
11897
- logger.info('Manage without it: c8ctl nano supervisor add|remove|restart|status|stop');
12435
+ logger.info('Manage without it: c8ctl nano supervisor add|remove|restart|reload|status|stop');
11898
12436
  }
11899
12437
 
11900
12438
  async function supervisorStatusCmd() {
@@ -11987,6 +12525,97 @@ async function supervisorRestartCmd(req) {
11987
12525
  else { logger.error(res.error); process.exit(1); }
11988
12526
  }
11989
12527
 
12528
+ /**
12529
+ * Stream a rolling hot reload of the fleet and log its progress. Shared by
12530
+ * `supervisor reload` and `workforce reload`. Sends `req` (a `reload` op with a
12531
+ * `target` token or a `targets` array) and resolves with the outcome that ended
12532
+ * the stream ('reloaded' | 'detached' | 'closed' | 'unreachable'). Ctrl-C
12533
+ * DETACHES the client — the daemon keeps reloading in the background.
12534
+ */
12535
+ async function streamSupervisorReload(socketPath, req, logger, { label = 'fleet' } = {}) {
12536
+ return await new Promise((resolve) => {
12537
+ let sock = null;
12538
+ let buf = '';
12539
+ let done = false;
12540
+ let onSigint = null;
12541
+ const cleanup = () => {
12542
+ if (onSigint) { try { process.removeListener('SIGINT', onSigint); } catch { /* ignore */ } }
12543
+ try { if (sock) sock.end(); } catch { /* ignore */ }
12544
+ };
12545
+ const finish = (result) => { if (done) return; done = true; cleanup(); resolve(result); };
12546
+
12547
+ supervisorConnect(socketPath).then((s) => {
12548
+ sock = s;
12549
+ sock.setEncoding('utf8');
12550
+ onSigint = () => {
12551
+ logger.info('Detached — supervisor keeps reloading in the background. Rerun `nano supervisor status` to check progress.');
12552
+ finish('detached');
12553
+ };
12554
+ process.on('SIGINT', onSigint);
12555
+
12556
+ sock.on('data', (chunk) => {
12557
+ buf += chunk;
12558
+ const { frames, rest } = decodeFrames(buf);
12559
+ buf = rest;
12560
+ for (const frame of frames) {
12561
+ if (!frame) continue;
12562
+ // Handle the terminal `reloaded` frame BEFORE the generic `ok:false`
12563
+ // request-error guard: `runReload` emits a partial failure as a
12564
+ // terminal `{ type:'reloaded', final:true, ok:false, reloaded, skipped }`
12565
+ // frame, so the bare `ok === false` guard would swallow it as a generic
12566
+ // "reload failed" and hide which workers reloaded/skipped (#253 review).
12567
+ // We still exit non-zero for `ok:false` so automation sees the partial.
12568
+ // Match ONLY `type:'reloaded'`, never a bare `frame.final`: generic
12569
+ // terminal error frames (e.g. `{ok:false, error:'a reload is already in
12570
+ // progress', final:true}`) are also `final` but carry no reloaded/skipped
12571
+ // lists, so this branch would print "No workers were reloaded" and hide
12572
+ // `frame.error` — they must fall through to the `ok === false` guard below.
12573
+ if (frame.type === 'reloaded') {
12574
+ const reloaded = Array.isArray(frame.reloaded) ? frame.reloaded : [];
12575
+ const skipped = Array.isArray(frame.skipped) ? frame.skipped : [];
12576
+ if (reloaded.length > 0) logger.info(`Reloaded ${reloaded.length} worker(s): ${reloaded.join(', ')}.`);
12577
+ else logger.warn('No workers were reloaded.');
12578
+ if (skipped.length > 0) logger.warn(`Skipped (gone/changed, or roll aborted after a failed reload): ${skipped.join(', ')}.`);
12579
+ finish(frame.ok === false ? 'error' : 'reloaded');
12580
+ return;
12581
+ }
12582
+ if (frame.ok === false) { logger.error(frame.error || 'reload failed'); finish('error'); return; }
12583
+ if (frame.type === 'reloading') {
12584
+ const n = Array.isArray(frame.targets) ? frame.targets.length : 0;
12585
+ logger.info(`Reloading ${n} worker(s) in ${label} one at a time (draining in-flight jobs first). Press Ctrl-C to detach.`);
12586
+ } else if (frame.event === 'worker-reload' && frame.id) {
12587
+ logger.info(` reloaded "${frame.id}" (adopted new code).`);
12588
+ }
12589
+ }
12590
+ });
12591
+ sock.on('error', () => finish('closed'));
12592
+ sock.on('close', () => finish('closed'));
12593
+ sock.write(encodeFrame(req));
12594
+ }).catch(() => finish('unreachable'));
12595
+ });
12596
+ }
12597
+
12598
+ /**
12599
+ * Hot-adopt new plugin code into the running fleet with zero downtime: roll
12600
+ * through the target workers, gracefully draining (finish in-flight jobs) and
12601
+ * respawning each so the new child re-reads the updated `c8ctl-plugin.js` from
12602
+ * disk. Defaults to the whole fleet. Note this adopts new WORKER code only; the
12603
+ * supervisor daemon itself keeps running its startup code until a full restart
12604
+ * (`supervisor stop && supervisor start`).
12605
+ */
12606
+ async function supervisorReloadCmd(req) {
12607
+ const logger = getLogger();
12608
+ // Default to the whole fleet: "adopt new code" naturally means every worker.
12609
+ const target = req.positional[1] || 'all';
12610
+ const running = await liveSupervisor();
12611
+ if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
12612
+ const socketPath = running.socket || getSupervisorSocketPath();
12613
+ const outcome = await streamSupervisorReload(socketPath, { op: 'reload', target }, logger, { label: 'the fleet' });
12614
+ if (outcome === 'unreachable') { logger.error('Could not reach the supervisor control socket to reload it.'); process.exit(1); }
12615
+ if (outcome === 'closed') { logger.error('The supervisor closed the connection before the reload finished (daemon crash or concurrent stop?) — the roll may be incomplete. Rerun `nano supervisor status` to check the fleet.'); process.exit(1); }
12616
+ if (outcome === 'error') process.exit(1);
12617
+ }
12618
+
11990
12619
  /**
11991
12620
  * Count the in-flight jobs across a fleet snapshot (array of
11992
12621
  * `summarizeSupervisorWorker` results) — the number an operator is waiting on
@@ -12786,6 +13415,9 @@ async function supervisorCommand(req, flags, ctx) {
12786
13415
  case 'restart':
12787
13416
  await supervisorRestartCmd(req);
12788
13417
  return;
13418
+ case 'reload':
13419
+ await supervisorReloadCmd(req);
13420
+ return;
12789
13421
  case 'stop':
12790
13422
  await supervisorStopCmd(coerceBool(flags?.force, false));
12791
13423
  return;
@@ -12794,7 +13426,7 @@ async function supervisorCommand(req, flags, ctx) {
12794
13426
  supervisorLogsCmd(req);
12795
13427
  return;
12796
13428
  default:
12797
- getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|stop|logs|attach`);
13429
+ getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach`);
12798
13430
  process.exit(1);
12799
13431
  }
12800
13432
  }
@@ -13743,6 +14375,41 @@ async function workforceStopCmd(req, flags, manifestName) {
13743
14375
  if (hadError) process.exit(1);
13744
14376
  }
13745
14377
 
14378
+ /**
14379
+ * Hot-adopt new plugin code into a workforce's running workers with zero
14380
+ * downtime: roll through the manifest-owned workers, gracefully draining and
14381
+ * respawning each so the new child re-reads the updated plugin from disk.
14382
+ * Mirrors `workforce stop`'s ownership resolution (longest-prefix match + a
14383
+ * live-profile collision guard) so it only ever reloads workers this manifest
14384
+ * owns.
14385
+ */
14386
+ async function workforceReloadCmd(req, flags, manifestName) {
14387
+ const logger = getLogger();
14388
+ const running = await liveSupervisor();
14389
+ if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
14390
+ const manifestNames = listWorkforceManifestNames();
14391
+ const { running: stillRunning, reachable, workers: live } = await fetchSupervisorWorkers();
14392
+ if (!stillRunning) { logger.warn('Supervisor is not running — nothing to reload.'); return; }
14393
+ if (!reachable) {
14394
+ logger.error('Supervisor is running but its status socket is unreachable — cannot enumerate workers.');
14395
+ process.exit(1);
14396
+ }
14397
+ const owned = live
14398
+ .filter((w) => w && typeof w.id === 'string' && isWorkforceOwnedWorker(w.id, manifestName, manifestNames))
14399
+ .filter((w) => {
14400
+ const embedded = workforceProfileFromWorkerName(manifestName, w.id);
14401
+ if (embedded != null && w.profile != null && w.profile !== embedded) return false;
14402
+ return true;
14403
+ })
14404
+ .map((w) => w.id);
14405
+ if (owned.length === 0) { logger.info(`No workers from workforce "${manifestName}" are running.`); return; }
14406
+ const socketPath = running.socket || getSupervisorSocketPath();
14407
+ const outcome = await streamSupervisorReload(socketPath, { op: 'reload', targets: owned }, logger, { label: `workforce "${manifestName}"` });
14408
+ if (outcome === 'unreachable') { logger.error('Could not reach the supervisor control socket to reload it.'); process.exit(1); }
14409
+ if (outcome === 'closed') { logger.error('The supervisor closed the connection before the reload finished (daemon crash or concurrent stop?) — the roll may be incomplete. Rerun `nano supervisor status` to check the fleet.'); process.exit(1); }
14410
+ if (outcome === 'error') process.exit(1);
14411
+ }
14412
+
13746
14413
  async function workforceCommand(req, flags) {
13747
14414
  const logger = getLogger();
13748
14415
  const action = (req.positional[0] || '').toLowerCase();
@@ -13780,8 +14447,11 @@ async function workforceCommand(req, flags) {
13780
14447
  case 'down':
13781
14448
  await workforceStopCmd(req, flags, manifestName);
13782
14449
  return;
14450
+ case 'reload':
14451
+ await workforceReloadCmd(req, flags, manifestName);
14452
+ return;
13783
14453
  default:
13784
- logger.error(`Unknown workforce action "${action}". Use: add|remove|list|start|status|stop`);
14454
+ logger.error(`Unknown workforce action "${action}". Use: add|remove|list|start|status|stop|reload`);
13785
14455
  process.exit(1);
13786
14456
  }
13787
14457
  }
@@ -15464,6 +16134,7 @@ export {
15464
16134
  startAgenticChannelWatchdog,
15465
16135
  };
15466
16136
  export { compareSemver, githubRepoSlug, filterReleasesSince, renderReleaseBody };
16137
+ export { probeAgentCliVersion };
15467
16138
  export {
15468
16139
  webConsoleUrl,
15469
16140
  consoleLinkLabel,
@@ -15626,6 +16297,8 @@ export {
15626
16297
  agenticStateForTarget,
15627
16298
  normalizeAgenticMessage,
15628
16299
  buildActivityPayload,
16300
+ activityMarkerReadyFor,
16301
+ waitForChildExit,
15629
16302
  supervisorWorkerActivityFile,
15630
16303
  WORK_FORWARD_FLAGS,
15631
16304
  installParentDeathWatchdog,
@@ -15634,6 +16307,8 @@ export {
15634
16307
  supervisorRequest,
15635
16308
  supervisorStartCmd,
15636
16309
  supervisorAddCmd,
16310
+ supervisorReloadCmd,
16311
+ workforceReloadCmd,
15637
16312
  runningSupervisor,
15638
16313
  readSupervisorState,
15639
16314
  clearSupervisorState,
@@ -15740,6 +16415,7 @@ export const metadata = {
15740
16415
  { command: 'c8ctl nano supervisor add decider', description: 'Add a supervised worker (forwarding work flags) to the running supervisor' },
15741
16416
  { command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
15742
16417
  { command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
16418
+ { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
15743
16419
  { command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
15744
16420
  { command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
15745
16421
  { command: 'c8ctl nano workforce add qwen --instances 2 --roles pr-review,feature', description: "Add an entry mapped to explicit job types (<rank>:pr-review, <rank>:feature, where <rank> is the qwen hire's rank at start) — does not mutate the hired profile" },
@@ -15748,6 +16424,7 @@ export const metadata = {
15748
16424
  { command: 'c8ctl nano workforce status --json', description: 'Manifest entries joined against live supervisor status (desired vs actual), machine-readable for the install script / CI' },
15749
16425
  { command: 'c8ctl nano workforce list', description: 'Print the default manifest and list the manifests that exist on this machine' },
15750
16426
  { command: 'c8ctl nano workforce stop', description: "Remove this manifest's workers; stop the daemon too if no supervised workers remain" },
16427
+ { command: 'c8ctl nano workforce reload', description: "Hot-adopt updated code into this manifest's workers (rolling graceful drain+respawn, zero downtime)" },
15751
16428
  ],
15752
16429
  },
15753
16430
  processos: {
@@ -15979,8 +16656,8 @@ function printUsage() {
15979
16656
  console.log(' c8ctl nano hire [--name <n>] [--rank <r>] [--command <c>] [--arg <switch> ...] [--model <m>] [--capabilities <a,b>] [--sandbox none|docker|podman] [--image <ref>] [--terminal pty|pipe] [--protocol pipe|acp] [--permission yolo|escalate|filter] [--env NAME=VALUE ...] [--list]');
15980
16657
  console.log(' c8ctl nano assign <profileName> <cap[,cap...]> [--name <n>] [--capabilities <a,b>]');
15981
16658
  console.log(' c8ctl nano work <profileName> [--auto [--auto-scope <p>]] [--arg <switch> ...] [--recovery-window <ms>] [--idle-timeout <ms>] [--job-timeout <ms>] [--poll-timeout <ms>] [--job-type <token> ...] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--secret-resolver host] [--min-free-mb <n>] [--clone-timeout <ms>] [--keep-runs] [--stream]');
15982
- console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|stop|logs|attach] ... (manage many workers from one terminal)');
15983
- console.log(' c8ctl nano workforce [add|remove|list|start|status|stop] ... [--manifest <manifest>] (declarative, reusable fleet manifests)');
16659
+ console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach] ... (manage many workers from one terminal)');
16660
+ console.log(' c8ctl nano workforce [add|remove|list|start|status|stop|reload] ... [--manifest <manifest>] (declarative, reusable fleet manifests)');
15984
16661
  console.log('');
15985
16662
  console.log('Subcommands:');
15986
16663
  console.log(' start Spawn an N-node local cluster wired to talk to each other on localhost');
@@ -15999,7 +16676,7 @@ function printUsage() {
15999
16676
  console.log(' assign Grant new capabilities (roles) to an existing hire (additive; comma-separated; workers hot-reload)');
16000
16677
  console.log(' work Run a hired profile as Nano job workers, polling for work until Ctrl-C');
16001
16678
  console.log(' supervisor Run/manage a fleet of workers from one terminal (detachable console + non-interactive control)');
16002
- console.log(' workforce Compose a reusable, declarative fleet manifest and reconcile it up/down (add|remove|list|start|status|stop)');
16679
+ console.log(' workforce Compose a reusable, declarative fleet manifest and reconcile it up/down (add|remove|list|start|status|stop|reload)');
16003
16680
  console.log('');
16004
16681
  console.log('Options:');
16005
16682
  console.log(' <nodes> Number of nodes to start (default 1)');