c8ctl-plugin-nano 1.44.8 → 1.44.9

package/README.md

@@ -352,7 +352,7 @@ one of:
 | `starting` | transient: the worker just spawned and hasn't resolved its channel target yet (pre-`connecting`) |
 | `connected` | presence is live on the hub — you should see this worker in the Cockpit |
 | `connecting` | resolved a hub, socket not open yet (or the hub is unreachable) |
-| `disconnected` | an established channel dropped (hub restart/outage) — it auto-reconnects; also set if the channel failed to start (bad URL/refused socket), in which case it stays disconnected until the worker restarts |
+| `disconnected` | an established channel dropped (hub restart/outage) — it auto-reconnects, and a worker-side **liveness watchdog** force-re-discovers the hub if it stays down (see below); also set if the channel failed to start (bad URL/refused socket), which is **not** auto-recovered by the watchdog (it only guards a channel that has connected) — fix the target and restart the worker |
 | `advisory` | nothing discoverable at the engine — **not** in the Cockpit; set `NANO_AGENTIC_URL` |
 | `off` | visibility disabled (`NANO_AGENTIC=off`) |
 | `?` | a live worker not yet reporting, or an older build predating these fields |
@@ -361,6 +361,22 @@ If workers show `advisory` (or stay `connecting`) while jobs still run, that's t
 "connected to the engine but empty Cockpit" case: point them at the app with
 `export NANO_AGENTIC_URL=http://<engine-host>:<appUi.port>` (e.g. `:3000`).
 
+**Liveness watchdog (auto-recovery from a wedged channel).** If the nano server
+restarts, crashes, or a network partition drops the connection *without* a clean
+close (a **half-open** socket), a worker's channel client can sit `disconnected`
+forever — the worker vanishes from the Nano Workers view / Cockpit and, before
+this, only a supervisor restart brought it back. Each worker now runs a
+belt-and-suspenders watchdog: once a channel that had connected stays down past a
+threshold (the client library's own reconnect never recovered it), the worker
+tears the wedged channel down and re-runs full hub discovery + reopen — no restart
+needed. The thresholds are tunable via env (sensible defaults; you rarely need
+these):
+
+```bash
+export NANO_AGENTIC_STALE_MS=60000     # force re-discovery if a drop hasn't recovered within 60s (default)
+export NANO_AGENTIC_WATCHDOG_MS=15000  # how often the watchdog checks channel liveness (default)
+```
+
 **Secure mode (opt-in).** For a deployment where you want the visibility channel
 authenticated (rather than open on the LAN), start the server **and** every worker
 box with the **same** `NANO_AGENTIC_SECRET` — same env-var name, same value on both

package/c8ctl-plugin.js

@@ -6089,6 +6089,138 @@ async function rediscoverAgenticUntilConnected({
   return null;
 }
 
+// Worker-side liveness watchdog defaults (jwulf/c8ctl-plugin-nano#144). A
+// previously-connected agentic channel that has been `disconnected` for longer
+// than the stale threshold — because the client lib's own reconnect never
+// brought it back (e.g. a half-open drop after a server restart/crash/partition,
+// or a reconnect that keeps failing) — is force-healed: the wedged channel is
+// torn down and full discovery + reopen is re-armed, instead of trusting the
+// client lib alone. Overridable via NANO_AGENTIC_STALE_MS / NANO_AGENTIC_WATCHDOG_MS.
+const DEFAULT_AGENTIC_STALE_MS = 60_000;
+const DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS = 15_000;
+
+/**
+ * Decide whether a worker's agentic channel is *stale* — i.e. it once connected,
+ * is no longer connected, and has stayed down past `staleAfterMs`. Pure so the
+ * watchdog's trigger condition is unit-testable without timers or sockets. A
+ * channel that never opened (`everConnected() === false`) is NOT stale — it is
+ * still doing its first connect, which the initial open / cold-start self-heal
+ * owns. `disconnectedSince` is null whenever the channel is up (or never went
+ * down), which also reads as not-stale.
+ *
+ * @param {{
+ *   connected: () => boolean,
+ *   everConnected: () => boolean,
+ *   disconnectedSince: () => (number|null),
+ *   now?: () => number,
+ *   staleAfterMs?: number,
+ * }} opts
+ * @returns {boolean}
+ */
+function agenticChannelIsStale({
+  connected,
+  everConnected,
+  disconnectedSince,
+  now = () => Date.now(),
+  staleAfterMs = DEFAULT_AGENTIC_STALE_MS,
+}) {
+  if (typeof connected !== 'function' || typeof everConnected !== 'function') return false;
+  if (!everConnected()) return false; // never opened → the initial connect owns it
+  if (connected()) return false; // healthy
+  const since = typeof disconnectedSince === 'function' ? disconnectedSince() : null;
+  if (since == null) return false; // no recorded drop → nothing to heal
+  return now() - since >= staleAfterMs;
+}
+
+/**
+ * Start the worker-side agentic-channel liveness watchdog (#144). On a fixed
+ * interval it asks {@link agenticChannelIsStale} whether the channel dropped and
+ * never recovered within the threshold; when it has, it fires `onStale()` (which
+ * tears the wedged channel down and re-runs discovery + reopen). It fires
+ * `onStale` **exactly once per stale episode** — a per-episode latch is re-armed
+ * only when the channel is next observed healthy (or gone), or when a heal
+ * throws (a failed recovery retries on the next tick), so a persistent stale
+ * condition with a successful heal does not retrigger the heal on every tick. Re-entrancy is
+ * guarded so a slow heal never overlaps a later tick. Timers, clock, and the
+ * channel accessors are injectable so this is unit-testable without real waits.
+ * Returns a `{ stop, tick }` handle — `stop()` clears the timer AND latches the
+ * watchdog stopped so any tick already scheduled or in flight around shutdown
+ * becomes a no-op before it can reach `onStale` (shutdown relies on this to
+ * prevent a stale-channel resurrection mid-teardown), and `tick()` runs a single
+ * check (tests drive it directly).
+ *
+ * @param {{
+ *   getChannel: () => (import('./work-channel.mjs').WorkChannel | null),
+ *   disconnectedSince: () => (number|null),
+ *   onStale: () => (void|Promise<void>),
+ *   staleAfterMs?: number,
+ *   intervalMs?: number,
+ *   now?: () => number,
+ *   setIntervalFn?: typeof setInterval,
+ *   clearIntervalFn?: typeof clearInterval,
+ *   logger?: object|null,
+ * }} opts
+ * @returns {{ stop: () => void, tick: () => Promise<void> }}
+ */
+function startAgenticChannelWatchdog({
+  getChannel,
+  disconnectedSince,
+  onStale,
+  staleAfterMs = DEFAULT_AGENTIC_STALE_MS,
+  intervalMs = DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS,
+  now = () => Date.now(),
+  setIntervalFn = setInterval,
+  clearIntervalFn = clearInterval,
+  logger = null,
+} = {}) {
+  let healing = false;
+  // Set once `stop()` runs so any tick already scheduled/in flight around
+  // shutdown becomes a no-op and can never re-open the channel mid-teardown
+  // (shutdown relies on `stop()` to prevent a stale-channel resurrection).
+  let stopped = false;
+  // Per-episode latch: fire `onStale` exactly once when a connected channel goes
+  // stale, and don't fire again until it recovers (a fresh episode). Without this
+  // the helper would re-heal on every tick whenever `onStale` does not itself
+  // clear the staleness signal, causing repeated teardown/re-discovery attempts.
+  let firedForEpisode = false;
+  const tick = async () => {
+    if (stopped || healing) return; // shutting down, or a heal is in flight — don't stack a second re-discovery
+    const ch = typeof getChannel === 'function' ? getChannel() : null;
+    // No channel object → the initial open or the cold-start self-heal loop owns
+    // recovery; the watchdog only guards a channel that HAS connected and stalled.
+    // A missing or non-stale (healthy / recovered / still-connecting) channel also
+    // ends any current stale episode, so re-arm the latch for the next one.
+    if (!ch || !agenticChannelIsStale({
+      connected: () => ch.connected(),
+      everConnected: () => ch.everConnected(),
+      disconnectedSince,
+      now,
+      staleAfterMs,
+    })) {
+      firedForEpisode = false;
+      return;
+    }
+    if (firedForEpisode) return; // already fired once for this stale episode
+    healing = true;
+    firedForEpisode = true;
+    try {
+      if (stopped) return; // shutdown raced us between the checks — do not heal
+      const since = disconnectedSince();
+      const downFor = since != null ? Math.round((now() - since) / 1000) : '?';
+      logger?.warn?.(`  agentic channel: no reconnect ${downFor}s after drop — forcing re-discovery (the client lib did not self-heal; likely a half-open drop).`);
+      await onStale?.();
+    } catch (err) {
+      firedForEpisode = false; // heal failed → re-arm so a later tick retries this episode
+      logger?.debug?.(`agentic watchdog heal failed: ${err?.message || err}`);
+    } finally {
+      healing = false;
+    }
+  };
+  const timer = setIntervalFn(() => { tick().catch(() => {}); }, intervalMs);
+  if (timer && typeof timer.unref === 'function') timer.unref();
+  return { stop: () => { stopped = true; try { clearIntervalFn(timer); } catch { /* best effort */ } }, tick };
+}
+
 /**
  * Collapse an agentic disconnect/failure detail into the single short string the
  * marker's `agentic.message` field carries (#99 contract). Accepts the close
@@ -6587,6 +6719,17 @@ async function workAgent(req, flags) {
   let workChannel = null;
   /** @type {import('./work-buffer.mjs').BufferMonitor | null} */
   let bufferMonitor = null;
+  // #144 liveness watchdog state. `agenticDisconnectedSince` is the epoch-ms the
+  // channel last dropped (null whenever it is up or has never opened); the
+  // watchdog uses it to force a full re-discovery + reopen when the client lib's
+  // own reconnect fails to bring a previously-connected channel back within the
+  // stale threshold. `agenticWatchdog` is the running timer handle (stopped on
+  // shutdown); `agenticSelfHealing` guards against two concurrent re-discovery
+  // loops (the cold-start one and a watchdog-triggered one).
+  let agenticDisconnectedSince = null;
+  /** @type {{ stop: () => void } | null} */
+  let agenticWatchdog = null;
+  let agenticSelfHealing = false;
   // Maintain `activeJobs` unconditionally: it feeds both the supervisor activity
   // file (gated inside writeActivity) AND the agentic presence frame's live
   // jobKey set, so a standalone worker (no NANO_SUPERVISOR_ACTIVITY_FILE) still
@@ -6696,11 +6839,21 @@ async function workAgent(req, flags) {
       // (before these listeners existed), connected() is false but everConnected()
       // is true — record that as `disconnected` rather than leaving it stuck at
       // `connecting`.
-      workChannel.onConnect(() => markAgentic('connected'));
-      workChannel.onReconnect(() => markAgentic('connected'));
-      workChannel.onDisconnect((info) => markAgentic('disconnected', normalizeAgenticMessage(info)));
-      if (workChannel.connected()) markAgentic('connected');
-      else if (workChannel.everConnected()) markAgentic('disconnected');
+      // #144: track the drop clock alongside presence — a (re)connect clears it,
+      // a disconnect starts it (first drop wins, so the watchdog measures from the
+      // ORIGINAL drop, not the latest of a reconnect storm). The watchdog reads
+      // this to decide when the client lib has failed to self-heal.
+      workChannel.onConnect(() => { markAgentic('connected'); agenticDisconnectedSince = null; });
+      workChannel.onReconnect(() => { markAgentic('connected'); agenticDisconnectedSince = null; });
+      workChannel.onDisconnect((info) => {
+        markAgentic('disconnected', normalizeAgenticMessage(info));
+        if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
+      });
+      if (workChannel.connected()) { markAgentic('connected'); agenticDisconnectedSince = null; }
+      else if (workChannel.everConnected()) {
+        markAgentic('disconnected');
+        if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
+      }
     } catch (err) {
       // Never let a channel failure stop the worker from doing its actual job.
       workChannel = null;
@@ -6732,16 +6885,16 @@ async function workAgent(req, flags) {
     }
   };
 
-  if (agenticCfg) {
-    await openAgenticChannel(agenticCfg);
-  } else if (agenticTarget.status === 'advisory') {
-    // (A) Self-heal a cold-start discovery miss (#133): discovery is one-shot at
-    // enrolment, so a worker that merely lost the cold-start race (e.g. a slow
-    // link-local candidate blew the budget) would otherwise run `advisory` for
-    // its whole lifetime — the only recovery being a restart. Keep re-discovering
-    // in the background on a jittered backoff and, on a later success, upgrade
-    // advisory→connected WITHOUT a restart, flipping the AGENTIC status surface.
-    // A shared cache lets a brief blip reuse the last known-good hub (#133-C).
+  // (A) Background self-heal loop, shared by the cold-start advisory path (#133)
+  // AND the #144 liveness watchdog. Re-run discovery on a jittered backoff and,
+  // on the first `connect` target, (re)open the channel WITHOUT a restart. The
+  // `agenticSelfHealing` guard makes it idempotent: the watchdog can call it
+  // after tearing a stale channel down without racing a still-running cold-start
+  // loop. A shared cache lets a brief blip reuse the last known-good hub (#133-C).
+  const armAgenticSelfHeal = () => {
+    if (agenticSelfHealing) return; // a re-discovery loop is already running
+    if (workChannel !== null) return; // a channel already exists — nothing to heal
+    agenticSelfHealing = true;
     const hubCache = new Map();
     rediscoverAgenticUntilConnected({
       resolveTarget: () => resolveAgenticTarget({ camunda, logger, cache: hubCache }),
@@ -6749,7 +6902,7 @@ async function workAgent(req, flags) {
         agenticCfg = target.config;
         agenticState = agenticStateForTarget(target, safeAgenticDisplayUrl);
         writeActivity();
-        logger.info('  agentic channel: background re-discovery succeeded — upgrading advisory → connecting.');
+        logger.info('  agentic channel: background re-discovery succeeded — (re)opening channel.');
         await openAgenticChannel(agenticCfg);
         // openAgenticChannel swallows its own open failures (it nulls
         // workChannel and returns rather than throwing), so a failed open must
@@ -6762,7 +6915,57 @@ async function workAgent(req, flags) {
       // Stop as soon as a channel exists (loop won this or a prior attempt did).
       shouldContinue: () => workChannel === null,
       logger,
-    }).catch(() => { /* best-effort self-heal — never surfaces an error */ });
+    })
+      .catch(() => { /* best-effort self-heal — never surfaces an error */ })
+      .finally(() => { agenticSelfHealing = false; });
+  };
+
+  // (B) #144 liveness watchdog: force-heal a wedged channel. When a channel that
+  // HAS connected drops and the client lib's own reconnect never brings it back
+  // within the stale threshold (a half-open drop after a server restart/crash/
+  // partition, or a reconnect that keeps failing), the client sits `disconnected`
+  // forever and the worker vanishes from the Workers view until a supervisor
+  // restart. This tears the wedged channel down (so `shouldContinue` re-arms) and
+  // re-runs full discovery + reopen instead of trusting the client lib alone.
+  const healStaleAgenticChannel = async () => {
+    const stale = workChannel;
+    if (!stale) return;
+    workChannel = null; // re-arms armAgenticSelfHeal()'s shouldContinue gate
+    agenticDisconnectedSince = null; // reset the clock; the fresh open restarts it
+    try { bufferMonitor?.stop(); } catch { /* best effort */ }
+    bufferMonitor = null;
+    markAgentic('disconnected', 'stale channel — re-discovering hub');
+    // Deregister + close the wedged client so it stops its own doomed reconnect
+    // attempts and we don't leak two clients once the fresh one connects.
+    try { await stale.stop('stale channel — re-discovering'); } catch { /* best effort */ }
+    armAgenticSelfHeal();
+  };
+
+  const startAgenticWatchdog = () => {
+    if (agenticWatchdog) return;
+    const staleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_STALE_MS, DEFAULT_AGENTIC_STALE_MS));
+    const intervalMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_WATCHDOG_MS, DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS));
+    agenticWatchdog = startAgenticChannelWatchdog({
+      getChannel: () => workChannel,
+      disconnectedSince: () => agenticDisconnectedSince,
+      onStale: healStaleAgenticChannel,
+      staleAfterMs,
+      intervalMs,
+      logger,
+    });
+  };
+
+  if (agenticCfg) {
+    await openAgenticChannel(agenticCfg);
+    // Guard the connected channel: if it later drops and the client lib can't
+    // recover it, the watchdog forces a full re-discovery + reopen (#144).
+    startAgenticWatchdog();
+  } else if (agenticTarget.status === 'advisory') {
+    // A cold-start discovery miss leaves the worker `advisory`; the self-heal
+    // loop upgrades it to `connected` without a restart (#133), and once a
+    // channel exists the watchdog keeps it alive across later drops (#144).
+    armAgenticSelfHeal();
+    startAgenticWatchdog();
   }
 
   // C3 (#42): the role's live-terminal mode — a full PTY (streamed on the relay
@@ -7382,6 +7585,9 @@ async function workAgent(req, flags) {
       logger.info(`Received ${signal} — stopping ${list.length} worker(s)...`);
       if (reaperTimer) clearInterval(reaperTimer);
       if (runDirTimer) clearInterval(runDirTimer);
+      // Stop the #144 liveness watchdog so it can't kick off a re-discovery
+      // mid-teardown (which would resurrect the channel we're about to close).
+      if (agenticWatchdog) { try { agenticWatchdog.stop(); } catch { /* best effort */ } agenticWatchdog = null; }
       const results = await Promise.all(list.map(drainWorker));
       const stopFailures = results.filter((ok) => !ok).length;
       if (stopFailures > 0) {
@@ -11751,6 +11957,8 @@ export {
   isLinkLocalAddress,
   rediscoverAgenticUntilConnected,
   defaultAgenticRediscoveryDelays,
+  agenticChannelIsStale,
+  startAgenticChannelWatchdog,
 };
 export { compareSemver, githubRepoSlug, filterReleasesSince, renderReleaseBody };
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "c8ctl-plugin-nano",
-  "version": "1.44.8",
+  "version": "1.44.9",
   "type": "module",
   "description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
   "main": "c8ctl-plugin.js",
@@ -57,12 +57,12 @@
   },
   "optionalDependencies": {
     "node-pty": "^1.0.0",
-    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.44.8"
+    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.44.9",
+    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.44.9",
+    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.44.9",
+    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.44.9",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.44.9",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.44.9",
+    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.44.9"
   }
 }

package/work-channel.mjs

@@ -281,15 +281,25 @@ export async function createWorkChannel(opts) {
     everConnected: () => hasConnected,
     buffered: () => client.buffered,
     async stop(reason = 'worker stopped') {
+      // Deregister to drop presence cleanly, then ALWAYS close the socket so the
+      // client stops its own reconnect loop. Closing only on a deregister error
+      // (the old behaviour) left a successfully-deregistered client half-open and
+      // still reconnecting — which is exactly the wedged/duplicate-client case the
+      // #144 stale-channel heal path relies on stop() to end.
+      // deregister() is fire-and-forget but may be thenable (register() is
+      // treated as one above), so guard BOTH a synchronous throw and an async
+      // rejection — an unhandled rejection during shutdown must never escape.
       try {
-        client.deregister(reason);
-      } catch (err) {
-        try {
+        Promise.resolve(client.deregister(reason)).catch((err) => {
           log.warn?.(`agentic deregister failed: ${err?.message || err}`);
-          client.close();
-        } catch {
-          /* best effort — never let shutdown hang on the channel */
-        }
+        });
+      } catch (err) {
+        log.warn?.(`agentic deregister failed: ${err?.message || err}`);
+      }
+      try {
+        client.close();
+      } catch {
+        /* best effort — never let shutdown hang on the channel */
       }
     },
   };