c8ctl-plugin-nano 1.44.10 → 1.44.12

package/README.md

@@ -377,6 +377,28 @@ export NANO_AGENTIC_STALE_MS=60000     # force re-discovery if a drop hasn't rec
 export NANO_AGENTIC_WATCHDOG_MS=15000  # how often the watchdog checks channel liveness (default)
 ```
 
+**Lossy-link hardening (reconnect churn that never re-lands presence).** A
+distinct failure mode shows up on a **lossy/roaming WiFi (or NAT) link**: the drop
+*is* detected (an abnormal closure, code `1006`), the client *does* reconnect and
+re-announce — yet presence never re-lands on the hub, so the worker stays absent
+even though its log shows it re-announcing. The sustained-drop watchdog above does
+not catch this, because each brief reconnect keeps resetting its drop clock. Two
+extra safeguards close the gap:
+
+- **Presence-keyed watchdog trigger.** A reconnect only counts as recovered once
+  the socket *holds* for a short grace window (so a `1006` blip that immediately
+  re-drops does not mask an unrecovered presence). When presence has not been
+  confirmed within a threshold — regardless of the reconnect flapping — the
+  watchdog forces the same full re-discovery + reopen.
+- **Jittered reconnect backoff.** Reconnect attempts are spread with equal-jitter
+  backoff so a fleet dropped on the same link does not reconnect in lockstep and
+  re-congest it.
+
+```bash
+export NANO_AGENTIC_PRESENCE_STALE_MS=60000  # force re-discovery if presence isn't re-confirmed within 60s (default)
+export NANO_AGENTIC_PRESENCE_GRACE_MS=5000   # how long a reconnect must hold before presence counts as landed (default)
+```
+
 **Secure mode (opt-in).** For a deployment where you want the visibility channel
 authenticated (rather than open on the LAN), start the server **and** every worker
 box with the **same** `NANO_AGENTIC_SECRET` — same env-var name, same value on both

package/c8ctl-plugin.js

@@ -53,6 +53,7 @@ import {
 import { createConnection, createServer } from 'node:net';
 import * as nodeNet from 'node:net';
 import { lookup as dnsLookup } from 'node:dns/promises';
+import * as nodeDns from 'node:dns';
 import { randomUUID, createHash, randomBytes } from 'node:crypto';
 import { homedir, platform as osPlatform, devNull, tmpdir, hostname } from 'node:os';
 import { join, isAbsolute, resolve as resolvePath, dirname, basename, sep } from 'node:path';
@@ -5463,7 +5464,11 @@ function runAgentJob(profile, job, opts = {}) {
       return Promise.resolve({ ok: false, exitCode: null, stdout: '', stderr: '', error: 'command-line args (--arg) are not supported for host execution on Windows; use a container sandbox or bake switches into the command', truncated: false, stderrTruncated: false });
     }
     const resultEnv = resultFile ? { [AGENT_RESULT_FILE_ENV]: resultFile } : {};
-    const harnessEnv = { ...process.env, ...staticEnv, ...secretEnv, ...agentEnv, ...extraEnv, ...resultEnv };
+    // Propagate IPv4-first DNS ordering into the forked agent via NODE_OPTIONS
+    // (jwulf/c8ctl-plugin-nano#151): the parent's process-wide setDefaultResultOrder
+    // can't reach the child, so its own Node runtime must read the flag at startup.
+    // Merges (never clobbers) any inherited/operator NODE_OPTIONS.
+    const harnessEnv = withIpv4FirstNodeOptions({ ...process.env, ...staticEnv, ...secretEnv, ...agentEnv, ...extraEnv, ...resultEnv });
 
     // A role opted into ACP (`protocol: acp`) drives its harness over the Agent
     // Client Protocol (JSON-RPC 2.0 over stdio) instead of the stdin/scrape pipe
@@ -5566,6 +5571,16 @@ function runAgentJob(profile, job, opts = {}) {
   for (const n of passThroughSecretNames) envArgs.push('-e', n);
   for (const k of Object.keys(staticEnv)) envArgs.push('-e', k);
 
+  // Propagate IPv4-first DNS ordering into the containerised agent's Node runtime
+  // via NODE_OPTIONS (jwulf/c8ctl-plugin-nano#151), merged (never clobbered) with
+  // any inherited/operator value. Forwarded by NAME like every other env so the
+  // value stays out of argv/`docker inspect`; `-e NODE_OPTIONS` is added only when
+  // it wasn't already forwarded via the static env above (avoid a duplicate flag).
+  const containerEnv = withIpv4FirstNodeOptions({ ...process.env, ...staticEnv, ...secretEnv, ...agentEnv, ...extraEnv, ...resultEnv });
+  if (!Object.prototype.hasOwnProperty.call(staticEnv, 'NODE_OPTIONS')) {
+    envArgs.push('-e', 'NODE_OPTIONS');
+  }
+
   const args = [
     'run', '--rm', '-i',
     '--name', containerName,
@@ -5588,7 +5603,7 @@ function runAgentJob(profile, job, opts = {}) {
     // Reserved harness env (AGENT_* + the result-file path) is layered AFTER
     // resolved secrets so a task-supplied secret NAME can never shadow it. In
     // container mode docker reads these values from our child env by NAME.
-    env: { ...process.env, ...staticEnv, ...secretEnv, ...agentEnv, ...extraEnv, ...resultEnv },
+    env: containerEnv,
     stdinData: payload,
     timeoutMs,
     idleTimeoutMs,
@@ -6259,6 +6274,20 @@ async function rediscoverAgenticUntilConnected({
 // client lib alone. Overridable via NANO_AGENTIC_STALE_MS / NANO_AGENTIC_WATCHDOG_MS.
 const DEFAULT_AGENTIC_STALE_MS = 60_000;
 const DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS = 15_000;
+// #147 presence-keyed staleness. #144's stale trigger keys on a SUSTAINED socket
+// drop (`disconnectedSince` ages past the threshold). On a lossy WiFi/NAT link
+// that drops `1006` repeatedly, the client lib reconnects and re-announces, but
+// each brief reconnect clears the drop clock — so the sustained-drop trigger
+// never fires even though presence never actually re-lands on the hub (the
+// worker stays absent from the Workers view). The presence-keyed trigger closes
+// that gap: a connection only counts as "presence confirmed" once it HOLDS for
+// the grace window, so a reconnect that immediately re-drops cannot mask an
+// unrecovered presence. When presence has not been confirmed within the stale
+// threshold — regardless of transient reconnect flapping — the watchdog forces a
+// full re-discovery + reopen. Overridable via NANO_AGENTIC_PRESENCE_STALE_MS /
+// NANO_AGENTIC_PRESENCE_GRACE_MS.
+const DEFAULT_AGENTIC_PRESENCE_STALE_MS = DEFAULT_AGENTIC_STALE_MS;
+const DEFAULT_AGENTIC_PRESENCE_GRACE_MS = 5_000;
 
 /**
  * Decide whether a worker's agentic channel is *stale* — i.e. it once connected,
@@ -6293,6 +6322,78 @@ function agenticChannelIsStale({
   return now() - since >= staleAfterMs;
 }
 
+/**
+ * Decide whether a worker's agentic channel is *presence-stale* (#147) — i.e. it
+ * once connected but its presence has NOT been re-confirmed on the hub within
+ * `staleAfterMs`, even though the socket may be intermittently reconnecting. This
+ * is the churn counterpart to {@link agenticChannelIsStale}: on a lossy link the
+ * client lib reconnects and re-announces after every `1006`, which keeps clearing
+ * the sustained-drop clock, so the socket-keyed trigger never fires — yet
+ * presence never actually lands. `presenceHealthySince` is the epoch-ms of the
+ * last time the channel was observed *stably* connected long enough for presence
+ * to be considered landed. The first open seeds it immediately (that open drains
+ * the buffered REGISTER, so presence lands with it); thereafter it advances only
+ * once a connection has *held* past a grace window (via `onPresenceHealthy`), so a
+ * reconnect that immediately re-drops does not count. Pure so the trigger is
+ * unit-testable
+ * without timers or sockets. A channel that never opened is not presence-stale
+ * (the initial connect owns it); a null `presenceHealthySince` (never confirmed,
+ * or reset by a heal) also reads as not-stale so a fresh open is given its grace.
+ *
+ * @param {{
+ *   everConnected: () => boolean,
+ *   presenceHealthySince?: () => (number|null),
+ *   now?: () => number,
+ *   staleAfterMs?: number,
+ * }} opts
+ * @returns {boolean}
+ */
+function agenticPresenceIsStale({
+  everConnected,
+  presenceHealthySince,
+  now = () => Date.now(),
+  staleAfterMs = DEFAULT_AGENTIC_PRESENCE_STALE_MS,
+}) {
+  if (typeof everConnected !== 'function') return false;
+  if (!everConnected()) return false; // never opened → the initial connect owns it
+  const since = typeof presenceHealthySince === 'function' ? presenceHealthySince() : null;
+  if (since == null) return false; // never confirmed / freshly reset → give the open its grace
+  return now() - since >= staleAfterMs;
+}
+
+/**
+ * Equal-jitter backoff (#147). Given a base backoff `baseMs`, keep half of it and
+ * randomise the other half: `baseMs/2 + rand()*baseMs/2`. A fleet of workers that
+ * all dropped on the same lossy link would otherwise reconnect in lockstep (the
+ * client lib's exponential backoff is deterministic), re-congesting the link and
+ * re-triggering the `1006` drops that stranded them. Spreading the reconnect
+ * attempts bounds that thundering-herd churn while preserving the exponential
+ * growth of the underlying policy. Pure (`rand` injectable) so it is testable.
+ *
+ * @param {number} baseMs the deterministic backoff the client lib computed
+ * @param {{ rand?: () => number }} [opts]
+ * @returns {number} the jittered delay in ms (0 when baseMs is non-positive)
+ */
+function jitteredDelay(baseMs, { rand = Math.random } = {}) {
+  const b = Number.isFinite(baseMs) && baseMs > 0 ? baseMs : 0;
+  if (b === 0) return 0;
+  const half = b / 2;
+  return Math.round(half + rand() * half);
+}
+
+/**
+ * Build a reconnect `schedule` function that applies {@link jitteredDelay} to the
+ * backoff the client lib passes, so the worker's reconnect attempts on a lossy
+ * link are de-synchronised (#147). Drops straight into `createWorkChannel`'s
+ * injectable `schedule` seam; the timer is injectable for tests.
+ *
+ * @param {{ rand?: () => number, timer?: (fn: () => void, ms: number) => any }} [opts]
+ * @returns {(fn: () => void, ms: number) => void}
+ */
+function makeJitteredReconnectSchedule({ rand = Math.random, timer = setTimeout } = {}) {
+  return (fn, ms) => { timer(fn, jitteredDelay(ms, { rand })); };
+}
+
 /**
  * Start the worker-side agentic-channel liveness watchdog (#144). On a fixed
  * interval it asks {@link agenticChannelIsStale} whether the channel dropped and
@@ -6310,9 +6411,23 @@ function agenticChannelIsStale({
  * prevent a stale-channel resurrection mid-teardown), and `tick()` runs a single
  * check (tests drive it directly).
  *
+ * #147 adds a second, presence-keyed trigger alongside the #144 sustained-drop
+ * one: each tick, a channel observed *stably* connected (connected for at least
+ * `presenceGraceMs` since `connectedSince()`) advances the presence-health clock
+ * via `onPresenceHealthy()`; when presence has not been confirmed within
+ * `presenceStaleAfterMs` — even while the socket flaps `1006` reconnects — the
+ * watchdog heals just as it does for a sustained drop. The presence accessors are
+ * optional: omitting them leaves the #147 trigger inert, so the #144 behaviour is
+ * unchanged.
+ *
  * @param {{
  *   getChannel: () => (import('./work-channel.mjs').WorkChannel | null),
  *   disconnectedSince: () => (number|null),
+ *   connectedSince?: () => (number|null),
+ *   presenceHealthySince?: () => (number|null),
+ *   onPresenceHealthy?: () => void,
+ *   presenceGraceMs?: number,
+ *   presenceStaleAfterMs?: number,
  *   onStale: () => (void|Promise<void>),
  *   staleAfterMs?: number,
  *   intervalMs?: number,
@@ -6327,6 +6442,11 @@ function startAgenticChannelWatchdog({
   getChannel,
   disconnectedSince,
   onStale,
+  connectedSince = null,
+  presenceHealthySince = null,
+  onPresenceHealthy = null,
+  presenceGraceMs = DEFAULT_AGENTIC_PRESENCE_GRACE_MS,
+  presenceStaleAfterMs = DEFAULT_AGENTIC_PRESENCE_STALE_MS,
   staleAfterMs = DEFAULT_AGENTIC_STALE_MS,
   intervalMs = DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS,
   now = () => Date.now(),
@@ -6347,17 +6467,39 @@ function startAgenticChannelWatchdog({
   const tick = async () => {
     if (stopped || healing) return; // shutting down, or a heal is in flight — don't stack a second re-discovery
     const ch = typeof getChannel === 'function' ? getChannel() : null;
+    // #147: advance the presence-health clock when the channel is observed
+    // STABLY connected (held for the grace window). A churny link that reconnects
+    // but immediately re-drops never accrues the grace, so its health clock ages
+    // out and the presence-keyed trigger below fires — unlike the sustained-drop
+    // trigger, which the churn keeps resetting. Inert unless the accessors are wired.
+    if (ch && typeof onPresenceHealthy === 'function' && typeof connectedSince === 'function') {
+      try {
+        const cs = connectedSince();
+        if (ch.connected() && cs != null && now() - cs >= presenceGraceMs) onPresenceHealthy();
+      } catch { /* best effort — never let a health probe break the tick */ }
+    }
+    // The channel is stale when EITHER trigger fires: #144's sustained socket
+    // drop, or #147's presence-not-confirmed-despite-reconnect-churn.
+    const stale = !!ch && (
+      agenticChannelIsStale({
+        connected: () => ch.connected(),
+        everConnected: () => ch.everConnected(),
+        disconnectedSince,
+        now,
+        staleAfterMs,
+      })
+      || agenticPresenceIsStale({
+        everConnected: () => ch.everConnected(),
+        presenceHealthySince,
+        now,
+        staleAfterMs: presenceStaleAfterMs,
+      })
+    );
     // No channel object → the initial open or the cold-start self-heal loop owns
     // recovery; the watchdog only guards a channel that HAS connected and stalled.
     // A missing or non-stale (healthy / recovered / still-connecting) channel also
     // ends any current stale episode, so re-arm the latch for the next one.
-    if (!ch || !agenticChannelIsStale({
-      connected: () => ch.connected(),
-      everConnected: () => ch.everConnected(),
-      disconnectedSince,
-      now,
-      staleAfterMs,
-    })) {
+    if (!stale) {
       firedForEpisode = false;
       return;
     }
@@ -6366,9 +6508,16 @@ function startAgenticChannelWatchdog({
     firedForEpisode = true;
     try {
       if (stopped) return; // shutdown raced us between the checks — do not heal
-      const since = disconnectedSince();
-      const downFor = since != null ? Math.round((now() - since) / 1000) : '?';
-      logger?.warn?.(`  agentic channel: no reconnect ${downFor}s after drop — forcing re-discovery (the client lib did not self-heal; likely a half-open drop).`);
+      const since = typeof disconnectedSince === 'function' ? disconnectedSince() : null;
+      // During reconnect churn `disconnectedSince` can be null (no sustained drop)
+      // or reset by the latest blip, so it under-reports the real staleness. Fall
+      // back to the presence-health clock — the signal we actually acted on — so
+      // the logged age reflects how long presence has genuinely been unconfirmed.
+      const staleSince = since != null
+        ? since
+        : (typeof presenceHealthySince === 'function' ? presenceHealthySince() : null);
+      const downFor = staleSince != null ? Math.round((now() - staleSince) / 1000) : '?';
+      logger?.warn?.(`  agentic channel: presence not confirmed within threshold (down ${downFor}s / reconnect churn) — forcing re-discovery (the client lib did not self-heal; likely a lossy-link 1006 churn or half-open drop).`);
       await onStale?.();
     } catch (err) {
       firedForEpisode = false; // heal failed → re-arm so a later tick retries this episode
@@ -6521,6 +6670,86 @@ function enableEngineHappyEyeballs(opts = {}) {
   return false;
 }
 
+// The Node DNS result-order token that ranks any A (IPv4) record ahead of an
+// AAAA (IPv6) one, so a host advertised over mDNS (`*.local`) that answers with
+// an unreachable IPv6 link-local `fe80::…` FIRST is still connected over its
+// reachable IPv4 A record. Used both process-wide (setDefaultResultOrder) and as
+// the `--dns-result-order` value propagated into the forked agent's NODE_OPTIONS.
+const DNS_RESULT_ORDER_IPV4_FIRST = 'ipv4first';
+
+/**
+ * Prefer IPv4 whenever an A record exists, PROCESS-WIDE, by flipping Node's
+ * default DNS result order to `ipv4first` (jwulf/c8ctl-plugin-nano#151). This is
+ * the harness-wide *class* fix that complements the engine-client Happy-Eyeballs
+ * enabler (#139): Happy-Eyeballs races families but can't help when the *only*
+ * answer picked is a single dead address, and it doesn't govern ordering; forcing
+ * `ipv4first` makes every `dns.lookup` (and therefore every outbound the worker
+ * and any in-process client make — the SDK's `activateJobs`, the raw-`fetch`
+ * `--auto` engine reads, agentic discovery) rank the reachable A record ahead of
+ * a dead AAAA. Because `dns.lookup` re-resolves per connection (Node keeps no
+ * process-wide lookup cache), a *running* worker that started against a bad
+ * resolution also heals the moment DNS is corrected — no supervisor restart — as
+ * the next long-poll / reconcile re-resolves under the new ordering.
+ *
+ * Fail-open and idempotent: a runtime without `setDefaultResultOrder` (or any
+ * throw) is swallowed so an ordering hint — an optimisation — can never block a
+ * worker from starting.
+ *
+ * @param {{ dns?: object, order?: string }} [opts] injection seam for tests
+ * @returns {boolean} true if the default DNS result order was (re)asserted
+ */
+function preferIpv4Resolution(opts = {}) {
+  try {
+    // Destructure INSIDE the try so a non-object arg (e.g. `null`) fails open
+    // like any other throw rather than blowing up before the guard.
+    const { dns = nodeDns, order = DNS_RESULT_ORDER_IPV4_FIRST } = opts || {};
+    if (dns && typeof dns.setDefaultResultOrder === 'function') {
+      dns.setDefaultResultOrder(order);
+      return true;
+    }
+  } catch {
+    // Fail-open: DNS ordering is a connectivity optimisation, never a start gate.
+  }
+  return false;
+}
+
+/**
+ * Merge `--dns-result-order=ipv4first` into an existing `NODE_OPTIONS` string so
+ * a forked agent child (and any node-based tool it spawns) inherits the same
+ * IPv4-first ordering as its parent worker (jwulf/c8ctl-plugin-nano#151) — a
+ * `dns.setDefaultResultOrder` call in THIS process can't reach the child, but the
+ * child's own Node runtime reads the flag from `NODE_OPTIONS` at startup.
+ *
+ * Preserves any other options already in `NODE_OPTIONS` (we append, never
+ * clobber) and, crucially, RESPECTS an operator who has *explicitly* pinned a
+ * `--dns-result-order` (e.g. `verbatim`): we only add ours when none is present,
+ * so we harden the default without overriding a deliberate choice.
+ *
+ * @param {string} [existing] the incoming NODE_OPTIONS value (may be empty)
+ * @returns {string} the NODE_OPTIONS value with an ordering flag guaranteed
+ */
+function ipv4FirstNodeOptions(existing = '') {
+  const flag = `--dns-result-order=${DNS_RESULT_ORDER_IPV4_FIRST}`;
+  const cur = typeof existing === 'string' ? existing.trim() : '';
+  // An explicit operator ordering (any `--dns-result-order=…`) wins untouched.
+  if (/--dns-result-order[=\s]/.test(cur)) return cur;
+  return cur ? `${cur} ${flag}` : flag;
+}
+
+/**
+ * Return a shallow copy of an env map whose `NODE_OPTIONS` carries the
+ * IPv4-first ordering flag, so the forked harness inherits it (#151). Applied at
+ * every harness spawn site (host pipe/PTY/ACP and container) so the ordering
+ * follows the agent regardless of transport. Never mutates the input.
+ *
+ * @param {Record<string,string>} [env] the env map about to be handed to a child
+ * @returns {Record<string,string>} a new env map with NODE_OPTIONS hardened
+ */
+function withIpv4FirstNodeOptions(env) {
+  const base = env && typeof env === 'object' ? env : {};
+  return { ...base, NODE_OPTIONS: ipv4FirstNodeOptions(base.NODE_OPTIONS) };
+}
+
 /**
  * work — turn a hire profile into live Nano job workers (one per job-type in
  * the rank×capability matrix) and poll for work in the foreground until Ctrl-C.
@@ -6768,6 +6997,14 @@ async function workAgent(req, flags) {
   // scaling the fleet to zero (jwulf/c8ctl-plugin-nano#139). Process-wide, so it
   // covers both the SDK's activateJobs and the raw-fetch --auto engine reads.
   enableEngineHappyEyeballs();
+  // Prefer IPv4 whenever an A record exists (jwulf/c8ctl-plugin-nano#151): the
+  // process-wide, class-level complement to Happy-Eyeballs. `ipv4first` ranks the
+  // reachable A record ahead of a dead AAAA (mDNS `fe80::` link-local), so BOTH
+  // the SDK activateJobs and the raw-fetch --auto reads resolve to it — and,
+  // since Node re-resolves per connection, a running worker heals the moment a
+  // bad DNS answer is corrected, without a supervisor restart. Set before the
+  // client is created so every outbound inherits it.
+  preferIpv4Resolution();
   const camunda = globalThis.c8ctl.createClient();
 
   // Broker REST endpoint for live linked-resource prompts (issue #63) and the
@@ -6888,6 +7125,16 @@ async function workAgent(req, flags) {
   // shutdown); `agenticSelfHealing` guards against two concurrent re-discovery
   // loops (the cold-start one and a watchdog-triggered one).
   let agenticDisconnectedSince = null;
+  // #147 presence-keyed watchdog state. `agenticConnectedSince` is the epoch-ms
+  // the channel last (re)connected (null while down); the watchdog uses it to
+  // require a stable connection to have held for a grace window before counting
+  // presence as confirmed. `agenticPresenceHealthyAt` is the epoch-ms presence
+  // was last confirmed healthy — advanced on the first connect (buffered REGISTER
+  // drains) and by the watchdog whenever a stable connection is observed, and
+  // reset by a heal. When it ages past the presence-stale threshold — even while
+  // the socket flaps `1006` reconnects — the watchdog forces a re-discovery.
+  let agenticConnectedSince = null;
+  let agenticPresenceHealthyAt = null;
   /** @type {{ stop: () => void } | null} */
   let agenticWatchdog = null;
   let agenticSelfHealing = false;
@@ -6985,6 +7232,11 @@ async function workAgent(req, flags) {
         token: cfg.token,
         credential: cfg.credential,
         bufferCapacity: cfg.bufferCapacity,
+        // #147: de-synchronise reconnect attempts with equal-jitter backoff so a
+        // fleet dropped on the same lossy link does not reconnect in lockstep and
+        // re-congest it. Wraps the client lib's own exponential policy (which has
+        // no jitter of its own); the base delays/factor stay the lib's defaults.
+        schedule: makeJitteredReconnectSchedule(),
         logger,
       });
       const shown = redactAgenticUrl(buildAgenticUrl(cfg.url, {}));
@@ -7004,16 +7256,38 @@ async function workAgent(req, flags) {
       // a disconnect starts it (first drop wins, so the watchdog measures from the
       // ORIGINAL drop, not the latest of a reconnect storm). The watchdog reads
       // this to decide when the client lib has failed to self-heal.
-      workChannel.onConnect(() => { markAgentic('connected'); agenticDisconnectedSince = null; });
-      workChannel.onReconnect(() => { markAgentic('connected'); agenticDisconnectedSince = null; });
+      workChannel.onConnect(() => {
+        markAgentic('connected');
+        agenticDisconnectedSince = null;
+        agenticConnectedSince = Date.now();
+        // First open drains the buffered REGISTER → presence lands; seed the
+        // presence-health clock so the #147 trigger measures from here (#147).
+        agenticPresenceHealthyAt = Date.now();
+      });
+      workChannel.onReconnect(() => {
+        markAgentic('connected');
+        agenticDisconnectedSince = null;
+        agenticConnectedSince = Date.now();
+        // Deliberately do NOT advance agenticPresenceHealthyAt here: a reconnect
+        // only CLAIMS presence (re-announces). On a lossy link the socket may
+        // re-drop `1006` before presence actually lands, so the watchdog confirms
+        // it only once a connection HOLDS for the grace window — a reconnect that
+        // immediately re-drops must not mask an unrecovered presence (#147).
+      });
       workChannel.onDisconnect((info) => {
         markAgentic('disconnected', normalizeAgenticMessage(info));
         if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
+        agenticConnectedSince = null;
       });
-      if (workChannel.connected()) { markAgentic('connected'); agenticDisconnectedSince = null; }
-      else if (workChannel.everConnected()) {
+      if (workChannel.connected()) {
+        markAgentic('connected');
+        agenticDisconnectedSince = null;
+        agenticConnectedSince = Date.now();
+        if (agenticPresenceHealthyAt == null) agenticPresenceHealthyAt = Date.now();
+      } else if (workChannel.everConnected()) {
         markAgentic('disconnected');
         if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
+        agenticConnectedSince = null;
       }
     } catch (err) {
       // Never let a channel failure stop the worker from doing its actual job.
@@ -7093,6 +7367,8 @@ async function workAgent(req, flags) {
     if (!stale) return;
     workChannel = null; // re-arms armAgenticSelfHeal()'s shouldContinue gate
     agenticDisconnectedSince = null; // reset the clock; the fresh open restarts it
+    agenticConnectedSince = null; // #147: the fresh open re-seeds it
+    agenticPresenceHealthyAt = null; // #147: the fresh open re-confirms presence
     try { bufferMonitor?.stop(); } catch { /* best effort */ }
     bufferMonitor = null;
     markAgentic('disconnected', 'stale channel — re-discovering hub');
@@ -7106,9 +7382,18 @@ async function workAgent(req, flags) {
     if (agenticWatchdog) return;
     const staleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_STALE_MS, DEFAULT_AGENTIC_STALE_MS));
     const intervalMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_WATCHDOG_MS, DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS));
+    const presenceStaleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_PRESENCE_STALE_MS, DEFAULT_AGENTIC_PRESENCE_STALE_MS));
+    const presenceGraceMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_PRESENCE_GRACE_MS, DEFAULT_AGENTIC_PRESENCE_GRACE_MS));
     agenticWatchdog = startAgenticChannelWatchdog({
       getChannel: () => workChannel,
       disconnectedSince: () => agenticDisconnectedSince,
+      // #147: presence-keyed trigger — heal reconnect-churn that never re-lands
+      // presence, not just a sustained socket drop.
+      connectedSince: () => agenticConnectedSince,
+      presenceHealthySince: () => agenticPresenceHealthyAt,
+      onPresenceHealthy: () => { agenticPresenceHealthyAt = Date.now(); },
+      presenceStaleAfterMs,
+      presenceGraceMs,
       onStale: healStaleAgenticChannel,
       staleAfterMs,
       intervalMs,
@@ -7506,9 +7791,12 @@ async function workAgent(req, flags) {
           if (isContainer) liveRunIds.delete(runId);
           if (runDir && !keepRuns) { try { rmSync(runDir, { recursive: true, force: true }); } catch { /* best effort */ } }
           if (runDir) liveRunDirs.delete(runDir);
-          // Detach the relay session's inbound-frame subscription so it never
-          // outlives the job or leaks a steer listener across jobs.
-          if (relaySession) { try { relaySession.close(); } catch { /* best effort */ } }
+          // Emit the relay session's `phase:close` lifecycle event and drain its
+          // outbound buffer before the job settles (so the live-terminal tail is
+          // flushed, nanobpm/nano-workforce#710), then detach its inbound-frame
+          // subscription so it never outlives the job or leaks a steer listener
+          // across jobs. Bounded internally — a hub outage never wedges completion.
+          if (relaySession) { try { await relaySession.close(); } catch { /* best effort */ } }
         }
 
         // Read the agent's structured result: the file it wrote, else a stdout
@@ -12154,6 +12442,9 @@ export {
   rediscoverAgenticUntilConnected,
   defaultAgenticRediscoveryDelays,
   agenticChannelIsStale,
+  agenticPresenceIsStale,
+  jitteredDelay,
+  makeJitteredReconnectSchedule,
   startAgenticChannelWatchdog,
 };
 export { compareSemver, githubRepoSlug, filterReleasesSince, renderReleaseBody };
@@ -12242,6 +12533,10 @@ export {
   readDeployedAgentJobTypes,
   resolveAutoJobTypes,
   enableEngineHappyEyeballs,
+  preferIpv4Resolution,
+  ipv4FirstNodeOptions,
+  withIpv4FirstNodeOptions,
+  DNS_RESULT_ORDER_IPV4_FIRST,
   workAgent,
   derivePollTimeoutMs,
   AGENT_TASK_NS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "c8ctl-plugin-nano",
-  "version": "1.44.10",
+  "version": "1.44.12",
   "type": "module",
   "description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
   "main": "c8ctl-plugin.js",
@@ -57,12 +57,12 @@
   },
   "optionalDependencies": {
     "node-pty": "^1.0.0",
-    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.44.10",
-    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.44.10",
-    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.44.10",
-    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.44.10",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.44.10",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.44.10",
-    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.44.10"
+    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.44.12",
+    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.44.12",
+    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.44.12",
+    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.44.12",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.44.12",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.44.12",
+    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.44.12"
   }
 }

package/work-relay.mjs

@@ -62,6 +62,36 @@ export const RELAY_OPEN_CHUNK = `${agenticTranscript.encodeTranscriptEvent({
   phase: 'open',
 })}\n`;
 
+/**
+ * The final produce frame emitted on a relay stream the instant a session is
+ * closed: the closing twin of {@link RELAY_OPEN_CHUNK} — a canonical
+ * `@nanobpm/agentic` lifecycle "close" transcript event.
+ *
+ * It carries no agent output — its sole job is to CLOSE the `job:<jobKey>`
+ * stream so the app can flush the durable transcript deterministically at job
+ * completion (nanobpm/nano-workforce#710), instead of relying only on the
+ * supersede/disconnect fallback (which can abandon the tail frames). Without it,
+ * a completed job's live-terminal transcript is truncated at the tail. Emitted
+ * from {@link RelaySession.close} before the outbound buffer is drained, so the
+ * close marker itself rides the same buffered, QoS-ordered relay lane as the
+ * agent's output (and survives a brief hub outage via C4's ring where possible).
+ * Newline-framed to match `RELAY_OPEN_CHUNK`, and derived through
+ * `encodeTranscriptEvent` (never hand-rolled) so the wire marker stays
+ * single-sourced in `@nanobpm/agentic`.
+ */
+export const RELAY_CLOSE_CHUNK = `${agenticTranscript.encodeTranscriptEvent({
+  kind: 'lifecycle',
+  phase: 'close',
+})}\n`;
+
+/** Default bound on the outbound-buffer drain at session close (ms). A hub
+ * outage must never wedge job completion, so the drain is always bounded: on
+ * timeout the caller completes anyway and the app's supersede/disconnect
+ * fallback still eventually flushes whatever arrived. */
+export const DEFAULT_DRAIN_TIMEOUT_MS = 2_000;
+/** Default poll cadence while awaiting `channel.buffered() → 0` (ms). */
+export const DEFAULT_DRAIN_POLL_MS = 25;
+
 /**
  * Resolve a role's terminal mode — whether the agent harness for this role gets
  * a full PTY or a plain pipe. Honors the vocab's per-role opt-in: a role may set
@@ -113,7 +143,7 @@ export function parseInboundRelayChunk(frame, stream) {
  * @property {string} stream the relay stream name (derived from the jobKey)
  * @property {(chunk: string|Uint8Array) => void} relay publish one framed, jobKey-tagged output chunk on the relay lane
  * @property {(write: (chunk: string) => void) => (() => void)} attachSteer wire inbound steer bytes for this stream to `write`; returns a detach fn
- * @property {() => void} close detach any steer subscription
+ * @property {() => Promise<{ closeEmitted: boolean, drained: boolean, timedOut: boolean }>} close emit the `phase:close` lifecycle event, detach any steer subscription, then drain the outbound buffer (bounded)
  */
 
 /**
@@ -132,9 +162,21 @@ export function parseInboundRelayChunk(frame, stream) {
  * @param {import('./work-channel.mjs').WorkChannel} opts.channel the C2 channel holder (NOT re-instantiated)
  * @param {string|number} opts.jobKey the activated job's key; tags every frame and names the stream
  * @param {{ warn?: Function, debug?: Function }} [opts.logger]
+ * @param {number} [opts.drainTimeoutMs] bound on the close-time outbound-buffer drain (ms); a hub outage must never wedge completion
+ * @param {number} [opts.drainPollMs] poll cadence while awaiting `channel.buffered() → 0` (ms)
+ * @param {(ms: number) => Promise<void>} [opts.sleep] injectable delay (tests); defaults to a `setTimeout` promise
+ * @param {() => number} [opts.now] injectable clock (tests); defaults to `Date.now`
  * @returns {RelaySession}
  */
-export function createRelaySession({ channel, jobKey, logger } = {}) {
+export function createRelaySession({
+  channel,
+  jobKey,
+  logger,
+  drainTimeoutMs = DEFAULT_DRAIN_TIMEOUT_MS,
+  drainPollMs = DEFAULT_DRAIN_POLL_MS,
+  sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  now = () => Date.now(),
+} = {}) {
   if (!channel || typeof channel.relayLane !== 'function') {
     throw new Error('createRelaySession requires a WorkChannel with a relayLane() accessor');
   }
@@ -207,8 +249,56 @@ export function createRelaySession({ channel, jobKey, logger } = {}) {
     return detach;
   };
 
+  // The outbound-buffer drain: poll `channel.buffered()` down to zero, bounded
+  // by drainTimeoutMs. A channel without a buffered() accessor (or one that
+  // throws) is treated as already drained — the drain must never block or crash
+  // completion. Returns whether the buffer emptied and whether we hit the bound.
+  const drain = async () => {
+    if (typeof channel.buffered !== 'function') return { drained: true, timedOut: false };
+    const deadline = now() + Math.max(0, Number(drainTimeoutMs) || 0);
+    for (;;) {
+      let pending;
+      try {
+        pending = channel.buffered();
+      } catch {
+        return { drained: true, timedOut: false };
+      }
+      if (!(Number(pending) > 0)) return { drained: true, timedOut: false };
+      if (now() >= deadline) {
+        try {
+          log.warn?.(`relay drain timed out for ${stream}: ${pending} frame(s) still buffered; completing anyway`);
+        } catch {
+          /* never let a logging failure escape the drain path */
+        }
+        return { drained: false, timedOut: true };
+      }
+      await sleep(Math.max(1, Number(drainPollMs) || 1));
+    }
+  };
+
+  // close() is idempotent: a second call returns the same settled promise
+  // without re-emitting the close marker or re-draining.
+  let closed = false;
+  let closedPromise = Promise.resolve({ closeEmitted: false, drained: true, timedOut: false });
   const close = () => {
+    if (closed) return closedPromise;
+    closed = true;
+    // Emit the closing lifecycle twin of RELAY_OPEN_CHUNK FIRST, so the app can
+    // flush the durable transcript deterministically at completion
+    // (nanobpm/nano-workforce#710). Routed through the internal relay(), which
+    // swallows sink errors, so emitting the close marker never fails the job. It
+    // rides the same buffered relay lane as the agent's output, and is included
+    // in the drain below.
+    relay(RELAY_CLOSE_CHUNK);
+    // Detach every steer subscription so none outlives the job.
     for (const detach of [...activeDetaches]) detach();
+    // Then drain the outbound buffer — a bounded await until the agent's tail
+    // bytes (and the close marker) are actually transmitted before the job
+    // settles. The bound is essential: a hub outage must not wedge completion,
+    // so on timeout we resolve anyway (the app's supersede/disconnect fallback
+    // still eventually flushes what arrived).
+    closedPromise = drain().then((res) => ({ closeEmitted: true, ...res }));
+    return closedPromise;
   };
 
   // Open the stream the instant the session exists, so the app correlates