c8ctl-plugin-nano 1.44.8 → 1.44.10

package/README.md

@@ -352,7 +352,7 @@ one of:
 | `starting` | transient: the worker just spawned and hasn't resolved its channel target yet (pre-`connecting`) |
 | `connected` | presence is live on the hub — you should see this worker in the Cockpit |
 | `connecting` | resolved a hub, socket not open yet (or the hub is unreachable) |
-| `disconnected` | an established channel dropped (hub restart/outage) — it auto-reconnects; also set if the channel failed to start (bad URL/refused socket), in which case it stays disconnected until the worker restarts |
+| `disconnected` | an established channel dropped (hub restart/outage) — it auto-reconnects, and a worker-side **liveness watchdog** force-re-discovers the hub if it stays down (see below); also set if the channel failed to start (bad URL/refused socket), which is **not** auto-recovered by the watchdog (it only guards a channel that has connected) — fix the target and restart the worker |
 | `advisory` | nothing discoverable at the engine — **not** in the Cockpit; set `NANO_AGENTIC_URL` |
 | `off` | visibility disabled (`NANO_AGENTIC=off`) |
 | `?` | a live worker not yet reporting, or an older build predating these fields |
@@ -361,6 +361,22 @@ If workers show `advisory` (or stay `connecting`) while jobs still run, that's t
 "connected to the engine but empty Cockpit" case: point them at the app with
 `export NANO_AGENTIC_URL=http://<engine-host>:<appUi.port>` (e.g. `:3000`).
 
+**Liveness watchdog (auto-recovery from a wedged channel).** If the nano server
+restarts, crashes, or a network partition drops the connection *without* a clean
+close (a **half-open** socket), a worker's channel client can sit `disconnected`
+forever — the worker vanishes from the Nano Workers view / Cockpit and, before
+this, only a supervisor restart brought it back. Each worker now runs a
+belt-and-suspenders watchdog: once a channel that had connected stays down past a
+threshold (the client library's own reconnect never recovered it), the worker
+tears the wedged channel down and re-runs full hub discovery + reopen — no restart
+needed. The thresholds are tunable via env (sensible defaults; you rarely need
+these):
+
+```bash
+export NANO_AGENTIC_STALE_MS=60000     # force re-discovery if a drop hasn't recovered within 60s (default)
+export NANO_AGENTIC_WATCHDOG_MS=15000  # how often the watchdog checks channel liveness (default)
+```
+
 **Secure mode (opt-in).** For a deployment where you want the visibility channel
 authenticated (rather than open on the LAN), start the server **and** every worker
 box with the **same** `NANO_AGENTIC_SECRET` — same env-var name, same value on both

package/c8ctl-plugin.js

@@ -2587,6 +2587,135 @@ const SANDBOXES = ['none', 'docker', 'podman'];
 // Only container-based sandboxes need an image / disk hygiene / a runtime bin.
 const CONTAINER_SANDBOXES = new Set(['docker', 'podman']);
 
+// Result-nudge (#678). A weak / non-Claude agent can finish a result-contract
+// job (exit 0, real work done) yet never emit the machine-readable result — the
+// downstream gateway then falls through to its default (e.g. a completed review
+// silently re-waits instead of converging). Rather than accept the empty result,
+// give the agent exactly ONE bounded "emit your result now" turn, feeding back
+// its own prior output, in the SAME workspace. This never redoes work; it only
+// recovers a dropped result. Bounded independently of the main run so a second
+// hang can't double a long idle window.
+const NUDGE_IDLE_TIMEOUT_MS = 120_000;
+const NUDGE_HARD_CAP_MS = 300_000;
+// Cap the prior transcript we echo back so a huge run can't blow the nudge prompt.
+// This is a UTF-16 code-unit (character) cap, not a byte cap: `String.slice`
+// counts code units, so with multi-byte output the byte size may be larger.
+const NUDGE_CONTEXT_CAP_CHARS = 24_000;
+
+// The bespoke prompt for the re-emit turn: derive the status from the work the
+// agent already did, write ONLY the result, change nothing else.
+function buildResultNudgePrompt(priorStdout, { hasResultFile = true } = {}) {
+  const ctx = typeof priorStdout === 'string' ? priorStdout.slice(-NUDGE_CONTEXT_CAP_CHARS) : '';
+  const intro = [
+    'You already completed the task in your previous turn, but you did NOT emit a',
+    'machine-readable result, so the orchestrator cannot read your status and the',
+    'run cannot advance.',
+    '',
+    'Do NOT redo the work, re-run tools, edit files, push, or open/modify a PR. Just',
+    'emit the result for the work you already did: a single flat JSON object of your',
+    'result variables (at minimum {"status":"..."}).',
+    '',
+  ];
+  // When the harness could not create the result file, AGENT_RESULT_FILE is NOT
+  // exported (runAgentJob only sets it for a truthy resultFile), so the usual
+  // "write to $AGENT_RESULT_FILE" instruction is impossible. Lead with the stdout
+  // sentinel in that case so weaker agents don't waste the turn chasing an unset
+  // env var; otherwise keep the file the primary path with the sentinel fallback.
+  const how = hasResultFile
+    ? [
+        'Write it to the file named by the AGENT_RESULT_FILE environment variable, e.g.:',
+        '',
+        '    printf \'%s\' \'{"status":"...","summary":"..."}\' > "$AGENT_RESULT_FILE"',
+        '',
+        'If you truly cannot write that file, print exactly one line: ::nano:result:: {json}',
+      ]
+    : [
+        'The AGENT_RESULT_FILE environment variable is unset/empty in this run, so you',
+        'CANNOT write a result file. Instead, print exactly one line to stdout:',
+        '',
+        '    ::nano:result:: {"status":"...","summary":"..."}',
+      ];
+  return [
+    ...intro,
+    ...how,
+    '',
+    'Your previous output (reference — derive the status/summary from it):',
+    '-----',
+    ctx,
+  ].join('\n');
+}
+
+// Given a finished agent run, decide whether it dropped its result and, if so,
+// perform ONE re-emit nudge via the injected `rerun(nudgeText)` (which re-invokes
+// the agent in the same workspace, writing to the same AGENT_RESULT_FILE).
+// `rerun` is injected so this is unit-testable without a real model. Returns the
+// (possibly appended) stdout and whether a nudge was attempted; the caller reads
+// the structured result from the result file / stdout afterwards as usual.
+// Cap a string to MAX_CAPTURE_BYTES of UTF-8 keeping the TAIL (so a trailing
+// `::nano:result::` sentinel / fenced JSON block survives the trim) and skipping
+// any partial leading continuation byte so we start on a char boundary. Mirrors
+// the per-stream capture cap for the nudge's post-concatenation stdout. Returns
+// `{ text, truncated }`.
+function capStdoutTail(s) {
+  const buf = Buffer.from(s, 'utf8');
+  if (buf.length <= MAX_CAPTURE_BYTES) return { text: s, truncated: false };
+  let start = buf.length - MAX_CAPTURE_BYTES;
+  while (start < buf.length && (buf[start] & 0xc0) === 0x80) start += 1;
+  return { text: buf.subarray(start).toString('utf8'), truncated: true };
+}
+
+async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger, logPrefix = '' }) {
+  const stdout0 = result && typeof result.stdout === 'string' ? result.stdout : '';
+  // A parsed result only counts as usable if it still carries at least one
+  // *effective* var after sanitizeResultVars strips the reserved / io.nanobpm.* /
+  // proto keys. An empty `{}` or a reserved-keys-only object leaves downstream
+  // gateways with no status/decision vars — exactly the dropped-result case the
+  // nudge exists to recover — so gate on effective vars, not raw object presence.
+  const hasUsableResult = (parsed) => Object.keys(sanitizeResultVars(parsed)).length > 0;
+  const already = readAgentResultFile(resultFile) ?? parseResultFromStdout(stdout0);
+  // Only nudge a clean run that produced NO usable result but DID produce output
+  // (silence means a crash/hang the idle path already handles, not a dropped result).
+  // A null `resultFile` (temp-dir creation failed) is NOT a reason to skip: the
+  // read-back already falls through to `parseResultFromStdout`, and the nudge
+  // prompt explicitly offers the `::nano:result::` stdout sentinel, so recovery
+  // still works in stdout-sentinel-only mode.
+  if (hasUsableResult(already) || !result?.ok || !stdout0.trim() || typeof rerun !== 'function') {
+    // No nudge: `stdout0` is the incoming stdout verbatim, so keep the returned
+    // `truncated` flag consistent with it — echo the incoming `result.truncated`
+    // rather than hardcoding false, so callers that trust the return value don't
+    // see an already-truncated stdout reported as untruncated.
+    return { stdout: stdout0, nudged: false, truncated: result?.truncated === true };
+  }
+  let nudge = null;
+  let nudgeError = null;
+  try { nudge = await rerun(buildResultNudgePrompt(stdout0, { hasResultFile: resultFile != null })); } catch (err) { nudge = null; nudgeError = err; }
+  const nudgeOut = nudge && typeof nudge.stdout === 'string' ? nudge.stdout : '';
+  // Re-apply the per-stream capture cap after concatenation: `result.stdout` is
+  // forwarded verbatim into the job vars (`output`) and the audit envelope, so
+  // the nudge must not let the combined output bypass MAX_CAPTURE_BYTES. Keep the
+  // tail so a `::nano:result::` sentinel emitted by the nudge survives the trim,
+  // and surface truncation so `result.truncated` stays honest.
+  const capped = capStdoutTail(nudgeOut ? `${stdout0}\n${nudgeOut}` : stdout0);
+  const stdout = capped.text;
+  // The returned `stdout` still starts with `stdout0`, so if the FIRST turn was
+  // already truncated the returned output is truncated regardless of whether the
+  // post-concatenation cap trimmed anything: an empty/short nudge leaves
+  // `capped.truncated` false even though `stdout0` was clipped. OR in the incoming
+  // flag so `truncated` stays consistent with the returned stdout (and with the
+  // no-nudge early return above).
+  const truncated = result?.truncated === true || capped.truncated;
+  const recovered = hasUsableResult(readAgentResultFile(resultFile) ?? parseResultFromStdout(stdout));
+  if (nudgeError && logger?.warn) {
+    logger.warn(`${logPrefix} re-emit nudge rerun threw — ${nudgeError?.message ?? nudgeError}`);
+  }
+  if (logger?.info) {
+    logger.info(recovered
+      ? `${logPrefix} no result on the first turn — recovered it via one re-emit nudge`
+      : `${logPrefix} no result on the first turn — re-emit nudge did not recover one`);
+  }
+  return { stdout, nudged: true, truncated };
+}
+
 function coerceBool(v, dflt = false) {
   if (typeof v === 'boolean') return v;
   if (v == null) return dflt;
@@ -5159,6 +5288,31 @@ function buildAgentPayload(profile, job, envelope) {
   };
 }
 
+// Build the exact bytes handed to the harness on stdin for a run (#678).
+//
+// Normal run: the JSON job envelope from `buildAgentPayload`. Every non-ACP
+// harness (pipe/PTY/container) reads that JSON off stdin, so the shape is a
+// contract — a re-emit nudge must NOT replace it with a bare string or the
+// harness fails to parse its job. So when a `nudgePayload` (the bounded "re-emit
+// your result" prompt) is present:
+//   - ACP delivers stdin verbatim as the `session/prompt` text, so it takes the
+//     raw nudge string.
+//   - Non-ACP keeps the JSON envelope and overrides its prompt fields (top-level
+//     `prompt` and the reserved `task.task.prompt`, i.e. `envelope.task.prompt`)
+//     so a harness that dispatches on either sees the nudge. The shared envelope
+//     is copied, never mutated.
+function buildAgentStdin(profile, job, envelope, { nudgePayload = null, acp = false } = {}) {
+  if (nudgePayload == null) return JSON.stringify(buildAgentPayload(profile, job, envelope));
+  const nudgeText = String(nudgePayload);
+  if (acp) return nudgeText;
+  const base = buildAgentPayload(profile, job, envelope);
+  base.prompt = nudgeText;
+  if (isPlainObject(base.task) && isPlainObject(base.task.task)) {
+    base.task = { ...base.task, task: { ...base.task.task, prompt: nudgeText } };
+  }
+  return JSON.stringify(base);
+}
+
 function baseAgentEnv(profile, job) {
   return {
     AGENT_PROFILE: profile.name,
@@ -5254,10 +5408,17 @@ function startLockExtender(job, windowMs, intervalMs, tag, logger) {
  * Both paths resolve to the same result contract.
  */
 function runAgentJob(profile, job, opts = {}) {
-  const { timeoutMs, idleTimeoutMs, recoveryWindowMs, envelope, sandbox = 'none', image, runId, secretEnv = {}, passThroughSecretNames = [], cwd, extraEnv = {}, profileEnv = {}, resultFile, stream = false, streamPrefix = '', onStreamOut, onStreamErr, args: commandArgs, terminal = 'pipe', protocol = 'pipe', permission = 'yolo', relaySession = null, ptyFactory } = opts;
+  const { timeoutMs, idleTimeoutMs, recoveryWindowMs, envelope, sandbox = 'none', image, runId, secretEnv = {}, passThroughSecretNames = [], cwd, extraEnv = {}, profileEnv = {}, resultFile, stream = false, streamPrefix = '', onStreamOut, onStreamErr, args: commandArgs, terminal = 'pipe', protocol = 'pipe', permission = 'yolo', relaySession = null, ptyFactory, nudgePayload = null } = opts;
   // #110: `protocol`/`permission` drive the ACP executor branch below. The
   // pipe/PTY paths are unchanged, so `protocol === 'pipe'` behaviour is identical.
-  const payload = JSON.stringify(buildAgentPayload(profile, job, envelope));
+  // A `nudgePayload` (#678) carries the bespoke "re-emit your result" prompt for a
+  // bounded second turn in the SAME workspace; everything else (env, cwd, command,
+  // result file) is identical to the main run. ACP delivers stdin as the prompt
+  // text so it takes the raw nudge; every non-ACP harness reads the JSON envelope
+  // off stdin, so there we keep the envelope and override its prompt fields
+  // (buildAgentStdin). Container sandboxes ignore `protocol` (pipe-only today).
+  const acpStdin = protocol === 'acp' && !CONTAINER_SANDBOXES.has(sandbox);
+  const payload = buildAgentStdin(profile, job, envelope, { nudgePayload, acp: acpStdin });
   const agentEnv = baseAgentEnv(profile, job);
   // The harness command line: the profile command plus its structured switches
   // (persisted `--arg`s, possibly extended at work time via opts.args), each
@@ -6089,6 +6250,138 @@ async function rediscoverAgenticUntilConnected({
   return null;
 }
 
+// Worker-side liveness watchdog defaults (jwulf/c8ctl-plugin-nano#144). A
+// previously-connected agentic channel that has been `disconnected` for longer
+// than the stale threshold — because the client lib's own reconnect never
+// brought it back (e.g. a half-open drop after a server restart/crash/partition,
+// or a reconnect that keeps failing) — is force-healed: the wedged channel is
+// torn down and full discovery + reopen is re-armed, instead of trusting the
+// client lib alone. Overridable via NANO_AGENTIC_STALE_MS / NANO_AGENTIC_WATCHDOG_MS.
+const DEFAULT_AGENTIC_STALE_MS = 60_000;
+const DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS = 15_000;
+
+/**
+ * Decide whether a worker's agentic channel is *stale* — i.e. it once connected,
+ * is no longer connected, and has stayed down past `staleAfterMs`. Pure so the
+ * watchdog's trigger condition is unit-testable without timers or sockets. A
+ * channel that never opened (`everConnected() === false`) is NOT stale — it is
+ * still doing its first connect, which the initial open / cold-start self-heal
+ * owns. `disconnectedSince` is null whenever the channel is up (or never went
+ * down), which also reads as not-stale.
+ *
+ * @param {{
+ *   connected: () => boolean,
+ *   everConnected: () => boolean,
+ *   disconnectedSince: () => (number|null),
+ *   now?: () => number,
+ *   staleAfterMs?: number,
+ * }} opts
+ * @returns {boolean}
+ */
+function agenticChannelIsStale({
+  connected,
+  everConnected,
+  disconnectedSince,
+  now = () => Date.now(),
+  staleAfterMs = DEFAULT_AGENTIC_STALE_MS,
+}) {
+  if (typeof connected !== 'function' || typeof everConnected !== 'function') return false;
+  if (!everConnected()) return false; // never opened → the initial connect owns it
+  if (connected()) return false; // healthy
+  const since = typeof disconnectedSince === 'function' ? disconnectedSince() : null;
+  if (since == null) return false; // no recorded drop → nothing to heal
+  return now() - since >= staleAfterMs;
+}
+
+/**
+ * Start the worker-side agentic-channel liveness watchdog (#144). On a fixed
+ * interval it asks {@link agenticChannelIsStale} whether the channel dropped and
+ * never recovered within the threshold; when it has, it fires `onStale()` (which
+ * tears the wedged channel down and re-runs discovery + reopen). It fires
+ * `onStale` **exactly once per stale episode** — a per-episode latch is re-armed
+ * only when the channel is next observed healthy (or gone), or when a heal
+ * throws (a failed recovery retries on the next tick), so a persistent stale
+ * condition with a successful heal does not retrigger the heal on every tick. Re-entrancy is
+ * guarded so a slow heal never overlaps a later tick. Timers, clock, and the
+ * channel accessors are injectable so this is unit-testable without real waits.
+ * Returns a `{ stop, tick }` handle — `stop()` clears the timer AND latches the
+ * watchdog stopped so any tick already scheduled or in flight around shutdown
+ * becomes a no-op before it can reach `onStale` (shutdown relies on this to
+ * prevent a stale-channel resurrection mid-teardown), and `tick()` runs a single
+ * check (tests drive it directly).
+ *
+ * @param {{
+ *   getChannel: () => (import('./work-channel.mjs').WorkChannel | null),
+ *   disconnectedSince: () => (number|null),
+ *   onStale: () => (void|Promise<void>),
+ *   staleAfterMs?: number,
+ *   intervalMs?: number,
+ *   now?: () => number,
+ *   setIntervalFn?: typeof setInterval,
+ *   clearIntervalFn?: typeof clearInterval,
+ *   logger?: object|null,
+ * }} opts
+ * @returns {{ stop: () => void, tick: () => Promise<void> }}
+ */
+function startAgenticChannelWatchdog({
+  getChannel,
+  disconnectedSince,
+  onStale,
+  staleAfterMs = DEFAULT_AGENTIC_STALE_MS,
+  intervalMs = DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS,
+  now = () => Date.now(),
+  setIntervalFn = setInterval,
+  clearIntervalFn = clearInterval,
+  logger = null,
+} = {}) {
+  let healing = false;
+  // Set once `stop()` runs so any tick already scheduled/in flight around
+  // shutdown becomes a no-op and can never re-open the channel mid-teardown
+  // (shutdown relies on `stop()` to prevent a stale-channel resurrection).
+  let stopped = false;
+  // Per-episode latch: fire `onStale` exactly once when a connected channel goes
+  // stale, and don't fire again until it recovers (a fresh episode). Without this
+  // the helper would re-heal on every tick whenever `onStale` does not itself
+  // clear the staleness signal, causing repeated teardown/re-discovery attempts.
+  let firedForEpisode = false;
+  const tick = async () => {
+    if (stopped || healing) return; // shutting down, or a heal is in flight — don't stack a second re-discovery
+    const ch = typeof getChannel === 'function' ? getChannel() : null;
+    // No channel object → the initial open or the cold-start self-heal loop owns
+    // recovery; the watchdog only guards a channel that HAS connected and stalled.
+    // A missing or non-stale (healthy / recovered / still-connecting) channel also
+    // ends any current stale episode, so re-arm the latch for the next one.
+    if (!ch || !agenticChannelIsStale({
+      connected: () => ch.connected(),
+      everConnected: () => ch.everConnected(),
+      disconnectedSince,
+      now,
+      staleAfterMs,
+    })) {
+      firedForEpisode = false;
+      return;
+    }
+    if (firedForEpisode) return; // already fired once for this stale episode
+    healing = true;
+    firedForEpisode = true;
+    try {
+      if (stopped) return; // shutdown raced us between the checks — do not heal
+      const since = disconnectedSince();
+      const downFor = since != null ? Math.round((now() - since) / 1000) : '?';
+      logger?.warn?.(`  agentic channel: no reconnect ${downFor}s after drop — forcing re-discovery (the client lib did not self-heal; likely a half-open drop).`);
+      await onStale?.();
+    } catch (err) {
+      firedForEpisode = false; // heal failed → re-arm so a later tick retries this episode
+      logger?.debug?.(`agentic watchdog heal failed: ${err?.message || err}`);
+    } finally {
+      healing = false;
+    }
+  };
+  const timer = setIntervalFn(() => { tick().catch(() => {}); }, intervalMs);
+  if (timer && typeof timer.unref === 'function') timer.unref();
+  return { stop: () => { stopped = true; try { clearIntervalFn(timer); } catch { /* best effort */ } }, tick };
+}
+
 /**
  * Collapse an agentic disconnect/failure detail into the single short string the
  * marker's `agentic.message` field carries (#99 contract). Accepts the close
@@ -6587,6 +6880,17 @@ async function workAgent(req, flags) {
   let workChannel = null;
   /** @type {import('./work-buffer.mjs').BufferMonitor | null} */
   let bufferMonitor = null;
+  // #144 liveness watchdog state. `agenticDisconnectedSince` is the epoch-ms the
+  // channel last dropped (null whenever it is up or has never opened); the
+  // watchdog uses it to force a full re-discovery + reopen when the client lib's
+  // own reconnect fails to bring a previously-connected channel back within the
+  // stale threshold. `agenticWatchdog` is the running timer handle (stopped on
+  // shutdown); `agenticSelfHealing` guards against two concurrent re-discovery
+  // loops (the cold-start one and a watchdog-triggered one).
+  let agenticDisconnectedSince = null;
+  /** @type {{ stop: () => void } | null} */
+  let agenticWatchdog = null;
+  let agenticSelfHealing = false;
   // Maintain `activeJobs` unconditionally: it feeds both the supervisor activity
   // file (gated inside writeActivity) AND the agentic presence frame's live
   // jobKey set, so a standalone worker (no NANO_SUPERVISOR_ACTIVITY_FILE) still
@@ -6696,11 +7000,21 @@ async function workAgent(req, flags) {
       // (before these listeners existed), connected() is false but everConnected()
       // is true — record that as `disconnected` rather than leaving it stuck at
       // `connecting`.
-      workChannel.onConnect(() => markAgentic('connected'));
-      workChannel.onReconnect(() => markAgentic('connected'));
-      workChannel.onDisconnect((info) => markAgentic('disconnected', normalizeAgenticMessage(info)));
-      if (workChannel.connected()) markAgentic('connected');
-      else if (workChannel.everConnected()) markAgentic('disconnected');
+      // #144: track the drop clock alongside presence — a (re)connect clears it,
+      // a disconnect starts it (first drop wins, so the watchdog measures from the
+      // ORIGINAL drop, not the latest of a reconnect storm). The watchdog reads
+      // this to decide when the client lib has failed to self-heal.
+      workChannel.onConnect(() => { markAgentic('connected'); agenticDisconnectedSince = null; });
+      workChannel.onReconnect(() => { markAgentic('connected'); agenticDisconnectedSince = null; });
+      workChannel.onDisconnect((info) => {
+        markAgentic('disconnected', normalizeAgenticMessage(info));
+        if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
+      });
+      if (workChannel.connected()) { markAgentic('connected'); agenticDisconnectedSince = null; }
+      else if (workChannel.everConnected()) {
+        markAgentic('disconnected');
+        if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
+      }
     } catch (err) {
       // Never let a channel failure stop the worker from doing its actual job.
       workChannel = null;
@@ -6732,16 +7046,16 @@ async function workAgent(req, flags) {
     }
   };
 
-  if (agenticCfg) {
-    await openAgenticChannel(agenticCfg);
-  } else if (agenticTarget.status === 'advisory') {
-    // (A) Self-heal a cold-start discovery miss (#133): discovery is one-shot at
-    // enrolment, so a worker that merely lost the cold-start race (e.g. a slow
-    // link-local candidate blew the budget) would otherwise run `advisory` for
-    // its whole lifetime — the only recovery being a restart. Keep re-discovering
-    // in the background on a jittered backoff and, on a later success, upgrade
-    // advisory→connected WITHOUT a restart, flipping the AGENTIC status surface.
-    // A shared cache lets a brief blip reuse the last known-good hub (#133-C).
+  // (A) Background self-heal loop, shared by the cold-start advisory path (#133)
+  // AND the #144 liveness watchdog. Re-run discovery on a jittered backoff and,
+  // on the first `connect` target, (re)open the channel WITHOUT a restart. The
+  // `agenticSelfHealing` guard makes it idempotent: the watchdog can call it
+  // after tearing a stale channel down without racing a still-running cold-start
+  // loop. A shared cache lets a brief blip reuse the last known-good hub (#133-C).
+  const armAgenticSelfHeal = () => {
+    if (agenticSelfHealing) return; // a re-discovery loop is already running
+    if (workChannel !== null) return; // a channel already exists — nothing to heal
+    agenticSelfHealing = true;
     const hubCache = new Map();
     rediscoverAgenticUntilConnected({
       resolveTarget: () => resolveAgenticTarget({ camunda, logger, cache: hubCache }),
@@ -6749,7 +7063,7 @@ async function workAgent(req, flags) {
         agenticCfg = target.config;
         agenticState = agenticStateForTarget(target, safeAgenticDisplayUrl);
         writeActivity();
-        logger.info('  agentic channel: background re-discovery succeeded — upgrading advisory → connecting.');
+        logger.info('  agentic channel: background re-discovery succeeded — (re)opening channel.');
         await openAgenticChannel(agenticCfg);
         // openAgenticChannel swallows its own open failures (it nulls
         // workChannel and returns rather than throwing), so a failed open must
@@ -6762,7 +7076,57 @@ async function workAgent(req, flags) {
       // Stop as soon as a channel exists (loop won this or a prior attempt did).
       shouldContinue: () => workChannel === null,
       logger,
-    }).catch(() => { /* best-effort self-heal — never surfaces an error */ });
+    })
+      .catch(() => { /* best-effort self-heal — never surfaces an error */ })
+      .finally(() => { agenticSelfHealing = false; });
+  };
+
+  // (B) #144 liveness watchdog: force-heal a wedged channel. When a channel that
+  // HAS connected drops and the client lib's own reconnect never brings it back
+  // within the stale threshold (a half-open drop after a server restart/crash/
+  // partition, or a reconnect that keeps failing), the client sits `disconnected`
+  // forever and the worker vanishes from the Workers view until a supervisor
+  // restart. This tears the wedged channel down (so `shouldContinue` re-arms) and
+  // re-runs full discovery + reopen instead of trusting the client lib alone.
+  const healStaleAgenticChannel = async () => {
+    const stale = workChannel;
+    if (!stale) return;
+    workChannel = null; // re-arms armAgenticSelfHeal()'s shouldContinue gate
+    agenticDisconnectedSince = null; // reset the clock; the fresh open restarts it
+    try { bufferMonitor?.stop(); } catch { /* best effort */ }
+    bufferMonitor = null;
+    markAgentic('disconnected', 'stale channel — re-discovering hub');
+    // Deregister + close the wedged client so it stops its own doomed reconnect
+    // attempts and we don't leak two clients once the fresh one connects.
+    try { await stale.stop('stale channel — re-discovering'); } catch { /* best effort */ }
+    armAgenticSelfHeal();
+  };
+
+  const startAgenticWatchdog = () => {
+    if (agenticWatchdog) return;
+    const staleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_STALE_MS, DEFAULT_AGENTIC_STALE_MS));
+    const intervalMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_WATCHDOG_MS, DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS));
+    agenticWatchdog = startAgenticChannelWatchdog({
+      getChannel: () => workChannel,
+      disconnectedSince: () => agenticDisconnectedSince,
+      onStale: healStaleAgenticChannel,
+      staleAfterMs,
+      intervalMs,
+      logger,
+    });
+  };
+
+  if (agenticCfg) {
+    await openAgenticChannel(agenticCfg);
+    // Guard the connected channel: if it later drops and the client lib can't
+    // recover it, the watchdog forces a full re-discovery + reopen (#144).
+    startAgenticWatchdog();
+  } else if (agenticTarget.status === 'advisory') {
+    // A cold-start discovery miss leaves the worker `advisory`; the self-heal
+    // loop upgrades it to `connected` without a restart (#133), and once a
+    // channel exists the watchdog keeps it alive across later drops (#144).
+    armAgenticSelfHeal();
+    startAgenticWatchdog();
   }
 
   // C3 (#42): the role's live-terminal mode — a full PTY (streamed on the relay
@@ -7052,7 +7416,7 @@ async function workAgent(req, flags) {
             liveRunDirs.add(resultDir);
           } catch { resultDir = null; resultFile = null; }
 
-          result = await runAgentJob(profile, job, {
+          const runOpts = {
             timeoutMs: effectiveHardCapMs,
             idleTimeoutMs: effectiveIdleTimeoutMs,
             recoveryWindowMs: effectiveRecoveryWindowMs,
@@ -7083,7 +7447,42 @@ async function workAgent(req, flags) {
             // spying never corrupts a structured/JSON output mode.
             onStreamOut: stream ? (line) => logger.info(line) : undefined,
             onStreamErr: stream ? (line) => logger.warn(line) : undefined,
-          });
+          };
+          result = await runAgentJob(profile, job, runOpts);
+
+          // Gap 2 (#678): a clean run that emitted no machine-readable result gets
+          // ONE bounded re-emit nudge in the same workspace, feeding back its own
+          // output, before we accept an empty result. Runs before finalizeGit so
+          // the workspace/result file are still live; the nudge changes no code.
+          // Not gated on `resultFile`: when the temp dir/file could not be created
+          // the result is recoverable only via the stdout `::nano:result::`
+          // sentinel, which `resolveAgentResultWithNudge` handles directly.
+          if (result.ok) {
+            const { stdout, nudged, truncated } = await resolveAgentResultWithNudge({
+              result,
+              resultFile,
+              logger,
+              logPrefix: `[${jobType}] job ${job.jobKey}:`,
+              rerun: (nudgeText) => runAgentJob(profile, job, {
+                ...runOpts,
+                nudgePayload: nudgeText,
+                stream: false,
+                idleTimeoutMs: Math.min(effectiveIdleTimeoutMs || NUDGE_IDLE_TIMEOUT_MS, NUDGE_IDLE_TIMEOUT_MS),
+                // Cap the recovery/probe window to the same 120s idle bound: it is
+                // used by createIdleLivenessMonitor as the probe window when >0, so
+                // inheriting the (possibly minutes-long) `effectiveRecoveryWindowMs`
+                // from runOpts would let a silent nudge run outlive the intended
+                // 120s idle bound and delay convergence on a wedged second turn.
+                recoveryWindowMs: Math.min(effectiveRecoveryWindowMs || NUDGE_IDLE_TIMEOUT_MS, NUDGE_IDLE_TIMEOUT_MS),
+                timeoutMs: Math.min(effectiveHardCapMs || NUDGE_HARD_CAP_MS, NUDGE_HARD_CAP_MS) || NUDGE_HARD_CAP_MS,
+              }),
+            });
+            result.stdout = stdout;
+            if (nudged) {
+              result.nudgedForResult = true;
+              if (truncated) result.truncated = true;
+            }
+          }
 
           // Finalize git only when the harness succeeded — never push a
           // half-finished workspace.
@@ -7382,6 +7781,9 @@ async function workAgent(req, flags) {
       logger.info(`Received ${signal} — stopping ${list.length} worker(s)...`);
       if (reaperTimer) clearInterval(reaperTimer);
       if (runDirTimer) clearInterval(runDirTimer);
+      // Stop the #144 liveness watchdog so it can't kick off a re-discovery
+      // mid-teardown (which would resurrect the channel we're about to close).
+      if (agenticWatchdog) { try { agenticWatchdog.stop(); } catch { /* best effort */ } agenticWatchdog = null; }
       const results = await Promise.all(list.map(drainWorker));
       const stopFailures = results.filter((ok) => !ok).length;
       if (stopFailures > 0) {
@@ -11751,6 +12153,8 @@ export {
   isLinkLocalAddress,
   rediscoverAgenticUntilConnected,
   defaultAgenticRediscoveryDelays,
+  agenticChannelIsStale,
+  startAgenticChannelWatchdog,
 };
 export { compareSemver, githubRepoSlug, filterReleasesSince, renderReleaseBody };
 export {
@@ -11783,11 +12187,14 @@ export {
   makeSecretResolver,
   hostEnvSecretResolver,
   buildAgentPayload,
+  buildAgentStdin,
   buildResultEnvelope,
   parseAgentResultObject,
   readAgentResultFile,
   parseResultFromStdout,
   sanitizeResultVars,
+  buildResultNudgePrompt,
+  resolveAgentResultWithNudge,
   parseEnvPairs,
   normalizeEnvMap,
   normalizeArgList,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "c8ctl-plugin-nano",
-  "version": "1.44.8",
+  "version": "1.44.10",
   "type": "module",
   "description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
   "main": "c8ctl-plugin.js",
@@ -57,12 +57,12 @@
   },
   "optionalDependencies": {
     "node-pty": "^1.0.0",
-    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.44.8",
-    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.44.8"
+    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.44.10",
+    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.44.10",
+    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.44.10",
+    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.44.10",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.44.10",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.44.10",
+    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.44.10"
   }
 }

package/work-channel.mjs

@@ -281,15 +281,25 @@ export async function createWorkChannel(opts) {
     everConnected: () => hasConnected,
     buffered: () => client.buffered,
     async stop(reason = 'worker stopped') {
+      // Deregister to drop presence cleanly, then ALWAYS close the socket so the
+      // client stops its own reconnect loop. Closing only on a deregister error
+      // (the old behaviour) left a successfully-deregistered client half-open and
+      // still reconnecting — which is exactly the wedged/duplicate-client case the
+      // #144 stale-channel heal path relies on stop() to end.
+      // deregister() is fire-and-forget but may be thenable (register() is
+      // treated as one above), so guard BOTH a synchronous throw and an async
+      // rejection — an unhandled rejection during shutdown must never escape.
       try {
-        client.deregister(reason);
-      } catch (err) {
-        try {
+        Promise.resolve(client.deregister(reason)).catch((err) => {
           log.warn?.(`agentic deregister failed: ${err?.message || err}`);
-          client.close();
-        } catch {
-          /* best effort — never let shutdown hang on the channel */
-        }
+        });
+      } catch (err) {
+        log.warn?.(`agentic deregister failed: ${err?.message || err}`);
+      }
+      try {
+        client.close();
+      } catch {
+        /* best effort — never let shutdown hang on the channel */
       }
     },
   };