c8ctl-plugin-nano 1.59.1 → 1.60.1

package/c8ctl-plugin.js

@@ -3323,6 +3323,23 @@ async function createAgenticEndpoint(opts) {
 // @param {typeof fetch} [opts.fetchImpl] injected fetch (tests)
 // @param {NodeJS.ProcessEnv} [opts.env]
 // @returns {Promise<{ deps: object, registry: object, settle: { complete: Function, fail: Function }, makeSupervisor: Function, Effect: object, Fiber: object }>}
+// Bind a settle seam to a SINGLE activation's lease token, captured from `job`
+// in closure scope. A settlement is thereby fenced with the exact activation
+// that ran — NOT a token re-read from a shared activeJobs map at settle time: a
+// same-key reactivation can overwrite that entry while an interrupted runner is
+// still unwinding, so a map lookup could fence the completion with the WRONG
+// (newer) token and clobber the new activation — the very lease bypass this
+// fence exists to prevent. `job.leaseToken` in the returned closures cannot drift.
+// @param {{ complete: Function, fail: Function }} settle  the raw settle seam
+// @param {{ jobKey: string, leaseToken?: string }} job    the activation whose lease fences the settlement
+// @returns {{ complete: (variables:any)=>any, fail: (opts?:object)=>any }}
+function bindJobSettle(settle, job) {
+  return {
+    complete: (variables) => settle.complete(job.jobKey, variables, job.leaseToken),
+    fail: (opts2) => settle.fail(job.jobKey, { ...(opts2 || {}), leaseToken: job.leaseToken }),
+  };
+}
+
 async function createSupervisorDeps(opts = {}) {
   const {
     runner,
@@ -3424,7 +3441,7 @@ async function createSupervisorDeps(opts = {}) {
   // the runtime uses (activate/extendLock), and any future port instrumentation
   // covers the settle path too.
   const settle = {
-    complete: (jobKey, variables) => rt.Effect.runPromise(engine.complete(jobKey, variables)),
+    complete: (jobKey, variables, leaseToken) => rt.Effect.runPromise(engine.complete(jobKey, variables, leaseToken)),
     fail: (jobKey, opts2) => rt.Effect.runPromise(engine.fail(jobKey, opts2)),
   };
 
@@ -6373,16 +6390,25 @@ function normalizeProjectApps(projects) {
 }
 
 /**
- * Probe whether an embedded app's `/agentic` endpoint answers a WebSocket
- * upgrade. Connects to `ws://<host>:<port>/agentic?token=…` (host defaults to
+ * Probe whether a `/agentic` endpoint answers a WebSocket upgrade. Connects to
+ * `ws(s)://<host>:<port><pathPrefix>/agentic?token=…` (host defaults to
  * `127.0.0.1`; a bare IPv6 literal is bracketed for the URL authority) and
  * resolves `true` only if the socket opens within `timeoutMs`; a refused
- * connection, the console proxy's deliberate `501`, a `404`, or a timeout all
- * resolve `false`. Self-cleaning — the probe socket is closed as soon as the
- * outcome is known. Never throws.
+ * connection, a `404`/`501`, or a timeout all resolve `false`. Self-cleaning —
+ * the probe socket is closed as soon as the outcome is known. Never throws.
  *
- * @param {number} port the app's direct agentic port (`appUi.port`)
- * @param {{ host?: string, token?: string, WebSocketImpl?: Function, timeoutMs?: number }} [opts]
+ * `pathPrefix` targets either the app's own port (empty prefix → `/agentic`) or
+ * the engine's console **app-view WebSocket tunnel** (engine #1054), where the
+ * prefix is `/console/app-view/<project>` so the channel rides the single engine
+ * port instead of the app's direct port (see {@link discoverAgenticHubs}).
+ *
+ * `secure` selects the WS scheme: `false` → `ws://` (plain), `true` → `wss://`.
+ * The tunnel leg rides the engine's own port, so an `https://` engine base
+ * requires a `wss://` upgrade — probing it with plain `ws://` would spuriously
+ * fail the tunnel and force the direct-port fallback (see {@link discoverAgenticHubs}).
+ *
+ * @param {number|string} port the port the WS connects to (app port, or engine port for the tunnel)
+ * @param {{ host?: string, token?: string, WebSocketImpl?: Function, timeoutMs?: number, pathPrefix?: string, secure?: boolean }} [opts]
  * @returns {Promise<boolean>}
  */
 function probeAgenticChannel(port, {
@@ -6390,9 +6416,12 @@ function probeAgenticChannel(port, {
   token = LOCAL_AGENTIC_TOKEN,
   WebSocketImpl = globalThis.WebSocket,
   timeoutMs = AGENTIC_DISCOVERY_TIMEOUT_MS,
+  pathPrefix = '',
+  secure = false,
 } = {}) {
   if (typeof WebSocketImpl !== 'function') return Promise.resolve(false);
-  const url = `ws://${wsHostPart(host)}:${port}/agentic?token=${encodeURIComponent(token)}`;
+  const wsScheme = secure ? 'wss' : 'ws';
+  const url = `${wsScheme}://${wsHostPart(host)}:${port}${pathPrefix}/agentic?token=${encodeURIComponent(token)}`;
   return new Promise((resolve) => {
     let done = false;
     let ws;
@@ -6499,8 +6528,8 @@ async function resolveProbeCandidates(host, { lookupImpl = dnsLookup } = {}) {
  * slow-to-open one can't stall the whole probe: a fast later candidate still
  * wins. Each per-host probe is bounded by `timeoutMs`. A single candidate skips
  * the racing machinery entirely (unchanged legacy path).
- * @param {number} port
- * @param {{ hosts?: string[], token?: string, timeoutMs?: number, wsProbe?: Function, staggerMs?: number }} [opts]
+ * @param {number|string} port
+ * @param {{ hosts?: string[], token?: string, timeoutMs?: number, wsProbe?: Function, staggerMs?: number, pathPrefix?: string, secure?: boolean }} [opts]
  * @returns {Promise<string|null>} the winning host, or null
  */
 async function raceProbeCandidates(port, {
@@ -6509,12 +6538,14 @@ async function raceProbeCandidates(port, {
   timeoutMs = AGENTIC_DISCOVERY_TIMEOUT_MS,
   wsProbe = probeAgenticChannel,
   staggerMs = 250,
+  pathPrefix = '',
+  secure = false,
 } = {}) {
   const list = Array.isArray(hosts) ? hosts.filter(Boolean) : [];
   if (list.length === 0) return null;
   if (list.length === 1) {
     try {
-      return (await wsProbe(port, { host: list[0], token, timeoutMs })) ? list[0] : null;
+      return (await wsProbe(port, { host: list[0], token, timeoutMs, pathPrefix, secure })) ? list[0] : null;
     } catch {
       return null;
     }
@@ -6531,7 +6562,7 @@ async function raceProbeCandidates(port, {
     };
     const start = (host) => {
       Promise.resolve()
-        .then(() => wsProbe(port, { host, token, timeoutMs }))
+        .then(() => wsProbe(port, { host, token, timeoutMs, pathPrefix, secure }))
         .catch(() => false)
         .then((ok) => {
           if (ok) done(host);
@@ -6547,23 +6578,32 @@ async function raceProbeCandidates(port, {
 
 /**
  * Auto-discover the embedded nwf agentic hub(s) reachable from an engine base
- * URL (#75, #96). Reads `GET <engine>/console/api/projects`, keeps the apps that
- * advertise an agentic UI port, and WS-probes each app's `/agentic` **on the
- * engine's own host** to confirm the channel is actually served there (bypassing
- * the WS-incapable console proxy). Works cross-machine on a trusted LAN: a
- * loopback engine probes `127.0.0.1`, a remote engine (e.g. `merlin.local`)
- * probes that same host — the port is taken from the projects API but the host is
- * always the engine's, so a rogue projects API can never steer a probe at the
- * worker's own loopback (#76). Gives the projects fetch and each WS probe
- * INDEPENDENT deadlines (#133) so a slow fetch can't starve the probe, prefers a
- * routable address over a link-local `fe80::` one (Happy-Eyeballs), and is
- * fail-open: any error — not a nano engine (Camunda), network failure, malformed
- * body, or a timeout — degrades to `[]` so the worker's real job is never
- * blocked.
+ * URL (#75, #96, #97). Reads `GET <engine>/console/api/projects`, keeps the apps
+ * that advertise an agentic UI port, and WS-probes each app's `/agentic` **on the
+ * engine's own host** to confirm the channel is actually served there. Works
+ * cross-machine on a trusted LAN: a loopback engine probes `127.0.0.1`, a remote
+ * engine (e.g. `merlin.local`) probes that same host — the port is taken from the
+ * projects API but the host is always the engine's, so a rogue projects API can
+ * never steer a probe at the worker's own loopback (#76).
+ *
+ * **Single-port hardening (#97, engine #1054):** each app is probed **tunnel-first** —
+ * `ws://<engineHost>:<enginePort>/console/app-view/<project>/agentic`, the console
+ * app-view WebSocket tunnel on the engine's *own* port (the same host:port the
+ * worker already reached for the projects read). A surviving tunnel hub is marked
+ * `via:'tunnel'` and needs only the engine port to be reachable — the app's direct
+ * port need not be LAN-open. If the tunnel probe fails (an engine that predates
+ * #1054 refuses the `/agentic` WS upgrade with `501`), it falls back to probing the
+ * app's **direct** port and marks the hub `via:'direct'` (unchanged #96 behaviour).
+ *
+ * Gives the projects fetch and each WS probe INDEPENDENT deadlines (#133) so a slow
+ * fetch can't starve the probe, prefers a routable address over a link-local `fe80::`
+ * one (Happy-Eyeballs), and is fail-open: any error — not a nano engine (Camunda),
+ * network failure, malformed body, or a timeout — degrades to `[]` so the worker's
+ * real job is never blocked.
  *
  * @param {string} engineBaseUrl the engine base URL (e.g. `http://merlin.local:8080`)
  * @param {{ token?: string, fetchImpl?: Function, wsProbe?: Function, lookupImpl?: Function, timeoutMs?: number, fetchTimeoutMs?: number, probeTimeoutMs?: number }} [opts]
- * @returns {Promise<Array<{ project: string, port: number, label?: string, host: string }>>}
+ * @returns {Promise<Array<{ project: string, port: number, label?: string, host: string, via: 'tunnel'|'direct', enginePort?: string, scheme?: string }>>}
  */
 async function discoverAgenticHubs(engineBaseUrl, {
   token = LOCAL_AGENTIC_TOKEN,
@@ -6586,12 +6626,15 @@ async function discoverAgenticHubs(engineBaseUrl, {
   // loopback services — which was the actual #76 concern (a rogue projects API
   // making the worker probe its own localhost). So the port comes from the
   // engine's projects API, but the HOST is always the engine's, never guessed.
-  let host;
+  let engineUrl;
   try {
-    host = new URL(base).hostname;
+    engineUrl = new URL(base);
   } catch {
     return [];
   }
+  const host = engineUrl.hostname;
+  const engineScheme = engineUrl.protocol; // 'http:' | 'https:'
+  const enginePort = engineUrl.port || (engineScheme === 'https:' ? '443' : '80');
   const probeHost = isLoopbackHost(host) ? '127.0.0.1' : host;
   // (C) Decoupled budgets (#133): the projects fetch and each WS probe get their
   // OWN independent deadline. Previously they shared one 2s budget, so a fetch
@@ -6623,13 +6666,32 @@ async function discoverAgenticHubs(engineBaseUrl, {
   const candidates = await resolveProbeCandidates(probeHost, { lookupImpl });
   const settled = await Promise.all(apps.map(async (app) => {
     try {
-      const winner = await raceProbeCandidates(app.port, {
+      // Tunnel-first (#97): confirm the channel over the console app-view WS
+      // tunnel on the ENGINE port — the same host:port the projects read just
+      // used — so the app's direct port need not be reachable. A pre-#1054
+      // engine 501s the upgrade and the probe fails; we then fall back to the
+      // app's direct port (unchanged #96 path).
+      const tunnelHost = await raceProbeCandidates(enginePort, {
+        hosts: candidates,
+        token,
+        timeoutMs: probeTimeoutMs,
+        wsProbe,
+        pathPrefix: `/console/app-view/${encodeURIComponent(app.project)}`,
+        // The tunnel rides the engine's own port, so match its TLS: an
+        // `https://` engine base upgrades over `wss://`, not `ws://` (else the
+        // tunnel probe spuriously fails and we drop to the direct port).
+        secure: engineScheme === 'https:',
+      });
+      if (tunnelHost) {
+        return { ...app, host: tunnelHost, via: 'tunnel', enginePort, scheme: engineScheme };
+      }
+      const directHost = await raceProbeCandidates(app.port, {
         hosts: candidates,
         token,
         timeoutMs: probeTimeoutMs,
         wsProbe,
       });
-      return winner ? { ...app, host: winner } : null;
+      return directHost ? { ...app, host: directHost, via: 'direct' } : null;
     } catch {
       return null;
     }
@@ -6646,8 +6708,11 @@ async function discoverAgenticHubs(engineBaseUrl, {
  *     half-configured. No discovery attempted.
  *   - `{ status: 'connect', config }` — a target to connect to. Either the
  *     explicit `NANO_AGENTIC_URL`/`agenticUrl` verbatim (no discovery), or the
- *     single discovered app's `ws://<engineHost>:<port>/agentic` (loopback for a
- *     local engine, the engine's LAN host for a remote one).
+ *     single discovered app: the console app-view WS tunnel on the engine port
+ *     (`<engineHost>:<enginePort>/console/app-view/<project>/agentic`, `via:'tunnel'`,
+ *     #97) when available, else the direct app port
+ *     (`ws://<engineHost>:<appPort>/agentic`, `via:'direct'`, #96) — loopback for a
+ *     local engine, the engine's LAN host for a remote one.
  *   - `{ status: 'ambiguous', message, candidates }` — two+ apps expose a
  *     channel. Hard stop for the worker: it must not silently pick one.
  *   - `{ status: 'advisory', message }` — nothing discoverable (zero matches,
@@ -6681,10 +6746,18 @@ async function resolveAgenticTarget({ camunda, cache, ...opts } = {}) {
   } catch { /* keep the loopback default */ }
 
   if (hubs.length === 1) {
-    const { project, port, host } = hubs[0];
+    const { project, port, host, via, enginePort, scheme } = hubs[0];
+    // A tunnel hub rides the console app-view WS bridge on the engine's own port
+    // (#97): `<scheme>//<engineHost>:<enginePort>/console/app-view/<project>`, to
+    // which `buildAgenticUrl` appends `/agentic`. A direct hub keeps the #96
+    // `http://<host>:<appPort>` form. `discovered.port` stays the app's advertised
+    // port either way (the hub identity); the local `via` records the route taken.
+    const url = via === 'tunnel'
+      ? `${scheme}//${wsHostPart(host)}:${enginePort}/console/app-view/${encodeURIComponent(project)}`
+      : `http://${wsHostPart(host)}:${port}`;
     const config = {
       ...base,
-      url: `http://${wsHostPart(host)}:${port}`,
+      url,
       discovered: { project, port, host },
     };
     // Cache the known-good hub so a later blip self-heals from cache (#133-C).
@@ -7715,10 +7788,18 @@ async function workAgent(req, flags) {
   // dispatch claim/release lifecycle keyed by this worker's instance drives the
   // cockpit's jobKeys — so the recorders no longer poke a per-process channel.
   const recordJobStart = (job, jobType) => {
-    activeJobs.set(String(job.jobKey), { type: jobType, since: Date.now(), retries: Number(job.retries) });
+    activeJobs.set(String(job.jobKey), { type: jobType, since: Date.now(), retries: Number(job.retries), leaseToken: job.leaseToken });
     writeActivity();
   };
   const recordJobEnd = (job) => {
+    // Delete only if the current entry is THIS activation. A same-key
+    // reactivation may have replaced the entry (new leaseToken) while this
+    // (interrupted) run unwinds — deleting then would erase the NEWER activation
+    // from status and can make force-stop miss its yield. The per-activation
+    // leaseToken is the identity; an unleased job (no token both sides) deletes as
+    // before.
+    const cur = activeJobs.get(String(job.jobKey));
+    if (cur && cur.leaseToken !== job.leaseToken) return;
     activeJobs.delete(String(job.jobKey));
     writeActivity();
   };
@@ -7910,6 +7991,14 @@ async function workAgent(req, flags) {
     run: async (job, abortSignal) => {
       const jobType = job.type;
       recordJobStart(job, jobType);
+      // Bind the settler to THIS activation's lease token, captured from the job
+      // in closure scope. A settlement is fenced with the exact activation that
+      // ran — NOT a token re-read from the shared activeJobs map at settle time: a
+      // same-key reactivation can overwrite that entry while an interrupted runner
+      // is still unwinding, so a map lookup could fence the completion with the
+      // WRONG (newer) token and clobber the new activation — the very lease bypass
+      // this fence exists to prevent. `job.leaseToken` in the closure cannot drift.
+      const settleJob = bindJobSettle(settle, job);
       try {
         logger.info(`[${jobType}] job ${job.jobKey} (instance ${job.processInstanceKey ?? '-'}) → ${buildAgentCommandLine(profile.command, effectiveArgs)}`);
 
@@ -7922,7 +8011,7 @@ async function workAgent(req, flags) {
             const freeMb = budget.free != null ? Math.round(budget.free / 1_048_576) : '?';
             const retries = Math.max(0, (Number(job.retries) || 1) - 1);
             logger.warn(`[${jobType}] job ${job.jobKey} shed — low disk (${freeMb}MB free); retries left ${retries}`);
-            return settle.fail(job.jobKey, { errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
+            return settleJob.fail({ errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
           }
         }
 
@@ -7954,7 +8043,7 @@ async function workAgent(req, flags) {
           const retries = Math.max(0, (Number(job.retries) || 1) - 1);
           const msg = err instanceof ProvisionError ? err.message : `prompt resource fetch failed: ${err.message}`;
           logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
-          return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
+          return settleJob.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
         }
 
         // Assemble + normalize the task envelope from headers (defaults) and
@@ -7965,7 +8054,7 @@ async function workAgent(req, flags) {
           const retries = Math.max(0, (Number(job.retries) || 1) - 1);
           const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
           logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
-          return settle.fail(job.jobKey, { errorMessage: msg, retries });
+          return settleJob.fail({ errorMessage: msg, retries });
         }
 
         // #130: per-task liveness overrides. Precedence: envelope override →
@@ -8031,7 +8120,7 @@ async function workAgent(req, flags) {
               : 'repository.url is missing';
             const msg = `incomplete repository envelope — ${why}; refusing to run in the launch/temp cwd (likely an orchestrator bug emitting a half-specified repository block)`;
             logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
-            return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
+            return settleJob.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
           }
         }
 
@@ -8086,7 +8175,7 @@ async function workAgent(req, flags) {
             const retries = Math.max(0, (Number(job.retries) || 1) - 1);
             const msg = err instanceof ProvisionError ? err.message : `provisioning error: ${err.message}`;
             logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
-            return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
+            return settleJob.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
           }
         } else if (!isContainer) {
           // Repo-less host job (issue #129, hardening 1): nothing is provisioned,
@@ -8112,7 +8201,7 @@ async function workAgent(req, flags) {
             const retries = Math.max(0, (Number(job.retries) || 1) - 1);
             const msg = `could not create a temp workspace under the worker namespace: ${err.message}`;
             logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
-            return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
+            return settleJob.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
           }
         }
 
@@ -8310,7 +8399,7 @@ async function workAgent(req, flags) {
           const resultKeys = Object.keys(resultVars);
           if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
           else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
-          return await settle.complete(job.jobKey, {
+          return await settleJob.complete({
             ...resultVars,
             [AGENT_RESULT_KEY]: resultEnvelope,
             output: result.stdout,
@@ -8327,7 +8416,7 @@ async function workAgent(req, flags) {
           || (result.stderr || '').trim() + (result.stderrTruncated && (result.stderr || '').trim() ? ' [stderr truncated]' : '')
           || (result.signal ? `terminated by signal ${result.signal}` : `exit code ${result.exitCode}`);
         logger.warn(`[${jobType}] job ${job.jobKey} failed (${detail}); retries left ${retries}`);
-        return await settle.fail(job.jobKey, {
+        return await settleJob.fail({
           errorMessage: `agent "${profile.name}" failed: ${detail}`.slice(0, 2000),
           retries,
           variables: { [AGENT_RESULT_KEY]: resultEnvelope },
@@ -8360,6 +8449,12 @@ async function workAgent(req, flags) {
       dispatch: { recoveryWindowMs, extendIntervalMs: lockExtendIntervalMs },
     },
   });
+  // The settle seam is used directly with an EXPLICIT lease token per call: the
+  // runner binds `settleJob` to its activation's `job.leaseToken` (race-free —
+  // see the runner), and the force-stop yield passes the token captured in
+  // `inflight`. We deliberately do NOT wrap `settle` to look the token up from the
+  // shared `activeJobs` map by jobKey, because a same-key reactivation could make
+  // that lookup fence a settlement with the wrong activation's token.
   settle = composed.settle;
   const {
     deps: supervisorDeps,
@@ -8555,7 +8650,7 @@ async function workAgent(req, flags) {
       logger.info(`Received ${signal} — aborting in-flight work and stopping worker...`);
       // Snapshot in-flight jobs (with their retry budget) BEFORE the interrupt
       // clears the ownership registry, so we can yield each one afterwards.
-      const inflight = [...activeJobs.entries()].map(([jobKey, info]) => ({ jobKey, retries: info?.retries }));
+      const inflight = [...activeJobs.entries()].map(([jobKey, info]) => ({ jobKey, retries: info?.retries, leaseToken: info?.leaseToken }));
       // Interrupt the runtime: this aborts each running job's AbortSignal (the
       // makeJobRunner seam) so runAgentJob killTree's the harness process group,
       // and runs dispatch's bracketed teardown (release ownership + slot). The
@@ -8569,12 +8664,13 @@ async function workAgent(req, flags) {
       // Yield each in-flight job so the broker re-activates it at once (retries
       // preserved — a force-stop doesn't consume an attempt). Best-effort: a
       // failed yield just lets the lock lapse (the honest fallback).
-      for (const { jobKey, retries } of inflight) {
+      for (const { jobKey, retries, leaseToken } of inflight) {
         try {
           await SupervisorEffect.runPromise(settle.fail(jobKey, {
             errorMessage: `worker force-stopped (${signal}); job yielded for retry`,
             retries: Number.isFinite(retries) && retries > 0 ? retries : 1,
             retryBackOff: 0,
+            leaseToken,
           }));
           logger.info(`  yielded job ${jobKey} for immediate retry.`);
         } catch (err) {
@@ -13805,6 +13901,7 @@ export {
   loadSupervisorRuntime,
   createAgenticEndpoint,
   createSupervisorDeps,
+  bindJobSettle,
   enableEngineHappyEyeballs,
   preferIpv4Resolution,
   isLikelyLocalNetworkTccBlock,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "c8ctl-plugin-nano",
-  "version": "1.59.1",
+  "version": "1.60.1",
   "type": "module",
   "description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
   "main": "c8ctl-plugin.js",
@@ -73,12 +73,12 @@
   },
   "optionalDependencies": {
     "node-pty": "^1.0.0",
-    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.59.1",
-    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.59.1",
-    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.59.1",
-    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.59.1",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.59.1",
-    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.59.1",
-    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.59.1"
+    "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.60.1",
+    "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.60.1",
+    "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.60.1",
+    "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.60.1",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.60.1",
+    "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.60.1",
+    "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.60.1"
   }
 }

package/supervisor-engine.mjs

@@ -36,6 +36,13 @@
  *     else the SAME calls issued raw via `fetchImpl` — the settle surface the
  *     supervisor's JobRunner uses once an agent finishes.
  *
+ * The three POST-ACTIVATION commands take an optional lease token that FENCES them
+ * against a superseded worker (the engine validates `updateJob` and requires it on
+ * `completeJob`/`failJob`): `extendLock(jobKey, ms, leaseToken?)`,
+ * `complete(jobKey, variables?, leaseToken?)`, and `fail(jobKey, { …, leaseToken? })`.
+ * On every path (SDK + raw) it is sent as the TOP-LEVEL camelCase `leaseToken` wire
+ * field and OMITTED when blank/absent (a non-leased job needs no fence).
+ *
  * Every method on this surface (`activate`→`activateJobs`,
  * `extendLock`→`updateJob`, `complete`→`completeJob`, `fail`→`failJob`) PREFERS
  * the injected `camunda` client and only hand-rolls the REST call as a
@@ -95,6 +102,18 @@ function isPlainObjectMap(v) {
   return proto === Object.prototype || proto === null;
 }
 
+/**
+ * A non-blank lease token — the opaque per-activation `leaseToken` the engine
+ * stamps on an agent job. When present it fences a post-activation command
+ * (`updateJob`/complete/fail) so a SUPERSEDED worker's call is rejected 409
+ * (`JobLeaseMismatch`) instead of silently mutating a job it no longer owns. A
+ * blank/absent token is OMITTED so the unfenced operator/bulk path (the engine
+ * treats a missing token as "always applies") is preserved untouched.
+ */
+function isNonBlankString(v) {
+  return typeof v === "string" && v.trim() !== "";
+}
+
 /**
  * Map one raw v2 activated-job record to the port's {@link ActivatedJob}. Keys are
  * strings in v2. A record missing `jobKey`/`type` violates the `ActivatedJob`
@@ -179,8 +198,8 @@ async function readErrorBody(res) {
  * @param {Record<string,string>|(() => (Record<string,string>|Promise<Record<string,string>>))} [opts.authHeaders] Ready-made auth header map, OR a resolver invoked per request (so rotating SDK auth — e.g. an OAuth bearer that refreshes — is re-derived each call rather than frozen). Wins over `token`.
  * @param {typeof fetch} [opts.fetchImpl] Injected `fetch` (defaults to the global; overridden in tests).
  * @param {number} [opts.requestTimeoutSlackMs] Extra ms added to a call's abort budget over its server long-poll (default 5000).
- * @param {{ activateJobs?: (input: { type: string, worker?: string, maxJobsToActivate: number, timeout: number, requestTimeout?: number }) => (Promise<{ jobs?: object[] }> & { cancel?: () => void }), updateJob?: (req: { jobKey: string, changeset: { timeout: number } }) => Promise<unknown>, completeJob?: (req: { jobKey: string, variables?: object }) => Promise<unknown>, failJob?: (req: { jobKey: string, retries?: number, errorMessage?: string, retryBackOff?: number, variables?: object }) => Promise<unknown> }} [opts.camunda] Optional Camunda SDK client. When present, each engine method prefers its typed SDK method over the raw fetch fallback: `activate`→`activateJobs` (`POST /v2/jobs/activation`, cancelled via the returned `CancelablePromise.cancel()`), `extendLock`→`updateJob` (`PATCH /v2/jobs/{jobKey}` `{ changeset: { timeout } }`), `complete`→`completeJob` (`POST /v2/jobs/{jobKey}/completion`), `fail`→`failJob` (`POST /v2/jobs/{jobKey}/failure`).
- * @returns {{ activate(req: ActivateRequest, signal?: AbortSignal): Promise<ReadonlyArray<ActivatedJob>>, extendLock(jobKey: string, ms: number): Promise<void>, complete(jobKey: string, variables?: object): Promise<void>, fail(jobKey: string, opts?: { retries?: number, errorMessage?: string, retryBackOff?: number, variables?: object }): Promise<void> }}
+ * @param {{ activateJobs?: (input: { type: string, worker?: string, maxJobsToActivate: number, timeout: number, requestTimeout?: number }) => (Promise<{ jobs?: object[] }> & { cancel?: () => void }), updateJob?: (req: { jobKey: string, changeset: { timeout: number }, leaseToken?: string }) => (Promise<unknown> & { cancel?: () => void }), completeJob?: (req: { jobKey: string, variables?: object, leaseToken?: string }) => Promise<unknown>, failJob?: (req: { jobKey: string, retries?: number, errorMessage?: string, retryBackOff?: number, variables?: object, leaseToken?: string }) => Promise<unknown> }} [opts.camunda] Optional Camunda SDK client. When present, each engine method prefers its typed SDK method over the raw fetch fallback: `activate`→`activateJobs` (`POST /v2/jobs/activation`, cancelled via the returned `CancelablePromise.cancel()`), `extendLock`→`updateJob` (`PATCH /v2/jobs/{jobKey}` `{ changeset: { timeout }, leaseToken? }`, likewise cancelled via `cancel()` when the extend's `AbortSignal` fires), `complete`→`completeJob` (`POST /v2/jobs/{jobKey}/completion`), `fail`→`failJob` (`POST /v2/jobs/{jobKey}/failure`). Each post-activation request carries the activation's top-level `leaseToken` (camelCase wire field) so the engine's lease fence accepts it.
+ * @returns {{ activate(req: ActivateRequest, signal?: AbortSignal): Promise<ReadonlyArray<ActivatedJob>>, extendLock(jobKey: string, ms: number, leaseToken?: string, signal?: AbortSignal): Promise<void>, complete(jobKey: string, variables?: object, leaseToken?: string): Promise<void>, fail(jobKey: string, opts?: { retries?: number, errorMessage?: string, retryBackOff?: number, variables?: object, leaseToken?: string }): Promise<void> }} Every post-activation method takes the activation's `leaseToken` (Camunda-v10 lease fence): the engine validates it before mutating a leased job, so `extendLock`/`complete`/`fail` must each carry the token from the activation that owns the lock — omitting it is rejected once a job is leased. `extendLock` also accepts an optional `AbortSignal` so a hung extend can be cancelled (not merely abandoned) when the caller's deadline fires.
  */
 export function createRawEngineClient(opts = {}) {
   const {
@@ -304,22 +323,53 @@ export function createRawEngineClient(opts = {}) {
       return jobs.map(mapJob);
     },
 
-    async extendLock(jobKey, ms) {
+    async extendLock(jobKey, ms, leaseToken, signal) {
       // Prefer the SDK's typed `updateJob` (operationId `updateJob` →
       // `PATCH /v2/jobs/{jobKey}` with `{ changeset: { timeout } }`) so the lock
       // extension tracks the engine contract instead of a hand-rolled URL. Fall
       // back to the SAME call issued raw when no SDK client is injected (keeps
       // this module wire-testable and usable standalone).
+      //
+      // `leaseToken` (when present) is a TOP-LEVEL sibling of `changeset`/`jobKey`
+      // — NOT nested in the changeset — matching the Camunda v10 `JobUpdateRequest`.
+      // The wire field is camelCase `leaseToken` (the engine's `JobUpdateRequest`
+      // deserializes it via `#[serde(rename = "leaseToken")]` into its internal
+      // `lease_token` field), so BOTH the SDK and this raw PATCH send the same
+      // camelCase key. It FENCES the extend: a superseded
+      // worker whose lease has been reassigned is rejected 409 (`JobLeaseMismatch`)
+      // rather than renewing a lock it no longer owns, which is what let a reclaimed
+      // agent job keep running and loop (empty transcript husks). Omitted when blank
+      // so the unfenced operator/bulk path is unchanged.
+      const fence = isNonBlankString(leaseToken) ? { leaseToken } : {};
       if (camunda && typeof camunda.updateJob === "function") {
+        // Mirror `activate`: wire the external abort `signal` (fired when the
+        // dispatch-side `Effect.timeout`/interruption cancels a hung beat) onto the
+        // SDK call's `CancelablePromise.cancel()` when it exposes one, so a wedged
+        // `updateJob` is actually cancelled rather than left in flight to (late)
+        // re-lock a job whose lease we've already treated as lost.
+        const p = camunda.updateJob({ changeset: { timeout: ms }, jobKey: String(jobKey), ...fence });
+        const cancel = typeof p?.cancel === "function" ? () => p.cancel() : () => {};
+        const onExtAbort = () => cancel();
+        if (signal) {
+          if (signal.aborted) cancel();
+          else signal.addEventListener("abort", onExtAbort, { once: true });
+        }
         try {
-          await camunda.updateJob({ changeset: { timeout: ms }, jobKey: String(jobKey) });
+          await p;
           return;
         } catch (err) {
           throw new Error(`extendLock ${jobKey}: SDK updateJob failed: ${err?.message ?? err}`, { cause: err });
+        } finally {
+          if (signal) signal.removeEventListener("abort", onExtAbort);
         }
       }
       const url = `${base}/jobs/${encodeURIComponent(jobKey)}`;
-      const res = await call(url, { method: "PATCH", body: JSON.stringify({ changeset: { timeout: ms } }) }, 15_000);
+      const res = await call(
+        url,
+        { method: "PATCH", body: JSON.stringify({ changeset: { timeout: ms }, ...fence }) },
+        15_000,
+        signal,
+      );
       if (!res || !res.ok) {
         const status = res ? res.status : "?";
         throw new Error(`extendLock ${jobKey}: HTTP ${status} from ${url}${res ? await readErrorBody(res) : ""}`);
@@ -338,23 +388,31 @@ export function createRawEngineClient(opts = {}) {
     // SDK rejection (e.g. a 409 when the lock lapsed and the job was reclaimed)
     // surfaces as a rejected promise for the port to map.
 
-    async complete(jobKey, variables) {
+    async complete(jobKey, variables, leaseToken) {
       // Prefer the SDK's typed `completeJob` (operationId `completeJob` →
       // `POST /v2/jobs/{jobKey}/completion` with `{ variables }`) when a `camunda`
       // SDK client is injected, else the SAME call issued raw via `fetchImpl`. The
       // result-variable map the model produced is merged onto the process
       // instance. C8 v2 answers 204 No Content on success.
+      //
+      // `leaseToken` (top-level, like the extend) FENCES the completion: the engine
+      // validates it with `required=true`, so for a LEASED job the matching token
+      // MUST be supplied — a superseded worker (whose token rotated when the job was
+      // re-activated) is rejected `JobLeaseMismatch`, never clobbering the newer
+      // activation's result. A non-leased job carries no `job.leaseToken`, so the
+      // field is omitted and no token is required.
       const vars = isPlainObjectMap(variables) ? { variables } : {};
+      const fence = isNonBlankString(leaseToken) ? { leaseToken } : {};
       if (camunda && typeof camunda.completeJob === "function") {
         try {
-          await camunda.completeJob({ jobKey: String(jobKey), ...vars });
+          await camunda.completeJob({ jobKey: String(jobKey), ...vars, ...fence });
           return;
         } catch (err) {
           throw new Error(`complete ${jobKey}: SDK completeJob failed: ${err?.message ?? err}`, { cause: err });
         }
       }
       const url = `${base}/jobs/${encodeURIComponent(jobKey)}/completion`;
-      const res = await call(url, { method: "POST", body: JSON.stringify(vars) }, 15_000);
+      const res = await call(url, { method: "POST", body: JSON.stringify({ ...vars, ...fence }) }, 15_000);
       if (!res || !res.ok) {
         const status = res ? res.status : "?";
         throw new Error(`complete ${jobKey}: HTTP ${status} from ${url}${res ? await readErrorBody(res) : ""}`);
@@ -365,7 +423,7 @@ export function createRawEngineClient(opts = {}) {
       // Tolerate a `null` opts the same as `undefined` (mirrors `complete`'s
       // null-safe `variables`), so a caller passing `null` never trips the
       // signature-destructure TypeError.
-      const { retries = 0, errorMessage, retryBackOff, variables } = opts || {};
+      const { retries = 0, errorMessage, retryBackOff, variables, leaseToken } = opts || {};
       // Normalize retries to a non-negative integer (mirrors `mapJob`), so a
       // string/float/negative never reaches the engine (or SDK) as an invalid
       // count. `retries > 0` re-queues for another attempt, `retries === 0`
@@ -377,6 +435,10 @@ export function createRawEngineClient(opts = {}) {
       if (errorMessage !== undefined && errorMessage !== null) extra.errorMessage = String(errorMessage);
       if (Number.isFinite(retryBackOff) && retryBackOff > 0) extra.retryBackOff = retryBackOff;
       if (isPlainObjectMap(variables)) extra.variables = variables;
+      // Same lease fencing as `complete` (engine validates `failJob` with
+      // `required=true`): a leased job's failure must carry the matching token or a
+      // superseded worker's fail is rejected; omitted for a non-leased job.
+      if (isNonBlankString(leaseToken)) extra.leaseToken = leaseToken;
       // Prefer the SDK's typed `failJob` (operationId `failJob` →
       // `POST /v2/jobs/{jobKey}/failure`) when a `camunda` SDK client is injected,
       // else the SAME call issued raw via `fetchImpl`. C8 v2 answers 204.