c8ctl-plugin-nano 1.73.0 → 1.73.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -62,11 +62,13 @@ const DEFAULT_CREATE_RETRY_MAX_MS = 30000;
62
62
  const DEFAULT_PRE_MINT_BUFFER_MAX = 1000;
63
63
  const DEFAULT_PRE_MINT_BUFFER_MAX_BYTES = 8_000_000;
64
64
 
65
- // A terminal COMPLETED update is driven directly (not via the best-effort append
66
- // queue) and RETRIED a bounded number of times: on a successful job end the caller
67
- // settles the job the instant complete() returns, so there is no reactivation to
68
- // retry a rejected terminal transition — a single swallowed failure would strand the
69
- // instance non-terminal forever (issue #230).
65
+ // On a successful job end, complete() drains pending appends and drives a final
66
+ // `status: 'IDLE'` update directly (not via the best-effort append queue) and
67
+ // RETRIES transient failures a bounded number of times. Per Camunda 8.10 and
68
+ // nanobpmn parity (#274), AgentInstanceUpdateStatusEnum is
69
+ // { IDLE, THINKING, TOOL_CALLING, TOOL_DISCOVERY } — UPDATE never accepts
70
+ // COMPLETED (that returns HTTP 422); the engine itself completes all active
71
+ // AgentInstances when the owning process instance ends.
70
72
  const DEFAULT_TERMINAL_RETRY_MAX = 3;
71
73
 
72
74
  // Bound on complete()'s last-chance create attempt (issue #230). The harness awaits
@@ -261,13 +263,13 @@ export function describeSdkError(err) {
261
263
  }
262
264
 
263
265
  /**
264
- * True when an HTTP status marks a PERMANENT rejection of the terminal
265
- * status→COMPLETED update (issue #275): a 4xx (a stale lease fence, an invalid
266
- * transition, a 0-turns husk the engine refuses to terminalize) will not succeed
267
- * on a blind retry, so the producer stops retrying it immediately instead of
268
- * burning the remaining attempts against a dying lease. 5xx / network / timeout
269
- * failures are potentially transient and still retry. `null`/unknown statuses are
270
- * treated as transient (retry) — fail-safe toward recovery, not toward giving up.
266
+ * True when an HTTP status marks a PERMANENT rejection of the final
267
+ * status→IDLE update (issue #275, #274): a 4xx (a stale lease fence, an invalid
268
+ * transition, a contract error like 422) will not succeed on a blind retry, so
269
+ * the producer stops retrying it immediately instead of burning the remaining
270
+ * attempts against a dying lease. 5xx / network / timeout failures are potentially
271
+ * transient and still retry. `null`/unknown statuses are treated as transient
272
+ * (retry) — fail-safe toward recovery, not toward giving up.
271
273
  *
272
274
  * Two 4xx statuses are the exception — NOT permanent, because a retry can still
273
275
  * succeed:
@@ -633,7 +635,8 @@ function contentForResult(result) {
633
635
  * - `ingest(rawUpdate)` — feed one raw ACP `session/update` (non-blocking; the SDK
634
636
  * append is enqueued so the ACP hot path never blocks).
635
637
  * - `complete(ok)` — drain pending appends, then (on a successful job end) update
636
- * the instance status to COMPLETED. `job.complete` is unchanged and orthogonal.
638
+ * the instance status to IDLE (the engine owns COMPLETED when the process
639
+ * instance ends; issue #274). `job.complete` is unchanged and orthogonal.
637
640
  *
638
641
  * @param {object} opts
639
642
  * @param {object} opts.camunda Host SDK client (createAgentInstance/updateAgentInstance).
@@ -1867,8 +1870,10 @@ export function createAgentInstanceProducer(opts = {}) {
1867
1870
  /**
1868
1871
  * End the AgentInstance lifecycle. Flushes any pending message turn, drains the
1869
1872
  * append queue, then — only on a SUCCESSFUL job end (`ok`) — updates the instance
1870
- * status to COMPLETED (there is no separate completeAgentInstance verb). On a
1871
- * failed run the instance is left non-terminal so a retry/reactivation continues
1873
+ * status to `IDLE` (a valid update-status; Camunda 8.10 and nanobpmn reject
1874
+ * `COMPLETED` on UPDATE with HTTP 422 because `COMPLETED` is engine-owned and
1875
+ * written when the owning process instance ends, #274). On a failed run the
1876
+ * instance is left at its last active status so a retry/reactivation continues
1872
1877
  * the same instance. Best-effort; never throws; `job.complete` is unaffected.
1873
1878
  */
1874
1879
  async complete(ok = true) {
@@ -1986,36 +1991,26 @@ export function createAgentInstanceProducer(opts = {}) {
1986
1991
  elementInstanceKey,
1987
1992
  jobKey,
1988
1993
  jobLease: leaseToken,
1989
- status: 'COMPLETED',
1994
+ status: 'IDLE',
1990
1995
  }),
1991
1996
  finalizeTimeoutMs,
1992
1997
  setTimer,
1993
1998
  );
1994
1999
  } catch (err) {
1995
- // This serialized enqueue IS the terminal transition on the drain-timeout
1996
- // path — the only actor that can still terminalize the instance here. The
1997
- // generic enqueue() catch would swallow it at debug with just err.message,
1998
- // losing the HTTP status/body/correlation on the very path that already
1999
- // warned reconciliation may be needed (issue #230). Emit the SAME shaped
2000
- // status/body/correlation warning the direct terminal-retry path uses
2001
- // (with its manual-reconciliation outcome) before it is swallowed.
2002
2000
  const d = describeSdkError(err);
2003
2001
  logger?.warn?.(
2004
- `AgentInstance ${agentInstanceKey}: serialized terminal COMPLETED update ` +
2002
+ `AgentInstance ${agentInstanceKey}: serialized final IDLE update ` +
2005
2003
  `${err?.__nanoTimeout ? `timed out (bounded at ${finalizeTimeoutMs}ms)` : 'failed'} — ` +
2006
- `status=${d.status ?? 'n/a'} body=${d.body ?? 'n/a'} message=${d.message ?? 'n/a'}; ` +
2007
- `MANUAL RECONCILIATION REQUIRED — the drain did not catch up and the job has ` +
2008
- `settled with no reactivation to retry this transition ${correlation()}.`,
2004
+ `status=${d.status ?? 'n/a'} body=${d.body ?? 'n/a'} message=${d.message ?? 'n/a'} ` +
2005
+ `${correlation()}.`,
2009
2006
  );
2010
2007
  }
2011
2008
  });
2012
2009
  logger?.warn?.(
2013
2010
  `AgentInstance ${agentInstanceKey}: history drain did not settle within ` +
2014
- `${finalizeTimeoutMs}ms — terminal COMPLETED update SERIALIZED behind the ` +
2011
+ `${finalizeTimeoutMs}ms — final IDLE update SERIALIZED behind the ` +
2015
2012
  `pending appends (best-effort, ordered) rather than racing them; its outcome ` +
2016
- `is not observed here and the job settles on return, so this background write ` +
2017
- `is expected to be fence-rejected once its lease is void — MANUAL ` +
2018
- `RECONCILIATION REQUIRED (terminal status NOT confirmed) ${correlation()}.`,
2013
+ `is not observed here and the job settles on return ${correlation()}.`,
2019
2014
  );
2020
2015
  // Emit the turn counter on this path too (issue #229/#232): the aggregate
2021
2016
  // append timeout is the hardest transcript failure to diagnose, so record how
@@ -2029,7 +2024,7 @@ export function createAgentInstanceProducer(opts = {}) {
2029
2024
  `so far over ${mins}m (drain still pending — count unobserved)` +
2030
2025
  `${appendsDropped > 0 ? `, ${appendsDropped} turn(s) DROPPED (append backlog cap — transcript truncated)` : ''}, ` +
2031
2026
  `${messageChunksDropped > 0 ? `${messageChunksDropped} streaming chunk(s) DROPPED (message buffer cap), ` : ''}` +
2032
- `terminal COMPLETED update serialized behind pending appends.`,
2027
+ `final IDLE update serialized behind pending appends.`,
2033
2028
  );
2034
2029
  }
2035
2030
  return;
@@ -2037,25 +2032,27 @@ export function createAgentInstanceProducer(opts = {}) {
2037
2032
  let completedOk = false;
2038
2033
  let terminalErr = null;
2039
2034
  let terminalTimedOut = false;
2040
- // Issue #275: latches when the terminal update was rejected with a PERMANENT
2035
+ // Issue #275: latches when the final status update was rejected with a PERMANENT
2041
2036
  // (4xx) status — surfaced in the diagnostics so a fence/validation rejection
2042
- // (the stuck-instance case) is distinguishable from a transient 5xx/outage.
2037
+ // is distinguishable from a transient 5xx/outage.
2043
2038
  let terminalPermanent = false;
2044
2039
  const terminalAttempts = Math.max(1, terminalRetryMax);
2045
2040
  if (ok) {
2046
2041
  for (let attempt = 1; attempt <= terminalAttempts; attempt += 1) {
2047
2042
  try {
2048
- // BOUND each terminal update (finalizeTimeoutMs). Like the create calls,
2043
+ // BOUND each final status update (finalizeTimeoutMs). Like the create calls,
2049
2044
  // the harness awaits complete() before it settles the job, so a hung
2050
2045
  // updateAgentInstance would otherwise hold the lease until it expires and
2051
2046
  // stall settlement during an engine outage (issue #230).
2047
+ // Issue #274: send `IDLE` (valid AgentInstanceUpdateStatusEnum), never
2048
+ // `COMPLETED` — the engine owns `COMPLETED` on process-instance end.
2052
2049
  await callWithin(
2053
2050
  camunda[SDK_UPDATE]({
2054
2051
  agentInstanceKey,
2055
2052
  elementInstanceKey,
2056
2053
  jobKey,
2057
2054
  jobLease: leaseToken,
2058
- status: 'COMPLETED',
2055
+ status: 'IDLE',
2059
2056
  }),
2060
2057
  finalizeTimeoutMs,
2061
2058
  setTimer,
@@ -2070,10 +2067,9 @@ export function createAgentInstanceProducer(opts = {}) {
2070
2067
  // burns another finalizeTimeoutMs against the lease, so STOP retrying on a
2071
2068
  // timeout (the underlying call is left to settle in the background). A
2072
2069
  // plain rejection is potentially transient, so those still retry — EXCEPT
2073
- // a permanent 4xx (issue #275): a 422/409/400 terminal rejection (a stale
2074
- // lease fence, an invalid transition, a 0-turns husk the engine refuses to
2075
- // terminalize) will not succeed on a blind retry, so break immediately
2076
- // rather than burn the remaining attempts against a dying lease.
2070
+ // a permanent 4xx (issue #275/#274): a 422/409/400 rejection will not
2071
+ // succeed on a blind retry, so break immediately rather than burn the
2072
+ // remaining attempts against a dying lease.
2077
2073
  if (err && err.__nanoTimeout) {
2078
2074
  terminalTimedOut = true;
2079
2075
  break;
@@ -2087,39 +2083,29 @@ export function createAgentInstanceProducer(opts = {}) {
2087
2083
  }
2088
2084
  const terminalOk = ok && completedOk;
2089
2085
  if (ok && !completedOk) {
2090
- // The terminal transition never confirmed (every retry rejected, or a hung
2091
- // request was bounded out). The job settles now with no reactivation to try
2092
- // again, so this is NOT self-healing — surface the SDK error details and flag
2093
- // that manual reconciliation is required (issue #230).
2094
2086
  logger?.warn?.(
2095
- `AgentInstance ${agentInstanceKey}: terminal status update to COMPLETED ` +
2087
+ `AgentInstance ${agentInstanceKey}: final status update to IDLE ` +
2096
2088
  `${terminalTimedOut
2097
2089
  ? `timed out (bounded at ${finalizeTimeoutMs}ms)`
2098
2090
  : terminalPermanent
2099
2091
  ? `rejected permanently (HTTP ${terminalErr?.status ?? '4xx'}) — not retried`
2100
2092
  : `failed after ${terminalAttempts} attempt(s)`} — ` +
2101
2093
  `status=${terminalErr?.status ?? 'n/a'} ` +
2102
- `body=${terminalErr?.body ?? 'n/a'} message=${terminalErr?.message ?? 'n/a'}; ` +
2103
- `MANUAL RECONCILIATION REQUIRED — the job settles now with no reactivation to retry ` +
2104
- `this transition ${correlation()}.`,
2094
+ `body=${terminalErr?.body ?? 'n/a'} message=${terminalErr?.message ?? 'n/a'} ` +
2095
+ `${correlation()}.`,
2105
2096
  );
2106
2097
  }
2107
- // #229/#232 turn counter — "N turn(s) appended over Xm, <transition>" — so the
2108
- // 0-turns husk ("created but nothing ingested") is distinguishable from a healthy
2109
- // run at a glance. Render the HONEST terminal transition: only claim
2110
- // `status→COMPLETED` when the terminal update actually confirmed (terminalOk).
2111
- // The `!ok` (failed job end) path IS reactivatable, so it says so; but the
2112
- // terminalOk-false path is a SUCCESSFUL job end whose COMPLETED update failed —
2113
- // the caller settles the job the instant complete() returns, so there is NO
2114
- // reactivation to continue it (that would contradict the manual-reconciliation
2115
- // warning). State only that it was left non-terminal and needs reconciliation.
2098
+ // #229/#232/#274 turn counter — "N turn(s) appended over Xm, <transition>" —
2099
+ // render the HONEST final status transition: `status→IDLE` when confirmed
2100
+ // (the engine owns COMPLETED when the process instance ends), or report the
2101
+ // failed IDLE update if it was rejected.
2116
2102
  const elapsedMs = activatedAt ? Math.max(0, now() - activatedAt) : 0;
2117
2103
  const mins = (elapsedMs / 60000).toFixed(1);
2118
2104
  const transition = !ok
2119
- ? 'left non-terminal (retry/reactivation continues it)'
2105
+ ? 'left active (retry/reactivation continues it)'
2120
2106
  : terminalOk
2121
- ? 'status→COMPLETED'
2122
- : 'COMPLETED update FAILED — left non-terminal; manual reconciliation required (no reactivation on a settled job)';
2107
+ ? 'status→IDLE (engine completes on process end)'
2108
+ : 'IDLE update FAILED (engine completes on process end)';
2123
2109
  // Surface the aggregate append backlog DROP count (issue #230): the warn-once
2124
2110
  // backlog log only says drops STARTED — the operator can't quantify the loss or
2125
2111
  // tell it apart from an engine append failure without the final tally. Fold it
package/c8ctl-plugin.js CHANGED
@@ -49,6 +49,7 @@ import {
49
49
  utimesSync,
50
50
  mkdtempSync,
51
51
  closeSync,
52
+ readlinkSync,
52
53
  watchFile,
53
54
  unwatchFile,
54
55
  } from 'node:fs';
@@ -2705,6 +2706,170 @@ function sampleSubtreeCpu(rootPid) {
2705
2706
  return sampleSubtreeCpuPs(rootPid);
2706
2707
  }
2707
2708
 
2709
+ // ---- Workspace-CPU liveness + in-flight tool holds (#313) ------------------
2710
+ // The #130 subtree probe only sees processes INSIDE the agent's own process
2711
+ // tree. Real work often happens OUTSIDE that tree: `cargo` builds through a
2712
+ // shared `sccache` server (a daemon parented to launchd/init, not the agent),
2713
+ // and a long foreground bash tool call can compile/test silently for minutes
2714
+ // (nanobpm/nano-bpm#1664 attempt 1 was idle-killed at 216 turns / 156 min with
2715
+ // a build genuinely running). Two extra liveness signals close that gap:
2716
+ // 1. `sampleWorkspaceCpu` — aggregate CPU of ANY process whose cwd is inside
2717
+ // the agent's workspace (the sccache/rustc case: the daemon's compile
2718
+ // workers chdir into the worktree they compile for).
2719
+ // 2. an in-flight-tool-call hold — while the agent has an ACP tool call still
2720
+ // open (a bash call it started has not reported a terminal status) it is
2721
+ // treated as active; the call's own timeout / the absolute hard cap still
2722
+ // bound a genuinely hung call.
2723
+ // Both ride the monitor below as extra samplers consulted only during an idle
2724
+ // window, so a truly quiescent agent still ages out exactly as before.
2725
+
2726
+ /** Default upper bound for the workspace-cwd scan (pid count). */
2727
+ const WORKSPACE_CPU_SCAN_LIMIT = 4096;
2728
+
2729
+ /**
2730
+ * Canonicalize a workspace root for cwd prefix checks. The kernel reports a
2731
+ * process cwd as the PHYSICAL path (`/proc/<pid>/cwd`, lsof), so a root reached
2732
+ * through a symlink (e.g. `C8CTL_NANO_HOME` under a symlinked dir, or macOS
2733
+ * `/tmp` → `/private/tmp`) must be realpath'd or every in-workspace process
2734
+ * fails the prefix check. Returns the list of distinct roots to accept (the
2735
+ * physical one plus the lexical one), or `[]` when the input is unusable.
2736
+ */
2737
+ function workspaceRoots(workspaceDir, realpath = realpathSync) {
2738
+ if (typeof workspaceDir !== 'string' || !workspaceDir) return [];
2739
+ const lexical = resolvePath(workspaceDir);
2740
+ let physical = lexical;
2741
+ try { physical = realpath(lexical); } catch { /* missing dir → lexical only */ }
2742
+ return physical === lexical ? [lexical] : [physical, lexical];
2743
+ }
2744
+
2745
+ /** True when `cwd` is one of `roots` or lies beneath one. */
2746
+ function cwdInRoots(cwd, roots) {
2747
+ if (typeof cwd !== 'string' || !cwd) return false;
2748
+ return roots.some((root) => cwd === root || cwd.startsWith(root.endsWith(sep) ? root : root + sep));
2749
+ }
2750
+
2751
+ /**
2752
+ * Linux: aggregate CPU (utime+stime ticks) of every process whose cwd resolves
2753
+ * inside `workspaceDir` — regardless of parentage — via `/proc/<pid>/cwd`
2754
+ * symlinks. Returns `{ processes, cpu }` (cpu in clock ticks, matching the
2755
+ * `/proc` subtree sampler), or `null` when the workspace is unusable (so the
2756
+ * caller degrades to subtree-only liveness rather than mis-killing). A process
2757
+ * counted here is NOT a descendant of the agent, so it is a pure CPU-progress
2758
+ * signal, never a presence signal. Bounded by `scanLimit` pids so a wedged
2759
+ * /proc read can't stall the watchdog. `readCwd`/`listPids` are injectable for
2760
+ * tests.
2761
+ */
2762
+ function sampleWorkspaceCpuProc(workspaceDir, { readCwd, listPids, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
2763
+ const roots = workspaceRoots(workspaceDir, realpath);
2764
+ if (roots.length === 0) return null;
2765
+ const read = typeof readCwd === 'function' ? readCwd : (pid) => {
2766
+ try { return readlinkSync(`/proc/${pid}/cwd`); } catch { return null; }
2767
+ };
2768
+ let entries;
2769
+ if (typeof listPids === 'function') entries = listPids();
2770
+ else {
2771
+ try { entries = readdirSync('/proc'); } catch { return null; }
2772
+ }
2773
+ let processes = 0;
2774
+ let totalCpu = 0;
2775
+ let scanned = 0;
2776
+ for (const name of entries) {
2777
+ if (scanned >= scanLimit) break;
2778
+ if (!/^\d+$/.test(name)) continue;
2779
+ scanned += 1;
2780
+ const pid = Number(name);
2781
+ if (pid === process.pid || excludePids.includes(pid)) continue;
2782
+ const cwd = read(pid);
2783
+ if (typeof cwd !== 'string' || !cwd) continue;
2784
+ if (!cwdInRoots(cwd, roots)) continue;
2785
+ let stat;
2786
+ try { stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { continue; }
2787
+ const rparen = stat.lastIndexOf(')');
2788
+ if (rparen < 0) continue;
2789
+ const tail = stat.slice(rparen + 1).trim().split(/\s+/);
2790
+ const utime = Number(tail[11]);
2791
+ const stime = Number(tail[12]);
2792
+ totalCpu += (Number.isFinite(utime) ? utime : 0) + (Number.isFinite(stime) ? stime : 0);
2793
+ processes += 1;
2794
+ }
2795
+ return { processes, cpu: totalCpu };
2796
+ }
2797
+
2798
+ /**
2799
+ * Non-Linux fallback (macOS): aggregate CPU of processes whose cwd is inside
2800
+ * `workspaceDir`, via a bounded `lsof` + `ps`. Heavier (two short-lived
2801
+ * spawns) but runs only during an idle window. Returns
2802
+ * `{ processes, cpu, resolutionMs: 1000 }` (cpu in whole seconds, so the
2803
+ * coarse-resolution flat-CPU rule applies) or `null` when unavailable.
2804
+ */
2805
+ function sampleWorkspaceCpuLsof(workspaceDir, { runLsof, runPs, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
2806
+ const roots = workspaceRoots(workspaceDir, realpath);
2807
+ if (roots.length === 0) return null;
2808
+ const lsof = typeof runLsof === 'function' ? runLsof : () => {
2809
+ // `lsof -d cwd -Fpn` lists the cwd of EVERY process as machine-readable
2810
+ // `p<pid>` / `n<path>` records. A path filter (`-- <root>`) would only match
2811
+ // processes whose cwd is EXACTLY the root, missing a compiler working in a
2812
+ // nested crate dir, so we collect all cwd records and apply the
2813
+ // root-or-descendant check ourselves. Bounded by timeout + maxBuffer; lsof
2814
+ // exits 1 on partial permission errors, which still yields usable stdout.
2815
+ const r = spawnSync('lsof', ['-n', '-P', '-w', '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
2816
+ if (r.error && r.error.code === 'ENOENT') return null; // lsof missing → degrade
2817
+ if (r.error && !r.stdout) return null; // timeout / overflow with nothing usable
2818
+ return typeof r.stdout === 'string' ? r.stdout : '';
2819
+ };
2820
+ let lsofOut;
2821
+ try { lsofOut = lsof(); } catch { return null; }
2822
+ if (typeof lsofOut !== 'string') return null;
2823
+ const pids = [];
2824
+ let curPid = null;
2825
+ for (const line of lsofOut.split('\n')) {
2826
+ const tag = line.charCodeAt(0);
2827
+ if (tag === 112 /* 'p' */) {
2828
+ const pid = Number(line.slice(1));
2829
+ curPid = Number.isFinite(pid) && pid > 0 && pid !== process.pid && !excludePids.includes(pid) ? pid : null;
2830
+ } else if (tag === 110 /* 'n' */ && curPid != null) {
2831
+ if (cwdInRoots(line.slice(1), roots) && !pids.includes(curPid)) pids.push(curPid);
2832
+ curPid = null;
2833
+ if (pids.length >= scanLimit) break;
2834
+ }
2835
+ }
2836
+ if (pids.length === 0) return { processes: 0, cpu: 0, resolutionMs: 1000 };
2837
+ const ps = typeof runPs === 'function' ? runPs : (list) => {
2838
+ const r = spawnSync('ps', ['-p', list.join(','), '-o', 'pid=,time='], { encoding: 'utf8', timeout: 5_000 });
2839
+ // ps exits 1 when some pids vanished between scans; partial stdout is valid.
2840
+ return typeof r.stdout === 'string' ? r.stdout : null;
2841
+ };
2842
+ let psOut;
2843
+ try { psOut = ps(pids); } catch { return null; }
2844
+ if (typeof psOut !== 'string') return null;
2845
+ let processes = 0;
2846
+ let totalCpu = 0;
2847
+ for (const line of psOut.split('\n')) {
2848
+ const m = line.trim().match(/^(\d+)\s+(\S+)$/);
2849
+ if (!m) continue;
2850
+ totalCpu += parsePsTime(m[2]);
2851
+ processes += 1;
2852
+ }
2853
+ return { processes, cpu: totalCpu, resolutionMs: 1000 };
2854
+ }
2855
+
2856
+ /**
2857
+ * Sample aggregate CPU of processes working inside `workspaceDir` regardless of
2858
+ * parentage (the sccache/shared-daemon case, #313). Linux reads `/proc`; macOS
2859
+ * falls back to `lsof`+`ps`; other platforms return `null` (unsupported → the
2860
+ * monitor degrades to subtree-only liveness). The result shape mirrors the
2861
+ * subtree sampler (`cpu` in ticks on Linux, whole seconds under `ps`) with
2862
+ * `processes` instead of `descendants`.
2863
+ */
2864
+ function sampleWorkspaceCpu(workspaceDir, { excludePids } = {}) {
2865
+ // The agent itself runs with cwd == workspace; its own CPU (an idle harness
2866
+ // ticking timers) is NOT out-of-tree work and must never defer its idle-kill.
2867
+ const exclude = Array.isArray(excludePids) ? excludePids.filter((p) => Number.isFinite(p) && p > 0) : [];
2868
+ if (process.platform === 'linux') return sampleWorkspaceCpuProc(workspaceDir, { excludePids: exclude });
2869
+ if (process.platform === 'darwin') return sampleWorkspaceCpuLsof(workspaceDir, { excludePids: exclude });
2870
+ return null; // win32 et al: unsupported → degrade, never mis-kill
2871
+ }
2872
+
2708
2873
  /**
2709
2874
  * Build a progress-aware idle-liveness monitor shared by the pipe/PTY/ACP
2710
2875
  * capture paths. `arm()` is called on every output chunk (and once at start) to
@@ -2730,37 +2895,155 @@ function sampleSubtreeCpu(rootPid) {
2730
2895
  * read under a coarse sampler must NOT kill; the fine-grained `/proc` path
2731
2896
  * carries no resolution and so ages out on the first flat window as before →
2732
2897
  * `onIdleKill()`.
2898
+ * #313 extensions:
2899
+ * - `extraSamplers`: additional activity probes (e.g. the workspace-cwd CPU
2900
+ * scan covering out-of-tree sccache/rustc daemons). Each returns
2901
+ * `{ processes, cpu, resolutionMs? } | null`; advancing CPU on ANY sampler
2902
+ * defers the kill. Extra samplers are pure progress signals — they never
2903
+ * satisfy the presence gate (a childless agent with a busy workspace daemon
2904
+ * still needs its own live descendant to count as present... EXCEPT that a
2905
+ * workspace daemon burning CPU IS evidence the agent's work is in flight, so
2906
+ * when an extra sampler shows advancing CPU the kill is deferred even with
2907
+ * no live descendant: the agent may be blocked reading a pipe its daemon
2908
+ * fills. A flat/zero extra sample never kills on its own — presence is
2909
+ * judged from the subtree as before).
2910
+ * - `isActivityHold`: an in-flight-tool-call latch (ACP). While it returns
2911
+ * true the agent has an open tool call, so silence is expected; the kill is
2912
+ * deferred window-by-window. The absolute `--job-timeout` hard cap (and the
2913
+ * tool call's own timeout, enforced by the agent) remain the bound for a
2914
+ * genuinely hung call.
2915
+ * - `logger`: when given, the monitor logs WHY it considered the agent active
2916
+ * or idle at each probe (diagnosability, #313).
2733
2917
  * The first elapse with live descendants takes a baseline (a single sample can't
2734
2918
  * show movement) and re-probes after the recovery window. The absolute
2735
2919
  * `--job-timeout` hard cap remains the ultimate backstop, untouched by this.
2736
2920
  */
2737
- function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu }) {
2921
+ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu, extraSamplers = null, isActivityHold = null, maxActivityHoldMs, logger = null }) {
2738
2922
  const enabled = idleTimeoutMs && idleTimeoutMs > 0;
2739
2923
  const probeWindow = recoveryWindowMs && recoveryWindowMs > 0 ? recoveryWindowMs : idleTimeoutMs;
2740
2924
  let timer = null;
2741
2925
  let prev = null; // last subtree CPU probe taken during the current silent stretch
2926
+ let prevExtra = null; // last per-extra-sampler CPU readings (array parallel to extraSamplers)
2927
+ let extraBaselined = false; // true once the extra samplers have a baseline to compare against
2928
+ let extraResolutionMs = 0; // coarsest resolution among live extra samples at the last probe
2742
2929
  let flatMs = 0; // cumulative time observed with flat CPU since the last movement/baseline
2743
2930
  let lastWindow = idleTimeoutMs; // duration of the currently pending probe window
2931
+ let extraFlatMs = 0; // cumulative flat time across the extra (workspace) samplers
2932
+ let holdMs = 0; // cumulative silent time deferred by the activity hold this stretch
2933
+ // #313: the activity hold must be FINITE — `nano work` defaults to no hard cap,
2934
+ // so an announced-then-hung tool call would otherwise defer forever. Default:
2935
+ // 4 idle windows (configurable). Past it, the hold is ignored and the normal
2936
+ // CPU probes decide (real silent work still defers via CPU progress).
2937
+ const holdCap = Number.isFinite(maxActivityHoldMs) && maxActivityHoldMs >= 0 ? maxActivityHoldMs : (idleTimeoutMs > 0 ? idleTimeoutMs * 4 : 0);
2744
2938
  const settled = () => (typeof isSettled === 'function' ? isSettled() : false);
2745
2939
  const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
2746
2940
  const schedule = (ms) => { lastWindow = ms; timer = setTimeout(onElapse, ms); };
2941
+ const log = (msg) => { try { logger?.info?.(`[idle-watchdog] ${msg}`); } catch { /* best effort */ } };
2942
+ const extras = Array.isArray(extraSamplers) ? extraSamplers.filter((s) => typeof s === 'function') : [];
2943
+ const held = () => { try { return typeof isActivityHold === 'function' && isActivityHold(); } catch { return false; } };
2747
2944
  function onElapse() {
2748
2945
  timer = null;
2749
2946
  if (settled()) return;
2750
2947
  const elapsed = lastWindow;
2948
+ // #313: an in-flight tool call (the agent started a bash call that has not
2949
+ // reported a terminal status) means the silence is expected — defer. The
2950
+ // call's own timeout / the absolute hard cap still bound a hung call.
2951
+ if (held()) {
2952
+ if (holdMs < holdCap) {
2953
+ holdMs += elapsed;
2954
+ log(`silent for >= ${elapsed}ms but a tool call is still in flight; deferring the idle-kill (hold ${holdMs}/${holdCap}ms)`);
2955
+ prev = null; prevExtra = null; extraBaselined = false; flatMs = 0; extraFlatMs = 0;
2956
+ schedule(probeWindow);
2957
+ return;
2958
+ }
2959
+ log(`a tool call has been in flight for >= ${holdMs}ms (hold cap ${holdCap}ms); falling back to CPU-progress probes`);
2960
+ }
2751
2961
  const pid = typeof getPid === 'function' ? getPid() : undefined;
2752
2962
  const sample = pid != null ? sampleCpu(pid) : null;
2753
- // Childless / unsampleable / dead tree → silence really is a hang: kill.
2754
- if (!sample || sample.descendants <= 0) { onIdleKill(); return; }
2963
+ // #313: consult the extra (workspace-cwd) samplers. Advancing CPU on any of
2964
+ // them is work attributable to this agent even when it lives outside the
2965
+ // agent's process tree (a shared sccache daemon), so it defers the kill —
2966
+ // INCLUDING when the subtree itself looks childless/quiescent.
2967
+ let extraActive = false;
2968
+ if (extras.length) {
2969
+ const next = [];
2970
+ extraResolutionMs = 0;
2971
+ for (let i = 0; i < extras.length; i += 1) {
2972
+ let s = null;
2973
+ try { s = extras[i](); } catch { s = null; }
2974
+ next.push(s);
2975
+ if (!s || !Number.isFinite(s.cpu)) continue;
2976
+ if (s.processes > 0 && Number.isFinite(s.resolutionMs)) extraResolutionMs = Math.max(extraResolutionMs, s.resolutionMs);
2977
+ const before = prevExtra && prevExtra[i];
2978
+ if (before != null && Number.isFinite(before.cpu) && s.cpu !== before.cpu) extraActive = true;
2979
+ }
2980
+ const hadBaseline = prevExtra != null;
2981
+ const prevReadings = prevExtra || [];
2982
+ prevExtra = next;
2983
+ if (extraActive) {
2984
+ log(`silent for >= ${elapsed}ms but a workspace process outside the agent tree is burning CPU (e.g. sccache/rustc); deferring the idle-kill`);
2985
+ prev = sample; flatMs = 0; extraFlatMs = 0;
2986
+ schedule(probeWindow);
2987
+ return;
2988
+ }
2989
+ // Flat time only counts while a workspace sampler stays LIVE (a reading
2990
+ // with processes > 0) across consecutive probes. Empty / unreadable
2991
+ // readings must not pre-pay a sampler's resolution allowance: when any
2992
+ // sampler transitions absent/unreadable → live, restart the accumulator
2993
+ // so a fresh process gets its full resolution window before a kill.
2994
+ const isLive = (r) => r != null && Number.isFinite(r.cpu) && r.processes > 0;
2995
+ let newlyLive = false; let continuedLive = false;
2996
+ for (let i = 0; i < next.length; i += 1) {
2997
+ if (!isLive(next[i])) continue;
2998
+ if (hadBaseline && isLive(prevReadings[i])) continuedLive = true; else newlyLive = true;
2999
+ }
3000
+ if (newlyLive || !continuedLive) extraFlatMs = 0;
3001
+ else extraFlatMs += elapsed;
3002
+ }
3003
+ // A live workspace process read flat, but a coarse sampler (whole-second
3004
+ // `ps`) can read flat across a sub-resolution window while still working:
3005
+ // only treat workspace CPU as idle once the flat time covers its resolution.
3006
+ const extraIdleConfirmed = extraFlatMs >= extraResolutionMs;
3007
+ // Childless / unsampleable / dead tree → silence is very likely a hang.
3008
+ // But with extra (workspace) samplers wired we cannot yet distinguish "no
3009
+ // workspace work" from "a workspace daemon whose baseline we have not
3010
+ // taken" on a single reading — so take ONE baseline probe window before
3011
+ // killing, exactly like the subtree first-elapse baseline below. Without
3012
+ // extra samplers (or once a baseline shows no workspace activity) the
3013
+ // legacy immediate kill applies.
3014
+ if (!sample || sample.descendants <= 0) {
3015
+ if (extras.length && !extraBaselined) {
3016
+ // Consume the baseline: the NEXT elapse compares against prevExtra and
3017
+ // either defers (workspace CPU moved) or kills (flat).
3018
+ extraBaselined = true;
3019
+ log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; baselining the workspace CPU probe and re-checking in ${probeWindow}ms`);
3020
+ schedule(probeWindow);
3021
+ return;
3022
+ }
3023
+ if (!extraIdleConfirmed) {
3024
+ log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; workspace CPU flat for ${extraFlatMs}ms < sampler resolution ${extraResolutionMs}ms, re-checking in ${probeWindow}ms`);
3025
+ schedule(probeWindow);
3026
+ return;
3027
+ }
3028
+ log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'} and no workspace CPU; idle-killing`);
3029
+ onIdleKill();
3030
+ return;
3031
+ }
2755
3032
  if (prev != null) {
2756
3033
  // Aggregate CPU advanced across the window → the tree is doing work; defer.
2757
- if (sample.cpu > prev.cpu) { prev = sample; flatMs = 0; schedule(probeWindow); return; }
3034
+ if (sample.cpu > prev.cpu) {
3035
+ log(`silent for >= ${elapsed}ms but subtree CPU advanced (${prev.cpu} → ${sample.cpu}); deferring the idle-kill`);
3036
+ prev = sample; flatMs = 0; schedule(probeWindow); return;
3037
+ }
2758
3038
  // Aggregate CPU REGRESSED → `sampleSubtreeCpu*` only sums CPU for live
2759
3039
  // PIDs, so the total can legitimately drop when a busy descendant exits
2760
3040
  // (its ticks stop counting) even though the subtree just made progress.
2761
3041
  // A regression is activity, not a stall: re-baseline and defer rather
2762
3042
  // than mis-killing a silent-but-busy job.
2763
- if (sample.cpu < prev.cpu) { prev = sample; flatMs = 0; schedule(probeWindow); return; }
3043
+ if (sample.cpu < prev.cpu) {
3044
+ log(`silent for >= ${elapsed}ms but subtree CPU regressed (${prev.cpu} → ${sample.cpu}; a busy descendant exited) — progress, not a stall; deferring the idle-kill`);
3045
+ prev = sample; flatMs = 0; schedule(probeWindow); return;
3046
+ }
2764
3047
  // Live descendants but CPU exactly unchanged across a full window → a
2765
3048
  // candidate stall. Only confirm it once we have observed flat CPU for at
2766
3049
  // least the sampler's CPU-time resolution: a probe window shorter than the
@@ -2768,12 +3051,17 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
2768
3051
  // low-churn subtree is still progressing but hasn't crossed the next tick.
2769
3052
  flatMs += elapsed;
2770
3053
  const resolutionMs = Number.isFinite(sample.resolutionMs) ? sample.resolutionMs : 0;
2771
- if (flatMs >= resolutionMs) { onIdleKill(); return; }
3054
+ if (flatMs >= resolutionMs && extraIdleConfirmed) {
3055
+ log(`silent for >= ${elapsed}ms with a live but CPU-flat subtree (${sample.descendants} descendant(s), cpu ${sample.cpu}) and no workspace CPU; idle-killing`);
3056
+ onIdleKill();
3057
+ return;
3058
+ }
2772
3059
  schedule(probeWindow);
2773
3060
  return;
2774
3061
  }
2775
3062
  // First elapse with live descendants: baseline, then re-probe (can't judge
2776
3063
  // progress from one sample).
3064
+ log(`silent for >= ${elapsed}ms with ${sample.descendants} live descendant(s); baselining CPU (${sample.cpu}) and re-probing in ${probeWindow}ms`);
2777
3065
  prev = sample;
2778
3066
  flatMs = 0;
2779
3067
  schedule(probeWindow);
@@ -2783,7 +3071,12 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
2783
3071
  arm() {
2784
3072
  if (!enabled || settled()) return;
2785
3073
  prev = null; // output resumed → a fresh silent stretch starts
3074
+ prevExtra = null;
3075
+ extraBaselined = false;
2786
3076
  flatMs = 0;
3077
+ extraFlatMs = 0;
3078
+ extraResolutionMs = 0;
3079
+ holdMs = 0;
2787
3080
  clear();
2788
3081
  schedule(idleTimeoutMs);
2789
3082
  },
@@ -8796,6 +9089,11 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
8796
9089
  idleTimeoutMs,
8797
9090
  recoveryWindowMs,
8798
9091
  isSettled: () => settled,
9092
+ logger: getLogger(),
9093
+ // #313: count CPU burned by any process whose cwd is inside the agent's
9094
+ // workspace (a shared sccache daemon lives outside the agent's tree), so
9095
+ // a silent compile holds off the idle-kill.
9096
+ extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
8799
9097
  onIdleKill: () => {
8800
9098
  try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
8801
9099
  finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
@@ -8968,6 +9266,11 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
8968
9266
  idleTimeoutMs,
8969
9267
  recoveryWindowMs,
8970
9268
  isSettled: () => settled,
9269
+ logger: getLogger(),
9270
+ // #313: count CPU burned by any process whose cwd is inside the agent's
9271
+ // workspace (a shared sccache daemon lives outside the agent's tree), so
9272
+ // a silent compile holds off the idle-kill.
9273
+ extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [term?.pid] })] : null,
8971
9274
  onIdleKill: () => {
8972
9275
  killTerm();
8973
9276
  finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
@@ -9342,6 +9645,28 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9342
9645
  // flips this; the floor never does.
9343
9646
  let sawRealTranscript = false;
9344
9647
 
9648
+ // #313: ACP tool calls currently in flight (`toolCallId` → true). A
9649
+ // `tool_call` notification with a non-terminal status opens the call; a
9650
+ // `tool_call`/`tool_call_update` with a terminal status closes it. While
9651
+ // any call is open the idle watchdog treats the agent as ACTIVE (a long
9652
+ // silent bash call is work, not a hang) — the call's own timeout and the
9653
+ // absolute job hard cap still bound a genuinely hung call.
9654
+ const inFlightToolCalls = new Map();
9655
+ const ACP_TOOL_TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled', 'canceled']);
9656
+ const noteToolCallUpdate = (update) => {
9657
+ if (!update || typeof update !== 'object') return;
9658
+ const kind = update.sessionUpdate || update.type;
9659
+ if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
9660
+ const id = update.toolCallId;
9661
+ if (id == null) return;
9662
+ const status = typeof update.status === 'string' ? update.status.toLowerCase() : '';
9663
+ if (ACP_TOOL_TERMINAL_STATUSES.has(status)) inFlightToolCalls.delete(id);
9664
+ else if (!inFlightToolCalls.has(id)) {
9665
+ inFlightToolCalls.set(id, true);
9666
+ try { logger.info?.(`[idle-watchdog] tool call in flight (${update.title || id}${status ? `, status ${status}` : ''}) — the idle-kill is held while it runs`); } catch { /* best effort */ }
9667
+ }
9668
+ };
9669
+
9345
9670
  // Live "spy" tee (--stream), line-buffered, mirroring the other paths.
9346
9671
  // Separate line buffers per lane (stdout-human vs stderr) so a partial line
9347
9672
  // on one lane never interleaves mid-line with the other — matching pipe/PTY.
@@ -9670,6 +9995,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9670
9995
  // A request or notification FROM the agent.
9671
9996
  if (typeof msg.method === 'string') {
9672
9997
  if (msg.method === 'session/update') {
9998
+ noteToolCallUpdate(msg.params?.update);
9673
9999
  emitTranscript(msg.params?.update);
9674
10000
  // #194: durable engine-native producer seam. In parallel with the
9675
10001
  // transcript-chunk relay overlay, hand the SAME raw `session/update` to
@@ -9720,6 +10046,16 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9720
10046
  idleTimeoutMs,
9721
10047
  recoveryWindowMs,
9722
10048
  isSettled: () => settled,
10049
+ logger,
10050
+ // #313: while an ACP tool call is in flight the agent is active even
10051
+ // when totally silent (a long foreground bash compile/test). Bounded by
10052
+ // the call's own timeout and the absolute job hard cap.
10053
+ isActivityHold: () => inFlightToolCalls.size > 0,
10054
+ // #313: also count CPU burned by processes whose cwd is inside the
10055
+ // agent's workspace, wherever they sit in the process tree — a shared
10056
+ // sccache daemon (parented to launchd/init, not the agent) compiling
10057
+ // this worktree is real work and must hold off the idle-kill.
10058
+ extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
9723
10059
  onIdleKill: () => {
9724
10060
  try { killTree(child); } catch { /* best effort */ }
9725
10061
  finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
@@ -21398,6 +21734,9 @@ export {
21398
21734
  spawnCapturePty,
21399
21735
  spawnCaptureAcp,
21400
21736
  sampleSubtreeCpu,
21737
+ sampleWorkspaceCpu,
21738
+ sampleWorkspaceCpuProc,
21739
+ sampleWorkspaceCpuLsof,
21401
21740
  createIdleLivenessMonitor,
21402
21741
  resolveLivenessOverrides,
21403
21742
  parsePsTime,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "c8ctl-plugin-nano",
3
- "version": "1.73.0",
3
+ "version": "1.73.2",
4
4
  "type": "module",
5
5
  "description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
6
6
  "main": "c8ctl-plugin.js",
@@ -76,12 +76,12 @@
76
76
  },
77
77
  "optionalDependencies": {
78
78
  "node-pty": "^1.0.0",
79
- "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.73.0",
80
- "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.73.0",
81
- "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.73.0",
82
- "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.73.0",
83
- "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.73.0",
84
- "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.73.0",
85
- "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.73.0"
79
+ "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.73.2",
80
+ "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.73.2",
81
+ "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.73.2",
82
+ "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.73.2",
83
+ "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.73.2",
84
+ "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.73.2",
85
+ "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.73.2"
86
86
  }
87
87
  }