c8ctl-plugin-nano 1.73.0 → 1.73.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-instance.mjs +46 -60
- package/c8ctl-plugin.js +345 -6
- package/package.json +8 -8
package/agent-instance.mjs
CHANGED
|
@@ -62,11 +62,13 @@ const DEFAULT_CREATE_RETRY_MAX_MS = 30000;
|
|
|
62
62
|
const DEFAULT_PRE_MINT_BUFFER_MAX = 1000;
|
|
63
63
|
const DEFAULT_PRE_MINT_BUFFER_MAX_BYTES = 8_000_000;
|
|
64
64
|
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
//
|
|
65
|
+
// On a successful job end, complete() drains pending appends and drives a final
|
|
66
|
+
// `status: 'IDLE'` update directly (not via the best-effort append queue) and
|
|
67
|
+
// RETRIES transient failures a bounded number of times. Per Camunda 8.10 and
|
|
68
|
+
// nanobpmn parity (#274), AgentInstanceUpdateStatusEnum is
|
|
69
|
+
// { IDLE, THINKING, TOOL_CALLING, TOOL_DISCOVERY } — UPDATE never accepts
|
|
70
|
+
// COMPLETED (that returns HTTP 422); the engine itself completes all active
|
|
71
|
+
// AgentInstances when the owning process instance ends.
|
|
70
72
|
const DEFAULT_TERMINAL_RETRY_MAX = 3;
|
|
71
73
|
|
|
72
74
|
// Bound on complete()'s last-chance create attempt (issue #230). The harness awaits
|
|
@@ -261,13 +263,13 @@ export function describeSdkError(err) {
|
|
|
261
263
|
}
|
|
262
264
|
|
|
263
265
|
/**
|
|
264
|
-
* True when an HTTP status marks a PERMANENT rejection of the
|
|
265
|
-
* status→
|
|
266
|
-
* transition, a
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
270
|
-
*
|
|
266
|
+
* True when an HTTP status marks a PERMANENT rejection of the final
|
|
267
|
+
* status→IDLE update (issue #275, #274): a 4xx (a stale lease fence, an invalid
|
|
268
|
+
* transition, a contract error like 422) will not succeed on a blind retry, so
|
|
269
|
+
* the producer stops retrying it immediately instead of burning the remaining
|
|
270
|
+
* attempts against a dying lease. 5xx / network / timeout failures are potentially
|
|
271
|
+
* transient and still retry. `null`/unknown statuses are treated as transient
|
|
272
|
+
* (retry) — fail-safe toward recovery, not toward giving up.
|
|
271
273
|
*
|
|
272
274
|
* Two 4xx statuses are the exception — NOT permanent, because a retry can still
|
|
273
275
|
* succeed:
|
|
@@ -633,7 +635,8 @@ function contentForResult(result) {
|
|
|
633
635
|
* - `ingest(rawUpdate)` — feed one raw ACP `session/update` (non-blocking; the SDK
|
|
634
636
|
* append is enqueued so the ACP hot path never blocks).
|
|
635
637
|
* - `complete(ok)` — drain pending appends, then (on a successful job end) update
|
|
636
|
-
* the instance status to
|
|
638
|
+
* the instance status to IDLE (the engine owns COMPLETED when the process
|
|
639
|
+
* instance ends; issue #274). `job.complete` is unchanged and orthogonal.
|
|
637
640
|
*
|
|
638
641
|
* @param {object} opts
|
|
639
642
|
* @param {object} opts.camunda Host SDK client (createAgentInstance/updateAgentInstance).
|
|
@@ -1867,8 +1870,10 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
1867
1870
|
/**
|
|
1868
1871
|
* End the AgentInstance lifecycle. Flushes any pending message turn, drains the
|
|
1869
1872
|
* append queue, then — only on a SUCCESSFUL job end (`ok`) — updates the instance
|
|
1870
|
-
* status to
|
|
1871
|
-
*
|
|
1873
|
+
* status to `IDLE` (a valid update-status; Camunda 8.10 and nanobpmn reject
|
|
1874
|
+
* `COMPLETED` on UPDATE with HTTP 422 because `COMPLETED` is engine-owned and
|
|
1875
|
+
* written when the owning process instance ends, #274). On a failed run the
|
|
1876
|
+
* instance is left at its last active status so a retry/reactivation continues
|
|
1872
1877
|
* the same instance. Best-effort; never throws; `job.complete` is unaffected.
|
|
1873
1878
|
*/
|
|
1874
1879
|
async complete(ok = true) {
|
|
@@ -1986,36 +1991,26 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
1986
1991
|
elementInstanceKey,
|
|
1987
1992
|
jobKey,
|
|
1988
1993
|
jobLease: leaseToken,
|
|
1989
|
-
status: '
|
|
1994
|
+
status: 'IDLE',
|
|
1990
1995
|
}),
|
|
1991
1996
|
finalizeTimeoutMs,
|
|
1992
1997
|
setTimer,
|
|
1993
1998
|
);
|
|
1994
1999
|
} catch (err) {
|
|
1995
|
-
// This serialized enqueue IS the terminal transition on the drain-timeout
|
|
1996
|
-
// path — the only actor that can still terminalize the instance here. The
|
|
1997
|
-
// generic enqueue() catch would swallow it at debug with just err.message,
|
|
1998
|
-
// losing the HTTP status/body/correlation on the very path that already
|
|
1999
|
-
// warned reconciliation may be needed (issue #230). Emit the SAME shaped
|
|
2000
|
-
// status/body/correlation warning the direct terminal-retry path uses
|
|
2001
|
-
// (with its manual-reconciliation outcome) before it is swallowed.
|
|
2002
2000
|
const d = describeSdkError(err);
|
|
2003
2001
|
logger?.warn?.(
|
|
2004
|
-
`AgentInstance ${agentInstanceKey}: serialized
|
|
2002
|
+
`AgentInstance ${agentInstanceKey}: serialized final IDLE update ` +
|
|
2005
2003
|
`${err?.__nanoTimeout ? `timed out (bounded at ${finalizeTimeoutMs}ms)` : 'failed'} — ` +
|
|
2006
|
-
`status=${d.status ?? 'n/a'} body=${d.body ?? 'n/a'} message=${d.message ?? 'n/a'}
|
|
2007
|
-
|
|
2008
|
-
`settled with no reactivation to retry this transition ${correlation()}.`,
|
|
2004
|
+
`status=${d.status ?? 'n/a'} body=${d.body ?? 'n/a'} message=${d.message ?? 'n/a'} ` +
|
|
2005
|
+
`${correlation()}.`,
|
|
2009
2006
|
);
|
|
2010
2007
|
}
|
|
2011
2008
|
});
|
|
2012
2009
|
logger?.warn?.(
|
|
2013
2010
|
`AgentInstance ${agentInstanceKey}: history drain did not settle within ` +
|
|
2014
|
-
`${finalizeTimeoutMs}ms —
|
|
2011
|
+
`${finalizeTimeoutMs}ms — final IDLE update SERIALIZED behind the ` +
|
|
2015
2012
|
`pending appends (best-effort, ordered) rather than racing them; its outcome ` +
|
|
2016
|
-
`is not observed here and the job settles on return
|
|
2017
|
-
`is expected to be fence-rejected once its lease is void — MANUAL ` +
|
|
2018
|
-
`RECONCILIATION REQUIRED (terminal status NOT confirmed) ${correlation()}.`,
|
|
2013
|
+
`is not observed here and the job settles on return ${correlation()}.`,
|
|
2019
2014
|
);
|
|
2020
2015
|
// Emit the turn counter on this path too (issue #229/#232): the aggregate
|
|
2021
2016
|
// append timeout is the hardest transcript failure to diagnose, so record how
|
|
@@ -2029,7 +2024,7 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
2029
2024
|
`so far over ${mins}m (drain still pending — count unobserved)` +
|
|
2030
2025
|
`${appendsDropped > 0 ? `, ${appendsDropped} turn(s) DROPPED (append backlog cap — transcript truncated)` : ''}, ` +
|
|
2031
2026
|
`${messageChunksDropped > 0 ? `${messageChunksDropped} streaming chunk(s) DROPPED (message buffer cap), ` : ''}` +
|
|
2032
|
-
`
|
|
2027
|
+
`final IDLE update serialized behind pending appends.`,
|
|
2033
2028
|
);
|
|
2034
2029
|
}
|
|
2035
2030
|
return;
|
|
@@ -2037,25 +2032,27 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
2037
2032
|
let completedOk = false;
|
|
2038
2033
|
let terminalErr = null;
|
|
2039
2034
|
let terminalTimedOut = false;
|
|
2040
|
-
// Issue #275: latches when the
|
|
2035
|
+
// Issue #275: latches when the final status update was rejected with a PERMANENT
|
|
2041
2036
|
// (4xx) status — surfaced in the diagnostics so a fence/validation rejection
|
|
2042
|
-
//
|
|
2037
|
+
// is distinguishable from a transient 5xx/outage.
|
|
2043
2038
|
let terminalPermanent = false;
|
|
2044
2039
|
const terminalAttempts = Math.max(1, terminalRetryMax);
|
|
2045
2040
|
if (ok) {
|
|
2046
2041
|
for (let attempt = 1; attempt <= terminalAttempts; attempt += 1) {
|
|
2047
2042
|
try {
|
|
2048
|
-
// BOUND each
|
|
2043
|
+
// BOUND each final status update (finalizeTimeoutMs). Like the create calls,
|
|
2049
2044
|
// the harness awaits complete() before it settles the job, so a hung
|
|
2050
2045
|
// updateAgentInstance would otherwise hold the lease until it expires and
|
|
2051
2046
|
// stall settlement during an engine outage (issue #230).
|
|
2047
|
+
// Issue #274: send `IDLE` (valid AgentInstanceUpdateStatusEnum), never
|
|
2048
|
+
// `COMPLETED` — the engine owns `COMPLETED` on process-instance end.
|
|
2052
2049
|
await callWithin(
|
|
2053
2050
|
camunda[SDK_UPDATE]({
|
|
2054
2051
|
agentInstanceKey,
|
|
2055
2052
|
elementInstanceKey,
|
|
2056
2053
|
jobKey,
|
|
2057
2054
|
jobLease: leaseToken,
|
|
2058
|
-
status: '
|
|
2055
|
+
status: 'IDLE',
|
|
2059
2056
|
}),
|
|
2060
2057
|
finalizeTimeoutMs,
|
|
2061
2058
|
setTimer,
|
|
@@ -2070,10 +2067,9 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
2070
2067
|
// burns another finalizeTimeoutMs against the lease, so STOP retrying on a
|
|
2071
2068
|
// timeout (the underlying call is left to settle in the background). A
|
|
2072
2069
|
// plain rejection is potentially transient, so those still retry — EXCEPT
|
|
2073
|
-
// a permanent 4xx (issue #275): a 422/409/400
|
|
2074
|
-
//
|
|
2075
|
-
//
|
|
2076
|
-
// rather than burn the remaining attempts against a dying lease.
|
|
2070
|
+
// a permanent 4xx (issue #275/#274): a 422/409/400 rejection will not
|
|
2071
|
+
// succeed on a blind retry, so break immediately rather than burn the
|
|
2072
|
+
// remaining attempts against a dying lease.
|
|
2077
2073
|
if (err && err.__nanoTimeout) {
|
|
2078
2074
|
terminalTimedOut = true;
|
|
2079
2075
|
break;
|
|
@@ -2087,39 +2083,29 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
2087
2083
|
}
|
|
2088
2084
|
const terminalOk = ok && completedOk;
|
|
2089
2085
|
if (ok && !completedOk) {
|
|
2090
|
-
// The terminal transition never confirmed (every retry rejected, or a hung
|
|
2091
|
-
// request was bounded out). The job settles now with no reactivation to try
|
|
2092
|
-
// again, so this is NOT self-healing — surface the SDK error details and flag
|
|
2093
|
-
// that manual reconciliation is required (issue #230).
|
|
2094
2086
|
logger?.warn?.(
|
|
2095
|
-
`AgentInstance ${agentInstanceKey}:
|
|
2087
|
+
`AgentInstance ${agentInstanceKey}: final status update to IDLE ` +
|
|
2096
2088
|
`${terminalTimedOut
|
|
2097
2089
|
? `timed out (bounded at ${finalizeTimeoutMs}ms)`
|
|
2098
2090
|
: terminalPermanent
|
|
2099
2091
|
? `rejected permanently (HTTP ${terminalErr?.status ?? '4xx'}) — not retried`
|
|
2100
2092
|
: `failed after ${terminalAttempts} attempt(s)`} — ` +
|
|
2101
2093
|
`status=${terminalErr?.status ?? 'n/a'} ` +
|
|
2102
|
-
`body=${terminalErr?.body ?? 'n/a'} message=${terminalErr?.message ?? 'n/a'}
|
|
2103
|
-
|
|
2104
|
-
`this transition ${correlation()}.`,
|
|
2094
|
+
`body=${terminalErr?.body ?? 'n/a'} message=${terminalErr?.message ?? 'n/a'} ` +
|
|
2095
|
+
`${correlation()}.`,
|
|
2105
2096
|
);
|
|
2106
2097
|
}
|
|
2107
|
-
// #229/#232 turn counter — "N turn(s) appended over Xm, <transition>" —
|
|
2108
|
-
//
|
|
2109
|
-
//
|
|
2110
|
-
//
|
|
2111
|
-
// The `!ok` (failed job end) path IS reactivatable, so it says so; but the
|
|
2112
|
-
// terminalOk-false path is a SUCCESSFUL job end whose COMPLETED update failed —
|
|
2113
|
-
// the caller settles the job the instant complete() returns, so there is NO
|
|
2114
|
-
// reactivation to continue it (that would contradict the manual-reconciliation
|
|
2115
|
-
// warning). State only that it was left non-terminal and needs reconciliation.
|
|
2098
|
+
// #229/#232/#274 turn counter — "N turn(s) appended over Xm, <transition>" —
|
|
2099
|
+
// render the HONEST final status transition: `status→IDLE` when confirmed
|
|
2100
|
+
// (the engine owns COMPLETED when the process instance ends), or report the
|
|
2101
|
+
// failed IDLE update if it was rejected.
|
|
2116
2102
|
const elapsedMs = activatedAt ? Math.max(0, now() - activatedAt) : 0;
|
|
2117
2103
|
const mins = (elapsedMs / 60000).toFixed(1);
|
|
2118
2104
|
const transition = !ok
|
|
2119
|
-
? 'left
|
|
2105
|
+
? 'left active (retry/reactivation continues it)'
|
|
2120
2106
|
: terminalOk
|
|
2121
|
-
? 'status→
|
|
2122
|
-
: '
|
|
2107
|
+
? 'status→IDLE (engine completes on process end)'
|
|
2108
|
+
: 'IDLE update FAILED (engine completes on process end)';
|
|
2123
2109
|
// Surface the aggregate append backlog DROP count (issue #230): the warn-once
|
|
2124
2110
|
// backlog log only says drops STARTED — the operator can't quantify the loss or
|
|
2125
2111
|
// tell it apart from an engine append failure without the final tally. Fold it
|
package/c8ctl-plugin.js
CHANGED
|
@@ -49,6 +49,7 @@ import {
|
|
|
49
49
|
utimesSync,
|
|
50
50
|
mkdtempSync,
|
|
51
51
|
closeSync,
|
|
52
|
+
readlinkSync,
|
|
52
53
|
watchFile,
|
|
53
54
|
unwatchFile,
|
|
54
55
|
} from 'node:fs';
|
|
@@ -2705,6 +2706,170 @@ function sampleSubtreeCpu(rootPid) {
|
|
|
2705
2706
|
return sampleSubtreeCpuPs(rootPid);
|
|
2706
2707
|
}
|
|
2707
2708
|
|
|
2709
|
+
// ---- Workspace-CPU liveness + in-flight tool holds (#313) ------------------
|
|
2710
|
+
// The #130 subtree probe only sees processes INSIDE the agent's own process
|
|
2711
|
+
// tree. Real work often happens OUTSIDE that tree: `cargo` builds through a
|
|
2712
|
+
// shared `sccache` server (a daemon parented to launchd/init, not the agent),
|
|
2713
|
+
// and a long foreground bash tool call can compile/test silently for minutes
|
|
2714
|
+
// (nanobpm/nano-bpm#1664 attempt 1 was idle-killed at 216 turns / 156 min with
|
|
2715
|
+
// a build genuinely running). Two extra liveness signals close that gap:
|
|
2716
|
+
// 1. `sampleWorkspaceCpu` — aggregate CPU of ANY process whose cwd is inside
|
|
2717
|
+
// the agent's workspace (the sccache/rustc case: the daemon's compile
|
|
2718
|
+
// workers chdir into the worktree they compile for).
|
|
2719
|
+
// 2. an in-flight-tool-call hold — while the agent has an ACP tool call still
|
|
2720
|
+
// open (a bash call it started has not reported a terminal status) it is
|
|
2721
|
+
// treated as active; the call's own timeout / the absolute hard cap still
|
|
2722
|
+
// bound a genuinely hung call.
|
|
2723
|
+
// Both ride the monitor below as extra samplers consulted only during an idle
|
|
2724
|
+
// window, so a truly quiescent agent still ages out exactly as before.
|
|
2725
|
+
|
|
2726
|
+
/** Default upper bound for the workspace-cwd scan (pid count). */
|
|
2727
|
+
const WORKSPACE_CPU_SCAN_LIMIT = 4096;
|
|
2728
|
+
|
|
2729
|
+
/**
|
|
2730
|
+
* Canonicalize a workspace root for cwd prefix checks. The kernel reports a
|
|
2731
|
+
* process cwd as the PHYSICAL path (`/proc/<pid>/cwd`, lsof), so a root reached
|
|
2732
|
+
* through a symlink (e.g. `C8CTL_NANO_HOME` under a symlinked dir, or macOS
|
|
2733
|
+
* `/tmp` → `/private/tmp`) must be realpath'd or every in-workspace process
|
|
2734
|
+
* fails the prefix check. Returns the list of distinct roots to accept (the
|
|
2735
|
+
* physical one plus the lexical one), or `[]` when the input is unusable.
|
|
2736
|
+
*/
|
|
2737
|
+
function workspaceRoots(workspaceDir, realpath = realpathSync) {
|
|
2738
|
+
if (typeof workspaceDir !== 'string' || !workspaceDir) return [];
|
|
2739
|
+
const lexical = resolvePath(workspaceDir);
|
|
2740
|
+
let physical = lexical;
|
|
2741
|
+
try { physical = realpath(lexical); } catch { /* missing dir → lexical only */ }
|
|
2742
|
+
return physical === lexical ? [lexical] : [physical, lexical];
|
|
2743
|
+
}
|
|
2744
|
+
|
|
2745
|
+
/** True when `cwd` is one of `roots` or lies beneath one. */
|
|
2746
|
+
function cwdInRoots(cwd, roots) {
|
|
2747
|
+
if (typeof cwd !== 'string' || !cwd) return false;
|
|
2748
|
+
return roots.some((root) => cwd === root || cwd.startsWith(root.endsWith(sep) ? root : root + sep));
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
/**
|
|
2752
|
+
* Linux: aggregate CPU (utime+stime ticks) of every process whose cwd resolves
|
|
2753
|
+
* inside `workspaceDir` — regardless of parentage — via `/proc/<pid>/cwd`
|
|
2754
|
+
* symlinks. Returns `{ processes, cpu }` (cpu in clock ticks, matching the
|
|
2755
|
+
* `/proc` subtree sampler), or `null` when the workspace is unusable (so the
|
|
2756
|
+
* caller degrades to subtree-only liveness rather than mis-killing). A process
|
|
2757
|
+
* counted here is NOT a descendant of the agent, so it is a pure CPU-progress
|
|
2758
|
+
* signal, never a presence signal. Bounded by `scanLimit` pids so a wedged
|
|
2759
|
+
* /proc read can't stall the watchdog. `readCwd`/`listPids` are injectable for
|
|
2760
|
+
* tests.
|
|
2761
|
+
*/
|
|
2762
|
+
function sampleWorkspaceCpuProc(workspaceDir, { readCwd, listPids, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
|
|
2763
|
+
const roots = workspaceRoots(workspaceDir, realpath);
|
|
2764
|
+
if (roots.length === 0) return null;
|
|
2765
|
+
const read = typeof readCwd === 'function' ? readCwd : (pid) => {
|
|
2766
|
+
try { return readlinkSync(`/proc/${pid}/cwd`); } catch { return null; }
|
|
2767
|
+
};
|
|
2768
|
+
let entries;
|
|
2769
|
+
if (typeof listPids === 'function') entries = listPids();
|
|
2770
|
+
else {
|
|
2771
|
+
try { entries = readdirSync('/proc'); } catch { return null; }
|
|
2772
|
+
}
|
|
2773
|
+
let processes = 0;
|
|
2774
|
+
let totalCpu = 0;
|
|
2775
|
+
let scanned = 0;
|
|
2776
|
+
for (const name of entries) {
|
|
2777
|
+
if (scanned >= scanLimit) break;
|
|
2778
|
+
if (!/^\d+$/.test(name)) continue;
|
|
2779
|
+
scanned += 1;
|
|
2780
|
+
const pid = Number(name);
|
|
2781
|
+
if (pid === process.pid || excludePids.includes(pid)) continue;
|
|
2782
|
+
const cwd = read(pid);
|
|
2783
|
+
if (typeof cwd !== 'string' || !cwd) continue;
|
|
2784
|
+
if (!cwdInRoots(cwd, roots)) continue;
|
|
2785
|
+
let stat;
|
|
2786
|
+
try { stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { continue; }
|
|
2787
|
+
const rparen = stat.lastIndexOf(')');
|
|
2788
|
+
if (rparen < 0) continue;
|
|
2789
|
+
const tail = stat.slice(rparen + 1).trim().split(/\s+/);
|
|
2790
|
+
const utime = Number(tail[11]);
|
|
2791
|
+
const stime = Number(tail[12]);
|
|
2792
|
+
totalCpu += (Number.isFinite(utime) ? utime : 0) + (Number.isFinite(stime) ? stime : 0);
|
|
2793
|
+
processes += 1;
|
|
2794
|
+
}
|
|
2795
|
+
return { processes, cpu: totalCpu };
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
/**
|
|
2799
|
+
* Non-Linux fallback (macOS): aggregate CPU of processes whose cwd is inside
|
|
2800
|
+
* `workspaceDir`, via a bounded `lsof` + `ps`. Heavier (two short-lived
|
|
2801
|
+
* spawns) but runs only during an idle window. Returns
|
|
2802
|
+
* `{ processes, cpu, resolutionMs: 1000 }` (cpu in whole seconds, so the
|
|
2803
|
+
* coarse-resolution flat-CPU rule applies) or `null` when unavailable.
|
|
2804
|
+
*/
|
|
2805
|
+
function sampleWorkspaceCpuLsof(workspaceDir, { runLsof, runPs, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
|
|
2806
|
+
const roots = workspaceRoots(workspaceDir, realpath);
|
|
2807
|
+
if (roots.length === 0) return null;
|
|
2808
|
+
const lsof = typeof runLsof === 'function' ? runLsof : () => {
|
|
2809
|
+
// `lsof -d cwd -Fpn` lists the cwd of EVERY process as machine-readable
|
|
2810
|
+
// `p<pid>` / `n<path>` records. A path filter (`-- <root>`) would only match
|
|
2811
|
+
// processes whose cwd is EXACTLY the root, missing a compiler working in a
|
|
2812
|
+
// nested crate dir, so we collect all cwd records and apply the
|
|
2813
|
+
// root-or-descendant check ourselves. Bounded by timeout + maxBuffer; lsof
|
|
2814
|
+
// exits 1 on partial permission errors, which still yields usable stdout.
|
|
2815
|
+
const r = spawnSync('lsof', ['-n', '-P', '-w', '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
|
|
2816
|
+
if (r.error && r.error.code === 'ENOENT') return null; // lsof missing → degrade
|
|
2817
|
+
if (r.error && !r.stdout) return null; // timeout / overflow with nothing usable
|
|
2818
|
+
return typeof r.stdout === 'string' ? r.stdout : '';
|
|
2819
|
+
};
|
|
2820
|
+
let lsofOut;
|
|
2821
|
+
try { lsofOut = lsof(); } catch { return null; }
|
|
2822
|
+
if (typeof lsofOut !== 'string') return null;
|
|
2823
|
+
const pids = [];
|
|
2824
|
+
let curPid = null;
|
|
2825
|
+
for (const line of lsofOut.split('\n')) {
|
|
2826
|
+
const tag = line.charCodeAt(0);
|
|
2827
|
+
if (tag === 112 /* 'p' */) {
|
|
2828
|
+
const pid = Number(line.slice(1));
|
|
2829
|
+
curPid = Number.isFinite(pid) && pid > 0 && pid !== process.pid && !excludePids.includes(pid) ? pid : null;
|
|
2830
|
+
} else if (tag === 110 /* 'n' */ && curPid != null) {
|
|
2831
|
+
if (cwdInRoots(line.slice(1), roots) && !pids.includes(curPid)) pids.push(curPid);
|
|
2832
|
+
curPid = null;
|
|
2833
|
+
if (pids.length >= scanLimit) break;
|
|
2834
|
+
}
|
|
2835
|
+
}
|
|
2836
|
+
if (pids.length === 0) return { processes: 0, cpu: 0, resolutionMs: 1000 };
|
|
2837
|
+
const ps = typeof runPs === 'function' ? runPs : (list) => {
|
|
2838
|
+
const r = spawnSync('ps', ['-p', list.join(','), '-o', 'pid=,time='], { encoding: 'utf8', timeout: 5_000 });
|
|
2839
|
+
// ps exits 1 when some pids vanished between scans; partial stdout is valid.
|
|
2840
|
+
return typeof r.stdout === 'string' ? r.stdout : null;
|
|
2841
|
+
};
|
|
2842
|
+
let psOut;
|
|
2843
|
+
try { psOut = ps(pids); } catch { return null; }
|
|
2844
|
+
if (typeof psOut !== 'string') return null;
|
|
2845
|
+
let processes = 0;
|
|
2846
|
+
let totalCpu = 0;
|
|
2847
|
+
for (const line of psOut.split('\n')) {
|
|
2848
|
+
const m = line.trim().match(/^(\d+)\s+(\S+)$/);
|
|
2849
|
+
if (!m) continue;
|
|
2850
|
+
totalCpu += parsePsTime(m[2]);
|
|
2851
|
+
processes += 1;
|
|
2852
|
+
}
|
|
2853
|
+
return { processes, cpu: totalCpu, resolutionMs: 1000 };
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
/**
|
|
2857
|
+
* Sample aggregate CPU of processes working inside `workspaceDir` regardless of
|
|
2858
|
+
* parentage (the sccache/shared-daemon case, #313). Linux reads `/proc`; macOS
|
|
2859
|
+
* falls back to `lsof`+`ps`; other platforms return `null` (unsupported → the
|
|
2860
|
+
* monitor degrades to subtree-only liveness). The result shape mirrors the
|
|
2861
|
+
* subtree sampler (`cpu` in ticks on Linux, whole seconds under `ps`) with
|
|
2862
|
+
* `processes` instead of `descendants`.
|
|
2863
|
+
*/
|
|
2864
|
+
function sampleWorkspaceCpu(workspaceDir, { excludePids } = {}) {
|
|
2865
|
+
// The agent itself runs with cwd == workspace; its own CPU (an idle harness
|
|
2866
|
+
// ticking timers) is NOT out-of-tree work and must never defer its idle-kill.
|
|
2867
|
+
const exclude = Array.isArray(excludePids) ? excludePids.filter((p) => Number.isFinite(p) && p > 0) : [];
|
|
2868
|
+
if (process.platform === 'linux') return sampleWorkspaceCpuProc(workspaceDir, { excludePids: exclude });
|
|
2869
|
+
if (process.platform === 'darwin') return sampleWorkspaceCpuLsof(workspaceDir, { excludePids: exclude });
|
|
2870
|
+
return null; // win32 et al: unsupported → degrade, never mis-kill
|
|
2871
|
+
}
|
|
2872
|
+
|
|
2708
2873
|
/**
|
|
2709
2874
|
* Build a progress-aware idle-liveness monitor shared by the pipe/PTY/ACP
|
|
2710
2875
|
* capture paths. `arm()` is called on every output chunk (and once at start) to
|
|
@@ -2730,37 +2895,155 @@ function sampleSubtreeCpu(rootPid) {
|
|
|
2730
2895
|
* read under a coarse sampler must NOT kill; the fine-grained `/proc` path
|
|
2731
2896
|
* carries no resolution and so ages out on the first flat window as before →
|
|
2732
2897
|
* `onIdleKill()`.
|
|
2898
|
+
* #313 extensions:
|
|
2899
|
+
* - `extraSamplers`: additional activity probes (e.g. the workspace-cwd CPU
|
|
2900
|
+
* scan covering out-of-tree sccache/rustc daemons). Each returns
|
|
2901
|
+
* `{ processes, cpu, resolutionMs? } | null`; advancing CPU on ANY sampler
|
|
2902
|
+
* defers the kill. Extra samplers are pure progress signals — they never
|
|
2903
|
+
* satisfy the presence gate (a childless agent with a busy workspace daemon
|
|
2904
|
+
* still needs its own live descendant to count as present... EXCEPT that a
|
|
2905
|
+
* workspace daemon burning CPU IS evidence the agent's work is in flight, so
|
|
2906
|
+
* when an extra sampler shows advancing CPU the kill is deferred even with
|
|
2907
|
+
* no live descendant: the agent may be blocked reading a pipe its daemon
|
|
2908
|
+
* fills. A flat/zero extra sample never kills on its own — presence is
|
|
2909
|
+
* judged from the subtree as before).
|
|
2910
|
+
* - `isActivityHold`: an in-flight-tool-call latch (ACP). While it returns
|
|
2911
|
+
* true the agent has an open tool call, so silence is expected; the kill is
|
|
2912
|
+
* deferred window-by-window. The absolute `--job-timeout` hard cap (and the
|
|
2913
|
+
* tool call's own timeout, enforced by the agent) remain the bound for a
|
|
2914
|
+
* genuinely hung call.
|
|
2915
|
+
* - `logger`: when given, the monitor logs WHY it considered the agent active
|
|
2916
|
+
* or idle at each probe (diagnosability, #313).
|
|
2733
2917
|
* The first elapse with live descendants takes a baseline (a single sample can't
|
|
2734
2918
|
* show movement) and re-probes after the recovery window. The absolute
|
|
2735
2919
|
* `--job-timeout` hard cap remains the ultimate backstop, untouched by this.
|
|
2736
2920
|
*/
|
|
2737
|
-
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu }) {
|
|
2921
|
+
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu, extraSamplers = null, isActivityHold = null, maxActivityHoldMs, logger = null }) {
|
|
2738
2922
|
const enabled = idleTimeoutMs && idleTimeoutMs > 0;
|
|
2739
2923
|
const probeWindow = recoveryWindowMs && recoveryWindowMs > 0 ? recoveryWindowMs : idleTimeoutMs;
|
|
2740
2924
|
let timer = null;
|
|
2741
2925
|
let prev = null; // last subtree CPU probe taken during the current silent stretch
|
|
2926
|
+
let prevExtra = null; // last per-extra-sampler CPU readings (array parallel to extraSamplers)
|
|
2927
|
+
let extraBaselined = false; // true once the extra samplers have a baseline to compare against
|
|
2928
|
+
let extraResolutionMs = 0; // coarsest resolution among live extra samples at the last probe
|
|
2742
2929
|
let flatMs = 0; // cumulative time observed with flat CPU since the last movement/baseline
|
|
2743
2930
|
let lastWindow = idleTimeoutMs; // duration of the currently pending probe window
|
|
2931
|
+
let extraFlatMs = 0; // cumulative flat time across the extra (workspace) samplers
|
|
2932
|
+
let holdMs = 0; // cumulative silent time deferred by the activity hold this stretch
|
|
2933
|
+
// #313: the activity hold must be FINITE — `nano work` defaults to no hard cap,
|
|
2934
|
+
// so an announced-then-hung tool call would otherwise defer forever. Default:
|
|
2935
|
+
// 4 idle windows (configurable). Past it, the hold is ignored and the normal
|
|
2936
|
+
// CPU probes decide (real silent work still defers via CPU progress).
|
|
2937
|
+
const holdCap = Number.isFinite(maxActivityHoldMs) && maxActivityHoldMs >= 0 ? maxActivityHoldMs : (idleTimeoutMs > 0 ? idleTimeoutMs * 4 : 0);
|
|
2744
2938
|
const settled = () => (typeof isSettled === 'function' ? isSettled() : false);
|
|
2745
2939
|
const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
|
|
2746
2940
|
const schedule = (ms) => { lastWindow = ms; timer = setTimeout(onElapse, ms); };
|
|
2941
|
+
const log = (msg) => { try { logger?.info?.(`[idle-watchdog] ${msg}`); } catch { /* best effort */ } };
|
|
2942
|
+
const extras = Array.isArray(extraSamplers) ? extraSamplers.filter((s) => typeof s === 'function') : [];
|
|
2943
|
+
const held = () => { try { return typeof isActivityHold === 'function' && isActivityHold(); } catch { return false; } };
|
|
2747
2944
|
function onElapse() {
|
|
2748
2945
|
timer = null;
|
|
2749
2946
|
if (settled()) return;
|
|
2750
2947
|
const elapsed = lastWindow;
|
|
2948
|
+
// #313: an in-flight tool call (the agent started a bash call that has not
|
|
2949
|
+
// reported a terminal status) means the silence is expected — defer. The
|
|
2950
|
+
// call's own timeout / the absolute hard cap still bound a hung call.
|
|
2951
|
+
if (held()) {
|
|
2952
|
+
if (holdMs < holdCap) {
|
|
2953
|
+
holdMs += elapsed;
|
|
2954
|
+
log(`silent for >= ${elapsed}ms but a tool call is still in flight; deferring the idle-kill (hold ${holdMs}/${holdCap}ms)`);
|
|
2955
|
+
prev = null; prevExtra = null; extraBaselined = false; flatMs = 0; extraFlatMs = 0;
|
|
2956
|
+
schedule(probeWindow);
|
|
2957
|
+
return;
|
|
2958
|
+
}
|
|
2959
|
+
log(`a tool call has been in flight for >= ${holdMs}ms (hold cap ${holdCap}ms); falling back to CPU-progress probes`);
|
|
2960
|
+
}
|
|
2751
2961
|
const pid = typeof getPid === 'function' ? getPid() : undefined;
|
|
2752
2962
|
const sample = pid != null ? sampleCpu(pid) : null;
|
|
2753
|
-
//
|
|
2754
|
-
|
|
2963
|
+
// #313: consult the extra (workspace-cwd) samplers. Advancing CPU on any of
|
|
2964
|
+
// them is work attributable to this agent even when it lives outside the
|
|
2965
|
+
// agent's process tree (a shared sccache daemon), so it defers the kill —
|
|
2966
|
+
// INCLUDING when the subtree itself looks childless/quiescent.
|
|
2967
|
+
let extraActive = false;
|
|
2968
|
+
if (extras.length) {
|
|
2969
|
+
const next = [];
|
|
2970
|
+
extraResolutionMs = 0;
|
|
2971
|
+
for (let i = 0; i < extras.length; i += 1) {
|
|
2972
|
+
let s = null;
|
|
2973
|
+
try { s = extras[i](); } catch { s = null; }
|
|
2974
|
+
next.push(s);
|
|
2975
|
+
if (!s || !Number.isFinite(s.cpu)) continue;
|
|
2976
|
+
if (s.processes > 0 && Number.isFinite(s.resolutionMs)) extraResolutionMs = Math.max(extraResolutionMs, s.resolutionMs);
|
|
2977
|
+
const before = prevExtra && prevExtra[i];
|
|
2978
|
+
if (before != null && Number.isFinite(before.cpu) && s.cpu !== before.cpu) extraActive = true;
|
|
2979
|
+
}
|
|
2980
|
+
const hadBaseline = prevExtra != null;
|
|
2981
|
+
const prevReadings = prevExtra || [];
|
|
2982
|
+
prevExtra = next;
|
|
2983
|
+
if (extraActive) {
|
|
2984
|
+
log(`silent for >= ${elapsed}ms but a workspace process outside the agent tree is burning CPU (e.g. sccache/rustc); deferring the idle-kill`);
|
|
2985
|
+
prev = sample; flatMs = 0; extraFlatMs = 0;
|
|
2986
|
+
schedule(probeWindow);
|
|
2987
|
+
return;
|
|
2988
|
+
}
|
|
2989
|
+
// Flat time only counts while a workspace sampler stays LIVE (a reading
|
|
2990
|
+
// with processes > 0) across consecutive probes. Empty / unreadable
|
|
2991
|
+
// readings must not pre-pay a sampler's resolution allowance: when any
|
|
2992
|
+
// sampler transitions absent/unreadable → live, restart the accumulator
|
|
2993
|
+
// so a fresh process gets its full resolution window before a kill.
|
|
2994
|
+
const isLive = (r) => r != null && Number.isFinite(r.cpu) && r.processes > 0;
|
|
2995
|
+
let newlyLive = false; let continuedLive = false;
|
|
2996
|
+
for (let i = 0; i < next.length; i += 1) {
|
|
2997
|
+
if (!isLive(next[i])) continue;
|
|
2998
|
+
if (hadBaseline && isLive(prevReadings[i])) continuedLive = true; else newlyLive = true;
|
|
2999
|
+
}
|
|
3000
|
+
if (newlyLive || !continuedLive) extraFlatMs = 0;
|
|
3001
|
+
else extraFlatMs += elapsed;
|
|
3002
|
+
}
|
|
3003
|
+
// A live workspace process read flat, but a coarse sampler (whole-second
|
|
3004
|
+
// `ps`) can read flat across a sub-resolution window while still working:
|
|
3005
|
+
// only treat workspace CPU as idle once the flat time covers its resolution.
|
|
3006
|
+
const extraIdleConfirmed = extraFlatMs >= extraResolutionMs;
|
|
3007
|
+
// Childless / unsampleable / dead tree → silence is very likely a hang.
|
|
3008
|
+
// But with extra (workspace) samplers wired we cannot yet distinguish "no
|
|
3009
|
+
// workspace work" from "a workspace daemon whose baseline we have not
|
|
3010
|
+
// taken" on a single reading — so take ONE baseline probe window before
|
|
3011
|
+
// killing, exactly like the subtree first-elapse baseline below. Without
|
|
3012
|
+
// extra samplers (or once a baseline shows no workspace activity) the
|
|
3013
|
+
// legacy immediate kill applies.
|
|
3014
|
+
if (!sample || sample.descendants <= 0) {
|
|
3015
|
+
if (extras.length && !extraBaselined) {
|
|
3016
|
+
// Consume the baseline: the NEXT elapse compares against prevExtra and
|
|
3017
|
+
// either defers (workspace CPU moved) or kills (flat).
|
|
3018
|
+
extraBaselined = true;
|
|
3019
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; baselining the workspace CPU probe and re-checking in ${probeWindow}ms`);
|
|
3020
|
+
schedule(probeWindow);
|
|
3021
|
+
return;
|
|
3022
|
+
}
|
|
3023
|
+
if (!extraIdleConfirmed) {
|
|
3024
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; workspace CPU flat for ${extraFlatMs}ms < sampler resolution ${extraResolutionMs}ms, re-checking in ${probeWindow}ms`);
|
|
3025
|
+
schedule(probeWindow);
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'} and no workspace CPU; idle-killing`);
|
|
3029
|
+
onIdleKill();
|
|
3030
|
+
return;
|
|
3031
|
+
}
|
|
2755
3032
|
if (prev != null) {
|
|
2756
3033
|
// Aggregate CPU advanced across the window → the tree is doing work; defer.
|
|
2757
|
-
if (sample.cpu > prev.cpu) {
|
|
3034
|
+
if (sample.cpu > prev.cpu) {
|
|
3035
|
+
log(`silent for >= ${elapsed}ms but subtree CPU advanced (${prev.cpu} → ${sample.cpu}); deferring the idle-kill`);
|
|
3036
|
+
prev = sample; flatMs = 0; schedule(probeWindow); return;
|
|
3037
|
+
}
|
|
2758
3038
|
// Aggregate CPU REGRESSED → `sampleSubtreeCpu*` only sums CPU for live
|
|
2759
3039
|
// PIDs, so the total can legitimately drop when a busy descendant exits
|
|
2760
3040
|
// (its ticks stop counting) even though the subtree just made progress.
|
|
2761
3041
|
// A regression is activity, not a stall: re-baseline and defer rather
|
|
2762
3042
|
// than mis-killing a silent-but-busy job.
|
|
2763
|
-
if (sample.cpu < prev.cpu) {
|
|
3043
|
+
if (sample.cpu < prev.cpu) {
|
|
3044
|
+
log(`silent for >= ${elapsed}ms but subtree CPU regressed (${prev.cpu} → ${sample.cpu}; a busy descendant exited) — progress, not a stall; deferring the idle-kill`);
|
|
3045
|
+
prev = sample; flatMs = 0; schedule(probeWindow); return;
|
|
3046
|
+
}
|
|
2764
3047
|
// Live descendants but CPU exactly unchanged across a full window → a
|
|
2765
3048
|
// candidate stall. Only confirm it once we have observed flat CPU for at
|
|
2766
3049
|
// least the sampler's CPU-time resolution: a probe window shorter than the
|
|
@@ -2768,12 +3051,17 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
|
|
|
2768
3051
|
// low-churn subtree is still progressing but hasn't crossed the next tick.
|
|
2769
3052
|
flatMs += elapsed;
|
|
2770
3053
|
const resolutionMs = Number.isFinite(sample.resolutionMs) ? sample.resolutionMs : 0;
|
|
2771
|
-
if (flatMs >= resolutionMs
|
|
3054
|
+
if (flatMs >= resolutionMs && extraIdleConfirmed) {
|
|
3055
|
+
log(`silent for >= ${elapsed}ms with a live but CPU-flat subtree (${sample.descendants} descendant(s), cpu ${sample.cpu}) and no workspace CPU; idle-killing`);
|
|
3056
|
+
onIdleKill();
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
2772
3059
|
schedule(probeWindow);
|
|
2773
3060
|
return;
|
|
2774
3061
|
}
|
|
2775
3062
|
// First elapse with live descendants: baseline, then re-probe (can't judge
|
|
2776
3063
|
// progress from one sample).
|
|
3064
|
+
log(`silent for >= ${elapsed}ms with ${sample.descendants} live descendant(s); baselining CPU (${sample.cpu}) and re-probing in ${probeWindow}ms`);
|
|
2777
3065
|
prev = sample;
|
|
2778
3066
|
flatMs = 0;
|
|
2779
3067
|
schedule(probeWindow);
|
|
@@ -2783,7 +3071,12 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
|
|
|
2783
3071
|
arm() {
|
|
2784
3072
|
if (!enabled || settled()) return;
|
|
2785
3073
|
prev = null; // output resumed → a fresh silent stretch starts
|
|
3074
|
+
prevExtra = null;
|
|
3075
|
+
extraBaselined = false;
|
|
2786
3076
|
flatMs = 0;
|
|
3077
|
+
extraFlatMs = 0;
|
|
3078
|
+
extraResolutionMs = 0;
|
|
3079
|
+
holdMs = 0;
|
|
2787
3080
|
clear();
|
|
2788
3081
|
schedule(idleTimeoutMs);
|
|
2789
3082
|
},
|
|
@@ -8796,6 +9089,11 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
|
|
|
8796
9089
|
idleTimeoutMs,
|
|
8797
9090
|
recoveryWindowMs,
|
|
8798
9091
|
isSettled: () => settled,
|
|
9092
|
+
logger: getLogger(),
|
|
9093
|
+
// #313: count CPU burned by any process whose cwd is inside the agent's
|
|
9094
|
+
// workspace (a shared sccache daemon lives outside the agent's tree), so
|
|
9095
|
+
// a silent compile holds off the idle-kill.
|
|
9096
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
|
|
8799
9097
|
onIdleKill: () => {
|
|
8800
9098
|
try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
|
|
8801
9099
|
finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
|
|
@@ -8968,6 +9266,11 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
8968
9266
|
idleTimeoutMs,
|
|
8969
9267
|
recoveryWindowMs,
|
|
8970
9268
|
isSettled: () => settled,
|
|
9269
|
+
logger: getLogger(),
|
|
9270
|
+
// #313: count CPU burned by any process whose cwd is inside the agent's
|
|
9271
|
+
// workspace (a shared sccache daemon lives outside the agent's tree), so
|
|
9272
|
+
// a silent compile holds off the idle-kill.
|
|
9273
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [term?.pid] })] : null,
|
|
8971
9274
|
onIdleKill: () => {
|
|
8972
9275
|
killTerm();
|
|
8973
9276
|
finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
|
|
@@ -9342,6 +9645,28 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9342
9645
|
// flips this; the floor never does.
|
|
9343
9646
|
let sawRealTranscript = false;
|
|
9344
9647
|
|
|
9648
|
+
// #313: ACP tool calls currently in flight (`toolCallId` → true). A
|
|
9649
|
+
// `tool_call` notification with a non-terminal status opens the call; a
|
|
9650
|
+
// `tool_call`/`tool_call_update` with a terminal status closes it. While
|
|
9651
|
+
// any call is open the idle watchdog treats the agent as ACTIVE (a long
|
|
9652
|
+
// silent bash call is work, not a hang) — the call's own timeout and the
|
|
9653
|
+
// absolute job hard cap still bound a genuinely hung call.
|
|
9654
|
+
const inFlightToolCalls = new Map();
|
|
9655
|
+
const ACP_TOOL_TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled', 'canceled']);
|
|
9656
|
+
const noteToolCallUpdate = (update) => {
|
|
9657
|
+
if (!update || typeof update !== 'object') return;
|
|
9658
|
+
const kind = update.sessionUpdate || update.type;
|
|
9659
|
+
if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
|
|
9660
|
+
const id = update.toolCallId;
|
|
9661
|
+
if (id == null) return;
|
|
9662
|
+
const status = typeof update.status === 'string' ? update.status.toLowerCase() : '';
|
|
9663
|
+
if (ACP_TOOL_TERMINAL_STATUSES.has(status)) inFlightToolCalls.delete(id);
|
|
9664
|
+
else if (!inFlightToolCalls.has(id)) {
|
|
9665
|
+
inFlightToolCalls.set(id, true);
|
|
9666
|
+
try { logger.info?.(`[idle-watchdog] tool call in flight (${update.title || id}${status ? `, status ${status}` : ''}) — the idle-kill is held while it runs`); } catch { /* best effort */ }
|
|
9667
|
+
}
|
|
9668
|
+
};
|
|
9669
|
+
|
|
9345
9670
|
// Live "spy" tee (--stream), line-buffered, mirroring the other paths.
|
|
9346
9671
|
// Separate line buffers per lane (stdout-human vs stderr) so a partial line
|
|
9347
9672
|
// on one lane never interleaves mid-line with the other — matching pipe/PTY.
|
|
@@ -9670,6 +9995,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9670
9995
|
// A request or notification FROM the agent.
|
|
9671
9996
|
if (typeof msg.method === 'string') {
|
|
9672
9997
|
if (msg.method === 'session/update') {
|
|
9998
|
+
noteToolCallUpdate(msg.params?.update);
|
|
9673
9999
|
emitTranscript(msg.params?.update);
|
|
9674
10000
|
// #194: durable engine-native producer seam. In parallel with the
|
|
9675
10001
|
// transcript-chunk relay overlay, hand the SAME raw `session/update` to
|
|
@@ -9720,6 +10046,16 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9720
10046
|
idleTimeoutMs,
|
|
9721
10047
|
recoveryWindowMs,
|
|
9722
10048
|
isSettled: () => settled,
|
|
10049
|
+
logger,
|
|
10050
|
+
// #313: while an ACP tool call is in flight the agent is active even
|
|
10051
|
+
// when totally silent (a long foreground bash compile/test). Bounded by
|
|
10052
|
+
// the call's own timeout and the absolute job hard cap.
|
|
10053
|
+
isActivityHold: () => inFlightToolCalls.size > 0,
|
|
10054
|
+
// #313: also count CPU burned by processes whose cwd is inside the
|
|
10055
|
+
// agent's workspace, wherever they sit in the process tree — a shared
|
|
10056
|
+
// sccache daemon (parented to launchd/init, not the agent) compiling
|
|
10057
|
+
// this worktree is real work and must hold off the idle-kill.
|
|
10058
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
|
|
9723
10059
|
onIdleKill: () => {
|
|
9724
10060
|
try { killTree(child); } catch { /* best effort */ }
|
|
9725
10061
|
finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
|
|
@@ -21398,6 +21734,9 @@ export {
|
|
|
21398
21734
|
spawnCapturePty,
|
|
21399
21735
|
spawnCaptureAcp,
|
|
21400
21736
|
sampleSubtreeCpu,
|
|
21737
|
+
sampleWorkspaceCpu,
|
|
21738
|
+
sampleWorkspaceCpuProc,
|
|
21739
|
+
sampleWorkspaceCpuLsof,
|
|
21401
21740
|
createIdleLivenessMonitor,
|
|
21402
21741
|
resolveLivenessOverrides,
|
|
21403
21742
|
parsePsTime,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "c8ctl-plugin-nano",
|
|
3
|
-
"version": "1.73.
|
|
3
|
+
"version": "1.73.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
|
|
6
6
|
"main": "c8ctl-plugin.js",
|
|
@@ -76,12 +76,12 @@
|
|
|
76
76
|
},
|
|
77
77
|
"optionalDependencies": {
|
|
78
78
|
"node-pty": "^1.0.0",
|
|
79
|
-
"@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.73.
|
|
80
|
-
"@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.73.
|
|
81
|
-
"@nanobpm/c8ctl-plugin-nano-linux-x64": "1.73.
|
|
82
|
-
"@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.73.
|
|
83
|
-
"@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.73.
|
|
84
|
-
"@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.73.
|
|
85
|
-
"@nanobpm/c8ctl-plugin-nano-win32-x64": "1.73.
|
|
79
|
+
"@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.73.2",
|
|
80
|
+
"@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.73.2",
|
|
81
|
+
"@nanobpm/c8ctl-plugin-nano-linux-x64": "1.73.2",
|
|
82
|
+
"@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.73.2",
|
|
83
|
+
"@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.73.2",
|
|
84
|
+
"@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.73.2",
|
|
85
|
+
"@nanobpm/c8ctl-plugin-nano-win32-x64": "1.73.2"
|
|
86
86
|
}
|
|
87
87
|
}
|