c8ctl-plugin-nano 1.73.0 → 1.73.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/c8ctl-plugin.js +345 -6
- package/package.json +8 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -49,6 +49,7 @@ import {
|
|
|
49
49
|
utimesSync,
|
|
50
50
|
mkdtempSync,
|
|
51
51
|
closeSync,
|
|
52
|
+
readlinkSync,
|
|
52
53
|
watchFile,
|
|
53
54
|
unwatchFile,
|
|
54
55
|
} from 'node:fs';
|
|
@@ -2705,6 +2706,170 @@ function sampleSubtreeCpu(rootPid) {
|
|
|
2705
2706
|
return sampleSubtreeCpuPs(rootPid);
|
|
2706
2707
|
}
|
|
2707
2708
|
|
|
2709
|
+
// ---- Workspace-CPU liveness + in-flight tool holds (#313) ------------------
|
|
2710
|
+
// The #130 subtree probe only sees processes INSIDE the agent's own process
|
|
2711
|
+
// tree. Real work often happens OUTSIDE that tree: `cargo` builds through a
|
|
2712
|
+
// shared `sccache` server (a daemon parented to launchd/init, not the agent),
|
|
2713
|
+
// and a long foreground bash tool call can compile/test silently for minutes
|
|
2714
|
+
// (nanobpm/nano-bpm#1664 attempt 1 was idle-killed at 216 turns / 156 min with
|
|
2715
|
+
// a build genuinely running). Two extra liveness signals close that gap:
|
|
2716
|
+
// 1. `sampleWorkspaceCpu` — aggregate CPU of ANY process whose cwd is inside
|
|
2717
|
+
// the agent's workspace (the sccache/rustc case: the daemon's compile
|
|
2718
|
+
// workers chdir into the worktree they compile for).
|
|
2719
|
+
// 2. an in-flight-tool-call hold — while the agent has an ACP tool call still
|
|
2720
|
+
// open (a bash call it started has not reported a terminal status) it is
|
|
2721
|
+
// treated as active; the call's own timeout / the absolute hard cap still
|
|
2722
|
+
// bound a genuinely hung call.
|
|
2723
|
+
// Both ride the monitor below as extra samplers consulted only during an idle
|
|
2724
|
+
// window, so a truly quiescent agent still ages out exactly as before.
|
|
2725
|
+
|
|
2726
|
+
/** Default upper bound for the workspace-cwd scan (pid count). */
|
|
2727
|
+
const WORKSPACE_CPU_SCAN_LIMIT = 4096;
|
|
2728
|
+
|
|
2729
|
+
/**
|
|
2730
|
+
* Canonicalize a workspace root for cwd prefix checks. The kernel reports a
|
|
2731
|
+
* process cwd as the PHYSICAL path (`/proc/<pid>/cwd`, lsof), so a root reached
|
|
2732
|
+
* through a symlink (e.g. `C8CTL_NANO_HOME` under a symlinked dir, or macOS
|
|
2733
|
+
* `/tmp` → `/private/tmp`) must be realpath'd or every in-workspace process
|
|
2734
|
+
* fails the prefix check. Returns the list of distinct roots to accept (the
|
|
2735
|
+
* physical one plus the lexical one), or `[]` when the input is unusable.
|
|
2736
|
+
*/
|
|
2737
|
+
function workspaceRoots(workspaceDir, realpath = realpathSync) {
|
|
2738
|
+
if (typeof workspaceDir !== 'string' || !workspaceDir) return [];
|
|
2739
|
+
const lexical = resolvePath(workspaceDir);
|
|
2740
|
+
let physical = lexical;
|
|
2741
|
+
try { physical = realpath(lexical); } catch { /* missing dir → lexical only */ }
|
|
2742
|
+
return physical === lexical ? [lexical] : [physical, lexical];
|
|
2743
|
+
}
|
|
2744
|
+
|
|
2745
|
+
/** True when `cwd` is one of `roots` or lies beneath one. */
|
|
2746
|
+
function cwdInRoots(cwd, roots) {
|
|
2747
|
+
if (typeof cwd !== 'string' || !cwd) return false;
|
|
2748
|
+
return roots.some((root) => cwd === root || cwd.startsWith(root.endsWith(sep) ? root : root + sep));
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
/**
|
|
2752
|
+
* Linux: aggregate CPU (utime+stime ticks) of every process whose cwd resolves
|
|
2753
|
+
* inside `workspaceDir` — regardless of parentage — via `/proc/<pid>/cwd`
|
|
2754
|
+
* symlinks. Returns `{ processes, cpu }` (cpu in clock ticks, matching the
|
|
2755
|
+
* `/proc` subtree sampler), or `null` when the workspace is unusable (so the
|
|
2756
|
+
* caller degrades to subtree-only liveness rather than mis-killing). A process
|
|
2757
|
+
* counted here is NOT a descendant of the agent, so it is a pure CPU-progress
|
|
2758
|
+
* signal, never a presence signal. Bounded by `scanLimit` pids so a wedged
|
|
2759
|
+
* /proc read can't stall the watchdog. `readCwd`/`listPids` are injectable for
|
|
2760
|
+
* tests.
|
|
2761
|
+
*/
|
|
2762
|
+
function sampleWorkspaceCpuProc(workspaceDir, { readCwd, listPids, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
|
|
2763
|
+
const roots = workspaceRoots(workspaceDir, realpath);
|
|
2764
|
+
if (roots.length === 0) return null;
|
|
2765
|
+
const read = typeof readCwd === 'function' ? readCwd : (pid) => {
|
|
2766
|
+
try { return readlinkSync(`/proc/${pid}/cwd`); } catch { return null; }
|
|
2767
|
+
};
|
|
2768
|
+
let entries;
|
|
2769
|
+
if (typeof listPids === 'function') entries = listPids();
|
|
2770
|
+
else {
|
|
2771
|
+
try { entries = readdirSync('/proc'); } catch { return null; }
|
|
2772
|
+
}
|
|
2773
|
+
let processes = 0;
|
|
2774
|
+
let totalCpu = 0;
|
|
2775
|
+
let scanned = 0;
|
|
2776
|
+
for (const name of entries) {
|
|
2777
|
+
if (scanned >= scanLimit) break;
|
|
2778
|
+
if (!/^\d+$/.test(name)) continue;
|
|
2779
|
+
scanned += 1;
|
|
2780
|
+
const pid = Number(name);
|
|
2781
|
+
if (pid === process.pid || excludePids.includes(pid)) continue;
|
|
2782
|
+
const cwd = read(pid);
|
|
2783
|
+
if (typeof cwd !== 'string' || !cwd) continue;
|
|
2784
|
+
if (!cwdInRoots(cwd, roots)) continue;
|
|
2785
|
+
let stat;
|
|
2786
|
+
try { stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); } catch { continue; }
|
|
2787
|
+
const rparen = stat.lastIndexOf(')');
|
|
2788
|
+
if (rparen < 0) continue;
|
|
2789
|
+
const tail = stat.slice(rparen + 1).trim().split(/\s+/);
|
|
2790
|
+
const utime = Number(tail[11]);
|
|
2791
|
+
const stime = Number(tail[12]);
|
|
2792
|
+
totalCpu += (Number.isFinite(utime) ? utime : 0) + (Number.isFinite(stime) ? stime : 0);
|
|
2793
|
+
processes += 1;
|
|
2794
|
+
}
|
|
2795
|
+
return { processes, cpu: totalCpu };
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
/**
|
|
2799
|
+
* Non-Linux fallback (macOS): aggregate CPU of processes whose cwd is inside
|
|
2800
|
+
* `workspaceDir`, via a bounded `lsof` + `ps`. Heavier (two short-lived
|
|
2801
|
+
* spawns) but runs only during an idle window. Returns
|
|
2802
|
+
* `{ processes, cpu, resolutionMs: 1000 }` (cpu in whole seconds, so the
|
|
2803
|
+
* coarse-resolution flat-CPU rule applies) or `null` when unavailable.
|
|
2804
|
+
*/
|
|
2805
|
+
function sampleWorkspaceCpuLsof(workspaceDir, { runLsof, runPs, realpath, excludePids = [], scanLimit = WORKSPACE_CPU_SCAN_LIMIT } = {}) {
|
|
2806
|
+
const roots = workspaceRoots(workspaceDir, realpath);
|
|
2807
|
+
if (roots.length === 0) return null;
|
|
2808
|
+
const lsof = typeof runLsof === 'function' ? runLsof : () => {
|
|
2809
|
+
// `lsof -d cwd -Fpn` lists the cwd of EVERY process as machine-readable
|
|
2810
|
+
// `p<pid>` / `n<path>` records. A path filter (`-- <root>`) would only match
|
|
2811
|
+
// processes whose cwd is EXACTLY the root, missing a compiler working in a
|
|
2812
|
+
// nested crate dir, so we collect all cwd records and apply the
|
|
2813
|
+
// root-or-descendant check ourselves. Bounded by timeout + maxBuffer; lsof
|
|
2814
|
+
// exits 1 on partial permission errors, which still yields usable stdout.
|
|
2815
|
+
const r = spawnSync('lsof', ['-n', '-P', '-w', '-d', 'cwd', '-Fpn'], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
|
|
2816
|
+
if (r.error && r.error.code === 'ENOENT') return null; // lsof missing → degrade
|
|
2817
|
+
if (r.error && !r.stdout) return null; // timeout / overflow with nothing usable
|
|
2818
|
+
return typeof r.stdout === 'string' ? r.stdout : '';
|
|
2819
|
+
};
|
|
2820
|
+
let lsofOut;
|
|
2821
|
+
try { lsofOut = lsof(); } catch { return null; }
|
|
2822
|
+
if (typeof lsofOut !== 'string') return null;
|
|
2823
|
+
const pids = [];
|
|
2824
|
+
let curPid = null;
|
|
2825
|
+
for (const line of lsofOut.split('\n')) {
|
|
2826
|
+
const tag = line.charCodeAt(0);
|
|
2827
|
+
if (tag === 112 /* 'p' */) {
|
|
2828
|
+
const pid = Number(line.slice(1));
|
|
2829
|
+
curPid = Number.isFinite(pid) && pid > 0 && pid !== process.pid && !excludePids.includes(pid) ? pid : null;
|
|
2830
|
+
} else if (tag === 110 /* 'n' */ && curPid != null) {
|
|
2831
|
+
if (cwdInRoots(line.slice(1), roots) && !pids.includes(curPid)) pids.push(curPid);
|
|
2832
|
+
curPid = null;
|
|
2833
|
+
if (pids.length >= scanLimit) break;
|
|
2834
|
+
}
|
|
2835
|
+
}
|
|
2836
|
+
if (pids.length === 0) return { processes: 0, cpu: 0, resolutionMs: 1000 };
|
|
2837
|
+
const ps = typeof runPs === 'function' ? runPs : (list) => {
|
|
2838
|
+
const r = spawnSync('ps', ['-p', list.join(','), '-o', 'pid=,time='], { encoding: 'utf8', timeout: 5_000 });
|
|
2839
|
+
// ps exits 1 when some pids vanished between scans; partial stdout is valid.
|
|
2840
|
+
return typeof r.stdout === 'string' ? r.stdout : null;
|
|
2841
|
+
};
|
|
2842
|
+
let psOut;
|
|
2843
|
+
try { psOut = ps(pids); } catch { return null; }
|
|
2844
|
+
if (typeof psOut !== 'string') return null;
|
|
2845
|
+
let processes = 0;
|
|
2846
|
+
let totalCpu = 0;
|
|
2847
|
+
for (const line of psOut.split('\n')) {
|
|
2848
|
+
const m = line.trim().match(/^(\d+)\s+(\S+)$/);
|
|
2849
|
+
if (!m) continue;
|
|
2850
|
+
totalCpu += parsePsTime(m[2]);
|
|
2851
|
+
processes += 1;
|
|
2852
|
+
}
|
|
2853
|
+
return { processes, cpu: totalCpu, resolutionMs: 1000 };
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
/**
|
|
2857
|
+
* Sample aggregate CPU of processes working inside `workspaceDir` regardless of
|
|
2858
|
+
* parentage (the sccache/shared-daemon case, #313). Linux reads `/proc`; macOS
|
|
2859
|
+
* falls back to `lsof`+`ps`; other platforms return `null` (unsupported → the
|
|
2860
|
+
* monitor degrades to subtree-only liveness). The result shape mirrors the
|
|
2861
|
+
* subtree sampler (`cpu` in ticks on Linux, whole seconds under `ps`) with
|
|
2862
|
+
* `processes` instead of `descendants`.
|
|
2863
|
+
*/
|
|
2864
|
+
function sampleWorkspaceCpu(workspaceDir, { excludePids } = {}) {
|
|
2865
|
+
// The agent itself runs with cwd == workspace; its own CPU (an idle harness
|
|
2866
|
+
// ticking timers) is NOT out-of-tree work and must never defer its idle-kill.
|
|
2867
|
+
const exclude = Array.isArray(excludePids) ? excludePids.filter((p) => Number.isFinite(p) && p > 0) : [];
|
|
2868
|
+
if (process.platform === 'linux') return sampleWorkspaceCpuProc(workspaceDir, { excludePids: exclude });
|
|
2869
|
+
if (process.platform === 'darwin') return sampleWorkspaceCpuLsof(workspaceDir, { excludePids: exclude });
|
|
2870
|
+
return null; // win32 et al: unsupported → degrade, never mis-kill
|
|
2871
|
+
}
|
|
2872
|
+
|
|
2708
2873
|
/**
|
|
2709
2874
|
* Build a progress-aware idle-liveness monitor shared by the pipe/PTY/ACP
|
|
2710
2875
|
* capture paths. `arm()` is called on every output chunk (and once at start) to
|
|
@@ -2730,37 +2895,155 @@ function sampleSubtreeCpu(rootPid) {
|
|
|
2730
2895
|
* read under a coarse sampler must NOT kill; the fine-grained `/proc` path
|
|
2731
2896
|
* carries no resolution and so ages out on the first flat window as before →
|
|
2732
2897
|
* `onIdleKill()`.
|
|
2898
|
+
* #313 extensions:
|
|
2899
|
+
* - `extraSamplers`: additional activity probes (e.g. the workspace-cwd CPU
|
|
2900
|
+
* scan covering out-of-tree sccache/rustc daemons). Each returns
|
|
2901
|
+
* `{ processes, cpu, resolutionMs? } | null`; advancing CPU on ANY sampler
|
|
2902
|
+
* defers the kill. Extra samplers are pure progress signals — they never
|
|
2903
|
+
* satisfy the presence gate (a childless agent with a busy workspace daemon
|
|
2904
|
+
* still needs its own live descendant to count as present... EXCEPT that a
|
|
2905
|
+
* workspace daemon burning CPU IS evidence the agent's work is in flight, so
|
|
2906
|
+
* when an extra sampler shows advancing CPU the kill is deferred even with
|
|
2907
|
+
* no live descendant: the agent may be blocked reading a pipe its daemon
|
|
2908
|
+
* fills. A flat/zero extra sample never kills on its own — presence is
|
|
2909
|
+
* judged from the subtree as before).
|
|
2910
|
+
* - `isActivityHold`: an in-flight-tool-call latch (ACP). While it returns
|
|
2911
|
+
* true the agent has an open tool call, so silence is expected; the kill is
|
|
2912
|
+
* deferred window-by-window. The absolute `--job-timeout` hard cap (and the
|
|
2913
|
+
* tool call's own timeout, enforced by the agent) remain the bound for a
|
|
2914
|
+
* genuinely hung call.
|
|
2915
|
+
* - `logger`: when given, the monitor logs WHY it considered the agent active
|
|
2916
|
+
* or idle at each probe (diagnosability, #313).
|
|
2733
2917
|
* The first elapse with live descendants takes a baseline (a single sample can't
|
|
2734
2918
|
* show movement) and re-probes after the recovery window. The absolute
|
|
2735
2919
|
* `--job-timeout` hard cap remains the ultimate backstop, untouched by this.
|
|
2736
2920
|
*/
|
|
2737
|
-
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu }) {
|
|
2921
|
+
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu, extraSamplers = null, isActivityHold = null, maxActivityHoldMs, logger = null }) {
|
|
2738
2922
|
const enabled = idleTimeoutMs && idleTimeoutMs > 0;
|
|
2739
2923
|
const probeWindow = recoveryWindowMs && recoveryWindowMs > 0 ? recoveryWindowMs : idleTimeoutMs;
|
|
2740
2924
|
let timer = null;
|
|
2741
2925
|
let prev = null; // last subtree CPU probe taken during the current silent stretch
|
|
2926
|
+
let prevExtra = null; // last per-extra-sampler CPU readings (array parallel to extraSamplers)
|
|
2927
|
+
let extraBaselined = false; // true once the extra samplers have a baseline to compare against
|
|
2928
|
+
let extraResolutionMs = 0; // coarsest resolution among live extra samples at the last probe
|
|
2742
2929
|
let flatMs = 0; // cumulative time observed with flat CPU since the last movement/baseline
|
|
2743
2930
|
let lastWindow = idleTimeoutMs; // duration of the currently pending probe window
|
|
2931
|
+
let extraFlatMs = 0; // cumulative flat time across the extra (workspace) samplers
|
|
2932
|
+
let holdMs = 0; // cumulative silent time deferred by the activity hold this stretch
|
|
2933
|
+
// #313: the activity hold must be FINITE — `nano work` defaults to no hard cap,
|
|
2934
|
+
// so an announced-then-hung tool call would otherwise defer forever. Default:
|
|
2935
|
+
// 4 idle windows (configurable). Past it, the hold is ignored and the normal
|
|
2936
|
+
// CPU probes decide (real silent work still defers via CPU progress).
|
|
2937
|
+
const holdCap = Number.isFinite(maxActivityHoldMs) && maxActivityHoldMs >= 0 ? maxActivityHoldMs : (idleTimeoutMs > 0 ? idleTimeoutMs * 4 : 0);
|
|
2744
2938
|
const settled = () => (typeof isSettled === 'function' ? isSettled() : false);
|
|
2745
2939
|
const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
|
|
2746
2940
|
const schedule = (ms) => { lastWindow = ms; timer = setTimeout(onElapse, ms); };
|
|
2941
|
+
const log = (msg) => { try { logger?.info?.(`[idle-watchdog] ${msg}`); } catch { /* best effort */ } };
|
|
2942
|
+
const extras = Array.isArray(extraSamplers) ? extraSamplers.filter((s) => typeof s === 'function') : [];
|
|
2943
|
+
const held = () => { try { return typeof isActivityHold === 'function' && isActivityHold(); } catch { return false; } };
|
|
2747
2944
|
function onElapse() {
|
|
2748
2945
|
timer = null;
|
|
2749
2946
|
if (settled()) return;
|
|
2750
2947
|
const elapsed = lastWindow;
|
|
2948
|
+
// #313: an in-flight tool call (the agent started a bash call that has not
|
|
2949
|
+
// reported a terminal status) means the silence is expected — defer. The
|
|
2950
|
+
// call's own timeout / the absolute hard cap still bound a hung call.
|
|
2951
|
+
if (held()) {
|
|
2952
|
+
if (holdMs < holdCap) {
|
|
2953
|
+
holdMs += elapsed;
|
|
2954
|
+
log(`silent for >= ${elapsed}ms but a tool call is still in flight; deferring the idle-kill (hold ${holdMs}/${holdCap}ms)`);
|
|
2955
|
+
prev = null; prevExtra = null; extraBaselined = false; flatMs = 0; extraFlatMs = 0;
|
|
2956
|
+
schedule(probeWindow);
|
|
2957
|
+
return;
|
|
2958
|
+
}
|
|
2959
|
+
log(`a tool call has been in flight for >= ${holdMs}ms (hold cap ${holdCap}ms); falling back to CPU-progress probes`);
|
|
2960
|
+
}
|
|
2751
2961
|
const pid = typeof getPid === 'function' ? getPid() : undefined;
|
|
2752
2962
|
const sample = pid != null ? sampleCpu(pid) : null;
|
|
2753
|
-
//
|
|
2754
|
-
|
|
2963
|
+
// #313: consult the extra (workspace-cwd) samplers. Advancing CPU on any of
|
|
2964
|
+
// them is work attributable to this agent even when it lives outside the
|
|
2965
|
+
// agent's process tree (a shared sccache daemon), so it defers the kill —
|
|
2966
|
+
// INCLUDING when the subtree itself looks childless/quiescent.
|
|
2967
|
+
let extraActive = false;
|
|
2968
|
+
if (extras.length) {
|
|
2969
|
+
const next = [];
|
|
2970
|
+
extraResolutionMs = 0;
|
|
2971
|
+
for (let i = 0; i < extras.length; i += 1) {
|
|
2972
|
+
let s = null;
|
|
2973
|
+
try { s = extras[i](); } catch { s = null; }
|
|
2974
|
+
next.push(s);
|
|
2975
|
+
if (!s || !Number.isFinite(s.cpu)) continue;
|
|
2976
|
+
if (s.processes > 0 && Number.isFinite(s.resolutionMs)) extraResolutionMs = Math.max(extraResolutionMs, s.resolutionMs);
|
|
2977
|
+
const before = prevExtra && prevExtra[i];
|
|
2978
|
+
if (before != null && Number.isFinite(before.cpu) && s.cpu !== before.cpu) extraActive = true;
|
|
2979
|
+
}
|
|
2980
|
+
const hadBaseline = prevExtra != null;
|
|
2981
|
+
const prevReadings = prevExtra || [];
|
|
2982
|
+
prevExtra = next;
|
|
2983
|
+
if (extraActive) {
|
|
2984
|
+
log(`silent for >= ${elapsed}ms but a workspace process outside the agent tree is burning CPU (e.g. sccache/rustc); deferring the idle-kill`);
|
|
2985
|
+
prev = sample; flatMs = 0; extraFlatMs = 0;
|
|
2986
|
+
schedule(probeWindow);
|
|
2987
|
+
return;
|
|
2988
|
+
}
|
|
2989
|
+
// Flat time only counts while a workspace sampler stays LIVE (a reading
|
|
2990
|
+
// with processes > 0) across consecutive probes. Empty / unreadable
|
|
2991
|
+
// readings must not pre-pay a sampler's resolution allowance: when any
|
|
2992
|
+
// sampler transitions absent/unreadable → live, restart the accumulator
|
|
2993
|
+
// so a fresh process gets its full resolution window before a kill.
|
|
2994
|
+
const isLive = (r) => r != null && Number.isFinite(r.cpu) && r.processes > 0;
|
|
2995
|
+
let newlyLive = false; let continuedLive = false;
|
|
2996
|
+
for (let i = 0; i < next.length; i += 1) {
|
|
2997
|
+
if (!isLive(next[i])) continue;
|
|
2998
|
+
if (hadBaseline && isLive(prevReadings[i])) continuedLive = true; else newlyLive = true;
|
|
2999
|
+
}
|
|
3000
|
+
if (newlyLive || !continuedLive) extraFlatMs = 0;
|
|
3001
|
+
else extraFlatMs += elapsed;
|
|
3002
|
+
}
|
|
3003
|
+
// A live workspace process read flat, but a coarse sampler (whole-second
|
|
3004
|
+
// `ps`) can read flat across a sub-resolution window while still working:
|
|
3005
|
+
// only treat workspace CPU as idle once the flat time covers its resolution.
|
|
3006
|
+
const extraIdleConfirmed = extraFlatMs >= extraResolutionMs;
|
|
3007
|
+
// Childless / unsampleable / dead tree → silence is very likely a hang.
|
|
3008
|
+
// But with extra (workspace) samplers wired we cannot yet distinguish "no
|
|
3009
|
+
// workspace work" from "a workspace daemon whose baseline we have not
|
|
3010
|
+
// taken" on a single reading — so take ONE baseline probe window before
|
|
3011
|
+
// killing, exactly like the subtree first-elapse baseline below. Without
|
|
3012
|
+
// extra samplers (or once a baseline shows no workspace activity) the
|
|
3013
|
+
// legacy immediate kill applies.
|
|
3014
|
+
if (!sample || sample.descendants <= 0) {
|
|
3015
|
+
if (extras.length && !extraBaselined) {
|
|
3016
|
+
// Consume the baseline: the NEXT elapse compares against prevExtra and
|
|
3017
|
+
// either defers (workspace CPU moved) or kills (flat).
|
|
3018
|
+
extraBaselined = true;
|
|
3019
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; baselining the workspace CPU probe and re-checking in ${probeWindow}ms`);
|
|
3020
|
+
schedule(probeWindow);
|
|
3021
|
+
return;
|
|
3022
|
+
}
|
|
3023
|
+
if (!extraIdleConfirmed) {
|
|
3024
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'}; workspace CPU flat for ${extraFlatMs}ms < sampler resolution ${extraResolutionMs}ms, re-checking in ${probeWindow}ms`);
|
|
3025
|
+
schedule(probeWindow);
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
log(`silent for >= ${elapsed}ms with ${!sample ? 'an unsampleable tree' : 'no live descendant'} and no workspace CPU; idle-killing`);
|
|
3029
|
+
onIdleKill();
|
|
3030
|
+
return;
|
|
3031
|
+
}
|
|
2755
3032
|
if (prev != null) {
|
|
2756
3033
|
// Aggregate CPU advanced across the window → the tree is doing work; defer.
|
|
2757
|
-
if (sample.cpu > prev.cpu) {
|
|
3034
|
+
if (sample.cpu > prev.cpu) {
|
|
3035
|
+
log(`silent for >= ${elapsed}ms but subtree CPU advanced (${prev.cpu} → ${sample.cpu}); deferring the idle-kill`);
|
|
3036
|
+
prev = sample; flatMs = 0; schedule(probeWindow); return;
|
|
3037
|
+
}
|
|
2758
3038
|
// Aggregate CPU REGRESSED → `sampleSubtreeCpu*` only sums CPU for live
|
|
2759
3039
|
// PIDs, so the total can legitimately drop when a busy descendant exits
|
|
2760
3040
|
// (its ticks stop counting) even though the subtree just made progress.
|
|
2761
3041
|
// A regression is activity, not a stall: re-baseline and defer rather
|
|
2762
3042
|
// than mis-killing a silent-but-busy job.
|
|
2763
|
-
if (sample.cpu < prev.cpu) {
|
|
3043
|
+
if (sample.cpu < prev.cpu) {
|
|
3044
|
+
log(`silent for >= ${elapsed}ms but subtree CPU regressed (${prev.cpu} → ${sample.cpu}; a busy descendant exited) — progress, not a stall; deferring the idle-kill`);
|
|
3045
|
+
prev = sample; flatMs = 0; schedule(probeWindow); return;
|
|
3046
|
+
}
|
|
2764
3047
|
// Live descendants but CPU exactly unchanged across a full window → a
|
|
2765
3048
|
// candidate stall. Only confirm it once we have observed flat CPU for at
|
|
2766
3049
|
// least the sampler's CPU-time resolution: a probe window shorter than the
|
|
@@ -2768,12 +3051,17 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
|
|
|
2768
3051
|
// low-churn subtree is still progressing but hasn't crossed the next tick.
|
|
2769
3052
|
flatMs += elapsed;
|
|
2770
3053
|
const resolutionMs = Number.isFinite(sample.resolutionMs) ? sample.resolutionMs : 0;
|
|
2771
|
-
if (flatMs >= resolutionMs
|
|
3054
|
+
if (flatMs >= resolutionMs && extraIdleConfirmed) {
|
|
3055
|
+
log(`silent for >= ${elapsed}ms with a live but CPU-flat subtree (${sample.descendants} descendant(s), cpu ${sample.cpu}) and no workspace CPU; idle-killing`);
|
|
3056
|
+
onIdleKill();
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
2772
3059
|
schedule(probeWindow);
|
|
2773
3060
|
return;
|
|
2774
3061
|
}
|
|
2775
3062
|
// First elapse with live descendants: baseline, then re-probe (can't judge
|
|
2776
3063
|
// progress from one sample).
|
|
3064
|
+
log(`silent for >= ${elapsed}ms with ${sample.descendants} live descendant(s); baselining CPU (${sample.cpu}) and re-probing in ${probeWindow}ms`);
|
|
2777
3065
|
prev = sample;
|
|
2778
3066
|
flatMs = 0;
|
|
2779
3067
|
schedule(probeWindow);
|
|
@@ -2783,7 +3071,12 @@ function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, on
|
|
|
2783
3071
|
arm() {
|
|
2784
3072
|
if (!enabled || settled()) return;
|
|
2785
3073
|
prev = null; // output resumed → a fresh silent stretch starts
|
|
3074
|
+
prevExtra = null;
|
|
3075
|
+
extraBaselined = false;
|
|
2786
3076
|
flatMs = 0;
|
|
3077
|
+
extraFlatMs = 0;
|
|
3078
|
+
extraResolutionMs = 0;
|
|
3079
|
+
holdMs = 0;
|
|
2787
3080
|
clear();
|
|
2788
3081
|
schedule(idleTimeoutMs);
|
|
2789
3082
|
},
|
|
@@ -8796,6 +9089,11 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
|
|
|
8796
9089
|
idleTimeoutMs,
|
|
8797
9090
|
recoveryWindowMs,
|
|
8798
9091
|
isSettled: () => settled,
|
|
9092
|
+
logger: getLogger(),
|
|
9093
|
+
// #313: count CPU burned by any process whose cwd is inside the agent's
|
|
9094
|
+
// workspace (a shared sccache daemon lives outside the agent's tree), so
|
|
9095
|
+
// a silent compile holds off the idle-kill.
|
|
9096
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
|
|
8799
9097
|
onIdleKill: () => {
|
|
8800
9098
|
try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
|
|
8801
9099
|
finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
|
|
@@ -8968,6 +9266,11 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
8968
9266
|
idleTimeoutMs,
|
|
8969
9267
|
recoveryWindowMs,
|
|
8970
9268
|
isSettled: () => settled,
|
|
9269
|
+
logger: getLogger(),
|
|
9270
|
+
// #313: count CPU burned by any process whose cwd is inside the agent's
|
|
9271
|
+
// workspace (a shared sccache daemon lives outside the agent's tree), so
|
|
9272
|
+
// a silent compile holds off the idle-kill.
|
|
9273
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [term?.pid] })] : null,
|
|
8971
9274
|
onIdleKill: () => {
|
|
8972
9275
|
killTerm();
|
|
8973
9276
|
finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
|
|
@@ -9342,6 +9645,28 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9342
9645
|
// flips this; the floor never does.
|
|
9343
9646
|
let sawRealTranscript = false;
|
|
9344
9647
|
|
|
9648
|
+
// #313: ACP tool calls currently in flight (`toolCallId` → true). A
|
|
9649
|
+
// `tool_call` notification with a non-terminal status opens the call; a
|
|
9650
|
+
// `tool_call`/`tool_call_update` with a terminal status closes it. While
|
|
9651
|
+
// any call is open the idle watchdog treats the agent as ACTIVE (a long
|
|
9652
|
+
// silent bash call is work, not a hang) — the call's own timeout and the
|
|
9653
|
+
// absolute job hard cap still bound a genuinely hung call.
|
|
9654
|
+
const inFlightToolCalls = new Map();
|
|
9655
|
+
const ACP_TOOL_TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled', 'canceled']);
|
|
9656
|
+
const noteToolCallUpdate = (update) => {
|
|
9657
|
+
if (!update || typeof update !== 'object') return;
|
|
9658
|
+
const kind = update.sessionUpdate || update.type;
|
|
9659
|
+
if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
|
|
9660
|
+
const id = update.toolCallId;
|
|
9661
|
+
if (id == null) return;
|
|
9662
|
+
const status = typeof update.status === 'string' ? update.status.toLowerCase() : '';
|
|
9663
|
+
if (ACP_TOOL_TERMINAL_STATUSES.has(status)) inFlightToolCalls.delete(id);
|
|
9664
|
+
else if (!inFlightToolCalls.has(id)) {
|
|
9665
|
+
inFlightToolCalls.set(id, true);
|
|
9666
|
+
try { logger.info?.(`[idle-watchdog] tool call in flight (${update.title || id}${status ? `, status ${status}` : ''}) — the idle-kill is held while it runs`); } catch { /* best effort */ }
|
|
9667
|
+
}
|
|
9668
|
+
};
|
|
9669
|
+
|
|
9345
9670
|
// Live "spy" tee (--stream), line-buffered, mirroring the other paths.
|
|
9346
9671
|
// Separate line buffers per lane (stdout-human vs stderr) so a partial line
|
|
9347
9672
|
// on one lane never interleaves mid-line with the other — matching pipe/PTY.
|
|
@@ -9670,6 +9995,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9670
9995
|
// A request or notification FROM the agent.
|
|
9671
9996
|
if (typeof msg.method === 'string') {
|
|
9672
9997
|
if (msg.method === 'session/update') {
|
|
9998
|
+
noteToolCallUpdate(msg.params?.update);
|
|
9673
9999
|
emitTranscript(msg.params?.update);
|
|
9674
10000
|
// #194: durable engine-native producer seam. In parallel with the
|
|
9675
10001
|
// transcript-chunk relay overlay, hand the SAME raw `session/update` to
|
|
@@ -9720,6 +10046,16 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
9720
10046
|
idleTimeoutMs,
|
|
9721
10047
|
recoveryWindowMs,
|
|
9722
10048
|
isSettled: () => settled,
|
|
10049
|
+
logger,
|
|
10050
|
+
// #313: while an ACP tool call is in flight the agent is active even
|
|
10051
|
+
// when totally silent (a long foreground bash compile/test). Bounded by
|
|
10052
|
+
// the call's own timeout and the absolute job hard cap.
|
|
10053
|
+
isActivityHold: () => inFlightToolCalls.size > 0,
|
|
10054
|
+
// #313: also count CPU burned by processes whose cwd is inside the
|
|
10055
|
+
// agent's workspace, wherever they sit in the process tree — a shared
|
|
10056
|
+
// sccache daemon (parented to launchd/init, not the agent) compiling
|
|
10057
|
+
// this worktree is real work and must hold off the idle-kill.
|
|
10058
|
+
extraSamplers: cwd ? [() => sampleWorkspaceCpu(cwd, { excludePids: [child?.pid] })] : null,
|
|
9723
10059
|
onIdleKill: () => {
|
|
9724
10060
|
try { killTree(child); } catch { /* best effort */ }
|
|
9725
10061
|
finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
|
|
@@ -21398,6 +21734,9 @@ export {
|
|
|
21398
21734
|
spawnCapturePty,
|
|
21399
21735
|
spawnCaptureAcp,
|
|
21400
21736
|
sampleSubtreeCpu,
|
|
21737
|
+
sampleWorkspaceCpu,
|
|
21738
|
+
sampleWorkspaceCpuProc,
|
|
21739
|
+
sampleWorkspaceCpuLsof,
|
|
21401
21740
|
createIdleLivenessMonitor,
|
|
21402
21741
|
resolveLivenessOverrides,
|
|
21403
21742
|
parsePsTime,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "c8ctl-plugin-nano",
|
|
3
|
-
"version": "1.73.
|
|
3
|
+
"version": "1.73.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
|
|
6
6
|
"main": "c8ctl-plugin.js",
|
|
@@ -76,12 +76,12 @@
|
|
|
76
76
|
},
|
|
77
77
|
"optionalDependencies": {
|
|
78
78
|
"node-pty": "^1.0.0",
|
|
79
|
-
"@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.73.
|
|
80
|
-
"@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.73.
|
|
81
|
-
"@nanobpm/c8ctl-plugin-nano-linux-x64": "1.73.
|
|
82
|
-
"@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.73.
|
|
83
|
-
"@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.73.
|
|
84
|
-
"@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.73.
|
|
85
|
-
"@nanobpm/c8ctl-plugin-nano-win32-x64": "1.73.
|
|
79
|
+
"@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.73.1",
|
|
80
|
+
"@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.73.1",
|
|
81
|
+
"@nanobpm/c8ctl-plugin-nano-linux-x64": "1.73.1",
|
|
82
|
+
"@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.73.1",
|
|
83
|
+
"@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.73.1",
|
|
84
|
+
"@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.73.1",
|
|
85
|
+
"@nanobpm/c8ctl-plugin-nano-win32-x64": "1.73.1"
|
|
86
86
|
}
|
|
87
87
|
}
|