caproom 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -15
- package/bin/caproom +406 -32
- package/bin/caproom-mcp.js +264 -0
- package/bin/caproom.js +21 -0
- package/bin/caproom.ps1 +471 -0
- package/package.json +10 -5
package/bin/caproom
CHANGED
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
# on macOS/Linux. macOS has no working RLIMIT_AS/DATA/RSS or launchd RSS
|
|
4
4
|
# enforcement (verified empirically — both are no-ops on modern macOS), so
|
|
5
5
|
# this uses whichever real enforcement mechanism is available:
|
|
6
|
-
# 1.
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
6
|
+
# 1. Host-native polling watchdog (process-tree RSS + SIGKILL) — the
|
|
7
|
+
# DEFAULT. Runs in your real environment: same PATH, auth, native
|
|
8
|
+
# binaries, tty. Small race window bounded by --interval.
|
|
9
|
+
# 2. Docker cgroup (--memory) — opt-in via --docker. Hard cap, zero race
|
|
10
|
+
# window, but runs the command inside a Linux container (native-module
|
|
11
|
+
# and toolchain drift; see README). Fails loudly if the daemon is down.
|
|
10
12
|
set -euo pipefail
|
|
11
13
|
|
|
12
14
|
usage() {
|
|
@@ -18,27 +20,41 @@ usage: caproom [--limit <mb>] [--image <docker-image>] [--interval <sec>] -- <co
|
|
|
18
20
|
caproom park <pid>
|
|
19
21
|
caproom wake <pid>
|
|
20
22
|
caproom status <pid>
|
|
23
|
+
caproom guard [--threshold <pct>] [--interval <sec>] <pid...>
|
|
21
24
|
caproom init <command> [--limit <mb>] [--grace <sec>]
|
|
22
25
|
|
|
23
26
|
--limit <mb> memory cap in MB (default: 4096)
|
|
24
|
-
--
|
|
25
|
-
backend (default: node:22-slim)
|
|
26
|
-
--interval <sec> watchdog poll interval in seconds, fallback backend only
|
|
27
|
-
(default: 0.2)
|
|
27
|
+
--interval <sec> watchdog poll interval in seconds (default: 0.2)
|
|
28
28
|
--grace <sec> seconds to wait after SIGTERM before SIGKILL, watchdog
|
|
29
29
|
backend only (default: 5) — gives the process a chance
|
|
30
30
|
to flush/save state before a hard kill
|
|
31
|
-
--
|
|
31
|
+
--docker opt in to the Docker cgroup backend instead of the default
|
|
32
|
+
host-native watchdog (needs the daemon running)
|
|
33
|
+
--image <name> docker image used by the --docker backend
|
|
34
|
+
(default: node:22-slim)
|
|
35
|
+
--force-watchdog no-op; the host-native watchdog IS the default — kept so
|
|
36
|
+
existing scripts and 'init' snippets keep working
|
|
32
37
|
|
|
33
|
-
park / wake — freeze an idle process so the kernel
|
|
34
|
-
memory without killing it.
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
park / wake — freeze an idle process so the kernel CAN reclaim/compress its
|
|
39
|
+
memory without killing it. Honest semantics: SIGSTOP only makes the pages
|
|
40
|
+
eligible — the kernel reclaims them lazily, when real memory pressure hits.
|
|
41
|
+
Park a 2GB agent on a quiet machine and it may stay ~2GB resident for hours.
|
|
42
|
+
Park is insurance against OOM, not immediate RAM return; use it for processes
|
|
43
|
+
too expensive to restart. `caproom park <pid>` (SIGSTOP), `caproom wake
|
|
38
44
|
<pid>` (SIGCONT) brings it back instantly, same state, no restart needed.
|
|
39
45
|
Any agent can call these directly — they're just SIGSTOP/SIGCONT, no daemon,
|
|
40
46
|
no tracking file required.
|
|
41
47
|
|
|
48
|
+
guard — watch SYSTEM-WIDE free memory (not any single process) and auto-park
|
|
49
|
+
tracked pids (SIGSTOP) when free mem drops below --threshold percent, before
|
|
50
|
+
the kernel OOM-killer has to pick a victim. Use it when unrelated heavy
|
|
51
|
+
processes (e.g. a GPU inference job in one terminal, a TTS job in another)
|
|
52
|
+
share a box and neither is individually over any --limit cap. Foreground,
|
|
53
|
+
blocking; exits once all watched pids have exited. Does not auto-wake —
|
|
54
|
+
`caproom wake <pid>` when memory pressure clears:
|
|
55
|
+
|
|
56
|
+
caproom guard --threshold 10 --interval 5 -- 12345 12346
|
|
57
|
+
|
|
42
58
|
init <command> — print a shell snippet that auto-caps <command> on every
|
|
43
59
|
invocation, so a new terminal tab is capped with no extra typing. Append the
|
|
44
60
|
output to your shell rc (~/.zshrc, ~/.bashrc):
|
|
@@ -50,8 +66,11 @@ env vars (override flags): CAPROOM_LIMIT_MB, CAPROOM_IMAGE, CAPROOM_INTERVAL, CA
|
|
|
50
66
|
examples:
|
|
51
67
|
caproom --limit 2048 -- npm run build
|
|
52
68
|
caproom --limit 512 -- claude --dangerously-skip-permissions -p "task"
|
|
69
|
+
caproom --limit 4096 --docker --image python:3.12-slim -- python train.py
|
|
53
70
|
caproom park 12345
|
|
54
71
|
caproom wake 12345
|
|
72
|
+
caproom top --json [--pid <pid>] [--park-min-mb <mb>]
|
|
73
|
+
caproom watch [--threshold-mb <mb>] [--auto-park] [--auto-wake-free-pct <pct>] [--json] <pid...>
|
|
55
74
|
caproom init claude --limit 6144 --grace 10
|
|
56
75
|
EOF
|
|
57
76
|
exit "$code"
|
|
@@ -86,7 +105,7 @@ cmd_park() {
|
|
|
86
105
|
[[ -z "$pid" ]] && { echo "usage: caproom park <pid>" >&2; exit 1; }
|
|
87
106
|
kill -0 "$pid" 2>/dev/null || { echo "caproom: no such pid $pid" >&2; exit 1; }
|
|
88
107
|
kill -STOP "$pid"
|
|
89
|
-
echo "caproom: pid $pid parked (SIGSTOP) —
|
|
108
|
+
echo "caproom: pid $pid parked (SIGSTOP) — pages now eligible for kernel reclaim, but the kernel acts only under real memory pressure; on a quiet machine RSS may not drop. wake with: caproom wake $pid" >&2
|
|
90
109
|
}
|
|
91
110
|
|
|
92
111
|
cmd_wake() {
|
|
@@ -103,10 +122,334 @@ cmd_status() {
|
|
|
103
122
|
ps -o pid,stat,rss,etime,command -p "$pid" 2>/dev/null || { echo "caproom: no such pid $pid" >&2; exit 1; }
|
|
104
123
|
}
|
|
105
124
|
|
|
125
|
+
# ---- process-tree inventory (top / watch foundation) ----
|
|
126
|
+
|
|
127
|
+
# One ps pass filling the SNAP_* global arrays for the current user.
|
|
128
|
+
read_snapshot() {
|
|
129
|
+
SNAP_PID=(); SNAP_PPID=(); SNAP_RSS=(); SNAP_ST=(); SNAP_ET=(); SNAP_CMD=()
|
|
130
|
+
local myuid uid pid ppid rss st et cmd
|
|
131
|
+
myuid="$(id -u)"
|
|
132
|
+
while read -r uid pid ppid rss st et cmd; do
|
|
133
|
+
[[ "$uid" != "$myuid" ]] && continue
|
|
134
|
+
SNAP_PID+=("$pid"); SNAP_PPID+=("$ppid"); SNAP_RSS+=("${rss:-0}")
|
|
135
|
+
SNAP_ST+=("${st:-?}"); SNAP_ET+=("${et:-0}"); SNAP_CMD+=("${cmd:-}")
|
|
136
|
+
done < <(ps -eo uid=,pid=,ppid=,rss=,state=,etime=,command=)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
# Walk the subtree of $1 over the existing SNAP_* arrays, filling
|
|
140
|
+
# TREE_PIDS / TREE_RSS_KB. Does NOT re-read ps — cheap enough to call
|
|
141
|
+
# once per tree root from a single snapshot.
|
|
142
|
+
walk_tree() {
|
|
143
|
+
local root="$1" cur i j
|
|
144
|
+
local -a lpids=("${SNAP_PPID[@]}") lq=()
|
|
145
|
+
TREE_PIDS=(); TREE_RSS_KB=0
|
|
146
|
+
lq=("$root")
|
|
147
|
+
while [[ ${#lq[@]} -gt 0 ]]; do
|
|
148
|
+
cur="${lq[0]}"
|
|
149
|
+
if [[ ${#lq[@]} -gt 1 ]]; then lq=("${lq[@]:1}"); else lq=(); fi
|
|
150
|
+
for i in "${!SNAP_PID[@]}"; do
|
|
151
|
+
if [[ "${SNAP_PID[$i]}" == "$cur" ]]; then
|
|
152
|
+
TREE_PIDS+=("$cur")
|
|
153
|
+
TREE_RSS_KB=$(( TREE_RSS_KB + SNAP_RSS[$i] ))
|
|
154
|
+
for j in "${!lpids[@]}"; do
|
|
155
|
+
if [[ "${lpids[$j]}" == "$cur" ]]; then
|
|
156
|
+
lq+=("${SNAP_PID[$j]}")
|
|
157
|
+
lpids[$j]=""
|
|
158
|
+
fi
|
|
159
|
+
done
|
|
160
|
+
break
|
|
161
|
+
fi
|
|
162
|
+
done
|
|
163
|
+
done
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
json_escape() {
|
|
167
|
+
local s="$1"
|
|
168
|
+
s="${s//\\/\\\\}"
|
|
169
|
+
s="${s//\"/\\\"}"
|
|
170
|
+
s="${s//$'\n'/ }"
|
|
171
|
+
s="${s//$'\r'/ }"
|
|
172
|
+
s="${s//$'\t'/ }"
|
|
173
|
+
printf '%s' "$s"
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
cmd_top() {
|
|
177
|
+
local json=0 park_min_kb=$(( 512 * 1024 )) filter_pid=""
|
|
178
|
+
while [[ $# -gt 0 ]]; do
|
|
179
|
+
case "$1" in
|
|
180
|
+
--json) json=1; shift ;;
|
|
181
|
+
--park-min-mb) park_min_kb=$(( $2 * 1024 )); shift 2 ;;
|
|
182
|
+
--pid) filter_pid="$2"; shift 2 ;;
|
|
183
|
+
*) echo "caproom top: unknown option $1" >&2; exit 1 ;;
|
|
184
|
+
esac
|
|
185
|
+
done
|
|
186
|
+
|
|
187
|
+
read_snapshot
|
|
188
|
+
[[ ${#SNAP_PID[@]} -eq 0 ]] && { [[ $json -eq 1 ]] && printf '{"schema":1,"ts":%s,"limit_mb_default":%s,"processes":[]}\n' "$(date +%s)" "${CAPROOM_LIMIT_MB:-4096}"; return 0; }
|
|
189
|
+
|
|
190
|
+
# Tree roots: parents outside the visible set (or init-reparented).
|
|
191
|
+
local -a roots=()
|
|
192
|
+
local i j p found
|
|
193
|
+
if [[ -n "$filter_pid" ]]; then
|
|
194
|
+
found=""
|
|
195
|
+
for i in "${!SNAP_PID[@]}"; do
|
|
196
|
+
[[ "${SNAP_PID[$i]}" == "$filter_pid" ]] && { roots+=("$filter_pid"); found=1; break; }
|
|
197
|
+
done
|
|
198
|
+
if [[ -z "$found" ]]; then
|
|
199
|
+
echo "caproom: no such pid $filter_pid (or not owned by you)" >&2
|
|
200
|
+
exit 1
|
|
201
|
+
fi
|
|
202
|
+
else
|
|
203
|
+
for i in "${!SNAP_PID[@]}"; do
|
|
204
|
+
[[ "${SNAP_PID[$i]}" == "$$" ]] && continue # never report ourselves
|
|
205
|
+
p="${SNAP_PPID[$i]}"
|
|
206
|
+
if [[ "$p" == "1" ]]; then roots+=("${SNAP_PID[$i]}"); continue; fi
|
|
207
|
+
found=""
|
|
208
|
+
for j in "${!SNAP_PID[@]}"; do
|
|
209
|
+
if [[ "${SNAP_PID[$j]}" == "$p" ]]; then found=1; break; fi
|
|
210
|
+
done
|
|
211
|
+
[[ -z "$found" ]] && roots+=("${SNAP_PID[$i]}")
|
|
212
|
+
done
|
|
213
|
+
fi
|
|
214
|
+
|
|
215
|
+
# Walk each root once; keep results in parallel arrays, then sort by
|
|
216
|
+
# tree RSS descending via a sortable temp stream.
|
|
217
|
+
local -a r_pid=() r_trss=() r_tpids=() r_st=() r_et=() r_cmd=() order=()
|
|
218
|
+
for p in "${roots[@]}"; do
|
|
219
|
+
walk_tree "$p"
|
|
220
|
+
local tjoin=""
|
|
221
|
+
[[ ${#TREE_PIDS[@]} -gt 0 ]] && tjoin="$(printf '%s,' "${TREE_PIDS[@]}")" && tjoin="${tjoin%,}"
|
|
222
|
+
for i in "${!SNAP_PID[@]}"; do
|
|
223
|
+
if [[ "${SNAP_PID[$i]}" == "$p" ]]; then
|
|
224
|
+
r_pid+=("$p"); r_trss+=("$TREE_RSS_KB")
|
|
225
|
+
r_tpids+=("$tjoin")
|
|
226
|
+
r_st+=("${SNAP_ST[$i]}"); r_et+=("${SNAP_ET[$i]}")
|
|
227
|
+
r_cmd+=("${SNAP_CMD[$i]}")
|
|
228
|
+
order+=("$(printf '%010d %d\n' "$TREE_RSS_KB" $(( ${#r_pid[@]} - 1 )))")
|
|
229
|
+
break
|
|
230
|
+
fi
|
|
231
|
+
done
|
|
232
|
+
done
|
|
233
|
+
|
|
234
|
+
if [[ $json -eq 1 ]]; then
|
|
235
|
+
local ts out='[' first=1 idx state cand reason kb
|
|
236
|
+
ts="$(date +%s)"
|
|
237
|
+
local -a sorted
|
|
238
|
+
sorted=($(printf '%s\n' "${order[@]:-}" | sort -rn | awk '{print $2}'))
|
|
239
|
+
for idx in "${sorted[@]:-}"; do
|
|
240
|
+
[[ -z "$idx" ]] && continue
|
|
241
|
+
kb="${r_trss[$idx]}"
|
|
242
|
+
local st0="${r_st[$idx]:0:1}"
|
|
243
|
+
case "$st0" in
|
|
244
|
+
T) state="parked" ;;
|
|
245
|
+
Z) state="zombie" ;;
|
|
246
|
+
*) state="running" ;;
|
|
247
|
+
esac
|
|
248
|
+
cand=false; reason=""
|
|
249
|
+
if [[ "$state" == "running" && ( "$st0" == "S" || "$st0" == "I" ) ]] && [[ "$kb" -ge "$park_min_kb" ]]; then
|
|
250
|
+
cand=true
|
|
251
|
+
reason="root sleeping + tree_rss ${kb}KB >= ${park_min_kb}KB park threshold"
|
|
252
|
+
fi
|
|
253
|
+
[[ $first -eq 1 ]] || out+=','
|
|
254
|
+
first=0
|
|
255
|
+
out+="{\"pid\":${r_pid[$idx]},\"cmd\":\"$(json_escape "${r_cmd[$idx]}")\",\"tree_rss_kb\":${kb},\"tree_pids\":[${r_tpids[$idx]}],\"state\":\"$state\",\"park_candidate\":$cand,\"reason\":\"$(json_escape "$reason")\"}"
|
|
256
|
+
done
|
|
257
|
+
out+=']'
|
|
258
|
+
printf '{"schema":1,"ts":%s,"limit_mb_default":%s,"processes":%s}\n' "$ts" "${CAPROOM_LIMIT_MB:-4096}" "$out"
|
|
259
|
+
else
|
|
260
|
+
local idx mb
|
|
261
|
+
printf '%-8s %10s %-8s %-9s %s\n' PID TREE_MB STATE ETIME COMMAND
|
|
262
|
+
local -a sorted
|
|
263
|
+
sorted=($(printf '%s\n' "${order[@]:-}" | sort -rn | awk '{print $2}'))
|
|
264
|
+
for idx in "${sorted[@]:-}"; do
|
|
265
|
+
[[ -z "$idx" ]] && continue
|
|
266
|
+
mb=$(( r_trss[idx] / 1024 ))
|
|
267
|
+
case "${r_st[$idx]}" in T) printf '%-8s %10s %-8s %-9s %s\n' "${r_pid[$idx]}" "$mb" PARKED "${r_et[$idx]}" "${r_cmd[$idx]:0:60}" ;; *) printf '%-8s %10s %-8s %-9s %s\n' "${r_pid[$idx]}" "$mb" "-" "${r_et[$idx]}" "${r_cmd[$idx]:0:60}" ;; esac
|
|
268
|
+
done
|
|
269
|
+
fi
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
cmd_watch() {
|
|
273
|
+
# Daemon: watch explicit pids, emit events when their TREE crosses a
|
|
274
|
+
# RSS threshold. --auto-park freezes breaching trees (SIGSTOP every pid
|
|
275
|
+
# in the snapshot) — only for pids passed explicitly, since stopping a
|
|
276
|
+
# mid-write process risks corruption; naming the pid IS the opt-in.
|
|
277
|
+
# --auto-wake-free-pct N undoes its own parks when free memory recovers.
|
|
278
|
+
local threshold_kb=$(( 2048 * 1024 )) interval=5 json=0 auto=0 wake_pct=""
|
|
279
|
+
local -a pids=()
|
|
280
|
+
while [[ $# -gt 0 ]]; do
|
|
281
|
+
case "$1" in
|
|
282
|
+
--threshold-mb) threshold_kb=$(( $2 * 1024 )); shift 2 ;;
|
|
283
|
+
--interval) interval="$2"; shift 2 ;;
|
|
284
|
+
--auto-park) auto=1; shift ;;
|
|
285
|
+
--auto-wake-free-pct) wake_pct="$2"; shift 2 ;;
|
|
286
|
+
--json) json=1; shift ;;
|
|
287
|
+
*) pids+=("$1"); shift ;;
|
|
288
|
+
esac
|
|
289
|
+
done
|
|
290
|
+
[[ ${#pids[@]} -eq 0 ]] && { echo "usage: caproom watch [--threshold-mb <mb>] [--interval <sec>] [--auto-park] [--auto-wake-free-pct <pct>] [--json] <pid...>" >&2; exit 1; }
|
|
291
|
+
|
|
292
|
+
local -a parked_by_us=() breaching=()
|
|
293
|
+
local mode
|
|
294
|
+
mode="watch"
|
|
295
|
+
[[ $auto -eq 1 ]] && mode="auto-park"
|
|
296
|
+
if [[ $json -eq 1 ]]; then
|
|
297
|
+
printf '{"schema":1,"event":"started","ts":%s,"mode":"%s","threshold_kb":%s,"pids":[%s]}\n' "$(date +%s)" "$mode" "$threshold_kb" "$(printf '%s,' "${pids[@]}" | sed 's/,$//')"
|
|
298
|
+
else
|
|
299
|
+
echo "caproom: watching ${#pids[@]} pid(s), tree threshold $(( threshold_kb / 1024 ))MB, poll ${interval}s$([[ $auto -eq 1 ]] && echo ', AUTO-PARK ARMED')$([[ -n "$wake_pct" ]] && echo ", auto-wake at >=${wake_pct}% free")" >&2
|
|
300
|
+
fi
|
|
301
|
+
|
|
302
|
+
while :; do
|
|
303
|
+
local -a alive=()
|
|
304
|
+
local pid i st0
|
|
305
|
+
for pid in "${pids[@]}"; do
|
|
306
|
+
kill -0 "$pid" 2>/dev/null && alive+=("$pid")
|
|
307
|
+
done
|
|
308
|
+
if [[ ${#alive[@]} -eq 0 ]]; then
|
|
309
|
+
[[ $json -eq 1 ]] && printf '{"schema":1,"event":"all-exited","ts":%s}\n' "$(date +%s)"
|
|
310
|
+
echo "caproom: watch: all watched pids exited" >&2
|
|
311
|
+
exit 0
|
|
312
|
+
fi
|
|
313
|
+
pids=("${alive[@]}")
|
|
314
|
+
|
|
315
|
+
# Auto-wake first: restore what WE parked once pressure clears.
|
|
316
|
+
if [[ -n "$wake_pct" && ${#parked_by_us[@]} -gt 0 ]]; then
|
|
317
|
+
local pct
|
|
318
|
+
pct=$(mem_free_pct)
|
|
319
|
+
if [[ "$pct" -ge "$wake_pct" ]]; then
|
|
320
|
+
local -a woke=()
|
|
321
|
+
for pid in "${parked_by_us[@]}"; do
|
|
322
|
+
if kill -0 "$pid" 2>/dev/null && kill -CONT "$pid" 2>/dev/null; then
|
|
323
|
+
woke+=("$pid")
|
|
324
|
+
if [[ $json -eq 1 ]]; then
|
|
325
|
+
printf '{"schema":1,"event":"woke","ts":%s,"pid":%s,"free_pct":%s}\n' "$(date +%s)" "$pid" "$pct"
|
|
326
|
+
else
|
|
327
|
+
echo "caproom: watch: free mem ${pct}% >= ${wake_pct}% — waking pid $pid" >&2
|
|
328
|
+
fi
|
|
329
|
+
fi
|
|
330
|
+
done
|
|
331
|
+
parked_by_us=()
|
|
332
|
+
fi
|
|
333
|
+
fi
|
|
334
|
+
|
|
335
|
+
read_snapshot
|
|
336
|
+
for pid in "${pids[@]}"; do
|
|
337
|
+
local found=""
|
|
338
|
+
for i in "${!SNAP_PID[@]}"; do
|
|
339
|
+
[[ "${SNAP_PID[$i]}" == "$pid" ]] && { found="$i"; break; }
|
|
340
|
+
done
|
|
341
|
+
[[ -z "$found" ]] && continue
|
|
342
|
+
st0="${SNAP_ST[$found]:0:1}"
|
|
343
|
+
[[ "$st0" == "T" || "$st0" == "Z" ]] && continue # already parked/dead
|
|
344
|
+
walk_tree "$pid"
|
|
345
|
+
if [[ "$TREE_RSS_KB" -ge "$threshold_kb" ]]; then
|
|
346
|
+
local is_breaching=""
|
|
347
|
+
local ev1
|
|
348
|
+
for ev1 in ${breaching[@]+"${breaching[@]}"}; do [[ "$ev1" == "$pid" ]] && is_breaching=1 && break; done
|
|
349
|
+
if [[ -n "$is_breaching" ]]; then continue; fi
|
|
350
|
+
breaching+=("$pid")
|
|
351
|
+
if [[ $auto -eq 1 ]]; then
|
|
352
|
+
local tp stopped=0
|
|
353
|
+
for tp in "${TREE_PIDS[@]}"; do
|
|
354
|
+
kill -STOP "$tp" 2>/dev/null && { parked_by_us+=("$tp"); stopped=$(( stopped + 1 )); }
|
|
355
|
+
done
|
|
356
|
+
if [[ $json -eq 1 ]]; then
|
|
357
|
+
printf '{"schema":1,"event":"parked","ts":%s,"pid":%s,"tree_rss_kb":%s,"tree_pids":[%s],"stopped":%s}\n' "$(date +%s)" "$pid" "$TREE_RSS_KB" "$(printf '%s,' "${TREE_PIDS[@]}" | sed 's/,$//')" "$stopped"
|
|
358
|
+
else
|
|
359
|
+
echo "caproom: watch: tree of pid $pid hit $(( TREE_RSS_KB / 1024 ))MB (>= $(( threshold_kb / 1024 ))MB) — PARKED tree (${stopped} pids, wake: caproom wake $pid)" >&2
|
|
360
|
+
fi
|
|
361
|
+
else
|
|
362
|
+
if [[ $json -eq 1 ]]; then
|
|
363
|
+
printf '{"schema":1,"event":"breach","ts":%s,"pid":%s,"tree_rss_kb":%s}\n' "$(date +%s)" "$pid" "$TREE_RSS_KB"
|
|
364
|
+
else
|
|
365
|
+
echo "caproom: watch: tree of pid $pid hit $(( TREE_RSS_KB / 1024 ))MB (>= $(( threshold_kb / 1024 ))MB) — no --auto-park, reporting only" >&2
|
|
366
|
+
fi
|
|
367
|
+
fi
|
|
368
|
+
else
|
|
369
|
+
local -a keep=()
|
|
370
|
+
local was_breaching=0 ev2
|
|
371
|
+
for ev2 in ${breaching[@]+"${breaching[@]}"}; do
|
|
372
|
+
if [[ "$ev2" == "$pid" ]]; then was_breaching=1; else keep+=("$ev2"); fi
|
|
373
|
+
done
|
|
374
|
+
if [[ $was_breaching -eq 1 ]]; then
|
|
375
|
+
breaching=()
|
|
376
|
+
local k2
|
|
377
|
+
for k2 in ${keep[@]+"${keep[@]}"}; do breaching+=("$k2"); done
|
|
378
|
+
if [[ $json -eq 1 ]]; then
|
|
379
|
+
printf '{"schema":1,"event":"recovered","ts":%s,"pid":%s,"tree_rss_kb":%s}\n' "$(date +%s)" "$pid" "$TREE_RSS_KB"
|
|
380
|
+
else
|
|
381
|
+
echo "caproom: watch: pid $pid back under threshold ($(( TREE_RSS_KB / 1024 ))MB)" >&2
|
|
382
|
+
fi
|
|
383
|
+
fi
|
|
384
|
+
fi
|
|
385
|
+
done
|
|
386
|
+
sleep "$interval"
|
|
387
|
+
done
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
mem_free_pct() { if [[ "$(uname)" == "Darwin" ]]; then
|
|
391
|
+
local page_size free inactive total_bytes avail_bytes
|
|
392
|
+
page_size=$(vm_stat | awk '/page size of/ {print $8}')
|
|
393
|
+
free=$(vm_stat | awk '/Pages free/ {gsub("\\.","",$3); print $3}')
|
|
394
|
+
inactive=$(vm_stat | awk '/Pages inactive/ {gsub("\\.","",$3); print $3}')
|
|
395
|
+
total_bytes=$(sysctl -n hw.memsize)
|
|
396
|
+
avail_bytes=$(( (free + inactive) * page_size ))
|
|
397
|
+
echo $(( avail_bytes * 100 / total_bytes ))
|
|
398
|
+
else
|
|
399
|
+
local avail_kb total_kb
|
|
400
|
+
avail_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
|
401
|
+
total_kb=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
|
402
|
+
echo $(( avail_kb * 100 / total_kb ))
|
|
403
|
+
fi
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
cmd_guard() {
|
|
407
|
+
local threshold=10
|
|
408
|
+
local interval=5
|
|
409
|
+
local pids=()
|
|
410
|
+
while [[ $# -gt 0 ]]; do
|
|
411
|
+
case "$1" in
|
|
412
|
+
--threshold) threshold="$2"; shift 2 ;;
|
|
413
|
+
--interval) interval="$2"; shift 2 ;;
|
|
414
|
+
--) shift ;;
|
|
415
|
+
*) pids+=("$1"); shift ;;
|
|
416
|
+
esac
|
|
417
|
+
done
|
|
418
|
+
[[ ${#pids[@]} -eq 0 ]] && { echo "usage: caproom guard [--threshold <pct>] [--interval <sec>] <pid...>" >&2; exit 1; }
|
|
419
|
+
echo "caproom: guarding ${#pids[@]} pid(s), park when system free mem < ${threshold}% (poll ${interval}s)" >&2
|
|
420
|
+
local parked=()
|
|
421
|
+
while :; do
|
|
422
|
+
local alive=()
|
|
423
|
+
local pid
|
|
424
|
+
for pid in "${pids[@]}"; do
|
|
425
|
+
kill -0 "$pid" 2>/dev/null && alive+=("$pid")
|
|
426
|
+
done
|
|
427
|
+
if [[ ${#alive[@]} -eq 0 ]]; then
|
|
428
|
+
echo "caproom: guard: all watched pids exited" >&2
|
|
429
|
+
exit 0
|
|
430
|
+
fi
|
|
431
|
+
pids=("${alive[@]}")
|
|
432
|
+
local pct
|
|
433
|
+
pct=$(mem_free_pct)
|
|
434
|
+
if [[ "$pct" -lt "$threshold" ]]; then
|
|
435
|
+
for pid in "${pids[@]}"; do
|
|
436
|
+
if [[ ! " ${parked[*]:-} " == *" $pid "* ]]; then
|
|
437
|
+
echo "caproom: system free mem ${pct}% < ${threshold}% threshold — about to blow, parking pid $pid (SIGSTOP)" >&2
|
|
438
|
+
kill -STOP "$pid" 2>/dev/null && parked+=("$pid")
|
|
439
|
+
fi
|
|
440
|
+
done
|
|
441
|
+
fi
|
|
442
|
+
sleep "$interval"
|
|
443
|
+
done
|
|
444
|
+
}
|
|
445
|
+
|
|
106
446
|
case "${1:-}" in
|
|
107
447
|
park) shift; cmd_park "$@"; exit 0 ;;
|
|
108
448
|
wake) shift; cmd_wake "$@"; exit 0 ;;
|
|
109
449
|
status) shift; cmd_status "$@"; exit 0 ;;
|
|
450
|
+
top) shift; cmd_top "$@"; exit 0 ;;
|
|
451
|
+
watch) shift; cmd_watch "$@"; exit 0 ;;
|
|
452
|
+
guard) shift; cmd_guard "$@"; exit 0 ;;
|
|
110
453
|
init) shift; cmd_init "$@"; exit 0 ;;
|
|
111
454
|
help|-h|--help) usage help ;;
|
|
112
455
|
esac
|
|
@@ -115,7 +458,7 @@ LIMIT_MB="${CAPROOM_LIMIT_MB:-4096}"
|
|
|
115
458
|
IMAGE="${CAPROOM_IMAGE:-node:22-slim}"
|
|
116
459
|
INTERVAL="${CAPROOM_INTERVAL:-0.2}"
|
|
117
460
|
GRACE="${CAPROOM_GRACE:-5}"
|
|
118
|
-
|
|
461
|
+
USE_DOCKER=0
|
|
119
462
|
|
|
120
463
|
while [[ $# -gt 0 ]]; do
|
|
121
464
|
case "$1" in
|
|
@@ -123,7 +466,10 @@ while [[ $# -gt 0 ]]; do
|
|
|
123
466
|
--image) IMAGE="$2"; shift 2 ;;
|
|
124
467
|
--interval) INTERVAL="$2"; shift 2 ;;
|
|
125
468
|
--grace) GRACE="$2"; shift 2 ;;
|
|
126
|
-
--
|
|
469
|
+
--docker) USE_DOCKER=1; shift ;;
|
|
470
|
+
# legacy no-op: the watchdog IS the default now; accepted so old
|
|
471
|
+
# scripts and init snippets keep working
|
|
472
|
+
--force-watchdog) shift ;;
|
|
127
473
|
--) shift; break ;;
|
|
128
474
|
-h|--help) usage help ;;
|
|
129
475
|
*) break ;;
|
|
@@ -132,10 +478,6 @@ done
|
|
|
132
478
|
|
|
133
479
|
[[ $# -eq 0 ]] && usage
|
|
134
480
|
|
|
135
|
-
docker_available() {
|
|
136
|
-
[[ "$FORCE_WATCHDOG" -eq 0 ]] && command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1
|
|
137
|
-
}
|
|
138
|
-
|
|
139
481
|
run_docker() {
|
|
140
482
|
echo "caproom: docker cgroup backend, limit=${LIMIT_MB}m image=${IMAGE}" >&2
|
|
141
483
|
exec docker run --rm -i \
|
|
@@ -143,26 +485,50 @@ run_docker() {
|
|
|
143
485
|
-v "$PWD:/work" -w /work "$IMAGE" "$@"
|
|
144
486
|
}
|
|
145
487
|
|
|
488
|
+
collect_tree() {
|
|
489
|
+
# Snapshot ps once and walk the descendant tree of $1 into TREE_PIDS /
|
|
490
|
+
# TREE_RSS_KB (see read_snapshot / walk_tree). Plain indexed arrays only
|
|
491
|
+
# so macOS's stock bash 3.2 works.
|
|
492
|
+
read_snapshot
|
|
493
|
+
walk_tree "$1"
|
|
494
|
+
}
|
|
495
|
+
|
|
146
496
|
run_watchdog() {
|
|
147
|
-
echo "caproom: watchdog backend (
|
|
497
|
+
echo "caproom: watchdog backend (host-native), limit=${LIMIT_MB}m poll=${INTERVAL}s (process-tree RSS)" >&2
|
|
148
498
|
local limit_kb=$(( LIMIT_MB * 1024 ))
|
|
149
499
|
"$@" &
|
|
150
500
|
local pid=$!
|
|
151
501
|
local exit_code=0
|
|
152
502
|
while kill -0 "$pid" 2>/dev/null; do
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
echo "caproom: pid $pid RSS ${
|
|
157
|
-
|
|
503
|
+
collect_tree "$pid"
|
|
504
|
+
if [[ "${#TREE_PIDS[@]}" -gt 0 && "$TREE_RSS_KB" -gt "$limit_kb" ]]; then
|
|
505
|
+
local overshoot=$(( TREE_RSS_KB * 100 / limit_kb ))
|
|
506
|
+
echo "caproom: pid $pid tree RSS ${TREE_RSS_KB}KB exceeded ${limit_kb}KB cap (+${overshoot}%) — killing tree (grace ${GRACE}s)" >&2
|
|
507
|
+
# Signal EVERY pid in the tree, not just the root: children that
|
|
508
|
+
# survive a root-only TERM get orphaned and keep allocating past
|
|
509
|
+
# the cap after caproom exits.
|
|
510
|
+
kill -TERM "${TREE_PIDS[@]}" 2>/dev/null || true
|
|
511
|
+
local -a breach_pids=("${TREE_PIDS[@]}")
|
|
158
512
|
local waited=0
|
|
159
513
|
while kill -0 "$pid" 2>/dev/null && [[ "$waited" -lt "$GRACE" ]]; do
|
|
160
514
|
sleep 1
|
|
161
515
|
waited=$(( waited + 1 ))
|
|
162
516
|
done
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
517
|
+
# Escalate against anything that ignored TERM — root OR child.
|
|
518
|
+
# Scanning the breach-time snapshot (not re-walking from the root)
|
|
519
|
+
# also catches the case where the root died but a stubborn child
|
|
520
|
+
# survived it. Children spawned DURING the grace window are not in
|
|
521
|
+
# the snapshot; same accepted gap as detached daemons generally.
|
|
522
|
+
local sp sweep=0
|
|
523
|
+
for sp in "${breach_pids[@]}"; do
|
|
524
|
+
if kill -0 "$sp" 2>/dev/null; then
|
|
525
|
+
kill -9 "$sp" 2>/dev/null || true
|
|
526
|
+
sweep=$(( sweep + 1 ))
|
|
527
|
+
fi
|
|
528
|
+
done
|
|
529
|
+
if [[ "$sweep" -gt 0 ]]; then
|
|
530
|
+
echo "caproom: SIGKILLed ${sweep} survivor(s) after grace — exit 137" >&2
|
|
531
|
+
wait "$pid" 2>/dev/null || true
|
|
166
532
|
exit 137
|
|
167
533
|
fi
|
|
168
534
|
wait "$pid" 2>/dev/null || exit_code=$?
|
|
@@ -175,8 +541,16 @@ run_watchdog() {
|
|
|
175
541
|
exit "$exit_code"
|
|
176
542
|
}
|
|
177
543
|
|
|
178
|
-
if
|
|
179
|
-
|
|
544
|
+
if [[ "$USE_DOCKER" -eq 1 ]]; then
|
|
545
|
+
# Explicit opt-in must fail loudly rather than silently downgrade —
|
|
546
|
+
# the caller asked for a hard cap, a silent watchdog switch would
|
|
547
|
+
# quietly change the guarantee they asked for.
|
|
548
|
+
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
|
|
549
|
+
run_docker "$@"
|
|
550
|
+
else
|
|
551
|
+
echo "caproom: --docker requested but the docker daemon is not reachable" >&2
|
|
552
|
+
exit 1
|
|
553
|
+
fi
|
|
180
554
|
else
|
|
181
555
|
run_watchdog "$@"
|
|
182
556
|
fi
|