@memnexus-ai/mx-agent-cli 0.1.230 → 0.1.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/doctor.test.d.ts +20 -0
- package/dist/__tests__/doctor.test.d.ts.map +1 -0
- package/dist/__tests__/doctor.test.js +849 -0
- package/dist/__tests__/doctor.test.js.map +1 -0
- package/dist/__tests__/fleet-data.test.d.ts +2 -0
- package/dist/__tests__/fleet-data.test.d.ts.map +1 -0
- package/dist/__tests__/fleet-data.test.js +340 -0
- package/dist/__tests__/fleet-data.test.js.map +1 -0
- package/dist/__tests__/fleet-tmux.test.d.ts +2 -0
- package/dist/__tests__/fleet-tmux.test.d.ts.map +1 -0
- package/dist/__tests__/fleet-tmux.test.js +242 -0
- package/dist/__tests__/fleet-tmux.test.js.map +1 -0
- package/dist/__tests__/worker-cap.test.d.ts +14 -0
- package/dist/__tests__/worker-cap.test.d.ts.map +1 -0
- package/dist/__tests__/worker-cap.test.js +97 -0
- package/dist/__tests__/worker-cap.test.js.map +1 -0
- package/dist/agent-config/agents/implementation-engineer.md +8 -0
- package/dist/agent-config/agents/qa-tester.md +13 -5
- package/dist/agent-config/shared/eval-tiered-rigor.md +4 -2
- package/dist/agent-config/skills/mx-pre-pr-check/SKILL.md +4 -0
- package/dist/commands/doctor.d.ts +203 -0
- package/dist/commands/doctor.d.ts.map +1 -0
- package/dist/commands/doctor.js +688 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/fleet-tmux.d.ts +68 -0
- package/dist/commands/fleet-tmux.d.ts.map +1 -0
- package/dist/commands/fleet-tmux.js +288 -0
- package/dist/commands/fleet-tmux.js.map +1 -0
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +36 -0
- package/dist/commands/start.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +87 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/fleet-data.d.ts +181 -0
- package/dist/lib/fleet-data.d.ts.map +1 -0
- package/dist/lib/fleet-data.js +375 -0
- package/dist/lib/fleet-data.js.map +1 -0
- package/dist/lib/fleet-render.d.ts +66 -0
- package/dist/lib/fleet-render.d.ts.map +1 -0
- package/dist/lib/fleet-render.js +184 -0
- package/dist/lib/fleet-render.js.map +1 -0
- package/dist/lib/tmux.d.ts +69 -0
- package/dist/lib/tmux.d.ts.map +1 -0
- package/dist/lib/tmux.js +115 -0
- package/dist/lib/tmux.js.map +1 -0
- package/dist/lib/vm-health.d.ts +562 -0
- package/dist/lib/vm-health.d.ts.map +1 -0
- package/dist/lib/vm-health.js +960 -0
- package/dist/lib/vm-health.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,960 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vm-health.ts — measured VM performance state for `mx-agent doctor` (#4498).
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* Per the 2026-07-24 post-mortem (MemNexus memory
|
|
6
|
+
* 70123fe2-44f1-4a76-b18f-cf527d71b59f, KI-WSL2-SWAP-EVICTS-VSCODE — these
|
|
7
|
+
* figures are quoted from it, not re-measured), the WSL2 VM took nine OOM kills
|
|
8
|
+
* between 22:59 and 23:56 UTC (node processes at ~1.3 GB anon-rss each),
|
|
9
|
+
* exhausted its 8 GiB swap (SwapFree reached 0 kB), and evicted the VS Code
|
|
10
|
+
* server family — fileWatcher 387 MB, extensionHost 210 MB, server-main 79 MB —
|
|
11
|
+
* into swap, leaving 3.4 GB swapped out. Linux does not page an idle process
|
|
12
|
+
* back proactively, so the editor stayed swapped out and keystrokes lagged for
|
|
13
|
+
* ~2 days. Diagnosing it took a manual post-mortem because nothing measured the
|
|
14
|
+
* VM's state.
|
|
15
|
+
*
|
|
16
|
+
* The root cause is host-side: WSL2 page reporting and auto memory reclaim hand
|
|
17
|
+
* idle guest pages back to Windows. The container cannot fix it — `/proc/sys` is
|
|
18
|
+
* read-only here, `swapoff` needs CAP_SYS_ADMIN, and `/mnt/c` is not mounted so
|
|
19
|
+
* `%UserProfile%\.wslconfig` is unreachable. What the container CAN do is detect
|
|
20
|
+
* the condition and name the remedy. That is this module's whole job.
|
|
21
|
+
*
|
|
22
|
+
* DESIGN CONTRACT
|
|
23
|
+
* 1. PURE PARSERS over text fixtures are exported and unit-tested against fixture
|
|
24
|
+
* strings — never against live system state.
|
|
25
|
+
* 2. COLLECTORS are thin wrappers that read one path and hand the text to a
|
|
26
|
+
* parser. Every collector returns `Collected<T>`; a failure yields
|
|
27
|
+
* `{ value: null, error }` so the check can print "unavailable" with the
|
|
28
|
+
* reason instead of crashing or inventing a number.
|
|
29
|
+
* 3. NO ESTIMATES. Every number rendered comes from a file we read this run.
|
|
30
|
+
* 4. CHEAP BY DEFAULT. The default path uses single reads of /proc/meminfo and
|
|
31
|
+
* /proc/vmstat plus statfs. It never walks /proc/<pid>/* across hundreds of
|
|
32
|
+
* PIDs — that allocates and syscalls under exactly the memory pressure being
|
|
33
|
+
* measured. Per-process work is either deferred to --verbose (see
|
|
34
|
+
* collectProcesses) or ENFORCED to at most MAX_BOUNDED_PIDS entries. The
|
|
35
|
+
* enforcement matters: a pgrep match count is an environmental assumption,
|
|
36
|
+
* not a bound, and this path runs at every session start on every team.
|
|
37
|
+
* 5. THE DELTA STORE IS UNTRUSTED CROSS-TEAM INPUT.
|
|
38
|
+
* `~/.mx-agent/doctor-last-reading/*.json` sits in the shared $HOME, OUTSIDE
|
|
39
|
+
* every worktree, and every agent team on this machine runs as the same uid.
|
|
40
|
+
* Any process on the box can write it. Its values are turned into text that
|
|
41
|
+
* is printed into agent context at session start, fleet-wide, so a malicious
|
|
42
|
+
* reading is a prompt-injection and terminal-escape channel. Treat anything
|
|
43
|
+
* read from that file exactly as this project treats any other cross-team
|
|
44
|
+
* content: as DATA, validated before use, never as trusted state. See
|
|
45
|
+
* loadLastReading — do not relax its validation, and do not add a field to
|
|
46
|
+
* DoctorReading without extending it.
|
|
47
|
+
*/
|
|
48
|
+
import { closeSync, constants, fstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, renameSync, statfsSync, unlinkSync, writeFileSync, } from 'fs';
|
|
49
|
+
import { execFileSync } from 'child_process';
|
|
50
|
+
import { homedir } from 'os';
|
|
51
|
+
import { join, dirname } from 'path';
|
|
52
|
+
// ── The host-side remedy ────────────────────────────────────────────────────
|
|
53
|
+
/**
|
|
54
|
+
* The exact `%UserProfile%\.wslconfig` settings block, printed VERBATIM when the
|
|
55
|
+
* swap or OOM condition is present. Single source of truth: the annotated
|
|
56
|
+
* shipped file `dev-tools/wsl/wslconfig-recommended.ini` must contain every one
|
|
57
|
+
* of these lines (enforced by doctor.test.ts), so the two cannot drift.
|
|
58
|
+
*
|
|
59
|
+
* WHY swap=4GB AND NOT swap=0 — this is the part a future reader will want to
|
|
60
|
+
* "helpfully" change, so it is written down here as well as in the README. The
|
|
61
|
+
* nine OOM kills the post-mortem recorded on 2026-07-24 happened WITH 8 GiB of
|
|
62
|
+
* swap present, and that swap was driven to SwapFree=0. (The 8 GiB is one figure
|
|
63
|
+
* we did re-confirm: /proc/meminfo still reported SwapTotal 8.0 GiB on
|
|
64
|
+
* 2026-07-25.) Swap existing was not the fault; swap being
|
|
65
|
+
* both too small to absorb the spike and large enough to host a long thrash was.
|
|
66
|
+
* Setting swap=0 deletes the kernel's pressure-relief buffer and makes OOM kills
|
|
67
|
+
* arrive SOONER — it trades the product owner's editor freezing for agent
|
|
68
|
+
* sessions being killed mid-write with unsaved work lost. That is the worse
|
|
69
|
+
* trade. 4 GiB is a deliberately smaller buffer: enough to absorb a transient
|
|
70
|
+
* spike, not enough to sustain a long thrash.
|
|
71
|
+
*
|
|
72
|
+
* The settings that actually address the documented root cause are
|
|
73
|
+
* pageReporting=false and autoMemoryReclaim=disabled — those are what stop WSL2
|
|
74
|
+
* returning idle pages to Windows, which is what evicted VS Code.
|
|
75
|
+
*
|
|
76
|
+
* `memory=22GB` is a tradeoff, not a settled number. The host has 32.0 GB
|
|
77
|
+
* installed / 31.8 GB usable, confirmed by the product owner on 2026-07-25 — it
|
|
78
|
+
* cannot be read from inside the container. Giving the VM more raises agent
|
|
79
|
+
* capacity but starves Windows, where the VS Code CLIENT runs; giving it less
|
|
80
|
+
* starves the Linux side, where the VS Code SERVER runs. Both ends produce the
|
|
81
|
+
* same typing lag for opposite reasons, and this command can only see the Linux
|
|
82
|
+
* end. dev-tools/wsl/README.md explains how to settle it by measurement.
|
|
83
|
+
*/
|
|
84
|
+
export const WSLCONFIG_REMEDY = `[wsl2]
|
|
85
|
+
memory=22GB
|
|
86
|
+
swap=4GB
|
|
87
|
+
pageReporting=false
|
|
88
|
+
|
|
89
|
+
[experimental]
|
|
90
|
+
autoMemoryReclaim=disabled`;
|
|
91
|
+
/**
|
|
92
|
+
* The swap size WSLCONFIG_REMEDY recommends, in kB (4 GiB). This is a POLICY
|
|
93
|
+
* CONSTANT — the recommendation from dev-tools/wsl/, not something measured — and
|
|
94
|
+
* the swap check compares the measured SwapTotal against it. Kept next to the
|
|
95
|
+
* remedy so the two can only ever be changed together.
|
|
96
|
+
*/
|
|
97
|
+
export const RECOMMENDED_SWAP_KB = 4 * 1024 * 1024;
|
|
98
|
+
/**
|
|
99
|
+
* Kernel tunable that belongs with the remedy but is NOT a .wslconfig setting.
|
|
100
|
+
* The default (60) lets the kernel page out active anonymous memory about as
|
|
101
|
+
* readily as it drops clean page cache; 10 makes it strongly prefer dropping
|
|
102
|
+
* cache, which is what keeps the editor resident. It cannot be set from inside
|
|
103
|
+
* this container (/proc/sys is read-only, and swapoff needs CAP_SYS_ADMIN) — it
|
|
104
|
+
* needs `/etc/wsl.conf` on the distro or an equivalent host-side step. See
|
|
105
|
+
* dev-tools/wsl/README.md.
|
|
106
|
+
*/
|
|
107
|
+
export const SWAPPINESS_REMEDY = 'vm.swappiness=10';
|
|
108
|
+
/** Drains swap immediately without a VM restart. Windows host, elevated shell. */
|
|
109
|
+
export const SWAP_DRAIN_COMMAND = 'wsl -d <distro> -u root sh -c "swapoff -a && swapon -a"';
|
|
110
|
+
/**
|
|
111
|
+
* The caveat that MUST travel with SWAP_DRAIN_COMMAND wherever it is printed.
|
|
112
|
+
*
|
|
113
|
+
* `swapoff -a` faults every swapped-out page back into RAM at once. Run under the
|
|
114
|
+
* memory pressure that filled swap in the first place, it triggers the very OOM
|
|
115
|
+
* kills our own documentation says lose unsaved agent work — so the command is
|
|
116
|
+
* most dangerous exactly when someone is most tempted to reach for it. The README
|
|
117
|
+
* carries this warning, but the copy people will actually paste is the one
|
|
118
|
+
* `doctor` prints, so the warning has to be printed there too.
|
|
119
|
+
*/
|
|
120
|
+
export const SWAP_DRAIN_CAVEAT = 'Run this ONLY while MemAvailable exceeds the swap currently in use (both are printed above). ' +
|
|
121
|
+
'It pulls every swapped page back into RAM at once; under real pressure that causes OOM kills.';
|
|
122
|
+
/** Where the annotated remedy and its verification steps live in this repo. */
|
|
123
|
+
export const REMEDY_DOC_PATH = 'dev-tools/wsl/README.md';
|
|
124
|
+
// ── Bounds on the default path ──────────────────────────────────────────────
|
|
125
|
+
//
|
|
126
|
+
// These exist because the default path runs at every session start on every
|
|
127
|
+
// team. A diagnostic that can be made slow by an unusual machine state is a
|
|
128
|
+
// diagnostic that makes a bad situation worse.
|
|
129
|
+
/**
|
|
130
|
+
* Timeout for a `pgrep` exec, in ms. Two pgrep calls happen per default run, so
|
|
131
|
+
* the worst case this contributes to session start is ~4 s rather than the ~10 s
|
|
132
|
+
* a 5 s timeout would allow. pgrep on a healthy box returns in single-digit ms;
|
|
133
|
+
* anything near this bound means the machine is already in trouble, which is
|
|
134
|
+
* precisely when we must not add to the delay.
|
|
135
|
+
*/
|
|
136
|
+
export const PGREP_TIMEOUT_MS = 2000;
|
|
137
|
+
/**
|
|
138
|
+
* Timeout for the `dmesg` exec, in ms. Verbose path only. Without it, a `dmesg`
|
|
139
|
+
* that blocks would hang the command indefinitely — `maxBuffer` bounds the
|
|
140
|
+
* output, not the wait.
|
|
141
|
+
*/
|
|
142
|
+
export const DMESG_TIMEOUT_MS = 5000;
|
|
143
|
+
/**
|
|
144
|
+
* Hard cap on how many PIDs the default path will read from /proc.
|
|
145
|
+
*
|
|
146
|
+
* `pgrep -f vscode-server` normally matches about 16 processes, but "normally"
|
|
147
|
+
* is an environmental assumption, not a bound. Any local process can spawn
|
|
148
|
+
* thousands of processes whose cmdline matches, and checkSwapOccupancy is NOT
|
|
149
|
+
* verbose-gated — so without this cap, every session start on every team would
|
|
150
|
+
* perform tens of thousands of /proc reads. 32 is comfortably above the observed
|
|
151
|
+
* count and low enough that the cost stays negligible. Truncation is reported
|
|
152
|
+
* rather than hidden, so a suspicious count is visible instead of silent.
|
|
153
|
+
*/
|
|
154
|
+
export const MAX_BOUNDED_PIDS = 32;
|
|
155
|
+
// ── /proc/meminfo ───────────────────────────────────────────────────────────
|
|
156
|
+
/**
|
|
157
|
+
* Parse /proc/meminfo into a map of field → kB. Fields without a `kB` unit
|
|
158
|
+
* (e.g. HugePages_Total) are parsed as bare numbers. Unrecognised lines are
|
|
159
|
+
* skipped rather than throwing — meminfo gains fields across kernel versions.
|
|
160
|
+
*/
|
|
161
|
+
export function parseMeminfo(text) {
|
|
162
|
+
const out = {};
|
|
163
|
+
for (const line of text.split('\n')) {
|
|
164
|
+
const m = /^(\w+)\(?(\w+)?\)?:\s+(\d+)(?:\s+kB)?\s*$/.exec(line.trim());
|
|
165
|
+
if (!m)
|
|
166
|
+
continue;
|
|
167
|
+
const key = m[2] ? `${m[1]}(${m[2]})` : m[1];
|
|
168
|
+
out[key] = Number(m[3]);
|
|
169
|
+
}
|
|
170
|
+
return out;
|
|
171
|
+
}
|
|
172
|
+
// ── /proc/vmstat ────────────────────────────────────────────────────────────
|
|
173
|
+
/**
|
|
174
|
+
* Parse /proc/vmstat into a map of counter → value. This is the cheap source for
|
|
175
|
+
* two numbers we would otherwise pay for: `oom_kill` is an exact count of OOM
|
|
176
|
+
* kills since boot (no dmesg, no permissions), and `pswpin`/`pswpout` are the
|
|
177
|
+
* cumulative page-swap counters that distinguish "swap is allocated but idle"
|
|
178
|
+
* from "swap is actively thrashing".
|
|
179
|
+
*/
|
|
180
|
+
export function parseVmstat(text) {
|
|
181
|
+
const out = {};
|
|
182
|
+
for (const line of text.split('\n')) {
|
|
183
|
+
const m = /^(\S+)\s+(-?\d+)\s*$/.exec(line.trim());
|
|
184
|
+
if (m)
|
|
185
|
+
out[m[1]] = Number(m[2]);
|
|
186
|
+
}
|
|
187
|
+
return out;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Maximum length of a process label we will print. The kernel caps `comm` at 15
|
|
191
|
+
* characters and OOM task names are similar, so this only ever truncates
|
|
192
|
+
* something abnormal.
|
|
193
|
+
*/
|
|
194
|
+
const MAX_LABEL_LENGTH = 64;
|
|
195
|
+
/**
|
|
196
|
+
* Make a process-supplied label safe to print.
|
|
197
|
+
*
|
|
198
|
+
* A process controls its own name via prctl(PR_SET_NAME), and the kernel escapes
|
|
199
|
+
* only newline and backslash when rendering it — a raw ESC byte passes straight
|
|
200
|
+
* through /proc. We print the top 15 processes by RSS under --verbose, so an
|
|
201
|
+
* unsanitised name is a terminal-escape channel. Strip C0 controls and DEL, then
|
|
202
|
+
* bound the length.
|
|
203
|
+
*
|
|
204
|
+
* WHAT THIS DOES NOT STRIP, so the bound is not mistaken for complete: C1
|
|
205
|
+
* controls (U+0080–U+009F) and Unicode bidirectional/format characters such as
|
|
206
|
+
* U+202E RIGHT-TO-LEFT OVERRIDE and U+200B ZERO WIDTH SPACE all survive. Those
|
|
207
|
+
* can still reorder or hide text in a terminal that honours them. Kernel `comm`
|
|
208
|
+
* is 15 bytes and would rarely carry them, and the consequence is confusing
|
|
209
|
+
* output rather than an executed escape sequence, so this is documented rather
|
|
210
|
+
* than widened — but do not read "sanitized" as "safe to render anywhere."
|
|
211
|
+
*/
|
|
212
|
+
export function sanitizeProcessLabel(raw) {
|
|
213
|
+
return raw.replace(/[\x00-\x1f\x7f]/g, '').trim().slice(0, MAX_LABEL_LENGTH);
|
|
214
|
+
}
|
|
215
|
+
/** Parse the three fields we need out of a /proc/<pid>/status text. */
|
|
216
|
+
export function parseProcStatus(text) {
|
|
217
|
+
const field = (name) => {
|
|
218
|
+
const m = new RegExp(`^${name}:\\s+(\\d+) kB$`, 'm').exec(text);
|
|
219
|
+
return m ? Number(m[1]) : null;
|
|
220
|
+
};
|
|
221
|
+
const nameMatch = /^Name:\s+(.*)$/m.exec(text);
|
|
222
|
+
return {
|
|
223
|
+
name: nameMatch ? sanitizeProcessLabel(nameMatch[1]) : '',
|
|
224
|
+
vmRssKb: field('VmRSS'),
|
|
225
|
+
vmSwapKb: field('VmSwap'),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
// ── Process classification ──────────────────────────────────────────────────
|
|
229
|
+
/**
|
|
230
|
+
* The VS Code server processes whose eviction to swap is what the product owner
|
|
231
|
+
* actually feels as keystroke lag. Matched on the cmdline because the kernel
|
|
232
|
+
* `comm` for all of them is just `MainThread` (verified in-container
|
|
233
|
+
* 2026-07-25) — `comm` cannot tell them apart.
|
|
234
|
+
*/
|
|
235
|
+
export const VSCODE_ROLE_PATTERNS = [
|
|
236
|
+
{ role: 'fileWatcher', needle: '--type=fileWatcher' },
|
|
237
|
+
{ role: 'extensionHost', needle: '--type=extensionHost' },
|
|
238
|
+
{ role: 'ptyHost', needle: '--type=ptyHost' },
|
|
239
|
+
{ role: 'server-main', needle: 'out/server-main.js' },
|
|
240
|
+
];
|
|
241
|
+
/**
|
|
242
|
+
* Identify the VS Code server role for a cmdline, or null. Only the roles above
|
|
243
|
+
* are named; any other vscode-server process is reported as 'vscode-server'.
|
|
244
|
+
*
|
|
245
|
+
* HEURISTIC, NOT AUTHORITATIVE. This is substring matching on a cmdline. An
|
|
246
|
+
* unrelated process whose command line happens to contain `--type=fileWatcher`
|
|
247
|
+
* or `vscode-server` will be labelled as one, and any process can choose its own
|
|
248
|
+
* command line. The labels are advisory context for a human reading the output —
|
|
249
|
+
* "which of these is the editor" — and nothing branches on them beyond wording.
|
|
250
|
+
* There is no security consequence: the PID list is bounded (MAX_BOUNDED_PIDS)
|
|
251
|
+
* and every label is sanitized before printing. Treat a surprising label as a
|
|
252
|
+
* prompt to look, not as a fact.
|
|
253
|
+
*/
|
|
254
|
+
export function vscodeRole(cmdline) {
|
|
255
|
+
for (const { role, needle } of VSCODE_ROLE_PATTERNS) {
|
|
256
|
+
if (cmdline.includes(needle))
|
|
257
|
+
return role;
|
|
258
|
+
}
|
|
259
|
+
if (cmdline.includes('vscode-server'))
|
|
260
|
+
return 'vscode-server';
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* True when the cmdline is an agent session root: argv[0] is the `claude`
|
|
265
|
+
* binary. Deliberately narrow — matching the substring "claude" anywhere would
|
|
266
|
+
* also catch every hook script under `.claude/hooks/`, which are shells, not
|
|
267
|
+
* sessions (verified: 3 such false positives in-container).
|
|
268
|
+
*
|
|
269
|
+
* HEURISTIC, NOT AUTHORITATIVE. Exact-argv0 matching is much tighter than the
|
|
270
|
+
* substring test in vscodeRole, but a process still controls its own argv, so
|
|
271
|
+
* anything invoked as `claude` counts and a session launched by another name
|
|
272
|
+
* does not. The session COUNT feeds the delta store, so treat it as an
|
|
273
|
+
* indicator, not an inventory. Same reasoning as vscodeRole: bounded and
|
|
274
|
+
* sanitized, so no security consequence.
|
|
275
|
+
*/
|
|
276
|
+
export function isAgentSession(cmdline) {
|
|
277
|
+
const argv0 = cmdline.split('\0')[0] || cmdline.split(' ')[0] || '';
|
|
278
|
+
const base = argv0.split('/').pop() ?? '';
|
|
279
|
+
return base === 'claude';
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Extract OOM events from a dmesg dump. Recognises the three shapes the kernel
|
|
283
|
+
* emits, in decreasing order of specificity:
|
|
284
|
+
* "Out of memory: Killed process 1234 (node) total-vm:..., anon-rss:1330000kB, ..."
|
|
285
|
+
* "oom-kill:constraint=CONSTRAINT_NONE,...,task=node,pid=1234,uid=1000"
|
|
286
|
+
* "node invoked oom-killer: gfp_mask=0x..., order=0, oom_score_adj=0"
|
|
287
|
+
* The first two are the same kill reported twice by newer kernels, so a kill is
|
|
288
|
+
* de-duplicated by (pid, name) when both appear within a second of each other.
|
|
289
|
+
*
|
|
290
|
+
* This is the VERBOSE path. The default path uses /proc/vmstat `oom_kill`, which
|
|
291
|
+
* is an exact since-boot count and costs one read; dmesg exists here only to
|
|
292
|
+
* attach timestamps and victim names to that count.
|
|
293
|
+
*/
|
|
294
|
+
export function parseOomEvents(dmesgText) {
|
|
295
|
+
const events = [];
|
|
296
|
+
for (const rawLine of dmesgText.split('\n')) {
|
|
297
|
+
const line = rawLine.trim();
|
|
298
|
+
if (!line)
|
|
299
|
+
continue;
|
|
300
|
+
const ts = /^\[\s*(\d+\.\d+)\]/.exec(line);
|
|
301
|
+
const monotonicSeconds = ts ? Number(ts[1]) : null;
|
|
302
|
+
const killed = /Out of memory: Killed process (\d+) \(([^)]*)\)/.exec(line);
|
|
303
|
+
if (killed) {
|
|
304
|
+
const anon = /anon-rss:(\d+)kB/.exec(line);
|
|
305
|
+
events.push({
|
|
306
|
+
monotonicSeconds,
|
|
307
|
+
kind: 'kill',
|
|
308
|
+
process: sanitizeProcessLabel(killed[2]),
|
|
309
|
+
pid: Number(killed[1]),
|
|
310
|
+
anonRssKb: anon ? Number(anon[1]) : null,
|
|
311
|
+
raw: line,
|
|
312
|
+
});
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
const oomKill = /oom-kill:.*?task=([^,]+),pid=(\d+)/.exec(line);
|
|
316
|
+
if (oomKill) {
|
|
317
|
+
events.push({
|
|
318
|
+
monotonicSeconds,
|
|
319
|
+
kind: 'kill',
|
|
320
|
+
process: sanitizeProcessLabel(oomKill[1]),
|
|
321
|
+
pid: Number(oomKill[2]),
|
|
322
|
+
anonRssKb: null,
|
|
323
|
+
raw: line,
|
|
324
|
+
});
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
const invoked = /(\S+) invoked oom-killer:/.exec(line);
|
|
328
|
+
if (invoked) {
|
|
329
|
+
events.push({
|
|
330
|
+
monotonicSeconds,
|
|
331
|
+
kind: 'invoked',
|
|
332
|
+
process: sanitizeProcessLabel(invoked[1]),
|
|
333
|
+
pid: null,
|
|
334
|
+
anonRssKb: null,
|
|
335
|
+
raw: line,
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
return dedupeKills(events);
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Newer kernels log the same kill twice ("oom-kill:" then "Out of memory: Killed
|
|
343
|
+
* process"). Collapse those to one event, keeping the richer record (the one
|
|
344
|
+
* carrying anon-rss).
|
|
345
|
+
*/
|
|
346
|
+
function dedupeKills(events) {
|
|
347
|
+
const out = [];
|
|
348
|
+
// Index of the most recent kill for each (pid, name), so this is one pass
|
|
349
|
+
// rather than a scan of everything seen so far per event. dmesg is a ring
|
|
350
|
+
// buffer we read whole — up to 32 MB — and the quadratic version would be felt
|
|
351
|
+
// on a box that had genuinely been OOM-killing, which is the case that matters.
|
|
352
|
+
const latestByVictim = new Map();
|
|
353
|
+
for (const e of events) {
|
|
354
|
+
if (e.kind !== 'kill') {
|
|
355
|
+
out.push(e);
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
const key = `${e.pid}:${e.process}`;
|
|
359
|
+
const priorIndex = latestByVictim.get(key);
|
|
360
|
+
const prior = priorIndex === undefined ? undefined : out[priorIndex];
|
|
361
|
+
// A twin is the SAME kill logged twice, which the kernel does within the
|
|
362
|
+
// same instant. A later kill of a recycled pid is a different event.
|
|
363
|
+
const isTwin = prior !== undefined &&
|
|
364
|
+
prior.monotonicSeconds !== null &&
|
|
365
|
+
e.monotonicSeconds !== null &&
|
|
366
|
+
Math.abs(prior.monotonicSeconds - e.monotonicSeconds) < 1;
|
|
367
|
+
if (!isTwin) {
|
|
368
|
+
out.push(e);
|
|
369
|
+
latestByVictim.set(key, out.length - 1);
|
|
370
|
+
}
|
|
371
|
+
else if (prior.anonRssKb === null && e.anonRssKb !== null) {
|
|
372
|
+
out[priorIndex] = e; // keep the richer of the two records
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return out;
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Slack allowed when checking a dmesg timestamp against uptime, in seconds.
|
|
379
|
+
* /proc/uptime and the log line are read at slightly different moments, so an
|
|
380
|
+
* event from the last instant can legitimately appear a hair beyond uptime.
|
|
381
|
+
*/
|
|
382
|
+
const UPTIME_BOUND_SLACK_SECONDS = 1;
|
|
383
|
+
/**
|
|
384
|
+
* Convert a dmesg monotonic timestamp to wall-clock ISO, given the current time
|
|
385
|
+
* and /proc/uptime seconds. Returns null rather than a date whenever we cannot
|
|
386
|
+
* compute an honest one:
|
|
387
|
+
*
|
|
388
|
+
* - the log line carried no timestamp; or
|
|
389
|
+
* - the timestamp is LATER THAN UPTIME, which is impossible. dmesg is a ring
|
|
390
|
+
* buffer and the monotonic clock resets at boot, so the two normally move
|
|
391
|
+
* together — but some WSL2 restart paths can leave stale lines behind a
|
|
392
|
+
* reset clock. Computing anyway produces a confident, fabricated future date
|
|
393
|
+
* (a line at [999999.0] with uptime 100 renders as 11 days from now). A
|
|
394
|
+
* number we know is impossible must never be rendered; the caller says the
|
|
395
|
+
* timestamp is inconsistent instead. See describeOomEventTime.
|
|
396
|
+
*/
|
|
397
|
+
export function monotonicToWallClock(monotonicSeconds, uptimeSeconds, nowMs) {
|
|
398
|
+
if (monotonicSeconds === null)
|
|
399
|
+
return null;
|
|
400
|
+
if (monotonicSeconds > uptimeSeconds + UPTIME_BOUND_SLACK_SECONDS)
|
|
401
|
+
return null;
|
|
402
|
+
const bootMs = nowMs - uptimeSeconds * 1000;
|
|
403
|
+
return new Date(bootMs + monotonicSeconds * 1000).toISOString();
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* The wall-clock phrase for one OOM event, or an explicit statement of why there
|
|
407
|
+
* is none. Pure, so every branch is testable without a live dmesg. Each failure
|
|
408
|
+
* mode gets its own wording — "unavailable" with a reason, never a blank or a
|
|
409
|
+
* plausible-looking guess.
|
|
410
|
+
*/
|
|
411
|
+
export function describeOomEventTime(event, uptimeSeconds, nowMs) {
|
|
412
|
+
if (uptimeSeconds === null)
|
|
413
|
+
return 'no wall-clock time (uptime unreadable)';
|
|
414
|
+
if (event.monotonicSeconds === null)
|
|
415
|
+
return 'no timestamp in the log line';
|
|
416
|
+
if (event.monotonicSeconds > uptimeSeconds + UPTIME_BOUND_SLACK_SECONDS) {
|
|
417
|
+
return `timestamp inconsistent with current uptime (log says ${event.monotonicSeconds}s since boot, uptime is ${uptimeSeconds}s) — the ring buffer predates this boot, so no date can be computed`;
|
|
418
|
+
}
|
|
419
|
+
return monotonicToWallClock(event.monotonicSeconds, uptimeSeconds, nowMs) ?? 'no timestamp in the log line';
|
|
420
|
+
}
|
|
421
|
+
/** Parse the first field of /proc/uptime (seconds since boot). */
|
|
422
|
+
export function parseUptimeSeconds(text) {
|
|
423
|
+
const m = /^\s*(\d+(?:\.\d+)?)/.exec(text);
|
|
424
|
+
return m ? Number(m[1]) : null;
|
|
425
|
+
}
|
|
426
|
+
// ── inotify ─────────────────────────────────────────────────────────────────
|
|
427
|
+
/** Count `inotify wd:...` lines in a /proc/<pid>/fdinfo/<fd> dump — one per watch. */
|
|
428
|
+
export function parseInotifyWatchCount(fdinfoText) {
|
|
429
|
+
let n = 0;
|
|
430
|
+
for (const line of fdinfoText.split('\n'))
|
|
431
|
+
if (line.startsWith('inotify '))
|
|
432
|
+
n++;
|
|
433
|
+
return n;
|
|
434
|
+
}
|
|
435
|
+
// ── cgroup memory cap ───────────────────────────────────────────────────────
|
|
436
|
+
/**
|
|
437
|
+
* Interpret a cgroup memory limit value. cgroup v2 writes the literal string
|
|
438
|
+
* `max`; v1 writes a sentinel near 2^63. Returns null for "unlimited".
|
|
439
|
+
*/
|
|
440
|
+
export function parseCgroupMemoryLimit(text) {
|
|
441
|
+
const raw = text.trim();
|
|
442
|
+
if (raw === 'max')
|
|
443
|
+
return null;
|
|
444
|
+
const n = Number(raw);
|
|
445
|
+
if (!Number.isFinite(n))
|
|
446
|
+
return null;
|
|
447
|
+
// v1's "no limit" sentinel is PAGE_COUNTER_MAX * PAGE_SIZE, just under 2^63.
|
|
448
|
+
// Anything at or above 2^62 is not a real container cap.
|
|
449
|
+
if (n >= 2 ** 62)
|
|
450
|
+
return null;
|
|
451
|
+
return n;
|
|
452
|
+
}
|
|
453
|
+
// ── Formatting helpers ──────────────────────────────────────────────────────
|
|
454
|
+
/** kB → human string, e.g. 8388608 → "8.0 GiB". */
|
|
455
|
+
export function formatKb(kb) {
|
|
456
|
+
if (kb >= 1024 * 1024)
|
|
457
|
+
return `${(kb / 1024 / 1024).toFixed(1)} GiB`;
|
|
458
|
+
if (kb >= 1024)
|
|
459
|
+
return `${(kb / 1024).toFixed(0)} MiB`;
|
|
460
|
+
return `${kb} kB`;
|
|
461
|
+
}
|
|
462
|
+
/** bytes → human string. */
|
|
463
|
+
export function formatBytes(bytes) {
|
|
464
|
+
return formatKb(Math.round(bytes / 1024));
|
|
465
|
+
}
|
|
466
|
+
/**
|
|
467
|
+
* Where the last reading lives. Follows the existing convention set by
|
|
468
|
+
* `monitor` (~/.mx-agent/monitor-last-check/<key>.txt): a per-feature directory
|
|
469
|
+
* under ~/.mx-agent with one file per key. No new store is introduced.
|
|
470
|
+
*/
|
|
471
|
+
export function doctorReadingPath(key = 'default') {
|
|
472
|
+
const safe = key.replace(/[^A-Za-z0-9._-]/g, '_');
|
|
473
|
+
return join(homedir(), '.mx-agent', 'doctor-last-reading', `${safe}.json`);
|
|
474
|
+
}
|
|
475
|
+
/**
|
|
476
|
+
* The only timestamp form accepted from the store: an ISO-8601 instant with
|
|
477
|
+
* exactly three fractional digits and a literal Z. Exactly 24 characters, all
|
|
478
|
+
* from a fixed safe alphabet. This is what `Date.prototype.toISOString` emits,
|
|
479
|
+
* so every reading WE write matches; anything else is rejected and the caller
|
|
480
|
+
* measures fresh.
|
|
481
|
+
*/
|
|
482
|
+
const ISO_INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
|
483
|
+
/**
|
|
484
|
+
* Hard cap on the delta store file, in bytes. A reading serializes to about 200
|
|
485
|
+
* bytes; 4 KiB is roughly 20x that and still trivially cheap to read.
|
|
486
|
+
*
|
|
487
|
+
* WHY A SIZE BOUND AND NOT JUST A CONTENT CHECK. Validating shape and content
|
|
488
|
+
* still means `readFileSync` + `JSON.parse` run on whatever is at that path
|
|
489
|
+
* first. QA planted a 300 MB junk file and measured collectDoctorResult taking
|
|
490
|
+
* 532 ms longer on that call — and nothing bounds it at 300 MB; it could be
|
|
491
|
+
* gigabytes. Same shared $HOME, same uid, same trust boundary as the injection
|
|
492
|
+
* vector: any process on the box can impose that cost on every session start on
|
|
493
|
+
* the machine, which is the opposite of this module's cheap-by-default
|
|
494
|
+
* principle on a box we are trying to keep responsive.
|
|
495
|
+
*
|
|
496
|
+
* This is the boundPids pattern applied to the other unbounded input: cap it,
|
|
497
|
+
* do not assume it. Over the cap the file is treated as corrupt, which returns
|
|
498
|
+
* null, which triggers a fresh measurement and rewrites a clean baseline — the
|
|
499
|
+
* store self-heals on the same call.
|
|
500
|
+
*/
|
|
501
|
+
export const MAX_READING_FILE_BYTES = 4096;
|
|
502
|
+
/** A stored counter must be a finite, non-negative, in-range number. */
|
|
503
|
+
function isSafeCount(v) {
|
|
504
|
+
return typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= Number.MAX_SAFE_INTEGER;
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Load the previous reading, or null when it is absent, unparseable, or fails
|
|
508
|
+
* validation.
|
|
509
|
+
*
|
|
510
|
+
* SECURITY — THIS FILE IS UNTRUSTED INPUT. See the module header. It lives in
|
|
511
|
+
* the shared $HOME outside every worktree, every team runs as the same uid, and
|
|
512
|
+
* the values flow into text printed at session start in every agent session on
|
|
513
|
+
* every team. So the shape is checked exhaustively rather than trusted:
|
|
514
|
+
*
|
|
515
|
+
* - `timestamp` must match ISO_INSTANT *and* survive a Date round-trip. That
|
|
516
|
+
* bounds it to 24 characters from a fixed alphabet, which removes the
|
|
517
|
+
* unbounded-string channel entirely (no newlines, no terminal escapes, no
|
|
518
|
+
* injected instructions).
|
|
519
|
+
* - Every numeric field must be a finite number in [0, MAX_SAFE_INTEGER].
|
|
520
|
+
* Negative values are rejected specifically: a stored `oomKills: -1` makes
|
|
521
|
+
* the `current > previous` comparison in detectRegressions true no matter
|
|
522
|
+
* what the machine reports, which is a guaranteed trigger.
|
|
523
|
+
* - The file must be a REGULAR FILE of at most MAX_READING_FILE_BYTES. Size is
|
|
524
|
+
* checked BEFORE reading, because shape validation cannot protect against a
|
|
525
|
+
* cost paid during the read itself. The regular-file requirement matters
|
|
526
|
+
* independently: a FIFO at this path makes a blocking open() wait forever,
|
|
527
|
+
* hanging session start for every team — the same class of hazard that took
|
|
528
|
+
* the disk check off the startup path.
|
|
529
|
+
*
|
|
530
|
+
* VALIDATE THE FILE DESCRIPTOR, NEVER THE PATH — this is the part to preserve.
|
|
531
|
+
* The first version of this bound called statSync(path) and then
|
|
532
|
+
* readFileSync(path). That resolves the path TWICE, and a co-tenant process
|
|
533
|
+
* (same uid, shared $HOME, the trust boundary this module's header declares) can
|
|
534
|
+
* rename the path between the two calls: stat sees the valid file, open gets the
|
|
535
|
+
* FIFO. Security review demonstrated it wedging a victim at iteration 4 of a
|
|
536
|
+
* swap loop. Worse, the damage outlived the attack — the racer restored a valid
|
|
537
|
+
* regular file and exited, and the victim stayed blocked, because a blocked
|
|
538
|
+
* open() pins the FIFO inode no matter what the path resolves to afterwards.
|
|
539
|
+
* Forensics would find a perfectly valid store file and no explanation.
|
|
540
|
+
*
|
|
541
|
+
* So: openSync once with O_NONBLOCK, then fstatSync on the DESCRIPTOR. There is
|
|
542
|
+
* no window to race, because there is no second name resolution. O_NONBLOCK
|
|
543
|
+
* makes open() on a FIFO return immediately instead of waiting for a writer,
|
|
544
|
+
* which closes that case whether or not anyone attempts the race. O_NOFOLLOW is
|
|
545
|
+
* deliberately NOT set: a symlink to a legitimate regular file is fine, and
|
|
546
|
+
* fstat judges the target it actually opened rather than the name used to reach
|
|
547
|
+
* it.
|
|
548
|
+
*
|
|
549
|
+
* None of the three fail-open layers would have helped here. A process blocked
|
|
550
|
+
* in a kernel syscall never throws, and readFileSync cannot be given a timeout.
|
|
551
|
+
*
|
|
552
|
+
* The returned object is REBUILT field by field, so unknown properties in the
|
|
553
|
+
* file are discarded rather than carried into the rest of the program.
|
|
554
|
+
*
|
|
555
|
+
* NOTE ON LAYERING — do not "simplify" this. A far-future but well-formed
|
|
556
|
+
* timestamp like 9999-01-01T00:00:00.000Z is ACCEPTED here on purpose: it is a
|
|
557
|
+
* syntactically valid, bounded ISO instant and not an injection vector, so it is
|
|
558
|
+
* not this function's business. It is rejected TEMPORALLY by the rate limiter in
|
|
559
|
+
* shouldRunStartupDoctorCheck. Two layers, two concerns: this one decides
|
|
560
|
+
* whether the data is well-formed, that one decides whether it is usable now.
|
|
561
|
+
* Merging them would make each harder to reason about.
|
|
562
|
+
*/
|
|
563
|
+
export function loadLastReading(path = doctorReadingPath()) {
|
|
564
|
+
let fd;
|
|
565
|
+
try {
|
|
566
|
+
// ONE name resolution, then everything else against the descriptor.
|
|
567
|
+
// O_NONBLOCK so a FIFO here returns instead of blocking in wait_for_partner.
|
|
568
|
+
// A missing path throws, which the catch turns into the same "no reading"
|
|
569
|
+
// result as a corrupt one.
|
|
570
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
571
|
+
// fstat on an OPEN descriptor cannot be raced: this is the inode we will
|
|
572
|
+
// read from, not whatever the path may name a moment from now.
|
|
573
|
+
const stats = fstatSync(fd);
|
|
574
|
+
if (!stats.isFile() || stats.size > MAX_READING_FILE_BYTES)
|
|
575
|
+
return null;
|
|
576
|
+
const buffer = Buffer.allocUnsafe(stats.size);
|
|
577
|
+
let filled = 0;
|
|
578
|
+
while (filled < stats.size) {
|
|
579
|
+
const bytes = readSync(fd, buffer, filled, stats.size - filled, filled);
|
|
580
|
+
if (bytes <= 0)
|
|
581
|
+
break; // truncated mid-read; parse what we have and fail there
|
|
582
|
+
filled += bytes;
|
|
583
|
+
}
|
|
584
|
+
const parsed = JSON.parse(buffer.subarray(0, filled).toString('utf-8'));
|
|
585
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
|
|
586
|
+
return null;
|
|
587
|
+
const r = parsed;
|
|
588
|
+
const timestamp = r.timestamp;
|
|
589
|
+
if (typeof timestamp !== 'string' || !ISO_INSTANT.test(timestamp))
|
|
590
|
+
return null;
|
|
591
|
+
const parsedDate = new Date(timestamp);
|
|
592
|
+
if (Number.isNaN(parsedDate.getTime()) || parsedDate.toISOString() !== timestamp)
|
|
593
|
+
return null;
|
|
594
|
+
const { oomKills, swapUsedKb, swapTotalKb, agentSessions, uptimeSeconds } = r;
|
|
595
|
+
if (!isSafeCount(oomKills) ||
|
|
596
|
+
!isSafeCount(swapUsedKb) ||
|
|
597
|
+
!isSafeCount(swapTotalKb) ||
|
|
598
|
+
!isSafeCount(agentSessions) ||
|
|
599
|
+
!isSafeCount(uptimeSeconds)) {
|
|
600
|
+
return null;
|
|
601
|
+
}
|
|
602
|
+
return { timestamp, oomKills, swapUsedKb, swapTotalKb, agentSessions, uptimeSeconds };
|
|
603
|
+
}
|
|
604
|
+
catch {
|
|
605
|
+
return null;
|
|
606
|
+
}
|
|
607
|
+
finally {
|
|
608
|
+
if (fd !== undefined) {
|
|
609
|
+
try {
|
|
610
|
+
closeSync(fd);
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
// Already closed, or never validly open — nothing to recover.
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Persist the current reading. Best-effort: never throws.
|
|
620
|
+
*
|
|
621
|
+
* Written to a process-unique temp file with `flag: 'wx'` (fail if it already
|
|
622
|
+
* exists) and `mode: 0o600`, then renamed into place. Three properties come out
|
|
623
|
+
* of that:
|
|
624
|
+
* - SYMLINK-SAFE. `writeFileSync` follows a symlink and would clobber whatever
|
|
625
|
+
* it points at; `renameSync` replaces the symlink itself. Since `doctor` now
|
|
626
|
+
* runs at every session start, a symlink planted at the store path would
|
|
627
|
+
* otherwise be a write primitive fired by every team.
|
|
628
|
+
* - ATOMIC. A reader never sees a half-written file.
|
|
629
|
+
* - PRIVATE. 0600 rather than the default 0644.
|
|
630
|
+
* The temp name includes the pid so concurrent sessions cannot collide on it and
|
|
631
|
+
* so `wx` can never be defeated by a pre-planted temp file; it is removed on any
|
|
632
|
+
* failure.
|
|
633
|
+
*/
|
|
634
|
+
export function saveReading(reading, path = doctorReadingPath()) {
|
|
635
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
636
|
+
try {
|
|
637
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
638
|
+
writeFileSync(tmp, JSON.stringify(reading, null, 2), { encoding: 'utf-8', mode: 0o600, flag: 'wx' });
|
|
639
|
+
renameSync(tmp, path);
|
|
640
|
+
}
|
|
641
|
+
catch {
|
|
642
|
+
// Best-effort — a read-only HOME must not break the diagnostic.
|
|
643
|
+
try {
|
|
644
|
+
unlinkSync(tmp);
|
|
645
|
+
}
|
|
646
|
+
catch {
|
|
647
|
+
// nothing to clean up
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* Noise floor for the swap-in-use delta, in kB (64 MiB). A POLICY CONSTANT, not
|
|
653
|
+
* a measurement. Swap in use drifts by a few hundred kB between consecutive runs
|
|
654
|
+
* on an otherwise idle box (observed while testing this command), and reporting
|
|
655
|
+
* that as "worse" is exactly the alarm fatigue the delta design exists to
|
|
656
|
+
* prevent. Two things still get reported regardless of size: swap going from
|
|
657
|
+
* completely unused to used at all — that is a state change, not drift — and any
|
|
658
|
+
* growth of 64 MiB or more.
|
|
659
|
+
*
|
|
660
|
+
* KNOWN LIMITATION — STRUCTURALLY BLIND TO A SLOW LEAK. Each run compares only
|
|
661
|
+
* against the IMMEDIATELY PRECEDING reading, and the baseline advances every
|
|
662
|
+
* time. So a leak that grows by less than 64 MiB between checks never fires, no
|
|
663
|
+
* matter how large it becomes in total: 60 MiB per hourly check is 1.4 GiB a day
|
|
664
|
+
* and reports nothing, because no single step clears the floor. Lowering the
|
|
665
|
+
* floor is not the fix — that reintroduces the drift noise it exists to
|
|
666
|
+
* suppress.
|
|
667
|
+
*
|
|
668
|
+
* The fix, if we decide it is worth building, is a second comparison against a
|
|
669
|
+
* LONGER-WINDOW baseline (for example a day-old reading kept alongside the most
|
|
670
|
+
* recent one), so cumulative growth is visible even when each step is small.
|
|
671
|
+
* Deliberately NOT built now: it is a real design change to the delta store, and
|
|
672
|
+
* no slow leak has been observed. Candidate for a later iteration.
|
|
673
|
+
*/
|
|
674
|
+
export const SWAP_GROWTH_NOISE_FLOOR_KB = 64 * 1024;
|
|
675
|
+
/**
|
|
676
|
+
* Compare two readings and return only the metrics that got WORSE. Returns []
|
|
677
|
+
* when there is no previous reading (first run is quiet by design) or when the
|
|
678
|
+
* VM restarted, since since-boot counters reset and a "decrease" would be
|
|
679
|
+
* meaningless. `swapTotalKb` is reported in BOTH directions: a drop means
|
|
680
|
+
* somebody applied a host-side change, which is news worth printing.
|
|
681
|
+
*/
|
|
682
|
+
export function detectRegressions(prev, cur) {
|
|
683
|
+
if (!prev)
|
|
684
|
+
return [];
|
|
685
|
+
const out = [];
|
|
686
|
+
const rebooted = cur.uptimeSeconds < prev.uptimeSeconds;
|
|
687
|
+
if (!rebooted && cur.oomKills > prev.oomKills) {
|
|
688
|
+
out.push({
|
|
689
|
+
metric: 'oomKills',
|
|
690
|
+
previous: prev.oomKills,
|
|
691
|
+
current: cur.oomKills,
|
|
692
|
+
message: `OOM kills rose from ${prev.oomKills} to ${cur.oomKills} since the last check (${prev.timestamp}).`,
|
|
693
|
+
});
|
|
694
|
+
}
|
|
695
|
+
const swapGrewMeaningfully = (prev.swapUsedKb === 0 && cur.swapUsedKb > 0) ||
|
|
696
|
+
cur.swapUsedKb - prev.swapUsedKb >= SWAP_GROWTH_NOISE_FLOOR_KB;
|
|
697
|
+
if (swapGrewMeaningfully) {
|
|
698
|
+
out.push({
|
|
699
|
+
metric: 'swapUsedKb',
|
|
700
|
+
previous: prev.swapUsedKb,
|
|
701
|
+
current: cur.swapUsedKb,
|
|
702
|
+
message: `Swap in use rose from ${formatKb(prev.swapUsedKb)} to ${formatKb(cur.swapUsedKb)} since the last check (${prev.timestamp}).`,
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
if (cur.swapTotalKb !== prev.swapTotalKb) {
|
|
706
|
+
out.push({
|
|
707
|
+
metric: 'swapTotalKb',
|
|
708
|
+
previous: prev.swapTotalKb,
|
|
709
|
+
current: cur.swapTotalKb,
|
|
710
|
+
message: `Swap size changed from ${formatKb(prev.swapTotalKb)} to ${formatKb(cur.swapTotalKb)} — the host config changed.`,
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
if (cur.agentSessions > prev.agentSessions) {
|
|
714
|
+
out.push({
|
|
715
|
+
metric: 'agentSessions',
|
|
716
|
+
previous: prev.agentSessions,
|
|
717
|
+
current: cur.agentSessions,
|
|
718
|
+
message: `Agent sessions rose from ${prev.agentSessions} to ${cur.agentSessions} since the last check (${prev.timestamp}).`,
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
return out;
|
|
722
|
+
}
|
|
723
|
+
function collect(fn) {
|
|
724
|
+
try {
|
|
725
|
+
return { value: fn() };
|
|
726
|
+
}
|
|
727
|
+
catch (err) {
|
|
728
|
+
return { value: null, error: err instanceof Error ? err.message : String(err) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
/** Single read. Default path. */
|
|
732
|
+
export function collectMeminfo(procRoot = '/proc') {
|
|
733
|
+
return collect(() => parseMeminfo(readFileSync(`${procRoot}/meminfo`, 'utf8')));
|
|
734
|
+
}
|
|
735
|
+
/** Single read. Default path. Source of the exact oom_kill count. */
|
|
736
|
+
export function collectVmstat(procRoot = '/proc') {
|
|
737
|
+
return collect(() => parseVmstat(readFileSync(`${procRoot}/vmstat`, 'utf8')));
|
|
738
|
+
}
|
|
739
|
+
/** Single read. Default path. */
|
|
740
|
+
export function collectUptimeSeconds(procRoot = '/proc') {
|
|
741
|
+
return collect(() => {
|
|
742
|
+
const v = parseUptimeSeconds(readFileSync(`${procRoot}/uptime`, 'utf8'));
|
|
743
|
+
if (v === null)
|
|
744
|
+
throw new Error('unparseable /proc/uptime');
|
|
745
|
+
return v;
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Read status + cmdline for an EXPLICIT, BOUNDED list of PIDs. This is the only
|
|
750
|
+
* per-process read on the default path, and the PID list comes from a single
|
|
751
|
+
* `pgrep` exec (kernel-side scan) rather than a Node-side walk of /proc.
|
|
752
|
+
* Processes that exit mid-read are skipped.
|
|
753
|
+
*/
|
|
754
|
+
export function collectProcessesByPid(pids, procRoot = '/proc') {
|
|
755
|
+
const samples = [];
|
|
756
|
+
for (const pid of pids) {
|
|
757
|
+
let status;
|
|
758
|
+
try {
|
|
759
|
+
status = readFileSync(`${procRoot}/${pid}/status`, 'utf8');
|
|
760
|
+
}
|
|
761
|
+
catch {
|
|
762
|
+
continue;
|
|
763
|
+
}
|
|
764
|
+
let cmdline = '';
|
|
765
|
+
try {
|
|
766
|
+
cmdline = readFileSync(`${procRoot}/${pid}/cmdline`, 'utf8');
|
|
767
|
+
}
|
|
768
|
+
catch {
|
|
769
|
+
// kernel threads have an empty cmdline
|
|
770
|
+
}
|
|
771
|
+
const spaced = cmdline.replace(/\0/g, ' ');
|
|
772
|
+
samples.push({
|
|
773
|
+
pid,
|
|
774
|
+
...parseProcStatus(status),
|
|
775
|
+
cmdline: spaced,
|
|
776
|
+
vscode: vscodeRole(spaced),
|
|
777
|
+
agentSession: isAgentSession(cmdline),
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
return samples;
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Resolve PIDs with a single `pgrep` exec. Returns `{ value: null, error }` when
|
|
784
|
+
* pgrep is absent or matches nothing distinguishable, so the caller can print
|
|
785
|
+
* "unavailable" rather than a zero it cannot justify.
|
|
786
|
+
*
|
|
787
|
+
* A pgrep exec is deliberately preferred over a Node-side /proc walk: the scan
|
|
788
|
+
* happens in the kernel/C, allocating nothing in this process, which matters
|
|
789
|
+
* because we run under the memory pressure we are measuring.
|
|
790
|
+
*
|
|
791
|
+
* The RESULT IS NOT BOUNDED — pgrep returns as many PIDs as match. Callers on
|
|
792
|
+
* the default path must apply boundPids() before reading /proc for each one.
|
|
793
|
+
*/
|
|
794
|
+
export function pgrepPids(args) {
|
|
795
|
+
try {
|
|
796
|
+
const out = execFileSync('pgrep', args, { encoding: 'utf8', timeout: PGREP_TIMEOUT_MS });
|
|
797
|
+
return { value: out.split('\n').map((s) => Number(s.trim())).filter((n) => Number.isInteger(n) && n > 0) };
|
|
798
|
+
}
|
|
799
|
+
catch (err) {
|
|
800
|
+
// pgrep exits 1 with empty output when nothing matched — that is a real
|
|
801
|
+
// measurement of zero, not a failure.
|
|
802
|
+
const e = err;
|
|
803
|
+
if (e && e.status === 1 && (e.stdout ?? '').trim() === '')
|
|
804
|
+
return { value: [] };
|
|
805
|
+
return { value: null, error: err instanceof Error ? err.message : String(err) };
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* Enforce the default path's per-process bound. Callers report `dropped` rather
|
|
810
|
+
* than silently truncating, so an abnormal match count surfaces as information
|
|
811
|
+
* instead of disappearing.
|
|
812
|
+
*/
|
|
813
|
+
export function boundPids(pids, limit = MAX_BOUNDED_PIDS) {
|
|
814
|
+
return { pids: pids.slice(0, limit), dropped: Math.max(0, pids.length - limit) };
|
|
815
|
+
}
|
|
816
|
+
/**
|
|
817
|
+
* FULL /proc walk — VERBOSE PATH ONLY, and capped. Returns the `limit` processes
|
|
818
|
+
* with the largest RSS. Never call this on the default path: iterating
|
|
819
|
+
* /proc/<pid>/status across hundreds of PIDs allocates and syscalls under
|
|
820
|
+
* exactly the memory pressure being measured.
|
|
821
|
+
*/
|
|
822
|
+
export function collectProcesses(limit = 20, procRoot = '/proc') {
|
|
823
|
+
return collect(() => {
|
|
824
|
+
const samples = [];
|
|
825
|
+
for (const entry of readdirSync(procRoot)) {
|
|
826
|
+
if (!/^\d+$/.test(entry))
|
|
827
|
+
continue;
|
|
828
|
+
let status;
|
|
829
|
+
try {
|
|
830
|
+
status = readFileSync(`${procRoot}/${entry}/status`, 'utf8');
|
|
831
|
+
}
|
|
832
|
+
catch {
|
|
833
|
+
continue; // exited mid-scan, or not ours
|
|
834
|
+
}
|
|
835
|
+
const parsed = parseProcStatus(status);
|
|
836
|
+
if (parsed.vmRssKb === null)
|
|
837
|
+
continue; // kernel thread — no memory map
|
|
838
|
+
let cmdline = '';
|
|
839
|
+
try {
|
|
840
|
+
cmdline = readFileSync(`${procRoot}/${entry}/cmdline`, 'utf8');
|
|
841
|
+
}
|
|
842
|
+
catch {
|
|
843
|
+
// keep the process with comm only
|
|
844
|
+
}
|
|
845
|
+
const spaced = cmdline.replace(/\0/g, ' ');
|
|
846
|
+
samples.push({
|
|
847
|
+
pid: Number(entry),
|
|
848
|
+
...parsed,
|
|
849
|
+
cmdline: spaced,
|
|
850
|
+
vscode: vscodeRole(spaced),
|
|
851
|
+
agentSession: isAgentSession(cmdline),
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
return samples.sort((a, b) => (b.vmRssKb ?? 0) - (a.vmRssKb ?? 0)).slice(0, limit);
|
|
855
|
+
});
|
|
856
|
+
}
|
|
857
|
+
/** Read the kernel ring buffer. Fails with EPERM when dmesg_restrict is set. */
|
|
858
|
+
export function collectDmesg() {
|
|
859
|
+
return collect(() => execFileSync('dmesg', { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: DMESG_TIMEOUT_MS }));
|
|
860
|
+
}
|
|
861
|
+
/** Two single reads. Default path — caps only, no walk. */
|
|
862
|
+
export function collectInotifyCaps(procRoot = '/proc') {
|
|
863
|
+
const readNum = (p) => {
|
|
864
|
+
try {
|
|
865
|
+
const n = Number(readFileSync(p, 'utf8').trim());
|
|
866
|
+
return Number.isFinite(n) ? n : null;
|
|
867
|
+
}
|
|
868
|
+
catch {
|
|
869
|
+
return null;
|
|
870
|
+
}
|
|
871
|
+
};
|
|
872
|
+
return {
|
|
873
|
+
maxInstances: readNum(`${procRoot}/sys/fs/inotify/max_user_instances`),
|
|
874
|
+
maxWatches: readNum(`${procRoot}/sys/fs/inotify/max_user_watches`),
|
|
875
|
+
};
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Count live inotify instances and watches by walking /proc/<pid>/fd for
|
|
879
|
+
* `anon_inode:inotify` symlinks and counting watch lines in the matching fdinfo.
|
|
880
|
+
* Regression guard for PR #4475 (the `.worktrees` watcher storm).
|
|
881
|
+
*
|
|
882
|
+
* VERBOSE PATH ONLY — this is a readdir per PID plus a read per matching fd, the
|
|
883
|
+
* most expensive thing in this module. There is no aggregate current-usage
|
|
884
|
+
* counter in /proc, so there is no cheap substitute; the default path reports
|
|
885
|
+
* the caps and says the live count needs --verbose.
|
|
886
|
+
*
|
|
887
|
+
* The counts cover only processes this UID can read, so they are a LOWER BOUND;
|
|
888
|
+
* `unreadableProcesses` reports how many were skipped, so the number is never
|
|
889
|
+
* presented as complete when it is not.
|
|
890
|
+
*/
|
|
891
|
+
export function collectInotifyUsage(procRoot = '/proc') {
|
|
892
|
+
return collect(() => {
|
|
893
|
+
let instances = 0;
|
|
894
|
+
let watches = 0;
|
|
895
|
+
let unreadableProcesses = 0;
|
|
896
|
+
for (const entry of readdirSync(procRoot)) {
|
|
897
|
+
if (!/^\d+$/.test(entry))
|
|
898
|
+
continue;
|
|
899
|
+
let fds;
|
|
900
|
+
try {
|
|
901
|
+
fds = readdirSync(`${procRoot}/${entry}/fd`);
|
|
902
|
+
}
|
|
903
|
+
catch {
|
|
904
|
+
unreadableProcesses++;
|
|
905
|
+
continue;
|
|
906
|
+
}
|
|
907
|
+
for (const fd of fds) {
|
|
908
|
+
let target;
|
|
909
|
+
try {
|
|
910
|
+
target = readlinkSync(`${procRoot}/${entry}/fd/${fd}`);
|
|
911
|
+
}
|
|
912
|
+
catch {
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
if (target !== 'anon_inode:inotify')
|
|
916
|
+
continue;
|
|
917
|
+
instances++;
|
|
918
|
+
try {
|
|
919
|
+
watches += parseInotifyWatchCount(readFileSync(`${procRoot}/${entry}/fdinfo/${fd}`, 'utf8'));
|
|
920
|
+
}
|
|
921
|
+
catch {
|
|
922
|
+
// instance counted; watches unknown for this fd
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
return { instances, watches, unreadableProcesses };
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
/** One statfs syscall. Default path. */
|
|
930
|
+
export function collectDisk(path) {
|
|
931
|
+
return collect(() => {
|
|
932
|
+
const s = statfsSync(path);
|
|
933
|
+
const totalBytes = Number(s.blocks) * Number(s.bsize);
|
|
934
|
+
const availableBytes = Number(s.bavail) * Number(s.bsize);
|
|
935
|
+
return {
|
|
936
|
+
path,
|
|
937
|
+
totalBytes,
|
|
938
|
+
availableBytes,
|
|
939
|
+
freePercent: totalBytes === 0 ? 0 : (availableBytes / totalBytes) * 100,
|
|
940
|
+
};
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
/**
|
|
944
|
+
* Read the container's memory cgroup cap, trying cgroup v2 then v1. A null value
|
|
945
|
+
* inside a successful Collected means the cap is unlimited.
|
|
946
|
+
*/
|
|
947
|
+
export function collectCgroupMemoryLimit() {
|
|
948
|
+
const paths = ['/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory/memory.limit_in_bytes'];
|
|
949
|
+
const errors = [];
|
|
950
|
+
for (const p of paths) {
|
|
951
|
+
try {
|
|
952
|
+
return { value: parseCgroupMemoryLimit(readFileSync(p, 'utf8')) };
|
|
953
|
+
}
|
|
954
|
+
catch (err) {
|
|
955
|
+
errors.push(`${p}: ${err instanceof Error ? err.message : String(err)}`);
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
return { value: null, error: `no readable memory cgroup file (${errors.join('; ')})` };
|
|
959
|
+
}
|
|
960
|
+
//# sourceMappingURL=vm-health.js.map
|