@bridge4dev/runner 0.53.0 → 0.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1183 @@
1
+ import { execFile } from 'node:child_process';
2
+ import crypto from 'node:crypto';
3
+ import fs from 'node:fs';
4
+ import os from 'node:os';
5
+ import path from 'node:path';
6
+ import { promisify } from 'node:util';
7
+ import { runnerIdentity, systemdUserEnv } from './environment.js';
8
+ import { log } from './log.js';
9
+ import { DEVBRIDGE_SLICE, memoryPolicy, readMemoryFacts, readSelfCgroup, readSwapTotalBytes, SESSION_CPU_WEIGHT, SESSIONS_SLICE, SESSIONS_SWAP_SHARE, sliceCgroupPath, } from './service-unit.js';
10
+ const execFileAsync = promisify(execFile);
11
+ /**
12
+ * A cgroup of its own for every agent session — the memory ceiling that
13
+ * `process-priority.ts` cannot give.
14
+ *
15
+ * Stage 1a (`process-priority.ts`) orders the CPU and nothing else. Memory is
16
+ * the mechanism behind three of the four incidents on other people's machines:
17
+ * one session allocates until the machine swaps itself to a standstill, and the
18
+ * only thing that ever stopped it was the ceiling on the WHOLE service — which
19
+ * kills whichever child the kernel picks, not the one that misbehaved. This
20
+ * module puts each session in `devbridge-session-<id>.scope` under
21
+ * `devbridge-sessions.slice`, so the session that runs out of memory is the
22
+ * session that dies, and its neighbours never notice.
23
+ *
24
+ * Every line below is a finding of the spike of 2026-09-07 (plan
25
+ * `agent-sessions-host-resources.md` §5.4.1), not a preference. The ones that
26
+ * cost the most to learn:
27
+ *
28
+ * - `MemorySwapMax` is BOUNDED, never unset. Without a bound there is no cage:
29
+ * a process under `MemoryMax=200M` allocated 2 GB and drained the host's
30
+ * entire swap (`memory.events` read `max 0, oom_kill 0`, swap 2047/2047 MB)
31
+ * — the exact incident this whole plan exists to prevent, reproduced by the
32
+ * supposed fix. `MemoryMax` bounds RESIDENT memory; the rest goes to swap,
33
+ * and swap belongs to the machine, not to the cgroup. 0.54.0 wrote `0`;
34
+ * since #387 it is a SHARE of the machine's swap (see
35
+ * {@link sessionSwapMaxBytes}), because without any swap the brake below
36
+ * has nothing to push pages into and stalls instead of slowing.
37
+ * - `MemoryHigh` UNDER `MemoryMax`, and far under it (#387). 0.54.0 shipped
38
+ * the hard line alone, at 2.5 GiB, with systemd's default `OOMPolicy=stop`:
39
+ * an honest `pnpm typecheck` (seven `tsc` at ~400 MB each) went over it, the
40
+ * kernel killed ONE `tsc`, and systemd then stopped the WHOLE scope — the
41
+ * session died with «exited with code 143», twice in an hour, on correct
42
+ * work. The owner's rule is that honest work never reaches a kill: the
43
+ * honest share is where the brake starts, the wall is the machine's measured
44
+ * headroom, and only a real runaway ever gets that far — and then
45
+ * `OOMPolicy=continue` costs it the one process, not the session.
46
+ * The spike's «60 s in, still alive at 100 MB» under `MemoryHigh` was a
47
+ * runaway with NO swap to be pushed into — that is the kernel's penalty
48
+ * (up to 2 s per 256 KB when nothing can be reclaimed), and it is why the
49
+ * swap share above is part of the same change, not a separate one.
50
+ * - no `--collect`. A scope killed by the OOM killer stays behind in `failed`,
51
+ * and `systemctl --user show <unit> -p Result` is the ONLY place the cause
52
+ * can be read: `Result=oom-kill`. `--collect` deletes the unit the instant it
53
+ * dies and takes the reason with it. Hence the order in
54
+ * {@link releaseSessionScope}: wait for the exit, read `Result`, log it, THEN
55
+ * `reset-failed`.
56
+ * - `--expand-environment=no`, but ONLY on systemd ≥ 254. The arguments carry
57
+ * the agent's prompt, and prompts contain `$`; without this flag a future
58
+ * systemd would substitute variables into the user's own words. systemd 254
59
+ * is where the switch was added, and it is also where its own NEWS says
60
+ * expansion «defaults to enabled for all execution types except `--scope`,
61
+ * where it defaults to off … for backward compatibility reasons». So on
62
+ * Ubuntu 22.04 (249), Debian 12 and RHEL 9 (252) the flag is both rejected
63
+ * (`unrecognized option`, exit 1, nothing spawned) and unnecessary. Passing
64
+ * it unconditionally cost those machines the cage entirely
65
+ * (QA-2026-09-07 MAJOR-2).
66
+ * - `--pty` is not compatible with `--scope` ("--pty/--pipe is not compatible
67
+ * in timer or --scope mode."), and does not need to be: the sign-in relay's
68
+ * `script -qec` runs INSIDE the scope byte for byte the same. `auth-relay.ts`
69
+ * is untouched by this module.
70
+ * - `CPUWeight` is set on the scope AND on `devbridge.slice`
71
+ * (`service-unit.ts`). The moment sessions leave the service's own cgroup,
72
+ * `nice(2)` stops protecting the daemon from its own children — nice orders
73
+ * tasks WITHIN one cgroup, and across cgroups `cpu.weight` decides, which is
74
+ * 100 everywhere by default. A cage without that drop-in would quietly undo
75
+ * stage 1a and bring back 16.08: daemon starved, four missed pongs, server
76
+ * Offline, 504 on every session.
77
+ *
78
+ * Rejected in the spike: `Delegate=yes` on the service plus the runner sorting
79
+ * PIDs into child cgroups itself. cgroup v2's "no internal processes" rule
80
+ * answers `Device or resource busy` on `cgroup.subtree_control`, the way around
81
+ * it moves the daemon into a sub-cgroup and needs a unit change and a RESTART
82
+ * (which kills every live session), and the spawn-then-move order lost the race
83
+ * anyway — the runaway was 100–150 MB in before it ever reached the cage.
84
+ */
85
+ const MIB = 1024 * 1024;
86
+ const GIB = 1024 * MIB;
87
+ /**
88
+ * The honest share of one session — where the BRAKE starts, not where it dies.
89
+ *
90
+ * Until #387 this was `MemoryMax`, the kill line. It is the same number and the
91
+ * same reasoning, moved one step down the ladder: above it the kernel reclaims
92
+ * and throttles (`MemoryHigh`), and the kill line is {@link sessionMemoryMaxBytes}.
93
+ *
94
+ * `min(2.5 GiB, service ceiling / 2)` — §3 of
95
+ * `docs/plans/shipped/agent-sessions-host-resources.md`. Both halves matter:
96
+ *
97
+ * - the absolute number is sized against the measurement, not against a round
98
+ * figure. The heaviest ordinary thing a session runs is a workspace
99
+ * `pnpm typecheck`, measured at a 1571 MB peak; three live `claude`
100
+ * processes are 512/577/646 MB. 2.5 GiB leaves ~60 % headroom over the
101
+ * worst measured case, which is the margin between "kills a runaway" and
102
+ * "kills the work";
103
+ * - the half-of-the-service half is what keeps ONE session from being able to
104
+ * fill the ceiling that covers all of them. On this machine the service
105
+ * ceiling is 7680M, so half is 3.75 GiB and the absolute number wins.
106
+ *
107
+ * The service ceiling is read off systemd (`systemctl --user show
108
+ * devbridge-runner -p MemoryMax`) rather than recomputed here, because what
109
+ * protects the machine is what systemd has in force, not what the drop-in on
110
+ * disk says — those two have already drifted apart once (§5.5 of the plan).
111
+ */
112
+ export const SESSION_MEMORY_HIGH_ABSOLUTE_BYTES = Math.round(2.5 * GIB);
113
+ /**
114
+ * Processes and threads per session.
115
+ *
116
+ * An agent forks a lot (a monorepo build is hundreds of short-lived processes),
117
+ * so this is not a memory limit in disguise — it is the fork-bomb stop. 512 is
118
+ * far above anything measured on a real session and far below the 8192 the
119
+ * whole service gets.
120
+ */
121
+ export const SESSION_TASKS_MAX = 512;
122
+ /** Prefix of every scope this module creates. The sweeper matches on it. */
123
+ export const SESSION_SCOPE_PREFIX = 'devbridge-session-';
124
+ /**
125
+ * The FLOOR the plan's formula does not have, and the reason it needs one.
126
+ *
127
+ * `min(2.5 GiB, ceiling / 2)` is right on a big machine and wrong on a small
128
+ * one: at the 2 GiB ceiling `memoryPolicy` gives the smallest supported box,
129
+ * half is 1 GiB — BELOW the 1571 MB a workspace `pnpm typecheck` was measured
130
+ * at. The cage would then kill honest work on exactly the machines least able
131
+ * to afford a failed run, and it would look like a flaky agent rather than a
132
+ * limit. A containment that fires on correct work is not containment.
133
+ *
134
+ * So the floor is set above the worst measured ordinary peak, not at a round
135
+ * number.
136
+ *
137
+ * It is a floor and not a switch: below ~2.4 GB of RAM the containing ceiling
138
+ * itself drops under this number, and there the last clamp in
139
+ * {@link sessionMemoryMaxBytes} wins and a session is capped below the floor.
140
+ * That is not the cage killing honest work — the slice would have killed the
141
+ * same `pnpm typecheck` a moment later anyway, because it holds the same
142
+ * number — so removing `MemoryMax` from the scope there would buy nothing and
143
+ * cost the one thing it does buy: the runaway dying instead of its neighbours
144
+ * (QA-2026-09-07 MINOR-6, where the earlier wording promised the opposite).
145
+ */
146
+ export const SESSION_MEMORY_HIGH_MIN_BYTES = 2 * GIB;
147
+ /**
148
+ * The wall, when nothing on the machine can say where it should be.
149
+ *
150
+ * Every real path reads the wall off systemd (the slice, else the service) or
151
+ * measures it (`memoryPolicy`). This is the last resort for a machine that gave
152
+ * neither — twice the honest share, so that even blind the wall sits above the
153
+ * brake by a margin an honest build fits into, and a runaway still meets one.
154
+ */
155
+ export const SESSION_MEMORY_WALL_FALLBACK_FACTOR = 2;
156
+ /**
157
+ * The pool is cut into this many honest shares. Three, because that is what
158
+ * `maxSessions` defaults to on a dev server, so three sessions can sit at their
159
+ * brakes without the pool itself overflowing — and one runaway then meets its
160
+ * own wall (`ceiling − share`) while a neighbour on its share still fits.
161
+ */
162
+ export const SESSION_SHARE_DIVISOR = 3;
163
+ /**
164
+ * On a machine too small to leave a share of room, the brake is this fraction
165
+ * of the wall. The same 0.8 `memoryPolicy` puts between the service's own
166
+ * `MemoryHigh` and `MemoryMax`, for the same reason: a band, however narrow,
167
+ * beats a bare kill line.
168
+ */
169
+ export const SESSION_BRAKE_OF_WALL = 0.8;
170
+ /**
171
+ * `clamp(ceiling / 2, 2 GiB, 2.5 GiB)`, never above the ceiling itself.
172
+ *
173
+ * Three bounds, each for its own reason — see
174
+ * {@link SESSION_MEMORY_MAX_ABSOLUTE_BYTES} for the upper two and
175
+ * {@link SESSION_MEMORY_MIN_BYTES} for the lower one. The last clamp matters
176
+ * on a tiny machine: a per-session number ABOVE the slice's own ceiling is not
177
+ * a limit at all, it is a bigger number that never applies, and printing it in
178
+ * `doctor` would be a straight lie about what is in force.
179
+ *
180
+ * The argument is the ceiling of the cgroup that CONTAINS the session, and
181
+ * since 0.54.0 that is `devbridge-sessions.slice` and not the service — the
182
+ * caller reads the slice and falls back to the service only when systemd has
183
+ * nothing to say about the slice. The two are the same number by construction
184
+ * today; they come apart the moment a drop-in fails to apply or is edited by
185
+ * hand, and then «never more than the slice» has to still be true
186
+ * (QA-2026-09-07 MINOR-5).
187
+ */
188
+ export function sessionMemoryHighBytes(containingMemoryMaxBytes) {
189
+ return sessionMemoryLadder(containingMemoryMaxBytes).highBytes;
190
+ }
191
+ /**
192
+ * The two numbers of the cage, computed in ONE place because each is wrong
193
+ * without the other (QA of #387 found both halves broken separately).
194
+ *
195
+ * ```
196
+ * brake = max(containing / 3, 2 GiB) capped by the containing ceiling
197
+ * wall = containing − brake never at or below the brake
198
+ * ```
199
+ *
200
+ * **Why a third, and no upper cap.** The owner's second note on #387: «ночью
201
+ * машина свободна (~9 ГБ), а сессии всё равно нельзя выйти за 2.5 ГБ… граница
202
+ * должна смотреть на то, сколько реально свободно». A fixed 2.5 GiB was right
203
+ * as a KILL line and is wrong as a brake: on a big machine it throttles honest
204
+ * work that the machine could have absorbed. A third of the pool is the largest
205
+ * share that still lets three sessions sit at their brakes inside the ceiling,
206
+ * which is what `maxSessions` defaults to. The 2 GiB floor stays — it is sized
207
+ * over the 1571 MB measured peak of a workspace `pnpm typecheck`, and a dev
208
+ * server that cannot run one of those is not a dev server.
209
+ *
210
+ * **Why the wall is `containing − brake` and not the ceiling itself.** 0.55.0's
211
+ * first shape put the wall AT the slice ceiling, and that is not a per-session
212
+ * wall at all: the kernel charges a leaf and every ancestor, so with any
213
+ * neighbour the SLICE overflows first, and then the victim is chosen across the
214
+ * whole slice. Measured on this host (a 300 MB slice, two 250 MB scopes): the
215
+ * scope that crept up survived to the end, and the **neighbour holding 200 MB
216
+ * under its own wall was killed** — slice `max 19, oom 1, oom_kill 1`, victim
217
+ * `max 0, oom 0, oom_kill 1`. Leaving one honest share of room under the
218
+ * ceiling is what makes the runaway meet its OWN wall first while a neighbour
219
+ * on its share still fits.
220
+ *
221
+ * **The tiny machine.** When the ceiling is so low that `containing − brake`
222
+ * would land at or under the brake, there is no room for isolation at all: the
223
+ * wall becomes the ceiling and the BRAKE is lowered to 80 % of it, so that
224
+ * there is still a braking band (`memoryPolicy` uses the same 0.8 for the
225
+ * service). Without this, a 4 GB machine got `MemoryHigh == MemoryMax` — the
226
+ * 0.54.0 kill line back in place, with the card claiming a brake.
227
+ */
228
+ export function sessionMemoryLadder(containingMemoryMaxBytes, measuredCeilingBytes = null) {
229
+ const containing = [containingMemoryMaxBytes, measuredCeilingBytes].find((value) => value !== null && Number.isFinite(value) && value > 0);
230
+ if (containing === undefined) {
231
+ // Nothing on this machine could say what contains the session. The
232
+ // reference share, and a wall well above it, so the ladder still exists.
233
+ return {
234
+ highBytes: SESSION_MEMORY_HIGH_ABSOLUTE_BYTES,
235
+ maxBytes: SESSION_MEMORY_HIGH_ABSOLUTE_BYTES * SESSION_MEMORY_WALL_FALLBACK_FACTOR,
236
+ };
237
+ }
238
+ const ceiling = Math.floor(containing);
239
+ const share = Math.max(Math.floor(ceiling / SESSION_SHARE_DIVISOR), SESSION_MEMORY_HIGH_MIN_BYTES);
240
+ // Never promise a session more than the cgroup that contains every session.
241
+ const brake = Math.max(1, Math.min(share, ceiling));
242
+ const wall = ceiling - brake;
243
+ if (wall <= brake) {
244
+ return {
245
+ highBytes: Math.max(1, Math.floor(ceiling * SESSION_BRAKE_OF_WALL)),
246
+ maxBytes: ceiling,
247
+ };
248
+ }
249
+ return { highBytes: brake, maxBytes: wall };
250
+ }
251
+ /**
252
+ * The wall — `MemoryMax` on the scope — and why it is the WHOLE containing
253
+ * ceiling and not a fraction of it (#387).
254
+ *
255
+ * The owner's rule: the wall is protection against a runaway, not the norm of
256
+ * work. Where the norm lives is {@link sessionMemoryHighBytes}; the wall has to
257
+ * be far enough above it that honest work never touches it, and «far» on a dev
258
+ * server means «what the machine can actually spare» — which is exactly the
259
+ * number the slice ceiling already is (`memoryPolicy`: `MemAvailable` plus what
260
+ * we hold, minus a reserve). Giving a session less than that is the 2.5 GiB
261
+ * mistake with a different number.
262
+ *
263
+ * With `OOMPolicy=continue` on the scope, reaching the wall costs a runaway its
264
+ * fattest process and nothing else; the neighbours are protected by the slice,
265
+ * which holds the same number over all of them together.
266
+ *
267
+ * `containing` is what systemd has in force on the slice (else the service);
268
+ * `measured` is what the policy would write there; the factor is the last
269
+ * resort. Never below the brake — a wall under the brake is a kill line again.
270
+ */
271
+ export function sessionMemoryMaxBytes(containingMemoryMaxBytes, measuredCeilingBytes = null) {
272
+ return sessionMemoryLadder(containingMemoryMaxBytes, measuredCeilingBytes).maxBytes;
273
+ }
274
+ /**
275
+ * Swap one session may push into — the reason the brake slows instead of stalls.
276
+ *
277
+ * Half of what the slice may use, the same «no single session takes the
278
+ * collective allowance» rule as the memory share. When systemd reports no
279
+ * bound on the slice (the drop-in never landed, MAJOR-3 shape) the share is cut
280
+ * from the machine's `SwapTotal` directly, with the same fraction the slice
281
+ * would have had, so the per-scope line is a real bound on its own. A machine
282
+ * with no swap gets 0, which is exactly 0.54.0's line — and exactly right:
283
+ * there is nothing to share.
284
+ */
285
+ export function sessionSwapMaxBytes(sliceSwapMaxBytes, hostSwapTotalBytes) {
286
+ if (sliceSwapMaxBytes !== null && Number.isFinite(sliceSwapMaxBytes)) {
287
+ return Math.max(0, Math.floor(sliceSwapMaxBytes / 2));
288
+ }
289
+ if (hostSwapTotalBytes !== null && Number.isFinite(hostSwapTotalBytes)) {
290
+ return Math.max(0, Math.floor((hostSwapTotalBytes * SESSIONS_SWAP_SHARE) / 2));
291
+ }
292
+ return 0;
293
+ }
294
+ // ─── unit names ──────────────────────────────────────────────────────
295
+ /**
296
+ * Room for the id inside a unit name. systemd's own limit is 255 bytes for the
297
+ * whole name; the prefix, the `.scope` suffix, a restart marker and a hash all
298
+ * come out of the same budget, so the id is cut well short of it.
299
+ */
300
+ const MAX_ID_IN_UNIT = 180;
301
+ /**
302
+ * The session id as systemd will accept it.
303
+ *
304
+ * Ids are UUIDs today and arbitrary text tomorrow — they arrive from the API,
305
+ * and `verify` keys its cage by a run id. systemd unit names take only
306
+ * `[A-Za-z0-9:_.-]`, so everything else is folded to `-`. Folding is lossy, and
307
+ * lossy is how two different sessions end up fighting over one scope, so
308
+ * anything that had to be changed or cut also gets eight hex digits of the
309
+ * original — deterministic, so the same session always names the same unit and
310
+ * the sweeper can still recognise it.
311
+ */
312
+ export function sanitizeCageId(id) {
313
+ const folded = id.replace(/[^A-Za-z0-9:_.-]/g, '-').replace(/^[.-]+/, '');
314
+ const cut = folded.slice(0, MAX_ID_IN_UNIT);
315
+ if (cut === id && cut.length > 0)
316
+ return cut;
317
+ const digest = crypto.createHash('sha256').update(id).digest('hex').slice(0, 8);
318
+ return `${cut || 'session'}-${digest}`;
319
+ }
320
+ /**
321
+ * The scope unit for one START of one session.
322
+ *
323
+ * `attempt` is not decoration. A scope the OOM killer took stays loaded in
324
+ * `failed` until something resets it, and `systemd-run --unit=` on a name that
325
+ * is still loaded fails outright — measured: «Unit devbridge-….scope was
326
+ * already loaded or has a fragment file», exit 1, nothing spawned. So a session
327
+ * that is restarted after an OOM (which is exactly when it IS restarted) would
328
+ * be unable to start at all if the name never changed.
329
+ * {@link releaseSessionScope} clears the failed unit and only then lets the
330
+ * counter fall back to 1, so the plain `devbridge-session-<id>.scope` is the
331
+ * normal case and the marker appears only while the old scope is still there.
332
+ */
333
+ export function sessionScopeUnit(id, attempt = 1) {
334
+ const base = `${SESSION_SCOPE_PREFIX}${sanitizeCageId(id)}`;
335
+ return attempt <= 1 ? `${base}.scope` : `${base}-r${attempt}.scope`;
336
+ }
337
+ // ─── capability detection ────────────────────────────────────────────
338
+ /** cgroup v2 unified, from `statfs` — the number behind `stat -fc %T`. */
339
+ const CGROUP2_SUPER_MAGIC = 0x63677270;
340
+ const TMPFS_MAGIC = 0x01021994;
341
+ /** What the live probe must read back out of its own cgroup: 64 MiB, exactly. */
342
+ const PROBE_MEMORY_MAX = 64 * MIB;
343
+ /**
344
+ * The systemd release that added `--expand-environment=`.
345
+ *
346
+ * Below it `systemd-run` answers `unrecognized option` and exits 1 — the probe
347
+ * then reads as «the cage did not hold» on a machine whose cgroups are perfect
348
+ * (Ubuntu 22.04 is 249, Debian 12 and RHEL 9 are 252). See the module header for
349
+ * why the flag is not needed there either.
350
+ */
351
+ export const EXPAND_ENVIRONMENT_MIN_SYSTEMD = 254;
352
+ function statfsType(target) {
353
+ try {
354
+ const type = fs.statfsSync(target).type;
355
+ if (type === CGROUP2_SUPER_MAGIC)
356
+ return 'cgroup2fs';
357
+ if (type === TMPFS_MAGIC)
358
+ return 'tmpfs';
359
+ return `0x${type.toString(16)}`;
360
+ }
361
+ catch {
362
+ return null;
363
+ }
364
+ }
365
+ function onPath(name) {
366
+ const dirs = (process.env['PATH'] ?? '').split(path.delimiter).filter(Boolean);
367
+ return dirs.some((dir) => {
368
+ try {
369
+ fs.accessSync(path.join(dir, name), fs.constants.X_OK);
370
+ return true;
371
+ }
372
+ catch {
373
+ return false;
374
+ }
375
+ });
376
+ }
377
+ /**
378
+ * Variables the child needs to reach the user's systemd, and nothing else.
379
+ *
380
+ * `systemd-run --user` finds the bus through `XDG_RUNTIME_DIR` (sd-bus falls
381
+ * back to `$XDG_RUNTIME_DIR/bus` when `DBUS_SESSION_BUS_ADDRESS` is unset), and
382
+ * that variable is already on both spawn allowlists — so merging this into a
383
+ * session's environment widens nothing. `DBUS_SESSION_BUS_ADDRESS` is passed on
384
+ * only when the daemon really has one of its own, never synthesised.
385
+ */
386
+ export function cageEnv() {
387
+ const out = {};
388
+ const dir = systemdUserEnv()['XDG_RUNTIME_DIR'];
389
+ if (dir)
390
+ out['XDG_RUNTIME_DIR'] = dir;
391
+ const bus = process.env['DBUS_SESSION_BUS_ADDRESS'];
392
+ if (bus)
393
+ out['DBUS_SESSION_BUS_ADDRESS'] = bus;
394
+ return out;
395
+ }
396
+ function userManagerCgroupControllers() {
397
+ const uid = runnerIdentity().uid;
398
+ if (uid < 0)
399
+ return null;
400
+ const file = path.join('/sys/fs/cgroup/user.slice', `user-${uid}.slice`, `user@${uid}.service`, 'cgroup.controllers');
401
+ try {
402
+ return fs.readFileSync(file, 'utf8').trim().split(/\s+/).filter(Boolean);
403
+ }
404
+ catch {
405
+ return null;
406
+ }
407
+ }
408
+ /**
409
+ * The only honest check there is: build a real cage and ask it what it holds.
410
+ *
411
+ * Reading `cgroup.controllers` and `systemd-run --version` proves the parts are
412
+ * on the machine, not that they work: under a foreign supervisor the files look
413
+ * perfect and the limit is silently never applied. So the probe starts a
414
+ * throwaway scope with `MemoryMax=64M` whose entire job is to `cat` its own
415
+ * `memory.max`, and the answer has to be `67108864` on the nose.
416
+ *
417
+ * `--collect` IS right here, and only here: this scope's cause of death is of no
418
+ * interest, and a probe that left units behind on every daemon start would be a
419
+ * litter machine.
420
+ */
421
+ async function runLiveProbe(options) {
422
+ const unit = `devbridge-cage-probe-${process.pid}`;
423
+ try {
424
+ const { stdout } = await execFileAsync('systemd-run', [
425
+ '--user',
426
+ '--scope',
427
+ '--quiet',
428
+ '--collect',
429
+ `--unit=${unit}`,
430
+ `--slice=${SESSIONS_SLICE}`,
431
+ // Exactly the switches `cageSpawn` will use, this one included: a probe
432
+ // that tests a different command line proves nothing about the real one.
433
+ ...(options.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
434
+ '-p',
435
+ `MemoryHigh=${PROBE_MEMORY_MAX / 2}`,
436
+ '-p',
437
+ `MemoryMax=${PROBE_MEMORY_MAX}`,
438
+ '-p',
439
+ 'MemorySwapMax=0',
440
+ // The one property an older systemd refuses for a scope — the probe is
441
+ // where that refusal has to surface, so the real spawn never meets it.
442
+ ...(options.oomPolicyFlag ? ['-p', OOM_POLICY_CONTINUE] : []),
443
+ '-p',
444
+ 'TasksMax=16',
445
+ '-p',
446
+ `CPUWeight=${SESSION_CPU_WEIGHT}`,
447
+ '--',
448
+ '/bin/sh',
449
+ '-c',
450
+ 'cat /sys/fs/cgroup$(cut -d: -f3 /proc/self/cgroup)/memory.max',
451
+ ],
452
+ // Generous: the same call took 0.07–2.5 s on an idle machine and 2741 ms
453
+ // under load. A probe that times out costs the machine its cage, so the
454
+ // budget is sized for the worst measurement, not the typical one.
455
+ { timeout: 20_000, env: { ...process.env, ...cageEnv() } });
456
+ return { memoryMax: stdout.trim(), error: null };
457
+ }
458
+ catch (error) {
459
+ // Kept, not swallowed. «Did not run» and «ran and did not hold» are
460
+ // different machines and different repairs, and one sentence about
461
+ // `memory.max` for both sent people to fix cgroups that were fine.
462
+ return { memoryMax: null, error: execErrorText(error) };
463
+ }
464
+ }
465
+ /**
466
+ * The first line `systemd-run` complained with, short enough for a card.
467
+ *
468
+ * `stderr` before `message`: node's own message is `Command failed: systemd-run
469
+ * …` with the whole command line in it, while stderr is the one sentence that
470
+ * names the cause (`unrecognized option '--expand-environment=no'`).
471
+ */
472
+ function execErrorText(error) {
473
+ const own = typeof error === 'object' && error !== null ? { ...error } : {};
474
+ const stderr = typeof own['stderr'] === 'string' ? own['stderr'] : '';
475
+ const text = stderr.trim() || (error instanceof Error ? error.message : String(error));
476
+ const first = text.split('\n').find((line) => line.trim().length > 0) ?? '';
477
+ return first.trim().slice(0, 200);
478
+ }
479
+ /**
480
+ * Which `systemd-run` this is. `systemd 255 (255.4-1ubuntu8.17)` → 255.
481
+ *
482
+ * The version decides one flag and nothing else, so an unreadable answer is not
483
+ * a refusal: it drops the flag, which is what every systemd below 254 does with
484
+ * `--scope` anyway.
485
+ */
486
+ async function readSystemdRunVersion() {
487
+ try {
488
+ const { stdout } = await execFileAsync('systemd-run', ['--version'], { timeout: 10_000 });
489
+ const match = /^systemd\s+(\d+)/m.exec(stdout);
490
+ const major = match?.[1] ? Number(match[1]) : Number.NaN;
491
+ return Number.isFinite(major) ? major : null;
492
+ }
493
+ catch {
494
+ return null;
495
+ }
496
+ }
497
+ async function showMemoryMax(unit) {
498
+ const value = await showByteCount(unit, 'MemoryMax');
499
+ return value !== null && value > 0 ? value : null;
500
+ }
501
+ /**
502
+ * A byte-count property as systemd has it in force. `infinity` and «no answer»
503
+ * are both null; 0 is kept, because for `MemorySwapMax` zero is the whole point.
504
+ */
505
+ async function showByteCount(unit, property) {
506
+ try {
507
+ const { stdout } = await execFileAsync('systemctl', ['--user', 'show', unit, '-p', property, '--value'], { timeout: 10_000, env: systemdUserEnv() });
508
+ const raw = stdout.trim();
509
+ if (!/^\d+$/.test(raw))
510
+ return null;
511
+ const value = Number(raw);
512
+ return Number.isSafeInteger(value) ? value : null;
513
+ }
514
+ catch {
515
+ return null;
516
+ }
517
+ }
518
+ /** The ceiling the policy would write to the slice today — the measured wall. */
519
+ function measuredMachineCeiling() {
520
+ const facts = readMemoryFacts();
521
+ return facts ? memoryPolicy(facts).maxBytes : null;
522
+ }
523
+ export const defaultCageProbe = {
524
+ cgroupFsType: () => statfsType('/sys/fs/cgroup'),
525
+ systemdRunOnPath: () => onPath('systemd-run'),
526
+ userBusPath: () => {
527
+ const dir = systemdUserEnv()['XDG_RUNTIME_DIR'];
528
+ if (!dir)
529
+ return null;
530
+ return fs.existsSync(path.join(dir, 'bus')) ? dir : null;
531
+ },
532
+ delegatedControllers: userManagerCgroupControllers,
533
+ systemdRunVersion: readSystemdRunVersion,
534
+ probeMemoryMax: runLiveProbe,
535
+ serviceMemoryMax: () => showMemoryMax('devbridge-runner'),
536
+ sessionsSliceMemoryMax: () => showMemoryMax(SESSIONS_SLICE),
537
+ sessionsSliceSwapMax: () => showByteCount(SESSIONS_SLICE, 'MemorySwapMax'),
538
+ hostSwapTotalBytes: readSwapTotalBytes,
539
+ machineCeilingBytes: measuredMachineCeiling,
540
+ canRenice: () => {
541
+ try {
542
+ // A no-op that still asks the kernel the question: setting our own
543
+ // priority to the value it already has. Allowed for every user, so a
544
+ // failure here means the syscall itself is refused (seccomp, a locked-down
545
+ // container) — which is the one case where even the fallback is a lie.
546
+ os.setPriority(0, os.getPriority(0));
547
+ return true;
548
+ }
549
+ catch {
550
+ return false;
551
+ }
552
+ },
553
+ };
554
+ export async function detectSessionCage(probe = defaultCageProbe) {
555
+ const fallback = (reason) => ({
556
+ mode: probe.canRenice() ? 'nice-only' : 'none',
557
+ reason,
558
+ memoryHighBytes: null,
559
+ memoryMaxBytes: null,
560
+ swapMaxBytes: null,
561
+ oomContinue: false,
562
+ serviceMemoryMaxBytes: null,
563
+ sessionsSliceMemoryMaxBytes: null,
564
+ expandEnvironmentFlag: false,
565
+ });
566
+ const fsType = probe.cgroupFsType();
567
+ // Not «contains cgroup2» — exactly cgroup2fs. `tmpfs` at this path is cgroup
568
+ // v1 or the hybrid layout, where `systemd-run --scope` accepts every `-p` and
569
+ // applies none of them.
570
+ if (fsType !== 'cgroup2fs') {
571
+ return fallback(`/sys/fs/cgroup is ${fsType ?? 'unreadable'}, not cgroup2fs (cgroup v1 or hybrid)`);
572
+ }
573
+ if (!probe.systemdRunOnPath())
574
+ return fallback('systemd-run is not on PATH');
575
+ if (!probe.userBusPath()) {
576
+ return fallback('no user systemd bus ($XDG_RUNTIME_DIR/bus) — started outside a user session?');
577
+ }
578
+ const controllers = probe.delegatedControllers();
579
+ // Unreadable is NOT a refusal: the path is only right for a systemd user
580
+ // manager, and the live probe below is the authority in every layout. A file
581
+ // we can read and which says «no memory» is a different matter — that is the
582
+ // delegation this cage is built on, missing.
583
+ if (controllers && !controllers.includes('memory')) {
584
+ return fallback(`the user manager has no delegated memory controller (${controllers.join(' ') || 'none'})`);
585
+ }
586
+ // Decided once, here, and carried in the facts: the probe and every later
587
+ // spawn have to build the SAME command line, or the thing that was proved to
588
+ // work is not the thing that runs.
589
+ const systemdVersion = await probe.systemdRunVersion();
590
+ const expandEnvironmentFlag = systemdVersion !== null && systemdVersion >= EXPAND_ENVIRONMENT_MIN_SYSTEMD;
591
+ // `OOMPolicy=` on a SCOPE is newer than the cage's other switches, and which
592
+ // release added it is not documented well enough to gate on a number (this
593
+ // host, systemd 255, takes it; an older systemd answers «Unknown assignment»
594
+ // and spawns NOTHING at all). So the probe decides: try with it, and on a
595
+ // refusal that names the property try once more without — that machine keeps
596
+ // its cage and loses only the «one process, not the session» half (#387).
597
+ let oomPolicyFlag = true;
598
+ let probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
599
+ if (probed.error !== null && refusesOomPolicy(probed.error)) {
600
+ oomPolicyFlag = false;
601
+ probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
602
+ }
603
+ if (probed.error !== null) {
604
+ // Not «the cage did not hold» — nothing was ever caged. Blaming `memory.max`
605
+ // for a refused command line sent people to fix delegation that was fine.
606
+ return fallback(`the test cage would not start: ${probed.error}`);
607
+ }
608
+ if (probed.memoryMax !== String(PROBE_MEMORY_MAX)) {
609
+ return fallback(`the test cage did not hold: memory.max read back as ${probed.memoryMax ?? 'nothing'}, wanted ${PROBE_MEMORY_MAX}`);
610
+ }
611
+ const serviceMemoryMaxBytes = await probe.serviceMemoryMax();
612
+ const sessionsSliceMemoryMaxBytes = await probe.sessionsSliceMemoryMax();
613
+ const sliceSwapMaxBytes = await probe.sessionsSliceSwapMax();
614
+ // The slice is what actually contains the session; the service is the
615
+ // fallback for a systemd that would not answer about the slice at all.
616
+ const containing = sessionsSliceMemoryMaxBytes ?? serviceMemoryMaxBytes;
617
+ const ladder = sessionMemoryLadder(containing, probe.machineCeilingBytes());
618
+ return {
619
+ mode: 'scope',
620
+ reason: '',
621
+ memoryHighBytes: ladder.highBytes,
622
+ memoryMaxBytes: ladder.maxBytes,
623
+ swapMaxBytes: sessionSwapMaxBytes(sliceSwapMaxBytes, probe.hostSwapTotalBytes()),
624
+ oomContinue: oomPolicyFlag,
625
+ serviceMemoryMaxBytes,
626
+ sessionsSliceMemoryMaxBytes,
627
+ expandEnvironmentFlag,
628
+ };
629
+ }
630
+ /** The `-p` assignment itself, in one place: the probe and the spawn must agree. */
631
+ const OOM_POLICY_CONTINUE = 'OOMPolicy=continue';
632
+ /**
633
+ * Did `systemd-run` refuse the command line BECAUSE of `OOMPolicy=`?
634
+ *
635
+ * systemd's wording for a property a unit type does not take is «Unknown
636
+ * assignment: OOMPolicy=continue» (`bus_append_unit_property_assignment`); a
637
+ * refusal that does not name the property is some other machine's problem and
638
+ * must not be retried into a cage with a weaker policy.
639
+ */
640
+ export function refusesOomPolicy(error) {
641
+ return /OOMPolicy/i.test(error);
642
+ }
643
+ /**
644
+ * Before the detector has run, nothing is wrapped.
645
+ *
646
+ * `nice-only` rather than `none` because stage 1a is unconditional — the renice
647
+ * happens at every spawn point whatever this module says. The value only ever
648
+ * appears where the daemon has not started (a `verify` from the command line, a
649
+ * test), and it means «no scope», which is the safe answer.
650
+ */
651
+ const UNPROBED = {
652
+ mode: 'nice-only',
653
+ reason: 'not probed yet',
654
+ memoryHighBytes: null,
655
+ memoryMaxBytes: null,
656
+ swapMaxBytes: null,
657
+ oomContinue: false,
658
+ serviceMemoryMaxBytes: null,
659
+ sessionsSliceMemoryMaxBytes: null,
660
+ expandEnvironmentFlag: false,
661
+ };
662
+ let detected = null;
663
+ /**
664
+ * Probe once, at daemon start, and remember the answer.
665
+ *
666
+ * The probe costs a process, so it is not something a spawn can afford to do:
667
+ * three sessions starting at once would mean three throwaway scopes before the
668
+ * first agent got a word out.
669
+ */
670
+ export async function initSessionCage(probe = defaultCageProbe) {
671
+ detected = await detectSessionCage(probe);
672
+ if (detected.mode === 'scope') {
673
+ log.info('session cage: each session gets its own cgroup', {
674
+ slice: SESSIONS_SLICE,
675
+ memoryHighMB: Math.round((detected.memoryHighBytes ?? 0) / MIB),
676
+ memoryMaxMB: Math.round((detected.memoryMaxBytes ?? 0) / MIB),
677
+ swapMaxMB: Math.round((detected.swapMaxBytes ?? 0) / MIB),
678
+ oomPolicy: detected.oomContinue ? 'continue' : 'stop (this systemd takes none on a scope)',
679
+ serviceMemoryMaxMB: detected.serviceMemoryMaxBytes
680
+ ? Math.round(detected.serviceMemoryMaxBytes / MIB)
681
+ : null,
682
+ sessionsSliceMemoryMaxMB: detected.sessionsSliceMemoryMaxBytes
683
+ ? Math.round(detected.sessionsSliceMemoryMaxBytes / MIB)
684
+ : null,
685
+ });
686
+ if (detected.sessionsSliceMemoryMaxBytes === null) {
687
+ // The half of the containment the probe cannot see. Loud, because from
688
+ // here on the machine looks caged and one session at a time is: what is
689
+ // missing is the ceiling over all of them together.
690
+ log.warn('session cage: no collective ceiling on the sessions slice', {
691
+ slice: SESSIONS_SLICE,
692
+ why: 'the drop-in has not been applied — systemd reports MemoryMax=infinity',
693
+ hint: 'run `devbridge-runner doctor --fix`',
694
+ });
695
+ }
696
+ }
697
+ else {
698
+ // Warn, not error. A machine without the cage is the machine we had
699
+ // yesterday; what must never happen is that it looks like the machine we
700
+ // wanted and quietly is not (plan §5.4.2).
701
+ log.warn('session cage: not available — falling back to priority only', {
702
+ mode: detected.mode,
703
+ reason: detected.reason,
704
+ });
705
+ }
706
+ return detected;
707
+ }
708
+ /** What the last {@link initSessionCage} found; the safe default before it ran. */
709
+ export function sessionCage() {
710
+ return detected ?? UNPROBED;
711
+ }
712
+ /**
713
+ * Live scope per cage id, so a restart cannot collide with a scope that the OOM
714
+ * killer left behind. Cleared by {@link releaseSessionScope} once the old unit
715
+ * is provably gone — see {@link sessionScopeUnit}.
716
+ */
717
+ const attempts = new Map();
718
+ /**
719
+ * The scope each caged id is CURRENTLY in, so the supervisor's memory watch can
720
+ * find the cgroup of a live session without the adapter having to hand it up.
721
+ * Set in {@link cageSpawn}, cleared in {@link releaseSessionScope}.
722
+ */
723
+ const liveUnits = new Map();
724
+ /** The scope a live caged id runs in, or null when it is not caged (or gone). */
725
+ export function sessionScopeUnitOf(id) {
726
+ return liveUnits.get(id) ?? null;
727
+ }
728
+ /**
729
+ * Wrap a command in its session's cage, or hand it back untouched.
730
+ *
731
+ * Untouched is the honest answer on every machine where the cage was not proved
732
+ * to work: `systemd-run` would accept the flags there and apply nothing, and a
733
+ * session that believes it is contained when it is not is worse than one that
734
+ * knows it is not.
735
+ *
736
+ * Pipes, exit codes, signals and stdin EOF behave exactly as with a direct
737
+ * spawn, because `systemd-run --scope` execs into the SAME pid — verified in the
738
+ * spike, including `detached: true` + `process.kill(-pid)` in `verify.ts`
739
+ * (`pgid === child.pid` still holds).
740
+ */
741
+ export function cageSpawn(input) {
742
+ const facts = sessionCage();
743
+ if (facts.mode !== 'scope' ||
744
+ facts.memoryMaxBytes === null ||
745
+ facts.memoryHighBytes === null ||
746
+ facts.swapMaxBytes === null) {
747
+ return { command: input.command, args: input.args, env: {}, unit: null };
748
+ }
749
+ const attempt = (attempts.get(input.id) ?? 0) + 1;
750
+ attempts.set(input.id, attempt);
751
+ const unit = sessionScopeUnit(input.id, attempt);
752
+ liveUnits.set(input.id, unit);
753
+ // A verdict nobody read belongs to the process that just ended, not to the
754
+ // one starting now: without this, a kill that was never surfaced would be
755
+ // pinned hours later on an unrelated crash of the next process (#387 QA).
756
+ deaths.delete(input.id);
757
+ oomKillsSeen.delete(input.id);
758
+ return {
759
+ command: 'systemd-run',
760
+ args: [
761
+ '--user',
762
+ '--scope',
763
+ '--quiet',
764
+ `--unit=${unit}`,
765
+ `--slice=${SESSIONS_SLICE}`,
766
+ // The agent's prompt travels in these arguments and prompts contain `$`.
767
+ // Only where systemd knows the switch — below 254 it is a hard error and
768
+ // `--scope` does no expansion anyway (module header, MAJOR-2).
769
+ ...(facts.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
770
+ // The ladder of #387, bottom to top. The brake: past the honest share the
771
+ // kernel reclaims and slows this session down, and nothing dies.
772
+ '-p',
773
+ `MemoryHigh=${facts.memoryHighBytes}`,
774
+ // The wall: the machine's measured headroom, far above the brake. Only a
775
+ // runaway gets here, and with `OOMPolicy=continue` it costs one process.
776
+ '-p',
777
+ `MemoryMax=${facts.memoryMaxBytes}`,
778
+ // THE line. Without a bound there is no cage at all: `MemoryMax` bounds
779
+ // resident memory and lets the rest fall into the host's swap, which is
780
+ // how a 200 MB cage allocated 2 GB and took the machine's swap with it.
781
+ // Bounded, never unset — and since #387 a share rather than 0, because
782
+ // the brake above has to have somewhere to push pages (module header).
783
+ '-p',
784
+ `MemorySwapMax=${facts.swapMaxBytes}`,
785
+ // Only where the probe proved this systemd takes it on a scope. Without
786
+ // it the kernel killing ONE process stops the WHOLE scope (`OOMPolicy`
787
+ // defaults to `stop`) — the exact mechanism that killed honest sessions
788
+ // in 0.54.0, and the same lesson `service-unit.ts` learned in QA-112.
789
+ ...(facts.oomContinue ? ['-p', OOM_POLICY_CONTINUE] : []),
790
+ '-p',
791
+ `TasksMax=${SESSION_TASKS_MAX}`,
792
+ '-p',
793
+ `CPUWeight=${SESSION_CPU_WEIGHT}`,
794
+ '--',
795
+ input.command,
796
+ ...input.args,
797
+ ],
798
+ env: cageEnv(),
799
+ unit,
800
+ };
801
+ }
802
+ /**
803
+ * Did this process die in the window before `systemd-run` handed over?
804
+ *
805
+ * Starting a scope took 0.07–2.5 s in the spike, and 2741 ms on a loaded
806
+ * machine. A stop inside that window signals `systemd-run` itself, before the
807
+ * `exec`, and the parent sees `{code: null, signal: 'SIGTERM'}` with nothing on
808
+ * stdout at all. That is «the session never started», not «the agent died
809
+ * silently» — told apart here so no supervisor has to infer it from an empty
810
+ * buffer.
811
+ */
812
+ export function killedBeforeExec(info) {
813
+ return info.caged && !info.sawOutput && info.code === null && info.signal === 'SIGTERM';
814
+ }
815
+ /**
816
+ * The pure half of {@link readScopeMemoryStatus}: the four files' text in, the
817
+ * status out. `memory.swap.current` is optional (no swap controller, cgroup v2
818
+ * without swap accounting); the rest are not.
819
+ */
820
+ export function parseScopeMemoryStatus(files) {
821
+ const count = (raw) => {
822
+ const trimmed = raw.trim();
823
+ if (!/^\d+$/.test(trimmed))
824
+ return null;
825
+ const value = Number(trimmed);
826
+ return Number.isSafeInteger(value) ? value : null;
827
+ };
828
+ const limit = (raw) => (raw.trim() === 'max' ? null : count(raw));
829
+ const event = (name) => {
830
+ const line = files.events.split('\n').find((l) => l.startsWith(`${name} `));
831
+ return line ? (count(line.slice(name.length + 1)) ?? 0) : 0;
832
+ };
833
+ const currentBytes = count(files.current);
834
+ if (currentBytes === null)
835
+ return null;
836
+ return {
837
+ currentBytes,
838
+ highBytes: limit(files.high),
839
+ maxBytes: limit(files.max),
840
+ swapCurrentBytes: files.swapCurrent === undefined ? 0 : (count(files.swapCurrent) ?? 0),
841
+ highEvents: event('high'),
842
+ oomKills: event('oom_kill'),
843
+ ownLimitOom: event('oom'),
844
+ };
845
+ }
846
+ /** Where a session scope's cgroup lives, or null off a user manager. */
847
+ function scopeCgroupDir(unit) {
848
+ const self = readSelfCgroup();
849
+ if (self === null)
850
+ return null;
851
+ const slice = sliceCgroupPath(self, SESSIONS_SLICE);
852
+ return slice === null ? null : path.join(slice, unit);
853
+ }
854
+ /**
855
+ * What one live session's cgroup holds and has been through. Null when the
856
+ * cgroup is not there (the scope ended, or this machine has no cage).
857
+ *
858
+ * Filesystem, not `systemctl show`: this runs on the supervisor's 30 s tick for
859
+ * every live session, and the bus is the thing that took 2.7 s under load.
860
+ */
861
+ export function readScopeMemoryStatus(unit, readFile = (p) => fs.readFileSync(p, 'utf8')) {
862
+ const dir = scopeCgroupDir(unit);
863
+ if (dir === null)
864
+ return null;
865
+ try {
866
+ let swapCurrent;
867
+ try {
868
+ swapCurrent = readFile(path.join(dir, 'memory.swap.current'));
869
+ }
870
+ catch {
871
+ swapCurrent = undefined;
872
+ }
873
+ return parseScopeMemoryStatus({
874
+ current: readFile(path.join(dir, 'memory.current')),
875
+ high: readFile(path.join(dir, 'memory.high')),
876
+ max: readFile(path.join(dir, 'memory.max')),
877
+ events: readFile(path.join(dir, 'memory.events')),
878
+ ...(swapCurrent === undefined ? {} : { swapCurrent }),
879
+ });
880
+ }
881
+ catch {
882
+ return null;
883
+ }
884
+ }
885
+ /**
886
+ * `oom_kill` as the watch last saw it, per id — so a kill that was already
887
+ * announced in the feed while the session lived is not blamed for a death that
888
+ * came later for another reason.
889
+ */
890
+ const oomKillsSeen = new Map();
891
+ export function markScopeOomKillsSeen(id, oomKills) {
892
+ oomKillsSeen.set(id, oomKills);
893
+ }
894
+ const deaths = new Map();
895
+ /**
896
+ * Snapshot the cgroup's verdict before systemd can take it away.
897
+ *
898
+ * Exported for the test seam only (`readStatus`): the real caller passes
899
+ * nothing and reads the live cgroup.
900
+ */
901
+ export function rememberDeath(id, unit, readStatus = readScopeMemoryStatus) {
902
+ const status = readStatus(unit);
903
+ const seen = oomKillsSeen.get(id) ?? 0;
904
+ oomKillsSeen.delete(id);
905
+ if (status === null || status.oomKills <= seen)
906
+ return;
907
+ deaths.set(id, {
908
+ oomKills: status.oomKills - seen,
909
+ maxBytes: status.maxBytes,
910
+ // `oom` moves only where the limit that was hit belongs. A kill with our
911
+ // own counter still at zero came from an ancestor — the pool.
912
+ ownWall: status.ownLimitOom > 0,
913
+ });
914
+ }
915
+ /**
916
+ * The other source of the same verdict, for the machine where the first one is
917
+ * already gone (#387 QA).
918
+ *
919
+ * On a systemd that takes no `OOMPolicy` on a scope, one kill stops the whole
920
+ * scope; systemd then removes the cgroup as soon as it is empty, and the
921
+ * snapshot above can find nothing at all. `Result=oom-kill` survives that —
922
+ * it is read a few milliseconds later from the unit, which is why
923
+ * {@link releaseSessionScope} is now awaited before a death is explained.
924
+ */
925
+ function rememberDeathFromResult(id) {
926
+ if (deaths.has(id))
927
+ return;
928
+ deaths.set(id, {
929
+ oomKills: 1,
930
+ maxBytes: sessionCage().memoryMaxBytes,
931
+ // systemd stopped the unit for an OOM inside it; which limit was hit is not
932
+ // in `Result`, and «your own wall» is the claim that must not be guessed.
933
+ ownWall: false,
934
+ });
935
+ }
936
+ /**
937
+ * One sentence for the error the person reads, when the kernel had a hand in
938
+ * this death — or null when it had not. Consumed: a session restarted after an
939
+ * OOM must not carry the old sentence into its next, unrelated failure.
940
+ *
941
+ * The text that reached people in 0.54.0 was «exited with code 143» once and
942
+ * «terminated by signal SIGKILL» the next time, for the same cause, and neither
943
+ * said the word memory. This is that word.
944
+ */
945
+ export function explainMemoryDeath(id) {
946
+ const death = deaths.get(id);
947
+ if (!death)
948
+ return null;
949
+ deaths.delete(id);
950
+ const killed = death.oomKills === 1 ? 'a process' : `${death.oomKills} processes`;
951
+ const wall = death.maxBytes === null ? '' : ` of ${Math.round(death.maxBytes / MIB)} MB`;
952
+ if (death.ownWall) {
953
+ return (`The session ran out of memory: it went over its own memory ceiling${wall} and the kernel killed ` +
954
+ `${killed} inside it. ` +
955
+ 'Resume it and avoid re-running the command that was in flight, or give the machine more memory or swap.');
956
+ }
957
+ return (`The machine ran out of memory for agent sessions — all sessions on it share one ceiling — and the kernel killed ` +
958
+ `${killed} in this one. This session was not necessarily the greedy one. ` +
959
+ 'Resume it; if it keeps happening, run fewer sessions at once or give the machine more memory or swap.');
960
+ }
961
+ const realSystemctl = async (args) => {
962
+ const { stdout, stderr } = await execFileAsync('systemctl', ['--user', ...args], {
963
+ timeout: 15_000,
964
+ env: systemdUserEnv(),
965
+ });
966
+ return { stdout, stderr };
967
+ };
968
+ function showValue(stdout, property) {
969
+ const line = stdout
970
+ .split('\n')
971
+ .map((l) => l.trim())
972
+ .find((l) => l.startsWith(`${property}=`));
973
+ return line ? line.slice(property.length + 1) : null;
974
+ }
975
+ /**
976
+ * Read why the scope ended, say so, and then let systemd forget it.
977
+ *
978
+ * This is the whole reason `--collect` is not passed. The order is fixed: the
979
+ * process has already exited, `Result` is read, the reason is logged, and only
980
+ * then is the unit reset — because `reset-failed` is what deletes the answer.
981
+ *
982
+ * Failure here is never fatal: a unit that could not be reset is swept at the
983
+ * next daemon start, and the counter in {@link cageSpawn} keeps the session
984
+ * startable in the meantime.
985
+ */
986
+ export async function releaseSessionScope(unit, id, systemctl = realSystemctl) {
987
+ const running = releaseSessionScopeInner(unit, id, systemctl);
988
+ if (id !== undefined) {
989
+ releasing.set(id, running);
990
+ void running.finally(() => {
991
+ if (releasing.get(id) === running)
992
+ releasing.delete(id);
993
+ });
994
+ }
995
+ return await running;
996
+ }
997
+ /**
998
+ * Releases in flight, per cage id, so a death can be explained with BOTH
999
+ * sources: the cgroup snapshot (synchronous, gone the moment systemd prunes the
1000
+ * group) and `Result=oom-kill` (authoritative, and a few milliseconds late).
1001
+ * Without the wait, the machine that needs the sentence most — the one whose
1002
+ * systemd stops the whole scope — was the one that never got it (#387 QA).
1003
+ */
1004
+ const releasing = new Map();
1005
+ /**
1006
+ * The sentence for a death, once the verdict is in. Null when the kernel had no
1007
+ * hand in it. Waits for the release of this session's scope, but never longer
1008
+ * than `capMs`: an answer that arrives after the person has read the error is
1009
+ * worth nothing, and a hung `systemctl` must not hold a failing session open.
1010
+ */
1011
+ export async function memoryDeathSentence(id, capMs = 3_000) {
1012
+ const pending = releasing.get(id);
1013
+ if (pending) {
1014
+ let timer;
1015
+ await Promise.race([
1016
+ pending.catch(() => null),
1017
+ new Promise((resolve) => {
1018
+ timer = setTimeout(resolve, capMs);
1019
+ timer.unref?.();
1020
+ }),
1021
+ ]);
1022
+ if (timer)
1023
+ clearTimeout(timer);
1024
+ }
1025
+ return explainMemoryDeath(id);
1026
+ }
1027
+ async function releaseSessionScopeInner(unit, id, systemctl) {
1028
+ if (!unit)
1029
+ return null;
1030
+ // Synchronously and FIRST: the cgroup's own counters are the one record of an
1031
+ // OOM kill that survives `OOMPolicy=continue` (the unit does not fail, so
1032
+ // `Result` says nothing), and the directory goes away the moment the scope is
1033
+ // empty. This is what the adapter's error text is built from (#387).
1034
+ if (id !== undefined)
1035
+ rememberDeath(id, unit);
1036
+ if (id !== undefined)
1037
+ liveUnits.delete(id);
1038
+ let result = null;
1039
+ try {
1040
+ const { stdout } = await systemctl(['show', unit, '-p', 'Result']);
1041
+ result = showValue(stdout, 'Result');
1042
+ }
1043
+ catch {
1044
+ // Already collected — a clean exit takes its scope with it.
1045
+ }
1046
+ if (result === 'oom-kill') {
1047
+ log.warn('session cage: the kernel killed this session for going over its memory ceiling', {
1048
+ unit,
1049
+ memoryMaxMB: Math.round((sessionCage().memoryMaxBytes ?? 0) / MIB),
1050
+ });
1051
+ // The cgroup snapshot above may have found nothing: on a systemd that stops
1052
+ // the whole scope, the group is gone by the time the process's `exit` fires.
1053
+ if (id !== undefined)
1054
+ rememberDeathFromResult(id);
1055
+ }
1056
+ let forgotten = false;
1057
+ try {
1058
+ await systemctl(['reset-failed', unit]);
1059
+ forgotten = true;
1060
+ }
1061
+ catch {
1062
+ // Nothing to reset is the normal case — but so is «the bus was not there»,
1063
+ // and those two look identical from here.
1064
+ }
1065
+ // Only when the unit is provably gone. Clearing the counter after a FAILED
1066
+ // `reset-failed` hands the next start of this session the same unit name,
1067
+ // which systemd answers with «was already loaded or has a fragment file» and
1068
+ // spawns nothing — the exact failure the counter exists to prevent
1069
+ // (QA-2026-09-07 MINOR-11).
1070
+ if (id !== undefined && forgotten)
1071
+ attempts.delete(id);
1072
+ return result;
1073
+ }
1074
+ /** Unit names of every `devbridge-session-*.scope` systemd still knows about. */
1075
+ export async function listSessionScopeUnits(systemctl = realSystemctl) {
1076
+ let stdout;
1077
+ try {
1078
+ ({ stdout } = await systemctl([
1079
+ 'list-units',
1080
+ '--all',
1081
+ '--plain',
1082
+ '--no-legend',
1083
+ `${SESSION_SCOPE_PREFIX}*.scope`,
1084
+ ]));
1085
+ }
1086
+ catch {
1087
+ return [];
1088
+ }
1089
+ return stdout
1090
+ .split('\n')
1091
+ .map((line) => line.trim().split(/\s+/)[0] ?? '')
1092
+ .filter((unit) => unit.startsWith(SESSION_SCOPE_PREFIX) && unit.endsWith('.scope'));
1093
+ }
1094
+ /** What `doctor` prints for each caged session. */
1095
+ export async function listSessionScopes(systemctl = realSystemctl) {
1096
+ const units = await listSessionScopeUnits(systemctl);
1097
+ const out = [];
1098
+ for (const unit of units) {
1099
+ let stdout = '';
1100
+ try {
1101
+ ({ stdout } = await systemctl([
1102
+ 'show',
1103
+ unit,
1104
+ '-p',
1105
+ 'MemoryMax',
1106
+ '-p',
1107
+ 'MemoryCurrent',
1108
+ '-p',
1109
+ 'TasksCurrent',
1110
+ '-p',
1111
+ 'Result',
1112
+ '-p',
1113
+ 'ActiveState',
1114
+ ]));
1115
+ }
1116
+ catch {
1117
+ // Gone between the list and the read — report the name and nothing else.
1118
+ }
1119
+ const num = (property) => {
1120
+ const value = Number(showValue(stdout, property));
1121
+ return Number.isFinite(value) ? value : null;
1122
+ };
1123
+ out.push({
1124
+ unit,
1125
+ memoryMaxBytes: num('MemoryMax'),
1126
+ memoryCurrentBytes: num('MemoryCurrent'),
1127
+ tasksCurrent: num('TasksCurrent'),
1128
+ result: showValue(stdout, 'Result'),
1129
+ activeState: showValue(stdout, 'ActiveState'),
1130
+ });
1131
+ }
1132
+ return out;
1133
+ }
1134
+ /**
1135
+ * Stop and forget every session scope that has no session behind it.
1136
+ *
1137
+ * At daemon start that is all of them by definition, and it is the point: a
1138
+ * scope outlives a killed daemon carrying the whole process tree with it, which
1139
+ * is the shape of the 10 h 51 min `ugrep` of 16.08. Stopping the scope takes the
1140
+ * tree, not just the process we happened to know about.
1141
+ *
1142
+ * `liveIds` exists so the same sweep can run later without killing work in
1143
+ * progress; only the CURRENT scope of a live session is spared, because an
1144
+ * earlier attempt of the same session is exactly the leftover we are here for.
1145
+ */
1146
+ export async function sweepOrphanSessionScopes(liveIds = [], systemctl = realSystemctl) {
1147
+ const spared = new Set();
1148
+ for (const id of liveIds)
1149
+ spared.add(sessionScopeUnit(id, attempts.get(id) ?? 1));
1150
+ const removed = [];
1151
+ for (const unit of await listSessionScopeUnits(systemctl)) {
1152
+ if (spared.has(unit))
1153
+ continue;
1154
+ try {
1155
+ await systemctl(['stop', unit]);
1156
+ }
1157
+ catch {
1158
+ // A failed scope has nothing to stop; `reset-failed` below is the part
1159
+ // that matters for it.
1160
+ }
1161
+ try {
1162
+ await systemctl(['reset-failed', unit]);
1163
+ }
1164
+ catch {
1165
+ // Already gone.
1166
+ }
1167
+ removed.push(unit);
1168
+ }
1169
+ if (removed.length > 0) {
1170
+ log.warn('session cage: removed scopes left behind by a previous run', {
1171
+ count: removed.length,
1172
+ units: removed.slice(0, 10),
1173
+ });
1174
+ }
1175
+ return removed;
1176
+ }
1177
+ /**
1178
+ * The slice names and the CPU share — re-exported so a caller that reasons about
1179
+ * the cage needs one import, while the values themselves stay next to the
1180
+ * drop-in that writes them (`service-unit.ts`).
1181
+ */
1182
+ export { SESSIONS_SLICE, DEVBRIDGE_SLICE, SESSION_CPU_WEIGHT };
1183
+ //# sourceMappingURL=session-cage.js.map