@bridge4dev/runner 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +105 -2
- package/dist/adapters/codex-protocol.d.ts +11 -0
- package/dist/adapters/codex-protocol.js +41 -3
- package/dist/adapters/codex.js +37 -0
- package/dist/host-load.d.ts +156 -0
- package/dist/host-load.js +223 -0
- package/dist/index.js +211 -40
- package/dist/process-priority.d.ts +55 -0
- package/dist/process-priority.js +99 -0
- package/dist/protocol.d.ts +25 -3
- package/dist/recipe-schema.d.ts +6 -6
- package/dist/self-update.js +43 -2
- package/dist/service-unit.d.ts +268 -10
- package/dist/service-unit.js +432 -53
- package/dist/session-cage.d.ts +508 -0
- package/dist/session-cage.js +1183 -0
- package/dist/supervisor.d.ts +106 -0
- package/dist/supervisor.js +243 -0
- package/dist/systemd-memory.d.ts +35 -0
- package/dist/systemd-memory.js +115 -0
- package/dist/verify.js +28 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,1183 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import crypto from 'node:crypto';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { promisify } from 'node:util';
|
|
7
|
+
import { runnerIdentity, systemdUserEnv } from './environment.js';
|
|
8
|
+
import { log } from './log.js';
|
|
9
|
+
import { DEVBRIDGE_SLICE, memoryPolicy, readMemoryFacts, readSelfCgroup, readSwapTotalBytes, SESSION_CPU_WEIGHT, SESSIONS_SLICE, SESSIONS_SWAP_SHARE, sliceCgroupPath, } from './service-unit.js';
|
|
10
|
+
const execFileAsync = promisify(execFile);
|
|
11
|
+
/**
|
|
12
|
+
* A cgroup of its own for every agent session — the memory ceiling that
|
|
13
|
+
* `process-priority.ts` cannot give.
|
|
14
|
+
*
|
|
15
|
+
* Stage 1a (`process-priority.ts`) orders the CPU and nothing else. Memory is
|
|
16
|
+
* the mechanism behind three of the four incidents on other people's machines:
|
|
17
|
+
* one session allocates until the machine swaps itself to a standstill, and the
|
|
18
|
+
* only thing that ever stopped it was the ceiling on the WHOLE service — which
|
|
19
|
+
* kills whichever child the kernel picks, not the one that misbehaved. This
|
|
20
|
+
* module puts each session in `devbridge-session-<id>.scope` under
|
|
21
|
+
* `devbridge-sessions.slice`, so the session that runs out of memory is the
|
|
22
|
+
* session that dies, and its neighbours never notice.
|
|
23
|
+
*
|
|
24
|
+
* Every line below is a finding of the spike of 2026-09-07 (plan
|
|
25
|
+
* `agent-sessions-host-resources.md` §5.4.1), not a preference. The ones that
|
|
26
|
+
* cost the most to learn:
|
|
27
|
+
*
|
|
28
|
+
* - `MemorySwapMax` is BOUNDED, never unset. Without a bound there is no cage:
|
|
29
|
+
* a process under `MemoryMax=200M` allocated 2 GB and drained the host's
|
|
30
|
+
* entire swap (`memory.events` read `max 0, oom_kill 0`, swap 2047/2047 MB)
|
|
31
|
+
* — the exact incident this whole plan exists to prevent, reproduced by the
|
|
32
|
+
* supposed fix. `MemoryMax` bounds RESIDENT memory; the rest goes to swap,
|
|
33
|
+
* and swap belongs to the machine, not to the cgroup. 0.54.0 wrote `0`;
|
|
34
|
+
* since #387 it is a SHARE of the machine's swap (see
|
|
35
|
+
* {@link sessionSwapMaxBytes}), because without any swap the brake below
|
|
36
|
+
* has nothing to push pages into and stalls instead of slowing.
|
|
37
|
+
* - `MemoryHigh` UNDER `MemoryMax`, and far under it (#387). 0.54.0 shipped
|
|
38
|
+
* the hard line alone, at 2.5 GiB, with systemd's default `OOMPolicy=stop`:
|
|
39
|
+
* an honest `pnpm typecheck` (seven `tsc` at ~400 MB each) went over it, the
|
|
40
|
+
* kernel killed ONE `tsc`, and systemd then stopped the WHOLE scope — the
|
|
41
|
+
* session died with «exited with code 143», twice in an hour, on correct
|
|
42
|
+
* work. The owner's rule is that honest work never reaches a kill: the
|
|
43
|
+
* honest share is where the brake starts, the wall is the machine's measured
|
|
44
|
+
* headroom, and only a real runaway ever gets that far — and then
|
|
45
|
+
* `OOMPolicy=continue` costs it the one process, not the session.
|
|
46
|
+
* The spike's «60 s in, still alive at 100 MB» under `MemoryHigh` was a
|
|
47
|
+
* runaway with NO swap to be pushed into — that is the kernel's penalty
|
|
48
|
+
* (up to 2 s per 256 KB when nothing can be reclaimed), and it is why the
|
|
49
|
+
* swap share above is part of the same change, not a separate one.
|
|
50
|
+
* - no `--collect`. A scope killed by the OOM killer stays behind in `failed`,
|
|
51
|
+
* and `systemctl --user show <unit> -p Result` is the ONLY place the cause
|
|
52
|
+
* can be read: `Result=oom-kill`. `--collect` deletes the unit the instant it
|
|
53
|
+
* dies and takes the reason with it. Hence the order in
|
|
54
|
+
* {@link releaseSessionScope}: wait for the exit, read `Result`, log it, THEN
|
|
55
|
+
* `reset-failed`.
|
|
56
|
+
* - `--expand-environment=no`, but ONLY on systemd ≥ 254. The arguments carry
|
|
57
|
+
* the agent's prompt, and prompts contain `$`; without this flag a future
|
|
58
|
+
* systemd would substitute variables into the user's own words. systemd 254
|
|
59
|
+
* is where the switch was added, and it is also where its own NEWS says
|
|
60
|
+
* expansion «defaults to enabled for all execution types except `--scope`,
|
|
61
|
+
* where it defaults to off … for backward compatibility reasons». So on
|
|
62
|
+
* Ubuntu 22.04 (249), Debian 12 and RHEL 9 (252) the flag is both rejected
|
|
63
|
+
* (`unrecognized option`, exit 1, nothing spawned) and unnecessary. Passing
|
|
64
|
+
* it unconditionally cost those machines the cage entirely
|
|
65
|
+
* (QA-2026-09-07 MAJOR-2).
|
|
66
|
+
* - `--pty` is not compatible with `--scope` ("--pty/--pipe is not compatible
|
|
67
|
+
* in timer or --scope mode."), and does not need to be: the sign-in relay's
|
|
68
|
+
* `script -qec` runs INSIDE the scope byte for byte the same. `auth-relay.ts`
|
|
69
|
+
* is untouched by this module.
|
|
70
|
+
* - `CPUWeight` is set on the scope AND on `devbridge.slice`
|
|
71
|
+
* (`service-unit.ts`). The moment sessions leave the service's own cgroup,
|
|
72
|
+
* `nice(2)` stops protecting the daemon from its own children — nice orders
|
|
73
|
+
* tasks WITHIN one cgroup, and across cgroups `cpu.weight` decides, which is
|
|
74
|
+
* 100 everywhere by default. A cage without that drop-in would quietly undo
|
|
75
|
+
* stage 1a and bring back 16.08: daemon starved, four missed pongs, server
|
|
76
|
+
* Offline, 504 on every session.
|
|
77
|
+
*
|
|
78
|
+
* Rejected in the spike: `Delegate=yes` on the service plus the runner sorting
|
|
79
|
+
* PIDs into child cgroups itself. cgroup v2's "no internal processes" rule
|
|
80
|
+
* answers `Device or resource busy` on `cgroup.subtree_control`, the way around
|
|
81
|
+
* it moves the daemon into a sub-cgroup and needs a unit change and a RESTART
|
|
82
|
+
* (which kills every live session), and the spawn-then-move order lost the race
|
|
83
|
+
* anyway — the runaway was 100–150 MB in before it ever reached the cage.
|
|
84
|
+
*/
|
|
85
|
+
const MIB = 1024 * 1024;
|
|
86
|
+
const GIB = 1024 * MIB;
|
|
87
|
+
/**
|
|
88
|
+
* The honest share of one session — where the BRAKE starts, not where it dies.
|
|
89
|
+
*
|
|
90
|
+
* Until #387 this was `MemoryMax`, the kill line. It is the same number and the
|
|
91
|
+
* same reasoning, moved one step down the ladder: above it the kernel reclaims
|
|
92
|
+
* and throttles (`MemoryHigh`), and the kill line is {@link sessionMemoryMaxBytes}.
|
|
93
|
+
*
|
|
94
|
+
* `min(2.5 GiB, service ceiling / 2)` — §3 of
|
|
95
|
+
* `docs/plans/shipped/agent-sessions-host-resources.md`. Both halves matter:
|
|
96
|
+
*
|
|
97
|
+
* - the absolute number is sized against the measurement, not against a round
|
|
98
|
+
* figure. The heaviest ordinary thing a session runs is a workspace
|
|
99
|
+
* `pnpm typecheck`, measured at a 1571 MB peak; three live `claude`
|
|
100
|
+
* processes are 512/577/646 MB. 2.5 GiB leaves ~60 % headroom over the
|
|
101
|
+
* worst measured case, which is the margin between "kills a runaway" and
|
|
102
|
+
* "kills the work";
|
|
103
|
+
* - the half-of-the-service half is what keeps ONE session from being able to
|
|
104
|
+
* fill the ceiling that covers all of them. On this machine the service
|
|
105
|
+
* ceiling is 7680M, so half is 3.75 GiB and the absolute number wins.
|
|
106
|
+
*
|
|
107
|
+
* The service ceiling is read off systemd (`systemctl --user show
|
|
108
|
+
* devbridge-runner -p MemoryMax`) rather than recomputed here, because what
|
|
109
|
+
* protects the machine is what systemd has in force, not what the drop-in on
|
|
110
|
+
* disk says — those two have already drifted apart once (§5.5 of the plan).
|
|
111
|
+
*/
|
|
112
|
+
export const SESSION_MEMORY_HIGH_ABSOLUTE_BYTES = Math.round(2.5 * GIB);
|
|
113
|
+
/**
|
|
114
|
+
* Processes and threads per session.
|
|
115
|
+
*
|
|
116
|
+
* An agent forks a lot (a monorepo build is hundreds of short-lived processes),
|
|
117
|
+
* so this is not a memory limit in disguise — it is the fork-bomb stop. 512 is
|
|
118
|
+
* far above anything measured on a real session and far below the 8192 the
|
|
119
|
+
* whole service gets.
|
|
120
|
+
*/
|
|
121
|
+
export const SESSION_TASKS_MAX = 512;
|
|
122
|
+
/** Prefix of every scope this module creates. The sweeper matches on it. */
|
|
123
|
+
export const SESSION_SCOPE_PREFIX = 'devbridge-session-';
|
|
124
|
+
/**
|
|
125
|
+
* The FLOOR the plan's formula does not have, and the reason it needs one.
|
|
126
|
+
*
|
|
127
|
+
* `min(2.5 GiB, ceiling / 2)` is right on a big machine and wrong on a small
|
|
128
|
+
* one: at the 2 GiB ceiling `memoryPolicy` gives the smallest supported box,
|
|
129
|
+
* half is 1 GiB — BELOW the 1571 MB a workspace `pnpm typecheck` was measured
|
|
130
|
+
* at. The cage would then kill honest work on exactly the machines least able
|
|
131
|
+
* to afford a failed run, and it would look like a flaky agent rather than a
|
|
132
|
+
* limit. A containment that fires on correct work is not containment.
|
|
133
|
+
*
|
|
134
|
+
* So the floor is set above the worst measured ordinary peak, not at a round
|
|
135
|
+
* number.
|
|
136
|
+
*
|
|
137
|
+
* It is a floor and not a switch: below ~2.4 GB of RAM the containing ceiling
|
|
138
|
+
* itself drops under this number, and there the last clamp in
|
|
139
|
+
* {@link sessionMemoryMaxBytes} wins and a session is capped below the floor.
|
|
140
|
+
* That is not the cage killing honest work — the slice would have killed the
|
|
141
|
+
* same `pnpm typecheck` a moment later anyway, because it holds the same
|
|
142
|
+
* number — so removing `MemoryMax` from the scope there would buy nothing and
|
|
143
|
+
* cost the one thing it does buy: the runaway dying instead of its neighbours
|
|
144
|
+
* (QA-2026-09-07 MINOR-6, where the earlier wording promised the opposite).
|
|
145
|
+
*/
|
|
146
|
+
export const SESSION_MEMORY_HIGH_MIN_BYTES = 2 * GIB;
|
|
147
|
+
/**
|
|
148
|
+
* The wall, when nothing on the machine can say where it should be.
|
|
149
|
+
*
|
|
150
|
+
* Every real path reads the wall off systemd (the slice, else the service) or
|
|
151
|
+
* measures it (`memoryPolicy`). This is the last resort for a machine that gave
|
|
152
|
+
* neither — twice the honest share, so that even blind the wall sits above the
|
|
153
|
+
* brake by a margin an honest build fits into, and a runaway still meets one.
|
|
154
|
+
*/
|
|
155
|
+
export const SESSION_MEMORY_WALL_FALLBACK_FACTOR = 2;
|
|
156
|
+
/**
|
|
157
|
+
* The pool is cut into this many honest shares. Three, because that is what
|
|
158
|
+
* `maxSessions` defaults to on a dev server, so three sessions can sit at their
|
|
159
|
+
* brakes without the pool itself overflowing — and one runaway then meets its
|
|
160
|
+
* own wall (`ceiling − share`) while a neighbour on its share still fits.
|
|
161
|
+
*/
|
|
162
|
+
export const SESSION_SHARE_DIVISOR = 3;
|
|
163
|
+
/**
|
|
164
|
+
* On a machine too small to leave a share of room, the brake is this fraction
|
|
165
|
+
* of the wall. The same 0.8 `memoryPolicy` puts between the service's own
|
|
166
|
+
* `MemoryHigh` and `MemoryMax`, for the same reason: a band, however narrow,
|
|
167
|
+
* beats a bare kill line.
|
|
168
|
+
*/
|
|
169
|
+
export const SESSION_BRAKE_OF_WALL = 0.8;
|
|
170
|
+
/**
|
|
171
|
+
* `clamp(ceiling / 2, 2 GiB, 2.5 GiB)`, never above the ceiling itself.
|
|
172
|
+
*
|
|
173
|
+
* Three bounds, each for its own reason — see
|
|
174
|
+
* {@link SESSION_MEMORY_MAX_ABSOLUTE_BYTES} for the upper two and
|
|
175
|
+
* {@link SESSION_MEMORY_MIN_BYTES} for the lower one. The last clamp matters
|
|
176
|
+
* on a tiny machine: a per-session number ABOVE the slice's own ceiling is not
|
|
177
|
+
* a limit at all, it is a bigger number that never applies, and printing it in
|
|
178
|
+
* `doctor` would be a straight lie about what is in force.
|
|
179
|
+
*
|
|
180
|
+
* The argument is the ceiling of the cgroup that CONTAINS the session, and
|
|
181
|
+
* since 0.54.0 that is `devbridge-sessions.slice` and not the service — the
|
|
182
|
+
* caller reads the slice and falls back to the service only when systemd has
|
|
183
|
+
* nothing to say about the slice. The two are the same number by construction
|
|
184
|
+
* today; they come apart the moment a drop-in fails to apply or is edited by
|
|
185
|
+
* hand, and then «never more than the slice» has to still be true
|
|
186
|
+
* (QA-2026-09-07 MINOR-5).
|
|
187
|
+
*/
|
|
188
|
+
export function sessionMemoryHighBytes(containingMemoryMaxBytes) {
|
|
189
|
+
return sessionMemoryLadder(containingMemoryMaxBytes).highBytes;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* The two numbers of the cage, computed in ONE place because each is wrong
|
|
193
|
+
* without the other (QA of #387 found both halves broken separately).
|
|
194
|
+
*
|
|
195
|
+
* ```
|
|
196
|
+
* brake = max(containing / 3, 2 GiB) capped by the containing ceiling
|
|
197
|
+
* wall = containing − brake never at or below the brake
|
|
198
|
+
* ```
|
|
199
|
+
*
|
|
200
|
+
* **Why a third, and no upper cap.** The owner's second note on #387: «ночью
|
|
201
|
+
* машина свободна (~9 ГБ), а сессии всё равно нельзя выйти за 2.5 ГБ… граница
|
|
202
|
+
* должна смотреть на то, сколько реально свободно». A fixed 2.5 GiB was right
|
|
203
|
+
* as a KILL line and is wrong as a brake: on a big machine it throttles honest
|
|
204
|
+
* work that the machine could have absorbed. A third of the pool is the largest
|
|
205
|
+
* share that still lets three sessions sit at their brakes inside the ceiling,
|
|
206
|
+
* which is what `maxSessions` defaults to. The 2 GiB floor stays — it is sized
|
|
207
|
+
* over the 1571 MB measured peak of a workspace `pnpm typecheck`, and a dev
|
|
208
|
+
* server that cannot run one of those is not a dev server.
|
|
209
|
+
*
|
|
210
|
+
* **Why the wall is `containing − brake` and not the ceiling itself.** 0.55.0's
|
|
211
|
+
* first shape put the wall AT the slice ceiling, and that is not a per-session
|
|
212
|
+
* wall at all: the kernel charges a leaf and every ancestor, so with any
|
|
213
|
+
* neighbour the SLICE overflows first, and then the victim is chosen across the
|
|
214
|
+
* whole slice. Measured on this host (a 300 MB slice, two 250 MB scopes): the
|
|
215
|
+
* scope that crept up survived to the end, and the **neighbour holding 200 MB
|
|
216
|
+
* under its own wall was killed** — slice `max 19, oom 1, oom_kill 1`, victim
|
|
217
|
+
* `max 0, oom 0, oom_kill 1`. Leaving one honest share of room under the
|
|
218
|
+
* ceiling is what makes the runaway meet its OWN wall first while a neighbour
|
|
219
|
+
* on its share still fits.
|
|
220
|
+
*
|
|
221
|
+
* **The tiny machine.** When the ceiling is so low that `containing − brake`
|
|
222
|
+
* would land at or under the brake, there is no room for isolation at all: the
|
|
223
|
+
* wall becomes the ceiling and the BRAKE is lowered to 80 % of it, so that
|
|
224
|
+
* there is still a braking band (`memoryPolicy` uses the same 0.8 for the
|
|
225
|
+
* service). Without this, a 4 GB machine got `MemoryHigh == MemoryMax` — the
|
|
226
|
+
* 0.54.0 kill line back in place, with the card claiming a brake.
|
|
227
|
+
*/
|
|
228
|
+
export function sessionMemoryLadder(containingMemoryMaxBytes, measuredCeilingBytes = null) {
|
|
229
|
+
const containing = [containingMemoryMaxBytes, measuredCeilingBytes].find((value) => value !== null && Number.isFinite(value) && value > 0);
|
|
230
|
+
if (containing === undefined) {
|
|
231
|
+
// Nothing on this machine could say what contains the session. The
|
|
232
|
+
// reference share, and a wall well above it, so the ladder still exists.
|
|
233
|
+
return {
|
|
234
|
+
highBytes: SESSION_MEMORY_HIGH_ABSOLUTE_BYTES,
|
|
235
|
+
maxBytes: SESSION_MEMORY_HIGH_ABSOLUTE_BYTES * SESSION_MEMORY_WALL_FALLBACK_FACTOR,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
const ceiling = Math.floor(containing);
|
|
239
|
+
const share = Math.max(Math.floor(ceiling / SESSION_SHARE_DIVISOR), SESSION_MEMORY_HIGH_MIN_BYTES);
|
|
240
|
+
// Never promise a session more than the cgroup that contains every session.
|
|
241
|
+
const brake = Math.max(1, Math.min(share, ceiling));
|
|
242
|
+
const wall = ceiling - brake;
|
|
243
|
+
if (wall <= brake) {
|
|
244
|
+
return {
|
|
245
|
+
highBytes: Math.max(1, Math.floor(ceiling * SESSION_BRAKE_OF_WALL)),
|
|
246
|
+
maxBytes: ceiling,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
return { highBytes: brake, maxBytes: wall };
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* The wall — `MemoryMax` on the scope — and why it is the WHOLE containing
|
|
253
|
+
* ceiling and not a fraction of it (#387).
|
|
254
|
+
*
|
|
255
|
+
* The owner's rule: the wall is protection against a runaway, not the norm of
|
|
256
|
+
* work. Where the norm lives is {@link sessionMemoryHighBytes}; the wall has to
|
|
257
|
+
* be far enough above it that honest work never touches it, and «far» on a dev
|
|
258
|
+
* server means «what the machine can actually spare» — which is exactly the
|
|
259
|
+
* number the slice ceiling already is (`memoryPolicy`: `MemAvailable` plus what
|
|
260
|
+
* we hold, minus a reserve). Giving a session less than that is the 2.5 GiB
|
|
261
|
+
* mistake with a different number.
|
|
262
|
+
*
|
|
263
|
+
* With `OOMPolicy=continue` on the scope, reaching the wall costs a runaway its
|
|
264
|
+
* fattest process and nothing else; the neighbours are protected by the slice,
|
|
265
|
+
* which holds the same number over all of them together.
|
|
266
|
+
*
|
|
267
|
+
* `containing` is what systemd has in force on the slice (else the service);
|
|
268
|
+
* `measured` is what the policy would write there; the factor is the last
|
|
269
|
+
* resort. Never below the brake — a wall under the brake is a kill line again.
|
|
270
|
+
*/
|
|
271
|
+
export function sessionMemoryMaxBytes(containingMemoryMaxBytes, measuredCeilingBytes = null) {
|
|
272
|
+
return sessionMemoryLadder(containingMemoryMaxBytes, measuredCeilingBytes).maxBytes;
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Swap one session may push into — the reason the brake slows instead of stalls.
|
|
276
|
+
*
|
|
277
|
+
* Half of what the slice may use, the same «no single session takes the
|
|
278
|
+
* collective allowance» rule as the memory share. When systemd reports no
|
|
279
|
+
* bound on the slice (the drop-in never landed, MAJOR-3 shape) the share is cut
|
|
280
|
+
* from the machine's `SwapTotal` directly, with the same fraction the slice
|
|
281
|
+
* would have had, so the per-scope line is a real bound on its own. A machine
|
|
282
|
+
* with no swap gets 0, which is exactly 0.54.0's line — and exactly right:
|
|
283
|
+
* there is nothing to share.
|
|
284
|
+
*/
|
|
285
|
+
export function sessionSwapMaxBytes(sliceSwapMaxBytes, hostSwapTotalBytes) {
|
|
286
|
+
if (sliceSwapMaxBytes !== null && Number.isFinite(sliceSwapMaxBytes)) {
|
|
287
|
+
return Math.max(0, Math.floor(sliceSwapMaxBytes / 2));
|
|
288
|
+
}
|
|
289
|
+
if (hostSwapTotalBytes !== null && Number.isFinite(hostSwapTotalBytes)) {
|
|
290
|
+
return Math.max(0, Math.floor((hostSwapTotalBytes * SESSIONS_SWAP_SHARE) / 2));
|
|
291
|
+
}
|
|
292
|
+
return 0;
|
|
293
|
+
}
|
|
294
|
+
// ─── unit names ──────────────────────────────────────────────────────
|
|
295
|
+
/**
|
|
296
|
+
* Room for the id inside a unit name. systemd's own limit is 255 bytes for the
|
|
297
|
+
* whole name; the prefix, the `.scope` suffix, a restart marker and a hash all
|
|
298
|
+
* come out of the same budget, so the id is cut well short of it.
|
|
299
|
+
*/
|
|
300
|
+
const MAX_ID_IN_UNIT = 180;
|
|
301
|
+
/**
|
|
302
|
+
* The session id as systemd will accept it.
|
|
303
|
+
*
|
|
304
|
+
* Ids are UUIDs today and arbitrary text tomorrow — they arrive from the API,
|
|
305
|
+
* and `verify` keys its cage by a run id. systemd unit names take only
|
|
306
|
+
* `[A-Za-z0-9:_.-]`, so everything else is folded to `-`. Folding is lossy, and
|
|
307
|
+
* lossy is how two different sessions end up fighting over one scope, so
|
|
308
|
+
* anything that had to be changed or cut also gets eight hex digits of the
|
|
309
|
+
* original — deterministic, so the same session always names the same unit and
|
|
310
|
+
* the sweeper can still recognise it.
|
|
311
|
+
*/
|
|
312
|
+
export function sanitizeCageId(id) {
|
|
313
|
+
const folded = id.replace(/[^A-Za-z0-9:_.-]/g, '-').replace(/^[.-]+/, '');
|
|
314
|
+
const cut = folded.slice(0, MAX_ID_IN_UNIT);
|
|
315
|
+
if (cut === id && cut.length > 0)
|
|
316
|
+
return cut;
|
|
317
|
+
const digest = crypto.createHash('sha256').update(id).digest('hex').slice(0, 8);
|
|
318
|
+
return `${cut || 'session'}-${digest}`;
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* The scope unit for one START of one session.
|
|
322
|
+
*
|
|
323
|
+
* `attempt` is not decoration. A scope the OOM killer took stays loaded in
|
|
324
|
+
* `failed` until something resets it, and `systemd-run --unit=` on a name that
|
|
325
|
+
* is still loaded fails outright — measured: «Unit devbridge-….scope was
|
|
326
|
+
* already loaded or has a fragment file», exit 1, nothing spawned. So a session
|
|
327
|
+
* that is restarted after an OOM (which is exactly when it IS restarted) would
|
|
328
|
+
* be unable to start at all if the name never changed.
|
|
329
|
+
* {@link releaseSessionScope} clears the failed unit and only then lets the
|
|
330
|
+
* counter fall back to 1, so the plain `devbridge-session-<id>.scope` is the
|
|
331
|
+
* normal case and the marker appears only while the old scope is still there.
|
|
332
|
+
*/
|
|
333
|
+
export function sessionScopeUnit(id, attempt = 1) {
|
|
334
|
+
const base = `${SESSION_SCOPE_PREFIX}${sanitizeCageId(id)}`;
|
|
335
|
+
return attempt <= 1 ? `${base}.scope` : `${base}-r${attempt}.scope`;
|
|
336
|
+
}
|
|
337
|
+
// ─── capability detection ────────────────────────────────────────────
|
|
338
|
+
/** cgroup v2 unified, from `statfs` — the number behind `stat -fc %T`. */
|
|
339
|
+
const CGROUP2_SUPER_MAGIC = 0x63677270;
|
|
340
|
+
const TMPFS_MAGIC = 0x01021994;
|
|
341
|
+
/** What the live probe must read back out of its own cgroup: 64 MiB, exactly. */
|
|
342
|
+
const PROBE_MEMORY_MAX = 64 * MIB;
|
|
343
|
+
/**
|
|
344
|
+
* The systemd release that added `--expand-environment=`.
|
|
345
|
+
*
|
|
346
|
+
* Below it `systemd-run` answers `unrecognized option` and exits 1 — the probe
|
|
347
|
+
* then reads as «the cage did not hold» on a machine whose cgroups are perfect
|
|
348
|
+
* (Ubuntu 22.04 is 249, Debian 12 and RHEL 9 are 252). See the module header for
|
|
349
|
+
* why the flag is not needed there either.
|
|
350
|
+
*/
|
|
351
|
+
export const EXPAND_ENVIRONMENT_MIN_SYSTEMD = 254;
|
|
352
|
+
function statfsType(target) {
|
|
353
|
+
try {
|
|
354
|
+
const type = fs.statfsSync(target).type;
|
|
355
|
+
if (type === CGROUP2_SUPER_MAGIC)
|
|
356
|
+
return 'cgroup2fs';
|
|
357
|
+
if (type === TMPFS_MAGIC)
|
|
358
|
+
return 'tmpfs';
|
|
359
|
+
return `0x${type.toString(16)}`;
|
|
360
|
+
}
|
|
361
|
+
catch {
|
|
362
|
+
return null;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
function onPath(name) {
|
|
366
|
+
const dirs = (process.env['PATH'] ?? '').split(path.delimiter).filter(Boolean);
|
|
367
|
+
return dirs.some((dir) => {
|
|
368
|
+
try {
|
|
369
|
+
fs.accessSync(path.join(dir, name), fs.constants.X_OK);
|
|
370
|
+
return true;
|
|
371
|
+
}
|
|
372
|
+
catch {
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Variables the child needs to reach the user's systemd, and nothing else.
|
|
379
|
+
*
|
|
380
|
+
* `systemd-run --user` finds the bus through `XDG_RUNTIME_DIR` (sd-bus falls
|
|
381
|
+
* back to `$XDG_RUNTIME_DIR/bus` when `DBUS_SESSION_BUS_ADDRESS` is unset), and
|
|
382
|
+
* that variable is already on both spawn allowlists — so merging this into a
|
|
383
|
+
* session's environment widens nothing. `DBUS_SESSION_BUS_ADDRESS` is passed on
|
|
384
|
+
* only when the daemon really has one of its own, never synthesised.
|
|
385
|
+
*/
|
|
386
|
+
export function cageEnv() {
|
|
387
|
+
const out = {};
|
|
388
|
+
const dir = systemdUserEnv()['XDG_RUNTIME_DIR'];
|
|
389
|
+
if (dir)
|
|
390
|
+
out['XDG_RUNTIME_DIR'] = dir;
|
|
391
|
+
const bus = process.env['DBUS_SESSION_BUS_ADDRESS'];
|
|
392
|
+
if (bus)
|
|
393
|
+
out['DBUS_SESSION_BUS_ADDRESS'] = bus;
|
|
394
|
+
return out;
|
|
395
|
+
}
|
|
396
|
+
function userManagerCgroupControllers() {
|
|
397
|
+
const uid = runnerIdentity().uid;
|
|
398
|
+
if (uid < 0)
|
|
399
|
+
return null;
|
|
400
|
+
const file = path.join('/sys/fs/cgroup/user.slice', `user-${uid}.slice`, `user@${uid}.service`, 'cgroup.controllers');
|
|
401
|
+
try {
|
|
402
|
+
return fs.readFileSync(file, 'utf8').trim().split(/\s+/).filter(Boolean);
|
|
403
|
+
}
|
|
404
|
+
catch {
|
|
405
|
+
return null;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* The only honest check there is: build a real cage and ask it what it holds.
|
|
410
|
+
*
|
|
411
|
+
* Reading `cgroup.controllers` and `systemd-run --version` proves the parts are
|
|
412
|
+
* on the machine, not that they work: under a foreign supervisor the files look
|
|
413
|
+
* perfect and the limit is silently never applied. So the probe starts a
|
|
414
|
+
* throwaway scope with `MemoryMax=64M` whose entire job is to `cat` its own
|
|
415
|
+
* `memory.max`, and the answer has to be `67108864` on the nose.
|
|
416
|
+
*
|
|
417
|
+
* `--collect` IS right here, and only here: this scope's cause of death is of no
|
|
418
|
+
* interest, and a probe that left units behind on every daemon start would be a
|
|
419
|
+
* litter machine.
|
|
420
|
+
*/
|
|
421
|
+
async function runLiveProbe(options) {
|
|
422
|
+
const unit = `devbridge-cage-probe-${process.pid}`;
|
|
423
|
+
try {
|
|
424
|
+
const { stdout } = await execFileAsync('systemd-run', [
|
|
425
|
+
'--user',
|
|
426
|
+
'--scope',
|
|
427
|
+
'--quiet',
|
|
428
|
+
'--collect',
|
|
429
|
+
`--unit=${unit}`,
|
|
430
|
+
`--slice=${SESSIONS_SLICE}`,
|
|
431
|
+
// Exactly the switches `cageSpawn` will use, this one included: a probe
|
|
432
|
+
// that tests a different command line proves nothing about the real one.
|
|
433
|
+
...(options.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
|
|
434
|
+
'-p',
|
|
435
|
+
`MemoryHigh=${PROBE_MEMORY_MAX / 2}`,
|
|
436
|
+
'-p',
|
|
437
|
+
`MemoryMax=${PROBE_MEMORY_MAX}`,
|
|
438
|
+
'-p',
|
|
439
|
+
'MemorySwapMax=0',
|
|
440
|
+
// The one property an older systemd refuses for a scope — the probe is
|
|
441
|
+
// where that refusal has to surface, so the real spawn never meets it.
|
|
442
|
+
...(options.oomPolicyFlag ? ['-p', OOM_POLICY_CONTINUE] : []),
|
|
443
|
+
'-p',
|
|
444
|
+
'TasksMax=16',
|
|
445
|
+
'-p',
|
|
446
|
+
`CPUWeight=${SESSION_CPU_WEIGHT}`,
|
|
447
|
+
'--',
|
|
448
|
+
'/bin/sh',
|
|
449
|
+
'-c',
|
|
450
|
+
'cat /sys/fs/cgroup$(cut -d: -f3 /proc/self/cgroup)/memory.max',
|
|
451
|
+
],
|
|
452
|
+
// Generous: the same call took 0.07–2.5 s on an idle machine and 2741 ms
|
|
453
|
+
// under load. A probe that times out costs the machine its cage, so the
|
|
454
|
+
// budget is sized for the worst measurement, not the typical one.
|
|
455
|
+
{ timeout: 20_000, env: { ...process.env, ...cageEnv() } });
|
|
456
|
+
return { memoryMax: stdout.trim(), error: null };
|
|
457
|
+
}
|
|
458
|
+
catch (error) {
|
|
459
|
+
// Kept, not swallowed. «Did not run» and «ran and did not hold» are
|
|
460
|
+
// different machines and different repairs, and one sentence about
|
|
461
|
+
// `memory.max` for both sent people to fix cgroups that were fine.
|
|
462
|
+
return { memoryMax: null, error: execErrorText(error) };
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* The first line `systemd-run` complained with, short enough for a card.
|
|
467
|
+
*
|
|
468
|
+
* `stderr` before `message`: node's own message is `Command failed: systemd-run
|
|
469
|
+
* …` with the whole command line in it, while stderr is the one sentence that
|
|
470
|
+
* names the cause (`unrecognized option '--expand-environment=no'`).
|
|
471
|
+
*/
|
|
472
|
+
function execErrorText(error) {
|
|
473
|
+
const own = typeof error === 'object' && error !== null ? { ...error } : {};
|
|
474
|
+
const stderr = typeof own['stderr'] === 'string' ? own['stderr'] : '';
|
|
475
|
+
const text = stderr.trim() || (error instanceof Error ? error.message : String(error));
|
|
476
|
+
const first = text.split('\n').find((line) => line.trim().length > 0) ?? '';
|
|
477
|
+
return first.trim().slice(0, 200);
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Which `systemd-run` this is. `systemd 255 (255.4-1ubuntu8.17)` → 255.
|
|
481
|
+
*
|
|
482
|
+
* The version decides one flag and nothing else, so an unreadable answer is not
|
|
483
|
+
* a refusal: it drops the flag, which is what every systemd below 254 does with
|
|
484
|
+
* `--scope` anyway.
|
|
485
|
+
*/
|
|
486
|
+
async function readSystemdRunVersion() {
|
|
487
|
+
try {
|
|
488
|
+
const { stdout } = await execFileAsync('systemd-run', ['--version'], { timeout: 10_000 });
|
|
489
|
+
const match = /^systemd\s+(\d+)/m.exec(stdout);
|
|
490
|
+
const major = match?.[1] ? Number(match[1]) : Number.NaN;
|
|
491
|
+
return Number.isFinite(major) ? major : null;
|
|
492
|
+
}
|
|
493
|
+
catch {
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
async function showMemoryMax(unit) {
|
|
498
|
+
const value = await showByteCount(unit, 'MemoryMax');
|
|
499
|
+
return value !== null && value > 0 ? value : null;
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* A byte-count property as systemd has it in force. `infinity` and «no answer»
|
|
503
|
+
* are both null; 0 is kept, because for `MemorySwapMax` zero is the whole point.
|
|
504
|
+
*/
|
|
505
|
+
async function showByteCount(unit, property) {
|
|
506
|
+
try {
|
|
507
|
+
const { stdout } = await execFileAsync('systemctl', ['--user', 'show', unit, '-p', property, '--value'], { timeout: 10_000, env: systemdUserEnv() });
|
|
508
|
+
const raw = stdout.trim();
|
|
509
|
+
if (!/^\d+$/.test(raw))
|
|
510
|
+
return null;
|
|
511
|
+
const value = Number(raw);
|
|
512
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
513
|
+
}
|
|
514
|
+
catch {
|
|
515
|
+
return null;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
/** The ceiling the policy would write to the slice today — the measured wall. */
|
|
519
|
+
function measuredMachineCeiling() {
|
|
520
|
+
const facts = readMemoryFacts();
|
|
521
|
+
return facts ? memoryPolicy(facts).maxBytes : null;
|
|
522
|
+
}
|
|
523
|
+
export const defaultCageProbe = {
|
|
524
|
+
cgroupFsType: () => statfsType('/sys/fs/cgroup'),
|
|
525
|
+
systemdRunOnPath: () => onPath('systemd-run'),
|
|
526
|
+
userBusPath: () => {
|
|
527
|
+
const dir = systemdUserEnv()['XDG_RUNTIME_DIR'];
|
|
528
|
+
if (!dir)
|
|
529
|
+
return null;
|
|
530
|
+
return fs.existsSync(path.join(dir, 'bus')) ? dir : null;
|
|
531
|
+
},
|
|
532
|
+
delegatedControllers: userManagerCgroupControllers,
|
|
533
|
+
systemdRunVersion: readSystemdRunVersion,
|
|
534
|
+
probeMemoryMax: runLiveProbe,
|
|
535
|
+
serviceMemoryMax: () => showMemoryMax('devbridge-runner'),
|
|
536
|
+
sessionsSliceMemoryMax: () => showMemoryMax(SESSIONS_SLICE),
|
|
537
|
+
sessionsSliceSwapMax: () => showByteCount(SESSIONS_SLICE, 'MemorySwapMax'),
|
|
538
|
+
hostSwapTotalBytes: readSwapTotalBytes,
|
|
539
|
+
machineCeilingBytes: measuredMachineCeiling,
|
|
540
|
+
canRenice: () => {
|
|
541
|
+
try {
|
|
542
|
+
// A no-op that still asks the kernel the question: setting our own
|
|
543
|
+
// priority to the value it already has. Allowed for every user, so a
|
|
544
|
+
// failure here means the syscall itself is refused (seccomp, a locked-down
|
|
545
|
+
// container) — which is the one case where even the fallback is a lie.
|
|
546
|
+
os.setPriority(0, os.getPriority(0));
|
|
547
|
+
return true;
|
|
548
|
+
}
|
|
549
|
+
catch {
|
|
550
|
+
return false;
|
|
551
|
+
}
|
|
552
|
+
},
|
|
553
|
+
};
|
|
554
|
+
export async function detectSessionCage(probe = defaultCageProbe) {
|
|
555
|
+
const fallback = (reason) => ({
|
|
556
|
+
mode: probe.canRenice() ? 'nice-only' : 'none',
|
|
557
|
+
reason,
|
|
558
|
+
memoryHighBytes: null,
|
|
559
|
+
memoryMaxBytes: null,
|
|
560
|
+
swapMaxBytes: null,
|
|
561
|
+
oomContinue: false,
|
|
562
|
+
serviceMemoryMaxBytes: null,
|
|
563
|
+
sessionsSliceMemoryMaxBytes: null,
|
|
564
|
+
expandEnvironmentFlag: false,
|
|
565
|
+
});
|
|
566
|
+
const fsType = probe.cgroupFsType();
|
|
567
|
+
// Not «contains cgroup2» — exactly cgroup2fs. `tmpfs` at this path is cgroup
|
|
568
|
+
// v1 or the hybrid layout, where `systemd-run --scope` accepts every `-p` and
|
|
569
|
+
// applies none of them.
|
|
570
|
+
if (fsType !== 'cgroup2fs') {
|
|
571
|
+
return fallback(`/sys/fs/cgroup is ${fsType ?? 'unreadable'}, not cgroup2fs (cgroup v1 or hybrid)`);
|
|
572
|
+
}
|
|
573
|
+
if (!probe.systemdRunOnPath())
|
|
574
|
+
return fallback('systemd-run is not on PATH');
|
|
575
|
+
if (!probe.userBusPath()) {
|
|
576
|
+
return fallback('no user systemd bus ($XDG_RUNTIME_DIR/bus) — started outside a user session?');
|
|
577
|
+
}
|
|
578
|
+
const controllers = probe.delegatedControllers();
|
|
579
|
+
// Unreadable is NOT a refusal: the path is only right for a systemd user
|
|
580
|
+
// manager, and the live probe below is the authority in every layout. A file
|
|
581
|
+
// we can read and which says «no memory» is a different matter — that is the
|
|
582
|
+
// delegation this cage is built on, missing.
|
|
583
|
+
if (controllers && !controllers.includes('memory')) {
|
|
584
|
+
return fallback(`the user manager has no delegated memory controller (${controllers.join(' ') || 'none'})`);
|
|
585
|
+
}
|
|
586
|
+
// Decided once, here, and carried in the facts: the probe and every later
|
|
587
|
+
// spawn have to build the SAME command line, or the thing that was proved to
|
|
588
|
+
// work is not the thing that runs.
|
|
589
|
+
const systemdVersion = await probe.systemdRunVersion();
|
|
590
|
+
const expandEnvironmentFlag = systemdVersion !== null && systemdVersion >= EXPAND_ENVIRONMENT_MIN_SYSTEMD;
|
|
591
|
+
// `OOMPolicy=` on a SCOPE is newer than the cage's other switches, and which
|
|
592
|
+
// release added it is not documented well enough to gate on a number (this
|
|
593
|
+
// host, systemd 255, takes it; an older systemd answers «Unknown assignment»
|
|
594
|
+
// and spawns NOTHING at all). So the probe decides: try with it, and on a
|
|
595
|
+
// refusal that names the property try once more without — that machine keeps
|
|
596
|
+
// its cage and loses only the «one process, not the session» half (#387).
|
|
597
|
+
let oomPolicyFlag = true;
|
|
598
|
+
let probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
|
|
599
|
+
if (probed.error !== null && refusesOomPolicy(probed.error)) {
|
|
600
|
+
oomPolicyFlag = false;
|
|
601
|
+
probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
|
|
602
|
+
}
|
|
603
|
+
if (probed.error !== null) {
|
|
604
|
+
// Not «the cage did not hold» — nothing was ever caged. Blaming `memory.max`
|
|
605
|
+
// for a refused command line sent people to fix delegation that was fine.
|
|
606
|
+
return fallback(`the test cage would not start: ${probed.error}`);
|
|
607
|
+
}
|
|
608
|
+
if (probed.memoryMax !== String(PROBE_MEMORY_MAX)) {
|
|
609
|
+
return fallback(`the test cage did not hold: memory.max read back as ${probed.memoryMax ?? 'nothing'}, wanted ${PROBE_MEMORY_MAX}`);
|
|
610
|
+
}
|
|
611
|
+
const serviceMemoryMaxBytes = await probe.serviceMemoryMax();
|
|
612
|
+
const sessionsSliceMemoryMaxBytes = await probe.sessionsSliceMemoryMax();
|
|
613
|
+
const sliceSwapMaxBytes = await probe.sessionsSliceSwapMax();
|
|
614
|
+
// The slice is what actually contains the session; the service is the
|
|
615
|
+
// fallback for a systemd that would not answer about the slice at all.
|
|
616
|
+
const containing = sessionsSliceMemoryMaxBytes ?? serviceMemoryMaxBytes;
|
|
617
|
+
const ladder = sessionMemoryLadder(containing, probe.machineCeilingBytes());
|
|
618
|
+
return {
|
|
619
|
+
mode: 'scope',
|
|
620
|
+
reason: '',
|
|
621
|
+
memoryHighBytes: ladder.highBytes,
|
|
622
|
+
memoryMaxBytes: ladder.maxBytes,
|
|
623
|
+
swapMaxBytes: sessionSwapMaxBytes(sliceSwapMaxBytes, probe.hostSwapTotalBytes()),
|
|
624
|
+
oomContinue: oomPolicyFlag,
|
|
625
|
+
serviceMemoryMaxBytes,
|
|
626
|
+
sessionsSliceMemoryMaxBytes,
|
|
627
|
+
expandEnvironmentFlag,
|
|
628
|
+
};
|
|
629
|
+
}
|
|
630
|
+
/** The `-p` assignment itself, in one place: the probe and the spawn must agree. */
|
|
631
|
+
const OOM_POLICY_CONTINUE = 'OOMPolicy=continue';
|
|
632
|
+
/**
|
|
633
|
+
* Did `systemd-run` refuse the command line BECAUSE of `OOMPolicy=`?
|
|
634
|
+
*
|
|
635
|
+
* systemd's wording for a property a unit type does not take is «Unknown
|
|
636
|
+
* assignment: OOMPolicy=continue» (`bus_append_unit_property_assignment`); a
|
|
637
|
+
* refusal that does not name the property is some other machine's problem and
|
|
638
|
+
* must not be retried into a cage with a weaker policy.
|
|
639
|
+
*/
|
|
640
|
+
export function refusesOomPolicy(error) {
|
|
641
|
+
return /OOMPolicy/i.test(error);
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Before the detector has run, nothing is wrapped.
|
|
645
|
+
*
|
|
646
|
+
* `nice-only` rather than `none` because stage 1a is unconditional — the renice
|
|
647
|
+
* happens at every spawn point whatever this module says. The value only ever
|
|
648
|
+
* appears where the daemon has not started (a `verify` from the command line, a
|
|
649
|
+
* test), and it means «no scope», which is the safe answer.
|
|
650
|
+
*/
|
|
651
|
+
const UNPROBED = {
|
|
652
|
+
mode: 'nice-only',
|
|
653
|
+
reason: 'not probed yet',
|
|
654
|
+
memoryHighBytes: null,
|
|
655
|
+
memoryMaxBytes: null,
|
|
656
|
+
swapMaxBytes: null,
|
|
657
|
+
oomContinue: false,
|
|
658
|
+
serviceMemoryMaxBytes: null,
|
|
659
|
+
sessionsSliceMemoryMaxBytes: null,
|
|
660
|
+
expandEnvironmentFlag: false,
|
|
661
|
+
};
|
|
662
|
+
let detected = null;
|
|
663
|
+
/**
|
|
664
|
+
* Probe once, at daemon start, and remember the answer.
|
|
665
|
+
*
|
|
666
|
+
* The probe costs a process, so it is not something a spawn can afford to do:
|
|
667
|
+
* three sessions starting at once would mean three throwaway scopes before the
|
|
668
|
+
* first agent got a word out.
|
|
669
|
+
*/
|
|
670
|
+
export async function initSessionCage(probe = defaultCageProbe) {
|
|
671
|
+
detected = await detectSessionCage(probe);
|
|
672
|
+
if (detected.mode === 'scope') {
|
|
673
|
+
log.info('session cage: each session gets its own cgroup', {
|
|
674
|
+
slice: SESSIONS_SLICE,
|
|
675
|
+
memoryHighMB: Math.round((detected.memoryHighBytes ?? 0) / MIB),
|
|
676
|
+
memoryMaxMB: Math.round((detected.memoryMaxBytes ?? 0) / MIB),
|
|
677
|
+
swapMaxMB: Math.round((detected.swapMaxBytes ?? 0) / MIB),
|
|
678
|
+
oomPolicy: detected.oomContinue ? 'continue' : 'stop (this systemd takes none on a scope)',
|
|
679
|
+
serviceMemoryMaxMB: detected.serviceMemoryMaxBytes
|
|
680
|
+
? Math.round(detected.serviceMemoryMaxBytes / MIB)
|
|
681
|
+
: null,
|
|
682
|
+
sessionsSliceMemoryMaxMB: detected.sessionsSliceMemoryMaxBytes
|
|
683
|
+
? Math.round(detected.sessionsSliceMemoryMaxBytes / MIB)
|
|
684
|
+
: null,
|
|
685
|
+
});
|
|
686
|
+
if (detected.sessionsSliceMemoryMaxBytes === null) {
|
|
687
|
+
// The half of the containment the probe cannot see. Loud, because from
|
|
688
|
+
// here on the machine looks caged and one session at a time is: what is
|
|
689
|
+
// missing is the ceiling over all of them together.
|
|
690
|
+
log.warn('session cage: no collective ceiling on the sessions slice', {
|
|
691
|
+
slice: SESSIONS_SLICE,
|
|
692
|
+
why: 'the drop-in has not been applied — systemd reports MemoryMax=infinity',
|
|
693
|
+
hint: 'run `devbridge-runner doctor --fix`',
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
else {
|
|
698
|
+
// Warn, not error. A machine without the cage is the machine we had
|
|
699
|
+
// yesterday; what must never happen is that it looks like the machine we
|
|
700
|
+
// wanted and quietly is not (plan §5.4.2).
|
|
701
|
+
log.warn('session cage: not available — falling back to priority only', {
|
|
702
|
+
mode: detected.mode,
|
|
703
|
+
reason: detected.reason,
|
|
704
|
+
});
|
|
705
|
+
}
|
|
706
|
+
return detected;
|
|
707
|
+
}
|
|
708
|
+
/** What the last {@link initSessionCage} found; the safe default before it ran. */
|
|
709
|
+
export function sessionCage() {
|
|
710
|
+
return detected ?? UNPROBED;
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Live scope per cage id, so a restart cannot collide with a scope that the OOM
|
|
714
|
+
* killer left behind. Cleared by {@link releaseSessionScope} once the old unit
|
|
715
|
+
* is provably gone — see {@link sessionScopeUnit}.
|
|
716
|
+
*/
|
|
717
|
+
const attempts = new Map();
|
|
718
|
+
/**
|
|
719
|
+
* The scope each caged id is CURRENTLY in, so the supervisor's memory watch can
|
|
720
|
+
* find the cgroup of a live session without the adapter having to hand it up.
|
|
721
|
+
* Set in {@link cageSpawn}, cleared in {@link releaseSessionScope}.
|
|
722
|
+
*/
|
|
723
|
+
const liveUnits = new Map();
|
|
724
|
+
/** The scope a live caged id runs in, or null when it is not caged (or gone). */
|
|
725
|
+
export function sessionScopeUnitOf(id) {
|
|
726
|
+
return liveUnits.get(id) ?? null;
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* Wrap a command in its session's cage, or hand it back untouched.
|
|
730
|
+
*
|
|
731
|
+
* Untouched is the honest answer on every machine where the cage was not proved
|
|
732
|
+
* to work: `systemd-run` would accept the flags there and apply nothing, and a
|
|
733
|
+
* session that believes it is contained when it is not is worse than one that
|
|
734
|
+
* knows it is not.
|
|
735
|
+
*
|
|
736
|
+
* Pipes, exit codes, signals and stdin EOF behave exactly as with a direct
|
|
737
|
+
* spawn, because `systemd-run --scope` execs into the SAME pid — verified in the
|
|
738
|
+
* spike, including `detached: true` + `process.kill(-pid)` in `verify.ts`
|
|
739
|
+
* (`pgid === child.pid` still holds).
|
|
740
|
+
*/
|
|
741
|
+
export function cageSpawn(input) {
|
|
742
|
+
const facts = sessionCage();
|
|
743
|
+
if (facts.mode !== 'scope' ||
|
|
744
|
+
facts.memoryMaxBytes === null ||
|
|
745
|
+
facts.memoryHighBytes === null ||
|
|
746
|
+
facts.swapMaxBytes === null) {
|
|
747
|
+
return { command: input.command, args: input.args, env: {}, unit: null };
|
|
748
|
+
}
|
|
749
|
+
const attempt = (attempts.get(input.id) ?? 0) + 1;
|
|
750
|
+
attempts.set(input.id, attempt);
|
|
751
|
+
const unit = sessionScopeUnit(input.id, attempt);
|
|
752
|
+
liveUnits.set(input.id, unit);
|
|
753
|
+
// A verdict nobody read belongs to the process that just ended, not to the
|
|
754
|
+
// one starting now: without this, a kill that was never surfaced would be
|
|
755
|
+
// pinned hours later on an unrelated crash of the next process (#387 QA).
|
|
756
|
+
deaths.delete(input.id);
|
|
757
|
+
oomKillsSeen.delete(input.id);
|
|
758
|
+
return {
|
|
759
|
+
command: 'systemd-run',
|
|
760
|
+
args: [
|
|
761
|
+
'--user',
|
|
762
|
+
'--scope',
|
|
763
|
+
'--quiet',
|
|
764
|
+
`--unit=${unit}`,
|
|
765
|
+
`--slice=${SESSIONS_SLICE}`,
|
|
766
|
+
// The agent's prompt travels in these arguments and prompts contain `$`.
|
|
767
|
+
// Only where systemd knows the switch — below 254 it is a hard error and
|
|
768
|
+
// `--scope` does no expansion anyway (module header, MAJOR-2).
|
|
769
|
+
...(facts.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
|
|
770
|
+
// The ladder of #387, bottom to top. The brake: past the honest share the
|
|
771
|
+
// kernel reclaims and slows this session down, and nothing dies.
|
|
772
|
+
'-p',
|
|
773
|
+
`MemoryHigh=${facts.memoryHighBytes}`,
|
|
774
|
+
// The wall: the machine's measured headroom, far above the brake. Only a
|
|
775
|
+
// runaway gets here, and with `OOMPolicy=continue` it costs one process.
|
|
776
|
+
'-p',
|
|
777
|
+
`MemoryMax=${facts.memoryMaxBytes}`,
|
|
778
|
+
// THE line. Without a bound there is no cage at all: `MemoryMax` bounds
|
|
779
|
+
// resident memory and lets the rest fall into the host's swap, which is
|
|
780
|
+
// how a 200 MB cage allocated 2 GB and took the machine's swap with it.
|
|
781
|
+
// Bounded, never unset — and since #387 a share rather than 0, because
|
|
782
|
+
// the brake above has to have somewhere to push pages (module header).
|
|
783
|
+
'-p',
|
|
784
|
+
`MemorySwapMax=${facts.swapMaxBytes}`,
|
|
785
|
+
// Only where the probe proved this systemd takes it on a scope. Without
|
|
786
|
+
// it the kernel killing ONE process stops the WHOLE scope (`OOMPolicy`
|
|
787
|
+
// defaults to `stop`) — the exact mechanism that killed honest sessions
|
|
788
|
+
// in 0.54.0, and the same lesson `service-unit.ts` learned in QA-112.
|
|
789
|
+
...(facts.oomContinue ? ['-p', OOM_POLICY_CONTINUE] : []),
|
|
790
|
+
'-p',
|
|
791
|
+
`TasksMax=${SESSION_TASKS_MAX}`,
|
|
792
|
+
'-p',
|
|
793
|
+
`CPUWeight=${SESSION_CPU_WEIGHT}`,
|
|
794
|
+
'--',
|
|
795
|
+
input.command,
|
|
796
|
+
...input.args,
|
|
797
|
+
],
|
|
798
|
+
env: cageEnv(),
|
|
799
|
+
unit,
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
/**
|
|
803
|
+
* Did this process die in the window before `systemd-run` handed over?
|
|
804
|
+
*
|
|
805
|
+
* Starting a scope took 0.07–2.5 s in the spike, and 2741 ms on a loaded
|
|
806
|
+
* machine. A stop inside that window signals `systemd-run` itself, before the
|
|
807
|
+
* `exec`, and the parent sees `{code: null, signal: 'SIGTERM'}` with nothing on
|
|
808
|
+
* stdout at all. That is «the session never started», not «the agent died
|
|
809
|
+
* silently» — told apart here so no supervisor has to infer it from an empty
|
|
810
|
+
* buffer.
|
|
811
|
+
*/
|
|
812
|
+
export function killedBeforeExec(info) {
|
|
813
|
+
return info.caged && !info.sawOutput && info.code === null && info.signal === 'SIGTERM';
|
|
814
|
+
}
|
|
815
|
+
/**
|
|
816
|
+
* The pure half of {@link readScopeMemoryStatus}: the four files' text in, the
|
|
817
|
+
* status out. `memory.swap.current` is optional (no swap controller, cgroup v2
|
|
818
|
+
* without swap accounting); the rest are not.
|
|
819
|
+
*/
|
|
820
|
+
export function parseScopeMemoryStatus(files) {
|
|
821
|
+
const count = (raw) => {
|
|
822
|
+
const trimmed = raw.trim();
|
|
823
|
+
if (!/^\d+$/.test(trimmed))
|
|
824
|
+
return null;
|
|
825
|
+
const value = Number(trimmed);
|
|
826
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
827
|
+
};
|
|
828
|
+
const limit = (raw) => (raw.trim() === 'max' ? null : count(raw));
|
|
829
|
+
const event = (name) => {
|
|
830
|
+
const line = files.events.split('\n').find((l) => l.startsWith(`${name} `));
|
|
831
|
+
return line ? (count(line.slice(name.length + 1)) ?? 0) : 0;
|
|
832
|
+
};
|
|
833
|
+
const currentBytes = count(files.current);
|
|
834
|
+
if (currentBytes === null)
|
|
835
|
+
return null;
|
|
836
|
+
return {
|
|
837
|
+
currentBytes,
|
|
838
|
+
highBytes: limit(files.high),
|
|
839
|
+
maxBytes: limit(files.max),
|
|
840
|
+
swapCurrentBytes: files.swapCurrent === undefined ? 0 : (count(files.swapCurrent) ?? 0),
|
|
841
|
+
highEvents: event('high'),
|
|
842
|
+
oomKills: event('oom_kill'),
|
|
843
|
+
ownLimitOom: event('oom'),
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
/** Where a session scope's cgroup lives, or null off a user manager. */
|
|
847
|
+
function scopeCgroupDir(unit) {
|
|
848
|
+
const self = readSelfCgroup();
|
|
849
|
+
if (self === null)
|
|
850
|
+
return null;
|
|
851
|
+
const slice = sliceCgroupPath(self, SESSIONS_SLICE);
|
|
852
|
+
return slice === null ? null : path.join(slice, unit);
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* What one live session's cgroup holds and has been through. Null when the
|
|
856
|
+
* cgroup is not there (the scope ended, or this machine has no cage).
|
|
857
|
+
*
|
|
858
|
+
* Filesystem, not `systemctl show`: this runs on the supervisor's 30 s tick for
|
|
859
|
+
* every live session, and the bus is the thing that took 2.7 s under load.
|
|
860
|
+
*/
|
|
861
|
+
export function readScopeMemoryStatus(unit, readFile = (p) => fs.readFileSync(p, 'utf8')) {
|
|
862
|
+
const dir = scopeCgroupDir(unit);
|
|
863
|
+
if (dir === null)
|
|
864
|
+
return null;
|
|
865
|
+
try {
|
|
866
|
+
let swapCurrent;
|
|
867
|
+
try {
|
|
868
|
+
swapCurrent = readFile(path.join(dir, 'memory.swap.current'));
|
|
869
|
+
}
|
|
870
|
+
catch {
|
|
871
|
+
swapCurrent = undefined;
|
|
872
|
+
}
|
|
873
|
+
return parseScopeMemoryStatus({
|
|
874
|
+
current: readFile(path.join(dir, 'memory.current')),
|
|
875
|
+
high: readFile(path.join(dir, 'memory.high')),
|
|
876
|
+
max: readFile(path.join(dir, 'memory.max')),
|
|
877
|
+
events: readFile(path.join(dir, 'memory.events')),
|
|
878
|
+
...(swapCurrent === undefined ? {} : { swapCurrent }),
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
catch {
|
|
882
|
+
return null;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* `oom_kill` as the watch last saw it, per id — so a kill that was already
|
|
887
|
+
* announced in the feed while the session lived is not blamed for a death that
|
|
888
|
+
* came later for another reason.
|
|
889
|
+
*/
|
|
890
|
+
const oomKillsSeen = new Map();
|
|
891
|
+
export function markScopeOomKillsSeen(id, oomKills) {
|
|
892
|
+
oomKillsSeen.set(id, oomKills);
|
|
893
|
+
}
|
|
894
|
+
const deaths = new Map();
|
|
895
|
+
/**
|
|
896
|
+
* Snapshot the cgroup's verdict before systemd can take it away.
|
|
897
|
+
*
|
|
898
|
+
* Exported for the test seam only (`readStatus`): the real caller passes
|
|
899
|
+
* nothing and reads the live cgroup.
|
|
900
|
+
*/
|
|
901
|
+
export function rememberDeath(id, unit, readStatus = readScopeMemoryStatus) {
|
|
902
|
+
const status = readStatus(unit);
|
|
903
|
+
const seen = oomKillsSeen.get(id) ?? 0;
|
|
904
|
+
oomKillsSeen.delete(id);
|
|
905
|
+
if (status === null || status.oomKills <= seen)
|
|
906
|
+
return;
|
|
907
|
+
deaths.set(id, {
|
|
908
|
+
oomKills: status.oomKills - seen,
|
|
909
|
+
maxBytes: status.maxBytes,
|
|
910
|
+
// `oom` moves only where the limit that was hit belongs. A kill with our
|
|
911
|
+
// own counter still at zero came from an ancestor — the pool.
|
|
912
|
+
ownWall: status.ownLimitOom > 0,
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
/**
|
|
916
|
+
* The other source of the same verdict, for the machine where the first one is
|
|
917
|
+
* already gone (#387 QA).
|
|
918
|
+
*
|
|
919
|
+
* On a systemd that takes no `OOMPolicy` on a scope, one kill stops the whole
|
|
920
|
+
* scope; systemd then removes the cgroup as soon as it is empty, and the
|
|
921
|
+
* snapshot above can find nothing at all. `Result=oom-kill` survives that —
|
|
922
|
+
* it is read a few milliseconds later from the unit, which is why
|
|
923
|
+
* {@link releaseSessionScope} is now awaited before a death is explained.
|
|
924
|
+
*/
|
|
925
|
+
function rememberDeathFromResult(id) {
|
|
926
|
+
if (deaths.has(id))
|
|
927
|
+
return;
|
|
928
|
+
deaths.set(id, {
|
|
929
|
+
oomKills: 1,
|
|
930
|
+
maxBytes: sessionCage().memoryMaxBytes,
|
|
931
|
+
// systemd stopped the unit for an OOM inside it; which limit was hit is not
|
|
932
|
+
// in `Result`, and «your own wall» is the claim that must not be guessed.
|
|
933
|
+
ownWall: false,
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* One sentence for the error the person reads, when the kernel had a hand in
|
|
938
|
+
* this death — or null when it had not. Consumed: a session restarted after an
|
|
939
|
+
* OOM must not carry the old sentence into its next, unrelated failure.
|
|
940
|
+
*
|
|
941
|
+
* The text that reached people in 0.54.0 was «exited with code 143» once and
|
|
942
|
+
* «terminated by signal SIGKILL» the next time, for the same cause, and neither
|
|
943
|
+
* said the word memory. This is that word.
|
|
944
|
+
*/
|
|
945
|
+
export function explainMemoryDeath(id) {
|
|
946
|
+
const death = deaths.get(id);
|
|
947
|
+
if (!death)
|
|
948
|
+
return null;
|
|
949
|
+
deaths.delete(id);
|
|
950
|
+
const killed = death.oomKills === 1 ? 'a process' : `${death.oomKills} processes`;
|
|
951
|
+
const wall = death.maxBytes === null ? '' : ` of ${Math.round(death.maxBytes / MIB)} MB`;
|
|
952
|
+
if (death.ownWall) {
|
|
953
|
+
return (`The session ran out of memory: it went over its own memory ceiling${wall} and the kernel killed ` +
|
|
954
|
+
`${killed} inside it. ` +
|
|
955
|
+
'Resume it and avoid re-running the command that was in flight, or give the machine more memory or swap.');
|
|
956
|
+
}
|
|
957
|
+
return (`The machine ran out of memory for agent sessions — all sessions on it share one ceiling — and the kernel killed ` +
|
|
958
|
+
`${killed} in this one. This session was not necessarily the greedy one. ` +
|
|
959
|
+
'Resume it; if it keeps happening, run fewer sessions at once or give the machine more memory or swap.');
|
|
960
|
+
}
|
|
961
|
+
const realSystemctl = async (args) => {
|
|
962
|
+
const { stdout, stderr } = await execFileAsync('systemctl', ['--user', ...args], {
|
|
963
|
+
timeout: 15_000,
|
|
964
|
+
env: systemdUserEnv(),
|
|
965
|
+
});
|
|
966
|
+
return { stdout, stderr };
|
|
967
|
+
};
|
|
968
|
+
function showValue(stdout, property) {
|
|
969
|
+
const line = stdout
|
|
970
|
+
.split('\n')
|
|
971
|
+
.map((l) => l.trim())
|
|
972
|
+
.find((l) => l.startsWith(`${property}=`));
|
|
973
|
+
return line ? line.slice(property.length + 1) : null;
|
|
974
|
+
}
|
|
975
|
+
/**
|
|
976
|
+
* Read why the scope ended, say so, and then let systemd forget it.
|
|
977
|
+
*
|
|
978
|
+
* This is the whole reason `--collect` is not passed. The order is fixed: the
|
|
979
|
+
* process has already exited, `Result` is read, the reason is logged, and only
|
|
980
|
+
* then is the unit reset — because `reset-failed` is what deletes the answer.
|
|
981
|
+
*
|
|
982
|
+
* Failure here is never fatal: a unit that could not be reset is swept at the
|
|
983
|
+
* next daemon start, and the counter in {@link cageSpawn} keeps the session
|
|
984
|
+
* startable in the meantime.
|
|
985
|
+
*/
|
|
986
|
+
export async function releaseSessionScope(unit, id, systemctl = realSystemctl) {
|
|
987
|
+
const running = releaseSessionScopeInner(unit, id, systemctl);
|
|
988
|
+
if (id !== undefined) {
|
|
989
|
+
releasing.set(id, running);
|
|
990
|
+
void running.finally(() => {
|
|
991
|
+
if (releasing.get(id) === running)
|
|
992
|
+
releasing.delete(id);
|
|
993
|
+
});
|
|
994
|
+
}
|
|
995
|
+
return await running;
|
|
996
|
+
}
|
|
997
|
+
/**
|
|
998
|
+
* Releases in flight, per cage id, so a death can be explained with BOTH
|
|
999
|
+
* sources: the cgroup snapshot (synchronous, gone the moment systemd prunes the
|
|
1000
|
+
* group) and `Result=oom-kill` (authoritative, and a few milliseconds late).
|
|
1001
|
+
* Without the wait, the machine that needs the sentence most — the one whose
|
|
1002
|
+
* systemd stops the whole scope — was the one that never got it (#387 QA).
|
|
1003
|
+
*/
|
|
1004
|
+
const releasing = new Map();
|
|
1005
|
+
/**
|
|
1006
|
+
* The sentence for a death, once the verdict is in. Null when the kernel had no
|
|
1007
|
+
* hand in it. Waits for the release of this session's scope, but never longer
|
|
1008
|
+
* than `capMs`: an answer that arrives after the person has read the error is
|
|
1009
|
+
* worth nothing, and a hung `systemctl` must not hold a failing session open.
|
|
1010
|
+
*/
|
|
1011
|
+
export async function memoryDeathSentence(id, capMs = 3_000) {
|
|
1012
|
+
const pending = releasing.get(id);
|
|
1013
|
+
if (pending) {
|
|
1014
|
+
let timer;
|
|
1015
|
+
await Promise.race([
|
|
1016
|
+
pending.catch(() => null),
|
|
1017
|
+
new Promise((resolve) => {
|
|
1018
|
+
timer = setTimeout(resolve, capMs);
|
|
1019
|
+
timer.unref?.();
|
|
1020
|
+
}),
|
|
1021
|
+
]);
|
|
1022
|
+
if (timer)
|
|
1023
|
+
clearTimeout(timer);
|
|
1024
|
+
}
|
|
1025
|
+
return explainMemoryDeath(id);
|
|
1026
|
+
}
|
|
1027
|
+
async function releaseSessionScopeInner(unit, id, systemctl) {
|
|
1028
|
+
if (!unit)
|
|
1029
|
+
return null;
|
|
1030
|
+
// Synchronously and FIRST: the cgroup's own counters are the one record of an
|
|
1031
|
+
// OOM kill that survives `OOMPolicy=continue` (the unit does not fail, so
|
|
1032
|
+
// `Result` says nothing), and the directory goes away the moment the scope is
|
|
1033
|
+
// empty. This is what the adapter's error text is built from (#387).
|
|
1034
|
+
if (id !== undefined)
|
|
1035
|
+
rememberDeath(id, unit);
|
|
1036
|
+
if (id !== undefined)
|
|
1037
|
+
liveUnits.delete(id);
|
|
1038
|
+
let result = null;
|
|
1039
|
+
try {
|
|
1040
|
+
const { stdout } = await systemctl(['show', unit, '-p', 'Result']);
|
|
1041
|
+
result = showValue(stdout, 'Result');
|
|
1042
|
+
}
|
|
1043
|
+
catch {
|
|
1044
|
+
// Already collected — a clean exit takes its scope with it.
|
|
1045
|
+
}
|
|
1046
|
+
if (result === 'oom-kill') {
|
|
1047
|
+
log.warn('session cage: the kernel killed this session for going over its memory ceiling', {
|
|
1048
|
+
unit,
|
|
1049
|
+
memoryMaxMB: Math.round((sessionCage().memoryMaxBytes ?? 0) / MIB),
|
|
1050
|
+
});
|
|
1051
|
+
// The cgroup snapshot above may have found nothing: on a systemd that stops
|
|
1052
|
+
// the whole scope, the group is gone by the time the process's `exit` fires.
|
|
1053
|
+
if (id !== undefined)
|
|
1054
|
+
rememberDeathFromResult(id);
|
|
1055
|
+
}
|
|
1056
|
+
let forgotten = false;
|
|
1057
|
+
try {
|
|
1058
|
+
await systemctl(['reset-failed', unit]);
|
|
1059
|
+
forgotten = true;
|
|
1060
|
+
}
|
|
1061
|
+
catch {
|
|
1062
|
+
// Nothing to reset is the normal case — but so is «the bus was not there»,
|
|
1063
|
+
// and those two look identical from here.
|
|
1064
|
+
}
|
|
1065
|
+
// Only when the unit is provably gone. Clearing the counter after a FAILED
|
|
1066
|
+
// `reset-failed` hands the next start of this session the same unit name,
|
|
1067
|
+
// which systemd answers with «was already loaded or has a fragment file» and
|
|
1068
|
+
// spawns nothing — the exact failure the counter exists to prevent
|
|
1069
|
+
// (QA-2026-09-07 MINOR-11).
|
|
1070
|
+
if (id !== undefined && forgotten)
|
|
1071
|
+
attempts.delete(id);
|
|
1072
|
+
return result;
|
|
1073
|
+
}
|
|
1074
|
+
/** Unit names of every `devbridge-session-*.scope` systemd still knows about. */
|
|
1075
|
+
export async function listSessionScopeUnits(systemctl = realSystemctl) {
|
|
1076
|
+
let stdout;
|
|
1077
|
+
try {
|
|
1078
|
+
({ stdout } = await systemctl([
|
|
1079
|
+
'list-units',
|
|
1080
|
+
'--all',
|
|
1081
|
+
'--plain',
|
|
1082
|
+
'--no-legend',
|
|
1083
|
+
`${SESSION_SCOPE_PREFIX}*.scope`,
|
|
1084
|
+
]));
|
|
1085
|
+
}
|
|
1086
|
+
catch {
|
|
1087
|
+
return [];
|
|
1088
|
+
}
|
|
1089
|
+
return stdout
|
|
1090
|
+
.split('\n')
|
|
1091
|
+
.map((line) => line.trim().split(/\s+/)[0] ?? '')
|
|
1092
|
+
.filter((unit) => unit.startsWith(SESSION_SCOPE_PREFIX) && unit.endsWith('.scope'));
|
|
1093
|
+
}
|
|
1094
|
+
/** What `doctor` prints for each caged session. */
|
|
1095
|
+
export async function listSessionScopes(systemctl = realSystemctl) {
|
|
1096
|
+
const units = await listSessionScopeUnits(systemctl);
|
|
1097
|
+
const out = [];
|
|
1098
|
+
for (const unit of units) {
|
|
1099
|
+
let stdout = '';
|
|
1100
|
+
try {
|
|
1101
|
+
({ stdout } = await systemctl([
|
|
1102
|
+
'show',
|
|
1103
|
+
unit,
|
|
1104
|
+
'-p',
|
|
1105
|
+
'MemoryMax',
|
|
1106
|
+
'-p',
|
|
1107
|
+
'MemoryCurrent',
|
|
1108
|
+
'-p',
|
|
1109
|
+
'TasksCurrent',
|
|
1110
|
+
'-p',
|
|
1111
|
+
'Result',
|
|
1112
|
+
'-p',
|
|
1113
|
+
'ActiveState',
|
|
1114
|
+
]));
|
|
1115
|
+
}
|
|
1116
|
+
catch {
|
|
1117
|
+
// Gone between the list and the read — report the name and nothing else.
|
|
1118
|
+
}
|
|
1119
|
+
const num = (property) => {
|
|
1120
|
+
const value = Number(showValue(stdout, property));
|
|
1121
|
+
return Number.isFinite(value) ? value : null;
|
|
1122
|
+
};
|
|
1123
|
+
out.push({
|
|
1124
|
+
unit,
|
|
1125
|
+
memoryMaxBytes: num('MemoryMax'),
|
|
1126
|
+
memoryCurrentBytes: num('MemoryCurrent'),
|
|
1127
|
+
tasksCurrent: num('TasksCurrent'),
|
|
1128
|
+
result: showValue(stdout, 'Result'),
|
|
1129
|
+
activeState: showValue(stdout, 'ActiveState'),
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1132
|
+
return out;
|
|
1133
|
+
}
|
|
1134
|
+
/**
|
|
1135
|
+
* Stop and forget every session scope that has no session behind it.
|
|
1136
|
+
*
|
|
1137
|
+
* At daemon start that is all of them by definition, and it is the point: a
|
|
1138
|
+
* scope outlives a killed daemon carrying the whole process tree with it, which
|
|
1139
|
+
* is the shape of the 10 h 51 min `ugrep` of 16.08. Stopping the scope takes the
|
|
1140
|
+
* tree, not just the process we happened to know about.
|
|
1141
|
+
*
|
|
1142
|
+
* `liveIds` exists so the same sweep can run later without killing work in
|
|
1143
|
+
* progress; only the CURRENT scope of a live session is spared, because an
|
|
1144
|
+
* earlier attempt of the same session is exactly the leftover we are here for.
|
|
1145
|
+
*/
|
|
1146
|
+
export async function sweepOrphanSessionScopes(liveIds = [], systemctl = realSystemctl) {
|
|
1147
|
+
const spared = new Set();
|
|
1148
|
+
for (const id of liveIds)
|
|
1149
|
+
spared.add(sessionScopeUnit(id, attempts.get(id) ?? 1));
|
|
1150
|
+
const removed = [];
|
|
1151
|
+
for (const unit of await listSessionScopeUnits(systemctl)) {
|
|
1152
|
+
if (spared.has(unit))
|
|
1153
|
+
continue;
|
|
1154
|
+
try {
|
|
1155
|
+
await systemctl(['stop', unit]);
|
|
1156
|
+
}
|
|
1157
|
+
catch {
|
|
1158
|
+
// A failed scope has nothing to stop; `reset-failed` below is the part
|
|
1159
|
+
// that matters for it.
|
|
1160
|
+
}
|
|
1161
|
+
try {
|
|
1162
|
+
await systemctl(['reset-failed', unit]);
|
|
1163
|
+
}
|
|
1164
|
+
catch {
|
|
1165
|
+
// Already gone.
|
|
1166
|
+
}
|
|
1167
|
+
removed.push(unit);
|
|
1168
|
+
}
|
|
1169
|
+
if (removed.length > 0) {
|
|
1170
|
+
log.warn('session cage: removed scopes left behind by a previous run', {
|
|
1171
|
+
count: removed.length,
|
|
1172
|
+
units: removed.slice(0, 10),
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
return removed;
|
|
1176
|
+
}
|
|
1177
|
+
/**
|
|
1178
|
+
* The slice names and the CPU share — re-exported so a caller that reasons about
|
|
1179
|
+
* the cage needs one import, while the values themselves stay next to the
|
|
1180
|
+
* drop-in that writes them (`service-unit.ts`).
|
|
1181
|
+
*/
|
|
1182
|
+
export { SESSIONS_SLICE, DEVBRIDGE_SLICE, SESSION_CPU_WEIGHT };
|
|
1183
|
+
//# sourceMappingURL=session-cage.js.map
|