@bridge4dev/runner 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +105 -2
- package/dist/adapters/codex-protocol.d.ts +11 -0
- package/dist/adapters/codex-protocol.js +41 -3
- package/dist/adapters/codex.js +37 -0
- package/dist/host-load.d.ts +156 -0
- package/dist/host-load.js +223 -0
- package/dist/index.js +211 -40
- package/dist/process-priority.d.ts +55 -0
- package/dist/process-priority.js +99 -0
- package/dist/protocol.d.ts +25 -3
- package/dist/recipe-schema.d.ts +6 -6
- package/dist/self-update.js +43 -2
- package/dist/service-unit.d.ts +268 -10
- package/dist/service-unit.js +432 -53
- package/dist/session-cage.d.ts +508 -0
- package/dist/session-cage.js +1183 -0
- package/dist/supervisor.d.ts +106 -0
- package/dist/supervisor.js +243 -0
- package/dist/systemd-memory.d.ts +35 -0
- package/dist/systemd-memory.js +115 -0
- package/dist/verify.js +28 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/service-unit.js
CHANGED
|
@@ -129,15 +129,69 @@ export function buildUnit(execStart, nodeBinary = process.execPath) {
|
|
|
129
129
|
* which is the only way to fix the servers that already have the bad numbers
|
|
130
130
|
* baked in — and it never overwrites a unit the operator edited by hand.
|
|
131
131
|
*/
|
|
132
|
-
export const LIMITS_VERSION =
|
|
132
|
+
export const LIMITS_VERSION = 5;
|
|
133
133
|
const LIMITS_MARKER = '# devbridge-limits-version:';
|
|
134
134
|
/** `zz-` so it sorts last: an operator's own drop-in should still win. */
|
|
135
135
|
const LIMITS_FILE = 'zz-devbridge-limits.conf';
|
|
136
|
+
/**
|
|
137
|
+
* Where the agent sessions live once they have a cage of their own.
|
|
138
|
+
*
|
|
139
|
+
* A `systemd-run --scope` is a SIBLING of the service, not a child of it: a
|
|
140
|
+
* session started that way leaves the service's `MemoryMax`, `CPUQuota`,
|
|
141
|
+
* `OOMPolicy=continue` and `KillMode=control-group` behind entirely. So the
|
|
142
|
+
* collective ceiling has to move with them, onto the slice — otherwise the cage
|
|
143
|
+
* per session would arrive at the price of the ceiling over all of them.
|
|
144
|
+
*
|
|
145
|
+
* The dash is systemd's hierarchy separator: `devbridge-sessions.slice` is a
|
|
146
|
+
* child of `devbridge.slice`, which is where the CPU share for everything the
|
|
147
|
+
* agents run is set.
|
|
148
|
+
*/
|
|
149
|
+
export const SESSIONS_SLICE = 'devbridge-sessions.slice';
|
|
150
|
+
export const DEVBRIDGE_SLICE = 'devbridge.slice';
|
|
151
|
+
/**
|
|
152
|
+
* Half the default weight, on `devbridge.slice` and on every session scope.
|
|
153
|
+
*
|
|
154
|
+
* Lives here rather than in `session-cage.ts` because the number has to be the
|
|
155
|
+
* SAME in both places — the slice sets the share of the agents against the
|
|
156
|
+
* daemon, the scope sets one session's share against another's — and a
|
|
157
|
+
* duplicated literal is how those two drift apart. Why 50 and what it replaces:
|
|
158
|
+
* `buildDevbridgeSliceOverride` below.
|
|
159
|
+
*/
|
|
160
|
+
export const SESSION_CPU_WEIGHT = 50;
|
|
136
161
|
export function limitsOverridePath(home = systemdUserHome()) {
|
|
137
162
|
return path.join(home, '.config', 'systemd', 'user', `${SERVICE_NAME}.service.d`, LIMITS_FILE);
|
|
138
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* Drop-in path for a slice unit that has no unit FILE at all.
|
|
166
|
+
*
|
|
167
|
+
* systemd synthesises `devbridge-sessions.slice` the first time something asks
|
|
168
|
+
* for it, and it reads drop-ins for the synthesised unit exactly as for a real
|
|
169
|
+
* one — verified on this machine: a `[Slice] MemoryMax=64M` drop-in with no
|
|
170
|
+
* fragment gave `MemoryMax=67108864` on the live slice. So there is no unit file
|
|
171
|
+
* to write and no unit file to keep in sync; the policy is the drop-in.
|
|
172
|
+
*/
|
|
173
|
+
export function sliceOverridePath(slice, home = systemdUserHome()) {
|
|
174
|
+
return path.join(home, '.config', 'systemd', 'user', `${slice}.d`, LIMITS_FILE);
|
|
175
|
+
}
|
|
176
|
+
export function sessionsSliceOverridePath(home = systemdUserHome()) {
|
|
177
|
+
return sliceOverridePath(SESSIONS_SLICE, home);
|
|
178
|
+
}
|
|
179
|
+
export function devbridgeSliceOverridePath(home = systemdUserHome()) {
|
|
180
|
+
return sliceOverridePath(DEVBRIDGE_SLICE, home);
|
|
181
|
+
}
|
|
139
182
|
const MIB = 1024 * 1024;
|
|
140
183
|
const GIB = 1024 * MIB;
|
|
184
|
+
/**
|
|
185
|
+
* How much of the machine's swap ALL sessions together may use (#387).
|
|
186
|
+
*
|
|
187
|
+
* Half, not all: swap belongs to the machine (the spike's 200 MB cage drained
|
|
188
|
+
* the host's 2 GB and the box hung), so the other half stays with the neighbours
|
|
189
|
+
* and the kernel. Each session gets half of this again (`sessionSwapMaxBytes`).
|
|
190
|
+
* On the 2 GB swap this machine has today that is 512 MB per session — enough
|
|
191
|
+
* for a build's overshoot, not for a runaway; the owner's note on #387 is that
|
|
192
|
+
* the swap itself has to grow for the brake to have real room.
|
|
193
|
+
*/
|
|
194
|
+
export const SESSIONS_SWAP_SHARE = 0.5;
|
|
141
195
|
/**
|
|
142
196
|
* How long after boot `MemAvailable` starts telling the truth.
|
|
143
197
|
*
|
|
@@ -149,7 +203,7 @@ const GIB = 1024 * MIB;
|
|
|
149
203
|
*/
|
|
150
204
|
export const BOOT_SETTLE_SEC = 600;
|
|
151
205
|
/**
|
|
152
|
-
* The ceiling is never written BELOW what the
|
|
206
|
+
* The ceiling is never written BELOW what the cgroups already hold.
|
|
153
207
|
*
|
|
154
208
|
* Until 0.39.0 this policy only ever REMOVED a limit, so applying it to a running
|
|
155
209
|
* service was free. Now it sets one, and systemd applies `memory.max` to a LIVE
|
|
@@ -158,11 +212,29 @@ export const BOOT_SETTLE_SEC = 600;
|
|
|
158
212
|
* running yet — but `doctor --fix` on a busy machine would otherwise take out
|
|
159
213
|
* somebody's session as a side effect of running a diagnostic.
|
|
160
214
|
*
|
|
215
|
+
* `max` of the two holders, not the sum: the SAME number is written to the
|
|
216
|
+
* service and to `devbridge-sessions.slice`, so it has to clear whichever of the
|
|
217
|
+
* two is fuller — each cgroup is measured against it on its own.
|
|
218
|
+
*
|
|
161
219
|
* The cost is that a machine fixed WHILE a runaway is in progress writes a ceiling
|
|
162
220
|
* above that runaway, which protects nothing. It resolves itself: the next daemon
|
|
163
|
-
* start measures
|
|
221
|
+
* start measures idle cgroups, and the drift check rewrites the inflated number.
|
|
164
222
|
*/
|
|
165
223
|
const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
224
|
+
/** What the ceiling about to be written has to clear, in either cgroup. */
|
|
225
|
+
export function managedUsageFloorBytes(facts) {
|
|
226
|
+
return (Math.max(facts.ownFloorBytes ?? facts.ownUsageBytes, facts.sessionsFloorBytes ?? facts.sessionsUsageBytes) * CEILING_HEADROOM_OVER_CURRENT);
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Below this a dev server cannot run one agent, so the policy never goes under it.
|
|
230
|
+
*
|
|
231
|
+
* Named rather than repeated because it is now used twice: as the lower clamp of
|
|
232
|
+
* the measured ceiling, and as the lower clamp of the ceiling written to the
|
|
233
|
+
* sessions slice on a machine the policy could not measure at all — where the
|
|
234
|
+
* only other number available is what that slice happens to hold this second,
|
|
235
|
+
* and `1.25 × 100 MB` is a cage, not a ceiling.
|
|
236
|
+
*/
|
|
237
|
+
const CEILING_FLOOR_BYTES = 2 * GIB;
|
|
166
238
|
/**
|
|
167
239
|
* The two numbers, and the incident that decides them.
|
|
168
240
|
*
|
|
@@ -183,8 +255,10 @@ const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
|
183
255
|
*
|
|
184
256
|
* So the percentage has to be of what the machine can SPARE, not of what it has:
|
|
185
257
|
*
|
|
186
|
-
* headroom = MemAvailable +
|
|
187
|
-
* rewrite would walk the ceiling down by what we already hold
|
|
258
|
+
* headroom = MemAvailable + everything WE hold (ours is added back, or every
|
|
259
|
+
* rewrite would walk the ceiling down by what we already hold —
|
|
260
|
+
* and since 0.54.0 «ours» is the daemon's cgroup PLUS the sessions
|
|
261
|
+
* slice, because the agents moved out of the daemon's one)
|
|
188
262
|
* MemoryMax = headroom − reserve
|
|
189
263
|
* MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
|
|
190
264
|
*
|
|
@@ -203,11 +277,18 @@ const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
|
203
277
|
* - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
|
|
204
278
|
* machine, and a ceiling of «everything» is the bug this function exists to fix.
|
|
205
279
|
*/
|
|
206
|
-
export function memoryPolicy(facts, minCeilingBytes = facts
|
|
207
|
-
const { totalBytes, availableBytes, ownUsageBytes, uptimeSec } = facts;
|
|
280
|
+
export function memoryPolicy(facts, minCeilingBytes = managedUsageFloorBytes(facts)) {
|
|
281
|
+
const { totalBytes, availableBytes, ownUsageBytes, sessionsUsageBytes, uptimeSec } = facts;
|
|
282
|
+
const swapMaxBytes = Math.floor(Math.max(0, facts.swapTotalBytes ?? 0) * SESSIONS_SWAP_SHARE);
|
|
208
283
|
const withFloor = (maxBytes, measured, starved = false) => {
|
|
209
284
|
const ceiling = Math.floor(Math.max(maxBytes, minCeilingBytes));
|
|
210
|
-
return {
|
|
285
|
+
return {
|
|
286
|
+
maxBytes: ceiling,
|
|
287
|
+
highBytes: Math.floor(ceiling * 0.8),
|
|
288
|
+
swapMaxBytes,
|
|
289
|
+
measured,
|
|
290
|
+
starved,
|
|
291
|
+
};
|
|
211
292
|
};
|
|
212
293
|
// Still booting: the neighbours have not claimed their memory yet, so measuring
|
|
213
294
|
// now would hand back almost the whole machine. Take a fraction of TOTAL
|
|
@@ -227,13 +308,17 @@ export function memoryPolicy(facts, minCeilingBytes = facts.ownUsageBytes * CEIL
|
|
|
227
308
|
// while the cgroup sits at its ceiling. Proportional on a big machine, absolute
|
|
228
309
|
// on a small one, because 15 % of 4 GB is not enough to stay reachable.
|
|
229
310
|
const reserve = Math.max(1.5 * GIB, totalBytes * 0.15);
|
|
230
|
-
|
|
311
|
+
// Both cgroups are added back, and for the same reason ours always was: what
|
|
312
|
+
// they hold is already OUT of `MemAvailable`, so leaving the sessions out
|
|
313
|
+
// makes a loaded machine look starved and walks the ceiling down under the
|
|
314
|
+
// very sessions it covers.
|
|
315
|
+
const headroom = availableBytes + ownUsageBytes + sessionsUsageBytes;
|
|
231
316
|
const cap = totalBytes * 0.85;
|
|
232
317
|
// `min` with the cap, not a bare 2 GiB: below ~2.4 GB of RAM the floor would be
|
|
233
318
|
// ABOVE the cap and `clamp` would silently return the cap anyway — with the
|
|
234
319
|
// reserve ignored and the «2 GB or nothing» promise quietly broken. Saying it
|
|
235
320
|
// here makes the tiny-machine answer deliberate instead of accidental.
|
|
236
|
-
const floor = Math.min(
|
|
321
|
+
const floor = Math.min(CEILING_FLOOR_BYTES, cap);
|
|
237
322
|
const wanted = headroom - reserve;
|
|
238
323
|
return withFloor(clamp(wanted, floor, cap), true, wanted < floor);
|
|
239
324
|
}
|
|
@@ -244,8 +329,8 @@ function clamp(value, low, high) {
|
|
|
244
329
|
function asMiB(bytes) {
|
|
245
330
|
return `${Math.max(1, Math.floor(bytes / MIB))}M`;
|
|
246
331
|
}
|
|
247
|
-
/** `
|
|
248
|
-
function
|
|
332
|
+
/** One `kB` field of `/proc/meminfo`, in bytes; null when absent or unreadable. */
|
|
333
|
+
function memInfoField(name) {
|
|
249
334
|
let text;
|
|
250
335
|
try {
|
|
251
336
|
text = fs.readFileSync('/proc/meminfo', 'utf8');
|
|
@@ -253,13 +338,20 @@ function readMemInfo() {
|
|
|
253
338
|
catch {
|
|
254
339
|
return null;
|
|
255
340
|
}
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const
|
|
262
|
-
|
|
341
|
+
const match = new RegExp(`^${name}:\\s+(\\d+) kB$`, 'm').exec(text);
|
|
342
|
+
return match?.[1] ? Number(match[1]) * 1024 : null;
|
|
343
|
+
}
|
|
344
|
+
/** `MemTotal`/`MemAvailable`/`SwapTotal` in bytes, or null where there is no `/proc`. */
|
|
345
|
+
function readMemInfo() {
|
|
346
|
+
const totalBytes = memInfoField('MemTotal');
|
|
347
|
+
const availableBytes = memInfoField('MemAvailable');
|
|
348
|
+
// 0 is a real answer here (no swap), unlike the two above.
|
|
349
|
+
const swapTotalBytes = memInfoField('SwapTotal') ?? 0;
|
|
350
|
+
return totalBytes && availableBytes ? { totalBytes, availableBytes, swapTotalBytes } : null;
|
|
351
|
+
}
|
|
352
|
+
/** `SwapTotal` of this machine, or null where `/proc/meminfo` will not say. */
|
|
353
|
+
export function readSwapTotalBytes() {
|
|
354
|
+
return memInfoField('SwapTotal');
|
|
263
355
|
}
|
|
264
356
|
/**
|
|
265
357
|
* What our own cgroup currently holds.
|
|
@@ -271,8 +363,7 @@ function readMemInfo() {
|
|
|
271
363
|
* already using. Returns 0 on cgroup v1 or in a container without the file, which
|
|
272
364
|
* is the safe direction: a slightly lower ceiling, never a higher one.
|
|
273
365
|
*/
|
|
274
|
-
function
|
|
275
|
-
let cgroup;
|
|
366
|
+
export function readSelfCgroup() {
|
|
276
367
|
try {
|
|
277
368
|
const line = fs
|
|
278
369
|
.readFileSync('/proc/self/cgroup', 'utf8')
|
|
@@ -280,11 +371,16 @@ function readOwnCgroupUsage() {
|
|
|
280
371
|
.find((l) => l.startsWith('0::'));
|
|
281
372
|
if (!line)
|
|
282
373
|
return null; // cgroup v1 — no v2 path to read
|
|
283
|
-
|
|
374
|
+
return line.slice('0::'.length).trim();
|
|
284
375
|
}
|
|
285
376
|
catch {
|
|
286
377
|
return null;
|
|
287
378
|
}
|
|
379
|
+
}
|
|
380
|
+
export function readOwnCgroupMemory() {
|
|
381
|
+
const cgroup = readSelfCgroup();
|
|
382
|
+
if (cgroup === null)
|
|
383
|
+
return null;
|
|
288
384
|
// `/proc/self` is the CALLER. For the daemon that is the service, and this is
|
|
289
385
|
// the hot path. For `doctor --fix` or `install-service` typed over ssh it is a
|
|
290
386
|
// `session-N.scope` holding a few MB — measuring that and calling it «what the
|
|
@@ -294,32 +390,120 @@ function readOwnCgroupUsage() {
|
|
|
294
390
|
// know» and let the caller supply the number.
|
|
295
391
|
if (!cgroup.endsWith(`/${SERVICE_NAME}.service`))
|
|
296
392
|
return null;
|
|
297
|
-
return
|
|
393
|
+
return readCgroupMemory(path.join('/sys/fs/cgroup', cgroup));
|
|
298
394
|
}
|
|
299
|
-
|
|
300
|
-
* What the cgroup holds that `MemAvailable` has NOT already counted.
|
|
301
|
-
*
|
|
302
|
-
* `memory.current` is `anon + file + kernel`, and `file` is page cache — which
|
|
303
|
-
* `MemAvailable` already lists as reclaimable. Adding the whole of
|
|
304
|
-
* `memory.current` back to `MemAvailable` therefore counts our page cache twice
|
|
305
|
-
* and inflates the ceiling by exactly that much: measured at +19 % on this host
|
|
306
|
-
* (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
|
|
307
|
-
* precisely when memory is tightest. Subtracting `file` keeps the part we really
|
|
308
|
-
* do hold and cannot give back on demand.
|
|
309
|
-
*/
|
|
310
|
-
export function readCgroupUnreclaimable(dir) {
|
|
395
|
+
export function readCgroupMemory(dir) {
|
|
311
396
|
try {
|
|
312
|
-
const
|
|
313
|
-
|
|
397
|
+
const raw = fs.readFileSync(path.join(dir, 'memory.current'), 'utf8').trim();
|
|
398
|
+
const current = Number(raw);
|
|
399
|
+
if (!/^\d+$/.test(raw) || !Number.isSafeInteger(current))
|
|
314
400
|
return null;
|
|
315
|
-
const
|
|
316
|
-
const
|
|
317
|
-
|
|
401
|
+
const stat = fs.readFileSync(path.join(dir, 'memory.stat'), 'utf8');
|
|
402
|
+
const file = /^file (\d+)$/m.exec(stat);
|
|
403
|
+
// `file` COUNTS shmem, and shmem is the one page cache the kernel cannot
|
|
404
|
+
// drop under `MemorySwapMax=0`: there is nowhere to put it. Taking it back
|
|
405
|
+
// out is the difference between «cache we can give back» and «cache that
|
|
406
|
+
// has to be killed for», and only the first belongs on the reclaimable side.
|
|
407
|
+
const shmem = /^shmem (\d+)$/m.exec(stat);
|
|
408
|
+
const cache = Math.max(0, (file?.[1] ? Number(file[1]) : 0) - (shmem?.[1] ? Number(shmem[1]) : 0));
|
|
409
|
+
return { currentBytes: current, unreclaimableBytes: Math.max(0, current - cache) };
|
|
318
410
|
}
|
|
319
411
|
catch {
|
|
320
412
|
return null;
|
|
321
413
|
}
|
|
322
414
|
}
|
|
415
|
+
export function readCgroupUnreclaimable(dir) {
|
|
416
|
+
return readCgroupMemory(dir)?.unreclaimableBytes ?? null;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Where a slice unit's cgroup lives, worked out from the cgroup we are in.
|
|
420
|
+
*
|
|
421
|
+
* systemd's dash rule spells the hierarchy out: `devbridge-sessions.slice` sits
|
|
422
|
+
* inside `devbridge.slice`, which sits directly under the user manager's own
|
|
423
|
+
* cgroup — verified on this host with a throwaway `--slice=dbqa-probe-sub.slice`,
|
|
424
|
+
* which landed in `user@0.service/dbqa.slice/dbqa-probe.slice/dbqa-probe-sub.slice`.
|
|
425
|
+
*
|
|
426
|
+
* The user manager's cgroup is found rather than assembled: the runner runs as
|
|
427
|
+
* root and as a dedicated user (`user@0.service`, `user@1001.service`), and the
|
|
428
|
+
* same guess-the-path mistake that `readOwnCgroupUsage` avoids would silently
|
|
429
|
+
* return 0 here — which is exactly the blindness BLOCKER-1 was.
|
|
430
|
+
*
|
|
431
|
+
* Null means «there is no user manager above us», and then there is no
|
|
432
|
+
* `--user` slice for the sessions to be in either.
|
|
433
|
+
*/
|
|
434
|
+
export function sliceCgroupPath(selfCgroup, slice) {
|
|
435
|
+
const segments = selfCgroup.split('/').filter(Boolean);
|
|
436
|
+
const managerAt = segments.findIndex((segment) => /^user@\d+\.service$/.test(segment));
|
|
437
|
+
if (managerAt < 0)
|
|
438
|
+
return null;
|
|
439
|
+
const name = slice.replace(/\.slice$/, '');
|
|
440
|
+
const parts = name.split('-');
|
|
441
|
+
// `a-b-c.slice` → `a.slice/a-b.slice/a-b-c.slice`, systemd's own nesting.
|
|
442
|
+
const chain = parts.map((_, index) => `${parts.slice(0, index + 1).join('-')}.slice`);
|
|
443
|
+
return path.join('/sys/fs/cgroup', ...segments.slice(0, managerAt + 1), ...chain);
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* What the agents are holding right now, outside the daemon's own cgroup — or
|
|
447
|
+
* null where this machine cannot say.
|
|
448
|
+
*
|
|
449
|
+
* Read from the filesystem rather than through `systemctl show`, because this is
|
|
450
|
+
* the hourly path inside the daemon and it has to stay synchronous — the same
|
|
451
|
+
* reason `readOwnCgroupUsage` reads `/proc`. Callers that are not the daemon can
|
|
452
|
+
* pass the number in; see {@link readMemoryFacts}.
|
|
453
|
+
*
|
|
454
|
+
* A missing directory is 0 and not «unknown»: systemd removes the cgroup of an
|
|
455
|
+
* empty slice, so «no directory» means «no session is holding anything». The two
|
|
456
|
+
* are kept apart because they now lead to opposite decisions — 0 lets a blind
|
|
457
|
+
* ceiling be written onto the slice, «unknown» forbids it (see
|
|
458
|
+
* {@link buildSessionsSliceOverride}).
|
|
459
|
+
*/
|
|
460
|
+
export function readSessionsSliceMemory() {
|
|
461
|
+
const self = readSelfCgroup();
|
|
462
|
+
if (self === null)
|
|
463
|
+
return null; // cgroup v1, or a container without the file
|
|
464
|
+
const dir = sliceCgroupPath(self, SESSIONS_SLICE);
|
|
465
|
+
if (dir === null)
|
|
466
|
+
return null; // no user manager above us — no `--user` slice
|
|
467
|
+
if (!fs.existsSync(dir))
|
|
468
|
+
return { currentBytes: 0, unreclaimableBytes: 0 };
|
|
469
|
+
return readCgroupMemory(dir);
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* The same tri-state, read out of `systemctl show` instead of the filesystem —
|
|
473
|
+
* the authoritative source for the paths a person types (`doctor --fix`,
|
|
474
|
+
* `install-service`), which run outside the daemon's cgroup.
|
|
475
|
+
*
|
|
476
|
+
* `[not set]` is the ambiguous answer and the reason `activeState` is asked for
|
|
477
|
+
* as well: systemd prints it for a slice that has no cgroup (nothing has ever
|
|
478
|
+
* run there) AND for one whose accounting is off, and those two must not lead to
|
|
479
|
+
* the same decision. Only «systemd loaded the unit and it is not even active» is
|
|
480
|
+
* a positive statement that nothing can be killed by what we write; everything
|
|
481
|
+
* else is «unknown», which writes no ceiling at all.
|
|
482
|
+
*
|
|
483
|
+
* A failed `systemctl` call is null on both counts — including the 10-second
|
|
484
|
+
* timeout, which fires exactly on the overloaded machine this policy protects.
|
|
485
|
+
*/
|
|
486
|
+
export function parseSliceUsage(memoryCurrent, activeState) {
|
|
487
|
+
const raw = memoryCurrent?.trim() ?? '';
|
|
488
|
+
const value = Number(raw);
|
|
489
|
+
if (raw.length > 0 && Number.isFinite(value) && value >= 0)
|
|
490
|
+
return value;
|
|
491
|
+
return activeState?.trim() === 'inactive' ? 0 : null;
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* The slice's number for the callers that take one, and «unknown» kept apart
|
|
495
|
+
* from «empty»: an unreadable live slice must never be replaced with zero.
|
|
496
|
+
*
|
|
497
|
+
* The FLOOR reading, not the headroom one — this feeds a ceiling that has to
|
|
498
|
+
* clear what the slice holds, and where the split is unknown the whole reading
|
|
499
|
+
* has to be assumed unreclaimable.
|
|
500
|
+
*/
|
|
501
|
+
export function readSessionsSliceUsageOrNull() {
|
|
502
|
+
const memory = readSessionsSliceMemory();
|
|
503
|
+
if (memory === null)
|
|
504
|
+
return null;
|
|
505
|
+
return memory.unreclaimableBytes ?? memory.currentBytes;
|
|
506
|
+
}
|
|
323
507
|
function readUptimeSec() {
|
|
324
508
|
try {
|
|
325
509
|
return Number(fs.readFileSync('/proc/uptime', 'utf8').split(/\s+/)[0]) || 0;
|
|
@@ -331,17 +515,37 @@ function readUptimeSec() {
|
|
|
331
515
|
/**
|
|
332
516
|
* Everything `memoryPolicy` needs, straight off this machine.
|
|
333
517
|
*
|
|
334
|
-
*
|
|
335
|
-
* `install-service` run in the operator's own
|
|
336
|
-
* service's usage from `/proc/self
|
|
337
|
-
*
|
|
338
|
-
*
|
|
518
|
+
* Both readings are passed in by callers that are not the daemon — `doctor` and
|
|
519
|
+
* `install-service` run in the operator's own `session-N.scope` and cannot read
|
|
520
|
+
* the service's usage from `/proc/self`; they ask systemd instead
|
|
521
|
+
* (`readMemoryFactsFromSystemd`). Returns null when either cgroup is unknowable,
|
|
522
|
+
* because guessing there is the one dangerous direction: it removes the floor
|
|
523
|
+
* that stops a live session from being killed on the next `daemon-reload`.
|
|
524
|
+
*
|
|
525
|
+
* The default reads both from the filesystem, which is right for the daemon:
|
|
526
|
+
* its own cgroup through `/proc/self`, and the sessions slice through the user
|
|
527
|
+
* manager's cgroup, which sits above the daemon and the CLI alike.
|
|
339
528
|
*/
|
|
340
|
-
export function readMemoryFacts(
|
|
529
|
+
export function readMemoryFacts(own = readOwnCgroupMemory(), sessions = readSessionsSliceMemory()) {
|
|
341
530
|
const info = readMemInfo();
|
|
342
|
-
|
|
531
|
+
// Either cgroup being unreadable is the whole answer: the ceiling computed
|
|
532
|
+
// here is written to BOTH units, so a number that clears one and not the
|
|
533
|
+
// other is the collective kill of QA-2026-09-07 BLOCKER-1 with extra steps.
|
|
534
|
+
if (!info || own === null || sessions === null)
|
|
343
535
|
return null;
|
|
344
|
-
return {
|
|
536
|
+
return {
|
|
537
|
+
...info,
|
|
538
|
+
// Headroom gets only what `MemAvailable` has not already counted, and an
|
|
539
|
+
// unknown split contributes nothing: guessing high here would raise the
|
|
540
|
+
// ceiling on a machine we cannot see.
|
|
541
|
+
ownUsageBytes: own.unreclaimableBytes ?? 0,
|
|
542
|
+
sessionsUsageBytes: sessions.unreclaimableBytes ?? 0,
|
|
543
|
+
// The floor errs the other way, and has to: an unknown split may be all
|
|
544
|
+
// anonymous memory, and a ceiling written under it kills.
|
|
545
|
+
ownFloorBytes: own.unreclaimableBytes ?? own.currentBytes,
|
|
546
|
+
sessionsFloorBytes: sessions.unreclaimableBytes ?? sessions.currentBytes,
|
|
547
|
+
uptimeSec: readUptimeSec(),
|
|
548
|
+
};
|
|
345
549
|
}
|
|
346
550
|
/**
|
|
347
551
|
* `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
|
|
@@ -416,6 +620,135 @@ export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMem
|
|
|
416
620
|
'TasksMax=8192',
|
|
417
621
|
].join('\n') + '\n');
|
|
418
622
|
}
|
|
623
|
+
const MANAGED_HEADER = [
|
|
624
|
+
`${LIMITS_MARKER} ${LIMITS_VERSION}`,
|
|
625
|
+
'# Managed by devbridge-runner. Put your own overrides in a file that sorts',
|
|
626
|
+
'# after this one, or edit the unit itself — neither is touched by updates.',
|
|
627
|
+
'',
|
|
628
|
+
];
|
|
629
|
+
/**
|
|
630
|
+
* The ceiling over ALL sessions, on the slice they were moved into.
|
|
631
|
+
*
|
|
632
|
+
* Same number as the service's, and deliberately so — but it is a COPY, not a
|
|
633
|
+
* move, and that is deliberate too. The plan asked for the ceiling to be carried
|
|
634
|
+
* across; shrinking the service's to «what the daemon alone needs» would be
|
|
635
|
+
* correct only on a machine where the cage actually took. On cgroup v1, without
|
|
636
|
+
* a user bus, under a foreign supervisor — every `nice-only` machine — the
|
|
637
|
+
* sessions are still CHILDREN of the service, and a service ceiling sized for
|
|
638
|
+
* the daemon would cap all of them at a few hundred MB. This file is written
|
|
639
|
+
* before anything has probed which of the two machines this is, so the safe
|
|
640
|
+
* shape is the same number twice: on a caged machine the slice is the ceiling
|
|
641
|
+
* that binds, on an uncaged one the service is, and neither machine is ever
|
|
642
|
+
* left with a ceiling that is too small for what is under it.
|
|
643
|
+
*
|
|
644
|
+
* The price is that a caged machine formally permits `service + slice`. It is
|
|
645
|
+
* not the guarantee `memoryPolicy` computes, and it is written down here rather
|
|
646
|
+
* than glossed over (QA-2026-09-07 MINOR-4).
|
|
647
|
+
*
|
|
648
|
+
* `MemorySwapMax` is a BOUND on the slice for the same reason it is on every
|
|
649
|
+
* scope: a ceiling on resident memory alone is not a ceiling, it is a swap pump
|
|
650
|
+
* (a 200 MB cage allocated 2 GB and drained the host's swap during the spike).
|
|
651
|
+
* Since #387 the bound is a share of the machine's swap rather than 0
|
|
652
|
+
* (`SESSIONS_SWAP_SHARE`), because each session now carries a `MemoryHigh`
|
|
653
|
+
* brake, and a brake with no swap under it stalls the session instead of
|
|
654
|
+
* slowing it. The unmeasured branch still writes 0: no measurement, no swap to
|
|
655
|
+
* hand out.
|
|
656
|
+
*
|
|
657
|
+
* No `MemoryHigh` here. The brake belongs on each SCOPE (`session-cage.ts`):
|
|
658
|
+
* soft pressure on the slice would throttle every session on the machine to
|
|
659
|
+
* keep one runaway alive a little longer.
|
|
660
|
+
*
|
|
661
|
+
* `sessionsUsageBytes` is the door that used to lead around all of the above.
|
|
662
|
+
* `facts` is null whenever the SERVICE's `MemoryCurrent` is unreadable — a
|
|
663
|
+
* stopped service, or a runner under a foreign supervisor — and this file then
|
|
664
|
+
* fell back to a flat `MemoryMax=55%`. But the sessions are SIBLINGS of the
|
|
665
|
+
* service, not its children: they survive `systemctl --user stop
|
|
666
|
+
* devbridge-runner`, so «the service is not running» says nothing at all about
|
|
667
|
+
* what the slice is holding, and `doctor --fix` on such a machine wrote 55 % of
|
|
668
|
+
* total onto a live slice and then called `daemon-reload` — the collective kill
|
|
669
|
+
* of QA-2026-09-07 BLOCKER-1 arriving through a door with no policy behind it.
|
|
670
|
+
*
|
|
671
|
+
* So the slice's own usage is read separately, and the promise made on
|
|
672
|
+
* `buildLimitsOverride` («the floor belongs to the policy, so every caller gets
|
|
673
|
+
* it and none can opt out») holds on this path too:
|
|
674
|
+
* - a number → the ceiling clears it by the same 1.25 the policy uses, and
|
|
675
|
+
* never drops below what one agent needs;
|
|
676
|
+
* - 0 → nothing is running there, so the blind fraction can kill
|
|
677
|
+
* nothing and stays;
|
|
678
|
+
* - null → this machine could not say, and NO ceiling is written at all.
|
|
679
|
+
* Leaving whatever is in force in force is strictly better than
|
|
680
|
+
* applying an unfounded number to a cgroup that may be full: the
|
|
681
|
+
* daemon rewrites the file with a measured ceiling the moment it
|
|
682
|
+
* can measure one (the drift check treats a file with no
|
|
683
|
+
* `MemoryMax` as outdated).
|
|
684
|
+
*/
|
|
685
|
+
export function buildSessionsSliceOverride(facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull()) {
|
|
686
|
+
const memory = facts ? memoryPolicy(facts) : null;
|
|
687
|
+
return ([
|
|
688
|
+
...MANAGED_HEADER,
|
|
689
|
+
'[Slice]',
|
|
690
|
+
...(memory
|
|
691
|
+
? [
|
|
692
|
+
`# ${memory.measured ? 'measured headroom' : 'still booting — conservative fraction of total'}`,
|
|
693
|
+
`MemoryMax=${asMiB(memory.maxBytes)}`,
|
|
694
|
+
]
|
|
695
|
+
: unmeasuredSliceCeiling(sessionsUsageBytes)),
|
|
696
|
+
// The line the cage is built on. See `session-cage.ts` and the note above.
|
|
697
|
+
`MemorySwapMax=${memory && memory.swapMaxBytes > 0 ? asMiB(memory.swapMaxBytes) : '0'}`,
|
|
698
|
+
'TasksMax=8192',
|
|
699
|
+
'MemoryAccounting=yes',
|
|
700
|
+
'TasksAccounting=yes',
|
|
701
|
+
'CPUAccounting=yes',
|
|
702
|
+
].join('\n') + '\n');
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* The `[Slice]` lines for a machine whose memory the policy could not measure.
|
|
706
|
+
* See {@link buildSessionsSliceOverride} for why each of the three answers is
|
|
707
|
+
* what it is.
|
|
708
|
+
*/
|
|
709
|
+
function unmeasuredSliceCeiling(sessionsUsageBytes) {
|
|
710
|
+
if (sessionsUsageBytes === null) {
|
|
711
|
+
return [
|
|
712
|
+
'# machine not measurable, and neither is this slice — no ceiling is written',
|
|
713
|
+
'# here at all: whatever systemd has in force stays in force, because a number',
|
|
714
|
+
'# with nothing behind it kills live sessions on the next daemon-reload.',
|
|
715
|
+
];
|
|
716
|
+
}
|
|
717
|
+
if (sessionsUsageBytes === 0) {
|
|
718
|
+
return [
|
|
719
|
+
'# machine not measurable; this slice holds nothing, so a blind fraction can',
|
|
720
|
+
'# kill nothing — the same one the service falls back to',
|
|
721
|
+
'MemoryMax=55%',
|
|
722
|
+
];
|
|
723
|
+
}
|
|
724
|
+
return [
|
|
725
|
+
`# machine not measurable — floored by the ${asMiB(sessionsUsageBytes)} this slice holds right now`,
|
|
726
|
+
`MemoryMax=${asMiB(Math.max(sessionsUsageBytes * CEILING_HEADROOM_OVER_CURRENT, CEILING_FLOOR_BYTES))}`,
|
|
727
|
+
];
|
|
728
|
+
}
|
|
729
|
+
/**
|
|
730
|
+
* The CPU share of everything the agents run, against the daemon's own.
|
|
731
|
+
*
|
|
732
|
+
* `nice(2)` orders tasks INSIDE one cgroup. The moment a session gets a scope of
|
|
733
|
+
* its own it is no longer inside the service's cgroup, and the split between the
|
|
734
|
+
* two is decided by `cpu.weight` — which is 100 everywhere by default,
|
|
735
|
+
* `app.slice` (where the service lives) included. Without this file the cage
|
|
736
|
+
* would silently undo stage 1a and hand back the failure of 16.08: the daemon
|
|
737
|
+
* starved by its own children, four missed heartbeats, the server Offline and
|
|
738
|
+
* 504 on every session. 50 against 100 leaves the daemon two thirds.
|
|
739
|
+
*
|
|
740
|
+
* `process-priority.ts` stays exactly as it is: it is what protects the daemon
|
|
741
|
+
* on cgroup v1 and on every machine where the cage does not apply.
|
|
742
|
+
*/
|
|
743
|
+
export function buildDevbridgeSliceOverride() {
|
|
744
|
+
return ([
|
|
745
|
+
...MANAGED_HEADER,
|
|
746
|
+
'[Slice]',
|
|
747
|
+
`CPUWeight=${SESSION_CPU_WEIGHT}`,
|
|
748
|
+
'CPUAccounting=yes',
|
|
749
|
+
'MemoryAccounting=yes',
|
|
750
|
+
].join('\n') + '\n');
|
|
751
|
+
}
|
|
419
752
|
/**
|
|
420
753
|
* Is the shipped resource policy missing or from an older runner?
|
|
421
754
|
*
|
|
@@ -427,9 +760,21 @@ export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMem
|
|
|
427
760
|
* version-based answer.
|
|
428
761
|
*/
|
|
429
762
|
export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome(), facts = null) {
|
|
763
|
+
// Three files since 0.54.0, and any one of them being stale means the policy
|
|
764
|
+
// is: the ceiling on the service protects the daemon, the ceiling on
|
|
765
|
+
// `devbridge-sessions.slice` protects the machine from the sessions that
|
|
766
|
+
// LEFT the service's cgroup, and the weight on `devbridge.slice` is what
|
|
767
|
+
// keeps the daemon ahead of them. A runner that shipped the cage without the
|
|
768
|
+
// weight would be strictly worse than one that shipped neither.
|
|
769
|
+
return (fileIsOutdated(readFile, limitsOverridePath(home), facts) ||
|
|
770
|
+
fileIsOutdated(readFile, sessionsSliceOverridePath(home), facts) ||
|
|
771
|
+
// No ceiling in this one, so nothing in it can drift.
|
|
772
|
+
fileIsOutdated(readFile, devbridgeSliceOverridePath(home), null));
|
|
773
|
+
}
|
|
774
|
+
function fileIsOutdated(readFile, target, facts) {
|
|
430
775
|
let contents;
|
|
431
776
|
try {
|
|
432
|
-
contents = readFile(
|
|
777
|
+
contents = readFile(target);
|
|
433
778
|
}
|
|
434
779
|
catch {
|
|
435
780
|
return true; // never written — every server that predates 0.21.0
|
|
@@ -457,6 +802,14 @@ export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'u
|
|
|
457
802
|
*/
|
|
458
803
|
const CEILING_DRIFT_TOLERANCE = 0.15;
|
|
459
804
|
function memoryCeilingHasDrifted(contents, facts) {
|
|
805
|
+
// The swap share drifts for its own reason, and on its own schedule: an
|
|
806
|
+
// operator who enlarges the machine's swapfile (the one lever that makes the
|
|
807
|
+
// brake a brake rather than a freeze) would otherwise keep the old, smaller
|
|
808
|
+
// share until the next `LIMITS_VERSION` — and each session would keep getting
|
|
809
|
+
// half of it (#387 QA). Zero to non-zero and back is always drift; above that
|
|
810
|
+
// the same tolerance as the ceiling.
|
|
811
|
+
if (swapShareHasDrifted(contents, facts))
|
|
812
|
+
return true;
|
|
460
813
|
// Never re-measure a machine that is still booting: the static pair it is
|
|
461
814
|
// holding is deliberate, and «drift» against a lie is not drift.
|
|
462
815
|
if (!memoryPolicy(facts).measured)
|
|
@@ -479,6 +832,19 @@ function memoryCeilingHasDrifted(contents, facts) {
|
|
|
479
832
|
const wanted = memoryPolicy(facts).maxBytes;
|
|
480
833
|
return Math.abs(writtenBytes - wanted) / wanted > CEILING_DRIFT_TOLERANCE;
|
|
481
834
|
}
|
|
835
|
+
function swapShareHasDrifted(contents, facts) {
|
|
836
|
+
const written = [...contents.matchAll(/^MemorySwapMax=(\d+)([KMG]?)$/gm)].at(-1);
|
|
837
|
+
// Not written at all: only the slice file carries this line, and the service
|
|
838
|
+
// file legitimately has none. Silence is not drift.
|
|
839
|
+
if (!written?.[1])
|
|
840
|
+
return false;
|
|
841
|
+
const scale = { '': 1, K: 1024, M: MIB, G: 1024 * MIB }[written[2] ?? ''] ?? MIB;
|
|
842
|
+
const writtenBytes = Number(written[1]) * scale;
|
|
843
|
+
const wanted = memoryPolicy(facts).swapMaxBytes;
|
|
844
|
+
if (writtenBytes === 0 || wanted === 0)
|
|
845
|
+
return writtenBytes !== wanted;
|
|
846
|
+
return Math.abs(writtenBytes - wanted) / wanted > CEILING_DRIFT_TOLERANCE;
|
|
847
|
+
}
|
|
482
848
|
/**
|
|
483
849
|
* Write the drop-in. Returns false when nothing needed doing.
|
|
484
850
|
*
|
|
@@ -487,14 +853,27 @@ function memoryCeilingHasDrifted(contents, facts) {
|
|
|
487
853
|
* arrive at the same number — otherwise a write whose floor was binding would be
|
|
488
854
|
* seen as drifted on the very next call and rewritten forever.
|
|
489
855
|
*/
|
|
490
|
-
export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts()) {
|
|
856
|
+
export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull()) {
|
|
857
|
+
// Unknown usage can be a busy service whose systemd query timed out. Do not
|
|
858
|
+
// replace an existing drop-in with a guessed percentage, or remove a limit
|
|
859
|
+
// by rewriting the file without its MemoryMax line. Defer the whole policy.
|
|
860
|
+
if (facts === null)
|
|
861
|
+
return false;
|
|
491
862
|
if (!force && !limitsOverrideIsOutdated(undefined, home, facts))
|
|
492
863
|
return false;
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
864
|
+
// All three or none. They are one policy split across three units only
|
|
865
|
+
// because systemd has no other way to say it, and a machine holding two of
|
|
866
|
+
// them is a machine whose sessions are capped but whose daemon is not
|
|
867
|
+
// prioritised — the regression described on `buildDevbridgeSliceOverride`.
|
|
868
|
+
write(limitsOverridePath(home), buildLimitsOverride(undefined, facts));
|
|
869
|
+
write(sessionsSliceOverridePath(home), buildSessionsSliceOverride(facts, sessionsUsageBytes));
|
|
870
|
+
write(devbridgeSliceOverridePath(home), buildDevbridgeSliceOverride());
|
|
496
871
|
return true;
|
|
497
872
|
}
|
|
873
|
+
function write(target, contents) {
|
|
874
|
+
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
875
|
+
fs.writeFileSync(target, contents, { mode: 0o644 });
|
|
876
|
+
}
|
|
498
877
|
/**
|
|
499
878
|
* Does the installed unit point at something that no longer exists?
|
|
500
879
|
*
|