@bridge4dev/runner 0.53.0 → 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +96 -1
- package/dist/adapters/codex-protocol.d.ts +11 -0
- package/dist/adapters/codex-protocol.js +41 -3
- package/dist/adapters/codex.js +4 -0
- package/dist/host-load.d.ts +156 -0
- package/dist/host-load.js +223 -0
- package/dist/index.js +190 -40
- package/dist/process-priority.d.ts +55 -0
- package/dist/process-priority.js +99 -0
- package/dist/protocol.d.ts +25 -3
- package/dist/recipe-schema.d.ts +6 -6
- package/dist/self-update.js +43 -2
- package/dist/service-unit.d.ts +232 -10
- package/dist/service-unit.js +372 -43
- package/dist/session-cage.d.ts +297 -0
- package/dist/session-cage.js +755 -0
- package/dist/supervisor.d.ts +57 -0
- package/dist/supervisor.js +72 -0
- package/dist/systemd-memory.d.ts +35 -0
- package/dist/systemd-memory.js +115 -0
- package/dist/verify.js +28 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/service-unit.js
CHANGED
|
@@ -129,13 +129,56 @@ export function buildUnit(execStart, nodeBinary = process.execPath) {
|
|
|
129
129
|
* which is the only way to fix the servers that already have the bad numbers
|
|
130
130
|
* baked in — and it never overwrites a unit the operator edited by hand.
|
|
131
131
|
*/
|
|
132
|
-
export const LIMITS_VERSION =
|
|
132
|
+
export const LIMITS_VERSION = 4;
|
|
133
133
|
const LIMITS_MARKER = '# devbridge-limits-version:';
|
|
134
134
|
/** `zz-` so it sorts last: an operator's own drop-in should still win. */
|
|
135
135
|
const LIMITS_FILE = 'zz-devbridge-limits.conf';
|
|
136
|
+
/**
|
|
137
|
+
* Where the agent sessions live once they have a cage of their own.
|
|
138
|
+
*
|
|
139
|
+
* A `systemd-run --scope` is a SIBLING of the service, not a child of it: a
|
|
140
|
+
* session started that way leaves the service's `MemoryMax`, `CPUQuota`,
|
|
141
|
+
* `OOMPolicy=continue` and `KillMode=control-group` behind entirely. So the
|
|
142
|
+
* collective ceiling has to move with them, onto the slice — otherwise the cage
|
|
143
|
+
* per session would arrive at the price of the ceiling over all of them.
|
|
144
|
+
*
|
|
145
|
+
* The dash is systemd's hierarchy separator: `devbridge-sessions.slice` is a
|
|
146
|
+
* child of `devbridge.slice`, which is where the CPU share for everything the
|
|
147
|
+
* agents run is set.
|
|
148
|
+
*/
|
|
149
|
+
export const SESSIONS_SLICE = 'devbridge-sessions.slice';
|
|
150
|
+
export const DEVBRIDGE_SLICE = 'devbridge.slice';
|
|
151
|
+
/**
|
|
152
|
+
* Half the default weight, on `devbridge.slice` and on every session scope.
|
|
153
|
+
*
|
|
154
|
+
* Lives here rather than in `session-cage.ts` because the number has to be the
|
|
155
|
+
* SAME in both places — the slice sets the share of the agents against the
|
|
156
|
+
* daemon, the scope sets one session's share against another's — and a
|
|
157
|
+
* duplicated literal is how those two drift apart. Why 50 and what it replaces:
|
|
158
|
+
* `buildDevbridgeSliceOverride` below.
|
|
159
|
+
*/
|
|
160
|
+
export const SESSION_CPU_WEIGHT = 50;
|
|
136
161
|
export function limitsOverridePath(home = systemdUserHome()) {
|
|
137
162
|
return path.join(home, '.config', 'systemd', 'user', `${SERVICE_NAME}.service.d`, LIMITS_FILE);
|
|
138
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* Drop-in path for a slice unit that has no unit FILE at all.
|
|
166
|
+
*
|
|
167
|
+
* systemd synthesises `devbridge-sessions.slice` the first time something asks
|
|
168
|
+
* for it, and it reads drop-ins for the synthesised unit exactly as for a real
|
|
169
|
+
* one — verified on this machine: a `[Slice] MemoryMax=64M` drop-in with no
|
|
170
|
+
* fragment gave `MemoryMax=67108864` on the live slice. So there is no unit file
|
|
171
|
+
* to write and no unit file to keep in sync; the policy is the drop-in.
|
|
172
|
+
*/
|
|
173
|
+
export function sliceOverridePath(slice, home = systemdUserHome()) {
|
|
174
|
+
return path.join(home, '.config', 'systemd', 'user', `${slice}.d`, LIMITS_FILE);
|
|
175
|
+
}
|
|
176
|
+
export function sessionsSliceOverridePath(home = systemdUserHome()) {
|
|
177
|
+
return sliceOverridePath(SESSIONS_SLICE, home);
|
|
178
|
+
}
|
|
179
|
+
export function devbridgeSliceOverridePath(home = systemdUserHome()) {
|
|
180
|
+
return sliceOverridePath(DEVBRIDGE_SLICE, home);
|
|
181
|
+
}
|
|
139
182
|
const MIB = 1024 * 1024;
|
|
140
183
|
const GIB = 1024 * MIB;
|
|
141
184
|
/**
|
|
@@ -149,7 +192,7 @@ const GIB = 1024 * MIB;
|
|
|
149
192
|
*/
|
|
150
193
|
export const BOOT_SETTLE_SEC = 600;
|
|
151
194
|
/**
|
|
152
|
-
* The ceiling is never written BELOW what the
|
|
195
|
+
* The ceiling is never written BELOW what the cgroups already hold.
|
|
153
196
|
*
|
|
154
197
|
* Until 0.39.0 this policy only ever REMOVED a limit, so applying it to a running
|
|
155
198
|
* service was free. Now it sets one, and systemd applies `memory.max` to a LIVE
|
|
@@ -158,11 +201,29 @@ export const BOOT_SETTLE_SEC = 600;
|
|
|
158
201
|
* running yet — but `doctor --fix` on a busy machine would otherwise take out
|
|
159
202
|
* somebody's session as a side effect of running a diagnostic.
|
|
160
203
|
*
|
|
204
|
+
* `max` of the two holders, not the sum: the SAME number is written to the
|
|
205
|
+
* service and to `devbridge-sessions.slice`, so it has to clear whichever of the
|
|
206
|
+
* two is fuller — each cgroup is measured against it on its own.
|
|
207
|
+
*
|
|
161
208
|
* The cost is that a machine fixed WHILE a runaway is in progress writes a ceiling
|
|
162
209
|
* above that runaway, which protects nothing. It resolves itself: the next daemon
|
|
163
|
-
* start measures
|
|
210
|
+
* start measures idle cgroups, and the drift check rewrites the inflated number.
|
|
164
211
|
*/
|
|
165
212
|
const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
213
|
+
/** What the ceiling about to be written has to clear, in either cgroup. */
|
|
214
|
+
export function managedUsageFloorBytes(facts) {
|
|
215
|
+
return (Math.max(facts.ownFloorBytes ?? facts.ownUsageBytes, facts.sessionsFloorBytes ?? facts.sessionsUsageBytes) * CEILING_HEADROOM_OVER_CURRENT);
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Below this a dev server cannot run one agent, so the policy never goes under it.
|
|
219
|
+
*
|
|
220
|
+
* Named rather than repeated because it is now used twice: as the lower clamp of
|
|
221
|
+
* the measured ceiling, and as the lower clamp of the ceiling written to the
|
|
222
|
+
* sessions slice on a machine the policy could not measure at all — where the
|
|
223
|
+
* only other number available is what that slice happens to hold this second,
|
|
224
|
+
* and `1.25 × 100 MB` is a cage, not a ceiling.
|
|
225
|
+
*/
|
|
226
|
+
const CEILING_FLOOR_BYTES = 2 * GIB;
|
|
166
227
|
/**
|
|
167
228
|
* The two numbers, and the incident that decides them.
|
|
168
229
|
*
|
|
@@ -183,8 +244,10 @@ const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
|
183
244
|
*
|
|
184
245
|
* So the percentage has to be of what the machine can SPARE, not of what it has:
|
|
185
246
|
*
|
|
186
|
-
* headroom = MemAvailable +
|
|
187
|
-
* rewrite would walk the ceiling down by what we already hold
|
|
247
|
+
* headroom = MemAvailable + everything WE hold (ours is added back, or every
|
|
248
|
+
* rewrite would walk the ceiling down by what we already hold —
|
|
249
|
+
* and since 0.54.0 «ours» is the daemon's cgroup PLUS the sessions
|
|
250
|
+
* slice, because the agents moved out of the daemon's one)
|
|
188
251
|
* MemoryMax = headroom − reserve
|
|
189
252
|
* MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
|
|
190
253
|
*
|
|
@@ -203,8 +266,8 @@ const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
|
203
266
|
* - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
|
|
204
267
|
* machine, and a ceiling of «everything» is the bug this function exists to fix.
|
|
205
268
|
*/
|
|
206
|
-
export function memoryPolicy(facts, minCeilingBytes = facts
|
|
207
|
-
const { totalBytes, availableBytes, ownUsageBytes, uptimeSec } = facts;
|
|
269
|
+
export function memoryPolicy(facts, minCeilingBytes = managedUsageFloorBytes(facts)) {
|
|
270
|
+
const { totalBytes, availableBytes, ownUsageBytes, sessionsUsageBytes, uptimeSec } = facts;
|
|
208
271
|
const withFloor = (maxBytes, measured, starved = false) => {
|
|
209
272
|
const ceiling = Math.floor(Math.max(maxBytes, minCeilingBytes));
|
|
210
273
|
return { maxBytes: ceiling, highBytes: Math.floor(ceiling * 0.8), measured, starved };
|
|
@@ -227,13 +290,17 @@ export function memoryPolicy(facts, minCeilingBytes = facts.ownUsageBytes * CEIL
|
|
|
227
290
|
// while the cgroup sits at its ceiling. Proportional on a big machine, absolute
|
|
228
291
|
// on a small one, because 15 % of 4 GB is not enough to stay reachable.
|
|
229
292
|
const reserve = Math.max(1.5 * GIB, totalBytes * 0.15);
|
|
230
|
-
|
|
293
|
+
// Both cgroups are added back, and for the same reason ours always was: what
|
|
294
|
+
// they hold is already OUT of `MemAvailable`, so leaving the sessions out
|
|
295
|
+
// makes a loaded machine look starved and walks the ceiling down under the
|
|
296
|
+
// very sessions it covers.
|
|
297
|
+
const headroom = availableBytes + ownUsageBytes + sessionsUsageBytes;
|
|
231
298
|
const cap = totalBytes * 0.85;
|
|
232
299
|
// `min` with the cap, not a bare 2 GiB: below ~2.4 GB of RAM the floor would be
|
|
233
300
|
// ABOVE the cap and `clamp` would silently return the cap anyway — with the
|
|
234
301
|
// reserve ignored and the «2 GB or nothing» promise quietly broken. Saying it
|
|
235
302
|
// here makes the tiny-machine answer deliberate instead of accidental.
|
|
236
|
-
const floor = Math.min(
|
|
303
|
+
const floor = Math.min(CEILING_FLOOR_BYTES, cap);
|
|
237
304
|
const wanted = headroom - reserve;
|
|
238
305
|
return withFloor(clamp(wanted, floor, cap), true, wanted < floor);
|
|
239
306
|
}
|
|
@@ -271,8 +338,7 @@ function readMemInfo() {
|
|
|
271
338
|
* already using. Returns 0 on cgroup v1 or in a container without the file, which
|
|
272
339
|
* is the safe direction: a slightly lower ceiling, never a higher one.
|
|
273
340
|
*/
|
|
274
|
-
function
|
|
275
|
-
let cgroup;
|
|
341
|
+
function readSelfCgroup() {
|
|
276
342
|
try {
|
|
277
343
|
const line = fs
|
|
278
344
|
.readFileSync('/proc/self/cgroup', 'utf8')
|
|
@@ -280,11 +346,16 @@ function readOwnCgroupUsage() {
|
|
|
280
346
|
.find((l) => l.startsWith('0::'));
|
|
281
347
|
if (!line)
|
|
282
348
|
return null; // cgroup v1 — no v2 path to read
|
|
283
|
-
|
|
349
|
+
return line.slice('0::'.length).trim();
|
|
284
350
|
}
|
|
285
351
|
catch {
|
|
286
352
|
return null;
|
|
287
353
|
}
|
|
354
|
+
}
|
|
355
|
+
export function readOwnCgroupMemory() {
|
|
356
|
+
const cgroup = readSelfCgroup();
|
|
357
|
+
if (cgroup === null)
|
|
358
|
+
return null;
|
|
288
359
|
// `/proc/self` is the CALLER. For the daemon that is the service, and this is
|
|
289
360
|
// the hot path. For `doctor --fix` or `install-service` typed over ssh it is a
|
|
290
361
|
// `session-N.scope` holding a few MB — measuring that and calling it «what the
|
|
@@ -294,32 +365,120 @@ function readOwnCgroupUsage() {
|
|
|
294
365
|
// know» and let the caller supply the number.
|
|
295
366
|
if (!cgroup.endsWith(`/${SERVICE_NAME}.service`))
|
|
296
367
|
return null;
|
|
297
|
-
return
|
|
368
|
+
return readCgroupMemory(path.join('/sys/fs/cgroup', cgroup));
|
|
298
369
|
}
|
|
299
|
-
|
|
300
|
-
* What the cgroup holds that `MemAvailable` has NOT already counted.
|
|
301
|
-
*
|
|
302
|
-
* `memory.current` is `anon + file + kernel`, and `file` is page cache — which
|
|
303
|
-
* `MemAvailable` already lists as reclaimable. Adding the whole of
|
|
304
|
-
* `memory.current` back to `MemAvailable` therefore counts our page cache twice
|
|
305
|
-
* and inflates the ceiling by exactly that much: measured at +19 % on this host
|
|
306
|
-
* (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
|
|
307
|
-
* precisely when memory is tightest. Subtracting `file` keeps the part we really
|
|
308
|
-
* do hold and cannot give back on demand.
|
|
309
|
-
*/
|
|
310
|
-
export function readCgroupUnreclaimable(dir) {
|
|
370
|
+
export function readCgroupMemory(dir) {
|
|
311
371
|
try {
|
|
312
|
-
const
|
|
313
|
-
|
|
372
|
+
const raw = fs.readFileSync(path.join(dir, 'memory.current'), 'utf8').trim();
|
|
373
|
+
const current = Number(raw);
|
|
374
|
+
if (!/^\d+$/.test(raw) || !Number.isSafeInteger(current))
|
|
314
375
|
return null;
|
|
315
|
-
const
|
|
316
|
-
const
|
|
317
|
-
|
|
376
|
+
const stat = fs.readFileSync(path.join(dir, 'memory.stat'), 'utf8');
|
|
377
|
+
const file = /^file (\d+)$/m.exec(stat);
|
|
378
|
+
// `file` COUNTS shmem, and shmem is the one page cache the kernel cannot
|
|
379
|
+
// drop under `MemorySwapMax=0`: there is nowhere to put it. Taking it back
|
|
380
|
+
// out is the difference between «cache we can give back» and «cache that
|
|
381
|
+
// has to be killed for», and only the first belongs on the reclaimable side.
|
|
382
|
+
const shmem = /^shmem (\d+)$/m.exec(stat);
|
|
383
|
+
const cache = Math.max(0, (file?.[1] ? Number(file[1]) : 0) - (shmem?.[1] ? Number(shmem[1]) : 0));
|
|
384
|
+
return { currentBytes: current, unreclaimableBytes: Math.max(0, current - cache) };
|
|
318
385
|
}
|
|
319
386
|
catch {
|
|
320
387
|
return null;
|
|
321
388
|
}
|
|
322
389
|
}
|
|
390
|
+
export function readCgroupUnreclaimable(dir) {
|
|
391
|
+
return readCgroupMemory(dir)?.unreclaimableBytes ?? null;
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Where a slice unit's cgroup lives, worked out from the cgroup we are in.
|
|
395
|
+
*
|
|
396
|
+
* systemd's dash rule spells the hierarchy out: `devbridge-sessions.slice` sits
|
|
397
|
+
* inside `devbridge.slice`, which sits directly under the user manager's own
|
|
398
|
+
* cgroup — verified on this host with a throwaway `--slice=dbqa-probe-sub.slice`,
|
|
399
|
+
* which landed in `user@0.service/dbqa.slice/dbqa-probe.slice/dbqa-probe-sub.slice`.
|
|
400
|
+
*
|
|
401
|
+
* The user manager's cgroup is found rather than assembled: the runner runs as
|
|
402
|
+
* root and as a dedicated user (`user@0.service`, `user@1001.service`), and the
|
|
403
|
+
* same guess-the-path mistake that `readOwnCgroupUsage` avoids would silently
|
|
404
|
+
* return 0 here — which is exactly the blindness BLOCKER-1 was.
|
|
405
|
+
*
|
|
406
|
+
* Null means «there is no user manager above us», and then there is no
|
|
407
|
+
* `--user` slice for the sessions to be in either.
|
|
408
|
+
*/
|
|
409
|
+
export function sliceCgroupPath(selfCgroup, slice) {
|
|
410
|
+
const segments = selfCgroup.split('/').filter(Boolean);
|
|
411
|
+
const managerAt = segments.findIndex((segment) => /^user@\d+\.service$/.test(segment));
|
|
412
|
+
if (managerAt < 0)
|
|
413
|
+
return null;
|
|
414
|
+
const name = slice.replace(/\.slice$/, '');
|
|
415
|
+
const parts = name.split('-');
|
|
416
|
+
// `a-b-c.slice` → `a.slice/a-b.slice/a-b-c.slice`, systemd's own nesting.
|
|
417
|
+
const chain = parts.map((_, index) => `${parts.slice(0, index + 1).join('-')}.slice`);
|
|
418
|
+
return path.join('/sys/fs/cgroup', ...segments.slice(0, managerAt + 1), ...chain);
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* What the agents are holding right now, outside the daemon's own cgroup — or
|
|
422
|
+
* null where this machine cannot say.
|
|
423
|
+
*
|
|
424
|
+
* Read from the filesystem rather than through `systemctl show`, because this is
|
|
425
|
+
* the hourly path inside the daemon and it has to stay synchronous — the same
|
|
426
|
+
* reason `readOwnCgroupUsage` reads `/proc`. Callers that are not the daemon can
|
|
427
|
+
* pass the number in; see {@link readMemoryFacts}.
|
|
428
|
+
*
|
|
429
|
+
* A missing directory is 0 and not «unknown»: systemd removes the cgroup of an
|
|
430
|
+
* empty slice, so «no directory» means «no session is holding anything». The two
|
|
431
|
+
* are kept apart because they now lead to opposite decisions — 0 lets a blind
|
|
432
|
+
* ceiling be written onto the slice, «unknown» forbids it (see
|
|
433
|
+
* {@link buildSessionsSliceOverride}).
|
|
434
|
+
*/
|
|
435
|
+
export function readSessionsSliceMemory() {
|
|
436
|
+
const self = readSelfCgroup();
|
|
437
|
+
if (self === null)
|
|
438
|
+
return null; // cgroup v1, or a container without the file
|
|
439
|
+
const dir = sliceCgroupPath(self, SESSIONS_SLICE);
|
|
440
|
+
if (dir === null)
|
|
441
|
+
return null; // no user manager above us — no `--user` slice
|
|
442
|
+
if (!fs.existsSync(dir))
|
|
443
|
+
return { currentBytes: 0, unreclaimableBytes: 0 };
|
|
444
|
+
return readCgroupMemory(dir);
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* The same tri-state, read out of `systemctl show` instead of the filesystem —
|
|
448
|
+
* the authoritative source for the paths a person types (`doctor --fix`,
|
|
449
|
+
* `install-service`), which run outside the daemon's cgroup.
|
|
450
|
+
*
|
|
451
|
+
* `[not set]` is the ambiguous answer and the reason `activeState` is asked for
|
|
452
|
+
* as well: systemd prints it for a slice that has no cgroup (nothing has ever
|
|
453
|
+
* run there) AND for one whose accounting is off, and those two must not lead to
|
|
454
|
+
* the same decision. Only «systemd loaded the unit and it is not even active» is
|
|
455
|
+
* a positive statement that nothing can be killed by what we write; everything
|
|
456
|
+
* else is «unknown», which writes no ceiling at all.
|
|
457
|
+
*
|
|
458
|
+
* A failed `systemctl` call is null on both counts — including the 10-second
|
|
459
|
+
* timeout, which fires exactly on the overloaded machine this policy protects.
|
|
460
|
+
*/
|
|
461
|
+
export function parseSliceUsage(memoryCurrent, activeState) {
|
|
462
|
+
const raw = memoryCurrent?.trim() ?? '';
|
|
463
|
+
const value = Number(raw);
|
|
464
|
+
if (raw.length > 0 && Number.isFinite(value) && value >= 0)
|
|
465
|
+
return value;
|
|
466
|
+
return activeState?.trim() === 'inactive' ? 0 : null;
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* The slice's number for the callers that take one, and «unknown» kept apart
|
|
470
|
+
* from «empty»: an unreadable live slice must never be replaced with zero.
|
|
471
|
+
*
|
|
472
|
+
* The FLOOR reading, not the headroom one — this feeds a ceiling that has to
|
|
473
|
+
* clear what the slice holds, and where the split is unknown the whole reading
|
|
474
|
+
* has to be assumed unreclaimable.
|
|
475
|
+
*/
|
|
476
|
+
export function readSessionsSliceUsageOrNull() {
|
|
477
|
+
const memory = readSessionsSliceMemory();
|
|
478
|
+
if (memory === null)
|
|
479
|
+
return null;
|
|
480
|
+
return memory.unreclaimableBytes ?? memory.currentBytes;
|
|
481
|
+
}
|
|
323
482
|
function readUptimeSec() {
|
|
324
483
|
try {
|
|
325
484
|
return Number(fs.readFileSync('/proc/uptime', 'utf8').split(/\s+/)[0]) || 0;
|
|
@@ -331,17 +490,37 @@ function readUptimeSec() {
|
|
|
331
490
|
/**
|
|
332
491
|
* Everything `memoryPolicy` needs, straight off this machine.
|
|
333
492
|
*
|
|
334
|
-
*
|
|
335
|
-
* `install-service` run in the operator's own
|
|
336
|
-
* service's usage from `/proc/self
|
|
337
|
-
*
|
|
338
|
-
*
|
|
493
|
+
* Both readings are passed in by callers that are not the daemon — `doctor` and
|
|
494
|
+
* `install-service` run in the operator's own `session-N.scope` and cannot read
|
|
495
|
+
* the service's usage from `/proc/self`; they ask systemd instead
|
|
496
|
+
* (`readMemoryFactsFromSystemd`). Returns null when either cgroup is unknowable,
|
|
497
|
+
* because guessing there is the one dangerous direction: it removes the floor
|
|
498
|
+
* that stops a live session from being killed on the next `daemon-reload`.
|
|
499
|
+
*
|
|
500
|
+
* The default reads both from the filesystem, which is right for the daemon:
|
|
501
|
+
* its own cgroup through `/proc/self`, and the sessions slice through the user
|
|
502
|
+
* manager's cgroup, which sits above the daemon and the CLI alike.
|
|
339
503
|
*/
|
|
340
|
-
export function readMemoryFacts(
|
|
504
|
+
export function readMemoryFacts(own = readOwnCgroupMemory(), sessions = readSessionsSliceMemory()) {
|
|
341
505
|
const info = readMemInfo();
|
|
342
|
-
|
|
506
|
+
// Either cgroup being unreadable is the whole answer: the ceiling computed
|
|
507
|
+
// here is written to BOTH units, so a number that clears one and not the
|
|
508
|
+
// other is the collective kill of QA-2026-09-07 BLOCKER-1 with extra steps.
|
|
509
|
+
if (!info || own === null || sessions === null)
|
|
343
510
|
return null;
|
|
344
|
-
return {
|
|
511
|
+
return {
|
|
512
|
+
...info,
|
|
513
|
+
// Headroom gets only what `MemAvailable` has not already counted, and an
|
|
514
|
+
// unknown split contributes nothing: guessing high here would raise the
|
|
515
|
+
// ceiling on a machine we cannot see.
|
|
516
|
+
ownUsageBytes: own.unreclaimableBytes ?? 0,
|
|
517
|
+
sessionsUsageBytes: sessions.unreclaimableBytes ?? 0,
|
|
518
|
+
// The floor errs the other way, and has to: an unknown split may be all
|
|
519
|
+
// anonymous memory, and a ceiling written under it kills.
|
|
520
|
+
ownFloorBytes: own.unreclaimableBytes ?? own.currentBytes,
|
|
521
|
+
sessionsFloorBytes: sessions.unreclaimableBytes ?? sessions.currentBytes,
|
|
522
|
+
uptimeSec: readUptimeSec(),
|
|
523
|
+
};
|
|
345
524
|
}
|
|
346
525
|
/**
|
|
347
526
|
* `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
|
|
@@ -416,6 +595,131 @@ export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMem
|
|
|
416
595
|
'TasksMax=8192',
|
|
417
596
|
].join('\n') + '\n');
|
|
418
597
|
}
|
|
598
|
+
const MANAGED_HEADER = [
|
|
599
|
+
`${LIMITS_MARKER} ${LIMITS_VERSION}`,
|
|
600
|
+
'# Managed by devbridge-runner. Put your own overrides in a file that sorts',
|
|
601
|
+
'# after this one, or edit the unit itself — neither is touched by updates.',
|
|
602
|
+
'',
|
|
603
|
+
];
|
|
604
|
+
/**
|
|
605
|
+
* The ceiling over ALL sessions, on the slice they were moved into.
|
|
606
|
+
*
|
|
607
|
+
* Same number as the service's, and deliberately so — but it is a COPY, not a
|
|
608
|
+
* move, and that is deliberate too. The plan asked for the ceiling to be carried
|
|
609
|
+
* across; shrinking the service's to «what the daemon alone needs» would be
|
|
610
|
+
* correct only on a machine where the cage actually took. On cgroup v1, without
|
|
611
|
+
* a user bus, under a foreign supervisor — every `nice-only` machine — the
|
|
612
|
+
* sessions are still CHILDREN of the service, and a service ceiling sized for
|
|
613
|
+
* the daemon would cap all of them at a few hundred MB. This file is written
|
|
614
|
+
* before anything has probed which of the two machines this is, so the safe
|
|
615
|
+
* shape is the same number twice: on a caged machine the slice is the ceiling
|
|
616
|
+
* that binds, on an uncaged one the service is, and neither machine is ever
|
|
617
|
+
* left with a ceiling that is too small for what is under it.
|
|
618
|
+
*
|
|
619
|
+
* The price is that a caged machine formally permits `service + slice`. It is
|
|
620
|
+
* not the guarantee `memoryPolicy` computes, and it is written down here rather
|
|
621
|
+
* than glossed over (QA-2026-09-07 MINOR-4).
|
|
622
|
+
*
|
|
623
|
+
* `MemorySwapMax=0` for the same reason it is on every scope: a ceiling on
|
|
624
|
+
* resident memory alone is not a ceiling, it is a swap pump (a 200 MB cage
|
|
625
|
+
* allocated 2 GB and drained the host's swap during the spike).
|
|
626
|
+
*
|
|
627
|
+
* No `MemoryHigh` here either. Soft pressure on the slice would throttle every
|
|
628
|
+
* session on the machine to keep one runaway alive a little longer — the exact
|
|
629
|
+
* trade the spike measured and rejected: 60 seconds of delays instead of a
|
|
630
|
+
* 380 ms honest death.
|
|
631
|
+
*
|
|
632
|
+
* `sessionsUsageBytes` is the door that used to lead around all of the above.
|
|
633
|
+
* `facts` is null whenever the SERVICE's `MemoryCurrent` is unreadable — a
|
|
634
|
+
* stopped service, or a runner under a foreign supervisor — and this file then
|
|
635
|
+
* fell back to a flat `MemoryMax=55%`. But the sessions are SIBLINGS of the
|
|
636
|
+
* service, not its children: they survive `systemctl --user stop
|
|
637
|
+
* devbridge-runner`, so «the service is not running» says nothing at all about
|
|
638
|
+
* what the slice is holding, and `doctor --fix` on such a machine wrote 55 % of
|
|
639
|
+
* total onto a live slice and then called `daemon-reload` — the collective kill
|
|
640
|
+
* of QA-2026-09-07 BLOCKER-1 arriving through a door with no policy behind it.
|
|
641
|
+
*
|
|
642
|
+
* So the slice's own usage is read separately, and the promise made on
|
|
643
|
+
* `buildLimitsOverride` («the floor belongs to the policy, so every caller gets
|
|
644
|
+
* it and none can opt out») holds on this path too:
|
|
645
|
+
* - a number → the ceiling clears it by the same 1.25 the policy uses, and
|
|
646
|
+
* never drops below what one agent needs;
|
|
647
|
+
* - 0 → nothing is running there, so the blind fraction can kill
|
|
648
|
+
* nothing and stays;
|
|
649
|
+
* - null → this machine could not say, and NO ceiling is written at all.
|
|
650
|
+
* Leaving whatever is in force in force is strictly better than
|
|
651
|
+
* applying an unfounded number to a cgroup that may be full: the
|
|
652
|
+
* daemon rewrites the file with a measured ceiling the moment it
|
|
653
|
+
* can measure one (the drift check treats a file with no
|
|
654
|
+
* `MemoryMax` as outdated).
|
|
655
|
+
*/
|
|
656
|
+
export function buildSessionsSliceOverride(facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull()) {
|
|
657
|
+
const memory = facts ? memoryPolicy(facts) : null;
|
|
658
|
+
return ([
|
|
659
|
+
...MANAGED_HEADER,
|
|
660
|
+
'[Slice]',
|
|
661
|
+
...(memory
|
|
662
|
+
? [
|
|
663
|
+
`# ${memory.measured ? 'measured headroom' : 'still booting — conservative fraction of total'}`,
|
|
664
|
+
`MemoryMax=${asMiB(memory.maxBytes)}`,
|
|
665
|
+
]
|
|
666
|
+
: unmeasuredSliceCeiling(sessionsUsageBytes)),
|
|
667
|
+
// The line the cage is built on. See `session-cage.ts`.
|
|
668
|
+
'MemorySwapMax=0',
|
|
669
|
+
'TasksMax=8192',
|
|
670
|
+
'MemoryAccounting=yes',
|
|
671
|
+
'TasksAccounting=yes',
|
|
672
|
+
'CPUAccounting=yes',
|
|
673
|
+
].join('\n') + '\n');
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* The `[Slice]` lines for a machine whose memory the policy could not measure.
|
|
677
|
+
* See {@link buildSessionsSliceOverride} for why each of the three answers is
|
|
678
|
+
* what it is.
|
|
679
|
+
*/
|
|
680
|
+
function unmeasuredSliceCeiling(sessionsUsageBytes) {
|
|
681
|
+
if (sessionsUsageBytes === null) {
|
|
682
|
+
return [
|
|
683
|
+
'# machine not measurable, and neither is this slice — no ceiling is written',
|
|
684
|
+
'# here at all: whatever systemd has in force stays in force, because a number',
|
|
685
|
+
'# with nothing behind it kills live sessions on the next daemon-reload.',
|
|
686
|
+
];
|
|
687
|
+
}
|
|
688
|
+
if (sessionsUsageBytes === 0) {
|
|
689
|
+
return [
|
|
690
|
+
'# machine not measurable; this slice holds nothing, so a blind fraction can',
|
|
691
|
+
'# kill nothing — the same one the service falls back to',
|
|
692
|
+
'MemoryMax=55%',
|
|
693
|
+
];
|
|
694
|
+
}
|
|
695
|
+
return [
|
|
696
|
+
`# machine not measurable — floored by the ${asMiB(sessionsUsageBytes)} this slice holds right now`,
|
|
697
|
+
`MemoryMax=${asMiB(Math.max(sessionsUsageBytes * CEILING_HEADROOM_OVER_CURRENT, CEILING_FLOOR_BYTES))}`,
|
|
698
|
+
];
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* The CPU share of everything the agents run, against the daemon's own.
|
|
702
|
+
*
|
|
703
|
+
* `nice(2)` orders tasks INSIDE one cgroup. The moment a session gets a scope of
|
|
704
|
+
* its own it is no longer inside the service's cgroup, and the split between the
|
|
705
|
+
* two is decided by `cpu.weight` — which is 100 everywhere by default,
|
|
706
|
+
* `app.slice` (where the service lives) included. Without this file the cage
|
|
707
|
+
* would silently undo stage 1a and hand back the failure of 16.08: the daemon
|
|
708
|
+
* starved by its own children, four missed heartbeats, the server Offline and
|
|
709
|
+
* 504 on every session. 50 against 100 leaves the daemon two thirds.
|
|
710
|
+
*
|
|
711
|
+
* `process-priority.ts` stays exactly as it is: it is what protects the daemon
|
|
712
|
+
* on cgroup v1 and on every machine where the cage does not apply.
|
|
713
|
+
*/
|
|
714
|
+
export function buildDevbridgeSliceOverride() {
|
|
715
|
+
return ([
|
|
716
|
+
...MANAGED_HEADER,
|
|
717
|
+
'[Slice]',
|
|
718
|
+
`CPUWeight=${SESSION_CPU_WEIGHT}`,
|
|
719
|
+
'CPUAccounting=yes',
|
|
720
|
+
'MemoryAccounting=yes',
|
|
721
|
+
].join('\n') + '\n');
|
|
722
|
+
}
|
|
419
723
|
/**
|
|
420
724
|
* Is the shipped resource policy missing or from an older runner?
|
|
421
725
|
*
|
|
@@ -427,9 +731,21 @@ export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMem
|
|
|
427
731
|
* version-based answer.
|
|
428
732
|
*/
|
|
429
733
|
export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome(), facts = null) {
|
|
734
|
+
// Three files since 0.54.0, and any one of them being stale means the policy
|
|
735
|
+
// is: the ceiling on the service protects the daemon, the ceiling on
|
|
736
|
+
// `devbridge-sessions.slice` protects the machine from the sessions that
|
|
737
|
+
// LEFT the service's cgroup, and the weight on `devbridge.slice` is what
|
|
738
|
+
// keeps the daemon ahead of them. A runner that shipped the cage without the
|
|
739
|
+
// weight would be strictly worse than one that shipped neither.
|
|
740
|
+
return (fileIsOutdated(readFile, limitsOverridePath(home), facts) ||
|
|
741
|
+
fileIsOutdated(readFile, sessionsSliceOverridePath(home), facts) ||
|
|
742
|
+
// No ceiling in this one, so nothing in it can drift.
|
|
743
|
+
fileIsOutdated(readFile, devbridgeSliceOverridePath(home), null));
|
|
744
|
+
}
|
|
745
|
+
function fileIsOutdated(readFile, target, facts) {
|
|
430
746
|
let contents;
|
|
431
747
|
try {
|
|
432
|
-
contents = readFile(
|
|
748
|
+
contents = readFile(target);
|
|
433
749
|
}
|
|
434
750
|
catch {
|
|
435
751
|
return true; // never written — every server that predates 0.21.0
|
|
@@ -487,14 +803,27 @@ function memoryCeilingHasDrifted(contents, facts) {
|
|
|
487
803
|
* arrive at the same number — otherwise a write whose floor was binding would be
|
|
488
804
|
* seen as drifted on the very next call and rewritten forever.
|
|
489
805
|
*/
|
|
490
|
-
export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts()) {
|
|
806
|
+
export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull()) {
|
|
807
|
+
// Unknown usage can be a busy service whose systemd query timed out. Do not
|
|
808
|
+
// replace an existing drop-in with a guessed percentage, or remove a limit
|
|
809
|
+
// by rewriting the file without its MemoryMax line. Defer the whole policy.
|
|
810
|
+
if (facts === null)
|
|
811
|
+
return false;
|
|
491
812
|
if (!force && !limitsOverrideIsOutdated(undefined, home, facts))
|
|
492
813
|
return false;
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
814
|
+
// All three or none. They are one policy split across three units only
|
|
815
|
+
// because systemd has no other way to say it, and a machine holding two of
|
|
816
|
+
// them is a machine whose sessions are capped but whose daemon is not
|
|
817
|
+
// prioritised — the regression described on `buildDevbridgeSliceOverride`.
|
|
818
|
+
write(limitsOverridePath(home), buildLimitsOverride(undefined, facts));
|
|
819
|
+
write(sessionsSliceOverridePath(home), buildSessionsSliceOverride(facts, sessionsUsageBytes));
|
|
820
|
+
write(devbridgeSliceOverridePath(home), buildDevbridgeSliceOverride());
|
|
496
821
|
return true;
|
|
497
822
|
}
|
|
823
|
+
function write(target, contents) {
|
|
824
|
+
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
825
|
+
fs.writeFileSync(target, contents, { mode: 0o644 });
|
|
826
|
+
}
|
|
498
827
|
/**
|
|
499
828
|
* Does the installed unit point at something that no longer exists?
|
|
500
829
|
*
|