@bridge4dev/runner 0.38.0 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +35 -4
- package/dist/attachments.js +35 -5
- package/dist/index.js +99 -10
- package/dist/service-unit.d.ts +122 -6
- package/dist/service-unit.js +300 -16
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/adapters/claude.js
CHANGED
|
@@ -287,6 +287,15 @@ class ClaudeSession {
|
|
|
287
287
|
mcpFallbackNotice = null;
|
|
288
288
|
/** Guards against overlapping capability probes. */
|
|
289
289
|
capabilitiesInFlight = false;
|
|
290
|
+
/**
|
|
291
|
+
* Has the first main-loop answer of this process already asked for the
|
|
292
|
+
* context window? (ticket #223)
|
|
293
|
+
*
|
|
294
|
+
* `q` is `readonly` — one adapter per CLI process — so this never needs
|
|
295
|
+
* resetting: a restarted process is a new instance with the flag back to
|
|
296
|
+
* false, which is exactly the moment the meter needs a fresh number.
|
|
297
|
+
*/
|
|
298
|
+
contextProbedAfterFirstReply = false;
|
|
290
299
|
mode;
|
|
291
300
|
/**
|
|
292
301
|
* A launch-time mode this workspace does not allow, remembered so the feed
|
|
@@ -559,6 +568,13 @@ class ClaudeSession {
|
|
|
559
568
|
// message would otherwise show no model list at all — while the control
|
|
560
569
|
// requests themselves work as soon as the CLI is up.
|
|
561
570
|
this.refreshCapabilities();
|
|
571
|
+
// Same reasoning for the context meter (ticket #223). It used to be asked
|
|
572
|
+
// for only at the END of a turn, so a session that had not finished its
|
|
573
|
+
// first one had no numbers at all and the gauge did not exist — worst in a
|
|
574
|
+
// session that parked a question, which can sit there for hours. This probe
|
|
575
|
+
// costs one control round-trip and gives the meter its denominator (the
|
|
576
|
+
// model's window) from the moment the process is up.
|
|
577
|
+
this.refreshContextUsage();
|
|
562
578
|
}
|
|
563
579
|
/**
|
|
564
580
|
* Write this session's MCP config to a 0600 file and return its path, or null
|
|
@@ -623,10 +639,14 @@ class ClaudeSession {
|
|
|
623
639
|
this.output.push(maskSecrets(event));
|
|
624
640
|
}
|
|
625
641
|
/**
|
|
626
|
-
* How full the context window is
|
|
627
|
-
*
|
|
628
|
-
*
|
|
629
|
-
*
|
|
642
|
+
* How full the context window is. Fire-and-forget with a timeout on purpose:
|
|
643
|
+
* this is one more control round-trip on a channel that can hang, and it must
|
|
644
|
+
* never be able to stall the event loop that carries the conversation.
|
|
645
|
+
*
|
|
646
|
+
* Called from four places, each answering a different gap: process start and
|
|
647
|
+
* first main-loop reply (ticket #223 — otherwise a session shows no gauge at
|
|
648
|
+
* all until its first turn ends), model change (the window's size differs 5x
|
|
649
|
+
* between 200k and 1M), and end of turn.
|
|
630
650
|
*/
|
|
631
651
|
refreshContextUsage() {
|
|
632
652
|
if (this.stopped)
|
|
@@ -1845,6 +1865,17 @@ class ClaudeSession {
|
|
|
1845
1865
|
});
|
|
1846
1866
|
}
|
|
1847
1867
|
}
|
|
1868
|
+
// Ticket #223. The startup probe fires before the CLI has read
|
|
1869
|
+
// anything, so its number is the empty window; this one lands after
|
|
1870
|
+
// the agent has actually said something and is the first figure
|
|
1871
|
+
// worth showing. Once per process, and only for the main loop: it is
|
|
1872
|
+
// a control round-trip on a channel that can hang, subagent chatter
|
|
1873
|
+
// is not this session's context, and the end-of-turn probe covers
|
|
1874
|
+
// everything after.
|
|
1875
|
+
if (!fromSubagent && !this.contextProbedAfterFirstReply) {
|
|
1876
|
+
this.contextProbedAfterFirstReply = true;
|
|
1877
|
+
this.refreshContextUsage();
|
|
1878
|
+
}
|
|
1848
1879
|
break;
|
|
1849
1880
|
}
|
|
1850
1881
|
case 'user': {
|
package/dist/attachments.js
CHANGED
|
@@ -18,9 +18,32 @@ const execFileAsync = promisify(execFile);
|
|
|
18
18
|
*/
|
|
19
19
|
/** Directory inside the worktree; also the line written to info/exclude. */
|
|
20
20
|
export const ATTACHMENT_DIR = '.devbridge/attachments';
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Hard ceiling per file; the API allows 10 MB for images and 50 MB otherwise.
|
|
23
|
+
*
|
|
24
|
+
* Mirror of `MAX_DOCUMENT_ATTACHMENT_SIZE_BYTES` in `@devbridge/shared` plus a
|
|
25
|
+
* megabyte of slack — the runner cannot import that package (see
|
|
26
|
+
* `recipe-schema.ts`: the published tarball would not resolve it). The slack is
|
|
27
|
+
* what keeps the two halves from disagreeing on a rounding: the API refuses
|
|
28
|
+
* first, with an error the person actually sees, and this only catches bytes
|
|
29
|
+
* that no upload path should have produced.
|
|
30
|
+
*
|
|
31
|
+
* Raise the API side and this one in the SAME runner release. A ceiling that
|
|
32
|
+
* moves only in DevBridge lets a file upload and then vanish: the refusal below
|
|
33
|
+
* is a warning line in the session, not something the uploader is shown.
|
|
34
|
+
*/
|
|
35
|
+
const MAX_ATTACHMENT_BYTES = 51 * 1024 * 1024;
|
|
36
|
+
/**
|
|
37
|
+
* A minute was plenty while the ceiling was 26 MB; at 50 MB it is a coin flip.
|
|
38
|
+
*
|
|
39
|
+
* The whole file is buffered by the API and pushed through nginx in one
|
|
40
|
+
* response, so this timeout covers the entire transfer, not idle time between
|
|
41
|
+
* packets. 60 s for 50 MB demands ~7 Mbit/s sustained end to end — ordinary for
|
|
42
|
+
* two VPSes, not ordinary for a dev server on a home uplink or behind a
|
|
43
|
+
* throttled proxy. Three minutes covers ~2.3 Mbit/s, and the cost of being
|
|
44
|
+
* generous is nil: a stalled download fails the same way, just later.
|
|
45
|
+
*/
|
|
46
|
+
const DOWNLOAD_TIMEOUT_MS = 180_000;
|
|
24
47
|
/**
|
|
25
48
|
* A file name that is safe to write and unambiguous to read.
|
|
26
49
|
*
|
|
@@ -153,9 +176,16 @@ export async function saveAttachments(input) {
|
|
|
153
176
|
pruneAttachmentDir(dir);
|
|
154
177
|
return { saved, failed };
|
|
155
178
|
}
|
|
156
|
-
/**
|
|
179
|
+
/**
|
|
180
|
+
* Keep at most this much history in one worktree's attachment folder.
|
|
181
|
+
*
|
|
182
|
+
* Sized in files, not in bytes: 512 MB was twenty attachments at the old 25 MB
|
|
183
|
+
* ceiling and would be ten at the new one — an afternoon of work, after which
|
|
184
|
+
* the folder starts evicting documents a running session may still be told to
|
|
185
|
+
* open. A gigabyte restores the same twenty-file depth.
|
|
186
|
+
*/
|
|
157
187
|
const ATTACHMENT_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
|
|
158
|
-
const ATTACHMENT_DIR_BUDGET_BYTES =
|
|
188
|
+
const ATTACHMENT_DIR_BUDGET_BYTES = 1024 * 1024 * 1024;
|
|
159
189
|
/**
|
|
160
190
|
* Delete old attachments from a session worktree.
|
|
161
191
|
*
|
package/dist/index.js
CHANGED
|
@@ -17,7 +17,7 @@ import { Supervisor } from './supervisor.js';
|
|
|
17
17
|
import { readStatusFile, isPidAlive, writeStatusFile, STATUS_FRESH_MS } from './status-file.js';
|
|
18
18
|
import { RunnerWsClient } from './ws-client.js';
|
|
19
19
|
import { RUNNER_VERSION } from './version.js';
|
|
20
|
-
import { buildUnit, cpuQuotaPercent, limitsOverrideIsOutdated, limitsOverridePath, unitExecTarget, unitPath, writeLimitsOverride, LIMITS_VERSION, } from './service-unit.js';
|
|
20
|
+
import { buildUnit, cpuQuotaPercent, limitsOverrideIsOutdated, limitsOverridePath, memoryPolicy, readMemoryFacts, unitExecTarget, unitPath, writeLimitsOverride, LIMITS_VERSION, } from './service-unit.js';
|
|
21
21
|
import { readOomKills, recordCrash, takeLastExit } from './crash-note.js';
|
|
22
22
|
import { agentAuthStatuses } from './auth-relay.js';
|
|
23
23
|
import { addSafeDirectory, agentConfigContour, dockerCheck, ensureAgentPath, firstUnreachableAncestor, hasSafeDirectory, inspectPath, knownWorkspacePaths, lingerEnabled, nodeCheck, otherHomeWithAgents, runnerIdentity, safeDirectoryCommand, systemctlHint, systemdUserBusReachable, systemdUserEnv, } from './environment.js';
|
|
@@ -359,7 +359,9 @@ function runnerCapabilities(apiUrlOverride) {
|
|
|
359
359
|
memTotalBytes: os.totalmem(),
|
|
360
360
|
memAvailableBytes: os.freemem(),
|
|
361
361
|
limitsVersion: LIMITS_VERSION,
|
|
362
|
-
|
|
362
|
+
// Facts passed on purpose: since 0.39.0 «current» also means the measured
|
|
363
|
+
// ceiling still fits this machine, not just that the version matches.
|
|
364
|
+
limitsCurrent: !limitsOverrideIsOutdated(undefined, undefined, readMemoryFacts()),
|
|
363
365
|
},
|
|
364
366
|
/**
|
|
365
367
|
* Identity of THIS process, so the API can tell a network blink from a
|
|
@@ -497,20 +499,32 @@ const RESTART_DELAY_MS = 1_500;
|
|
|
497
499
|
*/
|
|
498
500
|
async function repairResourceLimits() {
|
|
499
501
|
try {
|
|
500
|
-
|
|
502
|
+
const facts = readMemoryFacts();
|
|
503
|
+
if (facts && memoryPolicy(facts).starved) {
|
|
504
|
+
// The only signal the machine's owner will ever get that the neighbours
|
|
505
|
+
// have taken the box: the honest headroom was below the 2 GB floor, so the
|
|
506
|
+
// ceiling we are about to write sits above what is actually free.
|
|
507
|
+
log.warn('daemon: not enough free memory for a real ceiling — neighbours have the machine', {
|
|
508
|
+
availableMB: Math.round(facts.availableBytes / 1048576),
|
|
509
|
+
totalMB: Math.round(facts.totalBytes / 1048576),
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
if (!writeLimitsOverride(false, undefined, facts))
|
|
501
513
|
return;
|
|
502
514
|
log.warn('daemon: resource limits drop-in written — reloading systemd', {
|
|
503
515
|
path: limitsOverridePath(),
|
|
504
516
|
version: LIMITS_VERSION,
|
|
517
|
+
...(facts ? memoryPolicy(facts) : {}),
|
|
505
518
|
});
|
|
506
519
|
await execFileAsync('systemctl', ['--user', 'daemon-reload'], {
|
|
507
520
|
timeout: 15_000,
|
|
508
521
|
env: systemdUserEnv(),
|
|
509
522
|
});
|
|
510
523
|
// Deliberately no restart: `daemon-reload` alone is enough for these
|
|
511
|
-
// directives
|
|
512
|
-
// 2G→infinity
|
|
513
|
-
//
|
|
524
|
+
// directives — verified live twice, in both directions: 2026-07-30 removing a
|
|
525
|
+
// ceiling (MemoryMax 2G→infinity, OOMPolicy stop→continue) and 2026-08-12
|
|
526
|
+
// installing one (infinity→1234M), each with the PID unchanged. Restarting
|
|
527
|
+
// here would park every live session to apply a setting already in force.
|
|
514
528
|
log.info('daemon: resource limits applied without a restart');
|
|
515
529
|
}
|
|
516
530
|
catch (error) {
|
|
@@ -520,6 +534,51 @@ async function repairResourceLimits() {
|
|
|
520
534
|
});
|
|
521
535
|
}
|
|
522
536
|
}
|
|
537
|
+
/**
|
|
538
|
+
* How often the daemon re-measures the machine after the first time.
|
|
539
|
+
*
|
|
540
|
+
* Doing this only at startup left a hole big enough to matter. The runner is a
|
|
541
|
+
* user unit and comes up 20–60 seconds into a boot, which is inside
|
|
542
|
+
* `BOOT_SETTLE_SEC` — so a machine that reboots gets the blind static fraction
|
|
543
|
+
* and then keeps it for as long as the daemon lives, which can be weeks. On a
|
|
544
|
+
* dedicated 8 GB box that is 3.5 GB of soft brake against the 3.3 GB three
|
|
545
|
+
* sessions and a build actually cost: throttling ordinary work for no reason.
|
|
546
|
+
* Worse, `limitsCurrent` in `hello` is computed from FRESH facts, so the API
|
|
547
|
+
* would keep logging «limits outdated» on every reconnect for a machine that had
|
|
548
|
+
* no way to fix itself.
|
|
549
|
+
*
|
|
550
|
+
* Hourly is right because drift is slow by definition — it tracks what else is
|
|
551
|
+
* installed on the machine, not what it is doing this minute — and because the
|
|
552
|
+
* check writes nothing when the version matches and the ceiling still fits.
|
|
553
|
+
*/
|
|
554
|
+
const LIMITS_RECHECK_MS = 3_600_000;
|
|
555
|
+
/**
|
|
556
|
+
* What the SERVICE's cgroup holds, for the paths that are not the service.
|
|
557
|
+
*
|
|
558
|
+
* `readMemoryFacts()` defaults to reading `/proc/self`, which is right for the
|
|
559
|
+
* daemon and wrong for everything a person types: `doctor --fix` and
|
|
560
|
+
* `install-service` run in the operator's own `session-N.scope`, worth a few MB.
|
|
561
|
+
* Measuring that and calling it «what the runner holds» made the CLI compute a
|
|
562
|
+
* ceiling 34 % away from the daemon's answer on a real host — each path then saw
|
|
563
|
+
* the other as drift and rewrote the file, forever.
|
|
564
|
+
*
|
|
565
|
+
* `MemoryCurrent` is `memory.current`, so it still counts page cache the way
|
|
566
|
+
* `MemAvailable` does. Subtracting it is not worth a second systemd call here:
|
|
567
|
+
* the CLI paths run once, by hand, and erring toward a LOWER ceiling is the safe
|
|
568
|
+
* direction. Returns null when the service is not running or systemd cannot be
|
|
569
|
+
* reached — the caller then has no facts, and writes nothing rather than writing
|
|
570
|
+
* a ceiling with no floor under it.
|
|
571
|
+
*/
|
|
572
|
+
async function serviceMemoryCurrent() {
|
|
573
|
+
try {
|
|
574
|
+
const { stdout } = await execFileAsync('systemctl', ['--user', 'show', 'devbridge-runner', '-p', 'MemoryCurrent', '--value'], { timeout: 10_000, env: systemdUserEnv() });
|
|
575
|
+
const value = Number(stdout.trim());
|
|
576
|
+
return Number.isFinite(value) && value > 0 ? value : null;
|
|
577
|
+
}
|
|
578
|
+
catch {
|
|
579
|
+
return null;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
523
582
|
/**
|
|
524
583
|
* Ticket #119. `ClaudeAdapter.stop()` removes each session's MCP config file,
|
|
525
584
|
* but a kill -9, an OOM or a machine reboot leaves it behind — a live project
|
|
@@ -620,6 +679,10 @@ async function cmdDaemon() {
|
|
|
620
679
|
// that stays connected for weeks.
|
|
621
680
|
const pruneTimer = setInterval(() => supervisor.pruneJournals(), 6 * 3_600_000);
|
|
622
681
|
pruneTimer.unref();
|
|
682
|
+
// Re-measure the machine — see `LIMITS_RECHECK_MS`. Writes nothing in the
|
|
683
|
+
// normal case, so this is a file read and some arithmetic once an hour.
|
|
684
|
+
const limitsTimer = setInterval(() => void repairResourceLimits(), LIMITS_RECHECK_MS);
|
|
685
|
+
limitsTimer.unref();
|
|
623
686
|
const shutdown = (signal) => {
|
|
624
687
|
log.info(`daemon: ${signal} received, shutting down`);
|
|
625
688
|
supervisor.shutdown();
|
|
@@ -720,8 +783,9 @@ async function cmdInstallService() {
|
|
|
720
783
|
print(`Wrote ${target}`);
|
|
721
784
|
// Resource policy is a versioned drop-in, not part of the unit — see
|
|
722
785
|
// `buildLimitsOverride`. Forced here: a fresh install must have it even if a
|
|
723
|
-
// file from an older runner is already sitting there.
|
|
724
|
-
|
|
786
|
+
// file from an older runner is already sitting there. Facts come from the
|
|
787
|
+
// service rather than from `/proc/self`, which here is the installing shell.
|
|
788
|
+
writeLimitsOverride(true, undefined, readMemoryFacts(await serviceMemoryCurrent()));
|
|
725
789
|
print(`Wrote ${limitsOverridePath()}`);
|
|
726
790
|
if (!exec.viaCommand) {
|
|
727
791
|
// Worth saying out loud: a unit pinned to a file inside the package directory
|
|
@@ -1143,10 +1207,22 @@ async function cmdDoctor(args) {
|
|
|
1143
1207
|
print(` fits sessions ~${advised} (at ~1.5 GB per session under load)`);
|
|
1144
1208
|
print('');
|
|
1145
1209
|
print('Service limits');
|
|
1146
|
-
const
|
|
1210
|
+
const memFacts = readMemoryFacts(await serviceMemoryCurrent());
|
|
1211
|
+
const outdated = limitsOverrideIsOutdated(undefined, undefined, memFacts);
|
|
1147
1212
|
print(` drop-in ${limitsOverridePath()}`);
|
|
1148
1213
|
print(` version ${outdated ? `MISSING or OLD (want ${LIMITS_VERSION})` : LIMITS_VERSION}`);
|
|
1149
1214
|
print(` cpu quota ${quota === null ? 'none (fewer than 4 cpus)' : `${quota}%`}`);
|
|
1215
|
+
// Spelled out because the whole point of 0.39.0 is that this number is measured
|
|
1216
|
+
// rather than a percentage of the machine: on a server with neighbours, «80% of
|
|
1217
|
+
// total» sat above what was actually free and never engaged (gotcha #301).
|
|
1218
|
+
if (memFacts) {
|
|
1219
|
+
const policy = memoryPolicy(memFacts);
|
|
1220
|
+
const mb = (bytes) => `${Math.round(bytes / 1024 / 1024)} MB`;
|
|
1221
|
+
print(` memory ceiling ${mb(policy.maxBytes)} hard / ${mb(policy.highBytes)} soft`);
|
|
1222
|
+
print(` sized from ${policy.measured
|
|
1223
|
+
? `${mb(memFacts.availableBytes)} available + ${mb(memFacts.ownUsageBytes)} ours`
|
|
1224
|
+
: `55% of ${mb(memFacts.totalBytes)} — still booting, remeasured on the next start`}`);
|
|
1225
|
+
}
|
|
1150
1226
|
let effective;
|
|
1151
1227
|
try {
|
|
1152
1228
|
const { stdout } = await execFileAsync('systemctl', [
|
|
@@ -1222,9 +1298,22 @@ async function cmdDoctor(args) {
|
|
|
1222
1298
|
process.exit(1);
|
|
1223
1299
|
return;
|
|
1224
1300
|
}
|
|
1225
|
-
|
|
1301
|
+
const fixFacts = readMemoryFacts(await serviceMemoryCurrent());
|
|
1302
|
+
writeLimitsOverride(true, undefined, fixFacts);
|
|
1226
1303
|
print('');
|
|
1227
1304
|
print(`Wrote ${limitsOverridePath()}`);
|
|
1305
|
+
// Say it out loud when the ceiling had to be raised above the honest answer to
|
|
1306
|
+
// avoid killing whatever is running right now. Lowering a ceiling under load is
|
|
1307
|
+
// a decision, not a side effect of running a diagnostic — and without this line
|
|
1308
|
+
// the number written here looks unexplainably generous.
|
|
1309
|
+
if (fixFacts) {
|
|
1310
|
+
const honest = memoryPolicy(fixFacts, 0);
|
|
1311
|
+
const applied = memoryPolicy(fixFacts);
|
|
1312
|
+
if (applied.maxBytes > honest.maxBytes) {
|
|
1313
|
+
const mb = (b) => Math.round(b / 1024 / 1024);
|
|
1314
|
+
print(`note: ceiling raised to ${mb(applied.maxBytes)} MB — the service already holds ${mb(fixFacts.ownUsageBytes)} MB, and ${mb(honest.maxBytes)} MB would kill it on reload. Re-run when idle.`);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1228
1317
|
try {
|
|
1229
1318
|
await execFileAsync('systemctl', ['--user', 'daemon-reload'], { env: systemdUserEnv() });
|
|
1230
1319
|
print('systemctl --user daemon-reload — done.');
|
package/dist/service-unit.d.ts
CHANGED
|
@@ -57,8 +57,114 @@ export declare function buildUnit(execStart?: string, nodeBinary?: string): stri
|
|
|
57
57
|
* which is the only way to fix the servers that already have the bad numbers
|
|
58
58
|
* baked in — and it never overwrites a unit the operator edited by hand.
|
|
59
59
|
*/
|
|
60
|
-
export declare const LIMITS_VERSION =
|
|
60
|
+
export declare const LIMITS_VERSION = 3;
|
|
61
61
|
export declare function limitsOverridePath(home?: string): string;
|
|
62
|
+
/**
|
|
63
|
+
* What the memory policy needs to know about this machine. Read once and passed
|
|
64
|
+
* in, so the policy itself is a pure function that a test can drive with the
|
|
65
|
+
* numbers of a server it does not have.
|
|
66
|
+
*/
|
|
67
|
+
export interface MemoryFacts {
|
|
68
|
+
/** `MemTotal` — the whole machine. */
|
|
69
|
+
totalBytes: number;
|
|
70
|
+
/** `MemAvailable` — what can be handed out without swapping. Already excludes
|
|
71
|
+
* what the co-tenants hold, and already counts reclaimable page cache. */
|
|
72
|
+
availableBytes: number;
|
|
73
|
+
/**
|
|
74
|
+
* What our own cgroup holds that `MemAvailable` has not already counted —
|
|
75
|
+
* `memory.current` minus its page cache. Added back to the headroom, and used
|
|
76
|
+
* as the floor under the ceiling; see `memoryPolicy`.
|
|
77
|
+
*/
|
|
78
|
+
ownUsageBytes: number;
|
|
79
|
+
/** Seconds since boot. Below `BOOT_SETTLE_SEC` the measurement is a lie. */
|
|
80
|
+
uptimeSec: number;
|
|
81
|
+
}
|
|
82
|
+
export interface MemoryPolicy {
|
|
83
|
+
maxBytes: number;
|
|
84
|
+
highBytes: number;
|
|
85
|
+
/** false = the conservative static pair, because the machine was still booting. */
|
|
86
|
+
measured: boolean;
|
|
87
|
+
/**
|
|
88
|
+
* The honest headroom was below the 2 GB floor — the neighbours have eaten the
|
|
89
|
+
* machine and this ceiling sits ABOVE what is free. Not an error (a dev server
|
|
90
|
+
* that cannot run one agent is worse than a busy one), but the only signal the
|
|
91
|
+
* owner will ever get that the box is oversubscribed, so callers log it.
|
|
92
|
+
*/
|
|
93
|
+
starved: boolean;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* How long after boot `MemAvailable` starts telling the truth.
|
|
97
|
+
*
|
|
98
|
+
* The runner is a user unit ordered `After=network-online.target`, and the things
|
|
99
|
+
* it shares the machine with — dockerd and its containers, mysql, pm2 — come up
|
|
100
|
+
* around the same time. Measure at second 20 and the neighbours have not claimed
|
|
101
|
+
* their memory yet: on the 12 GB machine below that reads as 11 GB free and
|
|
102
|
+
* produces a ceiling that protects nothing.
|
|
103
|
+
*/
|
|
104
|
+
export declare const BOOT_SETTLE_SEC = 600;
|
|
105
|
+
/**
|
|
106
|
+
* The two numbers, and the incident that decides them.
|
|
107
|
+
*
|
|
108
|
+
* Until 0.21.0 the policy was a 2 GB hard ceiling with systemd's default
|
|
109
|
+
* `OOMPolicy=stop`: one agent over the line and the whole service died, taking
|
|
110
|
+
* every other session with it (QA-112, gotcha #100). The fix removed the ceiling
|
|
111
|
+
* entirely and left a soft `MemoryHigh=80%`, on this reasoning, quoted from the
|
|
112
|
+
* code it replaced: «the machine keeps a fifth of its memory for everything that
|
|
113
|
+
* is not this service».
|
|
114
|
+
*
|
|
115
|
+
* That reasoning holds on a DEDICATED dev server and fails on a shared one, which
|
|
116
|
+
* is what vmi2502773 is: 11.9 GB total with ~4 GB permanently held by 18 docker
|
|
117
|
+
* containers, mysql, flowise, chroma and pm2. There, 80 % of TOTAL is 9.5 GB while
|
|
118
|
+
* only ~8 GB is actually free — so the soft brake sat ABOVE the real headroom and
|
|
119
|
+
* could never engage. The machine swapped, `kcompactd0` blocked for over 120
|
|
120
|
+
* seconds and the box hung; 2026-08-05 it was a `claude` at 4.2 GB, 2026-08-12 a
|
|
121
|
+
* single grep at 6.8 GB with 160 MB left. Only a power cycle recovered it.
|
|
122
|
+
*
|
|
123
|
+
* So the percentage has to be of what the machine can SPARE, not of what it has:
|
|
124
|
+
*
|
|
125
|
+
* headroom = MemAvailable + our own usage (ours is added back, or every
|
|
126
|
+
* rewrite would walk the ceiling down by what we already hold)
|
|
127
|
+
* MemoryMax = headroom − reserve
|
|
128
|
+
* MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
|
|
129
|
+
*
|
|
130
|
+
* A hard ceiling is safe to have again only because the OTHER half of the 0.21.0
|
|
131
|
+
* fix stayed: `OOMPolicy=continue` is still set below, so the kernel killing one
|
|
132
|
+
* runaway child no longer tears down the service. Restoring the ceiling without
|
|
133
|
+
* that would re-open QA-112.
|
|
134
|
+
*
|
|
135
|
+
* The clamps are the two ways this can go wrong:
|
|
136
|
+
* - floor 2 GB: on a machine whose neighbours already ate everything, a computed
|
|
137
|
+
* ceiling of a few hundred MB would mean a dev server that cannot run one
|
|
138
|
+
* agent. Measured cost of ordinary work is 512–646 MB per `claude` and 1571 MB
|
|
139
|
+
* for a workspace `pnpm -r typecheck`, so anything under 2 GB is not a dev
|
|
140
|
+
* server at all — an unusable one is a worse failure than a busy one, the same
|
|
141
|
+
* call `cpuQuotaPercent` makes below;
|
|
142
|
+
* - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
|
|
143
|
+
* machine, and a ceiling of «everything» is the bug this function exists to fix.
|
|
144
|
+
*/
|
|
145
|
+
export declare function memoryPolicy(facts: MemoryFacts, minCeilingBytes?: number): MemoryPolicy;
|
|
146
|
+
/**
|
|
147
|
+
* What the cgroup holds that `MemAvailable` has NOT already counted.
|
|
148
|
+
*
|
|
149
|
+
* `memory.current` is `anon + file + kernel`, and `file` is page cache — which
|
|
150
|
+
* `MemAvailable` already lists as reclaimable. Adding the whole of
|
|
151
|
+
* `memory.current` back to `MemAvailable` therefore counts our page cache twice
|
|
152
|
+
* and inflates the ceiling by exactly that much: measured at +19 % on this host
|
|
153
|
+
* (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
|
|
154
|
+
* precisely when memory is tightest. Subtracting `file` keeps the part we really
|
|
155
|
+
* do hold and cannot give back on demand.
|
|
156
|
+
*/
|
|
157
|
+
export declare function readCgroupUnreclaimable(dir: string): number | null;
|
|
158
|
+
/**
|
|
159
|
+
* Everything `memoryPolicy` needs, straight off this machine.
|
|
160
|
+
*
|
|
161
|
+
* `ownUsageBytes` is passed in by callers that are not the daemon — `doctor` and
|
|
162
|
+
* `install-service` run in the operator's own cgroup and cannot read the
|
|
163
|
+
* service's usage from `/proc/self`. Returns null when the service's usage is
|
|
164
|
+
* unknowable, because guessing 0 there is the one dangerous direction: it removes
|
|
165
|
+
* the floor that stops a live session from being killed on `daemon-reload`.
|
|
166
|
+
*/
|
|
167
|
+
export declare function readMemoryFacts(ownUsageBytes?: number | null): MemoryFacts | null;
|
|
62
168
|
/**
|
|
63
169
|
* `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
|
|
64
170
|
* box unreachable, but never so little that ordinary work is throttled.
|
|
@@ -69,17 +175,27 @@ export declare function limitsOverridePath(home?: string): string;
|
|
|
69
175
|
* than a busy one.
|
|
70
176
|
*/
|
|
71
177
|
export declare function cpuQuotaPercent(cpuCount?: number): number | null;
|
|
72
|
-
export declare function buildLimitsOverride(cpuCount?: number): string;
|
|
178
|
+
export declare function buildLimitsOverride(cpuCount?: number, facts?: MemoryFacts | null): string;
|
|
73
179
|
/**
|
|
74
180
|
* Is the shipped resource policy missing or from an older runner?
|
|
75
181
|
*
|
|
76
182
|
* Deliberately version-based rather than content-based: an operator may add
|
|
77
183
|
* their own directives to our file, and re-writing on every start would fight
|
|
78
|
-
* them.
|
|
184
|
+
* them. The version number decides — plus, since 0.39.0 and only when `facts` are
|
|
185
|
+
* supplied, whether the measured ceiling still fits the machine (see
|
|
186
|
+
* `memoryCeilingHasDrifted`). Callers that pass no facts get the old, purely
|
|
187
|
+
* version-based answer.
|
|
188
|
+
*/
|
|
189
|
+
export declare function limitsOverrideIsOutdated(readFile?: (p: string) => string, home?: string, facts?: MemoryFacts | null): boolean;
|
|
190
|
+
/**
|
|
191
|
+
* Write the drop-in. Returns false when nothing needed doing.
|
|
192
|
+
*
|
|
193
|
+
* The «never below what we already hold» guard lives in `memoryPolicy` itself
|
|
194
|
+
* (`CEILING_HEADROOM_OVER_CURRENT`), so writing, reading and the drift check all
|
|
195
|
+
* arrive at the same number — otherwise a write whose floor was binding would be
|
|
196
|
+
* seen as drifted on the very next call and rewritten forever.
|
|
79
197
|
*/
|
|
80
|
-
export declare function
|
|
81
|
-
/** Write the drop-in. Returns false when nothing needed doing. */
|
|
82
|
-
export declare function writeLimitsOverride(force?: boolean, home?: string): boolean;
|
|
198
|
+
export declare function writeLimitsOverride(force?: boolean, home?: string, facts?: MemoryFacts | null): boolean;
|
|
83
199
|
/**
|
|
84
200
|
* Does the installed unit point at something that no longer exists?
|
|
85
201
|
*
|
package/dist/service-unit.js
CHANGED
|
@@ -129,13 +129,220 @@ export function buildUnit(execStart, nodeBinary = process.execPath) {
|
|
|
129
129
|
* which is the only way to fix the servers that already have the bad numbers
|
|
130
130
|
* baked in — and it never overwrites a unit the operator edited by hand.
|
|
131
131
|
*/
|
|
132
|
-
export const LIMITS_VERSION =
|
|
132
|
+
export const LIMITS_VERSION = 3;
|
|
133
133
|
const LIMITS_MARKER = '# devbridge-limits-version:';
|
|
134
134
|
/** `zz-` so it sorts last: an operator's own drop-in should still win. */
|
|
135
135
|
const LIMITS_FILE = 'zz-devbridge-limits.conf';
|
|
136
136
|
export function limitsOverridePath(home = systemdUserHome()) {
|
|
137
137
|
return path.join(home, '.config', 'systemd', 'user', `${SERVICE_NAME}.service.d`, LIMITS_FILE);
|
|
138
138
|
}
|
|
139
|
+
const MIB = 1024 * 1024;
|
|
140
|
+
const GIB = 1024 * MIB;
|
|
141
|
+
/**
|
|
142
|
+
* How long after boot `MemAvailable` starts telling the truth.
|
|
143
|
+
*
|
|
144
|
+
* The runner is a user unit ordered `After=network-online.target`, and the things
|
|
145
|
+
* it shares the machine with — dockerd and its containers, mysql, pm2 — come up
|
|
146
|
+
* around the same time. Measure at second 20 and the neighbours have not claimed
|
|
147
|
+
* their memory yet: on the 12 GB machine below that reads as 11 GB free and
|
|
148
|
+
* produces a ceiling that protects nothing.
|
|
149
|
+
*/
|
|
150
|
+
export const BOOT_SETTLE_SEC = 600;
|
|
151
|
+
/**
|
|
152
|
+
* The ceiling is never written BELOW what the cgroup already holds.
|
|
153
|
+
*
|
|
154
|
+
* Until 0.39.0 this policy only ever REMOVED a limit, so applying it to a running
|
|
155
|
+
* service was free. Now it sets one, and systemd applies `memory.max` to a LIVE
|
|
156
|
+
* cgroup on `daemon-reload`: a value under current usage makes the kernel reclaim
|
|
157
|
+
* and then kill inside the cgroup. At daemon start that is harmless — nothing is
|
|
158
|
+
* running yet — but `doctor --fix` on a busy machine would otherwise take out
|
|
159
|
+
* somebody's session as a side effect of running a diagnostic.
|
|
160
|
+
*
|
|
161
|
+
* The cost is that a machine fixed WHILE a runaway is in progress writes a ceiling
|
|
162
|
+
* above that runaway, which protects nothing. It resolves itself: the next daemon
|
|
163
|
+
* start measures an idle cgroup, and the drift check rewrites the inflated number.
|
|
164
|
+
*/
|
|
165
|
+
const CEILING_HEADROOM_OVER_CURRENT = 1.25;
|
|
166
|
+
/**
|
|
167
|
+
* The two numbers, and the incident that decides them.
|
|
168
|
+
*
|
|
169
|
+
* Until 0.21.0 the policy was a 2 GB hard ceiling with systemd's default
|
|
170
|
+
* `OOMPolicy=stop`: one agent over the line and the whole service died, taking
|
|
171
|
+
* every other session with it (QA-112, gotcha #100). The fix removed the ceiling
|
|
172
|
+
* entirely and left a soft `MemoryHigh=80%`, on this reasoning, quoted from the
|
|
173
|
+
* code it replaced: «the machine keeps a fifth of its memory for everything that
|
|
174
|
+
* is not this service».
|
|
175
|
+
*
|
|
176
|
+
* That reasoning holds on a DEDICATED dev server and fails on a shared one, which
|
|
177
|
+
* is what vmi2502773 is: 11.9 GB total with ~4 GB permanently held by 18 docker
|
|
178
|
+
* containers, mysql, flowise, chroma and pm2. There, 80 % of TOTAL is 9.5 GB while
|
|
179
|
+
* only ~8 GB is actually free — so the soft brake sat ABOVE the real headroom and
|
|
180
|
+
* could never engage. The machine swapped, `kcompactd0` blocked for over 120
|
|
181
|
+
* seconds and the box hung; 2026-08-05 it was a `claude` at 4.2 GB, 2026-08-12 a
|
|
182
|
+
* single grep at 6.8 GB with 160 MB left. Only a power cycle recovered it.
|
|
183
|
+
*
|
|
184
|
+
* So the percentage has to be of what the machine can SPARE, not of what it has:
|
|
185
|
+
*
|
|
186
|
+
* headroom = MemAvailable + our own usage (ours is added back, or every
|
|
187
|
+
* rewrite would walk the ceiling down by what we already hold)
|
|
188
|
+
* MemoryMax = headroom − reserve
|
|
189
|
+
* MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
|
|
190
|
+
*
|
|
191
|
+
* A hard ceiling is safe to have again only because the OTHER half of the 0.21.0
|
|
192
|
+
* fix stayed: `OOMPolicy=continue` is still set below, so the kernel killing one
|
|
193
|
+
* runaway child no longer tears down the service. Restoring the ceiling without
|
|
194
|
+
* that would re-open QA-112.
|
|
195
|
+
*
|
|
196
|
+
* The clamps are the two ways this can go wrong:
|
|
197
|
+
* - floor 2 GB: on a machine whose neighbours already ate everything, a computed
|
|
198
|
+
* ceiling of a few hundred MB would mean a dev server that cannot run one
|
|
199
|
+
* agent. Measured cost of ordinary work is 512–646 MB per `claude` and 1571 MB
|
|
200
|
+
* for a workspace `pnpm -r typecheck`, so anything under 2 GB is not a dev
|
|
201
|
+
* server at all — an unusable one is a worse failure than a busy one, the same
|
|
202
|
+
* call `cpuQuotaPercent` makes below;
|
|
203
|
+
* - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
|
|
204
|
+
* machine, and a ceiling of «everything» is the bug this function exists to fix.
|
|
205
|
+
*/
|
|
206
|
+
export function memoryPolicy(facts, minCeilingBytes = facts.ownUsageBytes * CEILING_HEADROOM_OVER_CURRENT) {
|
|
207
|
+
const { totalBytes, availableBytes, ownUsageBytes, uptimeSec } = facts;
|
|
208
|
+
const withFloor = (maxBytes, measured, starved = false) => {
|
|
209
|
+
const ceiling = Math.floor(Math.max(maxBytes, minCeilingBytes));
|
|
210
|
+
return { maxBytes: ceiling, highBytes: Math.floor(ceiling * 0.8), measured, starved };
|
|
211
|
+
};
|
|
212
|
+
// Still booting: the neighbours have not claimed their memory yet, so measuring
|
|
213
|
+
// now would hand back almost the whole machine. Take a fraction of TOTAL
|
|
214
|
+
// instead — it needs no measurement at all.
|
|
215
|
+
//
|
|
216
|
+
// 55 % is chosen against the machine that failed rather than against a tidy
|
|
217
|
+
// number: 55 % of 11.9 GB is 6.5 GB, which still leaves 1.4 GB after the 4 GB
|
|
218
|
+
// its neighbours hold, and puts the brake at 5.2 GB — engaged with plenty free.
|
|
219
|
+
// 60 % looked reasonable and left only 760 MB, which is not a margin. This
|
|
220
|
+
// number works blind, so it is sized for the crowded case, and it costs a
|
|
221
|
+
// dedicated box nothing: it only applies for the first ten minutes of uptime,
|
|
222
|
+
// before any session exists. The next daemon start measures and replaces it.
|
|
223
|
+
if (!Number.isFinite(uptimeSec) || uptimeSec < BOOT_SETTLE_SEC) {
|
|
224
|
+
return withFloor(totalBytes * 0.55, false);
|
|
225
|
+
}
|
|
226
|
+
// Enough for sshd, journald, the kernel and some page cache to keep working
|
|
227
|
+
// while the cgroup sits at its ceiling. Proportional on a big machine, absolute
|
|
228
|
+
// on a small one, because 15 % of 4 GB is not enough to stay reachable.
|
|
229
|
+
const reserve = Math.max(1.5 * GIB, totalBytes * 0.15);
|
|
230
|
+
const headroom = availableBytes + ownUsageBytes;
|
|
231
|
+
const cap = totalBytes * 0.85;
|
|
232
|
+
// `min` with the cap, not a bare 2 GiB: below ~2.4 GB of RAM the floor would be
|
|
233
|
+
// ABOVE the cap and `clamp` would silently return the cap anyway — with the
|
|
234
|
+
// reserve ignored and the «2 GB or nothing» promise quietly broken. Saying it
|
|
235
|
+
// here makes the tiny-machine answer deliberate instead of accidental.
|
|
236
|
+
const floor = Math.min(2 * GIB, cap);
|
|
237
|
+
const wanted = headroom - reserve;
|
|
238
|
+
return withFloor(clamp(wanted, floor, cap), true, wanted < floor);
|
|
239
|
+
}
|
|
240
|
+
function clamp(value, low, high) {
|
|
241
|
+
return Math.min(Math.max(value, low), high);
|
|
242
|
+
}
|
|
243
|
+
/** systemd takes a plain byte count, but a unit file is read by people. */
|
|
244
|
+
function asMiB(bytes) {
|
|
245
|
+
return `${Math.max(1, Math.floor(bytes / MIB))}M`;
|
|
246
|
+
}
|
|
247
|
+
/** `MemTotal`/`MemAvailable` in bytes, or null where there is no `/proc`. */
|
|
248
|
+
function readMemInfo() {
|
|
249
|
+
let text;
|
|
250
|
+
try {
|
|
251
|
+
text = fs.readFileSync('/proc/meminfo', 'utf8');
|
|
252
|
+
}
|
|
253
|
+
catch {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
const field = (name) => {
|
|
257
|
+
const match = new RegExp(`^${name}:\\s+(\\d+) kB$`, 'm').exec(text);
|
|
258
|
+
return match?.[1] ? Number(match[1]) * 1024 : null;
|
|
259
|
+
};
|
|
260
|
+
const totalBytes = field('MemTotal');
|
|
261
|
+
const availableBytes = field('MemAvailable');
|
|
262
|
+
return totalBytes && availableBytes ? { totalBytes, availableBytes } : null;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* What our own cgroup currently holds.
|
|
266
|
+
*
|
|
267
|
+
* Read through `/proc/self/cgroup` rather than assembling the path from the
|
|
268
|
+
* service name: the runner runs as root and as a dedicated user, under
|
|
269
|
+
* `user@0.service` and under `user@1001.service`, and guessing that path wrong
|
|
270
|
+
* silently returns 0 — which would quietly shrink the ceiling by whatever we are
|
|
271
|
+
* already using. Returns 0 on cgroup v1 or in a container without the file, which
|
|
272
|
+
* is the safe direction: a slightly lower ceiling, never a higher one.
|
|
273
|
+
*/
|
|
274
|
+
function readOwnCgroupUsage() {
|
|
275
|
+
let cgroup;
|
|
276
|
+
try {
|
|
277
|
+
const line = fs
|
|
278
|
+
.readFileSync('/proc/self/cgroup', 'utf8')
|
|
279
|
+
.split('\n')
|
|
280
|
+
.find((l) => l.startsWith('0::'));
|
|
281
|
+
if (!line)
|
|
282
|
+
return null; // cgroup v1 — no v2 path to read
|
|
283
|
+
cgroup = line.slice('0::'.length).trim();
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
// `/proc/self` is the CALLER. For the daemon that is the service, and this is
|
|
289
|
+
// the hot path. For `doctor --fix` or `install-service` typed over ssh it is a
|
|
290
|
+
// `session-N.scope` holding a few MB — measuring that and calling it «what the
|
|
291
|
+
// runner holds» produced a ceiling 34 % away from the daemon's own answer on
|
|
292
|
+
// this very host, so the two paths would each see the other as drift and
|
|
293
|
+
// rewrite the file forever. Anything that is not the service must say «I do not
|
|
294
|
+
// know» and let the caller supply the number.
|
|
295
|
+
if (!cgroup.endsWith(`/${SERVICE_NAME}.service`))
|
|
296
|
+
return null;
|
|
297
|
+
return readCgroupUnreclaimable(path.join('/sys/fs/cgroup', cgroup));
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* What the cgroup holds that `MemAvailable` has NOT already counted.
|
|
301
|
+
*
|
|
302
|
+
* `memory.current` is `anon + file + kernel`, and `file` is page cache — which
|
|
303
|
+
* `MemAvailable` already lists as reclaimable. Adding the whole of
|
|
304
|
+
* `memory.current` back to `MemAvailable` therefore counts our page cache twice
|
|
305
|
+
* and inflates the ceiling by exactly that much: measured at +19 % on this host
|
|
306
|
+
* (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
|
|
307
|
+
* precisely when memory is tightest. Subtracting `file` keeps the part we really
|
|
308
|
+
* do hold and cannot give back on demand.
|
|
309
|
+
*/
|
|
310
|
+
export function readCgroupUnreclaimable(dir) {
|
|
311
|
+
try {
|
|
312
|
+
const current = Number(fs.readFileSync(path.join(dir, 'memory.current'), 'utf8').trim());
|
|
313
|
+
if (!Number.isFinite(current))
|
|
314
|
+
return null;
|
|
315
|
+
const file = /^file (\d+)$/m.exec(fs.readFileSync(path.join(dir, 'memory.stat'), 'utf8'));
|
|
316
|
+
const cache = file?.[1] ? Number(file[1]) : 0;
|
|
317
|
+
return Math.max(0, current - cache);
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
return null;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
function readUptimeSec() {
|
|
324
|
+
try {
|
|
325
|
+
return Number(fs.readFileSync('/proc/uptime', 'utf8').split(/\s+/)[0]) || 0;
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
return 0;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Everything `memoryPolicy` needs, straight off this machine.
|
|
333
|
+
*
|
|
334
|
+
* `ownUsageBytes` is passed in by callers that are not the daemon — `doctor` and
|
|
335
|
+
* `install-service` run in the operator's own cgroup and cannot read the
|
|
336
|
+
* service's usage from `/proc/self`. Returns null when the service's usage is
|
|
337
|
+
* unknowable, because guessing 0 there is the one dangerous direction: it removes
|
|
338
|
+
* the floor that stops a live session from being killed on `daemon-reload`.
|
|
339
|
+
*/
|
|
340
|
+
export function readMemoryFacts(ownUsageBytes = readOwnCgroupUsage()) {
|
|
341
|
+
const info = readMemInfo();
|
|
342
|
+
if (!info || ownUsageBytes === null)
|
|
343
|
+
return null;
|
|
344
|
+
return { ...info, ownUsageBytes, uptimeSec: readUptimeSec() };
|
|
345
|
+
}
|
|
139
346
|
/**
|
|
140
347
|
* `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
|
|
141
348
|
* box unreachable, but never so little that ordinary work is throttled.
|
|
@@ -148,8 +355,16 @@ export function limitsOverridePath(home = systemdUserHome()) {
|
|
|
148
355
|
export function cpuQuotaPercent(cpuCount = os.cpus().length) {
|
|
149
356
|
return cpuCount >= 4 ? (cpuCount - 1) * 100 : null;
|
|
150
357
|
}
|
|
151
|
-
export function buildLimitsOverride(cpuCount = os.cpus().length) {
|
|
358
|
+
export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMemoryFacts()) {
|
|
152
359
|
const quota = cpuQuotaPercent(cpuCount);
|
|
360
|
+
// No `minCeilingBytes` parameter on purpose. It existed here for one revision
|
|
361
|
+
// and passed an explicit `0`, which silently DEFEATED the default in
|
|
362
|
+
// `memoryPolicy` — the writer produced a ceiling below live cgroup usage while
|
|
363
|
+
// the drift check and `doctor` both computed the floored one. Two consequences,
|
|
364
|
+
// both found in QA: `doctor --fix` could kill a session on a loaded machine,
|
|
365
|
+
// and the file was rewritten on every start because no fixed point existed.
|
|
366
|
+
// The floor belongs to the policy, so every caller gets it and none can opt out.
|
|
367
|
+
const memory = facts ? memoryPolicy(facts) : null;
|
|
153
368
|
return ([
|
|
154
369
|
`${LIMITS_MARKER} ${LIMITS_VERSION}`,
|
|
155
370
|
'# Managed by devbridge-runner. Put your own overrides in a file that sorts',
|
|
@@ -163,14 +378,34 @@ export function buildLimitsOverride(cpuCount = os.cpus().length) {
|
|
|
163
378
|
'StartLimitBurst=0',
|
|
164
379
|
'',
|
|
165
380
|
'[Service]',
|
|
166
|
-
// The whole point of the change
|
|
381
|
+
// The whole point of the 0.21.0 change, and the reason the ceiling below is
|
|
382
|
+
// safe to have at all: one child's OOM must not take the fleet.
|
|
167
383
|
'OOMPolicy=continue',
|
|
168
|
-
//
|
|
169
|
-
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
|
|
384
|
+
// Both values also override the `MemoryMax=2G` baked into units written
|
|
385
|
+
// before 0.21.0.
|
|
386
|
+
//
|
|
387
|
+
// The unmeasurable branch used to write `MemoryMax=` + `MemoryHigh=80%` —
|
|
388
|
+
// literally the pair this release exists to remove. Because an empty
|
|
389
|
+
// assignment RESETS a directive and a drop-in sorts after the unit, that
|
|
390
|
+
// branch would have stripped the ceiling the API's fallback unit now carries
|
|
391
|
+
// and left the machine weaker than if we had written nothing at all. A
|
|
392
|
+
// drop-in must never disarm the unit it overrides, so it falls back to the
|
|
393
|
+
// same static pair used during boot.
|
|
394
|
+
...(memory
|
|
395
|
+
? [
|
|
396
|
+
`# ${memory.measured ? 'measured headroom' : 'still booting — conservative fraction of total'}`,
|
|
397
|
+
`MemoryMax=${asMiB(memory.maxBytes)}`,
|
|
398
|
+
// Soft pressure below the hard stop: the kernel reclaims and throttles
|
|
399
|
+
// here, and only kills at MemoryMax. This is the line that failed on
|
|
400
|
+
// vmi2502773 — as a percentage of TOTAL it sat above what the machine
|
|
401
|
+
// could spare, so it never engaged.
|
|
402
|
+
`MemoryHigh=${asMiB(memory.highBytes)}`,
|
|
403
|
+
]
|
|
404
|
+
: [
|
|
405
|
+
'# machine not measurable — same conservative pair as the fallback unit',
|
|
406
|
+
'MemoryMax=55%',
|
|
407
|
+
'MemoryHigh=44%',
|
|
408
|
+
]),
|
|
174
409
|
// Either a computed quota, or an explicit reset — both of which clear the
|
|
175
410
|
// `CPUQuota=80%` baked into units written before 0.21.0.
|
|
176
411
|
quota === null ? 'CPUQuota=' : `CPUQuota=${quota}%`,
|
|
@@ -186,9 +421,12 @@ export function buildLimitsOverride(cpuCount = os.cpus().length) {
|
|
|
186
421
|
*
|
|
187
422
|
* Deliberately version-based rather than content-based: an operator may add
|
|
188
423
|
* their own directives to our file, and re-writing on every start would fight
|
|
189
|
-
* them.
|
|
424
|
+
* them. The version number decides — plus, since 0.39.0 and only when `facts` are
|
|
425
|
+
* supplied, whether the measured ceiling still fits the machine (see
|
|
426
|
+
* `memoryCeilingHasDrifted`). Callers that pass no facts get the old, purely
|
|
427
|
+
* version-based answer.
|
|
190
428
|
*/
|
|
191
|
-
export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome()) {
|
|
429
|
+
export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome(), facts = null) {
|
|
192
430
|
let contents;
|
|
193
431
|
try {
|
|
194
432
|
contents = readFile(limitsOverridePath(home));
|
|
@@ -200,15 +438,61 @@ export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'u
|
|
|
200
438
|
if (!line)
|
|
201
439
|
return true;
|
|
202
440
|
const version = Number.parseInt(line.slice(LIMITS_MARKER.length).trim(), 10);
|
|
203
|
-
|
|
441
|
+
if (!Number.isFinite(version) || version < LIMITS_VERSION)
|
|
442
|
+
return true;
|
|
443
|
+
return facts ? memoryCeilingHasDrifted(contents, facts) : false;
|
|
204
444
|
}
|
|
205
|
-
/**
|
|
206
|
-
|
|
207
|
-
|
|
445
|
+
/**
|
|
446
|
+
* Since 0.39.0 the ceiling is a measurement, and a measurement goes stale.
|
|
447
|
+
*
|
|
448
|
+
* A machine that gained a database and a dozen containers after the drop-in was
|
|
449
|
+
* written would keep a ceiling sized for the machine it used to be — which is the
|
|
450
|
+
* original bug wearing a different hat. So the version marker is no longer the
|
|
451
|
+
* only thing that can make the file outdated.
|
|
452
|
+
*
|
|
453
|
+
* The 15 % band is what keeps this from becoming a rewrite on every start:
|
|
454
|
+
* `MemAvailable` moves by hundreds of MB just from page cache, and re-writing our
|
|
455
|
+
* file that often would fight an operator who added their own directives to it
|
|
456
|
+
* (the reason the check was version-only to begin with).
|
|
457
|
+
*/
|
|
458
|
+
const CEILING_DRIFT_TOLERANCE = 0.15;
|
|
459
|
+
function memoryCeilingHasDrifted(contents, facts) {
|
|
460
|
+
// Never re-measure a machine that is still booting: the static pair it is
|
|
461
|
+
// holding is deliberate, and «drift» against a lie is not drift.
|
|
462
|
+
if (!memoryPolicy(facts).measured)
|
|
463
|
+
return false;
|
|
464
|
+
// LAST match, not the first: systemd applies the last assignment, and the file
|
|
465
|
+
// explicitly invites the operator to add their own lines. Reading the first one
|
|
466
|
+
// would measure drift against a directive that is not in force.
|
|
467
|
+
const written = [...contents.matchAll(/^MemoryMax=(\d+)([KMG]?)$/gm)].at(-1);
|
|
468
|
+
// Anything we cannot read as a byte count is replaced, and that is deliberate:
|
|
469
|
+
// an EMPTY `MemoryMax=` is the 0.21.0 shape this release exists to remove, and
|
|
470
|
+
// `MemoryMax=55%` is the blind pair we write when the machine is unmeasurable —
|
|
471
|
+
// both must give way the moment a real measurement is available. The operator's
|
|
472
|
+
// own directives belong in a file that sorts after this one, which the header of
|
|
473
|
+
// every generated file says; a value they appended HERE is still honoured,
|
|
474
|
+
// because a plain `4G` parses above and is compared like any other.
|
|
475
|
+
if (!written?.[1])
|
|
476
|
+
return true;
|
|
477
|
+
const scale = { '': 1, K: 1024, M: MIB, G: 1024 * MIB }[written[2] ?? ''] ?? MIB;
|
|
478
|
+
const writtenBytes = Number(written[1]) * scale;
|
|
479
|
+
const wanted = memoryPolicy(facts).maxBytes;
|
|
480
|
+
return Math.abs(writtenBytes - wanted) / wanted > CEILING_DRIFT_TOLERANCE;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Write the drop-in. Returns false when nothing needed doing.
|
|
484
|
+
*
|
|
485
|
+
* The «never below what we already hold» guard lives in `memoryPolicy` itself
|
|
486
|
+
* (`CEILING_HEADROOM_OVER_CURRENT`), so writing, reading and the drift check all
|
|
487
|
+
* arrive at the same number — otherwise a write whose floor was binding would be
|
|
488
|
+
* seen as drifted on the very next call and rewritten forever.
|
|
489
|
+
*/
|
|
490
|
+
export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts()) {
|
|
491
|
+
if (!force && !limitsOverrideIsOutdated(undefined, home, facts))
|
|
208
492
|
return false;
|
|
209
493
|
const target = limitsOverridePath(home);
|
|
210
494
|
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
211
|
-
fs.writeFileSync(target, buildLimitsOverride(), { mode: 0o644 });
|
|
495
|
+
fs.writeFileSync(target, buildLimitsOverride(undefined, facts), { mode: 0o644 });
|
|
212
496
|
return true;
|
|
213
497
|
}
|
|
214
498
|
/**
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const RUNNER_VERSION = "0.
|
|
1
|
+
export declare const RUNNER_VERSION = "0.40.0";
|
|
2
2
|
//# sourceMappingURL=version.d.ts.map
|
package/dist/version.js
CHANGED
package/package.json
CHANGED