@bridge4dev/runner 0.38.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -287,6 +287,15 @@ class ClaudeSession {
287
287
  mcpFallbackNotice = null;
288
288
  /** Guards against overlapping capability probes. */
289
289
  capabilitiesInFlight = false;
290
+ /**
291
+ * Has the first main-loop answer of this process already asked for the
292
+ * context window? (ticket #223)
293
+ *
294
+ * `q` is `readonly` — one adapter per CLI process — so this never needs
295
+ * resetting: a restarted process is a new instance with the flag back to
296
+ * false, which is exactly the moment the meter needs a fresh number.
297
+ */
298
+ contextProbedAfterFirstReply = false;
290
299
  mode;
291
300
  /**
292
301
  * A launch-time mode this workspace does not allow, remembered so the feed
@@ -559,6 +568,13 @@ class ClaudeSession {
559
568
  // message would otherwise show no model list at all — while the control
560
569
  // requests themselves work as soon as the CLI is up.
561
570
  this.refreshCapabilities();
571
+ // Same reasoning for the context meter (ticket #223). It used to be asked
572
+ // for only at the END of a turn, so a session that had not finished its
573
+ // first one had no numbers at all and the gauge did not exist — worst in a
574
+ // session that parked a question, which can sit there for hours. This probe
575
+ // costs one control round-trip and gives the meter its denominator (the
576
+ // model's window) from the moment the process is up.
577
+ this.refreshContextUsage();
562
578
  }
563
579
  /**
564
580
  * Write this session's MCP config to a 0600 file and return its path, or null
@@ -623,10 +639,14 @@ class ClaudeSession {
623
639
  this.output.push(maskSecrets(event));
624
640
  }
625
641
  /**
626
- * How full the context window is, after a finished turn. Fire-and-forget with
627
- * a timeout on purpose: this is one more control round-trip on a channel that
628
- * can hang, and it must never be able to stall the event loop that carries the
629
- * conversation.
642
+ * How full the context window is. Fire-and-forget with a timeout on purpose:
643
+ * this is one more control round-trip on a channel that can hang, and it must
644
+ * never be able to stall the event loop that carries the conversation.
645
+ *
646
+ * Called from four places, each answering a different gap: process start and
647
+ * first main-loop reply (ticket #223 — otherwise a session shows no gauge at
648
+ * all until its first turn ends), model change (the window's size differs 5x
649
+ * between 200k and 1M), and end of turn.
630
650
  */
631
651
  refreshContextUsage() {
632
652
  if (this.stopped)
@@ -1845,6 +1865,17 @@ class ClaudeSession {
1845
1865
  });
1846
1866
  }
1847
1867
  }
1868
+ // Ticket #223. The startup probe fires before the CLI has read
1869
+ // anything, so its number is the empty window; this one lands after
1870
+ // the agent has actually said something and is the first figure
1871
+ // worth showing. Once per process, and only for the main loop: it is
1872
+ // a control round-trip on a channel that can hang, subagent chatter
1873
+ // is not this session's context, and the end-of-turn probe covers
1874
+ // everything after.
1875
+ if (!fromSubagent && !this.contextProbedAfterFirstReply) {
1876
+ this.contextProbedAfterFirstReply = true;
1877
+ this.refreshContextUsage();
1878
+ }
1848
1879
  break;
1849
1880
  }
1850
1881
  case 'user': {
@@ -18,9 +18,32 @@ const execFileAsync = promisify(execFile);
18
18
  */
19
19
  /** Directory inside the worktree; also the line written to info/exclude. */
20
20
  export const ATTACHMENT_DIR = '.devbridge/attachments';
21
- /** Hard ceiling per file; the API allows 10 MB for images and 25 MB otherwise. */
22
- const MAX_ATTACHMENT_BYTES = 26 * 1024 * 1024;
23
- const DOWNLOAD_TIMEOUT_MS = 60_000;
21
+ /**
22
+ * Hard ceiling per file; the API allows 10 MB for images and 50 MB otherwise.
23
+ *
24
+ * Mirror of `MAX_DOCUMENT_ATTACHMENT_SIZE_BYTES` in `@devbridge/shared` plus a
25
+ * megabyte of slack — the runner cannot import that package (see
26
+ * `recipe-schema.ts`: the published tarball would not resolve it). The slack is
27
+ * what keeps the two halves from disagreeing on a rounding: the API refuses
28
+ * first, with an error the person actually sees, and this only catches bytes
29
+ * that no upload path should have produced.
30
+ *
31
+ * Raise the API side and this one in the SAME runner release. A ceiling that
32
+ * moves only in DevBridge lets a file upload and then vanish: the refusal below
33
+ * is a warning line in the session, not something the uploader is shown.
34
+ */
35
+ const MAX_ATTACHMENT_BYTES = 51 * 1024 * 1024;
36
+ /**
37
+ * A minute was plenty while the ceiling was 26 MB; at 50 MB it is a coin flip.
38
+ *
39
+ * The whole file is buffered by the API and pushed through nginx in one
40
+ * response, so this timeout covers the entire transfer, not idle time between
41
+ * packets. 60 s for 50 MB demands ~7 Mbit/s sustained end to end — ordinary for
42
+ * two VPSes, not ordinary for a dev server on a home uplink or behind a
43
+ * throttled proxy. Three minutes covers ~2.3 Mbit/s, and the cost of being
44
+ * generous is nil: a stalled download fails the same way, just later.
45
+ */
46
+ const DOWNLOAD_TIMEOUT_MS = 180_000;
24
47
  /**
25
48
  * A file name that is safe to write and unambiguous to read.
26
49
  *
@@ -153,9 +176,16 @@ export async function saveAttachments(input) {
153
176
  pruneAttachmentDir(dir);
154
177
  return { saved, failed };
155
178
  }
156
- /** Keep at most this much history in one worktree's attachment folder. */
179
+ /**
180
+ * Keep at most this much history in one worktree's attachment folder.
181
+ *
182
+ * Sized in files, not in bytes: 512 MB was twenty attachments at the old 25 MB
183
+ * ceiling and would be ten at the new one — an afternoon of work, after which
184
+ * the folder starts evicting documents a running session may still be told to
185
+ * open. A gigabyte restores the same twenty-file depth.
186
+ */
157
187
  const ATTACHMENT_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
158
- const ATTACHMENT_DIR_BUDGET_BYTES = 512 * 1024 * 1024;
188
+ const ATTACHMENT_DIR_BUDGET_BYTES = 1024 * 1024 * 1024;
159
189
  /**
160
190
  * Delete old attachments from a session worktree.
161
191
  *
package/dist/index.js CHANGED
@@ -17,7 +17,7 @@ import { Supervisor } from './supervisor.js';
17
17
  import { readStatusFile, isPidAlive, writeStatusFile, STATUS_FRESH_MS } from './status-file.js';
18
18
  import { RunnerWsClient } from './ws-client.js';
19
19
  import { RUNNER_VERSION } from './version.js';
20
- import { buildUnit, cpuQuotaPercent, limitsOverrideIsOutdated, limitsOverridePath, unitExecTarget, unitPath, writeLimitsOverride, LIMITS_VERSION, } from './service-unit.js';
20
+ import { buildUnit, cpuQuotaPercent, limitsOverrideIsOutdated, limitsOverridePath, memoryPolicy, readMemoryFacts, unitExecTarget, unitPath, writeLimitsOverride, LIMITS_VERSION, } from './service-unit.js';
21
21
  import { readOomKills, recordCrash, takeLastExit } from './crash-note.js';
22
22
  import { agentAuthStatuses } from './auth-relay.js';
23
23
  import { addSafeDirectory, agentConfigContour, dockerCheck, ensureAgentPath, firstUnreachableAncestor, hasSafeDirectory, inspectPath, knownWorkspacePaths, lingerEnabled, nodeCheck, otherHomeWithAgents, runnerIdentity, safeDirectoryCommand, systemctlHint, systemdUserBusReachable, systemdUserEnv, } from './environment.js';
@@ -359,7 +359,9 @@ function runnerCapabilities(apiUrlOverride) {
359
359
  memTotalBytes: os.totalmem(),
360
360
  memAvailableBytes: os.freemem(),
361
361
  limitsVersion: LIMITS_VERSION,
362
- limitsCurrent: !limitsOverrideIsOutdated(),
362
+ // Facts passed on purpose: since 0.39.0 «current» also means the measured
363
+ // ceiling still fits this machine, not just that the version matches.
364
+ limitsCurrent: !limitsOverrideIsOutdated(undefined, undefined, readMemoryFacts()),
363
365
  },
364
366
  /**
365
367
  * Identity of THIS process, so the API can tell a network blink from a
@@ -497,20 +499,32 @@ const RESTART_DELAY_MS = 1_500;
497
499
  */
498
500
  async function repairResourceLimits() {
499
501
  try {
500
- if (!writeLimitsOverride())
502
+ const facts = readMemoryFacts();
503
+ if (facts && memoryPolicy(facts).starved) {
504
+ // The only signal the machine's owner will ever get that the neighbours
505
+ // have taken the box: the honest headroom was below the 2 GB floor, so the
506
+ // ceiling we are about to write sits above what is actually free.
507
+ log.warn('daemon: not enough free memory for a real ceiling — neighbours have the machine', {
508
+ availableMB: Math.round(facts.availableBytes / 1048576),
509
+ totalMB: Math.round(facts.totalBytes / 1048576),
510
+ });
511
+ }
512
+ if (!writeLimitsOverride(false, undefined, facts))
501
513
  return;
502
514
  log.warn('daemon: resource limits drop-in written — reloading systemd', {
503
515
  path: limitsOverridePath(),
504
516
  version: LIMITS_VERSION,
517
+ ...(facts ? memoryPolicy(facts) : {}),
505
518
  });
506
519
  await execFileAsync('systemctl', ['--user', 'daemon-reload'], {
507
520
  timeout: 15_000,
508
521
  env: systemdUserEnv(),
509
522
  });
510
523
  // Deliberately no restart: `daemon-reload` alone is enough for these
511
- // directives (verified live — OOMPolicy went stop→continue and MemoryMax
512
- // 2G→infinity with the PID unchanged), and restarting here would park every
513
- // live session to apply a setting that is already in force.
524
+ // directives — verified live twice, in both directions: 2026-07-30 removing a
525
+ // ceiling (MemoryMax 2G→infinity, OOMPolicy stop→continue) and 2026-08-12
526
+ // installing one (infinity→1234M), each with the PID unchanged. Restarting
527
+ // here would park every live session to apply a setting already in force.
514
528
  log.info('daemon: resource limits applied without a restart');
515
529
  }
516
530
  catch (error) {
@@ -520,6 +534,51 @@ async function repairResourceLimits() {
520
534
  });
521
535
  }
522
536
  }
537
+ /**
538
+ * How often the daemon re-measures the machine after the first time.
539
+ *
540
+ * Doing this only at startup left a hole big enough to matter. The runner is a
541
+ * user unit and comes up 20–60 seconds into a boot, which is inside
542
+ * `BOOT_SETTLE_SEC` — so a machine that reboots gets the blind static fraction
543
+ * and then keeps it for as long as the daemon lives, which can be weeks. On a
544
+ * dedicated 8 GB box that is 3.5 GB of soft brake against the 3.3 GB three
545
+ * sessions and a build actually cost: throttling ordinary work for no reason.
546
+ * Worse, `limitsCurrent` in `hello` is computed from FRESH facts, so the API
547
+ * would keep logging «limits outdated» on every reconnect for a machine that had
548
+ * no way to fix itself.
549
+ *
550
+ * Hourly is right because drift is slow by definition — it tracks what else is
551
+ * installed on the machine, not what it is doing this minute — and because the
552
+ * check writes nothing when the version matches and the ceiling still fits.
553
+ */
554
+ const LIMITS_RECHECK_MS = 3_600_000;
555
+ /**
556
+ * What the SERVICE's cgroup holds, for the paths that are not the service.
557
+ *
558
+ * `readMemoryFacts()` defaults to reading `/proc/self`, which is right for the
559
+ * daemon and wrong for everything a person types: `doctor --fix` and
560
+ * `install-service` run in the operator's own `session-N.scope`, worth a few MB.
561
+ * Measuring that and calling it «what the runner holds» made the CLI compute a
562
+ * ceiling 34 % away from the daemon's answer on a real host — each path then saw
563
+ * the other as drift and rewrote the file, forever.
564
+ *
565
+ * `MemoryCurrent` is `memory.current`, so it still counts page cache the way
566
+ * `MemAvailable` does. Subtracting it is not worth a second systemd call here:
567
+ * the CLI paths run once, by hand, and erring toward a LOWER ceiling is the safe
568
+ * direction. Returns null when the service is not running or systemd cannot be
569
+ * reached — the caller then has no facts, and writes nothing rather than writing
570
+ * a ceiling with no floor under it.
571
+ */
572
+ async function serviceMemoryCurrent() {
573
+ try {
574
+ const { stdout } = await execFileAsync('systemctl', ['--user', 'show', 'devbridge-runner', '-p', 'MemoryCurrent', '--value'], { timeout: 10_000, env: systemdUserEnv() });
575
+ const value = Number(stdout.trim());
576
+ return Number.isFinite(value) && value > 0 ? value : null;
577
+ }
578
+ catch {
579
+ return null;
580
+ }
581
+ }
523
582
  /**
524
583
  * Ticket #119. `ClaudeAdapter.stop()` removes each session's MCP config file,
525
584
  * but a kill -9, an OOM or a machine reboot leaves it behind — a live project
@@ -620,6 +679,10 @@ async function cmdDaemon() {
620
679
  // that stays connected for weeks.
621
680
  const pruneTimer = setInterval(() => supervisor.pruneJournals(), 6 * 3_600_000);
622
681
  pruneTimer.unref();
682
+ // Re-measure the machine — see `LIMITS_RECHECK_MS`. Writes nothing in the
683
+ // normal case, so this is a file read and some arithmetic once an hour.
684
+ const limitsTimer = setInterval(() => void repairResourceLimits(), LIMITS_RECHECK_MS);
685
+ limitsTimer.unref();
623
686
  const shutdown = (signal) => {
624
687
  log.info(`daemon: ${signal} received, shutting down`);
625
688
  supervisor.shutdown();
@@ -720,8 +783,9 @@ async function cmdInstallService() {
720
783
  print(`Wrote ${target}`);
721
784
  // Resource policy is a versioned drop-in, not part of the unit — see
722
785
  // `buildLimitsOverride`. Forced here: a fresh install must have it even if a
723
- // file from an older runner is already sitting there.
724
- writeLimitsOverride(true);
786
+ // file from an older runner is already sitting there. Facts come from the
787
+ // service rather than from `/proc/self`, which here is the installing shell.
788
+ writeLimitsOverride(true, undefined, readMemoryFacts(await serviceMemoryCurrent()));
725
789
  print(`Wrote ${limitsOverridePath()}`);
726
790
  if (!exec.viaCommand) {
727
791
  // Worth saying out loud: a unit pinned to a file inside the package directory
@@ -1143,10 +1207,22 @@ async function cmdDoctor(args) {
1143
1207
  print(` fits sessions ~${advised} (at ~1.5 GB per session under load)`);
1144
1208
  print('');
1145
1209
  print('Service limits');
1146
- const outdated = limitsOverrideIsOutdated();
1210
+ const memFacts = readMemoryFacts(await serviceMemoryCurrent());
1211
+ const outdated = limitsOverrideIsOutdated(undefined, undefined, memFacts);
1147
1212
  print(` drop-in ${limitsOverridePath()}`);
1148
1213
  print(` version ${outdated ? `MISSING or OLD (want ${LIMITS_VERSION})` : LIMITS_VERSION}`);
1149
1214
  print(` cpu quota ${quota === null ? 'none (fewer than 4 cpus)' : `${quota}%`}`);
1215
+ // Spelled out because the whole point of 0.39.0 is that this number is measured
1216
+ // rather than a percentage of the machine: on a server with neighbours, «80% of
1217
+ // total» sat above what was actually free and never engaged (gotcha #301).
1218
+ if (memFacts) {
1219
+ const policy = memoryPolicy(memFacts);
1220
+ const mb = (bytes) => `${Math.round(bytes / 1024 / 1024)} MB`;
1221
+ print(` memory ceiling ${mb(policy.maxBytes)} hard / ${mb(policy.highBytes)} soft`);
1222
+ print(` sized from ${policy.measured
1223
+ ? `${mb(memFacts.availableBytes)} available + ${mb(memFacts.ownUsageBytes)} ours`
1224
+ : `55% of ${mb(memFacts.totalBytes)} — still booting, remeasured on the next start`}`);
1225
+ }
1150
1226
  let effective;
1151
1227
  try {
1152
1228
  const { stdout } = await execFileAsync('systemctl', [
@@ -1222,9 +1298,22 @@ async function cmdDoctor(args) {
1222
1298
  process.exit(1);
1223
1299
  return;
1224
1300
  }
1225
- writeLimitsOverride(true);
1301
+ const fixFacts = readMemoryFacts(await serviceMemoryCurrent());
1302
+ writeLimitsOverride(true, undefined, fixFacts);
1226
1303
  print('');
1227
1304
  print(`Wrote ${limitsOverridePath()}`);
1305
+ // Say it out loud when the ceiling had to be raised above the honest answer to
1306
+ // avoid killing whatever is running right now. Lowering a ceiling under load is
1307
+ // a decision, not a side effect of running a diagnostic — and without this line
1308
+ // the number written here looks unexplainably generous.
1309
+ if (fixFacts) {
1310
+ const honest = memoryPolicy(fixFacts, 0);
1311
+ const applied = memoryPolicy(fixFacts);
1312
+ if (applied.maxBytes > honest.maxBytes) {
1313
+ const mb = (b) => Math.round(b / 1024 / 1024);
1314
+ print(`note: ceiling raised to ${mb(applied.maxBytes)} MB — the service already holds ${mb(fixFacts.ownUsageBytes)} MB, and ${mb(honest.maxBytes)} MB would kill it on reload. Re-run when idle.`);
1315
+ }
1316
+ }
1228
1317
  try {
1229
1318
  await execFileAsync('systemctl', ['--user', 'daemon-reload'], { env: systemdUserEnv() });
1230
1319
  print('systemctl --user daemon-reload — done.');
@@ -57,8 +57,114 @@ export declare function buildUnit(execStart?: string, nodeBinary?: string): stri
57
57
  * which is the only way to fix the servers that already have the bad numbers
58
58
  * baked in — and it never overwrites a unit the operator edited by hand.
59
59
  */
60
- export declare const LIMITS_VERSION = 2;
60
+ export declare const LIMITS_VERSION = 3;
61
61
  export declare function limitsOverridePath(home?: string): string;
62
+ /**
63
+ * What the memory policy needs to know about this machine. Read once and passed
64
+ * in, so the policy itself is a pure function that a test can drive with the
65
+ * numbers of a server it does not have.
66
+ */
67
+ export interface MemoryFacts {
68
+ /** `MemTotal` — the whole machine. */
69
+ totalBytes: number;
70
+ /** `MemAvailable` — what can be handed out without swapping. Already excludes
71
+ * what the co-tenants hold, and already counts reclaimable page cache. */
72
+ availableBytes: number;
73
+ /**
74
+ * What our own cgroup holds that `MemAvailable` has not already counted —
75
+ * `memory.current` minus its page cache. Added back to the headroom, and used
76
+ * as the floor under the ceiling; see `memoryPolicy`.
77
+ */
78
+ ownUsageBytes: number;
79
+ /** Seconds since boot. Below `BOOT_SETTLE_SEC` the measurement is a lie. */
80
+ uptimeSec: number;
81
+ }
82
+ export interface MemoryPolicy {
83
+ maxBytes: number;
84
+ highBytes: number;
85
+ /** false = the conservative static pair, because the machine was still booting. */
86
+ measured: boolean;
87
+ /**
88
+ * The honest headroom was below the 2 GB floor — the neighbours have eaten the
89
+ * machine and this ceiling sits ABOVE what is free. Not an error (a dev server
90
+ * that cannot run one agent is worse than a busy one), but the only signal the
91
+ * owner will ever get that the box is oversubscribed, so callers log it.
92
+ */
93
+ starved: boolean;
94
+ }
95
+ /**
96
+ * How long after boot `MemAvailable` starts telling the truth.
97
+ *
98
+ * The runner is a user unit ordered `After=network-online.target`, and the things
99
+ * it shares the machine with — dockerd and its containers, mysql, pm2 — come up
100
+ * around the same time. Measure at second 20 and the neighbours have not claimed
101
+ * their memory yet: on the 12 GB machine below that reads as 11 GB free and
102
+ * produces a ceiling that protects nothing.
103
+ */
104
+ export declare const BOOT_SETTLE_SEC = 600;
105
+ /**
106
+ * The two numbers, and the incident that decides them.
107
+ *
108
+ * Until 0.21.0 the policy was a 2 GB hard ceiling with systemd's default
109
+ * `OOMPolicy=stop`: one agent over the line and the whole service died, taking
110
+ * every other session with it (QA-112, gotcha #100). The fix removed the ceiling
111
+ * entirely and left a soft `MemoryHigh=80%`, on this reasoning, quoted from the
112
+ * code it replaced: «the machine keeps a fifth of its memory for everything that
113
+ * is not this service».
114
+ *
115
+ * That reasoning holds on a DEDICATED dev server and fails on a shared one, which
116
+ * is what vmi2502773 is: 11.9 GB total with ~4 GB permanently held by 18 docker
117
+ * containers, mysql, flowise, chroma and pm2. There, 80 % of TOTAL is 9.5 GB while
118
+ * only ~8 GB is actually free — so the soft brake sat ABOVE the real headroom and
119
+ * could never engage. The machine swapped, `kcompactd0` blocked for over 120
120
+ * seconds and the box hung; 2026-08-05 it was a `claude` at 4.2 GB, 2026-08-12 a
121
+ * single grep at 6.8 GB with 160 MB left. Only a power cycle recovered it.
122
+ *
123
+ * So the percentage has to be of what the machine can SPARE, not of what it has:
124
+ *
125
+ * headroom = MemAvailable + our own usage (ours is added back, or every
126
+ * rewrite would walk the ceiling down by what we already hold)
127
+ * MemoryMax = headroom − reserve
128
+ * MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
129
+ *
130
+ * A hard ceiling is safe to have again only because the OTHER half of the 0.21.0
131
+ * fix stayed: `OOMPolicy=continue` is still set below, so the kernel killing one
132
+ * runaway child no longer tears down the service. Restoring the ceiling without
133
+ * that would re-open QA-112.
134
+ *
135
+ * The clamps are the two ways this can go wrong:
136
+ * - floor 2 GB: on a machine whose neighbours already ate everything, a computed
137
+ * ceiling of a few hundred MB would mean a dev server that cannot run one
138
+ * agent. Measured cost of ordinary work is 512–646 MB per `claude` and 1571 MB
139
+ * for a workspace `pnpm -r typecheck`, so anything under 2 GB is not a dev
140
+ * server at all — an unusable one is a worse failure than a busy one, the same
141
+ * call `cpuQuotaPercent` makes below;
142
+ * - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
143
+ * machine, and a ceiling of «everything» is the bug this function exists to fix.
144
+ */
145
+ export declare function memoryPolicy(facts: MemoryFacts, minCeilingBytes?: number): MemoryPolicy;
146
+ /**
147
+ * What the cgroup holds that `MemAvailable` has NOT already counted.
148
+ *
149
+ * `memory.current` is `anon + file + kernel`, and `file` is page cache — which
150
+ * `MemAvailable` already lists as reclaimable. Adding the whole of
151
+ * `memory.current` back to `MemAvailable` therefore counts our page cache twice
152
+ * and inflates the ceiling by exactly that much: measured at +19 % on this host
153
+ * (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
154
+ * precisely when memory is tightest. Subtracting `file` keeps the part we really
155
+ * do hold and cannot give back on demand.
156
+ */
157
+ export declare function readCgroupUnreclaimable(dir: string): number | null;
158
+ /**
159
+ * Everything `memoryPolicy` needs, straight off this machine.
160
+ *
161
+ * `ownUsageBytes` is passed in by callers that are not the daemon — `doctor` and
162
+ * `install-service` run in the operator's own cgroup and cannot read the
163
+ * service's usage from `/proc/self`. Returns null when the service's usage is
164
+ * unknowable, because guessing 0 there is the one dangerous direction: it removes
165
+ * the floor that stops a live session from being killed on `daemon-reload`.
166
+ */
167
+ export declare function readMemoryFacts(ownUsageBytes?: number | null): MemoryFacts | null;
62
168
  /**
63
169
  * `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
64
170
  * box unreachable, but never so little that ordinary work is throttled.
@@ -69,17 +175,27 @@ export declare function limitsOverridePath(home?: string): string;
69
175
  * than a busy one.
70
176
  */
71
177
  export declare function cpuQuotaPercent(cpuCount?: number): number | null;
72
- export declare function buildLimitsOverride(cpuCount?: number): string;
178
+ export declare function buildLimitsOverride(cpuCount?: number, facts?: MemoryFacts | null): string;
73
179
  /**
74
180
  * Is the shipped resource policy missing or from an older runner?
75
181
  *
76
182
  * Deliberately version-based rather than content-based: an operator may add
77
183
  * their own directives to our file, and re-writing on every start would fight
78
- * them. Only the version number decides.
184
+ * them. The version number decides — plus, since 0.39.0 and only when `facts` are
185
+ * supplied, whether the measured ceiling still fits the machine (see
186
+ * `memoryCeilingHasDrifted`). Callers that pass no facts get the old, purely
187
+ * version-based answer.
188
+ */
189
+ export declare function limitsOverrideIsOutdated(readFile?: (p: string) => string, home?: string, facts?: MemoryFacts | null): boolean;
190
+ /**
191
+ * Write the drop-in. Returns false when nothing needed doing.
192
+ *
193
+ * The «never below what we already hold» guard lives in `memoryPolicy` itself
194
+ * (`CEILING_HEADROOM_OVER_CURRENT`), so writing, reading and the drift check all
195
+ * arrive at the same number — otherwise a write whose floor was binding would be
196
+ * seen as drifted on the very next call and rewritten forever.
79
197
  */
80
- export declare function limitsOverrideIsOutdated(readFile?: (p: string) => string, home?: string): boolean;
81
- /** Write the drop-in. Returns false when nothing needed doing. */
82
- export declare function writeLimitsOverride(force?: boolean, home?: string): boolean;
198
+ export declare function writeLimitsOverride(force?: boolean, home?: string, facts?: MemoryFacts | null): boolean;
83
199
  /**
84
200
  * Does the installed unit point at something that no longer exists?
85
201
  *
@@ -129,13 +129,220 @@ export function buildUnit(execStart, nodeBinary = process.execPath) {
129
129
  * which is the only way to fix the servers that already have the bad numbers
130
130
  * baked in — and it never overwrites a unit the operator edited by hand.
131
131
  */
132
- export const LIMITS_VERSION = 2;
132
+ export const LIMITS_VERSION = 3;
133
133
  const LIMITS_MARKER = '# devbridge-limits-version:';
134
134
  /** `zz-` so it sorts last: an operator's own drop-in should still win. */
135
135
  const LIMITS_FILE = 'zz-devbridge-limits.conf';
136
136
  export function limitsOverridePath(home = systemdUserHome()) {
137
137
  return path.join(home, '.config', 'systemd', 'user', `${SERVICE_NAME}.service.d`, LIMITS_FILE);
138
138
  }
139
+ const MIB = 1024 * 1024;
140
+ const GIB = 1024 * MIB;
141
+ /**
142
+ * How long after boot `MemAvailable` starts telling the truth.
143
+ *
144
+ * The runner is a user unit ordered `After=network-online.target`, and the things
145
+ * it shares the machine with — dockerd and its containers, mysql, pm2 — come up
146
+ * around the same time. Measure at second 20 and the neighbours have not claimed
147
+ * their memory yet: on the 12 GB machine below that reads as 11 GB free and
148
+ * produces a ceiling that protects nothing.
149
+ */
150
+ export const BOOT_SETTLE_SEC = 600;
151
+ /**
152
+ * The ceiling is never written BELOW what the cgroup already holds.
153
+ *
154
+ * Until 0.39.0 this policy only ever REMOVED a limit, so applying it to a running
155
+ * service was free. Now it sets one, and systemd applies `memory.max` to a LIVE
156
+ * cgroup on `daemon-reload`: a value under current usage makes the kernel reclaim
157
+ * and then kill inside the cgroup. At daemon start that is harmless — nothing is
158
+ * running yet — but `doctor --fix` on a busy machine would otherwise take out
159
+ * somebody's session as a side effect of running a diagnostic.
160
+ *
161
+ * The cost is that a machine fixed WHILE a runaway is in progress writes a ceiling
162
+ * above that runaway, which protects nothing. It resolves itself: the next daemon
163
+ * start measures an idle cgroup, and the drift check rewrites the inflated number.
164
+ */
165
+ const CEILING_HEADROOM_OVER_CURRENT = 1.25;
166
+ /**
167
+ * The two numbers, and the incident that decides them.
168
+ *
169
+ * Until 0.21.0 the policy was a 2 GB hard ceiling with systemd's default
170
+ * `OOMPolicy=stop`: one agent over the line and the whole service died, taking
171
+ * every other session with it (QA-112, gotcha #100). The fix removed the ceiling
172
+ * entirely and left a soft `MemoryHigh=80%`, on this reasoning, quoted from the
173
+ * code it replaced: «the machine keeps a fifth of its memory for everything that
174
+ * is not this service».
175
+ *
176
+ * That reasoning holds on a DEDICATED dev server and fails on a shared one, which
177
+ * is what vmi2502773 is: 11.9 GB total with ~4 GB permanently held by 18 docker
178
+ * containers, mysql, flowise, chroma and pm2. There, 80 % of TOTAL is 9.5 GB while
179
+ * only ~8 GB is actually free — so the soft brake sat ABOVE the real headroom and
180
+ * could never engage. The machine swapped, `kcompactd0` blocked for over 120
181
+ * seconds and the box hung; 2026-08-05 it was a `claude` at 4.2 GB, 2026-08-12 a
182
+ * single grep at 6.8 GB with 160 MB left. Only a power cycle recovered it.
183
+ *
184
+ * So the percentage has to be of what the machine can SPARE, not of what it has:
185
+ *
186
+ * headroom = MemAvailable + our own usage (ours is added back, or every
187
+ * rewrite would walk the ceiling down by what we already hold)
188
+ * MemoryMax = headroom − reserve
189
+ * MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
190
+ *
191
+ * A hard ceiling is safe to have again only because the OTHER half of the 0.21.0
192
+ * fix stayed: `OOMPolicy=continue` is still set below, so the kernel killing one
193
+ * runaway child no longer tears down the service. Restoring the ceiling without
194
+ * that would re-open QA-112.
195
+ *
196
+ * The clamps are the two ways this can go wrong:
197
+ * - floor 2 GB: on a machine whose neighbours already ate everything, a computed
198
+ * ceiling of a few hundred MB would mean a dev server that cannot run one
199
+ * agent. Measured cost of ordinary work is 512–646 MB per `claude` and 1571 MB
200
+ * for a workspace `pnpm -r typecheck`, so anything under 2 GB is not a dev
201
+ * server at all — an unusable one is a worse failure than a busy one, the same
202
+ * call `cpuQuotaPercent` makes below;
203
+ * - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
204
+ * machine, and a ceiling of «everything» is the bug this function exists to fix.
205
+ */
206
+ export function memoryPolicy(facts, minCeilingBytes = facts.ownUsageBytes * CEILING_HEADROOM_OVER_CURRENT) {
207
+ const { totalBytes, availableBytes, ownUsageBytes, uptimeSec } = facts;
208
+ const withFloor = (maxBytes, measured, starved = false) => {
209
+ const ceiling = Math.floor(Math.max(maxBytes, minCeilingBytes));
210
+ return { maxBytes: ceiling, highBytes: Math.floor(ceiling * 0.8), measured, starved };
211
+ };
212
+ // Still booting: the neighbours have not claimed their memory yet, so measuring
213
+ // now would hand back almost the whole machine. Take a fraction of TOTAL
214
+ // instead — it needs no measurement at all.
215
+ //
216
+ // 55 % is chosen against the machine that failed rather than against a tidy
217
+ // number: 55 % of 11.9 GB is 6.5 GB, which still leaves 1.4 GB after the 4 GB
218
+ // its neighbours hold, and puts the brake at 5.2 GB — engaged with plenty free.
219
+ // 60 % looked reasonable and left only 760 MB, which is not a margin. This
220
+ // number works blind, so it is sized for the crowded case, and it costs a
221
+ // dedicated box nothing: it only applies for the first ten minutes of uptime,
222
+ // before any session exists. The next daemon start measures and replaces it.
223
+ if (!Number.isFinite(uptimeSec) || uptimeSec < BOOT_SETTLE_SEC) {
224
+ return withFloor(totalBytes * 0.55, false);
225
+ }
226
+ // Enough for sshd, journald, the kernel and some page cache to keep working
227
+ // while the cgroup sits at its ceiling. Proportional on a big machine, absolute
228
+ // on a small one, because 15 % of 4 GB is not enough to stay reachable.
229
+ const reserve = Math.max(1.5 * GIB, totalBytes * 0.15);
230
+ const headroom = availableBytes + ownUsageBytes;
231
+ const cap = totalBytes * 0.85;
232
+ // `min` with the cap, not a bare 2 GiB: below ~2.4 GB of RAM the floor would be
233
+ // ABOVE the cap and `clamp` would silently return the cap anyway — with the
234
+ // reserve ignored and the «2 GB or nothing» promise quietly broken. Saying it
235
+ // here makes the tiny-machine answer deliberate instead of accidental.
236
+ const floor = Math.min(2 * GIB, cap);
237
+ const wanted = headroom - reserve;
238
+ return withFloor(clamp(wanted, floor, cap), true, wanted < floor);
239
+ }
240
+ function clamp(value, low, high) {
241
+ return Math.min(Math.max(value, low), high);
242
+ }
243
+ /** systemd takes a plain byte count, but a unit file is read by people. */
244
+ function asMiB(bytes) {
245
+ return `${Math.max(1, Math.floor(bytes / MIB))}M`;
246
+ }
247
+ /** `MemTotal`/`MemAvailable` in bytes, or null where there is no `/proc`. */
248
+ function readMemInfo() {
249
+ let text;
250
+ try {
251
+ text = fs.readFileSync('/proc/meminfo', 'utf8');
252
+ }
253
+ catch {
254
+ return null;
255
+ }
256
+ const field = (name) => {
257
+ const match = new RegExp(`^${name}:\\s+(\\d+) kB$`, 'm').exec(text);
258
+ return match?.[1] ? Number(match[1]) * 1024 : null;
259
+ };
260
+ const totalBytes = field('MemTotal');
261
+ const availableBytes = field('MemAvailable');
262
+ return totalBytes && availableBytes ? { totalBytes, availableBytes } : null;
263
+ }
264
+ /**
265
+ * What our own cgroup currently holds.
266
+ *
267
+ * Read through `/proc/self/cgroup` rather than assembling the path from the
268
+ * service name: the runner runs as root and as a dedicated user, under
269
+ * `user@0.service` and under `user@1001.service`, and guessing that path wrong
270
+ * silently returns 0 — which would quietly shrink the ceiling by whatever we are
271
+ * already using. Returns 0 on cgroup v1 or in a container without the file, which
272
+ * is the safe direction: a slightly lower ceiling, never a higher one.
273
+ */
274
+ function readOwnCgroupUsage() {
275
+ let cgroup;
276
+ try {
277
+ const line = fs
278
+ .readFileSync('/proc/self/cgroup', 'utf8')
279
+ .split('\n')
280
+ .find((l) => l.startsWith('0::'));
281
+ if (!line)
282
+ return null; // cgroup v1 — no v2 path to read
283
+ cgroup = line.slice('0::'.length).trim();
284
+ }
285
+ catch {
286
+ return null;
287
+ }
288
+ // `/proc/self` is the CALLER. For the daemon that is the service, and this is
289
+ // the hot path. For `doctor --fix` or `install-service` typed over ssh it is a
290
+ // `session-N.scope` holding a few MB — measuring that and calling it «what the
291
+ // runner holds» produced a ceiling 34 % away from the daemon's own answer on
292
+ // this very host, so the two paths would each see the other as drift and
293
+ // rewrite the file forever. Anything that is not the service must say «I do not
294
+ // know» and let the caller supply the number.
295
+ if (!cgroup.endsWith(`/${SERVICE_NAME}.service`))
296
+ return null;
297
+ return readCgroupUnreclaimable(path.join('/sys/fs/cgroup', cgroup));
298
+ }
299
+ /**
300
+ * What the cgroup holds that `MemAvailable` has NOT already counted.
301
+ *
302
+ * `memory.current` is `anon + file + kernel`, and `file` is page cache — which
303
+ * `MemAvailable` already lists as reclaimable. Adding the whole of
304
+ * `memory.current` back to `MemAvailable` therefore counts our page cache twice
305
+ * and inflates the ceiling by exactly that much: measured at +19 % on this host
306
+ * (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
307
+ * precisely when memory is tightest. Subtracting `file` keeps the part we really
308
+ * do hold and cannot give back on demand.
309
+ */
310
+ export function readCgroupUnreclaimable(dir) {
311
+ try {
312
+ const current = Number(fs.readFileSync(path.join(dir, 'memory.current'), 'utf8').trim());
313
+ if (!Number.isFinite(current))
314
+ return null;
315
+ const file = /^file (\d+)$/m.exec(fs.readFileSync(path.join(dir, 'memory.stat'), 'utf8'));
316
+ const cache = file?.[1] ? Number(file[1]) : 0;
317
+ return Math.max(0, current - cache);
318
+ }
319
+ catch {
320
+ return null;
321
+ }
322
+ }
323
+ function readUptimeSec() {
324
+ try {
325
+ return Number(fs.readFileSync('/proc/uptime', 'utf8').split(/\s+/)[0]) || 0;
326
+ }
327
+ catch {
328
+ return 0;
329
+ }
330
+ }
331
+ /**
332
+ * Everything `memoryPolicy` needs, straight off this machine.
333
+ *
334
+ * `ownUsageBytes` is passed in by callers that are not the daemon — `doctor` and
335
+ * `install-service` run in the operator's own cgroup and cannot read the
336
+ * service's usage from `/proc/self`. Returns null when the service's usage is
337
+ * unknowable, because guessing 0 there is the one dangerous direction: it removes
338
+ * the floor that stops a live session from being killed on `daemon-reload`.
339
+ */
340
+ export function readMemoryFacts(ownUsageBytes = readOwnCgroupUsage()) {
341
+ const info = readMemInfo();
342
+ if (!info || ownUsageBytes === null)
343
+ return null;
344
+ return { ...info, ownUsageBytes, uptimeSec: readUptimeSec() };
345
+ }
139
346
  /**
140
347
  * `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
141
348
  * box unreachable, but never so little that ordinary work is throttled.
@@ -148,8 +355,16 @@ export function limitsOverridePath(home = systemdUserHome()) {
148
355
  export function cpuQuotaPercent(cpuCount = os.cpus().length) {
149
356
  return cpuCount >= 4 ? (cpuCount - 1) * 100 : null;
150
357
  }
151
- export function buildLimitsOverride(cpuCount = os.cpus().length) {
358
+ export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMemoryFacts()) {
152
359
  const quota = cpuQuotaPercent(cpuCount);
360
+ // No `minCeilingBytes` parameter on purpose. It existed here for one revision
361
+ // and passed an explicit `0`, which silently DEFEATED the default in
362
+ // `memoryPolicy` — the writer produced a ceiling below live cgroup usage while
363
+ // the drift check and `doctor` both computed the floored one. Two consequences,
364
+ // both found in QA: `doctor --fix` could kill a session on a loaded machine,
365
+ // and the file was rewritten on every start because no fixed point existed.
366
+ // The floor belongs to the policy, so every caller gets it and none can opt out.
367
+ const memory = facts ? memoryPolicy(facts) : null;
153
368
  return ([
154
369
  `${LIMITS_MARKER} ${LIMITS_VERSION}`,
155
370
  '# Managed by devbridge-runner. Put your own overrides in a file that sorts',
@@ -163,14 +378,34 @@ export function buildLimitsOverride(cpuCount = os.cpus().length) {
163
378
  'StartLimitBurst=0',
164
379
  '',
165
380
  '[Service]',
166
- // The whole point of the change: one child's OOM must not take the fleet.
381
+ // The whole point of the 0.21.0 change, and the reason the ceiling below is
382
+ // safe to have at all: one child's OOM must not take the fleet.
167
383
  'OOMPolicy=continue',
168
- // Clears `MemoryMax=2G` from units written before 0.21.0.
169
- 'MemoryMax=',
170
- // Soft pressure instead of a hard ceiling: the kernel reclaims and
171
- // throttles rather than killing, and the machine keeps a fifth of its
172
- // memory for everything that is not this service.
173
- 'MemoryHigh=80%',
384
+ // Both values also override the `MemoryMax=2G` baked into units written
385
+ // before 0.21.0.
386
+ //
387
+ // The unmeasurable branch used to write `MemoryMax=` + `MemoryHigh=80%` —
388
+ // literally the pair this release exists to remove. Because an empty
389
+ // assignment RESETS a directive and a drop-in sorts after the unit, that
390
+ // branch would have stripped the ceiling the API's fallback unit now carries
391
+ // and left the machine weaker than if we had written nothing at all. A
392
+ // drop-in must never disarm the unit it overrides, so it falls back to the
393
+ // same static pair used during boot.
394
+ ...(memory
395
+ ? [
396
+ `# ${memory.measured ? 'measured headroom' : 'still booting — conservative fraction of total'}`,
397
+ `MemoryMax=${asMiB(memory.maxBytes)}`,
398
+ // Soft pressure below the hard stop: the kernel reclaims and throttles
399
+ // here, and only kills at MemoryMax. This is the line that failed on
400
+ // vmi2502773 — as a percentage of TOTAL it sat above what the machine
401
+ // could spare, so it never engaged.
402
+ `MemoryHigh=${asMiB(memory.highBytes)}`,
403
+ ]
404
+ : [
405
+ '# machine not measurable — same conservative pair as the fallback unit',
406
+ 'MemoryMax=55%',
407
+ 'MemoryHigh=44%',
408
+ ]),
174
409
  // Either a computed quota, or an explicit reset — both of which clear the
175
410
  // `CPUQuota=80%` baked into units written before 0.21.0.
176
411
  quota === null ? 'CPUQuota=' : `CPUQuota=${quota}%`,
@@ -186,9 +421,12 @@ export function buildLimitsOverride(cpuCount = os.cpus().length) {
186
421
  *
187
422
  * Deliberately version-based rather than content-based: an operator may add
188
423
  * their own directives to our file, and re-writing on every start would fight
189
- * them. Only the version number decides.
424
+ * them. The version number decides — plus, since 0.39.0 and only when `facts` are
425
+ * supplied, whether the measured ceiling still fits the machine (see
426
+ * `memoryCeilingHasDrifted`). Callers that pass no facts get the old, purely
427
+ * version-based answer.
190
428
  */
191
- export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome()) {
429
+ export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome(), facts = null) {
192
430
  let contents;
193
431
  try {
194
432
  contents = readFile(limitsOverridePath(home));
@@ -200,15 +438,61 @@ export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'u
200
438
  if (!line)
201
439
  return true;
202
440
  const version = Number.parseInt(line.slice(LIMITS_MARKER.length).trim(), 10);
203
- return !Number.isFinite(version) || version < LIMITS_VERSION;
441
+ if (!Number.isFinite(version) || version < LIMITS_VERSION)
442
+ return true;
443
+ return facts ? memoryCeilingHasDrifted(contents, facts) : false;
204
444
  }
205
- /** Write the drop-in. Returns false when nothing needed doing. */
206
- export function writeLimitsOverride(force = false, home = systemdUserHome()) {
207
- if (!force && !limitsOverrideIsOutdated(undefined, home))
445
+ /**
446
+ * Since 0.39.0 the ceiling is a measurement, and a measurement goes stale.
447
+ *
448
+ * A machine that gained a database and a dozen containers after the drop-in was
449
+ * written would keep a ceiling sized for the machine it used to be — which is the
450
+ * original bug wearing a different hat. So the version marker is no longer the
451
+ * only thing that can make the file outdated.
452
+ *
453
+ * The 15 % band is what keeps this from becoming a rewrite on every start:
454
+ * `MemAvailable` moves by hundreds of MB just from page cache, and re-writing our
455
+ * file that often would fight an operator who added their own directives to it
456
+ * (the reason the check was version-only to begin with).
457
+ */
458
+ const CEILING_DRIFT_TOLERANCE = 0.15;
459
+ function memoryCeilingHasDrifted(contents, facts) {
460
+ // Never re-measure a machine that is still booting: the static pair it is
461
+ // holding is deliberate, and «drift» against a lie is not drift.
462
+ if (!memoryPolicy(facts).measured)
463
+ return false;
464
+ // LAST match, not the first: systemd applies the last assignment, and the file
465
+ // explicitly invites the operator to add their own lines. Reading the first one
466
+ // would measure drift against a directive that is not in force.
467
+ const written = [...contents.matchAll(/^MemoryMax=(\d+)([KMG]?)$/gm)].at(-1);
468
+ // Anything we cannot read as a byte count is replaced, and that is deliberate:
469
+ // an EMPTY `MemoryMax=` is the 0.21.0 shape this release exists to remove, and
470
+ // `MemoryMax=55%` is the blind pair we write when the machine is unmeasurable —
471
+ // both must give way the moment a real measurement is available. The operator's
472
+ // own directives belong in a file that sorts after this one, which the header of
473
+ // every generated file says; a value they appended HERE is still honoured,
474
+ // because a plain `4G` parses above and is compared like any other.
475
+ if (!written?.[1])
476
+ return true;
477
+ const scale = { '': 1, K: 1024, M: MIB, G: 1024 * MIB }[written[2] ?? ''] ?? MIB;
478
+ const writtenBytes = Number(written[1]) * scale;
479
+ const wanted = memoryPolicy(facts).maxBytes;
480
+ return Math.abs(writtenBytes - wanted) / wanted > CEILING_DRIFT_TOLERANCE;
481
+ }
482
+ /**
483
+ * Write the drop-in. Returns false when nothing needed doing.
484
+ *
485
+ * The «never below what we already hold» guard lives in `memoryPolicy` itself
486
+ * (`CEILING_HEADROOM_OVER_CURRENT`), so writing, reading and the drift check all
487
+ * arrive at the same number — otherwise a write whose floor was binding would be
488
+ * seen as drifted on the very next call and rewritten forever.
489
+ */
490
+ export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts()) {
491
+ if (!force && !limitsOverrideIsOutdated(undefined, home, facts))
208
492
  return false;
209
493
  const target = limitsOverridePath(home);
210
494
  fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
211
- fs.writeFileSync(target, buildLimitsOverride(), { mode: 0o644 });
495
+ fs.writeFileSync(target, buildLimitsOverride(undefined, facts), { mode: 0o644 });
212
496
  return true;
213
497
  }
214
498
  /**
package/dist/version.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const RUNNER_VERSION = "0.38.0";
1
+ export declare const RUNNER_VERSION = "0.40.0";
2
2
  //# sourceMappingURL=version.d.ts.map
package/dist/version.js CHANGED
@@ -1,3 +1,3 @@
1
1
  // Kept in sync with package.json by the release script (manual for now).
2
- export const RUNNER_VERSION = '0.38.0';
2
+ export const RUNNER_VERSION = '0.40.0';
3
3
  //# sourceMappingURL=version.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bridge4dev/runner",
3
- "version": "0.38.0",
3
+ "version": "0.40.0",
4
4
  "description": "DevBridge dev runner — connects a dev server to DevBridge and runs agent sessions (Claude Code / Codex)",
5
5
  "homepage": "https://bridge4.dev",
6
6
  "license": "MIT",