@bridge4dev/runner 0.56.0 → 0.58.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,13 +1,12 @@
1
- import { execFile } from 'node:child_process';
2
1
  import crypto from 'node:crypto';
3
2
  import fs from 'node:fs';
4
3
  import os from 'node:os';
5
4
  import path from 'node:path';
6
- import { promisify } from 'node:util';
5
+ import { cageAuthority, mayActOnSessionCages, mayStartScope, runSystemctl, runSystemdRun, } from './cage-authority.js';
7
6
  import { runnerIdentity, systemdUserEnv } from './environment.js';
8
7
  import { log } from './log.js';
9
- import { DEVBRIDGE_SLICE, memoryPolicy, readMemoryFacts, readSelfCgroup, readSwapTotalBytes, SESSION_CPU_WEIGHT, SESSIONS_SLICE, SESSIONS_SWAP_SHARE, sliceCgroupPath, } from './service-unit.js';
10
- const execFileAsync = promisify(execFile);
8
+ import { stateDir } from './paths.js';
9
+ import { DEVBRIDGE_SLICE, memoryPolicy, readCgroupMemory, readMemoryFacts, readSelfCgroup, readSwapTotalBytes, SESSION_CPU_WEIGHT, SESSIONS_SLICE, SESSIONS_SWAP_SHARE, sliceCgroupPath, } from './service-unit.js';
11
10
  /**
12
11
  * A cgroup of its own for every agent session — the memory ceiling that
13
12
  * `process-priority.ts` cannot give.
@@ -144,6 +143,40 @@ export const SESSION_SCOPE_PREFIX = 'devbridge-session-';
144
143
  * (QA-2026-09-07 MINOR-6, where the earlier wording promised the opposite).
145
144
  */
146
145
  export const SESSION_MEMORY_HIGH_MIN_BYTES = 2 * GIB;
146
+ /**
147
+ * The gap between the brake and the wall, and never less than this (#398 S2).
148
+ *
149
+ * The brake is where the kernel starts reclaiming; the wall is where it kills.
150
+ * With the two at the same number there is no brake at all — that is the 0.54.0
151
+ * kill line, and #387 is the whole story of what it costs. So every path that
152
+ * computes the pair keeps at least a band between them, and the band is also
153
+ * the step by which the stall detector may raise a brake.
154
+ *
155
+ * 256 MiB is sized against what one `tsc` of this monorepo holds (~400 MB
156
+ * measured) divided by the overshoot a build makes in one 5 s window: enough
157
+ * room for the allocation that was refused, not enough to hide a runaway.
158
+ *
159
+ * {@link sessionMemoryBandBytes} widens it on the machines where reaching the
160
+ * wall costs the whole session rather than one process — see there.
161
+ */
162
+ export const SESSION_MEMORY_BAND_BYTES = 256 * MIB;
163
+ /**
164
+ * The band on THIS machine.
165
+ *
166
+ * Where the probe found no `OOMPolicy=continue` (systemd 249 and friends), a
167
+ * kill inside the scope stops the whole scope: the session dies, not one
168
+ * process. The band there is not a nicety but the only warning the session
169
+ * gets, so it is a quarter of the brake and never under 512 MiB.
170
+ *
171
+ * Two of the fleet's fifteen caged machines answer `sessionOomContinue: false`
172
+ * (`search aigain`, `trizailab`, measured 10.09.2026), so this is not a
173
+ * hypothetical branch.
174
+ */
175
+ export function sessionMemoryBandBytes(brakeBytes, oomContinue) {
176
+ if (oomContinue)
177
+ return SESSION_MEMORY_BAND_BYTES;
178
+ return Math.max(512 * MIB, Math.floor(brakeBytes * 0.25));
179
+ }
147
180
  /**
148
181
  * The wall, when nothing on the machine can say where it should be.
149
182
  *
@@ -421,7 +454,7 @@ function userManagerCgroupControllers() {
421
454
  async function runLiveProbe(options) {
422
455
  const unit = `devbridge-cage-probe-${process.pid}`;
423
456
  try {
424
- const { stdout } = await execFileAsync('systemd-run', [
457
+ const { stdout } = await runSystemdRun([
425
458
  '--user',
426
459
  '--scope',
427
460
  '--quiet',
@@ -431,6 +464,11 @@ async function runLiveProbe(options) {
431
464
  // Exactly the switches `cageSpawn` will use, this one included: a probe
432
465
  // that tests a different command line proves nothing about the real one.
433
466
  ...(options.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
467
+ // The guarantee (#398 S3). In the probe for the same reason `OOMPolicy`
468
+ // is: a `-p` an older systemd refuses makes `systemd-run` exit 1 and
469
+ // spawn NOTHING, so a property that has never been tried here would
470
+ // cost that machine every session it tries to start.
471
+ ...(options.memoryLowFlag ? ['-p', `MemoryLow=${PROBE_MEMORY_MAX / 4}`] : []),
434
472
  '-p',
435
473
  `MemoryHigh=${PROBE_MEMORY_MAX / 2}`,
436
474
  '-p',
@@ -485,7 +523,7 @@ function execErrorText(error) {
485
523
  */
486
524
  async function readSystemdRunVersion() {
487
525
  try {
488
- const { stdout } = await execFileAsync('systemd-run', ['--version'], { timeout: 10_000 });
526
+ const { stdout } = await runSystemdRun(['--version'], { timeout: 10_000 });
489
527
  const match = /^systemd\s+(\d+)/m.exec(stdout);
490
528
  const major = match?.[1] ? Number(match[1]) : Number.NaN;
491
529
  return Number.isFinite(major) ? major : null;
@@ -504,7 +542,9 @@ async function showMemoryMax(unit) {
504
542
  */
505
543
  async function showByteCount(unit, property) {
506
544
  try {
507
- const { stdout } = await execFileAsync('systemctl', ['--user', 'show', unit, '-p', property, '--value'], { timeout: 10_000, env: systemdUserEnv() });
545
+ const { stdout } = await runSystemctl(['show', unit, '-p', property, '--value'], {
546
+ timeout: 10_000,
547
+ });
508
548
  const raw = stdout.trim();
509
549
  if (!/^\d+$/.test(raw))
510
550
  return null;
@@ -562,6 +602,7 @@ export async function detectSessionCage(probe = defaultCageProbe) {
562
602
  serviceMemoryMaxBytes: null,
563
603
  sessionsSliceMemoryMaxBytes: null,
564
604
  expandEnvironmentFlag: false,
605
+ memoryLowFlag: false,
565
606
  });
566
607
  const fsType = probe.cgroupFsType();
567
608
  // Not «contains cgroup2» — exactly cgroup2fs. `tmpfs` at this path is cgroup
@@ -595,10 +636,30 @@ export async function detectSessionCage(probe = defaultCageProbe) {
595
636
  // refusal that names the property try once more without — that machine keeps
596
637
  // its cage and loses only the «one process, not the session» half (#387).
597
638
  let oomPolicyFlag = true;
598
- let probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
639
+ /**
640
+ * `MemoryLow` on a SCOPE, tried the same way and for the same reason (#398 S3).
641
+ *
642
+ * The guarantee is a new `-p` on a command line that has been stable for two
643
+ * releases, and a `-p` an older systemd does not take makes `systemd-run` exit
644
+ * 1 and spawn NOTHING — which on that machine is not a missing feature, it is
645
+ * every session failing to start. The fleet runs systemd 249, 252 and 255, and
646
+ * only 255 could be tried here. So it is probed, and a refusal costs the
647
+ * machine its guarantees and nothing else.
648
+ */
649
+ let memoryLowFlag = true;
650
+ let probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag, memoryLowFlag });
599
651
  if (probed.error !== null && refusesOomPolicy(probed.error)) {
600
652
  oomPolicyFlag = false;
601
- probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
653
+ probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag, memoryLowFlag });
654
+ }
655
+ if (probed.error !== null && refusesMemoryLow(probed.error)) {
656
+ memoryLowFlag = false;
657
+ probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag, memoryLowFlag });
658
+ if (probed.error === null) {
659
+ log.warn('session cage: this systemd will not take MemoryLow on a scope — no guarantees here', {
660
+ why: 'the brake and the wall are unaffected; only MemoryLow is dropped',
661
+ });
662
+ }
602
663
  }
603
664
  if (probed.error !== null) {
604
665
  // Not «the cage did not hold» — nothing was ever caged. Blaming `memory.max`
@@ -625,6 +686,7 @@ export async function detectSessionCage(probe = defaultCageProbe) {
625
686
  serviceMemoryMaxBytes,
626
687
  sessionsSliceMemoryMaxBytes,
627
688
  expandEnvironmentFlag,
689
+ memoryLowFlag,
628
690
  };
629
691
  }
630
692
  /** The `-p` assignment itself, in one place: the probe and the spawn must agree. */
@@ -640,6 +702,10 @@ const OOM_POLICY_CONTINUE = 'OOMPolicy=continue';
640
702
  export function refusesOomPolicy(error) {
641
703
  return /OOMPolicy/i.test(error);
642
704
  }
705
+ /** The same question for `MemoryLow=`, and the same wording from systemd. */
706
+ export function refusesMemoryLow(error) {
707
+ return /MemoryLow/i.test(error);
708
+ }
643
709
  /**
644
710
  * Before the detector has run, nothing is wrapped.
645
711
  *
@@ -658,6 +724,7 @@ const UNPROBED = {
658
724
  serviceMemoryMaxBytes: null,
659
725
  sessionsSliceMemoryMaxBytes: null,
660
726
  expandEnvironmentFlag: false,
727
+ memoryLowFlag: false,
661
728
  };
662
729
  let detected = null;
663
730
  /**
@@ -667,8 +734,31 @@ let detected = null;
667
734
  * three sessions starting at once would mean three throwaway scopes before the
668
735
  * first agent got a word out.
669
736
  */
670
- export async function initSessionCage(probe = defaultCageProbe) {
671
- detected = await detectSessionCage(probe);
737
+ export async function initSessionCage(probe = defaultCageProbe, options = {}) {
738
+ const found = await detectSessionCage(probe);
739
+ /**
740
+ * A re-probe that comes back WORSE does not take the cage away (#398).
741
+ *
742
+ * The live probe is a `systemd-run`, and one transient failure — a busy bus,
743
+ * a momentary «Failed to connect» — answers `nice-only`. At daemon start that
744
+ * is the honest answer and the machine simply runs uncaged. On the HOURLY
745
+ * re-measure it is something else entirely: from that moment every new
746
+ * session on a perfectly good machine would start with no cage, until
747
+ * somebody restarted the runner. An hourly job must not be able to do that
748
+ * (found by the independent review of 10.09.2026).
749
+ *
750
+ * Losing the cage for real is possible — cgroups remounted, the user bus gone
751
+ * — and it is not silent: the line below says so, loudly, every hour, and the
752
+ * next daemon start settles it either way.
753
+ */
754
+ if (options.keepCageIfWorse && detected?.mode === 'scope' && found.mode !== 'scope') {
755
+ log.error('session cage: the re-measure lost the cage — keeping what the last probe proved', {
756
+ reason: found.reason,
757
+ hint: 'restart the runner if this machine really has no cgroups any more',
758
+ });
759
+ return detected;
760
+ }
761
+ detected = found;
672
762
  if (detected.mode === 'scope') {
673
763
  log.info('session cage: each session gets its own cgroup', {
674
764
  slice: SESSIONS_SLICE,
@@ -725,6 +815,30 @@ const liveUnits = new Map();
725
815
  export function sessionScopeUnitOf(id) {
726
816
  return liveUnits.get(id) ?? null;
727
817
  }
818
+ /**
819
+ * The pid of the agent process of each live session (#403).
820
+ *
821
+ * Written by the adapters right after the spawn — `systemd-run --scope` execs
822
+ * into the same pid, so this IS the process inside the cage — and read by the
823
+ * stall mechanism, which must never choose the agent as the command to stop.
824
+ *
825
+ * The structural rule it replaces («the agent is the one process whose parent
826
+ * is outside the scope») is true of the agent and true of one other thing: an
827
+ * MCP server whose launcher exited. Under the old rule such an orphan became a
828
+ * root, was protected on that account, and its own children — a browser, a
829
+ * language server — lost the protection they had. Knowing the pid tells the two
830
+ * apart, and it costs one map.
831
+ */
832
+ const agentPids = new Map();
833
+ /** Remember which process is the agent of this session. */
834
+ export function noteSessionAgentPid(id, pid) {
835
+ if (typeof pid === 'number' && pid > 0)
836
+ agentPids.set(id, pid);
837
+ }
838
+ /** The agent's pid for a live session, or null when nothing has said. */
839
+ export function sessionAgentPid(id) {
840
+ return agentPids.get(id) ?? null;
841
+ }
728
842
  /**
729
843
  * Wrap a command in its session's cage, or hand it back untouched.
730
844
  *
@@ -737,8 +851,30 @@ export function sessionScopeUnitOf(id) {
737
851
  * spawn, because `systemd-run --scope` execs into the SAME pid — verified in the
738
852
  * spike, including `detached: true` + `process.kill(-pid)` in `verify.ts`
739
853
  * (`pgid === child.pid` still holds).
854
+ *
855
+ * Untouched is also the answer for a process that has no right to put units on
856
+ * this machine (#403). It is the mildest of the doors in `cage-authority.ts`:
857
+ * the command still runs, it simply runs uncaged, which is exactly what happens
858
+ * on every machine without cgroups.
740
859
  */
741
860
  export function cageSpawn(input) {
861
+ if (!mayStartScope(SESSION_SCOPE_PREFIX)) {
862
+ log.warn('session cage: this process may not create cages, the command runs uncaged', {
863
+ id: input.id,
864
+ });
865
+ return { command: input.command, args: input.args, env: {}, unit: null };
866
+ }
867
+ return buildCagedSpawn(input);
868
+ }
869
+ /**
870
+ * The command line itself, with no question of who is allowed to run it.
871
+ *
872
+ * Split out from {@link cageSpawn} so the shape of the line stays testable: the
873
+ * suite proves what `systemd-run` would be handed, and the door above proves
874
+ * who may hand it. One function could not do both — a test process legitimately
875
+ * needs the first answer and must never get the second.
876
+ */
877
+ export function buildCagedSpawn(input) {
742
878
  const facts = sessionCage();
743
879
  if (facts.mode !== 'scope' ||
744
880
  facts.memoryMaxBytes === null ||
@@ -746,6 +882,20 @@ export function cageSpawn(input) {
746
882
  facts.swapMaxBytes === null) {
747
883
  return { command: input.command, args: input.args, env: {}, unit: null };
748
884
  }
885
+ /**
886
+ * The live numbers if anything can give them, the daemon-start snapshot
887
+ * otherwise (#398 S3, work 9).
888
+ *
889
+ * Without this a session started at 03:00 would run for its first half-minute
890
+ * on the formula of whenever the daemon last started, and then jump. The
891
+ * allocator is asked to compute as though this session already existed, so
892
+ * the number it is born with is the number its first tick will confirm.
893
+ */
894
+ const live = liveLadderSource?.(input.id) ?? null;
895
+ const highBytes = live?.highBytes ?? facts.memoryHighBytes;
896
+ const maxBytes = live?.maxBytes ?? facts.memoryMaxBytes;
897
+ const swapMaxBytes = live?.swapBytes ?? facts.swapMaxBytes;
898
+ const guaranteedBytes = live?.guaranteedBytes ?? null;
749
899
  const attempt = (attempts.get(input.id) ?? 0) + 1;
750
900
  attempts.set(input.id, attempt);
751
901
  const unit = sessionScopeUnit(input.id, attempt);
@@ -767,21 +917,28 @@ export function cageSpawn(input) {
767
917
  // Only where systemd knows the switch — below 254 it is a hard error and
768
918
  // `--scope` does no expansion anyway (module header, MAJOR-2).
769
919
  ...(facts.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
920
+ // The guarantee: what will not be reclaimed from this session when the
921
+ // slice as a whole comes under pressure (#398 S3). Only where the
922
+ // allocator could compute one — a number nobody stands behind is worse
923
+ // than none.
924
+ ...(guaranteedBytes === null || !facts.memoryLowFlag
925
+ ? []
926
+ : ['-p', `MemoryLow=${guaranteedBytes}`]),
770
927
  // The ladder of #387, bottom to top. The brake: past the honest share the
771
928
  // kernel reclaims and slows this session down, and nothing dies.
772
929
  '-p',
773
- `MemoryHigh=${facts.memoryHighBytes}`,
930
+ `MemoryHigh=${highBytes}`,
774
931
  // The wall: the machine's measured headroom, far above the brake. Only a
775
932
  // runaway gets here, and with `OOMPolicy=continue` it costs one process.
776
933
  '-p',
777
- `MemoryMax=${facts.memoryMaxBytes}`,
934
+ `MemoryMax=${maxBytes}`,
778
935
  // THE line. Without a bound there is no cage at all: `MemoryMax` bounds
779
936
  // resident memory and lets the rest fall into the host's swap, which is
780
937
  // how a 200 MB cage allocated 2 GB and took the machine's swap with it.
781
938
  // Bounded, never unset — and since #387 a share rather than 0, because
782
939
  // the brake above has to have somewhere to push pages (module header).
783
940
  '-p',
784
- `MemorySwapMax=${facts.swapMaxBytes}`,
941
+ `MemorySwapMax=${swapMaxBytes}`,
785
942
  // Only where the probe proved this systemd takes it on a scope. Without
786
943
  // it the kernel killing ONE process stops the WHOLE scope (`OOMPolicy`
787
944
  // defaults to `stop`) — the exact mechanism that killed honest sessions
@@ -843,8 +1000,14 @@ export function parseScopeMemoryStatus(files) {
843
1000
  ownLimitOom: event('oom'),
844
1001
  };
845
1002
  }
846
- /** Where a session scope's cgroup lives, or null off a user manager. */
847
- function scopeCgroupDir(unit) {
1003
+ /**
1004
+ * Where a session scope's cgroup lives, or null off a user manager.
1005
+ *
1006
+ * Exported for `session-stall.ts`, which reads three more files out of the same
1007
+ * directory (`memory.pressure`, `memory.stat`, `cgroup.procs`). One resolver,
1008
+ * so the two modules can never disagree about which cgroup a session is in.
1009
+ */
1010
+ export function scopeCgroupDir(unit) {
848
1011
  const self = readSelfCgroup();
849
1012
  if (self === null)
850
1013
  return null;
@@ -882,6 +1045,223 @@ export function readScopeMemoryStatus(unit, readFile = (p) => fs.readFileSync(p,
882
1045
  return null;
883
1046
  }
884
1047
  }
1048
+ /**
1049
+ * What a live session holds, for the allocator (#398 S3).
1050
+ *
1051
+ * `readCgroupMemory` rather than a second parser: it already takes page cache
1052
+ * out and puts `shmem` back, which is the difference between «cache we can give
1053
+ * back» and «memory something would have to be killed for», and getting that
1054
+ * wrong inflates every number by whatever the machine happened to have cached
1055
+ * (measured at +19 % on this host).
1056
+ */
1057
+ export function readScopeHold(unit) {
1058
+ const dir = scopeCgroupDir(unit);
1059
+ if (dir === null)
1060
+ return null;
1061
+ const reading = readCgroupMemory(dir);
1062
+ if (reading === null)
1063
+ return null;
1064
+ return {
1065
+ currentBytes: reading.currentBytes,
1066
+ holdBytes: reading.unreclaimableBytes ?? reading.currentBytes,
1067
+ };
1068
+ }
1069
+ /**
1070
+ * The slice's own two limits, from the filesystem rather than the bus.
1071
+ *
1072
+ * Live on purpose. What the probe found at daemon start is a snapshot, and the
1073
+ * pot moves: the hourly `repairResourceLimits` rewrites it from a fresh
1074
+ * measurement, and a machine whose neighbours grew or shrank gets a different
1075
+ * number. Reading two small files on the tick costs nothing and cannot be stale;
1076
+ * a `systemctl show` would cost a bus call measured at 2.7 s under load.
1077
+ */
1078
+ export function readSliceLimits(readFile = (p) => fs.readFileSync(p, 'utf8')) {
1079
+ /**
1080
+ * The slice's path, from the READER's own cgroup where that works — and from
1081
+ * the user id where it does not.
1082
+ *
1083
+ * `sliceCgroupPath(readSelfCgroup(), …)` answers null for anything that is
1084
+ * not itself under the user manager: a login-session scope, a `systemd-run`
1085
+ * scope, a bare shell. That is fine for the DAEMON, which always is, and
1086
+ * quite wrong for `doctor` — which the installer runs through `setpriv`,
1087
+ * keeping the installer's own cgroup. The result was that a freshly and
1088
+ * correctly installed machine reported «collective brake MISSING» on its very
1089
+ * first acceptance run. Found by the independent review of 10.09.2026.
1090
+ */
1091
+ const dir = sessionsSliceCgroupDir();
1092
+ if (dir === null)
1093
+ return null;
1094
+ const limit = (name) => {
1095
+ try {
1096
+ const raw = readFile(path.join(dir, name)).trim();
1097
+ if (raw === 'max' || !/^\d+$/.test(raw))
1098
+ return null;
1099
+ const value = Number(raw);
1100
+ return Number.isSafeInteger(value) ? value : null;
1101
+ }
1102
+ catch {
1103
+ return null;
1104
+ }
1105
+ };
1106
+ const potBytes = limit('memory.max');
1107
+ const collectiveBrakeBytes = limit('memory.high');
1108
+ // `memory.low` reads `0` when nothing protects the slice, and `limit()` turns
1109
+ // only `max` into null — so zero survives, which is the whole point here.
1110
+ const guaranteeBytes = limit('memory.low');
1111
+ if (potBytes === null && collectiveBrakeBytes === null)
1112
+ return null;
1113
+ return { potBytes, collectiveBrakeBytes, guaranteeBytes };
1114
+ }
1115
+ /**
1116
+ * Where `devbridge-sessions.slice` lives on this machine.
1117
+ *
1118
+ * Two ways of answering, and the second is what makes this usable from a
1119
+ * command the operator typed. The first is the reader's own cgroup, which is
1120
+ * exact whenever the reader is under the user manager. The second builds the
1121
+ * path from the user id, exactly as `userManagerCgroupControllers()` already
1122
+ * does — a `systemd-run` scope, a login session and a bare shell all get the
1123
+ * right answer that way, and the daemon gets the same one either way.
1124
+ */
1125
+ export function sessionsSliceCgroupDir(io = {}) {
1126
+ const selfCgroup = io.selfCgroup ?? readSelfCgroup;
1127
+ const exists = io.exists ?? ((p) => fs.existsSync(p));
1128
+ const self = selfCgroup();
1129
+ const fromSelf = self === null ? null : sliceCgroupPath(self, SESSIONS_SLICE);
1130
+ if (fromSelf !== null)
1131
+ return fromSelf;
1132
+ /**
1133
+ * Whose user manager, when the reader's own cgroup cannot say.
1134
+ *
1135
+ * Two candidates and the order matters (#398 S7, B6). The runner's OWN uid
1136
+ * comes first — that is the daemon and every ordinary CLI. But `doctor` is
1137
+ * routinely run as `sudo devbridge-runner doctor` on a machine installed with
1138
+ * `--user devbridge`, and then the process's uid is root's while the slice
1139
+ * lives under the service user's manager: a correctly installed machine
1140
+ * reported «collective brake MISSING» on its first acceptance run.
1141
+ *
1142
+ * The owner of the runner's state directory IS the service user, by
1143
+ * construction — the installer creates it as them — so it is the honest
1144
+ * second guess, and it costs one `stat`.
1145
+ */
1146
+ const uids = [io.selfUid?.() ?? runnerIdentity().uid, io.ownerUid?.() ?? stateDirOwnerUid()];
1147
+ for (const uid of uids) {
1148
+ if (uid < 0)
1149
+ continue;
1150
+ const guess = path.join('/sys/fs/cgroup/user.slice', `user-${uid}.slice`, `user@${uid}.service`, DEVBRIDGE_SLICE, SESSIONS_SLICE);
1151
+ try {
1152
+ if (exists(guess))
1153
+ return guess;
1154
+ }
1155
+ catch {
1156
+ // Unreadable is «not here», and the next candidate may still answer.
1157
+ }
1158
+ }
1159
+ return null;
1160
+ }
1161
+ /** The uid that owns the runner's state directory — the service user. */
1162
+ function stateDirOwnerUid() {
1163
+ try {
1164
+ return fs.statSync(stateDir()).uid;
1165
+ }
1166
+ catch {
1167
+ return -1;
1168
+ }
1169
+ }
1170
+ let liveLadderSource = null;
1171
+ export function setLiveLadderSource(source) {
1172
+ liveLadderSource = source;
1173
+ }
1174
+ /**
1175
+ * The two numbers this session may be TOLD about (#398 S5).
1176
+ *
1177
+ * The guarantee and the ceiling, and deliberately not the live share. The share
1178
+ * is recomputed every 30 s, so any value of it put into an environment variable
1179
+ * or into a system prompt is stale within half a minute — and a number the agent
1180
+ * believes and acts on after it has stopped being true is worse than no number.
1181
+ * The guarantee is a constant of the formula and the ceiling barely moves.
1182
+ *
1183
+ * Null on a machine with no cage: there is nothing to promise there, and «нет
1184
+ * cgroup — держим ноль» (#257) means we say nothing rather than invent a limit.
1185
+ */
1186
+ export function sessionMemoryFor(id) {
1187
+ const facts = sessionCage();
1188
+ if (facts.mode !== 'scope')
1189
+ return null;
1190
+ const live = liveLadderSource?.(id) ?? null;
1191
+ if (live) {
1192
+ return {
1193
+ guaranteedBytes: live.guaranteedBytes,
1194
+ maxBytes: live.maxBytes,
1195
+ swapBytes: live.swapBytes,
1196
+ };
1197
+ }
1198
+ if (facts.memoryMaxBytes === null)
1199
+ return null;
1200
+ return {
1201
+ // Before the allocator has anything to say, the honest floor is the brake
1202
+ // the daemon measured: it is what this session would be held to.
1203
+ guaranteedBytes: facts.memoryHighBytes ?? 0,
1204
+ maxBytes: facts.memoryMaxBytes,
1205
+ swapBytes: facts.swapMaxBytes ?? 0,
1206
+ };
1207
+ }
1208
+ /**
1209
+ * What the agent is told, and how — one text for both CLIs (#398 S5).
1210
+ *
1211
+ * A shared builder rather than two literals for the reason `DIRECT_BRANCH_RULE`
1212
+ * in `adapters/types.ts` is shared: the two system-prompt appends are hand-kept
1213
+ * copies of one another, and a third place for them to diverge was not worth
1214
+ * having.
1215
+ *
1216
+ * The live share is named as something the agent CANNOT know, on purpose. Half
1217
+ * of the incident this whole plan comes from was an agent raising its own heap
1218
+ * twice — to 4 GB, then to 6 GB, against a wall of 4296 MB — because nothing
1219
+ * had ever told it there was a wall.
1220
+ */
1221
+ export function sessionMemoryEnv(id) {
1222
+ const memory = sessionMemoryFor(id);
1223
+ if (memory === null)
1224
+ return {};
1225
+ const mb = (bytes) => String(Math.max(0, Math.floor(bytes / MIB)));
1226
+ return {
1227
+ DEVBRIDGE_SESSION_MEMORY_GUARANTEED_MB: mb(memory.guaranteedBytes),
1228
+ DEVBRIDGE_SESSION_MEMORY_MAX_MB: mb(memory.maxBytes),
1229
+ };
1230
+ }
1231
+ /** The sentence for the system prompt, or null on a machine with no cage. */
1232
+ export function sessionMemoryPromptLine(id) {
1233
+ const memory = sessionMemoryFor(id);
1234
+ if (memory === null)
1235
+ return null;
1236
+ const mb = (bytes) => Math.max(0, Math.round(bytes / MIB));
1237
+ const swap = memory.swapBytes > 0
1238
+ ? `This machine has swap, so going over your share slows you down.`
1239
+ : `This machine has NO swap, so going over your share does not slow you down — it stops you.`;
1240
+ /**
1241
+ * A guarantee is named only when one is actually in force (#398 S7, B3).
1242
+ *
1243
+ * `guaranteedBytes` is zero when nothing wrote `MemoryLow` — the systemd here
1244
+ * refuses the property, or the slice above has no protection of its own. The
1245
+ * agent plans around what it is told, so telling it about two gigabytes that
1246
+ * nobody reserved is worse than telling it nothing.
1247
+ */
1248
+ if (memory.guaranteedBytes <= 0) {
1249
+ return (`- Memory: this session can never exceed ${mb(memory.maxBytes)} MB ` +
1250
+ `(DEVBRIDGE_SESSION_MEMORY_MAX_MB), and it has NO reserved share on this machine — under ` +
1251
+ `pressure the kernel may reclaim anything it holds. How much you get at any moment is decided ` +
1252
+ `by the machine and changes; you cannot read it, so do not plan around a number. ${swap} ` +
1253
+ `If a command stands still for three minutes for want of memory, DevBridge stops that command ` +
1254
+ `(not the session) and tells you. Do not raise --max-old-space-size above the ceiling: it is ` +
1255
+ `enforced outside your process, and asking for more only makes the command fail sooner.`);
1256
+ }
1257
+ return (`- Memory: this session is guaranteed ${mb(memory.guaranteedBytes)} MB and can never exceed ` +
1258
+ `${mb(memory.maxBytes)} MB (DEVBRIDGE_SESSION_MEMORY_GUARANTEED_MB and ` +
1259
+ `DEVBRIDGE_SESSION_MEMORY_MAX_MB). How much you get above the guarantee at any moment is decided ` +
1260
+ `by the machine and changes; you cannot read it, so do not plan around a number. ${swap} ` +
1261
+ `If a command stands still for three minutes for want of memory, DevBridge stops that command ` +
1262
+ `(not the session) and tells you. Do not raise --max-old-space-size above the ceiling: it is ` +
1263
+ `enforced outside your process, and asking for more only makes the command fail sooner.`);
1264
+ }
885
1265
  /**
886
1266
  * `oom_kill` as the watch last saw it, per id — so a kill that was already
887
1267
  * announced in the feed while the session lived is not blamed for a death that
@@ -973,13 +1353,7 @@ export function explainMemoryDeath(id) {
973
1353
  return (`There was not enough memory for this session and ${stopped} — ` +
974
1354
  'send a message to carry on; if it keeps happening, run fewer sessions at once or give the machine more memory.');
975
1355
  }
976
- const realSystemctl = async (args) => {
977
- const { stdout, stderr } = await execFileAsync('systemctl', ['--user', ...args], {
978
- timeout: 15_000,
979
- env: systemdUserEnv(),
980
- });
981
- return { stdout, stderr };
982
- };
1356
+ const realSystemctl = async (args) => await runSystemctl(args);
983
1357
  function showValue(stdout, property) {
984
1358
  const line = stdout
985
1359
  .split('\n')
@@ -1001,6 +1375,9 @@ function showValue(stdout, property) {
1001
1375
  export async function releaseSessionScope(unit, id, systemctl = realSystemctl) {
1002
1376
  const running = releaseSessionScopeInner(unit, id, systemctl);
1003
1377
  if (id !== undefined) {
1378
+ // The process this named is gone with the scope; a pid outliving it would be
1379
+ // a pid that means something else by the time anybody reads it.
1380
+ agentPids.delete(id);
1004
1381
  releasing.set(id, running);
1005
1382
  void running.finally(() => {
1006
1383
  if (releasing.get(id) === running)
@@ -1051,9 +1428,19 @@ async function releaseSessionScopeInner(unit, id, systemctl) {
1051
1428
  if (id !== undefined)
1052
1429
  liveUnits.delete(id);
1053
1430
  let result = null;
1431
+ let tasksLeft = null;
1054
1432
  try {
1055
- const { stdout } = await systemctl(['show', unit, '-p', 'Result']);
1433
+ // Both properties in ONE call: this runs on every process exit, and the bus
1434
+ // was measured at 2.7 s under load — two round trips where one will do is a
1435
+ // cost paid on the path that is walked most.
1436
+ const { stdout } = await systemctl(['show', unit, '-p', 'Result', '-p', 'TasksCurrent']);
1056
1437
  result = showValue(stdout, 'Result');
1438
+ // Explicitly null-checked, not coerced: `Number(null)` is 0, and reading a
1439
+ // property systemd never printed as «no tasks left» would silently turn the
1440
+ // stop below off. systemd's own «unknown» is `[not set]`, which is NaN.
1441
+ const raw = showValue(stdout, 'TasksCurrent');
1442
+ const tasks = raw === null ? Number.NaN : Number(raw);
1443
+ tasksLeft = Number.isFinite(tasks) ? tasks : null;
1057
1444
  }
1058
1445
  catch {
1059
1446
  // Already collected — a clean exit takes its scope with it.
@@ -1068,6 +1455,41 @@ async function releaseSessionScopeInner(unit, id, systemctl) {
1068
1455
  if (id !== undefined)
1069
1456
  rememberDeathFromResult(id);
1070
1457
  }
1458
+ /**
1459
+ * The agent's process is gone and the scope still holds tasks — put the tree
1460
+ * out, here and nowhere else.
1461
+ *
1462
+ * Until 0.58.0 this function only ever `show`ed and `reset-failed` the unit,
1463
+ * and nothing anywhere stopped it: `park()` kills ONE pid (the CLI), so a
1464
+ * `pnpm typecheck` or a `vite build` the agent had started went on eating
1465
+ * memory for a session that no longer existed, and the next start of the same
1466
+ * session met «Unit … was already loaded» and died as
1467
+ * «codex app-server exited before the session was ready (code 1)» (gotcha
1468
+ * §479, measured 09.09 on vmi3219930: 3.4 GB free instead of 8.5).
1469
+ *
1470
+ * Deliberately in the RELEASE path rather than as a second call from the
1471
+ * supervisor's park: all three call sites (`claude.ts`, `codex-protocol.ts`,
1472
+ * `verify.ts`) already come through here on process exit, so one place keeps
1473
+ * both agents and the verify runs identical and cannot race a second release
1474
+ * of the same unit.
1475
+ *
1476
+ * `TasksCurrent > 0` is the gate rather than an unconditional stop, because
1477
+ * the normal case is an empty scope systemd has already collected, and
1478
+ * `stop` on it would be a bus call per session exit for nothing.
1479
+ */
1480
+ if (tasksLeft !== null && tasksLeft > 0) {
1481
+ log.warn('session cage: the agent is gone but its commands are not — stopping the scope', {
1482
+ unit,
1483
+ tasksLeft,
1484
+ });
1485
+ try {
1486
+ await systemctl(['stop', unit]);
1487
+ }
1488
+ catch {
1489
+ // A scope that ended between the read and the stop answers «not loaded».
1490
+ // The sweep on the watch tick is the second line of defence.
1491
+ }
1492
+ }
1071
1493
  let forgotten = false;
1072
1494
  try {
1073
1495
  await systemctl(['reset-failed', unit]);
@@ -1146,32 +1568,160 @@ export async function listSessionScopes(systemctl = realSystemctl) {
1146
1568
  }
1147
1569
  return out;
1148
1570
  }
1571
+ export function scopeOwnerLiveness(unit, io = {}) {
1572
+ const readFile = io.readFile ?? ((p) => fs.readFileSync(p, 'utf8'));
1573
+ const cgroupDir = io.cgroupDir ?? scopeCgroupDir;
1574
+ const isAlive = io.isAlive ??
1575
+ ((pid) => {
1576
+ try {
1577
+ process.kill(pid, 0);
1578
+ return true;
1579
+ }
1580
+ catch (error) {
1581
+ // `EPERM` means it exists and belongs to somebody else — alive either way.
1582
+ return error.code === 'EPERM';
1583
+ }
1584
+ });
1585
+ const dir = cgroupDir(unit);
1586
+ if (dir === null)
1587
+ return 'unknown';
1588
+ let pids;
1589
+ try {
1590
+ pids = readFile(path.join(dir, 'cgroup.procs'))
1591
+ .split('\n')
1592
+ .map((line) => Number(line.trim()))
1593
+ .filter((pid) => Number.isSafeInteger(pid) && pid > 0);
1594
+ }
1595
+ catch {
1596
+ return 'unknown';
1597
+ }
1598
+ if (pids.length === 0)
1599
+ return 'empty';
1600
+ let sawUnknown = false;
1601
+ for (const pid of pids) {
1602
+ let ppid;
1603
+ try {
1604
+ const status = readFile(path.join('/proc', String(pid), 'status'));
1605
+ const line = status.split('\n').find((l) => l.startsWith('PPid:'));
1606
+ const value = line === undefined ? Number.NaN : Number(line.slice('PPid:'.length).trim());
1607
+ ppid = Number.isSafeInteger(value) ? value : null;
1608
+ }
1609
+ catch {
1610
+ // Ended between the listing and the read — says nothing either way.
1611
+ continue;
1612
+ }
1613
+ if (ppid === null) {
1614
+ sawUnknown = true;
1615
+ continue;
1616
+ }
1617
+ // Reparented to init: nobody is waiting on this process. `0` is the kernel's
1618
+ // answer for a process that is going away.
1619
+ if (ppid <= 1)
1620
+ continue;
1621
+ /**
1622
+ * …and reparented to the USER MANAGER counts the same (measured 10.09.2026).
1623
+ *
1624
+ * `systemd --user` sets itself as a subreaper, so an orphan inside a user
1625
+ * scope does not land on pid 1 at all — it lands on the manager, whose pid
1626
+ * is an ordinary live pid. Read literally, «the parent is alive» then made
1627
+ * every orphan look supervised, and the sweep would never remove anything
1628
+ * again: the opposite failure to the one this whole change is about, and
1629
+ * just as silent.
1630
+ */
1631
+ let parentComm;
1632
+ try {
1633
+ parentComm = readFile(path.join('/proc', String(ppid), 'comm')).trim();
1634
+ }
1635
+ catch {
1636
+ // The parent went between the two reads: it is not supervising anything.
1637
+ continue;
1638
+ }
1639
+ if (parentComm === 'systemd' || parentComm === 'init')
1640
+ continue;
1641
+ if (isAlive(ppid))
1642
+ return 'owned';
1643
+ }
1644
+ return sawUnknown ? 'unknown' : 'orphaned';
1645
+ }
1149
1646
  /**
1150
- * Stop and forget every session scope that has no session behind it.
1151
- *
1152
- * At daemon start that is all of them by definition, and it is the point: a
1153
- * scope outlives a killed daemon carrying the whole process tree with it, which
1154
- * is the shape of the 10 h 51 min `ugrep` of 16.08. Stopping the scope takes the
1155
- * tree, not just the process we happened to know about.
1647
+ * Is this unit, RIGHT NOW, a cage somebody is running in?
1156
1648
  *
1157
- * `liveIds` exists so the same sweep can run later without killing work in
1158
- * progress; only the CURRENT scope of a live session is spared, because an
1159
- * earlier attempt of the same session is exactly the leftover we are here for.
1649
+ * A scan rather than a snapshot on purpose — the freshness is the point — and a
1650
+ * scan rather than `new Set(liveUnits.values()).has(unit)`, which built a set
1651
+ * per unit inside the sweep's loop for nothing. The register holds one entry
1652
+ * per live session; on the biggest machine of the fleet that is six.
1160
1653
  */
1161
- export async function sweepOrphanSessionScopes(liveIds = [], systemctl = realSystemctl) {
1162
- const spared = new Set();
1654
+ function isLiveCageUnit(unit) {
1655
+ for (const live of liveUnits.values())
1656
+ if (live === unit)
1657
+ return true;
1658
+ return false;
1659
+ }
1660
+ export async function sweepOrphanSessionScopes(liveIds = [], systemctl = realSystemctl, liveness = scopeOwnerLiveness) {
1661
+ /**
1662
+ * Asked here, and not left to the door in `cage-authority.ts` (#403).
1663
+ *
1664
+ * The loop below catches every failed `systemctl` on purpose — a scope that
1665
+ * is already `failed` has nothing to stop — so a refusal down there would be
1666
+ * swallowed and this function would report units as removed that it never
1667
+ * touched. The register `liveUnits` is empty in any process but the daemon's,
1668
+ * which is precisely why this function is the dangerous one: to a foreign
1669
+ * process every real cage on the machine looks like litter.
1670
+ */
1671
+ if (systemctl === realSystemctl && !mayActOnSessionCages()) {
1672
+ log.warn('session cage: this process may not sweep cages, nothing was touched', {
1673
+ role: cageAuthority(),
1674
+ });
1675
+ return [];
1676
+ }
1677
+ const spared = new Set(liveUnits.values());
1163
1678
  for (const id of liveIds)
1164
1679
  spared.add(sessionScopeUnit(id, attempts.get(id) ?? 1));
1165
1680
  const removed = [];
1166
1681
  for (const unit of await listSessionScopeUnits(systemctl)) {
1167
1682
  if (spared.has(unit))
1168
1683
  continue;
1169
- try {
1170
- await systemctl(['stop', unit]);
1684
+ /**
1685
+ * …and asked AGAIN, immediately before the stop.
1686
+ *
1687
+ * `spared` is a snapshot, and between it and this line there are two
1688
+ * `systemctl` calls — the listing, and every stop before this one — each of
1689
+ * which the bus has been measured to hold for 2.7 s under load. A session
1690
+ * that started inside that window would be in `list-units` and not in the
1691
+ * snapshot, and this call would kill it. Found by the independent review of
1692
+ * 10.09.2026.
1693
+ *
1694
+ * `liveUnits` is filled by `cageSpawn` BEFORE `systemd-run` is spawned, so
1695
+ * a scope cannot exist without its name being in there first.
1696
+ */
1697
+ if (isLiveCageUnit(unit))
1698
+ continue;
1699
+ /**
1700
+ * …and the kernel is asked too, because the register above only knows what
1701
+ * THIS process started (#403).
1702
+ *
1703
+ * `owned` means a live parent outside the cage is supervising what is
1704
+ * inside it — another daemon's session, or this machine's own runner seen
1705
+ * from a second process. Whatever it is, it is not litter, and stopping it
1706
+ * is the incident of 10.09.2026. `unknown` gets the same answer for the
1707
+ * same reason: the sweep may only act on what it can prove.
1708
+ */
1709
+ const state = liveness(unit);
1710
+ if (state === 'owned' || state === 'unknown') {
1711
+ log.warn('session cage: left a scope alone — somebody is running in it', {
1712
+ unit,
1713
+ liveness: state,
1714
+ });
1715
+ continue;
1171
1716
  }
1172
- catch {
1173
- // A failed scope has nothing to stop; `reset-failed` below is the part
1174
- // that matters for it.
1717
+ if (state !== 'empty') {
1718
+ try {
1719
+ await systemctl(['stop', unit]);
1720
+ }
1721
+ catch {
1722
+ // A failed scope has nothing to stop; `reset-failed` below is the part
1723
+ // that matters for it.
1724
+ }
1175
1725
  }
1176
1726
  try {
1177
1727
  await systemctl(['reset-failed', unit]);