@bridge4dev/runner 0.57.0 → 0.58.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +23 -4
- package/dist/adapters/codex-protocol.js +6 -1
- package/dist/adapters/codex.js +11 -2
- package/dist/adapters/types.js +14 -0
- package/dist/cage-authority.d.ts +118 -0
- package/dist/cage-authority.js +241 -0
- package/dist/config.d.ts +83 -5
- package/dist/config.js +59 -1
- package/dist/daemon-lock.d.ts +43 -0
- package/dist/daemon-lock.js +107 -0
- package/dist/host-load.d.ts +9 -0
- package/dist/host-load.js +9 -0
- package/dist/index.js +222 -20
- package/dist/policy.d.ts +9 -0
- package/dist/policy.js +71 -1
- package/dist/protocol.d.ts +43 -27
- package/dist/recipe-schema.d.ts +12 -12
- package/dist/regex-guard.js +6 -6
- package/dist/self-update.js +22 -1
- package/dist/service-unit.d.ts +35 -3
- package/dist/service-unit.js +82 -5
- package/dist/session-allocator.d.ts +259 -0
- package/dist/session-allocator.js +492 -0
- package/dist/session-cage.d.ts +229 -2
- package/dist/session-cage.js +590 -40
- package/dist/session-limits.d.ts +71 -0
- package/dist/session-limits.js +93 -0
- package/dist/session-stall.d.ts +353 -0
- package/dist/session-stall.js +760 -0
- package/dist/supervisor.d.ts +223 -33
- package/dist/supervisor.js +845 -94
- package/dist/systemd-memory.js +2 -5
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/session-cage.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import { execFile } from 'node:child_process';
|
|
2
1
|
import crypto from 'node:crypto';
|
|
3
2
|
import fs from 'node:fs';
|
|
4
3
|
import os from 'node:os';
|
|
5
4
|
import path from 'node:path';
|
|
6
|
-
import {
|
|
5
|
+
import { cageAuthority, mayActOnSessionCages, mayStartScope, runSystemctl, runSystemdRun, } from './cage-authority.js';
|
|
7
6
|
import { runnerIdentity, systemdUserEnv } from './environment.js';
|
|
8
7
|
import { log } from './log.js';
|
|
9
|
-
import {
|
|
10
|
-
|
|
8
|
+
import { stateDir } from './paths.js';
|
|
9
|
+
import { DEVBRIDGE_SLICE, memoryPolicy, readCgroupMemory, readMemoryFacts, readSelfCgroup, readSwapTotalBytes, SESSION_CPU_WEIGHT, SESSIONS_SLICE, SESSIONS_SWAP_SHARE, sliceCgroupPath, } from './service-unit.js';
|
|
11
10
|
/**
|
|
12
11
|
* A cgroup of its own for every agent session — the memory ceiling that
|
|
13
12
|
* `process-priority.ts` cannot give.
|
|
@@ -144,6 +143,40 @@ export const SESSION_SCOPE_PREFIX = 'devbridge-session-';
|
|
|
144
143
|
* (QA-2026-09-07 MINOR-6, where the earlier wording promised the opposite).
|
|
145
144
|
*/
|
|
146
145
|
export const SESSION_MEMORY_HIGH_MIN_BYTES = 2 * GIB;
|
|
146
|
+
/**
|
|
147
|
+
* The gap between the brake and the wall, and never less than this (#398 S2).
|
|
148
|
+
*
|
|
149
|
+
* The brake is where the kernel starts reclaiming; the wall is where it kills.
|
|
150
|
+
* With the two at the same number there is no brake at all — that is the 0.54.0
|
|
151
|
+
* kill line, and #387 is the whole story of what it costs. So every path that
|
|
152
|
+
* computes the pair keeps at least a band between them, and the band is also
|
|
153
|
+
* the step by which the stall detector may raise a brake.
|
|
154
|
+
*
|
|
155
|
+
* 256 MiB is sized against what one `tsc` of this monorepo holds (~400 MB
|
|
156
|
+
* measured) divided by the overshoot a build makes in one 5 s window: enough
|
|
157
|
+
* room for the allocation that was refused, not enough to hide a runaway.
|
|
158
|
+
*
|
|
159
|
+
* {@link sessionMemoryBandBytes} widens it on the machines where reaching the
|
|
160
|
+
* wall costs the whole session rather than one process — see there.
|
|
161
|
+
*/
|
|
162
|
+
export const SESSION_MEMORY_BAND_BYTES = 256 * MIB;
|
|
163
|
+
/**
|
|
164
|
+
* The band on THIS machine.
|
|
165
|
+
*
|
|
166
|
+
* Where the probe found no `OOMPolicy=continue` (systemd 249 and friends), a
|
|
167
|
+
* kill inside the scope stops the whole scope: the session dies, not one
|
|
168
|
+
* process. The band there is not a nicety but the only warning the session
|
|
169
|
+
* gets, so it is a quarter of the brake and never under 512 MiB.
|
|
170
|
+
*
|
|
171
|
+
* Two of the fleet's fifteen caged machines answer `sessionOomContinue: false`
|
|
172
|
+
* (`search aigain`, `trizailab`, measured 10.09.2026), so this is not a
|
|
173
|
+
* hypothetical branch.
|
|
174
|
+
*/
|
|
175
|
+
export function sessionMemoryBandBytes(brakeBytes, oomContinue) {
|
|
176
|
+
if (oomContinue)
|
|
177
|
+
return SESSION_MEMORY_BAND_BYTES;
|
|
178
|
+
return Math.max(512 * MIB, Math.floor(brakeBytes * 0.25));
|
|
179
|
+
}
|
|
147
180
|
/**
|
|
148
181
|
* The wall, when nothing on the machine can say where it should be.
|
|
149
182
|
*
|
|
@@ -421,7 +454,7 @@ function userManagerCgroupControllers() {
|
|
|
421
454
|
async function runLiveProbe(options) {
|
|
422
455
|
const unit = `devbridge-cage-probe-${process.pid}`;
|
|
423
456
|
try {
|
|
424
|
-
const { stdout } = await
|
|
457
|
+
const { stdout } = await runSystemdRun([
|
|
425
458
|
'--user',
|
|
426
459
|
'--scope',
|
|
427
460
|
'--quiet',
|
|
@@ -431,6 +464,11 @@ async function runLiveProbe(options) {
|
|
|
431
464
|
// Exactly the switches `cageSpawn` will use, this one included: a probe
|
|
432
465
|
// that tests a different command line proves nothing about the real one.
|
|
433
466
|
...(options.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
|
|
467
|
+
// The guarantee (#398 S3). In the probe for the same reason `OOMPolicy`
|
|
468
|
+
// is: a `-p` an older systemd refuses makes `systemd-run` exit 1 and
|
|
469
|
+
// spawn NOTHING, so a property that has never been tried here would
|
|
470
|
+
// cost that machine every session it tries to start.
|
|
471
|
+
...(options.memoryLowFlag ? ['-p', `MemoryLow=${PROBE_MEMORY_MAX / 4}`] : []),
|
|
434
472
|
'-p',
|
|
435
473
|
`MemoryHigh=${PROBE_MEMORY_MAX / 2}`,
|
|
436
474
|
'-p',
|
|
@@ -485,7 +523,7 @@ function execErrorText(error) {
|
|
|
485
523
|
*/
|
|
486
524
|
async function readSystemdRunVersion() {
|
|
487
525
|
try {
|
|
488
|
-
const { stdout } = await
|
|
526
|
+
const { stdout } = await runSystemdRun(['--version'], { timeout: 10_000 });
|
|
489
527
|
const match = /^systemd\s+(\d+)/m.exec(stdout);
|
|
490
528
|
const major = match?.[1] ? Number(match[1]) : Number.NaN;
|
|
491
529
|
return Number.isFinite(major) ? major : null;
|
|
@@ -504,7 +542,9 @@ async function showMemoryMax(unit) {
|
|
|
504
542
|
*/
|
|
505
543
|
async function showByteCount(unit, property) {
|
|
506
544
|
try {
|
|
507
|
-
const { stdout } = await
|
|
545
|
+
const { stdout } = await runSystemctl(['show', unit, '-p', property, '--value'], {
|
|
546
|
+
timeout: 10_000,
|
|
547
|
+
});
|
|
508
548
|
const raw = stdout.trim();
|
|
509
549
|
if (!/^\d+$/.test(raw))
|
|
510
550
|
return null;
|
|
@@ -562,6 +602,7 @@ export async function detectSessionCage(probe = defaultCageProbe) {
|
|
|
562
602
|
serviceMemoryMaxBytes: null,
|
|
563
603
|
sessionsSliceMemoryMaxBytes: null,
|
|
564
604
|
expandEnvironmentFlag: false,
|
|
605
|
+
memoryLowFlag: false,
|
|
565
606
|
});
|
|
566
607
|
const fsType = probe.cgroupFsType();
|
|
567
608
|
// Not «contains cgroup2» — exactly cgroup2fs. `tmpfs` at this path is cgroup
|
|
@@ -595,10 +636,30 @@ export async function detectSessionCage(probe = defaultCageProbe) {
|
|
|
595
636
|
// refusal that names the property try once more without — that machine keeps
|
|
596
637
|
// its cage and loses only the «one process, not the session» half (#387).
|
|
597
638
|
let oomPolicyFlag = true;
|
|
598
|
-
|
|
639
|
+
/**
|
|
640
|
+
* `MemoryLow` on a SCOPE, tried the same way and for the same reason (#398 S3).
|
|
641
|
+
*
|
|
642
|
+
* The guarantee is a new `-p` on a command line that has been stable for two
|
|
643
|
+
* releases, and a `-p` an older systemd does not take makes `systemd-run` exit
|
|
644
|
+
* 1 and spawn NOTHING — which on that machine is not a missing feature, it is
|
|
645
|
+
* every session failing to start. The fleet runs systemd 249, 252 and 255, and
|
|
646
|
+
* only 255 could be tried here. So it is probed, and a refusal costs the
|
|
647
|
+
* machine its guarantees and nothing else.
|
|
648
|
+
*/
|
|
649
|
+
let memoryLowFlag = true;
|
|
650
|
+
let probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag, memoryLowFlag });
|
|
599
651
|
if (probed.error !== null && refusesOomPolicy(probed.error)) {
|
|
600
652
|
oomPolicyFlag = false;
|
|
601
|
-
probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag });
|
|
653
|
+
probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag, memoryLowFlag });
|
|
654
|
+
}
|
|
655
|
+
if (probed.error !== null && refusesMemoryLow(probed.error)) {
|
|
656
|
+
memoryLowFlag = false;
|
|
657
|
+
probed = await probe.probeMemoryMax({ expandEnvironmentFlag, oomPolicyFlag, memoryLowFlag });
|
|
658
|
+
if (probed.error === null) {
|
|
659
|
+
log.warn('session cage: this systemd will not take MemoryLow on a scope — no guarantees here', {
|
|
660
|
+
why: 'the brake and the wall are unaffected; only MemoryLow is dropped',
|
|
661
|
+
});
|
|
662
|
+
}
|
|
602
663
|
}
|
|
603
664
|
if (probed.error !== null) {
|
|
604
665
|
// Not «the cage did not hold» — nothing was ever caged. Blaming `memory.max`
|
|
@@ -625,6 +686,7 @@ export async function detectSessionCage(probe = defaultCageProbe) {
|
|
|
625
686
|
serviceMemoryMaxBytes,
|
|
626
687
|
sessionsSliceMemoryMaxBytes,
|
|
627
688
|
expandEnvironmentFlag,
|
|
689
|
+
memoryLowFlag,
|
|
628
690
|
};
|
|
629
691
|
}
|
|
630
692
|
/** The `-p` assignment itself, in one place: the probe and the spawn must agree. */
|
|
@@ -640,6 +702,10 @@ const OOM_POLICY_CONTINUE = 'OOMPolicy=continue';
|
|
|
640
702
|
export function refusesOomPolicy(error) {
|
|
641
703
|
return /OOMPolicy/i.test(error);
|
|
642
704
|
}
|
|
705
|
+
/** The same question for `MemoryLow=`, and the same wording from systemd. */
|
|
706
|
+
export function refusesMemoryLow(error) {
|
|
707
|
+
return /MemoryLow/i.test(error);
|
|
708
|
+
}
|
|
643
709
|
/**
|
|
644
710
|
* Before the detector has run, nothing is wrapped.
|
|
645
711
|
*
|
|
@@ -658,6 +724,7 @@ const UNPROBED = {
|
|
|
658
724
|
serviceMemoryMaxBytes: null,
|
|
659
725
|
sessionsSliceMemoryMaxBytes: null,
|
|
660
726
|
expandEnvironmentFlag: false,
|
|
727
|
+
memoryLowFlag: false,
|
|
661
728
|
};
|
|
662
729
|
let detected = null;
|
|
663
730
|
/**
|
|
@@ -667,8 +734,31 @@ let detected = null;
|
|
|
667
734
|
* three sessions starting at once would mean three throwaway scopes before the
|
|
668
735
|
* first agent got a word out.
|
|
669
736
|
*/
|
|
670
|
-
export async function initSessionCage(probe = defaultCageProbe) {
|
|
671
|
-
|
|
737
|
+
export async function initSessionCage(probe = defaultCageProbe, options = {}) {
|
|
738
|
+
const found = await detectSessionCage(probe);
|
|
739
|
+
/**
|
|
740
|
+
* A re-probe that comes back WORSE does not take the cage away (#398).
|
|
741
|
+
*
|
|
742
|
+
* The live probe is a `systemd-run`, and one transient failure — a busy bus,
|
|
743
|
+
* a momentary «Failed to connect» — answers `nice-only`. At daemon start that
|
|
744
|
+
* is the honest answer and the machine simply runs uncaged. On the HOURLY
|
|
745
|
+
* re-measure it is something else entirely: from that moment every new
|
|
746
|
+
* session on a perfectly good machine would start with no cage, until
|
|
747
|
+
* somebody restarted the runner. An hourly job must not be able to do that
|
|
748
|
+
* (found by the independent review of 10.09.2026).
|
|
749
|
+
*
|
|
750
|
+
* Losing the cage for real is possible — cgroups remounted, the user bus gone
|
|
751
|
+
* — and it is not silent: the line below says so, loudly, every hour, and the
|
|
752
|
+
* next daemon start settles it either way.
|
|
753
|
+
*/
|
|
754
|
+
if (options.keepCageIfWorse && detected?.mode === 'scope' && found.mode !== 'scope') {
|
|
755
|
+
log.error('session cage: the re-measure lost the cage — keeping what the last probe proved', {
|
|
756
|
+
reason: found.reason,
|
|
757
|
+
hint: 'restart the runner if this machine really has no cgroups any more',
|
|
758
|
+
});
|
|
759
|
+
return detected;
|
|
760
|
+
}
|
|
761
|
+
detected = found;
|
|
672
762
|
if (detected.mode === 'scope') {
|
|
673
763
|
log.info('session cage: each session gets its own cgroup', {
|
|
674
764
|
slice: SESSIONS_SLICE,
|
|
@@ -725,6 +815,30 @@ const liveUnits = new Map();
|
|
|
725
815
|
export function sessionScopeUnitOf(id) {
|
|
726
816
|
return liveUnits.get(id) ?? null;
|
|
727
817
|
}
|
|
818
|
+
/**
|
|
819
|
+
* The pid of the agent process of each live session (#403).
|
|
820
|
+
*
|
|
821
|
+
* Written by the adapters right after the spawn — `systemd-run --scope` execs
|
|
822
|
+
* into the same pid, so this IS the process inside the cage — and read by the
|
|
823
|
+
* stall mechanism, which must never choose the agent as the command to stop.
|
|
824
|
+
*
|
|
825
|
+
* The structural rule it replaces («the agent is the one process whose parent
|
|
826
|
+
* is outside the scope») is true of the agent and true of one other thing: an
|
|
827
|
+
* MCP server whose launcher exited. Under the old rule such an orphan became a
|
|
828
|
+
* root, was protected on that account, and its own children — a browser, a
|
|
829
|
+
* language server — lost the protection they had. Knowing the pid tells the two
|
|
830
|
+
* apart, and it costs one map.
|
|
831
|
+
*/
|
|
832
|
+
const agentPids = new Map();
|
|
833
|
+
/** Remember which process is the agent of this session. */
|
|
834
|
+
export function noteSessionAgentPid(id, pid) {
|
|
835
|
+
if (typeof pid === 'number' && pid > 0)
|
|
836
|
+
agentPids.set(id, pid);
|
|
837
|
+
}
|
|
838
|
+
/** The agent's pid for a live session, or null when nothing has said. */
|
|
839
|
+
export function sessionAgentPid(id) {
|
|
840
|
+
return agentPids.get(id) ?? null;
|
|
841
|
+
}
|
|
728
842
|
/**
|
|
729
843
|
* Wrap a command in its session's cage, or hand it back untouched.
|
|
730
844
|
*
|
|
@@ -737,8 +851,30 @@ export function sessionScopeUnitOf(id) {
|
|
|
737
851
|
* spawn, because `systemd-run --scope` execs into the SAME pid — verified in the
|
|
738
852
|
* spike, including `detached: true` + `process.kill(-pid)` in `verify.ts`
|
|
739
853
|
* (`pgid === child.pid` still holds).
|
|
854
|
+
*
|
|
855
|
+
* Untouched is also the answer for a process that has no right to put units on
|
|
856
|
+
* this machine (#403). It is the mildest of the doors in `cage-authority.ts`:
|
|
857
|
+
* the command still runs, it simply runs uncaged, which is exactly what happens
|
|
858
|
+
* on every machine without cgroups.
|
|
740
859
|
*/
|
|
741
860
|
export function cageSpawn(input) {
|
|
861
|
+
if (!mayStartScope(SESSION_SCOPE_PREFIX)) {
|
|
862
|
+
log.warn('session cage: this process may not create cages, the command runs uncaged', {
|
|
863
|
+
id: input.id,
|
|
864
|
+
});
|
|
865
|
+
return { command: input.command, args: input.args, env: {}, unit: null };
|
|
866
|
+
}
|
|
867
|
+
return buildCagedSpawn(input);
|
|
868
|
+
}
|
|
869
|
+
/**
|
|
870
|
+
* The command line itself, with no question of who is allowed to run it.
|
|
871
|
+
*
|
|
872
|
+
* Split out from {@link cageSpawn} so the shape of the line stays testable: the
|
|
873
|
+
* suite proves what `systemd-run` would be handed, and the door above proves
|
|
874
|
+
* who may hand it. One function could not do both — a test process legitimately
|
|
875
|
+
* needs the first answer and must never get the second.
|
|
876
|
+
*/
|
|
877
|
+
export function buildCagedSpawn(input) {
|
|
742
878
|
const facts = sessionCage();
|
|
743
879
|
if (facts.mode !== 'scope' ||
|
|
744
880
|
facts.memoryMaxBytes === null ||
|
|
@@ -746,6 +882,20 @@ export function cageSpawn(input) {
|
|
|
746
882
|
facts.swapMaxBytes === null) {
|
|
747
883
|
return { command: input.command, args: input.args, env: {}, unit: null };
|
|
748
884
|
}
|
|
885
|
+
/**
|
|
886
|
+
* The live numbers if anything can give them, the daemon-start snapshot
|
|
887
|
+
* otherwise (#398 S3, work 9).
|
|
888
|
+
*
|
|
889
|
+
* Without this a session started at 03:00 would run for its first half-minute
|
|
890
|
+
* on the formula of whenever the daemon last started, and then jump. The
|
|
891
|
+
* allocator is asked to compute as though this session already existed, so
|
|
892
|
+
* the number it is born with is the number its first tick will confirm.
|
|
893
|
+
*/
|
|
894
|
+
const live = liveLadderSource?.(input.id) ?? null;
|
|
895
|
+
const highBytes = live?.highBytes ?? facts.memoryHighBytes;
|
|
896
|
+
const maxBytes = live?.maxBytes ?? facts.memoryMaxBytes;
|
|
897
|
+
const swapMaxBytes = live?.swapBytes ?? facts.swapMaxBytes;
|
|
898
|
+
const guaranteedBytes = live?.guaranteedBytes ?? null;
|
|
749
899
|
const attempt = (attempts.get(input.id) ?? 0) + 1;
|
|
750
900
|
attempts.set(input.id, attempt);
|
|
751
901
|
const unit = sessionScopeUnit(input.id, attempt);
|
|
@@ -767,21 +917,28 @@ export function cageSpawn(input) {
|
|
|
767
917
|
// Only where systemd knows the switch — below 254 it is a hard error and
|
|
768
918
|
// `--scope` does no expansion anyway (module header, MAJOR-2).
|
|
769
919
|
...(facts.expandEnvironmentFlag ? ['--expand-environment=no'] : []),
|
|
920
|
+
// The guarantee: what will not be reclaimed from this session when the
|
|
921
|
+
// slice as a whole comes under pressure (#398 S3). Only where the
|
|
922
|
+
// allocator could compute one — a number nobody stands behind is worse
|
|
923
|
+
// than none.
|
|
924
|
+
...(guaranteedBytes === null || !facts.memoryLowFlag
|
|
925
|
+
? []
|
|
926
|
+
: ['-p', `MemoryLow=${guaranteedBytes}`]),
|
|
770
927
|
// The ladder of #387, bottom to top. The brake: past the honest share the
|
|
771
928
|
// kernel reclaims and slows this session down, and nothing dies.
|
|
772
929
|
'-p',
|
|
773
|
-
`MemoryHigh=${
|
|
930
|
+
`MemoryHigh=${highBytes}`,
|
|
774
931
|
// The wall: the machine's measured headroom, far above the brake. Only a
|
|
775
932
|
// runaway gets here, and with `OOMPolicy=continue` it costs one process.
|
|
776
933
|
'-p',
|
|
777
|
-
`MemoryMax=${
|
|
934
|
+
`MemoryMax=${maxBytes}`,
|
|
778
935
|
// THE line. Without a bound there is no cage at all: `MemoryMax` bounds
|
|
779
936
|
// resident memory and lets the rest fall into the host's swap, which is
|
|
780
937
|
// how a 200 MB cage allocated 2 GB and took the machine's swap with it.
|
|
781
938
|
// Bounded, never unset — and since #387 a share rather than 0, because
|
|
782
939
|
// the brake above has to have somewhere to push pages (module header).
|
|
783
940
|
'-p',
|
|
784
|
-
`MemorySwapMax=${
|
|
941
|
+
`MemorySwapMax=${swapMaxBytes}`,
|
|
785
942
|
// Only where the probe proved this systemd takes it on a scope. Without
|
|
786
943
|
// it the kernel killing ONE process stops the WHOLE scope (`OOMPolicy`
|
|
787
944
|
// defaults to `stop`) — the exact mechanism that killed honest sessions
|
|
@@ -843,8 +1000,14 @@ export function parseScopeMemoryStatus(files) {
|
|
|
843
1000
|
ownLimitOom: event('oom'),
|
|
844
1001
|
};
|
|
845
1002
|
}
|
|
846
|
-
/**
|
|
847
|
-
|
|
1003
|
+
/**
|
|
1004
|
+
* Where a session scope's cgroup lives, or null off a user manager.
|
|
1005
|
+
*
|
|
1006
|
+
* Exported for `session-stall.ts`, which reads three more files out of the same
|
|
1007
|
+
* directory (`memory.pressure`, `memory.stat`, `cgroup.procs`). One resolver,
|
|
1008
|
+
* so the two modules can never disagree about which cgroup a session is in.
|
|
1009
|
+
*/
|
|
1010
|
+
export function scopeCgroupDir(unit) {
|
|
848
1011
|
const self = readSelfCgroup();
|
|
849
1012
|
if (self === null)
|
|
850
1013
|
return null;
|
|
@@ -882,6 +1045,223 @@ export function readScopeMemoryStatus(unit, readFile = (p) => fs.readFileSync(p,
|
|
|
882
1045
|
return null;
|
|
883
1046
|
}
|
|
884
1047
|
}
|
|
1048
|
+
/**
|
|
1049
|
+
* What a live session holds, for the allocator (#398 S3).
|
|
1050
|
+
*
|
|
1051
|
+
* `readCgroupMemory` rather than a second parser: it already takes page cache
|
|
1052
|
+
* out and puts `shmem` back, which is the difference between «cache we can give
|
|
1053
|
+
* back» and «memory something would have to be killed for», and getting that
|
|
1054
|
+
* wrong inflates every number by whatever the machine happened to have cached
|
|
1055
|
+
* (measured at +19 % on this host).
|
|
1056
|
+
*/
|
|
1057
|
+
export function readScopeHold(unit) {
|
|
1058
|
+
const dir = scopeCgroupDir(unit);
|
|
1059
|
+
if (dir === null)
|
|
1060
|
+
return null;
|
|
1061
|
+
const reading = readCgroupMemory(dir);
|
|
1062
|
+
if (reading === null)
|
|
1063
|
+
return null;
|
|
1064
|
+
return {
|
|
1065
|
+
currentBytes: reading.currentBytes,
|
|
1066
|
+
holdBytes: reading.unreclaimableBytes ?? reading.currentBytes,
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
/**
|
|
1070
|
+
* The slice's own two limits, from the filesystem rather than the bus.
|
|
1071
|
+
*
|
|
1072
|
+
* Live on purpose. What the probe found at daemon start is a snapshot, and the
|
|
1073
|
+
* pot moves: the hourly `repairResourceLimits` rewrites it from a fresh
|
|
1074
|
+
* measurement, and a machine whose neighbours grew or shrank gets a different
|
|
1075
|
+
* number. Reading two small files on the tick costs nothing and cannot be stale;
|
|
1076
|
+
* a `systemctl show` would cost a bus call measured at 2.7 s under load.
|
|
1077
|
+
*/
|
|
1078
|
+
export function readSliceLimits(readFile = (p) => fs.readFileSync(p, 'utf8')) {
|
|
1079
|
+
/**
|
|
1080
|
+
* The slice's path, from the READER's own cgroup where that works — and from
|
|
1081
|
+
* the user id where it does not.
|
|
1082
|
+
*
|
|
1083
|
+
* `sliceCgroupPath(readSelfCgroup(), …)` answers null for anything that is
|
|
1084
|
+
* not itself under the user manager: a login-session scope, a `systemd-run`
|
|
1085
|
+
* scope, a bare shell. That is fine for the DAEMON, which always is, and
|
|
1086
|
+
* quite wrong for `doctor` — which the installer runs through `setpriv`,
|
|
1087
|
+
* keeping the installer's own cgroup. The result was that a freshly and
|
|
1088
|
+
* correctly installed machine reported «collective brake MISSING» on its very
|
|
1089
|
+
* first acceptance run. Found by the independent review of 10.09.2026.
|
|
1090
|
+
*/
|
|
1091
|
+
const dir = sessionsSliceCgroupDir();
|
|
1092
|
+
if (dir === null)
|
|
1093
|
+
return null;
|
|
1094
|
+
const limit = (name) => {
|
|
1095
|
+
try {
|
|
1096
|
+
const raw = readFile(path.join(dir, name)).trim();
|
|
1097
|
+
if (raw === 'max' || !/^\d+$/.test(raw))
|
|
1098
|
+
return null;
|
|
1099
|
+
const value = Number(raw);
|
|
1100
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
1101
|
+
}
|
|
1102
|
+
catch {
|
|
1103
|
+
return null;
|
|
1104
|
+
}
|
|
1105
|
+
};
|
|
1106
|
+
const potBytes = limit('memory.max');
|
|
1107
|
+
const collectiveBrakeBytes = limit('memory.high');
|
|
1108
|
+
// `memory.low` reads `0` when nothing protects the slice, and `limit()` turns
|
|
1109
|
+
// only `max` into null — so zero survives, which is the whole point here.
|
|
1110
|
+
const guaranteeBytes = limit('memory.low');
|
|
1111
|
+
if (potBytes === null && collectiveBrakeBytes === null)
|
|
1112
|
+
return null;
|
|
1113
|
+
return { potBytes, collectiveBrakeBytes, guaranteeBytes };
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Where `devbridge-sessions.slice` lives on this machine.
|
|
1117
|
+
*
|
|
1118
|
+
* Two ways of answering, and the second is what makes this usable from a
|
|
1119
|
+
* command the operator typed. The first is the reader's own cgroup, which is
|
|
1120
|
+
* exact whenever the reader is under the user manager. The second builds the
|
|
1121
|
+
* path from the user id, exactly as `userManagerCgroupControllers()` already
|
|
1122
|
+
* does — a `systemd-run` scope, a login session and a bare shell all get the
|
|
1123
|
+
* right answer that way, and the daemon gets the same one either way.
|
|
1124
|
+
*/
|
|
1125
|
+
export function sessionsSliceCgroupDir(io = {}) {
|
|
1126
|
+
const selfCgroup = io.selfCgroup ?? readSelfCgroup;
|
|
1127
|
+
const exists = io.exists ?? ((p) => fs.existsSync(p));
|
|
1128
|
+
const self = selfCgroup();
|
|
1129
|
+
const fromSelf = self === null ? null : sliceCgroupPath(self, SESSIONS_SLICE);
|
|
1130
|
+
if (fromSelf !== null)
|
|
1131
|
+
return fromSelf;
|
|
1132
|
+
/**
|
|
1133
|
+
* Whose user manager, when the reader's own cgroup cannot say.
|
|
1134
|
+
*
|
|
1135
|
+
* Two candidates and the order matters (#398 S7, B6). The runner's OWN uid
|
|
1136
|
+
* comes first — that is the daemon and every ordinary CLI. But `doctor` is
|
|
1137
|
+
* routinely run as `sudo devbridge-runner doctor` on a machine installed with
|
|
1138
|
+
* `--user devbridge`, and then the process's uid is root's while the slice
|
|
1139
|
+
* lives under the service user's manager: a correctly installed machine
|
|
1140
|
+
* reported «collective brake MISSING» on its first acceptance run.
|
|
1141
|
+
*
|
|
1142
|
+
* The owner of the runner's state directory IS the service user, by
|
|
1143
|
+
* construction — the installer creates it as them — so it is the honest
|
|
1144
|
+
* second guess, and it costs one `stat`.
|
|
1145
|
+
*/
|
|
1146
|
+
const uids = [io.selfUid?.() ?? runnerIdentity().uid, io.ownerUid?.() ?? stateDirOwnerUid()];
|
|
1147
|
+
for (const uid of uids) {
|
|
1148
|
+
if (uid < 0)
|
|
1149
|
+
continue;
|
|
1150
|
+
const guess = path.join('/sys/fs/cgroup/user.slice', `user-${uid}.slice`, `user@${uid}.service`, DEVBRIDGE_SLICE, SESSIONS_SLICE);
|
|
1151
|
+
try {
|
|
1152
|
+
if (exists(guess))
|
|
1153
|
+
return guess;
|
|
1154
|
+
}
|
|
1155
|
+
catch {
|
|
1156
|
+
// Unreadable is «not here», and the next candidate may still answer.
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
return null;
|
|
1160
|
+
}
|
|
1161
|
+
/** The uid that owns the runner's state directory — the service user. */
|
|
1162
|
+
function stateDirOwnerUid() {
|
|
1163
|
+
try {
|
|
1164
|
+
return fs.statSync(stateDir()).uid;
|
|
1165
|
+
}
|
|
1166
|
+
catch {
|
|
1167
|
+
return -1;
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
let liveLadderSource = null;
|
|
1171
|
+
export function setLiveLadderSource(source) {
|
|
1172
|
+
liveLadderSource = source;
|
|
1173
|
+
}
|
|
1174
|
+
/**
|
|
1175
|
+
* The two numbers this session may be TOLD about (#398 S5).
|
|
1176
|
+
*
|
|
1177
|
+
* The guarantee and the ceiling, and deliberately not the live share. The share
|
|
1178
|
+
* is recomputed every 30 s, so any value of it put into an environment variable
|
|
1179
|
+
* or into a system prompt is stale within half a minute — and a number the agent
|
|
1180
|
+
* believes and acts on after it has stopped being true is worse than no number.
|
|
1181
|
+
* The guarantee is a constant of the formula and the ceiling barely moves.
|
|
1182
|
+
*
|
|
1183
|
+
* Null on a machine with no cage: there is nothing to promise there, and «нет
|
|
1184
|
+
* cgroup — держим ноль» (#257) means we say nothing rather than invent a limit.
|
|
1185
|
+
*/
|
|
1186
|
+
export function sessionMemoryFor(id) {
|
|
1187
|
+
const facts = sessionCage();
|
|
1188
|
+
if (facts.mode !== 'scope')
|
|
1189
|
+
return null;
|
|
1190
|
+
const live = liveLadderSource?.(id) ?? null;
|
|
1191
|
+
if (live) {
|
|
1192
|
+
return {
|
|
1193
|
+
guaranteedBytes: live.guaranteedBytes,
|
|
1194
|
+
maxBytes: live.maxBytes,
|
|
1195
|
+
swapBytes: live.swapBytes,
|
|
1196
|
+
};
|
|
1197
|
+
}
|
|
1198
|
+
if (facts.memoryMaxBytes === null)
|
|
1199
|
+
return null;
|
|
1200
|
+
return {
|
|
1201
|
+
// Before the allocator has anything to say, the honest floor is the brake
|
|
1202
|
+
// the daemon measured: it is what this session would be held to.
|
|
1203
|
+
guaranteedBytes: facts.memoryHighBytes ?? 0,
|
|
1204
|
+
maxBytes: facts.memoryMaxBytes,
|
|
1205
|
+
swapBytes: facts.swapMaxBytes ?? 0,
|
|
1206
|
+
};
|
|
1207
|
+
}
|
|
1208
|
+
/**
|
|
1209
|
+
* What the agent is told, and how — one text for both CLIs (#398 S5).
|
|
1210
|
+
*
|
|
1211
|
+
* A shared builder rather than two literals for the reason `DIRECT_BRANCH_RULE`
|
|
1212
|
+
* in `adapters/types.ts` is shared: the two system-prompt appends are hand-kept
|
|
1213
|
+
* copies of one another, and a third place for them to diverge was not worth
|
|
1214
|
+
* having.
|
|
1215
|
+
*
|
|
1216
|
+
* The live share is named as something the agent CANNOT know, on purpose. Half
|
|
1217
|
+
* of the incident this whole plan comes from was an agent raising its own heap
|
|
1218
|
+
* twice — to 4 GB, then to 6 GB, against a wall of 4296 MB — because nothing
|
|
1219
|
+
* had ever told it there was a wall.
|
|
1220
|
+
*/
|
|
1221
|
+
export function sessionMemoryEnv(id) {
|
|
1222
|
+
const memory = sessionMemoryFor(id);
|
|
1223
|
+
if (memory === null)
|
|
1224
|
+
return {};
|
|
1225
|
+
const mb = (bytes) => String(Math.max(0, Math.floor(bytes / MIB)));
|
|
1226
|
+
return {
|
|
1227
|
+
DEVBRIDGE_SESSION_MEMORY_GUARANTEED_MB: mb(memory.guaranteedBytes),
|
|
1228
|
+
DEVBRIDGE_SESSION_MEMORY_MAX_MB: mb(memory.maxBytes),
|
|
1229
|
+
};
|
|
1230
|
+
}
|
|
1231
|
+
/** The sentence for the system prompt, or null on a machine with no cage. */
|
|
1232
|
+
export function sessionMemoryPromptLine(id) {
|
|
1233
|
+
const memory = sessionMemoryFor(id);
|
|
1234
|
+
if (memory === null)
|
|
1235
|
+
return null;
|
|
1236
|
+
const mb = (bytes) => Math.max(0, Math.round(bytes / MIB));
|
|
1237
|
+
const swap = memory.swapBytes > 0
|
|
1238
|
+
? `This machine has swap, so going over your share slows you down.`
|
|
1239
|
+
: `This machine has NO swap, so going over your share does not slow you down — it stops you.`;
|
|
1240
|
+
/**
|
|
1241
|
+
* A guarantee is named only when one is actually in force (#398 S7, B3).
|
|
1242
|
+
*
|
|
1243
|
+
* `guaranteedBytes` is zero when nothing wrote `MemoryLow` — the systemd here
|
|
1244
|
+
* refuses the property, or the slice above has no protection of its own. The
|
|
1245
|
+
* agent plans around what it is told, so telling it about two gigabytes that
|
|
1246
|
+
* nobody reserved is worse than telling it nothing.
|
|
1247
|
+
*/
|
|
1248
|
+
if (memory.guaranteedBytes <= 0) {
|
|
1249
|
+
return (`- Memory: this session can never exceed ${mb(memory.maxBytes)} MB ` +
|
|
1250
|
+
`(DEVBRIDGE_SESSION_MEMORY_MAX_MB), and it has NO reserved share on this machine — under ` +
|
|
1251
|
+
`pressure the kernel may reclaim anything it holds. How much you get at any moment is decided ` +
|
|
1252
|
+
`by the machine and changes; you cannot read it, so do not plan around a number. ${swap} ` +
|
|
1253
|
+
`If a command stands still for three minutes for want of memory, DevBridge stops that command ` +
|
|
1254
|
+
`(not the session) and tells you. Do not raise --max-old-space-size above the ceiling: it is ` +
|
|
1255
|
+
`enforced outside your process, and asking for more only makes the command fail sooner.`);
|
|
1256
|
+
}
|
|
1257
|
+
return (`- Memory: this session is guaranteed ${mb(memory.guaranteedBytes)} MB and can never exceed ` +
|
|
1258
|
+
`${mb(memory.maxBytes)} MB (DEVBRIDGE_SESSION_MEMORY_GUARANTEED_MB and ` +
|
|
1259
|
+
`DEVBRIDGE_SESSION_MEMORY_MAX_MB). How much you get above the guarantee at any moment is decided ` +
|
|
1260
|
+
`by the machine and changes; you cannot read it, so do not plan around a number. ${swap} ` +
|
|
1261
|
+
`If a command stands still for three minutes for want of memory, DevBridge stops that command ` +
|
|
1262
|
+
`(not the session) and tells you. Do not raise --max-old-space-size above the ceiling: it is ` +
|
|
1263
|
+
`enforced outside your process, and asking for more only makes the command fail sooner.`);
|
|
1264
|
+
}
|
|
885
1265
|
/**
|
|
886
1266
|
* `oom_kill` as the watch last saw it, per id — so a kill that was already
|
|
887
1267
|
* announced in the feed while the session lived is not blamed for a death that
|
|
@@ -973,13 +1353,7 @@ export function explainMemoryDeath(id) {
|
|
|
973
1353
|
return (`There was not enough memory for this session and ${stopped} — ` +
|
|
974
1354
|
'send a message to carry on; if it keeps happening, run fewer sessions at once or give the machine more memory.');
|
|
975
1355
|
}
|
|
976
|
-
const realSystemctl = async (args) =>
|
|
977
|
-
const { stdout, stderr } = await execFileAsync('systemctl', ['--user', ...args], {
|
|
978
|
-
timeout: 15_000,
|
|
979
|
-
env: systemdUserEnv(),
|
|
980
|
-
});
|
|
981
|
-
return { stdout, stderr };
|
|
982
|
-
};
|
|
1356
|
+
const realSystemctl = async (args) => await runSystemctl(args);
|
|
983
1357
|
function showValue(stdout, property) {
|
|
984
1358
|
const line = stdout
|
|
985
1359
|
.split('\n')
|
|
@@ -1001,6 +1375,9 @@ function showValue(stdout, property) {
|
|
|
1001
1375
|
export async function releaseSessionScope(unit, id, systemctl = realSystemctl) {
|
|
1002
1376
|
const running = releaseSessionScopeInner(unit, id, systemctl);
|
|
1003
1377
|
if (id !== undefined) {
|
|
1378
|
+
// The process this named is gone with the scope; a pid outliving it would be
|
|
1379
|
+
// a pid that means something else by the time anybody reads it.
|
|
1380
|
+
agentPids.delete(id);
|
|
1004
1381
|
releasing.set(id, running);
|
|
1005
1382
|
void running.finally(() => {
|
|
1006
1383
|
if (releasing.get(id) === running)
|
|
@@ -1051,9 +1428,19 @@ async function releaseSessionScopeInner(unit, id, systemctl) {
|
|
|
1051
1428
|
if (id !== undefined)
|
|
1052
1429
|
liveUnits.delete(id);
|
|
1053
1430
|
let result = null;
|
|
1431
|
+
let tasksLeft = null;
|
|
1054
1432
|
try {
|
|
1055
|
-
|
|
1433
|
+
// Both properties in ONE call: this runs on every process exit, and the bus
|
|
1434
|
+
// was measured at 2.7 s under load — two round trips where one will do is a
|
|
1435
|
+
// cost paid on the path that is walked most.
|
|
1436
|
+
const { stdout } = await systemctl(['show', unit, '-p', 'Result', '-p', 'TasksCurrent']);
|
|
1056
1437
|
result = showValue(stdout, 'Result');
|
|
1438
|
+
// Explicitly null-checked, not coerced: `Number(null)` is 0, and reading a
|
|
1439
|
+
// property systemd never printed as «no tasks left» would silently turn the
|
|
1440
|
+
// stop below off. systemd's own «unknown» is `[not set]`, which is NaN.
|
|
1441
|
+
const raw = showValue(stdout, 'TasksCurrent');
|
|
1442
|
+
const tasks = raw === null ? Number.NaN : Number(raw);
|
|
1443
|
+
tasksLeft = Number.isFinite(tasks) ? tasks : null;
|
|
1057
1444
|
}
|
|
1058
1445
|
catch {
|
|
1059
1446
|
// Already collected — a clean exit takes its scope with it.
|
|
@@ -1068,6 +1455,41 @@ async function releaseSessionScopeInner(unit, id, systemctl) {
|
|
|
1068
1455
|
if (id !== undefined)
|
|
1069
1456
|
rememberDeathFromResult(id);
|
|
1070
1457
|
}
|
|
1458
|
+
/**
|
|
1459
|
+
* The agent's process is gone and the scope still holds tasks — put the tree
|
|
1460
|
+
* out, here and nowhere else.
|
|
1461
|
+
*
|
|
1462
|
+
* Until 0.58.0 this function only ever `show`ed and `reset-failed` the unit,
|
|
1463
|
+
* and nothing anywhere stopped it: `park()` kills ONE pid (the CLI), so a
|
|
1464
|
+
* `pnpm typecheck` or a `vite build` the agent had started went on eating
|
|
1465
|
+
* memory for a session that no longer existed, and the next start of the same
|
|
1466
|
+
* session met «Unit … was already loaded» and died as
|
|
1467
|
+
* «codex app-server exited before the session was ready (code 1)» (gotcha
|
|
1468
|
+
* §479, measured 09.09 on vmi3219930: 3.4 GB free instead of 8.5).
|
|
1469
|
+
*
|
|
1470
|
+
* Deliberately in the RELEASE path rather than as a second call from the
|
|
1471
|
+
* supervisor's park: all three call sites (`claude.ts`, `codex-protocol.ts`,
|
|
1472
|
+
* `verify.ts`) already come through here on process exit, so one place keeps
|
|
1473
|
+
* both agents and the verify runs identical and cannot race a second release
|
|
1474
|
+
* of the same unit.
|
|
1475
|
+
*
|
|
1476
|
+
* `TasksCurrent > 0` is the gate rather than an unconditional stop, because
|
|
1477
|
+
* the normal case is an empty scope systemd has already collected, and
|
|
1478
|
+
* `stop` on it would be a bus call per session exit for nothing.
|
|
1479
|
+
*/
|
|
1480
|
+
if (tasksLeft !== null && tasksLeft > 0) {
|
|
1481
|
+
log.warn('session cage: the agent is gone but its commands are not — stopping the scope', {
|
|
1482
|
+
unit,
|
|
1483
|
+
tasksLeft,
|
|
1484
|
+
});
|
|
1485
|
+
try {
|
|
1486
|
+
await systemctl(['stop', unit]);
|
|
1487
|
+
}
|
|
1488
|
+
catch {
|
|
1489
|
+
// A scope that ended between the read and the stop answers «not loaded».
|
|
1490
|
+
// The sweep on the watch tick is the second line of defence.
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1071
1493
|
let forgotten = false;
|
|
1072
1494
|
try {
|
|
1073
1495
|
await systemctl(['reset-failed', unit]);
|
|
@@ -1146,32 +1568,160 @@ export async function listSessionScopes(systemctl = realSystemctl) {
|
|
|
1146
1568
|
}
|
|
1147
1569
|
return out;
|
|
1148
1570
|
}
|
|
1571
|
+
export function scopeOwnerLiveness(unit, io = {}) {
|
|
1572
|
+
const readFile = io.readFile ?? ((p) => fs.readFileSync(p, 'utf8'));
|
|
1573
|
+
const cgroupDir = io.cgroupDir ?? scopeCgroupDir;
|
|
1574
|
+
const isAlive = io.isAlive ??
|
|
1575
|
+
((pid) => {
|
|
1576
|
+
try {
|
|
1577
|
+
process.kill(pid, 0);
|
|
1578
|
+
return true;
|
|
1579
|
+
}
|
|
1580
|
+
catch (error) {
|
|
1581
|
+
// `EPERM` means it exists and belongs to somebody else — alive either way.
|
|
1582
|
+
return error.code === 'EPERM';
|
|
1583
|
+
}
|
|
1584
|
+
});
|
|
1585
|
+
const dir = cgroupDir(unit);
|
|
1586
|
+
if (dir === null)
|
|
1587
|
+
return 'unknown';
|
|
1588
|
+
let pids;
|
|
1589
|
+
try {
|
|
1590
|
+
pids = readFile(path.join(dir, 'cgroup.procs'))
|
|
1591
|
+
.split('\n')
|
|
1592
|
+
.map((line) => Number(line.trim()))
|
|
1593
|
+
.filter((pid) => Number.isSafeInteger(pid) && pid > 0);
|
|
1594
|
+
}
|
|
1595
|
+
catch {
|
|
1596
|
+
return 'unknown';
|
|
1597
|
+
}
|
|
1598
|
+
if (pids.length === 0)
|
|
1599
|
+
return 'empty';
|
|
1600
|
+
let sawUnknown = false;
|
|
1601
|
+
for (const pid of pids) {
|
|
1602
|
+
let ppid;
|
|
1603
|
+
try {
|
|
1604
|
+
const status = readFile(path.join('/proc', String(pid), 'status'));
|
|
1605
|
+
const line = status.split('\n').find((l) => l.startsWith('PPid:'));
|
|
1606
|
+
const value = line === undefined ? Number.NaN : Number(line.slice('PPid:'.length).trim());
|
|
1607
|
+
ppid = Number.isSafeInteger(value) ? value : null;
|
|
1608
|
+
}
|
|
1609
|
+
catch {
|
|
1610
|
+
// Ended between the listing and the read — says nothing either way.
|
|
1611
|
+
continue;
|
|
1612
|
+
}
|
|
1613
|
+
if (ppid === null) {
|
|
1614
|
+
sawUnknown = true;
|
|
1615
|
+
continue;
|
|
1616
|
+
}
|
|
1617
|
+
// Reparented to init: nobody is waiting on this process. `0` is the kernel's
|
|
1618
|
+
// answer for a process that is going away.
|
|
1619
|
+
if (ppid <= 1)
|
|
1620
|
+
continue;
|
|
1621
|
+
/**
|
|
1622
|
+
* …and reparented to the USER MANAGER counts the same (measured 10.09.2026).
|
|
1623
|
+
*
|
|
1624
|
+
* `systemd --user` sets itself as a subreaper, so an orphan inside a user
|
|
1625
|
+
* scope does not land on pid 1 at all — it lands on the manager, whose pid
|
|
1626
|
+
* is an ordinary live pid. Read literally, «the parent is alive» then made
|
|
1627
|
+
* every orphan look supervised, and the sweep would never remove anything
|
|
1628
|
+
* again: the opposite failure to the one this whole change is about, and
|
|
1629
|
+
* just as silent.
|
|
1630
|
+
*/
|
|
1631
|
+
let parentComm;
|
|
1632
|
+
try {
|
|
1633
|
+
parentComm = readFile(path.join('/proc', String(ppid), 'comm')).trim();
|
|
1634
|
+
}
|
|
1635
|
+
catch {
|
|
1636
|
+
// The parent went between the two reads: it is not supervising anything.
|
|
1637
|
+
continue;
|
|
1638
|
+
}
|
|
1639
|
+
if (parentComm === 'systemd' || parentComm === 'init')
|
|
1640
|
+
continue;
|
|
1641
|
+
if (isAlive(ppid))
|
|
1642
|
+
return 'owned';
|
|
1643
|
+
}
|
|
1644
|
+
return sawUnknown ? 'unknown' : 'orphaned';
|
|
1645
|
+
}
|
|
1149
1646
|
/**
|
|
1150
|
-
*
|
|
1151
|
-
*
|
|
1152
|
-
* At daemon start that is all of them by definition, and it is the point: a
|
|
1153
|
-
* scope outlives a killed daemon carrying the whole process tree with it, which
|
|
1154
|
-
* is the shape of the 10 h 51 min `ugrep` of 16.08. Stopping the scope takes the
|
|
1155
|
-
* tree, not just the process we happened to know about.
|
|
1647
|
+
* Is this unit, RIGHT NOW, a cage somebody is running in?
|
|
1156
1648
|
*
|
|
1157
|
-
*
|
|
1158
|
-
*
|
|
1159
|
-
*
|
|
1649
|
+
* A scan rather than a snapshot on purpose — the freshness is the point — and a
|
|
1650
|
+
* scan rather than `new Set(liveUnits.values()).has(unit)`, which built a set
|
|
1651
|
+
* per unit inside the sweep's loop for nothing. The register holds one entry
|
|
1652
|
+
* per live session; on the biggest machine of the fleet that is six.
|
|
1160
1653
|
*/
|
|
1161
|
-
|
|
1162
|
-
const
|
|
1654
|
+
function isLiveCageUnit(unit) {
|
|
1655
|
+
for (const live of liveUnits.values())
|
|
1656
|
+
if (live === unit)
|
|
1657
|
+
return true;
|
|
1658
|
+
return false;
|
|
1659
|
+
}
|
|
1660
|
+
export async function sweepOrphanSessionScopes(liveIds = [], systemctl = realSystemctl, liveness = scopeOwnerLiveness) {
|
|
1661
|
+
/**
|
|
1662
|
+
* Asked here, and not left to the door in `cage-authority.ts` (#403).
|
|
1663
|
+
*
|
|
1664
|
+
* The loop below catches every failed `systemctl` on purpose — a scope that
|
|
1665
|
+
* is already `failed` has nothing to stop — so a refusal down there would be
|
|
1666
|
+
* swallowed and this function would report units as removed that it never
|
|
1667
|
+
* touched. The register `liveUnits` is empty in any process but the daemon's,
|
|
1668
|
+
* which is precisely why this function is the dangerous one: to a foreign
|
|
1669
|
+
* process every real cage on the machine looks like litter.
|
|
1670
|
+
*/
|
|
1671
|
+
if (systemctl === realSystemctl && !mayActOnSessionCages()) {
|
|
1672
|
+
log.warn('session cage: this process may not sweep cages, nothing was touched', {
|
|
1673
|
+
role: cageAuthority(),
|
|
1674
|
+
});
|
|
1675
|
+
return [];
|
|
1676
|
+
}
|
|
1677
|
+
const spared = new Set(liveUnits.values());
|
|
1163
1678
|
for (const id of liveIds)
|
|
1164
1679
|
spared.add(sessionScopeUnit(id, attempts.get(id) ?? 1));
|
|
1165
1680
|
const removed = [];
|
|
1166
1681
|
for (const unit of await listSessionScopeUnits(systemctl)) {
|
|
1167
1682
|
if (spared.has(unit))
|
|
1168
1683
|
continue;
|
|
1169
|
-
|
|
1170
|
-
|
|
1684
|
+
/**
|
|
1685
|
+
* …and asked AGAIN, immediately before the stop.
|
|
1686
|
+
*
|
|
1687
|
+
* `spared` is a snapshot, and between it and this line there are two
|
|
1688
|
+
* `systemctl` calls — the listing, and every stop before this one — each of
|
|
1689
|
+
* which the bus has been measured to hold for 2.7 s under load. A session
|
|
1690
|
+
* that started inside that window would be in `list-units` and not in the
|
|
1691
|
+
* snapshot, and this call would kill it. Found by the independent review of
|
|
1692
|
+
* 10.09.2026.
|
|
1693
|
+
*
|
|
1694
|
+
* `liveUnits` is filled by `cageSpawn` BEFORE `systemd-run` is spawned, so
|
|
1695
|
+
* a scope cannot exist without its name being in there first.
|
|
1696
|
+
*/
|
|
1697
|
+
if (isLiveCageUnit(unit))
|
|
1698
|
+
continue;
|
|
1699
|
+
/**
|
|
1700
|
+
* …and the kernel is asked too, because the register above only knows what
|
|
1701
|
+
* THIS process started (#403).
|
|
1702
|
+
*
|
|
1703
|
+
* `owned` means a live parent outside the cage is supervising what is
|
|
1704
|
+
* inside it — another daemon's session, or this machine's own runner seen
|
|
1705
|
+
* from a second process. Whatever it is, it is not litter, and stopping it
|
|
1706
|
+
* is the incident of 10.09.2026. `unknown` gets the same answer for the
|
|
1707
|
+
* same reason: the sweep may only act on what it can prove.
|
|
1708
|
+
*/
|
|
1709
|
+
const state = liveness(unit);
|
|
1710
|
+
if (state === 'owned' || state === 'unknown') {
|
|
1711
|
+
log.warn('session cage: left a scope alone — somebody is running in it', {
|
|
1712
|
+
unit,
|
|
1713
|
+
liveness: state,
|
|
1714
|
+
});
|
|
1715
|
+
continue;
|
|
1171
1716
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1717
|
+
if (state !== 'empty') {
|
|
1718
|
+
try {
|
|
1719
|
+
await systemctl(['stop', unit]);
|
|
1720
|
+
}
|
|
1721
|
+
catch {
|
|
1722
|
+
// A failed scope has nothing to stop; `reset-failed` below is the part
|
|
1723
|
+
// that matters for it.
|
|
1724
|
+
}
|
|
1175
1725
|
}
|
|
1176
1726
|
try {
|
|
1177
1727
|
await systemctl(['reset-failed', unit]);
|