@bridge4dev/runner 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +105 -2
- package/dist/adapters/codex-protocol.d.ts +11 -0
- package/dist/adapters/codex-protocol.js +41 -3
- package/dist/adapters/codex.js +37 -0
- package/dist/host-load.d.ts +156 -0
- package/dist/host-load.js +223 -0
- package/dist/index.js +211 -40
- package/dist/process-priority.d.ts +55 -0
- package/dist/process-priority.js +99 -0
- package/dist/protocol.d.ts +25 -3
- package/dist/recipe-schema.d.ts +6 -6
- package/dist/self-update.js +43 -2
- package/dist/service-unit.d.ts +268 -10
- package/dist/service-unit.js +432 -53
- package/dist/session-cage.d.ts +508 -0
- package/dist/session-cage.js +1183 -0
- package/dist/supervisor.d.ts +106 -0
- package/dist/supervisor.js +243 -0
- package/dist/systemd-memory.d.ts +35 -0
- package/dist/systemd-memory.js +115 -0
- package/dist/verify.js +28 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import os from 'node:os';
|
|
2
|
+
import { log } from './log.js';
|
|
3
|
+
/**
|
|
4
|
+
* Everything an agent starts runs at a lower CPU priority than the daemon that
|
|
5
|
+
* supervises it.
|
|
6
|
+
*
|
|
7
|
+
* The failure this exists for is the runner starving itself with its own
|
|
8
|
+
* children (16.08): a session's build saturated the box, the daemon lost four
|
|
9
|
+
* heartbeats in a row, the server went Offline and every session on that
|
|
10
|
+
* machine answered 504 — while the work it was doing was fine. The daemon's job
|
|
11
|
+
* during a heavy turn is a few milliseconds of socket traffic; it should not
|
|
12
|
+
* have to queue behind a `tsc` it launched itself.
|
|
13
|
+
*
|
|
14
|
+
* **What this gives.** Inside the service's own cgroup the daemon (nice 0) gets
|
|
15
|
+
* the processor before the agents (nice 10); neighbouring sessions, all at 10,
|
|
16
|
+
* still share it evenly between themselves. Children inherit the nice value at
|
|
17
|
+
* fork, so a `vitest` the agent starts through Bash — several levels down from
|
|
18
|
+
* the process we renice — is covered without us knowing about it.
|
|
19
|
+
*
|
|
20
|
+
* Inheritance is why every call site is the line straight after its `spawn`:
|
|
21
|
+
* measured here, a grandchild forked AFTER the call comes up at 10 and one
|
|
22
|
+
* forked in the microseconds before it stays at 0. An agent needs hundreds of
|
|
23
|
+
* milliseconds to boot before it forks anything, so that window is empty in
|
|
24
|
+
* practice — but it is a window, and it only grows if the call drifts down the
|
|
25
|
+
* function.
|
|
26
|
+
*
|
|
27
|
+
* **What it does NOT give.** Against processes OUTSIDE the cgroup (production
|
|
28
|
+
* in `system.slice`) it does nothing at all: across cgroups the split is
|
|
29
|
+
* decided by `cpu.weight`, and nice only orders tasks within one. It does not
|
|
30
|
+
* touch memory, which is the mechanism behind three of the four incidents on
|
|
31
|
+
* other people's machines. This is the approach to stage 2 (a scope per session
|
|
32
|
+
* with a memory ceiling) and the fallback for it on cgroup v1, where stage 2
|
|
33
|
+
* cannot work — not a replacement for it.
|
|
34
|
+
*
|
|
35
|
+
* Priority is a convenience, not correctness: nothing here throws. A session
|
|
36
|
+
* that runs at the wrong priority is a slower machine; a session that fails to
|
|
37
|
+
* start because renicing failed is a broken product.
|
|
38
|
+
*/
|
|
39
|
+
/**
|
|
40
|
+
* The nice value every process the runner spawns for an agent gets.
|
|
41
|
+
*
|
|
42
|
+
* 10 rather than 19: the point is to lose to the daemon and to anything the
|
|
43
|
+
* owner is doing by hand, not to be scheduled last behind every background cron
|
|
44
|
+
* on the box. The scheduler's weight table gives nice 10 about a ninth of the
|
|
45
|
+
* share of nice 0 under contention (1024 → 110), which is all the room the
|
|
46
|
+
* heartbeat needs — 19 would buy an order of magnitude more and cost a session
|
|
47
|
+
* its throughput whenever anything else on the machine woke up.
|
|
48
|
+
*/
|
|
49
|
+
export const NICE = 10;
|
|
50
|
+
/**
|
|
51
|
+
* EPERM is a property of the machine, not of the process — it means this kernel
|
|
52
|
+
* or container will not let us renice at all, and it will mean that for every
|
|
53
|
+
* spawn afterwards. Said once; the alternative is one warning per agent process
|
|
54
|
+
* for the life of the daemon, which is how a real line gets buried.
|
|
55
|
+
*/
|
|
56
|
+
let permissionWarned = false;
|
|
57
|
+
function errorCode(error) {
|
|
58
|
+
if (typeof error === 'object' && error !== null && 'code' in error) {
|
|
59
|
+
return String(error.code);
|
|
60
|
+
}
|
|
61
|
+
return '';
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Push one spawned process down to {@link NICE}. Never throws.
|
|
65
|
+
*
|
|
66
|
+
* Takes `number | undefined` because that is exactly what `child.pid` is: a
|
|
67
|
+
* spawn that failed has none, and the caller should not have to ask.
|
|
68
|
+
*/
|
|
69
|
+
export function lowerPriority(pid) {
|
|
70
|
+
if (pid === undefined)
|
|
71
|
+
return;
|
|
72
|
+
try {
|
|
73
|
+
os.setPriority(pid, NICE);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
const code = errorCode(error);
|
|
77
|
+
// ESRCH: the child was already gone — a binary that is not there exits
|
|
78
|
+
// before we get to it. Nothing happened and nothing is wrong, so nothing
|
|
79
|
+
// is said; the spawn failure itself is reported by whoever spawned it.
|
|
80
|
+
if (code === 'ESRCH')
|
|
81
|
+
return;
|
|
82
|
+
if (code === 'EPERM') {
|
|
83
|
+
if (permissionWarned)
|
|
84
|
+
return;
|
|
85
|
+
permissionWarned = true;
|
|
86
|
+
log.warn('priority: not allowed to renice agent processes on this machine', {
|
|
87
|
+
nice: NICE,
|
|
88
|
+
error: String(error),
|
|
89
|
+
});
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
log.warn('priority: could not lower a spawned process', {
|
|
93
|
+
pid,
|
|
94
|
+
nice: NICE,
|
|
95
|
+
error: String(error),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=process-priority.js.map
|
package/dist/protocol.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import type { HostLoadFrame } from './host-load.js';
|
|
2
3
|
export declare const SessionDescriptorSchema: z.ZodObject<{
|
|
3
4
|
id: z.ZodString;
|
|
4
5
|
kind: z.ZodEnum<["TICKET", "CHAT"]>;
|
|
@@ -1150,7 +1151,6 @@ export declare const GatewayFrameSchema: z.ZodDiscriminatedUnion<"type", [z.ZodO
|
|
|
1150
1151
|
branchPlan?: unknown;
|
|
1151
1152
|
}>;
|
|
1152
1153
|
}, "strip", z.ZodTypeAny, {
|
|
1153
|
-
type: "session_start";
|
|
1154
1154
|
session: {
|
|
1155
1155
|
mode: "ask" | "plan" | "auto" | "full";
|
|
1156
1156
|
agent: "CLAUDE" | "CODEX";
|
|
@@ -1201,8 +1201,8 @@ export declare const GatewayFrameSchema: z.ZodDiscriminatedUnion<"type", [z.ZodO
|
|
|
1201
1201
|
baseSha?: string | undefined;
|
|
1202
1202
|
} | undefined;
|
|
1203
1203
|
};
|
|
1204
|
-
}, {
|
|
1205
1204
|
type: "session_start";
|
|
1205
|
+
}, {
|
|
1206
1206
|
session: {
|
|
1207
1207
|
agent: "CLAUDE" | "CODEX";
|
|
1208
1208
|
id: string;
|
|
@@ -1248,6 +1248,7 @@ export declare const GatewayFrameSchema: z.ZodDiscriminatedUnion<"type", [z.ZodO
|
|
|
1248
1248
|
branchHint?: unknown;
|
|
1249
1249
|
branchPlan?: unknown;
|
|
1250
1250
|
};
|
|
1251
|
+
type: "session_start";
|
|
1251
1252
|
}>, z.ZodObject<{
|
|
1252
1253
|
type: z.ZodLiteral<"session_message">;
|
|
1253
1254
|
sessionId: z.ZodString;
|
|
@@ -1651,7 +1652,28 @@ export type RunnerFrame = {
|
|
|
1651
1652
|
from: string | null;
|
|
1652
1653
|
to: string;
|
|
1653
1654
|
};
|
|
1654
|
-
}
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* What this machine's own load looks like, right now (plan §5.3).
|
|
1658
|
+
*
|
|
1659
|
+
* A frame of its own rather than a field of `hello`, for the same reason
|
|
1660
|
+
* `agent_versions` is one: `hello` is composed once per process and replayed
|
|
1661
|
+
* on every reconnect, so a load put there would be frozen at daemon start —
|
|
1662
|
+
* a number that is always wrong except in the first second of the machine's
|
|
1663
|
+
* life. This one is measured on a timer and sent only when it moved.
|
|
1664
|
+
*
|
|
1665
|
+
* Nothing static travels here. `machine: {cpuCount, memTotalBytes,
|
|
1666
|
+
* memAvailableBytes}` is already in `hello`, and repeating facts is how two
|
|
1667
|
+
* sources of one truth start disagreeing. `cpuCount` is the single exception
|
|
1668
|
+
* and it earns its place: load1 without it cannot be read as a ratio, and a
|
|
1669
|
+
* consumer joining two frames to find out would eventually paint one
|
|
1670
|
+
* machine's load against another's core count.
|
|
1671
|
+
*
|
|
1672
|
+
* Fields and thresholds mirror `@devbridge/shared` — see `host-load.ts`.
|
|
1673
|
+
*/
|
|
1674
|
+
| ({
|
|
1675
|
+
type: 'host_load';
|
|
1676
|
+
} & HostLoadFrame) | {
|
|
1655
1677
|
type: 'pong';
|
|
1656
1678
|
};
|
|
1657
1679
|
//# sourceMappingURL=protocol.d.ts.map
|
package/dist/recipe-schema.d.ts
CHANGED
|
@@ -50,14 +50,14 @@ export declare const ProjectRecipePreviewSchema: z.ZodObject<{
|
|
|
50
50
|
}, "strip", z.ZodTypeAny, {
|
|
51
51
|
run: string;
|
|
52
52
|
url?: string | undefined;
|
|
53
|
-
project?: string | undefined;
|
|
54
53
|
stop?: string | undefined;
|
|
54
|
+
project?: string | undefined;
|
|
55
55
|
timeoutSec?: number | undefined;
|
|
56
56
|
}, {
|
|
57
57
|
run: string;
|
|
58
58
|
url?: string | undefined;
|
|
59
|
-
project?: string | undefined;
|
|
60
59
|
stop?: string | undefined;
|
|
60
|
+
project?: string | undefined;
|
|
61
61
|
timeoutSec?: number | undefined;
|
|
62
62
|
}>;
|
|
63
63
|
export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
@@ -200,14 +200,14 @@ export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
|
200
200
|
}, "strip", z.ZodTypeAny, {
|
|
201
201
|
run: string;
|
|
202
202
|
url?: string | undefined;
|
|
203
|
-
project?: string | undefined;
|
|
204
203
|
stop?: string | undefined;
|
|
204
|
+
project?: string | undefined;
|
|
205
205
|
timeoutSec?: number | undefined;
|
|
206
206
|
}, {
|
|
207
207
|
run: string;
|
|
208
208
|
url?: string | undefined;
|
|
209
|
-
project?: string | undefined;
|
|
210
209
|
stop?: string | undefined;
|
|
210
|
+
project?: string | undefined;
|
|
211
211
|
timeoutSec?: number | undefined;
|
|
212
212
|
}>>;
|
|
213
213
|
notes: z.ZodOptional<z.ZodString>;
|
|
@@ -243,8 +243,8 @@ export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
|
243
243
|
preview?: {
|
|
244
244
|
run: string;
|
|
245
245
|
url?: string | undefined;
|
|
246
|
-
project?: string | undefined;
|
|
247
246
|
stop?: string | undefined;
|
|
247
|
+
project?: string | undefined;
|
|
248
248
|
timeoutSec?: number | undefined;
|
|
249
249
|
} | undefined;
|
|
250
250
|
notes?: string | undefined;
|
|
@@ -257,8 +257,8 @@ export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
|
257
257
|
preview?: {
|
|
258
258
|
run: string;
|
|
259
259
|
url?: string | undefined;
|
|
260
|
-
project?: string | undefined;
|
|
261
260
|
stop?: string | undefined;
|
|
261
|
+
project?: string | undefined;
|
|
262
262
|
timeoutSec?: number | undefined;
|
|
263
263
|
} | undefined;
|
|
264
264
|
notes?: string | undefined;
|
package/dist/self-update.js
CHANGED
|
@@ -4,6 +4,7 @@ import os from 'node:os';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { promisify } from 'node:util';
|
|
6
6
|
import { findClaudeCli, USE_BUNDLED_CLAUDE } from './agent-binary.js';
|
|
7
|
+
import { systemdUserEnv } from './environment.js';
|
|
7
8
|
import { log } from './log.js';
|
|
8
9
|
import { stateDir } from './paths.js';
|
|
9
10
|
import { RUNNER_VERSION } from './version.js';
|
|
@@ -330,6 +331,8 @@ async function installGlobal(exec, source, prefix) {
|
|
|
330
331
|
* npm needs PATH/HOME and a writable cache; everything else is stripped, both to
|
|
331
332
|
* keep provider credentials out of a child process that touches the network and
|
|
332
333
|
* to stop a stray `npm_config_*` from redirecting the install.
|
|
334
|
+
*
|
|
335
|
+
* NOT for `systemctl` — see `systemctlEnv()` below.
|
|
333
336
|
*/
|
|
334
337
|
function npmEnv() {
|
|
335
338
|
const env = {
|
|
@@ -344,6 +347,44 @@ function npmEnv() {
|
|
|
344
347
|
env['XDG_CACHE_HOME'] = process.env['XDG_CACHE_HOME'];
|
|
345
348
|
return env;
|
|
346
349
|
}
|
|
350
|
+
/**
|
|
351
|
+
* What `systemctl --user` needs, and nothing more.
|
|
352
|
+
*
|
|
353
|
+
* `npmEnv()` was used here too, and that was the whole defect (plan §5.5): it
|
|
354
|
+
* strips `XDG_RUNTIME_DIR`, which is how the client finds this user's D-Bus
|
|
355
|
+
* socket. Without it systemd prints «Failed to connect to bus: No medium found»
|
|
356
|
+
* and exits **0** — so every self-update reported a `daemon-reload` that never
|
|
357
|
+
* happened, and the memory ceiling written into the drop-in a line earlier never
|
|
358
|
+
* came into effect. In the file 7680M, in effect 6.0G, and no error anywhere.
|
|
359
|
+
*
|
|
360
|
+
* The two environments are merged rather than swapped, because each one answers
|
|
361
|
+
* a question the other does not:
|
|
362
|
+
*
|
|
363
|
+
* - `systemdUserEnv()` alone would work, and is what the rest of the runner
|
|
364
|
+
* passes to `systemctl`. But it starts from the full `process.env`, and in
|
|
365
|
+
* the daemon that includes the agent provider keys. This module deliberately
|
|
366
|
+
* hands no child a credential it has no use for (there is a test for exactly
|
|
367
|
+
* that), and `systemctl` has no use for one.
|
|
368
|
+
* - `npmEnv()` alone is the bug.
|
|
369
|
+
*
|
|
370
|
+
* So: the minimal shape of `npmEnv()` — PATH to find the binary, HOME for
|
|
371
|
+
* completeness — plus the two bus variables `systemdUserEnv()` resolves. The
|
|
372
|
+
* `npm_config_*` half is left out; it means nothing to systemctl.
|
|
373
|
+
*/
|
|
374
|
+
function systemctlEnv() {
|
|
375
|
+
const resolved = systemdUserEnv();
|
|
376
|
+
const env = {
|
|
377
|
+
PATH: process.env['PATH'] ?? '/usr/local/bin:/usr/bin:/bin',
|
|
378
|
+
HOME: process.env['HOME'] ?? '',
|
|
379
|
+
};
|
|
380
|
+
const runtimeDir = resolved['XDG_RUNTIME_DIR'];
|
|
381
|
+
if (runtimeDir)
|
|
382
|
+
env['XDG_RUNTIME_DIR'] = runtimeDir;
|
|
383
|
+
const busAddress = resolved['DBUS_SESSION_BUS_ADDRESS'];
|
|
384
|
+
if (busAddress)
|
|
385
|
+
env['DBUS_SESSION_BUS_ADDRESS'] = busAddress;
|
|
386
|
+
return env;
|
|
387
|
+
}
|
|
347
388
|
export async function selfUpdate(options) {
|
|
348
389
|
const exec = options.exec ??
|
|
349
390
|
((file, args, opts) => execFileAsync(file, args, { timeout: opts.timeout, env: opts.env, maxBuffer: 4_000_000 }));
|
|
@@ -501,7 +542,7 @@ export async function selfUpdate(options) {
|
|
|
501
542
|
fs.writeFileSync(unitPath(), buildUnit(installed.command));
|
|
502
543
|
await exec('systemctl', ['--user', 'daemon-reload'], {
|
|
503
544
|
timeout: VERIFY_TIMEOUT_MS,
|
|
504
|
-
env:
|
|
545
|
+
env: systemctlEnv(),
|
|
505
546
|
});
|
|
506
547
|
log.warn('self-update: the service unit pointed at the previous location — rewritten', {
|
|
507
548
|
execStart: installed.command,
|
|
@@ -525,7 +566,7 @@ export async function selfUpdate(options) {
|
|
|
525
566
|
if ((options.writeLimits ?? writeLimitsOverride)()) {
|
|
526
567
|
await exec('systemctl', ['--user', 'daemon-reload'], {
|
|
527
568
|
timeout: VERIFY_TIMEOUT_MS,
|
|
528
|
-
env:
|
|
569
|
+
env: systemctlEnv(),
|
|
529
570
|
});
|
|
530
571
|
log.warn('self-update: resource limits drop-in written', {
|
|
531
572
|
path: limitsOverridePath(),
|
package/dist/service-unit.d.ts
CHANGED
|
@@ -57,8 +57,45 @@ export declare function buildUnit(execStart?: string, nodeBinary?: string): stri
|
|
|
57
57
|
* which is the only way to fix the servers that already have the bad numbers
|
|
58
58
|
* baked in — and it never overwrites a unit the operator edited by hand.
|
|
59
59
|
*/
|
|
60
|
-
export declare const LIMITS_VERSION =
|
|
60
|
+
export declare const LIMITS_VERSION = 5;
|
|
61
|
+
/**
|
|
62
|
+
* Where the agent sessions live once they have a cage of their own.
|
|
63
|
+
*
|
|
64
|
+
* A `systemd-run --scope` is a SIBLING of the service, not a child of it: a
|
|
65
|
+
* session started that way leaves the service's `MemoryMax`, `CPUQuota`,
|
|
66
|
+
* `OOMPolicy=continue` and `KillMode=control-group` behind entirely. So the
|
|
67
|
+
* collective ceiling has to move with them, onto the slice — otherwise the cage
|
|
68
|
+
* per session would arrive at the price of the ceiling over all of them.
|
|
69
|
+
*
|
|
70
|
+
* The dash is systemd's hierarchy separator: `devbridge-sessions.slice` is a
|
|
71
|
+
* child of `devbridge.slice`, which is where the CPU share for everything the
|
|
72
|
+
* agents run is set.
|
|
73
|
+
*/
|
|
74
|
+
export declare const SESSIONS_SLICE = "devbridge-sessions.slice";
|
|
75
|
+
export declare const DEVBRIDGE_SLICE = "devbridge.slice";
|
|
76
|
+
/**
|
|
77
|
+
* Half the default weight, on `devbridge.slice` and on every session scope.
|
|
78
|
+
*
|
|
79
|
+
* Lives here rather than in `session-cage.ts` because the number has to be the
|
|
80
|
+
* SAME in both places — the slice sets the share of the agents against the
|
|
81
|
+
* daemon, the scope sets one session's share against another's — and a
|
|
82
|
+
* duplicated literal is how those two drift apart. Why 50 and what it replaces:
|
|
83
|
+
* `buildDevbridgeSliceOverride` below.
|
|
84
|
+
*/
|
|
85
|
+
export declare const SESSION_CPU_WEIGHT = 50;
|
|
61
86
|
export declare function limitsOverridePath(home?: string): string;
|
|
87
|
+
/**
|
|
88
|
+
* Drop-in path for a slice unit that has no unit FILE at all.
|
|
89
|
+
*
|
|
90
|
+
* systemd synthesises `devbridge-sessions.slice` the first time something asks
|
|
91
|
+
* for it, and it reads drop-ins for the synthesised unit exactly as for a real
|
|
92
|
+
* one — verified on this machine: a `[Slice] MemoryMax=64M` drop-in with no
|
|
93
|
+
* fragment gave `MemoryMax=67108864` on the live slice. So there is no unit file
|
|
94
|
+
* to write and no unit file to keep in sync; the policy is the drop-in.
|
|
95
|
+
*/
|
|
96
|
+
export declare function sliceOverridePath(slice: string, home?: string): string;
|
|
97
|
+
export declare function sessionsSliceOverridePath(home?: string): string;
|
|
98
|
+
export declare function devbridgeSliceOverridePath(home?: string): string;
|
|
62
99
|
/**
|
|
63
100
|
* What the memory policy needs to know about this machine. Read once and passed
|
|
64
101
|
* in, so the policy itself is a pure function that a test can drive with the
|
|
@@ -76,12 +113,68 @@ export interface MemoryFacts {
|
|
|
76
113
|
* as the floor under the ceiling; see `memoryPolicy`.
|
|
77
114
|
*/
|
|
78
115
|
ownUsageBytes: number;
|
|
116
|
+
/**
|
|
117
|
+
* What a LOWER ceiling could not reclaim its way out of — the number the
|
|
118
|
+
* ceiling has to clear, and not the same question as the one above.
|
|
119
|
+
*
|
|
120
|
+
* Equal to `ownUsageBytes` whenever the split is known, which is every path
|
|
121
|
+
* that can read the cgroup's own `memory.stat`. It differs only where the
|
|
122
|
+
* split is unknown — systemd answered `MemoryCurrent` but the files behind it
|
|
123
|
+
* could not be read — and there it is the FULL reading: a ceiling written
|
|
124
|
+
* under an unknown cgroup must assume none of it can be given back.
|
|
125
|
+
*
|
|
126
|
+
* Page cache is deliberately NOT part of it when the split IS known: lowering
|
|
127
|
+
* `MemoryMax` under clean file pages makes the kernel reclaim them, not kill
|
|
128
|
+
* anything, so counting them would inflate every ceiling by whatever the
|
|
129
|
+
* machine happened to have cached (measured: +87 % on this host) — and the
|
|
130
|
+
* floor is applied AFTER the 85 %-of-total cap, so that inflation walks the
|
|
131
|
+
* ceiling straight past the cap this policy exists to enforce.
|
|
132
|
+
*/
|
|
133
|
+
ownFloorBytes?: number;
|
|
134
|
+
/**
|
|
135
|
+
* The same reading for `devbridge-sessions.slice`, and the reason it is a
|
|
136
|
+
* separate number rather than part of the one above.
|
|
137
|
+
*
|
|
138
|
+
* Since 0.54.0 the agents live in that slice, NOT in the service's cgroup, and
|
|
139
|
+
* the ceiling this policy computes is written to both units. So both cgroups
|
|
140
|
+
* are invisible to `MemAvailable` and both have to survive the write — a
|
|
141
|
+
* policy that measures only the daemon computes a ceiling from a machine it
|
|
142
|
+
* cannot see and then applies it to one it can kill (QA-2026-09-07 BLOCKER-1:
|
|
143
|
+
* the hourly re-measure wrote 2 GiB onto a slice holding 6 GiB, with
|
|
144
|
+
* `MemorySwapMax=0`, which is every session on the machine).
|
|
145
|
+
*
|
|
146
|
+
* 0 is the honest answer on a machine where the cage never took: there the
|
|
147
|
+
* sessions are still inside the service's cgroup and `ownUsageBytes` already
|
|
148
|
+
* counts them.
|
|
149
|
+
*/
|
|
150
|
+
sessionsUsageBytes: number;
|
|
151
|
+
/** The same distinction as {@link MemoryFacts.ownFloorBytes}, for the slice. */
|
|
152
|
+
sessionsFloorBytes?: number;
|
|
79
153
|
/** Seconds since boot. Below `BOOT_SETTLE_SEC` the measurement is a lie. */
|
|
80
154
|
uptimeSec: number;
|
|
155
|
+
/**
|
|
156
|
+
* `SwapTotal` — what the sessions' swap share is cut from (#387). Absent or 0
|
|
157
|
+
* means a machine without swap, and then the share is 0: the brake on such a
|
|
158
|
+
* machine can only drop page cache, and the wall is what stops a runaway.
|
|
159
|
+
*/
|
|
160
|
+
swapTotalBytes?: number;
|
|
81
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* How much of the machine's swap ALL sessions together may use (#387).
|
|
164
|
+
*
|
|
165
|
+
* Half, not all: swap belongs to the machine (the spike's 200 MB cage drained
|
|
166
|
+
* the host's 2 GB and the box hung), so the other half stays with the neighbours
|
|
167
|
+
* and the kernel. Each session gets half of this again (`sessionSwapMaxBytes`).
|
|
168
|
+
* On the 2 GB swap this machine has today that is 512 MB per session — enough
|
|
169
|
+
* for a build's overshoot, not for a runaway; the owner's note on #387 is that
|
|
170
|
+
* the swap itself has to grow for the brake to have real room.
|
|
171
|
+
*/
|
|
172
|
+
export declare const SESSIONS_SWAP_SHARE = 0.5;
|
|
82
173
|
export interface MemoryPolicy {
|
|
83
174
|
maxBytes: number;
|
|
84
175
|
highBytes: number;
|
|
176
|
+
/** `MemorySwapMax` for the sessions slice: `SESSIONS_SWAP_SHARE` of `SwapTotal`. */
|
|
177
|
+
swapMaxBytes: number;
|
|
85
178
|
/** false = the conservative static pair, because the machine was still booting. */
|
|
86
179
|
measured: boolean;
|
|
87
180
|
/**
|
|
@@ -102,6 +195,8 @@ export interface MemoryPolicy {
|
|
|
102
195
|
* produces a ceiling that protects nothing.
|
|
103
196
|
*/
|
|
104
197
|
export declare const BOOT_SETTLE_SEC = 600;
|
|
198
|
+
/** What the ceiling about to be written has to clear, in either cgroup. */
|
|
199
|
+
export declare function managedUsageFloorBytes(facts: MemoryFacts): number;
|
|
105
200
|
/**
|
|
106
201
|
* The two numbers, and the incident that decides them.
|
|
107
202
|
*
|
|
@@ -122,8 +217,10 @@ export declare const BOOT_SETTLE_SEC = 600;
|
|
|
122
217
|
*
|
|
123
218
|
* So the percentage has to be of what the machine can SPARE, not of what it has:
|
|
124
219
|
*
|
|
125
|
-
* headroom = MemAvailable +
|
|
126
|
-
* rewrite would walk the ceiling down by what we already hold
|
|
220
|
+
* headroom = MemAvailable + everything WE hold (ours is added back, or every
|
|
221
|
+
* rewrite would walk the ceiling down by what we already hold —
|
|
222
|
+
* and since 0.54.0 «ours» is the daemon's cgroup PLUS the sessions
|
|
223
|
+
* slice, because the agents moved out of the daemon's one)
|
|
127
224
|
* MemoryMax = headroom − reserve
|
|
128
225
|
* MemoryHigh = 80 % of MemoryMax (reclaim and throttle first, kill last)
|
|
129
226
|
*
|
|
@@ -143,6 +240,20 @@ export declare const BOOT_SETTLE_SEC = 600;
|
|
|
143
240
|
* machine, and a ceiling of «everything» is the bug this function exists to fix.
|
|
144
241
|
*/
|
|
145
242
|
export declare function memoryPolicy(facts: MemoryFacts, minCeilingBytes?: number): MemoryPolicy;
|
|
243
|
+
/** `SwapTotal` of this machine, or null where `/proc/meminfo` will not say. */
|
|
244
|
+
export declare function readSwapTotalBytes(): number | null;
|
|
245
|
+
/**
|
|
246
|
+
* What our own cgroup currently holds.
|
|
247
|
+
*
|
|
248
|
+
* Read through `/proc/self/cgroup` rather than assembling the path from the
|
|
249
|
+
* service name: the runner runs as root and as a dedicated user, under
|
|
250
|
+
* `user@0.service` and under `user@1001.service`, and guessing that path wrong
|
|
251
|
+
* silently returns 0 — which would quietly shrink the ceiling by whatever we are
|
|
252
|
+
* already using. Returns 0 on cgroup v1 or in a container without the file, which
|
|
253
|
+
* is the safe direction: a slightly lower ceiling, never a higher one.
|
|
254
|
+
*/
|
|
255
|
+
export declare function readSelfCgroup(): string | null;
|
|
256
|
+
export declare function readOwnCgroupMemory(): CgroupMemory | null;
|
|
146
257
|
/**
|
|
147
258
|
* What the cgroup holds that `MemAvailable` has NOT already counted.
|
|
148
259
|
*
|
|
@@ -154,17 +265,92 @@ export declare function memoryPolicy(facts: MemoryFacts, minCeilingBytes?: numbe
|
|
|
154
265
|
* precisely when memory is tightest. Subtracting `file` keeps the part we really
|
|
155
266
|
* do hold and cannot give back on demand.
|
|
156
267
|
*/
|
|
268
|
+
export interface CgroupMemory {
|
|
269
|
+
/** Everything the cgroup holds, page cache included. */
|
|
270
|
+
currentBytes: number;
|
|
271
|
+
/**
|
|
272
|
+
* The part of it a lower ceiling could not reclaim its way out of, or null
|
|
273
|
+
* when only the total is known — systemd answered `MemoryCurrent` but the
|
|
274
|
+
* cgroup's own files could not be read. Null is «unknown», never «zero»:
|
|
275
|
+
* the two lead to different ceilings, and only one of them is safe.
|
|
276
|
+
*/
|
|
277
|
+
unreclaimableBytes: number | null;
|
|
278
|
+
}
|
|
279
|
+
export declare function readCgroupMemory(dir: string): CgroupMemory | null;
|
|
157
280
|
export declare function readCgroupUnreclaimable(dir: string): number | null;
|
|
281
|
+
/**
|
|
282
|
+
* Where a slice unit's cgroup lives, worked out from the cgroup we are in.
|
|
283
|
+
*
|
|
284
|
+
* systemd's dash rule spells the hierarchy out: `devbridge-sessions.slice` sits
|
|
285
|
+
* inside `devbridge.slice`, which sits directly under the user manager's own
|
|
286
|
+
* cgroup — verified on this host with a throwaway `--slice=dbqa-probe-sub.slice`,
|
|
287
|
+
* which landed in `user@0.service/dbqa.slice/dbqa-probe.slice/dbqa-probe-sub.slice`.
|
|
288
|
+
*
|
|
289
|
+
* The user manager's cgroup is found rather than assembled: the runner runs as
|
|
290
|
+
* root and as a dedicated user (`user@0.service`, `user@1001.service`), and the
|
|
291
|
+
* same guess-the-path mistake that `readOwnCgroupUsage` avoids would silently
|
|
292
|
+
* return 0 here — which is exactly the blindness BLOCKER-1 was.
|
|
293
|
+
*
|
|
294
|
+
* Null means «there is no user manager above us», and then there is no
|
|
295
|
+
* `--user` slice for the sessions to be in either.
|
|
296
|
+
*/
|
|
297
|
+
export declare function sliceCgroupPath(selfCgroup: string, slice: string): string | null;
|
|
298
|
+
/**
|
|
299
|
+
* What the agents are holding right now, outside the daemon's own cgroup — or
|
|
300
|
+
* null where this machine cannot say.
|
|
301
|
+
*
|
|
302
|
+
* Read from the filesystem rather than through `systemctl show`, because this is
|
|
303
|
+
* the hourly path inside the daemon and it has to stay synchronous — the same
|
|
304
|
+
* reason `readOwnCgroupUsage` reads `/proc`. Callers that are not the daemon can
|
|
305
|
+
* pass the number in; see {@link readMemoryFacts}.
|
|
306
|
+
*
|
|
307
|
+
* A missing directory is 0 and not «unknown»: systemd removes the cgroup of an
|
|
308
|
+
* empty slice, so «no directory» means «no session is holding anything». The two
|
|
309
|
+
* are kept apart because they now lead to opposite decisions — 0 lets a blind
|
|
310
|
+
* ceiling be written onto the slice, «unknown» forbids it (see
|
|
311
|
+
* {@link buildSessionsSliceOverride}).
|
|
312
|
+
*/
|
|
313
|
+
export declare function readSessionsSliceMemory(): CgroupMemory | null;
|
|
314
|
+
/**
|
|
315
|
+
* The same tri-state, read out of `systemctl show` instead of the filesystem —
|
|
316
|
+
* the authoritative source for the paths a person types (`doctor --fix`,
|
|
317
|
+
* `install-service`), which run outside the daemon's cgroup.
|
|
318
|
+
*
|
|
319
|
+
* `[not set]` is the ambiguous answer and the reason `activeState` is asked for
|
|
320
|
+
* as well: systemd prints it for a slice that has no cgroup (nothing has ever
|
|
321
|
+
* run there) AND for one whose accounting is off, and those two must not lead to
|
|
322
|
+
* the same decision. Only «systemd loaded the unit and it is not even active» is
|
|
323
|
+
* a positive statement that nothing can be killed by what we write; everything
|
|
324
|
+
* else is «unknown», which writes no ceiling at all.
|
|
325
|
+
*
|
|
326
|
+
* A failed `systemctl` call is null on both counts — including the 10-second
|
|
327
|
+
* timeout, which fires exactly on the overloaded machine this policy protects.
|
|
328
|
+
*/
|
|
329
|
+
export declare function parseSliceUsage(memoryCurrent: string | null, activeState: string | null): number | null;
|
|
330
|
+
/**
|
|
331
|
+
* The slice's number for the callers that take one, and «unknown» kept apart
|
|
332
|
+
* from «empty»: an unreadable live slice must never be replaced with zero.
|
|
333
|
+
*
|
|
334
|
+
* The FLOOR reading, not the headroom one — this feeds a ceiling that has to
|
|
335
|
+
* clear what the slice holds, and where the split is unknown the whole reading
|
|
336
|
+
* has to be assumed unreclaimable.
|
|
337
|
+
*/
|
|
338
|
+
export declare function readSessionsSliceUsageOrNull(): number | null;
|
|
158
339
|
/**
|
|
159
340
|
* Everything `memoryPolicy` needs, straight off this machine.
|
|
160
341
|
*
|
|
161
|
-
*
|
|
162
|
-
* `install-service` run in the operator's own
|
|
163
|
-
* service's usage from `/proc/self
|
|
164
|
-
*
|
|
165
|
-
*
|
|
342
|
+
* Both readings are passed in by callers that are not the daemon — `doctor` and
|
|
343
|
+
* `install-service` run in the operator's own `session-N.scope` and cannot read
|
|
344
|
+
* the service's usage from `/proc/self`; they ask systemd instead
|
|
345
|
+
* (`readMemoryFactsFromSystemd`). Returns null when either cgroup is unknowable,
|
|
346
|
+
* because guessing there is the one dangerous direction: it removes the floor
|
|
347
|
+
* that stops a live session from being killed on the next `daemon-reload`.
|
|
348
|
+
*
|
|
349
|
+
* The default reads both from the filesystem, which is right for the daemon:
|
|
350
|
+
* its own cgroup through `/proc/self`, and the sessions slice through the user
|
|
351
|
+
* manager's cgroup, which sits above the daemon and the CLI alike.
|
|
166
352
|
*/
|
|
167
|
-
export declare function readMemoryFacts(
|
|
353
|
+
export declare function readMemoryFacts(own?: CgroupMemory | null, sessions?: CgroupMemory | null): MemoryFacts | null;
|
|
168
354
|
/**
|
|
169
355
|
* `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
|
|
170
356
|
* box unreachable, but never so little that ordinary work is throttled.
|
|
@@ -176,6 +362,78 @@ export declare function readMemoryFacts(ownUsageBytes?: number | null): MemoryFa
|
|
|
176
362
|
*/
|
|
177
363
|
export declare function cpuQuotaPercent(cpuCount?: number): number | null;
|
|
178
364
|
export declare function buildLimitsOverride(cpuCount?: number, facts?: MemoryFacts | null): string;
|
|
365
|
+
/**
|
|
366
|
+
* The ceiling over ALL sessions, on the slice they were moved into.
|
|
367
|
+
*
|
|
368
|
+
* Same number as the service's, and deliberately so — but it is a COPY, not a
|
|
369
|
+
* move, and that is deliberate too. The plan asked for the ceiling to be carried
|
|
370
|
+
* across; shrinking the service's to «what the daemon alone needs» would be
|
|
371
|
+
* correct only on a machine where the cage actually took. On cgroup v1, without
|
|
372
|
+
* a user bus, under a foreign supervisor — every `nice-only` machine — the
|
|
373
|
+
* sessions are still CHILDREN of the service, and a service ceiling sized for
|
|
374
|
+
* the daemon would cap all of them at a few hundred MB. This file is written
|
|
375
|
+
* before anything has probed which of the two machines this is, so the safe
|
|
376
|
+
* shape is the same number twice: on a caged machine the slice is the ceiling
|
|
377
|
+
* that binds, on an uncaged one the service is, and neither machine is ever
|
|
378
|
+
* left with a ceiling that is too small for what is under it.
|
|
379
|
+
*
|
|
380
|
+
* The price is that a caged machine formally permits `service + slice`. It is
|
|
381
|
+
* not the guarantee `memoryPolicy` computes, and it is written down here rather
|
|
382
|
+
* than glossed over (QA-2026-09-07 MINOR-4).
|
|
383
|
+
*
|
|
384
|
+
* `MemorySwapMax` is a BOUND on the slice for the same reason it is on every
|
|
385
|
+
* scope: a ceiling on resident memory alone is not a ceiling, it is a swap pump
|
|
386
|
+
* (a 200 MB cage allocated 2 GB and drained the host's swap during the spike).
|
|
387
|
+
* Since #387 the bound is a share of the machine's swap rather than 0
|
|
388
|
+
* (`SESSIONS_SWAP_SHARE`), because each session now carries a `MemoryHigh`
|
|
389
|
+
* brake, and a brake with no swap under it stalls the session instead of
|
|
390
|
+
* slowing it. The unmeasured branch still writes 0: no measurement, no swap to
|
|
391
|
+
* hand out.
|
|
392
|
+
*
|
|
393
|
+
* No `MemoryHigh` here. The brake belongs on each SCOPE (`session-cage.ts`):
|
|
394
|
+
* soft pressure on the slice would throttle every session on the machine to
|
|
395
|
+
* keep one runaway alive a little longer.
|
|
396
|
+
*
|
|
397
|
+
* `sessionsUsageBytes` is the door that used to lead around all of the above.
|
|
398
|
+
* `facts` is null whenever the SERVICE's `MemoryCurrent` is unreadable — a
|
|
399
|
+
* stopped service, or a runner under a foreign supervisor — and this file then
|
|
400
|
+
* fell back to a flat `MemoryMax=55%`. But the sessions are SIBLINGS of the
|
|
401
|
+
* service, not its children: they survive `systemctl --user stop
|
|
402
|
+
* devbridge-runner`, so «the service is not running» says nothing at all about
|
|
403
|
+
* what the slice is holding, and `doctor --fix` on such a machine wrote 55 % of
|
|
404
|
+
* total onto a live slice and then called `daemon-reload` — the collective kill
|
|
405
|
+
* of QA-2026-09-07 BLOCKER-1 arriving through a door with no policy behind it.
|
|
406
|
+
*
|
|
407
|
+
* So the slice's own usage is read separately, and the promise made on
|
|
408
|
+
* `buildLimitsOverride` («the floor belongs to the policy, so every caller gets
|
|
409
|
+
* it and none can opt out») holds on this path too:
|
|
410
|
+
* - a number → the ceiling clears it by the same 1.25 the policy uses, and
|
|
411
|
+
* never drops below what one agent needs;
|
|
412
|
+
* - 0 → nothing is running there, so the blind fraction can kill
|
|
413
|
+
* nothing and stays;
|
|
414
|
+
* - null → this machine could not say, and NO ceiling is written at all.
|
|
415
|
+
* Leaving whatever is in force in force is strictly better than
|
|
416
|
+
* applying an unfounded number to a cgroup that may be full: the
|
|
417
|
+
* daemon rewrites the file with a measured ceiling the moment it
|
|
418
|
+
* can measure one (the drift check treats a file with no
|
|
419
|
+
* `MemoryMax` as outdated).
|
|
420
|
+
*/
|
|
421
|
+
export declare function buildSessionsSliceOverride(facts?: MemoryFacts | null, sessionsUsageBytes?: number | null): string;
|
|
422
|
+
/**
|
|
423
|
+
* The CPU share of everything the agents run, against the daemon's own.
|
|
424
|
+
*
|
|
425
|
+
* `nice(2)` orders tasks INSIDE one cgroup. The moment a session gets a scope of
|
|
426
|
+
* its own it is no longer inside the service's cgroup, and the split between the
|
|
427
|
+
* two is decided by `cpu.weight` — which is 100 everywhere by default,
|
|
428
|
+
* `app.slice` (where the service lives) included. Without this file the cage
|
|
429
|
+
* would silently undo stage 1a and hand back the failure of 16.08: the daemon
|
|
430
|
+
* starved by its own children, four missed heartbeats, the server Offline and
|
|
431
|
+
* 504 on every session. 50 against 100 leaves the daemon two thirds.
|
|
432
|
+
*
|
|
433
|
+
* `process-priority.ts` stays exactly as it is: it is what protects the daemon
|
|
434
|
+
* on cgroup v1 and on every machine where the cage does not apply.
|
|
435
|
+
*/
|
|
436
|
+
export declare function buildDevbridgeSliceOverride(): string;
|
|
179
437
|
/**
|
|
180
438
|
* Is the shipped resource policy missing or from an older runner?
|
|
181
439
|
*
|
|
@@ -195,7 +453,7 @@ export declare function limitsOverrideIsOutdated(readFile?: (p: string) => strin
|
|
|
195
453
|
* arrive at the same number — otherwise a write whose floor was binding would be
|
|
196
454
|
* seen as drifted on the very next call and rewritten forever.
|
|
197
455
|
*/
|
|
198
|
-
export declare function writeLimitsOverride(force?: boolean, home?: string, facts?: MemoryFacts | null): boolean;
|
|
456
|
+
export declare function writeLimitsOverride(force?: boolean, home?: string, facts?: MemoryFacts | null, sessionsUsageBytes?: number | null): boolean;
|
|
199
457
|
/**
|
|
200
458
|
* Does the installed unit point at something that no longer exists?
|
|
201
459
|
*
|