@bridge4dev/runner 0.53.0 → 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +96 -1
- package/dist/adapters/codex-protocol.d.ts +11 -0
- package/dist/adapters/codex-protocol.js +41 -3
- package/dist/adapters/codex.js +4 -0
- package/dist/host-load.d.ts +156 -0
- package/dist/host-load.js +223 -0
- package/dist/index.js +190 -40
- package/dist/process-priority.d.ts +55 -0
- package/dist/process-priority.js +99 -0
- package/dist/protocol.d.ts +25 -3
- package/dist/recipe-schema.d.ts +6 -6
- package/dist/self-update.js +43 -2
- package/dist/service-unit.d.ts +232 -10
- package/dist/service-unit.js +372 -43
- package/dist/session-cage.d.ts +297 -0
- package/dist/session-cage.js +755 -0
- package/dist/supervisor.d.ts +57 -0
- package/dist/supervisor.js +72 -0
- package/dist/systemd-memory.d.ts +35 -0
- package/dist/systemd-memory.js +115 -0
- package/dist/verify.js +28 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -17,12 +17,14 @@ import { Supervisor } from './supervisor.js';
|
|
|
17
17
|
import { readStatusFile, isPidAlive, writeStatusFile, STATUS_FRESH_MS } from './status-file.js';
|
|
18
18
|
import { RunnerWsClient } from './ws-client.js';
|
|
19
19
|
import { RUNNER_VERSION } from './version.js';
|
|
20
|
-
import { buildUnit, cpuQuotaPercent, limitsOverrideIsOutdated, limitsOverridePath, memoryPolicy, readMemoryFacts, unitExecTarget, unitPath, writeLimitsOverride, LIMITS_VERSION, } from './service-unit.js';
|
|
20
|
+
import { buildUnit, cpuQuotaPercent, devbridgeSliceOverridePath, limitsOverrideIsOutdated, limitsOverridePath, memoryPolicy, readMemoryFacts, sessionsSliceOverridePath, unitExecTarget, unitPath, writeLimitsOverride, DEVBRIDGE_SLICE, LIMITS_VERSION, SESSION_CPU_WEIGHT, SESSIONS_SLICE, } from './service-unit.js';
|
|
21
|
+
import { initSessionCage, listSessionScopes, sessionCage, sweepOrphanSessionScopes, SESSION_TASKS_MAX, } from './session-cage.js';
|
|
21
22
|
import { SEARCH_GUARD_ENABLED, claudeSettingsPath, installSearchGuard, removeSearchGuard, searchGuardCommand, searchGuardHome, searchGuardHookPath, searchGuardStatus, } from './claude-settings.js';
|
|
22
23
|
import { readOomKills, recordCrash, takeLastExit } from './crash-note.js';
|
|
23
24
|
import { agentAuthStatuses } from './auth-relay.js';
|
|
24
25
|
import { addSafeDirectory, agentConfigContour, dockerCheck, ensureAgentPath, firstUnreachableAncestor, hasSafeDirectory, inspectPath, knownWorkspacePaths, lingerEnabled, nodeCheck, otherHomeWithAgents, runnerIdentity, safeDirectoryCommand, systemctlHint, systemdUserBusReachable, systemdUserEnv, } from './environment.js';
|
|
25
26
|
import { mcpConfigDir } from './paths.js';
|
|
27
|
+
import { readMemoryFactsFromSystemd } from './systemd-memory.js';
|
|
26
28
|
const execFileAsync = promisify(execFile);
|
|
27
29
|
/**
|
|
28
30
|
* Identity of this process, and the note the previous one left.
|
|
@@ -405,6 +407,37 @@ function runnerCapabilities(apiUrlOverride) {
|
|
|
405
407
|
// Facts passed on purpose: since 0.39.0 «current» also means the measured
|
|
406
408
|
// ceiling still fits this machine, not just that the version matches.
|
|
407
409
|
limitsCurrent: !limitsOverrideIsOutdated(undefined, undefined, readMemoryFacts()),
|
|
410
|
+
/**
|
|
411
|
+
* How the sessions on this machine are actually contained (plan §5.4.3).
|
|
412
|
+
*
|
|
413
|
+
* `scope` — a cgroup and a memory ceiling per session; `nice-only` — the
|
|
414
|
+
* machine cannot do that (cgroup v1, no user bus, a foreign supervisor)
|
|
415
|
+
* and only the CPU priority applies; `none` — not even that.
|
|
416
|
+
*
|
|
417
|
+
* Reported rather than assumed because the containment fails QUIETLY:
|
|
418
|
+
* `systemd-run` accepts every limit on a cgroup v1 machine and applies
|
|
419
|
+
* none of them. Without this field «it did not work at the client's» is
|
|
420
|
+
* something to be guessed at; with it the card can say so.
|
|
421
|
+
*/
|
|
422
|
+
sessionCage: sessionCage().mode,
|
|
423
|
+
// Why not `scope`, in one sentence, for the card to show under it.
|
|
424
|
+
// Empty string when the cage is on.
|
|
425
|
+
sessionCageReason: sessionCage().reason,
|
|
426
|
+
// What ONE session may hold, in bytes. Null when there is no cage —
|
|
427
|
+
// then the only ceiling is the service's, over all of them together.
|
|
428
|
+
sessionMemoryMaxBytes: sessionCage().memoryMaxBytes,
|
|
429
|
+
/**
|
|
430
|
+
* The ceiling over ALL sessions, as systemd has it in force — not as the
|
|
431
|
+
* drop-in on disk says.
|
|
432
|
+
*
|
|
433
|
+
* The live probe proves the personal ceiling and nothing else: a
|
|
434
|
+
* `MemoryMax` on a scope applies whatever the parent slice holds. So a
|
|
435
|
+
* machine whose `daemon-reload` never landed reported `scope` and
|
|
436
|
+
* `limitsCurrent: true` while three sessions of 2.5 GB ran over a slice
|
|
437
|
+
* at `infinity`, and the card said «Sessions capped»
|
|
438
|
+
* (QA-2026-09-07 MAJOR-3). Null is that machine, and the card says so.
|
|
439
|
+
*/
|
|
440
|
+
sessionsMemoryMaxBytes: sessionCage().sessionsSliceMemoryMaxBytes,
|
|
408
441
|
},
|
|
409
442
|
/**
|
|
410
443
|
* Identity of THIS process, so the API can tell a network blink from a
|
|
@@ -482,6 +515,11 @@ async function cmdPair(args) {
|
|
|
482
515
|
}
|
|
483
516
|
const apiUrl = (argValue(args, '--api') ?? 'https://api.bridge4.dev').replace(/\/$/, '');
|
|
484
517
|
const name = argValue(args, '--name') ?? os.hostname();
|
|
518
|
+
// Probed here as well as at daemon start, so the very first record of this
|
|
519
|
+
// server already carries the truth about its containment instead of the safe
|
|
520
|
+
// default. Costs one throwaway process, once.
|
|
521
|
+
await initSessionCage();
|
|
522
|
+
const capabilities = runnerCapabilities(apiUrl);
|
|
485
523
|
const response = await fetch(`${apiUrl}/api/v1/dev/servers/claim`, {
|
|
486
524
|
method: 'POST',
|
|
487
525
|
headers: { 'content-type': 'application/json' },
|
|
@@ -490,7 +528,7 @@ async function cmdPair(args) {
|
|
|
490
528
|
name,
|
|
491
529
|
runnerVersion: RUNNER_VERSION,
|
|
492
530
|
osInfo: `${os.type()} ${os.release()} ${os.arch()}`.slice(0, 200),
|
|
493
|
-
capabilities
|
|
531
|
+
capabilities,
|
|
494
532
|
}),
|
|
495
533
|
});
|
|
496
534
|
const body = (await response.json().catch(() => null));
|
|
@@ -553,8 +591,17 @@ const RESTART_DELAY_MS = 1_500;
|
|
|
553
591
|
*/
|
|
554
592
|
async function repairResourceLimits() {
|
|
555
593
|
try {
|
|
556
|
-
const facts =
|
|
557
|
-
if (facts
|
|
594
|
+
const { facts, sessionsUsageBytes } = await readMemoryFactsFromSystemd();
|
|
595
|
+
if (facts === null) {
|
|
596
|
+
// Neither systemd nor the cgroup files could answer, which on the daemon's
|
|
597
|
+
// own path means the machine is in a shape this policy cannot reason about
|
|
598
|
+
// at all. Leaving what is in force in force is the safe half of the
|
|
599
|
+
// decision; the loud line is the other half, because the alternative is a
|
|
600
|
+
// machine that silently keeps an old ceiling forever.
|
|
601
|
+
log.warn('daemon: resource limits unchanged — memory usage could not be measured; retrying in an hour', { recheckMs: LIMITS_RECHECK_MS });
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
if (memoryPolicy(facts).starved) {
|
|
558
605
|
// The only signal the machine's owner will ever get that the neighbours
|
|
559
606
|
// have taken the box: the honest headroom was below the 2 GB floor, so the
|
|
560
607
|
// ceiling we are about to write sits above what is actually free.
|
|
@@ -563,12 +610,12 @@ async function repairResourceLimits() {
|
|
|
563
610
|
totalMB: Math.round(facts.totalBytes / 1048576),
|
|
564
611
|
});
|
|
565
612
|
}
|
|
566
|
-
if (!writeLimitsOverride(false, undefined, facts))
|
|
613
|
+
if (!writeLimitsOverride(false, undefined, facts, sessionsUsageBytes))
|
|
567
614
|
return;
|
|
568
615
|
log.warn('daemon: resource limits drop-in written — reloading systemd', {
|
|
569
616
|
path: limitsOverridePath(),
|
|
570
617
|
version: LIMITS_VERSION,
|
|
571
|
-
...
|
|
618
|
+
...memoryPolicy(facts),
|
|
572
619
|
});
|
|
573
620
|
await execFileAsync('systemctl', ['--user', 'daemon-reload'], {
|
|
574
621
|
timeout: 15_000,
|
|
@@ -657,33 +704,6 @@ function repairSearchGuard() {
|
|
|
657
704
|
* check writes nothing when the version matches and the ceiling still fits.
|
|
658
705
|
*/
|
|
659
706
|
const LIMITS_RECHECK_MS = 3_600_000;
|
|
660
|
-
/**
|
|
661
|
-
* What the SERVICE's cgroup holds, for the paths that are not the service.
|
|
662
|
-
*
|
|
663
|
-
* `readMemoryFacts()` defaults to reading `/proc/self`, which is right for the
|
|
664
|
-
* daemon and wrong for everything a person types: `doctor --fix` and
|
|
665
|
-
* `install-service` run in the operator's own `session-N.scope`, worth a few MB.
|
|
666
|
-
* Measuring that and calling it «what the runner holds» made the CLI compute a
|
|
667
|
-
* ceiling 34 % away from the daemon's answer on a real host — each path then saw
|
|
668
|
-
* the other as drift and rewrote the file, forever.
|
|
669
|
-
*
|
|
670
|
-
* `MemoryCurrent` is `memory.current`, so it still counts page cache the way
|
|
671
|
-
* `MemAvailable` does. Subtracting it is not worth a second systemd call here:
|
|
672
|
-
* the CLI paths run once, by hand, and erring toward a LOWER ceiling is the safe
|
|
673
|
-
* direction. Returns null when the service is not running or systemd cannot be
|
|
674
|
-
* reached — the caller then has no facts, and writes nothing rather than writing
|
|
675
|
-
* a ceiling with no floor under it.
|
|
676
|
-
*/
|
|
677
|
-
async function serviceMemoryCurrent() {
|
|
678
|
-
try {
|
|
679
|
-
const { stdout } = await execFileAsync('systemctl', ['--user', 'show', 'devbridge-runner', '-p', 'MemoryCurrent', '--value'], { timeout: 10_000, env: systemdUserEnv() });
|
|
680
|
-
const value = Number(stdout.trim());
|
|
681
|
-
return Number.isFinite(value) && value > 0 ? value : null;
|
|
682
|
-
}
|
|
683
|
-
catch {
|
|
684
|
-
return null;
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
707
|
/**
|
|
688
708
|
* Ticket #119. `ClaudeAdapter.stop()` removes each session's MCP config file,
|
|
689
709
|
* but a kill -9, an OOM or a machine reboot leaves it behind — a live project
|
|
@@ -719,7 +739,17 @@ async function cmdDaemon() {
|
|
|
719
739
|
const config = requireConfig();
|
|
720
740
|
log.info('daemon starting', { version: RUNNER_VERSION, server: config.server.name });
|
|
721
741
|
sweepOrphanedMcpConfigs();
|
|
742
|
+
// Nothing of ours is running yet, so every `devbridge-session-*.scope` on this
|
|
743
|
+
// machine belongs to a process that is gone. Stopping the scope takes the
|
|
744
|
+
// whole tree under it — which is the answer to the `ugrep` that outlived its
|
|
745
|
+
// session by 10 h 51 min on 16.08, and to every scope the OOM killer left in
|
|
746
|
+
// `failed` (a name systemd will otherwise refuse to reuse).
|
|
747
|
+
await sweepOrphanSessionScopes();
|
|
722
748
|
await repairResourceLimits();
|
|
749
|
+
// After the drop-ins are on disk and reloaded, never before: the probe below
|
|
750
|
+
// creates `devbridge-sessions.slice`, and a slice first loaded without its
|
|
751
|
+
// policy would hold no ceiling until the next daemon-reload.
|
|
752
|
+
await initSessionCage();
|
|
723
753
|
repairSearchGuard();
|
|
724
754
|
// A Claude token this runner captured through the sign-in relay. Applied
|
|
725
755
|
// BEFORE any adapter exists, because `scrubbedEnv()` copies it out of this
|
|
@@ -892,8 +922,29 @@ async function cmdInstallService() {
|
|
|
892
922
|
// `buildLimitsOverride`. Forced here: a fresh install must have it even if a
|
|
893
923
|
// file from an older runner is already sitting there. Facts come from the
|
|
894
924
|
// service rather than from `/proc/self`, which here is the installing shell.
|
|
895
|
-
|
|
896
|
-
|
|
925
|
+
// The slice reading goes with them: re-installing over a machine that is
|
|
926
|
+
// running sessions right now must not write a ceiling under them.
|
|
927
|
+
//
|
|
928
|
+
// Unmeasurable is a WARNING here, not a refusal, and the order above is why:
|
|
929
|
+
// the unit is already on disk, so the machine gets a runner either way. The
|
|
930
|
+
// states that produce «unmeasurable» — a failed service, a service mid
|
|
931
|
+
// auto-restart, an unhappy user bus — are exactly the states someone runs
|
|
932
|
+
// `install.sh --repair` from, and a repair that installs nothing because it
|
|
933
|
+
// could not size a ceiling is a repair that does not repair.
|
|
934
|
+
const installMem = await readMemoryFactsFromSystemd();
|
|
935
|
+
if (installMem.facts === null) {
|
|
936
|
+
print('warning: could not measure this machine — resource limits were left as they are. ' +
|
|
937
|
+
'Nothing was removed; the daemon re-measures hourly, or run `devbridge-runner doctor --fix`.');
|
|
938
|
+
}
|
|
939
|
+
else {
|
|
940
|
+
writeLimitsOverride(true, undefined, installMem.facts, installMem.sessionsUsageBytes);
|
|
941
|
+
print(`Wrote ${limitsOverridePath()}`);
|
|
942
|
+
// Two more files since 0.54.0: the sessions no longer live inside the
|
|
943
|
+
// service's cgroup, so the ceiling over all of them and the CPU share that
|
|
944
|
+
// keeps the daemon ahead of them belong to the slices (`session-cage.ts`).
|
|
945
|
+
print(`Wrote ${sessionsSliceOverridePath()}`);
|
|
946
|
+
print(`Wrote ${devbridgeSliceOverridePath()}`);
|
|
947
|
+
}
|
|
897
948
|
// Привратник шаблонов поиска — та же логика, что и у политики ресурсов:
|
|
898
949
|
// свежая установка должна получить его сразу. Пишем в ЧУЖОЙ файл настроек,
|
|
899
950
|
// поэтому молча пропускаем, если тронуть его нельзя (`claude-settings.ts`).
|
|
@@ -1330,7 +1381,7 @@ async function cmdDoctor(args) {
|
|
|
1330
1381
|
print(` fits sessions ~${advised} (at ~1.5 GB per session under load)`);
|
|
1331
1382
|
print('');
|
|
1332
1383
|
print('Service limits');
|
|
1333
|
-
const memFacts =
|
|
1384
|
+
const { facts: memFacts, sessionsUsageBytes: memSessionsUsage } = await readMemoryFactsFromSystemd();
|
|
1334
1385
|
const outdated = limitsOverrideIsOutdated(undefined, undefined, memFacts);
|
|
1335
1386
|
print(` drop-in ${limitsOverridePath()}`);
|
|
1336
1387
|
print(` version ${outdated ? `MISSING or OLD (want ${LIMITS_VERSION})` : LIMITS_VERSION}`);
|
|
@@ -1346,6 +1397,62 @@ async function cmdDoctor(args) {
|
|
|
1346
1397
|
? `${mb(memFacts.availableBytes)} available + ${mb(memFacts.ownUsageBytes)} ours`
|
|
1347
1398
|
: `55% of ${mb(memFacts.totalBytes)} — still booting, remeasured on the next start`}`);
|
|
1348
1399
|
}
|
|
1400
|
+
else {
|
|
1401
|
+
// Не молча: без фактов о машине политика не считается вовсе, и `--fix`
|
|
1402
|
+
// тогда НИЧЕГО не пишет — ни на сервис, ни на слайс. Строка про слайс
|
|
1403
|
+
// остаётся полезной сама по себе: она говорит, что стоит на кону, если
|
|
1404
|
+
// измерение так и не появится.
|
|
1405
|
+
print(" memory ceiling not computed — this machine's usage is unreadable");
|
|
1406
|
+
print(' --fix leaves every limits file exactly as it is');
|
|
1407
|
+
print(` sessions slice ${memSessionsUsage === null
|
|
1408
|
+
? 'unreadable too'
|
|
1409
|
+
: memSessionsUsage === 0
|
|
1410
|
+
? 'holds nothing right now'
|
|
1411
|
+
: `holds ${Math.round(memSessionsUsage / 1024 / 1024)} MB — a ceiling under that would kill it`}`);
|
|
1412
|
+
}
|
|
1413
|
+
// The cage, and the reason it matters more than the numbers above it: those
|
|
1414
|
+
// are the ceiling over the WHOLE service, and a session that has left the
|
|
1415
|
+
// service's cgroup is not under them any more. If this section says anything
|
|
1416
|
+
// but `scope`, the only containment on this machine is the CPU priority.
|
|
1417
|
+
const cage = await initSessionCage();
|
|
1418
|
+
print('');
|
|
1419
|
+
print('Session cage (a cgroup per session)');
|
|
1420
|
+
print(` mode ${cage.mode}`);
|
|
1421
|
+
if (cage.reason)
|
|
1422
|
+
print(` why ${cage.reason}`);
|
|
1423
|
+
print(` slice ${SESSIONS_SLICE} (under ${DEVBRIDGE_SLICE}, CPUWeight=${SESSION_CPU_WEIGHT})`);
|
|
1424
|
+
print(` drop-ins ${sessionsSliceOverridePath()}`);
|
|
1425
|
+
print(` ${devbridgeSliceOverridePath()}`);
|
|
1426
|
+
if (cage.mode === 'scope') {
|
|
1427
|
+
const mb = (bytes) => bytes === null ? 'infinity' : `${Math.round(bytes / 1024 / 1024)} MB`;
|
|
1428
|
+
print(` per session MemoryMax ${mb(cage.memoryMaxBytes)}, MemorySwapMax 0, TasksMax ${SESSION_TASKS_MAX}`);
|
|
1429
|
+
// What systemd has IN FORCE on the slice, beside the paths of the files
|
|
1430
|
+
// that were supposed to put it there. «В файле 7680M, действует 6.0G» is the
|
|
1431
|
+
// lesson of §5.5, and the drop-in of a slice can fail to land in exactly the
|
|
1432
|
+
// same way: a reload that never happened leaves every session personally
|
|
1433
|
+
// capped and collectively unlimited (QA-2026-09-07 MAJOR-3).
|
|
1434
|
+
print(` all sessions MemoryMax ${mb(cage.sessionsSliceMemoryMaxBytes)} in force on ${SESSIONS_SLICE}`);
|
|
1435
|
+
if (cage.sessionsSliceMemoryMaxBytes === null) {
|
|
1436
|
+
print(' NOT CAPPED — the drop-in above has not been applied.');
|
|
1437
|
+
print(` Apply with: ${systemctlHint('daemon-reload')}`);
|
|
1438
|
+
}
|
|
1439
|
+
print(` service ceiling ${mb(cage.serviceMemoryMaxBytes)} (the per-session figure is half of the containing ceiling, capped)`);
|
|
1440
|
+
print(` expand-env flag ${cage.expandEnvironmentFlag ? 'passed (systemd ≥ 254)' : 'not passed — this systemd does not know it, and --scope does not expand anyway'}`);
|
|
1441
|
+
}
|
|
1442
|
+
else {
|
|
1443
|
+
print(' per session none — only the CPU priority from `process-priority.ts` applies');
|
|
1444
|
+
}
|
|
1445
|
+
const scopes = await listSessionScopes();
|
|
1446
|
+
if (scopes.length === 0) {
|
|
1447
|
+
print(' live scopes none');
|
|
1448
|
+
}
|
|
1449
|
+
else {
|
|
1450
|
+
print(` live scopes ${scopes.length}`);
|
|
1451
|
+
for (const scope of scopes) {
|
|
1452
|
+
const mb = (bytes) => bytes === null ? '?' : `${Math.round(bytes / 1024 / 1024)} MB`;
|
|
1453
|
+
print(` ${scope.unit} ${scope.activeState ?? '?'} ${mb(scope.memoryCurrentBytes)} of ${mb(scope.memoryMaxBytes)} tasks ${scope.tasksCurrent ?? '?'}${scope.result && scope.result !== 'success' ? ` ${scope.result}` : ''}`);
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1349
1456
|
let effective;
|
|
1350
1457
|
try {
|
|
1351
1458
|
const { stdout } = await execFileAsync('systemctl', [
|
|
@@ -1372,6 +1479,19 @@ async function cmdDoctor(args) {
|
|
|
1372
1479
|
print('Effective (systemd)');
|
|
1373
1480
|
for (const line of effective)
|
|
1374
1481
|
print(` ${line}`);
|
|
1482
|
+
// «В файле 7680M, действует 6.0G». Ровно это и получалось, пока самообновление
|
|
1483
|
+
// звало `systemctl --user daemon-reload` без `XDG_RUNTIME_DIR`: systemd отвечал
|
|
1484
|
+
// «Failed to connect to bus» и выходил с кодом 0, drop-in переписывался, а
|
|
1485
|
+
// systemd продолжал жить по старому юниту (план §5.5). Причину починили, но
|
|
1486
|
+
// расхождение может прийти и иначе — правкой юнита руками, оборванным
|
|
1487
|
+
// обновлением, — и без этой строки строки выше выглядят как то, что
|
|
1488
|
+
// действует, хотя это уже не так.
|
|
1489
|
+
if ((await systemctlProperty('NeedDaemonReload')) === 'yes') {
|
|
1490
|
+
print('');
|
|
1491
|
+
print('STALE — systemd has NOT re-read the unit since it changed on disk,');
|
|
1492
|
+
print(' so the numbers above are the OLD ones, not what the drop-in says.');
|
|
1493
|
+
print(` Apply with: ${systemctlHint('daemon-reload')}`);
|
|
1494
|
+
}
|
|
1375
1495
|
// Привратник шаблонов поиска. Печатается всегда, в том числе когда он ВЫКЛЮЧЕН
|
|
1376
1496
|
// в коде: «стоит, хотя выключен» — это и есть та машина, которую пропустило
|
|
1377
1497
|
// обновление, и увидеть её больше неоткуда.
|
|
@@ -1438,10 +1558,24 @@ async function cmdDoctor(args) {
|
|
|
1438
1558
|
process.exit(1);
|
|
1439
1559
|
return;
|
|
1440
1560
|
}
|
|
1441
|
-
const fixFacts =
|
|
1442
|
-
writeLimitsOverride(true, undefined, fixFacts);
|
|
1561
|
+
const { facts: fixFacts, sessionsUsageBytes: fixSessionsUsage } = await readMemoryFactsFromSystemd();
|
|
1443
1562
|
print('');
|
|
1444
|
-
|
|
1563
|
+
// Unmeasurable leaves the files exactly as they are and says so — the one
|
|
1564
|
+
// thing this must never do is replace a ceiling in force with a guess, or
|
|
1565
|
+
// rewrite the file without its `MemoryMax` line, which RESETS the ceiling.
|
|
1566
|
+
// The rest of the repair still runs: `doctor --fix` is also the search guard,
|
|
1567
|
+
// the unit file and the daemon-reload, and a machine that cannot be measured
|
|
1568
|
+
// this second still needs those. The non-zero exit at the end is what keeps
|
|
1569
|
+
// «could not do the main thing» from reading as success to automation.
|
|
1570
|
+
if (fixFacts === null) {
|
|
1571
|
+
print('Resource limits: LEFT UNCHANGED — this machine could not be measured.');
|
|
1572
|
+
print(' Nothing was removed. Re-run once `systemctl --user` answers again.');
|
|
1573
|
+
}
|
|
1574
|
+
else {
|
|
1575
|
+
// Keep the service and the sessions under the same measured policy.
|
|
1576
|
+
writeLimitsOverride(true, undefined, fixFacts, fixSessionsUsage);
|
|
1577
|
+
print(`Wrote ${limitsOverridePath()}`);
|
|
1578
|
+
}
|
|
1445
1579
|
const guardOutcome = applySearchGuard(print);
|
|
1446
1580
|
if (guardOutcome === 'installed') {
|
|
1447
1581
|
print(`Installed search-pattern guard into ${claudeSettingsPath(searchGuardHome())}`);
|
|
@@ -1458,7 +1592,19 @@ async function cmdDoctor(args) {
|
|
|
1458
1592
|
const applied = memoryPolicy(fixFacts);
|
|
1459
1593
|
if (applied.maxBytes > honest.maxBytes) {
|
|
1460
1594
|
const mb = (b) => Math.round(b / 1024 / 1024);
|
|
1461
|
-
|
|
1595
|
+
// Which cgroup forced the ceiling up, not just «the service»: on a loaded
|
|
1596
|
+
// machine it is the sessions slice, and naming the service there sent the
|
|
1597
|
+
// reader looking at a daemon holding 400 MB (QA-2026-09-07 BLOCKER-1).
|
|
1598
|
+
// The SAME numbers the floor is computed from (`managedUsageFloorBytes`),
|
|
1599
|
+
// not the headroom ones: where the split is unknown those two differ, and
|
|
1600
|
+
// a note sourced from the wrong pair names the wrong cgroup and a smaller
|
|
1601
|
+
// number than the one that actually forced the ceiling up.
|
|
1602
|
+
const ownFloor = fixFacts.ownFloorBytes ?? fixFacts.ownUsageBytes;
|
|
1603
|
+
const sessionsFloor = fixFacts.sessionsFloorBytes ?? fixFacts.sessionsUsageBytes;
|
|
1604
|
+
const holder = sessionsFloor > ownFloor
|
|
1605
|
+
? `the sessions on this machine already hold ${mb(sessionsFloor)} MB`
|
|
1606
|
+
: `the service already holds ${mb(ownFloor)} MB`;
|
|
1607
|
+
print(`note: ceiling raised to ${mb(applied.maxBytes)} MB — ${holder}, and ${mb(honest.maxBytes)} MB would kill it on reload. Re-run when idle.`);
|
|
1462
1608
|
}
|
|
1463
1609
|
}
|
|
1464
1610
|
try {
|
|
@@ -1469,6 +1615,10 @@ async function cmdDoctor(args) {
|
|
|
1469
1615
|
catch (error) {
|
|
1470
1616
|
fail(`daemon-reload failed: ${String(error instanceof Error ? error.message : error)}`);
|
|
1471
1617
|
}
|
|
1618
|
+
// Everything else was repaired; the limits were not. Said with the exit code
|
|
1619
|
+
// as well as with the line above, because `doctor --fix` is run by scripts.
|
|
1620
|
+
if (fixFacts === null)
|
|
1621
|
+
process.exit(1);
|
|
1472
1622
|
}
|
|
1473
1623
|
// ─── set-token ───────────────────────────────────────────────────────
|
|
1474
1624
|
function cmdSetToken(args) {
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Everything an agent starts runs at a lower CPU priority than the daemon that
|
|
3
|
+
* supervises it.
|
|
4
|
+
*
|
|
5
|
+
* The failure this exists for is the runner starving itself with its own
|
|
6
|
+
* children (16.08): a session's build saturated the box, the daemon lost four
|
|
7
|
+
* heartbeats in a row, the server went Offline and every session on that
|
|
8
|
+
* machine answered 504 — while the work it was doing was fine. The daemon's job
|
|
9
|
+
* during a heavy turn is a few milliseconds of socket traffic; it should not
|
|
10
|
+
* have to queue behind a `tsc` it launched itself.
|
|
11
|
+
*
|
|
12
|
+
* **What this gives.** Inside the service's own cgroup the daemon (nice 0) gets
|
|
13
|
+
* the processor before the agents (nice 10); neighbouring sessions, all at 10,
|
|
14
|
+
* still share it evenly between themselves. Children inherit the nice value at
|
|
15
|
+
* fork, so a `vitest` the agent starts through Bash — several levels down from
|
|
16
|
+
* the process we renice — is covered without us knowing about it.
|
|
17
|
+
*
|
|
18
|
+
* Inheritance is why every call site is the line straight after its `spawn`:
|
|
19
|
+
* measured here, a grandchild forked AFTER the call comes up at 10 and one
|
|
20
|
+
* forked in the microseconds before it stays at 0. An agent needs hundreds of
|
|
21
|
+
* milliseconds to boot before it forks anything, so that window is empty in
|
|
22
|
+
* practice — but it is a window, and it only grows if the call drifts down the
|
|
23
|
+
* function.
|
|
24
|
+
*
|
|
25
|
+
* **What it does NOT give.** Against processes OUTSIDE the cgroup (production
|
|
26
|
+
* in `system.slice`) it does nothing at all: across cgroups the split is
|
|
27
|
+
* decided by `cpu.weight`, and nice only orders tasks within one. It does not
|
|
28
|
+
* touch memory, which is the mechanism behind three of the four incidents on
|
|
29
|
+
* other people's machines. This is the approach to stage 2 (a scope per session
|
|
30
|
+
* with a memory ceiling) and the fallback for it on cgroup v1, where stage 2
|
|
31
|
+
* cannot work — not a replacement for it.
|
|
32
|
+
*
|
|
33
|
+
* Priority is a convenience, not correctness: nothing here throws. A session
|
|
34
|
+
* that runs at the wrong priority is a slower machine; a session that fails to
|
|
35
|
+
* start because renicing failed is a broken product.
|
|
36
|
+
*/
|
|
37
|
+
/**
|
|
38
|
+
* The nice value every process the runner spawns for an agent gets.
|
|
39
|
+
*
|
|
40
|
+
* 10 rather than 19: the point is to lose to the daemon and to anything the
|
|
41
|
+
* owner is doing by hand, not to be scheduled last behind every background cron
|
|
42
|
+
* on the box. The scheduler's weight table gives nice 10 about a ninth of the
|
|
43
|
+
* share of nice 0 under contention (1024 → 110), which is all the room the
|
|
44
|
+
* heartbeat needs — 19 would buy an order of magnitude more and cost a session
|
|
45
|
+
* its throughput whenever anything else on the machine woke up.
|
|
46
|
+
*/
|
|
47
|
+
export declare const NICE = 10;
|
|
48
|
+
/**
|
|
49
|
+
* Push one spawned process down to {@link NICE}. Never throws.
|
|
50
|
+
*
|
|
51
|
+
* Takes `number | undefined` because that is exactly what `child.pid` is: a
|
|
52
|
+
* spawn that failed has none, and the caller should not have to ask.
|
|
53
|
+
*/
|
|
54
|
+
export declare function lowerPriority(pid: number | undefined): void;
|
|
55
|
+
//# sourceMappingURL=process-priority.d.ts.map
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import os from 'node:os';
|
|
2
|
+
import { log } from './log.js';
|
|
3
|
+
/**
|
|
4
|
+
* Everything an agent starts runs at a lower CPU priority than the daemon that
|
|
5
|
+
* supervises it.
|
|
6
|
+
*
|
|
7
|
+
* The failure this exists for is the runner starving itself with its own
|
|
8
|
+
* children (16.08): a session's build saturated the box, the daemon lost four
|
|
9
|
+
* heartbeats in a row, the server went Offline and every session on that
|
|
10
|
+
* machine answered 504 — while the work it was doing was fine. The daemon's job
|
|
11
|
+
* during a heavy turn is a few milliseconds of socket traffic; it should not
|
|
12
|
+
* have to queue behind a `tsc` it launched itself.
|
|
13
|
+
*
|
|
14
|
+
* **What this gives.** Inside the service's own cgroup the daemon (nice 0) gets
|
|
15
|
+
* the processor before the agents (nice 10); neighbouring sessions, all at 10,
|
|
16
|
+
* still share it evenly between themselves. Children inherit the nice value at
|
|
17
|
+
* fork, so a `vitest` the agent starts through Bash — several levels down from
|
|
18
|
+
* the process we renice — is covered without us knowing about it.
|
|
19
|
+
*
|
|
20
|
+
* Inheritance is why every call site is the line straight after its `spawn`:
|
|
21
|
+
* measured here, a grandchild forked AFTER the call comes up at 10 and one
|
|
22
|
+
* forked in the microseconds before it stays at 0. An agent needs hundreds of
|
|
23
|
+
* milliseconds to boot before it forks anything, so that window is empty in
|
|
24
|
+
* practice — but it is a window, and it only grows if the call drifts down the
|
|
25
|
+
* function.
|
|
26
|
+
*
|
|
27
|
+
* **What it does NOT give.** Against processes OUTSIDE the cgroup (production
|
|
28
|
+
* in `system.slice`) it does nothing at all: across cgroups the split is
|
|
29
|
+
* decided by `cpu.weight`, and nice only orders tasks within one. It does not
|
|
30
|
+
* touch memory, which is the mechanism behind three of the four incidents on
|
|
31
|
+
* other people's machines. This is the approach to stage 2 (a scope per session
|
|
32
|
+
* with a memory ceiling) and the fallback for it on cgroup v1, where stage 2
|
|
33
|
+
* cannot work — not a replacement for it.
|
|
34
|
+
*
|
|
35
|
+
* Priority is a convenience, not correctness: nothing here throws. A session
|
|
36
|
+
* that runs at the wrong priority is a slower machine; a session that fails to
|
|
37
|
+
* start because renicing failed is a broken product.
|
|
38
|
+
*/
|
|
39
|
+
/**
|
|
40
|
+
* The nice value every process the runner spawns for an agent gets.
|
|
41
|
+
*
|
|
42
|
+
* 10 rather than 19: the point is to lose to the daemon and to anything the
|
|
43
|
+
* owner is doing by hand, not to be scheduled last behind every background cron
|
|
44
|
+
* on the box. The scheduler's weight table gives nice 10 about a ninth of the
|
|
45
|
+
* share of nice 0 under contention (1024 → 110), which is all the room the
|
|
46
|
+
* heartbeat needs — 19 would buy an order of magnitude more and cost a session
|
|
47
|
+
* its throughput whenever anything else on the machine woke up.
|
|
48
|
+
*/
|
|
49
|
+
export const NICE = 10;
|
|
50
|
+
/**
|
|
51
|
+
* EPERM is a property of the machine, not of the process — it means this kernel
|
|
52
|
+
* or container will not let us renice at all, and it will mean that for every
|
|
53
|
+
* spawn afterwards. Said once; the alternative is one warning per agent process
|
|
54
|
+
* for the life of the daemon, which is how a real line gets buried.
|
|
55
|
+
*/
|
|
56
|
+
let permissionWarned = false;
|
|
57
|
+
function errorCode(error) {
|
|
58
|
+
if (typeof error === 'object' && error !== null && 'code' in error) {
|
|
59
|
+
return String(error.code);
|
|
60
|
+
}
|
|
61
|
+
return '';
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Push one spawned process down to {@link NICE}. Never throws.
|
|
65
|
+
*
|
|
66
|
+
* Takes `number | undefined` because that is exactly what `child.pid` is: a
|
|
67
|
+
* spawn that failed has none, and the caller should not have to ask.
|
|
68
|
+
*/
|
|
69
|
+
export function lowerPriority(pid) {
|
|
70
|
+
if (pid === undefined)
|
|
71
|
+
return;
|
|
72
|
+
try {
|
|
73
|
+
os.setPriority(pid, NICE);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
const code = errorCode(error);
|
|
77
|
+
// ESRCH: the child was already gone — a binary that is not there exits
|
|
78
|
+
// before we get to it. Nothing happened and nothing is wrong, so nothing
|
|
79
|
+
// is said; the spawn failure itself is reported by whoever spawned it.
|
|
80
|
+
if (code === 'ESRCH')
|
|
81
|
+
return;
|
|
82
|
+
if (code === 'EPERM') {
|
|
83
|
+
if (permissionWarned)
|
|
84
|
+
return;
|
|
85
|
+
permissionWarned = true;
|
|
86
|
+
log.warn('priority: not allowed to renice agent processes on this machine', {
|
|
87
|
+
nice: NICE,
|
|
88
|
+
error: String(error),
|
|
89
|
+
});
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
log.warn('priority: could not lower a spawned process', {
|
|
93
|
+
pid,
|
|
94
|
+
nice: NICE,
|
|
95
|
+
error: String(error),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=process-priority.js.map
|
package/dist/protocol.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import type { HostLoadFrame } from './host-load.js';
|
|
2
3
|
export declare const SessionDescriptorSchema: z.ZodObject<{
|
|
3
4
|
id: z.ZodString;
|
|
4
5
|
kind: z.ZodEnum<["TICKET", "CHAT"]>;
|
|
@@ -1150,7 +1151,6 @@ export declare const GatewayFrameSchema: z.ZodDiscriminatedUnion<"type", [z.ZodO
|
|
|
1150
1151
|
branchPlan?: unknown;
|
|
1151
1152
|
}>;
|
|
1152
1153
|
}, "strip", z.ZodTypeAny, {
|
|
1153
|
-
type: "session_start";
|
|
1154
1154
|
session: {
|
|
1155
1155
|
mode: "ask" | "plan" | "auto" | "full";
|
|
1156
1156
|
agent: "CLAUDE" | "CODEX";
|
|
@@ -1201,8 +1201,8 @@ export declare const GatewayFrameSchema: z.ZodDiscriminatedUnion<"type", [z.ZodO
|
|
|
1201
1201
|
baseSha?: string | undefined;
|
|
1202
1202
|
} | undefined;
|
|
1203
1203
|
};
|
|
1204
|
-
}, {
|
|
1205
1204
|
type: "session_start";
|
|
1205
|
+
}, {
|
|
1206
1206
|
session: {
|
|
1207
1207
|
agent: "CLAUDE" | "CODEX";
|
|
1208
1208
|
id: string;
|
|
@@ -1248,6 +1248,7 @@ export declare const GatewayFrameSchema: z.ZodDiscriminatedUnion<"type", [z.ZodO
|
|
|
1248
1248
|
branchHint?: unknown;
|
|
1249
1249
|
branchPlan?: unknown;
|
|
1250
1250
|
};
|
|
1251
|
+
type: "session_start";
|
|
1251
1252
|
}>, z.ZodObject<{
|
|
1252
1253
|
type: z.ZodLiteral<"session_message">;
|
|
1253
1254
|
sessionId: z.ZodString;
|
|
@@ -1651,7 +1652,28 @@ export type RunnerFrame = {
|
|
|
1651
1652
|
from: string | null;
|
|
1652
1653
|
to: string;
|
|
1653
1654
|
};
|
|
1654
|
-
}
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* What this machine's own load looks like, right now (plan §5.3).
|
|
1658
|
+
*
|
|
1659
|
+
* A frame of its own rather than a field of `hello`, for the same reason
|
|
1660
|
+
* `agent_versions` is one: `hello` is composed once per process and replayed
|
|
1661
|
+
* on every reconnect, so a load put there would be frozen at daemon start —
|
|
1662
|
+
* a number that is always wrong except in the first second of the machine's
|
|
1663
|
+
* life. This one is measured on a timer and sent only when it moved.
|
|
1664
|
+
*
|
|
1665
|
+
* Nothing static travels here. `machine: {cpuCount, memTotalBytes,
|
|
1666
|
+
* memAvailableBytes}` is already in `hello`, and repeating facts is how two
|
|
1667
|
+
* sources of one truth start disagreeing. `cpuCount` is the single exception
|
|
1668
|
+
* and it earns its place: load1 without it cannot be read as a ratio, and a
|
|
1669
|
+
* consumer joining two frames to find out would eventually paint one
|
|
1670
|
+
* machine's load against another's core count.
|
|
1671
|
+
*
|
|
1672
|
+
* Fields and thresholds mirror `@devbridge/shared` — see `host-load.ts`.
|
|
1673
|
+
*/
|
|
1674
|
+
| ({
|
|
1675
|
+
type: 'host_load';
|
|
1676
|
+
} & HostLoadFrame) | {
|
|
1655
1677
|
type: 'pong';
|
|
1656
1678
|
};
|
|
1657
1679
|
//# sourceMappingURL=protocol.d.ts.map
|
package/dist/recipe-schema.d.ts
CHANGED
|
@@ -50,14 +50,14 @@ export declare const ProjectRecipePreviewSchema: z.ZodObject<{
|
|
|
50
50
|
}, "strip", z.ZodTypeAny, {
|
|
51
51
|
run: string;
|
|
52
52
|
url?: string | undefined;
|
|
53
|
-
project?: string | undefined;
|
|
54
53
|
stop?: string | undefined;
|
|
54
|
+
project?: string | undefined;
|
|
55
55
|
timeoutSec?: number | undefined;
|
|
56
56
|
}, {
|
|
57
57
|
run: string;
|
|
58
58
|
url?: string | undefined;
|
|
59
|
-
project?: string | undefined;
|
|
60
59
|
stop?: string | undefined;
|
|
60
|
+
project?: string | undefined;
|
|
61
61
|
timeoutSec?: number | undefined;
|
|
62
62
|
}>;
|
|
63
63
|
export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
@@ -200,14 +200,14 @@ export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
|
200
200
|
}, "strip", z.ZodTypeAny, {
|
|
201
201
|
run: string;
|
|
202
202
|
url?: string | undefined;
|
|
203
|
-
project?: string | undefined;
|
|
204
203
|
stop?: string | undefined;
|
|
204
|
+
project?: string | undefined;
|
|
205
205
|
timeoutSec?: number | undefined;
|
|
206
206
|
}, {
|
|
207
207
|
run: string;
|
|
208
208
|
url?: string | undefined;
|
|
209
|
-
project?: string | undefined;
|
|
210
209
|
stop?: string | undefined;
|
|
210
|
+
project?: string | undefined;
|
|
211
211
|
timeoutSec?: number | undefined;
|
|
212
212
|
}>>;
|
|
213
213
|
notes: z.ZodOptional<z.ZodString>;
|
|
@@ -243,8 +243,8 @@ export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
|
243
243
|
preview?: {
|
|
244
244
|
run: string;
|
|
245
245
|
url?: string | undefined;
|
|
246
|
-
project?: string | undefined;
|
|
247
246
|
stop?: string | undefined;
|
|
247
|
+
project?: string | undefined;
|
|
248
248
|
timeoutSec?: number | undefined;
|
|
249
249
|
} | undefined;
|
|
250
250
|
notes?: string | undefined;
|
|
@@ -257,8 +257,8 @@ export declare const ProjectRecipeSchema: z.ZodObject<{
|
|
|
257
257
|
preview?: {
|
|
258
258
|
run: string;
|
|
259
259
|
url?: string | undefined;
|
|
260
|
-
project?: string | undefined;
|
|
261
260
|
stop?: string | undefined;
|
|
261
|
+
project?: string | undefined;
|
|
262
262
|
timeoutSec?: number | undefined;
|
|
263
263
|
} | undefined;
|
|
264
264
|
notes?: string | undefined;
|