@bridge4dev/runner 0.65.0 → 0.65.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/codex.js +25 -2
- package/dist/daemon-exit.d.ts +40 -0
- package/dist/daemon-exit.js +29 -0
- package/dist/index.js +15 -17
- package/dist/session-cage.d.ts +49 -0
- package/dist/session-cage.js +238 -12
- package/dist/session-stall.d.ts +25 -0
- package/dist/session-stall.js +84 -0
- package/dist/supervisor.d.ts +66 -2
- package/dist/supervisor.js +164 -5
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/ws-client.js +14 -0
- package/package.json +1 -1
package/dist/adapters/codex.js
CHANGED
|
@@ -2119,12 +2119,35 @@ class CodexSession {
|
|
|
2119
2119
|
// codex is chatty on stderr (bubblewrap notices, MCP transport spam). Log
|
|
2120
2120
|
// locally, never turn it into session events.
|
|
2121
2121
|
log.warn('codex: stderr', { text: maskString(trimmed).slice(0, 500) });
|
|
2122
|
-
|
|
2122
|
+
/**
|
|
2123
|
+
* …with one exception, kept for the death below (#431). A process that
|
|
2124
|
+
* never reached `initialize` said exactly one thing on its way out, and
|
|
2125
|
+
* that sentence is the difference between «code 1» and a cause: on 16.09 it
|
|
2126
|
+
* was systemd's «Unit … was already loaded», and the person got the code.
|
|
2127
|
+
*
|
|
2128
|
+
* Only until the session is ready: after that this is never read again, and
|
|
2129
|
+
* codex is chatty enough on stderr for «never read» to be worth not
|
|
2130
|
+
* computing (independent review of #431).
|
|
2131
|
+
*/
|
|
2132
|
+
if (this.ready)
|
|
2133
|
+
return;
|
|
2134
|
+
const last = trimmed
|
|
2135
|
+
.split('\n')
|
|
2136
|
+
.filter((line) => line.trim())
|
|
2137
|
+
.pop();
|
|
2138
|
+
if (last !== undefined)
|
|
2139
|
+
this.lastStderrLine = last.trim();
|
|
2140
|
+
}
|
|
2141
|
+
/** The last thing this process said on stderr — see {@link onStderr}. */
|
|
2142
|
+
lastStderrLine = '';
|
|
2123
2143
|
onExit(info) {
|
|
2124
2144
|
if (!this.stopped && !this.ready) {
|
|
2145
|
+
// The cause, when the process left one. Masked and cut like every other
|
|
2146
|
+
// line that comes out of a child process here.
|
|
2147
|
+
const said = this.lastStderrLine ? `: ${maskString(this.lastStderrLine).slice(0, 200)}` : '';
|
|
2125
2148
|
this.emit({
|
|
2126
2149
|
type: 'error',
|
|
2127
|
-
message: `codex app-server exited before the session was ready (code ${info.code ?? 'null'})`,
|
|
2150
|
+
message: `codex app-server exited before the session was ready (code ${info.code ?? 'null'})${said}`,
|
|
2128
2151
|
// #373: the process is what ended, not a turn. `stopped` is already
|
|
2129
2152
|
// excluded above, so this is always an exit nobody here asked for.
|
|
2130
2153
|
processGone: true,
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one way out of the daemon process (#431).
|
|
3
|
+
*
|
|
4
|
+
* Its own module, and not four lines inside `cmdDaemon`, for one reason: the
|
|
5
|
+
* defect this ticket is about WAS those four lines. `supervisor.shutdown();
|
|
6
|
+
* ws.stop(); process.exit(0);` in a single tick killed one process per session
|
|
7
|
+
* and left everything that process had started running inside a cgroup systemd
|
|
8
|
+
* would then refuse to reuse by name. Nothing in the suite could see that,
|
|
9
|
+
* because no test in this package imports `index.ts` — found by the independent
|
|
10
|
+
* review of #431, which also pointed out that the next edit to that file would
|
|
11
|
+
* be just as invisible.
|
|
12
|
+
*
|
|
13
|
+
* So the order is code with a test on it rather than a comment:
|
|
14
|
+
*
|
|
15
|
+
* 1. **`shutdown()`** — its synchronous half runs before it returns: the cards
|
|
16
|
+
* a person was still being asked are withdrawn, the line saying what this
|
|
17
|
+
* restart costs goes into the journal, the agents are killed.
|
|
18
|
+
* 2. **the socket is closed** — after the events above needed it open, and
|
|
19
|
+
* before the wait below, which must not be a window in which an `event_ack`
|
|
20
|
+
* or a `session_start` reaches a supervisor that has let go of everything.
|
|
21
|
+
* 3. **the promise is awaited** — the cages, and nothing else. What does not
|
|
22
|
+
* finish inside the supervisor's own budget continues on systemd's side:
|
|
23
|
+
* the stop has been accepted by then.
|
|
24
|
+
* 4. **`exit`**, always — including when the wait ends in a rejection. A
|
|
25
|
+
* daemon that would not die because a bus call failed is worse than one
|
|
26
|
+
* that leaves a scope behind.
|
|
27
|
+
*/
|
|
28
|
+
export interface DaemonExitDeps {
|
|
29
|
+
/** The supervisor's shutdown. The synchronous half must run before it returns. */
|
|
30
|
+
shutdown: () => Promise<void>;
|
|
31
|
+
/** Close the socket, both directions. */
|
|
32
|
+
stopWs: () => void;
|
|
33
|
+
/** Write the status file, so `doctor` sees a daemon that is going down. */
|
|
34
|
+
noteStatus?: () => void;
|
|
35
|
+
/** End the process. Injected so the order above can be proved. */
|
|
36
|
+
exit: (code: number) => void;
|
|
37
|
+
}
|
|
38
|
+
/** Build the exit door. The returned function is safe to call more than once. */
|
|
39
|
+
export declare function daemonExit(deps: DaemonExitDeps): (code: number) => void;
|
|
40
|
+
//# sourceMappingURL=daemon-exit.d.ts.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { log } from './log.js';
|
|
2
|
+
/** Build the exit door. The returned function is safe to call more than once. */
|
|
3
|
+
export function daemonExit(deps) {
|
|
4
|
+
let leaving = false;
|
|
5
|
+
return (code) => {
|
|
6
|
+
/**
|
|
7
|
+
* A second signal does not shorten the first departure.
|
|
8
|
+
*
|
|
9
|
+
* Two `SIGTERM`s, or «Update runner» racing a restart command, used to run
|
|
10
|
+
* the whole thing twice: a second `verify.shutdown()` signalling a group
|
|
11
|
+
* that is already dying, and an `exit` from the second call cutting the
|
|
12
|
+
* first call's wait for its cages — the very wait this door exists for.
|
|
13
|
+
*/
|
|
14
|
+
if (leaving) {
|
|
15
|
+
log.warn('daemon: already leaving, this second request changes nothing', { code });
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
leaving = true;
|
|
19
|
+
const cages = deps.shutdown();
|
|
20
|
+
deps.stopWs();
|
|
21
|
+
deps.noteStatus?.();
|
|
22
|
+
void cages
|
|
23
|
+
.catch((error) => {
|
|
24
|
+
log.warn('daemon: the cages did not all stop cleanly', { error: String(error) });
|
|
25
|
+
})
|
|
26
|
+
.finally(() => deps.exit(code));
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=daemon-exit.js.map
|
package/dist/index.js
CHANGED
|
@@ -25,6 +25,7 @@ import { buildUnit, cpuQuotaPercent, devbridgeSliceOverridePath, limitsOverrideI
|
|
|
25
25
|
import { defaultCageProbe, initSessionCage, readSliceLimits, listSessionScopes, sessionCage, sweepOrphanSessionScopes, SESSION_TASKS_MAX, } from './session-cage.js';
|
|
26
26
|
import { SEARCH_GUARD_ENABLED, claudeSettingsPath, installSearchGuard, removeSearchGuard, searchGuardCommand, searchGuardHome, searchGuardHookPath, searchGuardStatus, } from './claude-settings.js';
|
|
27
27
|
import { readOomKills, recordCrash, takeLastExit } from './crash-note.js';
|
|
28
|
+
import { daemonExit } from './daemon-exit.js';
|
|
28
29
|
import { agentAuthStatuses } from './auth-relay.js';
|
|
29
30
|
import { addSafeDirectory, agentConfigContour, dockerCheck, ensureAgentPath, firstUnreachableAncestor, hasSafeDirectory, inspectPath, knownWorkspacePaths, lingerEnabled, nodeCheck, otherHomeWithAgents, runnerIdentity, safeDirectoryCommand, systemctlHint, systemdUserBusReachable, } from './environment.js';
|
|
30
31
|
import { mcpConfigDir } from './paths.js';
|
|
@@ -1019,11 +1020,7 @@ async function cmdDaemon() {
|
|
|
1019
1020
|
from: outcome.fromVersion,
|
|
1020
1021
|
to: outcome.toVersion ?? 'unknown',
|
|
1021
1022
|
});
|
|
1022
|
-
const timer = setTimeout(() =>
|
|
1023
|
-
supervisor.shutdown();
|
|
1024
|
-
ws.stop();
|
|
1025
|
-
process.exit(0);
|
|
1026
|
-
}, RESTART_DELAY_MS);
|
|
1023
|
+
const timer = setTimeout(() => leave(0), RESTART_DELAY_MS);
|
|
1027
1024
|
timer.unref();
|
|
1028
1025
|
},
|
|
1029
1026
|
// #396: the same exit, asked for directly rather than as the tail of an
|
|
@@ -1035,11 +1032,7 @@ async function cmdDaemon() {
|
|
|
1035
1032
|
sessions: supervisor.activeSessionIds.length,
|
|
1036
1033
|
told: Boolean(note),
|
|
1037
1034
|
});
|
|
1038
|
-
const timer = setTimeout(() =>
|
|
1039
|
-
supervisor.shutdown();
|
|
1040
|
-
ws.stop();
|
|
1041
|
-
process.exit(0);
|
|
1042
|
-
}, RESTART_DELAY_MS);
|
|
1035
|
+
const timer = setTimeout(() => leave(0), RESTART_DELAY_MS);
|
|
1043
1036
|
timer.unref();
|
|
1044
1037
|
},
|
|
1045
1038
|
});
|
|
@@ -1052,13 +1045,21 @@ async function cmdDaemon() {
|
|
|
1052
1045
|
activeSessionIds: supervisor.activeSessionIds,
|
|
1053
1046
|
updatedAt: new Date().toISOString(),
|
|
1054
1047
|
});
|
|
1048
|
+
/**
|
|
1049
|
+
* The one way out of this process — the order lives in `daemon-exit.ts`,
|
|
1050
|
+
* where a test can hold it (#431).
|
|
1051
|
+
*/
|
|
1052
|
+
const leave = daemonExit({
|
|
1053
|
+
shutdown: () => supervisor.shutdown(),
|
|
1054
|
+
stopWs: () => ws.stop(),
|
|
1055
|
+
noteStatus: updateStatus,
|
|
1056
|
+
exit: (code) => process.exit(code),
|
|
1057
|
+
});
|
|
1055
1058
|
ws.on('open', updateStatus);
|
|
1056
1059
|
ws.on('close', updateStatus);
|
|
1057
1060
|
ws.on('revoked', (reason) => {
|
|
1058
1061
|
log.error(`daemon: access revoked (${reason}) — exiting`);
|
|
1059
|
-
|
|
1060
|
-
updateStatus();
|
|
1061
|
-
process.exit(3);
|
|
1062
|
+
leave(3);
|
|
1062
1063
|
});
|
|
1063
1064
|
const statusTimer = setInterval(updateStatus, 30_000);
|
|
1064
1065
|
statusTimer.unref();
|
|
@@ -1072,10 +1073,7 @@ async function cmdDaemon() {
|
|
|
1072
1073
|
limitsTimer.unref();
|
|
1073
1074
|
const shutdown = (signal) => {
|
|
1074
1075
|
log.info(`daemon: ${signal} received, shutting down`);
|
|
1075
|
-
|
|
1076
|
-
ws.stop();
|
|
1077
|
-
updateStatus();
|
|
1078
|
-
process.exit(0);
|
|
1076
|
+
leave(0);
|
|
1079
1077
|
};
|
|
1080
1078
|
process.on('SIGINT', () => shutdown('SIGINT'));
|
|
1081
1079
|
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
package/dist/session-cage.d.ts
CHANGED
|
@@ -300,6 +300,13 @@ export declare function sanitizeCageId(id: string): string;
|
|
|
300
300
|
* normal case and the marker appears only while the old scope is still there.
|
|
301
301
|
*/
|
|
302
302
|
export declare function sessionScopeUnit(id: string, attempt?: number): string;
|
|
303
|
+
/**
|
|
304
|
+
* Is there a cgroup by this unit's name — that is, processes still in it?
|
|
305
|
+
*
|
|
306
|
+
* `dirOf` is the test seam: the answer depends on a cgroup tree, and a suite
|
|
307
|
+
* has none. Everything else about this function is one `stat`.
|
|
308
|
+
*/
|
|
309
|
+
export declare function cageCgroupExists(unit: string, dirOf?: (unit: string) => string | null): boolean;
|
|
303
310
|
/**
|
|
304
311
|
* The systemd release that added `--expand-environment=`.
|
|
305
312
|
*
|
|
@@ -433,6 +440,16 @@ export declare function buildCagedSpawn(input: {
|
|
|
433
440
|
id: string;
|
|
434
441
|
command: string;
|
|
435
442
|
args: string[];
|
|
443
|
+
},
|
|
444
|
+
/**
|
|
445
|
+
* The two things this function does to the machine besides building a command
|
|
446
|
+
* line, as seams (#431): whether a cage of that name still exists, and what to
|
|
447
|
+
* do about it. A suite has neither a cgroup tree nor a systemd to answer.
|
|
448
|
+
*/
|
|
449
|
+
io?: {
|
|
450
|
+
cageExists?: (unit: string) => boolean;
|
|
451
|
+
cageLiveness?: (unit: string) => ScopeLiveness;
|
|
452
|
+
stopCage?: (unit: string) => void;
|
|
436
453
|
}): CagedSpawn;
|
|
437
454
|
/**
|
|
438
455
|
* Did this process die in the window before `systemd-run` handed over?
|
|
@@ -628,6 +645,23 @@ export declare function markScopeOomKillsSeen(id: string, oomKills: number): voi
|
|
|
628
645
|
* thing and must not word it differently.
|
|
629
646
|
*/
|
|
630
647
|
export declare function stoppedProcesses(count: number, where?: 'it' | 'this one'): string;
|
|
648
|
+
/**
|
|
649
|
+
* What the feed says when the runner takes a session's cage down with itself
|
|
650
|
+
* (#431).
|
|
651
|
+
*
|
|
652
|
+
* Next to {@link stoppedProcesses} and for the same reason: two places that
|
|
653
|
+
* count the same thing must not word it differently. This one counts COMMANDS,
|
|
654
|
+
* because that is what a person can act on — «7 processes» is the same build
|
|
655
|
+
* said in a way nobody can use.
|
|
656
|
+
*
|
|
657
|
+
* The middle sentence is the part that matters. A command that outlives the
|
|
658
|
+
* process which started it cannot hand its output back to anybody: the pipe
|
|
659
|
+
* died with the agent, and the agent that comes back is a different process
|
|
660
|
+
* with a different tool call. Stopping it is not a loss, it is the refusal to
|
|
661
|
+
* keep a zombie — but the person has to be told, or they will wait for a build
|
|
662
|
+
* that nobody is going to report.
|
|
663
|
+
*/
|
|
664
|
+
export declare function restartStoppedCommands(count: number): string;
|
|
631
665
|
/**
|
|
632
666
|
* Snapshot the cgroup's verdict before systemd can take it away.
|
|
633
667
|
*
|
|
@@ -669,6 +703,21 @@ export declare function releaseSessionScope(unit: string | null, id?: string, sy
|
|
|
669
703
|
* worth nothing, and a hung `systemctl` must not hold a failing session open.
|
|
670
704
|
*/
|
|
671
705
|
export declare function memoryDeathSentence(id: string, capMs?: number): Promise<string | null>;
|
|
706
|
+
/**
|
|
707
|
+
* Stop a cage and let systemd forget its name — the other half of killing the
|
|
708
|
+
* agent (#431, грабля §524).
|
|
709
|
+
*
|
|
710
|
+
* {@link releaseSessionScope} is the path for a process that ENDED: it reads
|
|
711
|
+
* the verdict first and only stops what is left. This one is for the two
|
|
712
|
+
* moments where there is nothing to read and no time to read it — the daemon
|
|
713
|
+
* going down, and a start that found its own name still taken. It asks for the
|
|
714
|
+
* stop and nothing else.
|
|
715
|
+
*
|
|
716
|
+
* Worth knowing on the shutdown path: once `stop` has been accepted, the job
|
|
717
|
+
* belongs to systemd. A caller that gives up waiting still gets the cage
|
|
718
|
+
* stopped — it simply does not get to see it happen.
|
|
719
|
+
*/
|
|
720
|
+
export declare function stopSessionScope(unit: string | null, systemctl?: Systemctl): Promise<boolean>;
|
|
672
721
|
/** Unit names of every `devbridge-session-*.scope` systemd still knows about. */
|
|
673
722
|
export declare function listSessionScopeUnits(systemctl?: Systemctl): Promise<string[]>;
|
|
674
723
|
export interface SessionScopeInfo {
|
package/dist/session-cage.js
CHANGED
|
@@ -367,6 +367,45 @@ export function sessionScopeUnit(id, attempt = 1) {
|
|
|
367
367
|
const base = `${SESSION_SCOPE_PREFIX}${sanitizeCageId(id)}`;
|
|
368
368
|
return attempt <= 1 ? `${base}.scope` : `${base}-r${attempt}.scope`;
|
|
369
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* How many names one start may walk through before giving up and letting
|
|
372
|
+
* systemd refuse it (#431). Three is two more than has ever been needed.
|
|
373
|
+
*/
|
|
374
|
+
const CAGE_NAME_MAX_TRIES = 3;
|
|
375
|
+
/**
|
|
376
|
+
* Is there a cgroup by this unit's name — that is, processes still in it?
|
|
377
|
+
*
|
|
378
|
+
* `dirOf` is the test seam: the answer depends on a cgroup tree, and a suite
|
|
379
|
+
* has none. Everything else about this function is one `stat`.
|
|
380
|
+
*/
|
|
381
|
+
export function cageCgroupExists(unit, dirOf = scopeCgroupDir) {
|
|
382
|
+
const dir = dirOf(unit);
|
|
383
|
+
if (dir === null)
|
|
384
|
+
return false;
|
|
385
|
+
try {
|
|
386
|
+
return fs.existsSync(dir);
|
|
387
|
+
}
|
|
388
|
+
catch {
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Put out the cage that is holding a name this session needs, and say so if it
|
|
394
|
+
* would not go.
|
|
395
|
+
*
|
|
396
|
+
* Loud on failure, because the fallback is the dangerous one: the start moves
|
|
397
|
+
* to `-r2` either way, and if the old tree is STILL running under the old name
|
|
398
|
+
* the agent that comes back will start the same command beside it — two runs of
|
|
399
|
+
* one build in one folder (independent review of #431). Nothing here can undo
|
|
400
|
+
* that; the machine's log is what lets a person see it happened.
|
|
401
|
+
*/
|
|
402
|
+
function defaultStopCage(unit) {
|
|
403
|
+
void stopSessionScope(unit).then((stopped) => {
|
|
404
|
+
if (!stopped) {
|
|
405
|
+
log.warn('session cage: the cage holding this name would not stop', { unit });
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
}
|
|
370
409
|
// ─── capability detection ────────────────────────────────────────────
|
|
371
410
|
/** cgroup v2 unified, from `statfs` — the number behind `stat -fc %T`. */
|
|
372
411
|
const CGROUP2_SUPER_MAGIC = 0x63677270;
|
|
@@ -874,7 +913,13 @@ export function cageSpawn(input) {
|
|
|
874
913
|
* who may hand it. One function could not do both — a test process legitimately
|
|
875
914
|
* needs the first answer and must never get the second.
|
|
876
915
|
*/
|
|
877
|
-
export function buildCagedSpawn(input
|
|
916
|
+
export function buildCagedSpawn(input,
|
|
917
|
+
/**
|
|
918
|
+
* The two things this function does to the machine besides building a command
|
|
919
|
+
* line, as seams (#431): whether a cage of that name still exists, and what to
|
|
920
|
+
* do about it. A suite has neither a cgroup tree nor a systemd to answer.
|
|
921
|
+
*/
|
|
922
|
+
io = {}) {
|
|
878
923
|
const facts = sessionCage();
|
|
879
924
|
if (facts.mode !== 'scope' ||
|
|
880
925
|
facts.memoryMaxBytes === null ||
|
|
@@ -896,9 +941,64 @@ export function buildCagedSpawn(input) {
|
|
|
896
941
|
const maxBytes = live?.maxBytes ?? facts.memoryMaxBytes;
|
|
897
942
|
const swapMaxBytes = live?.swapBytes ?? facts.swapMaxBytes;
|
|
898
943
|
const guaranteedBytes = live?.guaranteedBytes ?? null;
|
|
899
|
-
|
|
944
|
+
let attempt = (attempts.get(input.id) ?? 0) + 1;
|
|
945
|
+
let unit = sessionScopeUnit(input.id, attempt);
|
|
946
|
+
/**
|
|
947
|
+
* The last line of defence: a name that is STILL taken (#431).
|
|
948
|
+
*
|
|
949
|
+
* The counter above is memory, and memory is exactly what a restarted daemon
|
|
950
|
+
* does not have: after `systemctl restart` it starts at one again and asks
|
|
951
|
+
* for the plain `devbridge-session-<id>.scope` — the name the previous life
|
|
952
|
+
* of this session may still be holding with a build nobody killed. systemd
|
|
953
|
+
* answers «was already loaded or has a fragment file» and exits 1, and the
|
|
954
|
+
* session dies before it starts (the whole of #431).
|
|
955
|
+
*
|
|
956
|
+
* Asked of the cgroup tree rather than of systemd on purpose: this runs on
|
|
957
|
+
* the start of every session, and the bus has been measured at 2.7 s under
|
|
958
|
+
* load. A directory that exists means processes are in it, which is the case
|
|
959
|
+
* that matters; a unit left `failed` and EMPTY has no directory, and the
|
|
960
|
+
* sweep at daemon start clears those by name.
|
|
961
|
+
*
|
|
962
|
+
* And the tree is not left to run under a different name. It cannot hand its
|
|
963
|
+
* output to anybody — the pipes died with the process that started it — while
|
|
964
|
+
* the agent coming back will start the same command again: two `docker
|
|
965
|
+
* compose` runs of one project in one folder is a worse outcome than the
|
|
966
|
+
* failure this guard exists to prevent.
|
|
967
|
+
*/
|
|
968
|
+
const exists = io.cageExists ?? cageCgroupExists;
|
|
969
|
+
const liveness = io.cageLiveness ?? scopeOwnerLiveness;
|
|
970
|
+
const stop = io.stopCage ?? defaultStopCage;
|
|
971
|
+
for (let tries = 0; tries < CAGE_NAME_MAX_TRIES && exists(unit); tries += 1) {
|
|
972
|
+
/**
|
|
973
|
+
* Asked of the kernel, not of this process's memory (independent review of
|
|
974
|
+
* #431).
|
|
975
|
+
*
|
|
976
|
+
* The first version compared the name against `liveUnits` and stopped
|
|
977
|
+
* anything that was not in it. That register holds ONE unit per session, so
|
|
978
|
+
* in the shape it was meant to protect — two processes of one session,
|
|
979
|
+
* #401 — the live cage is the one MISSING from it, and the guard would have
|
|
980
|
+
* put out an agent somebody was using. `scopeOwnerLiveness` answers the
|
|
981
|
+
* question that actually matters, on evidence: `orphaned` and `empty` are
|
|
982
|
+
* the previous life of this session, and nothing else may be touched.
|
|
983
|
+
*/
|
|
984
|
+
const state = liveness(unit);
|
|
985
|
+
if (state === 'orphaned' || state === 'empty') {
|
|
986
|
+
log.warn('session cage: the name of this cage is still taken, stopping what is left in it', {
|
|
987
|
+
unit,
|
|
988
|
+
liveness: state,
|
|
989
|
+
});
|
|
990
|
+
stop(unit);
|
|
991
|
+
}
|
|
992
|
+
else {
|
|
993
|
+
log.warn('session cage: the name of this cage is taken by something alive, leaving it', {
|
|
994
|
+
unit,
|
|
995
|
+
liveness: state,
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
attempt += 1;
|
|
999
|
+
unit = sessionScopeUnit(input.id, attempt);
|
|
1000
|
+
}
|
|
900
1001
|
attempts.set(input.id, attempt);
|
|
901
|
-
const unit = sessionScopeUnit(input.id, attempt);
|
|
902
1002
|
liveUnits.set(input.id, unit);
|
|
903
1003
|
// A verdict nobody read belongs to the process that just ended, not to the
|
|
904
1004
|
// one starting now: without this, a kill that was never surfaced would be
|
|
@@ -1281,6 +1381,31 @@ export function stoppedProcesses(count, where = 'it') {
|
|
|
1281
1381
|
? `a process in ${where} was stopped`
|
|
1282
1382
|
: `${count} processes in ${where} were stopped`;
|
|
1283
1383
|
}
|
|
1384
|
+
/**
|
|
1385
|
+
* What the feed says when the runner takes a session's cage down with itself
|
|
1386
|
+
* (#431).
|
|
1387
|
+
*
|
|
1388
|
+
* Next to {@link stoppedProcesses} and for the same reason: two places that
|
|
1389
|
+
* count the same thing must not word it differently. This one counts COMMANDS,
|
|
1390
|
+
* because that is what a person can act on — «7 processes» is the same build
|
|
1391
|
+
* said in a way nobody can use.
|
|
1392
|
+
*
|
|
1393
|
+
* The middle sentence is the part that matters. A command that outlives the
|
|
1394
|
+
* process which started it cannot hand its output back to anybody: the pipe
|
|
1395
|
+
* died with the agent, and the agent that comes back is a different process
|
|
1396
|
+
* with a different tool call. Stopping it is not a loss, it is the refusal to
|
|
1397
|
+
* keep a zombie — but the person has to be told, or they will wait for a build
|
|
1398
|
+
* that nobody is going to report.
|
|
1399
|
+
*/
|
|
1400
|
+
export function restartStoppedCommands(count) {
|
|
1401
|
+
return count === 1
|
|
1402
|
+
? 'The runner was restarted, so the command this session had running was stopped. ' +
|
|
1403
|
+
'Nothing can hand its output back now — start it again if you still need it. ' +
|
|
1404
|
+
'The session itself is fine: its conversation was not touched.'
|
|
1405
|
+
: `The runner was restarted, so the ${count} commands this session had running were stopped. ` +
|
|
1406
|
+
'Nothing can hand their output back now — start them again if you still need them. ' +
|
|
1407
|
+
'The session itself is fine: its conversation was not touched.';
|
|
1408
|
+
}
|
|
1284
1409
|
const deaths = new Map();
|
|
1285
1410
|
/**
|
|
1286
1411
|
* Snapshot the cgroup's verdict before systemd can take it away.
|
|
@@ -1373,11 +1498,28 @@ function showValue(stdout, property) {
|
|
|
1373
1498
|
* startable in the meantime.
|
|
1374
1499
|
*/
|
|
1375
1500
|
export async function releaseSessionScope(unit, id, systemctl = realSystemctl) {
|
|
1376
|
-
|
|
1501
|
+
/**
|
|
1502
|
+
* Is the scope being released the one this session is CURRENTLY in?
|
|
1503
|
+
*
|
|
1504
|
+
* It usually is, and then everything below behaves as it always did. It is
|
|
1505
|
+
* not when the agent was relaunched while the previous process was still on
|
|
1506
|
+
* its way out — the one-shot relaunches empty the slot when the event stream
|
|
1507
|
+
* ends, and the CLI's `exit` lands a second or two later. By then the
|
|
1508
|
+
* register names the LIVE cage, and wiping it by session id threw away the
|
|
1509
|
+
* only record of it: the memory watch lost the cgroup, the sweep lost a
|
|
1510
|
+
* spare, and since #431 `shutdown()` reads exactly this to decide whose cage
|
|
1511
|
+
* to put out — «this session has no cage» would have left the live one
|
|
1512
|
+
* running, which is the defect this ticket is about. Found by the
|
|
1513
|
+
* independent review of #431.
|
|
1514
|
+
*/
|
|
1515
|
+
const current = unit !== null && id !== undefined && liveUnits.get(id) === unit;
|
|
1516
|
+
const running = releaseSessionScopeInner(unit, id, systemctl, current);
|
|
1377
1517
|
if (id !== undefined) {
|
|
1378
1518
|
// The process this named is gone with the scope; a pid outliving it would be
|
|
1379
|
-
// a pid that means something else by the time anybody reads it.
|
|
1380
|
-
|
|
1519
|
+
// a pid that means something else by the time anybody reads it. Only when
|
|
1520
|
+
// this IS that process — see `current` above.
|
|
1521
|
+
if (current)
|
|
1522
|
+
agentPids.delete(id);
|
|
1381
1523
|
releasing.set(id, running);
|
|
1382
1524
|
void running.finally(() => {
|
|
1383
1525
|
if (releasing.get(id) === running)
|
|
@@ -1416,7 +1558,9 @@ export async function memoryDeathSentence(id, capMs = 3_000) {
|
|
|
1416
1558
|
}
|
|
1417
1559
|
return explainMemoryDeath(id);
|
|
1418
1560
|
}
|
|
1419
|
-
async function releaseSessionScopeInner(unit, id, systemctl
|
|
1561
|
+
async function releaseSessionScopeInner(unit, id, systemctl,
|
|
1562
|
+
/** Is this the cage the session is in right now — see {@link releaseSessionScope}. */
|
|
1563
|
+
current) {
|
|
1420
1564
|
if (!unit)
|
|
1421
1565
|
return null;
|
|
1422
1566
|
// Synchronously and FIRST: the cgroup's own counters are the one record of an
|
|
@@ -1425,7 +1569,7 @@ async function releaseSessionScopeInner(unit, id, systemctl) {
|
|
|
1425
1569
|
// empty. This is what the adapter's error text is built from (#387).
|
|
1426
1570
|
if (id !== undefined)
|
|
1427
1571
|
rememberDeath(id, unit);
|
|
1428
|
-
if (id !== undefined)
|
|
1572
|
+
if (id !== undefined && current)
|
|
1429
1573
|
liveUnits.delete(id);
|
|
1430
1574
|
let result = null;
|
|
1431
1575
|
let tasksLeft = null;
|
|
@@ -1504,10 +1648,45 @@ async function releaseSessionScopeInner(unit, id, systemctl) {
|
|
|
1504
1648
|
// which systemd answers with «was already loaded or has a fragment file» and
|
|
1505
1649
|
// spawns nothing — the exact failure the counter exists to prevent
|
|
1506
1650
|
// (QA-2026-09-07 MINOR-11).
|
|
1507
|
-
if (id !== undefined && forgotten)
|
|
1651
|
+
if (id !== undefined && forgotten && current)
|
|
1508
1652
|
attempts.delete(id);
|
|
1509
1653
|
return result;
|
|
1510
1654
|
}
|
|
1655
|
+
/**
|
|
1656
|
+
* Stop a cage and let systemd forget its name — the other half of killing the
|
|
1657
|
+
* agent (#431, грабля §524).
|
|
1658
|
+
*
|
|
1659
|
+
* {@link releaseSessionScope} is the path for a process that ENDED: it reads
|
|
1660
|
+
* the verdict first and only stops what is left. This one is for the two
|
|
1661
|
+
* moments where there is nothing to read and no time to read it — the daemon
|
|
1662
|
+
* going down, and a start that found its own name still taken. It asks for the
|
|
1663
|
+
* stop and nothing else.
|
|
1664
|
+
*
|
|
1665
|
+
* Worth knowing on the shutdown path: once `stop` has been accepted, the job
|
|
1666
|
+
* belongs to systemd. A caller that gives up waiting still gets the cage
|
|
1667
|
+
* stopped — it simply does not get to see it happen.
|
|
1668
|
+
*/
|
|
1669
|
+
export async function stopSessionScope(unit, systemctl = realSystemctl) {
|
|
1670
|
+
if (!unit)
|
|
1671
|
+
return false;
|
|
1672
|
+
let stopped = false;
|
|
1673
|
+
try {
|
|
1674
|
+
await systemctl(['stop', unit]);
|
|
1675
|
+
stopped = true;
|
|
1676
|
+
}
|
|
1677
|
+
catch (error) {
|
|
1678
|
+
// A scope that ended by itself answers «not loaded», and a process with no
|
|
1679
|
+
// right to act on cages answers with a refusal. Neither is worth a session.
|
|
1680
|
+
log.debug('session cage: could not stop the scope', { unit, error: String(error) });
|
|
1681
|
+
}
|
|
1682
|
+
try {
|
|
1683
|
+
await systemctl(['reset-failed', unit]);
|
|
1684
|
+
}
|
|
1685
|
+
catch {
|
|
1686
|
+
// Nothing to forget: the usual answer for a scope that stopped cleanly.
|
|
1687
|
+
}
|
|
1688
|
+
return stopped;
|
|
1689
|
+
}
|
|
1511
1690
|
/** Unit names of every `devbridge-session-*.scope` systemd still knows about. */
|
|
1512
1691
|
export async function listSessionScopeUnits(systemctl = realSystemctl) {
|
|
1513
1692
|
let stdout;
|
|
@@ -1568,6 +1747,17 @@ export async function listSessionScopes(systemctl = realSystemctl) {
|
|
|
1568
1747
|
}
|
|
1569
1748
|
return out;
|
|
1570
1749
|
}
|
|
1750
|
+
/**
|
|
1751
|
+
* Did this read fail because the process is gone, or because we could not read?
|
|
1752
|
+
*
|
|
1753
|
+
* `ENOENT` is /proc's answer for a pid that ended; `ESRCH` is the same answer
|
|
1754
|
+
* from the other syscalls. Everything else — permissions, descriptors, a broken
|
|
1755
|
+
* read — is ignorance, and ignorance is never evidence of litter.
|
|
1756
|
+
*/
|
|
1757
|
+
function processIsGone(error) {
|
|
1758
|
+
const code = error?.code;
|
|
1759
|
+
return code === 'ENOENT' || code === 'ESRCH';
|
|
1760
|
+
}
|
|
1571
1761
|
export function scopeOwnerLiveness(unit, io = {}) {
|
|
1572
1762
|
const readFile = io.readFile ?? ((p) => fs.readFileSync(p, 'utf8'));
|
|
1573
1763
|
const cgroupDir = io.cgroupDir ?? scopeCgroupDir;
|
|
@@ -1597,6 +1787,19 @@ export function scopeOwnerLiveness(unit, io = {}) {
|
|
|
1597
1787
|
}
|
|
1598
1788
|
if (pids.length === 0)
|
|
1599
1789
|
return 'empty';
|
|
1790
|
+
/**
|
|
1791
|
+
* Who is INSIDE, as a set — the difference between «a live parent» and «a
|
|
1792
|
+
* live parent outside the cage» (#431).
|
|
1793
|
+
*
|
|
1794
|
+
* An agent's command is never one process: `bash -lc 'docker compose …'` is a
|
|
1795
|
+
* tree, and every process below its root has a living parent — its own
|
|
1796
|
+
* neighbour in this cgroup. Without this set the question below answered
|
|
1797
|
+
* «somebody is supervising them» for every abandoned build on the machine,
|
|
1798
|
+
* so the sweep left it alone for ever, the cage kept its name, and the next
|
|
1799
|
+
* start of that session died as «(code 1)» (measured 16.09.2026). The single
|
|
1800
|
+
* orphan of §479 answered correctly only because it had no children.
|
|
1801
|
+
*/
|
|
1802
|
+
const inside = new Set(pids);
|
|
1600
1803
|
let sawUnknown = false;
|
|
1601
1804
|
for (const pid of pids) {
|
|
1602
1805
|
let ppid;
|
|
@@ -1606,8 +1809,23 @@ export function scopeOwnerLiveness(unit, io = {}) {
|
|
|
1606
1809
|
const value = line === undefined ? Number.NaN : Number(line.slice('PPid:'.length).trim());
|
|
1607
1810
|
ppid = Number.isSafeInteger(value) ? value : null;
|
|
1608
1811
|
}
|
|
1609
|
-
catch {
|
|
1610
|
-
|
|
1812
|
+
catch (error) {
|
|
1813
|
+
/**
|
|
1814
|
+
* Two different answers wearing one coat, and the difference decides
|
|
1815
|
+
* whether a live session may be stopped (независимая проверка #431).
|
|
1816
|
+
*
|
|
1817
|
+
* `ENOENT`/`ESRCH` is the kernel saying the process ended between the
|
|
1818
|
+
* listing and the read — it says nothing either way, and the scan goes
|
|
1819
|
+
* on. Anything else (`EACCES` under `hidepid`, `EMFILE` on a busy
|
|
1820
|
+
* daemon, `EIO`) is a /proc we could not read, and «could not read» has
|
|
1821
|
+
* exactly one safe reading in this function: `unknown`, which is never
|
|
1822
|
+
* swept. Until the tree rule above, a live session survived this by
|
|
1823
|
+
* accident — its neighbours' parents were alive; now the agent's own
|
|
1824
|
+
* line is the only voice for «somebody is working here», and losing it
|
|
1825
|
+
* must not read as «litter».
|
|
1826
|
+
*/
|
|
1827
|
+
if (!processIsGone(error))
|
|
1828
|
+
sawUnknown = true;
|
|
1611
1829
|
continue;
|
|
1612
1830
|
}
|
|
1613
1831
|
if (ppid === null) {
|
|
@@ -1618,6 +1836,10 @@ export function scopeOwnerLiveness(unit, io = {}) {
|
|
|
1618
1836
|
// answer for a process that is going away.
|
|
1619
1837
|
if (ppid <= 1)
|
|
1620
1838
|
continue;
|
|
1839
|
+
// A parent in the SAME cage is a neighbour, not an owner: it is part of the
|
|
1840
|
+
// same abandoned tree, and its own parent is asked about in its own turn.
|
|
1841
|
+
if (inside.has(ppid))
|
|
1842
|
+
continue;
|
|
1621
1843
|
/**
|
|
1622
1844
|
* …and reparented to the USER MANAGER counts the same (measured 10.09.2026).
|
|
1623
1845
|
*
|
|
@@ -1632,8 +1854,12 @@ export function scopeOwnerLiveness(unit, io = {}) {
|
|
|
1632
1854
|
try {
|
|
1633
1855
|
parentComm = readFile(path.join('/proc', String(ppid), 'comm')).trim();
|
|
1634
1856
|
}
|
|
1635
|
-
catch {
|
|
1857
|
+
catch (error) {
|
|
1636
1858
|
// The parent went between the two reads: it is not supervising anything.
|
|
1859
|
+
// Anything other than «it is gone» is a /proc we could not read — see the
|
|
1860
|
+
// catch above; the same rule, for the same reason.
|
|
1861
|
+
if (!processIsGone(error))
|
|
1862
|
+
sawUnknown = true;
|
|
1637
1863
|
continue;
|
|
1638
1864
|
}
|
|
1639
1865
|
if (parentComm === 'systemd' || parentComm === 'init')
|
package/dist/session-stall.d.ts
CHANGED
|
@@ -305,6 +305,31 @@ export declare function readScopeProcesses(unit: string, readFile?: (p: string)
|
|
|
305
305
|
* over that a `sh`, and killing the leaf leaves the parent to start another.
|
|
306
306
|
*/
|
|
307
307
|
export declare function pickKillCandidate(processes: ScopeProcess[], agentPid?: number | null): KillCandidate | null;
|
|
308
|
+
/**
|
|
309
|
+
* How many COMMANDS this session has running, counted the way a person counts
|
|
310
|
+
* them (#431).
|
|
311
|
+
*
|
|
312
|
+
* Used on the way out: the runner is about to take the cage down with it, and
|
|
313
|
+
* the line it writes into the feed has to say how much work that costs. The
|
|
314
|
+
* number must therefore mean what the sentence says – «commands», not
|
|
315
|
+
* «processes»: `bash -lc 'docker compose up'` is three or four processes and
|
|
316
|
+
* exactly one command, and a person who is told «7 processes were stopped»
|
|
317
|
+
* learns nothing they can act on.
|
|
318
|
+
*
|
|
319
|
+
* The exclusions are {@link pickKillCandidate}'s, for the same reasons and with
|
|
320
|
+
* the same reading of `agentPid`:
|
|
321
|
+
*
|
|
322
|
+
* - **the agent is not a command.** It is the root of the tree, and stopping
|
|
323
|
+
* it is what the restart is.
|
|
324
|
+
* - **MCP servers and everything under them are not commands.** They are the
|
|
325
|
+
* agent's own plumbing; the session did not ask for them and nothing of the
|
|
326
|
+
* person's work is lost with them.
|
|
327
|
+
*
|
|
328
|
+
* `git` is NOT excluded here, though it is excluded there: that rule exists so
|
|
329
|
+
* the memory brake does not leave `.git/index.lock` behind, and this function
|
|
330
|
+
* stops nothing – it counts what is about to go either way.
|
|
331
|
+
*/
|
|
332
|
+
export declare function countRunningCommands(processes: ScopeProcess[], agentPid?: number | null): number;
|
|
308
333
|
/**
|
|
309
334
|
* Is this pid still the process it was when we decided to stop it?
|
|
310
335
|
*
|
package/dist/session-stall.js
CHANGED
|
@@ -668,6 +668,90 @@ export function pickKillCandidate(processes, agentPid = null) {
|
|
|
668
668
|
}
|
|
669
669
|
return best;
|
|
670
670
|
}
|
|
671
|
+
/**
|
|
672
|
+
* How many COMMANDS this session has running, counted the way a person counts
|
|
673
|
+
* them (#431).
|
|
674
|
+
*
|
|
675
|
+
* Used on the way out: the runner is about to take the cage down with it, and
|
|
676
|
+
* the line it writes into the feed has to say how much work that costs. The
|
|
677
|
+
* number must therefore mean what the sentence says – «commands», not
|
|
678
|
+
* «processes»: `bash -lc 'docker compose up'` is three or four processes and
|
|
679
|
+
* exactly one command, and a person who is told «7 processes were stopped»
|
|
680
|
+
* learns nothing they can act on.
|
|
681
|
+
*
|
|
682
|
+
* The exclusions are {@link pickKillCandidate}'s, for the same reasons and with
|
|
683
|
+
* the same reading of `agentPid`:
|
|
684
|
+
*
|
|
685
|
+
* - **the agent is not a command.** It is the root of the tree, and stopping
|
|
686
|
+
* it is what the restart is.
|
|
687
|
+
* - **MCP servers and everything under them are not commands.** They are the
|
|
688
|
+
* agent's own plumbing; the session did not ask for them and nothing of the
|
|
689
|
+
* person's work is lost with them.
|
|
690
|
+
*
|
|
691
|
+
* `git` is NOT excluded here, though it is excluded there: that rule exists so
|
|
692
|
+
* the memory brake does not leave `.git/index.lock` behind, and this function
|
|
693
|
+
* stops nothing – it counts what is about to go either way.
|
|
694
|
+
*/
|
|
695
|
+
export function countRunningCommands(processes, agentPid = null) {
|
|
696
|
+
if (processes.length === 0)
|
|
697
|
+
return 0;
|
|
698
|
+
const byPid = new Map(processes.map((p) => [p.pid, p]));
|
|
699
|
+
const children = new Map();
|
|
700
|
+
for (const p of processes) {
|
|
701
|
+
if (!byPid.has(p.ppid))
|
|
702
|
+
continue;
|
|
703
|
+
const list = children.get(p.ppid);
|
|
704
|
+
if (list)
|
|
705
|
+
list.push(p.pid);
|
|
706
|
+
else
|
|
707
|
+
children.set(p.ppid, [p.pid]);
|
|
708
|
+
}
|
|
709
|
+
const roots = processes.filter((p) => !byPid.has(p.ppid)).map((p) => p.pid);
|
|
710
|
+
/**
|
|
711
|
+
* The agent, and ONLY the agent, when its pid is known (independent review of
|
|
712
|
+
* #431).
|
|
713
|
+
*
|
|
714
|
+
* `pickKillCandidate` protects every parentless process, and there that is
|
|
715
|
+
* right: it chooses something to KILL, and a command wrongly spared costs a
|
|
716
|
+
* retry while the agent wrongly killed costs the session. Counting is the
|
|
717
|
+
* other way round. A cage has more than one parentless process whenever the
|
|
718
|
+
* agent left something behind it — `setsid`, `nohup pnpm dev &`, a double
|
|
719
|
+
* fork: the child stays in the cgroup and is reparented to the user manager,
|
|
720
|
+
* OUTSIDE the cage. Measured on this machine 16.09.2026: such a command was
|
|
721
|
+
* counted as zero, so the person was told nothing at all while it was being
|
|
722
|
+
* stopped — the exact silence this line exists to break (#427).
|
|
723
|
+
*/
|
|
724
|
+
const excluded = new Set(agentPid === null ? roots : [agentPid]);
|
|
725
|
+
const spread = (pid) => {
|
|
726
|
+
for (const child of children.get(pid) ?? []) {
|
|
727
|
+
if (excluded.has(child))
|
|
728
|
+
continue;
|
|
729
|
+
excluded.add(child);
|
|
730
|
+
spread(child);
|
|
731
|
+
}
|
|
732
|
+
};
|
|
733
|
+
for (const p of processes) {
|
|
734
|
+
if (!p.mcp)
|
|
735
|
+
continue;
|
|
736
|
+
excluded.add(p.pid);
|
|
737
|
+
// …but never from the agent: its own command line names the MCP config, so
|
|
738
|
+
// spreading from it would hide every command it ever started.
|
|
739
|
+
const isAgent = agentPid === null ? roots.includes(p.pid) : p.pid === agentPid;
|
|
740
|
+
if (!isAgent)
|
|
741
|
+
spread(p.pid);
|
|
742
|
+
}
|
|
743
|
+
let commands = 0;
|
|
744
|
+
for (const p of processes) {
|
|
745
|
+
if (excluded.has(p.pid))
|
|
746
|
+
continue;
|
|
747
|
+
// Only the top of each surviving subtree: a process whose parent is also a
|
|
748
|
+
// command is part of that command, not another one.
|
|
749
|
+
if (byPid.has(p.ppid) && !excluded.has(p.ppid))
|
|
750
|
+
continue;
|
|
751
|
+
commands += 1;
|
|
752
|
+
}
|
|
753
|
+
return commands;
|
|
754
|
+
}
|
|
671
755
|
/**
|
|
672
756
|
* Is this pid still the process it was when we decided to stop it?
|
|
673
757
|
*
|
package/dist/supervisor.d.ts
CHANGED
|
@@ -186,6 +186,19 @@ export interface SupervisorOptions {
|
|
|
186
186
|
* it never calls it twice at once. Production sweeps for real.
|
|
187
187
|
*/
|
|
188
188
|
sweepOrphanSessionScopes?: (liveIds: Iterable<string>) => Promise<string[]>;
|
|
189
|
+
/**
|
|
190
|
+
* #431: taking a session's cage down together with its agent. A test has no
|
|
191
|
+
* systemd to answer, and the daemon must not need one to be testable here.
|
|
192
|
+
*/
|
|
193
|
+
stopSessionScope?: (unit: string | null) => Promise<boolean>;
|
|
194
|
+
/** How long those stops are given. The real one is 3 s — no suite waits it out. */
|
|
195
|
+
cageStopBudgetMs?: number;
|
|
196
|
+
/**
|
|
197
|
+
* Which pid in a cage is the agent. Production reads the register the spawn
|
|
198
|
+
* filled; without the seam every test counted commands through the fallback
|
|
199
|
+
* branch, i.e. never the one the daemon uses (independent review of #431).
|
|
200
|
+
*/
|
|
201
|
+
sessionAgentPid?: (id: string) => number | null;
|
|
189
202
|
/**
|
|
190
203
|
* #398 S2: the stall mechanism's four doors to the machine.
|
|
191
204
|
*
|
|
@@ -765,6 +778,16 @@ export declare class Supervisor {
|
|
|
765
778
|
*/
|
|
766
779
|
private publishBackgroundHeartbeats;
|
|
767
780
|
private publishHostLoad;
|
|
781
|
+
/**
|
|
782
|
+
* Set by {@link shutdown}, and read wherever this supervisor could still be
|
|
783
|
+
* ASKED to do something after it has let go of everything (#431).
|
|
784
|
+
*
|
|
785
|
+
* It exists because the daemon now outlives `shutdown()` by up to three
|
|
786
|
+
* seconds — the time its cages are given to stop. Until that change every
|
|
787
|
+
* caller exited in the same tick and «after shutdown» was not a state
|
|
788
|
+
* anything could be in.
|
|
789
|
+
*/
|
|
790
|
+
private shuttingDown;
|
|
768
791
|
private onFrame;
|
|
769
792
|
private startSession;
|
|
770
793
|
/**
|
|
@@ -1601,8 +1624,49 @@ export declare class Supervisor {
|
|
|
1601
1624
|
*/
|
|
1602
1625
|
private sendEvent;
|
|
1603
1626
|
private reportStatus;
|
|
1604
|
-
/**
|
|
1605
|
-
|
|
1627
|
+
/**
|
|
1628
|
+
* How long the whole fleet of cages is given to stop before this process goes
|
|
1629
|
+
* anyway (#431).
|
|
1630
|
+
*
|
|
1631
|
+
* The bus has been measured at 2.7 s under load and the stops run in
|
|
1632
|
+
* parallel, so three seconds is room for all of them on a bad day. Going over
|
|
1633
|
+
* it costs nothing: `stop` has been ACCEPTED by then, and systemd finishes
|
|
1634
|
+
* the job whether or not anybody is still here to watch. systemd's own
|
|
1635
|
+
* patience with this service is `TimeoutStopSec`, far above this.
|
|
1636
|
+
*/
|
|
1637
|
+
private static readonly CAGE_STOP_BUDGET_MS;
|
|
1638
|
+
/**
|
|
1639
|
+
* Graceful daemon shutdown: kill agents, take their cages with them, keep
|
|
1640
|
+
* sessions resumable server-side.
|
|
1641
|
+
*
|
|
1642
|
+
* **Everything that has to survive this process happens BEFORE the first
|
|
1643
|
+
* `await`** — the withdrawn cards, the feed line, and the word to each agent
|
|
1644
|
+
* that it is over (a signal for Codex; for Claude, stdin closed and the SDK's
|
|
1645
|
+
* own `close()`, whose SIGTERM is a timer this process will not live to see —
|
|
1646
|
+
* on a caged machine what actually ends it is the stop below). A caller that
|
|
1647
|
+
* does not wait (the suite does not) gets exactly the shutdown it always got;
|
|
1648
|
+
* what it misses is only the stop of the cgroups. Written this way rather
|
|
1649
|
+
* than as two methods because «kill the agent» and «put its cage out» are one
|
|
1650
|
+
* action, and the shape of the code is what stops them drifting apart again
|
|
1651
|
+
* (грабля §479).
|
|
1652
|
+
*
|
|
1653
|
+
* What the caller owes in return: close the socket between the two halves.
|
|
1654
|
+
* The events above need it open, and the seconds below must not be a window
|
|
1655
|
+
* in which a `session_start` arrives at a supervisor that has already let go
|
|
1656
|
+
* of everything (`index.ts`).
|
|
1657
|
+
*/
|
|
1658
|
+
shutdown(): Promise<void>;
|
|
1659
|
+
/**
|
|
1660
|
+
* Put out every cage this daemon was holding, in parallel and on a clock.
|
|
1661
|
+
*
|
|
1662
|
+
* Until 0.65.0 nothing did this: `releaseSessionScope` hangs off the agent
|
|
1663
|
+
* process's `exit`, and on a restart the daemon is gone before that event
|
|
1664
|
+
* fires. What stayed behind was a whole tree — a build, its docker client,
|
|
1665
|
+
* their children — holding the name of a scope systemd will then refuse to
|
|
1666
|
+
* reuse, so the FIRST start of that session after the restart died as
|
|
1667
|
+
* «exited before the session was ready (code 1)» (#431).
|
|
1668
|
+
*/
|
|
1669
|
+
private stopCages;
|
|
1606
1670
|
}
|
|
1607
1671
|
/**
|
|
1608
1672
|
* The first message the agent gets.
|
package/dist/supervisor.js
CHANGED
|
@@ -28,9 +28,9 @@ import { cageAuthority } from './cage-authority.js';
|
|
|
28
28
|
import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
|
|
29
29
|
import { rememberWorkspacePath } from './environment.js';
|
|
30
30
|
import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
|
|
31
|
-
import { markScopeOomKillsSeen, readScopeHold, readScopeMemoryStatus, readSliceLimits, sessionAgentPid, sessionCage, sessionScopeUnitOf, setLiveLadderSource, stoppedProcesses, sweepOrphanSessionScopes, } from './session-cage.js';
|
|
31
|
+
import { markScopeOomKillsSeen, readScopeHold, readScopeMemoryStatus, readSliceLimits, restartStoppedCommands, sessionAgentPid, sessionCage, sessionScopeUnitOf, setLiveLadderSource, stopSessionScope, stoppedProcesses, sweepOrphanSessionScopes, } from './session-cage.js';
|
|
32
32
|
import { allocateSessionMemory, planLimitMove, SESSION_GUARANTEE_BYTES, } from './session-allocator.js';
|
|
33
|
-
import { freshStallState, pickKillCandidate, readScopeProcesses, readStallSample, setScopeProperties, signalSubtree, stallStep, STALL_GRACE_MS, STALL_MUTE_AFTER_LOWER_MS, STALL_SIGKILL_AFTER_MS, STALL_WINDOW_MS, } from './session-stall.js';
|
|
33
|
+
import { countRunningCommands, freshStallState, pickKillCandidate, readScopeProcesses, readStallSample, setScopeProperties, signalSubtree, stallStep, STALL_GRACE_MS, STALL_MUTE_AFTER_LOWER_MS, STALL_SIGKILL_AFTER_MS, STALL_WINDOW_MS, } from './session-stall.js';
|
|
34
34
|
import { machineReserveBytes } from './service-unit.js';
|
|
35
35
|
import { sessionLimitsChangedEnough, sessionLimitsHeartbeatDue, SESSION_LIMITS_HEARTBEAT_MS, } from './session-limits.js';
|
|
36
36
|
import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
|
|
@@ -1693,7 +1693,28 @@ export class Supervisor {
|
|
|
1693
1693
|
this.lastHostLoadSentAt = now;
|
|
1694
1694
|
}
|
|
1695
1695
|
}
|
|
1696
|
+
/**
|
|
1697
|
+
* Set by {@link shutdown}, and read wherever this supervisor could still be
|
|
1698
|
+
* ASKED to do something after it has let go of everything (#431).
|
|
1699
|
+
*
|
|
1700
|
+
* It exists because the daemon now outlives `shutdown()` by up to three
|
|
1701
|
+
* seconds — the time its cages are given to stop. Until that change every
|
|
1702
|
+
* caller exited in the same tick and «after shutdown» was not a state
|
|
1703
|
+
* anything could be in.
|
|
1704
|
+
*/
|
|
1705
|
+
shuttingDown = false;
|
|
1696
1706
|
async onFrame(frame) {
|
|
1707
|
+
/**
|
|
1708
|
+
* Nothing from the wire is answered once the shutdown has started.
|
|
1709
|
+
*
|
|
1710
|
+
* The socket is closed right after the synchronous half of `shutdown()`,
|
|
1711
|
+
* but closing is a handshake: a frame already in flight still lands. The
|
|
1712
|
+
* client refuses to deliver it (`ws-client.ts`) and this is the second
|
|
1713
|
+
* door, on the side that would act on it — an `event_ack` here deletes the
|
|
1714
|
+
* journal of a session this supervisor has already released.
|
|
1715
|
+
*/
|
|
1716
|
+
if (this.shuttingDown)
|
|
1717
|
+
return;
|
|
1697
1718
|
switch (frame.type) {
|
|
1698
1719
|
case 'hello_ack':
|
|
1699
1720
|
this.helloAcked = true;
|
|
@@ -1866,6 +1887,21 @@ export class Supervisor {
|
|
|
1866
1887
|
* the ask any more.
|
|
1867
1888
|
*/
|
|
1868
1889
|
withdrawReason = 'not_held') {
|
|
1890
|
+
/**
|
|
1891
|
+
* Nothing is started once this runner is going down (#431).
|
|
1892
|
+
*
|
|
1893
|
+
* The wire is already shut by then, but this door has a second caller: the
|
|
1894
|
+
* tail of `pumpEvents`, which runs a microtask after the agent's stream
|
|
1895
|
+
* ends — and since the shutdown waits for its cages, the process is still
|
|
1896
|
+
* here to run it. What it would build first is a worktree and a branch, and
|
|
1897
|
+
* what it would build last is a fresh cage, moments before `process.exit`.
|
|
1898
|
+
*/
|
|
1899
|
+
if (this.shuttingDown) {
|
|
1900
|
+
log.warn('supervisor: not starting a session, this runner is going down', {
|
|
1901
|
+
sessionId: descriptor.id,
|
|
1902
|
+
});
|
|
1903
|
+
return;
|
|
1904
|
+
}
|
|
1869
1905
|
const existing = this.sessions.get(descriptor.id);
|
|
1870
1906
|
if (existing) {
|
|
1871
1907
|
// A resume (higher epoch) can land while the previous life is still
|
|
@@ -2192,6 +2228,22 @@ export class Supervisor {
|
|
|
2192
2228
|
});
|
|
2193
2229
|
return LAUNCH_REFUSED;
|
|
2194
2230
|
}
|
|
2231
|
+
/**
|
|
2232
|
+
* …and no agent at all once the daemon is on its way out (#431).
|
|
2233
|
+
*
|
|
2234
|
+
* The five one-shot relaunches live in the tail of `pumpEvents`, which runs
|
|
2235
|
+
* a microtask after the event stream of the dying process ends — and since
|
|
2236
|
+
* the shutdown waits for its cages, the process is still here to run it.
|
|
2237
|
+
* A launch there would put a NEW cage on the machine moments before
|
|
2238
|
+
* `process.exit`, leaving behind exactly the orphan this ticket is about.
|
|
2239
|
+
* Found by the independent review of #431.
|
|
2240
|
+
*/
|
|
2241
|
+
if (this.shuttingDown) {
|
|
2242
|
+
log.warn('supervisor: not launching an agent, this runner is going down', {
|
|
2243
|
+
sessionId: descriptor.id,
|
|
2244
|
+
});
|
|
2245
|
+
return LAUNCH_REFUSED;
|
|
2246
|
+
}
|
|
2195
2247
|
// An exhausted USD budget must not relaunch $0.01-floor processes (QA-96 F4).
|
|
2196
2248
|
// Codex reports no cost at all, so its costUsd never leaves 0 — gating on it
|
|
2197
2249
|
// would be a limit that can never fire while the UI shows $0.00. Those
|
|
@@ -7676,8 +7728,40 @@ export class Supervisor {
|
|
|
7676
7728
|
// changed, and the interval in the constructor is the net under both.
|
|
7677
7729
|
this.publishSlots();
|
|
7678
7730
|
}
|
|
7679
|
-
/**
|
|
7680
|
-
|
|
7731
|
+
/**
|
|
7732
|
+
* How long the whole fleet of cages is given to stop before this process goes
|
|
7733
|
+
* anyway (#431).
|
|
7734
|
+
*
|
|
7735
|
+
* The bus has been measured at 2.7 s under load and the stops run in
|
|
7736
|
+
* parallel, so three seconds is room for all of them on a bad day. Going over
|
|
7737
|
+
* it costs nothing: `stop` has been ACCEPTED by then, and systemd finishes
|
|
7738
|
+
* the job whether or not anybody is still here to watch. systemd's own
|
|
7739
|
+
* patience with this service is `TimeoutStopSec`, far above this.
|
|
7740
|
+
*/
|
|
7741
|
+
static CAGE_STOP_BUDGET_MS = 3_000;
|
|
7742
|
+
/**
|
|
7743
|
+
* Graceful daemon shutdown: kill agents, take their cages with them, keep
|
|
7744
|
+
* sessions resumable server-side.
|
|
7745
|
+
*
|
|
7746
|
+
* **Everything that has to survive this process happens BEFORE the first
|
|
7747
|
+
* `await`** — the withdrawn cards, the feed line, and the word to each agent
|
|
7748
|
+
* that it is over (a signal for Codex; for Claude, stdin closed and the SDK's
|
|
7749
|
+
* own `close()`, whose SIGTERM is a timer this process will not live to see —
|
|
7750
|
+
* on a caged machine what actually ends it is the stop below). A caller that
|
|
7751
|
+
* does not wait (the suite does not) gets exactly the shutdown it always got;
|
|
7752
|
+
* what it misses is only the stop of the cgroups. Written this way rather
|
|
7753
|
+
* than as two methods because «kill the agent» and «put its cage out» are one
|
|
7754
|
+
* action, and the shape of the code is what stops them drifting apart again
|
|
7755
|
+
* (грабля §479).
|
|
7756
|
+
*
|
|
7757
|
+
* What the caller owes in return: close the socket between the two halves.
|
|
7758
|
+
* The events above need it open, and the seconds below must not be a window
|
|
7759
|
+
* in which a `session_start` arrives at a supervisor that has already let go
|
|
7760
|
+
* of everything (`index.ts`).
|
|
7761
|
+
*/
|
|
7762
|
+
async shutdown() {
|
|
7763
|
+
// First line: every door that could put work back on this machine reads it.
|
|
7764
|
+
this.shuttingDown = true;
|
|
7681
7765
|
clearInterval(this.slotsTimer);
|
|
7682
7766
|
clearInterval(this.hostLoadTimer);
|
|
7683
7767
|
clearInterval(this.stallTimer);
|
|
@@ -7686,6 +7770,10 @@ export class Supervisor {
|
|
|
7686
7770
|
clearTimeout(this.agentCleanupFirstTimer);
|
|
7687
7771
|
clearInterval(this.agentCleanupTimer);
|
|
7688
7772
|
this.authRelay.cancel();
|
|
7773
|
+
const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
|
|
7774
|
+
const readProcesses = this.opts.readScopeProcesses ?? readScopeProcesses;
|
|
7775
|
+
const agentPidOf = this.opts.sessionAgentPid ?? sessionAgentPid;
|
|
7776
|
+
const cages = [];
|
|
7689
7777
|
for (const running of this.sessions.values()) {
|
|
7690
7778
|
this.clearBudgetTimers(running);
|
|
7691
7779
|
// BEFORE stop(), and from here rather than from the adapter: the adapter
|
|
@@ -7707,15 +7795,86 @@ export class Supervisor {
|
|
|
7707
7795
|
error: String(error),
|
|
7708
7796
|
});
|
|
7709
7797
|
}
|
|
7798
|
+
/**
|
|
7799
|
+
* What this restart costs, in the person's units, before anything is
|
|
7800
|
+
* killed (#431).
|
|
7801
|
+
*
|
|
7802
|
+
* The count comes from the cage itself rather than from anything this
|
|
7803
|
+
* process believes: the runner does not track what the agent started —
|
|
7804
|
+
* that is the whole shape of the problem — and the cgroup does. Read
|
|
7805
|
+
* BEFORE `stop()`, because a moment later there is nothing left to count.
|
|
7806
|
+
*
|
|
7807
|
+
* Silent when the number is zero: an agent that was only thinking loses
|
|
7808
|
+
* nothing, and «0 commands were stopped» is a line a person has to read
|
|
7809
|
+
* and cannot use (D8).
|
|
7810
|
+
*/
|
|
7811
|
+
const unit = unitOf(running.descriptor.id);
|
|
7812
|
+
if (unit !== null) {
|
|
7813
|
+
cages.push(unit);
|
|
7814
|
+
try {
|
|
7815
|
+
const commands = countRunningCommands(readProcesses(unit), agentPidOf(running.descriptor.id));
|
|
7816
|
+
if (commands > 0) {
|
|
7817
|
+
this.sendEvent(running, 'notice', {
|
|
7818
|
+
level: 'warn',
|
|
7819
|
+
text: restartStoppedCommands(commands),
|
|
7820
|
+
});
|
|
7821
|
+
}
|
|
7822
|
+
}
|
|
7823
|
+
catch (error) {
|
|
7824
|
+
// /proc said nothing readable. The cage is still stopped below — the
|
|
7825
|
+
// person simply does not get the number.
|
|
7826
|
+
log.warn('supervisor: could not count what this session was running', {
|
|
7827
|
+
sessionId: running.descriptor.id,
|
|
7828
|
+
error: String(error),
|
|
7829
|
+
});
|
|
7830
|
+
}
|
|
7831
|
+
}
|
|
7710
7832
|
// The reason matters here: this path also runs for «Update runner», and a
|
|
7711
7833
|
// card that vanishes during an update must say why (session 12).
|
|
7712
|
-
|
|
7834
|
+
//
|
|
7835
|
+
// Guarded like its neighbours in this loop: one adapter that throws on
|
|
7836
|
+
// its way out must not cost every OTHER session the stop of its cage —
|
|
7837
|
+
// the lines below it are what puts them out (independent review of #431).
|
|
7838
|
+
try {
|
|
7839
|
+
running.session?.stop('runner_restarted');
|
|
7840
|
+
}
|
|
7841
|
+
catch (error) {
|
|
7842
|
+
log.warn('supervisor: an agent did not take its stop cleanly', {
|
|
7843
|
+
sessionId: running.descriptor.id,
|
|
7844
|
+
error: String(error),
|
|
7845
|
+
});
|
|
7846
|
+
}
|
|
7713
7847
|
}
|
|
7714
7848
|
this.sessions.clear();
|
|
7715
7849
|
// A build that was mid-flight is abandoned, not judged: its row stays
|
|
7716
7850
|
// RUNNING until the API's sweep turns it into LOST. A verdict nobody
|
|
7717
7851
|
// observed must never become PASSED.
|
|
7718
7852
|
this.verify.shutdown();
|
|
7853
|
+
await this.stopCages(cages);
|
|
7854
|
+
}
|
|
7855
|
+
/**
|
|
7856
|
+
* Put out every cage this daemon was holding, in parallel and on a clock.
|
|
7857
|
+
*
|
|
7858
|
+
* Until 0.65.0 nothing did this: `releaseSessionScope` hangs off the agent
|
|
7859
|
+
* process's `exit`, and on a restart the daemon is gone before that event
|
|
7860
|
+
* fires. What stayed behind was a whole tree — a build, its docker client,
|
|
7861
|
+
* their children — holding the name of a scope systemd will then refuse to
|
|
7862
|
+
* reuse, so the FIRST start of that session after the restart died as
|
|
7863
|
+
* «exited before the session was ready (code 1)» (#431).
|
|
7864
|
+
*/
|
|
7865
|
+
async stopCages(units) {
|
|
7866
|
+
if (units.length === 0)
|
|
7867
|
+
return;
|
|
7868
|
+
const stop = this.opts.stopSessionScope ?? stopSessionScope;
|
|
7869
|
+
log.info('supervisor: stopping the cages of this runner', { count: units.length });
|
|
7870
|
+
const all = Promise.allSettled(units.map((unit) => stop(unit))).then(() => undefined);
|
|
7871
|
+
let timer;
|
|
7872
|
+
const budget = new Promise((resolve) => {
|
|
7873
|
+
timer = setTimeout(resolve, this.opts.cageStopBudgetMs ?? Supervisor.CAGE_STOP_BUDGET_MS);
|
|
7874
|
+
});
|
|
7875
|
+
await Promise.race([all, budget]);
|
|
7876
|
+
if (timer)
|
|
7877
|
+
clearTimeout(timer);
|
|
7719
7878
|
}
|
|
7720
7879
|
}
|
|
7721
7880
|
/**
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const RUNNER_VERSION = "0.65.
|
|
1
|
+
export declare const RUNNER_VERSION = "0.65.1";
|
|
2
2
|
//# sourceMappingURL=version.d.ts.map
|
package/dist/version.js
CHANGED
package/dist/ws-client.js
CHANGED
|
@@ -137,6 +137,20 @@ export class RunnerWsClient {
|
|
|
137
137
|
this.lastInboundAt = Date.now();
|
|
138
138
|
});
|
|
139
139
|
socket.on('message', (data) => {
|
|
140
|
+
/**
|
|
141
|
+
* A client that was stopped answers nothing, in either direction (#431).
|
|
142
|
+
*
|
|
143
|
+
* `close()` is the START of a handshake, not a cut: the socket lives on
|
|
144
|
+
* for a round trip, and this listener used to keep delivering frames off
|
|
145
|
+
* it. That was harmless while every caller of `stop()` exited in the same
|
|
146
|
+
* tick; since the daemon waits for its cages to be put out, the window is
|
|
147
|
+
* seconds wide, and the supervisor on the other side of `frame` has
|
|
148
|
+
* already let go of every session. An `event_ack` landing there DELETES
|
|
149
|
+
* the journal of a live session (found by the independent review of
|
|
150
|
+
* #431); a `session_start` starts an agent nobody will ever stop.
|
|
151
|
+
*/
|
|
152
|
+
if (this.stopped)
|
|
153
|
+
return;
|
|
140
154
|
this.lastInboundAt = Date.now();
|
|
141
155
|
let parsed;
|
|
142
156
|
try {
|
package/package.json
CHANGED