@bridge4dev/runner 0.54.0 → 0.55.0

package/dist/supervisor.d.ts

@@ -5,6 +5,7 @@ import { installAgent } from './agent-install.js';
 import { pruneNativeClaudeVersions } from './agent-cleanup.js';
 import { type AgentVersionsMeasurement } from './agent-versions.js';
 import { type HostLoadFrame } from './host-load.js';
+import { type ScopeMemoryStatus } from './session-cage.js';
 import type { RunnerWsClient } from './ws-client.js';
 import type { SessionDescriptor } from './protocol.js';
 import type { AgentAdapter } from './adapters/types.js';
@@ -146,6 +147,14 @@ export interface SupervisorOptions {
      * still reports» would be an untested promise. Never set in production.
      */
     hostLoadHeartbeatMs?: number;
+    /**
+     * #387: the memory watch's two readers, so a test can put a caged session
+     * over its brake without a cgroup under it. Production reads the cage module.
+     */
+    sessionScopeUnitOf?: (id: string) => string | null;
+    readScopeMemoryStatus?: (unit: string) => ScopeMemoryStatus | null;
+    /** How long over the brake before the second, «do something» warning. A test seam. */
+    cageBrakeLongMs?: number;
 }
 export declare class Supervisor {
     private readonly ws;
@@ -222,6 +231,23 @@ export declare class Supervisor {
     private readonly verify;
     private readonly verifyReports;
     constructor(ws: RunnerWsClient, opts: SupervisorOptions);
+    /**
+     * How long a session may sit over its brake before the feed says more than
+     * «slower» (#387, step 3). Five minutes is a starting point, not a
+     * measurement: an honest `pnpm typecheck` overshoots for tens of seconds, a
+     * runaway for ever, and the number that tells them apart on real machines is
+     * still to be read off `memory.events`. Sized so the first warning is never
+     * the only one a night-time run leaves behind.
+     */
+    private static readonly CAGE_BRAKE_LONG_MS;
+    /**
+     * Calm ticks before the feed says the braking is over.
+     *
+     * `memory.high` holds usage AT the line, so a single sample under it means
+     * nothing. Two (a minute at the 30 s cadence) is the difference between an
+     * episode that ended and a build that breathed.
+     */
+    private static readonly CAGE_CALM_TICKS_TO_RELEASE;
     /** How often the agent versions are re-derived. See the constructor. */
     private static readonly AGENT_VERSIONS_INTERVAL_MS;
     private readonly agentVersionsTimer;
@@ -402,6 +428,29 @@ export declare class Supervisor {
      * reconnect the API would have no measurement at all while this side believed
      * it had one.
      */
+    /**
+     * Tell the person what the kernel is doing to each caged session (#387).
+     *
+     * Three things the cgroup can say, each with its own line in the feed:
+     *
+     *  - the brake engaged (`memory.current` over `memory.high`): the session is
+     *    being slowed down, not killed. Said once per episode, and again when it
+     *    has lasted `CAGE_BRAKE_LONG_MS` — that second line is the one that
+     *    suggests an action, because by then «slower» may mean «stuck»;
+     *  - the brake released: said once, so a person who saw the warning knows the
+     *    wait is over without reading the numbers;
+     *  - `oom_kill` moved: the kernel killed a process inside the session. With
+     *    `OOMPolicy=continue` the session itself is still here, and the person
+     *    has to be told that the command that was running is probably what died.
+     *
+     * Until this tick existed the runner knew all three and wrote them to its own
+     * log only; the feed said «exited with code 143» and nothing about memory.
+     *
+     * A parked session has no process and no cgroup, and is skipped without a
+     * read. A session whose cgroup cannot be read (no cage on this machine, or
+     * the scope already gone) is skipped the same way: the watch never guesses.
+     */
+    private watchSessionCages;
     private publishHostLoad;
     private onFrame;
     private startSession;

package/dist/supervisor.js

@@ -26,6 +26,7 @@ import { agentByDbValue } from './agent-registry.js';
 import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
 import { rememberWorkspacePath } from './environment.js';
 import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
+import { markScopeOomKillsSeen, readScopeMemoryStatus, sessionScopeUnitOf, } from './session-cage.js';
 import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
 import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, MAX_BUSY_SESSIONS, previewRewind, pruneCheckpoints, } from './checkpoints.js';
 import { DeliverMessageArgsSchema, QuestionAnswerArgsSchema } from './protocol.js';
@@ -223,9 +224,31 @@ export class Supervisor {
          * actually moved (or the heartbeat came due).
          */
         this.hostLoadHeartbeatMs = opts.hostLoadHeartbeatMs ?? HOST_LOAD_HEARTBEAT_MS;
-        this.hostLoadTimer = setInterval(() => this.publishHostLoad(), opts.hostLoadSampleMs ?? HOST_LOAD_SAMPLE_INTERVAL_MS);
+        this.hostLoadTimer = setInterval(() => {
+            this.publishHostLoad();
+            // Same cadence, same reason: what a session's cgroup is going through is
+            // something that happens to it, and only a tick can notice (#387).
+            this.watchSessionCages();
+        }, opts.hostLoadSampleMs ?? HOST_LOAD_SAMPLE_INTERVAL_MS);
         this.hostLoadTimer.unref?.();
     }
+    /**
+     * How long a session may sit over its brake before the feed says more than
+     * «slower» (#387, step 3). Five minutes is a starting point, not a
+     * measurement: an honest `pnpm typecheck` overshoots for tens of seconds, a
+     * runaway for ever, and the number that tells them apart on real machines is
+     * still to be read off `memory.events`. Sized so the first warning is never
+     * the only one a night-time run leaves behind.
+     */
+    static CAGE_BRAKE_LONG_MS = 5 * 60_000;
+    /**
+     * Calm ticks before the feed says the braking is over.
+     *
+     * `memory.high` holds usage AT the line, so a single sample under it means
+     * nothing. Two (a minute at the 30 s cadence) is the difference between an
+     * episode that ended and a build that breathed.
+     */
+    static CAGE_CALM_TICKS_TO_RELEASE = 2;
     /** How often the agent versions are re-derived. See the constructor. */
     static AGENT_VERSIONS_INTERVAL_MS = 60 * 60 * 1_000;
     agentVersionsTimer;
@@ -620,6 +643,154 @@ export class Supervisor {
      * reconnect the API would have no measurement at all while this side believed
      * it had one.
      */
+    /**
+     * Tell the person what the kernel is doing to each caged session (#387).
+     *
+     * Three things the cgroup can say, each with its own line in the feed:
+     *
+     *  - the brake engaged (`memory.current` over `memory.high`): the session is
+     *    being slowed down, not killed. Said once per episode, and again when it
+     *    has lasted `CAGE_BRAKE_LONG_MS` — that second line is the one that
+     *    suggests an action, because by then «slower» may mean «stuck»;
+     *  - the brake released: said once, so a person who saw the warning knows the
+     *    wait is over without reading the numbers;
+     *  - `oom_kill` moved: the kernel killed a process inside the session. With
+     *    `OOMPolicy=continue` the session itself is still here, and the person
+     *    has to be told that the command that was running is probably what died.
+     *
+     * Until this tick existed the runner knew all three and wrote them to its own
+     * log only; the feed said «exited with code 143» and nothing about memory.
+     *
+     * A parked session has no process and no cgroup, and is skipped without a
+     * read. A session whose cgroup cannot be read (no cage on this machine, or
+     * the scope already gone) is skipped the same way: the watch never guesses.
+     */
+    watchSessionCages() {
+        const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
+        const read = this.opts.readScopeMemoryStatus ?? readScopeMemoryStatus;
+        const longMs = this.opts.cageBrakeLongMs ?? Supervisor.CAGE_BRAKE_LONG_MS;
+        const now = Date.now();
+        const mb = (bytes) => `${Math.round(bytes / (1024 * 1024))} MB`;
+        for (const running of this.sessions.values()) {
+            if (running.session === null)
+                continue;
+            const unit = unitOf(running.descriptor.id);
+            if (unit === null)
+                continue;
+            const status = read(unit);
+            if (status === null)
+                continue;
+            const seen = running.cageWatch;
+            /**
+             * A cgroup this watch has not seen before starts from zero.
+             *
+             * Two ways that happens, and the second is why the counters are checked
+             * as well as the name: a session relaunched after its scope was released
+             * gets the SAME unit name back (`sessionScopeUnit` falls back to the
+             * plain name once the failed unit is reset), so «same name» does not mean
+             * «same cgroup». Counters that went backwards can only be a new one —
+             * without this the episode of the dead process carried over, and the
+             * feed opened with «memory is back under the share» for a process that
+             * had never been over it (#387 QA).
+             */
+            const carriedOver = seen !== undefined &&
+                seen.unit === unit &&
+                status.oomKills >= seen.oomKills &&
+                status.highEvents >= seen.highEvents &&
+                status.ownLimitOom >= seen.ownOom;
+            const base = carriedOver
+                ? seen
+                : {
+                    unit,
+                    oomKills: 0,
+                    ownOom: 0,
+                    highEvents: 0,
+                    brakedSince: null,
+                    warnedLong: false,
+                    calmTicks: 0,
+                };
+            if (status.oomKills > base.oomKills) {
+                // WHOSE limit was hit decides the sentence. `oom` moves only in the
+                // cgroup whose own ceiling was reached; a kill with our counter still
+                // where it was came from the slice over ALL sessions, and telling this
+                // session it went over its ceiling would be false (measured on this
+                // host: the killed neighbour read `max 0, oom 0, oom_kill 1`).
+                const ownWall = status.ownLimitOom > base.ownOom;
+                const killed = status.oomKills - base.oomKills === 1
+                    ? 'a process'
+                    : `${status.oomKills - base.oomKills} processes`;
+                this.sendEvent(running, 'notice', {
+                    level: 'warn',
+                    text: ownWall
+                        ? `This session went over its memory ceiling${status.maxBytes === null ? '' : ` of ${mb(status.maxBytes)}`} and the kernel killed ${killed} inside it. ` +
+                            'The session itself is still running; the command that was in flight most likely died with it — check its output before going on.'
+                        : `The machine ran out of memory for agent sessions — they share one ceiling — and the kernel killed ${killed} in this session. ` +
+                            `This session was using ${mb(status.currentBytes)}${status.maxBytes === null ? '' : ` of its own ${mb(status.maxBytes)}`}, so it was not necessarily the greedy one. ` +
+                            'It is still running; the command that was in flight most likely died with it.',
+                });
+                // So a death minutes later is not blamed on a kill the feed already carries.
+                markScopeOomKillsSeen(running.descriptor.id, status.oomKills);
+            }
+            /**
+             * «Being braked» is the EVENT counter first, the level second.
+             *
+             * `memory.high` works by holding usage at the line: a 30 s sample of
+             * `current > high` lands over it and under it in turn, which produced a
+             * warning and an all-clear on alternate ticks for one long build. The
+             * `high` counter only ever moves when the kernel actually throttled this
+             * cgroup, so a delta is the honest answer to «is it being slowed down
+             * right now», and the level is kept as the answer for a session that is
+             * sitting over the line without allocating.
+             */
+            const throttled = status.highEvents > base.highEvents ||
+                (status.highBytes !== null && status.currentBytes > status.highBytes);
+            let { brakedSince, warnedLong, calmTicks } = base;
+            if (throttled) {
+                calmTicks = 0;
+                if (brakedSince === null) {
+                    brakedSince = now;
+                    warnedLong = false;
+                    this.sendEvent(running, 'notice', {
+                        level: 'warn',
+                        text: `This session is over its memory share (${mb(status.highBytes ?? 0)}, using ${mb(status.currentBytes)}), so the kernel is slowing it down instead of killing it. ` +
+                            'Heavy commands will finish slower until memory is freed.',
+                    });
+                }
+                else if (!warnedLong && now - brakedSince >= longMs) {
+                    warnedLong = true;
+                    this.sendEvent(running, 'notice', {
+                        level: 'warn',
+                        text: `Still over its memory share after ${Math.round(longMs / 60_000)} min (using ${mb(status.currentBytes)} of a ${mb(status.highBytes ?? 0)} share). ` +
+                            `If the current command is stuck rather than slow, stop it or pause the session.${status.maxBytes === null
+                                ? ''
+                                : ` The hard ceiling, where a process would be killed, is ${mb(status.maxBytes)}.`}`,
+                    });
+                }
+            }
+            else if (brakedSince !== null) {
+                // One calm sample is noise — see above. Two in a row is an episode that
+                // really ended, and only then is the all-clear worth a line in the feed.
+                calmTicks += 1;
+                if (calmTicks >= Supervisor.CAGE_CALM_TICKS_TO_RELEASE) {
+                    brakedSince = null;
+                    warnedLong = false;
+                    this.sendEvent(running, 'notice', {
+                        level: 'info',
+                        text: `Memory is back under the session's share (${mb(status.highBytes ?? 0)}); it runs at full speed again.`,
+                    });
+                }
+            }
+            running.cageWatch = {
+                unit,
+                oomKills: status.oomKills,
+                ownOom: status.ownLimitOom,
+                highEvents: status.highEvents,
+                brakedSince,
+                warnedLong,
+                calmTicks,
+            };
+        }
+    }
     publishHostLoad() {
         const sample = (this.opts.readHostLoad ?? readHostLoad)();
         if (!sample)

package/dist/version.d.ts

@@ -1,2 +1,2 @@
-export declare const RUNNER_VERSION = "0.54.0";
+export declare const RUNNER_VERSION = "0.55.0";
 //# sourceMappingURL=version.d.ts.map

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in sync with package.json by the release script (manual for now).
-export const RUNNER_VERSION = '0.54.0';
+export const RUNNER_VERSION = '0.55.0';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bridge4dev/runner",
-  "version": "0.54.0",
+  "version": "0.55.0",
   "description": "DevBridge dev runner — connects a dev server to DevBridge and runs agent sessions (Claude Code / Codex)",
   "homepage": "https://bridge4.dev",
   "license": "MIT",