@bridge4dev/runner 0.53.0 → 0.54.0

package/dist/supervisor.d.ts

@@ -4,6 +4,7 @@ import { selfUpdate, type SelfUpdateOutcome } from './self-update.js';
 import { installAgent } from './agent-install.js';
 import { pruneNativeClaudeVersions } from './agent-cleanup.js';
 import { type AgentVersionsMeasurement } from './agent-versions.js';
+import { type HostLoadFrame } from './host-load.js';
 import type { RunnerWsClient } from './ws-client.js';
 import type { SessionDescriptor } from './protocol.js';
 import type { AgentAdapter } from './adapters/types.js';
@@ -122,6 +123,29 @@ export interface SupervisorOptions {
      * in production.
      */
     stopSettleMs?: number;
+    /**
+     * Test seam for the host-load measurement (§5.3).
+     *
+     * A seam because the real one reads THIS machine's `/proc`, and a test that
+     * had to arrange the kernel into «load 12.4, swap 98 % used» would not be
+     * written — so the rule this frame exists for (send only what moved) would
+     * go unverified on the one machine shape that matters.
+     */
+    readHostLoad?: () => HostLoadFrame | null;
+    /**
+     * How often `/proc` is sampled, in ms — a test seam over
+     * `HOST_LOAD_SAMPLE_INTERVAL_MS`, for the same reason `agentCleanupMs`
+     * exists: the real cadence is 30 s and this suite runs on real timers (fake
+     * ones fire the WS client's liveness watchdog and kill the socket). Never
+     * set in production.
+     */
+    hostLoadSampleMs?: number;
+    /**
+     * The heartbeat window, in ms — a test seam over `HOST_LOAD_HEARTBEAT_MS`.
+     * The real one is five minutes; without this seam «an unchanged machine
+     * still reports» would be an untested promise. Never set in production.
+     */
+    hostLoadHeartbeatMs?: number;
 }
 export declare class Supervisor {
     private readonly ws;
@@ -346,6 +370,39 @@ export declare class Supervisor {
      * it had room. Both were caught by the independent QA review of this change.
      */
     private publishSlots;
+    /** The heartbeat window actually used — the constant, or a test's own. */
+    private readonly hostLoadHeartbeatMs;
+    private readonly hostLoadTimer;
+    /** The last measurement the API actually took from us, or `null` for «nothing yet». */
+    private lastPublishedHostLoad;
+    /** When that frame went out, by this machine's clock. `0` = never. */
+    private lastHostLoadSentAt;
+    /**
+     * Tell the API what this machine's load looks like — when it is worth telling.
+     *
+     * Three ways a frame goes out, and each covers a hole the others leave:
+     *
+     *  - the value MOVED past a threshold (`hostLoadChangedEnough`) — the reason
+     *    the frame exists, and the only one that makes the card timely;
+     *  - the heartbeat came due — an idle machine still has to say «still here,
+     *    still idle», because the API expires the measurement after two minutes
+     *    and silence would otherwise turn «nothing is happening» into «we have no
+     *    idea» on a perfectly healthy card;
+     *  - nothing was ever sent on this connection (`hello_ack` clears the memory)
+     *    — a fresh socket knows nothing about what the last one was told.
+     *
+     * `null` from the sampler is a complete answer: not Linux, `/proc` masked, a
+     * kernel too old for `MemAvailable`. Nothing is sent and nothing is logged —
+     * a machine that cannot measure itself leaves the card saying «no data»,
+     * which is exactly what is true.
+     *
+     * Recorded ONLY when the socket took it, same as `publishSlots`: a frame
+     * dropped by a dead socket must not be remembered as sent, or a machine whose
+     * load never moves again would go silent until the heartbeat — and on a
+     * reconnect the API would have no measurement at all while this side believed
+     * it had one.
+     */
+    private publishHostLoad;
     private onFrame;
     private startSession;
     /**

package/dist/supervisor.js

@@ -25,6 +25,7 @@ import { pruneNativeClaudeVersions } from './agent-cleanup.js';
 import { agentByDbValue } from './agent-registry.js';
 import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
 import { rememberWorkspacePath } from './environment.js';
+import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
 import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
 import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, MAX_BUSY_SESSIONS, previewRewind, pruneCheckpoints, } from './checkpoints.js';
 import { DeliverMessageArgsSchema, QuestionAnswerArgsSchema } from './protocol.js';
@@ -209,6 +210,21 @@ export class Supervisor {
         this.agentCleanupFirstTimer.unref?.();
         this.agentCleanupTimer = setInterval(() => this.sweepOldAgentVersions(), cleanupEvery);
         this.agentCleanupTimer.unref?.();
+        /**
+         * The machine's own load, sampled on a timer for the reason the seat report
+         * is: nothing on this side is an EVENT. Load is not something the runner
+         * does, it is something that happens to it — a `pnpm build` a person
+         * started over ssh, a neighbour container, this machine's own three agents
+         * — and there is no call site to hook. A tick that re-reads the truth is
+         * the only shape that cannot be forgotten.
+         *
+         * Nearly free, and quiet by default: two small files are read, and
+         * `publishHostLoad` returns without touching the socket unless the number
+         * actually moved (or the heartbeat came due).
+         */
+        this.hostLoadHeartbeatMs = opts.hostLoadHeartbeatMs ?? HOST_LOAD_HEARTBEAT_MS;
+        this.hostLoadTimer = setInterval(() => this.publishHostLoad(), opts.hostLoadSampleMs ?? HOST_LOAD_SAMPLE_INTERVAL_MS);
+        this.hostLoadTimer.unref?.();
     }
     /** How often the agent versions are re-derived. See the constructor. */
     static AGENT_VERSIONS_INTERVAL_MS = 60 * 60 * 1_000;
@@ -572,6 +588,53 @@ export class Supervisor {
             this.lastPublishedSlots = fingerprint;
         }
     }
+    /** The heartbeat window actually used — the constant, or a test's own. */
+    hostLoadHeartbeatMs;
+    hostLoadTimer;
+    /** The last measurement the API actually took from us, or `null` for «nothing yet». */
+    lastPublishedHostLoad = null;
+    /** When that frame went out, by this machine's clock. `0` = never. */
+    lastHostLoadSentAt = 0;
+    /**
+     * Tell the API what this machine's load looks like — when it is worth telling.
+     *
+     * Three ways a frame goes out, and each covers a hole the others leave:
+     *
+     *  - the value MOVED past a threshold (`hostLoadChangedEnough`) — the reason
+     *    the frame exists, and the only one that makes the card timely;
+     *  - the heartbeat came due — an idle machine still has to say «still here,
+     *    still idle», because the API expires the measurement after two minutes
+     *    and silence would otherwise turn «nothing is happening» into «we have no
+     *    idea» on a perfectly healthy card;
+     *  - nothing was ever sent on this connection (`hello_ack` clears the memory)
+     *    — a fresh socket knows nothing about what the last one was told.
+     *
+     * `null` from the sampler is a complete answer: not Linux, `/proc` masked, a
+     * kernel too old for `MemAvailable`. Nothing is sent and nothing is logged —
+     * a machine that cannot measure itself leaves the card saying «no data»,
+     * which is exactly what is true.
+     *
+     * Recorded ONLY when the socket took it, same as `publishSlots`: a frame
+     * dropped by a dead socket must not be remembered as sent, or a machine whose
+     * load never moves again would go silent until the heartbeat — and on a
+     * reconnect the API would have no measurement at all while this side believed
+     * it had one.
+     */
+    publishHostLoad() {
+        const sample = (this.opts.readHostLoad ?? readHostLoad)();
+        if (!sample)
+            return;
+        const now = Date.now();
+        const sinceLastSent = now - this.lastHostLoadSentAt;
+        // A backwards clock step is «due», not «early» — see `hostLoadHeartbeatDue`.
+        const heartbeatDue = hostLoadHeartbeatDue(sinceLastSent, this.hostLoadHeartbeatMs);
+        if (!heartbeatDue && !hostLoadChangedEnough(this.lastPublishedHostLoad, sample))
+            return;
+        if (this.ws.send({ type: 'host_load', ...sample })) {
+            this.lastPublishedHostLoad = sample;
+            this.lastHostLoadSentAt = now;
+        }
+    }
     async onFrame(frame) {
         switch (frame.type) {
             case 'hello_ack':
@@ -580,9 +643,17 @@ export class Supervisor {
                 // because it is only true while the socket is. Forget what we told the
                 // old one so the first tick after this reconnect actually sends.
                 this.lastPublishedSlots = '';
+                // Same rule for the load: the API keeps the measurement in Redis beside
+                // the socket, with a TTL shorter than our heartbeat, so a machine that
+                // has just come back has no load on its card at all. Forget what the
+                // last connection was told and say it again immediately — a card that
+                // is right only after five minutes of silence is not right.
+                this.lastPublishedHostLoad = null;
+                this.lastHostLoadSentAt = 0;
                 this.setMaxSessions(frame.maxSessions);
                 await this.reconcile(frame.sessions);
                 this.publishSlots();
+                this.publishHostLoad();
                 // A build that finished while the socket was down has its verdict
                 // sitting on disk. This is the moment it can be delivered.
                 this.flushVerifyReports();
@@ -5805,6 +5876,7 @@ export class Supervisor {
     /** Graceful daemon shutdown: kill agents, keep sessions resumable server-side. */
     shutdown() {
         clearInterval(this.slotsTimer);
+        clearInterval(this.hostLoadTimer);
         clearInterval(this.agentVersionsTimer);
         clearTimeout(this.agentCleanupFirstTimer);
         clearInterval(this.agentCleanupTimer);

package/dist/systemd-memory.d.ts

@@ -0,0 +1,35 @@
+import { type CgroupMemory, type MemoryFacts } from './service-unit.js';
+/**
+ * Read the unit's group, never the CLI's own `/proc/self` group. A missing
+ * reading is not an empty service: a timed-out user bus can belong to a busy
+ * machine. Raw `memory.current` deliberately survives beside the split, so the
+ * safety floor never depends on an estimate of how much can be reclaimed.
+ */
+export declare function unitMemoryReading(output: string, cgroupRoot?: string): CgroupMemory | null;
+/**
+ * Both units, measured through systemd, with the filesystem underneath.
+ *
+ * One helper so `doctor`, `doctor --fix`, `install-service` and the daemon's
+ * hourly re-measure cannot drift apart in what they measure — which is how the
+ * CLI once computed a ceiling 34 % away from the daemon's and the two rewrote
+ * the file forever.
+ *
+ * The fallback is not a second opinion, it is the same reading taken another
+ * way, and it is safe for every caller precisely because each reader
+ * self-identifies: `readOwnCgroupMemory` hands back null for any process that
+ * is not the service itself, so the daemon measures itself when the bus is
+ * unhappy and a CLI never mistakes its own `session-N.scope` for the service.
+ * Without it, one slow `systemctl` meant the daemon wrote no policy at all —
+ * for an hour, on the overloaded machine this policy exists to protect.
+ */
+export declare function readMemoryFactsFromSystemd(): Promise<MemoryReadings>;
+export interface MemoryReadings {
+    facts: MemoryFacts | null;
+    sessionsUsageBytes: number | null;
+}
+/** The half of the above that has no bus in it, so it can be tested. */
+export declare function memoryReadings(service: CgroupMemory | null, sessions: CgroupMemory | null, fromFilesystem?: {
+    own: () => CgroupMemory | null;
+    sessions: () => CgroupMemory | null;
+}): MemoryReadings;
+//# sourceMappingURL=systemd-memory.d.ts.map

package/dist/systemd-memory.js

@@ -0,0 +1,115 @@
+import { execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { systemdUserEnv } from './environment.js';
+import { readCgroupMemory, readMemoryFacts, readOwnCgroupMemory, readSessionsSliceMemory, SERVICE_NAME, SESSIONS_SLICE, } from './service-unit.js';
+const execFileAsync = promisify(execFile);
+function bytes(raw) {
+    if (raw === undefined || !/^\d+$/.test(raw.trim()))
+        return null;
+    const value = Number(raw);
+    // systemd prints UINT64_MAX for «not set» on some versions, which is above
+    // `MAX_SAFE_INTEGER` and would otherwise arrive as a plausible byte count.
+    return Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+/**
+ * Read the unit's group, never the CLI's own `/proc/self` group. A missing
+ * reading is not an empty service: a timed-out user bus can belong to a busy
+ * machine. Raw `memory.current` deliberately survives beside the split, so the
+ * safety floor never depends on an estimate of how much can be reclaimed.
+ */
+export function unitMemoryReading(output, cgroupRoot = '/sys/fs/cgroup') {
+    const properties = new Map(output.split('\n').map((line) => {
+        const separator = line.indexOf('=');
+        return [line.slice(0, separator), line.slice(separator + 1).trim()];
+    }));
+    // An empty answer is not a fact about the unit: everything below reads meaning
+    // into what systemd said, so there has to have been an answer first. A dead
+    // bus prints nothing and stays «unknown». `ControlGroup` is not required to be
+    // PRESENT — only to be empty where it decides emptiness below — so a systemd
+    // that does not print it leaves «unknown» too, and the filesystem reader
+    // behind this one covers that machine.
+    if (!properties.has('ActiveState'))
+        return null;
+    const current = bytes(properties.get('MemoryCurrent'));
+    const group = properties.get('ControlGroup');
+    if (group !== undefined && group.startsWith('/') && group !== '/') {
+        const root = path.resolve(cgroupRoot);
+        const directory = path.resolve(root, `.${group}`);
+        if (directory.startsWith(`${root}${path.sep}`)) {
+            try {
+                const measured = readCgroupMemory(directory);
+                if (measured !== null)
+                    return measured;
+                const raw = bytes(fs.readFileSync(path.join(directory, 'memory.current'), 'utf8'));
+                // Without `memory.stat` the split between «cache we can give back» and
+                // «memory that has to be killed for» is unknown, and null says so: the
+                // floor then assumes all of it, the headroom none of it.
+                if (raw !== null)
+                    return { currentBytes: raw, unreclaimableBytes: null };
+            }
+            catch {
+                // Unsupported controller, removed cgroup or unreadable file: unknown.
+            }
+        }
+    }
+    if (current !== null)
+        return { currentBytes: current, unreclaimableBytes: null };
+    // A unit with NO cgroup holds nothing a new ceiling could kill, whatever its
+    // ActiveState says — and that is the whole point of not asking for
+    // `inactive` here. `failed` and `activating (auto-restart)` report exactly
+    // the shape `inactive` does (verified on systemd 255: `ControlGroup=`,
+    // `MemoryCurrent=[not set]`), and those are precisely the states a machine is
+    // in when someone runs `install.sh --repair` or `doctor --fix` on it. Reading
+    // them as «unknown» turned the cure into a refusal on the only machines that
+    // need it.
+    return group === '' ? { currentBytes: 0, unreclaimableBytes: 0 } : null;
+}
+async function readUnitUsage(unit) {
+    try {
+        const { stdout } = await execFileAsync('systemctl', ['--user', 'show', unit, '-p', 'MemoryCurrent', '-p', 'ActiveState', '-p', 'ControlGroup'], { timeout: 10_000, env: systemdUserEnv() });
+        return unitMemoryReading(stdout);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Both units, measured through systemd, with the filesystem underneath.
+ *
+ * One helper so `doctor`, `doctor --fix`, `install-service` and the daemon's
+ * hourly re-measure cannot drift apart in what they measure — which is how the
+ * CLI once computed a ceiling 34 % away from the daemon's and the two rewrote
+ * the file forever.
+ *
+ * The fallback is not a second opinion, it is the same reading taken another
+ * way, and it is safe for every caller precisely because each reader
+ * self-identifies: `readOwnCgroupMemory` hands back null for any process that
+ * is not the service itself, so the daemon measures itself when the bus is
+ * unhappy and a CLI never mistakes its own `session-N.scope` for the service.
+ * Without it, one slow `systemctl` meant the daemon wrote no policy at all —
+ * for an hour, on the overloaded machine this policy exists to protect.
+ */
+export async function readMemoryFactsFromSystemd() {
+    const [service, sessions] = await Promise.all([
+        readUnitUsage(SERVICE_NAME),
+        readUnitUsage(SESSIONS_SLICE),
+    ]);
+    return memoryReadings(service, sessions);
+}
+/** The half of the above that has no bus in it, so it can be tested. */
+export function memoryReadings(service, sessions, fromFilesystem = { own: readOwnCgroupMemory, sessions: readSessionsSliceMemory }) {
+    const serviceMemory = service ?? fromFilesystem.own();
+    const sessionsMemory = sessions ?? fromFilesystem.sessions();
+    return {
+        facts: readMemoryFacts(serviceMemory, sessionsMemory),
+        // The floor reading, and «unknown» kept apart from «empty»: this number is
+        // the only thing standing between a live slice and the ceiling written by
+        // a command typed on a machine whose service cannot be measured.
+        sessionsUsageBytes: sessionsMemory === null
+            ? null
+            : (sessionsMemory.unreclaimableBytes ?? sessionsMemory.currentBytes),
+    };
+}
+//# sourceMappingURL=systemd-memory.js.map

package/dist/verify.js

@@ -5,6 +5,8 @@ import { promisify } from 'node:util';
 import { log } from './log.js';
 import { maskString } from './policy.js';
 import { evaluateRecipeCommand } from './policy.js';
+import { lowerPriority } from './process-priority.js';
+import { cageSpawn, releaseSessionScope } from './session-cage.js';
 import { recipeFingerprint } from './recipe.js';
 import { StringDecoder } from 'node:string_decoder';
 import { stateDir } from './paths.js';
@@ -301,6 +303,7 @@ export class VerifyRunner {
             logPath,
             logBytes: 0,
             child: null,
+            scopeUnit: null,
             cancelled: false,
             branch: input.branch,
             commitSha: input.commitSha,
@@ -540,6 +543,11 @@ export class VerifyRunner {
                 settled = true;
                 clearTimeout(timer);
                 run.child = null;
+                // Read why the scope ended and let systemd forget it, in that order —
+                // a scope the OOM killer took stays in `failed` and would refuse the
+                // very same unit name to the next step (`session-cage.ts`).
+                void releaseSessionScope(run.scopeUnit, `verify-${run.runId}`);
+                run.scopeUnit = null;
                 resolve({
                     step: run.currentStep ?? '',
                     durationMs: this.now() - startedAt,
@@ -547,14 +555,26 @@ export class VerifyRunner {
                     ...(timedOut ? { timedOut: true } : {}),
                 });
             };
+            // A verification step is the heaviest thing on the machine that has an id
+            // of its own, so it gets a cage of its own too — keyed by the run rather
+            // than by a session, because that is the identity a verify run has. The
+            // scope survives `detached: true`: `systemd-run --scope` execs in the
+            // same pid, so `setsid` still makes the child its own group leader and
+            // `process.kill(-pid)` below still reaches the compilers.
+            const caged = cageSpawn({
+                id: `verify-${run.runId}`,
+                command: '/bin/sh',
+                args: ['-c', step.run],
+            });
+            run.scopeUnit = caged.unit;
             let child;
             try {
                 // `detached` so the whole process tree gets the signal: a build script
                 // is a shell that spawns compilers, and killing only the shell leaves
                 // them running with the disk and the CPU.
-                child = spawn('/bin/sh', ['-c', step.run], {
+                child = spawn(caged.command, caged.args, {
                     cwd: run.cwd,
-                    env: buildEnv(step.env),
+                    env: { ...buildEnv(step.env), ...caged.env },
                     detached: true,
                     stdio: ['ignore', 'pipe', 'pipe'],
                 });
@@ -565,6 +585,12 @@ export class VerifyRunner {
                 return;
             }
             run.child = child;
+            // A recipe step is the heaviest thing the runner starts on its own — a
+            // full build or a full test run — and the one with nobody waiting on a
+            // keystroke. It goes behind the daemon and level with the agent sessions;
+            // the whole process group inherits it, which is the point, because
+            // `detached` means the compilers are down there and not here.
+            lowerPriority(child.pid);
             // A StringDecoder per stream, not `buffer.toString('utf8')` per chunk.
             //
             // A pipe read ends wherever the kernel filled the buffer, which is

package/dist/version.d.ts

@@ -1,2 +1,2 @@
-export declare const RUNNER_VERSION = "0.53.0";
+export declare const RUNNER_VERSION = "0.54.0";
 //# sourceMappingURL=version.d.ts.map

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in sync with package.json by the release script (manual for now).
-export const RUNNER_VERSION = '0.53.0';
+export const RUNNER_VERSION = '0.54.0';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bridge4dev/runner",
-  "version": "0.53.0",
+  "version": "0.54.0",
   "description": "DevBridge dev runner — connects a dev server to DevBridge and runs agent sessions (Claude Code / Codex)",
   "homepage": "https://bridge4.dev",
   "license": "MIT",