@bridge4dev/runner 0.53.0 → 0.55.0

package/dist/adapters/claude.js

@@ -1,9 +1,12 @@
 import { query, } from '@anthropic-ai/claude-agent-sdk';
+import { spawn } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
 import { AsyncQueue } from '../async-queue.js';
 import { log } from '../log.js';
 import { mcpConfigPath } from '../paths.js';
+import { lowerPriority } from '../process-priority.js';
+import { cageSpawn, memoryDeathSentence, releaseSessionScope } from '../session-cage.js';
 import { evaluateToolUse, maskSecrets, maskString, } from '../policy.js';
 import { availableModes, cardDescription, DIRECT_BRANCH_RULE, MODE_REFUSED_TEXT, MODE_WITHDRAWN_TEXT, policyContextFor, DEVBRIDGE_MCP_SERVER_NAME, } from './types.js';
 import { percentFromUtilization, RATE_WINDOW_MINUTES, rateWindowKey } from './rate-limits.js';
@@ -11,6 +14,8 @@ import { applyUsagePercentages, parseUsageText, probeUsageText } from './claude-
 import { claudeExecutableOption, sessionClaudePath } from '../agent-binary.js';
 /** How often `/usage` may be read. Free, but still a process. */
 const USAGE_PROBE_INTERVAL_MS = 3 * 60 * 1000;
+/** Same 2KB the SDK keeps: enough for the CLI's last words, not a log sink. */
+const STDERR_TAIL_LIMIT = 2048;
 import { answerSummary, answerValue, discussMessage, invalidationMessage, mirrorOptions, newAskId, MAX_OPTIONS, MAX_QUESTIONS, OPTION_TEXT_LIMIT, QUESTION_TEXT_LIMIT, } from './questions.js';
 // Claude adapter over the Agent SDK. Three live-verified gotchas (plan §2):
 //  1. Bare tool names in `allowedTools` auto-approve BEFORE canUseTool — we
@@ -293,6 +298,17 @@ class ClaudeSession {
     mcpConfigFile = null;
     /** Set when the write failed and the key went into argv after all. */
     mcpFallbackNotice = null;
+    /**
+     * The last few KB the CLI wrote to stderr, kept because we spawn it ourselves.
+     *
+     * The SDK's own spawn keeps this tail and appends it to the error it throws
+     * when the process dies (`. stderr: …`); a custom `spawnClaudeCodeProcess`
+     * gets no such service, and that text is load-bearing here — «no conversation
+     * found» and «No message found with message.uuid» reach `classifyRunError`
+     * exactly that way. Keeping our own copy is what makes the priority hook cost
+     * nothing diagnostically.
+     */
+    stderrTail = '';
     /** Guards against overlapping capability probes. */
     capabilitiesInFlight = false;
     /**
@@ -508,6 +524,12 @@ class ClaudeSession {
             // bundled binary, exactly as before. After C3 this pins the system
             // `claude`, which is the file the card measures and the button installs.
             ...claudeExecutableOption(),
+            // Stage 1a of the host-resources plan: the CLI, and therefore everything
+            // it starts, is spawned at nice 10 so the daemon that supervises it keeps
+            // the processor. The SDK has no post-spawn hook — replacing the spawn is
+            // the only way in, which is why `spawnAgentProcess` has to re-create the
+            // one thing the default spawn did for us (the stderr tail).
+            spawnClaudeCodeProcess: (spawnOptions) => this.spawnAgentProcess(spawnOptions),
             /**
              * Everything the machine's own Claude has (owner's call, 2026-07-30).
              *
@@ -623,6 +645,75 @@ class ClaudeSession {
         // model's window) from the moment the process is up.
         this.refreshContextUsage();
     }
+    /**
+     * Start the Claude CLI ourselves, one notch below the daemon.
+     *
+     * The SDK offers no «after spawn» callback, so the only place to renice the
+     * process is a spawn we own. Two things the default spawn did have to be done
+     * here, and neither is optional:
+     *
+     *  - **stderr must be read.** With `stdio: 'pipe'` and nobody draining it, the
+     *    CLI blocks on a full pipe after ~64KB — a session that hangs for no
+     *    visible reason. That is a worse outcome than anything nice(2) can buy.
+     *  - **the tail must be kept**, because the SDK appends it to the error it
+     *    throws on process death and `classifyRunError` reads that text. Without
+     *    it, «the resume id is unknown» comes back as a bare exit code and the
+     *    supervisor's recovery never fires.
+     *
+     * `signal` is the SDK's own forwarded abort, not the caller's: it fires only
+     * after stdin EOF and the ~2s grace, so handing it to `spawn()` kills a CLI
+     * that already had its chance to shut down cleanly.
+     */
+    spawnAgentProcess(options) {
+        // The same hook carries stage 2: the CLI, and everything it starts, goes
+        // into this session's own cgroup with its own memory ceiling. On a machine
+        // where the cage was not proved to work `cageSpawn` hands the command back
+        // untouched, and the renice below is the whole of the containment — which
+        // is exactly the state of every machine before this release.
+        const caged = cageSpawn({
+            id: this.spec.sessionId,
+            command: options.command,
+            args: options.args,
+        });
+        const child = spawn(caged.command, caged.args, {
+            cwd: options.cwd,
+            env: { ...options.env, ...caged.env },
+            stdio: ['pipe', 'pipe', 'pipe'],
+            signal: options.signal,
+            windowsHide: true,
+        });
+        // `systemd-run --scope` execs into the same pid and nice survives `exec`,
+        // so this still lands on the CLI itself.
+        lowerPriority(child.pid);
+        // Read `Result` and clear the unit once the process is gone. Only an
+        // `exit` listener: stdout belongs to the SDK, and attaching a reader to it
+        // here would put the stream in flowing mode and steal the conversation.
+        child.on('exit', () => {
+            void releaseSessionScope(caged.unit, this.spec.sessionId);
+        });
+        // `setEncoding` puts a StringDecoder on the stream, so a multi-byte
+        // character split across two reads survives — the same reason `verify.ts`
+        // decodes rather than `toString`s.
+        child.stderr.setEncoding('utf8');
+        child.stderr.on('data', (chunk) => this.appendStderr(chunk));
+        // A read error on stderr is not a session error: the process itself is
+        // still on stdout, which is where the conversation lives.
+        child.stderr.on('error', (error) => {
+            log.debug('claude: stderr read failed', { error: String(error) });
+        });
+        return child;
+    }
+    appendStderr(chunk) {
+        if (!chunk)
+            return;
+        const next = this.stderrTail + chunk;
+        this.stderrTail = next.length > STDERR_TAIL_LIMIT ? next.slice(-STDERR_TAIL_LIMIT) : next;
+    }
+    /** What the CLI said on its way out, in the SDK's own wording. */
+    stderrSuffix() {
+        const tail = this.stderrTail.trim();
+        return tail ? `. stderr: ${tail}` : '';
+    }
     /**
      * Write this session's MCP config to a 0600 file and return its path, or null
      * if anything went wrong (the caller then keeps the old inline shape).
@@ -2347,11 +2438,23 @@ class ClaudeSession {
             }
         }
         catch (error) {
-            const message = maskString(String(error instanceof Error ? error.message : error));
+            // The stderr tail rides along because we spawn the CLI ourselves now (see
+            // `spawnAgentProcess`) — the SDK used to add it and cannot any more. This
+            // catch IS the process falling over, which is the only case the SDK
+            // appended it to as well.
+            const message = maskString(String(error instanceof Error ? error.message : error) + this.stderrSuffix());
             const code = errorCode(message);
+            // #387: «exited with code 143» and «terminated by signal SIGKILL» were the
+            // two faces of one cause — the kernel's OOM killer inside this session's
+            // cgroup — and neither said so. The cage remembers the kill counter at
+            // exit; when it moved, the sentence about memory rides on the error.
+            // Awaited, not read: on a systemd that stops the whole scope the cgroup
+            // is already gone when the process's `exit` fires, and the only surviving
+            // record is `Result=oom-kill`, which the release is still fetching.
+            const memory = await memoryDeathSentence(this.spec.sessionId);
             this.emit({
                 type: 'error',
-                message: classifyRunError(message),
+                message: memory ? `${classifyRunError(message)} ${memory}` : classifyRunError(message),
                 ...(code ? { code } : {}),
                 // #373: this catch is the CLI process falling over under the read loop —
                 // the only place in this adapter where that is what happened. Our own

package/dist/adapters/codex-protocol.d.ts

@@ -30,6 +30,13 @@ export interface AppServerOptions {
     args: string[];
     cwd?: string;
     env: Record<string, string>;
+    /**
+     * Whose session this app-server belongs to — the name of its cgroup cage
+     * (`session-cage.ts`). Required rather than optional: a spawn with no id
+     * cannot be caged, and a session running outside the cage while everything
+     * reports that the cage is on is the one failure this feature must not have.
+     */
+    sessionId: string;
     onNotification: (method: string, params: Record<string, unknown>) => void;
     onServerRequest: (request: ServerRequest) => void;
     /** Fired once when the child is gone — the session's end-of-stream signal. */
@@ -47,6 +54,10 @@ export declare class AppServerClient {
     private nextId;
     private stdoutBuffer;
     private exited;
+    /** The scope this process runs in, or null when the machine has no cage. */
+    private readonly scopeUnit;
+    /** Has the far end said anything at all? Reads the start window — see below. */
+    private sawOutput;
     constructor(opts: AppServerOptions);
     get pid(): number | undefined;
     get alive(): boolean;

package/dist/adapters/codex-protocol.js

@@ -1,5 +1,7 @@
 import { spawn } from 'node:child_process';
 import { log } from '../log.js';
+import { lowerPriority } from '../process-priority.js';
+import { cageSpawn, killedBeforeExec, releaseSessionScope } from '../session-cage.js';
 export class RpcError extends Error {
     code;
     method;
@@ -36,15 +38,32 @@ export class AppServerClient {
     nextId = 1;
     stdoutBuffer = '';
     exited = false;
+    /** The scope this process runs in, or null when the machine has no cage. */
+    scopeUnit;
+    /** Has the far end said anything at all? Reads the start window — see below. */
+    sawOutput = false;
     constructor(opts) {
         this.opts = opts;
-        this.child = spawn(opts.command, opts.args, {
+        // `codex app-server` is the root of everything this session will run, so
+        // this is where the cage goes: the scope holds the whole tree below it, and
+        // the ceiling is per session rather than per machine.
+        const caged = cageSpawn({ id: opts.sessionId, command: opts.command, args: opts.args });
+        this.scopeUnit = caged.unit;
+        this.child = spawn(caged.command, caged.args, {
             ...(opts.cwd ? { cwd: opts.cwd } : {}),
-            env: opts.env,
+            env: { ...opts.env, ...caged.env },
             stdio: ['pipe', 'pipe', 'pipe'],
         });
+        // Before a single byte of protocol, and still worth doing inside a scope:
+        // nice is inherited at fork AND across `exec`, so renicing `systemd-run`
+        // renices the app-server it turns into. On a machine with no cage this is
+        // the only containment there is.
+        lowerPriority(this.child.pid);
         this.child.stdout.setEncoding('utf8');
-        this.child.stdout.on('data', (chunk) => this.onStdout(chunk));
+        this.child.stdout.on('data', (chunk) => {
+            this.sawOutput = true;
+            this.onStdout(chunk);
+        });
         this.child.stderr.setEncoding('utf8');
         this.child.stderr.on('data', (chunk) => opts.onStderr?.(chunk));
         this.child.on('error', (error) => {
@@ -67,6 +86,25 @@ export class AppServerClient {
         if (this.exited)
             return;
         this.exited = true;
+        if (killedBeforeExec({
+            code: info.code,
+            signal: info.signal,
+            sawOutput: this.sawOutput,
+            caged: this.scopeUnit !== null,
+        })) {
+            // Not a silent death of the agent. Starting a scope takes 0.07–2.5 s (up
+            // to 2741 ms on a loaded machine), and a stop inside that window signals
+            // `systemd-run` before it has exec'd — so the process that got SIGTERM
+            // was the wrapper, and `codex app-server` never existed. Said plainly
+            // here, because from the outside it is indistinguishable from a crash
+            // with an empty stdout.
+            log.warn('codex: the session was stopped before its process started', {
+                unit: this.scopeUnit,
+            });
+        }
+        // Read the cause, THEN let systemd forget the unit: `Result=oom-kill` is
+        // only readable while the failed scope is still there (`session-cage.ts`).
+        void releaseSessionScope(this.scopeUnit, this.opts.sessionId);
         this.opts.onExit(info);
     }
     failAll(error) {

package/dist/adapters/codex.js

@@ -1,6 +1,7 @@
 import { AsyncQueue } from '../async-queue.js';
 import { log } from '../log.js';
 import { evaluateToolUse, maskSecrets, maskString, } from '../policy.js';
+import { memoryDeathSentence } from '../session-cage.js';
 import { RUNNER_VERSION } from '../version.js';
 import { repairCodexAuth } from './codex-home.js';
 import { AppServerClient, asRecord, num, RpcError, RpcTimeoutError, str, } from './codex-protocol.js';
@@ -220,6 +221,10 @@ class CodexSession {
                 command: CODEX_BIN,
                 args: ['app-server'],
                 cwd: spec.cwd,
+                // Names this session's cgroup cage (`session-cage.ts`). Not part of
+                // `wiring`: an injected client is a scripted object with no process
+                // under it, and there is nothing to cage.
+                sessionId: spec.sessionId,
                 ...wiring,
             });
         if (this.modeRefusedAtLaunch) {
@@ -1862,8 +1867,36 @@ class CodexSession {
                 processGone: true,
             });
         }
+        if (!this.stopped)
+            this.sayIfMemoryKilledIt();
         this.finish();
     }
+    /**
+     * «codex app-server exited (code null)» is the same sentence whether the CLI
+     * crashed or the kernel took it for memory, and #387 is the ticket about
+     * exactly that silence. The cage knows which it was; here it reaches the
+     * person.
+     *
+     * A follow-up `notice` rather than a longer error message, because both death
+     * paths in this adapter are synchronous by design (one is a notification
+     * handler, the other a catch that closes the output queue on the next line),
+     * and the verdict needs an await — `Result=oom-kill` is read after the exit.
+     */
+    sayIfMemoryKilledIt() {
+        if (this.memoryVerdictAsked)
+            return;
+        this.memoryVerdictAsked = true;
+        void memoryDeathSentence(this.spec.sessionId)
+            .then((sentence) => {
+            if (sentence)
+                this.emit({ type: 'notice', level: 'warn', text: sentence });
+        })
+            .catch(() => {
+            // A verdict we could not fetch is no verdict; the error text stands.
+        });
+    }
+    /** One verdict per process — both death paths can fire for the same exit. */
+    memoryVerdictAsked = false;
     // ─── Capabilities ──────────────────────────────────────────────────
     refreshCapabilities() {
         if (this.capabilitiesInFlight || this.stopped)
@@ -2116,6 +2149,10 @@ class CodexSession {
             return code;
         }
         this.emit({ type: 'error', message: `Codex session error: ${message.slice(0, 1_000)}` });
+        // The generic door: a pending request rejected because the app-server died
+        // (`failAll`). If the kernel is why it died, say so (#387).
+        if (/app-server exited|SIGKILL/i.test(message))
+            this.sayIfMemoryKilledIt();
         return null;
     }
     /**

package/dist/host-load.d.ts

@@ -0,0 +1,156 @@
+/**
+ * What this machine can say about its own load, read straight from `/proc`.
+ *
+ * Until this existed, an overloaded dev server was visible in exactly one
+ * place: `sar` on the machine itself. The product could not say «this machine
+ * is on its knees», so it said nothing, and the first symptom a user saw was a
+ * session that had gone quiet (plan `agent-sessions-host-resources.md` §5.3).
+ *
+ * Two files and nothing else. That is a security property, not an
+ * implementation detail: this runner reports to a server, so what it may read
+ * for telemetry is listed here in full — `/proc/loadavg` and `/proc/meminfo`,
+ * both world-readable, neither containing a path, a command line, an
+ * environment variable or a process name. No `/proc/<pid>` walk, no `ps`.
+ *
+ * Mirror of `packages/shared/src/schemas/host-load.ts` and the `HOST_LOAD_*`
+ * block of `packages/shared/src/constants/runner.ts` — the DevBridge side is
+ * the source of truth, exactly like `levels.ts` mirrors the level thresholds
+ * and `protocol.ts` mirrors the API's wire types. Copied rather than imported
+ * on purpose: this package is published to npm on its own and installed by
+ * users who have no DevBridge workspace, so a `@devbridge/shared` import would
+ * make the published tarball unresolvable (see `recipe-schema.ts`).
+ *
+ * CHECKED: `host-load.test.ts` reads the shared file and compares every number
+ * below against it. A threshold edited on one side only would make the runner
+ * send frames the API throws away — or go silent while a card waits for one.
+ */
+/**
+ * How often the runner LOOKS at `/proc`. Not how often it sends.
+ *
+ * Fine enough that a card is never more than half a minute stale, coarse
+ * enough that reading two small files costs nothing measurable.
+ */
+export declare const HOST_LOAD_SAMPLE_INTERVAL_MS = 30000;
+/**
+ * The heartbeat: an unchanged machine still reports this often.
+ *
+ * Without it a quiet runner would go silent and the API's key would expire,
+ * turning «nothing is happening» into «we have no idea» on the card. With it,
+ * silence really does mean the runner stopped talking.
+ *
+ * Bound to the API's `HOST_LOAD_TTL_MS` (120 s), and the order is the design:
+ * a heartbeat LONGER than the TTL means a healthy quiet machine shows «no
+ * data» for the gap between them. The first draft had 5 minutes against a
+ * 2-minute TTL — three silent minutes out of every five.
+ */
+export declare const HOST_LOAD_HEARTBEAT_MS = 60000;
+/**
+ * How much load1 must move before an otherwise unchanged sample earns a frame.
+ *
+ * Same principle as `publishSlots`: a tick that would repeat what the server
+ * already knows is not sent at all. 0.5 is below the resolution at which a
+ * person reads the number and well above the noise of an idle machine.
+ */
+export declare const HOST_LOAD_MIN_DELTA_LOAD1 = 0.5;
+/** The same idea for memory, as a fraction — see `hostLoadChangedEnough`. */
+export declare const HOST_LOAD_MIN_DELTA_MEM_RATIO = 0.05;
+/**
+ * One measurement of this machine. Only the moving parts.
+ *
+ * `machine: {cpuCount, memTotalBytes, memAvailableBytes}` already travels in
+ * `hello` (`index.ts`), so nothing static is repeated here. `cpuCount` is the
+ * exception and it earns its place: load1 is meaningless without it, and a
+ * consumer that had to join two sources to answer «is 12.4 a lot?» would
+ * eventually paint one machine's load against another's core count.
+ */
+export interface HostLoadFrame {
+    /** `/proc/loadavg`, the three windows as the kernel reports them. */
+    load1: number;
+    load5: number;
+    load15: number;
+    /** Processors, so load1 can be read as a ratio without a second lookup. */
+    cpuCount: number;
+    /**
+     * `MemAvailable`, not `MemFree` — the kernel's own estimate of what a new
+     * allocation could actually get. `MemFree` on a healthy Linux box is near
+     * zero by design (page cache), and a card drawn from it would cry wolf on
+     * every machine, every minute.
+     */
+    memAvailableBytes: number;
+    /** Zero on a machine with no swap; consumers must not divide blindly. */
+    swapTotalBytes: number;
+    swapFreeBytes: number;
+    /**
+     * When THIS MACHINE measured it, ISO.
+     *
+     * Load-bearing, same as in `agent_versions`: frames without a session id take
+     * the gateway's unordered fast path, so two of these can be handled out of
+     * order. The card dates the number by this field, never by arrival.
+     */
+    at: string;
+}
+/** Test seams. Production calls `readHostLoad()` with nothing. */
+export interface HostLoadSources {
+    /**
+     * Where `/proc` is mounted. A seam so the parser can be driven by fixtures:
+     * a test that had to arrange the real kernel into a state would not be
+     * written, and the states worth covering (no swap, no `MemAvailable`, no
+     * file at all) cannot be arranged at all.
+     */
+    procDir?: string;
+    /** Processors. Seam for the same reason — a test cannot change its own. */
+    cpuCount?: number;
+    /** The clock behind `at`. */
+    now?: () => Date;
+}
+/**
+ * Measure this machine, or say honestly that we cannot.
+ *
+ * `null` is a first-class answer and every caller must treat it as «send
+ * nothing». There is deliberately no `process.platform` check in front of it:
+ * the absence of the files IS the check, and it is the wider one — it also
+ * covers a Linux container with `/proc` masked or mounted `hidepid`, which a
+ * platform test would sail straight past into an exception. The runner is
+ * Linux-only by its installer, but «only runs on Linux» must never mean
+ * «crashes anywhere else»: this is called from a timer, and an exception here
+ * would take the whole session supervisor with it.
+ */
+export declare function readHostLoad(sources?: HostLoadSources): HostLoadFrame | null;
+/**
+ * Is the heartbeat due, given how long ago the last frame actually went out?
+ *
+ * The second half of the answer is the whole reason this is a function: a
+ * backwards clock step makes «how long ago» NEGATIVE, and a machine whose load
+ * never moves would then say nothing for the entire offset — an hour of silence
+ * on a machine that may be on fire. A clock correction is due, not early.
+ *
+ * The API makes the same allowance for the same step, and has to: it drops
+ * frames older than the one it holds unless the step is larger than a
+ * measurement's lifetime (`HOST_LOAD_TTL_MS`, `runner-gateway.ts`). Half of this
+ * fix on one side of the socket is no fix at all.
+ */
+export declare function hostLoadHeartbeatDue(sinceLastSentMs: number, heartbeatMs: number): boolean;
+/**
+ * Is this sample worth a frame at all? The «a quiet runner says nothing» rule.
+ *
+ * Same shape as `publishSlots`: what the server already knows is not repeated.
+ * The alternative is 2 880 identical frames a day per machine, each one waking
+ * the gateway, a Redis write and every open dashboard socket.
+ *
+ * The memory threshold is 5 % of the machine's TOTAL memory, not 5 % of what
+ * is currently available. Two reasons, and both were the deciding one:
+ *
+ *  - a fraction of `memAvailable` shrinks as the machine fills, so the frames
+ *    would get chattiest exactly when the machine is least able to afford it —
+ *    5 % of the last 200 MB is a 10 MB trigger;
+ *  - `memTotal` is fixed per machine, so «the card moves when a GB moves» reads
+ *    the same on every server instead of meaning something different on each.
+ *
+ * Swap counts too, though the plan names only load and memory. It is one of the
+ * three verdicts behind «overloaded», and it is the only one with no other
+ * trigger: a machine that starts paging with its load and its `MemAvailable`
+ * both flat would otherwise wait up to five minutes for the heartbeat to say
+ * so, and paging is the state whose wall-clock cost is unbounded.
+ */
+export declare function hostLoadChangedEnough(previous: HostLoadFrame | null, next: HostLoadFrame, memTotalBytes?: number): boolean;
+//# sourceMappingURL=host-load.d.ts.map