@bridge4dev/runner 0.37.0 → 0.39.0

package/dist/service-unit.js

@@ -129,13 +129,220 @@ export function buildUnit(execStart, nodeBinary = process.execPath) {
  * which is the only way to fix the servers that already have the bad numbers
  * baked in — and it never overwrites a unit the operator edited by hand.
  */
-export const LIMITS_VERSION = 2;
+export const LIMITS_VERSION = 3;
 const LIMITS_MARKER = '# devbridge-limits-version:';
 /** `zz-` so it sorts last: an operator's own drop-in should still win. */
 const LIMITS_FILE = 'zz-devbridge-limits.conf';
 export function limitsOverridePath(home = systemdUserHome()) {
     return path.join(home, '.config', 'systemd', 'user', `${SERVICE_NAME}.service.d`, LIMITS_FILE);
 }
+const MIB = 1024 * 1024;
+const GIB = 1024 * MIB;
+/**
+ * How long after boot `MemAvailable` starts telling the truth.
+ *
+ * The runner is a user unit ordered `After=network-online.target`, and the things
+ * it shares the machine with — dockerd and its containers, mysql, pm2 — come up
+ * around the same time. Measure at second 20 and the neighbours have not claimed
+ * their memory yet: on the 12 GB machine below that reads as 11 GB free and
+ * produces a ceiling that protects nothing.
+ */
+export const BOOT_SETTLE_SEC = 600;
+/**
+ * The ceiling is never written BELOW what the cgroup already holds.
+ *
+ * Until 0.39.0 this policy only ever REMOVED a limit, so applying it to a running
+ * service was free. Now it sets one, and systemd applies `memory.max` to a LIVE
+ * cgroup on `daemon-reload`: a value under current usage makes the kernel reclaim
+ * and then kill inside the cgroup. At daemon start that is harmless — nothing is
+ * running yet — but `doctor --fix` on a busy machine would otherwise take out
+ * somebody's session as a side effect of running a diagnostic.
+ *
+ * The cost is that a machine fixed WHILE a runaway is in progress writes a ceiling
+ * above that runaway, which protects nothing. It resolves itself: the next daemon
+ * start measures an idle cgroup, and the drift check rewrites the inflated number.
+ */
+const CEILING_HEADROOM_OVER_CURRENT = 1.25;
+/**
+ * The two numbers, and the incident that decides them.
+ *
+ * Until 0.21.0 the policy was a 2 GB hard ceiling with systemd's default
+ * `OOMPolicy=stop`: one agent over the line and the whole service died, taking
+ * every other session with it (QA-112, gotcha #100). The fix removed the ceiling
+ * entirely and left a soft `MemoryHigh=80%`, on this reasoning, quoted from the
+ * code it replaced: «the machine keeps a fifth of its memory for everything that
+ * is not this service».
+ *
+ * That reasoning holds on a DEDICATED dev server and fails on a shared one, which
+ * is what vmi2502773 is: 11.9 GB total with ~4 GB permanently held by 18 docker
+ * containers, mysql, flowise, chroma and pm2. There, 80 % of TOTAL is 9.5 GB while
+ * only ~8 GB is actually free — so the soft brake sat ABOVE the real headroom and
+ * could never engage. The machine swapped, `kcompactd0` blocked for over 120
+ * seconds and the box hung; 2026-08-05 it was a `claude` at 4.2 GB, 2026-08-12 a
+ * single grep at 6.8 GB with 160 MB left. Only a power cycle recovered it.
+ *
+ * So the percentage has to be of what the machine can SPARE, not of what it has:
+ *
+ *   headroom = MemAvailable + our own usage   (ours is added back, or every
+ *              rewrite would walk the ceiling down by what we already hold)
+ *   MemoryMax  = headroom − reserve
+ *   MemoryHigh = 80 % of MemoryMax            (reclaim and throttle first, kill last)
+ *
+ * A hard ceiling is safe to have again only because the OTHER half of the 0.21.0
+ * fix stayed: `OOMPolicy=continue` is still set below, so the kernel killing one
+ * runaway child no longer tears down the service. Restoring the ceiling without
+ * that would re-open QA-112.
+ *
+ * The clamps are the two ways this can go wrong:
+ *  - floor 2 GB: on a machine whose neighbours already ate everything, a computed
+ *    ceiling of a few hundred MB would mean a dev server that cannot run one
+ *    agent. Measured cost of ordinary work is 512–646 MB per `claude` and 1571 MB
+ *    for a workspace `pnpm -r typecheck`, so anything under 2 GB is not a dev
+ *    server at all — an unusable one is a worse failure than a busy one, the same
+ *    call `cpuQuotaPercent` makes below;
+ *  - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
+ *    machine, and a ceiling of «everything» is the bug this function exists to fix.
+ */
+export function memoryPolicy(facts, minCeilingBytes = facts.ownUsageBytes * CEILING_HEADROOM_OVER_CURRENT) {
+    const { totalBytes, availableBytes, ownUsageBytes, uptimeSec } = facts;
+    const withFloor = (maxBytes, measured, starved = false) => {
+        const ceiling = Math.floor(Math.max(maxBytes, minCeilingBytes));
+        return { maxBytes: ceiling, highBytes: Math.floor(ceiling * 0.8), measured, starved };
+    };
+    // Still booting: the neighbours have not claimed their memory yet, so measuring
+    // now would hand back almost the whole machine. Take a fraction of TOTAL
+    // instead — it needs no measurement at all.
+    //
+    // 55 % is chosen against the machine that failed rather than against a tidy
+    // number: 55 % of 11.9 GB is 6.5 GB, which still leaves 1.4 GB after the 4 GB
+    // its neighbours hold, and puts the brake at 5.2 GB — engaged with plenty free.
+    // 60 % looked reasonable and left only 760 MB, which is not a margin. This
+    // number works blind, so it is sized for the crowded case, and it costs a
+    // dedicated box nothing: it only applies for the first ten minutes of uptime,
+    // before any session exists. The next daemon start measures and replaces it.
+    if (!Number.isFinite(uptimeSec) || uptimeSec < BOOT_SETTLE_SEC) {
+        return withFloor(totalBytes * 0.55, false);
+    }
+    // Enough for sshd, journald, the kernel and some page cache to keep working
+    // while the cgroup sits at its ceiling. Proportional on a big machine, absolute
+    // on a small one, because 15 % of 4 GB is not enough to stay reachable.
+    const reserve = Math.max(1.5 * GIB, totalBytes * 0.15);
+    const headroom = availableBytes + ownUsageBytes;
+    const cap = totalBytes * 0.85;
+    // `min` with the cap, not a bare 2 GiB: below ~2.4 GB of RAM the floor would be
+    // ABOVE the cap and `clamp` would silently return the cap anyway — with the
+    // reserve ignored and the «2 GB or nothing» promise quietly broken. Saying it
+    // here makes the tiny-machine answer deliberate instead of accidental.
+    const floor = Math.min(2 * GIB, cap);
+    const wanted = headroom - reserve;
+    return withFloor(clamp(wanted, floor, cap), true, wanted < floor);
+}
+function clamp(value, low, high) {
+    return Math.min(Math.max(value, low), high);
+}
+/** systemd takes a plain byte count, but a unit file is read by people. */
+function asMiB(bytes) {
+    return `${Math.max(1, Math.floor(bytes / MIB))}M`;
+}
+/** `MemTotal`/`MemAvailable` in bytes, or null where there is no `/proc`. */
+function readMemInfo() {
+    let text;
+    try {
+        text = fs.readFileSync('/proc/meminfo', 'utf8');
+    }
+    catch {
+        return null;
+    }
+    const field = (name) => {
+        const match = new RegExp(`^${name}:\\s+(\\d+) kB$`, 'm').exec(text);
+        return match?.[1] ? Number(match[1]) * 1024 : null;
+    };
+    const totalBytes = field('MemTotal');
+    const availableBytes = field('MemAvailable');
+    return totalBytes && availableBytes ? { totalBytes, availableBytes } : null;
+}
+/**
+ * What our own cgroup currently holds.
+ *
+ * Read through `/proc/self/cgroup` rather than assembling the path from the
+ * service name: the runner runs as root and as a dedicated user, under
+ * `user@0.service` and under `user@1001.service`, and guessing that path wrong
+ * silently returns 0 — which would quietly shrink the ceiling by whatever we are
+ * already using. Returns 0 on cgroup v1 or in a container without the file, which
+ * is the safe direction: a slightly lower ceiling, never a higher one.
+ */
+function readOwnCgroupUsage() {
+    let cgroup;
+    try {
+        const line = fs
+            .readFileSync('/proc/self/cgroup', 'utf8')
+            .split('\n')
+            .find((l) => l.startsWith('0::'));
+        if (!line)
+            return null; // cgroup v1 — no v2 path to read
+        cgroup = line.slice('0::'.length).trim();
+    }
+    catch {
+        return null;
+    }
+    // `/proc/self` is the CALLER. For the daemon that is the service, and this is
+    // the hot path. For `doctor --fix` or `install-service` typed over ssh it is a
+    // `session-N.scope` holding a few MB — measuring that and calling it «what the
+    // runner holds» produced a ceiling 34 % away from the daemon's own answer on
+    // this very host, so the two paths would each see the other as drift and
+    // rewrite the file forever. Anything that is not the service must say «I do not
+    // know» and let the caller supply the number.
+    if (!cgroup.endsWith(`/${SERVICE_NAME}.service`))
+        return null;
+    return readCgroupUnreclaimable(path.join('/sys/fs/cgroup', cgroup));
+}
+/**
+ * What the cgroup holds that `MemAvailable` has NOT already counted.
+ *
+ * `memory.current` is `anon + file + kernel`, and `file` is page cache — which
+ * `MemAvailable` already lists as reclaimable. Adding the whole of
+ * `memory.current` back to `MemAvailable` therefore counts our page cache twice
+ * and inflates the ceiling by exactly that much: measured at +19 % on this host
+ * (1.1 GB of cache in a 2.3 GB cgroup), and the cache is largest during builds —
+ * precisely when memory is tightest. Subtracting `file` keeps the part we really
+ * do hold and cannot give back on demand.
+ */
+export function readCgroupUnreclaimable(dir) {
+    try {
+        const current = Number(fs.readFileSync(path.join(dir, 'memory.current'), 'utf8').trim());
+        if (!Number.isFinite(current))
+            return null;
+        const file = /^file (\d+)$/m.exec(fs.readFileSync(path.join(dir, 'memory.stat'), 'utf8'));
+        const cache = file?.[1] ? Number(file[1]) : 0;
+        return Math.max(0, current - cache);
+    }
+    catch {
+        return null;
+    }
+}
+function readUptimeSec() {
+    try {
+        return Number(fs.readFileSync('/proc/uptime', 'utf8').split(/\s+/)[0]) || 0;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Everything `memoryPolicy` needs, straight off this machine.
+ *
+ * `ownUsageBytes` is passed in by callers that are not the daemon — `doctor` and
+ * `install-service` run in the operator's own cgroup and cannot read the
+ * service's usage from `/proc/self`. Returns null when the service's usage is
+ * unknowable, because guessing 0 there is the one dangerous direction: it removes
+ * the floor that stops a live session from being killed on `daemon-reload`.
+ */
+export function readMemoryFacts(ownUsageBytes = readOwnCgroupUsage()) {
+    const info = readMemInfo();
+    if (!info || ownUsageBytes === null)
+        return null;
+    return { ...info, ownUsageBytes, uptimeSec: readUptimeSec() };
+}
 /**
  * `CPUQuota` worth keeping: enough headroom that a runaway build cannot make the
  * box unreachable, but never so little that ordinary work is throttled.
@@ -148,8 +355,16 @@ export function limitsOverridePath(home = systemdUserHome()) {
 export function cpuQuotaPercent(cpuCount = os.cpus().length) {
     return cpuCount >= 4 ? (cpuCount - 1) * 100 : null;
 }
-export function buildLimitsOverride(cpuCount = os.cpus().length) {
+export function buildLimitsOverride(cpuCount = os.cpus().length, facts = readMemoryFacts()) {
     const quota = cpuQuotaPercent(cpuCount);
+    // No `minCeilingBytes` parameter on purpose. It existed here for one revision
+    // and passed an explicit `0`, which silently DEFEATED the default in
+    // `memoryPolicy` — the writer produced a ceiling below live cgroup usage while
+    // the drift check and `doctor` both computed the floored one. Two consequences,
+    // both found in QA: `doctor --fix` could kill a session on a loaded machine,
+    // and the file was rewritten on every start because no fixed point existed.
+    // The floor belongs to the policy, so every caller gets it and none can opt out.
+    const memory = facts ? memoryPolicy(facts) : null;
     return ([
         `${LIMITS_MARKER} ${LIMITS_VERSION}`,
         '# Managed by devbridge-runner. Put your own overrides in a file that sorts',
@@ -163,14 +378,34 @@ export function buildLimitsOverride(cpuCount = os.cpus().length) {
         'StartLimitBurst=0',
         '',
         '[Service]',
-        // The whole point of the change: one child's OOM must not take the fleet.
+        // The whole point of the 0.21.0 change, and the reason the ceiling below is
+        // safe to have at all: one child's OOM must not take the fleet.
         'OOMPolicy=continue',
-        // Clears `MemoryMax=2G` from units written before 0.21.0.
-        'MemoryMax=',
-        // Soft pressure instead of a hard ceiling: the kernel reclaims and
-        // throttles rather than killing, and the machine keeps a fifth of its
-        // memory for everything that is not this service.
-        'MemoryHigh=80%',
+        // Both values also override the `MemoryMax=2G` baked into units written
+        // before 0.21.0.
+        //
+        // The unmeasurable branch used to write `MemoryMax=` + `MemoryHigh=80%` —
+        // literally the pair this release exists to remove. Because an empty
+        // assignment RESETS a directive and a drop-in sorts after the unit, that
+        // branch would have stripped the ceiling the API's fallback unit now carries
+        // and left the machine weaker than if we had written nothing at all. A
+        // drop-in must never disarm the unit it overrides, so it falls back to the
+        // same static pair used during boot.
+        ...(memory
+            ? [
+                `# ${memory.measured ? 'measured headroom' : 'still booting — conservative fraction of total'}`,
+                `MemoryMax=${asMiB(memory.maxBytes)}`,
+                // Soft pressure below the hard stop: the kernel reclaims and throttles
+                // here, and only kills at MemoryMax. This is the line that failed on
+                // vmi2502773 — as a percentage of TOTAL it sat above what the machine
+                // could spare, so it never engaged.
+                `MemoryHigh=${asMiB(memory.highBytes)}`,
+            ]
+            : [
+                '# machine not measurable — same conservative pair as the fallback unit',
+                'MemoryMax=55%',
+                'MemoryHigh=44%',
+            ]),
         // Either a computed quota, or an explicit reset — both of which clear the
         // `CPUQuota=80%` baked into units written before 0.21.0.
         quota === null ? 'CPUQuota=' : `CPUQuota=${quota}%`,
@@ -186,9 +421,12 @@ export function buildLimitsOverride(cpuCount = os.cpus().length) {
  *
  * Deliberately version-based rather than content-based: an operator may add
  * their own directives to our file, and re-writing on every start would fight
- * them. Only the version number decides.
+ * them. The version number decides — plus, since 0.39.0 and only when `facts` are
+ * supplied, whether the measured ceiling still fits the machine (see
+ * `memoryCeilingHasDrifted`). Callers that pass no facts get the old, purely
+ * version-based answer.
  */
-export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome()) {
+export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'utf8'), home = systemdUserHome(), facts = null) {
     let contents;
     try {
         contents = readFile(limitsOverridePath(home));
@@ -200,15 +438,61 @@ export function limitsOverrideIsOutdated(readFile = (p) => fs.readFileSync(p, 'u
     if (!line)
         return true;
     const version = Number.parseInt(line.slice(LIMITS_MARKER.length).trim(), 10);
-    return !Number.isFinite(version) || version < LIMITS_VERSION;
+    if (!Number.isFinite(version) || version < LIMITS_VERSION)
+        return true;
+    return facts ? memoryCeilingHasDrifted(contents, facts) : false;
 }
-/** Write the drop-in. Returns false when nothing needed doing. */
-export function writeLimitsOverride(force = false, home = systemdUserHome()) {
-    if (!force && !limitsOverrideIsOutdated(undefined, home))
+/**
+ * Since 0.39.0 the ceiling is a measurement, and a measurement goes stale.
+ *
+ * A machine that gained a database and a dozen containers after the drop-in was
+ * written would keep a ceiling sized for the machine it used to be — which is the
+ * original bug wearing a different hat. So the version marker is no longer the
+ * only thing that can make the file outdated.
+ *
+ * The 15 % band is what keeps this from becoming a rewrite on every start:
+ * `MemAvailable` moves by hundreds of MB just from page cache, and re-writing our
+ * file that often would fight an operator who added their own directives to it
+ * (the reason the check was version-only to begin with).
+ */
+const CEILING_DRIFT_TOLERANCE = 0.15;
+function memoryCeilingHasDrifted(contents, facts) {
+    // Never re-measure a machine that is still booting: the static pair it is
+    // holding is deliberate, and «drift» against a lie is not drift.
+    if (!memoryPolicy(facts).measured)
+        return false;
+    // LAST match, not the first: systemd applies the last assignment, and the file
+    // explicitly invites the operator to add their own lines. Reading the first one
+    // would measure drift against a directive that is not in force.
+    const written = [...contents.matchAll(/^MemoryMax=(\d+)([KMG]?)$/gm)].at(-1);
+    // Anything we cannot read as a byte count is replaced, and that is deliberate:
+    // an EMPTY `MemoryMax=` is the 0.21.0 shape this release exists to remove, and
+    // `MemoryMax=55%` is the blind pair we write when the machine is unmeasurable —
+    // both must give way the moment a real measurement is available. The operator's
+    // own directives belong in a file that sorts after this one, which the header of
+    // every generated file says; a value they appended HERE is still honoured,
+    // because a plain `4G` parses above and is compared like any other.
+    if (!written?.[1])
+        return true;
+    const scale = { '': 1, K: 1024, M: MIB, G: 1024 * MIB }[written[2] ?? ''] ?? MIB;
+    const writtenBytes = Number(written[1]) * scale;
+    const wanted = memoryPolicy(facts).maxBytes;
+    return Math.abs(writtenBytes - wanted) / wanted > CEILING_DRIFT_TOLERANCE;
+}
+/**
+ * Write the drop-in. Returns false when nothing needed doing.
+ *
+ * The «never below what we already hold» guard lives in `memoryPolicy` itself
+ * (`CEILING_HEADROOM_OVER_CURRENT`), so writing, reading and the drift check all
+ * arrive at the same number — otherwise a write whose floor was binding would be
+ * seen as drifted on the very next call and rewritten forever.
+ */
+export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts()) {
+    if (!force && !limitsOverrideIsOutdated(undefined, home, facts))
         return false;
     const target = limitsOverridePath(home);
     fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
-    fs.writeFileSync(target, buildLimitsOverride(), { mode: 0o644 });
+    fs.writeFileSync(target, buildLimitsOverride(undefined, facts), { mode: 0o644 });
     return true;
 }
 /**

package/dist/supervisor.d.ts

@@ -19,6 +19,12 @@ export interface SupervisorOptions {
     runnerToken?: string;
     /** Test seam for the `self_update` command. */
     selfUpdate?: typeof selfUpdate;
+    /**
+     * How long a freshly started agent process may say nothing before the session
+     * says so out loud (ticket #225). Test seam — the default is a minute, and a
+     * test that had to wait one would not be written.
+     */
+    startupSilenceMs?: number;
     /**
      * Called after a successful update, once the reply is on the wire. The daemon
      * exits here and systemd starts the new build; without a handler the runner
@@ -113,11 +119,53 @@ export declare class Supervisor {
      * free CHAT session with no prompt at all (the agent boots, reports its
      * capabilities and waits for the first message).
      *
-     * Returns whether an agent process actually started: a caller holding a user
-     * message needs to know, because a refused launch means the message has to
-     * stay queued rather than be marked delivered (session 9).
+     * Returns what came of it: a caller holding a user message needs to know,
+     * because anything but `ok` means the message has to stay queued rather than
+     * be marked delivered (session 9).
+     *
+     * NEVER throws (ticket #225). Everything from reading the project prompt to
+     * the adapter's own constructor runs inside one guard, because the caller
+     * chain above cannot tell the difference between «did not start» and
+     * «threw»: the message path swallows the exception into a log line, and the
+     * reconnect path lets it abort the restore of every OTHER session on the
+     * machine. A launch that fails is a state this session reports, not an
+     * exception somebody else has to remember to catch.
      */
     private launchAgent;
+    /**
+     * The agent process could not be started at all (ticket #225).
+     *
+     * Three things have to happen here, and until this ticket none of them did:
+     * the reason is said WHERE THE PERSON IS LOOKING (an `error` event is the
+     * feed's red line), the session stops claiming to be working, and the stack
+     * reaches journald for whoever has to fix the machine. The status is the
+     * honest one for a session with no process — the agent is not running, and a
+     * session left in `RUNNING` shows a stop button for a turn that does not
+     * exist.
+     *
+     * Never rethrows: this IS the handling. The caller gets `crashed` and decides
+     * what to do with the message it was holding.
+     */
+    private launchCrashed;
+    /** How long a freshly started agent may say nothing before we say so. */
+    private static readonly STARTUP_SILENCE_MS;
+    /**
+     * Watch for the first word out of a process we just started (ticket #225).
+     *
+     * «The adapter object exists» is not «the agent is running». A CLI that hangs
+     * before its first frame — a stuck hook, an MCP server that never answers, a
+     * transcript it cannot read — produces no events, no error and no exit, and
+     * the session sits in `RUNNING` forever. On a healthy launch the first event
+     * arrives in about two seconds, so a minute of silence is not a slow start,
+     * it is something worth saying out loud.
+     *
+     * Says it and stops there: no kill. A long conversation has the right to boot
+     * slowly, and killing it would cost the person the very turn they are waiting
+     * for.
+     */
+    private watchForFirstSignOfLife;
+    /** The process spoke, or went away — either way the watch is over. */
+    private clearStartupWatch;
     /**
      * The project's own prompt file, read fresh for THIS agent process.
      *