@bridge4dev/runner 0.57.0 → 0.58.2

package/dist/service-unit.js

@@ -129,7 +129,7 @@ export function buildUnit(execStart, nodeBinary = process.execPath) {
  * which is the only way to fix the servers that already have the bad numbers
  * baked in — and it never overwrites a unit the operator edited by hand.
  */
-export const LIMITS_VERSION = 5;
+export const LIMITS_VERSION = 6;
 const LIMITS_MARKER = '# devbridge-limits-version:';
 /** `zz-` so it sorts last: an operator's own drop-in should still win. */
 const LIMITS_FILE = 'zz-devbridge-limits.conf';
@@ -277,6 +277,21 @@ const CEILING_FLOOR_BYTES = 2 * GIB;
  *  - cap 85 % of total: on an idle dedicated box `MemAvailable` is nearly the whole
  *    machine, and a ceiling of «everything» is the bug this function exists to fix.
  */
+/**
+ * What the machine keeps for itself, whatever the agents are doing.
+ *
+ * Enough for sshd, journald, the kernel and some page cache to keep working
+ * while our cgroups sit at their ceiling. Proportional on a big machine,
+ * absolute on a small one, because 15 % of 4 GB is not enough to stay reachable.
+ *
+ * Hoisted out of {@link memoryPolicy} for #398 S3: the live allocator has to
+ * subtract exactly the same reserve when it decides how much of the machine is
+ * free right now, and a second copy of this number would drift from this one on
+ * the first machine where it mattered.
+ */
+export function machineReserveBytes(totalBytes) {
+    return Math.max(1.5 * GIB, totalBytes * 0.15);
+}
 export function memoryPolicy(facts, minCeilingBytes = managedUsageFloorBytes(facts)) {
     const { totalBytes, availableBytes, ownUsageBytes, sessionsUsageBytes, uptimeSec } = facts;
     const swapMaxBytes = Math.floor(Math.max(0, facts.swapTotalBytes ?? 0) * SESSIONS_SWAP_SHARE);
@@ -307,7 +322,7 @@ export function memoryPolicy(facts, minCeilingBytes = managedUsageFloorBytes(fac
     // Enough for sshd, journald, the kernel and some page cache to keep working
     // while the cgroup sits at its ceiling. Proportional on a big machine, absolute
     // on a small one, because 15 % of 4 GB is not enough to stay reachable.
-    const reserve = Math.max(1.5 * GIB, totalBytes * 0.15);
+    const reserve = machineReserveBytes(totalBytes);
     // Both cgroups are added back, and for the same reason ours always was: what
     // they hold is already OUT of `MemAvailable`, so leaving the sessions out
     // makes a loaded machine look starved and walks the ceiling down under the
@@ -682,7 +697,7 @@ const MANAGED_HEADER = [
  *                can measure one (the drift check treats a file with no
  *                `MemoryMax` as outdated).
  */
-export function buildSessionsSliceOverride(facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull()) {
+export function buildSessionsSliceOverride(facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull(), maxSessions = DEFAULT_MAX_SESSIONS) {
     const memory = facts ? memoryPolicy(facts) : null;
     return ([
         ...MANAGED_HEADER,
@@ -691,6 +706,41 @@ export function buildSessionsSliceOverride(facts = readMemoryFacts(), sessionsUs
             ? [
                 `# ${memory.measured ? 'measured headroom' : 'still booting — conservative fraction of total'}`,
                 `MemoryMax=${asMiB(memory.maxBytes)}`,
+                /**
+                 * The COLLECTIVE brake, and the thing that makes #398 S3 safe.
+                 *
+                 * Since S3 each session's own brake is deliberately oversold — every
+                 * session may grow into everything that is free — so the sum of the
+                 * brakes is larger than this slice. That is only safe while THIS
+                 * line exists: as the sessions together approach the ceiling the
+                 * kernel slows all of them and reclaims first from whoever is above
+                 * their `MemoryLow`, and the hard ceiling above, where the OOM
+                 * killer picks a victim across the whole slice, is never reached.
+                 *
+                 * Measured on this host before it existed: the slice read
+                 * `MemoryHigh=infinity`, overflowing it ended in a kill rather than
+                 * throttling, and the one killed was the neighbour sitting under its
+                 * own limit (`memory.events` of the victim: `high 0, max 0, oom 0,
+                 * oom_kill 1`). The allocator reads this property back and falls
+                 * back to a non-oversold formula when it is not in force.
+                 */
+                `MemoryHigh=${asMiB(Math.floor(memory.maxBytes * SLICE_BRAKE_OF_CEILING))}`,
+                /**
+                 * …and the protection the guarantees are built on.
+                 *
+                 * `memory.low` of a child is bounded by its ancestors' protection:
+                 * without this line a session's own `MemoryLow` protects nothing at
+                 * all. Deliberately a CONSTANT (seats × one guarantee), not the sum
+                 * of the guarantees of the sessions that happen to be live: the sum
+                 * changes on every start and finish, the slice is never rewritten
+                 * live (a blocker QA found on 07.09 — a rewritten slice kills the
+                 * sessions under it), so a number that depended on the live roster
+                 * would be true for the hour after this file is written and a lie
+                 * for the rest of it. What separates one session from another lives
+                 * on their own scopes; this line only gives them the right to be
+                 * protected at all.
+                 */
+                `MemoryLow=${asMiB(Math.min(Math.max(1, maxSessions) * SESSION_GUARANTEE_BYTES, memory.maxBytes))}`,
             ]
             : unmeasuredSliceCeiling(sessionsUsageBytes)),
         // The line the cage is built on. See `session-cage.ts` and the note above.
@@ -701,6 +751,31 @@ export function buildSessionsSliceOverride(facts = readMemoryFacts(), sessionsUs
         'CPUAccounting=yes',
     ].join('\n') + '\n');
 }
+/**
+ * Where the collective brake sits under the collective ceiling.
+ *
+ * Nine tenths: close enough that the sessions get almost the whole pot before
+ * anything is throttled, far enough that the kernel has a tenth of the pot in
+ * which to slow everyone down and reclaim, instead of arriving at the hard
+ * ceiling and choosing a victim.
+ */
+export const SLICE_BRAKE_OF_CEILING = 0.9;
+/**
+ * The guarantee one session gets, in the one place the slice needs to know it.
+ *
+ * Copied rather than imported from `session-allocator.ts`: that module imports
+ * `session-cage.ts`, which imports this one, and the cycle would be real. The
+ * two are compared by a test (`service-unit.test.ts`) so they cannot drift.
+ */
+const SESSION_GUARANTEE_BYTES = 2048 * MIB;
+/**
+ * Seats assumed when nobody has said otherwise.
+ *
+ * Three, which is what `DevServer.maxSessions` defaults to in the database. The
+ * runner's own config may say fewer; the slice's protection is sized for the
+ * seats the machine sells, not for the sessions that happen to be running.
+ */
+export const DEFAULT_MAX_SESSIONS = 3;
 /**
  * The `[Slice]` lines for a machine whose memory the policy could not measure.
  * See {@link buildSessionsSliceOverride} for why each of the three answers is
@@ -853,7 +928,9 @@ function swapShareHasDrifted(contents, facts) {
  * arrive at the same number — otherwise a write whose floor was binding would be
  * seen as drifted on the very next call and rewritten forever.
  */
-export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull()) {
+export function writeLimitsOverride(force = false, home = systemdUserHome(), facts = readMemoryFacts(), sessionsUsageBytes = readSessionsSliceUsageOrNull(), 
+/** Seats this machine sells — sizes the slice's collective `MemoryLow` (#398 S3). */
+maxSessions = DEFAULT_MAX_SESSIONS) {
     // Unknown usage can be a busy service whose systemd query timed out. Do not
     // replace an existing drop-in with a guessed percentage, or remove a limit
     // by rewriting the file without its MemoryMax line. Defer the whole policy.
@@ -866,7 +943,7 @@ export function writeLimitsOverride(force = false, home = systemdUserHome(), fac
     // them is a machine whose sessions are capped but whose daemon is not
     // prioritised — the regression described on `buildDevbridgeSliceOverride`.
     write(limitsOverridePath(home), buildLimitsOverride(undefined, facts));
-    write(sessionsSliceOverridePath(home), buildSessionsSliceOverride(facts, sessionsUsageBytes));
+    write(sessionsSliceOverridePath(home), buildSessionsSliceOverride(facts, sessionsUsageBytes, maxSessions));
     write(devbridgeSliceOverridePath(home), buildDevbridgeSliceOverride());
     return true;
 }

package/dist/session-allocator.d.ts

@@ -0,0 +1,259 @@
+/**
+ * How much memory each live session may use, decided from what is actually free
+ * (#398 S3, ticket #398).
+ *
+ * The number this replaces was `max(pot / 3, 2 GiB)`, computed ONCE at daemon
+ * start and never again. The divisor came from `maxSessions` defaulting to three
+ * and from nothing else, so a lone session on an idle machine got a third of the
+ * pot and the other two thirds sat unused — the owner's complaint in #398, word
+ * for word: «ночью машина свободна ~9 ГБ, а сессии всё равно нельзя выйти за
+ * 2.5 ГБ».
+ *
+ * **The decisive idea is three levels, and it has to be understood before any of
+ * the arithmetic below.** Today a session has two numbers, the brake and the
+ * wall, and both of the jobs — «do not crowd your neighbour» and «do not kill
+ * the machine» — are being asked of them at once. That is unsatisfiable: for a
+ * lone session to take nearly the whole pot its brake must be nearly the whole
+ * pot, and then two such sessions overflow the pot between them without either
+ * being braked — and what overflows is the SLICE, where the victim is chosen by
+ * the kernel and the one that dies is not the one at fault (measured, #387).
+ *
+ * So there are three, the way every grown-up scheduler has three:
+ *
+ *  - **the guarantee** (`MemoryLow`) — small and untouchable. The sum of the
+ *    guarantees has to fit in the pot, and this is the ONLY place a sum is
+ *    checked. It answers «how much will not be taken from me».
+ *  - **the brake** (`MemoryHigh`) — large and DELIBERATELY oversold. Every
+ *    session may grow into everything that is free right now. The sum of the
+ *    brakes is greater than the pot, and that is not a mistake.
+ *  - **the collective brake on the slice** (`MemoryHigh` on
+ *    `devbridge-sessions.slice`, `service-unit.ts`) — the thing that makes the
+ *    overselling safe. As the sessions together approach the pot the kernel
+ *    slows all of them and reclaims first from whoever is above their guarantee,
+ *    so the hard ceiling — where the OOM killer picks — is never reached.
+ *  - **the wall** (`MemoryMax`) — still one guarantee below the pot, so a lone
+ *    runaway meets ITS OWN wall before the shared pot overflows (rule of #387).
+ *
+ * Overselling the brakes WITHOUT the collective brake on the slice is a bug, not
+ * a solution. The two halves are one change, and {@link allocateSessionMemory}
+ * refuses to oversell when the caller reports that the slice has no live
+ * `MemoryHigh` — see {@link AllocatorMachine.collectiveBrake}.
+ */
+/**
+ * What one session is guaranteed and will not have taken away.
+ *
+ * 2 GiB, sized over the 1571 MB peak measured for a workspace `pnpm typecheck`
+ * in this monorepo. A dev server that cannot run one of those is not a dev
+ * server, so this is the number below which a session is not worth starting.
+ */
+export declare const SESSION_GUARANTEE_BYTES: number;
+/**
+ * Growth allowance over what a session already holds.
+ *
+ * The same 1.25 `service-unit.ts` uses for the ceiling it writes over a cgroup's
+ * current usage, and for the same reason: a limit at exactly what is held is a
+ * limit that is already breached.
+ */
+export declare const SESSION_HEADROOM_OVER_HOLD = 1.25;
+/**
+ * The floor under the BRAKE, over what a session already holds — 1.05, and it
+ * is a different number from the one above on purpose (#398 §8, #403).
+ *
+ * The wall may sit a quarter above the hold: it is a hard limit, and headroom
+ * there costs nothing. The brake may not, because it is followed every tick: at
+ * 1.25 a session that merely holds its ground has its brake RAISED on every
+ * pass, `addMemoryTo` reads that as «the machine found it more memory», and the
+ * stall deadline resets for ever — measured in the suite, four tests waiting
+ * for a stop that could no longer come.
+ *
+ * 1.05 is enough for the one thing this floor is for: `memory.current` counts
+ * page cache, so a brake at exactly the anonymous hold is already breached.
+ */
+export declare const SESSION_BRAKE_OVER_HOLD = 1.05;
+/**
+ * The step the DIVIDED guarantee is rounded down to — 64 MiB (#403).
+ *
+ * `MemoryLow` is the one number here with no dead zone of its own, on purpose:
+ * a guarantee is what a person is promised, and a promise that lags a tick
+ * behind the machine is worse than one that follows it. But the divided
+ * guarantee is `pool / n`, and the pool follows `MemAvailable`, which breathes:
+ * measured on a hungry machine with two sessions, a pot of 3800 MiB and 11 MiB
+ * of breathing wrote `MemoryLow` on ten ticks out of ten where the code before
+ * these fixes wrote it on one.
+ *
+ * Rounding down rather than to nearest, because the sum of the guarantees is
+ * the one sum in this module that has to fit inside the pot.
+ */
+export declare const GUARANTEE_STEP_BYTES: number;
+/** Below this pot the machine is honestly single-session — see the formula. */
+export declare function hungryPotBytes(baseBand: number): number;
+/** The band this machine uses before any session-specific widening. */
+export declare function baseBandBytes(oomContinue: boolean): number;
+/** One live session, as the allocator needs to see it. */
+export interface AllocatorSession {
+    id: string;
+    /**
+     * `memory.current − (file − shmem)`: what the session holds and cannot simply
+     * give back. Page cache is excluded because lowering a limit over clean file
+     * pages makes the kernel reclaim them, not kill anything — counting them would
+     * inflate every number by whatever the machine happened to have cached.
+     */
+    holdBytes: number;
+    /** `memory.current`, the whole reading — what `MemAvailable` cannot see. */
+    currentBytes: number;
+}
+/**
+ * What the machine's OWNER has said, in `config.toml` (#398 S6).
+ *
+ * Every field optional, and every absent one means «the shipped default». The
+ * knobs exist because this formula travels to dev servers nobody here has ever
+ * seen: the two previous memory rules were both reasonable on the author's
+ * machine and destructive somewhere else, and an owner who has to get work done
+ * tonight needs a way back that is not «downgrade the runner».
+ */
+export interface AllocatorKnobs {
+    /** `false` — the pre-0.58.0 fixed third, computed once and never moved. */
+    adaptive?: boolean;
+    /** The guarantee, bytes. Default {@link SESSION_GUARANTEE_BYTES}. */
+    guaranteeBytes?: number;
+    /** A ceiling on the ceiling, bytes: no session grows past this whatever is free. */
+    sessionMaxBytes?: number;
+}
+/** What the machine says about itself right now. */
+export interface AllocatorMachine {
+    /** `MemoryMax` systemd has in force on the sessions slice. Never rewritten live. */
+    potBytes: number | null;
+    /** `MemAvailable`. */
+    availableBytes: number;
+    /** `MemTotal`. */
+    totalBytes: number;
+    /** What the machine keeps for itself — `machineReserveBytes(totalBytes)`. */
+    reserveBytes: number;
+    /** Does a kill inside a scope cost one process or the whole session? */
+    oomContinue: boolean;
+    /**
+     * Is there a LIVE `MemoryHigh` on the sessions slice, read back off systemd?
+     *
+     * The hard invariant of this module. Overselling the brakes is safe only while
+     * the collective brake exists; without it, overflowing the slice ends in a
+     * kill rather than in throttling, and the one killed is the neighbour sitting
+     * under its own limit — measured on this host, victim `memory.events`:
+     * `high 0, max 0, oom 0, oom_kill 1`.
+     */
+    collectiveBrake: boolean;
+    /** Swap one session may push into, as `session-cage.ts` computes it. */
+    swapBytes: number;
+}
+/** The three numbers of one session, plus the band they were built with. */
+export interface SessionAllocation {
+    id: string;
+    guaranteedBytes: number;
+    brakeBytes: number;
+    wallBytes: number;
+    swapBytes: number;
+    bandBytes: number;
+    /** This session's guarantee does not fit in the pot beside the others (D9). */
+    tight: boolean;
+}
+export interface Allocation {
+    sessions: SessionAllocation[];
+    /** The live pot — what is actually available to sessions right now. */
+    poolBytes: number;
+    /** What the sessions hold between them. */
+    heldBytes: number;
+    /** What nobody holds. */
+    freeBytes: number;
+    /** How many guarantees fit in the live pot. */
+    guaranteesFit: number;
+    /** The conservative formula was used because the slice has no collective brake. */
+    conservative: boolean;
+    /** The machine is honestly single-session: the pot is below `2G + band`. */
+    hungry: boolean;
+}
+/**
+ * The whole decision, as a pure function of the numbers.
+ *
+ * Kept pure and kept here, away from every timer and socket, because what it has
+ * to be is CORRECT on machines nobody has: the formula travels with the runner
+ * to every customer's dev server, and the last two times a memory rule shipped
+ * it was reasonable on the author's machine and destructive somewhere else
+ * (#387: the victim was the neighbour; `service-unit.ts:314`: a 4 GB machine got
+ * `MemoryHigh == MemoryMax`). So the fleet's profiles are fixtures, and the
+ * eight properties below are tests rather than comments.
+ */
+export declare function allocateSessionMemory(machine: AllocatorMachine, sessions: AllocatorSession[], knobs?: AllocatorKnobs): Allocation;
+/**
+ * What was last written to one session's scope, and how long it has been asking
+ * to shrink.
+ *
+ * The asymmetry below is not caution, it is a measurement: `MemoryMax` lowered
+ * below what a process holds ends it in seconds (verified on this host — a scope
+ * holding 200 MiB, `MemoryMax` set to 64 MiB, the process read «Killed» and the
+ * unit went `inactive`). An oversold limit is safe; a killed honest session is
+ * not recoverable.
+ */
+export interface WrittenLimits {
+    brakeBytes: number;
+    wallBytes: number;
+    guaranteedBytes: number;
+    /**
+     * `MemorySwapMax`, which moves only when the MACHINE changes under us.
+     *
+     * It is here because of a live example: on 09.09.2026 a swap file appeared on
+     * vmi3219930 and every session on it went on being told `MemorySwapMax=0`,
+     * because the cage measures the machine once at daemon start. Without swap the
+     * brake stops instead of slowing, so that machine kept the exact failure this
+     * whole plan is about while the means to avoid it sat unused on its disk.
+     */
+    swapBytes: number;
+    /** Consecutive ticks on which the session's own numbers asked to come down. */
+    shrinkTicks: number;
+    /**
+     * What the session held when these numbers were written.
+     *
+     * The difference between «the session shrank» and «a neighbour arrived», and
+     * without it there was no difference at all: the wall came down whenever the
+     * formula wanted less, whatever the reason — which the design and the comment
+     * beside it both forbid. Found by the independent review of 10.09.2026.
+     */
+    holdBytes: number;
+}
+/** Ticks in a row a session must ask to shrink before its wall comes down. */
+export declare const SHRINK_TICKS_TO_LOWER = 4;
+/**
+ * Neither number moves for less than this — `max(256 MiB, 10 %)`.
+ *
+ * Every write is a `systemctl` call, and the bus was measured at 2.7 s under
+ * load. A dead zone is what keeps a pot that breathes by a few megabytes from
+ * costing a bus call per session per tick.
+ */
+export declare function deadZoneBytes(value: number): number;
+export interface LimitMove {
+    brakeBytes: number;
+    wallBytes: number;
+    guaranteedBytes: number;
+    swapBytes: number;
+    holdBytes: number;
+    shrinkTicks: number;
+    /** Properties to write, empty when nothing moved enough to be worth a call. */
+    properties: string[];
+    /** The brake came DOWN — the stall detector has to be muted for a while. */
+    lowered: boolean;
+}
+/**
+ * What to actually write, given what is already in force.
+ *
+ * Up is free and immediate: memory nobody is using costs nobody anything.
+ * Down is where every rule lives.
+ */
+export declare function planLimitMove(written: WrittenLimits | null, wanted: SessionAllocation, holdBytes: number, 
+/**
+ * Does this systemd take `MemoryLow=` at all? (#398 S3)
+ *
+ * False costs the machine its guarantees and nothing else. It has to be
+ * honoured HERE and not only at spawn, because every property goes to systemd
+ * in ONE `set-property` call: a single directive this systemd refuses would
+ * take the brake and the wall down with it, every tick, for every session.
+ */
+memoryLow?: boolean): LimitMove;
+//# sourceMappingURL=session-allocator.d.ts.map