@bridge4dev/runner 0.57.0 → 0.58.2

package/dist/supervisor.d.ts

@@ -5,7 +5,9 @@ import { installAgent } from './agent-install.js';
 import { pruneNativeClaudeVersions } from './agent-cleanup.js';
 import { type AgentVersionsMeasurement } from './agent-versions.js';
 import { type HostLoadFrame } from './host-load.js';
-import { type ScopeMemoryStatus } from './session-cage.js';
+import { type ScopeHold, type ScopeMemoryStatus, type SliceLimits } from './session-cage.js';
+import { type AllocatorKnobs } from './session-allocator.js';
+import { type ScopeProcess, type StallSample } from './session-stall.js';
 import { createCheckpoint } from './checkpoints.js';
 import type { RunnerWsClient } from './ws-client.js';
 import type { SessionDescriptor } from './protocol.js';
@@ -165,8 +167,49 @@ export interface SupervisorOptions {
      */
     sessionScopeUnitOf?: (id: string) => string | null;
     readScopeMemoryStatus?: (unit: string) => ScopeMemoryStatus | null;
-    /** How long over the brake before the second, «do something» warning. A test seam. */
-    cageBrakeLongMs?: number;
+    /**
+     * #398 S1: the orphan sweep, so a test can prove the tick calls it — and that
+     * it never calls it twice at once. Production sweeps for real.
+     */
+    sweepOrphanSessionScopes?: (liveIds: Iterable<string>) => Promise<string[]>;
+    /**
+     * #398 S2: the stall mechanism's four doors to the machine.
+     *
+     * All four are seams for the same reason the cage's two are: putting a real
+     * session into a real stall needs a real kernel refusing real memory, and no
+     * suite can arrange that. Production reads `/sys` and `/proc` and talks to
+     * systemd.
+     */
+    readStallSample?: (unit: string, at: number) => StallSample | null;
+    readScopeProcesses?: (unit: string) => ScopeProcess[];
+    setScopeProperties?: (unit: string, properties: string[]) => Promise<boolean>;
+    signalSubtree?: (pids: Array<number | {
+        pid: number;
+        startedAtTicks: number;
+    }>, signal: NodeJS.Signals) => number;
+    /** The detector's cadence, ms. Real one is 5 s — no suite waits that out. */
+    stallSampleMs?: number;
+    /** How long a standing session is given. Real one is 3 minutes (D6). */
+    stallGraceMs?: number;
+    /**
+     * #398 S6: what the machine's owner asked for in `config.toml`.
+     *
+     * Read once in `cmdDaemon` and passed down, which is this package's idiom for
+     * config — nothing outside `index.ts` imports `config.js`, so a module that
+     * read the file itself would be the first exception and the first place for
+     * two answers about one setting to appear.
+     */
+    memoryKnobs?: AllocatorKnobs;
+    /** `report-only` leaves the deadline running and stops nothing (#398 S6). */
+    memoryStallAction?: 'stop-command' | 'report-only';
+    /**
+     * #398 S3: the allocator's two readings of the machine, so a test can put a
+     * session on a crowded or an empty box without one. Production reads cgroupfs.
+     */
+    readSliceLimits?: () => SliceLimits | null;
+    readScopeHold?: (unit: string) => ScopeHold | null;
+    /** #398 S4: the heartbeat for the memory frame. Real one is a minute. */
+    sessionLimitsHeartbeatMs?: number;
 }
 export declare class Supervisor {
     private readonly ws;
@@ -245,23 +288,6 @@ export declare class Supervisor {
     private readonly verify;
     private readonly verifyReports;
     constructor(ws: RunnerWsClient, opts: SupervisorOptions);
-    /**
-     * How long a session may sit over its brake before the feed says more than
-     * «slower» (#387, step 3). Five minutes is a starting point, not a
-     * measurement: an honest `pnpm typecheck` overshoots for tens of seconds, a
-     * runaway for ever, and the number that tells them apart on real machines is
-     * still to be read off `memory.events`. Sized so the first warning is never
-     * the only one a night-time run leaves behind.
-     */
-    private static readonly CAGE_BRAKE_LONG_MS;
-    /**
-     * Calm ticks before the feed says the braking is over.
-     *
-     * `memory.high` holds usage AT the line, so a single sample under it means
-     * nothing. Two (a minute at the 30 s cadence) is the difference between an
-     * episode that ended and a build that breathed.
-     */
-    private static readonly CAGE_CALM_TICKS_TO_RELEASE;
     /** How often the agent versions are re-derived. See the constructor. */
     private static readonly AGENT_VERSIONS_INTERVAL_MS;
     private readonly agentVersionsTimer;
@@ -413,6 +439,13 @@ export declare class Supervisor {
     /** The heartbeat window actually used — the constant, or a test's own. */
     private readonly hostLoadHeartbeatMs;
     private readonly hostLoadTimer;
+    /** #398 S2: the stall detector's clock, and the deadline it enforces. */
+    private readonly stallTimer;
+    private readonly stallGraceMs;
+    private readonly stallSampleMs;
+    private readonly limitsHeartbeatMs;
+    /** A stall tick is in flight — the next one waits rather than overlaps. */
+    private watchingStalls;
     /** The last measurement the API actually took from us, or `null` for «nothing yet». */
     private lastPublishedHostLoad;
     /** When that frame went out, by this machine's clock. `0` = never. */
@@ -443,28 +476,185 @@ export declare class Supervisor {
      * it had one.
      */
     /**
-     * Tell the person what the kernel is doing to each caged session (#387).
+     * Tell the person when the KERNEL killed something inside a caged session.
+     *
+     * One thing, since #398 S2 — `oom_kill` moved: the kernel stopped a process
+     * inside the session. With `OOMPolicy=continue` the session itself is still
+     * here, and the person has to be told that the command that was running is
+     * probably what died.
      *
-     * Three things the cgroup can say, each with its own line in the feed:
+     * **The three braking lines this tick used to write are gone** (decision D8 of
+     * `session-memory-fair-share.md`): «has hit its memory share», «has been over
+     * its memory share for 5 min», and the all-clear. Two reasons, and the second
+     * is the decisive one:
      *
-     *  - the brake engaged (`memory.current` over `memory.high`): the session is
-     *    being slowed down, not killed. Said once per episode, and again when it
-     *    has lasted `CAGE_BRAKE_LONG_MS` — that second line is the one that
-     *    suggests an action, because by then «slower» may mean «stuck»;
-     *  - the brake released: said once, so a person who saw the warning knows the
-     *    wait is over without reading the numbers;
-     *  - `oom_kill` moved: the kernel killed a process inside the session. With
-     *    `OOMPolicy=continue` the session itself is still here, and the person
-     *    has to be told that the command that was running is probably what died.
+     *  - they were state, not events. Being over the brake is a condition that
+     *    comes and goes; its place is the activity strip under the session and the
+     *    machine card, not the history a person scrolls back through;
+     *  - they disagreed with the mechanism that replaced them. The deadline in
+     *    `session-stall.ts` runs out after three minutes, so the five-minute line
+     *    could never have been printed at all.
      *
-     * Until this tick existed the runner knew all three and wrote them to its own
-     * log only; the feed said «exited with code 143» and nothing about memory.
+     * What a person sees instead: nothing while the machine copes, and exactly one
+     * line when a command was actually stopped.
      *
      * A parked session has no process and no cgroup, and is skipped without a
      * read. A session whose cgroup cannot be read (no cage on this machine, or
      * the scope already gone) is skipped the same way: the watch never guesses.
      */
     private watchSessionCages;
+    /** A sweep is in flight — see {@link sweepOrphanCages}. */
+    private sweepingCages;
+    /**
+     * Stop every session scope with nobody behind it, on the watch tick (#398 S1).
+     *
+     * The sweep itself spares every LIVE cage by its own register, sessions and
+     * `verify` runs alike (`session-cage.ts`), so what is passed here is belt and
+     * braces rather than the guard — but passing it says at the call site which
+     * ids this supervisor believes are alive, and a divergence between the two
+     * would be a bug worth seeing in a test.
+     *
+     * Re-entrancy matters more than it looks: the sweep is a `systemctl` call and
+     * the bus was measured at 2.7 s under load, so on a machine where the tick is
+     * 30 s and the bus is slow two sweeps could otherwise overlap and each would
+     * see the other's half-stopped units.
+     *
+     * Never fatal, and deliberately silent on failure: the daemon carries a fatal
+     * `unhandledRejection` handler, and losing every session because a bus call
+     * failed would be a far worse trade than an orphan living one more tick.
+     */
+    private sweepOrphanCages;
+    /**
+     * The stall mechanism, once per window (#398 S2, D6, gotcha §480).
+     *
+     * The order is the whole design, and each step exists because the one before
+     * it was not enough:
+     *
+     *  1. **T0 — add.** If the machine has memory free, the brake goes up and the
+     *     session carries on. Nothing is written to the feed: the mechanism
+     *     worked, nothing changed for the person, the build is still going (D8).
+     *  2. **T1 — the deadline.** Three minutes, announced as state (the strip
+     *     under the session says how long is left) and never as a line in the
+     *     history. This is the window in which a person can step in without
+     *     paying for it with an entry they will scroll past for ever.
+     *  3. **T2 — stop the biggest command.** Not the session, not the agent: the
+     *     command. The agent stays alive, sees its command stopped, is told why,
+     *     and answers the person itself. Waiting for a human instead is what cost
+     *     the incident its hour — at night nobody comes.
+     *  4. **T3 — nothing to stop.** When all that is left in the cage is the agent
+     *     and its tools, this mechanism has no right to any of them. State again,
+     *     no line: the person decides with «Stop» or «Pause».
+     *
+     * Never throws out of here. The daemon carries a fatal `unhandledRejection`
+     * handler, so one unhandled error would end every session on the machine.
+     */
+    private watchSessionStalls;
+    private stepOneStall;
+    /**
+     * Ask the allocator for more, now, because this session has stopped moving (T0).
+     *
+     * Since S3 there is no separate ladder here: `MemoryHigh` has exactly TWO
+     * writers, and they are the same code — the allocator on its tick and the
+     * allocator on this signal. Two independent writers of one number is how a
+     * limit ends up being whatever the last tick happened to think.
+     *
+     * The allocator needs no special case for a stall: a standing session's `hold`
+     * is by definition all the way up against its brake, so `hold + free` asks for
+     * everything the machine has spare. If there is nothing spare, nothing moves,
+     * and the deadline below is the answer instead.
+     */
+    private addMemoryTo;
+    /** T2 and T3 — stop the biggest command of this session, or nothing at all. */
+    private stopBiggestCommand;
+    /**
+     * Say something to the agent on the runner's own initiative, without stepping
+     * on a card the person has not answered (#398 S7, B7).
+     *
+     * `session.send()` looks like a neutral channel and is not one: with a
+     * question open, `ClaudeSession.send` answers it «discuss» and the API
+     * records `question_resolved {source: 'user'}` — a decision attributed to a
+     * person who never made it — while `CodexSession.send` declines a held plan
+     * the same way. Every other initiative of this supervisor already checks for
+     * an open card; the memory mechanism was the one that did not.
+     *
+     * Held rather than dropped: the agent is parked on the card, so it cannot run
+     * anything until the person answers anyway, and once they do the note is the
+     * first thing it reads.
+     */
+    private tellAgent;
+    /** Deliver what waited for a card to close — see {@link tellAgent}. */
+    private flushPendingAgentNote;
+    /** How many sessions are running in a cage right now (#398 S7, B9). */
+    private liveCagedSessions;
+    /** SIGKILL for a subtree that did not take SIGTERM within the grace. */
+    private escalatePendingKill;
+    /** An allocator pass is in flight — the next one waits rather than overlaps. */
+    private allocating;
+    /** The last pass's machine-wide numbers, for the card and the session banner. */
+    private lastAllocation;
+    /**
+     * The machine the last allocation was computed for, kept so the card's own
+     * number can be derived without measuring the machine a SECOND time (#403).
+     */
+    private lastAllocatorMachine;
+    /** The last frame the API actually took from us, or null for «nothing yet». */
+    private lastPublishedLimits;
+    /** When that frame went out, by this machine's clock. `0` = never. */
+    private lastLimitsSentAt;
+    /**
+     * Decide, and write, how much memory each live session may use (#398 S3).
+     *
+     * Runs on the 30 s watch tick and on the stall detector's signal, and nowhere
+     * else: `MemoryHigh` must have exactly one implementation of «what should this
+     * be», or the two writers disagree and the limit becomes whichever ran last.
+     *
+     * Everything that could be wrong about the machine is read fresh here rather
+     * than taken from the daemon-start probe: the pot moves when the hourly
+     * re-measure rewrites the slice, the collective brake decides whether
+     * overselling is even legal, and a neighbouring application growing is
+     * precisely the case #398 was opened about.
+     */
+    private runAllocator;
+    /** The machine as the allocator needs to see it, from live readings only. */
+    private allocatorMachine;
+    /**
+     * What a session STARTING right now should be given (#398 S3, work 9).
+     *
+     * The allocator is asked to compute as though this session already existed,
+     * so a session born at 03:00 gets the number its first tick will confirm
+     * rather than the one the daemon measured whenever it last started.
+     */
+    private ladderForNewSession;
+    /**
+     * Tell the API how this machine is dividing its memory — when it moved (#398 S4).
+     *
+     * Built from the pass the allocator has just finished rather than measured
+     * again: two readings of a moving machine taken a moment apart would let the
+     * card and the scope disagree about the same session, and «the card said 4 GB
+     * and the cgroup said 2» is the exact shape of the bug that made
+     * `limitsCurrent` untrustworthy once already.
+     *
+     * Recorded ONLY when the socket took it, same as `publishHostLoad`: a frame
+     * dropped by a dead socket must not be remembered as sent.
+     */
+    /**
+     * The number on the machine card: what ONE session gets when the machine is
+     * free (decision D4 of `runner-cage-authority.md`, 10.09.2026).
+     *
+     * The card used to carry the ceiling a newcomer would get RIGHT NOW, from the
+     * current crowd — which is honest per second and useless per person: on a
+     * 12 GiB machine with two sessions of 3 GiB it read ~6 GiB, it moved whenever
+     * a neighbour started or stopped, and the sentence beside it said «up to X on
+     * an idle machine», which is a different number entirely. A card is read to
+     * answer «what does this machine give a session», and that question has a
+     * still answer.
+     *
+     * Computed, not measured: the pot and the owner's knobs are all it takes, so
+     * this makes no second reading of a moving machine — the one the shape of
+     * `publishSessionLimits` warns against in its own header.
+     */
+    private idleSessionCeiling;
+    private publishSessionLimits;
     private publishHostLoad;
     private onFrame;
     private startSession;