@bridge4dev/runner 0.64.1 → 0.65.1

package/dist/supervisor.js

@@ -28,9 +28,9 @@ import { cageAuthority } from './cage-authority.js';
 import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
 import { rememberWorkspacePath } from './environment.js';
 import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
-import { markScopeOomKillsSeen, readScopeHold, readScopeMemoryStatus, readSliceLimits, sessionAgentPid, sessionCage, sessionScopeUnitOf, setLiveLadderSource, stoppedProcesses, sweepOrphanSessionScopes, } from './session-cage.js';
+import { markScopeOomKillsSeen, readScopeHold, readScopeMemoryStatus, readSliceLimits, restartStoppedCommands, sessionAgentPid, sessionCage, sessionScopeUnitOf, setLiveLadderSource, stopSessionScope, stoppedProcesses, sweepOrphanSessionScopes, } from './session-cage.js';
 import { allocateSessionMemory, planLimitMove, SESSION_GUARANTEE_BYTES, } from './session-allocator.js';
-import { freshStallState, pickKillCandidate, readScopeProcesses, readStallSample, setScopeProperties, signalSubtree, stallStep, STALL_GRACE_MS, STALL_MUTE_AFTER_LOWER_MS, STALL_SIGKILL_AFTER_MS, STALL_WINDOW_MS, } from './session-stall.js';
+import { countRunningCommands, freshStallState, pickKillCandidate, readScopeProcesses, readStallSample, setScopeProperties, signalSubtree, stallStep, STALL_GRACE_MS, STALL_MUTE_AFTER_LOWER_MS, STALL_SIGKILL_AFTER_MS, STALL_WINDOW_MS, } from './session-stall.js';
 import { machineReserveBytes } from './service-unit.js';
 import { sessionLimitsChangedEnough, sessionLimitsHeartbeatDue, SESSION_LIMITS_HEARTBEAT_MS, } from './session-limits.js';
 import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
@@ -1693,7 +1693,28 @@ export class Supervisor {
             this.lastHostLoadSentAt = now;
         }
     }
+    /**
+     * Set by {@link shutdown}, and read wherever this supervisor could still be
+     * ASKED to do something after it has let go of everything (#431).
+     *
+     * It exists because the daemon now outlives `shutdown()` by up to three
+     * seconds — the time its cages are given to stop. Until that change every
+     * caller exited in the same tick and «after shutdown» was not a state
+     * anything could be in.
+     */
+    shuttingDown = false;
     async onFrame(frame) {
+        /**
+         * Nothing from the wire is answered once the shutdown has started.
+         *
+         * The socket is closed right after the synchronous half of `shutdown()`,
+         * but closing is a handshake: a frame already in flight still lands. The
+         * client refuses to deliver it (`ws-client.ts`) and this is the second
+         * door, on the side that would act on it — an `event_ack` here deletes the
+         * journal of a session this supervisor has already released.
+         */
+        if (this.shuttingDown)
+            return;
         switch (frame.type) {
             case 'hello_ack':
                 this.helloAcked = true;
@@ -1866,6 +1887,21 @@ export class Supervisor {
      * the ask any more.
      */
     withdrawReason = 'not_held') {
+        /**
+         * Nothing is started once this runner is going down (#431).
+         *
+         * The wire is already shut by then, but this door has a second caller: the
+         * tail of `pumpEvents`, which runs a microtask after the agent's stream
+         * ends — and since the shutdown waits for its cages, the process is still
+         * here to run it. What it would build first is a worktree and a branch, and
+         * what it would build last is a fresh cage, moments before `process.exit`.
+         */
+        if (this.shuttingDown) {
+            log.warn('supervisor: not starting a session, this runner is going down', {
+                sessionId: descriptor.id,
+            });
+            return;
+        }
         const existing = this.sessions.get(descriptor.id);
         if (existing) {
             // A resume (higher epoch) can land while the previous life is still
@@ -2192,6 +2228,22 @@ export class Supervisor {
             });
             return LAUNCH_REFUSED;
         }
+        /**
+         * …and no agent at all once the daemon is on its way out (#431).
+         *
+         * The five one-shot relaunches live in the tail of `pumpEvents`, which runs
+         * a microtask after the event stream of the dying process ends — and since
+         * the shutdown waits for its cages, the process is still here to run it.
+         * A launch there would put a NEW cage on the machine moments before
+         * `process.exit`, leaving behind exactly the orphan this ticket is about.
+         * Found by the independent review of #431.
+         */
+        if (this.shuttingDown) {
+            log.warn('supervisor: not launching an agent, this runner is going down', {
+                sessionId: descriptor.id,
+            });
+            return LAUNCH_REFUSED;
+        }
         // An exhausted USD budget must not relaunch $0.01-floor processes (QA-96 F4).
         // Codex reports no cost at all, so its costUsd never leaves 0 — gating on it
         // would be a limit that can never fire while the UI shows $0.00. Those
@@ -3404,6 +3456,39 @@ export class Supervisor {
         clearTimeout(running.compactionWatchdog);
         delete running.compactionWatchdog;
     }
+    /**
+     * A card that interrupted a RESTING session has been answered: go back to
+     * rest instead of reporting a turn (#382).
+     *
+     * Which card it was does not matter — a Codex helper's approval, a Claude
+     * background subagent's, an ask either of them parked while the session was
+     * already the person's. What matters is that no turn of this session was
+     * running when the card went out, so there is no turn to go back to and
+     * nothing that would end one: on Codex the helper's own ending is explicitly
+     * not the session's, so «Working» stood until somebody typed.
+     *
+     * Only when the burst is over — both card sets empty — because answering one
+     * of three still leaves the session parked on the other two. The frame
+     * carries the background count like every other status report, so the badge
+     * and the Inbox see «resting, with helpers» rather than «your turn».
+     *
+     * Returns true when it handled the resolution, so the callers' «the human
+     * answered, bill again» branches stay out of it.
+     */
+    restAfterCard(running) {
+        const back = running.restBeforeCard;
+        if (!back)
+            return false;
+        if (running.openPermissions.size > 0 || running.openQuestions.size > 0)
+            return true;
+        // `reportStatus` clears the memory — the status it sends is the newest
+        // truth about this session, whatever it is.
+        this.reportStatus(running.descriptor.id, back, {
+            costUsd: running.costUsd,
+            activeMs: Supervisor.spentMs(running),
+        });
+        return true;
+    }
     /**
      * Record how many subagents are alive, and say so when it matters (#236).
      *
@@ -3817,6 +3902,8 @@ export class Supervisor {
                     source: event.source,
                     reason: event.reason,
                 });
+                if (this.restAfterCard(running))
+                    return;
                 if (event.source === 'user' &&
                     event.allow &&
                     running.lastReported === 'WAITING_PERMISSION') {
@@ -3844,6 +3931,11 @@ export class Supervisor {
                 // is the bug session 7 removed (five of the first twelve prod sessions
                 // died having spent their budget waiting for a human).
                 running.openQuestions.add(event.askId);
+                // #382, the same as a permission card: an ask from work that outlives
+                // the turn finds the session at rest, and the answer must put it back.
+                if (running.lastReported === 'WAITING_INPUT' || running.lastReported === 'REVIEW') {
+                    running.restBeforeCard = running.lastReported;
+                }
                 this.sendEvent(running, 'question', {
                     askId: event.askId,
                     questions: event.questions,
@@ -3872,6 +3964,8 @@ export class Supervisor {
                 // `openQuestions.size` matters: both agents can park several asks at
                 // once, and reporting RUNNING while another card is still waiting would
                 // bill a human's thinking time all over again (QA-106 M4).
+                if (this.restAfterCard(running))
+                    return;
                 if (event.source === 'user' &&
                     running.openQuestions.size === 0 &&
                     running.lastReported === 'WAITING_INPUT') {
@@ -7563,6 +7657,13 @@ export class Supervisor {
             // The API infers WAITING_PERMISSION from the event itself, so this never
             // goes through reportStatus — but the budget clock still has to stop, or
             // an ask-mode session bills every second the human spends reading the card.
+            //
+            // #382: remember what the card interrupted. A card from work that
+            // outlives the turn finds the session at rest, and the answer must put it
+            // back there rather than into a turn nobody is running.
+            if (running.lastReported === 'WAITING_INPUT' || running.lastReported === 'REVIEW') {
+                running.restBeforeCard = running.lastReported;
+            }
             running.lastReported = 'WAITING_PERMISSION';
             this.syncBudgetClock(running);
         }
@@ -7572,6 +7673,8 @@ export class Supervisor {
         const running = this.sessions.get(sessionId);
         if (running) {
             running.lastReported = status;
+            // #382: any status report is newer than the rest a card interrupted.
+            delete running.restBeforeCard;
             // Coming to rest ends the busy period the once-per-turn notices were
             // limited to: the next one is about a new answer and deserves saying.
             if (!Supervisor.MID_TURN_STATUSES.includes(status)) {
@@ -7625,8 +7728,40 @@ export class Supervisor {
         // changed, and the interval in the constructor is the net under both.
         this.publishSlots();
     }
-    /** Graceful daemon shutdown: kill agents, keep sessions resumable server-side. */
-    shutdown() {
+    /**
+     * How long the whole fleet of cages is given to stop before this process goes
+     * anyway (#431).
+     *
+     * The bus has been measured at 2.7 s under load and the stops run in
+     * parallel, so three seconds is room for all of them on a bad day. Going over
+     * it costs nothing: `stop` has been ACCEPTED by then, and systemd finishes
+     * the job whether or not anybody is still here to watch. systemd's own
+     * patience with this service is `TimeoutStopSec`, far above this.
+     */
+    static CAGE_STOP_BUDGET_MS = 3_000;
+    /**
+     * Graceful daemon shutdown: kill agents, take their cages with them, keep
+     * sessions resumable server-side.
+     *
+     * **Everything that has to survive this process happens BEFORE the first
+     * `await`** — the withdrawn cards, the feed line, and the word to each agent
+     * that it is over (a signal for Codex; for Claude, stdin closed and the SDK's
+     * own `close()`, whose SIGTERM is a timer this process will not live to see —
+     * on a caged machine what actually ends it is the stop below). A caller that
+     * does not wait (the suite does not) gets exactly the shutdown it always got;
+     * what it misses is only the stop of the cgroups. Written this way rather
+     * than as two methods because «kill the agent» and «put its cage out» are one
+     * action, and the shape of the code is what stops them drifting apart again
+     * (грабля §479).
+     *
+     * What the caller owes in return: close the socket between the two halves.
+     * The events above need it open, and the seconds below must not be a window
+     * in which a `session_start` arrives at a supervisor that has already let go
+     * of everything (`index.ts`).
+     */
+    async shutdown() {
+        // First line: every door that could put work back on this machine reads it.
+        this.shuttingDown = true;
         clearInterval(this.slotsTimer);
         clearInterval(this.hostLoadTimer);
         clearInterval(this.stallTimer);
@@ -7635,6 +7770,10 @@ export class Supervisor {
         clearTimeout(this.agentCleanupFirstTimer);
         clearInterval(this.agentCleanupTimer);
         this.authRelay.cancel();
+        const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
+        const readProcesses = this.opts.readScopeProcesses ?? readScopeProcesses;
+        const agentPidOf = this.opts.sessionAgentPid ?? sessionAgentPid;
+        const cages = [];
         for (const running of this.sessions.values()) {
             this.clearBudgetTimers(running);
             // BEFORE stop(), and from here rather than from the adapter: the adapter
@@ -7656,15 +7795,86 @@ export class Supervisor {
                     error: String(error),
                 });
             }
+            /**
+             * What this restart costs, in the person's units, before anything is
+             * killed (#431).
+             *
+             * The count comes from the cage itself rather than from anything this
+             * process believes: the runner does not track what the agent started —
+             * that is the whole shape of the problem — and the cgroup does. Read
+             * BEFORE `stop()`, because a moment later there is nothing left to count.
+             *
+             * Silent when the number is zero: an agent that was only thinking loses
+             * nothing, and «0 commands were stopped» is a line a person has to read
+             * and cannot use (D8).
+             */
+            const unit = unitOf(running.descriptor.id);
+            if (unit !== null) {
+                cages.push(unit);
+                try {
+                    const commands = countRunningCommands(readProcesses(unit), agentPidOf(running.descriptor.id));
+                    if (commands > 0) {
+                        this.sendEvent(running, 'notice', {
+                            level: 'warn',
+                            text: restartStoppedCommands(commands),
+                        });
+                    }
+                }
+                catch (error) {
+                    // /proc said nothing readable. The cage is still stopped below — the
+                    // person simply does not get the number.
+                    log.warn('supervisor: could not count what this session was running', {
+                        sessionId: running.descriptor.id,
+                        error: String(error),
+                    });
+                }
+            }
             // The reason matters here: this path also runs for «Update runner», and a
             // card that vanishes during an update must say why (session 12).
-            running.session?.stop('runner_restarted');
+            //
+            // Guarded like its neighbours in this loop: one adapter that throws on
+            // its way out must not cost every OTHER session the stop of its cage —
+            // the lines below it are what puts them out (independent review of #431).
+            try {
+                running.session?.stop('runner_restarted');
+            }
+            catch (error) {
+                log.warn('supervisor: an agent did not take its stop cleanly', {
+                    sessionId: running.descriptor.id,
+                    error: String(error),
+                });
+            }
         }
         this.sessions.clear();
         // A build that was mid-flight is abandoned, not judged: its row stays
         // RUNNING until the API's sweep turns it into LOST. A verdict nobody
         // observed must never become PASSED.
         this.verify.shutdown();
+        await this.stopCages(cages);
+    }
+    /**
+     * Put out every cage this daemon was holding, in parallel and on a clock.
+     *
+     * Until 0.65.0 nothing did this: `releaseSessionScope` hangs off the agent
+     * process's `exit`, and on a restart the daemon is gone before that event
+     * fires. What stayed behind was a whole tree — a build, its docker client,
+     * their children — holding the name of a scope systemd will then refuse to
+     * reuse, so the FIRST start of that session after the restart died as
+     * «exited before the session was ready (code 1)» (#431).
+     */
+    async stopCages(units) {
+        if (units.length === 0)
+            return;
+        const stop = this.opts.stopSessionScope ?? stopSessionScope;
+        log.info('supervisor: stopping the cages of this runner', { count: units.length });
+        const all = Promise.allSettled(units.map((unit) => stop(unit))).then(() => undefined);
+        let timer;
+        const budget = new Promise((resolve) => {
+            timer = setTimeout(resolve, this.opts.cageStopBudgetMs ?? Supervisor.CAGE_STOP_BUDGET_MS);
+        });
+        await Promise.race([all, budget]);
+        if (timer)
+            clearTimeout(timer);
     }
 }
 /**

package/dist/version.d.ts

@@ -1,2 +1,2 @@
-export declare const RUNNER_VERSION = "0.64.1";
+export declare const RUNNER_VERSION = "0.65.1";
 //# sourceMappingURL=version.d.ts.map

package/dist/version.js

@@ -1,3 +1,3 @@
 // Kept in sync with package.json by the release script (manual for now).
-export const RUNNER_VERSION = '0.64.1';
+export const RUNNER_VERSION = '0.65.1';
 //# sourceMappingURL=version.js.map

package/dist/ws-client.js

@@ -137,6 +137,20 @@ export class RunnerWsClient {
             this.lastInboundAt = Date.now();
         });
         socket.on('message', (data) => {
+            /**
+             * A client that was stopped answers nothing, in either direction (#431).
+             *
+             * `close()` is the START of a handshake, not a cut: the socket lives on
+             * for a round trip, and this listener used to keep delivering frames off
+             * it. That was harmless while every caller of `stop()` exited in the same
+             * tick; since the daemon waits for its cages to be put out, the window is
+             * seconds wide, and the supervisor on the other side of `frame` has
+             * already let go of every session. An `event_ack` landing there DELETES
+             * the journal of a live session (found by the independent review of
+             * #431); a `session_start` starts an agent nobody will ever stop.
+             */
+            if (this.stopped)
+                return;
             this.lastInboundAt = Date.now();
             let parsed;
             try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bridge4dev/runner",
-  "version": "0.64.1",
+  "version": "0.65.1",
   "description": "DevBridge dev runner — connects a dev server to DevBridge and runs agent sessions (Claude Code / Codex)",
   "homepage": "https://bridge4.dev",
   "license": "MIT",