@bridge4dev/runner 0.56.0 → 0.58.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import fs from 'node:fs';
2
+ import os from 'node:os';
2
3
  import path from 'node:path';
3
4
  import { log } from './log.js';
4
5
  import { CONTEXT_USAGE_MIN_DELTA_RATIO, CONTEXT_USAGE_MIN_DELTA_TOKENS, RATE_LIMITS_RESEND_INTERVAL_MS, } from './levels.js';
@@ -23,14 +24,20 @@ import { installAgent } from './agent-install.js';
23
24
  import { autoUpdateRefusal, claimAgentAutoUpdate } from './agent-auto-update.js';
24
25
  import { pruneNativeClaudeVersions } from './agent-cleanup.js';
25
26
  import { agentByDbValue } from './agent-registry.js';
27
+ import { cageAuthority } from './cage-authority.js';
26
28
  import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
27
29
  import { rememberWorkspacePath } from './environment.js';
28
30
  import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
29
- import { markScopeOomKillsSeen, readScopeMemoryStatus, sessionScopeUnitOf, stoppedProcesses, } from './session-cage.js';
31
+ import { markScopeOomKillsSeen, readScopeHold, readScopeMemoryStatus, readSliceLimits, sessionAgentPid, sessionCage, sessionScopeUnitOf, setLiveLadderSource, stoppedProcesses, sweepOrphanSessionScopes, } from './session-cage.js';
32
+ import { allocateSessionMemory, planLimitMove, SESSION_GUARANTEE_BYTES, } from './session-allocator.js';
33
+ import { freshStallState, pickKillCandidate, readScopeProcesses, readStallSample, setScopeProperties, signalSubtree, stallStep, STALL_GRACE_MS, STALL_MUTE_AFTER_LOWER_MS, STALL_SIGKILL_AFTER_MS, STALL_WINDOW_MS, } from './session-stall.js';
34
+ import { machineReserveBytes } from './service-unit.js';
35
+ import { sessionLimitsChangedEnough, sessionLimitsHeartbeatDue, SESSION_LIMITS_HEARTBEAT_MS, } from './session-limits.js';
30
36
  import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
31
37
  import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, MAX_BUSY_SESSIONS, previewRewind, pruneCheckpoints, } from './checkpoints.js';
32
38
  import { DeliverMessageArgsSchema, QuestionAnswerArgsSchema } from './protocol.js';
33
39
  import { availableModes, MODE_REFUSED_TEXT } from './adapters/types.js';
40
+ import { answersAsMessage } from './adapters/questions.js';
34
41
  /** Refusals shared by every checkpoint command (ticket #126). */
35
42
  const CHECKPOINTS_OFF = 'Restore points are switched off on this server ([checkpoints] enabled = false)';
36
43
  const AGENT_BUSY = 'The agent is still working — stop the turn first';
@@ -231,26 +238,45 @@ export class Supervisor {
231
238
  // Same cadence, same reason: what a session's cgroup is going through is
232
239
  // something that happens to it, and only a tick can notice (#387).
233
240
  this.watchSessionCages();
241
+ // And the same argument one step further (#398, S1): a scope with nobody
242
+ // behind it is also something that HAPPENS, and until 0.58.0 the only
243
+ // sweep ran at daemon start — so an orphan held its memory until the next
244
+ // restart. `releaseSessionScope` is the primary path now; this is the net
245
+ // under it, for the exit that never fired at all.
246
+ void this.sweepOrphanCages();
247
+ // …and the third thing that only a tick can notice (#398 S3): how much of
248
+ // the machine is free RIGHT NOW, and therefore how much each session may
249
+ // use. The number this replaced was computed once, at daemon start.
250
+ void this.runAllocator(Date.now());
234
251
  }, opts.hostLoadSampleMs ?? HOST_LOAD_SAMPLE_INTERVAL_MS);
235
252
  this.hostLoadTimer.unref?.();
253
+ /**
254
+ * The stall detector, on its own faster clock (#398 S2).
255
+ *
256
+ * A second timer rather than more work on the 30 s one, and the reason is
257
+ * arithmetic: the deadline a standing session gets is three minutes, the
258
+ * verdict needs two consecutive windows, and at 30 s a session would spend a
259
+ * third of its deadline being diagnosed. Five seconds costs five small file
260
+ * reads per live caged session — nothing next to what the tick above already
261
+ * does — and only for sessions that have a cgroup at all.
262
+ */
263
+ this.limitsHeartbeatMs = opts.sessionLimitsHeartbeatMs ?? SESSION_LIMITS_HEARTBEAT_MS;
264
+ this.stallGraceMs = opts.stallGraceMs ?? STALL_GRACE_MS;
265
+ this.stallSampleMs = opts.stallSampleMs ?? STALL_WINDOW_MS;
266
+ this.stallTimer = setInterval(() => {
267
+ void this.watchSessionStalls();
268
+ }, this.stallSampleMs);
269
+ this.stallTimer.unref?.();
270
+ /**
271
+ * A session starting mid-tick asks the allocator for its numbers rather
272
+ * than taking the daemon-start snapshot (#398 S3, work 9).
273
+ *
274
+ * Registered here and cleared in `shutdown()`: the hook lives on the cage
275
+ * module, which cannot import this one, and a stale hook pointing at a dead
276
+ * supervisor would be worse than none.
277
+ */
278
+ setLiveLadderSource((id) => this.ladderForNewSession(id));
236
279
  }
237
- /**
238
- * How long a session may sit over its brake before the feed says more than
239
- * «slower» (#387, step 3). Five minutes is a starting point, not a
240
- * measurement: an honest `pnpm typecheck` overshoots for tens of seconds, a
241
- * runaway for ever, and the number that tells them apart on real machines is
242
- * still to be read off `memory.events`. Sized so the first warning is never
243
- * the only one a night-time run leaves behind.
244
- */
245
- static CAGE_BRAKE_LONG_MS = 5 * 60_000;
246
- /**
247
- * Calm ticks before the feed says the braking is over.
248
- *
249
- * `memory.high` holds usage AT the line, so a single sample under it means
250
- * nothing. Two (a minute at the 30 s cadence) is the difference between an
251
- * episode that ended and a build that breathed.
252
- */
253
- static CAGE_CALM_TICKS_TO_RELEASE = 2;
254
280
  /** How often the agent versions are re-derived. See the constructor. */
255
281
  static AGENT_VERSIONS_INTERVAL_MS = 60 * 60 * 1_000;
256
282
  agentVersionsTimer;
@@ -616,6 +642,13 @@ export class Supervisor {
616
642
  /** The heartbeat window actually used — the constant, or a test's own. */
617
643
  hostLoadHeartbeatMs;
618
644
  hostLoadTimer;
645
+ /** #398 S2: the stall detector's clock, and the deadline it enforces. */
646
+ stallTimer;
647
+ stallGraceMs;
648
+ stallSampleMs;
649
+ limitsHeartbeatMs;
650
+ /** A stall tick is in flight — the next one waits rather than overlaps. */
651
+ watchingStalls = false;
619
652
  /** The last measurement the API actually took from us, or `null` for «nothing yet». */
620
653
  lastPublishedHostLoad = null;
621
654
  /** When that frame went out, by this machine's clock. `0` = never. */
@@ -646,22 +679,27 @@ export class Supervisor {
646
679
  * it had one.
647
680
  */
648
681
  /**
649
- * Tell the person what the kernel is doing to each caged session (#387).
682
+ * Tell the person when the KERNEL killed something inside a caged session.
683
+ *
684
+ * One thing, since #398 S2 — `oom_kill` moved: the kernel stopped a process
685
+ * inside the session. With `OOMPolicy=continue` the session itself is still
686
+ * here, and the person has to be told that the command that was running is
687
+ * probably what died.
650
688
  *
651
- * Three things the cgroup can say, each with its own line in the feed:
689
+ * **The three braking lines this tick used to write are gone** (decision D8 of
690
+ * `session-memory-fair-share.md`): «has hit its memory share», «has been over
691
+ * its memory share for 5 min», and the all-clear. Two reasons, and the second
692
+ * is the decisive one:
652
693
  *
653
- * - the brake engaged (`memory.current` over `memory.high`): the session is
654
- * being slowed down, not killed. Said once per episode, and again when it
655
- * has lasted `CAGE_BRAKE_LONG_MS` — that second line is the one that
656
- * suggests an action, because by then «slower» may mean «stuck»;
657
- * - the brake released: said once, so a person who saw the warning knows the
658
- * wait is over without reading the numbers;
659
- * - `oom_kill` moved: the kernel killed a process inside the session. With
660
- * `OOMPolicy=continue` the session itself is still here, and the person
661
- * has to be told that the command that was running is probably what died.
694
+ * - they were state, not events. Being over the brake is a condition that
695
+ * comes and goes; its place is the activity strip under the session and the
696
+ * machine card, not the history a person scrolls back through;
697
+ * - they disagreed with the mechanism that replaced them. The deadline in
698
+ * `session-stall.ts` runs out after three minutes, so the five-minute line
699
+ * could never have been printed at all.
662
700
  *
663
- * Until this tick existed the runner knew all three and wrote them to its own
664
- * log only; the feed said «exited with code 143» and nothing about memory.
701
+ * What a person sees instead: nothing while the machine copes, and exactly one
702
+ * line when a command was actually stopped.
665
703
  *
666
704
  * A parked session has no process and no cgroup, and is skipped without a
667
705
  * read. A session whose cgroup cannot be read (no cage on this machine, or
@@ -670,8 +708,6 @@ export class Supervisor {
670
708
  watchSessionCages() {
671
709
  const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
672
710
  const read = this.opts.readScopeMemoryStatus ?? readScopeMemoryStatus;
673
- const longMs = this.opts.cageBrakeLongMs ?? Supervisor.CAGE_BRAKE_LONG_MS;
674
- const now = Date.now();
675
711
  const mb = (bytes) => `${Math.round(bytes / (1024 * 1024))} MB`;
676
712
  for (const running of this.sessions.values()) {
677
713
  if (running.session === null)
@@ -702,15 +738,7 @@ export class Supervisor {
702
738
  status.ownLimitOom >= seen.ownOom;
703
739
  const base = carriedOver
704
740
  ? seen
705
- : {
706
- unit,
707
- oomKills: 0,
708
- ownOom: 0,
709
- highEvents: 0,
710
- brakedSince: null,
711
- warnedLong: false,
712
- calmTicks: 0,
713
- };
741
+ : { unit, oomKills: 0, ownOom: 0, highEvents: 0 };
714
742
  if (status.oomKills > base.oomKills) {
715
743
  // WHOSE limit was hit decides the sentence. `oom` moves only in the
716
744
  // cgroup whose own ceiling was reached; a kill with our counter still
@@ -730,66 +758,778 @@ export class Supervisor {
730
758
  // So a death minutes later is not blamed on a kill the feed already carries.
731
759
  markScopeOomKillsSeen(running.descriptor.id, status.oomKills);
732
760
  }
733
- /**
734
- * «Being braked» is the EVENT counter first, the level second.
735
- *
736
- * `memory.high` works by holding usage at the line: a 30 s sample of
737
- * `current > high` lands over it and under it in turn, which produced a
738
- * warning and an all-clear on alternate ticks for one long build. The
739
- * `high` counter only ever moves when the kernel actually throttled this
740
- * cgroup, so a delta is the honest answer to «is it being slowed down
741
- * right now», and the level is kept as the answer for a session that is
742
- * sitting over the line without allocating.
743
- */
744
- const throttled = status.highEvents > base.highEvents ||
745
- (status.highBytes !== null && status.currentBytes > status.highBytes);
746
- // `memory.high` reads `max` on a machine where the share never applied —
747
- // then there is no number to name, and «its 0 MB share» is worse than no
748
- // number at all.
749
- const share = status.highBytes === null ? '' : ` ${mb(status.highBytes)}`;
750
- let { brakedSince, warnedLong, calmTicks } = base;
751
- if (throttled) {
752
- calmTicks = 0;
753
- if (brakedSince === null) {
754
- brakedSince = now;
755
- warnedLong = false;
756
- this.sendEvent(running, 'notice', {
757
- level: 'warn',
758
- text: `This session has hit its${share} memory share — commands run slower until it uses less memory.`,
759
- });
761
+ running.cageWatch = {
762
+ unit,
763
+ oomKills: status.oomKills,
764
+ ownOom: status.ownLimitOom,
765
+ highEvents: status.highEvents,
766
+ };
767
+ }
768
+ }
769
+ /** A sweep is in flight — see {@link sweepOrphanCages}. */
770
+ sweepingCages = false;
771
+ /**
772
+ * Stop every session scope with nobody behind it, on the watch tick (#398 S1).
773
+ *
774
+ * The sweep itself spares every LIVE cage by its own register, sessions and
775
+ * `verify` runs alike (`session-cage.ts`), so what is passed here is belt and
776
+ * braces rather than the guard — but passing it says at the call site which
777
+ * ids this supervisor believes are alive, and a divergence between the two
778
+ * would be a bug worth seeing in a test.
779
+ *
780
+ * Re-entrancy matters more than it looks: the sweep is a `systemctl` call and
781
+ * the bus was measured at 2.7 s under load, so on a machine where the tick is
782
+ * 30 s and the bus is slow two sweeps could otherwise overlap and each would
783
+ * see the other's half-stopped units.
784
+ *
785
+ * Never fatal, and deliberately silent on failure: the daemon carries a fatal
786
+ * `unhandledRejection` handler, and losing every session because a bus call
787
+ * failed would be a far worse trade than an orphan living one more tick.
788
+ */
789
+ async sweepOrphanCages() {
790
+ if (this.sweepingCages)
791
+ return;
792
+ const injected = this.opts.sweepOrphanSessionScopes;
793
+ /**
794
+ * Two gates, and the first one is the one that had to be rebuilt (#403).
795
+ *
796
+ * The guard here used to be «the cage has been probed», on the reasoning
797
+ * that no test probes it and every daemon does. On 10.09.2026 a new test
798
+ * probed it, and this sweep — from a vitest worker, with an empty register
799
+ * of live cages — stopped three live sessions belonging to other people.
800
+ * The lesson is not «that test was wrong»: a guard that depends on what
801
+ * OTHER people's future code happens to do cannot hold, and it failed
802
+ * silently, because an assumption cannot fail loudly.
803
+ *
804
+ * So the gate is now a right that has to be taken: only the process that
805
+ * claimed `daemon` at start-up sweeps, and a test process cannot claim it.
806
+ * `cage-authority.ts` refuses the underlying `systemctl` as well — this
807
+ * gate is the one that keeps the call from being made at all.
808
+ *
809
+ * The second gate is unchanged and is only thrift: on a machine with no
810
+ * cage there is nothing of ours under that prefix.
811
+ */
812
+ if (!injected && cageAuthority() !== 'daemon')
813
+ return;
814
+ if (!injected && sessionCage().mode !== 'scope')
815
+ return;
816
+ this.sweepingCages = true;
817
+ try {
818
+ const sweep = injected ?? sweepOrphanSessionScopes;
819
+ await sweep([...this.sessions.keys()]);
820
+ }
821
+ catch (error) {
822
+ log.debug('supervisor: the orphan cage sweep did not run', { error: String(error) });
823
+ }
824
+ finally {
825
+ this.sweepingCages = false;
826
+ }
827
+ }
828
+ /**
829
+ * The stall mechanism, once per window (#398 S2, D6, gotcha §480).
830
+ *
831
+ * The order is the whole design, and each step exists because the one before
832
+ * it was not enough:
833
+ *
834
+ * 1. **T0 — add.** If the machine has memory free, the brake goes up and the
835
+ * session carries on. Nothing is written to the feed: the mechanism
836
+ * worked, nothing changed for the person, the build is still going (D8).
837
+ * 2. **T1 — the deadline.** Three minutes, announced as state (the strip
838
+ * under the session says how long is left) and never as a line in the
839
+ * history. This is the window in which a person can step in without
840
+ * paying for it with an entry they will scroll past for ever.
841
+ * 3. **T2 — stop the biggest command.** Not the session, not the agent: the
842
+ * command. The agent stays alive, sees its command stopped, is told why,
843
+ * and answers the person itself. Waiting for a human instead is what cost
844
+ * the incident its hour — at night nobody comes.
845
+ * 4. **T3 — nothing to stop.** When all that is left in the cage is the agent
846
+ * and its tools, this mechanism has no right to any of them. State again,
847
+ * no line: the person decides with «Stop» or «Pause».
848
+ *
849
+ * Never throws out of here. The daemon carries a fatal `unhandledRejection`
850
+ * handler, so one unhandled error would end every session on the machine.
851
+ */
852
+ async watchSessionStalls() {
853
+ if (this.watchingStalls)
854
+ return;
855
+ this.watchingStalls = true;
856
+ try {
857
+ const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
858
+ const sampleOf = this.opts.readStallSample ?? readStallSample;
859
+ const now = Date.now();
860
+ for (const running of this.sessions.values()) {
861
+ const unit = running.session === null ? null : unitOf(running.descriptor.id);
862
+ if (unit === null) {
863
+ /**
864
+ * Parked, or never caged. A stall belongs to a live process; keeping
865
+ * the state would age into a deadline against a session that is not
866
+ * running at all.
867
+ *
868
+ * …and so do the limits (#403). `running.limits` means «what systemd
869
+ * was told about THIS scope», and a parked session has no scope: on
870
+ * resume `cageSpawn` was starting the new one with yesterday's
871
+ * numbers, `sessionMemoryEnv` was telling the agent a ceiling nobody
872
+ * held, and the wall could only come down through four shrink ticks
873
+ * that a fresh session does not have.
874
+ */
875
+ delete running.stall;
876
+ delete running.limits;
877
+ continue;
760
878
  }
761
- else if (!warnedLong && now - brakedSince >= longMs) {
762
- warnedLong = true;
763
- this.sendEvent(running, 'notice', {
764
- level: 'warn',
765
- text: `This session has been over its${share} memory share for ${Math.round(longMs / 60_000)} min and is now using ${mb(status.currentBytes)} — ` +
766
- 'if a command is stuck rather than slow, stop the turn or pause the session.',
879
+ try {
880
+ await this.stepOneStall(running, unit, sampleOf, now);
881
+ }
882
+ catch (error) {
883
+ log.warn('supervisor: the memory stall check failed for a session', {
884
+ sessionId: running.descriptor.id,
885
+ error: String(error instanceof Error ? error.message : error),
767
886
  });
768
887
  }
769
888
  }
770
- else if (brakedSince !== null) {
771
- // One calm sample is noise — see above. Two in a row is an episode that
772
- // really ended, and only then is the all-clear worth a line in the feed.
773
- calmTicks += 1;
774
- if (calmTicks >= Supervisor.CAGE_CALM_TICKS_TO_RELEASE) {
775
- brakedSince = null;
776
- warnedLong = false;
777
- this.sendEvent(running, 'notice', {
778
- level: 'info',
779
- text: `This session is back under its${share} memory share — it runs at full speed again.`,
780
- });
889
+ }
890
+ finally {
891
+ this.watchingStalls = false;
892
+ }
893
+ }
894
+ async stepOneStall(running, unit, sampleOf, now) {
895
+ // The pending SIGKILL first, and before anything can return early: a subtree
896
+ // that ignored SIGTERM must get the second signal even on a tick where the
897
+ // cgroup could not be read at all.
898
+ this.escalatePendingKill(running, now);
899
+ /**
900
+ * Stamped at the READ, not at the start of the pass (#398 S7, B13).
901
+ *
902
+ * `now` is one clock for the whole sweep, and the sweep walks every live
903
+ * session doing file reads — so the last session's sample carried the first
904
+ * session's timestamp. The share this feeds is `Δpressure / windowMs`, and
905
+ * an offset between the two ends of a window is exactly the error that
906
+ * makes it wrong. `readHostLoad` has always stamped inside itself.
907
+ */
908
+ const sample = sampleOf(unit, Date.now());
909
+ if (sample === null)
910
+ return;
911
+ const previous = running.stall && running.stall.detector.unit === unit
912
+ ? running.stall
913
+ : {
914
+ detector: freshStallState(unit),
915
+ sample: null,
916
+ remainingMs: null,
917
+ kill: null,
918
+ reason: null,
919
+ };
920
+ const verdict = stallStep(previous.detector, sample, {
921
+ graceMs: this.stallGraceMs,
922
+ windowMs: this.stallSampleMs,
923
+ now,
924
+ // Alone in the slice, «somebody else's brake» has no somebody else — see
925
+ // the discriminator in `session-stall.ts` (#398 S7, B9).
926
+ soleSession: this.liveCagedSessions() <= 1,
927
+ });
928
+ const stall = {
929
+ ...previous,
930
+ detector: verdict.state,
931
+ sample,
932
+ remainingMs: verdict.remainingMs,
933
+ };
934
+ running.stall = stall;
935
+ if (!verdict.stuck) {
936
+ /**
937
+ * The session is moving again, so whatever the strip was saying about its
938
+ * memory stops being true — and state that outlives its cause is exactly
939
+ * what decision D8 moved out of the feed to avoid.
940
+ *
941
+ * Found by the independent review of 10.09.2026: `reason` was set and
942
+ * never cleared, so one episode made the strip say «low on memory» for
943
+ * the rest of the session's life. Kept while a subtree is still being
944
+ * stopped: the line about the stopped command is true until it is gone.
945
+ */
946
+ if (stall.kill === null)
947
+ stall.reason = null;
948
+ return;
949
+ }
950
+ // T0. Free memory on the machine is the cheapest answer there is, and it
951
+ // costs the person nothing to know about.
952
+ if (await this.addMemoryTo(running, sample, now)) {
953
+ stall.reason = 'added';
954
+ stall.detector = { ...stall.detector, hotWindows: 0, stalledSince: null, markBytes: 0 };
955
+ stall.remainingMs = null;
956
+ return;
957
+ }
958
+ /**
959
+ * The owner's switch (#398 S6). `report-only` leaves everything else
960
+ * exactly as it is — the deadline still runs, the strip still shows it, the
961
+ * person can still act — and only DevBridge stops acting on their behalf.
962
+ *
963
+ * Said at the START of the countdown and not at the end of it (#403). The
964
+ * strip renders the reason ahead of the clock, so while this was set only
965
+ * on expiry the owner of a report-only machine spent three minutes being
966
+ * promised «2:14 until a command is stopped» — a countdown to an event
967
+ * their own setting had already ruled out.
968
+ */
969
+ const action = this.opts.memoryStallAction ?? 'stop-command';
970
+ if (action === 'report-only')
971
+ stall.reason = 'report-only';
972
+ // T1. The deadline runs; the strip shows it. One victim at a time: while a
973
+ // subtree is still being stopped, nothing else is chosen.
974
+ if (!verdict.expired || stall.kill !== null)
975
+ return;
976
+ if (action === 'report-only') {
977
+ // Its own word, not «nothing to stop»: those are different facts about the
978
+ // machine, and telling the owner who chose this setting that there is
979
+ // nothing to stop would be a falsehood about their own machine (found by
980
+ // the independent review of 10.09.2026).
981
+ //
982
+ // Once per episode: compared against what the state held BEFORE this pass,
983
+ // because the line above has already set it for this one.
984
+ if (previous.reason !== 'report-only') {
985
+ log.warn('session memory: a session is standing still, and this machine is set to report only', {
986
+ sessionId: running.descriptor.id,
987
+ holdingMB: Math.round(sample.currentBytes / (1024 * 1024)),
988
+ });
989
+ }
990
+ }
991
+ else {
992
+ await this.stopBiggestCommand(running, unit, sample, now);
993
+ }
994
+ // Whatever we did or did not do, the clock starts again: the next attempt is
995
+ // a full deadline away, not five seconds.
996
+ stall.detector = { ...stall.detector, stalledSince: now, markBytes: sample.currentBytes };
997
+ stall.remainingMs = this.stallGraceMs;
998
+ }
999
+ /**
1000
+ * Ask the allocator for more, now, because this session has stopped moving (T0).
1001
+ *
1002
+ * Since S3 there is no separate ladder here: `MemoryHigh` has exactly TWO
1003
+ * writers, and they are the same code — the allocator on its tick and the
1004
+ * allocator on this signal. Two independent writers of one number is how a
1005
+ * limit ends up being whatever the last tick happened to think.
1006
+ *
1007
+ * The allocator needs no special case for a stall: a standing session's `hold`
1008
+ * is by definition all the way up against its brake, so `hold + free` asks for
1009
+ * everything the machine has spare. If there is nothing spare, nothing moves,
1010
+ * and the deadline below is the answer instead.
1011
+ */
1012
+ async addMemoryTo(running, sample, now) {
1013
+ // The brake as the CGROUP has it, not as this supervisor remembers writing
1014
+ // it: on the first pass there is nothing remembered, and «did this session
1015
+ // get more room» has to be answerable then too.
1016
+ const before = sample.brakeBytes;
1017
+ await this.runAllocator(now);
1018
+ const after = running.limits?.brakeBytes ?? null;
1019
+ if (before === null || after === null)
1020
+ return false;
1021
+ return after > before;
1022
+ }
1023
+ /** T2 and T3 — stop the biggest command of this session, or nothing at all. */
1024
+ async stopBiggestCommand(running, unit, sample, now) {
1025
+ const stall = running.stall;
1026
+ if (!stall)
1027
+ return;
1028
+ const processes = (this.opts.readScopeProcesses ?? readScopeProcesses)(unit);
1029
+ // Which pid is the agent, from the cage that spawned it — see
1030
+ // `noteSessionAgentPid`. Null on a machine with no cage, and then
1031
+ // `pickKillCandidate` falls back to «every parentless process».
1032
+ const candidate = pickKillCandidate(processes, sessionAgentPid(running.descriptor.id));
1033
+ const mb = (bytes) => Math.round(bytes / (1024 * 1024));
1034
+ if (candidate === null) {
1035
+ // T3. Everything left in the cage is the agent itself, its MCP servers or
1036
+ // `git`. This mechanism may not touch any of them, and by D8 it does not
1037
+ // write a line either — the strip says so while it is true.
1038
+ if (stall.reason !== 'nothing-to-stop') {
1039
+ log.warn('session memory: a session is out of memory and has nothing but the agent to stop', {
1040
+ sessionId: running.descriptor.id,
1041
+ holdingMB: mb(sample.currentBytes),
1042
+ });
1043
+ }
1044
+ stall.reason = 'nothing-to-stop';
1045
+ return;
1046
+ }
1047
+ const signal = this.opts.signalSubtree ?? signalSubtree;
1048
+ const sent = signal(candidate.pids, 'SIGTERM');
1049
+ stall.kill = {
1050
+ identities: candidate.identities,
1051
+ name: candidate.name,
1052
+ at: now + STALL_SIGKILL_AFTER_MS,
1053
+ };
1054
+ stall.reason = 'stopped-command';
1055
+ log.warn('session memory: stopped the biggest command of a session that had stopped moving', {
1056
+ sessionId: running.descriptor.id,
1057
+ command: candidate.name,
1058
+ processes: sent,
1059
+ commandMB: mb(candidate.rssBytes),
1060
+ sessionMB: mb(sample.currentBytes),
1061
+ });
1062
+ // The one line a person gets out of the whole episode (D8). The command's
1063
+ // NAME only — its arguments carry keys and tokens.
1064
+ this.sendEvent(running, 'notice', {
1065
+ level: 'warn',
1066
+ text: `This session ran out of memory, so DevBridge stopped its biggest command — ` +
1067
+ `${candidate.name}, holding ${mb(candidate.rssBytes)} MB of the session's ${mb(sample.currentBytes)} MB. ` +
1068
+ 'The session itself is still here.',
1069
+ });
1070
+ // And the agent is told, so it does not simply run the same thing again (D7).
1071
+ this.tellAgent(running, `DevBridge stopped your \`${candidate.name}\` command: this session ran out of memory. ` +
1072
+ `It was holding ${mb(candidate.rssBytes)} MB, the session's share is ` +
1073
+ `${sample.brakeBytes === null ? 'unknown' : `${mb(sample.brakeBytes)} MB`}, and there was nothing free ` +
1074
+ 'on the machine to add. Running the same command again will end the same way — make it need less ' +
1075
+ '(fewer parallel jobs, a smaller heap, a narrower scope) or split the work up. Do not raise ' +
1076
+ '`--max-old-space-size`: the ceiling is enforced outside your process.');
1077
+ }
1078
+ /**
1079
+ * Say something to the agent on the runner's own initiative, without stepping
1080
+ * on a card the person has not answered (#398 S7, B7).
1081
+ *
1082
+ * `session.send()` looks like a neutral channel and is not one: with a
1083
+ * question open, `ClaudeSession.send` answers it «discuss» and the API
1084
+ * records `question_resolved {source: 'user'}` — a decision attributed to a
1085
+ * person who never made it — while `CodexSession.send` declines a held plan
1086
+ * the same way. Every other initiative of this supervisor already checks for
1087
+ * an open card; the memory mechanism was the one that did not.
1088
+ *
1089
+ * Held rather than dropped: the agent is parked on the card, so it cannot run
1090
+ * anything until the person answers anyway, and once they do the note is the
1091
+ * first thing it reads.
1092
+ */
1093
+ tellAgent(running, text) {
1094
+ if (running.openQuestions.size > 0 || running.openPermissions.size > 0) {
1095
+ running.pendingAgentNote = running.pendingAgentNote
1096
+ ? `${running.pendingAgentNote}\n\n${text}`
1097
+ : text;
1098
+ return;
1099
+ }
1100
+ running.session?.send(text);
1101
+ }
1102
+ /** Deliver what waited for a card to close — see {@link tellAgent}. */
1103
+ flushPendingAgentNote(running) {
1104
+ const note = running.pendingAgentNote;
1105
+ if (note === undefined)
1106
+ return;
1107
+ if (running.openQuestions.size > 0 || running.openPermissions.size > 0)
1108
+ return;
1109
+ delete running.pendingAgentNote;
1110
+ running.session?.send(note);
1111
+ }
1112
+ /** How many sessions are running in a cage right now (#398 S7, B9). */
1113
+ liveCagedSessions() {
1114
+ const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
1115
+ let count = 0;
1116
+ for (const running of this.sessions.values()) {
1117
+ if (running.session === null)
1118
+ continue;
1119
+ if (unitOf(running.descriptor.id) !== null)
1120
+ count += 1;
1121
+ }
1122
+ return count;
1123
+ }
1124
+ /** SIGKILL for a subtree that did not take SIGTERM within the grace. */
1125
+ escalatePendingKill(running, now) {
1126
+ const kill = running.stall?.kill;
1127
+ if (!kill || now < kill.at)
1128
+ return;
1129
+ const signal = this.opts.signalSubtree ?? signalSubtree;
1130
+ /**
1131
+ * Identities, not bare pids. Thirty seconds is long enough on a busy machine
1132
+ * for a pid freed by the SIGTERM to be handed to something else, and a
1133
+ * SIGKILL to the wrong process is the worst thing in this whole change —
1134
+ * named as such by the independent review of 10.09.2026.
1135
+ */
1136
+ const sent = signal(kill.identities, 'SIGKILL');
1137
+ if (sent > 0) {
1138
+ log.warn('session memory: the stopped command did not go on its own', {
1139
+ sessionId: running.descriptor.id,
1140
+ command: kill.name,
1141
+ processes: sent,
1142
+ });
1143
+ }
1144
+ else if (kill.identities.length > 0) {
1145
+ /**
1146
+ * Nothing was signalled, and that is worth a line (#403).
1147
+ *
1148
+ * Two ways to get here and they are different machines. Either the
1149
+ * processes ended on their own during the grace — the good outcome, and
1150
+ * the common one — or their identity could not be read when the victim
1151
+ * was chosen (`startedAtTicks: -1`), in which case `isSameProcess` refuses
1152
+ * every one of them and the promised SIGKILL never happens at all. Under
1153
+ * `if (sent > 0)` both were silent, so «stopped, then escalated» could
1154
+ * quietly be neither.
1155
+ */
1156
+ log.info('session memory: nothing was left to SIGKILL', {
1157
+ sessionId: running.descriptor.id,
1158
+ command: kill.name,
1159
+ identities: kill.identities.length,
1160
+ unreadable: kill.identities.filter((i) => i.startedAtTicks < 0).length,
1161
+ });
1162
+ }
1163
+ if (running.stall)
1164
+ running.stall.kill = null;
1165
+ }
1166
+ /** An allocator pass is in flight — the next one waits rather than overlaps. */
1167
+ allocating = false;
1168
+ /** The last pass's machine-wide numbers, for the card and the session banner. */
1169
+ lastAllocation = null;
1170
+ /**
1171
+ * The machine the last allocation was computed for, kept so the card's own
1172
+ * number can be derived without measuring the machine a SECOND time (#403).
1173
+ */
1174
+ lastAllocatorMachine = null;
1175
+ /** The last frame the API actually took from us, or null for «nothing yet». */
1176
+ lastPublishedLimits = null;
1177
+ /** When that frame went out, by this machine's clock. `0` = never. */
1178
+ lastLimitsSentAt = 0;
1179
+ /**
1180
+ * Decide, and write, how much memory each live session may use (#398 S3).
1181
+ *
1182
+ * Runs on the 30 s watch tick and on the stall detector's signal, and nowhere
1183
+ * else: `MemoryHigh` must have exactly one implementation of «what should this
1184
+ * be», or the two writers disagree and the limit becomes whichever ran last.
1185
+ *
1186
+ * Everything that could be wrong about the machine is read fresh here rather
1187
+ * than taken from the daemon-start probe: the pot moves when the hourly
1188
+ * re-measure rewrites the slice, the collective brake decides whether
1189
+ * overselling is even legal, and a neighbouring application growing is
1190
+ * precisely the case #398 was opened about.
1191
+ */
1192
+ async runAllocator(now) {
1193
+ if (this.allocating)
1194
+ return;
1195
+ const facts = sessionCage();
1196
+ /**
1197
+ * No explicit «is this machine caged» gate, and it stays that way (#398 S7,
1198
+ * B5 — reviewed, tried, and rejected on 10.09.2026).
1199
+ *
1200
+ * The finding is real as far as it goes: `sessionScopeUnitOf` answers null
1201
+ * for sessions started on an uncaged machine, but NOT for sessions that
1202
+ * were already running when the machine lost its cage — for those the
1203
+ * register still holds a unit name, and this loop would go on writing
1204
+ * limits to scopes that may no longer be anybody's.
1205
+ *
1206
+ * A gate on `sessionCage().mode` is nonetheless the wrong fix, and the
1207
+ * suite says so out loud: every memory test here drives the allocator
1208
+ * through its injected readers WITHOUT probing a cage, so such a gate turns
1209
+ * thirty tests off rather than making them pass — which is the shape of a
1210
+ * change that looks green and proves nothing.
1211
+ *
1212
+ * What actually happens in the case the finding describes is small and
1213
+ * self-correcting: the write fails, `setScopeProperties` logs it and
1214
+ * returns false, and nothing is corrupted. Left as is, deliberately.
1215
+ */
1216
+ const slice = (this.opts.readSliceLimits ?? readSliceLimits)();
1217
+ const load = (this.opts.readHostLoad ?? readHostLoad)();
1218
+ if (slice === null || load === null)
1219
+ return;
1220
+ const holdOf = this.opts.readScopeHold ?? readScopeHold;
1221
+ const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
1222
+ const live = [];
1223
+ for (const running of this.sessions.values()) {
1224
+ if (running.session === null)
1225
+ continue;
1226
+ const unit = unitOf(running.descriptor.id);
1227
+ if (unit === null)
1228
+ continue;
1229
+ const hold = holdOf(unit);
1230
+ if (hold === null)
1231
+ continue;
1232
+ live.push({ running, unit, hold });
1233
+ }
1234
+ // Computed even when nobody is running, and that is not a waste: half of
1235
+ // what the card shows is about the MACHINE — the live pot, what is held by
1236
+ // things that are not DevBridge, how many guarantees would fit — and an idle
1237
+ // machine has to be able to say it too. With no sessions the loop below
1238
+ // simply has nothing to write.
1239
+ const machine = this.allocatorMachine(slice, load.memAvailableBytes, facts.swapMaxBytes ?? 0);
1240
+ const allocation = allocateSessionMemory(machine, live.map(({ running, hold }) => ({
1241
+ id: running.descriptor.id,
1242
+ holdBytes: hold.holdBytes,
1243
+ currentBytes: hold.currentBytes,
1244
+ })), this.opts.memoryKnobs ?? {});
1245
+ this.lastAllocation = allocation;
1246
+ this.lastAllocatorMachine = machine;
1247
+ this.allocating = true;
1248
+ try {
1249
+ const write = this.opts.setScopeProperties ?? setScopeProperties;
1250
+ const readStatus = this.opts.readScopeMemoryStatus ?? readScopeMemoryStatus;
1251
+ for (const { running, unit, hold } of live) {
1252
+ const wanted = allocation.sessions.find((s) => s.id === running.descriptor.id);
1253
+ if (!wanted)
1254
+ continue;
1255
+ /**
1256
+ * Reconcile before planning (work 10 of the stage).
1257
+ *
1258
+ * What this supervisor believes it wrote and what the scope actually
1259
+ * holds are two different facts, and they have come apart before — a
1260
+ * drop-in that never applied while the card said «capped». The cgroup is
1261
+ * the authority, and reading it costs nothing here: the status is two
1262
+ * small files the watch already reads.
1263
+ */
1264
+ const inForce = readStatus(unit);
1265
+ let written = running.limits ?? null;
1266
+ if (inForce !== null && written !== null) {
1267
+ const realBrake = inForce.highBytes;
1268
+ const realWall = inForce.maxBytes;
1269
+ if ((realBrake !== null && realBrake !== written.brakeBytes) ||
1270
+ (realWall !== null && realWall !== written.wallBytes)) {
1271
+ log.warn('session memory: the scope does not hold what this runner wrote', {
1272
+ sessionId: running.descriptor.id,
1273
+ believedBrakeMB: Math.round(written.brakeBytes / (1024 * 1024)),
1274
+ actualBrakeMB: realBrake === null ? null : Math.round(realBrake / (1024 * 1024)),
1275
+ believedWallMB: Math.round(written.wallBytes / (1024 * 1024)),
1276
+ actualWallMB: realWall === null ? null : Math.round(realWall / (1024 * 1024)),
1277
+ });
1278
+ written = {
1279
+ ...written,
1280
+ ...(realBrake === null ? {} : { brakeBytes: realBrake }),
1281
+ ...(realWall === null ? {} : { wallBytes: realWall }),
1282
+ };
1283
+ }
1284
+ }
1285
+ const move = planLimitMove(written, wanted, hold.holdBytes, facts.memoryLowFlag);
1286
+ if (move.properties.length > 0) {
1287
+ if (!(await write(unit, move.properties)))
1288
+ continue;
1289
+ if (move.lowered) {
1290
+ /**
1291
+ * We just narrowed this session's brake, and that manufactures the
1292
+ * exact signal the stall detector looks for: measured on this host,
1293
+ * a process honestly writing at 6.4 MB/s stopped dead the instant
1294
+ * its `MemoryHigh` was narrowed on the live scope. Without this the
1295
+ * mechanism would catch its own edit and stop a command for it.
1296
+ */
1297
+ const stall = running.stall;
1298
+ if (stall) {
1299
+ stall.detector = {
1300
+ ...stall.detector,
1301
+ mutedUntil: now + STALL_MUTE_AFTER_LOWER_MS,
1302
+ hotWindows: 0,
1303
+ stalledSince: null,
1304
+ };
1305
+ stall.remainingMs = null;
1306
+ }
1307
+ }
781
1308
  }
1309
+ running.limits = {
1310
+ brakeBytes: move.brakeBytes,
1311
+ wallBytes: move.wallBytes,
1312
+ guaranteedBytes: move.guaranteedBytes,
1313
+ swapBytes: move.swapBytes,
1314
+ holdBytes: move.holdBytes,
1315
+ shrinkTicks: move.shrinkTicks,
1316
+ };
782
1317
  }
783
- running.cageWatch = {
784
- unit,
785
- oomKills: status.oomKills,
786
- ownOom: status.ownLimitOom,
787
- highEvents: status.highEvents,
788
- brakedSince,
789
- warnedLong,
790
- calmTicks,
1318
+ }
1319
+ finally {
1320
+ this.allocating = false;
1321
+ }
1322
+ // After the writes, never before: the frame reports what is IN FORCE, and
1323
+ // reporting an intention that then failed to apply is how a card starts
1324
+ // lying about a machine.
1325
+ this.publishSessionLimits(now, allocation);
1326
+ }
1327
+ /** The machine as the allocator needs to see it, from live readings only. */
1328
+ allocatorMachine(slice, availableBytes, swapBytes) {
1329
+ const totalBytes = os.totalmem();
1330
+ return {
1331
+ potBytes: slice.potBytes,
1332
+ availableBytes,
1333
+ totalBytes,
1334
+ reserveBytes: machineReserveBytes(totalBytes),
1335
+ oomContinue: sessionCage().oomContinue,
1336
+ // The hard invariant: overselling the brakes is legal only while the
1337
+ // slice carries a live collective brake. Read back off cgroupfs, never
1338
+ // assumed from what we wrote to the drop-in.
1339
+ collectiveBrake: slice.collectiveBrakeBytes !== null,
1340
+ swapBytes,
1341
+ };
1342
+ }
1343
+ /**
1344
+ * What a session STARTING right now should be given (#398 S3, work 9).
1345
+ *
1346
+ * The allocator is asked to compute as though this session already existed,
1347
+ * so a session born at 03:00 gets the number its first tick will confirm
1348
+ * rather than the one the daemon measured whenever it last started.
1349
+ */
1350
+ ladderForNewSession(id) {
1351
+ /**
1352
+ * A session that ALREADY exists is answered with what is in force for it,
1353
+ * not with what a newcomer would get.
1354
+ *
1355
+ * Found by the independent review of 10.09.2026, and it was not only the
1356
+ * cage that asked: `policyContextFor` calls `sessionMemoryFor` on every
1357
+ * Bash tool call, so the heap gate was judging commands against a ceiling
1358
+ * computed as though the asking session did not exist — which drops its own
1359
+ * `memory.current` out of the live pot and makes the number much smaller
1360
+ * exactly while the session is busy. The gate would then refuse a heap the
1361
+ * cage would have allowed.
1362
+ *
1363
+ * The newcomer arithmetic below is for the one caller that really has no
1364
+ * session yet: `cageSpawn`, at the moment of the start.
1365
+ */
1366
+ const inForce = this.sessions.get(id)?.limits ?? null;
1367
+ if (inForce) {
1368
+ return {
1369
+ highBytes: inForce.brakeBytes,
1370
+ maxBytes: inForce.wallBytes,
1371
+ swapBytes: inForce.swapBytes,
1372
+ guaranteedBytes: inForce.guaranteedBytes,
791
1373
  };
792
1374
  }
1375
+ // Reached only from `cageSpawn`, which has already established that this
1376
+ // machine has a cage — see `runAllocator` for why there is no second gate.
1377
+ const facts = sessionCage();
1378
+ const slice = (this.opts.readSliceLimits ?? readSliceLimits)();
1379
+ const load = (this.opts.readHostLoad ?? readHostLoad)();
1380
+ if (slice === null || load === null)
1381
+ return null;
1382
+ const holdOf = this.opts.readScopeHold ?? readScopeHold;
1383
+ const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
1384
+ const sessions = [];
1385
+ for (const running of this.sessions.values()) {
1386
+ if (running.session === null || running.descriptor.id === id)
1387
+ continue;
1388
+ const unit = unitOf(running.descriptor.id);
1389
+ const hold = unit === null ? null : holdOf(unit);
1390
+ if (hold === null)
1391
+ continue;
1392
+ sessions.push({
1393
+ id: running.descriptor.id,
1394
+ holdBytes: hold.holdBytes,
1395
+ currentBytes: hold.currentBytes,
1396
+ });
1397
+ }
1398
+ // The starting session holds nothing yet, and saying so is the point: it
1399
+ // must not be given the pot as though it already had it.
1400
+ sessions.push({ id, holdBytes: 0, currentBytes: 0 });
1401
+ const allocation = allocateSessionMemory(this.allocatorMachine(slice, load.memAvailableBytes, facts.swapMaxBytes ?? 0), sessions, this.opts.memoryKnobs ?? {});
1402
+ const mine = allocation.sessions.find((s) => s.id === id);
1403
+ if (!mine)
1404
+ return null;
1405
+ return {
1406
+ highBytes: mine.brakeBytes,
1407
+ maxBytes: mine.wallBytes,
1408
+ swapBytes: mine.swapBytes,
1409
+ guaranteedBytes: mine.guaranteedBytes,
1410
+ };
1411
+ }
1412
+ /**
1413
+ * Tell the API how this machine is dividing its memory — when it moved (#398 S4).
1414
+ *
1415
+ * Built from the pass the allocator has just finished rather than measured
1416
+ * again: two readings of a moving machine taken a moment apart would let the
1417
+ * card and the scope disagree about the same session, and «the card said 4 GB
1418
+ * and the cgroup said 2» is the exact shape of the bug that made
1419
+ * `limitsCurrent` untrustworthy once already.
1420
+ *
1421
+ * Recorded ONLY when the socket took it, same as `publishHostLoad`: a frame
1422
+ * dropped by a dead socket must not be remembered as sent.
1423
+ */
1424
+ /**
1425
+ * The number on the machine card: what ONE session gets when the machine is
1426
+ * free (decision D4 of `runner-cage-authority.md`, 10.09.2026).
1427
+ *
1428
+ * The card used to carry the ceiling a newcomer would get RIGHT NOW, from the
1429
+ * current crowd — which is honest per second and useless per person: on a
1430
+ * 12 GiB machine with two sessions of 3 GiB it read ~6 GiB, it moved whenever
1431
+ * a neighbour started or stopped, and the sentence beside it said «up to X on
1432
+ * an idle machine», which is a different number entirely. A card is read to
1433
+ * answer «what does this machine give a session», and that question has a
1434
+ * still answer.
1435
+ *
1436
+ * Computed, not measured: the pot and the owner's knobs are all it takes, so
1437
+ * this makes no second reading of a moving machine — the one the shape of
1438
+ * `publishSessionLimits` warns against in its own header.
1439
+ */
1440
+ idleSessionCeiling(allocation) {
1441
+ const machine = this.lastAllocatorMachine;
1442
+ // No pot means no slice ceiling to be idle against, and then the pool the
1443
+ // allocation already used is the honest answer.
1444
+ if (machine === null || machine.potBytes === null)
1445
+ return Math.floor(allocation.poolBytes);
1446
+ const idle = allocateSessionMemory(
1447
+ // «Free» is `available` big enough that the pot itself is the binding
1448
+ // limit: the pool is `min(pot, available − reserve)` floored at the
1449
+ // guarantee, so anything at or above `pot + reserve` gives a full pot.
1450
+ { ...machine, availableBytes: machine.potBytes + machine.reserveBytes }, [{ id: '__card__', holdBytes: 0, currentBytes: 0 }], this.opts.memoryKnobs ?? {});
1451
+ const only = idle.sessions[0];
1452
+ return Math.floor(only?.brakeBytes ?? allocation.poolBytes);
1453
+ }
1454
+ publishSessionLimits(now, allocation) {
1455
+ if (allocation === null)
1456
+ return;
1457
+ const load = (this.opts.readHostLoad ?? readHostLoad)();
1458
+ if (load === null)
1459
+ return;
1460
+ /**
1461
+ * Stamped HERE, not from the tick's own clock.
1462
+ *
1463
+ * One pass of the stall watch can publish twice — once from the allocator
1464
+ * the stalled session asked for, once from the tick — and both were being
1465
+ * stamped with the single `now` the pass began with. The gateway's ordering
1466
+ * guard is `measuredAt <= last`, so the second frame was dropped while this
1467
+ * side recorded it as sent, and the difference stayed invisible until the
1468
+ * heartbeat a minute later. `host_load` has never had the bug because
1469
+ * `readHostLoad` stamps inside itself, once per publish. Found by the
1470
+ * independent review of 10.09.2026.
1471
+ */
1472
+ const measuredAt = Math.max(now, this.lastLimitsSentAt + 1);
1473
+ // What is held on this machine by everything that is not us. The card had
1474
+ // nowhere to get this, and it is the main question of #398.
1475
+ const otherHeld = Math.max(0, os.totalmem() - load.memAvailableBytes - allocation.heldBytes);
1476
+ const frame = {
1477
+ at: new Date(measuredAt).toISOString(),
1478
+ poolBytes: Math.floor(allocation.poolBytes),
1479
+ heldBytes: Math.floor(allocation.heldBytes),
1480
+ otherHeldBytes: Math.floor(otherHeld),
1481
+ /**
1482
+ * What is PROMISED, and only where the machine can keep the promise
1483
+ * (#398 S7, B3): zero when the slice carries no `memory.low` of its own,
1484
+ * because then nothing is reserved for anybody and the card must not say
1485
+ * otherwise. Measured on this host: the file held 0 while the chip read
1486
+ * «2.0 GB guaranteed».
1487
+ */
1488
+ guaranteeBytes: ((this.opts.readSliceLimits ?? readSliceLimits)()?.guaranteeBytes ?? 0) > 0
1489
+ ? (this.opts.memoryKnobs?.guaranteeBytes ?? SESSION_GUARANTEE_BYTES)
1490
+ : 0,
1491
+ // Asked of the allocator, not derived from the pot: only it knows the
1492
+ // owner's knobs, the band this machine uses and the single-session branch.
1493
+ // On an IDLE machine, though — see `idleSessionCeiling`.
1494
+ sessionCeilingBytes: this.idleSessionCeiling(allocation),
1495
+ guaranteesFit: allocation.guaranteesFit,
1496
+ seats: this.maxSessions,
1497
+ swapPerSessionBytes: sessionCage().swapMaxBytes ?? 0,
1498
+ collectiveBrake: !allocation.conservative,
1499
+ conservative: allocation.conservative,
1500
+ hungry: allocation.hungry,
1501
+ sessions: allocation.sessions.map((s) => {
1502
+ const running = this.sessions.get(s.id);
1503
+ /**
1504
+ * What is IN FORCE, not what the allocator wanted.
1505
+ *
1506
+ * `running.limits` is what systemd was actually told; the allocation is
1507
+ * an intention, and the two come apart every time a write fails or a
1508
+ * movement rule refuses to lower something. A card drawn from the
1509
+ * intention would claim a session had a share no scope has ever held —
1510
+ * the kind of lie that made `limitsCurrent` untrustworthy once already.
1511
+ * Found by the independent review of 10.09.2026.
1512
+ */
1513
+ const written = running?.limits ?? null;
1514
+ return {
1515
+ sessionId: s.id,
1516
+ guaranteedBytes: Math.floor(written?.guaranteedBytes ?? s.guaranteedBytes),
1517
+ brakeBytes: Math.floor(written?.brakeBytes ?? s.brakeBytes),
1518
+ wallBytes: Math.floor(written?.wallBytes ?? s.wallBytes),
1519
+ holdBytes: Math.floor(written?.holdBytes ?? 0),
1520
+ tight: s.tight,
1521
+ stallRemainingMs: running?.stall?.remainingMs ?? null,
1522
+ reason: running?.stall?.reason ?? null,
1523
+ };
1524
+ }),
1525
+ };
1526
+ const heartbeatDue = sessionLimitsHeartbeatDue(now - this.lastLimitsSentAt, this.limitsHeartbeatMs);
1527
+ if (!heartbeatDue && !sessionLimitsChangedEnough(this.lastPublishedLimits, frame))
1528
+ return;
1529
+ if (this.ws.send({ type: 'session_limits', ...frame })) {
1530
+ this.lastPublishedLimits = frame;
1531
+ this.lastLimitsSentAt = measuredAt;
1532
+ }
793
1533
  }
794
1534
  publishHostLoad() {
795
1535
  const sample = (this.opts.readHostLoad ?? readHostLoad)();
@@ -821,10 +1561,16 @@ export class Supervisor {
821
1561
  // is right only after five minutes of silence is not right.
822
1562
  this.lastPublishedHostLoad = null;
823
1563
  this.lastHostLoadSentAt = 0;
1564
+ // …and for the memory numbers, which the API keeps the same way (#398 S4).
1565
+ this.lastPublishedLimits = null;
1566
+ this.lastLimitsSentAt = 0;
824
1567
  this.setMaxSessions(frame.maxSessions);
825
1568
  await this.reconcile(frame.sessions);
826
1569
  this.publishSlots();
827
1570
  this.publishHostLoad();
1571
+ // Deliberately not awaited: this frame also carries the session
1572
+ // reconciliation, and a slow cgroup read must not hold it up.
1573
+ void this.runAllocator(Date.now());
828
1574
  // A build that finished while the socket was down has its verdict
829
1575
  // sitting on disk. This is the moment it can be delivered.
830
1576
  this.flushVerifyReports();
@@ -1017,6 +1763,9 @@ export class Supervisor {
1017
1763
  levels: freshLevels(),
1018
1764
  stopRequested: false,
1019
1765
  parkRequested: false,
1766
+ // Set here rather than at the first `await`: frames are dispatched
1767
+ // concurrently, so anything assigned later is already too late (#401).
1768
+ starting: true,
1020
1769
  pendingMessages: [],
1021
1770
  activeMs: descriptor.activeMsBase,
1022
1771
  extraBudgetMinutes: descriptor.extraBudgetMinutes,
@@ -1070,71 +1819,84 @@ export class Supervisor {
1070
1819
  }
1071
1820
  }
1072
1821
  try {
1073
- const prepared = await this.prepareWorkspace(descriptor);
1074
- running.branch = prepared.branch;
1075
- running.worktreePath = prepared.worktreePath;
1076
- // Only when WE created the branch: the fork point is a fact the API can
1077
- // learn nowhere else, and it pins the first answer it gets.
1078
- if (prepared.baseSha)
1079
- running.baseSha = prepared.baseSha;
1080
- if (prepared.baseBranch)
1081
- running.baseBranch = prepared.baseBranch;
1082
- }
1083
- catch (error) {
1084
- this.workspacePrepareFailed(running, error, 'Failed to prepare git worktree');
1085
- return;
1086
- }
1087
- // A session that is already past STARTING (re-sent because this runner
1088
- // asked for it) resumes on the next message instead of replaying its
1089
- // original prompt.
1090
- if (descriptor.status === 'STARTING') {
1091
- // Ticket #126: the point before the agent has touched anything. It is
1092
- // anchored to seq 0 — the synthetic opening bubble the dashboard puts in
1093
- // front of every feed — so "put it all back" is reachable from the very
1094
- // first thing on the page.
1095
- await this.captureCheckpoint(running, 'TURN', 0);
1096
- if (this.isStale(running))
1097
- return;
1098
- // A launch that failed has already said so and reported a status the
1099
- // person can act on. Flushing the queue into it would only walk the same
1100
- // failure again, once per waiting message (ticket #225).
1101
- if (!this.launchAgent(running, composeInitialPrompt(descriptor), null).ok)
1822
+ try {
1823
+ const prepared = await this.prepareWorkspace(descriptor);
1824
+ running.branch = prepared.branch;
1825
+ running.worktreePath = prepared.worktreePath;
1826
+ // Only when WE created the branch: the fork point is a fact the API can
1827
+ // learn nowhere else, and it pins the first answer it gets.
1828
+ if (prepared.baseSha)
1829
+ running.baseSha = prepared.baseSha;
1830
+ if (prepared.baseBranch)
1831
+ running.baseBranch = prepared.baseBranch;
1832
+ }
1833
+ catch (error) {
1834
+ this.workspacePrepareFailed(running, error, 'Failed to prepare git worktree');
1102
1835
  return;
1103
- }
1104
- else {
1105
- if (descriptor.epoch > 0) {
1106
- // The API owns the resume transition; the feed marker has to come from
1107
- // here because the runner is the only writer of the event seq.
1108
- //
1109
- // Ticket #177: which of the two sentences is true depends on whether
1110
- // there is a conversation to go back to. `providerSessionId` is the
1111
- // agent's own name for it, and it is what the next launch hands to
1112
- // `--resume` / `thread/resume`. Without it the next process starts the
1113
- // conversation over — which is a real loss, and promising «continue
1114
- // where the agent left off» there is the one thing the feed must not do.
1115
- // It happens for real: a process that dies before it reports its session
1116
- // id (the SIGABRT this ticket came from) leaves the row with none.
1117
- //
1118
- // Ticket #370: the sentence about the agent's MEMORY waits for proof.
1119
- // It used to be written here, on the strength of a stored id and before
1120
- // the CLI had been asked anything — and on Athanor it appeared sixty
1121
- // seconds before `thread/resume` timed out and took the session with it.
1122
- // What is honest at this moment is that the runner is back; whether the
1123
- // conversation reopens is answered by the process, in `provider_session`.
1124
- if (descriptor.providerSessionId)
1125
- running.resumeClaimPending = true;
1126
- this.sendEvent(running, 'system_note', {
1127
- text: descriptor.providerSessionId
1128
- ? 'Session resumed — send a message and the agent picks its conversation back up.'
1129
- : 'Session resumed, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
1836
+ }
1837
+ // A session that is already past STARTING (re-sent because this runner
1838
+ // asked for it) resumes on the next message instead of replaying its
1839
+ // original prompt.
1840
+ if (descriptor.status === 'STARTING') {
1841
+ // Ticket #126: the point before the agent has touched anything. It is
1842
+ // anchored to seq 0 — the synthetic opening bubble the dashboard puts in
1843
+ // front of every feed — so "put it all back" is reachable from the very
1844
+ // first thing on the page.
1845
+ await this.captureCheckpoint(running, 'TURN', 0);
1846
+ if (this.isStale(running))
1847
+ return;
1848
+ // A launch that failed has already said so and reported a status the
1849
+ // person can act on. Flushing the queue into it would only walk the same
1850
+ // failure again, once per waiting message (ticket #225).
1851
+ if (!this.launchAgent(running, composeInitialPrompt(descriptor), null).ok)
1852
+ return;
1853
+ }
1854
+ else {
1855
+ if (descriptor.epoch > 0) {
1856
+ // The API owns the resume transition; the feed marker has to come from
1857
+ // here because the runner is the only writer of the event seq.
1858
+ //
1859
+ // Ticket #177: which of the two sentences is true depends on whether
1860
+ // there is a conversation to go back to. `providerSessionId` is the
1861
+ // agent's own name for it, and it is what the next launch hands to
1862
+ // `--resume` / `thread/resume`. Without it the next process starts the
1863
+ // conversation over — which is a real loss, and promising «continue
1864
+ // where the agent left off» there is the one thing the feed must not do.
1865
+ // It happens for real: a process that dies before it reports its session
1866
+ // id (the SIGABRT this ticket came from) leaves the row with none.
1867
+ //
1868
+ // Ticket #370: the sentence about the agent's MEMORY waits for proof.
1869
+ // It used to be written here, on the strength of a stored id and before
1870
+ // the CLI had been asked anything — and on Athanor it appeared sixty
1871
+ // seconds before `thread/resume` timed out and took the session with it.
1872
+ // What is honest at this moment is that the runner is back; whether the
1873
+ // conversation reopens is answered by the process, in `provider_session`.
1874
+ if (descriptor.providerSessionId)
1875
+ running.resumeClaimPending = true;
1876
+ this.sendEvent(running, 'system_note', {
1877
+ text: descriptor.providerSessionId
1878
+ ? 'Session resumed — send a message and the agent picks its conversation back up.'
1879
+ : 'Session resumed, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
1880
+ });
1881
+ }
1882
+ running.lastReported = descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT';
1883
+ this.reportStatus(descriptor.id, running.lastReported, {
1884
+ branch: running.branch,
1885
+ worktreePath: running.worktreePath,
1886
+ activeMs: running.activeMs,
1130
1887
  });
1131
1888
  }
1132
- running.lastReported = descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT';
1133
- this.reportStatus(descriptor.id, running.lastReported, {
1134
- branch: running.branch,
1135
- worktreePath: running.worktreePath,
1136
- activeMs: running.activeMs,
1137
- });
1889
+ }
1890
+ finally {
1891
+ /**
1892
+ * The startup gate opens again on EVERY exit, including the failed ones.
1893
+ *
1894
+ * #401: this flag is what makes `acceptUserMessage` hold a message until
1895
+ * the first process exists. Left standing by an early return it would
1896
+ * hold the person's words in the queue for the life of the session, so
1897
+ * it is cleared here rather than at each `return` above.
1898
+ */
1899
+ running.starting = false;
1138
1900
  }
1139
1901
  this.flushPendingMessages(running);
1140
1902
  }
@@ -1227,6 +1989,40 @@ export class Supervisor {
1227
1989
  const adapter = this.opts.adapters[descriptor.agent];
1228
1990
  if (!adapter || !running.worktreePath || !running.branch)
1229
1991
  return LAUNCH_REFUSED;
1992
+ /**
1993
+ * One process per session, enforced where the slot is written (#401).
1994
+ *
1995
+ * `running.session` is a single field, and until this check it was assigned
1996
+ * unconditionally: a second launch overwrote the reference to a process
1997
+ * that was still running, and nobody held it any more. Everything the
1998
+ * session then did — a message, an answer to a question, `stop`, the memory
1999
+ * watch — addressed the survivor, while the orphan kept working, kept its
2000
+ * own cgroup and outlived even the service restart.
2001
+ *
2002
+ * Safe as an entry check because this method is SYNCHRONOUS from here to
2003
+ * the assignment: nothing yields in between, so «free now» is still true
2004
+ * when the slot is filled.
2005
+ *
2006
+ * Every legitimate caller reaches this with an empty slot, by one of three
2007
+ * arguments: the four one-shot relaunches run in the tail of `pumpEvents`,
2008
+ * where the slot was emptied because the process is gone; `deliverMessage`
2009
+ * only gets here past its own `running.session` test and is serialised by
2010
+ * the session's delivery chain; and the two opening doors hold `starting`
2011
+ * for exactly as long as it takes them to decide, which is what keeps the
2012
+ * queue from starting an agent out from under them.
2013
+ *
2014
+ * A refusal here is therefore not a race being caught — it is a bug, and
2015
+ * the log line says so. Read it together with `starting`: this guard alone
2016
+ * would turn «two processes» into «the session's own task never sent», the
2017
+ * quieter of the two failures.
2018
+ */
2019
+ if (running.session) {
2020
+ log.error('supervisor: refusing to launch a second agent for one session', {
2021
+ sessionId: descriptor.id,
2022
+ processSeq: running.processSeq,
2023
+ });
2024
+ return LAUNCH_REFUSED;
2025
+ }
1230
2026
  // An exhausted USD budget must not relaunch $0.01-floor processes (QA-96 F4).
1231
2027
  // Codex reports no cost at all, so its costUsd never leaves 0 — gating on it
1232
2028
  // would be a limit that can never fire while the UI shows $0.00. Those
@@ -1693,8 +2489,35 @@ export class Supervisor {
1693
2489
  });
1694
2490
  }
1695
2491
  // Stream ended — the agent process is gone.
1696
- if (running.session !== session)
1697
- return; // superseded (shouldn't happen in v0)
2492
+ if (running.session !== session) {
2493
+ /**
2494
+ * Somebody else holds the slot while this process's stream ends.
2495
+ *
2496
+ * Since #401 the launch REFUSES to overwrite a live process, and the slot
2497
+ * is emptied only where the process is known to be gone — the line below
2498
+ * and `launchCrashed`. So this branch no longer means «a live agent was
2499
+ * orphaned»; it means the entry was rebuilt under this stream, and the
2500
+ * cleanup below belongs to whoever holds it now. Said out loud rather
2501
+ * than returned in silence: the old comment here called the case
2502
+ * impossible, and it was the shape four production sessions took.
2503
+ */
2504
+ log.warn('supervisor: event stream ended for a process that no longer holds the slot', {
2505
+ sessionId: descriptor.id,
2506
+ processSeq: running.processSeq,
2507
+ });
2508
+ return;
2509
+ }
2510
+ /**
2511
+ * The slot describes a LIVE process, so it is emptied the moment there is
2512
+ * none — here, and not by whoever launches next (#401).
2513
+ *
2514
+ * Three of the one-shot relaunches below (`freshRetry`, `rewindRetry`,
2515
+ * `authRetry`) used to call `launchAgent` with the dead adapter still in
2516
+ * the slot and rely on being overwritten. That overwrite is exactly what
2517
+ * the launch now refuses, so without this line the recovery paths would
2518
+ * refuse themselves. Nothing between here and them reads the field.
2519
+ */
2520
+ running.session = null;
1698
2521
  // Backstop for a process that died without going through `stop()` (a crash,
1699
2522
  // an SDK error, an agent that exited mid-question): the adapter never got
1700
2523
  // to withdraw its cards, and a card nobody can answer must not stay live.
@@ -1768,7 +2591,9 @@ export class Supervisor {
1768
2591
  const { priorStatus } = running.modeRelaunch;
1769
2592
  delete running.modeRelaunch;
1770
2593
  running.parkRequested = false;
1771
- running.session = null;
2594
+ // The slot was emptied where the process was found gone (#401) — this
2595
+ // branch used to be the only one that remembered to do it, and the three
2596
+ // above did not. One place, so they cannot disagree again.
1772
2597
  running.costBaseUsd = running.costUsd; // the next process starts from here
1773
2598
  const mode = running.mode;
1774
2599
  // «Your conversation is kept» is only true when there IS one to keep
@@ -1832,7 +2657,7 @@ export class Supervisor {
1832
2657
  !isTerminal(running.lastReported) &&
1833
2658
  running.descriptor.providerSessionId) {
1834
2659
  // Idle process ended (parked or died between turns) — stay resumable.
1835
- running.session = null;
2660
+ // The slot itself was emptied above, where the process was found gone.
1836
2661
  running.parkRequested = false;
1837
2662
  /**
1838
2663
  * The subagents died with it (QA-2026-08-16 M-5).
@@ -2653,6 +3478,8 @@ export class Supervisor {
2653
3478
  return;
2654
3479
  case 'permission_resolved':
2655
3480
  running.openPermissions.delete(event.requestId);
3481
+ // …and the same for a permission card — see `tellAgent`.
3482
+ this.flushPendingAgentNote(running);
2656
3483
  this.sendEvent(running, 'permission_resolved', {
2657
3484
  requestId: event.requestId,
2658
3485
  allow: event.allow,
@@ -2698,6 +3525,8 @@ export class Supervisor {
2698
3525
  return;
2699
3526
  case 'question_resolved':
2700
3527
  running.openQuestions.delete(event.askId);
3528
+ // Anything the runner itself wanted to say while the card was open.
3529
+ this.flushPendingAgentNote(running);
2701
3530
  this.sendEvent(running, 'question_resolved', {
2702
3531
  askId: event.askId,
2703
3532
  outcome: event.outcome,
@@ -2880,8 +3709,26 @@ export class Supervisor {
2880
3709
  // words below have now been published once, and a redelivery must not
2881
3710
  // publish them again.
2882
3711
  running.answeredAsks.add(frame.askId);
3712
+ /**
3713
+ * The reply, whichever control the person used to give it (#401).
3714
+ *
3715
+ * `frame.text` alone was not it: that field carries the bottom input of the
3716
+ * card («Discuss instead»), and the dashboard sends the ORDINARY answer as
3717
+ * `answers` with no text at all. So a branch that rescued only the text
3718
+ * lost the entire reply of everyone who answered by tapping the options —
3719
+ * silently, because the card stayed live and said nothing. That is what
3720
+ * happened to the owner on 08.09.2026 and it is the second half of this
3721
+ * ticket: a question that could not be answered AND an answer that
3722
+ * vanished.
3723
+ *
3724
+ * Both halves, joined rather than one or the other: the frame allows a
3725
+ * person to have typed AND picked, and dropping either would be the same
3726
+ * defect on a narrower path. The sentence comes first because that is the
3727
+ * order the card puts them in.
3728
+ */
3729
+ const rescued = [typed, answersAsMessage(frame.answers ?? [])].filter(Boolean).join('\n');
2883
3730
  this.sendEvent(running, 'system_note', {
2884
- text: typed
3731
+ text: rescued
2885
3732
  ? 'That question is no longer open — sending your reply as an ordinary message instead.'
2886
3733
  : 'That question is no longer open — the agent has already moved on.',
2887
3734
  });
@@ -2889,8 +3736,8 @@ export class Supervisor {
2889
3736
  // restart: the card in the browser outlived the process that asked, and
2890
3737
  // showing the user's own message in the feed while nothing receives it is
2891
3738
  // the exact failure `acceptUserMessage` was hardened against (QA-106 M2).
2892
- if (typed) {
2893
- this.acceptUserMessage(frame.sessionId, typed);
3739
+ if (rescued) {
3740
+ this.acceptUserMessage(frame.sessionId, rescued);
2894
3741
  return 'not_open';
2895
3742
  }
2896
3743
  // Nothing to deliver — but the API optimistically flipped the session to
@@ -2980,11 +3827,22 @@ export class Supervisor {
2980
3827
  ...(attachments?.length ? { attachments } : {}),
2981
3828
  });
2982
3829
  const originSeq = echoed.seq;
2983
- if (!running.worktreePath || Supervisor.isPaused(running)) {
3830
+ if (running.starting || !running.worktreePath || Supervisor.isPaused(running)) {
2984
3831
  // Session is still being prepared — deliver after launch (QA-96 F3).
2985
3832
  // Attachments travel as metadata and are downloaded at delivery time,
2986
3833
  // which is the first moment the worktree is guaranteed to exist.
2987
3834
  //
3835
+ // #401: `starting` is the half of that promise the condition used to be
3836
+ // missing. The folder is ready in milliseconds and the first process only
3837
+ // seconds later, so testing the folder alone let a message through into a
3838
+ // session that had no agent yet — and the delivery path then started one
3839
+ // of its own, beside the one `startSession` was about to start. Both
3840
+ // halves are named because they fail apart: a session past STARTING has a
3841
+ // folder and no `starting`, and waking it with a message is correct.
3842
+ //
3843
+ // Nothing is lost by waiting: `flushPendingMessages` runs on the way out
3844
+ // of `startSession`, after the launch, and delivers this queue in order.
3845
+ //
2988
3846
  // Or the session is held under a clock (#196). The API refuses live
2989
3847
  // messages for a paused session, but not every path goes through that
2990
3848
  // check — the outbox flushes on reconnect, and the git service posts its
@@ -3452,7 +4310,8 @@ export class Supervisor {
3452
4310
  // rewind resumes the session the point names rather than whichever one the
3453
4311
  // session happens to be on now.
3454
4312
  const anchor = this.currentAnchor(running);
3455
- const result = await createCheckpoint({
4313
+ const takeCheckpoint = this.opts.createCheckpoint ?? createCheckpoint;
4314
+ const result = await takeCheckpoint({
3456
4315
  worktreePath,
3457
4316
  sessionId: running.descriptor.id,
3458
4317
  kind,
@@ -3539,6 +4398,33 @@ export class Supervisor {
3539
4398
  // pause. Held work stays held until the clock is off.
3540
4399
  if (Supervisor.isPaused(running))
3541
4400
  return;
4401
+ /**
4402
+ * …and by the same argument, none of them knew about a session that has not
4403
+ * finished starting (#401, found by the independent QA of this fix).
4404
+ *
4405
+ * The queue is drained from six places, and two of them fire on somebody
4406
+ * else's news: `drainSessionsWaitingForCapacity` runs whenever ANY session
4407
+ * on this runner frees a slot, and the pause release runs when a clock comes
4408
+ * off. Either can land in the middle of another session's opening, where the
4409
+ * slot is legitimately empty — so delivery would start the agent with the
4410
+ * queued message, and the opening's own launch would then be refused by the
4411
+ * guard in `launchAgent`. The session's task would never be handed over at
4412
+ * all, and the only trace would be one line in the daemon's log.
4413
+ *
4414
+ * That is a QUIETER failure than the two processes this ticket started
4415
+ * from, so the check belongs here, at the one place the queue turns into
4416
+ * delivery, and not at each of the six callers.
4417
+ *
4418
+ * This does NOT make the `starting` test in `acceptUserMessage` redundant:
4419
+ * that one decides whether a message joins the queue at all, and without it
4420
+ * a message would go straight down the delivery chain and never be seen
4421
+ * here. Two different questions, both needed.
4422
+ *
4423
+ * The opening procedure drains its own queue on the way out, after clearing
4424
+ * the flag — `startSession` and the reconnect branch both do it.
4425
+ */
4426
+ if (running.starting)
4427
+ return;
3542
4428
  const pending = running.pendingMessages.splice(0);
3543
4429
  if (pending.length === 0)
3544
4430
  return;
@@ -4450,6 +5336,9 @@ export class Supervisor {
4450
5336
  levels: freshLevels(),
4451
5337
  stopRequested: false,
4452
5338
  parkRequested: false,
5339
+ // Registered before the worktree await, so the gate has to be up
5340
+ // before it too (#401).
5341
+ starting: true,
4453
5342
  pendingMessages: [],
4454
5343
  // Seed from the API, not 0: a runner restart used to hand the session
4455
5344
  // a full fresh budget silently.
@@ -4475,110 +5364,132 @@ export class Supervisor {
4475
5364
  running.pendingMessages.push(...running.journal.pending());
4476
5365
  this.sessions.set(descriptor.id, running);
4477
5366
  try {
4478
- // A session being restored after a runner restart already has its
4479
- // branch, so the API sends `CONTINUE` — the NEW guard inside would
4480
- // otherwise fire on the runner's own previous work.
4481
- const prepared = await this.prepareWorkspace(descriptor);
4482
- running.branch = prepared.branch;
4483
- running.worktreePath = prepared.worktreePath;
4484
- if (prepared.baseSha)
4485
- running.baseSha = prepared.baseSha;
4486
- if (prepared.baseBranch)
4487
- running.baseBranch = prepared.baseBranch;
4488
- }
4489
- catch (error) {
4490
- // The same treatment as the start door (#360). Two doors doing one
4491
- // thing is how the pause bug of #373 stayed half-fixed for a
4492
- // release; this one used to mask its text differently AND emit no
4493
- // feed event at all, so a session that could not be restored went
4494
- // FAILED with nothing to read anywhere.
4495
- this.workspacePrepareFailed(running, error, 'Failed to restore session worktree');
4496
- continue;
4497
- }
4498
- /**
4499
- * Was a turn actually in flight when the process died?
4500
- *
4501
- * `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
4502
- * others mean the agent was already waiting for a human, and there is
4503
- * nothing to continue. REVIEW is deliberately excluded — the work is
4504
- * finished and waiting to be looked at.
4505
- */
4506
- const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
4507
- const resumeId = descriptor.providerSessionId;
4508
- // Ticket #177: `resumeId` is required, not merely nice to have. Without
4509
- // it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
4510
- // — "continue from where you stopped, re-check what you were in the
4511
- // middle of" — would be addressed to an agent that remembers none of
4512
- // it. A process killed before it reported its session id (the SIGABRT
4513
- // this ticket came from) leaves the row in exactly that state.
4514
- // Ticket #196: a paused session is never continued automatically. The
4515
- // row still says RUNNING — a pause interrupts the turn but is not a
4516
- // status — so without this the reconnect would read «mid-turn» and
4517
- // relaunch the agent with «continue from where you stopped», which is
4518
- // the exact opposite of what the clock was set for.
4519
- const willContinue = wasMidTurn &&
4520
- !Supervisor.isPaused(running) &&
4521
- Boolean(resumeId) &&
4522
- claimAutoResume(descriptor.id);
4523
- // The note stays either way (owner's call): an interruption is a fact
4524
- // about the session and must not disappear just because we recovered
4525
- // from it. Only the instruction at the end changes — telling someone to
4526
- // send a message while the agent is already working again would be a lie.
4527
- this.sendEvent(running, 'system_note', {
4528
- text: willContinue
4529
- ? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
4530
- : resumeId
4531
- ? 'Runner reconnected. The session was resumed — send a message to continue.'
4532
- : 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
4533
- });
4534
- if (willContinue) {
4535
- // Resumed through the PROVIDER session, so the agent keeps its whole
4536
- // conversation; the prompt is only the nudge a human would otherwise
4537
- // have to type. Exactly what «продолжай» did by hand — no new class of
4538
- // risk, and the same ceiling protects against a crash loop doing it
4539
- // forever (see `auto-resume.ts`).
4540
- const continued = this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId);
4541
- if (continued.ok) {
4542
- this.reportStatus(descriptor.id, 'RUNNING', {});
4543
- this.flushPendingMessages(running);
5367
+ try {
5368
+ // A session being restored after a runner restart already has its
5369
+ // branch, so the API sends `CONTINUE` — the NEW guard inside would
5370
+ // otherwise fire on the runner's own previous work.
5371
+ const prepared = await this.prepareWorkspace(descriptor);
5372
+ running.branch = prepared.branch;
5373
+ running.worktreePath = prepared.worktreePath;
5374
+ if (prepared.baseSha)
5375
+ running.baseSha = prepared.baseSha;
5376
+ if (prepared.baseBranch)
5377
+ running.baseBranch = prepared.baseBranch;
5378
+ }
5379
+ catch (error) {
5380
+ // The same treatment as the start door (#360). Two doors doing one
5381
+ // thing is how the pause bug of #373 stayed half-fixed for a
5382
+ // release; this one used to mask its text differently AND emit no
5383
+ // feed event at all, so a session that could not be restored went
5384
+ // FAILED with nothing to read anywhere.
5385
+ this.workspacePrepareFailed(running, error, 'Failed to restore session worktree');
4544
5386
  continue;
4545
5387
  }
4546
- // Could not start — an exhausted budget, or a launch that crashed
4547
- // (ticket #225: this is the exact line the incident died on, and the
4548
- // throw took the WHOLE restore loop with it). Fall through to the old
4549
- // behaviour and say so honestly; a crash has already put its own
4550
- // reason in the feed, so this note would only repeat it.
4551
- if (continued.reason === 'refused') {
4552
- this.sendEvent(running, 'system_note', {
4553
- text: 'Could not continue automatically — send a message to pick the work back up.',
4554
- });
5388
+ /**
5389
+ * Was a turn actually in flight when the process died?
5390
+ *
5391
+ * `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
5392
+ * others mean the agent was already waiting for a human, and there is
5393
+ * nothing to continue. REVIEW is deliberately excluded — the work is
5394
+ * finished and waiting to be looked at.
5395
+ */
5396
+ const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
5397
+ const resumeId = descriptor.providerSessionId;
5398
+ // Ticket #177: `resumeId` is required, not merely nice to have. Without
5399
+ // it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
5400
+ // — "continue from where you stopped, re-check what you were in the
5401
+ // middle of" — would be addressed to an agent that remembers none of
5402
+ // it. A process killed before it reported its session id (the SIGABRT
5403
+ // this ticket came from) leaves the row in exactly that state.
5404
+ // Ticket #196: a paused session is never continued automatically. The
5405
+ // row still says RUNNING — a pause interrupts the turn but is not a
5406
+ // status — so without this the reconnect would read «mid-turn» and
5407
+ // relaunch the agent with «continue from where you stopped», which is
5408
+ // the exact opposite of what the clock was set for.
5409
+ const willContinue = wasMidTurn &&
5410
+ !Supervisor.isPaused(running) &&
5411
+ Boolean(resumeId) &&
5412
+ claimAutoResume(descriptor.id);
5413
+ // The note stays either way (owner's call): an interruption is a fact
5414
+ // about the session and must not disappear just because we recovered
5415
+ // from it. Only the instruction at the end changes — telling someone to
5416
+ // send a message while the agent is already working again would be a lie.
5417
+ this.sendEvent(running, 'system_note', {
5418
+ text: willContinue
5419
+ ? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
5420
+ : resumeId
5421
+ ? 'Runner reconnected. The session was resumed — send a message to continue.'
5422
+ : 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
5423
+ });
5424
+ if (willContinue) {
5425
+ // Resumed through the PROVIDER session, so the agent keeps its whole
5426
+ // conversation; the prompt is only the nudge a human would otherwise
5427
+ // have to type. Exactly what «продолжай» did by hand — no new class of
5428
+ // risk, and the same ceiling protects against a crash loop doing it
5429
+ // forever (see `auto-resume.ts`).
5430
+ const continued = this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId);
5431
+ if (continued.ok) {
5432
+ this.reportStatus(descriptor.id, 'RUNNING', {});
5433
+ // The opening is over — hand the queue back before draining it,
5434
+ // because `flushPendingMessages` refuses a session that is still
5435
+ // starting (#401). The `finally` below is the net for the exits
5436
+ // that never get here.
5437
+ running.starting = false;
5438
+ this.flushPendingMessages(running);
5439
+ continue;
5440
+ }
5441
+ // Could not start — an exhausted budget, or a launch that crashed
5442
+ // (ticket #225: this is the exact line the incident died on, and the
5443
+ // throw took the WHOLE restore loop with it). Fall through to the old
5444
+ // behaviour and say so honestly; a crash has already put its own
5445
+ // reason in the feed, so this note would only repeat it.
5446
+ if (continued.reason === 'refused') {
5447
+ this.sendEvent(running, 'system_note', {
5448
+ text: 'Could not continue automatically — send a message to pick the work back up.',
5449
+ });
5450
+ }
4555
5451
  }
5452
+ /**
5453
+ * REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
5454
+ * mid-turn statuses are downgraded to "waiting for the user".
5455
+ *
5456
+ * REVIEW is now REPORTED rather than skipped — ticket #356, and the
5457
+ * news in that frame is not the status, which has not moved. It is
5458
+ * the ZERO riding on it.
5459
+ *
5460
+ * `running.backgroundTasks` is seeded to 0 above, because a runner
5461
+ * restart takes every subagent with it. But the number the API holds
5462
+ * is written by the runner ALONE, and it is only ever written by a
5463
+ * frame that carries the field — which `reportStatus` attaches only
5464
+ * for a tracked session. Skipping the report here left the API
5465
+ * believing whatever the dead process last said, for good: the badge
5466
+ * would keep saying «Agents», the Inbox would keep hiding the card,
5467
+ * and no later frame would ever correct either. `setBackgroundTasks`
5468
+ * cannot help — it compares against the in-memory 0 and returns
5469
+ * early, having nothing to announce.
5470
+ *
5471
+ * Same-status frames are legal (`canDevSessionTransition` answers
5472
+ * `true` for `from === to`), so this costs one no-op write and buys
5473
+ * back a session that would otherwise have been lost.
5474
+ */
5475
+ this.reportStatus(descriptor.id, statusForReport(running), {});
5476
+ // As above: the flag comes off first, or the drain below is a no-op.
5477
+ running.starting = false;
5478
+ this.flushPendingMessages(running);
5479
+ }
5480
+ finally {
5481
+ /**
5482
+ * The same gate as the start door, for the same reason (#401).
5483
+ *
5484
+ * This branch registers the entry BEFORE awaiting the worktree —
5485
+ * deliberately, so a racing message is buffered rather than dropped
5486
+ * (QA-96 F3) — and then decides for itself whether to bring the
5487
+ * agent back up. A message let through in between would make that
5488
+ * decision instead, and the feed would go on to say the session
5489
+ * could not be continued automatically while it plainly was.
5490
+ */
5491
+ running.starting = false;
4556
5492
  }
4557
- /**
4558
- * REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
4559
- * mid-turn statuses are downgraded to "waiting for the user".
4560
- *
4561
- * REVIEW is now REPORTED rather than skipped — ticket #356, and the
4562
- * news in that frame is not the status, which has not moved. It is
4563
- * the ZERO riding on it.
4564
- *
4565
- * `running.backgroundTasks` is seeded to 0 above, because a runner
4566
- * restart takes every subagent with it. But the number the API holds
4567
- * is written by the runner ALONE, and it is only ever written by a
4568
- * frame that carries the field — which `reportStatus` attaches only
4569
- * for a tracked session. Skipping the report here left the API
4570
- * believing whatever the dead process last said, for good: the badge
4571
- * would keep saying «Agents», the Inbox would keep hiding the card,
4572
- * and no later frame would ever correct either. `setBackgroundTasks`
4573
- * cannot help — it compares against the in-memory 0 and returns
4574
- * early, having nothing to announce.
4575
- *
4576
- * Same-status frames are legal (`canDevSessionTransition` answers
4577
- * `true` for `from === to`), so this costs one no-op write and buys
4578
- * back a session that would otherwise have been lost.
4579
- */
4580
- this.reportStatus(descriptor.id, statusForReport(running), {});
4581
- this.flushPendingMessages(running);
4582
5493
  }
4583
5494
  else if (descriptor.status === 'WAITING_INPUT') {
4584
5495
  // A session that never had a turn (a free session still waiting for
@@ -6079,6 +6990,8 @@ export class Supervisor {
6079
6990
  shutdown() {
6080
6991
  clearInterval(this.slotsTimer);
6081
6992
  clearInterval(this.hostLoadTimer);
6993
+ clearInterval(this.stallTimer);
6994
+ setLiveLadderSource(null);
6082
6995
  clearInterval(this.agentVersionsTimer);
6083
6996
  clearTimeout(this.agentCleanupFirstTimer);
6084
6997
  clearInterval(this.agentCleanupTimer);