@bridge4dev/runner 0.56.0 → 0.58.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +23 -4
- package/dist/adapters/codex-protocol.js +6 -1
- package/dist/adapters/codex.js +11 -2
- package/dist/adapters/questions.d.ts +15 -0
- package/dist/adapters/questions.js +32 -0
- package/dist/adapters/types.js +14 -0
- package/dist/cage-authority.d.ts +118 -0
- package/dist/cage-authority.js +241 -0
- package/dist/config.d.ts +83 -5
- package/dist/config.js +59 -1
- package/dist/daemon-lock.d.ts +43 -0
- package/dist/daemon-lock.js +107 -0
- package/dist/host-load.d.ts +9 -0
- package/dist/host-load.js +9 -0
- package/dist/index.js +222 -20
- package/dist/policy.d.ts +9 -0
- package/dist/policy.js +68 -0
- package/dist/protocol.d.ts +43 -27
- package/dist/recipe-schema.d.ts +12 -12
- package/dist/self-update.js +22 -1
- package/dist/service-unit.d.ts +35 -3
- package/dist/service-unit.js +82 -5
- package/dist/session-allocator.d.ts +259 -0
- package/dist/session-allocator.js +492 -0
- package/dist/session-cage.d.ts +229 -2
- package/dist/session-cage.js +590 -40
- package/dist/session-limits.d.ts +71 -0
- package/dist/session-limits.js +93 -0
- package/dist/session-stall.d.ts +353 -0
- package/dist/session-stall.js +760 -0
- package/dist/supervisor.d.ts +235 -33
- package/dist/supervisor.js +1178 -265
- package/dist/systemd-memory.js +2 -5
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/supervisor.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
2
3
|
import path from 'node:path';
|
|
3
4
|
import { log } from './log.js';
|
|
4
5
|
import { CONTEXT_USAGE_MIN_DELTA_RATIO, CONTEXT_USAGE_MIN_DELTA_TOKENS, RATE_LIMITS_RESEND_INTERVAL_MS, } from './levels.js';
|
|
@@ -23,14 +24,20 @@ import { installAgent } from './agent-install.js';
|
|
|
23
24
|
import { autoUpdateRefusal, claimAgentAutoUpdate } from './agent-auto-update.js';
|
|
24
25
|
import { pruneNativeClaudeVersions } from './agent-cleanup.js';
|
|
25
26
|
import { agentByDbValue } from './agent-registry.js';
|
|
27
|
+
import { cageAuthority } from './cage-authority.js';
|
|
26
28
|
import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
|
|
27
29
|
import { rememberWorkspacePath } from './environment.js';
|
|
28
30
|
import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
|
|
29
|
-
import { markScopeOomKillsSeen, readScopeMemoryStatus, sessionScopeUnitOf, stoppedProcesses, } from './session-cage.js';
|
|
31
|
+
import { markScopeOomKillsSeen, readScopeHold, readScopeMemoryStatus, readSliceLimits, sessionAgentPid, sessionCage, sessionScopeUnitOf, setLiveLadderSource, stoppedProcesses, sweepOrphanSessionScopes, } from './session-cage.js';
|
|
32
|
+
import { allocateSessionMemory, planLimitMove, SESSION_GUARANTEE_BYTES, } from './session-allocator.js';
|
|
33
|
+
import { freshStallState, pickKillCandidate, readScopeProcesses, readStallSample, setScopeProperties, signalSubtree, stallStep, STALL_GRACE_MS, STALL_MUTE_AFTER_LOWER_MS, STALL_SIGKILL_AFTER_MS, STALL_WINDOW_MS, } from './session-stall.js';
|
|
34
|
+
import { machineReserveBytes } from './service-unit.js';
|
|
35
|
+
import { sessionLimitsChangedEnough, sessionLimitsHeartbeatDue, SESSION_LIMITS_HEARTBEAT_MS, } from './session-limits.js';
|
|
30
36
|
import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
|
|
31
37
|
import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, MAX_BUSY_SESSIONS, previewRewind, pruneCheckpoints, } from './checkpoints.js';
|
|
32
38
|
import { DeliverMessageArgsSchema, QuestionAnswerArgsSchema } from './protocol.js';
|
|
33
39
|
import { availableModes, MODE_REFUSED_TEXT } from './adapters/types.js';
|
|
40
|
+
import { answersAsMessage } from './adapters/questions.js';
|
|
34
41
|
/** Refusals shared by every checkpoint command (ticket #126). */
|
|
35
42
|
const CHECKPOINTS_OFF = 'Restore points are switched off on this server ([checkpoints] enabled = false)';
|
|
36
43
|
const AGENT_BUSY = 'The agent is still working — stop the turn first';
|
|
@@ -231,26 +238,45 @@ export class Supervisor {
|
|
|
231
238
|
// Same cadence, same reason: what a session's cgroup is going through is
|
|
232
239
|
// something that happens to it, and only a tick can notice (#387).
|
|
233
240
|
this.watchSessionCages();
|
|
241
|
+
// And the same argument one step further (#398, S1): a scope with nobody
|
|
242
|
+
// behind it is also something that HAPPENS, and until 0.58.0 the only
|
|
243
|
+
// sweep ran at daemon start — so an orphan held its memory until the next
|
|
244
|
+
// restart. `releaseSessionScope` is the primary path now; this is the net
|
|
245
|
+
// under it, for the exit that never fired at all.
|
|
246
|
+
void this.sweepOrphanCages();
|
|
247
|
+
// …and the third thing that only a tick can notice (#398 S3): how much of
|
|
248
|
+
// the machine is free RIGHT NOW, and therefore how much each session may
|
|
249
|
+
// use. The number this replaced was computed once, at daemon start.
|
|
250
|
+
void this.runAllocator(Date.now());
|
|
234
251
|
}, opts.hostLoadSampleMs ?? HOST_LOAD_SAMPLE_INTERVAL_MS);
|
|
235
252
|
this.hostLoadTimer.unref?.();
|
|
253
|
+
/**
|
|
254
|
+
* The stall detector, on its own faster clock (#398 S2).
|
|
255
|
+
*
|
|
256
|
+
* A second timer rather than more work on the 30 s one, and the reason is
|
|
257
|
+
* arithmetic: the deadline a standing session gets is three minutes, the
|
|
258
|
+
* verdict needs two consecutive windows, and at 30 s a session would spend a
|
|
259
|
+
* third of its deadline being diagnosed. Five seconds costs five small file
|
|
260
|
+
* reads per live caged session — nothing next to what the tick above already
|
|
261
|
+
* does — and only for sessions that have a cgroup at all.
|
|
262
|
+
*/
|
|
263
|
+
this.limitsHeartbeatMs = opts.sessionLimitsHeartbeatMs ?? SESSION_LIMITS_HEARTBEAT_MS;
|
|
264
|
+
this.stallGraceMs = opts.stallGraceMs ?? STALL_GRACE_MS;
|
|
265
|
+
this.stallSampleMs = opts.stallSampleMs ?? STALL_WINDOW_MS;
|
|
266
|
+
this.stallTimer = setInterval(() => {
|
|
267
|
+
void this.watchSessionStalls();
|
|
268
|
+
}, this.stallSampleMs);
|
|
269
|
+
this.stallTimer.unref?.();
|
|
270
|
+
/**
|
|
271
|
+
* A session starting mid-tick asks the allocator for its numbers rather
|
|
272
|
+
* than taking the daemon-start snapshot (#398 S3, work 9).
|
|
273
|
+
*
|
|
274
|
+
* Registered here and cleared in `shutdown()`: the hook lives on the cage
|
|
275
|
+
* module, which cannot import this one, and a stale hook pointing at a dead
|
|
276
|
+
* supervisor would be worse than none.
|
|
277
|
+
*/
|
|
278
|
+
setLiveLadderSource((id) => this.ladderForNewSession(id));
|
|
236
279
|
}
|
|
237
|
-
/**
|
|
238
|
-
* How long a session may sit over its brake before the feed says more than
|
|
239
|
-
* «slower» (#387, step 3). Five minutes is a starting point, not a
|
|
240
|
-
* measurement: an honest `pnpm typecheck` overshoots for tens of seconds, a
|
|
241
|
-
* runaway for ever, and the number that tells them apart on real machines is
|
|
242
|
-
* still to be read off `memory.events`. Sized so the first warning is never
|
|
243
|
-
* the only one a night-time run leaves behind.
|
|
244
|
-
*/
|
|
245
|
-
static CAGE_BRAKE_LONG_MS = 5 * 60_000;
|
|
246
|
-
/**
|
|
247
|
-
* Calm ticks before the feed says the braking is over.
|
|
248
|
-
*
|
|
249
|
-
* `memory.high` holds usage AT the line, so a single sample under it means
|
|
250
|
-
* nothing. Two (a minute at the 30 s cadence) is the difference between an
|
|
251
|
-
* episode that ended and a build that breathed.
|
|
252
|
-
*/
|
|
253
|
-
static CAGE_CALM_TICKS_TO_RELEASE = 2;
|
|
254
280
|
/** How often the agent versions are re-derived. See the constructor. */
|
|
255
281
|
static AGENT_VERSIONS_INTERVAL_MS = 60 * 60 * 1_000;
|
|
256
282
|
agentVersionsTimer;
|
|
@@ -616,6 +642,13 @@ export class Supervisor {
|
|
|
616
642
|
/** The heartbeat window actually used — the constant, or a test's own. */
|
|
617
643
|
hostLoadHeartbeatMs;
|
|
618
644
|
hostLoadTimer;
|
|
645
|
+
/** #398 S2: the stall detector's clock, and the deadline it enforces. */
|
|
646
|
+
stallTimer;
|
|
647
|
+
stallGraceMs;
|
|
648
|
+
stallSampleMs;
|
|
649
|
+
limitsHeartbeatMs;
|
|
650
|
+
/** A stall tick is in flight — the next one waits rather than overlaps. */
|
|
651
|
+
watchingStalls = false;
|
|
619
652
|
/** The last measurement the API actually took from us, or `null` for «nothing yet». */
|
|
620
653
|
lastPublishedHostLoad = null;
|
|
621
654
|
/** When that frame went out, by this machine's clock. `0` = never. */
|
|
@@ -646,22 +679,27 @@ export class Supervisor {
|
|
|
646
679
|
* it had one.
|
|
647
680
|
*/
|
|
648
681
|
/**
|
|
649
|
-
* Tell the person
|
|
682
|
+
* Tell the person when the KERNEL killed something inside a caged session.
|
|
683
|
+
*
|
|
684
|
+
* One thing, since #398 S2 — `oom_kill` moved: the kernel stopped a process
|
|
685
|
+
* inside the session. With `OOMPolicy=continue` the session itself is still
|
|
686
|
+
* here, and the person has to be told that the command that was running is
|
|
687
|
+
* probably what died.
|
|
650
688
|
*
|
|
651
|
-
*
|
|
689
|
+
* **The three braking lines this tick used to write are gone** (decision D8 of
|
|
690
|
+
* `session-memory-fair-share.md`): «has hit its memory share», «has been over
|
|
691
|
+
* its memory share for 5 min», and the all-clear. Two reasons, and the second
|
|
692
|
+
* is the decisive one:
|
|
652
693
|
*
|
|
653
|
-
* -
|
|
654
|
-
*
|
|
655
|
-
*
|
|
656
|
-
*
|
|
657
|
-
*
|
|
658
|
-
*
|
|
659
|
-
* - `oom_kill` moved: the kernel killed a process inside the session. With
|
|
660
|
-
* `OOMPolicy=continue` the session itself is still here, and the person
|
|
661
|
-
* has to be told that the command that was running is probably what died.
|
|
694
|
+
* - they were state, not events. Being over the brake is a condition that
|
|
695
|
+
* comes and goes; its place is the activity strip under the session and the
|
|
696
|
+
* machine card, not the history a person scrolls back through;
|
|
697
|
+
* - they disagreed with the mechanism that replaced them. The deadline in
|
|
698
|
+
* `session-stall.ts` runs out after three minutes, so the five-minute line
|
|
699
|
+
* could never have been printed at all.
|
|
662
700
|
*
|
|
663
|
-
*
|
|
664
|
-
*
|
|
701
|
+
* What a person sees instead: nothing while the machine copes, and exactly one
|
|
702
|
+
* line when a command was actually stopped.
|
|
665
703
|
*
|
|
666
704
|
* A parked session has no process and no cgroup, and is skipped without a
|
|
667
705
|
* read. A session whose cgroup cannot be read (no cage on this machine, or
|
|
@@ -670,8 +708,6 @@ export class Supervisor {
|
|
|
670
708
|
watchSessionCages() {
|
|
671
709
|
const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
|
|
672
710
|
const read = this.opts.readScopeMemoryStatus ?? readScopeMemoryStatus;
|
|
673
|
-
const longMs = this.opts.cageBrakeLongMs ?? Supervisor.CAGE_BRAKE_LONG_MS;
|
|
674
|
-
const now = Date.now();
|
|
675
711
|
const mb = (bytes) => `${Math.round(bytes / (1024 * 1024))} MB`;
|
|
676
712
|
for (const running of this.sessions.values()) {
|
|
677
713
|
if (running.session === null)
|
|
@@ -702,15 +738,7 @@ export class Supervisor {
|
|
|
702
738
|
status.ownLimitOom >= seen.ownOom;
|
|
703
739
|
const base = carriedOver
|
|
704
740
|
? seen
|
|
705
|
-
: {
|
|
706
|
-
unit,
|
|
707
|
-
oomKills: 0,
|
|
708
|
-
ownOom: 0,
|
|
709
|
-
highEvents: 0,
|
|
710
|
-
brakedSince: null,
|
|
711
|
-
warnedLong: false,
|
|
712
|
-
calmTicks: 0,
|
|
713
|
-
};
|
|
741
|
+
: { unit, oomKills: 0, ownOom: 0, highEvents: 0 };
|
|
714
742
|
if (status.oomKills > base.oomKills) {
|
|
715
743
|
// WHOSE limit was hit decides the sentence. `oom` moves only in the
|
|
716
744
|
// cgroup whose own ceiling was reached; a kill with our counter still
|
|
@@ -730,66 +758,778 @@ export class Supervisor {
|
|
|
730
758
|
// So a death minutes later is not blamed on a kill the feed already carries.
|
|
731
759
|
markScopeOomKillsSeen(running.descriptor.id, status.oomKills);
|
|
732
760
|
}
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
761
|
+
running.cageWatch = {
|
|
762
|
+
unit,
|
|
763
|
+
oomKills: status.oomKills,
|
|
764
|
+
ownOom: status.ownLimitOom,
|
|
765
|
+
highEvents: status.highEvents,
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
/** A sweep is in flight — see {@link sweepOrphanCages}. */
|
|
770
|
+
sweepingCages = false;
|
|
771
|
+
/**
|
|
772
|
+
* Stop every session scope with nobody behind it, on the watch tick (#398 S1).
|
|
773
|
+
*
|
|
774
|
+
* The sweep itself spares every LIVE cage by its own register, sessions and
|
|
775
|
+
* `verify` runs alike (`session-cage.ts`), so what is passed here is belt and
|
|
776
|
+
* braces rather than the guard — but passing it says at the call site which
|
|
777
|
+
* ids this supervisor believes are alive, and a divergence between the two
|
|
778
|
+
* would be a bug worth seeing in a test.
|
|
779
|
+
*
|
|
780
|
+
* Re-entrancy matters more than it looks: the sweep is a `systemctl` call and
|
|
781
|
+
* the bus was measured at 2.7 s under load, so on a machine where the tick is
|
|
782
|
+
* 30 s and the bus is slow two sweeps could otherwise overlap and each would
|
|
783
|
+
* see the other's half-stopped units.
|
|
784
|
+
*
|
|
785
|
+
* Never fatal, and deliberately silent on failure: the daemon carries a fatal
|
|
786
|
+
* `unhandledRejection` handler, and losing every session because a bus call
|
|
787
|
+
* failed would be a far worse trade than an orphan living one more tick.
|
|
788
|
+
*/
|
|
789
|
+
async sweepOrphanCages() {
|
|
790
|
+
if (this.sweepingCages)
|
|
791
|
+
return;
|
|
792
|
+
const injected = this.opts.sweepOrphanSessionScopes;
|
|
793
|
+
/**
|
|
794
|
+
* Two gates, and the first one is the one that had to be rebuilt (#403).
|
|
795
|
+
*
|
|
796
|
+
* The guard here used to be «the cage has been probed», on the reasoning
|
|
797
|
+
* that no test probes it and every daemon does. On 10.09.2026 a new test
|
|
798
|
+
* probed it, and this sweep — from a vitest worker, with an empty register
|
|
799
|
+
* of live cages — stopped three live sessions belonging to other people.
|
|
800
|
+
* The lesson is not «that test was wrong»: a guard that depends on what
|
|
801
|
+
* OTHER people's future code happens to do cannot hold, and it failed
|
|
802
|
+
* silently, because an assumption cannot fail loudly.
|
|
803
|
+
*
|
|
804
|
+
* So the gate is now a right that has to be taken: only the process that
|
|
805
|
+
* claimed `daemon` at start-up sweeps, and a test process cannot claim it.
|
|
806
|
+
* `cage-authority.ts` refuses the underlying `systemctl` as well — this
|
|
807
|
+
* gate is the one that keeps the call from being made at all.
|
|
808
|
+
*
|
|
809
|
+
* The second gate is unchanged and is only thrift: on a machine with no
|
|
810
|
+
* cage there is nothing of ours under that prefix.
|
|
811
|
+
*/
|
|
812
|
+
if (!injected && cageAuthority() !== 'daemon')
|
|
813
|
+
return;
|
|
814
|
+
if (!injected && sessionCage().mode !== 'scope')
|
|
815
|
+
return;
|
|
816
|
+
this.sweepingCages = true;
|
|
817
|
+
try {
|
|
818
|
+
const sweep = injected ?? sweepOrphanSessionScopes;
|
|
819
|
+
await sweep([...this.sessions.keys()]);
|
|
820
|
+
}
|
|
821
|
+
catch (error) {
|
|
822
|
+
log.debug('supervisor: the orphan cage sweep did not run', { error: String(error) });
|
|
823
|
+
}
|
|
824
|
+
finally {
|
|
825
|
+
this.sweepingCages = false;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* The stall mechanism, once per window (#398 S2, D6, gotcha §480).
|
|
830
|
+
*
|
|
831
|
+
* The order is the whole design, and each step exists because the one before
|
|
832
|
+
* it was not enough:
|
|
833
|
+
*
|
|
834
|
+
* 1. **T0 — add.** If the machine has memory free, the brake goes up and the
|
|
835
|
+
* session carries on. Nothing is written to the feed: the mechanism
|
|
836
|
+
* worked, nothing changed for the person, the build is still going (D8).
|
|
837
|
+
* 2. **T1 — the deadline.** Three minutes, announced as state (the strip
|
|
838
|
+
* under the session says how long is left) and never as a line in the
|
|
839
|
+
* history. This is the window in which a person can step in without
|
|
840
|
+
* paying for it with an entry they will scroll past for ever.
|
|
841
|
+
* 3. **T2 — stop the biggest command.** Not the session, not the agent: the
|
|
842
|
+
* command. The agent stays alive, sees its command stopped, is told why,
|
|
843
|
+
* and answers the person itself. Waiting for a human instead is what cost
|
|
844
|
+
* the incident its hour — at night nobody comes.
|
|
845
|
+
* 4. **T3 — nothing to stop.** When all that is left in the cage is the agent
|
|
846
|
+
* and its tools, this mechanism has no right to any of them. State again,
|
|
847
|
+
* no line: the person decides with «Stop» or «Pause».
|
|
848
|
+
*
|
|
849
|
+
* Never throws out of here. The daemon carries a fatal `unhandledRejection`
|
|
850
|
+
* handler, so one unhandled error would end every session on the machine.
|
|
851
|
+
*/
|
|
852
|
+
async watchSessionStalls() {
|
|
853
|
+
if (this.watchingStalls)
|
|
854
|
+
return;
|
|
855
|
+
this.watchingStalls = true;
|
|
856
|
+
try {
|
|
857
|
+
const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
|
|
858
|
+
const sampleOf = this.opts.readStallSample ?? readStallSample;
|
|
859
|
+
const now = Date.now();
|
|
860
|
+
for (const running of this.sessions.values()) {
|
|
861
|
+
const unit = running.session === null ? null : unitOf(running.descriptor.id);
|
|
862
|
+
if (unit === null) {
|
|
863
|
+
/**
|
|
864
|
+
* Parked, or never caged. A stall belongs to a live process; keeping
|
|
865
|
+
* the state would age into a deadline against a session that is not
|
|
866
|
+
* running at all.
|
|
867
|
+
*
|
|
868
|
+
* …and so do the limits (#403). `running.limits` means «what systemd
|
|
869
|
+
* was told about THIS scope», and a parked session has no scope: on
|
|
870
|
+
* resume `cageSpawn` was starting the new one with yesterday's
|
|
871
|
+
* numbers, `sessionMemoryEnv` was telling the agent a ceiling nobody
|
|
872
|
+
* held, and the wall could only come down through four shrink ticks
|
|
873
|
+
* that a fresh session does not have.
|
|
874
|
+
*/
|
|
875
|
+
delete running.stall;
|
|
876
|
+
delete running.limits;
|
|
877
|
+
continue;
|
|
760
878
|
}
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
879
|
+
try {
|
|
880
|
+
await this.stepOneStall(running, unit, sampleOf, now);
|
|
881
|
+
}
|
|
882
|
+
catch (error) {
|
|
883
|
+
log.warn('supervisor: the memory stall check failed for a session', {
|
|
884
|
+
sessionId: running.descriptor.id,
|
|
885
|
+
error: String(error instanceof Error ? error.message : error),
|
|
767
886
|
});
|
|
768
887
|
}
|
|
769
888
|
}
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
889
|
+
}
|
|
890
|
+
finally {
|
|
891
|
+
this.watchingStalls = false;
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
async stepOneStall(running, unit, sampleOf, now) {
|
|
895
|
+
// The pending SIGKILL first, and before anything can return early: a subtree
|
|
896
|
+
// that ignored SIGTERM must get the second signal even on a tick where the
|
|
897
|
+
// cgroup could not be read at all.
|
|
898
|
+
this.escalatePendingKill(running, now);
|
|
899
|
+
/**
|
|
900
|
+
* Stamped at the READ, not at the start of the pass (#398 S7, B13).
|
|
901
|
+
*
|
|
902
|
+
* `now` is one clock for the whole sweep, and the sweep walks every live
|
|
903
|
+
* session doing file reads — so the last session's sample carried the first
|
|
904
|
+
* session's timestamp. The share this feeds is `Δpressure / windowMs`, and
|
|
905
|
+
* an offset between the two ends of a window is exactly the error that
|
|
906
|
+
* makes it wrong. `readHostLoad` has always stamped inside itself.
|
|
907
|
+
*/
|
|
908
|
+
const sample = sampleOf(unit, Date.now());
|
|
909
|
+
if (sample === null)
|
|
910
|
+
return;
|
|
911
|
+
const previous = running.stall && running.stall.detector.unit === unit
|
|
912
|
+
? running.stall
|
|
913
|
+
: {
|
|
914
|
+
detector: freshStallState(unit),
|
|
915
|
+
sample: null,
|
|
916
|
+
remainingMs: null,
|
|
917
|
+
kill: null,
|
|
918
|
+
reason: null,
|
|
919
|
+
};
|
|
920
|
+
const verdict = stallStep(previous.detector, sample, {
|
|
921
|
+
graceMs: this.stallGraceMs,
|
|
922
|
+
windowMs: this.stallSampleMs,
|
|
923
|
+
now,
|
|
924
|
+
// Alone in the slice, «somebody else's brake» has no somebody else — see
|
|
925
|
+
// the discriminator in `session-stall.ts` (#398 S7, B9).
|
|
926
|
+
soleSession: this.liveCagedSessions() <= 1,
|
|
927
|
+
});
|
|
928
|
+
const stall = {
|
|
929
|
+
...previous,
|
|
930
|
+
detector: verdict.state,
|
|
931
|
+
sample,
|
|
932
|
+
remainingMs: verdict.remainingMs,
|
|
933
|
+
};
|
|
934
|
+
running.stall = stall;
|
|
935
|
+
if (!verdict.stuck) {
|
|
936
|
+
/**
|
|
937
|
+
* The session is moving again, so whatever the strip was saying about its
|
|
938
|
+
* memory stops being true — and state that outlives its cause is exactly
|
|
939
|
+
* what decision D8 moved out of the feed to avoid.
|
|
940
|
+
*
|
|
941
|
+
* Found by the independent review of 10.09.2026: `reason` was set and
|
|
942
|
+
* never cleared, so one episode made the strip say «low on memory» for
|
|
943
|
+
* the rest of the session's life. Kept while a subtree is still being
|
|
944
|
+
* stopped: the line about the stopped command is true until it is gone.
|
|
945
|
+
*/
|
|
946
|
+
if (stall.kill === null)
|
|
947
|
+
stall.reason = null;
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
// T0. Free memory on the machine is the cheapest answer there is, and it
|
|
951
|
+
// costs the person nothing to know about.
|
|
952
|
+
if (await this.addMemoryTo(running, sample, now)) {
|
|
953
|
+
stall.reason = 'added';
|
|
954
|
+
stall.detector = { ...stall.detector, hotWindows: 0, stalledSince: null, markBytes: 0 };
|
|
955
|
+
stall.remainingMs = null;
|
|
956
|
+
return;
|
|
957
|
+
}
|
|
958
|
+
/**
|
|
959
|
+
* The owner's switch (#398 S6). `report-only` leaves everything else
|
|
960
|
+
* exactly as it is — the deadline still runs, the strip still shows it, the
|
|
961
|
+
* person can still act — and only DevBridge stops acting on their behalf.
|
|
962
|
+
*
|
|
963
|
+
* Said at the START of the countdown and not at the end of it (#403). The
|
|
964
|
+
* strip renders the reason ahead of the clock, so while this was set only
|
|
965
|
+
* on expiry the owner of a report-only machine spent three minutes being
|
|
966
|
+
* promised «2:14 until a command is stopped» — a countdown to an event
|
|
967
|
+
* their own setting had already ruled out.
|
|
968
|
+
*/
|
|
969
|
+
const action = this.opts.memoryStallAction ?? 'stop-command';
|
|
970
|
+
if (action === 'report-only')
|
|
971
|
+
stall.reason = 'report-only';
|
|
972
|
+
// T1. The deadline runs; the strip shows it. One victim at a time: while a
|
|
973
|
+
// subtree is still being stopped, nothing else is chosen.
|
|
974
|
+
if (!verdict.expired || stall.kill !== null)
|
|
975
|
+
return;
|
|
976
|
+
if (action === 'report-only') {
|
|
977
|
+
// Its own word, not «nothing to stop»: those are different facts about the
|
|
978
|
+
// machine, and telling the owner who chose this setting that there is
|
|
979
|
+
// nothing to stop would be a falsehood about their own machine (found by
|
|
980
|
+
// the independent review of 10.09.2026).
|
|
981
|
+
//
|
|
982
|
+
// Once per episode: compared against what the state held BEFORE this pass,
|
|
983
|
+
// because the line above has already set it for this one.
|
|
984
|
+
if (previous.reason !== 'report-only') {
|
|
985
|
+
log.warn('session memory: a session is standing still, and this machine is set to report only', {
|
|
986
|
+
sessionId: running.descriptor.id,
|
|
987
|
+
holdingMB: Math.round(sample.currentBytes / (1024 * 1024)),
|
|
988
|
+
});
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
else {
|
|
992
|
+
await this.stopBiggestCommand(running, unit, sample, now);
|
|
993
|
+
}
|
|
994
|
+
// Whatever we did or did not do, the clock starts again: the next attempt is
|
|
995
|
+
// a full deadline away, not five seconds.
|
|
996
|
+
stall.detector = { ...stall.detector, stalledSince: now, markBytes: sample.currentBytes };
|
|
997
|
+
stall.remainingMs = this.stallGraceMs;
|
|
998
|
+
}
|
|
999
|
+
/**
|
|
1000
|
+
* Ask the allocator for more, now, because this session has stopped moving (T0).
|
|
1001
|
+
*
|
|
1002
|
+
* Since S3 there is no separate ladder here: `MemoryHigh` has exactly TWO
|
|
1003
|
+
* writers, and they are the same code — the allocator on its tick and the
|
|
1004
|
+
* allocator on this signal. Two independent writers of one number is how a
|
|
1005
|
+
* limit ends up being whatever the last tick happened to think.
|
|
1006
|
+
*
|
|
1007
|
+
* The allocator needs no special case for a stall: a standing session's `hold`
|
|
1008
|
+
* is by definition all the way up against its brake, so `hold + free` asks for
|
|
1009
|
+
* everything the machine has spare. If there is nothing spare, nothing moves,
|
|
1010
|
+
* and the deadline below is the answer instead.
|
|
1011
|
+
*/
|
|
1012
|
+
async addMemoryTo(running, sample, now) {
|
|
1013
|
+
// The brake as the CGROUP has it, not as this supervisor remembers writing
|
|
1014
|
+
// it: on the first pass there is nothing remembered, and «did this session
|
|
1015
|
+
// get more room» has to be answerable then too.
|
|
1016
|
+
const before = sample.brakeBytes;
|
|
1017
|
+
await this.runAllocator(now);
|
|
1018
|
+
const after = running.limits?.brakeBytes ?? null;
|
|
1019
|
+
if (before === null || after === null)
|
|
1020
|
+
return false;
|
|
1021
|
+
return after > before;
|
|
1022
|
+
}
|
|
1023
|
+
/** T2 and T3 — stop the biggest command of this session, or nothing at all. */
|
|
1024
|
+
async stopBiggestCommand(running, unit, sample, now) {
|
|
1025
|
+
const stall = running.stall;
|
|
1026
|
+
if (!stall)
|
|
1027
|
+
return;
|
|
1028
|
+
const processes = (this.opts.readScopeProcesses ?? readScopeProcesses)(unit);
|
|
1029
|
+
// Which pid is the agent, from the cage that spawned it — see
|
|
1030
|
+
// `noteSessionAgentPid`. Null on a machine with no cage, and then
|
|
1031
|
+
// `pickKillCandidate` falls back to «every parentless process».
|
|
1032
|
+
const candidate = pickKillCandidate(processes, sessionAgentPid(running.descriptor.id));
|
|
1033
|
+
const mb = (bytes) => Math.round(bytes / (1024 * 1024));
|
|
1034
|
+
if (candidate === null) {
|
|
1035
|
+
// T3. Everything left in the cage is the agent itself, its MCP servers or
|
|
1036
|
+
// `git`. This mechanism may not touch any of them, and by D8 it does not
|
|
1037
|
+
// write a line either — the strip says so while it is true.
|
|
1038
|
+
if (stall.reason !== 'nothing-to-stop') {
|
|
1039
|
+
log.warn('session memory: a session is out of memory and has nothing but the agent to stop', {
|
|
1040
|
+
sessionId: running.descriptor.id,
|
|
1041
|
+
holdingMB: mb(sample.currentBytes),
|
|
1042
|
+
});
|
|
1043
|
+
}
|
|
1044
|
+
stall.reason = 'nothing-to-stop';
|
|
1045
|
+
return;
|
|
1046
|
+
}
|
|
1047
|
+
const signal = this.opts.signalSubtree ?? signalSubtree;
|
|
1048
|
+
const sent = signal(candidate.pids, 'SIGTERM');
|
|
1049
|
+
stall.kill = {
|
|
1050
|
+
identities: candidate.identities,
|
|
1051
|
+
name: candidate.name,
|
|
1052
|
+
at: now + STALL_SIGKILL_AFTER_MS,
|
|
1053
|
+
};
|
|
1054
|
+
stall.reason = 'stopped-command';
|
|
1055
|
+
log.warn('session memory: stopped the biggest command of a session that had stopped moving', {
|
|
1056
|
+
sessionId: running.descriptor.id,
|
|
1057
|
+
command: candidate.name,
|
|
1058
|
+
processes: sent,
|
|
1059
|
+
commandMB: mb(candidate.rssBytes),
|
|
1060
|
+
sessionMB: mb(sample.currentBytes),
|
|
1061
|
+
});
|
|
1062
|
+
// The one line a person gets out of the whole episode (D8). The command's
|
|
1063
|
+
// NAME only — its arguments carry keys and tokens.
|
|
1064
|
+
this.sendEvent(running, 'notice', {
|
|
1065
|
+
level: 'warn',
|
|
1066
|
+
text: `This session ran out of memory, so DevBridge stopped its biggest command — ` +
|
|
1067
|
+
`${candidate.name}, holding ${mb(candidate.rssBytes)} MB of the session's ${mb(sample.currentBytes)} MB. ` +
|
|
1068
|
+
'The session itself is still here.',
|
|
1069
|
+
});
|
|
1070
|
+
// And the agent is told, so it does not simply run the same thing again (D7).
|
|
1071
|
+
this.tellAgent(running, `DevBridge stopped your \`${candidate.name}\` command: this session ran out of memory. ` +
|
|
1072
|
+
`It was holding ${mb(candidate.rssBytes)} MB, the session's share is ` +
|
|
1073
|
+
`${sample.brakeBytes === null ? 'unknown' : `${mb(sample.brakeBytes)} MB`}, and there was nothing free ` +
|
|
1074
|
+
'on the machine to add. Running the same command again will end the same way — make it need less ' +
|
|
1075
|
+
'(fewer parallel jobs, a smaller heap, a narrower scope) or split the work up. Do not raise ' +
|
|
1076
|
+
'`--max-old-space-size`: the ceiling is enforced outside your process.');
|
|
1077
|
+
}
|
|
1078
|
+
/**
|
|
1079
|
+
* Say something to the agent on the runner's own initiative, without stepping
|
|
1080
|
+
* on a card the person has not answered (#398 S7, B7).
|
|
1081
|
+
*
|
|
1082
|
+
* `session.send()` looks like a neutral channel and is not one: with a
|
|
1083
|
+
* question open, `ClaudeSession.send` answers it «discuss» and the API
|
|
1084
|
+
* records `question_resolved {source: 'user'}` — a decision attributed to a
|
|
1085
|
+
* person who never made it — while `CodexSession.send` declines a held plan
|
|
1086
|
+
* the same way. Every other initiative of this supervisor already checks for
|
|
1087
|
+
* an open card; the memory mechanism was the one that did not.
|
|
1088
|
+
*
|
|
1089
|
+
* Held rather than dropped: the agent is parked on the card, so it cannot run
|
|
1090
|
+
* anything until the person answers anyway, and once they do the note is the
|
|
1091
|
+
* first thing it reads.
|
|
1092
|
+
*/
|
|
1093
|
+
tellAgent(running, text) {
|
|
1094
|
+
if (running.openQuestions.size > 0 || running.openPermissions.size > 0) {
|
|
1095
|
+
running.pendingAgentNote = running.pendingAgentNote
|
|
1096
|
+
? `${running.pendingAgentNote}\n\n${text}`
|
|
1097
|
+
: text;
|
|
1098
|
+
return;
|
|
1099
|
+
}
|
|
1100
|
+
running.session?.send(text);
|
|
1101
|
+
}
|
|
1102
|
+
/** Deliver what waited for a card to close — see {@link tellAgent}. */
|
|
1103
|
+
flushPendingAgentNote(running) {
|
|
1104
|
+
const note = running.pendingAgentNote;
|
|
1105
|
+
if (note === undefined)
|
|
1106
|
+
return;
|
|
1107
|
+
if (running.openQuestions.size > 0 || running.openPermissions.size > 0)
|
|
1108
|
+
return;
|
|
1109
|
+
delete running.pendingAgentNote;
|
|
1110
|
+
running.session?.send(note);
|
|
1111
|
+
}
|
|
1112
|
+
/** How many sessions are running in a cage right now (#398 S7, B9). */
|
|
1113
|
+
liveCagedSessions() {
|
|
1114
|
+
const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
|
|
1115
|
+
let count = 0;
|
|
1116
|
+
for (const running of this.sessions.values()) {
|
|
1117
|
+
if (running.session === null)
|
|
1118
|
+
continue;
|
|
1119
|
+
if (unitOf(running.descriptor.id) !== null)
|
|
1120
|
+
count += 1;
|
|
1121
|
+
}
|
|
1122
|
+
return count;
|
|
1123
|
+
}
|
|
1124
|
+
/** SIGKILL for a subtree that did not take SIGTERM within the grace. */
|
|
1125
|
+
escalatePendingKill(running, now) {
|
|
1126
|
+
const kill = running.stall?.kill;
|
|
1127
|
+
if (!kill || now < kill.at)
|
|
1128
|
+
return;
|
|
1129
|
+
const signal = this.opts.signalSubtree ?? signalSubtree;
|
|
1130
|
+
/**
|
|
1131
|
+
* Identities, not bare pids. Thirty seconds is long enough on a busy machine
|
|
1132
|
+
* for a pid freed by the SIGTERM to be handed to something else, and a
|
|
1133
|
+
* SIGKILL to the wrong process is the worst thing in this whole change —
|
|
1134
|
+
* named as such by the independent review of 10.09.2026.
|
|
1135
|
+
*/
|
|
1136
|
+
const sent = signal(kill.identities, 'SIGKILL');
|
|
1137
|
+
if (sent > 0) {
|
|
1138
|
+
log.warn('session memory: the stopped command did not go on its own', {
|
|
1139
|
+
sessionId: running.descriptor.id,
|
|
1140
|
+
command: kill.name,
|
|
1141
|
+
processes: sent,
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
1144
|
+
else if (kill.identities.length > 0) {
|
|
1145
|
+
/**
|
|
1146
|
+
* Nothing was signalled, and that is worth a line (#403).
|
|
1147
|
+
*
|
|
1148
|
+
* Two ways to get here and they are different machines. Either the
|
|
1149
|
+
* processes ended on their own during the grace — the good outcome, and
|
|
1150
|
+
* the common one — or their identity could not be read when the victim
|
|
1151
|
+
* was chosen (`startedAtTicks: -1`), in which case `isSameProcess` refuses
|
|
1152
|
+
* every one of them and the promised SIGKILL never happens at all. Under
|
|
1153
|
+
* `if (sent > 0)` both were silent, so «stopped, then escalated» could
|
|
1154
|
+
* quietly be neither.
|
|
1155
|
+
*/
|
|
1156
|
+
log.info('session memory: nothing was left to SIGKILL', {
|
|
1157
|
+
sessionId: running.descriptor.id,
|
|
1158
|
+
command: kill.name,
|
|
1159
|
+
identities: kill.identities.length,
|
|
1160
|
+
unreadable: kill.identities.filter((i) => i.startedAtTicks < 0).length,
|
|
1161
|
+
});
|
|
1162
|
+
}
|
|
1163
|
+
if (running.stall)
|
|
1164
|
+
running.stall.kill = null;
|
|
1165
|
+
}
|
|
1166
|
+
/** An allocator pass is in flight — the next one waits rather than overlaps. */
|
|
1167
|
+
allocating = false;
|
|
1168
|
+
/** The last pass's machine-wide numbers, for the card and the session banner. */
|
|
1169
|
+
lastAllocation = null;
|
|
1170
|
+
/**
|
|
1171
|
+
* The machine the last allocation was computed for, kept so the card's own
|
|
1172
|
+
* number can be derived without measuring the machine a SECOND time (#403).
|
|
1173
|
+
*/
|
|
1174
|
+
lastAllocatorMachine = null;
|
|
1175
|
+
/** The last frame the API actually took from us, or null for «nothing yet». */
|
|
1176
|
+
lastPublishedLimits = null;
|
|
1177
|
+
/** When that frame went out, by this machine's clock. `0` = never. */
|
|
1178
|
+
lastLimitsSentAt = 0;
|
|
1179
|
+
/**
|
|
1180
|
+
* Decide, and write, how much memory each live session may use (#398 S3).
|
|
1181
|
+
*
|
|
1182
|
+
* Runs on the 30 s watch tick and on the stall detector's signal, and nowhere
|
|
1183
|
+
* else: `MemoryHigh` must have exactly one implementation of «what should this
|
|
1184
|
+
* be», or the two writers disagree and the limit becomes whichever ran last.
|
|
1185
|
+
*
|
|
1186
|
+
* Everything that could be wrong about the machine is read fresh here rather
|
|
1187
|
+
* than taken from the daemon-start probe: the pot moves when the hourly
|
|
1188
|
+
* re-measure rewrites the slice, the collective brake decides whether
|
|
1189
|
+
* overselling is even legal, and a neighbouring application growing is
|
|
1190
|
+
* precisely the case #398 was opened about.
|
|
1191
|
+
*/
|
|
1192
|
+
async runAllocator(now) {
|
|
1193
|
+
if (this.allocating)
|
|
1194
|
+
return;
|
|
1195
|
+
const facts = sessionCage();
|
|
1196
|
+
/**
|
|
1197
|
+
* No explicit «is this machine caged» gate, and it stays that way (#398 S7,
|
|
1198
|
+
* B5 — reviewed, tried, and rejected on 10.09.2026).
|
|
1199
|
+
*
|
|
1200
|
+
* The finding is real as far as it goes: `sessionScopeUnitOf` answers null
|
|
1201
|
+
* for sessions started on an uncaged machine, but NOT for sessions that
|
|
1202
|
+
* were already running when the machine lost its cage — for those the
|
|
1203
|
+
* register still holds a unit name, and this loop would go on writing
|
|
1204
|
+
* limits to scopes that may no longer be anybody's.
|
|
1205
|
+
*
|
|
1206
|
+
* A gate on `sessionCage().mode` is nonetheless the wrong fix, and the
|
|
1207
|
+
* suite says so out loud: every memory test here drives the allocator
|
|
1208
|
+
* through its injected readers WITHOUT probing a cage, so such a gate turns
|
|
1209
|
+
* thirty tests off rather than making them pass — which is the shape of a
|
|
1210
|
+
* change that looks green and proves nothing.
|
|
1211
|
+
*
|
|
1212
|
+
* What actually happens in the case the finding describes is small and
|
|
1213
|
+
* self-correcting: the write fails, `setScopeProperties` logs it and
|
|
1214
|
+
* returns false, and nothing is corrupted. Left as is, deliberately.
|
|
1215
|
+
*/
|
|
1216
|
+
const slice = (this.opts.readSliceLimits ?? readSliceLimits)();
|
|
1217
|
+
const load = (this.opts.readHostLoad ?? readHostLoad)();
|
|
1218
|
+
if (slice === null || load === null)
|
|
1219
|
+
return;
|
|
1220
|
+
const holdOf = this.opts.readScopeHold ?? readScopeHold;
|
|
1221
|
+
const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
|
|
1222
|
+
const live = [];
|
|
1223
|
+
for (const running of this.sessions.values()) {
|
|
1224
|
+
if (running.session === null)
|
|
1225
|
+
continue;
|
|
1226
|
+
const unit = unitOf(running.descriptor.id);
|
|
1227
|
+
if (unit === null)
|
|
1228
|
+
continue;
|
|
1229
|
+
const hold = holdOf(unit);
|
|
1230
|
+
if (hold === null)
|
|
1231
|
+
continue;
|
|
1232
|
+
live.push({ running, unit, hold });
|
|
1233
|
+
}
|
|
1234
|
+
// Computed even when nobody is running, and that is not a waste: half of
|
|
1235
|
+
// what the card shows is about the MACHINE — the live pot, what is held by
|
|
1236
|
+
// things that are not DevBridge, how many guarantees would fit — and an idle
|
|
1237
|
+
// machine has to be able to say it too. With no sessions the loop below
|
|
1238
|
+
// simply has nothing to write.
|
|
1239
|
+
const machine = this.allocatorMachine(slice, load.memAvailableBytes, facts.swapMaxBytes ?? 0);
|
|
1240
|
+
const allocation = allocateSessionMemory(machine, live.map(({ running, hold }) => ({
|
|
1241
|
+
id: running.descriptor.id,
|
|
1242
|
+
holdBytes: hold.holdBytes,
|
|
1243
|
+
currentBytes: hold.currentBytes,
|
|
1244
|
+
})), this.opts.memoryKnobs ?? {});
|
|
1245
|
+
this.lastAllocation = allocation;
|
|
1246
|
+
this.lastAllocatorMachine = machine;
|
|
1247
|
+
this.allocating = true;
|
|
1248
|
+
try {
|
|
1249
|
+
const write = this.opts.setScopeProperties ?? setScopeProperties;
|
|
1250
|
+
const readStatus = this.opts.readScopeMemoryStatus ?? readScopeMemoryStatus;
|
|
1251
|
+
for (const { running, unit, hold } of live) {
|
|
1252
|
+
const wanted = allocation.sessions.find((s) => s.id === running.descriptor.id);
|
|
1253
|
+
if (!wanted)
|
|
1254
|
+
continue;
|
|
1255
|
+
/**
|
|
1256
|
+
* Reconcile before planning (work 10 of the stage).
|
|
1257
|
+
*
|
|
1258
|
+
* What this supervisor believes it wrote and what the scope actually
|
|
1259
|
+
* holds are two different facts, and they have come apart before — a
|
|
1260
|
+
* drop-in that never applied while the card said «capped». The cgroup is
|
|
1261
|
+
* the authority, and reading it costs nothing here: the status is two
|
|
1262
|
+
* small files the watch already reads.
|
|
1263
|
+
*/
|
|
1264
|
+
const inForce = readStatus(unit);
|
|
1265
|
+
let written = running.limits ?? null;
|
|
1266
|
+
if (inForce !== null && written !== null) {
|
|
1267
|
+
const realBrake = inForce.highBytes;
|
|
1268
|
+
const realWall = inForce.maxBytes;
|
|
1269
|
+
if ((realBrake !== null && realBrake !== written.brakeBytes) ||
|
|
1270
|
+
(realWall !== null && realWall !== written.wallBytes)) {
|
|
1271
|
+
log.warn('session memory: the scope does not hold what this runner wrote', {
|
|
1272
|
+
sessionId: running.descriptor.id,
|
|
1273
|
+
believedBrakeMB: Math.round(written.brakeBytes / (1024 * 1024)),
|
|
1274
|
+
actualBrakeMB: realBrake === null ? null : Math.round(realBrake / (1024 * 1024)),
|
|
1275
|
+
believedWallMB: Math.round(written.wallBytes / (1024 * 1024)),
|
|
1276
|
+
actualWallMB: realWall === null ? null : Math.round(realWall / (1024 * 1024)),
|
|
1277
|
+
});
|
|
1278
|
+
written = {
|
|
1279
|
+
...written,
|
|
1280
|
+
...(realBrake === null ? {} : { brakeBytes: realBrake }),
|
|
1281
|
+
...(realWall === null ? {} : { wallBytes: realWall }),
|
|
1282
|
+
};
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
const move = planLimitMove(written, wanted, hold.holdBytes, facts.memoryLowFlag);
|
|
1286
|
+
if (move.properties.length > 0) {
|
|
1287
|
+
if (!(await write(unit, move.properties)))
|
|
1288
|
+
continue;
|
|
1289
|
+
if (move.lowered) {
|
|
1290
|
+
/**
|
|
1291
|
+
* We just narrowed this session's brake, and that manufactures the
|
|
1292
|
+
* exact signal the stall detector looks for: measured on this host,
|
|
1293
|
+
* a process honestly writing at 6.4 MB/s stopped dead the instant
|
|
1294
|
+
* its `MemoryHigh` was narrowed on the live scope. Without this the
|
|
1295
|
+
* mechanism would catch its own edit and stop a command for it.
|
|
1296
|
+
*/
|
|
1297
|
+
const stall = running.stall;
|
|
1298
|
+
if (stall) {
|
|
1299
|
+
stall.detector = {
|
|
1300
|
+
...stall.detector,
|
|
1301
|
+
mutedUntil: now + STALL_MUTE_AFTER_LOWER_MS,
|
|
1302
|
+
hotWindows: 0,
|
|
1303
|
+
stalledSince: null,
|
|
1304
|
+
};
|
|
1305
|
+
stall.remainingMs = null;
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
781
1308
|
}
|
|
1309
|
+
running.limits = {
|
|
1310
|
+
brakeBytes: move.brakeBytes,
|
|
1311
|
+
wallBytes: move.wallBytes,
|
|
1312
|
+
guaranteedBytes: move.guaranteedBytes,
|
|
1313
|
+
swapBytes: move.swapBytes,
|
|
1314
|
+
holdBytes: move.holdBytes,
|
|
1315
|
+
shrinkTicks: move.shrinkTicks,
|
|
1316
|
+
};
|
|
782
1317
|
}
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
1318
|
+
}
|
|
1319
|
+
finally {
|
|
1320
|
+
this.allocating = false;
|
|
1321
|
+
}
|
|
1322
|
+
// After the writes, never before: the frame reports what is IN FORCE, and
|
|
1323
|
+
// reporting an intention that then failed to apply is how a card starts
|
|
1324
|
+
// lying about a machine.
|
|
1325
|
+
this.publishSessionLimits(now, allocation);
|
|
1326
|
+
}
|
|
1327
|
+
/** The machine as the allocator needs to see it, from live readings only. */
|
|
1328
|
+
allocatorMachine(slice, availableBytes, swapBytes) {
|
|
1329
|
+
const totalBytes = os.totalmem();
|
|
1330
|
+
return {
|
|
1331
|
+
potBytes: slice.potBytes,
|
|
1332
|
+
availableBytes,
|
|
1333
|
+
totalBytes,
|
|
1334
|
+
reserveBytes: machineReserveBytes(totalBytes),
|
|
1335
|
+
oomContinue: sessionCage().oomContinue,
|
|
1336
|
+
// The hard invariant: overselling the brakes is legal only while the
|
|
1337
|
+
// slice carries a live collective brake. Read back off cgroupfs, never
|
|
1338
|
+
// assumed from what we wrote to the drop-in.
|
|
1339
|
+
collectiveBrake: slice.collectiveBrakeBytes !== null,
|
|
1340
|
+
swapBytes,
|
|
1341
|
+
};
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* What a session STARTING right now should be given (#398 S3, work 9).
|
|
1345
|
+
*
|
|
1346
|
+
* The allocator is asked to compute as though this session already existed,
|
|
1347
|
+
* so a session born at 03:00 gets the number its first tick will confirm
|
|
1348
|
+
* rather than the one the daemon measured whenever it last started.
|
|
1349
|
+
*/
|
|
1350
|
+
ladderForNewSession(id) {
|
|
1351
|
+
/**
|
|
1352
|
+
* A session that ALREADY exists is answered with what is in force for it,
|
|
1353
|
+
* not with what a newcomer would get.
|
|
1354
|
+
*
|
|
1355
|
+
* Found by the independent review of 10.09.2026, and it was not only the
|
|
1356
|
+
* cage that asked: `policyContextFor` calls `sessionMemoryFor` on every
|
|
1357
|
+
* Bash tool call, so the heap gate was judging commands against a ceiling
|
|
1358
|
+
* computed as though the asking session did not exist — which drops its own
|
|
1359
|
+
* `memory.current` out of the live pot and makes the number much smaller
|
|
1360
|
+
* exactly while the session is busy. The gate would then refuse a heap the
|
|
1361
|
+
* cage would have allowed.
|
|
1362
|
+
*
|
|
1363
|
+
* The newcomer arithmetic below is for the one caller that really has no
|
|
1364
|
+
* session yet: `cageSpawn`, at the moment of the start.
|
|
1365
|
+
*/
|
|
1366
|
+
const inForce = this.sessions.get(id)?.limits ?? null;
|
|
1367
|
+
if (inForce) {
|
|
1368
|
+
return {
|
|
1369
|
+
highBytes: inForce.brakeBytes,
|
|
1370
|
+
maxBytes: inForce.wallBytes,
|
|
1371
|
+
swapBytes: inForce.swapBytes,
|
|
1372
|
+
guaranteedBytes: inForce.guaranteedBytes,
|
|
791
1373
|
};
|
|
792
1374
|
}
|
|
1375
|
+
// Reached only from `cageSpawn`, which has already established that this
|
|
1376
|
+
// machine has a cage — see `runAllocator` for why there is no second gate.
|
|
1377
|
+
const facts = sessionCage();
|
|
1378
|
+
const slice = (this.opts.readSliceLimits ?? readSliceLimits)();
|
|
1379
|
+
const load = (this.opts.readHostLoad ?? readHostLoad)();
|
|
1380
|
+
if (slice === null || load === null)
|
|
1381
|
+
return null;
|
|
1382
|
+
const holdOf = this.opts.readScopeHold ?? readScopeHold;
|
|
1383
|
+
const unitOf = this.opts.sessionScopeUnitOf ?? sessionScopeUnitOf;
|
|
1384
|
+
const sessions = [];
|
|
1385
|
+
for (const running of this.sessions.values()) {
|
|
1386
|
+
if (running.session === null || running.descriptor.id === id)
|
|
1387
|
+
continue;
|
|
1388
|
+
const unit = unitOf(running.descriptor.id);
|
|
1389
|
+
const hold = unit === null ? null : holdOf(unit);
|
|
1390
|
+
if (hold === null)
|
|
1391
|
+
continue;
|
|
1392
|
+
sessions.push({
|
|
1393
|
+
id: running.descriptor.id,
|
|
1394
|
+
holdBytes: hold.holdBytes,
|
|
1395
|
+
currentBytes: hold.currentBytes,
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
// The starting session holds nothing yet, and saying so is the point: it
|
|
1399
|
+
// must not be given the pot as though it already had it.
|
|
1400
|
+
sessions.push({ id, holdBytes: 0, currentBytes: 0 });
|
|
1401
|
+
const allocation = allocateSessionMemory(this.allocatorMachine(slice, load.memAvailableBytes, facts.swapMaxBytes ?? 0), sessions, this.opts.memoryKnobs ?? {});
|
|
1402
|
+
const mine = allocation.sessions.find((s) => s.id === id);
|
|
1403
|
+
if (!mine)
|
|
1404
|
+
return null;
|
|
1405
|
+
return {
|
|
1406
|
+
highBytes: mine.brakeBytes,
|
|
1407
|
+
maxBytes: mine.wallBytes,
|
|
1408
|
+
swapBytes: mine.swapBytes,
|
|
1409
|
+
guaranteedBytes: mine.guaranteedBytes,
|
|
1410
|
+
};
|
|
1411
|
+
}
|
|
1412
|
+
/**
|
|
1413
|
+
* Tell the API how this machine is dividing its memory — when it moved (#398 S4).
|
|
1414
|
+
*
|
|
1415
|
+
* Built from the pass the allocator has just finished rather than measured
|
|
1416
|
+
* again: two readings of a moving machine taken a moment apart would let the
|
|
1417
|
+
* card and the scope disagree about the same session, and «the card said 4 GB
|
|
1418
|
+
* and the cgroup said 2» is the exact shape of the bug that made
|
|
1419
|
+
* `limitsCurrent` untrustworthy once already.
|
|
1420
|
+
*
|
|
1421
|
+
* Recorded ONLY when the socket took it, same as `publishHostLoad`: a frame
|
|
1422
|
+
* dropped by a dead socket must not be remembered as sent.
|
|
1423
|
+
*/
|
|
1424
|
+
/**
|
|
1425
|
+
* The number on the machine card: what ONE session gets when the machine is
|
|
1426
|
+
* free (decision D4 of `runner-cage-authority.md`, 10.09.2026).
|
|
1427
|
+
*
|
|
1428
|
+
* The card used to carry the ceiling a newcomer would get RIGHT NOW, from the
|
|
1429
|
+
* current crowd — which is honest per second and useless per person: on a
|
|
1430
|
+
* 12 GiB machine with two sessions of 3 GiB it read ~6 GiB, it moved whenever
|
|
1431
|
+
* a neighbour started or stopped, and the sentence beside it said «up to X on
|
|
1432
|
+
* an idle machine», which is a different number entirely. A card is read to
|
|
1433
|
+
* answer «what does this machine give a session», and that question has a
|
|
1434
|
+
* still answer.
|
|
1435
|
+
*
|
|
1436
|
+
* Computed, not measured: the pot and the owner's knobs are all it takes, so
|
|
1437
|
+
* this makes no second reading of a moving machine — the one the shape of
|
|
1438
|
+
* `publishSessionLimits` warns against in its own header.
|
|
1439
|
+
*/
|
|
1440
|
+
idleSessionCeiling(allocation) {
|
|
1441
|
+
const machine = this.lastAllocatorMachine;
|
|
1442
|
+
// No pot means no slice ceiling to be idle against, and then the pool the
|
|
1443
|
+
// allocation already used is the honest answer.
|
|
1444
|
+
if (machine === null || machine.potBytes === null)
|
|
1445
|
+
return Math.floor(allocation.poolBytes);
|
|
1446
|
+
const idle = allocateSessionMemory(
|
|
1447
|
+
// «Free» is `available` big enough that the pot itself is the binding
|
|
1448
|
+
// limit: the pool is `min(pot, available − reserve)` floored at the
|
|
1449
|
+
// guarantee, so anything at or above `pot + reserve` gives a full pot.
|
|
1450
|
+
{ ...machine, availableBytes: machine.potBytes + machine.reserveBytes }, [{ id: '__card__', holdBytes: 0, currentBytes: 0 }], this.opts.memoryKnobs ?? {});
|
|
1451
|
+
const only = idle.sessions[0];
|
|
1452
|
+
return Math.floor(only?.brakeBytes ?? allocation.poolBytes);
|
|
1453
|
+
}
|
|
1454
|
+
publishSessionLimits(now, allocation) {
|
|
1455
|
+
if (allocation === null)
|
|
1456
|
+
return;
|
|
1457
|
+
const load = (this.opts.readHostLoad ?? readHostLoad)();
|
|
1458
|
+
if (load === null)
|
|
1459
|
+
return;
|
|
1460
|
+
/**
|
|
1461
|
+
* Stamped HERE, not from the tick's own clock.
|
|
1462
|
+
*
|
|
1463
|
+
* One pass of the stall watch can publish twice — once from the allocator
|
|
1464
|
+
* the stalled session asked for, once from the tick — and both were being
|
|
1465
|
+
* stamped with the single `now` the pass began with. The gateway's ordering
|
|
1466
|
+
* guard is `measuredAt <= last`, so the second frame was dropped while this
|
|
1467
|
+
* side recorded it as sent, and the difference stayed invisible until the
|
|
1468
|
+
* heartbeat a minute later. `host_load` has never had the bug because
|
|
1469
|
+
* `readHostLoad` stamps inside itself, once per publish. Found by the
|
|
1470
|
+
* independent review of 10.09.2026.
|
|
1471
|
+
*/
|
|
1472
|
+
const measuredAt = Math.max(now, this.lastLimitsSentAt + 1);
|
|
1473
|
+
// What is held on this machine by everything that is not us. The card had
|
|
1474
|
+
// nowhere to get this, and it is the main question of #398.
|
|
1475
|
+
const otherHeld = Math.max(0, os.totalmem() - load.memAvailableBytes - allocation.heldBytes);
|
|
1476
|
+
const frame = {
|
|
1477
|
+
at: new Date(measuredAt).toISOString(),
|
|
1478
|
+
poolBytes: Math.floor(allocation.poolBytes),
|
|
1479
|
+
heldBytes: Math.floor(allocation.heldBytes),
|
|
1480
|
+
otherHeldBytes: Math.floor(otherHeld),
|
|
1481
|
+
/**
|
|
1482
|
+
* What is PROMISED, and only where the machine can keep the promise
|
|
1483
|
+
* (#398 S7, B3): zero when the slice carries no `memory.low` of its own,
|
|
1484
|
+
* because then nothing is reserved for anybody and the card must not say
|
|
1485
|
+
* otherwise. Measured on this host: the file held 0 while the chip read
|
|
1486
|
+
* «2.0 GB guaranteed».
|
|
1487
|
+
*/
|
|
1488
|
+
guaranteeBytes: ((this.opts.readSliceLimits ?? readSliceLimits)()?.guaranteeBytes ?? 0) > 0
|
|
1489
|
+
? (this.opts.memoryKnobs?.guaranteeBytes ?? SESSION_GUARANTEE_BYTES)
|
|
1490
|
+
: 0,
|
|
1491
|
+
// Asked of the allocator, not derived from the pot: only it knows the
|
|
1492
|
+
// owner's knobs, the band this machine uses and the single-session branch.
|
|
1493
|
+
// On an IDLE machine, though — see `idleSessionCeiling`.
|
|
1494
|
+
sessionCeilingBytes: this.idleSessionCeiling(allocation),
|
|
1495
|
+
guaranteesFit: allocation.guaranteesFit,
|
|
1496
|
+
seats: this.maxSessions,
|
|
1497
|
+
swapPerSessionBytes: sessionCage().swapMaxBytes ?? 0,
|
|
1498
|
+
collectiveBrake: !allocation.conservative,
|
|
1499
|
+
conservative: allocation.conservative,
|
|
1500
|
+
hungry: allocation.hungry,
|
|
1501
|
+
sessions: allocation.sessions.map((s) => {
|
|
1502
|
+
const running = this.sessions.get(s.id);
|
|
1503
|
+
/**
|
|
1504
|
+
* What is IN FORCE, not what the allocator wanted.
|
|
1505
|
+
*
|
|
1506
|
+
* `running.limits` is what systemd was actually told; the allocation is
|
|
1507
|
+
* an intention, and the two come apart every time a write fails or a
|
|
1508
|
+
* movement rule refuses to lower something. A card drawn from the
|
|
1509
|
+
* intention would claim a session had a share no scope has ever held —
|
|
1510
|
+
* the kind of lie that made `limitsCurrent` untrustworthy once already.
|
|
1511
|
+
* Found by the independent review of 10.09.2026.
|
|
1512
|
+
*/
|
|
1513
|
+
const written = running?.limits ?? null;
|
|
1514
|
+
return {
|
|
1515
|
+
sessionId: s.id,
|
|
1516
|
+
guaranteedBytes: Math.floor(written?.guaranteedBytes ?? s.guaranteedBytes),
|
|
1517
|
+
brakeBytes: Math.floor(written?.brakeBytes ?? s.brakeBytes),
|
|
1518
|
+
wallBytes: Math.floor(written?.wallBytes ?? s.wallBytes),
|
|
1519
|
+
holdBytes: Math.floor(written?.holdBytes ?? 0),
|
|
1520
|
+
tight: s.tight,
|
|
1521
|
+
stallRemainingMs: running?.stall?.remainingMs ?? null,
|
|
1522
|
+
reason: running?.stall?.reason ?? null,
|
|
1523
|
+
};
|
|
1524
|
+
}),
|
|
1525
|
+
};
|
|
1526
|
+
const heartbeatDue = sessionLimitsHeartbeatDue(now - this.lastLimitsSentAt, this.limitsHeartbeatMs);
|
|
1527
|
+
if (!heartbeatDue && !sessionLimitsChangedEnough(this.lastPublishedLimits, frame))
|
|
1528
|
+
return;
|
|
1529
|
+
if (this.ws.send({ type: 'session_limits', ...frame })) {
|
|
1530
|
+
this.lastPublishedLimits = frame;
|
|
1531
|
+
this.lastLimitsSentAt = measuredAt;
|
|
1532
|
+
}
|
|
793
1533
|
}
|
|
794
1534
|
publishHostLoad() {
|
|
795
1535
|
const sample = (this.opts.readHostLoad ?? readHostLoad)();
|
|
@@ -821,10 +1561,16 @@ export class Supervisor {
|
|
|
821
1561
|
// is right only after five minutes of silence is not right.
|
|
822
1562
|
this.lastPublishedHostLoad = null;
|
|
823
1563
|
this.lastHostLoadSentAt = 0;
|
|
1564
|
+
// …and for the memory numbers, which the API keeps the same way (#398 S4).
|
|
1565
|
+
this.lastPublishedLimits = null;
|
|
1566
|
+
this.lastLimitsSentAt = 0;
|
|
824
1567
|
this.setMaxSessions(frame.maxSessions);
|
|
825
1568
|
await this.reconcile(frame.sessions);
|
|
826
1569
|
this.publishSlots();
|
|
827
1570
|
this.publishHostLoad();
|
|
1571
|
+
// Deliberately not awaited: this frame also carries the session
|
|
1572
|
+
// reconciliation, and a slow cgroup read must not hold it up.
|
|
1573
|
+
void this.runAllocator(Date.now());
|
|
828
1574
|
// A build that finished while the socket was down has its verdict
|
|
829
1575
|
// sitting on disk. This is the moment it can be delivered.
|
|
830
1576
|
this.flushVerifyReports();
|
|
@@ -1017,6 +1763,9 @@ export class Supervisor {
|
|
|
1017
1763
|
levels: freshLevels(),
|
|
1018
1764
|
stopRequested: false,
|
|
1019
1765
|
parkRequested: false,
|
|
1766
|
+
// Set here rather than at the first `await`: frames are dispatched
|
|
1767
|
+
// concurrently, so anything assigned later is already too late (#401).
|
|
1768
|
+
starting: true,
|
|
1020
1769
|
pendingMessages: [],
|
|
1021
1770
|
activeMs: descriptor.activeMsBase,
|
|
1022
1771
|
extraBudgetMinutes: descriptor.extraBudgetMinutes,
|
|
@@ -1070,71 +1819,84 @@ export class Supervisor {
|
|
|
1070
1819
|
}
|
|
1071
1820
|
}
|
|
1072
1821
|
try {
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
}
|
|
1087
|
-
// A session that is already past STARTING (re-sent because this runner
|
|
1088
|
-
// asked for it) resumes on the next message instead of replaying its
|
|
1089
|
-
// original prompt.
|
|
1090
|
-
if (descriptor.status === 'STARTING') {
|
|
1091
|
-
// Ticket #126: the point before the agent has touched anything. It is
|
|
1092
|
-
// anchored to seq 0 — the synthetic opening bubble the dashboard puts in
|
|
1093
|
-
// front of every feed — so "put it all back" is reachable from the very
|
|
1094
|
-
// first thing on the page.
|
|
1095
|
-
await this.captureCheckpoint(running, 'TURN', 0);
|
|
1096
|
-
if (this.isStale(running))
|
|
1097
|
-
return;
|
|
1098
|
-
// A launch that failed has already said so and reported a status the
|
|
1099
|
-
// person can act on. Flushing the queue into it would only walk the same
|
|
1100
|
-
// failure again, once per waiting message (ticket #225).
|
|
1101
|
-
if (!this.launchAgent(running, composeInitialPrompt(descriptor), null).ok)
|
|
1822
|
+
try {
|
|
1823
|
+
const prepared = await this.prepareWorkspace(descriptor);
|
|
1824
|
+
running.branch = prepared.branch;
|
|
1825
|
+
running.worktreePath = prepared.worktreePath;
|
|
1826
|
+
// Only when WE created the branch: the fork point is a fact the API can
|
|
1827
|
+
// learn nowhere else, and it pins the first answer it gets.
|
|
1828
|
+
if (prepared.baseSha)
|
|
1829
|
+
running.baseSha = prepared.baseSha;
|
|
1830
|
+
if (prepared.baseBranch)
|
|
1831
|
+
running.baseBranch = prepared.baseBranch;
|
|
1832
|
+
}
|
|
1833
|
+
catch (error) {
|
|
1834
|
+
this.workspacePrepareFailed(running, error, 'Failed to prepare git worktree');
|
|
1102
1835
|
return;
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
//
|
|
1109
|
-
//
|
|
1110
|
-
//
|
|
1111
|
-
//
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
//
|
|
1116
|
-
//
|
|
1117
|
-
//
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1836
|
+
}
|
|
1837
|
+
// A session that is already past STARTING (re-sent because this runner
|
|
1838
|
+
// asked for it) resumes on the next message instead of replaying its
|
|
1839
|
+
// original prompt.
|
|
1840
|
+
if (descriptor.status === 'STARTING') {
|
|
1841
|
+
// Ticket #126: the point before the agent has touched anything. It is
|
|
1842
|
+
// anchored to seq 0 — the synthetic opening bubble the dashboard puts in
|
|
1843
|
+
// front of every feed — so "put it all back" is reachable from the very
|
|
1844
|
+
// first thing on the page.
|
|
1845
|
+
await this.captureCheckpoint(running, 'TURN', 0);
|
|
1846
|
+
if (this.isStale(running))
|
|
1847
|
+
return;
|
|
1848
|
+
// A launch that failed has already said so and reported a status the
|
|
1849
|
+
// person can act on. Flushing the queue into it would only walk the same
|
|
1850
|
+
// failure again, once per waiting message (ticket #225).
|
|
1851
|
+
if (!this.launchAgent(running, composeInitialPrompt(descriptor), null).ok)
|
|
1852
|
+
return;
|
|
1853
|
+
}
|
|
1854
|
+
else {
|
|
1855
|
+
if (descriptor.epoch > 0) {
|
|
1856
|
+
// The API owns the resume transition; the feed marker has to come from
|
|
1857
|
+
// here because the runner is the only writer of the event seq.
|
|
1858
|
+
//
|
|
1859
|
+
// Ticket #177: which of the two sentences is true depends on whether
|
|
1860
|
+
// there is a conversation to go back to. `providerSessionId` is the
|
|
1861
|
+
// agent's own name for it, and it is what the next launch hands to
|
|
1862
|
+
// `--resume` / `thread/resume`. Without it the next process starts the
|
|
1863
|
+
// conversation over — which is a real loss, and promising «continue
|
|
1864
|
+
// where the agent left off» there is the one thing the feed must not do.
|
|
1865
|
+
// It happens for real: a process that dies before it reports its session
|
|
1866
|
+
// id (the SIGABRT this ticket came from) leaves the row with none.
|
|
1867
|
+
//
|
|
1868
|
+
// Ticket #370: the sentence about the agent's MEMORY waits for proof.
|
|
1869
|
+
// It used to be written here, on the strength of a stored id and before
|
|
1870
|
+
// the CLI had been asked anything — and on Athanor it appeared sixty
|
|
1871
|
+
// seconds before `thread/resume` timed out and took the session with it.
|
|
1872
|
+
// What is honest at this moment is that the runner is back; whether the
|
|
1873
|
+
// conversation reopens is answered by the process, in `provider_session`.
|
|
1874
|
+
if (descriptor.providerSessionId)
|
|
1875
|
+
running.resumeClaimPending = true;
|
|
1876
|
+
this.sendEvent(running, 'system_note', {
|
|
1877
|
+
text: descriptor.providerSessionId
|
|
1878
|
+
? 'Session resumed — send a message and the agent picks its conversation back up.'
|
|
1879
|
+
: 'Session resumed, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
|
|
1880
|
+
});
|
|
1881
|
+
}
|
|
1882
|
+
running.lastReported = descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT';
|
|
1883
|
+
this.reportStatus(descriptor.id, running.lastReported, {
|
|
1884
|
+
branch: running.branch,
|
|
1885
|
+
worktreePath: running.worktreePath,
|
|
1886
|
+
activeMs: running.activeMs,
|
|
1130
1887
|
});
|
|
1131
1888
|
}
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1889
|
+
}
|
|
1890
|
+
finally {
|
|
1891
|
+
/**
|
|
1892
|
+
* The startup gate opens again on EVERY exit, including the failed ones.
|
|
1893
|
+
*
|
|
1894
|
+
* #401: this flag is what makes `acceptUserMessage` hold a message until
|
|
1895
|
+
* the first process exists. Left standing by an early return it would
|
|
1896
|
+
* hold the person's words in the queue for the life of the session, so
|
|
1897
|
+
* it is cleared here rather than at each `return` above.
|
|
1898
|
+
*/
|
|
1899
|
+
running.starting = false;
|
|
1138
1900
|
}
|
|
1139
1901
|
this.flushPendingMessages(running);
|
|
1140
1902
|
}
|
|
@@ -1227,6 +1989,40 @@ export class Supervisor {
|
|
|
1227
1989
|
const adapter = this.opts.adapters[descriptor.agent];
|
|
1228
1990
|
if (!adapter || !running.worktreePath || !running.branch)
|
|
1229
1991
|
return LAUNCH_REFUSED;
|
|
1992
|
+
/**
|
|
1993
|
+
* One process per session, enforced where the slot is written (#401).
|
|
1994
|
+
*
|
|
1995
|
+
* `running.session` is a single field, and until this check it was assigned
|
|
1996
|
+
* unconditionally: a second launch overwrote the reference to a process
|
|
1997
|
+
* that was still running, and nobody held it any more. Everything the
|
|
1998
|
+
* session then did — a message, an answer to a question, `stop`, the memory
|
|
1999
|
+
* watch — addressed the survivor, while the orphan kept working, kept its
|
|
2000
|
+
* own cgroup and outlived even the service restart.
|
|
2001
|
+
*
|
|
2002
|
+
* Safe as an entry check because this method is SYNCHRONOUS from here to
|
|
2003
|
+
* the assignment: nothing yields in between, so «free now» is still true
|
|
2004
|
+
* when the slot is filled.
|
|
2005
|
+
*
|
|
2006
|
+
* Every legitimate caller reaches this with an empty slot, by one of three
|
|
2007
|
+
* arguments: the four one-shot relaunches run in the tail of `pumpEvents`,
|
|
2008
|
+
* where the slot was emptied because the process is gone; `deliverMessage`
|
|
2009
|
+
* only gets here past its own `running.session` test and is serialised by
|
|
2010
|
+
* the session's delivery chain; and the two opening doors hold `starting`
|
|
2011
|
+
* for exactly as long as it takes them to decide, which is what keeps the
|
|
2012
|
+
* queue from starting an agent out from under them.
|
|
2013
|
+
*
|
|
2014
|
+
* A refusal here is therefore not a race being caught — it is a bug, and
|
|
2015
|
+
* the log line says so. Read it together with `starting`: this guard alone
|
|
2016
|
+
* would turn «two processes» into «the session's own task never sent», the
|
|
2017
|
+
* quieter of the two failures.
|
|
2018
|
+
*/
|
|
2019
|
+
if (running.session) {
|
|
2020
|
+
log.error('supervisor: refusing to launch a second agent for one session', {
|
|
2021
|
+
sessionId: descriptor.id,
|
|
2022
|
+
processSeq: running.processSeq,
|
|
2023
|
+
});
|
|
2024
|
+
return LAUNCH_REFUSED;
|
|
2025
|
+
}
|
|
1230
2026
|
// An exhausted USD budget must not relaunch $0.01-floor processes (QA-96 F4).
|
|
1231
2027
|
// Codex reports no cost at all, so its costUsd never leaves 0 — gating on it
|
|
1232
2028
|
// would be a limit that can never fire while the UI shows $0.00. Those
|
|
@@ -1693,8 +2489,35 @@ export class Supervisor {
|
|
|
1693
2489
|
});
|
|
1694
2490
|
}
|
|
1695
2491
|
// Stream ended — the agent process is gone.
|
|
1696
|
-
if (running.session !== session)
|
|
1697
|
-
|
|
2492
|
+
if (running.session !== session) {
|
|
2493
|
+
/**
|
|
2494
|
+
* Somebody else holds the slot while this process's stream ends.
|
|
2495
|
+
*
|
|
2496
|
+
* Since #401 the launch REFUSES to overwrite a live process, and the slot
|
|
2497
|
+
* is emptied only where the process is known to be gone — the line below
|
|
2498
|
+
* and `launchCrashed`. So this branch no longer means «a live agent was
|
|
2499
|
+
* orphaned»; it means the entry was rebuilt under this stream, and the
|
|
2500
|
+
* cleanup below belongs to whoever holds it now. Said out loud rather
|
|
2501
|
+
* than returned in silence: the old comment here called the case
|
|
2502
|
+
* impossible, and it was the shape four production sessions took.
|
|
2503
|
+
*/
|
|
2504
|
+
log.warn('supervisor: event stream ended for a process that no longer holds the slot', {
|
|
2505
|
+
sessionId: descriptor.id,
|
|
2506
|
+
processSeq: running.processSeq,
|
|
2507
|
+
});
|
|
2508
|
+
return;
|
|
2509
|
+
}
|
|
2510
|
+
/**
|
|
2511
|
+
* The slot describes a LIVE process, so it is emptied the moment there is
|
|
2512
|
+
* none — here, and not by whoever launches next (#401).
|
|
2513
|
+
*
|
|
2514
|
+
* Three of the one-shot relaunches below (`freshRetry`, `rewindRetry`,
|
|
2515
|
+
* `authRetry`) used to call `launchAgent` with the dead adapter still in
|
|
2516
|
+
* the slot and rely on being overwritten. That overwrite is exactly what
|
|
2517
|
+
* the launch now refuses, so without this line the recovery paths would
|
|
2518
|
+
* refuse themselves. Nothing between here and them reads the field.
|
|
2519
|
+
*/
|
|
2520
|
+
running.session = null;
|
|
1698
2521
|
// Backstop for a process that died without going through `stop()` (a crash,
|
|
1699
2522
|
// an SDK error, an agent that exited mid-question): the adapter never got
|
|
1700
2523
|
// to withdraw its cards, and a card nobody can answer must not stay live.
|
|
@@ -1768,7 +2591,9 @@ export class Supervisor {
|
|
|
1768
2591
|
const { priorStatus } = running.modeRelaunch;
|
|
1769
2592
|
delete running.modeRelaunch;
|
|
1770
2593
|
running.parkRequested = false;
|
|
1771
|
-
|
|
2594
|
+
// The slot was emptied where the process was found gone (#401) — this
|
|
2595
|
+
// branch used to be the only one that remembered to do it, and the three
|
|
2596
|
+
// above did not. One place, so they cannot disagree again.
|
|
1772
2597
|
running.costBaseUsd = running.costUsd; // the next process starts from here
|
|
1773
2598
|
const mode = running.mode;
|
|
1774
2599
|
// «Your conversation is kept» is only true when there IS one to keep
|
|
@@ -1832,7 +2657,7 @@ export class Supervisor {
|
|
|
1832
2657
|
!isTerminal(running.lastReported) &&
|
|
1833
2658
|
running.descriptor.providerSessionId) {
|
|
1834
2659
|
// Idle process ended (parked or died between turns) — stay resumable.
|
|
1835
|
-
|
|
2660
|
+
// The slot itself was emptied above, where the process was found gone.
|
|
1836
2661
|
running.parkRequested = false;
|
|
1837
2662
|
/**
|
|
1838
2663
|
* The subagents died with it (QA-2026-08-16 M-5).
|
|
@@ -2653,6 +3478,8 @@ export class Supervisor {
|
|
|
2653
3478
|
return;
|
|
2654
3479
|
case 'permission_resolved':
|
|
2655
3480
|
running.openPermissions.delete(event.requestId);
|
|
3481
|
+
// …and the same for a permission card — see `tellAgent`.
|
|
3482
|
+
this.flushPendingAgentNote(running);
|
|
2656
3483
|
this.sendEvent(running, 'permission_resolved', {
|
|
2657
3484
|
requestId: event.requestId,
|
|
2658
3485
|
allow: event.allow,
|
|
@@ -2698,6 +3525,8 @@ export class Supervisor {
|
|
|
2698
3525
|
return;
|
|
2699
3526
|
case 'question_resolved':
|
|
2700
3527
|
running.openQuestions.delete(event.askId);
|
|
3528
|
+
// Anything the runner itself wanted to say while the card was open.
|
|
3529
|
+
this.flushPendingAgentNote(running);
|
|
2701
3530
|
this.sendEvent(running, 'question_resolved', {
|
|
2702
3531
|
askId: event.askId,
|
|
2703
3532
|
outcome: event.outcome,
|
|
@@ -2880,8 +3709,26 @@ export class Supervisor {
|
|
|
2880
3709
|
// words below have now been published once, and a redelivery must not
|
|
2881
3710
|
// publish them again.
|
|
2882
3711
|
running.answeredAsks.add(frame.askId);
|
|
3712
|
+
/**
|
|
3713
|
+
* The reply, whichever control the person used to give it (#401).
|
|
3714
|
+
*
|
|
3715
|
+
* `frame.text` alone was not it: that field carries the bottom input of the
|
|
3716
|
+
* card («Discuss instead»), and the dashboard sends the ORDINARY answer as
|
|
3717
|
+
* `answers` with no text at all. So a branch that rescued only the text
|
|
3718
|
+
* lost the entire reply of everyone who answered by tapping the options —
|
|
3719
|
+
* silently, because the card stayed live and said nothing. That is what
|
|
3720
|
+
* happened to the owner on 08.09.2026 and it is the second half of this
|
|
3721
|
+
* ticket: a question that could not be answered AND an answer that
|
|
3722
|
+
* vanished.
|
|
3723
|
+
*
|
|
3724
|
+
* Both halves, joined rather than one or the other: the frame allows a
|
|
3725
|
+
* person to have typed AND picked, and dropping either would be the same
|
|
3726
|
+
* defect on a narrower path. The sentence comes first because that is the
|
|
3727
|
+
* order the card puts them in.
|
|
3728
|
+
*/
|
|
3729
|
+
const rescued = [typed, answersAsMessage(frame.answers ?? [])].filter(Boolean).join('\n');
|
|
2883
3730
|
this.sendEvent(running, 'system_note', {
|
|
2884
|
-
text:
|
|
3731
|
+
text: rescued
|
|
2885
3732
|
? 'That question is no longer open — sending your reply as an ordinary message instead.'
|
|
2886
3733
|
: 'That question is no longer open — the agent has already moved on.',
|
|
2887
3734
|
});
|
|
@@ -2889,8 +3736,8 @@ export class Supervisor {
|
|
|
2889
3736
|
// restart: the card in the browser outlived the process that asked, and
|
|
2890
3737
|
// showing the user's own message in the feed while nothing receives it is
|
|
2891
3738
|
// the exact failure `acceptUserMessage` was hardened against (QA-106 M2).
|
|
2892
|
-
if (
|
|
2893
|
-
this.acceptUserMessage(frame.sessionId,
|
|
3739
|
+
if (rescued) {
|
|
3740
|
+
this.acceptUserMessage(frame.sessionId, rescued);
|
|
2894
3741
|
return 'not_open';
|
|
2895
3742
|
}
|
|
2896
3743
|
// Nothing to deliver — but the API optimistically flipped the session to
|
|
@@ -2980,11 +3827,22 @@ export class Supervisor {
|
|
|
2980
3827
|
...(attachments?.length ? { attachments } : {}),
|
|
2981
3828
|
});
|
|
2982
3829
|
const originSeq = echoed.seq;
|
|
2983
|
-
if (!running.worktreePath || Supervisor.isPaused(running)) {
|
|
3830
|
+
if (running.starting || !running.worktreePath || Supervisor.isPaused(running)) {
|
|
2984
3831
|
// Session is still being prepared — deliver after launch (QA-96 F3).
|
|
2985
3832
|
// Attachments travel as metadata and are downloaded at delivery time,
|
|
2986
3833
|
// which is the first moment the worktree is guaranteed to exist.
|
|
2987
3834
|
//
|
|
3835
|
+
// #401: `starting` is the half of that promise the condition used to be
|
|
3836
|
+
// missing. The folder is ready in milliseconds and the first process only
|
|
3837
|
+
// seconds later, so testing the folder alone let a message through into a
|
|
3838
|
+
// session that had no agent yet — and the delivery path then started one
|
|
3839
|
+
// of its own, beside the one `startSession` was about to start. Both
|
|
3840
|
+
// halves are named because they fail apart: a session past STARTING has a
|
|
3841
|
+
// folder and no `starting`, and waking it with a message is correct.
|
|
3842
|
+
//
|
|
3843
|
+
// Nothing is lost by waiting: `flushPendingMessages` runs on the way out
|
|
3844
|
+
// of `startSession`, after the launch, and delivers this queue in order.
|
|
3845
|
+
//
|
|
2988
3846
|
// Or the session is held under a clock (#196). The API refuses live
|
|
2989
3847
|
// messages for a paused session, but not every path goes through that
|
|
2990
3848
|
// check — the outbox flushes on reconnect, and the git service posts its
|
|
@@ -3452,7 +4310,8 @@ export class Supervisor {
|
|
|
3452
4310
|
// rewind resumes the session the point names rather than whichever one the
|
|
3453
4311
|
// session happens to be on now.
|
|
3454
4312
|
const anchor = this.currentAnchor(running);
|
|
3455
|
-
const
|
|
4313
|
+
const takeCheckpoint = this.opts.createCheckpoint ?? createCheckpoint;
|
|
4314
|
+
const result = await takeCheckpoint({
|
|
3456
4315
|
worktreePath,
|
|
3457
4316
|
sessionId: running.descriptor.id,
|
|
3458
4317
|
kind,
|
|
@@ -3539,6 +4398,33 @@ export class Supervisor {
|
|
|
3539
4398
|
// pause. Held work stays held until the clock is off.
|
|
3540
4399
|
if (Supervisor.isPaused(running))
|
|
3541
4400
|
return;
|
|
4401
|
+
/**
|
|
4402
|
+
* …and by the same argument, none of them knew about a session that has not
|
|
4403
|
+
* finished starting (#401, found by the independent QA of this fix).
|
|
4404
|
+
*
|
|
4405
|
+
* The queue is drained from six places, and two of them fire on somebody
|
|
4406
|
+
* else's news: `drainSessionsWaitingForCapacity` runs whenever ANY session
|
|
4407
|
+
* on this runner frees a slot, and the pause release runs when a clock comes
|
|
4408
|
+
* off. Either can land in the middle of another session's opening, where the
|
|
4409
|
+
* slot is legitimately empty — so delivery would start the agent with the
|
|
4410
|
+
* queued message, and the opening's own launch would then be refused by the
|
|
4411
|
+
* guard in `launchAgent`. The session's task would never be handed over at
|
|
4412
|
+
* all, and the only trace would be one line in the daemon's log.
|
|
4413
|
+
*
|
|
4414
|
+
* That is a QUIETER failure than the two processes this ticket started
|
|
4415
|
+
* from, so the check belongs here, at the one place the queue turns into
|
|
4416
|
+
* delivery, and not at each of the six callers.
|
|
4417
|
+
*
|
|
4418
|
+
* This does NOT make the `starting` test in `acceptUserMessage` redundant:
|
|
4419
|
+
* that one decides whether a message joins the queue at all, and without it
|
|
4420
|
+
* a message would go straight down the delivery chain and never be seen
|
|
4421
|
+
* here. Two different questions, both needed.
|
|
4422
|
+
*
|
|
4423
|
+
* The opening procedure drains its own queue on the way out, after clearing
|
|
4424
|
+
* the flag — `startSession` and the reconnect branch both do it.
|
|
4425
|
+
*/
|
|
4426
|
+
if (running.starting)
|
|
4427
|
+
return;
|
|
3542
4428
|
const pending = running.pendingMessages.splice(0);
|
|
3543
4429
|
if (pending.length === 0)
|
|
3544
4430
|
return;
|
|
@@ -4450,6 +5336,9 @@ export class Supervisor {
|
|
|
4450
5336
|
levels: freshLevels(),
|
|
4451
5337
|
stopRequested: false,
|
|
4452
5338
|
parkRequested: false,
|
|
5339
|
+
// Registered before the worktree await, so the gate has to be up
|
|
5340
|
+
// before it too (#401).
|
|
5341
|
+
starting: true,
|
|
4453
5342
|
pendingMessages: [],
|
|
4454
5343
|
// Seed from the API, not 0: a runner restart used to hand the session
|
|
4455
5344
|
// a full fresh budget silently.
|
|
@@ -4475,110 +5364,132 @@ export class Supervisor {
|
|
|
4475
5364
|
running.pendingMessages.push(...running.journal.pending());
|
|
4476
5365
|
this.sessions.set(descriptor.id, running);
|
|
4477
5366
|
try {
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
}
|
|
4498
|
-
/**
|
|
4499
|
-
* Was a turn actually in flight when the process died?
|
|
4500
|
-
*
|
|
4501
|
-
* `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
|
|
4502
|
-
* others mean the agent was already waiting for a human, and there is
|
|
4503
|
-
* nothing to continue. REVIEW is deliberately excluded — the work is
|
|
4504
|
-
* finished and waiting to be looked at.
|
|
4505
|
-
*/
|
|
4506
|
-
const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
|
|
4507
|
-
const resumeId = descriptor.providerSessionId;
|
|
4508
|
-
// Ticket #177: `resumeId` is required, not merely nice to have. Without
|
|
4509
|
-
// it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
|
|
4510
|
-
// — "continue from where you stopped, re-check what you were in the
|
|
4511
|
-
// middle of" — would be addressed to an agent that remembers none of
|
|
4512
|
-
// it. A process killed before it reported its session id (the SIGABRT
|
|
4513
|
-
// this ticket came from) leaves the row in exactly that state.
|
|
4514
|
-
// Ticket #196: a paused session is never continued automatically. The
|
|
4515
|
-
// row still says RUNNING — a pause interrupts the turn but is not a
|
|
4516
|
-
// status — so without this the reconnect would read «mid-turn» and
|
|
4517
|
-
// relaunch the agent with «continue from where you stopped», which is
|
|
4518
|
-
// the exact opposite of what the clock was set for.
|
|
4519
|
-
const willContinue = wasMidTurn &&
|
|
4520
|
-
!Supervisor.isPaused(running) &&
|
|
4521
|
-
Boolean(resumeId) &&
|
|
4522
|
-
claimAutoResume(descriptor.id);
|
|
4523
|
-
// The note stays either way (owner's call): an interruption is a fact
|
|
4524
|
-
// about the session and must not disappear just because we recovered
|
|
4525
|
-
// from it. Only the instruction at the end changes — telling someone to
|
|
4526
|
-
// send a message while the agent is already working again would be a lie.
|
|
4527
|
-
this.sendEvent(running, 'system_note', {
|
|
4528
|
-
text: willContinue
|
|
4529
|
-
? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
|
|
4530
|
-
: resumeId
|
|
4531
|
-
? 'Runner reconnected. The session was resumed — send a message to continue.'
|
|
4532
|
-
: 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
|
|
4533
|
-
});
|
|
4534
|
-
if (willContinue) {
|
|
4535
|
-
// Resumed through the PROVIDER session, so the agent keeps its whole
|
|
4536
|
-
// conversation; the prompt is only the nudge a human would otherwise
|
|
4537
|
-
// have to type. Exactly what «продолжай» did by hand — no new class of
|
|
4538
|
-
// risk, and the same ceiling protects against a crash loop doing it
|
|
4539
|
-
// forever (see `auto-resume.ts`).
|
|
4540
|
-
const continued = this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId);
|
|
4541
|
-
if (continued.ok) {
|
|
4542
|
-
this.reportStatus(descriptor.id, 'RUNNING', {});
|
|
4543
|
-
this.flushPendingMessages(running);
|
|
5367
|
+
try {
|
|
5368
|
+
// A session being restored after a runner restart already has its
|
|
5369
|
+
// branch, so the API sends `CONTINUE` — the NEW guard inside would
|
|
5370
|
+
// otherwise fire on the runner's own previous work.
|
|
5371
|
+
const prepared = await this.prepareWorkspace(descriptor);
|
|
5372
|
+
running.branch = prepared.branch;
|
|
5373
|
+
running.worktreePath = prepared.worktreePath;
|
|
5374
|
+
if (prepared.baseSha)
|
|
5375
|
+
running.baseSha = prepared.baseSha;
|
|
5376
|
+
if (prepared.baseBranch)
|
|
5377
|
+
running.baseBranch = prepared.baseBranch;
|
|
5378
|
+
}
|
|
5379
|
+
catch (error) {
|
|
5380
|
+
// The same treatment as the start door (#360). Two doors doing one
|
|
5381
|
+
// thing is how the pause bug of #373 stayed half-fixed for a
|
|
5382
|
+
// release; this one used to mask its text differently AND emit no
|
|
5383
|
+
// feed event at all, so a session that could not be restored went
|
|
5384
|
+
// FAILED with nothing to read anywhere.
|
|
5385
|
+
this.workspacePrepareFailed(running, error, 'Failed to restore session worktree');
|
|
4544
5386
|
continue;
|
|
4545
5387
|
}
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
5388
|
+
/**
|
|
5389
|
+
* Was a turn actually in flight when the process died?
|
|
5390
|
+
*
|
|
5391
|
+
* `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
|
|
5392
|
+
* others mean the agent was already waiting for a human, and there is
|
|
5393
|
+
* nothing to continue. REVIEW is deliberately excluded — the work is
|
|
5394
|
+
* finished and waiting to be looked at.
|
|
5395
|
+
*/
|
|
5396
|
+
const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
|
|
5397
|
+
const resumeId = descriptor.providerSessionId;
|
|
5398
|
+
// Ticket #177: `resumeId` is required, not merely nice to have. Without
|
|
5399
|
+
// it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
|
|
5400
|
+
// — "continue from where you stopped, re-check what you were in the
|
|
5401
|
+
// middle of" — would be addressed to an agent that remembers none of
|
|
5402
|
+
// it. A process killed before it reported its session id (the SIGABRT
|
|
5403
|
+
// this ticket came from) leaves the row in exactly that state.
|
|
5404
|
+
// Ticket #196: a paused session is never continued automatically. The
|
|
5405
|
+
// row still says RUNNING — a pause interrupts the turn but is not a
|
|
5406
|
+
// status — so without this the reconnect would read «mid-turn» and
|
|
5407
|
+
// relaunch the agent with «continue from where you stopped», which is
|
|
5408
|
+
// the exact opposite of what the clock was set for.
|
|
5409
|
+
const willContinue = wasMidTurn &&
|
|
5410
|
+
!Supervisor.isPaused(running) &&
|
|
5411
|
+
Boolean(resumeId) &&
|
|
5412
|
+
claimAutoResume(descriptor.id);
|
|
5413
|
+
// The note stays either way (owner's call): an interruption is a fact
|
|
5414
|
+
// about the session and must not disappear just because we recovered
|
|
5415
|
+
// from it. Only the instruction at the end changes — telling someone to
|
|
5416
|
+
// send a message while the agent is already working again would be a lie.
|
|
5417
|
+
this.sendEvent(running, 'system_note', {
|
|
5418
|
+
text: willContinue
|
|
5419
|
+
? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
|
|
5420
|
+
: resumeId
|
|
5421
|
+
? 'Runner reconnected. The session was resumed — send a message to continue.'
|
|
5422
|
+
: 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
|
|
5423
|
+
});
|
|
5424
|
+
if (willContinue) {
|
|
5425
|
+
// Resumed through the PROVIDER session, so the agent keeps its whole
|
|
5426
|
+
// conversation; the prompt is only the nudge a human would otherwise
|
|
5427
|
+
// have to type. Exactly what «продолжай» did by hand — no new class of
|
|
5428
|
+
// risk, and the same ceiling protects against a crash loop doing it
|
|
5429
|
+
// forever (see `auto-resume.ts`).
|
|
5430
|
+
const continued = this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId);
|
|
5431
|
+
if (continued.ok) {
|
|
5432
|
+
this.reportStatus(descriptor.id, 'RUNNING', {});
|
|
5433
|
+
// The opening is over — hand the queue back before draining it,
|
|
5434
|
+
// because `flushPendingMessages` refuses a session that is still
|
|
5435
|
+
// starting (#401). The `finally` below is the net for the exits
|
|
5436
|
+
// that never get here.
|
|
5437
|
+
running.starting = false;
|
|
5438
|
+
this.flushPendingMessages(running);
|
|
5439
|
+
continue;
|
|
5440
|
+
}
|
|
5441
|
+
// Could not start — an exhausted budget, or a launch that crashed
|
|
5442
|
+
// (ticket #225: this is the exact line the incident died on, and the
|
|
5443
|
+
// throw took the WHOLE restore loop with it). Fall through to the old
|
|
5444
|
+
// behaviour and say so honestly; a crash has already put its own
|
|
5445
|
+
// reason in the feed, so this note would only repeat it.
|
|
5446
|
+
if (continued.reason === 'refused') {
|
|
5447
|
+
this.sendEvent(running, 'system_note', {
|
|
5448
|
+
text: 'Could not continue automatically — send a message to pick the work back up.',
|
|
5449
|
+
});
|
|
5450
|
+
}
|
|
4555
5451
|
}
|
|
5452
|
+
/**
|
|
5453
|
+
* REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
|
|
5454
|
+
* mid-turn statuses are downgraded to "waiting for the user".
|
|
5455
|
+
*
|
|
5456
|
+
* REVIEW is now REPORTED rather than skipped — ticket #356, and the
|
|
5457
|
+
* news in that frame is not the status, which has not moved. It is
|
|
5458
|
+
* the ZERO riding on it.
|
|
5459
|
+
*
|
|
5460
|
+
* `running.backgroundTasks` is seeded to 0 above, because a runner
|
|
5461
|
+
* restart takes every subagent with it. But the number the API holds
|
|
5462
|
+
* is written by the runner ALONE, and it is only ever written by a
|
|
5463
|
+
* frame that carries the field — which `reportStatus` attaches only
|
|
5464
|
+
* for a tracked session. Skipping the report here left the API
|
|
5465
|
+
* believing whatever the dead process last said, for good: the badge
|
|
5466
|
+
* would keep saying «Agents», the Inbox would keep hiding the card,
|
|
5467
|
+
* and no later frame would ever correct either. `setBackgroundTasks`
|
|
5468
|
+
* cannot help — it compares against the in-memory 0 and returns
|
|
5469
|
+
* early, having nothing to announce.
|
|
5470
|
+
*
|
|
5471
|
+
* Same-status frames are legal (`canDevSessionTransition` answers
|
|
5472
|
+
* `true` for `from === to`), so this costs one no-op write and buys
|
|
5473
|
+
* back a session that would otherwise have been lost.
|
|
5474
|
+
*/
|
|
5475
|
+
this.reportStatus(descriptor.id, statusForReport(running), {});
|
|
5476
|
+
// As above: the flag comes off first, or the drain below is a no-op.
|
|
5477
|
+
running.starting = false;
|
|
5478
|
+
this.flushPendingMessages(running);
|
|
5479
|
+
}
|
|
5480
|
+
finally {
|
|
5481
|
+
/**
|
|
5482
|
+
* The same gate as the start door, for the same reason (#401).
|
|
5483
|
+
*
|
|
5484
|
+
* This branch registers the entry BEFORE awaiting the worktree —
|
|
5485
|
+
* deliberately, so a racing message is buffered rather than dropped
|
|
5486
|
+
* (QA-96 F3) — and then decides for itself whether to bring the
|
|
5487
|
+
* agent back up. A message let through in between would make that
|
|
5488
|
+
* decision instead, and the feed would go on to say the session
|
|
5489
|
+
* could not be continued automatically while it plainly was.
|
|
5490
|
+
*/
|
|
5491
|
+
running.starting = false;
|
|
4556
5492
|
}
|
|
4557
|
-
/**
|
|
4558
|
-
* REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
|
|
4559
|
-
* mid-turn statuses are downgraded to "waiting for the user".
|
|
4560
|
-
*
|
|
4561
|
-
* REVIEW is now REPORTED rather than skipped — ticket #356, and the
|
|
4562
|
-
* news in that frame is not the status, which has not moved. It is
|
|
4563
|
-
* the ZERO riding on it.
|
|
4564
|
-
*
|
|
4565
|
-
* `running.backgroundTasks` is seeded to 0 above, because a runner
|
|
4566
|
-
* restart takes every subagent with it. But the number the API holds
|
|
4567
|
-
* is written by the runner ALONE, and it is only ever written by a
|
|
4568
|
-
* frame that carries the field — which `reportStatus` attaches only
|
|
4569
|
-
* for a tracked session. Skipping the report here left the API
|
|
4570
|
-
* believing whatever the dead process last said, for good: the badge
|
|
4571
|
-
* would keep saying «Agents», the Inbox would keep hiding the card,
|
|
4572
|
-
* and no later frame would ever correct either. `setBackgroundTasks`
|
|
4573
|
-
* cannot help — it compares against the in-memory 0 and returns
|
|
4574
|
-
* early, having nothing to announce.
|
|
4575
|
-
*
|
|
4576
|
-
* Same-status frames are legal (`canDevSessionTransition` answers
|
|
4577
|
-
* `true` for `from === to`), so this costs one no-op write and buys
|
|
4578
|
-
* back a session that would otherwise have been lost.
|
|
4579
|
-
*/
|
|
4580
|
-
this.reportStatus(descriptor.id, statusForReport(running), {});
|
|
4581
|
-
this.flushPendingMessages(running);
|
|
4582
5493
|
}
|
|
4583
5494
|
else if (descriptor.status === 'WAITING_INPUT') {
|
|
4584
5495
|
// A session that never had a turn (a free session still waiting for
|
|
@@ -6079,6 +6990,8 @@ export class Supervisor {
|
|
|
6079
6990
|
shutdown() {
|
|
6080
6991
|
clearInterval(this.slotsTimer);
|
|
6081
6992
|
clearInterval(this.hostLoadTimer);
|
|
6993
|
+
clearInterval(this.stallTimer);
|
|
6994
|
+
setLiveLadderSource(null);
|
|
6082
6995
|
clearInterval(this.agentVersionsTimer);
|
|
6083
6996
|
clearTimeout(this.agentCleanupFirstTimer);
|
|
6084
6997
|
clearInterval(this.agentCleanupTimer);
|