@runuai/host 0.9.146 → 0.9.148

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,6 +34,11 @@ import { getDb, schema } from "./db";
34
34
  import { mockAgentFactory } from "./agents/mock";
35
35
  import { realAgentFactory } from "./agents/factory";
36
36
  import { resolveAgentMode } from "./agents/mode";
37
+ import {
38
+ clearPersistedCodexThread,
39
+ persistedAgentIds,
40
+ stopPersistedAgentSession,
41
+ } from "./agents/session-state";
37
42
  import {
38
43
  type AgentEvent,
39
44
  type AgentSession,
@@ -223,14 +228,27 @@ interface Channel {
223
228
  /** Sessions that loaded MCP config before the latest rewrite. Busy sessions
224
229
  * stay here until their turn completes; each id is recycled exactly once. */
225
230
  browserStaleSessions: Set<string>;
231
+ /** Launch-setting edits reuse the idle-safe stale-session path, but carry
232
+ * their own lifecycle so only those replacements emit agent.session. */
233
+ rosterSessionChanges: Map<string, RosterSessionChange>;
226
234
  /** Browser-config rewrites whose install/config preconditions are not ready
227
235
  * yet. These sessions must eventually reload the definition, but recycling
228
236
  * them before the browser is usable only trades one broken session for
229
237
  * another. */
230
238
  browserPendingStaleSessions: Set<string>;
239
+ /** Engine kinds newly covered by an already-enabled browser setup. The
240
+ * aggregate setup result can then recycle only sessions for those kinds. */
241
+ browserConfigAffectedKinds: Set<string>;
231
242
  /** One reconcile loop per channel. It covers ordinary missing agents and
232
243
  * stale-session recycling without overlapping factory creates. */
233
244
  reconcileReady: Promise<void> | null;
245
+ /** Forced removal closes begun by refreshChannel. ensureSessions joins this
246
+ * fence before spawning so one outbox consumer cannot overlap another. */
247
+ rosterRefreshReady: Promise<void> | null;
248
+ /** First-start cleanup of durable rows whose ids are absent from the fresh
249
+ * cloud roster (the host may have restarted after their removal). */
250
+ persistedRosterChecked: boolean;
251
+ persistedRosterReady: Promise<void> | null;
234
252
  /** A reconcile request that arrived while the loop was in flight. */
235
253
  reconcileAgain: boolean;
236
254
  /** Set before teardown awaits anything. Every async continuation checks both
@@ -260,6 +278,14 @@ interface Channel {
260
278
  * turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
261
279
  * half-turn instead of delivering it to @-mentioned peers. */
262
280
  interrupted: Set<string>;
281
+ /** Cloud turn boundaries whose runner generation was retired by recovery.
282
+ * Kept separately from activeTurns because remove/edit may overtake the
283
+ * async recovery after generation-local turn state has been cleared. */
284
+ retiredTurnBoundaries: Set<string>;
285
+ /** Async account recoveries fenced by roster removal/edit. A same-settings
286
+ * remove/re-add is invisible to launch-tuple comparison, so the recovery
287
+ * itself carries the supersession bit until it releases the spawn slot. */
288
+ recoveryByAgent: Map<string, RecoveryFence>;
263
289
  /** Per-agent respawn counter — bounded so a broken agent can't
264
290
  * loop forever rewriting its config. */
265
291
  respawns: Map<string, number>;
@@ -288,6 +314,9 @@ interface Channel {
288
314
  mcpConfigDirtyEpoch: number;
289
315
  /** Dirty epoch covered by the last successful gateway publication. */
290
316
  mcpConfigAppliedEpoch: number;
317
+ /** Engine kinds whose provisioned config directories dirtied the managed
318
+ * gateway files at the current unapplied epoch. */
319
+ mcpProvisionAffectedKinds: Set<string>;
291
320
  /** Monotonic version of the managed config files agent processes read at
292
321
  * startup. A factory create that straddles a rewrite is discarded and
293
322
  * retried before it can be bound as the live generation. */
@@ -309,6 +338,9 @@ interface Channel {
309
338
  * drains them so an old channel cannot resume against a reused container
310
339
  * name and overwrite the replacement generation's durable transport. */
311
340
  factoryOperations: Set<Promise<unknown>>;
341
+ /** The create currently owning an agent id, used to fence a removal until a
342
+ * stale in-flight process has observed the new roster and closed itself. */
343
+ factoryOperationByAgent: Map<string, Promise<AgentSession | null>>;
312
344
  /** Terminal-event recovery/rotation can remove a session before awaiting
313
345
  * its close. Track the enclosing handler so teardown still drains that
314
346
  * external operation before the container name can be reused. */
@@ -328,6 +360,16 @@ interface AcceptedPrompt {
328
360
  text: string;
329
361
  }
330
362
 
363
+ interface RosterSessionChange {
364
+ reason: string;
365
+ state: "pending-close" | "restarting";
366
+ abortPublished: boolean;
367
+ }
368
+
369
+ interface RecoveryFence {
370
+ superseded: boolean;
371
+ }
372
+
331
373
  /** Hard cap on automatic respawns per agent per channel lifetime. */
332
374
  const MAX_RESPAWNS_PER_AGENT = 5;
333
375
  /** ADR-076: hard cap on account rotations per agent between successful turns. */
@@ -545,31 +587,100 @@ export class Orchestrator {
545
587
  /** Fold a fresh channel spec into a live channel (ADR-049). */
546
588
  private refreshChannel(channel: Channel, spec: ChannelEnsureInput): void {
547
589
  const browserWasEnabled = channel.browserTesting;
590
+ const previousRoster = channel.roster;
548
591
  const previousMcpShape = JSON.stringify({
549
592
  connections: channel.mcpConnections,
550
- kinds: [...new Set(channel.roster.map((agent) => agent.kind))].sort(),
593
+ kinds: [...new Set(previousRoster.map((agent) => agent.kind))].sort(),
551
594
  });
552
- const known = new Set(channel.roster.map((a) => a.id));
595
+ const previousById = new Map(previousRoster.map((agent) => [agent.id, agent]));
596
+ const nextById = new Map(spec.agents.map((agent) => [agent.id, agent]));
597
+ const previousKinds = new Set(previousRoster.map((agent) => agent.kind));
598
+ const nextKinds = new Set(spec.agents.map((agent) => agent.kind));
599
+ const removed = previousRoster.filter((agent) => !nextById.has(agent.id));
600
+ const added = spec.agents.filter((agent) => !previousById.has(agent.id));
601
+ const changed = spec.agents.flatMap((agent) => {
602
+ const previous = previousById.get(agent.id);
603
+ if (!previous) return [];
604
+ const reason = rosterLaunchChangeReason(previous, agent);
605
+ return reason ? [{ agent, reason }] : [];
606
+ });
607
+
553
608
  // Replace EXISTING agents' data with the fresh spec — a mid-task roster edit
554
609
  // can change permissions, model, brief, role or skills, and without this the
555
610
  // stale snapshot persists (e.g. granting `uai` CLI permissions mid-task never
556
- // took effect: writeAgentCli kept seeing the old empty list). Swapping the
557
- // object (not merging) also drops fields that were removed. The token/model a
558
- // LIVE session already carries only changes on its next respawn.
559
- const bySpecId = new Map(spec.agents.map((a) => [a.id, a]));
560
- channel.roster = channel.roster.map((a) => bySpecId.get(a.id) ?? a);
561
- for (const agent of spec.agents) {
562
- if (!known.has(agent.id)) {
563
- channel.roster.push(agent);
564
- // Per-agent link/document skills, like getOrCreateChannel does at
565
- // channel birth. Best-effort.
566
- writeAgentSkills(channel.taskId, agent);
611
+ // took effect: writeAgentCli kept seeing the old empty list). Replacing the
612
+ // array exactly also makes absence authoritative for live removal.
613
+ channel.roster = [...spec.agents];
614
+ for (const agent of added) {
615
+ // Per-agent link/document skills, like getOrCreateChannel does at
616
+ // channel birth. Best-effort.
617
+ writeAgentSkills(channel.taskId, agent);
618
+ }
619
+ for (const { agent, reason } of changed) {
620
+ const recoveryAbortPublished = this.supersedeRecovery(
621
+ channel,
622
+ agent.id,
623
+ );
624
+ // Clear before any reconcile can construct a replacement. A session can
625
+ // already be absent from the map while its durable runner/thread row is
626
+ // still live, so this must not depend on in-memory session presence.
627
+ try {
628
+ clearPersistedCodexThread(channel.taskId, agent.id);
629
+ } catch (err) {
630
+ console.warn(
631
+ `[orchestrator] ${channel.taskId}/${agent.id}: could not clear persisted Codex thread for roster change: ${
632
+ err instanceof Error ? err.message : String(err)
633
+ }`,
634
+ );
567
635
  }
636
+ channel.accountByAgent.delete(agent.id);
637
+ channel.rotations.delete(agent.id);
638
+ // Crash/recovery budgets belong to the retired launch generation. A
639
+ // newly selected engine/model/effort must get an immediate clean start,
640
+ // not inherit the predecessor's cooldown window.
641
+ channel.respawns.delete(agent.id);
642
+ channel.respawnLastAt.delete(agent.id);
643
+ const priorChange = channel.rosterSessionChanges.get(agent.id);
644
+ channel.rosterSessionChanges.set(agent.id, {
645
+ reason,
646
+ state:
647
+ priorChange?.state === "restarting"
648
+ ? "restarting"
649
+ : "pending-close",
650
+ abortPublished:
651
+ priorChange?.abortPublished === true || recoveryAbortPublished,
652
+ });
653
+ channel.browserStaleSessions.add(agent.id);
654
+ }
655
+ if (removed.length > 0) {
656
+ const previousRefresh = channel.rosterRefreshReady ?? Promise.resolve();
657
+ const removals = removed.map((agent) =>
658
+ this.removeRosterAgent(channel, agent.id),
659
+ );
660
+ let refresh: Promise<void>;
661
+ refresh = Promise.allSettled([previousRefresh, ...removals])
662
+ .then(() => undefined)
663
+ .finally(() => {
664
+ if (channel.rosterRefreshReady === refresh) {
665
+ channel.rosterRefreshReady = null;
666
+ }
667
+ });
668
+ channel.rosterRefreshReady = refresh;
568
669
  }
569
670
  channel.humans = spec.humans ?? [];
570
671
  channel.mode = spec.mode === "secretary" ? "secretary" : "open";
571
672
  channel.secretaryAgentId = spec.secretaryAgentId;
572
673
  channel.browserTesting = spec.browserTesting === true;
674
+ if (channel.browserTesting) {
675
+ const affectedKinds = browserWasEnabled
676
+ ? [...nextKinds].filter((kind) => !previousKinds.has(kind))
677
+ : [...nextKinds];
678
+ for (const kind of affectedKinds) {
679
+ channel.browserConfigAffectedKinds.add(kind);
680
+ }
681
+ } else {
682
+ channel.browserConfigAffectedKinds.clear();
683
+ }
573
684
  channel.routingShadow = spec.routing?.shadow === true;
574
685
  channel.workspacePath = spec.workspacePath;
575
686
  channel.sharedFiles = spec.sharedFiles ?? "ro";
@@ -626,6 +737,162 @@ export class Orchestrator {
626
737
  }
627
738
  }
628
739
 
740
+ /** Remove every live/in-flight trace of one agent without deleting its
741
+ * durable session directory. Removal is intentionally immediate mid-turn. */
742
+ private async removeRosterAgent(
743
+ channel: Channel,
744
+ agentId: string,
745
+ ): Promise<void> {
746
+ const session = channel.sessions.get(agentId);
747
+ const inFlightCreate = channel.factoryOperationByAgent.get(agentId);
748
+ const lifecycleAtRemoval = channel.rosterSessionChanges.get(agentId);
749
+ channel.sessions.delete(agentId);
750
+ // Do not release another lifecycle owner's spawn slot. Recovery/reconcile
751
+ // paths clear it in their own finally blocks after rechecking the desired
752
+ // roster; releasing it here lets a rapid re-add start a second same-id
753
+ // factory while the first is still provisioning.
754
+
755
+ // Recovery may already have retired the runner-local turn state while an
756
+ // async provision/broker call was in flight. Consume that saved boundary
757
+ // before looking at the live generation's maps.
758
+ this.supersedeRecovery(channel, agentId);
759
+ this.emitRetiredTurnAbort(channel, agentId);
760
+ // A removed partial turn can never receive its natural terminal event.
761
+ // Close that cloud boundary as aborted before the old session's identity
762
+ // guard begins suppressing its final adapter events.
763
+ if (
764
+ channel.activeTurns.has(agentId) ||
765
+ channel.openTurns.has(agentId) ||
766
+ channel.acceptedPrompts.has(agentId)
767
+ ) {
768
+ channel.interrupted.add(agentId);
769
+ }
770
+ this.retireRunnerGeneration(channel, agentId);
771
+ channel.preambles.delete(agentId);
772
+ channel.firstTurns.delete(agentId);
773
+ channel.browserStaleSessions.delete(agentId);
774
+ channel.rosterSessionChanges.delete(agentId);
775
+ channel.browserPendingStaleSessions.delete(agentId);
776
+ channel.pendingPrompts.delete(agentId);
777
+ channel.runtimeDeferredPrompts.delete(agentId);
778
+ channel.runtimeDeferredAgents.delete(agentId);
779
+ channel.interrupted.delete(agentId);
780
+ channel.retiredTurnBoundaries.delete(agentId);
781
+ channel.respawns.delete(agentId);
782
+ channel.respawnLastAt.delete(agentId);
783
+ channel.everBound.delete(agentId);
784
+ channel.accountByAgent.delete(agentId);
785
+ channel.lastPrompt.delete(agentId);
786
+ channel.rotations.delete(agentId);
787
+
788
+ if (session) {
789
+ try {
790
+ // Delete-before-close above keeps the adapter's terminal event
791
+ // identity-guarded and prevents an outbox consumer overlap.
792
+ await session.close();
793
+ } catch {
794
+ // Already dead — close is idempotent for our adapters.
795
+ }
796
+ }
797
+ if (inFlightCreate) {
798
+ // createAndBindStableSessionCore rechecks the desired roster after the
799
+ // factory await and closes the stale process instead of binding it.
800
+ await inFlightCreate.catch(() => null);
801
+ }
802
+ if (!this.isActiveChannel(channel)) return;
803
+ // A newer ensure can re-add the same id while the old close/create drains.
804
+ // That spec owns the status now; do not stop its bound generation or emit
805
+ // a stale removed edge.
806
+ if (channel.roster.some((agent) => agent.id === agentId)) {
807
+ this.restoreReaddedRosterLifecycle(
808
+ channel,
809
+ agentId,
810
+ lifecycleAtRemoval,
811
+ );
812
+ void this.reconcileIfStarted(channel).catch((err: unknown) => {
813
+ console.warn(
814
+ `[orchestrator] ${channel.taskId}/${agentId}: re-added roster reconcile failed: ${
815
+ err instanceof Error ? err.message : String(err)
816
+ }`,
817
+ );
818
+ });
819
+ return;
820
+ }
821
+ let durableCleanupFailed = false;
822
+ try {
823
+ await this.stopPersistedRunnerIfPresent(channel, agentId);
824
+ } catch (err) {
825
+ durableCleanupFailed = true;
826
+ // The same ensure that joined this removal fence will retry through the
827
+ // persisted-roster recovery pass instead of trusting a poisoned result.
828
+ channel.persistedRosterChecked = false;
829
+ console.warn(
830
+ `[orchestrator] ${channel.taskId}/${agentId}: removed durable session cleanup failed: ${
831
+ err instanceof Error ? err.message : String(err)
832
+ }`,
833
+ );
834
+ }
835
+ if (!this.isActiveChannel(channel)) return;
836
+ if (channel.roster.some((agent) => agent.id === agentId)) {
837
+ this.restoreReaddedRosterLifecycle(
838
+ channel,
839
+ agentId,
840
+ lifecycleAtRemoval,
841
+ );
842
+ void this.reconcileIfStarted(channel).catch((err: unknown) => {
843
+ console.warn(
844
+ `[orchestrator] ${channel.taskId}/${agentId}: post-stop re-add reconcile failed: ${
845
+ err instanceof Error ? err.message : String(err)
846
+ }`,
847
+ );
848
+ });
849
+ return;
850
+ }
851
+ // Slow rotation/account continuations captured before the roster swap may
852
+ // have reached their finally blocks while the close was in flight.
853
+ channel.accountByAgent.delete(agentId);
854
+ channel.rotations.delete(agentId);
855
+ channel.respawns.delete(agentId);
856
+ channel.respawnLastAt.delete(agentId);
857
+ channel.lastPrompt.delete(agentId);
858
+ if (!this.isActiveChannel(channel)) return;
859
+ if (durableCleanupFailed) return;
860
+ this.emitHost({
861
+ kind: "agent.session",
862
+ taskId: channel.taskId,
863
+ agentId,
864
+ state: "removed",
865
+ });
866
+ console.log(
867
+ `[orchestrator] ${channel.taskId}/${agentId}: stopped removed roster session`,
868
+ );
869
+ }
870
+
871
+ /** A remove superseded while an earlier edit was already restarting must
872
+ * still close that visible lifecycle with ready for the re-added roster. */
873
+ private restoreReaddedRosterLifecycle(
874
+ channel: Channel,
875
+ agentId: string,
876
+ removedChange: RosterSessionChange | undefined,
877
+ ): void {
878
+ if (
879
+ removedChange?.state !== "restarting" ||
880
+ channel.rosterSessionChanges.has(agentId)
881
+ ) {
882
+ return;
883
+ }
884
+ if (channel.sessions.has(agentId)) {
885
+ this.emitHost({
886
+ kind: "agent.session",
887
+ taskId: channel.taskId,
888
+ agentId,
889
+ state: "ready",
890
+ });
891
+ return;
892
+ }
893
+ channel.rosterSessionChanges.set(agentId, removedChange);
894
+ }
895
+
629
896
  /** Materialize a browser flag enabled after the channel already started. */
630
897
  private async enableBrowserTesting(channel: Channel): Promise<void> {
631
898
  try {
@@ -709,8 +976,13 @@ export class Orchestrator {
709
976
  browserSetupCoverage: null,
710
977
  browserRetryTimer: null,
711
978
  browserStaleSessions: new Set(),
979
+ rosterSessionChanges: new Map(),
712
980
  browserPendingStaleSessions: new Set(),
981
+ browserConfigAffectedKinds: new Set(),
713
982
  reconcileReady: null,
983
+ rosterRefreshReady: null,
984
+ persistedRosterChecked: false,
985
+ persistedRosterReady: null,
714
986
  reconcileAgain: false,
715
987
  closed: false,
716
988
  openTurns: new Set(),
@@ -720,6 +992,8 @@ export class Orchestrator {
720
992
  runtimeDeferredPrompts: new Map(),
721
993
  runtimeDeferredAgents: new Set(),
722
994
  interrupted: new Set(),
995
+ retiredTurnBoundaries: new Set(),
996
+ recoveryByAgent: new Map(),
723
997
  respawns: new Map(),
724
998
  respawnLastAt: new Map(),
725
999
  humans: spec.humans ?? [],
@@ -729,12 +1003,14 @@ export class Orchestrator {
729
1003
  mcpConfigWrite: null,
730
1004
  mcpConfigDirtyEpoch: 1,
731
1005
  mcpConfigAppliedEpoch: 0,
1006
+ mcpProvisionAffectedKinds: new Set(),
732
1007
  configGeneration: 0,
733
1008
  spawning: new Set(),
734
1009
  everBound: new Set(),
735
1010
  routingShadow: spec.routing?.shadow === true,
736
1011
  workspacePath: spec.workspacePath,
737
1012
  factoryOperations: new Set(),
1013
+ factoryOperationByAgent: new Map(),
738
1014
  eventOperations: new Set(),
739
1015
  accountByAgent: new Map(),
740
1016
  lastPrompt: new Map(),
@@ -761,6 +1037,58 @@ export class Orchestrator {
761
1037
  // copy cannot land after browser/MCP materialization and erase config.
762
1038
  await taskRecoveryComplete(channel.taskId);
763
1039
  if (!this.isActiveChannel(channel)) return false;
1040
+ if (channel.rosterRefreshReady) {
1041
+ await channel.rosterRefreshReady;
1042
+ if (!this.isActiveChannel(channel)) return false;
1043
+ }
1044
+ if (!channel.persistedRosterChecked) {
1045
+ let removedIds: string[];
1046
+ try {
1047
+ const rosterIds = new Set(channel.roster.map((agent) => agent.id));
1048
+ removedIds = persistedAgentIds(channel.taskId).filter(
1049
+ (agentId) => !rosterIds.has(agentId),
1050
+ );
1051
+ } catch (err) {
1052
+ console.warn(
1053
+ `[orchestrator] ${channel.taskId}: persisted roster scan failed: ${
1054
+ err instanceof Error ? err.message : String(err)
1055
+ }`,
1056
+ );
1057
+ throw err;
1058
+ }
1059
+ channel.persistedRosterChecked = true;
1060
+ if (removedIds.length > 0) {
1061
+ const operation = this.stopPersistedRemovedAgents(
1062
+ channel,
1063
+ removedIds,
1064
+ );
1065
+ let tracked: Promise<void>;
1066
+ tracked = operation.then(
1067
+ () => {
1068
+ if (channel.persistedRosterReady === tracked) {
1069
+ channel.persistedRosterReady = null;
1070
+ }
1071
+ },
1072
+ (err: unknown) => {
1073
+ if (channel.persistedRosterReady === tracked) {
1074
+ channel.persistedRosterReady = null;
1075
+ channel.persistedRosterChecked = false;
1076
+ }
1077
+ console.warn(
1078
+ `[orchestrator] ${channel.taskId}: persisted roster cleanup failed: ${
1079
+ err instanceof Error ? err.message : String(err)
1080
+ }`,
1081
+ );
1082
+ throw err;
1083
+ },
1084
+ );
1085
+ channel.persistedRosterReady = tracked;
1086
+ }
1087
+ }
1088
+ if (channel.persistedRosterReady) {
1089
+ await channel.persistedRosterReady;
1090
+ if (!this.isActiveChannel(channel)) return false;
1091
+ }
764
1092
  if (!backgroundContainerWorkAllowed()) return false;
765
1093
  // Memoized: every caller awaits the SAME in-flight start, so a concurrent
766
1094
  // deliver() can't observe "ready" while the sessions map is still empty
@@ -825,6 +1153,48 @@ export class Orchestrator {
825
1153
  return ready;
826
1154
  }
827
1155
 
1156
+ /** A host restart loses the in-memory old roster, not its durable runners.
1157
+ * Stop persisted ids that the authoritative fresh spec no longer contains. */
1158
+ private async stopPersistedRemovedAgents(
1159
+ channel: Channel,
1160
+ removedIds: readonly string[],
1161
+ ): Promise<void> {
1162
+ const environment = await this.agentEnvironment(channel);
1163
+ for (const agentId of removedIds) {
1164
+ if (!this.isActiveChannel(channel)) return;
1165
+ // The roster may have advanced while environment reconstruction was in
1166
+ // flight. Last arrival wins; never stop an id the current spec wants.
1167
+ if (channel.roster.some((agent) => agent.id === agentId)) continue;
1168
+ const stoppedDir = await stopPersistedAgentSession(
1169
+ channel.taskId,
1170
+ agentId,
1171
+ environment,
1172
+ );
1173
+ if (channel.roster.some((agent) => agent.id === agentId)) continue;
1174
+ if (!stoppedDir || !this.isActiveChannel(channel)) continue;
1175
+ this.emitHost({
1176
+ kind: "agent.session",
1177
+ taskId: channel.taskId,
1178
+ agentId,
1179
+ state: "removed",
1180
+ });
1181
+ console.log(
1182
+ `[orchestrator] ${channel.taskId}/${agentId}: stopped removed durable session during roster recovery`,
1183
+ );
1184
+ }
1185
+ }
1186
+
1187
+ /** Stop a durable predecessor even when its row is already marked closed.
1188
+ * Closed is not proof of runner death: markClosed precedes the stop write. */
1189
+ private async stopPersistedRunnerIfPresent(
1190
+ channel: Channel,
1191
+ agentId: string,
1192
+ ): Promise<string | null> {
1193
+ if (!persistedAgentIds(channel.taskId).includes(agentId)) return null;
1194
+ const environment = await this.agentEnvironment(channel);
1195
+ return stopPersistedAgentSession(channel.taskId, agentId, environment);
1196
+ }
1197
+
828
1198
  /**
829
1199
  * Background browser work may recycle sessions only after the memoized
830
1200
  * initial start has finished. Waiting here prevents live enable/retry from
@@ -873,11 +1243,17 @@ export class Orchestrator {
873
1243
  private noteProvisionMutation(
874
1244
  channel: Channel,
875
1245
  waitForBrowserReadiness: boolean,
1246
+ affectedKinds: ReadonlySet<string>,
876
1247
  ): void {
877
1248
  if (!this.isActiveChannel(channel)) return;
878
1249
  channel.configGeneration += 1;
879
1250
  channel.mcpConfigDirtyEpoch += 1;
880
- for (const agentId of channel.sessions.keys()) {
1251
+ for (const kind of affectedKinds) {
1252
+ channel.mcpProvisionAffectedKinds.add(kind);
1253
+ }
1254
+ for (const [agentId] of channel.sessions) {
1255
+ const kind = channel.roster.find((agent) => agent.id === agentId)?.kind;
1256
+ if (!kind || !affectedKinds.has(kind)) continue;
881
1257
  if (waitForBrowserReadiness) {
882
1258
  channel.browserPendingStaleSessions.add(agentId);
883
1259
  } else {
@@ -910,6 +1286,42 @@ export class Orchestrator {
910
1286
  });
911
1287
  }
912
1288
 
1289
+ /** Kinds whose account-copy inputs changed since the last attempted pass. */
1290
+ private changedProvisionKinds(
1291
+ previousFingerprint: string | undefined,
1292
+ nextFingerprint: string,
1293
+ requestedKinds: readonly string[],
1294
+ ): string[] {
1295
+ if (!previousFingerprint) return [...requestedKinds];
1296
+ type Fingerprint = {
1297
+ kinds?: string[];
1298
+ accounts?: Array<{
1299
+ kind: string;
1300
+ id: string;
1301
+ hostDir: string;
1302
+ containerDir: string;
1303
+ }>;
1304
+ };
1305
+ try {
1306
+ const previous = JSON.parse(previousFingerprint) as Fingerprint;
1307
+ const next = JSON.parse(nextFingerprint) as Fingerprint;
1308
+ return requestedKinds.filter((kind) => {
1309
+ if (!previous.kinds?.includes(kind)) return true;
1310
+ const previousAccounts = (previous.accounts ?? []).filter(
1311
+ (account) => account.kind === kind,
1312
+ );
1313
+ const nextAccounts = (next.accounts ?? []).filter(
1314
+ (account) => account.kind === kind,
1315
+ );
1316
+ return JSON.stringify(previousAccounts) !== JSON.stringify(nextAccounts);
1317
+ });
1318
+ } catch {
1319
+ // A legacy/partial fingerprint is not trustworthy enough to narrow a
1320
+ // write; copy and invalidate the requested kinds conservatively.
1321
+ return [...requestedKinds];
1322
+ }
1323
+ }
1324
+
913
1325
  private async ensureBrowserSetup(
914
1326
  channel: Channel,
915
1327
  hasCodex: boolean,
@@ -936,11 +1348,20 @@ export class Orchestrator {
936
1348
  ];
937
1349
  const requestedProvisionFingerprint =
938
1350
  this.provisionFingerprint(requestedKinds, channel.engineAccountScope);
939
- const mayProvisionConfig = requestedKinds.some((kind) =>
1351
+ const changedProvisionKinds = this.changedProvisionKinds(
1352
+ channel.provisionAttemptedFingerprint,
1353
+ requestedProvisionFingerprint,
1354
+ requestedKinds,
1355
+ );
1356
+ const provisionKinds = forceProvision
1357
+ ? requestedKinds
1358
+ : changedProvisionKinds;
1359
+ const mutableProvisionKinds = provisionKinds.filter((kind) =>
940
1360
  resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
941
1361
  (account) => !account.isDefault && Boolean(account.configDir),
942
1362
  ),
943
1363
  );
1364
+ const mayProvisionConfig = mutableProvisionKinds.length > 0;
944
1365
 
945
1366
  const inFlight = channel.browserSetup;
946
1367
  if (inFlight) {
@@ -1011,14 +1432,14 @@ export class Orchestrator {
1011
1432
  const provisioned = channel.agentHandle
1012
1433
  ? await provisionEngineAccounts(
1013
1434
  channel.containerName,
1014
- requestedKinds,
1435
+ provisionKinds,
1015
1436
  {},
1016
1437
  channel.agentHandle,
1017
1438
  channel.engineAccountScope,
1018
1439
  )
1019
1440
  : await provisionEngineAccounts(
1020
1441
  channel.containerName,
1021
- requestedKinds,
1442
+ provisionKinds,
1022
1443
  {},
1023
1444
  undefined,
1024
1445
  channel.engineAccountScope,
@@ -1050,6 +1471,7 @@ export class Orchestrator {
1050
1471
  this.noteProvisionMutation(
1051
1472
  channel,
1052
1473
  channel.browserTesting,
1474
+ new Set(mutableProvisionKinds),
1053
1475
  );
1054
1476
  } else {
1055
1477
  this.fenceConfigGeneration(channel);
@@ -1107,10 +1529,23 @@ export class Orchestrator {
1107
1529
  channel.browserReady = setup.ready && setup.configured;
1108
1530
  if (setup.changed) {
1109
1531
  channel.configGeneration += 1;
1110
- for (const agentId of channel.sessions.keys()) {
1532
+ const affectedKinds = channel.browserConfigAffectedKinds;
1533
+ const affectedIds =
1534
+ affectedKinds.size > 0
1535
+ ? channel.roster
1536
+ .filter((agent) => affectedKinds.has(agent.kind))
1537
+ .map((agent) => agent.id)
1538
+ : [...channel.sessions.keys()];
1539
+ for (const agentId of affectedIds) {
1540
+ if (!channel.sessions.has(agentId)) continue;
1111
1541
  channel.browserPendingStaleSessions.add(agentId);
1112
1542
  }
1113
1543
  }
1544
+ if (setup.configured) {
1545
+ for (const kind of requestedKinds) {
1546
+ channel.browserConfigAffectedKinds.delete(kind);
1547
+ }
1548
+ }
1114
1549
  if (channel.browserReady) {
1115
1550
  for (const agentId of channel.browserPendingStaleSessions) {
1116
1551
  channel.browserStaleSessions.add(agentId);
@@ -1259,6 +1694,9 @@ export class Orchestrator {
1259
1694
  ...(grokHomes.length > 0 ? { grokHomes } : {}),
1260
1695
  });
1261
1696
  const dirtyEpoch = channel.mcpConfigDirtyEpoch;
1697
+ const provisionAffectedKinds = new Set(
1698
+ channel.mcpProvisionAffectedKinds,
1699
+ );
1262
1700
  const repairingProvisionedConfig =
1263
1701
  channel.mcpConfigAppliedEpoch !== dirtyEpoch &&
1264
1702
  channel.mcpConfigFingerprint === fingerprint;
@@ -1321,6 +1759,9 @@ export class Orchestrator {
1321
1759
  // the files while docker exec was in flight, the newer epoch remains
1322
1760
  // uncovered and the serialized ensure immediately publishes another pass.
1323
1761
  channel.mcpConfigAppliedEpoch = dirtyEpoch;
1762
+ if (channel.mcpConfigDirtyEpoch === dirtyEpoch) {
1763
+ channel.mcpProvisionAffectedKinds.clear();
1764
+ }
1324
1765
  // Agent CLIs read MCP servers once, at process start — and durable
1325
1766
  // sessions (ADR-061) make processes long-lived, so without this a
1326
1767
  // connection added mid-task stays invisible indefinitely. Recycle IDLE
@@ -1338,7 +1779,10 @@ export class Orchestrator {
1338
1779
  // Provisioning marked the sessions that existed around its copy. Mark
1339
1780
  // again after the repair so a factory that bound during this write is
1340
1781
  // covered too (Set de-duplicates the earlier marks).
1341
- this.markSessionsStale(channel);
1782
+ const affected = channel.roster
1783
+ .filter((agent) => provisionAffectedKinds.has(agent.kind))
1784
+ .map((agent) => agent.id);
1785
+ this.markSessionsStale(channel, affected);
1342
1786
  } else if (!hadPrevious && hadLiveSessions) {
1343
1787
  this.markSessionsStale(channel);
1344
1788
  } else if (hadPrevious && connectionsChanged) {
@@ -1372,7 +1816,16 @@ export class Orchestrator {
1372
1816
  /** Close stale sessions that are idle, without recursively reconciling. */
1373
1817
  private async closeIdleStaleSessions(channel: Channel): Promise<number> {
1374
1818
  let recycled = 0;
1375
- for (const agentId of [...channel.browserStaleSessions]) {
1819
+ let mcpRecycled = 0;
1820
+ // The roster-change tracker is authoritative. Recovery paths written for
1821
+ // ordinary config recycling may clear browserStaleSessions while an edit
1822
+ // is waiting on their spawn slot; never let a pending-close edit bind its
1823
+ // replacement without first publishing the actual restart boundary.
1824
+ const staleAgentIds = new Set([
1825
+ ...channel.browserStaleSessions,
1826
+ ...channel.rosterSessionChanges.keys(),
1827
+ ]);
1828
+ for (const agentId of staleAgentIds) {
1376
1829
  if (!this.isActiveChannel(channel)) return recycled;
1377
1830
  if (
1378
1831
  channel.activeTurns.has(agentId) ||
@@ -1383,31 +1836,155 @@ export class Orchestrator {
1383
1836
  }
1384
1837
  const session = channel.sessions.get(agentId);
1385
1838
  // A session that exited on its own is already absent. Its eventual
1386
- // replacement is fresh, so the old generation is no longer stale.
1839
+ // replacement is fresh for MCP purposes. A roster change still has to
1840
+ // stop a possibly-live durable predecessor before that replacement can
1841
+ // bind under new launch settings.
1387
1842
  if (!session) {
1388
- channel.browserStaleSessions.delete(agentId);
1843
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
1844
+ if (rosterChange) {
1845
+ await this.closeMissingRosterChangedSession(
1846
+ channel,
1847
+ agentId,
1848
+ rosterChange,
1849
+ );
1850
+ recycled += 1;
1851
+ } else {
1852
+ channel.browserStaleSessions.delete(agentId);
1853
+ }
1389
1854
  continue;
1390
1855
  }
1391
1856
  channel.browserStaleSessions.delete(agentId);
1392
1857
  channel.sessions.delete(agentId);
1393
- try {
1394
- // Delete before close. Event handlers are identity-guarded, so the
1395
- // old session's synthetic exit cannot consume respawn budget or delete
1396
- // the replacement that follows.
1397
- await session.close();
1398
- } catch {
1399
- // Already dead — close is idempotent for our adapters.
1858
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
1859
+ if (rosterChange) {
1860
+ await this.closeRosterChangedSession(
1861
+ channel,
1862
+ agentId,
1863
+ session,
1864
+ rosterChange,
1865
+ );
1866
+ } else {
1867
+ try {
1868
+ // Delete before close. Event handlers are identity-guarded, so the
1869
+ // old session's synthetic exit cannot consume respawn budget or delete
1870
+ // the replacement that follows.
1871
+ await session.close();
1872
+ } catch {
1873
+ // Already dead — close is idempotent for our adapters.
1874
+ }
1875
+ mcpRecycled += 1;
1400
1876
  }
1401
1877
  recycled += 1;
1402
1878
  }
1403
- if (recycled > 0) {
1879
+ if (mcpRecycled > 0) {
1404
1880
  console.log(
1405
- `[orchestrator] ${channel.taskId}: recycled ${recycled} idle session(s) for MCP config change`,
1881
+ `[orchestrator] ${channel.taskId}: recycled ${mcpRecycled} idle session(s) for MCP config change`,
1406
1882
  );
1407
1883
  }
1408
1884
  return recycled;
1409
1885
  }
1410
1886
 
1887
+ /** Stop one generation for a live launch-setting edit and publish the
1888
+ * roster-specific lifecycle edge only if that edit still owns the id. */
1889
+ private async closeRosterChangedSession(
1890
+ channel: Channel,
1891
+ agentId: string,
1892
+ session: AgentSession,
1893
+ change: RosterSessionChange,
1894
+ ): Promise<void> {
1895
+ // Durable attach cannot reconstruct whether a pre-restart runner is
1896
+ // already mid-turn. Publish the terminal boundary before every forced
1897
+ // launch close; it is a no-op for an actually idle cloud stream.
1898
+ this.publishRosterChangeAbort(channel, agentId, change);
1899
+ try {
1900
+ await session.close();
1901
+ } catch {
1902
+ // Already dead — close is idempotent for our adapters.
1903
+ }
1904
+ try {
1905
+ await this.stopPersistedRunnerIfPresent(channel, agentId);
1906
+ // Clear again after close. A thread/started notification already queued
1907
+ // by the old Codex process must not restore its resume id between the
1908
+ // eager roster clear and replacement construction.
1909
+ clearPersistedCodexThread(channel.taskId, agentId);
1910
+ } catch (err) {
1911
+ if (
1912
+ this.isActiveChannel(channel) &&
1913
+ channel.rosterSessionChanges.get(agentId) === change
1914
+ ) {
1915
+ channel.browserStaleSessions.add(agentId);
1916
+ }
1917
+ throw err;
1918
+ }
1919
+ if (
1920
+ !this.isActiveChannel(channel) ||
1921
+ channel.rosterSessionChanges.get(agentId) !== change ||
1922
+ !channel.roster.some((agent) => agent.id === agentId)
1923
+ ) {
1924
+ return;
1925
+ }
1926
+ const alreadyRestarting = change.state === "restarting";
1927
+ change.state = "restarting";
1928
+ if (!alreadyRestarting) {
1929
+ this.emitHost({
1930
+ kind: "agent.session",
1931
+ taskId: channel.taskId,
1932
+ agentId,
1933
+ state: "restarting",
1934
+ });
1935
+ }
1936
+ console.log(
1937
+ `[orchestrator] ${channel.taskId}/${agentId}: recycled session for live roster ${change.reason}`,
1938
+ );
1939
+ }
1940
+
1941
+ /** Finish a launch edit whose adapter already disappeared, but whose
1942
+ * durable predecessor may still own the CLI process/outbox. */
1943
+ private async closeMissingRosterChangedSession(
1944
+ channel: Channel,
1945
+ agentId: string,
1946
+ change: RosterSessionChange,
1947
+ ): Promise<void> {
1948
+ this.publishRosterChangeAbort(channel, agentId, change);
1949
+ const stoppedDir = await this.stopPersistedRunnerIfPresent(channel, agentId);
1950
+ clearPersistedCodexThread(channel.taskId, agentId);
1951
+ if (
1952
+ !this.isActiveChannel(channel) ||
1953
+ channel.rosterSessionChanges.get(agentId) !== change ||
1954
+ !channel.roster.some((agent) => agent.id === agentId)
1955
+ ) {
1956
+ return;
1957
+ }
1958
+ channel.browserStaleSessions.delete(agentId);
1959
+ if (!stoppedDir) {
1960
+ // No old generation exists to recycle (one-shot engines between turns,
1961
+ // crash cooldown, or an agent not spawned yet). The cloud set the row to
1962
+ // Restarting when it persisted the edit, so clear that status now even
1963
+ // though there is no close/restart edge to publish first.
1964
+ channel.rosterSessionChanges.delete(agentId);
1965
+ this.emitHost({
1966
+ kind: "agent.session",
1967
+ taskId: channel.taskId,
1968
+ agentId,
1969
+ state: "ready",
1970
+ });
1971
+ return;
1972
+ }
1973
+ const alreadyRestarting = change.state === "restarting";
1974
+ change.state = "restarting";
1975
+ if (!alreadyRestarting) {
1976
+ this.emitHost({
1977
+ kind: "agent.session",
1978
+ taskId: channel.taskId,
1979
+ agentId,
1980
+ state: "restarting",
1981
+ });
1982
+ }
1983
+ console.log(
1984
+ `[orchestrator] ${channel.taskId}/${agentId}: stopped detached durable session for live roster ${change.reason}`,
1985
+ );
1986
+ }
1987
+
1411
1988
  /** Serialize all missing-session and stale-session reconciliation. */
1412
1989
  private async reconcileSessions(channel: Channel): Promise<void> {
1413
1990
  if (!this.isActiveChannel(channel)) return;
@@ -1520,6 +2097,10 @@ export class Orchestrator {
1520
2097
  // Another recovery path may have filled this slot while the browser
1521
2098
  // setup was awaiting.
1522
2099
  if (channel.sessions.has(agent.id)) continue;
2100
+ if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
2101
+ channel.reconcileAgain = true;
2102
+ continue;
2103
+ }
1523
2104
  const session = await this.createAndBindStableSession(
1524
2105
  channel,
1525
2106
  agent.id,
@@ -1530,7 +2111,7 @@ export class Orchestrator {
1530
2111
  executionProfile: this.executionProfileFor(channel, agent.id),
1531
2112
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
1532
2113
  channel,
1533
- agent.id,
2114
+ agent,
1534
2115
  ),
1535
2116
  agentEnv: this.accountAgentEnv(
1536
2117
  channel,
@@ -1571,16 +2152,30 @@ export class Orchestrator {
1571
2152
 
1572
2153
  private attachCompatibilityKeyFor(
1573
2154
  channel: Channel,
1574
- agentId: string,
2155
+ agent: RosterAgent,
1575
2156
  ): string | undefined {
1576
- if (!this.isSecretary(channel, agentId)) return undefined;
1577
2157
  // Key the durable runner to the policy actually passed to the adapter.
1578
2158
  // The restricted profile is dormant today, but giving it the normal-tools
1579
2159
  // generation would make a future opt-out able to attach a runner launched
1580
2160
  // under tool denial.
1581
- return this.executionProfileFor(channel, agentId) === "communicator"
1582
- ? RESTRICTED_SECRETARY_SESSION_ATTACH_KEY
1583
- : SECRETARY_SESSION_ATTACH_KEY;
2161
+ const policy = this.isSecretary(channel, agent.id)
2162
+ ? this.executionProfileFor(channel, agent.id) === "communicator"
2163
+ ? RESTRICTED_SECRETARY_SESSION_ATTACH_KEY
2164
+ : SECRETARY_SESSION_ATTACH_KEY
2165
+ : undefined;
2166
+ const model = agent.model ?? "";
2167
+ const effort = agent.effort ?? "";
2168
+ // Preserve legacy null/policy-only attachment for agents that choose no
2169
+ // launch flags. Explicit model/effort gets a stable key so a host restart
2170
+ // cannot attach a runner created under different settings.
2171
+ if (!model && !effort) return policy;
2172
+ return JSON.stringify([
2173
+ "roster-launch-v1",
2174
+ agent.kind,
2175
+ model,
2176
+ effort,
2177
+ policy ?? "",
2178
+ ]);
1584
2179
  }
1585
2180
 
1586
2181
  private isSecretary(channel: Channel, agentId: string): boolean {
@@ -1618,6 +2213,10 @@ export class Orchestrator {
1618
2213
  * container env to carry it. The account's env and the uai base win. */
1619
2214
  floor: Record<string, string> = {},
1620
2215
  ): Record<string, string> {
2216
+ if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
2217
+ channel.accountByAgent.delete(agent.id);
2218
+ return { ...floor, ...base };
2219
+ }
1621
2220
  const account = pickEngineAccount(agent.kind, {
1622
2221
  scope: channel.engineAccountScope,
1623
2222
  });
@@ -1662,6 +2261,18 @@ export class Orchestrator {
1662
2261
  ): void {
1663
2262
  channel.sessions.set(agentId, session);
1664
2263
  channel.everBound.add(agentId);
2264
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
2265
+ if (rosterChange) {
2266
+ channel.rosterSessionChanges.delete(agentId);
2267
+ if (rosterChange.state === "restarting") {
2268
+ this.emitHost({
2269
+ kind: "agent.session",
2270
+ taskId: channel.taskId,
2271
+ agentId,
2272
+ state: "ready",
2273
+ });
2274
+ }
2275
+ }
1665
2276
  // The launch screen's `ready` rides the bind, not the tail of the start
1666
2277
  // loop: whichever path brings the last roster agent up — the initial
1667
2278
  // start, a reconcile that won the race with it, a respawn after Resume —
@@ -1729,16 +2340,29 @@ export class Orchestrator {
1729
2340
  channel: Channel,
1730
2341
  agentId: string,
1731
2342
  args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
2343
+ recovery?: RecoveryFence,
1732
2344
  ): Promise<AgentSession | null> {
1733
2345
  const operation = this.createAndBindStableSessionCore(
1734
2346
  channel,
1735
2347
  agentId,
1736
2348
  args,
2349
+ recovery,
1737
2350
  );
1738
2351
  channel.factoryOperations.add(operation);
2352
+ channel.factoryOperationByAgent.set(agentId, operation);
1739
2353
  operation.then(
1740
- () => channel.factoryOperations.delete(operation),
1741
- () => channel.factoryOperations.delete(operation),
2354
+ () => {
2355
+ channel.factoryOperations.delete(operation);
2356
+ if (channel.factoryOperationByAgent.get(agentId) === operation) {
2357
+ channel.factoryOperationByAgent.delete(agentId);
2358
+ }
2359
+ },
2360
+ () => {
2361
+ channel.factoryOperations.delete(operation);
2362
+ if (channel.factoryOperationByAgent.get(agentId) === operation) {
2363
+ channel.factoryOperationByAgent.delete(agentId);
2364
+ }
2365
+ },
1742
2366
  );
1743
2367
  return operation;
1744
2368
  }
@@ -1791,8 +2415,27 @@ export class Orchestrator {
1791
2415
  channel: Channel,
1792
2416
  agentId: string,
1793
2417
  args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
2418
+ recovery?: RecoveryFence,
1794
2419
  ): Promise<AgentSession | null> {
2420
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2421
+ channel.reconcileAgain = true;
2422
+ return null;
2423
+ }
2424
+ if (
2425
+ channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
2426
+ ) {
2427
+ channel.reconcileAgain = true;
2428
+ return null;
2429
+ }
2430
+ if (!this.hasDesiredLaunchSettings(channel, agentId, args.agent)) {
2431
+ channel.reconcileAgain = true;
2432
+ return null;
2433
+ }
1795
2434
  const environment = await this.agentEnvironment(channel);
2435
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2436
+ channel.reconcileAgain = true;
2437
+ return null;
2438
+ }
1796
2439
  // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
1797
2440
  // account share one CODEX_HOME, and codex's new sqlite state runtime
1798
2441
  // cannot be opened by two concurrent processes — the second agent dies
@@ -1809,12 +2452,57 @@ export class Orchestrator {
1809
2452
  ? await this.isolateCodexHome(environment, agentId, args)
1810
2453
  : args;
1811
2454
  while (this.isActiveChannel(channel)) {
2455
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2456
+ channel.reconcileAgain = true;
2457
+ return null;
2458
+ }
2459
+ if (
2460
+ channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
2461
+ ) {
2462
+ channel.reconcileAgain = true;
2463
+ return null;
2464
+ }
2465
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2466
+ channel.reconcileAgain = true;
2467
+ return null;
2468
+ }
1812
2469
  const generation = channel.configGeneration;
1813
2470
  const session = await this.factory.create({ ...boundArgs, environment });
1814
2471
  if (!this.isActiveChannel(channel)) {
1815
2472
  await session.close().catch(() => {});
1816
2473
  return null;
1817
2474
  }
2475
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2476
+ await session.close().catch(() => {});
2477
+ channel.reconcileAgain = true;
2478
+ return null;
2479
+ }
2480
+ const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
2481
+ if (pendingRosterClose?.state === "pending-close") {
2482
+ await this.closeRosterChangedSession(
2483
+ channel,
2484
+ agentId,
2485
+ session,
2486
+ pendingRosterClose,
2487
+ );
2488
+ channel.reconcileAgain = true;
2489
+ return null;
2490
+ }
2491
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2492
+ const change = channel.rosterSessionChanges.get(agentId);
2493
+ if (change) {
2494
+ await this.closeRosterChangedSession(
2495
+ channel,
2496
+ agentId,
2497
+ session,
2498
+ change,
2499
+ );
2500
+ } else {
2501
+ await session.close().catch(() => {});
2502
+ }
2503
+ channel.reconcileAgain = true;
2504
+ return null;
2505
+ }
1818
2506
  if (generation === channel.configGeneration) {
1819
2507
  // Keep the final generation check, stale cleanup, and map bind in one
1820
2508
  // synchronous continuation. A config writer that resumes immediately
@@ -1831,6 +2519,17 @@ export class Orchestrator {
1831
2519
  return null;
1832
2520
  }
1833
2521
 
2522
+ private hasDesiredLaunchSettings(
2523
+ channel: Channel,
2524
+ agentId: string,
2525
+ launched: RosterAgent,
2526
+ ): boolean {
2527
+ const desired = channel.roster.find((agent) => agent.id === agentId);
2528
+ return Boolean(
2529
+ desired && rosterLaunchChangeReason(launched, desired) === null,
2530
+ );
2531
+ }
2532
+
1834
2533
  private agentEnvironment(
1835
2534
  channel: Channel,
1836
2535
  ): Promise<TaskEnvironmentHandle<unknown>> {
@@ -2029,6 +2728,10 @@ export class Orchestrator {
2029
2728
  if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
2030
2729
  continue;
2031
2730
  }
2731
+ if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
2732
+ channel.reconcileAgain = true;
2733
+ continue;
2734
+ }
2032
2735
  channel.spawning.add(agent.id);
2033
2736
  try {
2034
2737
  const session = await this.createAndBindStableSession(
@@ -2041,7 +2744,7 @@ export class Orchestrator {
2041
2744
  executionProfile: this.executionProfileFor(channel, agent.id),
2042
2745
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2043
2746
  channel,
2044
- agent.id,
2747
+ agent,
2045
2748
  ),
2046
2749
  agentEnv: this.accountAgentEnv(
2047
2750
  channel,
@@ -2185,6 +2888,78 @@ export class Orchestrator {
2185
2888
  return remaining;
2186
2889
  }
2187
2890
 
2891
+ /** Whether a retired generation still owns a cloud turn boundary. An
2892
+ * explicit interrupt already publishes its own aborted completion. */
2893
+ private needsAbortedTurnBoundary(
2894
+ channel: Channel,
2895
+ agentId: string,
2896
+ ): boolean {
2897
+ return (
2898
+ !channel.interrupted.has(agentId) &&
2899
+ (channel.activeTurns.has(agentId) ||
2900
+ channel.openTurns.has(agentId) ||
2901
+ (channel.acceptedPrompts.get(agentId)?.length ?? 0) > 0)
2902
+ );
2903
+ }
2904
+
2905
+ private retainRetiredTurnBoundary(
2906
+ channel: Channel,
2907
+ agentId: string,
2908
+ needed: boolean,
2909
+ ): void {
2910
+ if (needed) channel.retiredTurnBoundaries.add(agentId);
2911
+ }
2912
+
2913
+ /** Publish a retained terminal edge at most once. Recovery can be overtaken
2914
+ * by remove/edit after its generation-local turn maps have been cleared. */
2915
+ private emitRetiredTurnAbort(channel: Channel, agentId: string): boolean {
2916
+ if (!channel.retiredTurnBoundaries.delete(agentId)) return false;
2917
+ this.emitHost({
2918
+ kind: "agent.turn_complete",
2919
+ taskId: channel.taskId,
2920
+ agentId,
2921
+ aborted: true,
2922
+ });
2923
+ return true;
2924
+ }
2925
+
2926
+ private supersedeRecovery(channel: Channel, agentId: string): boolean {
2927
+ const recovery = channel.recoveryByAgent.get(agentId);
2928
+ if (!recovery) return false;
2929
+ recovery.superseded = true;
2930
+ // A live edit/removal now owns the terminal edge. Publish it before the
2931
+ // roster lifecycle can emit restarting/removed or bind a replacement.
2932
+ return this.emitRetiredTurnAbort(channel, agentId);
2933
+ }
2934
+
2935
+ private publishRosterChangeAbort(
2936
+ channel: Channel,
2937
+ agentId: string,
2938
+ change: RosterSessionChange,
2939
+ ): void {
2940
+ if (change.abortPublished) return;
2941
+ if (!this.emitRetiredTurnAbort(channel, agentId)) {
2942
+ this.emitHost({
2943
+ kind: "agent.turn_complete",
2944
+ taskId: channel.taskId,
2945
+ agentId,
2946
+ aborted: true,
2947
+ });
2948
+ }
2949
+ change.abortPublished = true;
2950
+ }
2951
+
2952
+ private recoveryIsCurrent(
2953
+ channel: Channel,
2954
+ agentId: string,
2955
+ recovery: RecoveryFence,
2956
+ ): boolean {
2957
+ return (
2958
+ channel.recoveryByAgent.get(agentId) === recovery &&
2959
+ !recovery.superseded
2960
+ );
2961
+ }
2962
+
2188
2963
  /**
2189
2964
  * Retire turn-local state owned by a runner generation before replacement.
2190
2965
  *
@@ -2618,6 +3393,13 @@ export class Orchestrator {
2618
3393
  if (channel.sessions.get(agentId) !== session) break;
2619
3394
  // The turn died with the session — drop its turn-state flags so a
2620
3395
  // respawned session starts clean.
3396
+ const abortBoundaryOnRosterChange =
3397
+ this.needsAbortedTurnBoundary(channel, agentId);
3398
+ this.retainRetiredTurnBoundary(
3399
+ channel,
3400
+ agentId,
3401
+ abortBoundaryOnRosterChange,
3402
+ );
2621
3403
  this.retireRunnerGeneration(channel, agentId);
2622
3404
  // Claude under load (especially Docker Desktop macOS) occasionally
2623
3405
  // unlinks ~/.claude.json mid-write during atomic config rewrites,
@@ -2627,13 +3409,32 @@ export class Orchestrator {
2627
3409
  // session so the user keeps working instead of staring at a dead
2628
3410
  // chat.
2629
3411
  const agent = channel.roster.find((a) => a.id === agentId);
3412
+ if (channel.rosterSessionChanges.has(agentId)) {
3413
+ // The authoritative roster already wants a different launch. Do not
3414
+ // rotate/recover this obsolete generation with its captured account;
3415
+ // preserve the live adapter so the normal stale-session recycler
3416
+ // closes it (and fences late Codex notifications). The cloud still
3417
+ // needs a terminal edge for any partial turn before restarting.
3418
+ this.reconcileErroredRosterChange(
3419
+ channel,
3420
+ agentId,
3421
+ event.message,
3422
+ abortBoundaryOnRosterChange,
3423
+ );
3424
+ break;
3425
+ }
2630
3426
  // ADR-076: a rate-limit/usage-cap error fails over to another account
2631
3427
  // for this kind (cool the current one, recreate the session on the
2632
3428
  // next, re-deliver the in-flight prompt). No-op with <2 accounts.
2633
3429
  if (
2634
3430
  event.retryable === "rate_limit" &&
2635
3431
  agent &&
2636
- (await this.rotateAccount(channel, agentId, agent))
3432
+ (await this.rotateAccount(
3433
+ channel,
3434
+ agentId,
3435
+ agent,
3436
+ abortBoundaryOnRosterChange,
3437
+ ))
2637
3438
  ) {
2638
3439
  break;
2639
3440
  }
@@ -2650,13 +3451,71 @@ export class Orchestrator {
2650
3451
  `[orchestrator] ${channel.taskId}/${agentId}: ${agent.kind} credential-invalid pattern matched: ${event.message.slice(0, 300)}`,
2651
3452
  );
2652
3453
  const accountId = channel.accountByAgent.get(agentId) ?? null;
2653
- const requested = await requestCredentialRefresh({
2654
- kind: agent.kind,
2655
- accountId: accountId && !accountId.startsWith("default:") ? accountId : null,
2656
- scope: channel.engineAccountScope,
2657
- });
3454
+ // The broker is a slow await between a failed turn and its recovery.
3455
+ // Own the spawn slot so a concurrent roster edit cannot publish
3456
+ // ready for a replacement before this old generation publishes its
3457
+ // aborted boundary (events carry no generation id in the cloud).
3458
+ const ownsBrokerSlot = !channel.spawning.has(agentId);
3459
+ if (ownsBrokerSlot) channel.spawning.add(agentId);
3460
+ let requested = false;
3461
+ try {
3462
+ requested = await requestCredentialRefresh({
3463
+ kind: agent.kind,
3464
+ accountId:
3465
+ accountId && !accountId.startsWith("default:")
3466
+ ? accountId
3467
+ : null,
3468
+ scope: channel.engineAccountScope,
3469
+ });
3470
+ } catch (err) {
3471
+ console.warn(
3472
+ `[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
3473
+ err instanceof Error ? err.message : String(err)
3474
+ }`,
3475
+ );
3476
+ } finally {
3477
+ if (ownsBrokerSlot) channel.spawning.delete(agentId);
3478
+ }
3479
+ // Removal/edit/recovery may have retired this generation while the
3480
+ // broker request was in flight. Never let its continuation mutate a
3481
+ // replacement or repopulate state for an absent roster id.
3482
+ if (channel.sessions.get(agentId) !== session) {
3483
+ // An edit can complete its restart while the broker request is
3484
+ // still outstanding, consuming the roster-change marker at bind.
3485
+ // The captured launch tuple still proves that this failed turn
3486
+ // belongs to the retired generation and needs its aborted edge.
3487
+ const stillRostered = channel.roster.some(
3488
+ (candidate) => candidate.id === agentId,
3489
+ );
3490
+ if (
3491
+ stillRostered &&
3492
+ (!channel.sessions.has(agentId) ||
3493
+ !this.hasDesiredLaunchSettings(channel, agentId, agent))
3494
+ ) {
3495
+ this.reconcileErroredRosterChange(
3496
+ channel,
3497
+ agentId,
3498
+ event.message,
3499
+ abortBoundaryOnRosterChange,
3500
+ );
3501
+ }
3502
+ break;
3503
+ }
3504
+ if (channel.rosterSessionChanges.has(agentId)) {
3505
+ this.reconcileErroredRosterChange(
3506
+ channel,
3507
+ agentId,
3508
+ event.message,
3509
+ abortBoundaryOnRosterChange,
3510
+ );
3511
+ break;
3512
+ }
2658
3513
  if (requested) {
2659
- await this.refreshAgentToken(channel, agent);
3514
+ await this.refreshAgentToken(
3515
+ channel,
3516
+ agent,
3517
+ abortBoundaryOnRosterChange,
3518
+ );
2660
3519
  break;
2661
3520
  }
2662
3521
  }
@@ -2670,7 +3529,11 @@ export class Orchestrator {
2670
3529
  console.warn(
2671
3530
  `[orchestrator] ${channel.taskId}/${agentId}: claude auth-revoked pattern matched: ${event.message.slice(0, 300)}`,
2672
3531
  );
2673
- await this.refreshAgentToken(channel, agent);
3532
+ await this.refreshAgentToken(
3533
+ channel,
3534
+ agent,
3535
+ abortBoundaryOnRosterChange,
3536
+ );
2674
3537
  break;
2675
3538
  }
2676
3539
  const configMissing = CLAUDE_CONFIG_MISSING_PATTERNS.some((re) =>
@@ -2681,10 +3544,15 @@ export class Orchestrator {
2681
3544
  // than replaying the failed one. Do not leave it looking in-flight to
2682
3545
  // a later account refresh.
2683
3546
  channel.lastPrompt.delete(agentId);
2684
- await this.recoverClaudeAgent(channel, agentId);
3547
+ await this.recoverClaudeAgent(
3548
+ channel,
3549
+ agentId,
3550
+ abortBoundaryOnRosterChange,
3551
+ );
2685
3552
  break;
2686
3553
  }
2687
3554
  channel.lastPrompt.delete(agentId);
3555
+ channel.retiredTurnBoundaries.delete(agentId);
2688
3556
  this.emitHost({
2689
3557
  kind: "agent.exit",
2690
3558
  taskId: channel.taskId,
@@ -2730,7 +3598,11 @@ export class Orchestrator {
2730
3598
  });
2731
3599
  if (remainingTurns === 0) {
2732
3600
  channel.lastPrompt.delete(agentId);
2733
- this.startNextPendingPrompt(channel, agentId);
3601
+ // A queued Secretary reply must run on the replacement settings,
3602
+ // not extend the obsolete generation past its first safe boundary.
3603
+ if (!channel.rosterSessionChanges.has(agentId)) {
3604
+ this.startNextPendingPrompt(channel, agentId);
3605
+ }
2734
3606
  }
2735
3607
  // A config repair never interrupts a turn. The stale id remains queued
2736
3608
  // until this exact boundary, then reconcile closes and replaces it.
@@ -2761,18 +3633,73 @@ export class Orchestrator {
2761
3633
  if (channel.sessions.get(agentId) !== session) break;
2762
3634
  // Same zombie hazard as the error path — a session whose process
2763
3635
  // ended (even cleanly) can never carry another turn.
3636
+ const abortBoundaryOnRosterChange =
3637
+ this.needsAbortedTurnBoundary(channel, agentId);
2764
3638
  this.retireRunnerGeneration(channel, agentId);
2765
3639
  channel.lastPrompt.delete(agentId);
2766
3640
  channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
2767
3641
  channel.respawnLastAt.set(agentId, Date.now());
2768
3642
  channel.sessions.delete(agentId);
2769
- channel.browserStaleSessions.delete(agentId);
3643
+ const rosterChangePending = channel.rosterSessionChanges.has(agentId);
3644
+ if (rosterChangePending && abortBoundaryOnRosterChange) {
3645
+ this.publishRosterChangeAbort(
3646
+ channel,
3647
+ agentId,
3648
+ channel.rosterSessionChanges.get(agentId)!,
3649
+ );
3650
+ }
3651
+ if (!rosterChangePending) channel.browserStaleSessions.delete(agentId);
2770
3652
  channel.browserPendingStaleSessions.delete(agentId);
2771
3653
  this.reconcilePendingSecretary(channel, agentId);
3654
+ if (rosterChangePending) {
3655
+ void this.reconcileSessions(channel).catch((err: unknown) => {
3656
+ console.warn(
3657
+ `[orchestrator] ${channel.taskId}/${agentId}: roster-change exit reconcile failed: ${
3658
+ err instanceof Error ? err.message : String(err)
3659
+ }`,
3660
+ );
3661
+ });
3662
+ }
2772
3663
  break;
2773
3664
  }
2774
3665
  }
2775
3666
 
3667
+ /** An engine error can race a live launch edit after a partial turn. Keep
3668
+ * the obsolete adapter owned until the stale recycler closes it, but close
3669
+ * the cloud turn boundary immediately so restarting/ready cannot leave the
3670
+ * crew UI stuck in a working state. */
3671
+ private reconcileErroredRosterChange(
3672
+ channel: Channel,
3673
+ agentId: string,
3674
+ reason: string,
3675
+ abortBoundary: boolean,
3676
+ ): void {
3677
+ channel.lastPrompt.delete(agentId);
3678
+ channel.browserPendingStaleSessions.delete(agentId);
3679
+ if (abortBoundary) {
3680
+ const change = channel.rosterSessionChanges.get(agentId);
3681
+ if (change) this.publishRosterChangeAbort(channel, agentId, change);
3682
+ else this.emitRetiredTurnAbort(channel, agentId);
3683
+ }
3684
+ void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
3685
+ console.warn(
3686
+ `[orchestrator] ${channel.taskId}/${agentId}: roster-change error (${reason}) reconcile failed: ${
3687
+ err instanceof Error ? err.message : String(err)
3688
+ }`,
3689
+ );
3690
+ });
3691
+ }
3692
+
3693
+ /** Removal owns an asynchronous close/durable-stop fence. Recovery paths
3694
+ * overtaken by remove/re-add must join it before they can spawn the re-added
3695
+ * id, otherwise removal can stop the replacement generation. */
3696
+ private async reconcileAfterRosterRefresh(channel: Channel): Promise<void> {
3697
+ const rosterRefresh = channel.rosterRefreshReady;
3698
+ if (rosterRefresh) await rosterRefresh;
3699
+ if (!this.isActiveChannel(channel)) return;
3700
+ await this.reconcileSessions(channel);
3701
+ }
3702
+
2776
3703
  /** A daemon loss is infrastructure recovery, not an agent crash. Preserve
2777
3704
  * accepted work, drop the dead generation, and leave respawn/exit accounting
2778
3705
  * untouched; the runtime-ready hook recreates it deterministically. */
@@ -2797,9 +3724,20 @@ export class Orchestrator {
2797
3724
  channel.runtimeDeferredAgents.add(agentId);
2798
3725
  }
2799
3726
  channel.sessions.delete(agentId);
2800
- channel.browserStaleSessions.delete(agentId);
3727
+ if (!channel.rosterSessionChanges.has(agentId)) {
3728
+ channel.browserStaleSessions.delete(agentId);
3729
+ }
2801
3730
  channel.browserPendingStaleSessions.delete(agentId);
2802
3731
  await session.close().catch(() => {});
3732
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
3733
+ if (rosterChange && this.isActiveChannel(channel)) {
3734
+ await this.closeRosterChangedSession(
3735
+ channel,
3736
+ agentId,
3737
+ session,
3738
+ rosterChange,
3739
+ );
3740
+ }
2803
3741
  }
2804
3742
 
2805
3743
  // -- session recovery -----------------------------------------------------
@@ -2812,6 +3750,7 @@ export class Orchestrator {
2812
3750
  private async recoverClaudeAgent(
2813
3751
  channel: Channel,
2814
3752
  agentId: string,
3753
+ abortBoundaryOnRosterChange = false,
2815
3754
  ): Promise<void> {
2816
3755
  if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
2817
3756
 
@@ -2823,6 +3762,7 @@ export class Orchestrator {
2823
3762
  channel.sessions.delete(agentId);
2824
3763
  channel.browserStaleSessions.delete(agentId);
2825
3764
  channel.browserPendingStaleSessions.delete(agentId);
3765
+ channel.retiredTurnBoundaries.delete(agentId);
2826
3766
  this.emitHost({
2827
3767
  kind: "agent.exit",
2828
3768
  taskId: channel.taskId,
@@ -2841,6 +3781,15 @@ export class Orchestrator {
2841
3781
  channel.containerName,
2842
3782
  channel.agentHandle ?? undefined,
2843
3783
  );
3784
+ if (channel.rosterSessionChanges.has(agentId)) {
3785
+ this.reconcileErroredRosterChange(
3786
+ channel,
3787
+ agentId,
3788
+ "Claude config recovery superseded by live roster edit",
3789
+ abortBoundaryOnRosterChange,
3790
+ );
3791
+ return;
3792
+ }
2844
3793
 
2845
3794
  // Delete the old generation before awaiting close. Its adapter emits a
2846
3795
  // terminal event during teardown; bindSession's identity guard must see
@@ -2848,7 +3797,9 @@ export class Orchestrator {
2848
3797
  // a second replacement path.
2849
3798
  const old = channel.sessions.get(agentId);
2850
3799
  channel.sessions.delete(agentId);
2851
- channel.browserStaleSessions.delete(agentId);
3800
+ if (!channel.rosterSessionChanges.has(agentId)) {
3801
+ channel.browserStaleSessions.delete(agentId);
3802
+ }
2852
3803
  channel.browserPendingStaleSessions.delete(agentId);
2853
3804
  this.retireRunnerGeneration(channel, agentId);
2854
3805
  if (old) {
@@ -2859,6 +3810,27 @@ export class Orchestrator {
2859
3810
  }
2860
3811
  }
2861
3812
  if (!this.isActiveChannel(channel)) return;
3813
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
3814
+ if (rosterChange) {
3815
+ // The generic recovery owned the close before the roster edit arrived.
3816
+ // close() is idempotent; route its completed boundary through the
3817
+ // roster lifecycle so the cloud still observes restarting -> ready.
3818
+ this.reconcileErroredRosterChange(
3819
+ channel,
3820
+ agentId,
3821
+ "Claude config recovery superseded by live roster edit",
3822
+ abortBoundaryOnRosterChange,
3823
+ );
3824
+ if (old) {
3825
+ await this.closeRosterChangedSession(
3826
+ channel,
3827
+ agentId,
3828
+ old,
3829
+ rosterChange,
3830
+ );
3831
+ }
3832
+ return;
3833
+ }
2862
3834
 
2863
3835
  const agent = channel.roster.find((a) => a.id === agentId);
2864
3836
  const task = getHostTask(channel.taskId);
@@ -2887,18 +3859,40 @@ export class Orchestrator {
2887
3859
  executionProfile: this.executionProfileFor(channel, agentId),
2888
3860
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2889
3861
  channel,
2890
- agentId,
3862
+ agent,
2891
3863
  ),
2892
3864
  agentEnv: boundAccount
2893
3865
  ? { ...boundAccount.execEnv, ...base }
2894
3866
  : this.accountAgentEnv(channel, agent, base),
2895
3867
  });
2896
- if (!session) return;
3868
+ if (!session) {
3869
+ if (channel.rosterSessionChanges.has(agentId)) {
3870
+ this.reconcileErroredRosterChange(
3871
+ channel,
3872
+ agentId,
3873
+ "Claude config recovery superseded by live roster edit",
3874
+ abortBoundaryOnRosterChange,
3875
+ );
3876
+ }
3877
+ return;
3878
+ }
3879
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
3880
+ if (channel.rosterSessionChanges.has(agentId)) {
3881
+ this.reconcileErroredRosterChange(
3882
+ channel,
3883
+ agentId,
3884
+ "Claude config recovery superseded by live roster edit",
3885
+ abortBoundaryOnRosterChange,
3886
+ );
3887
+ }
3888
+ return;
3889
+ }
2897
3890
  this.startNextPendingPrompt(channel, agentId);
2898
3891
 
2899
3892
  const message = restored
2900
3893
  ? `${agentId} restarted — config file was missing in the container, restored from the host.`
2901
3894
  : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
3895
+ channel.retiredTurnBoundaries.delete(agentId);
2902
3896
  this.emitHost({
2903
3897
  kind: "agent.exit",
2904
3898
  taskId: channel.taskId,
@@ -2940,8 +3934,18 @@ export class Orchestrator {
2940
3934
  channel: Channel,
2941
3935
  agentId: string,
2942
3936
  agent: RosterAgent,
3937
+ abortBoundaryOnRosterChange = false,
2943
3938
  ): Promise<boolean> {
2944
3939
  if (!this.isActiveChannel(channel)) return true;
3940
+ if (channel.rosterSessionChanges.has(agentId)) {
3941
+ this.reconcileErroredRosterChange(
3942
+ channel,
3943
+ agentId,
3944
+ "account rotation superseded by live roster edit",
3945
+ abortBoundaryOnRosterChange,
3946
+ );
3947
+ return true;
3948
+ }
2945
3949
  // A concurrent terminal event for the same session is already being
2946
3950
  // handled by the path that owns this spawn slot.
2947
3951
  if (channel.spawning.has(agentId)) return true;
@@ -2964,6 +3968,8 @@ export class Orchestrator {
2964
3968
  const task = getHostTask(channel.taskId);
2965
3969
  if (!task || task.statusMirror !== "running") return false;
2966
3970
 
3971
+ const recovery: RecoveryFence = { superseded: false };
3972
+ channel.recoveryByAgent.set(agentId, recovery);
2967
3973
  let replacement: AgentSession | null = null;
2968
3974
  channel.spawning.add(agentId);
2969
3975
  try {
@@ -2974,6 +3980,11 @@ export class Orchestrator {
2974
3980
  channel.sessions.delete(agentId);
2975
3981
  channel.browserStaleSessions.delete(agentId);
2976
3982
  channel.browserPendingStaleSessions.delete(agentId);
3983
+ this.retainRetiredTurnBoundary(
3984
+ channel,
3985
+ agentId,
3986
+ abortBoundaryOnRosterChange,
3987
+ );
2977
3988
  this.retireRunnerGeneration(channel, agentId);
2978
3989
  if (old) {
2979
3990
  try {
@@ -2983,6 +3994,25 @@ export class Orchestrator {
2983
3994
  }
2984
3995
  }
2985
3996
  if (!this.isActiveChannel(channel)) return true;
3997
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
3998
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
3999
+ if (rosterChange) {
4000
+ this.reconcileErroredRosterChange(
4001
+ channel,
4002
+ agentId,
4003
+ "account rotation superseded by live roster edit",
4004
+ abortBoundaryOnRosterChange,
4005
+ );
4006
+ if (old) {
4007
+ await this.closeRosterChangedSession(
4008
+ channel,
4009
+ agentId,
4010
+ old,
4011
+ rosterChange,
4012
+ );
4013
+ }
4014
+ return true;
4015
+ }
2986
4016
 
2987
4017
  // The serialized materialization gate owns both account provisioning
2988
4018
  // and, when enabled, browser migration.
@@ -2992,8 +4022,21 @@ export class Orchestrator {
2992
4022
  true,
2993
4023
  );
2994
4024
  if (!this.isActiveChannel(channel)) return true;
4025
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
2995
4026
  await this.ensureMcpConfig(channel);
2996
4027
  if (!this.isActiveChannel(channel)) return true;
4028
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4029
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4030
+ if (channel.rosterSessionChanges.has(agentId)) {
4031
+ this.reconcileErroredRosterChange(
4032
+ channel,
4033
+ agentId,
4034
+ "account rotation superseded by live roster edit",
4035
+ abortBoundaryOnRosterChange,
4036
+ );
4037
+ }
4038
+ return true;
4039
+ }
2997
4040
 
2998
4041
  const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
2999
4042
  const cliSecret = loadTaskCliSecret(channel.taskId);
@@ -3018,12 +4061,34 @@ export class Orchestrator {
3018
4061
  executionProfile: this.executionProfileFor(channel, agentId),
3019
4062
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
3020
4063
  channel,
3021
- agentId,
4064
+ agent,
3022
4065
  ),
3023
4066
  agentEnv: { ...next.execEnv, ...base },
3024
4067
  },
4068
+ recovery,
3025
4069
  );
3026
- if (!replacement) return true;
4070
+ if (!replacement) {
4071
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4072
+ this.reconcileErroredRosterChange(
4073
+ channel,
4074
+ agentId,
4075
+ "account rotation superseded by live roster edit",
4076
+ abortBoundaryOnRosterChange,
4077
+ );
4078
+ }
4079
+ return true;
4080
+ }
4081
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4082
+ if (channel.rosterSessionChanges.has(agentId)) {
4083
+ this.reconcileErroredRosterChange(
4084
+ channel,
4085
+ agentId,
4086
+ "account rotation superseded by live roster edit",
4087
+ abortBoundaryOnRosterChange,
4088
+ );
4089
+ }
4090
+ return true;
4091
+ }
3027
4092
 
3028
4093
  this.emitSystemNote(
3029
4094
  channel.taskId,
@@ -3033,14 +4098,35 @@ export class Orchestrator {
3033
4098
  // Re-deliver the in-flight prompt so the interrupted turn resumes.
3034
4099
  const prompt = channel.lastPrompt.get(agentId);
3035
4100
  if (prompt) {
4101
+ channel.retiredTurnBoundaries.delete(agentId);
3036
4102
  this.startPrompt(channel, agentId, replacement, prompt);
3037
4103
  } else {
4104
+ this.emitRetiredTurnAbort(channel, agentId);
3038
4105
  this.startNextPendingPrompt(channel, agentId);
3039
4106
  }
3040
4107
  } finally {
3041
4108
  channel.spawning.delete(agentId);
4109
+ if (channel.recoveryByAgent.get(agentId) === recovery) {
4110
+ channel.recoveryByAgent.delete(agentId);
4111
+ }
3042
4112
  if (!replacement) {
4113
+ if (!channel.roster.some((candidate) => candidate.id === agentId)) {
4114
+ channel.accountByAgent.delete(agentId);
4115
+ channel.rotations.delete(agentId);
4116
+ }
3043
4117
  this.reconcilePendingSecretary(channel, agentId);
4118
+ if (
4119
+ this.isActiveChannel(channel) &&
4120
+ (recovery.superseded || channel.rosterSessionChanges.has(agentId))
4121
+ ) {
4122
+ void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
4123
+ console.warn(
4124
+ `[orchestrator] ${channel.taskId}/${agentId}: post-rotation roster reconcile failed: ${
4125
+ err instanceof Error ? err.message : String(err)
4126
+ }`,
4127
+ );
4128
+ });
4129
+ }
3044
4130
  }
3045
4131
  }
3046
4132
 
@@ -3077,10 +4163,23 @@ export class Orchestrator {
3077
4163
  agent: RosterAgent,
3078
4164
  account: ResolvedEngineAccount,
3079
4165
  note: string,
4166
+ abortBoundaryOnRosterChange = false,
3080
4167
  ): Promise<boolean> {
4168
+ if (channel.rosterSessionChanges.has(agentId)) {
4169
+ this.reconcileErroredRosterChange(
4170
+ channel,
4171
+ agentId,
4172
+ "credential respawn superseded by live roster edit",
4173
+ abortBoundaryOnRosterChange,
4174
+ );
4175
+ return true;
4176
+ }
4177
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return true;
3081
4178
  const task = getHostTask(channel.taskId);
3082
4179
  if (!task || task.statusMirror !== "running") return false;
3083
4180
 
4181
+ const recovery: RecoveryFence = { superseded: false };
4182
+ channel.recoveryByAgent.set(agentId, recovery);
3084
4183
  let replacement: AgentSession | null = null;
3085
4184
  channel.spawning.add(agentId);
3086
4185
  try {
@@ -3102,6 +4201,25 @@ export class Orchestrator {
3102
4201
  }
3103
4202
  }
3104
4203
  if (!this.isActiveChannel(channel)) return true;
4204
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4205
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
4206
+ if (rosterChange) {
4207
+ this.reconcileErroredRosterChange(
4208
+ channel,
4209
+ agentId,
4210
+ "credential respawn superseded by live roster edit",
4211
+ abortBoundaryOnRosterChange,
4212
+ );
4213
+ if (old) {
4214
+ await this.closeRosterChangedSession(
4215
+ channel,
4216
+ agentId,
4217
+ old,
4218
+ rosterChange,
4219
+ );
4220
+ }
4221
+ return true;
4222
+ }
3105
4223
 
3106
4224
  await this.ensureBrowserSetup(
3107
4225
  channel,
@@ -3109,8 +4227,21 @@ export class Orchestrator {
3109
4227
  true,
3110
4228
  );
3111
4229
  if (!this.isActiveChannel(channel)) return true;
4230
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
3112
4231
  await this.ensureMcpConfig(channel);
3113
4232
  if (!this.isActiveChannel(channel)) return true;
4233
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4234
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4235
+ if (channel.rosterSessionChanges.has(agentId)) {
4236
+ this.reconcileErroredRosterChange(
4237
+ channel,
4238
+ agentId,
4239
+ "credential respawn superseded by live roster edit",
4240
+ abortBoundaryOnRosterChange,
4241
+ );
4242
+ }
4243
+ return true;
4244
+ }
3114
4245
 
3115
4246
  const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
3116
4247
  const cliSecret = loadTaskCliSecret(channel.taskId);
@@ -3125,32 +4256,79 @@ export class Orchestrator {
3125
4256
  channel.accountByAgent.set(agentId, account.id);
3126
4257
  noteEngineAccountUsed(account.id);
3127
4258
 
3128
- replacement = await this.createAndBindStableSession(channel, agentId, {
3129
- taskId: channel.taskId,
3130
- agent,
3131
- systemPreamble: channel.preambles.get(agentId) ?? "",
3132
- executionProfile: this.executionProfileFor(channel, agentId),
3133
- attachCompatibilityKey: this.attachCompatibilityKeyFor(
3134
- channel,
3135
- agentId,
3136
- ),
3137
- agentEnv: { ...account.execEnv, ...base },
3138
- });
3139
- if (!replacement) return true;
4259
+ replacement = await this.createAndBindStableSession(
4260
+ channel,
4261
+ agentId,
4262
+ {
4263
+ taskId: channel.taskId,
4264
+ agent,
4265
+ systemPreamble: channel.preambles.get(agentId) ?? "",
4266
+ executionProfile: this.executionProfileFor(channel, agentId),
4267
+ attachCompatibilityKey: this.attachCompatibilityKeyFor(
4268
+ channel,
4269
+ agent,
4270
+ ),
4271
+ agentEnv: { ...account.execEnv, ...base },
4272
+ },
4273
+ recovery,
4274
+ );
4275
+ if (!replacement) {
4276
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4277
+ this.reconcileErroredRosterChange(
4278
+ channel,
4279
+ agentId,
4280
+ "credential respawn superseded by live roster edit",
4281
+ abortBoundaryOnRosterChange,
4282
+ );
4283
+ }
4284
+ return true;
4285
+ }
4286
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4287
+ if (channel.rosterSessionChanges.has(agentId)) {
4288
+ this.reconcileErroredRosterChange(
4289
+ channel,
4290
+ agentId,
4291
+ "credential respawn superseded by live roster edit",
4292
+ abortBoundaryOnRosterChange,
4293
+ );
4294
+ }
4295
+ return true;
4296
+ }
3140
4297
 
3141
4298
  this.emitSystemNote(channel.taskId, note);
3142
4299
 
3143
4300
  // Re-deliver the in-flight prompt so the interrupted turn resumes.
3144
4301
  const prompt = channel.lastPrompt.get(agentId);
3145
4302
  if (prompt) {
4303
+ channel.retiredTurnBoundaries.delete(agentId);
3146
4304
  this.startPrompt(channel, agentId, replacement, prompt);
3147
4305
  } else {
4306
+ this.emitRetiredTurnAbort(channel, agentId);
3148
4307
  this.startNextPendingPrompt(channel, agentId);
3149
4308
  }
3150
4309
  } finally {
3151
4310
  channel.spawning.delete(agentId);
4311
+ if (channel.recoveryByAgent.get(agentId) === recovery) {
4312
+ channel.recoveryByAgent.delete(agentId);
4313
+ }
3152
4314
  if (!replacement) {
4315
+ if (!channel.roster.some((candidate) => candidate.id === agentId)) {
4316
+ channel.accountByAgent.delete(agentId);
4317
+ channel.rotations.delete(agentId);
4318
+ }
3153
4319
  this.reconcilePendingSecretary(channel, agentId);
4320
+ if (
4321
+ this.isActiveChannel(channel) &&
4322
+ (recovery.superseded || channel.rosterSessionChanges.has(agentId))
4323
+ ) {
4324
+ void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
4325
+ console.warn(
4326
+ `[orchestrator] ${channel.taskId}/${agentId}: post-respawn roster reconcile failed: ${
4327
+ err instanceof Error ? err.message : String(err)
4328
+ }`,
4329
+ );
4330
+ });
4331
+ }
3154
4332
  }
3155
4333
  }
3156
4334
 
@@ -3179,14 +4357,21 @@ export class Orchestrator {
3179
4357
  private async refreshAgentToken(
3180
4358
  channel: Channel,
3181
4359
  agent: RosterAgent,
4360
+ abortBoundaryOnRosterChange?: boolean,
3182
4361
  ): Promise<void> {
3183
4362
  const agentId = agent.id;
3184
4363
  if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
4364
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return;
4365
+ const abortBoundary =
4366
+ abortBoundaryOnRosterChange ??
4367
+ this.needsAbortedTurnBoundary(channel, agentId);
4368
+ this.retainRetiredTurnBoundary(channel, agentId, abortBoundary);
3185
4369
 
3186
4370
  const tries = (channel.respawns.get(agentId) ?? 0) + 1;
3187
4371
  channel.respawns.set(agentId, tries);
3188
4372
  channel.respawnLastAt.set(agentId, Date.now());
3189
4373
  if (tries > MAX_RESPAWNS_PER_AGENT) {
4374
+ channel.retiredTurnBoundaries.delete(agentId);
3190
4375
  this.emitHost({
3191
4376
  kind: "agent.exit",
3192
4377
  taskId: channel.taskId,
@@ -3203,7 +4388,10 @@ export class Orchestrator {
3203
4388
  const account =
3204
4389
  accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
3205
4390
  accounts[0];
3206
- if (!account) return;
4391
+ if (!account) {
4392
+ this.emitRetiredTurnAbort(channel, agentId);
4393
+ return;
4394
+ }
3207
4395
 
3208
4396
  await this.respawnAgentOnAccount(
3209
4397
  channel,
@@ -3211,6 +4399,7 @@ export class Orchestrator {
3211
4399
  agent,
3212
4400
  account,
3213
4401
  `${agentId}: ${agent.kind} credential refreshed — restarted the agent.`,
4402
+ abortBoundary,
3214
4403
  );
3215
4404
  }
3216
4405
 
@@ -3252,9 +4441,21 @@ export class Orchestrator {
3252
4441
  async refreshEngineAccountAgents(kind: string): Promise<void> {
3253
4442
  for (const channel of this.channels.values()) {
3254
4443
  if (!this.isActiveChannel(channel)) continue;
3255
- for (const agent of channel.roster) {
3256
- if (agent.kind !== kind || !channel.sessions.has(agent.id)) continue;
3257
- channel.respawns.set(agent.id, 0);
4444
+ const agentIds = channel.roster
4445
+ .filter((agent) => agent.kind === kind)
4446
+ .map((agent) => agent.id);
4447
+ for (const agentId of agentIds) {
4448
+ const agent = channel.roster.find(
4449
+ (candidate) => candidate.id === agentId,
4450
+ );
4451
+ if (
4452
+ !agent ||
4453
+ agent.kind !== kind ||
4454
+ !channel.sessions.has(agentId)
4455
+ ) {
4456
+ continue;
4457
+ }
4458
+ channel.respawns.set(agentId, 0);
3258
4459
  try {
3259
4460
  await this.refreshAgentToken(channel, agent);
3260
4461
  } catch (err) {
@@ -3299,7 +4500,10 @@ export class Orchestrator {
3299
4500
  this.channels.delete(taskId);
3300
4501
  this.clearBrowserRetry(ch);
3301
4502
  ch.browserStaleSessions.clear();
4503
+ ch.rosterSessionChanges.clear();
3302
4504
  ch.browserPendingStaleSessions.clear();
4505
+ ch.browserConfigAffectedKinds.clear();
4506
+ ch.mcpProvisionAffectedKinds.clear();
3303
4507
  ch.activeTurns.clear();
3304
4508
  ch.acceptedPrompts.clear();
3305
4509
  ch.pendingPrompts.clear();
@@ -3307,17 +4511,22 @@ export class Orchestrator {
3307
4511
  ch.runtimeDeferredAgents.clear();
3308
4512
  ch.openTurns.clear();
3309
4513
  ch.interrupted.clear();
4514
+ ch.retiredTurnBoundaries.clear();
4515
+ ch.recoveryByAgent.clear();
3310
4516
  ch.reconcileAgain = false;
3311
4517
  const drains = [
3312
4518
  ch.browserSetup,
3313
4519
  ch.mcpConfigWrite,
3314
4520
  ch.reconcileReady,
4521
+ ch.rosterRefreshReady,
4522
+ ch.persistedRosterReady,
3315
4523
  ch.sessionsReady,
3316
4524
  ...ch.factoryOperations,
3317
4525
  ...ch.eventOperations,
3318
4526
  ].filter((promise): promise is Promise<unknown> => promise !== null);
3319
4527
  const sessions = [...ch.sessions.values()];
3320
4528
  ch.sessions.clear();
4529
+ ch.factoryOperationByAgent.clear();
3321
4530
  const closes = sessions.map((session) =>
3322
4531
  session.close().catch(() => {
3323
4532
  // Teardown is best-effort; every adapter close is intended idempotent.
@@ -3441,6 +4650,18 @@ export class Orchestrator {
3441
4650
  }
3442
4651
  }
3443
4652
 
4653
+ /** Launch-time settings only. Wire-compatible absence spellings are equal. */
4654
+ function rosterLaunchChangeReason(
4655
+ previous: RosterAgent,
4656
+ next: RosterAgent,
4657
+ ): string | null {
4658
+ const changed: string[] = [];
4659
+ if (previous.kind !== next.kind) changed.push("engine");
4660
+ if ((previous.model ?? "") !== (next.model ?? "")) changed.push("model");
4661
+ if ((previous.effort ?? "") !== (next.effort ?? "")) changed.push("effort");
4662
+ return changed.length > 0 ? `${changed.join("/")} change` : null;
4663
+ }
4664
+
3444
4665
  /** Exact host-wire role validation; never trims or canonicalises agent ids. */
3445
4666
  export function hasValidSecretarySelection(
3446
4667
  spec: Pick<