@runuai/host 0.9.147 → 0.9.149

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/db/migrations/0024_task_env_mode.sql +9 -0
  2. package/db/migrations/0025_project_env_authority.sql +147 -0
  3. package/db/migrations/0026_project_env_replica.sql +150 -0
  4. package/db/migrations/meta/_journal.json +21 -0
  5. package/db/schema.ts +312 -2
  6. package/images/standard/Dockerfile +44 -6
  7. package/images/standard/container/uai-init +85 -2
  8. package/images/standard/container/uai-task-forwarder.mjs +303 -0
  9. package/lib/agent-cli.ts +18 -1
  10. package/lib/agent.ts +99 -34
  11. package/lib/agents/attach-compatibility.ts +51 -0
  12. package/lib/agents/codex.ts +137 -13
  13. package/lib/agents/cursor.ts +15 -1
  14. package/lib/agents/durable-proc.ts +2 -0
  15. package/lib/agents/grok.ts +8 -0
  16. package/lib/agents/kimi.ts +1 -0
  17. package/lib/agents/machine-durable.ts +17 -0
  18. package/lib/agents/opencode.ts +1 -0
  19. package/lib/agents/proc.ts +6 -1
  20. package/lib/agents/session-state.ts +107 -0
  21. package/lib/agents/transport.ts +125 -58
  22. package/lib/agents/types.ts +3 -3
  23. package/lib/browser-testing.ts +6 -0
  24. package/lib/codex-auth.ts +2 -0
  25. package/lib/credential-store.ts +141 -6
  26. package/lib/engine-accounts.ts +1 -0
  27. package/lib/env-file.ts +7 -0
  28. package/lib/env-schema.ts +841 -51
  29. package/lib/git-diff.ts +1 -0
  30. package/lib/git-identity.ts +2 -0
  31. package/lib/github-tokens.ts +1 -0
  32. package/lib/host-env.ts +98 -18
  33. package/lib/lease.ts +11 -3
  34. package/lib/machine-gateway-tunnel.ts +195 -17
  35. package/lib/machine-project-env-loader.ts +152 -0
  36. package/lib/managed-runtime.ts +270 -51
  37. package/lib/mcp-gateway.ts +777 -26
  38. package/lib/orchestrator.ts +1681 -138
  39. package/lib/project-env-authority.ts +1319 -0
  40. package/lib/project-env-feature.ts +74 -0
  41. package/lib/project-env-loader.ts +237 -0
  42. package/lib/project-env-replica.ts +2094 -0
  43. package/lib/project-env-schema-capture.ts +342 -0
  44. package/lib/project-env-staging.ts +148 -0
  45. package/lib/project-env-task-policy.ts +176 -0
  46. package/lib/runtime-archive.ts +79 -16
  47. package/lib/runtime-authority.ts +44 -0
  48. package/lib/runtime-state.ts +175 -10
  49. package/lib/secrets-proxy/address.ts +72 -0
  50. package/lib/secrets-proxy/audit.ts +30 -0
  51. package/lib/secrets-proxy/ca-store.ts +200 -0
  52. package/lib/secrets-proxy/ca.ts +470 -0
  53. package/lib/secrets-proxy/container-steps.ts +138 -0
  54. package/lib/secrets-proxy/der.ts +273 -0
  55. package/lib/secrets-proxy/engine.ts +1633 -0
  56. package/lib/secrets-proxy/host.ts +93 -0
  57. package/lib/secrets-proxy/hostname.ts +60 -0
  58. package/lib/secrets-proxy/placeholders.ts +196 -0
  59. package/lib/secrets-proxy/policy.ts +141 -0
  60. package/lib/secrets-proxy/readiness.ts +148 -0
  61. package/lib/secrets-proxy/runtime.ts +582 -0
  62. package/lib/secrets-proxy/scrub.ts +559 -0
  63. package/lib/secrets-proxy/sidecar-view.ts +139 -0
  64. package/lib/secrets-proxy/task-scripts.ts +262 -0
  65. package/lib/secrets-proxy/task-trust.ts +574 -0
  66. package/lib/secrets-proxy/trust-watch.ts +80 -0
  67. package/lib/shared-files-s3.ts +3 -0
  68. package/lib/sidecar-guard.ts +58 -0
  69. package/lib/sidecar-resume.ts +51 -3
  70. package/lib/standard-image.ts +50 -12
  71. package/lib/task-env-forwarder.ts +228 -0
  72. package/lib/task-env-source.ts +193 -0
  73. package/lib/task-environment/apple-container.ts +45 -11
  74. package/lib/task-environment/docker.ts +51 -12
  75. package/lib/task-environment/index.ts +6 -2
  76. package/lib/task-environment/machine-task-up.ts +258 -25
  77. package/lib/task-environment/machine.ts +80 -22
  78. package/lib/task-environment/registry.ts +3 -2
  79. package/lib/task-environment/self-identity.ts +20 -0
  80. package/lib/task-environment/self.ts +92 -16
  81. package/lib/task-environment/types.ts +63 -0
  82. package/lib/task-gateway-port.ts +22 -0
  83. package/lib/task-gateway-protocol.ts +190 -0
  84. package/lib/task-gateway-recovery.ts +44 -0
  85. package/lib/task-gateway-state.ts +497 -0
  86. package/lib/transcript.ts +1 -0
  87. package/lib/ulid.ts +10 -1
  88. package/package.json +13 -1
  89. package/scripts/agent/env-hook.sh +626 -0
  90. package/scripts/agent/task-up.sh +476 -27
  91. package/scripts/agent/uai-env-exec.sh +26 -0
  92. package/src/cli.ts +26 -3
  93. package/src/index.ts +273 -19
  94. package/src/main.ts +941 -48
  95. package/src/project-env-latest.ts +130 -0
  96. package/src/project-env-policy.ts +281 -0
  97. package/src/project-env-wording.ts +47 -0
  98. package/src/protocol.ts +2239 -15
@@ -34,6 +34,11 @@ import { getDb, schema } from "./db";
34
34
  import { mockAgentFactory } from "./agents/mock";
35
35
  import { realAgentFactory } from "./agents/factory";
36
36
  import { resolveAgentMode } from "./agents/mode";
37
+ import {
38
+ clearPersistedCodexThread,
39
+ persistedAgentIds,
40
+ stopPersistedAgentSession,
41
+ } from "./agents/session-state";
37
42
  import {
38
43
  type AgentEvent,
39
44
  type AgentSession,
@@ -45,7 +50,12 @@ import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
45
50
  import { isSafeHostTaskId } from "./task-identity";
46
51
  import { projectEnvForTask } from "./host-env";
47
52
  import { shadowRoute } from "./routing/shadow";
48
- import { getHostTask, hostTaskProjectIds, upsertHostTask } from "./runtime-state";
53
+ import {
54
+ getHostTask,
55
+ hostTaskEnvMode,
56
+ hostTaskProjectIds,
57
+ upsertHostTask,
58
+ } from "./runtime-state";
49
59
  import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
50
60
  import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
51
61
  import { dockerCli } from "./docker-exec";
@@ -78,7 +88,20 @@ import {
78
88
  appleTaskStoreVolumeName,
79
89
  parseAppleTaskEnvironmentLocator,
80
90
  } from "./task-environment/apple-container";
81
- import { parseDockerTaskEnvironmentLocator } from "./task-environment/docker";
91
+ import {
92
+ DOCKER_TASK_ENVIRONMENT_PROVIDER,
93
+ parseDockerTaskEnvironmentLocator,
94
+ } from "./task-environment/docker";
95
+ import { APPLE_TASK_ENVIRONMENT_PROVIDER } from "./task-environment/apple-container";
96
+ import { activeSecretsProxy } from "./secrets-proxy/host";
97
+ import { usableProxiedKeys } from "./secrets-proxy/policy";
98
+ import { forgetTaskProxyReadiness } from "./secrets-proxy/readiness";
99
+ import {
100
+ invalidateTaskWrites,
101
+ applyTaskRootSteps,
102
+ taskWritesQuiescing,
103
+ type RecoveryProxyFailure,
104
+ } from "./secrets-proxy/task-trust";
82
105
  import type {
83
106
  TaskEnvironmentDescriptor,
84
107
  TaskEnvironmentHandle,
@@ -92,9 +115,15 @@ import {
92
115
  type SkillExec,
93
116
  writeAgentSkills,
94
117
  } from "./skills";
95
- import { ensureMachineGatewayTunnel } from "./machine-gateway-tunnel";
118
+ import {
119
+ ensureMachineGatewayTunnel,
120
+ ensureMachineGatewayTunnelReady,
121
+ MACHINE_GATEWAY_TUNNEL_PORT,
122
+ } from "./machine-gateway-tunnel";
123
+ import { ensureContainerTaskEnvForwarder } from "./task-env-forwarder";
96
124
  import {
97
125
  CONTAINER_CLI_PATH,
126
+ agentCliArgv,
98
127
  agentCliEnv,
99
128
  apiUrlFromCloudUrl,
100
129
  loadTaskCliSecret,
@@ -121,6 +150,12 @@ import {
121
150
  type ResolvedEngineAccount,
122
151
  } from "./engine-accounts";
123
152
  import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
153
+ import {
154
+ reactivateTaskGatewayTask,
155
+ removeTaskGatewayTask,
156
+ suspendTaskGatewayTask,
157
+ type TaskGatewayRegistration,
158
+ } from "./task-gateway-state";
124
159
  import {
125
160
  PREVIEW_SIDECAR_TASK_LABEL,
126
161
  stopPreviewSidecars,
@@ -145,6 +180,7 @@ import {
145
180
  SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
146
181
  } from "../src/protocol";
147
182
  import { isEnvironmentTransportProvider } from "./task-environment/types";
183
+ import { forgetTaskEnvSnapshot } from "./task-env-source";
148
184
 
149
185
  export type HostEventSubscriber = (event: HostEvent) => void;
150
186
 
@@ -223,14 +259,27 @@ interface Channel {
223
259
  /** Sessions that loaded MCP config before the latest rewrite. Busy sessions
224
260
  * stay here until their turn completes; each id is recycled exactly once. */
225
261
  browserStaleSessions: Set<string>;
262
+ /** Launch-setting edits reuse the idle-safe stale-session path, but carry
263
+ * their own lifecycle so only those replacements emit agent.session. */
264
+ rosterSessionChanges: Map<string, RosterSessionChange>;
226
265
  /** Browser-config rewrites whose install/config preconditions are not ready
227
266
  * yet. These sessions must eventually reload the definition, but recycling
228
267
  * them before the browser is usable only trades one broken session for
229
268
  * another. */
230
269
  browserPendingStaleSessions: Set<string>;
270
+ /** Engine kinds newly covered by an already-enabled browser setup. The
271
+ * aggregate setup result can then recycle only sessions for those kinds. */
272
+ browserConfigAffectedKinds: Set<string>;
231
273
  /** One reconcile loop per channel. It covers ordinary missing agents and
232
274
  * stale-session recycling without overlapping factory creates. */
233
275
  reconcileReady: Promise<void> | null;
276
+ /** Forced removal closes begun by refreshChannel. ensureSessions joins this
277
+ * fence before spawning so one outbox consumer cannot overlap another. */
278
+ rosterRefreshReady: Promise<void> | null;
279
+ /** First-start cleanup of durable rows whose ids are absent from the fresh
280
+ * cloud roster (the host may have restarted after their removal). */
281
+ persistedRosterChecked: boolean;
282
+ persistedRosterReady: Promise<void> | null;
234
283
  /** A reconcile request that arrived while the loop was in flight. */
235
284
  reconcileAgain: boolean;
236
285
  /** Set before teardown awaits anything. Every async continuation checks both
@@ -260,6 +309,14 @@ interface Channel {
260
309
  * turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
261
310
  * half-turn instead of delivering it to @-mentioned peers. */
262
311
  interrupted: Set<string>;
312
+ /** Cloud turn boundaries whose runner generation was retired by recovery.
313
+ * Kept separately from activeTurns because remove/edit may overtake the
314
+ * async recovery after generation-local turn state has been cleared. */
315
+ retiredTurnBoundaries: Set<string>;
316
+ /** Async account recoveries fenced by roster removal/edit. A same-settings
317
+ * remove/re-add is invisible to launch-tuple comparison, so the recovery
318
+ * itself carries the supersession bit until it releases the spawn slot. */
319
+ recoveryByAgent: Map<string, RecoveryFence>;
263
320
  /** Per-agent respawn counter — bounded so a broken agent can't
264
321
  * loop forever rewriting its config. */
265
322
  respawns: Map<string, number>;
@@ -288,6 +345,9 @@ interface Channel {
288
345
  mcpConfigDirtyEpoch: number;
289
346
  /** Dirty epoch covered by the last successful gateway publication. */
290
347
  mcpConfigAppliedEpoch: number;
348
+ /** Engine kinds whose provisioned config directories dirtied the managed
349
+ * gateway files at the current unapplied epoch. */
350
+ mcpProvisionAffectedKinds: Set<string>;
291
351
  /** Monotonic version of the managed config files agent processes read at
292
352
  * startup. A factory create that straddles a rewrite is discarded and
293
353
  * retried before it can be bound as the live generation. */
@@ -309,6 +369,9 @@ interface Channel {
309
369
  * drains them so an old channel cannot resume against a reused container
310
370
  * name and overwrite the replacement generation's durable transport. */
311
371
  factoryOperations: Set<Promise<unknown>>;
372
+ /** The create currently owning an agent id, used to fence a removal until a
373
+ * stale in-flight process has observed the new roster and closed itself. */
374
+ factoryOperationByAgent: Map<string, Promise<AgentSession | null>>;
312
375
  /** Terminal-event recovery/rotation can remove a session before awaiting
313
376
  * its close. Track the enclosing handler so teardown still drains that
314
377
  * external operation before the container name can be reused. */
@@ -328,6 +391,16 @@ interface AcceptedPrompt {
328
391
  text: string;
329
392
  }
330
393
 
394
+ interface RosterSessionChange {
395
+ reason: string;
396
+ state: "pending-close" | "restarting";
397
+ abortPublished: boolean;
398
+ }
399
+
400
+ interface RecoveryFence {
401
+ superseded: boolean;
402
+ }
403
+
331
404
  /** Hard cap on automatic respawns per agent per channel lifetime. */
332
405
  const MAX_RESPAWNS_PER_AGENT = 5;
333
406
  /** ADR-076: hard cap on account rotations per agent between successful turns. */
@@ -545,31 +618,100 @@ export class Orchestrator {
545
618
  /** Fold a fresh channel spec into a live channel (ADR-049). */
546
619
  private refreshChannel(channel: Channel, spec: ChannelEnsureInput): void {
547
620
  const browserWasEnabled = channel.browserTesting;
621
+ const previousRoster = channel.roster;
548
622
  const previousMcpShape = JSON.stringify({
549
623
  connections: channel.mcpConnections,
550
- kinds: [...new Set(channel.roster.map((agent) => agent.kind))].sort(),
624
+ kinds: [...new Set(previousRoster.map((agent) => agent.kind))].sort(),
551
625
  });
552
- const known = new Set(channel.roster.map((a) => a.id));
626
+ const previousById = new Map(previousRoster.map((agent) => [agent.id, agent]));
627
+ const nextById = new Map(spec.agents.map((agent) => [agent.id, agent]));
628
+ const previousKinds = new Set(previousRoster.map((agent) => agent.kind));
629
+ const nextKinds = new Set(spec.agents.map((agent) => agent.kind));
630
+ const removed = previousRoster.filter((agent) => !nextById.has(agent.id));
631
+ const added = spec.agents.filter((agent) => !previousById.has(agent.id));
632
+ const changed = spec.agents.flatMap((agent) => {
633
+ const previous = previousById.get(agent.id);
634
+ if (!previous) return [];
635
+ const reason = rosterLaunchChangeReason(previous, agent);
636
+ return reason ? [{ agent, reason }] : [];
637
+ });
638
+
553
639
  // Replace EXISTING agents' data with the fresh spec — a mid-task roster edit
554
640
  // can change permissions, model, brief, role or skills, and without this the
555
641
  // stale snapshot persists (e.g. granting `uai` CLI permissions mid-task never
556
- // took effect: writeAgentCli kept seeing the old empty list). Swapping the
557
- // object (not merging) also drops fields that were removed. The token/model a
558
- // LIVE session already carries only changes on its next respawn.
559
- const bySpecId = new Map(spec.agents.map((a) => [a.id, a]));
560
- channel.roster = channel.roster.map((a) => bySpecId.get(a.id) ?? a);
561
- for (const agent of spec.agents) {
562
- if (!known.has(agent.id)) {
563
- channel.roster.push(agent);
564
- // Per-agent link/document skills, like getOrCreateChannel does at
565
- // channel birth. Best-effort.
566
- writeAgentSkills(channel.taskId, agent);
642
+ // took effect: writeAgentCli kept seeing the old empty list). Replacing the
643
+ // array exactly also makes absence authoritative for live removal.
644
+ channel.roster = [...spec.agents];
645
+ for (const agent of added) {
646
+ // Per-agent link/document skills, like getOrCreateChannel does at
647
+ // channel birth. Best-effort.
648
+ writeAgentSkills(channel.taskId, agent);
649
+ }
650
+ for (const { agent, reason } of changed) {
651
+ const recoveryAbortPublished = this.supersedeRecovery(
652
+ channel,
653
+ agent.id,
654
+ );
655
+ // Clear before any reconcile can construct a replacement. A session can
656
+ // already be absent from the map while its durable runner/thread row is
657
+ // still live, so this must not depend on in-memory session presence.
658
+ try {
659
+ clearPersistedCodexThread(channel.taskId, agent.id);
660
+ } catch (err) {
661
+ console.warn(
662
+ `[orchestrator] ${channel.taskId}/${agent.id}: could not clear persisted Codex thread for roster change: ${
663
+ err instanceof Error ? err.message : String(err)
664
+ }`,
665
+ );
567
666
  }
667
+ channel.accountByAgent.delete(agent.id);
668
+ channel.rotations.delete(agent.id);
669
+ // Crash/recovery budgets belong to the retired launch generation. A
670
+ // newly selected engine/model/effort must get an immediate clean start,
671
+ // not inherit the predecessor's cooldown window.
672
+ channel.respawns.delete(agent.id);
673
+ channel.respawnLastAt.delete(agent.id);
674
+ const priorChange = channel.rosterSessionChanges.get(agent.id);
675
+ channel.rosterSessionChanges.set(agent.id, {
676
+ reason,
677
+ state:
678
+ priorChange?.state === "restarting"
679
+ ? "restarting"
680
+ : "pending-close",
681
+ abortPublished:
682
+ priorChange?.abortPublished === true || recoveryAbortPublished,
683
+ });
684
+ channel.browserStaleSessions.add(agent.id);
685
+ }
686
+ if (removed.length > 0) {
687
+ const previousRefresh = channel.rosterRefreshReady ?? Promise.resolve();
688
+ const removals = removed.map((agent) =>
689
+ this.removeRosterAgent(channel, agent.id),
690
+ );
691
+ let refresh: Promise<void>;
692
+ refresh = Promise.allSettled([previousRefresh, ...removals])
693
+ .then(() => undefined)
694
+ .finally(() => {
695
+ if (channel.rosterRefreshReady === refresh) {
696
+ channel.rosterRefreshReady = null;
697
+ }
698
+ });
699
+ channel.rosterRefreshReady = refresh;
568
700
  }
569
701
  channel.humans = spec.humans ?? [];
570
702
  channel.mode = spec.mode === "secretary" ? "secretary" : "open";
571
703
  channel.secretaryAgentId = spec.secretaryAgentId;
572
704
  channel.browserTesting = spec.browserTesting === true;
705
+ if (channel.browserTesting) {
706
+ const affectedKinds = browserWasEnabled
707
+ ? [...nextKinds].filter((kind) => !previousKinds.has(kind))
708
+ : [...nextKinds];
709
+ for (const kind of affectedKinds) {
710
+ channel.browserConfigAffectedKinds.add(kind);
711
+ }
712
+ } else {
713
+ channel.browserConfigAffectedKinds.clear();
714
+ }
573
715
  channel.routingShadow = spec.routing?.shadow === true;
574
716
  channel.workspacePath = spec.workspacePath;
575
717
  channel.sharedFiles = spec.sharedFiles ?? "ro";
@@ -597,6 +739,7 @@ export class Orchestrator {
597
739
  channel.sharedFiles,
598
740
  spec.mode,
599
741
  spec.secretaryAgentId,
742
+ taskProxiedKeys(channel.taskId),
600
743
  ),
601
744
  );
602
745
  }
@@ -626,6 +769,162 @@ export class Orchestrator {
626
769
  }
627
770
  }
628
771
 
772
+ /** Remove every live/in-flight trace of one agent without deleting its
773
+ * durable session directory. Removal is intentionally immediate mid-turn. */
774
+ private async removeRosterAgent(
775
+ channel: Channel,
776
+ agentId: string,
777
+ ): Promise<void> {
778
+ const session = channel.sessions.get(agentId);
779
+ const inFlightCreate = channel.factoryOperationByAgent.get(agentId);
780
+ const lifecycleAtRemoval = channel.rosterSessionChanges.get(agentId);
781
+ channel.sessions.delete(agentId);
782
+ // Do not release another lifecycle owner's spawn slot. Recovery/reconcile
783
+ // paths clear it in their own finally blocks after rechecking the desired
784
+ // roster; releasing it here lets a rapid re-add start a second same-id
785
+ // factory while the first is still provisioning.
786
+
787
+ // Recovery may already have retired the runner-local turn state while an
788
+ // async provision/broker call was in flight. Consume that saved boundary
789
+ // before looking at the live generation's maps.
790
+ this.supersedeRecovery(channel, agentId);
791
+ this.emitRetiredTurnAbort(channel, agentId);
792
+ // A removed partial turn can never receive its natural terminal event.
793
+ // Close that cloud boundary as aborted before the old session's identity
794
+ // guard begins suppressing its final adapter events.
795
+ if (
796
+ channel.activeTurns.has(agentId) ||
797
+ channel.openTurns.has(agentId) ||
798
+ channel.acceptedPrompts.has(agentId)
799
+ ) {
800
+ channel.interrupted.add(agentId);
801
+ }
802
+ this.retireRunnerGeneration(channel, agentId);
803
+ channel.preambles.delete(agentId);
804
+ channel.firstTurns.delete(agentId);
805
+ channel.browserStaleSessions.delete(agentId);
806
+ channel.rosterSessionChanges.delete(agentId);
807
+ channel.browserPendingStaleSessions.delete(agentId);
808
+ channel.pendingPrompts.delete(agentId);
809
+ channel.runtimeDeferredPrompts.delete(agentId);
810
+ channel.runtimeDeferredAgents.delete(agentId);
811
+ channel.interrupted.delete(agentId);
812
+ channel.retiredTurnBoundaries.delete(agentId);
813
+ channel.respawns.delete(agentId);
814
+ channel.respawnLastAt.delete(agentId);
815
+ channel.everBound.delete(agentId);
816
+ channel.accountByAgent.delete(agentId);
817
+ channel.lastPrompt.delete(agentId);
818
+ channel.rotations.delete(agentId);
819
+
820
+ if (session) {
821
+ try {
822
+ // Delete-before-close above keeps the adapter's terminal event
823
+ // identity-guarded and prevents an outbox consumer overlap.
824
+ await session.close();
825
+ } catch {
826
+ // Already dead — close is idempotent for our adapters.
827
+ }
828
+ }
829
+ if (inFlightCreate) {
830
+ // createAndBindStableSessionCore rechecks the desired roster after the
831
+ // factory await and closes the stale process instead of binding it.
832
+ await inFlightCreate.catch(() => null);
833
+ }
834
+ if (!this.isActiveChannel(channel)) return;
835
+ // A newer ensure can re-add the same id while the old close/create drains.
836
+ // That spec owns the status now; do not stop its bound generation or emit
837
+ // a stale removed edge.
838
+ if (channel.roster.some((agent) => agent.id === agentId)) {
839
+ this.restoreReaddedRosterLifecycle(
840
+ channel,
841
+ agentId,
842
+ lifecycleAtRemoval,
843
+ );
844
+ void this.reconcileIfStarted(channel).catch((err: unknown) => {
845
+ console.warn(
846
+ `[orchestrator] ${channel.taskId}/${agentId}: re-added roster reconcile failed: ${
847
+ err instanceof Error ? err.message : String(err)
848
+ }`,
849
+ );
850
+ });
851
+ return;
852
+ }
853
+ let durableCleanupFailed = false;
854
+ try {
855
+ await this.stopPersistedRunnerIfPresent(channel, agentId);
856
+ } catch (err) {
857
+ durableCleanupFailed = true;
858
+ // The same ensure that joined this removal fence will retry through the
859
+ // persisted-roster recovery pass instead of trusting a poisoned result.
860
+ channel.persistedRosterChecked = false;
861
+ console.warn(
862
+ `[orchestrator] ${channel.taskId}/${agentId}: removed durable session cleanup failed: ${
863
+ err instanceof Error ? err.message : String(err)
864
+ }`,
865
+ );
866
+ }
867
+ if (!this.isActiveChannel(channel)) return;
868
+ if (channel.roster.some((agent) => agent.id === agentId)) {
869
+ this.restoreReaddedRosterLifecycle(
870
+ channel,
871
+ agentId,
872
+ lifecycleAtRemoval,
873
+ );
874
+ void this.reconcileIfStarted(channel).catch((err: unknown) => {
875
+ console.warn(
876
+ `[orchestrator] ${channel.taskId}/${agentId}: post-stop re-add reconcile failed: ${
877
+ err instanceof Error ? err.message : String(err)
878
+ }`,
879
+ );
880
+ });
881
+ return;
882
+ }
883
+ // Slow rotation/account continuations captured before the roster swap may
884
+ // have reached their finally blocks while the close was in flight.
885
+ channel.accountByAgent.delete(agentId);
886
+ channel.rotations.delete(agentId);
887
+ channel.respawns.delete(agentId);
888
+ channel.respawnLastAt.delete(agentId);
889
+ channel.lastPrompt.delete(agentId);
890
+ if (!this.isActiveChannel(channel)) return;
891
+ if (durableCleanupFailed) return;
892
+ this.emitHost({
893
+ kind: "agent.session",
894
+ taskId: channel.taskId,
895
+ agentId,
896
+ state: "removed",
897
+ });
898
+ console.log(
899
+ `[orchestrator] ${channel.taskId}/${agentId}: stopped removed roster session`,
900
+ );
901
+ }
902
+
903
+ /** A remove superseded while an earlier edit was already restarting must
904
+ * still close that visible lifecycle with ready for the re-added roster. */
905
+ private restoreReaddedRosterLifecycle(
906
+ channel: Channel,
907
+ agentId: string,
908
+ removedChange: RosterSessionChange | undefined,
909
+ ): void {
910
+ if (
911
+ removedChange?.state !== "restarting" ||
912
+ channel.rosterSessionChanges.has(agentId)
913
+ ) {
914
+ return;
915
+ }
916
+ if (channel.sessions.has(agentId)) {
917
+ this.emitHost({
918
+ kind: "agent.session",
919
+ taskId: channel.taskId,
920
+ agentId,
921
+ state: "ready",
922
+ });
923
+ return;
924
+ }
925
+ channel.rosterSessionChanges.set(agentId, removedChange);
926
+ }
927
+
629
928
  /** Materialize a browser flag enabled after the channel already started. */
630
929
  private async enableBrowserTesting(channel: Channel): Promise<void> {
631
930
  try {
@@ -670,6 +969,7 @@ export class Orchestrator {
670
969
  spec.sharedFiles ?? "ro",
671
970
  spec.mode,
672
971
  spec.secretaryAgentId,
972
+ taskProxiedKeys(taskId),
673
973
  ),
674
974
  );
675
975
  // ADR-046: materialise this agent's skills to its per-agent SKILL.md in
@@ -709,8 +1009,13 @@ export class Orchestrator {
709
1009
  browserSetupCoverage: null,
710
1010
  browserRetryTimer: null,
711
1011
  browserStaleSessions: new Set(),
1012
+ rosterSessionChanges: new Map(),
712
1013
  browserPendingStaleSessions: new Set(),
1014
+ browserConfigAffectedKinds: new Set(),
713
1015
  reconcileReady: null,
1016
+ rosterRefreshReady: null,
1017
+ persistedRosterChecked: false,
1018
+ persistedRosterReady: null,
714
1019
  reconcileAgain: false,
715
1020
  closed: false,
716
1021
  openTurns: new Set(),
@@ -720,6 +1025,8 @@ export class Orchestrator {
720
1025
  runtimeDeferredPrompts: new Map(),
721
1026
  runtimeDeferredAgents: new Set(),
722
1027
  interrupted: new Set(),
1028
+ retiredTurnBoundaries: new Set(),
1029
+ recoveryByAgent: new Map(),
723
1030
  respawns: new Map(),
724
1031
  respawnLastAt: new Map(),
725
1032
  humans: spec.humans ?? [],
@@ -729,12 +1036,14 @@ export class Orchestrator {
729
1036
  mcpConfigWrite: null,
730
1037
  mcpConfigDirtyEpoch: 1,
731
1038
  mcpConfigAppliedEpoch: 0,
1039
+ mcpProvisionAffectedKinds: new Set(),
732
1040
  configGeneration: 0,
733
1041
  spawning: new Set(),
734
1042
  everBound: new Set(),
735
1043
  routingShadow: spec.routing?.shadow === true,
736
1044
  workspacePath: spec.workspacePath,
737
1045
  factoryOperations: new Set(),
1046
+ factoryOperationByAgent: new Map(),
738
1047
  eventOperations: new Set(),
739
1048
  accountByAgent: new Map(),
740
1049
  lastPrompt: new Map(),
@@ -761,6 +1070,58 @@ export class Orchestrator {
761
1070
  // copy cannot land after browser/MCP materialization and erase config.
762
1071
  await taskRecoveryComplete(channel.taskId);
763
1072
  if (!this.isActiveChannel(channel)) return false;
1073
+ if (channel.rosterRefreshReady) {
1074
+ await channel.rosterRefreshReady;
1075
+ if (!this.isActiveChannel(channel)) return false;
1076
+ }
1077
+ if (!channel.persistedRosterChecked) {
1078
+ let removedIds: string[];
1079
+ try {
1080
+ const rosterIds = new Set(channel.roster.map((agent) => agent.id));
1081
+ removedIds = persistedAgentIds(channel.taskId).filter(
1082
+ (agentId) => !rosterIds.has(agentId),
1083
+ );
1084
+ } catch (err) {
1085
+ console.warn(
1086
+ `[orchestrator] ${channel.taskId}: persisted roster scan failed: ${
1087
+ err instanceof Error ? err.message : String(err)
1088
+ }`,
1089
+ );
1090
+ throw err;
1091
+ }
1092
+ channel.persistedRosterChecked = true;
1093
+ if (removedIds.length > 0) {
1094
+ const operation = this.stopPersistedRemovedAgents(
1095
+ channel,
1096
+ removedIds,
1097
+ );
1098
+ let tracked: Promise<void>;
1099
+ tracked = operation.then(
1100
+ () => {
1101
+ if (channel.persistedRosterReady === tracked) {
1102
+ channel.persistedRosterReady = null;
1103
+ }
1104
+ },
1105
+ (err: unknown) => {
1106
+ if (channel.persistedRosterReady === tracked) {
1107
+ channel.persistedRosterReady = null;
1108
+ channel.persistedRosterChecked = false;
1109
+ }
1110
+ console.warn(
1111
+ `[orchestrator] ${channel.taskId}: persisted roster cleanup failed: ${
1112
+ err instanceof Error ? err.message : String(err)
1113
+ }`,
1114
+ );
1115
+ throw err;
1116
+ },
1117
+ );
1118
+ channel.persistedRosterReady = tracked;
1119
+ }
1120
+ }
1121
+ if (channel.persistedRosterReady) {
1122
+ await channel.persistedRosterReady;
1123
+ if (!this.isActiveChannel(channel)) return false;
1124
+ }
764
1125
  if (!backgroundContainerWorkAllowed()) return false;
765
1126
  // Memoized: every caller awaits the SAME in-flight start, so a concurrent
766
1127
  // deliver() can't observe "ready" while the sessions map is still empty
@@ -825,6 +1186,48 @@ export class Orchestrator {
825
1186
  return ready;
826
1187
  }
827
1188
 
1189
+ /** A host restart loses the in-memory old roster, not its durable runners.
1190
+ * Stop persisted ids that the authoritative fresh spec no longer contains. */
1191
+ private async stopPersistedRemovedAgents(
1192
+ channel: Channel,
1193
+ removedIds: readonly string[],
1194
+ ): Promise<void> {
1195
+ const environment = await this.agentEnvironment(channel);
1196
+ for (const agentId of removedIds) {
1197
+ if (!this.isActiveChannel(channel)) return;
1198
+ // The roster may have advanced while environment reconstruction was in
1199
+ // flight. Last arrival wins; never stop an id the current spec wants.
1200
+ if (channel.roster.some((agent) => agent.id === agentId)) continue;
1201
+ const stoppedDir = await stopPersistedAgentSession(
1202
+ channel.taskId,
1203
+ agentId,
1204
+ environment,
1205
+ );
1206
+ if (channel.roster.some((agent) => agent.id === agentId)) continue;
1207
+ if (!stoppedDir || !this.isActiveChannel(channel)) continue;
1208
+ this.emitHost({
1209
+ kind: "agent.session",
1210
+ taskId: channel.taskId,
1211
+ agentId,
1212
+ state: "removed",
1213
+ });
1214
+ console.log(
1215
+ `[orchestrator] ${channel.taskId}/${agentId}: stopped removed durable session during roster recovery`,
1216
+ );
1217
+ }
1218
+ }
1219
+
1220
+ /** Stop a durable predecessor even when its row is already marked closed.
1221
+ * Closed is not proof of runner death: markClosed precedes the stop write. */
1222
+ private async stopPersistedRunnerIfPresent(
1223
+ channel: Channel,
1224
+ agentId: string,
1225
+ ): Promise<string | null> {
1226
+ if (!persistedAgentIds(channel.taskId).includes(agentId)) return null;
1227
+ const environment = await this.agentEnvironment(channel);
1228
+ return stopPersistedAgentSession(channel.taskId, agentId, environment);
1229
+ }
1230
+
828
1231
  /**
829
1232
  * Background browser work may recycle sessions only after the memoized
830
1233
  * initial start has finished. Waiting here prevents live enable/retry from
@@ -873,11 +1276,17 @@ export class Orchestrator {
873
1276
  private noteProvisionMutation(
874
1277
  channel: Channel,
875
1278
  waitForBrowserReadiness: boolean,
1279
+ affectedKinds: ReadonlySet<string>,
876
1280
  ): void {
877
1281
  if (!this.isActiveChannel(channel)) return;
878
1282
  channel.configGeneration += 1;
879
1283
  channel.mcpConfigDirtyEpoch += 1;
880
- for (const agentId of channel.sessions.keys()) {
1284
+ for (const kind of affectedKinds) {
1285
+ channel.mcpProvisionAffectedKinds.add(kind);
1286
+ }
1287
+ for (const [agentId] of channel.sessions) {
1288
+ const kind = channel.roster.find((agent) => agent.id === agentId)?.kind;
1289
+ if (!kind || !affectedKinds.has(kind)) continue;
881
1290
  if (waitForBrowserReadiness) {
882
1291
  channel.browserPendingStaleSessions.add(agentId);
883
1292
  } else {
@@ -910,6 +1319,42 @@ export class Orchestrator {
910
1319
  });
911
1320
  }
912
1321
 
1322
+ /** Kinds whose account-copy inputs changed since the last attempted pass. */
1323
+ private changedProvisionKinds(
1324
+ previousFingerprint: string | undefined,
1325
+ nextFingerprint: string,
1326
+ requestedKinds: readonly string[],
1327
+ ): string[] {
1328
+ if (!previousFingerprint) return [...requestedKinds];
1329
+ type Fingerprint = {
1330
+ kinds?: string[];
1331
+ accounts?: Array<{
1332
+ kind: string;
1333
+ id: string;
1334
+ hostDir: string;
1335
+ containerDir: string;
1336
+ }>;
1337
+ };
1338
+ try {
1339
+ const previous = JSON.parse(previousFingerprint) as Fingerprint;
1340
+ const next = JSON.parse(nextFingerprint) as Fingerprint;
1341
+ return requestedKinds.filter((kind) => {
1342
+ if (!previous.kinds?.includes(kind)) return true;
1343
+ const previousAccounts = (previous.accounts ?? []).filter(
1344
+ (account) => account.kind === kind,
1345
+ );
1346
+ const nextAccounts = (next.accounts ?? []).filter(
1347
+ (account) => account.kind === kind,
1348
+ );
1349
+ return JSON.stringify(previousAccounts) !== JSON.stringify(nextAccounts);
1350
+ });
1351
+ } catch {
1352
+ // A legacy/partial fingerprint is not trustworthy enough to narrow a
1353
+ // write; copy and invalidate the requested kinds conservatively.
1354
+ return [...requestedKinds];
1355
+ }
1356
+ }
1357
+
913
1358
  private async ensureBrowserSetup(
914
1359
  channel: Channel,
915
1360
  hasCodex: boolean,
@@ -936,11 +1381,20 @@ export class Orchestrator {
936
1381
  ];
937
1382
  const requestedProvisionFingerprint =
938
1383
  this.provisionFingerprint(requestedKinds, channel.engineAccountScope);
939
- const mayProvisionConfig = requestedKinds.some((kind) =>
1384
+ const changedProvisionKinds = this.changedProvisionKinds(
1385
+ channel.provisionAttemptedFingerprint,
1386
+ requestedProvisionFingerprint,
1387
+ requestedKinds,
1388
+ );
1389
+ const provisionKinds = forceProvision
1390
+ ? requestedKinds
1391
+ : changedProvisionKinds;
1392
+ const mutableProvisionKinds = provisionKinds.filter((kind) =>
940
1393
  resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
941
1394
  (account) => !account.isDefault && Boolean(account.configDir),
942
1395
  ),
943
1396
  );
1397
+ const mayProvisionConfig = mutableProvisionKinds.length > 0;
944
1398
 
945
1399
  const inFlight = channel.browserSetup;
946
1400
  if (inFlight) {
@@ -1011,14 +1465,14 @@ export class Orchestrator {
1011
1465
  const provisioned = channel.agentHandle
1012
1466
  ? await provisionEngineAccounts(
1013
1467
  channel.containerName,
1014
- requestedKinds,
1468
+ provisionKinds,
1015
1469
  {},
1016
1470
  channel.agentHandle,
1017
1471
  channel.engineAccountScope,
1018
1472
  )
1019
1473
  : await provisionEngineAccounts(
1020
1474
  channel.containerName,
1021
- requestedKinds,
1475
+ provisionKinds,
1022
1476
  {},
1023
1477
  undefined,
1024
1478
  channel.engineAccountScope,
@@ -1050,6 +1504,7 @@ export class Orchestrator {
1050
1504
  this.noteProvisionMutation(
1051
1505
  channel,
1052
1506
  channel.browserTesting,
1507
+ new Set(mutableProvisionKinds),
1053
1508
  );
1054
1509
  } else {
1055
1510
  this.fenceConfigGeneration(channel);
@@ -1107,10 +1562,23 @@ export class Orchestrator {
1107
1562
  channel.browserReady = setup.ready && setup.configured;
1108
1563
  if (setup.changed) {
1109
1564
  channel.configGeneration += 1;
1110
- for (const agentId of channel.sessions.keys()) {
1565
+ const affectedKinds = channel.browserConfigAffectedKinds;
1566
+ const affectedIds =
1567
+ affectedKinds.size > 0
1568
+ ? channel.roster
1569
+ .filter((agent) => affectedKinds.has(agent.kind))
1570
+ .map((agent) => agent.id)
1571
+ : [...channel.sessions.keys()];
1572
+ for (const agentId of affectedIds) {
1573
+ if (!channel.sessions.has(agentId)) continue;
1111
1574
  channel.browserPendingStaleSessions.add(agentId);
1112
1575
  }
1113
1576
  }
1577
+ if (setup.configured) {
1578
+ for (const kind of requestedKinds) {
1579
+ channel.browserConfigAffectedKinds.delete(kind);
1580
+ }
1581
+ }
1114
1582
  if (channel.browserReady) {
1115
1583
  for (const agentId of channel.browserPendingStaleSessions) {
1116
1584
  channel.browserStaleSessions.add(agentId);
@@ -1259,6 +1727,9 @@ export class Orchestrator {
1259
1727
  ...(grokHomes.length > 0 ? { grokHomes } : {}),
1260
1728
  });
1261
1729
  const dirtyEpoch = channel.mcpConfigDirtyEpoch;
1730
+ const provisionAffectedKinds = new Set(
1731
+ channel.mcpProvisionAffectedKinds,
1732
+ );
1262
1733
  const repairingProvisionedConfig =
1263
1734
  channel.mcpConfigAppliedEpoch !== dirtyEpoch &&
1264
1735
  channel.mcpConfigFingerprint === fingerprint;
@@ -1321,6 +1792,9 @@ export class Orchestrator {
1321
1792
  // the files while docker exec was in flight, the newer epoch remains
1322
1793
  // uncovered and the serialized ensure immediately publishes another pass.
1323
1794
  channel.mcpConfigAppliedEpoch = dirtyEpoch;
1795
+ if (channel.mcpConfigDirtyEpoch === dirtyEpoch) {
1796
+ channel.mcpProvisionAffectedKinds.clear();
1797
+ }
1324
1798
  // Agent CLIs read MCP servers once, at process start — and durable
1325
1799
  // sessions (ADR-061) make processes long-lived, so without this a
1326
1800
  // connection added mid-task stays invisible indefinitely. Recycle IDLE
@@ -1338,7 +1812,10 @@ export class Orchestrator {
1338
1812
  // Provisioning marked the sessions that existed around its copy. Mark
1339
1813
  // again after the repair so a factory that bound during this write is
1340
1814
  // covered too (Set de-duplicates the earlier marks).
1341
- this.markSessionsStale(channel);
1815
+ const affected = channel.roster
1816
+ .filter((agent) => provisionAffectedKinds.has(agent.kind))
1817
+ .map((agent) => agent.id);
1818
+ this.markSessionsStale(channel, affected);
1342
1819
  } else if (!hadPrevious && hadLiveSessions) {
1343
1820
  this.markSessionsStale(channel);
1344
1821
  } else if (hadPrevious && connectionsChanged) {
@@ -1372,7 +1849,16 @@ export class Orchestrator {
1372
1849
  /** Close stale sessions that are idle, without recursively reconciling. */
1373
1850
  private async closeIdleStaleSessions(channel: Channel): Promise<number> {
1374
1851
  let recycled = 0;
1375
- for (const agentId of [...channel.browserStaleSessions]) {
1852
+ let mcpRecycled = 0;
1853
+ // The roster-change tracker is authoritative. Recovery paths written for
1854
+ // ordinary config recycling may clear browserStaleSessions while an edit
1855
+ // is waiting on their spawn slot; never let a pending-close edit bind its
1856
+ // replacement without first publishing the actual restart boundary.
1857
+ const staleAgentIds = new Set([
1858
+ ...channel.browserStaleSessions,
1859
+ ...channel.rosterSessionChanges.keys(),
1860
+ ]);
1861
+ for (const agentId of staleAgentIds) {
1376
1862
  if (!this.isActiveChannel(channel)) return recycled;
1377
1863
  if (
1378
1864
  channel.activeTurns.has(agentId) ||
@@ -1383,31 +1869,155 @@ export class Orchestrator {
1383
1869
  }
1384
1870
  const session = channel.sessions.get(agentId);
1385
1871
  // A session that exited on its own is already absent. Its eventual
1386
- // replacement is fresh, so the old generation is no longer stale.
1872
+ // replacement is fresh for MCP purposes. A roster change still has to
1873
+ // stop a possibly-live durable predecessor before that replacement can
1874
+ // bind under new launch settings.
1387
1875
  if (!session) {
1388
- channel.browserStaleSessions.delete(agentId);
1876
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
1877
+ if (rosterChange) {
1878
+ await this.closeMissingRosterChangedSession(
1879
+ channel,
1880
+ agentId,
1881
+ rosterChange,
1882
+ );
1883
+ recycled += 1;
1884
+ } else {
1885
+ channel.browserStaleSessions.delete(agentId);
1886
+ }
1389
1887
  continue;
1390
1888
  }
1391
1889
  channel.browserStaleSessions.delete(agentId);
1392
1890
  channel.sessions.delete(agentId);
1393
- try {
1394
- // Delete before close. Event handlers are identity-guarded, so the
1395
- // old session's synthetic exit cannot consume respawn budget or delete
1396
- // the replacement that follows.
1397
- await session.close();
1398
- } catch {
1399
- // Already dead — close is idempotent for our adapters.
1891
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
1892
+ if (rosterChange) {
1893
+ await this.closeRosterChangedSession(
1894
+ channel,
1895
+ agentId,
1896
+ session,
1897
+ rosterChange,
1898
+ );
1899
+ } else {
1900
+ try {
1901
+ // Delete before close. Event handlers are identity-guarded, so the
1902
+ // old session's synthetic exit cannot consume respawn budget or delete
1903
+ // the replacement that follows.
1904
+ await session.close();
1905
+ } catch {
1906
+ // Already dead — close is idempotent for our adapters.
1907
+ }
1908
+ mcpRecycled += 1;
1400
1909
  }
1401
1910
  recycled += 1;
1402
1911
  }
1403
- if (recycled > 0) {
1912
+ if (mcpRecycled > 0) {
1404
1913
  console.log(
1405
- `[orchestrator] ${channel.taskId}: recycled ${recycled} idle session(s) for MCP config change`,
1914
+ `[orchestrator] ${channel.taskId}: recycled ${mcpRecycled} idle session(s) for MCP config change`,
1406
1915
  );
1407
1916
  }
1408
1917
  return recycled;
1409
1918
  }
1410
1919
 
1920
+ /** Stop one generation for a live launch-setting edit and publish the
1921
+ * roster-specific lifecycle edge only if that edit still owns the id. */
1922
+ private async closeRosterChangedSession(
1923
+ channel: Channel,
1924
+ agentId: string,
1925
+ session: AgentSession,
1926
+ change: RosterSessionChange,
1927
+ ): Promise<void> {
1928
+ // Durable attach cannot reconstruct whether a pre-restart runner is
1929
+ // already mid-turn. Publish the terminal boundary before every forced
1930
+ // launch close; it is a no-op for an actually idle cloud stream.
1931
+ this.publishRosterChangeAbort(channel, agentId, change);
1932
+ try {
1933
+ await session.close();
1934
+ } catch {
1935
+ // Already dead — close is idempotent for our adapters.
1936
+ }
1937
+ try {
1938
+ await this.stopPersistedRunnerIfPresent(channel, agentId);
1939
+ // Clear again after close. A thread/started notification already queued
1940
+ // by the old Codex process must not restore its resume id between the
1941
+ // eager roster clear and replacement construction.
1942
+ clearPersistedCodexThread(channel.taskId, agentId);
1943
+ } catch (err) {
1944
+ if (
1945
+ this.isActiveChannel(channel) &&
1946
+ channel.rosterSessionChanges.get(agentId) === change
1947
+ ) {
1948
+ channel.browserStaleSessions.add(agentId);
1949
+ }
1950
+ throw err;
1951
+ }
1952
+ if (
1953
+ !this.isActiveChannel(channel) ||
1954
+ channel.rosterSessionChanges.get(agentId) !== change ||
1955
+ !channel.roster.some((agent) => agent.id === agentId)
1956
+ ) {
1957
+ return;
1958
+ }
1959
+ const alreadyRestarting = change.state === "restarting";
1960
+ change.state = "restarting";
1961
+ if (!alreadyRestarting) {
1962
+ this.emitHost({
1963
+ kind: "agent.session",
1964
+ taskId: channel.taskId,
1965
+ agentId,
1966
+ state: "restarting",
1967
+ });
1968
+ }
1969
+ console.log(
1970
+ `[orchestrator] ${channel.taskId}/${agentId}: recycled session for live roster ${change.reason}`,
1971
+ );
1972
+ }
1973
+
1974
+ /** Finish a launch edit whose adapter already disappeared, but whose
1975
+ * durable predecessor may still own the CLI process/outbox. */
1976
+ private async closeMissingRosterChangedSession(
1977
+ channel: Channel,
1978
+ agentId: string,
1979
+ change: RosterSessionChange,
1980
+ ): Promise<void> {
1981
+ this.publishRosterChangeAbort(channel, agentId, change);
1982
+ const stoppedDir = await this.stopPersistedRunnerIfPresent(channel, agentId);
1983
+ clearPersistedCodexThread(channel.taskId, agentId);
1984
+ if (
1985
+ !this.isActiveChannel(channel) ||
1986
+ channel.rosterSessionChanges.get(agentId) !== change ||
1987
+ !channel.roster.some((agent) => agent.id === agentId)
1988
+ ) {
1989
+ return;
1990
+ }
1991
+ channel.browserStaleSessions.delete(agentId);
1992
+ if (!stoppedDir) {
1993
+ // No old generation exists to recycle (one-shot engines between turns,
1994
+ // crash cooldown, or an agent not spawned yet). The cloud set the row to
1995
+ // Restarting when it persisted the edit, so clear that status now even
1996
+ // though there is no close/restart edge to publish first.
1997
+ channel.rosterSessionChanges.delete(agentId);
1998
+ this.emitHost({
1999
+ kind: "agent.session",
2000
+ taskId: channel.taskId,
2001
+ agentId,
2002
+ state: "ready",
2003
+ });
2004
+ return;
2005
+ }
2006
+ const alreadyRestarting = change.state === "restarting";
2007
+ change.state = "restarting";
2008
+ if (!alreadyRestarting) {
2009
+ this.emitHost({
2010
+ kind: "agent.session",
2011
+ taskId: channel.taskId,
2012
+ agentId,
2013
+ state: "restarting",
2014
+ });
2015
+ }
2016
+ console.log(
2017
+ `[orchestrator] ${channel.taskId}/${agentId}: stopped detached durable session for live roster ${change.reason}`,
2018
+ );
2019
+ }
2020
+
1411
2021
  /** Serialize all missing-session and stale-session reconciliation. */
1412
2022
  private async reconcileSessions(channel: Channel): Promise<void> {
1413
2023
  if (!this.isActiveChannel(channel)) return;
@@ -1520,6 +2130,10 @@ export class Orchestrator {
1520
2130
  // Another recovery path may have filled this slot while the browser
1521
2131
  // setup was awaiting.
1522
2132
  if (channel.sessions.has(agent.id)) continue;
2133
+ if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
2134
+ channel.reconcileAgain = true;
2135
+ continue;
2136
+ }
1523
2137
  const session = await this.createAndBindStableSession(
1524
2138
  channel,
1525
2139
  agent.id,
@@ -1530,7 +2144,8 @@ export class Orchestrator {
1530
2144
  executionProfile: this.executionProfileFor(channel, agent.id),
1531
2145
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
1532
2146
  channel,
1533
- agent.id,
2147
+ agent,
2148
+ task,
1534
2149
  ),
1535
2150
  agentEnv: this.accountAgentEnv(
1536
2151
  channel,
@@ -1543,7 +2158,7 @@ export class Orchestrator {
1543
2158
  cliSecret,
1544
2159
  this.isPromptLevelSecretary(channel, agent.id),
1545
2160
  ),
1546
- this.machineProjectEnv(task),
2161
+ task,
1547
2162
  ),
1548
2163
  },
1549
2164
  );
@@ -1571,16 +2186,33 @@ export class Orchestrator {
1571
2186
 
1572
2187
  private attachCompatibilityKeyFor(
1573
2188
  channel: Channel,
1574
- agentId: string,
2189
+ agent: RosterAgent,
2190
+ task: { envMode?: string | null },
1575
2191
  ): string | undefined {
1576
- if (!this.isSecretary(channel, agentId)) return undefined;
1577
2192
  // Key the durable runner to the policy actually passed to the adapter.
1578
2193
  // The restricted profile is dormant today, but giving it the normal-tools
1579
2194
  // generation would make a future opt-out able to attach a runner launched
1580
2195
  // under tool denial.
1581
- return this.executionProfileFor(channel, agentId) === "communicator"
1582
- ? RESTRICTED_SECRETARY_SESSION_ATTACH_KEY
1583
- : SECRETARY_SESSION_ATTACH_KEY;
2196
+ const policy = this.isSecretary(channel, agent.id)
2197
+ ? this.executionProfileFor(channel, agent.id) === "communicator"
2198
+ ? RESTRICTED_SECRETARY_SESSION_ATTACH_KEY
2199
+ : SECRETARY_SESSION_ATTACH_KEY
2200
+ : undefined;
2201
+ const model = agent.model ?? "";
2202
+ const effort = agent.effort ?? "";
2203
+ // Preserve legacy null/policy-only attachment for agents that choose no
2204
+ // launch flags. Explicit model/effort gets a stable key so a host restart
2205
+ // cannot attach a runner created under different settings.
2206
+ const envMode = hostTaskEnvMode(task);
2207
+ if (envMode === "baked" && !model && !effort) return policy;
2208
+ return JSON.stringify([
2209
+ envMode === "baked" ? "roster-launch-v1" : "roster-launch-v2",
2210
+ ...(envMode === "loader" ? [envMode] : []),
2211
+ agent.kind,
2212
+ model,
2213
+ effort,
2214
+ policy ?? "",
2215
+ ]);
1584
2216
  }
1585
2217
 
1586
2218
  private isSecretary(channel: Channel, agentId: string): boolean {
@@ -1614,10 +2246,17 @@ export class Orchestrator {
1614
2246
  channel: Channel,
1615
2247
  agent: RosterAgent,
1616
2248
  base: Record<string, string>,
1617
- /** Lowest precedence: project env for tasks whose environment has no
1618
- * container env to carry it. The account's env and the uai base win. */
1619
- floor: Record<string, string> = {},
2249
+ task: {
2250
+ environmentProvider: string | null;
2251
+ projectIds?: string | null;
2252
+ envMode?: string | null;
2253
+ },
1620
2254
  ): Record<string, string> {
2255
+ const floor = this.machineProjectEnv(task);
2256
+ if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
2257
+ channel.accountByAgent.delete(agent.id);
2258
+ return { ...floor, ...base };
2259
+ }
1621
2260
  const account = pickEngineAccount(agent.kind, {
1622
2261
  scope: channel.engineAccountScope,
1623
2262
  });
@@ -1631,18 +2270,22 @@ export class Orchestrator {
1631
2270
  }
1632
2271
 
1633
2272
  /**
1634
- * ADR-015 project env for a machine task (ADR-121): Docker tasks receive it
1635
- * from the compose env task-up renders, but a machine has no container, so
1636
- * it rides every agent session's env instead. Read at each spawn, so a
1637
- * respawn picks up values set since the task started. Empty for container
1638
- * tasks — their env already carries it and must not be doubled into argv.
2273
+ * Rollout compatibility for a machine/self task created before ADR-153.
2274
+ * Those runtimes had no container env: project values rode every agent
2275
+ * spawn. Loader tasks must never take this path; they fetch through the
2276
+ * gateway, and adding values here would freeze them and bypass declarations.
1639
2277
  */
1640
2278
  private machineProjectEnv(task: {
1641
2279
  environmentProvider: string | null;
1642
2280
  projectIds?: string | null;
2281
+ envMode?: string | null;
1643
2282
  }): Record<string, string> {
1644
- // Machine OR sidecar (ADR-133): neither has a compose env to ride.
1645
- if (!isEnvironmentTransportProvider(task.environmentProvider)) return {};
2283
+ if (
2284
+ hostTaskEnvMode(task) !== "baked" ||
2285
+ !isEnvironmentTransportProvider(task.environmentProvider)
2286
+ ) {
2287
+ return {};
2288
+ }
1646
2289
  return projectEnvForTask(hostTaskProjectIds(task));
1647
2290
  }
1648
2291
 
@@ -1662,6 +2305,18 @@ export class Orchestrator {
1662
2305
  ): void {
1663
2306
  channel.sessions.set(agentId, session);
1664
2307
  channel.everBound.add(agentId);
2308
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
2309
+ if (rosterChange) {
2310
+ channel.rosterSessionChanges.delete(agentId);
2311
+ if (rosterChange.state === "restarting") {
2312
+ this.emitHost({
2313
+ kind: "agent.session",
2314
+ taskId: channel.taskId,
2315
+ agentId,
2316
+ state: "ready",
2317
+ });
2318
+ }
2319
+ }
1665
2320
  // The launch screen's `ready` rides the bind, not the tail of the start
1666
2321
  // loop: whichever path brings the last roster agent up — the initial
1667
2322
  // start, a reconcile that won the race with it, a respawn after Resume —
@@ -1729,16 +2384,29 @@ export class Orchestrator {
1729
2384
  channel: Channel,
1730
2385
  agentId: string,
1731
2386
  args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
2387
+ recovery?: RecoveryFence,
1732
2388
  ): Promise<AgentSession | null> {
1733
2389
  const operation = this.createAndBindStableSessionCore(
1734
2390
  channel,
1735
2391
  agentId,
1736
2392
  args,
2393
+ recovery,
1737
2394
  );
1738
2395
  channel.factoryOperations.add(operation);
2396
+ channel.factoryOperationByAgent.set(agentId, operation);
1739
2397
  operation.then(
1740
- () => channel.factoryOperations.delete(operation),
1741
- () => channel.factoryOperations.delete(operation),
2398
+ () => {
2399
+ channel.factoryOperations.delete(operation);
2400
+ if (channel.factoryOperationByAgent.get(agentId) === operation) {
2401
+ channel.factoryOperationByAgent.delete(agentId);
2402
+ }
2403
+ },
2404
+ () => {
2405
+ channel.factoryOperations.delete(operation);
2406
+ if (channel.factoryOperationByAgent.get(agentId) === operation) {
2407
+ channel.factoryOperationByAgent.delete(agentId);
2408
+ }
2409
+ },
1742
2410
  );
1743
2411
  return operation;
1744
2412
  }
@@ -1762,6 +2430,7 @@ export class Orchestrator {
1762
2430
  try {
1763
2431
  const res = await environment.exec({
1764
2432
  argv: ["/bin/sh", "-c", codexHomeSeedCommand(home, perAgent)],
2433
+ projectEnvProfile: "maintenance",
1765
2434
  user: "root",
1766
2435
  env: RUNTIME_AUTHORITY_ENV,
1767
2436
  timeoutMs: 30_000,
@@ -1791,8 +2460,27 @@ export class Orchestrator {
1791
2460
  channel: Channel,
1792
2461
  agentId: string,
1793
2462
  args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
2463
+ recovery?: RecoveryFence,
1794
2464
  ): Promise<AgentSession | null> {
2465
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2466
+ channel.reconcileAgain = true;
2467
+ return null;
2468
+ }
2469
+ if (
2470
+ channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
2471
+ ) {
2472
+ channel.reconcileAgain = true;
2473
+ return null;
2474
+ }
2475
+ if (!this.hasDesiredLaunchSettings(channel, agentId, args.agent)) {
2476
+ channel.reconcileAgain = true;
2477
+ return null;
2478
+ }
1795
2479
  const environment = await this.agentEnvironment(channel);
2480
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2481
+ channel.reconcileAgain = true;
2482
+ return null;
2483
+ }
1796
2484
  // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
1797
2485
  // account share one CODEX_HOME, and codex's new sqlite state runtime
1798
2486
  // cannot be opened by two concurrent processes — the second agent dies
@@ -1809,12 +2497,57 @@ export class Orchestrator {
1809
2497
  ? await this.isolateCodexHome(environment, agentId, args)
1810
2498
  : args;
1811
2499
  while (this.isActiveChannel(channel)) {
2500
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2501
+ channel.reconcileAgain = true;
2502
+ return null;
2503
+ }
2504
+ if (
2505
+ channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
2506
+ ) {
2507
+ channel.reconcileAgain = true;
2508
+ return null;
2509
+ }
2510
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2511
+ channel.reconcileAgain = true;
2512
+ return null;
2513
+ }
1812
2514
  const generation = channel.configGeneration;
1813
2515
  const session = await this.factory.create({ ...boundArgs, environment });
1814
2516
  if (!this.isActiveChannel(channel)) {
1815
2517
  await session.close().catch(() => {});
1816
2518
  return null;
1817
2519
  }
2520
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2521
+ await session.close().catch(() => {});
2522
+ channel.reconcileAgain = true;
2523
+ return null;
2524
+ }
2525
+ const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
2526
+ if (pendingRosterClose?.state === "pending-close") {
2527
+ await this.closeRosterChangedSession(
2528
+ channel,
2529
+ agentId,
2530
+ session,
2531
+ pendingRosterClose,
2532
+ );
2533
+ channel.reconcileAgain = true;
2534
+ return null;
2535
+ }
2536
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2537
+ const change = channel.rosterSessionChanges.get(agentId);
2538
+ if (change) {
2539
+ await this.closeRosterChangedSession(
2540
+ channel,
2541
+ agentId,
2542
+ session,
2543
+ change,
2544
+ );
2545
+ } else {
2546
+ await session.close().catch(() => {});
2547
+ }
2548
+ channel.reconcileAgain = true;
2549
+ return null;
2550
+ }
1818
2551
  if (generation === channel.configGeneration) {
1819
2552
  // Keep the final generation check, stale cleanup, and map bind in one
1820
2553
  // synchronous continuation. A config writer that resumes immediately
@@ -1831,6 +2564,17 @@ export class Orchestrator {
1831
2564
  return null;
1832
2565
  }
1833
2566
 
2567
+ private hasDesiredLaunchSettings(
2568
+ channel: Channel,
2569
+ agentId: string,
2570
+ launched: RosterAgent,
2571
+ ): boolean {
2572
+ const desired = channel.roster.find((agent) => agent.id === agentId);
2573
+ return Boolean(
2574
+ desired && rosterLaunchChangeReason(launched, desired) === null,
2575
+ );
2576
+ }
2577
+
1834
2578
  private agentEnvironment(
1835
2579
  channel: Channel,
1836
2580
  ): Promise<TaskEnvironmentHandle<unknown>> {
@@ -1878,6 +2622,7 @@ export class Orchestrator {
1878
2622
  const argv = cmd as [string, ...string[]];
1879
2623
  const result = await environment.exec({
1880
2624
  argv,
2625
+ projectEnvProfile: "maintenance",
1881
2626
  cwd,
1882
2627
  user: "node",
1883
2628
  env: RUNTIME_AUTHORITY_ENV,
@@ -2029,6 +2774,10 @@ export class Orchestrator {
2029
2774
  if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
2030
2775
  continue;
2031
2776
  }
2777
+ if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
2778
+ channel.reconcileAgain = true;
2779
+ continue;
2780
+ }
2032
2781
  channel.spawning.add(agent.id);
2033
2782
  try {
2034
2783
  const session = await this.createAndBindStableSession(
@@ -2037,13 +2786,14 @@ export class Orchestrator {
2037
2786
  {
2038
2787
  taskId: channel.taskId,
2039
2788
  agent,
2040
- systemPreamble: channel.preambles.get(agent.id) ?? "",
2041
- executionProfile: this.executionProfileFor(channel, agent.id),
2042
- attachCompatibilityKey: this.attachCompatibilityKeyFor(
2043
- channel,
2044
- agent.id,
2045
- ),
2046
- agentEnv: this.accountAgentEnv(
2789
+ systemPreamble: channel.preambles.get(agent.id) ?? "",
2790
+ executionProfile: this.executionProfileFor(channel, agent.id),
2791
+ attachCompatibilityKey: this.attachCompatibilityKeyFor(
2792
+ channel,
2793
+ agent,
2794
+ task,
2795
+ ),
2796
+ agentEnv: this.accountAgentEnv(
2047
2797
  channel,
2048
2798
  agent,
2049
2799
  agentCliEnv(
@@ -2054,7 +2804,7 @@ export class Orchestrator {
2054
2804
  cliSecret,
2055
2805
  this.isPromptLevelSecretary(channel, agent.id),
2056
2806
  ),
2057
- this.machineProjectEnv(task),
2807
+ task,
2058
2808
  ),
2059
2809
  },
2060
2810
  );
@@ -2185,6 +2935,78 @@ export class Orchestrator {
2185
2935
  return remaining;
2186
2936
  }
2187
2937
 
2938
+ /** Whether a retired generation still owns a cloud turn boundary. An
2939
+ * explicit interrupt already publishes its own aborted completion. */
2940
+ private needsAbortedTurnBoundary(
2941
+ channel: Channel,
2942
+ agentId: string,
2943
+ ): boolean {
2944
+ return (
2945
+ !channel.interrupted.has(agentId) &&
2946
+ (channel.activeTurns.has(agentId) ||
2947
+ channel.openTurns.has(agentId) ||
2948
+ (channel.acceptedPrompts.get(agentId)?.length ?? 0) > 0)
2949
+ );
2950
+ }
2951
+
2952
+ private retainRetiredTurnBoundary(
2953
+ channel: Channel,
2954
+ agentId: string,
2955
+ needed: boolean,
2956
+ ): void {
2957
+ if (needed) channel.retiredTurnBoundaries.add(agentId);
2958
+ }
2959
+
2960
+ /** Publish a retained terminal edge at most once. Recovery can be overtaken
2961
+ * by remove/edit after its generation-local turn maps have been cleared. */
2962
+ private emitRetiredTurnAbort(channel: Channel, agentId: string): boolean {
2963
+ if (!channel.retiredTurnBoundaries.delete(agentId)) return false;
2964
+ this.emitHost({
2965
+ kind: "agent.turn_complete",
2966
+ taskId: channel.taskId,
2967
+ agentId,
2968
+ aborted: true,
2969
+ });
2970
+ return true;
2971
+ }
2972
+
2973
+ private supersedeRecovery(channel: Channel, agentId: string): boolean {
2974
+ const recovery = channel.recoveryByAgent.get(agentId);
2975
+ if (!recovery) return false;
2976
+ recovery.superseded = true;
2977
+ // A live edit/removal now owns the terminal edge. Publish it before the
2978
+ // roster lifecycle can emit restarting/removed or bind a replacement.
2979
+ return this.emitRetiredTurnAbort(channel, agentId);
2980
+ }
2981
+
2982
+ private publishRosterChangeAbort(
2983
+ channel: Channel,
2984
+ agentId: string,
2985
+ change: RosterSessionChange,
2986
+ ): void {
2987
+ if (change.abortPublished) return;
2988
+ if (!this.emitRetiredTurnAbort(channel, agentId)) {
2989
+ this.emitHost({
2990
+ kind: "agent.turn_complete",
2991
+ taskId: channel.taskId,
2992
+ agentId,
2993
+ aborted: true,
2994
+ });
2995
+ }
2996
+ change.abortPublished = true;
2997
+ }
2998
+
2999
+ private recoveryIsCurrent(
3000
+ channel: Channel,
3001
+ agentId: string,
3002
+ recovery: RecoveryFence,
3003
+ ): boolean {
3004
+ return (
3005
+ channel.recoveryByAgent.get(agentId) === recovery &&
3006
+ !recovery.superseded
3007
+ );
3008
+ }
3009
+
2188
3010
  /**
2189
3011
  * Retire turn-local state owned by a runner generation before replacement.
2190
3012
  *
@@ -2429,7 +3251,8 @@ export class Orchestrator {
2429
3251
  exec: async (argv) => {
2430
3252
  if (!handle || !canExec) return null;
2431
3253
  const r = await handle.exec({
2432
- argv,
3254
+ argv: agentCliArgv(handle, argv),
3255
+ projectEnvProfile: "task",
2433
3256
  cwd: scratchDir,
2434
3257
  user: "node",
2435
3258
  env,
@@ -2618,6 +3441,13 @@ export class Orchestrator {
2618
3441
  if (channel.sessions.get(agentId) !== session) break;
2619
3442
  // The turn died with the session — drop its turn-state flags so a
2620
3443
  // respawned session starts clean.
3444
+ const abortBoundaryOnRosterChange =
3445
+ this.needsAbortedTurnBoundary(channel, agentId);
3446
+ this.retainRetiredTurnBoundary(
3447
+ channel,
3448
+ agentId,
3449
+ abortBoundaryOnRosterChange,
3450
+ );
2621
3451
  this.retireRunnerGeneration(channel, agentId);
2622
3452
  // Claude under load (especially Docker Desktop macOS) occasionally
2623
3453
  // unlinks ~/.claude.json mid-write during atomic config rewrites,
@@ -2627,13 +3457,32 @@ export class Orchestrator {
2627
3457
  // session so the user keeps working instead of staring at a dead
2628
3458
  // chat.
2629
3459
  const agent = channel.roster.find((a) => a.id === agentId);
3460
+ if (channel.rosterSessionChanges.has(agentId)) {
3461
+ // The authoritative roster already wants a different launch. Do not
3462
+ // rotate/recover this obsolete generation with its captured account;
3463
+ // preserve the live adapter so the normal stale-session recycler
3464
+ // closes it (and fences late Codex notifications). The cloud still
3465
+ // needs a terminal edge for any partial turn before restarting.
3466
+ this.reconcileErroredRosterChange(
3467
+ channel,
3468
+ agentId,
3469
+ event.message,
3470
+ abortBoundaryOnRosterChange,
3471
+ );
3472
+ break;
3473
+ }
2630
3474
  // ADR-076: a rate-limit/usage-cap error fails over to another account
2631
3475
  // for this kind (cool the current one, recreate the session on the
2632
3476
  // next, re-deliver the in-flight prompt). No-op with <2 accounts.
2633
3477
  if (
2634
3478
  event.retryable === "rate_limit" &&
2635
3479
  agent &&
2636
- (await this.rotateAccount(channel, agentId, agent))
3480
+ (await this.rotateAccount(
3481
+ channel,
3482
+ agentId,
3483
+ agent,
3484
+ abortBoundaryOnRosterChange,
3485
+ ))
2637
3486
  ) {
2638
3487
  break;
2639
3488
  }
@@ -2650,13 +3499,71 @@ export class Orchestrator {
2650
3499
  `[orchestrator] ${channel.taskId}/${agentId}: ${agent.kind} credential-invalid pattern matched: ${event.message.slice(0, 300)}`,
2651
3500
  );
2652
3501
  const accountId = channel.accountByAgent.get(agentId) ?? null;
2653
- const requested = await requestCredentialRefresh({
2654
- kind: agent.kind,
2655
- accountId: accountId && !accountId.startsWith("default:") ? accountId : null,
2656
- scope: channel.engineAccountScope,
2657
- });
3502
+ // The broker is a slow await between a failed turn and its recovery.
3503
+ // Own the spawn slot so a concurrent roster edit cannot publish
3504
+ // ready for a replacement before this old generation publishes its
3505
+ // aborted boundary (events carry no generation id in the cloud).
3506
+ const ownsBrokerSlot = !channel.spawning.has(agentId);
3507
+ if (ownsBrokerSlot) channel.spawning.add(agentId);
3508
+ let requested = false;
3509
+ try {
3510
+ requested = await requestCredentialRefresh({
3511
+ kind: agent.kind,
3512
+ accountId:
3513
+ accountId && !accountId.startsWith("default:")
3514
+ ? accountId
3515
+ : null,
3516
+ scope: channel.engineAccountScope,
3517
+ });
3518
+ } catch (err) {
3519
+ console.warn(
3520
+ `[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
3521
+ err instanceof Error ? err.message : String(err)
3522
+ }`,
3523
+ );
3524
+ } finally {
3525
+ if (ownsBrokerSlot) channel.spawning.delete(agentId);
3526
+ }
3527
+ // Removal/edit/recovery may have retired this generation while the
3528
+ // broker request was in flight. Never let its continuation mutate a
3529
+ // replacement or repopulate state for an absent roster id.
3530
+ if (channel.sessions.get(agentId) !== session) {
3531
+ // An edit can complete its restart while the broker request is
3532
+ // still outstanding, consuming the roster-change marker at bind.
3533
+ // The captured launch tuple still proves that this failed turn
3534
+ // belongs to the retired generation and needs its aborted edge.
3535
+ const stillRostered = channel.roster.some(
3536
+ (candidate) => candidate.id === agentId,
3537
+ );
3538
+ if (
3539
+ stillRostered &&
3540
+ (!channel.sessions.has(agentId) ||
3541
+ !this.hasDesiredLaunchSettings(channel, agentId, agent))
3542
+ ) {
3543
+ this.reconcileErroredRosterChange(
3544
+ channel,
3545
+ agentId,
3546
+ event.message,
3547
+ abortBoundaryOnRosterChange,
3548
+ );
3549
+ }
3550
+ break;
3551
+ }
3552
+ if (channel.rosterSessionChanges.has(agentId)) {
3553
+ this.reconcileErroredRosterChange(
3554
+ channel,
3555
+ agentId,
3556
+ event.message,
3557
+ abortBoundaryOnRosterChange,
3558
+ );
3559
+ break;
3560
+ }
2658
3561
  if (requested) {
2659
- await this.refreshAgentToken(channel, agent);
3562
+ await this.refreshAgentToken(
3563
+ channel,
3564
+ agent,
3565
+ abortBoundaryOnRosterChange,
3566
+ );
2660
3567
  break;
2661
3568
  }
2662
3569
  }
@@ -2670,7 +3577,11 @@ export class Orchestrator {
2670
3577
  console.warn(
2671
3578
  `[orchestrator] ${channel.taskId}/${agentId}: claude auth-revoked pattern matched: ${event.message.slice(0, 300)}`,
2672
3579
  );
2673
- await this.refreshAgentToken(channel, agent);
3580
+ await this.refreshAgentToken(
3581
+ channel,
3582
+ agent,
3583
+ abortBoundaryOnRosterChange,
3584
+ );
2674
3585
  break;
2675
3586
  }
2676
3587
  const configMissing = CLAUDE_CONFIG_MISSING_PATTERNS.some((re) =>
@@ -2681,10 +3592,15 @@ export class Orchestrator {
2681
3592
  // than replaying the failed one. Do not leave it looking in-flight to
2682
3593
  // a later account refresh.
2683
3594
  channel.lastPrompt.delete(agentId);
2684
- await this.recoverClaudeAgent(channel, agentId);
3595
+ await this.recoverClaudeAgent(
3596
+ channel,
3597
+ agentId,
3598
+ abortBoundaryOnRosterChange,
3599
+ );
2685
3600
  break;
2686
3601
  }
2687
3602
  channel.lastPrompt.delete(agentId);
3603
+ channel.retiredTurnBoundaries.delete(agentId);
2688
3604
  this.emitHost({
2689
3605
  kind: "agent.exit",
2690
3606
  taskId: channel.taskId,
@@ -2730,7 +3646,11 @@ export class Orchestrator {
2730
3646
  });
2731
3647
  if (remainingTurns === 0) {
2732
3648
  channel.lastPrompt.delete(agentId);
2733
- this.startNextPendingPrompt(channel, agentId);
3649
+ // A queued Secretary reply must run on the replacement settings,
3650
+ // not extend the obsolete generation past its first safe boundary.
3651
+ if (!channel.rosterSessionChanges.has(agentId)) {
3652
+ this.startNextPendingPrompt(channel, agentId);
3653
+ }
2734
3654
  }
2735
3655
  // A config repair never interrupts a turn. The stale id remains queued
2736
3656
  // until this exact boundary, then reconcile closes and replaces it.
@@ -2761,18 +3681,73 @@ export class Orchestrator {
2761
3681
  if (channel.sessions.get(agentId) !== session) break;
2762
3682
  // Same zombie hazard as the error path — a session whose process
2763
3683
  // ended (even cleanly) can never carry another turn.
3684
+ const abortBoundaryOnRosterChange =
3685
+ this.needsAbortedTurnBoundary(channel, agentId);
2764
3686
  this.retireRunnerGeneration(channel, agentId);
2765
3687
  channel.lastPrompt.delete(agentId);
2766
3688
  channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
2767
3689
  channel.respawnLastAt.set(agentId, Date.now());
2768
3690
  channel.sessions.delete(agentId);
2769
- channel.browserStaleSessions.delete(agentId);
3691
+ const rosterChangePending = channel.rosterSessionChanges.has(agentId);
3692
+ if (rosterChangePending && abortBoundaryOnRosterChange) {
3693
+ this.publishRosterChangeAbort(
3694
+ channel,
3695
+ agentId,
3696
+ channel.rosterSessionChanges.get(agentId)!,
3697
+ );
3698
+ }
3699
+ if (!rosterChangePending) channel.browserStaleSessions.delete(agentId);
2770
3700
  channel.browserPendingStaleSessions.delete(agentId);
2771
3701
  this.reconcilePendingSecretary(channel, agentId);
3702
+ if (rosterChangePending) {
3703
+ void this.reconcileSessions(channel).catch((err: unknown) => {
3704
+ console.warn(
3705
+ `[orchestrator] ${channel.taskId}/${agentId}: roster-change exit reconcile failed: ${
3706
+ err instanceof Error ? err.message : String(err)
3707
+ }`,
3708
+ );
3709
+ });
3710
+ }
2772
3711
  break;
2773
3712
  }
2774
3713
  }
2775
3714
 
3715
+ /** An engine error can race a live launch edit after a partial turn. Keep
3716
+ * the obsolete adapter owned until the stale recycler closes it, but close
3717
+ * the cloud turn boundary immediately so restarting/ready cannot leave the
3718
+ * crew UI stuck in a working state. */
3719
+ private reconcileErroredRosterChange(
3720
+ channel: Channel,
3721
+ agentId: string,
3722
+ reason: string,
3723
+ abortBoundary: boolean,
3724
+ ): void {
3725
+ channel.lastPrompt.delete(agentId);
3726
+ channel.browserPendingStaleSessions.delete(agentId);
3727
+ if (abortBoundary) {
3728
+ const change = channel.rosterSessionChanges.get(agentId);
3729
+ if (change) this.publishRosterChangeAbort(channel, agentId, change);
3730
+ else this.emitRetiredTurnAbort(channel, agentId);
3731
+ }
3732
+ void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
3733
+ console.warn(
3734
+ `[orchestrator] ${channel.taskId}/${agentId}: roster-change error (${reason}) reconcile failed: ${
3735
+ err instanceof Error ? err.message : String(err)
3736
+ }`,
3737
+ );
3738
+ });
3739
+ }
3740
+
3741
+ /** Removal owns an asynchronous close/durable-stop fence. Recovery paths
3742
+ * overtaken by remove/re-add must join it before they can spawn the re-added
3743
+ * id, otherwise removal can stop the replacement generation. */
3744
+ private async reconcileAfterRosterRefresh(channel: Channel): Promise<void> {
3745
+ const rosterRefresh = channel.rosterRefreshReady;
3746
+ if (rosterRefresh) await rosterRefresh;
3747
+ if (!this.isActiveChannel(channel)) return;
3748
+ await this.reconcileSessions(channel);
3749
+ }
3750
+
2776
3751
  /** A daemon loss is infrastructure recovery, not an agent crash. Preserve
2777
3752
  * accepted work, drop the dead generation, and leave respawn/exit accounting
2778
3753
  * untouched; the runtime-ready hook recreates it deterministically. */
@@ -2797,9 +3772,20 @@ export class Orchestrator {
2797
3772
  channel.runtimeDeferredAgents.add(agentId);
2798
3773
  }
2799
3774
  channel.sessions.delete(agentId);
2800
- channel.browserStaleSessions.delete(agentId);
3775
+ if (!channel.rosterSessionChanges.has(agentId)) {
3776
+ channel.browserStaleSessions.delete(agentId);
3777
+ }
2801
3778
  channel.browserPendingStaleSessions.delete(agentId);
2802
3779
  await session.close().catch(() => {});
3780
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
3781
+ if (rosterChange && this.isActiveChannel(channel)) {
3782
+ await this.closeRosterChangedSession(
3783
+ channel,
3784
+ agentId,
3785
+ session,
3786
+ rosterChange,
3787
+ );
3788
+ }
2803
3789
  }
2804
3790
 
2805
3791
  // -- session recovery -----------------------------------------------------
@@ -2812,6 +3798,7 @@ export class Orchestrator {
2812
3798
  private async recoverClaudeAgent(
2813
3799
  channel: Channel,
2814
3800
  agentId: string,
3801
+ abortBoundaryOnRosterChange = false,
2815
3802
  ): Promise<void> {
2816
3803
  if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
2817
3804
 
@@ -2823,6 +3810,7 @@ export class Orchestrator {
2823
3810
  channel.sessions.delete(agentId);
2824
3811
  channel.browserStaleSessions.delete(agentId);
2825
3812
  channel.browserPendingStaleSessions.delete(agentId);
3813
+ channel.retiredTurnBoundaries.delete(agentId);
2826
3814
  this.emitHost({
2827
3815
  kind: "agent.exit",
2828
3816
  taskId: channel.taskId,
@@ -2841,6 +3829,15 @@ export class Orchestrator {
2841
3829
  channel.containerName,
2842
3830
  channel.agentHandle ?? undefined,
2843
3831
  );
3832
+ if (channel.rosterSessionChanges.has(agentId)) {
3833
+ this.reconcileErroredRosterChange(
3834
+ channel,
3835
+ agentId,
3836
+ "Claude config recovery superseded by live roster edit",
3837
+ abortBoundaryOnRosterChange,
3838
+ );
3839
+ return;
3840
+ }
2844
3841
 
2845
3842
  // Delete the old generation before awaiting close. Its adapter emits a
2846
3843
  // terminal event during teardown; bindSession's identity guard must see
@@ -2848,7 +3845,9 @@ export class Orchestrator {
2848
3845
  // a second replacement path.
2849
3846
  const old = channel.sessions.get(agentId);
2850
3847
  channel.sessions.delete(agentId);
2851
- channel.browserStaleSessions.delete(agentId);
3848
+ if (!channel.rosterSessionChanges.has(agentId)) {
3849
+ channel.browserStaleSessions.delete(agentId);
3850
+ }
2852
3851
  channel.browserPendingStaleSessions.delete(agentId);
2853
3852
  this.retireRunnerGeneration(channel, agentId);
2854
3853
  if (old) {
@@ -2859,6 +3858,27 @@ export class Orchestrator {
2859
3858
  }
2860
3859
  }
2861
3860
  if (!this.isActiveChannel(channel)) return;
3861
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
3862
+ if (rosterChange) {
3863
+ // The generic recovery owned the close before the roster edit arrived.
3864
+ // close() is idempotent; route its completed boundary through the
3865
+ // roster lifecycle so the cloud still observes restarting -> ready.
3866
+ this.reconcileErroredRosterChange(
3867
+ channel,
3868
+ agentId,
3869
+ "Claude config recovery superseded by live roster edit",
3870
+ abortBoundaryOnRosterChange,
3871
+ );
3872
+ if (old) {
3873
+ await this.closeRosterChangedSession(
3874
+ channel,
3875
+ agentId,
3876
+ old,
3877
+ rosterChange,
3878
+ );
3879
+ }
3880
+ return;
3881
+ }
2862
3882
 
2863
3883
  const agent = channel.roster.find((a) => a.id === agentId);
2864
3884
  const task = getHostTask(channel.taskId);
@@ -2887,18 +3907,45 @@ export class Orchestrator {
2887
3907
  executionProfile: this.executionProfileFor(channel, agentId),
2888
3908
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2889
3909
  channel,
2890
- agentId,
3910
+ agent,
3911
+ task,
2891
3912
  ),
2892
3913
  agentEnv: boundAccount
2893
- ? { ...boundAccount.execEnv, ...base }
2894
- : this.accountAgentEnv(channel, agent, base),
3914
+ ? {
3915
+ ...this.machineProjectEnv(task),
3916
+ ...boundAccount.execEnv,
3917
+ ...base,
3918
+ }
3919
+ : this.accountAgentEnv(channel, agent, base, task),
2895
3920
  });
2896
- if (!session) return;
3921
+ if (!session) {
3922
+ if (channel.rosterSessionChanges.has(agentId)) {
3923
+ this.reconcileErroredRosterChange(
3924
+ channel,
3925
+ agentId,
3926
+ "Claude config recovery superseded by live roster edit",
3927
+ abortBoundaryOnRosterChange,
3928
+ );
3929
+ }
3930
+ return;
3931
+ }
3932
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
3933
+ if (channel.rosterSessionChanges.has(agentId)) {
3934
+ this.reconcileErroredRosterChange(
3935
+ channel,
3936
+ agentId,
3937
+ "Claude config recovery superseded by live roster edit",
3938
+ abortBoundaryOnRosterChange,
3939
+ );
3940
+ }
3941
+ return;
3942
+ }
2897
3943
  this.startNextPendingPrompt(channel, agentId);
2898
3944
 
2899
3945
  const message = restored
2900
3946
  ? `${agentId} restarted — config file was missing in the container, restored from the host.`
2901
3947
  : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
3948
+ channel.retiredTurnBoundaries.delete(agentId);
2902
3949
  this.emitHost({
2903
3950
  kind: "agent.exit",
2904
3951
  taskId: channel.taskId,
@@ -2940,8 +3987,18 @@ export class Orchestrator {
2940
3987
  channel: Channel,
2941
3988
  agentId: string,
2942
3989
  agent: RosterAgent,
3990
+ abortBoundaryOnRosterChange = false,
2943
3991
  ): Promise<boolean> {
2944
3992
  if (!this.isActiveChannel(channel)) return true;
3993
+ if (channel.rosterSessionChanges.has(agentId)) {
3994
+ this.reconcileErroredRosterChange(
3995
+ channel,
3996
+ agentId,
3997
+ "account rotation superseded by live roster edit",
3998
+ abortBoundaryOnRosterChange,
3999
+ );
4000
+ return true;
4001
+ }
2945
4002
  // A concurrent terminal event for the same session is already being
2946
4003
  // handled by the path that owns this spawn slot.
2947
4004
  if (channel.spawning.has(agentId)) return true;
@@ -2964,6 +4021,8 @@ export class Orchestrator {
2964
4021
  const task = getHostTask(channel.taskId);
2965
4022
  if (!task || task.statusMirror !== "running") return false;
2966
4023
 
4024
+ const recovery: RecoveryFence = { superseded: false };
4025
+ channel.recoveryByAgent.set(agentId, recovery);
2967
4026
  let replacement: AgentSession | null = null;
2968
4027
  channel.spawning.add(agentId);
2969
4028
  try {
@@ -2974,6 +4033,11 @@ export class Orchestrator {
2974
4033
  channel.sessions.delete(agentId);
2975
4034
  channel.browserStaleSessions.delete(agentId);
2976
4035
  channel.browserPendingStaleSessions.delete(agentId);
4036
+ this.retainRetiredTurnBoundary(
4037
+ channel,
4038
+ agentId,
4039
+ abortBoundaryOnRosterChange,
4040
+ );
2977
4041
  this.retireRunnerGeneration(channel, agentId);
2978
4042
  if (old) {
2979
4043
  try {
@@ -2983,6 +4047,25 @@ export class Orchestrator {
2983
4047
  }
2984
4048
  }
2985
4049
  if (!this.isActiveChannel(channel)) return true;
4050
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4051
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
4052
+ if (rosterChange) {
4053
+ this.reconcileErroredRosterChange(
4054
+ channel,
4055
+ agentId,
4056
+ "account rotation superseded by live roster edit",
4057
+ abortBoundaryOnRosterChange,
4058
+ );
4059
+ if (old) {
4060
+ await this.closeRosterChangedSession(
4061
+ channel,
4062
+ agentId,
4063
+ old,
4064
+ rosterChange,
4065
+ );
4066
+ }
4067
+ return true;
4068
+ }
2986
4069
 
2987
4070
  // The serialized materialization gate owns both account provisioning
2988
4071
  // and, when enabled, browser migration.
@@ -2992,8 +4075,21 @@ export class Orchestrator {
2992
4075
  true,
2993
4076
  );
2994
4077
  if (!this.isActiveChannel(channel)) return true;
4078
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
2995
4079
  await this.ensureMcpConfig(channel);
2996
4080
  if (!this.isActiveChannel(channel)) return true;
4081
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4082
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4083
+ if (channel.rosterSessionChanges.has(agentId)) {
4084
+ this.reconcileErroredRosterChange(
4085
+ channel,
4086
+ agentId,
4087
+ "account rotation superseded by live roster edit",
4088
+ abortBoundaryOnRosterChange,
4089
+ );
4090
+ }
4091
+ return true;
4092
+ }
2997
4093
 
2998
4094
  const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
2999
4095
  const cliSecret = loadTaskCliSecret(channel.taskId);
@@ -3018,12 +4114,39 @@ export class Orchestrator {
3018
4114
  executionProfile: this.executionProfileFor(channel, agentId),
3019
4115
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
3020
4116
  channel,
3021
- agentId,
4117
+ agent,
4118
+ task,
3022
4119
  ),
3023
- agentEnv: { ...next.execEnv, ...base },
4120
+ agentEnv: {
4121
+ ...this.machineProjectEnv(task),
4122
+ ...next.execEnv,
4123
+ ...base,
4124
+ },
3024
4125
  },
4126
+ recovery,
3025
4127
  );
3026
- if (!replacement) return true;
4128
+ if (!replacement) {
4129
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4130
+ this.reconcileErroredRosterChange(
4131
+ channel,
4132
+ agentId,
4133
+ "account rotation superseded by live roster edit",
4134
+ abortBoundaryOnRosterChange,
4135
+ );
4136
+ }
4137
+ return true;
4138
+ }
4139
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4140
+ if (channel.rosterSessionChanges.has(agentId)) {
4141
+ this.reconcileErroredRosterChange(
4142
+ channel,
4143
+ agentId,
4144
+ "account rotation superseded by live roster edit",
4145
+ abortBoundaryOnRosterChange,
4146
+ );
4147
+ }
4148
+ return true;
4149
+ }
3027
4150
 
3028
4151
  this.emitSystemNote(
3029
4152
  channel.taskId,
@@ -3033,14 +4156,35 @@ export class Orchestrator {
3033
4156
  // Re-deliver the in-flight prompt so the interrupted turn resumes.
3034
4157
  const prompt = channel.lastPrompt.get(agentId);
3035
4158
  if (prompt) {
4159
+ channel.retiredTurnBoundaries.delete(agentId);
3036
4160
  this.startPrompt(channel, agentId, replacement, prompt);
3037
4161
  } else {
4162
+ this.emitRetiredTurnAbort(channel, agentId);
3038
4163
  this.startNextPendingPrompt(channel, agentId);
3039
4164
  }
3040
4165
  } finally {
3041
4166
  channel.spawning.delete(agentId);
4167
+ if (channel.recoveryByAgent.get(agentId) === recovery) {
4168
+ channel.recoveryByAgent.delete(agentId);
4169
+ }
3042
4170
  if (!replacement) {
4171
+ if (!channel.roster.some((candidate) => candidate.id === agentId)) {
4172
+ channel.accountByAgent.delete(agentId);
4173
+ channel.rotations.delete(agentId);
4174
+ }
3043
4175
  this.reconcilePendingSecretary(channel, agentId);
4176
+ if (
4177
+ this.isActiveChannel(channel) &&
4178
+ (recovery.superseded || channel.rosterSessionChanges.has(agentId))
4179
+ ) {
4180
+ void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
4181
+ console.warn(
4182
+ `[orchestrator] ${channel.taskId}/${agentId}: post-rotation roster reconcile failed: ${
4183
+ err instanceof Error ? err.message : String(err)
4184
+ }`,
4185
+ );
4186
+ });
4187
+ }
3044
4188
  }
3045
4189
  }
3046
4190
 
@@ -3077,10 +4221,23 @@ export class Orchestrator {
3077
4221
  agent: RosterAgent,
3078
4222
  account: ResolvedEngineAccount,
3079
4223
  note: string,
4224
+ abortBoundaryOnRosterChange = false,
3080
4225
  ): Promise<boolean> {
4226
+ if (channel.rosterSessionChanges.has(agentId)) {
4227
+ this.reconcileErroredRosterChange(
4228
+ channel,
4229
+ agentId,
4230
+ "credential respawn superseded by live roster edit",
4231
+ abortBoundaryOnRosterChange,
4232
+ );
4233
+ return true;
4234
+ }
4235
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return true;
3081
4236
  const task = getHostTask(channel.taskId);
3082
4237
  if (!task || task.statusMirror !== "running") return false;
3083
4238
 
4239
+ const recovery: RecoveryFence = { superseded: false };
4240
+ channel.recoveryByAgent.set(agentId, recovery);
3084
4241
  let replacement: AgentSession | null = null;
3085
4242
  channel.spawning.add(agentId);
3086
4243
  try {
@@ -3102,6 +4259,25 @@ export class Orchestrator {
3102
4259
  }
3103
4260
  }
3104
4261
  if (!this.isActiveChannel(channel)) return true;
4262
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4263
+ const rosterChange = channel.rosterSessionChanges.get(agentId);
4264
+ if (rosterChange) {
4265
+ this.reconcileErroredRosterChange(
4266
+ channel,
4267
+ agentId,
4268
+ "credential respawn superseded by live roster edit",
4269
+ abortBoundaryOnRosterChange,
4270
+ );
4271
+ if (old) {
4272
+ await this.closeRosterChangedSession(
4273
+ channel,
4274
+ agentId,
4275
+ old,
4276
+ rosterChange,
4277
+ );
4278
+ }
4279
+ return true;
4280
+ }
3105
4281
 
3106
4282
  await this.ensureBrowserSetup(
3107
4283
  channel,
@@ -3109,8 +4285,21 @@ export class Orchestrator {
3109
4285
  true,
3110
4286
  );
3111
4287
  if (!this.isActiveChannel(channel)) return true;
4288
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
3112
4289
  await this.ensureMcpConfig(channel);
3113
4290
  if (!this.isActiveChannel(channel)) return true;
4291
+ if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
4292
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4293
+ if (channel.rosterSessionChanges.has(agentId)) {
4294
+ this.reconcileErroredRosterChange(
4295
+ channel,
4296
+ agentId,
4297
+ "credential respawn superseded by live roster edit",
4298
+ abortBoundaryOnRosterChange,
4299
+ );
4300
+ }
4301
+ return true;
4302
+ }
3114
4303
 
3115
4304
  const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
3116
4305
  const cliSecret = loadTaskCliSecret(channel.taskId);
@@ -3125,32 +4314,84 @@ export class Orchestrator {
3125
4314
  channel.accountByAgent.set(agentId, account.id);
3126
4315
  noteEngineAccountUsed(account.id);
3127
4316
 
3128
- replacement = await this.createAndBindStableSession(channel, agentId, {
3129
- taskId: channel.taskId,
3130
- agent,
3131
- systemPreamble: channel.preambles.get(agentId) ?? "",
3132
- executionProfile: this.executionProfileFor(channel, agentId),
3133
- attachCompatibilityKey: this.attachCompatibilityKeyFor(
3134
- channel,
3135
- agentId,
3136
- ),
3137
- agentEnv: { ...account.execEnv, ...base },
3138
- });
3139
- if (!replacement) return true;
4317
+ replacement = await this.createAndBindStableSession(
4318
+ channel,
4319
+ agentId,
4320
+ {
4321
+ taskId: channel.taskId,
4322
+ agent,
4323
+ systemPreamble: channel.preambles.get(agentId) ?? "",
4324
+ executionProfile: this.executionProfileFor(channel, agentId),
4325
+ attachCompatibilityKey: this.attachCompatibilityKeyFor(
4326
+ channel,
4327
+ agent,
4328
+ task,
4329
+ ),
4330
+ agentEnv: {
4331
+ ...this.machineProjectEnv(task),
4332
+ ...account.execEnv,
4333
+ ...base,
4334
+ },
4335
+ },
4336
+ recovery,
4337
+ );
4338
+ if (!replacement) {
4339
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4340
+ this.reconcileErroredRosterChange(
4341
+ channel,
4342
+ agentId,
4343
+ "credential respawn superseded by live roster edit",
4344
+ abortBoundaryOnRosterChange,
4345
+ );
4346
+ }
4347
+ return true;
4348
+ }
4349
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
4350
+ if (channel.rosterSessionChanges.has(agentId)) {
4351
+ this.reconcileErroredRosterChange(
4352
+ channel,
4353
+ agentId,
4354
+ "credential respawn superseded by live roster edit",
4355
+ abortBoundaryOnRosterChange,
4356
+ );
4357
+ }
4358
+ return true;
4359
+ }
3140
4360
 
3141
4361
  this.emitSystemNote(channel.taskId, note);
3142
4362
 
3143
4363
  // Re-deliver the in-flight prompt so the interrupted turn resumes.
3144
4364
  const prompt = channel.lastPrompt.get(agentId);
3145
4365
  if (prompt) {
4366
+ channel.retiredTurnBoundaries.delete(agentId);
3146
4367
  this.startPrompt(channel, agentId, replacement, prompt);
3147
4368
  } else {
4369
+ this.emitRetiredTurnAbort(channel, agentId);
3148
4370
  this.startNextPendingPrompt(channel, agentId);
3149
4371
  }
3150
4372
  } finally {
3151
4373
  channel.spawning.delete(agentId);
4374
+ if (channel.recoveryByAgent.get(agentId) === recovery) {
4375
+ channel.recoveryByAgent.delete(agentId);
4376
+ }
3152
4377
  if (!replacement) {
4378
+ if (!channel.roster.some((candidate) => candidate.id === agentId)) {
4379
+ channel.accountByAgent.delete(agentId);
4380
+ channel.rotations.delete(agentId);
4381
+ }
3153
4382
  this.reconcilePendingSecretary(channel, agentId);
4383
+ if (
4384
+ this.isActiveChannel(channel) &&
4385
+ (recovery.superseded || channel.rosterSessionChanges.has(agentId))
4386
+ ) {
4387
+ void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
4388
+ console.warn(
4389
+ `[orchestrator] ${channel.taskId}/${agentId}: post-respawn roster reconcile failed: ${
4390
+ err instanceof Error ? err.message : String(err)
4391
+ }`,
4392
+ );
4393
+ });
4394
+ }
3154
4395
  }
3155
4396
  }
3156
4397
 
@@ -3179,14 +4420,21 @@ export class Orchestrator {
3179
4420
  private async refreshAgentToken(
3180
4421
  channel: Channel,
3181
4422
  agent: RosterAgent,
4423
+ abortBoundaryOnRosterChange?: boolean,
3182
4424
  ): Promise<void> {
3183
4425
  const agentId = agent.id;
3184
4426
  if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
4427
+ if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return;
4428
+ const abortBoundary =
4429
+ abortBoundaryOnRosterChange ??
4430
+ this.needsAbortedTurnBoundary(channel, agentId);
4431
+ this.retainRetiredTurnBoundary(channel, agentId, abortBoundary);
3185
4432
 
3186
4433
  const tries = (channel.respawns.get(agentId) ?? 0) + 1;
3187
4434
  channel.respawns.set(agentId, tries);
3188
4435
  channel.respawnLastAt.set(agentId, Date.now());
3189
4436
  if (tries > MAX_RESPAWNS_PER_AGENT) {
4437
+ channel.retiredTurnBoundaries.delete(agentId);
3190
4438
  this.emitHost({
3191
4439
  kind: "agent.exit",
3192
4440
  taskId: channel.taskId,
@@ -3203,7 +4451,10 @@ export class Orchestrator {
3203
4451
  const account =
3204
4452
  accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
3205
4453
  accounts[0];
3206
- if (!account) return;
4454
+ if (!account) {
4455
+ this.emitRetiredTurnAbort(channel, agentId);
4456
+ return;
4457
+ }
3207
4458
 
3208
4459
  await this.respawnAgentOnAccount(
3209
4460
  channel,
@@ -3211,6 +4462,7 @@ export class Orchestrator {
3211
4462
  agent,
3212
4463
  account,
3213
4464
  `${agentId}: ${agent.kind} credential refreshed — restarted the agent.`,
4465
+ abortBoundary,
3214
4466
  );
3215
4467
  }
3216
4468
 
@@ -3252,9 +4504,21 @@ export class Orchestrator {
3252
4504
  async refreshEngineAccountAgents(kind: string): Promise<void> {
3253
4505
  for (const channel of this.channels.values()) {
3254
4506
  if (!this.isActiveChannel(channel)) continue;
3255
- for (const agent of channel.roster) {
3256
- if (agent.kind !== kind || !channel.sessions.has(agent.id)) continue;
3257
- channel.respawns.set(agent.id, 0);
4507
+ const agentIds = channel.roster
4508
+ .filter((agent) => agent.kind === kind)
4509
+ .map((agent) => agent.id);
4510
+ for (const agentId of agentIds) {
4511
+ const agent = channel.roster.find(
4512
+ (candidate) => candidate.id === agentId,
4513
+ );
4514
+ if (
4515
+ !agent ||
4516
+ agent.kind !== kind ||
4517
+ !channel.sessions.has(agentId)
4518
+ ) {
4519
+ continue;
4520
+ }
4521
+ channel.respawns.set(agentId, 0);
3258
4522
  try {
3259
4523
  await this.refreshAgentToken(channel, agent);
3260
4524
  } catch (err) {
@@ -3299,7 +4563,10 @@ export class Orchestrator {
3299
4563
  this.channels.delete(taskId);
3300
4564
  this.clearBrowserRetry(ch);
3301
4565
  ch.browserStaleSessions.clear();
4566
+ ch.rosterSessionChanges.clear();
3302
4567
  ch.browserPendingStaleSessions.clear();
4568
+ ch.browserConfigAffectedKinds.clear();
4569
+ ch.mcpProvisionAffectedKinds.clear();
3303
4570
  ch.activeTurns.clear();
3304
4571
  ch.acceptedPrompts.clear();
3305
4572
  ch.pendingPrompts.clear();
@@ -3307,17 +4574,22 @@ export class Orchestrator {
3307
4574
  ch.runtimeDeferredAgents.clear();
3308
4575
  ch.openTurns.clear();
3309
4576
  ch.interrupted.clear();
4577
+ ch.retiredTurnBoundaries.clear();
4578
+ ch.recoveryByAgent.clear();
3310
4579
  ch.reconcileAgain = false;
3311
4580
  const drains = [
3312
4581
  ch.browserSetup,
3313
4582
  ch.mcpConfigWrite,
3314
4583
  ch.reconcileReady,
4584
+ ch.rosterRefreshReady,
4585
+ ch.persistedRosterReady,
3315
4586
  ch.sessionsReady,
3316
4587
  ...ch.factoryOperations,
3317
4588
  ...ch.eventOperations,
3318
4589
  ].filter((promise): promise is Promise<unknown> => promise !== null);
3319
4590
  const sessions = [...ch.sessions.values()];
3320
4591
  ch.sessions.clear();
4592
+ ch.factoryOperationByAgent.clear();
3321
4593
  const closes = sessions.map((session) =>
3322
4594
  session.close().catch(() => {
3323
4595
  // Teardown is best-effort; every adapter close is intended idempotent.
@@ -3350,18 +4622,38 @@ export class Orchestrator {
3350
4622
  * tombstone prevents any agent process from respawning during maintenance. */
3351
4623
  async quarantineChannelForRecovery(taskId: string): Promise<void> {
3352
4624
  const currentSpec = this.channelSpecs.get(taskId);
4625
+ const suspended = suspendTaskGatewayTask(taskId);
4626
+ if (suspended) this.suspendedTaskGateways.set(taskId, suspended);
3353
4627
  await this.closeChannel(taskId);
3354
4628
  if (currentSpec && !this.channelSpecs.has(taskId)) {
3355
4629
  this.channelSpecs.set(taskId, currentSpec);
3356
4630
  }
3357
4631
  }
3358
4632
 
4633
+ /** Reopen only the authenticated task gateway for sanctioned recovery
4634
+ * work. The channel tombstone remains in place until allowChannel(), so
4635
+ * agents cannot respawn while uai-init's dependency worker refreshes env. */
4636
+ reactivateTaskGatewayForRecovery(taskId: string): void {
4637
+ const suspended = this.suspendedTaskGateways.get(taskId);
4638
+ if (!suspended) return;
4639
+ if (!reactivateTaskGatewayTask(suspended)) {
4640
+ throw new Error(
4641
+ `task ${taskId} gateway generation could not be reactivated for recovery`,
4642
+ );
4643
+ }
4644
+ }
4645
+
3359
4646
  /** A successful task-up is the only transition that reopens a tombstoned
3360
4647
  * task id after stop/teardown. */
3361
4648
  allowChannel(taskId: string): void {
3362
4649
  // A caller that forgot the task-up fence cannot reopen the id while old
3363
4650
  // external operations are still draining.
3364
4651
  if (this.channelClosures.has(taskId)) return;
4652
+ const suspended = this.suspendedTaskGateways.get(taskId);
4653
+ if (suspended) {
4654
+ reactivateTaskGatewayTask(suspended);
4655
+ this.suspendedTaskGateways.delete(taskId);
4656
+ }
3365
4657
  this.blockedTasks.delete(taskId);
3366
4658
  }
3367
4659
 
@@ -3378,6 +4670,10 @@ export class Orchestrator {
3378
4670
  * (ADR-125) reads this — the cloud decides idleness from what it can see,
3379
4671
  * but only the host can see a turn that is mid-flight RIGHT NOW. */
3380
4672
  readonly #channelActivity = new Map<string, number>();
4673
+ private readonly suspendedTaskGateways = new Map<
4674
+ string,
4675
+ TaskGatewayRegistration
4676
+ >();
3381
4677
 
3382
4678
  markChannelActivity(taskId: string): void {
3383
4679
  this.#channelActivity.set(taskId, Date.now());
@@ -3403,7 +4699,11 @@ export class Orchestrator {
3403
4699
  if (!task) return { ok: false, error: "unknown task" };
3404
4700
  const project = task.composeProject;
3405
4701
  if (!project) return { ok: false, error: "task has no running stack" };
3406
- // Close sessions first so nothing races to respawn them mid-stop.
4702
+ // Revoke env/proxy access before a potentially long session drain, then
4703
+ // close sessions so nothing races to respawn them mid-stop.
4704
+ const suspendedGateway = suspendTaskGatewayTask(taskId);
4705
+ // ADR-153: no longer served; end its open proxy connections too.
4706
+ activeSecretsProxy()?.runtime.stopTask(taskId);
3407
4707
  await this.closeChannel(taskId);
3408
4708
  try {
3409
4709
  await stopPreviewSidecars(taskId);
@@ -3424,11 +4724,13 @@ export class Orchestrator {
3424
4724
  try {
3425
4725
  const environment = await reconstructPersistedTaskEnvironment(task);
3426
4726
  if (!environment) {
4727
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
3427
4728
  this.allowChannel(taskId);
3428
4729
  return { ok: false, error: "task has no persisted environment" };
3429
4730
  }
3430
4731
  await environment.stop();
3431
4732
  } catch (err) {
4733
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
3432
4734
  this.allowChannel(taskId);
3433
4735
  return {
3434
4736
  ok: false,
@@ -3436,11 +4738,41 @@ export class Orchestrator {
3436
4738
  };
3437
4739
  }
3438
4740
  upsertHostTask(taskId, { statusMirror: "stopped" });
4741
+ // The next start installs the CA again before anything reads the env;
4742
+ // no write still queued for the stopped container may land or record.
4743
+ invalidateTaskWrites(taskId);
4744
+ forgetTaskProxyReadiness(taskId);
4745
+ try {
4746
+ removeTaskGatewayTask(taskId);
4747
+ } catch (error) {
4748
+ // The runtime is stopped and the in-memory generation is already
4749
+ // inactive. A leftover durable file also loads inactive; explicit
4750
+ // Resume rotates it through task-up.
4751
+ console.warn(
4752
+ `[task-gateway] ${taskId}: paused with stale inactive gateway state: ${
4753
+ error instanceof Error ? error.message : String(error)
4754
+ }`,
4755
+ );
4756
+ }
4757
+ // Nothing serves the task's env now: no value of it stays in memory.
4758
+ forgetTaskEnvSnapshot(taskId);
3439
4759
  clearTaskGatewayAcl(taskId);
3440
4760
  return { ok: true };
3441
4761
  }
3442
4762
  }
3443
4763
 
4764
+ /** Launch-time settings only. Wire-compatible absence spellings are equal. */
4765
+ function rosterLaunchChangeReason(
4766
+ previous: RosterAgent,
4767
+ next: RosterAgent,
4768
+ ): string | null {
4769
+ const changed: string[] = [];
4770
+ if (previous.kind !== next.kind) changed.push("engine");
4771
+ if ((previous.model ?? "") !== (next.model ?? "")) changed.push("model");
4772
+ if ((previous.effort ?? "") !== (next.effort ?? "")) changed.push("effort");
4773
+ return changed.length > 0 ? `${changed.join("/")} change` : null;
4774
+ }
4775
+
3444
4776
  /** Exact host-wire role validation; never trims or canonicalises agent ids. */
3445
4777
  export function hasValidSecretarySelection(
3446
4778
  spec: Pick<
@@ -3581,6 +4913,7 @@ async function repairClaudeConfigInContainer(
3581
4913
  if (environment) {
3582
4914
  const write = await environment.exec({
3583
4915
  argv: ["/bin/sh", "-c", repairScript],
4916
+ projectEnvProfile: "maintenance",
3584
4917
  env: RUNTIME_AUTHORITY_ENV,
3585
4918
  stdin: content,
3586
4919
  timeoutMs: 30_000,
@@ -3677,6 +5010,28 @@ export function assembleFirstTurnPrompt(
3677
5010
  * to every turn; only `initialPrompt` is delivered as the first user
3678
5011
  * turn (see `assembleFirstTurnPrompt`).
3679
5012
  */
5013
+ /** The task's usable proxied keys right now (ADR-153 §5): its agents are
5014
+ * told which env vars hold placeholders, and for which domains; a withheld
5015
+ * key is absent from the task, so it is not mentioned. */
5016
+ function taskProxiedKeys(taskId: string): { key: string; domains: string[] }[] {
5017
+ // The same gate as the placeholder and the injection.
5018
+ return usableProxiedKeys(taskId).map((key) => ({ key: key.key, domains: [...key.domains] }));
5019
+ }
5020
+
5021
+ /** ADR-153 §5's line, with the exact key-to-domain mapping. */
5022
+ export function proxiedEnvLine(keys: readonly { readonly key: string; readonly domains: readonly string[] }[]): string {
5023
+ const listed = [...keys]
5024
+ .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
5025
+ .map((entry) => `\`${entry.key}\` (${entry.domains.join(", ")})`)
5026
+ .join(", ");
5027
+ const domainCount = new Set(keys.flatMap((entry) => entry.domains)).size;
5028
+ return (
5029
+ `These env vars hold placeholders, not the real secrets: ${listed}. Uai's proxy puts the real value in on ` +
5030
+ `HTTPS requests to ${domainCount === 1 ? "that domain" : "those domains"}, so use them as they are — don't ` +
5031
+ `replace, re-fetch or "fix" them. They won't work anywhere else.`
5032
+ );
5033
+ }
5034
+
3680
5035
  export function buildSystemPreamble(
3681
5036
  roster: Roster,
3682
5037
  agent: RosterAgent,
@@ -3689,6 +5044,7 @@ export function buildSystemPreamble(
3689
5044
  sharedFiles?: string,
3690
5045
  mode?: "open" | "secretary",
3691
5046
  secretaryAgentId?: string,
5047
+ proxiedKeys?: readonly { readonly key: string; readonly domains: readonly string[] }[],
3692
5048
  ): string {
3693
5049
  const channelList = roster
3694
5050
  .map((a) =>
@@ -4134,6 +5490,10 @@ export function buildSystemPreamble(
4134
5490
  "",
4135
5491
  ]
4136
5492
  : []),
5493
+ // ADR-153 §5: proxied keys reach the task as placeholders.
5494
+ ...(proxiedKeys !== undefined && proxiedKeys.length > 0
5495
+ ? ["## Proxied env vars", "", proxiedEnvLine(proxiedKeys), ""]
5496
+ : []),
4137
5497
  // ADR-048: tell every agent with persona-granted permissions about its
4138
5498
  // `uai` CLI. Secretary mode adds dispatch but does not invent any other
4139
5499
  // permission: an empty persona permission set remains empty.
@@ -4329,7 +5689,15 @@ function beginRuntimeRecovery(
4329
5689
  const predecessor = recoveryInFlight;
4330
5690
  const sweep = async (): Promise<boolean> => {
4331
5691
  try {
4332
- return await recoverRunningTasks({ lifecycle, isCurrent });
5692
+ return await recoverRunningTasks({
5693
+ lifecycle,
5694
+ // Boot recovery deliberately keeps its per-task barriers instead of
5695
+ // entering lifecycle slots, but still needs the live orchestrator to
5696
+ // suspend/reactivate gateway generations around uai-init.
5697
+ consumerLifecycle:
5698
+ lifecycle ?? globalForOrchestrator.__uaiOrchestrator,
5699
+ isCurrent,
5700
+ });
4333
5701
  } catch (error) {
4334
5702
  console.error(
4335
5703
  "[orchestrator] recovery: unhandled failure",
@@ -5750,6 +7118,9 @@ export async function recoverRunningTasks(opts?: {
5750
7118
  /** Post-boot recovery must queue behind live taskUp/taskDown operations.
5751
7119
  * Boot recovery omits this and uses its earlier per-task barriers instead. */
5752
7120
  lifecycle?: Orchestrator;
7121
+ /** Consumer quarantine/admission hooks without changing boot recovery's
7122
+ * barrier-based sequencing. Production boot supplies the live singleton. */
7123
+ consumerLifecycle?: Orchestrator;
5753
7124
  /** Test seam; production waits for the shared image/CLI-volume reconcile. */
5754
7125
  maintenanceReady?: Promise<void>;
5755
7126
  /** A daemon-generation fence. False cancels every remaining side effect. */
@@ -5769,6 +7140,7 @@ export async function recoverRunningTasks(opts?: {
5769
7140
  const result = await recoveryPass(
5770
7141
  pendingTaskIds,
5771
7142
  opts?.lifecycle,
7143
+ opts?.consumerLifecycle,
5772
7144
  opts?.maintenanceReady ?? agentClisReady,
5773
7145
  opts?.isCurrent ?? (() => true),
5774
7146
  );
@@ -5813,6 +7185,7 @@ interface RecoveryPassResult {
5813
7185
  async function recoveryPass(
5814
7186
  onlyTaskIds?: ReadonlySet<string>,
5815
7187
  lifecycle?: Orchestrator,
7188
+ consumerLifecycle?: Orchestrator,
5816
7189
  maintenanceReady: Promise<void> = agentClisReady,
5817
7190
  isCurrent: () => boolean = () => true,
5818
7191
  ): Promise<RecoveryPassResult> {
@@ -5870,6 +7243,7 @@ async function recoveryPass(
5870
7243
  task,
5871
7244
  maintenanceReady,
5872
7245
  isCurrent,
7246
+ consumerLifecycle,
5873
7247
  );
5874
7248
  if (recovered) {
5875
7249
  if (!lifecycle) finishTaskRecovery(task.taskId);
@@ -6064,15 +7438,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
6064
7438
  /** Keep the MCP gateway reverse tunnel alive for a machine channel. The
6065
7439
  * handle exposes tunnel argv only on the machine provider; anything else is
6066
7440
  * a quiet no-op. */
6067
- function ensureMachineGatewayTunnelFor(
7441
+ async function ensureMachineGatewayTunnelFor(
6068
7442
  taskId: string,
6069
7443
  handle: TaskEnvironmentHandle<unknown> | null,
6070
- ): void {
7444
+ waitUntilReady = false,
7445
+ ): Promise<void> {
6071
7446
  const argv = handle?.gatewayTunnelSshArgs;
6072
7447
  if (!argv) return;
6073
- ensureMachineGatewayTunnel(taskId, {
6074
- gatewayTunnelSshArgs: argv.bind(handle),
6075
- });
7448
+ const source = { gatewayTunnelSshArgs: argv.bind(handle) };
7449
+ const loaderPort =
7450
+ handle.descriptor.envMode === "loader"
7451
+ ? MACHINE_GATEWAY_TUNNEL_PORT
7452
+ : undefined;
7453
+ if (waitUntilReady && loaderPort !== undefined) {
7454
+ await ensureMachineGatewayTunnelReady(taskId, source, loaderPort);
7455
+ return;
7456
+ }
7457
+ ensureMachineGatewayTunnel(taskId, source, loaderPort);
6076
7458
  }
6077
7459
 
6078
7460
  async function materializeAgentCli(
@@ -6107,25 +7489,105 @@ async function recoverPersistedTaskEnvironment(
6107
7489
  const environment = await reconstructPersistedTaskEnvironment(task);
6108
7490
  if (environment === null) return true;
6109
7491
  if (!isCurrent()) return true;
6110
- const result = await environment.recover({
6111
- maintenanceReady,
6112
- isCurrent,
6113
- ...(lifecycle
6114
- ? {
6115
- quarantineConsumers: () =>
6116
- lifecycle.quarantineChannelForRecovery(task.taskId),
6117
- allowConsumers: () => lifecycle.allowChannel(task.taskId),
6118
- ensureConsumersStarted: async () => {
6119
- await lifecycle.ensureStarted(task.taskId);
6120
- },
6121
- }
6122
- : {}),
6123
- });
6124
- return result.outcome === "recovered";
7492
+ // ADR-153: a recovery may restart the task's container or VM, which takes
7493
+ // the proxy's CA, chain and names with it, and its forwarder. Before the
7494
+ // provider touches it, nothing recorded counts any more and no write queued
7495
+ // or under way may land: a reconcile in flight ends with its exec under
7496
+ // way, and none pins again until this recovery's own slot has the
7497
+ // forwarder running.
7498
+ invalidateTaskWrites(task.taskId);
7499
+ forgetTaskProxyReadiness(task.taskId);
7500
+ // A running machine's editor can submit a command before any agent channel
7501
+ // is reconstructed. Re-arm the authenticated gateway tunnel at recovery,
7502
+ // not only at the first session spawn, so that command can refresh env.
7503
+ void ensureMachineGatewayTunnelFor(task.taskId, environment);
7504
+ let consumersPrepared = false;
7505
+ let tunnelReadinessFailed = false;
7506
+ const prepareConsumers = async (): Promise<void> => {
7507
+ if (!isCurrent()) return;
7508
+ try {
7509
+ await ensureMachineGatewayTunnelFor(task.taskId, environment, true);
7510
+ } catch (error) {
7511
+ tunnelReadinessFailed = true;
7512
+ console.warn(
7513
+ `[machine-tunnel] ${task.taskId}: recovery deferred until the gateway tunnel is ready (${error instanceof Error ? error.message : String(error)})`,
7514
+ );
7515
+ throw error;
7516
+ }
7517
+ // ADR-153 "Order", in one slot of the task's write queue: the CA (or,
7518
+ // with the proxy off, the pins out), then the forwarder, then the pins,
7519
+ // so no name is pinned with nothing listening behind it. The forwarder
7520
+ // carries the generation's secret into the task as root, and the
7521
+ // reactivation opens the gateway to it: neither once this recovery is
7522
+ // stale or the runner is shutting down. Every exec is fenced, so
7523
+ // shutdown waits for one under way rather than exiting under it, and
7524
+ // refuses (throws) any that would start after.
7525
+ if (!isCurrent()) return;
7526
+ const secretsProxy = activeSecretsProxy();
7527
+ let proxyFailure: RecoveryProxyFailure | null;
7528
+ try {
7529
+ proxyFailure = await applyTaskRootSteps(
7530
+ environment,
7531
+ environment.descriptor.envMode === "loader" ? { runtime: secretsProxy?.runtime ?? null } : null,
7532
+ (fenced) => ensureContainerTaskEnvForwarder(fenced),
7533
+ isCurrent,
7534
+ );
7535
+ } catch (error) {
7536
+ // Made stale meanwhile, its exec refused: recover() handles a stale
7537
+ // recovery itself.
7538
+ if (!isCurrent()) return;
7539
+ throw error;
7540
+ }
7541
+ if (proxyFailure !== null) {
7542
+ console.warn(
7543
+ `[secrets-proxy] ${task.taskId}: recovery could not reconcile the proxy's pins and trust: ${
7544
+ proxyFailure.error instanceof Error ? proxyFailure.error.message : String(proxyFailure.error)
7545
+ }`,
7546
+ );
7547
+ // A failure leaves the task up without the proxy: the trust watcher
7548
+ // retries, and its proxied keys stay withheld until the CA and the
7549
+ // pins are in.
7550
+ secretsProxy?.runtime.requestTrustDelivery();
7551
+ }
7552
+ if (!isCurrent()) return;
7553
+ // Shutdown began during the launch: not prepared, so the consumers stay
7554
+ // unadmitted, as for any failed preparation.
7555
+ if (taskWritesQuiescing()) throw new Error("the runner is shutting down");
7556
+ lifecycle?.reactivateTaskGatewayForRecovery(task.taskId);
7557
+ consumersPrepared = true;
7558
+ };
7559
+ try {
7560
+ const result = await environment.recover({
7561
+ maintenanceReady,
7562
+ isCurrent,
7563
+ ...(lifecycle
7564
+ ? {
7565
+ quarantineConsumers: () =>
7566
+ lifecycle.quarantineChannelForRecovery(task.taskId),
7567
+ prepareConsumers,
7568
+ allowConsumers: () => lifecycle.allowChannel(task.taskId),
7569
+ ensureConsumersStarted: async () => {
7570
+ await lifecycle.ensureStarted(task.taskId);
7571
+ },
7572
+ }
7573
+ : {}),
7574
+ });
7575
+ if (result.outcome === "recovered" && !consumersPrepared && isCurrent()) {
7576
+ const current = getHostTask(task.taskId);
7577
+ if (current?.statusMirror && isActive(current.statusMirror)) {
7578
+ await prepareConsumers();
7579
+ }
7580
+ }
7581
+ return result.outcome === "recovered";
7582
+ } catch (error) {
7583
+ if (tunnelReadinessFailed) return false;
7584
+ throw error;
7585
+ }
6125
7586
  }
6126
7587
 
6127
7588
  interface RecoveryTaskConsumers {
6128
7589
  quarantineChannelForRecovery(taskId: string): Promise<void>;
7590
+ prepareChannelForRecovery(taskId: string): Promise<void>;
6129
7591
  allowChannel(taskId: string): void;
6130
7592
  ensureStarted(taskId: string): Promise<boolean | void>;
6131
7593
  }
@@ -6588,6 +8050,7 @@ export async function recoverAppleTaskEnvironment(
6588
8050
  '[ "$(/usr/bin/stat -c %U /opt/uai/store/pnpm)" = "node" ] || { echo "store dir is not node-owned" >&2; exit 1; }; ' +
6589
8051
  "fi",
6590
8052
  ],
8053
+ projectEnvProfile: "maintenance",
6591
8054
  user: "root",
6592
8055
  env: {},
6593
8056
  timeoutMs: 30_000,
@@ -6614,23 +8077,47 @@ export async function recoverAppleTaskEnvironment(
6614
8077
  );
6615
8078
  return settleFromStatus(afterOverlay);
6616
8079
  }
8080
+ // The dependency worker inside uai-init performs its one loader sync. Its
8081
+ // root forwarder and authenticated generation must be live first, while
8082
+ // public agent admission remains quarantined until initialization ends.
8083
+ let consumersPreparedForInit = false;
8084
+ const reclosePreparedConsumers = async (): Promise<void> => {
8085
+ if (!consumersPreparedForInit) return;
8086
+ consumersPreparedForInit = false;
8087
+ await context.quarantineConsumers?.();
8088
+ };
8089
+ if (context.prepareConsumers) {
8090
+ await context.prepareConsumers();
8091
+ consumersPreparedForInit = true;
8092
+ }
8093
+ if (!context.isCurrent()) {
8094
+ await reclosePreparedConsumers();
8095
+ return { outcome: "deferred", detail: "runtime generation superseded" };
8096
+ }
6617
8097
  // Dependencies + code-server. The in-container deadline (TERM then KILL)
6618
8098
  // means a host-side expiry can never leave the initializer mutating the
6619
8099
  // workspace while agents use it; exit 78 is a runtime-authority violation
6620
8100
  // and quarantines exactly like at task-up.
6621
- const init = await environment.exec({
6622
- argv: [
6623
- "/usr/bin/timeout",
6624
- "--signal=TERM",
6625
- "--kill-after=15s",
6626
- "570s",
6627
- "/usr/local/bin/uai-init",
6628
- ],
6629
- env: {},
6630
- timeoutMs: 10 * 60_000,
6631
- maxOutputBytes: 1024 * 1024,
6632
- });
8101
+ const init = await environment
8102
+ .exec({
8103
+ argv: [
8104
+ "/usr/bin/timeout",
8105
+ "--signal=TERM",
8106
+ "--kill-after=15s",
8107
+ "570s",
8108
+ "/usr/local/bin/uai-init",
8109
+ ],
8110
+ projectEnvProfile: "maintenance",
8111
+ env: {},
8112
+ timeoutMs: 10 * 60_000,
8113
+ maxOutputBytes: 1024 * 1024,
8114
+ })
8115
+ .catch(async (error: unknown) => {
8116
+ await reclosePreparedConsumers();
8117
+ throw error;
8118
+ });
6633
8119
  if (!context.isCurrent()) {
8120
+ await reclosePreparedConsumers();
6634
8121
  return { outcome: "deferred", detail: "runtime generation superseded" };
6635
8122
  }
6636
8123
  if (
@@ -6639,6 +8126,7 @@ export async function recoverAppleTaskEnvironment(
6639
8126
  init.stdoutTruncated ||
6640
8127
  init.stderrTruncated
6641
8128
  ) {
8129
+ await reclosePreparedConsumers();
6642
8130
  // The host-side client died (deadline, overflow, forced settlement)
6643
8131
  // without a real exit code, so the in-container initializer may STILL be
6644
8132
  // mutating dependencies for several minutes. Never expose the task on an
@@ -6661,6 +8149,7 @@ export async function recoverAppleTaskEnvironment(
6661
8149
  return settleFromStatus(after);
6662
8150
  }
6663
8151
  if (init.exitCode === 78) {
8152
+ await reclosePreparedConsumers();
6664
8153
  await environment.stop();
6665
8154
  const after = await environment.status();
6666
8155
  if (after.state === "running" || after.state === "unknown") {
@@ -6679,6 +8168,7 @@ export async function recoverAppleTaskEnvironment(
6679
8168
  return settleFromStatus(after);
6680
8169
  }
6681
8170
  if (init.exitCode === 127) {
8171
+ await reclosePreparedConsumers();
6682
8172
  // Command-not-found from a read-only bind mount the guest cannot touch:
6683
8173
  // the initializer itself is unrunnable — live 2026-08-26, containers
6684
8174
  // whose versioned asset tree a host update had pruned looped here as
@@ -6709,6 +8199,7 @@ export async function recoverAppleTaskEnvironment(
6709
8199
  );
6710
8200
  }
6711
8201
  if (!context.isCurrent()) {
8202
+ await reclosePreparedConsumers();
6712
8203
  return { outcome: "deferred", detail: "runtime generation superseded" };
6713
8204
  }
6714
8205
  db_setRuntime(descriptor.taskId, { codeServerPort: null });
@@ -6762,6 +8253,7 @@ async function recoverDockerTaskEnvironment(
6762
8253
 
6763
8254
  const lifecycleCallbacks = [
6764
8255
  context.quarantineConsumers,
8256
+ context.prepareConsumers,
6765
8257
  context.allowConsumers,
6766
8258
  context.ensureConsumersStarted,
6767
8259
  ];
@@ -6777,6 +8269,12 @@ async function recoverDockerTaskEnvironment(
6777
8269
  }
6778
8270
  await context.quarantineConsumers!();
6779
8271
  },
8272
+ prepareChannelForRecovery: async (taskId) => {
8273
+ if (taskId !== descriptor.taskId) {
8274
+ throw new Error("task environment recovery consumer identity mismatch");
8275
+ }
8276
+ await context.prepareConsumers!();
8277
+ },
6780
8278
  allowChannel: (taskId) => {
6781
8279
  if (taskId !== descriptor.taskId) {
6782
8280
  throw new Error("task environment recovery consumer identity mismatch");
@@ -6984,6 +8482,13 @@ async function recoverOneDockerTask(
6984
8482
  );
6985
8483
  if (!isCurrent()) return true;
6986
8484
  if (lifecycle) {
8485
+ await lifecycle.prepareChannelForRecovery(task.taskId);
8486
+ if (!isCurrent()) {
8487
+ // Preparation may have reactivated this task's gateway generation.
8488
+ // A superseded pass must close that authority again before returning.
8489
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
8490
+ return true;
8491
+ }
6987
8492
  lifecycle.allowChannel(task.taskId);
6988
8493
  await lifecycle.ensureStarted(task.taskId);
6989
8494
  if (!isCurrent()) {
@@ -7119,6 +8624,23 @@ async function recoverOneDockerTask(
7119
8624
  }
7120
8625
  if (!isCurrent()) return true;
7121
8626
  }
8627
+ // uai-init's dependency worker loads the task env once. Recreate the
8628
+ // forwarder and reactivate only its authenticated gateway before that
8629
+ // worker runs; the channel itself stays tombstoned until recovery finishes.
8630
+ let consumersPreparedForInit = false;
8631
+ const reclosePreparedConsumers = async (): Promise<void> => {
8632
+ if (!consumersPreparedForInit || !lifecycle) return;
8633
+ consumersPreparedForInit = false;
8634
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
8635
+ };
8636
+ if (lifecycle) {
8637
+ await lifecycle.prepareChannelForRecovery(task.taskId);
8638
+ consumersPreparedForInit = true;
8639
+ if (!isCurrent()) {
8640
+ await reclosePreparedConsumers();
8641
+ return true;
8642
+ }
8643
+ }
7122
8644
  // uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
7123
8645
  // repo. dockerCli's 30s default would SIGKILL it mid-install.
7124
8646
  const initResult = await dockerCli(
@@ -7210,9 +8732,16 @@ async function recoverOneDockerTask(
7210
8732
  RECOVERY_INIT_PATH,
7211
8733
  ],
7212
8734
  { timeoutMs: 10 * 60_000 },
7213
- );
7214
- if (!isCurrent()) return true;
8735
+ ).catch(async (error: unknown) => {
8736
+ await reclosePreparedConsumers();
8737
+ throw error;
8738
+ });
8739
+ if (!isCurrent()) {
8740
+ await reclosePreparedConsumers();
8741
+ return true;
8742
+ }
7215
8743
  if (initResult.status === 78) {
8744
+ await reclosePreparedConsumers();
7216
8745
  const quarantined = await quarantineRunningContainer(containerName);
7217
8746
  if (!isCurrent()) return true;
7218
8747
  if (quarantined === "unreachable") return false;
@@ -7235,9 +8764,20 @@ async function recoverOneDockerTask(
7235
8764
  `[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
7236
8765
  );
7237
8766
  }
7238
- const portResult = await dockerPort(containerName, 8080);
7239
- if (!isCurrent()) return true;
7240
- if (!portResult.reachable) return false;
8767
+ const portResult = await dockerPort(containerName, 8080).catch(
8768
+ async (error: unknown) => {
8769
+ await reclosePreparedConsumers();
8770
+ throw error;
8771
+ },
8772
+ );
8773
+ if (!isCurrent()) {
8774
+ await reclosePreparedConsumers();
8775
+ return true;
8776
+ }
8777
+ if (!portResult.reachable) {
8778
+ await reclosePreparedConsumers();
8779
+ return false;
8780
+ }
7241
8781
  const port = portResult.port;
7242
8782
  db_setRuntime(task.taskId, {
7243
8783
  codeServerPort: port,
@@ -7254,7 +8794,10 @@ async function recoverOneDockerTask(
7254
8794
  console.log(
7255
8795
  `[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
7256
8796
  );
7257
- if (!isCurrent()) return true;
8797
+ if (!isCurrent()) {
8798
+ await reclosePreparedConsumers();
8799
+ return true;
8800
+ }
7258
8801
  if (lifecycle) {
7259
8802
  lifecycle.allowChannel(task.taskId);
7260
8803
  await lifecycle.ensureStarted(task.taskId);