@runuai/host 0.8.42 → 0.8.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,7 +50,11 @@ import {
50
50
  loadTaskCliSecret,
51
51
  writeAgentCli,
52
52
  } from "./agent-cli";
53
- import { setupBrowserTesting } from "./browser-testing";
53
+ import {
54
+ DEFAULT_CODEX_HOME,
55
+ setupBrowserTesting,
56
+ type BrowserSetup,
57
+ } from "./browser-testing";
54
58
  import { injectCodexIntoContainer } from "./codex-auth";
55
59
  import {
56
60
  cooldownEngineAccount,
@@ -60,6 +64,7 @@ import {
60
64
  resolveEngineAccounts,
61
65
  } from "./engine-accounts";
62
66
  import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
67
+ import { stopPreviewSidecars } from "./preview-sidecar";
63
68
  import { env } from "./env";
64
69
  import type {
65
70
  ChannelEnsureInput,
@@ -90,9 +95,60 @@ interface Channel {
90
95
  * awaits docker work before populating it). Reset to null on failure so the
91
96
  * next ensure retries. */
92
97
  sessionsReady: Promise<boolean> | null;
98
+ /** ADR-053: true only when both the browser install and managed MCP configs
99
+ * are ready. A task still starts while false; setup retries in the
100
+ * background on a bounded timer. */
101
+ browserReady: boolean;
102
+ /** The one setup pass currently touching this container. Reconcile, retry,
103
+ * and initial start all join this promise instead of racing config files. */
104
+ browserSetup: Promise<BrowserSetup> | null;
105
+ /** Codex account homes covered by the in-flight setup. A roster/account
106
+ * change that adds a home queues one stronger pass after it. */
107
+ browserSetupCodexHomes: string[] | null;
108
+ /** Engine kinds provisioned by the in-flight setup. A roster generation
109
+ * that adds a kind cannot join a weaker provisioning pass. */
110
+ browserSetupEngineKinds: string[] | null;
111
+ /** Exact account roster covered by the in-flight provisioning pass. */
112
+ browserSetupProvisionFingerprint: string | null;
113
+ /** Mutable coverage of the in-flight materialization pass. Joiners retain
114
+ * this object through cleanup and can distinguish requested browser work
115
+ * from a browser attempt that actually began. */
116
+ browserSetupCoverage: {
117
+ browserRequested: boolean;
118
+ browserAttempted: boolean;
119
+ skippedBecauseDisabled: boolean;
120
+ provisionAttempted: boolean;
121
+ } | null;
122
+ /** Last account roster whose provisioning was attempted in this container.
123
+ * Missing-session reconciliation must not hot-loop an unchanged failed
124
+ * copy and invalidate the peer sessions it is trying to repair. */
125
+ provisionAttemptedFingerprint?: string;
126
+ /** An incomplete account copy that must be retried on the bounded timer. */
127
+ provisionRetryFingerprint?: string;
128
+ /** Self-scheduled retry after a failed install/configuration pass. */
129
+ browserRetryTimer: ReturnType<typeof setTimeout> | null;
130
+ /** Sessions that loaded MCP config before the latest rewrite. Busy sessions
131
+ * stay here until their turn completes; each id is recycled exactly once. */
132
+ browserStaleSessions: Set<string>;
133
+ /** Browser-config rewrites whose install/config preconditions are not ready
134
+ * yet. These sessions must eventually reload the definition, but recycling
135
+ * them before the browser is usable only trades one broken session for
136
+ * another. */
137
+ browserPendingStaleSessions: Set<string>;
138
+ /** One reconcile loop per channel. It covers ordinary missing agents and
139
+ * stale-session recycling without overlapping factory creates. */
140
+ reconcileReady: Promise<void> | null;
141
+ /** A reconcile request that arrived while the loop was in flight. */
142
+ reconcileAgain: boolean;
143
+ /** Set before teardown awaits anything. Every async continuation checks both
144
+ * this flag and the channel-map identity before mutating or spawning. */
145
+ closed: boolean;
93
146
  /** Agents with turn output already streamed for the current turn — i.e. the
94
147
  * cloud has buffered content for them (see ChannelRouter.turnBuffer). */
95
148
  openTurns: Set<string>;
149
+ /** Agents with a delivered turn that has not reached a terminal event yet.
150
+ * Unlike openTurns, this covers thinking/tool/permission time before text. */
151
+ activeTurns: Map<string, number>;
96
152
  /** Agents whose current turn was interrupted (ESC) — their next
97
153
  * turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
98
154
  * half-turn instead of delivering it to @-mentioned peers. */
@@ -116,9 +172,30 @@ interface Channel {
116
172
  * ensure runs on every message). Live sessions read MCP config at spawn,
117
173
  * so a mid-task write takes effect on the next (re)spawn. */
118
174
  mcpConfigFingerprint?: string;
175
+ /** Serialize gateway config writes so two ensures cannot publish stale
176
+ * content and then memoize the newer fingerprint over it. */
177
+ mcpConfigWrite: Promise<void> | null;
178
+ /** Account provisioning can overwrite per-engine config without changing
179
+ * the desired gateway fingerprint. A monotonic epoch prevents an older
180
+ * in-flight writer from clearing a newer provisioning mutation. */
181
+ mcpConfigDirtyEpoch: number;
182
+ /** Dirty epoch covered by the last successful gateway publication. */
183
+ mcpConfigAppliedEpoch: number;
184
+ /** Monotonic version of the managed config files agent processes read at
185
+ * startup. A factory create that straddles a rewrite is discarded and
186
+ * retried before it can be bound as the live generation. */
187
+ configGeneration: number;
119
188
  /** Agents with a reconcile-spawn in flight (ADR-049 mid-task adds) — guards
120
189
  * against a concurrent ensure double-spawning the same new agent. */
121
190
  spawning: Set<string>;
191
+ /** Factory creates are externally side-effecting before they bind. Teardown
192
+ * drains them so an old channel cannot resume against a reused container
193
+ * name and overwrite the replacement generation's durable transport. */
194
+ factoryOperations: Set<Promise<unknown>>;
195
+ /** Terminal-event recovery/rotation can remove a session before awaiting
196
+ * its close. Track the enclosing handler so teardown still drains that
197
+ * external operation before the container name can be reused. */
198
+ eventOperations: Set<Promise<unknown>>;
122
199
  /** ADR-076: which engine account each agent's live session is bound to, so a
123
200
  * rate-limit can cool THAT account and rotate off it. */
124
201
  accountByAgent: Map<string, string>;
@@ -136,6 +213,10 @@ const MAX_RESPAWNS_PER_AGENT = 5;
136
213
  const MAX_ROTATIONS_PER_AGENT = 6;
137
214
  /** A burned respawn budget resets after this quiet period (see reconcile). */
138
215
  const RESPAWN_COOLDOWN_MS = 10 * 60_000;
216
+ /** ADR-053: gap between browser-install retries. Ensures fire every couple of
217
+ * seconds; a container-gone failure returns instantly, so without this the
218
+ * retry would spin npx continuously for the life of a broken task. */
219
+ const BROWSER_RETRY_COOLDOWN_MS = 60_000;
139
220
 
140
221
  /** Substrings in an agent's error output that mean "config was
141
222
  * unlinked between runs" — repair-and-respawn covers the common
@@ -146,9 +227,18 @@ const CLAUDE_CONFIG_MISSING_PATTERNS = [
146
227
  /\/\.claude\.json/i,
147
228
  ];
148
229
 
149
- class Orchestrator {
230
+ export class Orchestrator {
150
231
  private readonly channels = new Map<string, Channel>();
151
232
  private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
233
+ /** Teardown tombstones prevent a cloud ensure from reopening a task between
234
+ * session close and container stop/removal. Cleared only by task-up. */
235
+ private readonly blockedTasks = new Set<string>();
236
+ /** Every teardown caller for a task joins the same container-reuse fence. */
237
+ private readonly channelClosures = new Map<string, Promise<void>>();
238
+ /** Host lifecycle commands can arrive concurrently over the bridge/local
239
+ * UI. Serialize each task through container creation/destruction so a
240
+ * later command cannot overtake an earlier one after channel close. */
241
+ private readonly taskLifecycleTails = new Map<string, Promise<void>>();
152
242
  private readonly hostSubscribers = new Set<HostEventSubscriber>();
153
243
 
154
244
  constructor(private readonly factory: AgentSessionFactory) {}
@@ -160,6 +250,36 @@ class Orchestrator {
160
250
  return () => this.hostSubscribers.delete(fn);
161
251
  }
162
252
 
253
+ /** Run one task lifecycle mutation in arrival order. Different task ids
254
+ * remain fully concurrent. The queue tail never rejects, so one failed
255
+ * command cannot poison later recovery. */
256
+ async runTaskLifecycle<T>(
257
+ taskId: string,
258
+ operation: () => Promise<T>,
259
+ ): Promise<T> {
260
+ const previous =
261
+ this.taskLifecycleTails.get(taskId) ?? Promise.resolve();
262
+ let release!: () => void;
263
+ const current = new Promise<void>((resolve) => {
264
+ release = resolve;
265
+ });
266
+ const tail = previous.catch(() => {}).then(() => current);
267
+ this.taskLifecycleTails.set(taskId, tail);
268
+ await previous.catch(() => {});
269
+ try {
270
+ // Boot recovery can still be starting/copying into this task's stable
271
+ // container name while bridge and local-UI commands are already live.
272
+ // Join that per-task barrier before any queued create/destroy mutation.
273
+ await taskRecoveryComplete(taskId);
274
+ return await operation();
275
+ } finally {
276
+ release();
277
+ if (this.taskLifecycleTails.get(taskId) === tail) {
278
+ this.taskLifecycleTails.delete(taskId);
279
+ }
280
+ }
281
+ }
282
+
163
283
  /** Emit a host-originated system note into a task's channel (ADR-027). */
164
284
  emitSystemNote(taskId: string, text: string): void {
165
285
  this.emitHost({ kind: "system.note", taskId, text });
@@ -179,6 +299,7 @@ class Orchestrator {
179
299
 
180
300
  registerChannelSpec(spec: ChannelEnsureInput): void {
181
301
  this.channelSpecs.set(spec.taskId, spec);
302
+ if (this.blockedTasks.has(spec.taskId)) return;
182
303
  // ADR-049: the cloud re-sends the spec on every message AND right after a
183
304
  // mid-task roster/participant change. If the channel is already live,
184
305
  // fold the fresh spec in: append new roster agents (their sessions spawn
@@ -190,6 +311,11 @@ class Orchestrator {
190
311
 
191
312
  /** Fold a fresh channel spec into a live channel (ADR-049). */
192
313
  private refreshChannel(channel: Channel, spec: ChannelEnsureInput): void {
314
+ const browserWasEnabled = channel.browserTesting;
315
+ const previousMcpShape = JSON.stringify({
316
+ connections: channel.mcpConnections,
317
+ kinds: [...new Set(channel.roster.map((agent) => agent.kind))].sort(),
318
+ });
193
319
  const known = new Set(channel.roster.map((a) => a.id));
194
320
  // Replace EXISTING agents' data with the fresh spec — a mid-task roster edit
195
321
  // can change permissions, model, brief, role or skills, and without this the
@@ -211,6 +337,13 @@ class Orchestrator {
211
337
  channel.browserTesting = spec.browserTesting === true;
212
338
  channel.sharedFiles = spec.sharedFiles ?? "ro";
213
339
  channel.mcpConnections = spec.mcpConnections ?? [];
340
+ const nextMcpShape = JSON.stringify({
341
+ connections: channel.mcpConnections,
342
+ kinds: [...new Set(channel.roster.map((agent) => agent.kind))].sort(),
343
+ });
344
+ if (nextMcpShape !== previousMcpShape) {
345
+ channel.mcpConfigDirtyEpoch += 1;
346
+ }
214
347
  for (const agent of channel.roster) {
215
348
  channel.preambles.set(
216
349
  agent.id,
@@ -227,9 +360,52 @@ class Orchestrator {
227
360
  ),
228
361
  );
229
362
  }
363
+ if (!browserWasEnabled && channel.browserTesting) {
364
+ void this.enableBrowserTesting(channel);
365
+ } else if (browserWasEnabled && !channel.browserTesting) {
366
+ if (channel.provisionRetryFingerprint) {
367
+ // Browser readiness is no longer required, but a failed extra-account
368
+ // copy still needs its bounded retry.
369
+ this.scheduleBrowserRetry(channel);
370
+ } else {
371
+ this.clearBrowserRetry(channel);
372
+ }
373
+ // Config files copied while browser readiness was required no longer
374
+ // need to wait for a download. They still need a safe process reload.
375
+ for (const agentId of channel.browserPendingStaleSessions) {
376
+ channel.browserStaleSessions.add(agentId);
377
+ }
378
+ channel.browserPendingStaleSessions.clear();
379
+ void this.reconcileIfStarted(channel).catch((err: unknown) => {
380
+ console.warn(
381
+ `[orchestrator] ${channel.taskId}: browser-disable reconcile failed: ${
382
+ err instanceof Error ? err.message : String(err)
383
+ }`,
384
+ );
385
+ });
386
+ }
387
+ }
388
+
389
+ /** Materialize a browser flag enabled after the channel already started. */
390
+ private async enableBrowserTesting(channel: Channel): Promise<void> {
391
+ try {
392
+ await this.ensureBrowserSetup(
393
+ channel,
394
+ channel.roster.some((agent) => agent.kind === "codex"),
395
+ Boolean(channel.provisionRetryFingerprint),
396
+ );
397
+ await this.reconcileIfStarted(channel);
398
+ } catch (err) {
399
+ console.warn(
400
+ `[browser] task ${channel.taskId}: enable failed: ${
401
+ err instanceof Error ? err.message : String(err)
402
+ }`,
403
+ );
404
+ }
230
405
  }
231
406
 
232
407
  private async getOrCreateChannel(taskId: string): Promise<Channel | null> {
408
+ if (this.blockedTasks.has(taskId)) return null;
233
409
  const existing = this.channels.get(taskId);
234
410
  if (existing) return existing;
235
411
 
@@ -275,7 +451,20 @@ class Orchestrator {
275
451
  preambles,
276
452
  firstTurns,
277
453
  sessionsReady: null,
454
+ browserReady: false,
455
+ browserSetup: null,
456
+ browserSetupCodexHomes: null,
457
+ browserSetupEngineKinds: null,
458
+ browserSetupProvisionFingerprint: null,
459
+ browserSetupCoverage: null,
460
+ browserRetryTimer: null,
461
+ browserStaleSessions: new Set(),
462
+ browserPendingStaleSessions: new Set(),
463
+ reconcileReady: null,
464
+ reconcileAgain: false,
465
+ closed: false,
278
466
  openTurns: new Set(),
467
+ activeTurns: new Map(),
279
468
  interrupted: new Set(),
280
469
  respawns: new Map(),
281
470
  respawnLastAt: new Map(),
@@ -283,7 +472,13 @@ class Orchestrator {
283
472
  browserTesting: spec.browserTesting === true,
284
473
  sharedFiles: spec.sharedFiles ?? "ro",
285
474
  mcpConnections: spec.mcpConnections ?? [],
475
+ mcpConfigWrite: null,
476
+ mcpConfigDirtyEpoch: 1,
477
+ mcpConfigAppliedEpoch: 0,
478
+ configGeneration: 0,
286
479
  spawning: new Set(),
480
+ factoryOperations: new Set(),
481
+ eventOperations: new Set(),
287
482
  accountByAgent: new Map(),
288
483
  lastPrompt: new Map(),
289
484
  rotations: new Map(),
@@ -304,6 +499,11 @@ class Orchestrator {
304
499
  * Returns whether sessions are ready.
305
500
  */
306
501
  private async ensureSessions(channel: Channel): Promise<boolean> {
502
+ // Boot recovery may be restarting this exact container and copying the
503
+ // full Codex directory. Wait only for this task's recovery barrier so that
504
+ // copy cannot land after browser/MCP materialization and erase config.
505
+ await taskRecoveryComplete(channel.taskId);
506
+ if (!this.isActiveChannel(channel)) return false;
307
507
  // Memoized: every caller awaits the SAME in-flight start, so a concurrent
308
508
  // deliver() can't observe "ready" while the sessions map is still empty
309
509
  // (startSessions awaits docker work before populating it — the old boolean
@@ -314,7 +514,7 @@ class Orchestrator {
314
514
  // next ensure retries (e.g. the task wasn't `running` yet).
315
515
  channel.sessionsReady.then(
316
516
  (ok) => {
317
- if (!ok) channel.sessionsReady = null;
517
+ if (!ok && this.isActiveChannel(channel)) channel.sessionsReady = null;
318
518
  },
319
519
  (err: unknown) => {
320
520
  // A start failure retries on the next ensure — but it must be
@@ -324,58 +524,536 @@ class Orchestrator {
324
524
  `[orchestrator] ${channel.taskId}: session start failed: ` +
325
525
  `${err instanceof Error ? (err.stack ?? err.message) : String(err)}`,
326
526
  );
327
- channel.sessionsReady = null;
527
+ if (this.isActiveChannel(channel)) channel.sessionsReady = null;
328
528
  },
329
529
  );
330
530
  }
331
531
  const ready = await channel.sessionsReady;
532
+ if (!this.isActiveChannel(channel)) return false;
533
+ // A live browser-enable or retry starts outside this ensure path. Join its
534
+ // serialized setup before reconciliation/delivery so the next message
535
+ // cannot reach a session that still holds the old MCP definition.
536
+ if (ready && channel.browserSetup) {
537
+ await this.ensureBrowserSetup(
538
+ channel,
539
+ channel.roster.some((agent) => agent.kind === "codex"),
540
+ );
541
+ if (!this.isActiveChannel(channel)) return false;
542
+ }
332
543
  // ADR-049 reconcile pass: spawn any roster agent that has no live session
333
544
  // yet — the initial batch, host-restart recovery, and mid-task adds all
334
545
  // converge here. No-op when every roster agent has a session.
335
546
  if (ready) {
336
- await this.reconcileSessions(channel);
337
547
  // ADR-057: keep the container's MCP configs current so a connection
338
- // added mid-task lands (effective at each session's next spawn).
548
+ // or engine added mid-task lands before its missing session spawns.
339
549
  await this.ensureMcpConfig(channel);
550
+ if (!this.isActiveChannel(channel)) return false;
551
+ await this.reconcileSessions(channel);
340
552
  }
341
553
  return ready;
342
554
  }
343
555
 
556
+ /**
557
+ * Background browser work may recycle sessions only after the memoized
558
+ * initial start has finished. Waiting here prevents live enable/retry from
559
+ * reconciling the same missing agents while startSessions is creating them.
560
+ */
561
+ private async reconcileIfStarted(channel: Channel): Promise<void> {
562
+ const starting = channel.sessionsReady;
563
+ if (!starting) return;
564
+ const ready = await starting;
565
+ if (ready && this.isActiveChannel(channel)) {
566
+ await this.reconcileSessions(channel);
567
+ }
568
+ }
569
+
570
+ /** Is this exact channel generation still the task's live channel? */
571
+ private isActiveChannel(channel: Channel): boolean {
572
+ return !channel.closed && this.channels.get(channel.taskId) === channel;
573
+ }
574
+
575
+ /**
576
+ * The single account/browser-materialization gate for a channel.
577
+ *
578
+ * Initial start, a missing-session reconcile, live enable, rotation, and the
579
+ * failure retry all join the same promise. Even browser-disabled account
580
+ * copies use this gate: a false→true toggle queues migration after the older
581
+ * copy, so that copy cannot land last and erase the managed browser block.
582
+ */
583
+ private codexHomes(hasCodex: boolean): string[] {
584
+ if (!hasCodex) return [];
585
+ const homes = resolveEngineAccounts("codex")
586
+ .map((account) => account.configDir?.containerDir)
587
+ .filter((home): home is string => Boolean(home));
588
+ return [...new Set([DEFAULT_CODEX_HOME, ...homes])];
589
+ }
590
+
591
+ private noteProvisionMutation(
592
+ channel: Channel,
593
+ waitForBrowserReadiness: boolean,
594
+ ): void {
595
+ if (!this.isActiveChannel(channel)) return;
596
+ channel.configGeneration += 1;
597
+ channel.mcpConfigDirtyEpoch += 1;
598
+ for (const agentId of channel.sessions.keys()) {
599
+ if (waitForBrowserReadiness) {
600
+ channel.browserPendingStaleSessions.add(agentId);
601
+ } else {
602
+ channel.browserStaleSessions.add(agentId);
603
+ }
604
+ }
605
+ }
606
+
607
+ private fenceConfigGeneration(channel: Channel): void {
608
+ if (this.isActiveChannel(channel)) channel.configGeneration += 1;
609
+ }
610
+
611
+ private provisionFingerprint(kinds: readonly string[]): string {
612
+ const accounts = kinds.flatMap((kind) =>
613
+ resolveEngineAccounts(kind)
614
+ .filter((account) => !account.isDefault && account.configDir)
615
+ .map((account) => ({
616
+ kind,
617
+ id: account.id,
618
+ hostDir: account.configDir!.hostDir,
619
+ containerDir: account.configDir!.containerDir,
620
+ })),
621
+ );
622
+ accounts.sort((left, right) =>
623
+ JSON.stringify(left).localeCompare(JSON.stringify(right)),
624
+ );
625
+ return JSON.stringify({
626
+ kinds: [...new Set(kinds)].sort(),
627
+ accounts,
628
+ });
629
+ }
630
+
631
+ private async ensureBrowserSetup(
632
+ channel: Channel,
633
+ hasCodex: boolean,
634
+ forceProvision = false,
635
+ ): Promise<BrowserSetup> {
636
+ const unavailable: BrowserSetup = {
637
+ ready: false,
638
+ configured: false,
639
+ changed: false,
640
+ };
641
+ if (!this.isActiveChannel(channel)) return unavailable;
642
+ const requestedBrowser = channel.browserTesting;
643
+ const requestedHasCodex =
644
+ hasCodex || channel.roster.some((agent) => agent.kind === "codex");
645
+ const requestedHomes = requestedBrowser
646
+ ? this.codexHomes(requestedHasCodex)
647
+ : [];
648
+ const requestedKinds = [
649
+ ...new Set(channel.roster.map((agent) => agent.kind)),
650
+ ];
651
+ const requestedProvisionFingerprint =
652
+ this.provisionFingerprint(requestedKinds);
653
+ const mayProvisionConfig = requestedKinds.some((kind) =>
654
+ resolveEngineAccounts(kind).some(
655
+ (account) => !account.isDefault && Boolean(account.configDir),
656
+ ),
657
+ );
658
+
659
+ const inFlight = channel.browserSetup;
660
+ if (inFlight) {
661
+ const coveredHomes = channel.browserSetupCodexHomes ?? [];
662
+ const coveredKinds = channel.browserSetupEngineKinds ?? [];
663
+ const coveredProvisionFingerprint =
664
+ channel.browserSetupProvisionFingerprint;
665
+ const coverage = channel.browserSetupCoverage;
666
+ const setup = await inFlight;
667
+ if (!this.isActiveChannel(channel)) return unavailable;
668
+ const latestBrowser = channel.browserTesting;
669
+ const latestHasCodex =
670
+ requestedHasCodex ||
671
+ channel.roster.some((agent) => agent.kind === "codex");
672
+ const latestHomes = latestBrowser
673
+ ? this.codexHomes(latestHasCodex)
674
+ : [];
675
+ const latestKinds = [
676
+ ...new Set(channel.roster.map((agent) => agent.kind)),
677
+ ];
678
+ const latestProvisionFingerprint =
679
+ this.provisionFingerprint(latestKinds);
680
+ if (
681
+ (latestBrowser &&
682
+ coverage?.browserAttempted !== true &&
683
+ coverage?.browserRequested === false) ||
684
+ (latestBrowser && coverage?.skippedBecauseDisabled === true) ||
685
+ latestHomes.some((home) => !coveredHomes.includes(home)) ||
686
+ latestKinds.some((kind) => !coveredKinds.includes(kind)) ||
687
+ latestProvisionFingerprint !== coveredProvisionFingerprint ||
688
+ (forceProvision && coverage?.provisionAttempted !== true)
689
+ ) {
690
+ return this.ensureBrowserSetup(
691
+ channel,
692
+ latestHasCodex,
693
+ forceProvision,
694
+ );
695
+ }
696
+ return setup;
697
+ }
698
+
699
+ const coverage = {
700
+ browserRequested: requestedBrowser,
701
+ browserAttempted: false,
702
+ skippedBecauseDisabled: false,
703
+ provisionAttempted: false,
704
+ };
705
+ const core = (async (): Promise<BrowserSetup> => {
706
+ let setup: BrowserSetup;
707
+ try {
708
+ // Account copies and MCP migration share one serialized generation.
709
+ // Otherwise a concurrent provision can overwrite config.toml after
710
+ // migration and immediately strand an extra-account Codex session.
711
+ const shouldProvision =
712
+ forceProvision ||
713
+ channel.provisionAttemptedFingerprint !==
714
+ requestedProvisionFingerprint;
715
+ if (shouldProvision) {
716
+ coverage.provisionAttempted = true;
717
+ if (mayProvisionConfig) {
718
+ // Fence a factory that started before the copy without declaring a
719
+ // disk mutation until the aggregate result says bytes may have
720
+ // landed. A total failure must not churn live peers.
721
+ this.fenceConfigGeneration(channel);
722
+ }
723
+ let copiedOrUnknown = false;
724
+ try {
725
+ const provisioned = await provisionEngineAccounts(
726
+ channel.containerName,
727
+ requestedKinds,
728
+ );
729
+ copiedOrUnknown =
730
+ provisioned.copied > 0 || provisioned.mayHaveMutated;
731
+ if (provisioned.failed > 0) {
732
+ channel.provisionRetryFingerprint =
733
+ requestedProvisionFingerprint;
734
+ } else {
735
+ channel.provisionRetryFingerprint = undefined;
736
+ }
737
+ } catch (err) {
738
+ // A thrown multi-command copy may have landed before failing.
739
+ copiedOrUnknown = true;
740
+ channel.provisionRetryFingerprint =
741
+ requestedProvisionFingerprint;
742
+ throw err;
743
+ } finally {
744
+ // Cache the ATTEMPT even when incomplete. Immediate reconcile
745
+ // loops may replace missing agents, but only the bounded retry
746
+ // timer may repeat a failed account copy.
747
+ channel.provisionAttemptedFingerprint =
748
+ requestedProvisionFingerprint;
749
+ if (mayProvisionConfig) {
750
+ if (copiedOrUnknown) {
751
+ // A process that starts during a real/partial copy observes
752
+ // this second bump, while gateway config is reasserted.
753
+ this.noteProvisionMutation(
754
+ channel,
755
+ channel.browserTesting,
756
+ );
757
+ } else {
758
+ this.fenceConfigGeneration(channel);
759
+ }
760
+ }
761
+ }
762
+ }
763
+ if (
764
+ !this.isActiveChannel(channel) ||
765
+ !requestedBrowser ||
766
+ !channel.browserTesting
767
+ ) {
768
+ if (
769
+ this.isActiveChannel(channel) &&
770
+ channel.provisionRetryFingerprint
771
+ ) {
772
+ this.scheduleBrowserRetry(channel);
773
+ }
774
+ coverage.skippedBecauseDisabled =
775
+ this.isActiveChannel(channel) &&
776
+ (!requestedBrowser || !channel.browserTesting);
777
+ return unavailable;
778
+ }
779
+ coverage.browserAttempted = true;
780
+ setup = await setupBrowserTesting(
781
+ channel.taskId,
782
+ channel.containerName,
783
+ requestedHasCodex,
784
+ requestedHomes,
785
+ );
786
+ } catch (err) {
787
+ console.warn(
788
+ `[browser] task ${channel.taskId}: setup failed: ${
789
+ err instanceof Error ? err.message : String(err)
790
+ }`,
791
+ );
792
+ setup = unavailable;
793
+ }
794
+
795
+ if (!this.isActiveChannel(channel)) return unavailable;
796
+
797
+ channel.browserReady = setup.ready && setup.configured;
798
+ if (setup.changed) {
799
+ channel.configGeneration += 1;
800
+ for (const agentId of channel.sessions.keys()) {
801
+ channel.browserPendingStaleSessions.add(agentId);
802
+ }
803
+ }
804
+ if (channel.browserReady) {
805
+ for (const agentId of channel.browserPendingStaleSessions) {
806
+ channel.browserStaleSessions.add(agentId);
807
+ }
808
+ channel.browserPendingStaleSessions.clear();
809
+ }
810
+
811
+ if (channel.browserReady && !channel.provisionRetryFingerprint) {
812
+ this.clearBrowserRetry(channel);
813
+ console.log(`[browser] task ${channel.taskId}: browser ready`);
814
+ } else {
815
+ this.scheduleBrowserRetry(channel);
816
+ }
817
+ return setup;
818
+ })();
819
+
820
+ let run: Promise<BrowserSetup>;
821
+ run = core.finally(() => {
822
+ if (channel.browserSetup === run) {
823
+ channel.browserSetup = null;
824
+ channel.browserSetupCodexHomes = null;
825
+ channel.browserSetupEngineKinds = null;
826
+ channel.browserSetupProvisionFingerprint = null;
827
+ channel.browserSetupCoverage = null;
828
+ }
829
+ });
830
+ channel.browserSetup = run;
831
+ channel.browserSetupCodexHomes = requestedHomes;
832
+ channel.browserSetupEngineKinds = requestedKinds;
833
+ channel.browserSetupProvisionFingerprint = requestedProvisionFingerprint;
834
+ channel.browserSetupCoverage = coverage;
835
+ const setup = await run;
836
+ if (!this.isActiveChannel(channel)) return unavailable;
837
+ const latestBrowser = channel.browserTesting;
838
+ const latestHasCodex =
839
+ requestedHasCodex ||
840
+ channel.roster.some((agent) => agent.kind === "codex");
841
+ const latestHomes = latestBrowser
842
+ ? this.codexHomes(latestHasCodex)
843
+ : [];
844
+ const latestKinds = [
845
+ ...new Set(channel.roster.map((agent) => agent.kind)),
846
+ ];
847
+ const latestProvisionFingerprint = this.provisionFingerprint(latestKinds);
848
+ if (
849
+ (latestBrowser &&
850
+ !coverage.browserAttempted &&
851
+ !coverage.browserRequested) ||
852
+ (latestBrowser && coverage.skippedBecauseDisabled) ||
853
+ latestHomes.some((home) => !requestedHomes.includes(home)) ||
854
+ latestKinds.some((kind) => !requestedKinds.includes(kind)) ||
855
+ latestProvisionFingerprint !== requestedProvisionFingerprint ||
856
+ (forceProvision && !coverage.provisionAttempted)
857
+ ) {
858
+ return this.ensureBrowserSetup(channel, latestHasCodex, forceProvision);
859
+ }
860
+ return setup;
861
+ }
862
+
863
+ private clearBrowserRetry(channel: Channel): void {
864
+ if (channel.browserRetryTimer) {
865
+ clearTimeout(channel.browserRetryTimer);
866
+ channel.browserRetryTimer = null;
867
+ }
868
+ }
869
+
870
+ /** A failed setup gets a real retry even if no message/ensure arrives. */
871
+ private scheduleBrowserRetry(channel: Channel): void {
872
+ if (
873
+ (!channel.browserTesting && !channel.provisionRetryFingerprint) ||
874
+ !this.isActiveChannel(channel) ||
875
+ channel.browserRetryTimer
876
+ ) {
877
+ return;
878
+ }
879
+ const timer = setTimeout(() => {
880
+ if (channel.browserRetryTimer === timer) {
881
+ channel.browserRetryTimer = null;
882
+ }
883
+ if (!this.isActiveChannel(channel)) return;
884
+ const forceProvision = Boolean(channel.provisionRetryFingerprint);
885
+ void this.ensureBrowserSetup(
886
+ channel,
887
+ channel.roster.some((a) => a.kind === "codex"),
888
+ forceProvision,
889
+ ).then(() => {
890
+ if (this.isActiveChannel(channel)) return this.reconcileIfStarted(channel);
891
+ }).catch((err: unknown) => {
892
+ console.warn(
893
+ `[browser] task ${channel.taskId}: retry reconcile failed: ${
894
+ err instanceof Error ? err.message : String(err)
895
+ }`,
896
+ );
897
+ });
898
+ }, BROWSER_RETRY_COOLDOWN_MS);
899
+ channel.browserRetryTimer = timer;
900
+ }
901
+
344
902
  /** Write the task's MCP configs when the connection set changed. */
345
903
  private async ensureMcpConfig(channel: Channel): Promise<void> {
346
- const fingerprint = JSON.stringify(channel.mcpConnections.map((c) => c.id));
347
- if (channel.mcpConfigFingerprint === fingerprint) return;
904
+ if (!this.isActiveChannel(channel)) return;
905
+ const inFlight = channel.mcpConfigWrite;
906
+ if (inFlight) {
907
+ await inFlight;
908
+ // Recompute after the serialized writer: the channel spec may have
909
+ // changed while we waited, in which case this caller owns the next pass.
910
+ if (this.isActiveChannel(channel)) await this.ensureMcpConfig(channel);
911
+ return;
912
+ }
913
+
914
+ let run: Promise<void>;
915
+ run = this.writeMcpConfigPass(channel).finally(() => {
916
+ if (channel.mcpConfigWrite === run) channel.mcpConfigWrite = null;
917
+ });
918
+ channel.mcpConfigWrite = run;
919
+ await run;
920
+ }
921
+
922
+ private async writeMcpConfigPass(channel: Channel): Promise<void> {
923
+ if (!this.isActiveChannel(channel)) return;
924
+ const connections = channel.mcpConnections.map((connection) => ({
925
+ id: connection.id,
926
+ slug: connection.slug,
927
+ }));
928
+ const engineKinds = [
929
+ ...new Set(channel.roster.map((agent) => agent.kind)),
930
+ ].sort();
931
+ const codexHomes = engineKinds.includes("codex")
932
+ ? this.codexHomes(true)
933
+ : [];
934
+ const fingerprint = JSON.stringify({
935
+ connections,
936
+ engineKinds,
937
+ codexHomes,
938
+ });
939
+ const dirtyEpoch = channel.mcpConfigDirtyEpoch;
940
+ const repairingProvisionedConfig =
941
+ channel.mcpConfigAppliedEpoch !== dirtyEpoch &&
942
+ channel.mcpConfigFingerprint === fingerprint;
943
+ if (
944
+ channel.mcpConfigFingerprint === fingerprint &&
945
+ channel.mcpConfigAppliedEpoch === dirtyEpoch
946
+ ) {
947
+ return;
948
+ }
348
949
  const hadPrevious = channel.mcpConfigFingerprint !== undefined;
349
- channel.mcpConfigFingerprint = fingerprint;
350
- await setupMcpTaskConfig(
950
+ let previous:
951
+ | {
952
+ connections: Array<{ id: string; slug: string }>;
953
+ engineKinds: string[];
954
+ codexHomes?: string[];
955
+ }
956
+ | undefined;
957
+ try {
958
+ previous = channel.mcpConfigFingerprint
959
+ ? (JSON.parse(channel.mcpConfigFingerprint) as typeof previous)
960
+ : undefined;
961
+ } catch {
962
+ previous = undefined;
963
+ }
964
+ const connectionsChanged =
965
+ JSON.stringify(previous?.connections) !== JSON.stringify(connections);
966
+ const addedKinds = new Set(
967
+ engineKinds.filter((kind) => !previous?.engineKinds?.includes(kind)),
968
+ );
969
+ const hadLiveSessions = channel.sessions.size > 0;
970
+ const configured = await setupMcpTaskConfig(
351
971
  channel.taskId,
352
972
  channel.containerName,
353
- channel.mcpConnections,
354
- channel.roster.map((a) => a.kind),
973
+ connections,
974
+ engineKinds,
975
+ codexHomes,
355
976
  );
977
+ if (!configured || !this.isActiveChannel(channel)) return;
978
+ channel.configGeneration += 1;
979
+ channel.mcpConfigFingerprint = fingerprint;
980
+ // Only acknowledge the epoch this write observed. If provisioning dirtied
981
+ // the files while docker exec was in flight, the newer epoch remains
982
+ // uncovered and the serialized ensure immediately publishes another pass.
983
+ channel.mcpConfigAppliedEpoch = dirtyEpoch;
356
984
  // Agent CLIs read MCP servers once, at process start — and durable
357
985
  // sessions (ADR-061) make processes long-lived, so without this a
358
986
  // connection added mid-task stays invisible indefinitely. Recycle IDLE
359
987
  // sessions so the change actually reaches the agents; busy agents keep
360
- // their turn (and in-memory context) and pick it up on a later change or
361
- // natural respawn. Trade-off: a recycled agent loses its in-memory
988
+ // their turn (and in-memory context), remain marked stale, and recycle at
989
+ // turn_complete. Trade-off: a recycled agent loses its in-memory
362
990
  // context and is re-briefed from chat — same as any respawn, and better
363
991
  // than never seeing the connection the user just added. Skipped on the
364
- // channel's first fingerprint (sessions just spawned with this config).
365
- if (hadPrevious) await this.recycleIdleSessions(channel);
992
+ // channel's first successful fingerprint unless sessions already spawned
993
+ // after a failed write and therefore need to reload the recovered config.
994
+ // A kind-only change is written before missing-agent reconciliation, so
995
+ // existing engines keep the same files; only an already-live session of
996
+ // the newly-added kind could be stale.
997
+ if (repairingProvisionedConfig) {
998
+ // Provisioning marked the sessions that existed around its copy. Mark
999
+ // again after the repair so a factory that bound during this write is
1000
+ // covered too (Set de-duplicates the earlier marks).
1001
+ this.markSessionsStale(channel);
1002
+ } else if (!hadPrevious && hadLiveSessions) {
1003
+ this.markSessionsStale(channel);
1004
+ } else if (hadPrevious && connectionsChanged) {
1005
+ this.markSessionsStale(channel);
1006
+ } else if (hadPrevious && addedKinds.size > 0) {
1007
+ const affected = channel.roster
1008
+ .filter((agent) => addedKinds.has(agent.kind))
1009
+ .map((agent) => agent.id);
1010
+ this.markSessionsStale(channel, affected);
1011
+ }
1012
+ }
1013
+
1014
+ /**
1015
+ * Mark current sessions stale after an MCP config change. Publication stays
1016
+ * pure: callers reconcile only after the writer promise has cleared, avoiding
1017
+ * writer -> reconcile -> same-writer self-deadlock. Busy sessions remain
1018
+ * marked until their turn_complete event.
1019
+ */
1020
+ private markSessionsStale(
1021
+ channel: Channel,
1022
+ agentIds: Iterable<string> = channel.sessions.keys(),
1023
+ ): void {
1024
+ if (!this.isActiveChannel(channel)) return;
1025
+ for (const agentId of agentIds) {
1026
+ if (channel.sessions.has(agentId)) {
1027
+ channel.browserStaleSessions.add(agentId);
1028
+ }
1029
+ }
366
1030
  }
367
1031
 
368
- /** Close + respawn every session with no open turn (MCP set changed). */
369
- private async recycleIdleSessions(channel: Channel): Promise<void> {
1032
+ /** Close stale sessions that are idle, without recursively reconciling. */
1033
+ private async closeIdleStaleSessions(channel: Channel): Promise<number> {
370
1034
  let recycled = 0;
371
- for (const [agentId, session] of [...channel.sessions]) {
372
- if (channel.openTurns.has(agentId)) continue; // never mid-turn
373
- if (channel.spawning.has(agentId)) continue;
1035
+ for (const agentId of [...channel.browserStaleSessions]) {
1036
+ if (!this.isActiveChannel(channel)) return recycled;
1037
+ if (
1038
+ channel.activeTurns.has(agentId) ||
1039
+ channel.openTurns.has(agentId) ||
1040
+ channel.spawning.has(agentId)
1041
+ ) {
1042
+ continue;
1043
+ }
1044
+ const session = channel.sessions.get(agentId);
1045
+ // A session that exited on its own is already absent. Its eventual
1046
+ // replacement is fresh, so the old generation is no longer stale.
1047
+ if (!session) {
1048
+ channel.browserStaleSessions.delete(agentId);
1049
+ continue;
1050
+ }
1051
+ channel.browserStaleSessions.delete(agentId);
374
1052
  channel.sessions.delete(agentId);
375
1053
  try {
376
- // close() flips the adapter's `closed` flag first, so the session's
377
- // exit is suppressed no budget increment, no agent.exit noise, no
378
- // race with the respawn below (recoverClaudeAgent's pattern).
1054
+ // Delete before close. Event handlers are identity-guarded, so the
1055
+ // old session's synthetic exit cannot consume respawn budget or delete
1056
+ // the replacement that follows.
379
1057
  await session.close();
380
1058
  } catch {
381
1059
  // Already dead — close is idempotent for our adapters.
@@ -386,82 +1064,124 @@ class Orchestrator {
386
1064
  console.log(
387
1065
  `[orchestrator] ${channel.taskId}: recycled ${recycled} idle session(s) for MCP config change`,
388
1066
  );
389
- await this.reconcileSessions(channel);
390
1067
  }
1068
+ return recycled;
391
1069
  }
392
1070
 
393
- /** Spawn sessions for roster agents added after the initial start. */
1071
+ /** Serialize all missing-session and stale-session reconciliation. */
394
1072
  private async reconcileSessions(channel: Channel): Promise<void> {
395
- const missing = channel.roster.filter((agent) => {
396
- if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
397
- return false;
398
- }
399
- // Crash-loop budget: an agent whose session keeps dying stops being
400
- // respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
401
- // but a budget that has been cold for RESPAWN_COOLDOWN_MS resets, so
402
- // the task heals itself once the cause (a missing CLI on the shared
403
- // volume, a broken config) is fixed, instead of staying dead until a
404
- // host restart.
405
- if ((channel.respawns.get(agent.id) ?? 0) > MAX_RESPAWNS_PER_AGENT) {
406
- const lastAt = channel.respawnLastAt.get(agent.id) ?? 0;
407
- if (Date.now() - lastAt < RESPAWN_COOLDOWN_MS) return false;
408
- channel.respawns.set(agent.id, 0);
409
- }
410
- return true;
411
- });
412
- if (missing.length === 0) return;
1073
+ if (!this.isActiveChannel(channel)) return;
1074
+ if (channel.reconcileReady) {
1075
+ channel.reconcileAgain = true;
1076
+ return channel.reconcileReady;
1077
+ }
413
1078
 
414
- const task = getHostTask(channel.taskId);
415
- if (!task || task.statusMirror !== "running") return;
1079
+ const core = (async () => {
1080
+ do {
1081
+ channel.reconcileAgain = false;
1082
+ await this.reconcileSessionsInner(channel);
1083
+ } while (channel.reconcileAgain && this.isActiveChannel(channel));
1084
+ })();
1085
+ let run: Promise<void>;
1086
+ run = core.finally(() => {
1087
+ if (channel.reconcileReady === run) channel.reconcileReady = null;
1088
+ });
1089
+ channel.reconcileReady = run;
1090
+ return run;
1091
+ }
416
1092
 
417
- for (const agent of missing) channel.spawning.add(agent.id);
418
- try {
419
- // Same per-agent materialisation the initial start does: package
420
- // skills (idempotent — only the new agents' installs run), and the
421
- // shared `uai` CLI rewritten for the full roster.
422
- await installPackageSkills(channel.taskId, missing);
423
- if (channel.browserTesting) {
424
- await setupBrowserTesting(
425
- channel.taskId,
426
- channel.containerName,
427
- channel.roster.some((a) => a.kind === "codex"),
428
- );
429
- }
430
- const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
431
- const cliSecret = loadTaskCliSecret(channel.taskId);
432
- writeAgentCli(channel.taskId, channel.roster, apiUrl);
1093
+ private async reconcileSessionsInner(channel: Channel): Promise<void> {
1094
+ while (this.isActiveChannel(channel)) {
1095
+ await this.closeIdleStaleSessions(channel);
1096
+ if (!this.isActiveChannel(channel)) return;
1097
+
1098
+ // Snapshot before any slow work. An agent added while this generation is
1099
+ // setting up is handled by the next loop, with a setup pass that includes
1100
+ // its engine (especially the Claude-only -> Codex transition).
1101
+ const roster = [...channel.roster];
1102
+ const missing = roster.filter((agent) => {
1103
+ if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
1104
+ return false;
1105
+ }
1106
+ // Crash-loop budget: an agent whose session keeps dying stops being
1107
+ // respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment)
1108
+ // but a cold budget resets so the task can heal after the cause clears.
1109
+ if ((channel.respawns.get(agent.id) ?? 0) > MAX_RESPAWNS_PER_AGENT) {
1110
+ const lastAt = channel.respawnLastAt.get(agent.id) ?? 0;
1111
+ if (Date.now() - lastAt < RESPAWN_COOLDOWN_MS) return false;
1112
+ channel.respawns.set(agent.id, 0);
1113
+ }
1114
+ return true;
1115
+ });
1116
+ if (missing.length === 0) return;
433
1117
 
434
- // ADR-076: extra config-dir accounts for the newly-spawning agents.
435
- await provisionEngineAccounts(
436
- channel.containerName,
437
- missing.map((a) => a.kind),
438
- );
1118
+ const task = getHostTask(channel.taskId);
1119
+ if (!task || task.statusMirror !== "running") return;
439
1120
 
440
- for (const agent of missing) {
441
- const session = await this.factory.create({
442
- taskId: channel.taskId,
443
- agent,
444
- containerName: channel.containerName,
445
- systemPreamble: channel.preambles.get(agent.id) ?? "",
446
- agentEnv: this.accountAgentEnv(
1121
+ for (const agent of missing) channel.spawning.add(agent.id);
1122
+ try {
1123
+ // Same per-agent materialisation the initial start does. Browser setup
1124
+ // is awaited before EVERY missing-session spawn: that reasserts configs
1125
+ // clobbered by resume/auth injection and covers roster generation.
1126
+ await installPackageSkills(channel.taskId, missing);
1127
+ if (!this.isActiveChannel(channel)) return;
1128
+
1129
+ // One materialization gate owns provisioning and (when enabled)
1130
+ // browser migration. This remains serialized across a live false→true
1131
+ // transition, so an older account copy cannot land after migration.
1132
+ await this.ensureBrowserSetup(
1133
+ channel,
1134
+ roster.some((a) => a.kind === "codex"),
1135
+ );
1136
+ if (!this.isActiveChannel(channel)) return;
1137
+ // Provisioning may have refreshed an extra account's whole config
1138
+ // directory. Reassert gateway entries in every Codex home before any
1139
+ // process for this generation can start.
1140
+ await this.ensureMcpConfig(channel);
1141
+ if (!this.isActiveChannel(channel)) return;
1142
+ // Browser and gateway definitions are both loaded only at process
1143
+ // start. A writer marks the previous generation stale; close its idle
1144
+ // subset now regardless of the browser feature flag.
1145
+ await this.closeIdleStaleSessions(channel);
1146
+ if (!this.isActiveChannel(channel)) return;
1147
+
1148
+ const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
1149
+ const cliSecret = loadTaskCliSecret(channel.taskId);
1150
+ writeAgentCli(channel.taskId, roster, apiUrl);
1151
+
1152
+ for (const agent of missing) {
1153
+ if (!this.isActiveChannel(channel)) return;
1154
+ // Another recovery path may have filled this slot while the browser
1155
+ // setup was awaiting.
1156
+ if (channel.sessions.has(agent.id)) continue;
1157
+ const session = await this.createAndBindStableSession(
447
1158
  channel,
448
- agent,
449
- agentCliEnv(
450
- channel.taskId,
1159
+ agent.id,
1160
+ {
1161
+ taskId: channel.taskId,
451
1162
  agent,
452
- task.ownerUserId,
453
- apiUrl,
454
- cliSecret,
455
- ),
456
- ),
457
- });
458
- channel.sessions.set(agent.id, session);
459
- session.onEvent((event) => {
460
- void this.handleAgentEvent(channel, agent.id, event);
461
- });
1163
+ containerName: channel.containerName,
1164
+ systemPreamble: channel.preambles.get(agent.id) ?? "",
1165
+ agentEnv: this.accountAgentEnv(
1166
+ channel,
1167
+ agent,
1168
+ agentCliEnv(
1169
+ channel.taskId,
1170
+ agent,
1171
+ task.ownerUserId,
1172
+ apiUrl,
1173
+ cliSecret,
1174
+ ),
1175
+ ),
1176
+ },
1177
+ );
1178
+ if (!session) return;
1179
+ }
1180
+ } finally {
1181
+ for (const agent of missing) channel.spawning.delete(agent.id);
462
1182
  }
463
- } finally {
464
- for (const agent of missing) channel.spawning.delete(agent.id);
1183
+ // Loop once more: roster additions and sessions retired by a config
1184
+ // rewrite each get a setup pass for their exact roster generation.
465
1185
  }
466
1186
  }
467
1187
 
@@ -487,9 +1207,105 @@ class Orchestrator {
487
1207
  return { ...base, ...account.execEnv };
488
1208
  }
489
1209
 
1210
+ /** Install a session with an event handler scoped to this exact generation. */
1211
+ private bindSession(
1212
+ channel: Channel,
1213
+ agentId: string,
1214
+ session: AgentSession,
1215
+ ): void {
1216
+ channel.sessions.set(agentId, session);
1217
+ session.onEvent((event) => {
1218
+ // Recycle/teardown deletes the old session before calling close().
1219
+ // Adapters deliberately emit a final exit event; without this identity
1220
+ // guard it can delete a replacement or consume its respawn budget.
1221
+ if (
1222
+ !this.isActiveChannel(channel) ||
1223
+ channel.sessions.get(agentId) !== session
1224
+ ) {
1225
+ return;
1226
+ }
1227
+ const operation = this.handleAgentEvent(channel, agentId, event).catch(
1228
+ (err: unknown) => {
1229
+ console.warn(
1230
+ `[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
1231
+ err instanceof Error ? (err.stack ?? err.message) : String(err)
1232
+ }`,
1233
+ );
1234
+ },
1235
+ );
1236
+ channel.eventOperations.add(operation);
1237
+ operation.then(
1238
+ () => channel.eventOperations.delete(operation),
1239
+ () => channel.eventOperations.delete(operation),
1240
+ );
1241
+ });
1242
+ }
1243
+
1244
+ /**
1245
+ * Create against a stable managed-config generation. Agent processes load
1246
+ * MCP definitions during factory startup but are not yet present in the
1247
+ * sessions map, so a concurrent rewrite cannot mark them stale. Detect that
1248
+ * straddle explicitly, close the ambiguous process, and retry.
1249
+ */
1250
+ private createAndBindStableSession(
1251
+ channel: Channel,
1252
+ agentId: string,
1253
+ args: Parameters<AgentSessionFactory["create"]>[0],
1254
+ ): Promise<AgentSession | null> {
1255
+ const operation = this.createAndBindStableSessionCore(
1256
+ channel,
1257
+ agentId,
1258
+ args,
1259
+ );
1260
+ channel.factoryOperations.add(operation);
1261
+ operation.then(
1262
+ () => channel.factoryOperations.delete(operation),
1263
+ () => channel.factoryOperations.delete(operation),
1264
+ );
1265
+ return operation;
1266
+ }
1267
+
1268
+ private async createAndBindStableSessionCore(
1269
+ channel: Channel,
1270
+ agentId: string,
1271
+ args: Parameters<AgentSessionFactory["create"]>[0],
1272
+ ): Promise<AgentSession | null> {
1273
+ while (this.isActiveChannel(channel)) {
1274
+ const generation = channel.configGeneration;
1275
+ const session = await this.factory.create(args);
1276
+ if (!this.isActiveChannel(channel)) {
1277
+ await session.close().catch(() => {});
1278
+ return null;
1279
+ }
1280
+ if (generation === channel.configGeneration) {
1281
+ // Keep the final generation check, stale cleanup, and map bind in one
1282
+ // synchronous continuation. A config writer that resumes immediately
1283
+ // afterward now sees this session and marks it stale; it can no longer
1284
+ // change the generation in the await→bind gap while the process is
1285
+ // invisible.
1286
+ channel.browserStaleSessions.delete(agentId);
1287
+ channel.browserPendingStaleSessions.delete(agentId);
1288
+ this.bindSession(channel, agentId, session);
1289
+ return session;
1290
+ }
1291
+ await session.close().catch(() => {});
1292
+ }
1293
+ return null;
1294
+ }
1295
+
490
1296
  private async startSessions(channel: Channel): Promise<boolean> {
491
1297
  const task = getHostTask(channel.taskId);
492
- if (!task || task.statusMirror !== "running") return false;
1298
+ if (
1299
+ !task ||
1300
+ task.statusMirror !== "running" ||
1301
+ !this.isActiveChannel(channel)
1302
+ ) {
1303
+ return false;
1304
+ }
1305
+ // Freeze the generation before any slow docker work. A roster add while
1306
+ // setup awaits must reconcile under its own engine-aware browser pass,
1307
+ // not slip into this factory loop under the old generation's config.
1308
+ const initialRoster = [...channel.roster];
493
1309
 
494
1310
  // Set the task creator's git author identity in the container (ADR-029).
495
1311
  // The SSH key itself is installed earlier by task-up.sh (host clone +
@@ -497,6 +1313,7 @@ class Orchestrator {
497
1313
  // async (docker exec via dockerCli) so it doesn't block the event loop —
498
1314
  // this runs on every channel ensure, including each post-restart reconnect.
499
1315
  await setupTaskGitIdentity(channel.taskId, task.ownerName, task.ownerEmail);
1316
+ if (!this.isActiveChannel(channel)) return false;
500
1317
 
501
1318
  // GitHub auth for `gh` (ADR-027): mint + inject the access token and
502
1319
  // (re)start its refresh schedule. Done here — not only at task-up — so it
@@ -518,22 +1335,26 @@ class Orchestrator {
518
1335
  // turn. Idempotent + best-effort (returns fast when there are none); a slow
519
1336
  // clone/install briefly delays start, which is acceptable for skill-bearing
520
1337
  // tasks. Never throws.
521
- await installPackageSkills(channel.taskId, channel.roster);
522
-
523
- // ADR-053: wire the Playwright MCP browser (configs + backgrounded
524
- // Chromium install) before agents spawn. Idempotent + best-effort.
525
- if (channel.browserTesting) {
526
- await setupBrowserTesting(
527
- channel.taskId,
528
- channel.containerName,
529
- channel.roster.some((a) => a.kind === "codex"),
530
- );
531
- }
1338
+ await installPackageSkills(channel.taskId, initialRoster);
1339
+ if (!this.isActiveChannel(channel)) return false;
1340
+
1341
+ // ADR-053: wire the Playwright MCP browser before agents spawn configs
1342
+ // AND the Chromium install, awaited, because the only other place the
1343
+ // download could happen is inside the MCP server's startup handshake,
1344
+ // which the client caps at 10s (Codex) / 30s (Claude Code). Idempotent
1345
+ // and ~0.5s once the build is present; a failure arms its own 60s retry
1346
+ // rather than depending on another delivery/ensure to arrive.
1347
+ await this.ensureBrowserSetup(
1348
+ channel,
1349
+ initialRoster.some((a) => a.kind === "codex"),
1350
+ );
1351
+ if (!this.isActiveChannel(channel)) return false;
532
1352
 
533
1353
  // ADR-057: the owner's MCP connections, reached through the host gateway
534
1354
  // (tokens never enter the container) — written BEFORE agents spawn so
535
1355
  // first sessions load them.
536
1356
  await this.ensureMcpConfig(channel);
1357
+ if (!this.isActiveChannel(channel)) return false;
537
1358
 
538
1359
  // ADR-048: write the in-container `uai` CLI (apiUrl only, no token) into the
539
1360
  // workspace. Each agent's OWN task token — carrying only ITS permissions — is
@@ -543,30 +1364,40 @@ class Orchestrator {
543
1364
  const cliSecret = loadTaskCliSecret(channel.taskId);
544
1365
  writeAgentCli(channel.taskId, channel.roster, apiUrl);
545
1366
 
546
- // ADR-076: copy EXTRA config-dir engine accounts into their per-account
547
- // container dirs (chowned) before sessions spawn — the default account's
548
- // dir is already handled by task-up.sh. Best-effort per account.
549
- await provisionEngineAccounts(
550
- channel.containerName,
551
- channel.roster.map((a) => a.kind),
552
- );
553
-
554
- for (const agent of channel.roster) {
555
- const session = await this.factory.create({
556
- taskId: channel.taskId,
557
- agent,
558
- containerName: channel.containerName,
559
- systemPreamble: channel.preambles.get(agent.id) ?? "",
560
- agentEnv: this.accountAgentEnv(
1367
+ for (const agent of initialRoster) {
1368
+ if (!this.isActiveChannel(channel)) return false;
1369
+ // A prior partial start or an independently guarded recovery may already
1370
+ // own this slot. Never replace it with an untracked second generation.
1371
+ if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
1372
+ continue;
1373
+ }
1374
+ channel.spawning.add(agent.id);
1375
+ try {
1376
+ const session = await this.createAndBindStableSession(
561
1377
  channel,
562
- agent,
563
- agentCliEnv(channel.taskId, agent, task.ownerUserId, apiUrl, cliSecret),
564
- ),
565
- });
566
- channel.sessions.set(agent.id, session);
567
- session.onEvent((event) => {
568
- void this.handleAgentEvent(channel, agent.id, event);
569
- });
1378
+ agent.id,
1379
+ {
1380
+ taskId: channel.taskId,
1381
+ agent,
1382
+ containerName: channel.containerName,
1383
+ systemPreamble: channel.preambles.get(agent.id) ?? "",
1384
+ agentEnv: this.accountAgentEnv(
1385
+ channel,
1386
+ agent,
1387
+ agentCliEnv(
1388
+ channel.taskId,
1389
+ agent,
1390
+ task.ownerUserId,
1391
+ apiUrl,
1392
+ cliSecret,
1393
+ ),
1394
+ ),
1395
+ },
1396
+ );
1397
+ if (!session) return false;
1398
+ } finally {
1399
+ channel.spawning.delete(agent.id);
1400
+ }
570
1401
  }
571
1402
 
572
1403
  // First-turn delivery is owned by the CLOUD (channel-router.ensureStarted),
@@ -590,6 +1421,32 @@ class Orchestrator {
590
1421
  if (channel) await this.ensureSessions(channel);
591
1422
  }
592
1423
 
1424
+ private incrementActiveTurns(channel: Channel, agentId: string): void {
1425
+ channel.activeTurns.set(
1426
+ agentId,
1427
+ (channel.activeTurns.get(agentId) ?? 0) + 1,
1428
+ );
1429
+ }
1430
+
1431
+ /** Session output proves at least one active turn, but deltas/tools from the
1432
+ * same turn must not inflate the delivery count. */
1433
+ private markActiveTurn(channel: Channel, agentId: string): void {
1434
+ if (!channel.activeTurns.has(agentId)) {
1435
+ channel.activeTurns.set(agentId, 1);
1436
+ }
1437
+ }
1438
+
1439
+ /** Consume one queued/active turn and report how many remain. */
1440
+ private decrementActiveTurns(channel: Channel, agentId: string): number {
1441
+ const remaining = Math.max(
1442
+ 0,
1443
+ (channel.activeTurns.get(agentId) ?? 1) - 1,
1444
+ );
1445
+ if (remaining === 0) channel.activeTurns.delete(agentId);
1446
+ else channel.activeTurns.set(agentId, remaining);
1447
+ return remaining;
1448
+ }
1449
+
593
1450
  // -- inbound: a human (or routed peer) message ----------------------------
594
1451
 
595
1452
  /**
@@ -622,7 +1479,17 @@ class Orchestrator {
622
1479
  // ADR-076: remember the in-flight prompt so an account rotation on a
623
1480
  // rate-limit can re-deliver it to the fresh session.
624
1481
  channel.lastPrompt.set(agentId, prompt);
625
- void session.send(prompt);
1482
+ this.incrementActiveTurns(channel, agentId);
1483
+ void session.send(prompt).catch((err: unknown) => {
1484
+ if (channel.sessions.get(agentId) === session) {
1485
+ this.decrementActiveTurns(channel, agentId);
1486
+ }
1487
+ console.warn(
1488
+ `[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
1489
+ err instanceof Error ? err.message : String(err)
1490
+ }`,
1491
+ );
1492
+ });
626
1493
  return { ok: true };
627
1494
  }
628
1495
 
@@ -673,8 +1540,10 @@ class Orchestrator {
673
1540
  agentId: string,
674
1541
  event: AgentEvent,
675
1542
  ): Promise<void> {
1543
+ if (!this.isActiveChannel(channel)) return;
676
1544
  switch (event.type) {
677
1545
  case "message_delta": {
1546
+ this.markActiveTurn(channel, agentId);
678
1547
  channel.openTurns.add(agentId);
679
1548
  this.emitHost({
680
1549
  kind: "agent.message_delta",
@@ -685,6 +1554,7 @@ class Orchestrator {
685
1554
  break;
686
1555
  }
687
1556
  case "message_complete": {
1557
+ this.markActiveTurn(channel, agentId);
688
1558
  channel.openTurns.add(agentId);
689
1559
  this.emitHost({
690
1560
  kind: "agent.message_complete",
@@ -696,6 +1566,7 @@ class Orchestrator {
696
1566
  break;
697
1567
  }
698
1568
  case "tool_call": {
1569
+ this.markActiveTurn(channel, agentId);
699
1570
  this.emitHost({
700
1571
  kind: "agent.tool_call",
701
1572
  taskId: channel.taskId,
@@ -706,6 +1577,7 @@ class Orchestrator {
706
1577
  break;
707
1578
  }
708
1579
  case "permission_request": {
1580
+ this.markActiveTurn(channel, agentId);
709
1581
  this.emitHost({
710
1582
  kind: "agent.permission_request",
711
1583
  taskId: channel.taskId,
@@ -733,6 +1605,7 @@ class Orchestrator {
733
1605
  case "error": {
734
1606
  // The turn died with the session — drop its turn-state flags so a
735
1607
  // respawned session starts clean.
1608
+ channel.activeTurns.delete(agentId);
736
1609
  channel.openTurns.delete(agentId);
737
1610
  channel.interrupted.delete(agentId);
738
1611
  // Claude under load (especially Docker Desktop macOS) occasionally
@@ -780,6 +1653,8 @@ class Orchestrator {
780
1653
  channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
781
1654
  channel.respawnLastAt.set(agentId, Date.now());
782
1655
  channel.sessions.delete(agentId);
1656
+ channel.browserStaleSessions.delete(agentId);
1657
+ channel.browserPendingStaleSessions.delete(agentId);
783
1658
  break;
784
1659
  }
785
1660
  case "turn_complete": {
@@ -788,6 +1663,7 @@ class Orchestrator {
788
1663
  // An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
789
1664
  // the cloud discards the buffer instead of handing it to peers.
790
1665
  const aborted = channel.interrupted.delete(agentId);
1666
+ const remainingTurns = this.decrementActiveTurns(channel, agentId);
791
1667
  channel.openTurns.delete(agentId);
792
1668
  // ADR-076: a turn that completed means the current account is healthy —
793
1669
  // reset its rotation budget so a LATER rate-limit gets a fresh failover.
@@ -799,14 +1675,33 @@ class Orchestrator {
799
1675
  aborted,
800
1676
  usage: event.usage,
801
1677
  });
1678
+ // A config repair never interrupts a turn. The stale id remains queued
1679
+ // until this exact boundary, then reconcile closes and replaces it.
1680
+ if (
1681
+ remainingTurns === 0 &&
1682
+ channel.browserStaleSessions.has(agentId)
1683
+ ) {
1684
+ void this.reconcileSessions(channel).catch((err: unknown) => {
1685
+ console.warn(
1686
+ `[orchestrator] ${channel.taskId}: stale-session reconcile failed: ${
1687
+ err instanceof Error ? err.message : String(err)
1688
+ }`,
1689
+ );
1690
+ });
1691
+ }
802
1692
  break;
803
1693
  }
804
1694
  case "exit":
805
1695
  // Same zombie hazard as the error path — a session whose process
806
1696
  // ended (even cleanly) can never carry another turn.
1697
+ channel.activeTurns.delete(agentId);
1698
+ channel.openTurns.delete(agentId);
1699
+ channel.interrupted.delete(agentId);
807
1700
  channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
808
1701
  channel.respawnLastAt.set(agentId, Date.now());
809
1702
  channel.sessions.delete(agentId);
1703
+ channel.browserStaleSessions.delete(agentId);
1704
+ channel.browserPendingStaleSessions.delete(agentId);
810
1705
  break;
811
1706
  }
812
1707
  }
@@ -822,11 +1717,16 @@ class Orchestrator {
822
1717
  channel: Channel,
823
1718
  agentId: string,
824
1719
  ): Promise<void> {
1720
+ if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
1721
+
825
1722
  const tries = (channel.respawns.get(agentId) ?? 0) + 1;
826
1723
  channel.respawns.set(agentId, tries);
827
1724
  channel.respawnLastAt.set(agentId, Date.now());
828
1725
 
829
1726
  if (tries > MAX_RESPAWNS_PER_AGENT) {
1727
+ channel.sessions.delete(agentId);
1728
+ channel.browserStaleSessions.delete(agentId);
1729
+ channel.browserPendingStaleSessions.delete(agentId);
830
1730
  this.emitHost({
831
1731
  kind: "agent.exit",
832
1732
  taskId: channel.taskId,
@@ -839,40 +1739,84 @@ class Orchestrator {
839
1739
  return;
840
1740
  }
841
1741
 
842
- const restored = repairClaudeConfigInContainer(channel.containerName);
1742
+ channel.spawning.add(agentId);
1743
+ try {
1744
+ const restored = repairClaudeConfigInContainer(channel.containerName);
843
1745
 
844
- // Tear the dead session down and build a fresh one in its place.
845
- const old = channel.sessions.get(agentId);
846
- if (old) {
847
- try {
848
- await old.close();
849
- } catch {
850
- // Already exited — close is idempotent for our adapters.
1746
+ // Delete the old generation before awaiting close. Its adapter emits a
1747
+ // terminal event during teardown; bindSession's identity guard must see
1748
+ // it as stale, and reconciliation must see `spawning` instead of opening
1749
+ // a second replacement path.
1750
+ const old = channel.sessions.get(agentId);
1751
+ channel.sessions.delete(agentId);
1752
+ channel.browserStaleSessions.delete(agentId);
1753
+ channel.browserPendingStaleSessions.delete(agentId);
1754
+ if (old) {
1755
+ try {
1756
+ await old.close();
1757
+ } catch {
1758
+ // Already exited — close is idempotent for our adapters.
1759
+ }
851
1760
  }
852
- }
1761
+ if (!this.isActiveChannel(channel)) return;
853
1762
 
854
- const agent = channel.roster.find((a) => a.id === agentId);
855
- if (!agent) return;
856
- const session = await this.factory.create({
857
- taskId: channel.taskId,
858
- agent,
859
- containerName: channel.containerName,
860
- systemPreamble: channel.preambles.get(agentId) ?? "",
861
- });
862
- channel.sessions.set(agentId, session);
863
- session.onEvent((event) => {
864
- void this.handleAgentEvent(channel, agentId, event);
865
- });
1763
+ const agent = channel.roster.find((a) => a.id === agentId);
1764
+ const task = getHostTask(channel.taskId);
1765
+ if (!agent || !task || task.statusMirror !== "running") return;
866
1766
 
867
- const message = restored
868
- ? `${agentId} restarted — config file was missing in the container, restored from the host.`
869
- : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
870
- this.emitHost({
871
- kind: "agent.exit",
872
- taskId: channel.taskId,
873
- agentId,
874
- reason: message,
875
- });
1767
+ const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
1768
+ const cliSecret = loadTaskCliSecret(channel.taskId);
1769
+ const base = agentCliEnv(
1770
+ channel.taskId,
1771
+ agent,
1772
+ task.ownerUserId,
1773
+ apiUrl,
1774
+ cliSecret,
1775
+ );
1776
+ const boundAccountId = channel.accountByAgent.get(agentId);
1777
+ const boundAccount = boundAccountId
1778
+ ? resolveEngineAccounts(agent.kind).find(
1779
+ (account) => account.id === boundAccountId,
1780
+ )
1781
+ : undefined;
1782
+ const session = await this.createAndBindStableSession(channel, agentId, {
1783
+ taskId: channel.taskId,
1784
+ agent,
1785
+ containerName: channel.containerName,
1786
+ systemPreamble: channel.preambles.get(agentId) ?? "",
1787
+ agentEnv: boundAccount
1788
+ ? { ...base, ...boundAccount.execEnv }
1789
+ : this.accountAgentEnv(channel, agent, base),
1790
+ });
1791
+ if (!session) return;
1792
+
1793
+ const message = restored
1794
+ ? `${agentId} restarted — config file was missing in the container, restored from the host.`
1795
+ : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
1796
+ this.emitHost({
1797
+ kind: "agent.exit",
1798
+ taskId: channel.taskId,
1799
+ agentId,
1800
+ reason: message,
1801
+ });
1802
+ } finally {
1803
+ channel.spawning.delete(agentId);
1804
+ // A concurrent config writer can mark the newly-bound recovery stale
1805
+ // while this spawn slot is still held. Reconciliation skips spawning
1806
+ // ids by design, so release must explicitly give that mark another turn.
1807
+ if (
1808
+ this.isActiveChannel(channel) &&
1809
+ channel.browserStaleSessions.has(agentId)
1810
+ ) {
1811
+ void this.reconcileSessions(channel).catch((err: unknown) => {
1812
+ console.warn(
1813
+ `[orchestrator] ${channel.taskId}/${agentId}: post-recovery reconcile failed: ${
1814
+ err instanceof Error ? err.message : String(err)
1815
+ }`,
1816
+ );
1817
+ });
1818
+ }
1819
+ }
876
1820
  }
877
1821
 
878
1822
  /**
@@ -888,6 +1832,10 @@ class Orchestrator {
888
1832
  agentId: string,
889
1833
  agent: RosterAgent,
890
1834
  ): Promise<boolean> {
1835
+ if (!this.isActiveChannel(channel)) return true;
1836
+ // A concurrent terminal event for the same session is already being
1837
+ // handled by the path that owns this spawn slot.
1838
+ if (channel.spawning.has(agentId)) return true;
891
1839
  if (resolveEngineAccounts(agent.kind).length < 2) return false;
892
1840
 
893
1841
  const budget = (channel.rotations.get(agentId) ?? 0) + 1;
@@ -904,53 +1852,100 @@ class Orchestrator {
904
1852
  const task = getHostTask(channel.taskId);
905
1853
  if (!task || task.statusMirror !== "running") return false;
906
1854
 
907
- // Tear the rate-limited session down (its `closed` flag suppresses its own
908
- // exit, so no respawn-budget noise races this rotation).
909
- const old = channel.sessions.get(agentId);
910
- channel.sessions.delete(agentId);
911
- if (old) {
912
- try {
913
- await old.close();
914
- } catch {
915
- /* already gone — close is idempotent for our adapters */
1855
+ let replacement: AgentSession | null = null;
1856
+ channel.spawning.add(agentId);
1857
+ try {
1858
+ // Tear the rate-limited session down. Delete first so its final adapter
1859
+ // event is generation-stale, while `spawning` keeps reconciliation from
1860
+ // opening a second replacement path during the slow provisioning/setup.
1861
+ const old = channel.sessions.get(agentId);
1862
+ channel.sessions.delete(agentId);
1863
+ channel.browserStaleSessions.delete(agentId);
1864
+ channel.browserPendingStaleSessions.delete(agentId);
1865
+ if (old) {
1866
+ try {
1867
+ await old.close();
1868
+ } catch {
1869
+ /* already gone — close is idempotent for our adapters */
1870
+ }
916
1871
  }
917
- }
1872
+ if (!this.isActiveChannel(channel)) return true;
1873
+
1874
+ // The serialized materialization gate owns both account provisioning
1875
+ // and, when enabled, browser migration.
1876
+ await this.ensureBrowserSetup(
1877
+ channel,
1878
+ channel.roster.some((candidate) => candidate.kind === "codex"),
1879
+ true,
1880
+ );
1881
+ if (!this.isActiveChannel(channel)) return true;
1882
+ await this.ensureMcpConfig(channel);
1883
+ if (!this.isActiveChannel(channel)) return true;
918
1884
 
919
- // Extra config-dir accounts need their dir copied in before use.
920
- await provisionEngineAccounts(channel.containerName, [agent.kind]);
1885
+ const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
1886
+ const cliSecret = loadTaskCliSecret(channel.taskId);
1887
+ const base = agentCliEnv(
1888
+ channel.taskId,
1889
+ agent,
1890
+ task.ownerUserId,
1891
+ apiUrl,
1892
+ cliSecret,
1893
+ );
1894
+ channel.accountByAgent.set(agentId, next.id);
1895
+ noteEngineAccountUsed(next.id);
921
1896
 
922
- const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
923
- const cliSecret = loadTaskCliSecret(channel.taskId);
924
- const base = agentCliEnv(
925
- channel.taskId,
926
- agent,
927
- task.ownerUserId,
928
- apiUrl,
929
- cliSecret,
930
- );
931
- channel.accountByAgent.set(agentId, next.id);
932
- noteEngineAccountUsed(next.id);
933
-
934
- const session = await this.factory.create({
935
- taskId: channel.taskId,
936
- agent,
937
- containerName: channel.containerName,
938
- systemPreamble: channel.preambles.get(agentId) ?? "",
939
- agentEnv: { ...base, ...next.execEnv },
940
- });
941
- channel.sessions.set(agentId, session);
942
- session.onEvent((event) => {
943
- void this.handleAgentEvent(channel, agentId, event);
944
- });
1897
+ replacement = await this.createAndBindStableSession(
1898
+ channel,
1899
+ agentId,
1900
+ {
1901
+ taskId: channel.taskId,
1902
+ agent,
1903
+ containerName: channel.containerName,
1904
+ systemPreamble: channel.preambles.get(agentId) ?? "",
1905
+ agentEnv: { ...base, ...next.execEnv },
1906
+ },
1907
+ );
1908
+ if (!replacement) return true;
945
1909
 
946
- this.emitSystemNote(
947
- channel.taskId,
948
- `${agentId}: hit a rate limit — switched to another ${agent.kind} account and retried.`,
949
- );
1910
+ this.emitSystemNote(
1911
+ channel.taskId,
1912
+ `${agentId}: hit a rate limit — switched to another ${agent.kind} account and retried.`,
1913
+ );
1914
+
1915
+ // Re-deliver the in-flight prompt so the interrupted turn resumes.
1916
+ const prompt = channel.lastPrompt.get(agentId);
1917
+ if (prompt) {
1918
+ this.incrementActiveTurns(channel, agentId);
1919
+ void replacement.send(prompt).catch((err: unknown) => {
1920
+ if (channel.sessions.get(agentId) === replacement) {
1921
+ this.decrementActiveTurns(channel, agentId);
1922
+ }
1923
+ console.warn(
1924
+ `[orchestrator] ${channel.taskId}/${agentId}: rotated send failed: ${
1925
+ err instanceof Error ? err.message : String(err)
1926
+ }`,
1927
+ );
1928
+ });
1929
+ }
1930
+ } finally {
1931
+ channel.spawning.delete(agentId);
1932
+ }
950
1933
 
951
- // Re-deliver the in-flight prompt so the interrupted turn resumes.
952
- const prompt = channel.lastPrompt.get(agentId);
953
- if (prompt) void session.send(prompt);
1934
+ // Browser/config setup may have marked other live generations stale. Run
1935
+ // their safe-idle recycle only after this agent's replacement is bound.
1936
+ if (
1937
+ replacement &&
1938
+ this.isActiveChannel(channel) &&
1939
+ channel.browserStaleSessions.size > 0
1940
+ ) {
1941
+ void this.reconcileSessions(channel).catch((err: unknown) => {
1942
+ console.warn(
1943
+ `[orchestrator] ${channel.taskId}: post-rotation reconcile failed: ${
1944
+ err instanceof Error ? err.message : String(err)
1945
+ }`,
1946
+ );
1947
+ });
1948
+ }
954
1949
  return true;
955
1950
  }
956
1951
 
@@ -970,11 +1965,69 @@ class Orchestrator {
970
1965
  }
971
1966
 
972
1967
  /** Tear a channel down (task killed). */
973
- async closeChannel(taskId: string): Promise<void> {
1968
+ closeChannel(taskId: string): Promise<void> {
1969
+ const existing = this.channelClosures.get(taskId);
1970
+ if (existing) return existing;
1971
+
1972
+ this.blockedTasks.add(taskId);
1973
+ this.channelSpecs.delete(taskId);
974
1974
  const ch = this.channels.get(taskId);
975
- if (!ch) return;
976
- for (const session of ch.sessions.values()) await session.close();
1975
+ if (!ch) return Promise.resolve();
1976
+ // Invalidate the generation BEFORE the first await. Browser setup and the
1977
+ // start/reconcile factory continuations all check this identity, so none
1978
+ // can repopulate an orphan channel while its sessions close.
1979
+ ch.closed = true;
977
1980
  this.channels.delete(taskId);
1981
+ this.clearBrowserRetry(ch);
1982
+ ch.browserStaleSessions.clear();
1983
+ ch.browserPendingStaleSessions.clear();
1984
+ ch.activeTurns.clear();
1985
+ ch.openTurns.clear();
1986
+ ch.interrupted.clear();
1987
+ ch.reconcileAgain = false;
1988
+ const drains = [
1989
+ ch.browserSetup,
1990
+ ch.mcpConfigWrite,
1991
+ ch.reconcileReady,
1992
+ ch.sessionsReady,
1993
+ ...ch.factoryOperations,
1994
+ ...ch.eventOperations,
1995
+ ].filter((promise): promise is Promise<unknown> => promise !== null);
1996
+ const sessions = [...ch.sessions.values()];
1997
+ ch.sessions.clear();
1998
+ const closes = sessions.map((session) =>
1999
+ session.close().catch(() => {
2000
+ // Teardown is best-effort; every adapter close is intended idempotent.
2001
+ }),
2002
+ );
2003
+ // Draining externally side-effecting work is part of the container-name
2004
+ // reuse fence. An old docker cp/config writer/factory must finish before
2005
+ // taskDown can return and a later taskUp can create the same container.
2006
+ let closing: Promise<void>;
2007
+ closing = Promise.allSettled([...drains, ...closes])
2008
+ .then(() => undefined)
2009
+ .finally(() => {
2010
+ if (this.channelClosures.get(taskId) === closing) {
2011
+ this.channelClosures.delete(taskId);
2012
+ }
2013
+ });
2014
+ this.channelClosures.set(taskId, closing);
2015
+ return closing;
2016
+ }
2017
+
2018
+ /** Task-up must join teardown before it can recreate the same compose and
2019
+ * container names. This does not clear the tombstone; only success does. */
2020
+ waitForChannelClose(taskId: string): Promise<void> {
2021
+ return this.channelClosures.get(taskId) ?? Promise.resolve();
2022
+ }
2023
+
2024
+ /** A successful task-up is the only transition that reopens a tombstoned
2025
+ * task id after stop/teardown. */
2026
+ allowChannel(taskId: string): void {
2027
+ // A caller that forgot the task-up fence cannot reopen the id while old
2028
+ // external operations are still draining.
2029
+ if (this.channelClosures.has(taskId)) return;
2030
+ this.blockedTasks.delete(taskId);
978
2031
  }
979
2032
 
980
2033
  /**
@@ -986,22 +2039,50 @@ class Orchestrator {
986
2039
  * `stopped` is not an ACTIVE status, host recovery won't restart it.
987
2040
  */
988
2041
  async stopTask(taskId: string): Promise<{ ok: boolean; error?: string }> {
2042
+ return this.runTaskLifecycle(taskId, () =>
2043
+ this.stopTaskWithinLifecycle(taskId),
2044
+ );
2045
+ }
2046
+
2047
+ private async stopTaskWithinLifecycle(
2048
+ taskId: string,
2049
+ ): Promise<{ ok: boolean; error?: string }> {
989
2050
  const task = getHostTask(taskId);
990
2051
  if (!task) return { ok: false, error: "unknown task" };
991
2052
  const project = task.composeProject;
992
2053
  if (!project) return { ok: false, error: "task has no running stack" };
993
2054
  // Close sessions first so nothing races to respawn them mid-stop.
994
2055
  await this.closeChannel(taskId);
2056
+ try {
2057
+ await stopPreviewSidecars(taskId);
2058
+ } catch (err) {
2059
+ console.warn(
2060
+ `[orchestrator] ${taskId}: preview cleanup before stop failed: ${
2061
+ err instanceof Error ? err.message : String(err)
2062
+ }`,
2063
+ );
2064
+ }
995
2065
  const ps = await dockerCli([
996
2066
  "ps",
997
2067
  "-q",
998
2068
  "--filter",
999
2069
  `label=com.docker.compose.project=${project}`,
1000
2070
  ]);
2071
+ if (ps.status !== 0) {
2072
+ // An empty stdout is not evidence that the stack is gone when docker
2073
+ // itself failed. Reopen the channel and leave the runtime mirror alone
2074
+ // so cloud reconciliation cannot turn a live container into "stopped".
2075
+ this.allowChannel(taskId);
2076
+ return {
2077
+ ok: false,
2078
+ error: ps.stderr.trim().slice(0, 200) || "docker ps failed",
2079
+ };
2080
+ }
1001
2081
  const ids = ps.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
1002
2082
  if (ids.length > 0) {
1003
2083
  const stopped = await dockerCli(["stop", ...ids], { timeoutMs: 60_000 });
1004
2084
  if (stopped.status !== 0) {
2085
+ this.allowChannel(taskId);
1005
2086
  return {
1006
2087
  ok: false,
1007
2088
  error: stopped.stderr.trim().slice(0, 200) || "docker stop failed",
@@ -1411,7 +2492,7 @@ export function buildSystemPreamble(
1411
2492
  ? [
1412
2493
  "## Browser",
1413
2494
  "",
1414
- "This container has a headless Chromium available through the",
2495
+ "This container has Chromium available through the",
1415
2496
  "`browser` MCP server (Playwright). Use it to VERIFY UI work",
1416
2497
  "end-to-end — the dev server you're building runs in this same",
1417
2498
  "container, so navigate to `http://localhost:<port>` directly.",
@@ -1422,9 +2503,10 @@ export function buildSystemPreamble(
1422
2503
  "in your reply as `[image: /workspace/.uai/attachments/<name>.png]`.",
1423
2504
  "Your browser runs on a virtual display the humans can WATCH live",
1424
2505
  "(the \"browser\" preview) — nothing for you to do about that.",
1425
- "The first browser launch in a fresh container may take a minute",
1426
- "while Chromium's system deps finish installing retry once if",
1427
- "it fails immediately after task start.",
2506
+ "The browser is ready to use just call it. If a call ever fails,",
2507
+ "the failure is real: report it and say what you could not verify.",
2508
+ "Do NOT fall back to a static HTML harness and describe it as",
2509
+ "verified — that is worse than saying the browser was unavailable.",
1428
2510
  "",
1429
2511
  ]
1430
2512
  : []),
@@ -1575,6 +2657,39 @@ export function getOrchestrator(): Orchestrator {
1575
2657
 
1576
2658
  let recoveryPromise: Promise<void> = Promise.resolve();
1577
2659
 
2660
+ interface TaskRecoveryBarrier {
2661
+ promise: Promise<void>;
2662
+ resolve(): void;
2663
+ }
2664
+
2665
+ const taskRecoveryBarriers = new Map<string, TaskRecoveryBarrier>();
2666
+
2667
+ function openTaskRecoveryBarrier(taskId: string): void {
2668
+ if (taskRecoveryBarriers.has(taskId)) return;
2669
+ let resolve!: () => void;
2670
+ const promise = new Promise<void>((done) => {
2671
+ resolve = done;
2672
+ });
2673
+ taskRecoveryBarriers.set(taskId, { promise, resolve });
2674
+ }
2675
+
2676
+ function finishTaskRecovery(taskId: string): void {
2677
+ const barrier = taskRecoveryBarriers.get(taskId);
2678
+ if (!barrier) return;
2679
+ taskRecoveryBarriers.delete(taskId);
2680
+ barrier.resolve();
2681
+ }
2682
+
2683
+ function finishAllTaskRecovery(): void {
2684
+ for (const taskId of [...taskRecoveryBarriers.keys()]) {
2685
+ finishTaskRecovery(taskId);
2686
+ }
2687
+ }
2688
+
2689
+ export function taskRecoveryComplete(taskId: string): Promise<void> {
2690
+ return taskRecoveryBarriers.get(taskId)?.promise ?? Promise.resolve();
2691
+ }
2692
+
1578
2693
  /**
1579
2694
  * Resolves once boot-time recovery has finished (starting the orchestrator —
1580
2695
  * and with it recovery — if that hasn't happened yet). Never rejects.
@@ -1698,47 +2813,67 @@ export async function recoverRunningTasks(opts?: {
1698
2813
  }): Promise<void> {
1699
2814
  const retryMs = opts?.retryMs ?? 20_000;
1700
2815
  const maxPasses = opts?.maxPasses ?? 45;
2816
+ let pendingTaskIds: ReadonlySet<string> | undefined;
1701
2817
  // After a machine reboot the host service and the Docker daemon start
1702
2818
  // concurrently; a pass that finds docker unreachable resolves nothing and
1703
2819
  // must retry once the daemon is up — the whole point of recovery is to
1704
2820
  // restart the exited containers that reboot left behind.
1705
2821
  for (let pass = 1; ; pass++) {
1706
- const deferred = await recoveryPass();
1707
- if (deferred === 0) return;
2822
+ const deferred = await recoveryPass(pendingTaskIds);
2823
+ if (deferred.size === 0) {
2824
+ finishAllTaskRecovery();
2825
+ return;
2826
+ }
2827
+ pendingTaskIds = deferred;
1708
2828
  if (pass >= maxPasses) {
1709
2829
  console.warn(
1710
- `[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred} task(s) unresolved)`,
2830
+ `[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
1711
2831
  );
2832
+ finishAllTaskRecovery();
1712
2833
  return;
1713
2834
  }
1714
2835
  if (pass === 1) {
1715
2836
  console.log(
1716
- `[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred} task(s)`,
2837
+ `[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
1717
2838
  );
1718
2839
  }
1719
2840
  await new Promise((resolve) => setTimeout(resolve, retryMs));
1720
2841
  }
1721
2842
  }
1722
2843
 
1723
- /** One recovery sweep. Returns how many tasks were deferred on an
1724
- * unreachable docker (0 = everything resolved one way or another). */
1725
- async function recoveryPass(): Promise<number> {
1726
- let deferred = 0;
2844
+ /** One recovery sweep. Returns only task ids deferred by unreachable Docker;
2845
+ * successful ids are never revisited by a later retry pass. */
2846
+ async function recoveryPass(
2847
+ onlyTaskIds?: ReadonlySet<string>,
2848
+ ): Promise<Set<string>> {
2849
+ const deferred = new Set<string>();
1727
2850
  try {
1728
2851
  const db = getDb();
1729
- const rows = db
2852
+ const activeRows = db
1730
2853
  .select()
1731
2854
  .from(schema.hostTasks)
1732
2855
  .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
1733
2856
  .all();
1734
- if (rows.length === 0) return 0;
2857
+ const rows = onlyTaskIds
2858
+ ? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
2859
+ : activeRows;
2860
+ const activeTaskIds = new Set(rows.map((task) => task.taskId));
2861
+ const expectedTaskIds =
2862
+ onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
2863
+ for (const taskId of expectedTaskIds) {
2864
+ if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
2865
+ }
2866
+ for (const task of rows) openTaskRecoveryBarrier(task.taskId);
2867
+ if (rows.length === 0) return deferred;
1735
2868
  console.log(
1736
2869
  `[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
1737
2870
  );
1738
2871
  for (const task of rows) {
1739
2872
  try {
1740
- if (!(await recoverOneTask(task))) deferred += 1;
2873
+ if (await recoverOneTask(task)) finishTaskRecovery(task.taskId);
2874
+ else deferred.add(task.taskId);
1741
2875
  } catch (err) {
2876
+ finishTaskRecovery(task.taskId);
1742
2877
  console.error(
1743
2878
  `[orchestrator] recovery: ${task.taskId} failed:`,
1744
2879
  err instanceof Error ? err.message : err,
@@ -1746,6 +2881,7 @@ async function recoveryPass(): Promise<number> {
1746
2881
  }
1747
2882
  }
1748
2883
  } catch (err) {
2884
+ finishAllTaskRecovery();
1749
2885
  console.error(
1750
2886
  "[orchestrator] recovery: top-level failure",
1751
2887
  err instanceof Error ? err.message : err,
@@ -1875,5 +3011,3 @@ function db_setRuntime(
1875
3011
  ): void {
1876
3012
  upsertHostTask(taskId, extras);
1877
3013
  }
1878
-
1879
- export type { Orchestrator };