@runuai/host 0.8.42 → 0.8.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/browser-testing.ts +1056 -90
- package/lib/codex-auth.ts +22 -15
- package/lib/engine-accounts.ts +23 -4
- package/lib/mcp-config-lock.ts +2 -0
- package/lib/mcp-gateway.ts +393 -21
- package/lib/orchestrator.ts +1351 -217
- package/lib/preview-sidecar.ts +123 -3
- package/package.json +1 -1
- package/scripts/agent/task-down.sh +16 -0
- package/src/index.ts +215 -97
- package/src/main.ts +19 -10
package/lib/orchestrator.ts
CHANGED
|
@@ -50,7 +50,11 @@ import {
|
|
|
50
50
|
loadTaskCliSecret,
|
|
51
51
|
writeAgentCli,
|
|
52
52
|
} from "./agent-cli";
|
|
53
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
DEFAULT_CODEX_HOME,
|
|
55
|
+
setupBrowserTesting,
|
|
56
|
+
type BrowserSetup,
|
|
57
|
+
} from "./browser-testing";
|
|
54
58
|
import { injectCodexIntoContainer } from "./codex-auth";
|
|
55
59
|
import {
|
|
56
60
|
cooldownEngineAccount,
|
|
@@ -60,6 +64,7 @@ import {
|
|
|
60
64
|
resolveEngineAccounts,
|
|
61
65
|
} from "./engine-accounts";
|
|
62
66
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
67
|
+
import { stopPreviewSidecars } from "./preview-sidecar";
|
|
63
68
|
import { env } from "./env";
|
|
64
69
|
import type {
|
|
65
70
|
ChannelEnsureInput,
|
|
@@ -90,9 +95,60 @@ interface Channel {
|
|
|
90
95
|
* awaits docker work before populating it). Reset to null on failure so the
|
|
91
96
|
* next ensure retries. */
|
|
92
97
|
sessionsReady: Promise<boolean> | null;
|
|
98
|
+
/** ADR-053: true only when both the browser install and managed MCP configs
|
|
99
|
+
* are ready. A task still starts while false; setup retries in the
|
|
100
|
+
* background on a bounded timer. */
|
|
101
|
+
browserReady: boolean;
|
|
102
|
+
/** The one setup pass currently touching this container. Reconcile, retry,
|
|
103
|
+
* and initial start all join this promise instead of racing config files. */
|
|
104
|
+
browserSetup: Promise<BrowserSetup> | null;
|
|
105
|
+
/** Codex account homes covered by the in-flight setup. A roster/account
|
|
106
|
+
* change that adds a home queues one stronger pass after it. */
|
|
107
|
+
browserSetupCodexHomes: string[] | null;
|
|
108
|
+
/** Engine kinds provisioned by the in-flight setup. A roster generation
|
|
109
|
+
* that adds a kind cannot join a weaker provisioning pass. */
|
|
110
|
+
browserSetupEngineKinds: string[] | null;
|
|
111
|
+
/** Exact account roster covered by the in-flight provisioning pass. */
|
|
112
|
+
browserSetupProvisionFingerprint: string | null;
|
|
113
|
+
/** Mutable coverage of the in-flight materialization pass. Joiners retain
|
|
114
|
+
* this object through cleanup and can distinguish requested browser work
|
|
115
|
+
* from a browser attempt that actually began. */
|
|
116
|
+
browserSetupCoverage: {
|
|
117
|
+
browserRequested: boolean;
|
|
118
|
+
browserAttempted: boolean;
|
|
119
|
+
skippedBecauseDisabled: boolean;
|
|
120
|
+
provisionAttempted: boolean;
|
|
121
|
+
} | null;
|
|
122
|
+
/** Last account roster whose provisioning was attempted in this container.
|
|
123
|
+
* Missing-session reconciliation must not hot-loop an unchanged failed
|
|
124
|
+
* copy and invalidate the peer sessions it is trying to repair. */
|
|
125
|
+
provisionAttemptedFingerprint?: string;
|
|
126
|
+
/** An incomplete account copy that must be retried on the bounded timer. */
|
|
127
|
+
provisionRetryFingerprint?: string;
|
|
128
|
+
/** Self-scheduled retry after a failed install/configuration pass. */
|
|
129
|
+
browserRetryTimer: ReturnType<typeof setTimeout> | null;
|
|
130
|
+
/** Sessions that loaded MCP config before the latest rewrite. Busy sessions
|
|
131
|
+
* stay here until their turn completes; each id is recycled exactly once. */
|
|
132
|
+
browserStaleSessions: Set<string>;
|
|
133
|
+
/** Browser-config rewrites whose install/config preconditions are not ready
|
|
134
|
+
* yet. These sessions must eventually reload the definition, but recycling
|
|
135
|
+
* them before the browser is usable only trades one broken session for
|
|
136
|
+
* another. */
|
|
137
|
+
browserPendingStaleSessions: Set<string>;
|
|
138
|
+
/** One reconcile loop per channel. It covers ordinary missing agents and
|
|
139
|
+
* stale-session recycling without overlapping factory creates. */
|
|
140
|
+
reconcileReady: Promise<void> | null;
|
|
141
|
+
/** A reconcile request that arrived while the loop was in flight. */
|
|
142
|
+
reconcileAgain: boolean;
|
|
143
|
+
/** Set before teardown awaits anything. Every async continuation checks both
|
|
144
|
+
* this flag and the channel-map identity before mutating or spawning. */
|
|
145
|
+
closed: boolean;
|
|
93
146
|
/** Agents with turn output already streamed for the current turn — i.e. the
|
|
94
147
|
* cloud has buffered content for them (see ChannelRouter.turnBuffer). */
|
|
95
148
|
openTurns: Set<string>;
|
|
149
|
+
/** Agents with a delivered turn that has not reached a terminal event yet.
|
|
150
|
+
* Unlike openTurns, this covers thinking/tool/permission time before text. */
|
|
151
|
+
activeTurns: Map<string, number>;
|
|
96
152
|
/** Agents whose current turn was interrupted (ESC) — their next
|
|
97
153
|
* turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
|
|
98
154
|
* half-turn instead of delivering it to @-mentioned peers. */
|
|
@@ -116,9 +172,30 @@ interface Channel {
|
|
|
116
172
|
* ensure runs on every message). Live sessions read MCP config at spawn,
|
|
117
173
|
* so a mid-task write takes effect on the next (re)spawn. */
|
|
118
174
|
mcpConfigFingerprint?: string;
|
|
175
|
+
/** Serialize gateway config writes so two ensures cannot publish stale
|
|
176
|
+
* content and then memoize the newer fingerprint over it. */
|
|
177
|
+
mcpConfigWrite: Promise<void> | null;
|
|
178
|
+
/** Account provisioning can overwrite per-engine config without changing
|
|
179
|
+
* the desired gateway fingerprint. A monotonic epoch prevents an older
|
|
180
|
+
* in-flight writer from clearing a newer provisioning mutation. */
|
|
181
|
+
mcpConfigDirtyEpoch: number;
|
|
182
|
+
/** Dirty epoch covered by the last successful gateway publication. */
|
|
183
|
+
mcpConfigAppliedEpoch: number;
|
|
184
|
+
/** Monotonic version of the managed config files agent processes read at
|
|
185
|
+
* startup. A factory create that straddles a rewrite is discarded and
|
|
186
|
+
* retried before it can be bound as the live generation. */
|
|
187
|
+
configGeneration: number;
|
|
119
188
|
/** Agents with a reconcile-spawn in flight (ADR-049 mid-task adds) — guards
|
|
120
189
|
* against a concurrent ensure double-spawning the same new agent. */
|
|
121
190
|
spawning: Set<string>;
|
|
191
|
+
/** Factory creates are externally side-effecting before they bind. Teardown
|
|
192
|
+
* drains them so an old channel cannot resume against a reused container
|
|
193
|
+
* name and overwrite the replacement generation's durable transport. */
|
|
194
|
+
factoryOperations: Set<Promise<unknown>>;
|
|
195
|
+
/** Terminal-event recovery/rotation can remove a session before awaiting
|
|
196
|
+
* its close. Track the enclosing handler so teardown still drains that
|
|
197
|
+
* external operation before the container name can be reused. */
|
|
198
|
+
eventOperations: Set<Promise<unknown>>;
|
|
122
199
|
/** ADR-076: which engine account each agent's live session is bound to, so a
|
|
123
200
|
* rate-limit can cool THAT account and rotate off it. */
|
|
124
201
|
accountByAgent: Map<string, string>;
|
|
@@ -136,6 +213,10 @@ const MAX_RESPAWNS_PER_AGENT = 5;
|
|
|
136
213
|
const MAX_ROTATIONS_PER_AGENT = 6;
|
|
137
214
|
/** A burned respawn budget resets after this quiet period (see reconcile). */
|
|
138
215
|
const RESPAWN_COOLDOWN_MS = 10 * 60_000;
|
|
216
|
+
/** ADR-053: gap between browser-install retries. Ensures fire every couple of
|
|
217
|
+
* seconds; a container-gone failure returns instantly, so without this the
|
|
218
|
+
* retry would spin npx continuously for the life of a broken task. */
|
|
219
|
+
const BROWSER_RETRY_COOLDOWN_MS = 60_000;
|
|
139
220
|
|
|
140
221
|
/** Substrings in an agent's error output that mean "config was
|
|
141
222
|
* unlinked between runs" — repair-and-respawn covers the common
|
|
@@ -146,9 +227,18 @@ const CLAUDE_CONFIG_MISSING_PATTERNS = [
|
|
|
146
227
|
/\/\.claude\.json/i,
|
|
147
228
|
];
|
|
148
229
|
|
|
149
|
-
class Orchestrator {
|
|
230
|
+
export class Orchestrator {
|
|
150
231
|
private readonly channels = new Map<string, Channel>();
|
|
151
232
|
private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
|
|
233
|
+
/** Teardown tombstones prevent a cloud ensure from reopening a task between
|
|
234
|
+
* session close and container stop/removal. Cleared only by task-up. */
|
|
235
|
+
private readonly blockedTasks = new Set<string>();
|
|
236
|
+
/** Every teardown caller for a task joins the same container-reuse fence. */
|
|
237
|
+
private readonly channelClosures = new Map<string, Promise<void>>();
|
|
238
|
+
/** Host lifecycle commands can arrive concurrently over the bridge/local
|
|
239
|
+
* UI. Serialize each task through container creation/destruction so a
|
|
240
|
+
* later command cannot overtake an earlier one after channel close. */
|
|
241
|
+
private readonly taskLifecycleTails = new Map<string, Promise<void>>();
|
|
152
242
|
private readonly hostSubscribers = new Set<HostEventSubscriber>();
|
|
153
243
|
|
|
154
244
|
constructor(private readonly factory: AgentSessionFactory) {}
|
|
@@ -160,6 +250,36 @@ class Orchestrator {
|
|
|
160
250
|
return () => this.hostSubscribers.delete(fn);
|
|
161
251
|
}
|
|
162
252
|
|
|
253
|
+
/** Run one task lifecycle mutation in arrival order. Different task ids
|
|
254
|
+
* remain fully concurrent. The queue tail never rejects, so one failed
|
|
255
|
+
* command cannot poison later recovery. */
|
|
256
|
+
async runTaskLifecycle<T>(
|
|
257
|
+
taskId: string,
|
|
258
|
+
operation: () => Promise<T>,
|
|
259
|
+
): Promise<T> {
|
|
260
|
+
const previous =
|
|
261
|
+
this.taskLifecycleTails.get(taskId) ?? Promise.resolve();
|
|
262
|
+
let release!: () => void;
|
|
263
|
+
const current = new Promise<void>((resolve) => {
|
|
264
|
+
release = resolve;
|
|
265
|
+
});
|
|
266
|
+
const tail = previous.catch(() => {}).then(() => current);
|
|
267
|
+
this.taskLifecycleTails.set(taskId, tail);
|
|
268
|
+
await previous.catch(() => {});
|
|
269
|
+
try {
|
|
270
|
+
// Boot recovery can still be starting/copying into this task's stable
|
|
271
|
+
// container name while bridge and local-UI commands are already live.
|
|
272
|
+
// Join that per-task barrier before any queued create/destroy mutation.
|
|
273
|
+
await taskRecoveryComplete(taskId);
|
|
274
|
+
return await operation();
|
|
275
|
+
} finally {
|
|
276
|
+
release();
|
|
277
|
+
if (this.taskLifecycleTails.get(taskId) === tail) {
|
|
278
|
+
this.taskLifecycleTails.delete(taskId);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
163
283
|
/** Emit a host-originated system note into a task's channel (ADR-027). */
|
|
164
284
|
emitSystemNote(taskId: string, text: string): void {
|
|
165
285
|
this.emitHost({ kind: "system.note", taskId, text });
|
|
@@ -179,6 +299,7 @@ class Orchestrator {
|
|
|
179
299
|
|
|
180
300
|
registerChannelSpec(spec: ChannelEnsureInput): void {
|
|
181
301
|
this.channelSpecs.set(spec.taskId, spec);
|
|
302
|
+
if (this.blockedTasks.has(spec.taskId)) return;
|
|
182
303
|
// ADR-049: the cloud re-sends the spec on every message AND right after a
|
|
183
304
|
// mid-task roster/participant change. If the channel is already live,
|
|
184
305
|
// fold the fresh spec in: append new roster agents (their sessions spawn
|
|
@@ -190,6 +311,11 @@ class Orchestrator {
|
|
|
190
311
|
|
|
191
312
|
/** Fold a fresh channel spec into a live channel (ADR-049). */
|
|
192
313
|
private refreshChannel(channel: Channel, spec: ChannelEnsureInput): void {
|
|
314
|
+
const browserWasEnabled = channel.browserTesting;
|
|
315
|
+
const previousMcpShape = JSON.stringify({
|
|
316
|
+
connections: channel.mcpConnections,
|
|
317
|
+
kinds: [...new Set(channel.roster.map((agent) => agent.kind))].sort(),
|
|
318
|
+
});
|
|
193
319
|
const known = new Set(channel.roster.map((a) => a.id));
|
|
194
320
|
// Replace EXISTING agents' data with the fresh spec — a mid-task roster edit
|
|
195
321
|
// can change permissions, model, brief, role or skills, and without this the
|
|
@@ -211,6 +337,13 @@ class Orchestrator {
|
|
|
211
337
|
channel.browserTesting = spec.browserTesting === true;
|
|
212
338
|
channel.sharedFiles = spec.sharedFiles ?? "ro";
|
|
213
339
|
channel.mcpConnections = spec.mcpConnections ?? [];
|
|
340
|
+
const nextMcpShape = JSON.stringify({
|
|
341
|
+
connections: channel.mcpConnections,
|
|
342
|
+
kinds: [...new Set(channel.roster.map((agent) => agent.kind))].sort(),
|
|
343
|
+
});
|
|
344
|
+
if (nextMcpShape !== previousMcpShape) {
|
|
345
|
+
channel.mcpConfigDirtyEpoch += 1;
|
|
346
|
+
}
|
|
214
347
|
for (const agent of channel.roster) {
|
|
215
348
|
channel.preambles.set(
|
|
216
349
|
agent.id,
|
|
@@ -227,9 +360,52 @@ class Orchestrator {
|
|
|
227
360
|
),
|
|
228
361
|
);
|
|
229
362
|
}
|
|
363
|
+
if (!browserWasEnabled && channel.browserTesting) {
|
|
364
|
+
void this.enableBrowserTesting(channel);
|
|
365
|
+
} else if (browserWasEnabled && !channel.browserTesting) {
|
|
366
|
+
if (channel.provisionRetryFingerprint) {
|
|
367
|
+
// Browser readiness is no longer required, but a failed extra-account
|
|
368
|
+
// copy still needs its bounded retry.
|
|
369
|
+
this.scheduleBrowserRetry(channel);
|
|
370
|
+
} else {
|
|
371
|
+
this.clearBrowserRetry(channel);
|
|
372
|
+
}
|
|
373
|
+
// Config files copied while browser readiness was required no longer
|
|
374
|
+
// need to wait for a download. They still need a safe process reload.
|
|
375
|
+
for (const agentId of channel.browserPendingStaleSessions) {
|
|
376
|
+
channel.browserStaleSessions.add(agentId);
|
|
377
|
+
}
|
|
378
|
+
channel.browserPendingStaleSessions.clear();
|
|
379
|
+
void this.reconcileIfStarted(channel).catch((err: unknown) => {
|
|
380
|
+
console.warn(
|
|
381
|
+
`[orchestrator] ${channel.taskId}: browser-disable reconcile failed: ${
|
|
382
|
+
err instanceof Error ? err.message : String(err)
|
|
383
|
+
}`,
|
|
384
|
+
);
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/** Materialize a browser flag enabled after the channel already started. */
|
|
390
|
+
private async enableBrowserTesting(channel: Channel): Promise<void> {
|
|
391
|
+
try {
|
|
392
|
+
await this.ensureBrowserSetup(
|
|
393
|
+
channel,
|
|
394
|
+
channel.roster.some((agent) => agent.kind === "codex"),
|
|
395
|
+
Boolean(channel.provisionRetryFingerprint),
|
|
396
|
+
);
|
|
397
|
+
await this.reconcileIfStarted(channel);
|
|
398
|
+
} catch (err) {
|
|
399
|
+
console.warn(
|
|
400
|
+
`[browser] task ${channel.taskId}: enable failed: ${
|
|
401
|
+
err instanceof Error ? err.message : String(err)
|
|
402
|
+
}`,
|
|
403
|
+
);
|
|
404
|
+
}
|
|
230
405
|
}
|
|
231
406
|
|
|
232
407
|
private async getOrCreateChannel(taskId: string): Promise<Channel | null> {
|
|
408
|
+
if (this.blockedTasks.has(taskId)) return null;
|
|
233
409
|
const existing = this.channels.get(taskId);
|
|
234
410
|
if (existing) return existing;
|
|
235
411
|
|
|
@@ -275,7 +451,20 @@ class Orchestrator {
|
|
|
275
451
|
preambles,
|
|
276
452
|
firstTurns,
|
|
277
453
|
sessionsReady: null,
|
|
454
|
+
browserReady: false,
|
|
455
|
+
browserSetup: null,
|
|
456
|
+
browserSetupCodexHomes: null,
|
|
457
|
+
browserSetupEngineKinds: null,
|
|
458
|
+
browserSetupProvisionFingerprint: null,
|
|
459
|
+
browserSetupCoverage: null,
|
|
460
|
+
browserRetryTimer: null,
|
|
461
|
+
browserStaleSessions: new Set(),
|
|
462
|
+
browserPendingStaleSessions: new Set(),
|
|
463
|
+
reconcileReady: null,
|
|
464
|
+
reconcileAgain: false,
|
|
465
|
+
closed: false,
|
|
278
466
|
openTurns: new Set(),
|
|
467
|
+
activeTurns: new Map(),
|
|
279
468
|
interrupted: new Set(),
|
|
280
469
|
respawns: new Map(),
|
|
281
470
|
respawnLastAt: new Map(),
|
|
@@ -283,7 +472,13 @@ class Orchestrator {
|
|
|
283
472
|
browserTesting: spec.browserTesting === true,
|
|
284
473
|
sharedFiles: spec.sharedFiles ?? "ro",
|
|
285
474
|
mcpConnections: spec.mcpConnections ?? [],
|
|
475
|
+
mcpConfigWrite: null,
|
|
476
|
+
mcpConfigDirtyEpoch: 1,
|
|
477
|
+
mcpConfigAppliedEpoch: 0,
|
|
478
|
+
configGeneration: 0,
|
|
286
479
|
spawning: new Set(),
|
|
480
|
+
factoryOperations: new Set(),
|
|
481
|
+
eventOperations: new Set(),
|
|
287
482
|
accountByAgent: new Map(),
|
|
288
483
|
lastPrompt: new Map(),
|
|
289
484
|
rotations: new Map(),
|
|
@@ -304,6 +499,11 @@ class Orchestrator {
|
|
|
304
499
|
* Returns whether sessions are ready.
|
|
305
500
|
*/
|
|
306
501
|
private async ensureSessions(channel: Channel): Promise<boolean> {
|
|
502
|
+
// Boot recovery may be restarting this exact container and copying the
|
|
503
|
+
// full Codex directory. Wait only for this task's recovery barrier so that
|
|
504
|
+
// copy cannot land after browser/MCP materialization and erase config.
|
|
505
|
+
await taskRecoveryComplete(channel.taskId);
|
|
506
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
307
507
|
// Memoized: every caller awaits the SAME in-flight start, so a concurrent
|
|
308
508
|
// deliver() can't observe "ready" while the sessions map is still empty
|
|
309
509
|
// (startSessions awaits docker work before populating it — the old boolean
|
|
@@ -314,7 +514,7 @@ class Orchestrator {
|
|
|
314
514
|
// next ensure retries (e.g. the task wasn't `running` yet).
|
|
315
515
|
channel.sessionsReady.then(
|
|
316
516
|
(ok) => {
|
|
317
|
-
if (!ok) channel.sessionsReady = null;
|
|
517
|
+
if (!ok && this.isActiveChannel(channel)) channel.sessionsReady = null;
|
|
318
518
|
},
|
|
319
519
|
(err: unknown) => {
|
|
320
520
|
// A start failure retries on the next ensure — but it must be
|
|
@@ -324,58 +524,536 @@ class Orchestrator {
|
|
|
324
524
|
`[orchestrator] ${channel.taskId}: session start failed: ` +
|
|
325
525
|
`${err instanceof Error ? (err.stack ?? err.message) : String(err)}`,
|
|
326
526
|
);
|
|
327
|
-
channel.sessionsReady = null;
|
|
527
|
+
if (this.isActiveChannel(channel)) channel.sessionsReady = null;
|
|
328
528
|
},
|
|
329
529
|
);
|
|
330
530
|
}
|
|
331
531
|
const ready = await channel.sessionsReady;
|
|
532
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
533
|
+
// A live browser-enable or retry starts outside this ensure path. Join its
|
|
534
|
+
// serialized setup before reconciliation/delivery so the next message
|
|
535
|
+
// cannot reach a session that still holds the old MCP definition.
|
|
536
|
+
if (ready && channel.browserSetup) {
|
|
537
|
+
await this.ensureBrowserSetup(
|
|
538
|
+
channel,
|
|
539
|
+
channel.roster.some((agent) => agent.kind === "codex"),
|
|
540
|
+
);
|
|
541
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
542
|
+
}
|
|
332
543
|
// ADR-049 reconcile pass: spawn any roster agent that has no live session
|
|
333
544
|
// yet — the initial batch, host-restart recovery, and mid-task adds all
|
|
334
545
|
// converge here. No-op when every roster agent has a session.
|
|
335
546
|
if (ready) {
|
|
336
|
-
await this.reconcileSessions(channel);
|
|
337
547
|
// ADR-057: keep the container's MCP configs current so a connection
|
|
338
|
-
// added mid-task lands
|
|
548
|
+
// or engine added mid-task lands before its missing session spawns.
|
|
339
549
|
await this.ensureMcpConfig(channel);
|
|
550
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
551
|
+
await this.reconcileSessions(channel);
|
|
340
552
|
}
|
|
341
553
|
return ready;
|
|
342
554
|
}
|
|
343
555
|
|
|
556
|
+
/**
|
|
557
|
+
* Background browser work may recycle sessions only after the memoized
|
|
558
|
+
* initial start has finished. Waiting here prevents live enable/retry from
|
|
559
|
+
* reconciling the same missing agents while startSessions is creating them.
|
|
560
|
+
*/
|
|
561
|
+
private async reconcileIfStarted(channel: Channel): Promise<void> {
|
|
562
|
+
const starting = channel.sessionsReady;
|
|
563
|
+
if (!starting) return;
|
|
564
|
+
const ready = await starting;
|
|
565
|
+
if (ready && this.isActiveChannel(channel)) {
|
|
566
|
+
await this.reconcileSessions(channel);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/** Is this exact channel generation still the task's live channel? */
|
|
571
|
+
private isActiveChannel(channel: Channel): boolean {
|
|
572
|
+
return !channel.closed && this.channels.get(channel.taskId) === channel;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* The single account/browser-materialization gate for a channel.
|
|
577
|
+
*
|
|
578
|
+
* Initial start, a missing-session reconcile, live enable, rotation, and the
|
|
579
|
+
* failure retry all join the same promise. Even browser-disabled account
|
|
580
|
+
* copies use this gate: a false→true toggle queues migration after the older
|
|
581
|
+
* copy, so that copy cannot land last and erase the managed browser block.
|
|
582
|
+
*/
|
|
583
|
+
private codexHomes(hasCodex: boolean): string[] {
|
|
584
|
+
if (!hasCodex) return [];
|
|
585
|
+
const homes = resolveEngineAccounts("codex")
|
|
586
|
+
.map((account) => account.configDir?.containerDir)
|
|
587
|
+
.filter((home): home is string => Boolean(home));
|
|
588
|
+
return [...new Set([DEFAULT_CODEX_HOME, ...homes])];
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
private noteProvisionMutation(
|
|
592
|
+
channel: Channel,
|
|
593
|
+
waitForBrowserReadiness: boolean,
|
|
594
|
+
): void {
|
|
595
|
+
if (!this.isActiveChannel(channel)) return;
|
|
596
|
+
channel.configGeneration += 1;
|
|
597
|
+
channel.mcpConfigDirtyEpoch += 1;
|
|
598
|
+
for (const agentId of channel.sessions.keys()) {
|
|
599
|
+
if (waitForBrowserReadiness) {
|
|
600
|
+
channel.browserPendingStaleSessions.add(agentId);
|
|
601
|
+
} else {
|
|
602
|
+
channel.browserStaleSessions.add(agentId);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
private fenceConfigGeneration(channel: Channel): void {
|
|
608
|
+
if (this.isActiveChannel(channel)) channel.configGeneration += 1;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
private provisionFingerprint(kinds: readonly string[]): string {
|
|
612
|
+
const accounts = kinds.flatMap((kind) =>
|
|
613
|
+
resolveEngineAccounts(kind)
|
|
614
|
+
.filter((account) => !account.isDefault && account.configDir)
|
|
615
|
+
.map((account) => ({
|
|
616
|
+
kind,
|
|
617
|
+
id: account.id,
|
|
618
|
+
hostDir: account.configDir!.hostDir,
|
|
619
|
+
containerDir: account.configDir!.containerDir,
|
|
620
|
+
})),
|
|
621
|
+
);
|
|
622
|
+
accounts.sort((left, right) =>
|
|
623
|
+
JSON.stringify(left).localeCompare(JSON.stringify(right)),
|
|
624
|
+
);
|
|
625
|
+
return JSON.stringify({
|
|
626
|
+
kinds: [...new Set(kinds)].sort(),
|
|
627
|
+
accounts,
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
private async ensureBrowserSetup(
|
|
632
|
+
channel: Channel,
|
|
633
|
+
hasCodex: boolean,
|
|
634
|
+
forceProvision = false,
|
|
635
|
+
): Promise<BrowserSetup> {
|
|
636
|
+
const unavailable: BrowserSetup = {
|
|
637
|
+
ready: false,
|
|
638
|
+
configured: false,
|
|
639
|
+
changed: false,
|
|
640
|
+
};
|
|
641
|
+
if (!this.isActiveChannel(channel)) return unavailable;
|
|
642
|
+
const requestedBrowser = channel.browserTesting;
|
|
643
|
+
const requestedHasCodex =
|
|
644
|
+
hasCodex || channel.roster.some((agent) => agent.kind === "codex");
|
|
645
|
+
const requestedHomes = requestedBrowser
|
|
646
|
+
? this.codexHomes(requestedHasCodex)
|
|
647
|
+
: [];
|
|
648
|
+
const requestedKinds = [
|
|
649
|
+
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
650
|
+
];
|
|
651
|
+
const requestedProvisionFingerprint =
|
|
652
|
+
this.provisionFingerprint(requestedKinds);
|
|
653
|
+
const mayProvisionConfig = requestedKinds.some((kind) =>
|
|
654
|
+
resolveEngineAccounts(kind).some(
|
|
655
|
+
(account) => !account.isDefault && Boolean(account.configDir),
|
|
656
|
+
),
|
|
657
|
+
);
|
|
658
|
+
|
|
659
|
+
const inFlight = channel.browserSetup;
|
|
660
|
+
if (inFlight) {
|
|
661
|
+
const coveredHomes = channel.browserSetupCodexHomes ?? [];
|
|
662
|
+
const coveredKinds = channel.browserSetupEngineKinds ?? [];
|
|
663
|
+
const coveredProvisionFingerprint =
|
|
664
|
+
channel.browserSetupProvisionFingerprint;
|
|
665
|
+
const coverage = channel.browserSetupCoverage;
|
|
666
|
+
const setup = await inFlight;
|
|
667
|
+
if (!this.isActiveChannel(channel)) return unavailable;
|
|
668
|
+
const latestBrowser = channel.browserTesting;
|
|
669
|
+
const latestHasCodex =
|
|
670
|
+
requestedHasCodex ||
|
|
671
|
+
channel.roster.some((agent) => agent.kind === "codex");
|
|
672
|
+
const latestHomes = latestBrowser
|
|
673
|
+
? this.codexHomes(latestHasCodex)
|
|
674
|
+
: [];
|
|
675
|
+
const latestKinds = [
|
|
676
|
+
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
677
|
+
];
|
|
678
|
+
const latestProvisionFingerprint =
|
|
679
|
+
this.provisionFingerprint(latestKinds);
|
|
680
|
+
if (
|
|
681
|
+
(latestBrowser &&
|
|
682
|
+
coverage?.browserAttempted !== true &&
|
|
683
|
+
coverage?.browserRequested === false) ||
|
|
684
|
+
(latestBrowser && coverage?.skippedBecauseDisabled === true) ||
|
|
685
|
+
latestHomes.some((home) => !coveredHomes.includes(home)) ||
|
|
686
|
+
latestKinds.some((kind) => !coveredKinds.includes(kind)) ||
|
|
687
|
+
latestProvisionFingerprint !== coveredProvisionFingerprint ||
|
|
688
|
+
(forceProvision && coverage?.provisionAttempted !== true)
|
|
689
|
+
) {
|
|
690
|
+
return this.ensureBrowserSetup(
|
|
691
|
+
channel,
|
|
692
|
+
latestHasCodex,
|
|
693
|
+
forceProvision,
|
|
694
|
+
);
|
|
695
|
+
}
|
|
696
|
+
return setup;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
const coverage = {
|
|
700
|
+
browserRequested: requestedBrowser,
|
|
701
|
+
browserAttempted: false,
|
|
702
|
+
skippedBecauseDisabled: false,
|
|
703
|
+
provisionAttempted: false,
|
|
704
|
+
};
|
|
705
|
+
const core = (async (): Promise<BrowserSetup> => {
|
|
706
|
+
let setup: BrowserSetup;
|
|
707
|
+
try {
|
|
708
|
+
// Account copies and MCP migration share one serialized generation.
|
|
709
|
+
// Otherwise a concurrent provision can overwrite config.toml after
|
|
710
|
+
// migration and immediately strand an extra-account Codex session.
|
|
711
|
+
const shouldProvision =
|
|
712
|
+
forceProvision ||
|
|
713
|
+
channel.provisionAttemptedFingerprint !==
|
|
714
|
+
requestedProvisionFingerprint;
|
|
715
|
+
if (shouldProvision) {
|
|
716
|
+
coverage.provisionAttempted = true;
|
|
717
|
+
if (mayProvisionConfig) {
|
|
718
|
+
// Fence a factory that started before the copy without declaring a
|
|
719
|
+
// disk mutation until the aggregate result says bytes may have
|
|
720
|
+
// landed. A total failure must not churn live peers.
|
|
721
|
+
this.fenceConfigGeneration(channel);
|
|
722
|
+
}
|
|
723
|
+
let copiedOrUnknown = false;
|
|
724
|
+
try {
|
|
725
|
+
const provisioned = await provisionEngineAccounts(
|
|
726
|
+
channel.containerName,
|
|
727
|
+
requestedKinds,
|
|
728
|
+
);
|
|
729
|
+
copiedOrUnknown =
|
|
730
|
+
provisioned.copied > 0 || provisioned.mayHaveMutated;
|
|
731
|
+
if (provisioned.failed > 0) {
|
|
732
|
+
channel.provisionRetryFingerprint =
|
|
733
|
+
requestedProvisionFingerprint;
|
|
734
|
+
} else {
|
|
735
|
+
channel.provisionRetryFingerprint = undefined;
|
|
736
|
+
}
|
|
737
|
+
} catch (err) {
|
|
738
|
+
// A thrown multi-command copy may have landed before failing.
|
|
739
|
+
copiedOrUnknown = true;
|
|
740
|
+
channel.provisionRetryFingerprint =
|
|
741
|
+
requestedProvisionFingerprint;
|
|
742
|
+
throw err;
|
|
743
|
+
} finally {
|
|
744
|
+
// Cache the ATTEMPT even when incomplete. Immediate reconcile
|
|
745
|
+
// loops may replace missing agents, but only the bounded retry
|
|
746
|
+
// timer may repeat a failed account copy.
|
|
747
|
+
channel.provisionAttemptedFingerprint =
|
|
748
|
+
requestedProvisionFingerprint;
|
|
749
|
+
if (mayProvisionConfig) {
|
|
750
|
+
if (copiedOrUnknown) {
|
|
751
|
+
// A process that starts during a real/partial copy observes
|
|
752
|
+
// this second bump, while gateway config is reasserted.
|
|
753
|
+
this.noteProvisionMutation(
|
|
754
|
+
channel,
|
|
755
|
+
channel.browserTesting,
|
|
756
|
+
);
|
|
757
|
+
} else {
|
|
758
|
+
this.fenceConfigGeneration(channel);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
if (
|
|
764
|
+
!this.isActiveChannel(channel) ||
|
|
765
|
+
!requestedBrowser ||
|
|
766
|
+
!channel.browserTesting
|
|
767
|
+
) {
|
|
768
|
+
if (
|
|
769
|
+
this.isActiveChannel(channel) &&
|
|
770
|
+
channel.provisionRetryFingerprint
|
|
771
|
+
) {
|
|
772
|
+
this.scheduleBrowserRetry(channel);
|
|
773
|
+
}
|
|
774
|
+
coverage.skippedBecauseDisabled =
|
|
775
|
+
this.isActiveChannel(channel) &&
|
|
776
|
+
(!requestedBrowser || !channel.browserTesting);
|
|
777
|
+
return unavailable;
|
|
778
|
+
}
|
|
779
|
+
coverage.browserAttempted = true;
|
|
780
|
+
setup = await setupBrowserTesting(
|
|
781
|
+
channel.taskId,
|
|
782
|
+
channel.containerName,
|
|
783
|
+
requestedHasCodex,
|
|
784
|
+
requestedHomes,
|
|
785
|
+
);
|
|
786
|
+
} catch (err) {
|
|
787
|
+
console.warn(
|
|
788
|
+
`[browser] task ${channel.taskId}: setup failed: ${
|
|
789
|
+
err instanceof Error ? err.message : String(err)
|
|
790
|
+
}`,
|
|
791
|
+
);
|
|
792
|
+
setup = unavailable;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
if (!this.isActiveChannel(channel)) return unavailable;
|
|
796
|
+
|
|
797
|
+
channel.browserReady = setup.ready && setup.configured;
|
|
798
|
+
if (setup.changed) {
|
|
799
|
+
channel.configGeneration += 1;
|
|
800
|
+
for (const agentId of channel.sessions.keys()) {
|
|
801
|
+
channel.browserPendingStaleSessions.add(agentId);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
if (channel.browserReady) {
|
|
805
|
+
for (const agentId of channel.browserPendingStaleSessions) {
|
|
806
|
+
channel.browserStaleSessions.add(agentId);
|
|
807
|
+
}
|
|
808
|
+
channel.browserPendingStaleSessions.clear();
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
if (channel.browserReady && !channel.provisionRetryFingerprint) {
|
|
812
|
+
this.clearBrowserRetry(channel);
|
|
813
|
+
console.log(`[browser] task ${channel.taskId}: browser ready`);
|
|
814
|
+
} else {
|
|
815
|
+
this.scheduleBrowserRetry(channel);
|
|
816
|
+
}
|
|
817
|
+
return setup;
|
|
818
|
+
})();
|
|
819
|
+
|
|
820
|
+
let run: Promise<BrowserSetup>;
|
|
821
|
+
run = core.finally(() => {
|
|
822
|
+
if (channel.browserSetup === run) {
|
|
823
|
+
channel.browserSetup = null;
|
|
824
|
+
channel.browserSetupCodexHomes = null;
|
|
825
|
+
channel.browserSetupEngineKinds = null;
|
|
826
|
+
channel.browserSetupProvisionFingerprint = null;
|
|
827
|
+
channel.browserSetupCoverage = null;
|
|
828
|
+
}
|
|
829
|
+
});
|
|
830
|
+
channel.browserSetup = run;
|
|
831
|
+
channel.browserSetupCodexHomes = requestedHomes;
|
|
832
|
+
channel.browserSetupEngineKinds = requestedKinds;
|
|
833
|
+
channel.browserSetupProvisionFingerprint = requestedProvisionFingerprint;
|
|
834
|
+
channel.browserSetupCoverage = coverage;
|
|
835
|
+
const setup = await run;
|
|
836
|
+
if (!this.isActiveChannel(channel)) return unavailable;
|
|
837
|
+
const latestBrowser = channel.browserTesting;
|
|
838
|
+
const latestHasCodex =
|
|
839
|
+
requestedHasCodex ||
|
|
840
|
+
channel.roster.some((agent) => agent.kind === "codex");
|
|
841
|
+
const latestHomes = latestBrowser
|
|
842
|
+
? this.codexHomes(latestHasCodex)
|
|
843
|
+
: [];
|
|
844
|
+
const latestKinds = [
|
|
845
|
+
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
846
|
+
];
|
|
847
|
+
const latestProvisionFingerprint = this.provisionFingerprint(latestKinds);
|
|
848
|
+
if (
|
|
849
|
+
(latestBrowser &&
|
|
850
|
+
!coverage.browserAttempted &&
|
|
851
|
+
!coverage.browserRequested) ||
|
|
852
|
+
(latestBrowser && coverage.skippedBecauseDisabled) ||
|
|
853
|
+
latestHomes.some((home) => !requestedHomes.includes(home)) ||
|
|
854
|
+
latestKinds.some((kind) => !requestedKinds.includes(kind)) ||
|
|
855
|
+
latestProvisionFingerprint !== requestedProvisionFingerprint ||
|
|
856
|
+
(forceProvision && !coverage.provisionAttempted)
|
|
857
|
+
) {
|
|
858
|
+
return this.ensureBrowserSetup(channel, latestHasCodex, forceProvision);
|
|
859
|
+
}
|
|
860
|
+
return setup;
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
private clearBrowserRetry(channel: Channel): void {
|
|
864
|
+
if (channel.browserRetryTimer) {
|
|
865
|
+
clearTimeout(channel.browserRetryTimer);
|
|
866
|
+
channel.browserRetryTimer = null;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/** A failed setup gets a real retry even if no message/ensure arrives. */
|
|
871
|
+
private scheduleBrowserRetry(channel: Channel): void {
|
|
872
|
+
if (
|
|
873
|
+
(!channel.browserTesting && !channel.provisionRetryFingerprint) ||
|
|
874
|
+
!this.isActiveChannel(channel) ||
|
|
875
|
+
channel.browserRetryTimer
|
|
876
|
+
) {
|
|
877
|
+
return;
|
|
878
|
+
}
|
|
879
|
+
const timer = setTimeout(() => {
|
|
880
|
+
if (channel.browserRetryTimer === timer) {
|
|
881
|
+
channel.browserRetryTimer = null;
|
|
882
|
+
}
|
|
883
|
+
if (!this.isActiveChannel(channel)) return;
|
|
884
|
+
const forceProvision = Boolean(channel.provisionRetryFingerprint);
|
|
885
|
+
void this.ensureBrowserSetup(
|
|
886
|
+
channel,
|
|
887
|
+
channel.roster.some((a) => a.kind === "codex"),
|
|
888
|
+
forceProvision,
|
|
889
|
+
).then(() => {
|
|
890
|
+
if (this.isActiveChannel(channel)) return this.reconcileIfStarted(channel);
|
|
891
|
+
}).catch((err: unknown) => {
|
|
892
|
+
console.warn(
|
|
893
|
+
`[browser] task ${channel.taskId}: retry reconcile failed: ${
|
|
894
|
+
err instanceof Error ? err.message : String(err)
|
|
895
|
+
}`,
|
|
896
|
+
);
|
|
897
|
+
});
|
|
898
|
+
}, BROWSER_RETRY_COOLDOWN_MS);
|
|
899
|
+
channel.browserRetryTimer = timer;
|
|
900
|
+
}
|
|
901
|
+
|
|
344
902
|
/** Write the task's MCP configs when the connection set changed. */
|
|
345
903
|
private async ensureMcpConfig(channel: Channel): Promise<void> {
|
|
346
|
-
|
|
347
|
-
|
|
904
|
+
if (!this.isActiveChannel(channel)) return;
|
|
905
|
+
const inFlight = channel.mcpConfigWrite;
|
|
906
|
+
if (inFlight) {
|
|
907
|
+
await inFlight;
|
|
908
|
+
// Recompute after the serialized writer: the channel spec may have
|
|
909
|
+
// changed while we waited, in which case this caller owns the next pass.
|
|
910
|
+
if (this.isActiveChannel(channel)) await this.ensureMcpConfig(channel);
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
let run: Promise<void>;
|
|
915
|
+
run = this.writeMcpConfigPass(channel).finally(() => {
|
|
916
|
+
if (channel.mcpConfigWrite === run) channel.mcpConfigWrite = null;
|
|
917
|
+
});
|
|
918
|
+
channel.mcpConfigWrite = run;
|
|
919
|
+
await run;
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
private async writeMcpConfigPass(channel: Channel): Promise<void> {
|
|
923
|
+
if (!this.isActiveChannel(channel)) return;
|
|
924
|
+
const connections = channel.mcpConnections.map((connection) => ({
|
|
925
|
+
id: connection.id,
|
|
926
|
+
slug: connection.slug,
|
|
927
|
+
}));
|
|
928
|
+
const engineKinds = [
|
|
929
|
+
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
930
|
+
].sort();
|
|
931
|
+
const codexHomes = engineKinds.includes("codex")
|
|
932
|
+
? this.codexHomes(true)
|
|
933
|
+
: [];
|
|
934
|
+
const fingerprint = JSON.stringify({
|
|
935
|
+
connections,
|
|
936
|
+
engineKinds,
|
|
937
|
+
codexHomes,
|
|
938
|
+
});
|
|
939
|
+
const dirtyEpoch = channel.mcpConfigDirtyEpoch;
|
|
940
|
+
const repairingProvisionedConfig =
|
|
941
|
+
channel.mcpConfigAppliedEpoch !== dirtyEpoch &&
|
|
942
|
+
channel.mcpConfigFingerprint === fingerprint;
|
|
943
|
+
if (
|
|
944
|
+
channel.mcpConfigFingerprint === fingerprint &&
|
|
945
|
+
channel.mcpConfigAppliedEpoch === dirtyEpoch
|
|
946
|
+
) {
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
348
949
|
const hadPrevious = channel.mcpConfigFingerprint !== undefined;
|
|
349
|
-
|
|
350
|
-
|
|
950
|
+
let previous:
|
|
951
|
+
| {
|
|
952
|
+
connections: Array<{ id: string; slug: string }>;
|
|
953
|
+
engineKinds: string[];
|
|
954
|
+
codexHomes?: string[];
|
|
955
|
+
}
|
|
956
|
+
| undefined;
|
|
957
|
+
try {
|
|
958
|
+
previous = channel.mcpConfigFingerprint
|
|
959
|
+
? (JSON.parse(channel.mcpConfigFingerprint) as typeof previous)
|
|
960
|
+
: undefined;
|
|
961
|
+
} catch {
|
|
962
|
+
previous = undefined;
|
|
963
|
+
}
|
|
964
|
+
const connectionsChanged =
|
|
965
|
+
JSON.stringify(previous?.connections) !== JSON.stringify(connections);
|
|
966
|
+
const addedKinds = new Set(
|
|
967
|
+
engineKinds.filter((kind) => !previous?.engineKinds?.includes(kind)),
|
|
968
|
+
);
|
|
969
|
+
const hadLiveSessions = channel.sessions.size > 0;
|
|
970
|
+
const configured = await setupMcpTaskConfig(
|
|
351
971
|
channel.taskId,
|
|
352
972
|
channel.containerName,
|
|
353
|
-
|
|
354
|
-
|
|
973
|
+
connections,
|
|
974
|
+
engineKinds,
|
|
975
|
+
codexHomes,
|
|
355
976
|
);
|
|
977
|
+
if (!configured || !this.isActiveChannel(channel)) return;
|
|
978
|
+
channel.configGeneration += 1;
|
|
979
|
+
channel.mcpConfigFingerprint = fingerprint;
|
|
980
|
+
// Only acknowledge the epoch this write observed. If provisioning dirtied
|
|
981
|
+
// the files while docker exec was in flight, the newer epoch remains
|
|
982
|
+
// uncovered and the serialized ensure immediately publishes another pass.
|
|
983
|
+
channel.mcpConfigAppliedEpoch = dirtyEpoch;
|
|
356
984
|
// Agent CLIs read MCP servers once, at process start — and durable
|
|
357
985
|
// sessions (ADR-061) make processes long-lived, so without this a
|
|
358
986
|
// connection added mid-task stays invisible indefinitely. Recycle IDLE
|
|
359
987
|
// sessions so the change actually reaches the agents; busy agents keep
|
|
360
|
-
// their turn (and in-memory context)
|
|
361
|
-
//
|
|
988
|
+
// their turn (and in-memory context), remain marked stale, and recycle at
|
|
989
|
+
// turn_complete. Trade-off: a recycled agent loses its in-memory
|
|
362
990
|
// context and is re-briefed from chat — same as any respawn, and better
|
|
363
991
|
// than never seeing the connection the user just added. Skipped on the
|
|
364
|
-
// channel's first fingerprint
|
|
365
|
-
|
|
992
|
+
// channel's first successful fingerprint unless sessions already spawned
|
|
993
|
+
// after a failed write and therefore need to reload the recovered config.
|
|
994
|
+
// A kind-only change is written before missing-agent reconciliation, so
|
|
995
|
+
// existing engines keep the same files; only an already-live session of
|
|
996
|
+
// the newly-added kind could be stale.
|
|
997
|
+
if (repairingProvisionedConfig) {
|
|
998
|
+
// Provisioning marked the sessions that existed around its copy. Mark
|
|
999
|
+
// again after the repair so a factory that bound during this write is
|
|
1000
|
+
// covered too (Set de-duplicates the earlier marks).
|
|
1001
|
+
this.markSessionsStale(channel);
|
|
1002
|
+
} else if (!hadPrevious && hadLiveSessions) {
|
|
1003
|
+
this.markSessionsStale(channel);
|
|
1004
|
+
} else if (hadPrevious && connectionsChanged) {
|
|
1005
|
+
this.markSessionsStale(channel);
|
|
1006
|
+
} else if (hadPrevious && addedKinds.size > 0) {
|
|
1007
|
+
const affected = channel.roster
|
|
1008
|
+
.filter((agent) => addedKinds.has(agent.kind))
|
|
1009
|
+
.map((agent) => agent.id);
|
|
1010
|
+
this.markSessionsStale(channel, affected);
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/**
|
|
1015
|
+
* Mark current sessions stale after an MCP config change. Publication stays
|
|
1016
|
+
* pure: callers reconcile only after the writer promise has cleared, avoiding
|
|
1017
|
+
* writer -> reconcile -> same-writer self-deadlock. Busy sessions remain
|
|
1018
|
+
* marked until their turn_complete event.
|
|
1019
|
+
*/
|
|
1020
|
+
private markSessionsStale(
|
|
1021
|
+
channel: Channel,
|
|
1022
|
+
agentIds: Iterable<string> = channel.sessions.keys(),
|
|
1023
|
+
): void {
|
|
1024
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1025
|
+
for (const agentId of agentIds) {
|
|
1026
|
+
if (channel.sessions.has(agentId)) {
|
|
1027
|
+
channel.browserStaleSessions.add(agentId);
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
366
1030
|
}
|
|
367
1031
|
|
|
368
|
-
/** Close
|
|
369
|
-
private async
|
|
1032
|
+
/** Close stale sessions that are idle, without recursively reconciling. */
|
|
1033
|
+
private async closeIdleStaleSessions(channel: Channel): Promise<number> {
|
|
370
1034
|
let recycled = 0;
|
|
371
|
-
for (const
|
|
372
|
-
if (
|
|
373
|
-
if (
|
|
1035
|
+
for (const agentId of [...channel.browserStaleSessions]) {
|
|
1036
|
+
if (!this.isActiveChannel(channel)) return recycled;
|
|
1037
|
+
if (
|
|
1038
|
+
channel.activeTurns.has(agentId) ||
|
|
1039
|
+
channel.openTurns.has(agentId) ||
|
|
1040
|
+
channel.spawning.has(agentId)
|
|
1041
|
+
) {
|
|
1042
|
+
continue;
|
|
1043
|
+
}
|
|
1044
|
+
const session = channel.sessions.get(agentId);
|
|
1045
|
+
// A session that exited on its own is already absent. Its eventual
|
|
1046
|
+
// replacement is fresh, so the old generation is no longer stale.
|
|
1047
|
+
if (!session) {
|
|
1048
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
channel.browserStaleSessions.delete(agentId);
|
|
374
1052
|
channel.sessions.delete(agentId);
|
|
375
1053
|
try {
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
//
|
|
1054
|
+
// Delete before close. Event handlers are identity-guarded, so the
|
|
1055
|
+
// old session's synthetic exit cannot consume respawn budget or delete
|
|
1056
|
+
// the replacement that follows.
|
|
379
1057
|
await session.close();
|
|
380
1058
|
} catch {
|
|
381
1059
|
// Already dead — close is idempotent for our adapters.
|
|
@@ -386,82 +1064,124 @@ class Orchestrator {
|
|
|
386
1064
|
console.log(
|
|
387
1065
|
`[orchestrator] ${channel.taskId}: recycled ${recycled} idle session(s) for MCP config change`,
|
|
388
1066
|
);
|
|
389
|
-
await this.reconcileSessions(channel);
|
|
390
1067
|
}
|
|
1068
|
+
return recycled;
|
|
391
1069
|
}
|
|
392
1070
|
|
|
393
|
-
/**
|
|
1071
|
+
/** Serialize all missing-session and stale-session reconciliation. */
|
|
394
1072
|
private async reconcileSessions(channel: Channel): Promise<void> {
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
// respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
|
|
401
|
-
// but a budget that has been cold for RESPAWN_COOLDOWN_MS resets, so
|
|
402
|
-
// the task heals itself once the cause (a missing CLI on the shared
|
|
403
|
-
// volume, a broken config) is fixed, instead of staying dead until a
|
|
404
|
-
// host restart.
|
|
405
|
-
if ((channel.respawns.get(agent.id) ?? 0) > MAX_RESPAWNS_PER_AGENT) {
|
|
406
|
-
const lastAt = channel.respawnLastAt.get(agent.id) ?? 0;
|
|
407
|
-
if (Date.now() - lastAt < RESPAWN_COOLDOWN_MS) return false;
|
|
408
|
-
channel.respawns.set(agent.id, 0);
|
|
409
|
-
}
|
|
410
|
-
return true;
|
|
411
|
-
});
|
|
412
|
-
if (missing.length === 0) return;
|
|
1073
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1074
|
+
if (channel.reconcileReady) {
|
|
1075
|
+
channel.reconcileAgain = true;
|
|
1076
|
+
return channel.reconcileReady;
|
|
1077
|
+
}
|
|
413
1078
|
|
|
414
|
-
const
|
|
415
|
-
|
|
1079
|
+
const core = (async () => {
|
|
1080
|
+
do {
|
|
1081
|
+
channel.reconcileAgain = false;
|
|
1082
|
+
await this.reconcileSessionsInner(channel);
|
|
1083
|
+
} while (channel.reconcileAgain && this.isActiveChannel(channel));
|
|
1084
|
+
})();
|
|
1085
|
+
let run: Promise<void>;
|
|
1086
|
+
run = core.finally(() => {
|
|
1087
|
+
if (channel.reconcileReady === run) channel.reconcileReady = null;
|
|
1088
|
+
});
|
|
1089
|
+
channel.reconcileReady = run;
|
|
1090
|
+
return run;
|
|
1091
|
+
}
|
|
416
1092
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
1093
|
+
private async reconcileSessionsInner(channel: Channel): Promise<void> {
|
|
1094
|
+
while (this.isActiveChannel(channel)) {
|
|
1095
|
+
await this.closeIdleStaleSessions(channel);
|
|
1096
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1097
|
+
|
|
1098
|
+
// Snapshot before any slow work. An agent added while this generation is
|
|
1099
|
+
// setting up is handled by the next loop, with a setup pass that includes
|
|
1100
|
+
// its engine (especially the Claude-only -> Codex transition).
|
|
1101
|
+
const roster = [...channel.roster];
|
|
1102
|
+
const missing = roster.filter((agent) => {
|
|
1103
|
+
if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
|
|
1104
|
+
return false;
|
|
1105
|
+
}
|
|
1106
|
+
// Crash-loop budget: an agent whose session keeps dying stops being
|
|
1107
|
+
// respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
|
|
1108
|
+
// but a cold budget resets so the task can heal after the cause clears.
|
|
1109
|
+
if ((channel.respawns.get(agent.id) ?? 0) > MAX_RESPAWNS_PER_AGENT) {
|
|
1110
|
+
const lastAt = channel.respawnLastAt.get(agent.id) ?? 0;
|
|
1111
|
+
if (Date.now() - lastAt < RESPAWN_COOLDOWN_MS) return false;
|
|
1112
|
+
channel.respawns.set(agent.id, 0);
|
|
1113
|
+
}
|
|
1114
|
+
return true;
|
|
1115
|
+
});
|
|
1116
|
+
if (missing.length === 0) return;
|
|
433
1117
|
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
channel.containerName,
|
|
437
|
-
missing.map((a) => a.kind),
|
|
438
|
-
);
|
|
1118
|
+
const task = getHostTask(channel.taskId);
|
|
1119
|
+
if (!task || task.statusMirror !== "running") return;
|
|
439
1120
|
|
|
440
|
-
for (const agent of missing)
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
1121
|
+
for (const agent of missing) channel.spawning.add(agent.id);
|
|
1122
|
+
try {
|
|
1123
|
+
// Same per-agent materialisation the initial start does. Browser setup
|
|
1124
|
+
// is awaited before EVERY missing-session spawn: that reasserts configs
|
|
1125
|
+
// clobbered by resume/auth injection and covers roster generation.
|
|
1126
|
+
await installPackageSkills(channel.taskId, missing);
|
|
1127
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1128
|
+
|
|
1129
|
+
// One materialization gate owns provisioning and (when enabled)
|
|
1130
|
+
// browser migration. This remains serialized across a live false→true
|
|
1131
|
+
// transition, so an older account copy cannot land after migration.
|
|
1132
|
+
await this.ensureBrowserSetup(
|
|
1133
|
+
channel,
|
|
1134
|
+
roster.some((a) => a.kind === "codex"),
|
|
1135
|
+
);
|
|
1136
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1137
|
+
// Provisioning may have refreshed an extra account's whole config
|
|
1138
|
+
// directory. Reassert gateway entries in every Codex home before any
|
|
1139
|
+
// process for this generation can start.
|
|
1140
|
+
await this.ensureMcpConfig(channel);
|
|
1141
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1142
|
+
// Browser and gateway definitions are both loaded only at process
|
|
1143
|
+
// start. A writer marks the previous generation stale; close its idle
|
|
1144
|
+
// subset now regardless of the browser feature flag.
|
|
1145
|
+
await this.closeIdleStaleSessions(channel);
|
|
1146
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1147
|
+
|
|
1148
|
+
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
1149
|
+
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
1150
|
+
writeAgentCli(channel.taskId, roster, apiUrl);
|
|
1151
|
+
|
|
1152
|
+
for (const agent of missing) {
|
|
1153
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1154
|
+
// Another recovery path may have filled this slot while the browser
|
|
1155
|
+
// setup was awaiting.
|
|
1156
|
+
if (channel.sessions.has(agent.id)) continue;
|
|
1157
|
+
const session = await this.createAndBindStableSession(
|
|
447
1158
|
channel,
|
|
448
|
-
agent,
|
|
449
|
-
|
|
450
|
-
channel.taskId,
|
|
1159
|
+
agent.id,
|
|
1160
|
+
{
|
|
1161
|
+
taskId: channel.taskId,
|
|
451
1162
|
agent,
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
1163
|
+
containerName: channel.containerName,
|
|
1164
|
+
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
1165
|
+
agentEnv: this.accountAgentEnv(
|
|
1166
|
+
channel,
|
|
1167
|
+
agent,
|
|
1168
|
+
agentCliEnv(
|
|
1169
|
+
channel.taskId,
|
|
1170
|
+
agent,
|
|
1171
|
+
task.ownerUserId,
|
|
1172
|
+
apiUrl,
|
|
1173
|
+
cliSecret,
|
|
1174
|
+
),
|
|
1175
|
+
),
|
|
1176
|
+
},
|
|
1177
|
+
);
|
|
1178
|
+
if (!session) return;
|
|
1179
|
+
}
|
|
1180
|
+
} finally {
|
|
1181
|
+
for (const agent of missing) channel.spawning.delete(agent.id);
|
|
462
1182
|
}
|
|
463
|
-
|
|
464
|
-
for
|
|
1183
|
+
// Loop once more: roster additions and sessions retired by a config
|
|
1184
|
+
// rewrite each get a setup pass for their exact roster generation.
|
|
465
1185
|
}
|
|
466
1186
|
}
|
|
467
1187
|
|
|
@@ -487,9 +1207,105 @@ class Orchestrator {
|
|
|
487
1207
|
return { ...base, ...account.execEnv };
|
|
488
1208
|
}
|
|
489
1209
|
|
|
1210
|
+
/** Install a session with an event handler scoped to this exact generation. */
|
|
1211
|
+
private bindSession(
|
|
1212
|
+
channel: Channel,
|
|
1213
|
+
agentId: string,
|
|
1214
|
+
session: AgentSession,
|
|
1215
|
+
): void {
|
|
1216
|
+
channel.sessions.set(agentId, session);
|
|
1217
|
+
session.onEvent((event) => {
|
|
1218
|
+
// Recycle/teardown deletes the old session before calling close().
|
|
1219
|
+
// Adapters deliberately emit a final exit event; without this identity
|
|
1220
|
+
// guard it can delete a replacement or consume its respawn budget.
|
|
1221
|
+
if (
|
|
1222
|
+
!this.isActiveChannel(channel) ||
|
|
1223
|
+
channel.sessions.get(agentId) !== session
|
|
1224
|
+
) {
|
|
1225
|
+
return;
|
|
1226
|
+
}
|
|
1227
|
+
const operation = this.handleAgentEvent(channel, agentId, event).catch(
|
|
1228
|
+
(err: unknown) => {
|
|
1229
|
+
console.warn(
|
|
1230
|
+
`[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
|
|
1231
|
+
err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
1232
|
+
}`,
|
|
1233
|
+
);
|
|
1234
|
+
},
|
|
1235
|
+
);
|
|
1236
|
+
channel.eventOperations.add(operation);
|
|
1237
|
+
operation.then(
|
|
1238
|
+
() => channel.eventOperations.delete(operation),
|
|
1239
|
+
() => channel.eventOperations.delete(operation),
|
|
1240
|
+
);
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
/**
|
|
1245
|
+
* Create against a stable managed-config generation. Agent processes load
|
|
1246
|
+
* MCP definitions during factory startup but are not yet present in the
|
|
1247
|
+
* sessions map, so a concurrent rewrite cannot mark them stale. Detect that
|
|
1248
|
+
* straddle explicitly, close the ambiguous process, and retry.
|
|
1249
|
+
*/
|
|
1250
|
+
private createAndBindStableSession(
|
|
1251
|
+
channel: Channel,
|
|
1252
|
+
agentId: string,
|
|
1253
|
+
args: Parameters<AgentSessionFactory["create"]>[0],
|
|
1254
|
+
): Promise<AgentSession | null> {
|
|
1255
|
+
const operation = this.createAndBindStableSessionCore(
|
|
1256
|
+
channel,
|
|
1257
|
+
agentId,
|
|
1258
|
+
args,
|
|
1259
|
+
);
|
|
1260
|
+
channel.factoryOperations.add(operation);
|
|
1261
|
+
operation.then(
|
|
1262
|
+
() => channel.factoryOperations.delete(operation),
|
|
1263
|
+
() => channel.factoryOperations.delete(operation),
|
|
1264
|
+
);
|
|
1265
|
+
return operation;
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
private async createAndBindStableSessionCore(
|
|
1269
|
+
channel: Channel,
|
|
1270
|
+
agentId: string,
|
|
1271
|
+
args: Parameters<AgentSessionFactory["create"]>[0],
|
|
1272
|
+
): Promise<AgentSession | null> {
|
|
1273
|
+
while (this.isActiveChannel(channel)) {
|
|
1274
|
+
const generation = channel.configGeneration;
|
|
1275
|
+
const session = await this.factory.create(args);
|
|
1276
|
+
if (!this.isActiveChannel(channel)) {
|
|
1277
|
+
await session.close().catch(() => {});
|
|
1278
|
+
return null;
|
|
1279
|
+
}
|
|
1280
|
+
if (generation === channel.configGeneration) {
|
|
1281
|
+
// Keep the final generation check, stale cleanup, and map bind in one
|
|
1282
|
+
// synchronous continuation. A config writer that resumes immediately
|
|
1283
|
+
// afterward now sees this session and marks it stale; it can no longer
|
|
1284
|
+
// change the generation in the await→bind gap while the process is
|
|
1285
|
+
// invisible.
|
|
1286
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1287
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
1288
|
+
this.bindSession(channel, agentId, session);
|
|
1289
|
+
return session;
|
|
1290
|
+
}
|
|
1291
|
+
await session.close().catch(() => {});
|
|
1292
|
+
}
|
|
1293
|
+
return null;
|
|
1294
|
+
}
|
|
1295
|
+
|
|
490
1296
|
private async startSessions(channel: Channel): Promise<boolean> {
|
|
491
1297
|
const task = getHostTask(channel.taskId);
|
|
492
|
-
if (
|
|
1298
|
+
if (
|
|
1299
|
+
!task ||
|
|
1300
|
+
task.statusMirror !== "running" ||
|
|
1301
|
+
!this.isActiveChannel(channel)
|
|
1302
|
+
) {
|
|
1303
|
+
return false;
|
|
1304
|
+
}
|
|
1305
|
+
// Freeze the generation before any slow docker work. A roster add while
|
|
1306
|
+
// setup awaits must reconcile under its own engine-aware browser pass,
|
|
1307
|
+
// not slip into this factory loop under the old generation's config.
|
|
1308
|
+
const initialRoster = [...channel.roster];
|
|
493
1309
|
|
|
494
1310
|
// Set the task creator's git author identity in the container (ADR-029).
|
|
495
1311
|
// The SSH key itself is installed earlier by task-up.sh (host clone +
|
|
@@ -497,6 +1313,7 @@ class Orchestrator {
|
|
|
497
1313
|
// async (docker exec via dockerCli) so it doesn't block the event loop —
|
|
498
1314
|
// this runs on every channel ensure, including each post-restart reconnect.
|
|
499
1315
|
await setupTaskGitIdentity(channel.taskId, task.ownerName, task.ownerEmail);
|
|
1316
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
500
1317
|
|
|
501
1318
|
// GitHub auth for `gh` (ADR-027): mint + inject the access token and
|
|
502
1319
|
// (re)start its refresh schedule. Done here — not only at task-up — so it
|
|
@@ -518,22 +1335,26 @@ class Orchestrator {
|
|
|
518
1335
|
// turn. Idempotent + best-effort (returns fast when there are none); a slow
|
|
519
1336
|
// clone/install briefly delays start, which is acceptable for skill-bearing
|
|
520
1337
|
// tasks. Never throws.
|
|
521
|
-
await installPackageSkills(channel.taskId,
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
//
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
1338
|
+
await installPackageSkills(channel.taskId, initialRoster);
|
|
1339
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1340
|
+
|
|
1341
|
+
// ADR-053: wire the Playwright MCP browser before agents spawn — configs
|
|
1342
|
+
// AND the Chromium install, awaited, because the only other place the
|
|
1343
|
+
// download could happen is inside the MCP server's startup handshake,
|
|
1344
|
+
// which the client caps at 10s (Codex) / 30s (Claude Code). Idempotent
|
|
1345
|
+
// and ~0.5s once the build is present; a failure arms its own 60s retry
|
|
1346
|
+
// rather than depending on another delivery/ensure to arrive.
|
|
1347
|
+
await this.ensureBrowserSetup(
|
|
1348
|
+
channel,
|
|
1349
|
+
initialRoster.some((a) => a.kind === "codex"),
|
|
1350
|
+
);
|
|
1351
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
532
1352
|
|
|
533
1353
|
// ADR-057: the owner's MCP connections, reached through the host gateway
|
|
534
1354
|
// (tokens never enter the container) — written BEFORE agents spawn so
|
|
535
1355
|
// first sessions load them.
|
|
536
1356
|
await this.ensureMcpConfig(channel);
|
|
1357
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
537
1358
|
|
|
538
1359
|
// ADR-048: write the in-container `uai` CLI (apiUrl only, no token) into the
|
|
539
1360
|
// workspace. Each agent's OWN task token — carrying only ITS permissions — is
|
|
@@ -543,30 +1364,40 @@ class Orchestrator {
|
|
|
543
1364
|
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
544
1365
|
writeAgentCli(channel.taskId, channel.roster, apiUrl);
|
|
545
1366
|
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
channel.
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
taskId: channel.taskId,
|
|
557
|
-
agent,
|
|
558
|
-
containerName: channel.containerName,
|
|
559
|
-
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
560
|
-
agentEnv: this.accountAgentEnv(
|
|
1367
|
+
for (const agent of initialRoster) {
|
|
1368
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1369
|
+
// A prior partial start or an independently guarded recovery may already
|
|
1370
|
+
// own this slot. Never replace it with an untracked second generation.
|
|
1371
|
+
if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
|
|
1372
|
+
continue;
|
|
1373
|
+
}
|
|
1374
|
+
channel.spawning.add(agent.id);
|
|
1375
|
+
try {
|
|
1376
|
+
const session = await this.createAndBindStableSession(
|
|
561
1377
|
channel,
|
|
562
|
-
agent,
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
1378
|
+
agent.id,
|
|
1379
|
+
{
|
|
1380
|
+
taskId: channel.taskId,
|
|
1381
|
+
agent,
|
|
1382
|
+
containerName: channel.containerName,
|
|
1383
|
+
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
1384
|
+
agentEnv: this.accountAgentEnv(
|
|
1385
|
+
channel,
|
|
1386
|
+
agent,
|
|
1387
|
+
agentCliEnv(
|
|
1388
|
+
channel.taskId,
|
|
1389
|
+
agent,
|
|
1390
|
+
task.ownerUserId,
|
|
1391
|
+
apiUrl,
|
|
1392
|
+
cliSecret,
|
|
1393
|
+
),
|
|
1394
|
+
),
|
|
1395
|
+
},
|
|
1396
|
+
);
|
|
1397
|
+
if (!session) return false;
|
|
1398
|
+
} finally {
|
|
1399
|
+
channel.spawning.delete(agent.id);
|
|
1400
|
+
}
|
|
570
1401
|
}
|
|
571
1402
|
|
|
572
1403
|
// First-turn delivery is owned by the CLOUD (channel-router.ensureStarted),
|
|
@@ -590,6 +1421,32 @@ class Orchestrator {
|
|
|
590
1421
|
if (channel) await this.ensureSessions(channel);
|
|
591
1422
|
}
|
|
592
1423
|
|
|
1424
|
+
private incrementActiveTurns(channel: Channel, agentId: string): void {
|
|
1425
|
+
channel.activeTurns.set(
|
|
1426
|
+
agentId,
|
|
1427
|
+
(channel.activeTurns.get(agentId) ?? 0) + 1,
|
|
1428
|
+
);
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
/** Session output proves at least one active turn, but deltas/tools from the
|
|
1432
|
+
* same turn must not inflate the delivery count. */
|
|
1433
|
+
private markActiveTurn(channel: Channel, agentId: string): void {
|
|
1434
|
+
if (!channel.activeTurns.has(agentId)) {
|
|
1435
|
+
channel.activeTurns.set(agentId, 1);
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
/** Consume one queued/active turn and report how many remain. */
|
|
1440
|
+
private decrementActiveTurns(channel: Channel, agentId: string): number {
|
|
1441
|
+
const remaining = Math.max(
|
|
1442
|
+
0,
|
|
1443
|
+
(channel.activeTurns.get(agentId) ?? 1) - 1,
|
|
1444
|
+
);
|
|
1445
|
+
if (remaining === 0) channel.activeTurns.delete(agentId);
|
|
1446
|
+
else channel.activeTurns.set(agentId, remaining);
|
|
1447
|
+
return remaining;
|
|
1448
|
+
}
|
|
1449
|
+
|
|
593
1450
|
// -- inbound: a human (or routed peer) message ----------------------------
|
|
594
1451
|
|
|
595
1452
|
/**
|
|
@@ -622,7 +1479,17 @@ class Orchestrator {
|
|
|
622
1479
|
// ADR-076: remember the in-flight prompt so an account rotation on a
|
|
623
1480
|
// rate-limit can re-deliver it to the fresh session.
|
|
624
1481
|
channel.lastPrompt.set(agentId, prompt);
|
|
625
|
-
|
|
1482
|
+
this.incrementActiveTurns(channel, agentId);
|
|
1483
|
+
void session.send(prompt).catch((err: unknown) => {
|
|
1484
|
+
if (channel.sessions.get(agentId) === session) {
|
|
1485
|
+
this.decrementActiveTurns(channel, agentId);
|
|
1486
|
+
}
|
|
1487
|
+
console.warn(
|
|
1488
|
+
`[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
|
|
1489
|
+
err instanceof Error ? err.message : String(err)
|
|
1490
|
+
}`,
|
|
1491
|
+
);
|
|
1492
|
+
});
|
|
626
1493
|
return { ok: true };
|
|
627
1494
|
}
|
|
628
1495
|
|
|
@@ -673,8 +1540,10 @@ class Orchestrator {
|
|
|
673
1540
|
agentId: string,
|
|
674
1541
|
event: AgentEvent,
|
|
675
1542
|
): Promise<void> {
|
|
1543
|
+
if (!this.isActiveChannel(channel)) return;
|
|
676
1544
|
switch (event.type) {
|
|
677
1545
|
case "message_delta": {
|
|
1546
|
+
this.markActiveTurn(channel, agentId);
|
|
678
1547
|
channel.openTurns.add(agentId);
|
|
679
1548
|
this.emitHost({
|
|
680
1549
|
kind: "agent.message_delta",
|
|
@@ -685,6 +1554,7 @@ class Orchestrator {
|
|
|
685
1554
|
break;
|
|
686
1555
|
}
|
|
687
1556
|
case "message_complete": {
|
|
1557
|
+
this.markActiveTurn(channel, agentId);
|
|
688
1558
|
channel.openTurns.add(agentId);
|
|
689
1559
|
this.emitHost({
|
|
690
1560
|
kind: "agent.message_complete",
|
|
@@ -696,6 +1566,7 @@ class Orchestrator {
|
|
|
696
1566
|
break;
|
|
697
1567
|
}
|
|
698
1568
|
case "tool_call": {
|
|
1569
|
+
this.markActiveTurn(channel, agentId);
|
|
699
1570
|
this.emitHost({
|
|
700
1571
|
kind: "agent.tool_call",
|
|
701
1572
|
taskId: channel.taskId,
|
|
@@ -706,6 +1577,7 @@ class Orchestrator {
|
|
|
706
1577
|
break;
|
|
707
1578
|
}
|
|
708
1579
|
case "permission_request": {
|
|
1580
|
+
this.markActiveTurn(channel, agentId);
|
|
709
1581
|
this.emitHost({
|
|
710
1582
|
kind: "agent.permission_request",
|
|
711
1583
|
taskId: channel.taskId,
|
|
@@ -733,6 +1605,7 @@ class Orchestrator {
|
|
|
733
1605
|
case "error": {
|
|
734
1606
|
// The turn died with the session — drop its turn-state flags so a
|
|
735
1607
|
// respawned session starts clean.
|
|
1608
|
+
channel.activeTurns.delete(agentId);
|
|
736
1609
|
channel.openTurns.delete(agentId);
|
|
737
1610
|
channel.interrupted.delete(agentId);
|
|
738
1611
|
// Claude under load (especially Docker Desktop macOS) occasionally
|
|
@@ -780,6 +1653,8 @@ class Orchestrator {
|
|
|
780
1653
|
channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
|
|
781
1654
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
782
1655
|
channel.sessions.delete(agentId);
|
|
1656
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1657
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
783
1658
|
break;
|
|
784
1659
|
}
|
|
785
1660
|
case "turn_complete": {
|
|
@@ -788,6 +1663,7 @@ class Orchestrator {
|
|
|
788
1663
|
// An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
|
|
789
1664
|
// the cloud discards the buffer instead of handing it to peers.
|
|
790
1665
|
const aborted = channel.interrupted.delete(agentId);
|
|
1666
|
+
const remainingTurns = this.decrementActiveTurns(channel, agentId);
|
|
791
1667
|
channel.openTurns.delete(agentId);
|
|
792
1668
|
// ADR-076: a turn that completed means the current account is healthy —
|
|
793
1669
|
// reset its rotation budget so a LATER rate-limit gets a fresh failover.
|
|
@@ -799,14 +1675,33 @@ class Orchestrator {
|
|
|
799
1675
|
aborted,
|
|
800
1676
|
usage: event.usage,
|
|
801
1677
|
});
|
|
1678
|
+
// A config repair never interrupts a turn. The stale id remains queued
|
|
1679
|
+
// until this exact boundary, then reconcile closes and replaces it.
|
|
1680
|
+
if (
|
|
1681
|
+
remainingTurns === 0 &&
|
|
1682
|
+
channel.browserStaleSessions.has(agentId)
|
|
1683
|
+
) {
|
|
1684
|
+
void this.reconcileSessions(channel).catch((err: unknown) => {
|
|
1685
|
+
console.warn(
|
|
1686
|
+
`[orchestrator] ${channel.taskId}: stale-session reconcile failed: ${
|
|
1687
|
+
err instanceof Error ? err.message : String(err)
|
|
1688
|
+
}`,
|
|
1689
|
+
);
|
|
1690
|
+
});
|
|
1691
|
+
}
|
|
802
1692
|
break;
|
|
803
1693
|
}
|
|
804
1694
|
case "exit":
|
|
805
1695
|
// Same zombie hazard as the error path — a session whose process
|
|
806
1696
|
// ended (even cleanly) can never carry another turn.
|
|
1697
|
+
channel.activeTurns.delete(agentId);
|
|
1698
|
+
channel.openTurns.delete(agentId);
|
|
1699
|
+
channel.interrupted.delete(agentId);
|
|
807
1700
|
channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
|
|
808
1701
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
809
1702
|
channel.sessions.delete(agentId);
|
|
1703
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1704
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
810
1705
|
break;
|
|
811
1706
|
}
|
|
812
1707
|
}
|
|
@@ -822,11 +1717,16 @@ class Orchestrator {
|
|
|
822
1717
|
channel: Channel,
|
|
823
1718
|
agentId: string,
|
|
824
1719
|
): Promise<void> {
|
|
1720
|
+
if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
|
|
1721
|
+
|
|
825
1722
|
const tries = (channel.respawns.get(agentId) ?? 0) + 1;
|
|
826
1723
|
channel.respawns.set(agentId, tries);
|
|
827
1724
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
828
1725
|
|
|
829
1726
|
if (tries > MAX_RESPAWNS_PER_AGENT) {
|
|
1727
|
+
channel.sessions.delete(agentId);
|
|
1728
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1729
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
830
1730
|
this.emitHost({
|
|
831
1731
|
kind: "agent.exit",
|
|
832
1732
|
taskId: channel.taskId,
|
|
@@ -839,40 +1739,84 @@ class Orchestrator {
|
|
|
839
1739
|
return;
|
|
840
1740
|
}
|
|
841
1741
|
|
|
842
|
-
|
|
1742
|
+
channel.spawning.add(agentId);
|
|
1743
|
+
try {
|
|
1744
|
+
const restored = repairClaudeConfigInContainer(channel.containerName);
|
|
843
1745
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
1746
|
+
// Delete the old generation before awaiting close. Its adapter emits a
|
|
1747
|
+
// terminal event during teardown; bindSession's identity guard must see
|
|
1748
|
+
// it as stale, and reconciliation must see `spawning` instead of opening
|
|
1749
|
+
// a second replacement path.
|
|
1750
|
+
const old = channel.sessions.get(agentId);
|
|
1751
|
+
channel.sessions.delete(agentId);
|
|
1752
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1753
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
1754
|
+
if (old) {
|
|
1755
|
+
try {
|
|
1756
|
+
await old.close();
|
|
1757
|
+
} catch {
|
|
1758
|
+
// Already exited — close is idempotent for our adapters.
|
|
1759
|
+
}
|
|
851
1760
|
}
|
|
852
|
-
|
|
1761
|
+
if (!this.isActiveChannel(channel)) return;
|
|
853
1762
|
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
taskId: channel.taskId,
|
|
858
|
-
agent,
|
|
859
|
-
containerName: channel.containerName,
|
|
860
|
-
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
861
|
-
});
|
|
862
|
-
channel.sessions.set(agentId, session);
|
|
863
|
-
session.onEvent((event) => {
|
|
864
|
-
void this.handleAgentEvent(channel, agentId, event);
|
|
865
|
-
});
|
|
1763
|
+
const agent = channel.roster.find((a) => a.id === agentId);
|
|
1764
|
+
const task = getHostTask(channel.taskId);
|
|
1765
|
+
if (!agent || !task || task.statusMirror !== "running") return;
|
|
866
1766
|
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
1767
|
+
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
1768
|
+
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
1769
|
+
const base = agentCliEnv(
|
|
1770
|
+
channel.taskId,
|
|
1771
|
+
agent,
|
|
1772
|
+
task.ownerUserId,
|
|
1773
|
+
apiUrl,
|
|
1774
|
+
cliSecret,
|
|
1775
|
+
);
|
|
1776
|
+
const boundAccountId = channel.accountByAgent.get(agentId);
|
|
1777
|
+
const boundAccount = boundAccountId
|
|
1778
|
+
? resolveEngineAccounts(agent.kind).find(
|
|
1779
|
+
(account) => account.id === boundAccountId,
|
|
1780
|
+
)
|
|
1781
|
+
: undefined;
|
|
1782
|
+
const session = await this.createAndBindStableSession(channel, agentId, {
|
|
1783
|
+
taskId: channel.taskId,
|
|
1784
|
+
agent,
|
|
1785
|
+
containerName: channel.containerName,
|
|
1786
|
+
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
1787
|
+
agentEnv: boundAccount
|
|
1788
|
+
? { ...base, ...boundAccount.execEnv }
|
|
1789
|
+
: this.accountAgentEnv(channel, agent, base),
|
|
1790
|
+
});
|
|
1791
|
+
if (!session) return;
|
|
1792
|
+
|
|
1793
|
+
const message = restored
|
|
1794
|
+
? `${agentId} restarted — config file was missing in the container, restored from the host.`
|
|
1795
|
+
: `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
|
|
1796
|
+
this.emitHost({
|
|
1797
|
+
kind: "agent.exit",
|
|
1798
|
+
taskId: channel.taskId,
|
|
1799
|
+
agentId,
|
|
1800
|
+
reason: message,
|
|
1801
|
+
});
|
|
1802
|
+
} finally {
|
|
1803
|
+
channel.spawning.delete(agentId);
|
|
1804
|
+
// A concurrent config writer can mark the newly-bound recovery stale
|
|
1805
|
+
// while this spawn slot is still held. Reconciliation skips spawning
|
|
1806
|
+
// ids by design, so release must explicitly give that mark another turn.
|
|
1807
|
+
if (
|
|
1808
|
+
this.isActiveChannel(channel) &&
|
|
1809
|
+
channel.browserStaleSessions.has(agentId)
|
|
1810
|
+
) {
|
|
1811
|
+
void this.reconcileSessions(channel).catch((err: unknown) => {
|
|
1812
|
+
console.warn(
|
|
1813
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-recovery reconcile failed: ${
|
|
1814
|
+
err instanceof Error ? err.message : String(err)
|
|
1815
|
+
}`,
|
|
1816
|
+
);
|
|
1817
|
+
});
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
876
1820
|
}
|
|
877
1821
|
|
|
878
1822
|
/**
|
|
@@ -888,6 +1832,10 @@ class Orchestrator {
|
|
|
888
1832
|
agentId: string,
|
|
889
1833
|
agent: RosterAgent,
|
|
890
1834
|
): Promise<boolean> {
|
|
1835
|
+
if (!this.isActiveChannel(channel)) return true;
|
|
1836
|
+
// A concurrent terminal event for the same session is already being
|
|
1837
|
+
// handled by the path that owns this spawn slot.
|
|
1838
|
+
if (channel.spawning.has(agentId)) return true;
|
|
891
1839
|
if (resolveEngineAccounts(agent.kind).length < 2) return false;
|
|
892
1840
|
|
|
893
1841
|
const budget = (channel.rotations.get(agentId) ?? 0) + 1;
|
|
@@ -904,53 +1852,100 @@ class Orchestrator {
|
|
|
904
1852
|
const task = getHostTask(channel.taskId);
|
|
905
1853
|
if (!task || task.statusMirror !== "running") return false;
|
|
906
1854
|
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
1855
|
+
let replacement: AgentSession | null = null;
|
|
1856
|
+
channel.spawning.add(agentId);
|
|
1857
|
+
try {
|
|
1858
|
+
// Tear the rate-limited session down. Delete first so its final adapter
|
|
1859
|
+
// event is generation-stale, while `spawning` keeps reconciliation from
|
|
1860
|
+
// opening a second replacement path during the slow provisioning/setup.
|
|
1861
|
+
const old = channel.sessions.get(agentId);
|
|
1862
|
+
channel.sessions.delete(agentId);
|
|
1863
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1864
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
1865
|
+
if (old) {
|
|
1866
|
+
try {
|
|
1867
|
+
await old.close();
|
|
1868
|
+
} catch {
|
|
1869
|
+
/* already gone — close is idempotent for our adapters */
|
|
1870
|
+
}
|
|
916
1871
|
}
|
|
917
|
-
|
|
1872
|
+
if (!this.isActiveChannel(channel)) return true;
|
|
1873
|
+
|
|
1874
|
+
// The serialized materialization gate owns both account provisioning
|
|
1875
|
+
// and, when enabled, browser migration.
|
|
1876
|
+
await this.ensureBrowserSetup(
|
|
1877
|
+
channel,
|
|
1878
|
+
channel.roster.some((candidate) => candidate.kind === "codex"),
|
|
1879
|
+
true,
|
|
1880
|
+
);
|
|
1881
|
+
if (!this.isActiveChannel(channel)) return true;
|
|
1882
|
+
await this.ensureMcpConfig(channel);
|
|
1883
|
+
if (!this.isActiveChannel(channel)) return true;
|
|
918
1884
|
|
|
919
|
-
|
|
920
|
-
|
|
1885
|
+
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
1886
|
+
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
1887
|
+
const base = agentCliEnv(
|
|
1888
|
+
channel.taskId,
|
|
1889
|
+
agent,
|
|
1890
|
+
task.ownerUserId,
|
|
1891
|
+
apiUrl,
|
|
1892
|
+
cliSecret,
|
|
1893
|
+
);
|
|
1894
|
+
channel.accountByAgent.set(agentId, next.id);
|
|
1895
|
+
noteEngineAccountUsed(next.id);
|
|
921
1896
|
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
const session = await this.factory.create({
|
|
935
|
-
taskId: channel.taskId,
|
|
936
|
-
agent,
|
|
937
|
-
containerName: channel.containerName,
|
|
938
|
-
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
939
|
-
agentEnv: { ...base, ...next.execEnv },
|
|
940
|
-
});
|
|
941
|
-
channel.sessions.set(agentId, session);
|
|
942
|
-
session.onEvent((event) => {
|
|
943
|
-
void this.handleAgentEvent(channel, agentId, event);
|
|
944
|
-
});
|
|
1897
|
+
replacement = await this.createAndBindStableSession(
|
|
1898
|
+
channel,
|
|
1899
|
+
agentId,
|
|
1900
|
+
{
|
|
1901
|
+
taskId: channel.taskId,
|
|
1902
|
+
agent,
|
|
1903
|
+
containerName: channel.containerName,
|
|
1904
|
+
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
1905
|
+
agentEnv: { ...base, ...next.execEnv },
|
|
1906
|
+
},
|
|
1907
|
+
);
|
|
1908
|
+
if (!replacement) return true;
|
|
945
1909
|
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
1910
|
+
this.emitSystemNote(
|
|
1911
|
+
channel.taskId,
|
|
1912
|
+
`${agentId}: hit a rate limit — switched to another ${agent.kind} account and retried.`,
|
|
1913
|
+
);
|
|
1914
|
+
|
|
1915
|
+
// Re-deliver the in-flight prompt so the interrupted turn resumes.
|
|
1916
|
+
const prompt = channel.lastPrompt.get(agentId);
|
|
1917
|
+
if (prompt) {
|
|
1918
|
+
this.incrementActiveTurns(channel, agentId);
|
|
1919
|
+
void replacement.send(prompt).catch((err: unknown) => {
|
|
1920
|
+
if (channel.sessions.get(agentId) === replacement) {
|
|
1921
|
+
this.decrementActiveTurns(channel, agentId);
|
|
1922
|
+
}
|
|
1923
|
+
console.warn(
|
|
1924
|
+
`[orchestrator] ${channel.taskId}/${agentId}: rotated send failed: ${
|
|
1925
|
+
err instanceof Error ? err.message : String(err)
|
|
1926
|
+
}`,
|
|
1927
|
+
);
|
|
1928
|
+
});
|
|
1929
|
+
}
|
|
1930
|
+
} finally {
|
|
1931
|
+
channel.spawning.delete(agentId);
|
|
1932
|
+
}
|
|
950
1933
|
|
|
951
|
-
//
|
|
952
|
-
|
|
953
|
-
if (
|
|
1934
|
+
// Browser/config setup may have marked other live generations stale. Run
|
|
1935
|
+
// their safe-idle recycle only after this agent's replacement is bound.
|
|
1936
|
+
if (
|
|
1937
|
+
replacement &&
|
|
1938
|
+
this.isActiveChannel(channel) &&
|
|
1939
|
+
channel.browserStaleSessions.size > 0
|
|
1940
|
+
) {
|
|
1941
|
+
void this.reconcileSessions(channel).catch((err: unknown) => {
|
|
1942
|
+
console.warn(
|
|
1943
|
+
`[orchestrator] ${channel.taskId}: post-rotation reconcile failed: ${
|
|
1944
|
+
err instanceof Error ? err.message : String(err)
|
|
1945
|
+
}`,
|
|
1946
|
+
);
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
954
1949
|
return true;
|
|
955
1950
|
}
|
|
956
1951
|
|
|
@@ -970,11 +1965,69 @@ class Orchestrator {
|
|
|
970
1965
|
}
|
|
971
1966
|
|
|
972
1967
|
/** Tear a channel down (task killed). */
|
|
973
|
-
|
|
1968
|
+
closeChannel(taskId: string): Promise<void> {
|
|
1969
|
+
const existing = this.channelClosures.get(taskId);
|
|
1970
|
+
if (existing) return existing;
|
|
1971
|
+
|
|
1972
|
+
this.blockedTasks.add(taskId);
|
|
1973
|
+
this.channelSpecs.delete(taskId);
|
|
974
1974
|
const ch = this.channels.get(taskId);
|
|
975
|
-
if (!ch) return;
|
|
976
|
-
|
|
1975
|
+
if (!ch) return Promise.resolve();
|
|
1976
|
+
// Invalidate the generation BEFORE the first await. Browser setup and the
|
|
1977
|
+
// start/reconcile factory continuations all check this identity, so none
|
|
1978
|
+
// can repopulate an orphan channel while its sessions close.
|
|
1979
|
+
ch.closed = true;
|
|
977
1980
|
this.channels.delete(taskId);
|
|
1981
|
+
this.clearBrowserRetry(ch);
|
|
1982
|
+
ch.browserStaleSessions.clear();
|
|
1983
|
+
ch.browserPendingStaleSessions.clear();
|
|
1984
|
+
ch.activeTurns.clear();
|
|
1985
|
+
ch.openTurns.clear();
|
|
1986
|
+
ch.interrupted.clear();
|
|
1987
|
+
ch.reconcileAgain = false;
|
|
1988
|
+
const drains = [
|
|
1989
|
+
ch.browserSetup,
|
|
1990
|
+
ch.mcpConfigWrite,
|
|
1991
|
+
ch.reconcileReady,
|
|
1992
|
+
ch.sessionsReady,
|
|
1993
|
+
...ch.factoryOperations,
|
|
1994
|
+
...ch.eventOperations,
|
|
1995
|
+
].filter((promise): promise is Promise<unknown> => promise !== null);
|
|
1996
|
+
const sessions = [...ch.sessions.values()];
|
|
1997
|
+
ch.sessions.clear();
|
|
1998
|
+
const closes = sessions.map((session) =>
|
|
1999
|
+
session.close().catch(() => {
|
|
2000
|
+
// Teardown is best-effort; every adapter close is intended idempotent.
|
|
2001
|
+
}),
|
|
2002
|
+
);
|
|
2003
|
+
// Draining externally side-effecting work is part of the container-name
|
|
2004
|
+
// reuse fence. An old docker cp/config writer/factory must finish before
|
|
2005
|
+
// taskDown can return and a later taskUp can create the same container.
|
|
2006
|
+
let closing: Promise<void>;
|
|
2007
|
+
closing = Promise.allSettled([...drains, ...closes])
|
|
2008
|
+
.then(() => undefined)
|
|
2009
|
+
.finally(() => {
|
|
2010
|
+
if (this.channelClosures.get(taskId) === closing) {
|
|
2011
|
+
this.channelClosures.delete(taskId);
|
|
2012
|
+
}
|
|
2013
|
+
});
|
|
2014
|
+
this.channelClosures.set(taskId, closing);
|
|
2015
|
+
return closing;
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
/** Task-up must join teardown before it can recreate the same compose and
|
|
2019
|
+
* container names. This does not clear the tombstone; only success does. */
|
|
2020
|
+
waitForChannelClose(taskId: string): Promise<void> {
|
|
2021
|
+
return this.channelClosures.get(taskId) ?? Promise.resolve();
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
/** A successful task-up is the only transition that reopens a tombstoned
|
|
2025
|
+
* task id after stop/teardown. */
|
|
2026
|
+
allowChannel(taskId: string): void {
|
|
2027
|
+
// A caller that forgot the task-up fence cannot reopen the id while old
|
|
2028
|
+
// external operations are still draining.
|
|
2029
|
+
if (this.channelClosures.has(taskId)) return;
|
|
2030
|
+
this.blockedTasks.delete(taskId);
|
|
978
2031
|
}
|
|
979
2032
|
|
|
980
2033
|
/**
|
|
@@ -986,22 +2039,50 @@ class Orchestrator {
|
|
|
986
2039
|
* `stopped` is not an ACTIVE status, host recovery won't restart it.
|
|
987
2040
|
*/
|
|
988
2041
|
async stopTask(taskId: string): Promise<{ ok: boolean; error?: string }> {
|
|
2042
|
+
return this.runTaskLifecycle(taskId, () =>
|
|
2043
|
+
this.stopTaskWithinLifecycle(taskId),
|
|
2044
|
+
);
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
private async stopTaskWithinLifecycle(
|
|
2048
|
+
taskId: string,
|
|
2049
|
+
): Promise<{ ok: boolean; error?: string }> {
|
|
989
2050
|
const task = getHostTask(taskId);
|
|
990
2051
|
if (!task) return { ok: false, error: "unknown task" };
|
|
991
2052
|
const project = task.composeProject;
|
|
992
2053
|
if (!project) return { ok: false, error: "task has no running stack" };
|
|
993
2054
|
// Close sessions first so nothing races to respawn them mid-stop.
|
|
994
2055
|
await this.closeChannel(taskId);
|
|
2056
|
+
try {
|
|
2057
|
+
await stopPreviewSidecars(taskId);
|
|
2058
|
+
} catch (err) {
|
|
2059
|
+
console.warn(
|
|
2060
|
+
`[orchestrator] ${taskId}: preview cleanup before stop failed: ${
|
|
2061
|
+
err instanceof Error ? err.message : String(err)
|
|
2062
|
+
}`,
|
|
2063
|
+
);
|
|
2064
|
+
}
|
|
995
2065
|
const ps = await dockerCli([
|
|
996
2066
|
"ps",
|
|
997
2067
|
"-q",
|
|
998
2068
|
"--filter",
|
|
999
2069
|
`label=com.docker.compose.project=${project}`,
|
|
1000
2070
|
]);
|
|
2071
|
+
if (ps.status !== 0) {
|
|
2072
|
+
// An empty stdout is not evidence that the stack is gone when docker
|
|
2073
|
+
// itself failed. Reopen the channel and leave the runtime mirror alone
|
|
2074
|
+
// so cloud reconciliation cannot turn a live container into "stopped".
|
|
2075
|
+
this.allowChannel(taskId);
|
|
2076
|
+
return {
|
|
2077
|
+
ok: false,
|
|
2078
|
+
error: ps.stderr.trim().slice(0, 200) || "docker ps failed",
|
|
2079
|
+
};
|
|
2080
|
+
}
|
|
1001
2081
|
const ids = ps.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
1002
2082
|
if (ids.length > 0) {
|
|
1003
2083
|
const stopped = await dockerCli(["stop", ...ids], { timeoutMs: 60_000 });
|
|
1004
2084
|
if (stopped.status !== 0) {
|
|
2085
|
+
this.allowChannel(taskId);
|
|
1005
2086
|
return {
|
|
1006
2087
|
ok: false,
|
|
1007
2088
|
error: stopped.stderr.trim().slice(0, 200) || "docker stop failed",
|
|
@@ -1411,7 +2492,7 @@ export function buildSystemPreamble(
|
|
|
1411
2492
|
? [
|
|
1412
2493
|
"## Browser",
|
|
1413
2494
|
"",
|
|
1414
|
-
"This container has
|
|
2495
|
+
"This container has Chromium available through the",
|
|
1415
2496
|
"`browser` MCP server (Playwright). Use it to VERIFY UI work",
|
|
1416
2497
|
"end-to-end — the dev server you're building runs in this same",
|
|
1417
2498
|
"container, so navigate to `http://localhost:<port>` directly.",
|
|
@@ -1422,9 +2503,10 @@ export function buildSystemPreamble(
|
|
|
1422
2503
|
"in your reply as `[image: /workspace/.uai/attachments/<name>.png]`.",
|
|
1423
2504
|
"Your browser runs on a virtual display the humans can WATCH live",
|
|
1424
2505
|
"(the \"browser\" preview) — nothing for you to do about that.",
|
|
1425
|
-
"The
|
|
1426
|
-
"
|
|
1427
|
-
"
|
|
2506
|
+
"The browser is ready to use — just call it. If a call ever fails,",
|
|
2507
|
+
"the failure is real: report it and say what you could not verify.",
|
|
2508
|
+
"Do NOT fall back to a static HTML harness and describe it as",
|
|
2509
|
+
"verified — that is worse than saying the browser was unavailable.",
|
|
1428
2510
|
"",
|
|
1429
2511
|
]
|
|
1430
2512
|
: []),
|
|
@@ -1575,6 +2657,39 @@ export function getOrchestrator(): Orchestrator {
|
|
|
1575
2657
|
|
|
1576
2658
|
let recoveryPromise: Promise<void> = Promise.resolve();
|
|
1577
2659
|
|
|
2660
|
+
interface TaskRecoveryBarrier {
|
|
2661
|
+
promise: Promise<void>;
|
|
2662
|
+
resolve(): void;
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
const taskRecoveryBarriers = new Map<string, TaskRecoveryBarrier>();
|
|
2666
|
+
|
|
2667
|
+
function openTaskRecoveryBarrier(taskId: string): void {
|
|
2668
|
+
if (taskRecoveryBarriers.has(taskId)) return;
|
|
2669
|
+
let resolve!: () => void;
|
|
2670
|
+
const promise = new Promise<void>((done) => {
|
|
2671
|
+
resolve = done;
|
|
2672
|
+
});
|
|
2673
|
+
taskRecoveryBarriers.set(taskId, { promise, resolve });
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
function finishTaskRecovery(taskId: string): void {
|
|
2677
|
+
const barrier = taskRecoveryBarriers.get(taskId);
|
|
2678
|
+
if (!barrier) return;
|
|
2679
|
+
taskRecoveryBarriers.delete(taskId);
|
|
2680
|
+
barrier.resolve();
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
function finishAllTaskRecovery(): void {
|
|
2684
|
+
for (const taskId of [...taskRecoveryBarriers.keys()]) {
|
|
2685
|
+
finishTaskRecovery(taskId);
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2689
|
+
export function taskRecoveryComplete(taskId: string): Promise<void> {
|
|
2690
|
+
return taskRecoveryBarriers.get(taskId)?.promise ?? Promise.resolve();
|
|
2691
|
+
}
|
|
2692
|
+
|
|
1578
2693
|
/**
|
|
1579
2694
|
* Resolves once boot-time recovery has finished (starting the orchestrator —
|
|
1580
2695
|
* and with it recovery — if that hasn't happened yet). Never rejects.
|
|
@@ -1698,47 +2813,67 @@ export async function recoverRunningTasks(opts?: {
|
|
|
1698
2813
|
}): Promise<void> {
|
|
1699
2814
|
const retryMs = opts?.retryMs ?? 20_000;
|
|
1700
2815
|
const maxPasses = opts?.maxPasses ?? 45;
|
|
2816
|
+
let pendingTaskIds: ReadonlySet<string> | undefined;
|
|
1701
2817
|
// After a machine reboot the host service and the Docker daemon start
|
|
1702
2818
|
// concurrently; a pass that finds docker unreachable resolves nothing and
|
|
1703
2819
|
// must retry once the daemon is up — the whole point of recovery is to
|
|
1704
2820
|
// restart the exited containers that reboot left behind.
|
|
1705
2821
|
for (let pass = 1; ; pass++) {
|
|
1706
|
-
const deferred = await recoveryPass();
|
|
1707
|
-
if (deferred === 0)
|
|
2822
|
+
const deferred = await recoveryPass(pendingTaskIds);
|
|
2823
|
+
if (deferred.size === 0) {
|
|
2824
|
+
finishAllTaskRecovery();
|
|
2825
|
+
return;
|
|
2826
|
+
}
|
|
2827
|
+
pendingTaskIds = deferred;
|
|
1708
2828
|
if (pass >= maxPasses) {
|
|
1709
2829
|
console.warn(
|
|
1710
|
-
`[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred} task(s) unresolved)`,
|
|
2830
|
+
`[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
|
|
1711
2831
|
);
|
|
2832
|
+
finishAllTaskRecovery();
|
|
1712
2833
|
return;
|
|
1713
2834
|
}
|
|
1714
2835
|
if (pass === 1) {
|
|
1715
2836
|
console.log(
|
|
1716
|
-
`[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred} task(s)`,
|
|
2837
|
+
`[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
|
|
1717
2838
|
);
|
|
1718
2839
|
}
|
|
1719
2840
|
await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
1720
2841
|
}
|
|
1721
2842
|
}
|
|
1722
2843
|
|
|
1723
|
-
/** One recovery sweep. Returns
|
|
1724
|
-
*
|
|
1725
|
-
async function recoveryPass(
|
|
1726
|
-
|
|
2844
|
+
/** One recovery sweep. Returns only task ids deferred by unreachable Docker;
|
|
2845
|
+
* successful ids are never revisited by a later retry pass. */
|
|
2846
|
+
async function recoveryPass(
|
|
2847
|
+
onlyTaskIds?: ReadonlySet<string>,
|
|
2848
|
+
): Promise<Set<string>> {
|
|
2849
|
+
const deferred = new Set<string>();
|
|
1727
2850
|
try {
|
|
1728
2851
|
const db = getDb();
|
|
1729
|
-
const
|
|
2852
|
+
const activeRows = db
|
|
1730
2853
|
.select()
|
|
1731
2854
|
.from(schema.hostTasks)
|
|
1732
2855
|
.where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
|
|
1733
2856
|
.all();
|
|
1734
|
-
|
|
2857
|
+
const rows = onlyTaskIds
|
|
2858
|
+
? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
|
|
2859
|
+
: activeRows;
|
|
2860
|
+
const activeTaskIds = new Set(rows.map((task) => task.taskId));
|
|
2861
|
+
const expectedTaskIds =
|
|
2862
|
+
onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
|
|
2863
|
+
for (const taskId of expectedTaskIds) {
|
|
2864
|
+
if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
|
|
2865
|
+
}
|
|
2866
|
+
for (const task of rows) openTaskRecoveryBarrier(task.taskId);
|
|
2867
|
+
if (rows.length === 0) return deferred;
|
|
1735
2868
|
console.log(
|
|
1736
2869
|
`[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
|
|
1737
2870
|
);
|
|
1738
2871
|
for (const task of rows) {
|
|
1739
2872
|
try {
|
|
1740
|
-
if (
|
|
2873
|
+
if (await recoverOneTask(task)) finishTaskRecovery(task.taskId);
|
|
2874
|
+
else deferred.add(task.taskId);
|
|
1741
2875
|
} catch (err) {
|
|
2876
|
+
finishTaskRecovery(task.taskId);
|
|
1742
2877
|
console.error(
|
|
1743
2878
|
`[orchestrator] recovery: ${task.taskId} failed:`,
|
|
1744
2879
|
err instanceof Error ? err.message : err,
|
|
@@ -1746,6 +2881,7 @@ async function recoveryPass(): Promise<number> {
|
|
|
1746
2881
|
}
|
|
1747
2882
|
}
|
|
1748
2883
|
} catch (err) {
|
|
2884
|
+
finishAllTaskRecovery();
|
|
1749
2885
|
console.error(
|
|
1750
2886
|
"[orchestrator] recovery: top-level failure",
|
|
1751
2887
|
err instanceof Error ? err.message : err,
|
|
@@ -1875,5 +3011,3 @@ function db_setRuntime(
|
|
|
1875
3011
|
): void {
|
|
1876
3012
|
upsertHostTask(taskId, extras);
|
|
1877
3013
|
}
|
|
1878
|
-
|
|
1879
|
-
export type { Orchestrator };
|