@runuai/host 0.9.146 → 0.9.148
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agent-cli.ts +17 -2
- package/lib/agents/attach-compatibility.ts +51 -0
- package/lib/agents/codex.ts +128 -12
- package/lib/agents/durable-proc.ts +2 -0
- package/lib/agents/machine-durable.ts +15 -0
- package/lib/agents/session-state.ts +107 -0
- package/lib/agents/transport.ts +116 -53
- package/lib/agents/types.ts +3 -3
- package/lib/orchestrator.ts +1294 -73
- package/lib/stopped-task-status-queue.ts +24 -6
- package/lib/ulid.ts +10 -1
- package/package.json +1 -1
- package/src/main.ts +18 -6
- package/src/protocol.ts +35 -3
package/lib/orchestrator.ts
CHANGED
|
@@ -34,6 +34,11 @@ import { getDb, schema } from "./db";
|
|
|
34
34
|
import { mockAgentFactory } from "./agents/mock";
|
|
35
35
|
import { realAgentFactory } from "./agents/factory";
|
|
36
36
|
import { resolveAgentMode } from "./agents/mode";
|
|
37
|
+
import {
|
|
38
|
+
clearPersistedCodexThread,
|
|
39
|
+
persistedAgentIds,
|
|
40
|
+
stopPersistedAgentSession,
|
|
41
|
+
} from "./agents/session-state";
|
|
37
42
|
import {
|
|
38
43
|
type AgentEvent,
|
|
39
44
|
type AgentSession,
|
|
@@ -223,14 +228,27 @@ interface Channel {
|
|
|
223
228
|
/** Sessions that loaded MCP config before the latest rewrite. Busy sessions
|
|
224
229
|
* stay here until their turn completes; each id is recycled exactly once. */
|
|
225
230
|
browserStaleSessions: Set<string>;
|
|
231
|
+
/** Launch-setting edits reuse the idle-safe stale-session path, but carry
|
|
232
|
+
* their own lifecycle so only those replacements emit agent.session. */
|
|
233
|
+
rosterSessionChanges: Map<string, RosterSessionChange>;
|
|
226
234
|
/** Browser-config rewrites whose install/config preconditions are not ready
|
|
227
235
|
* yet. These sessions must eventually reload the definition, but recycling
|
|
228
236
|
* them before the browser is usable only trades one broken session for
|
|
229
237
|
* another. */
|
|
230
238
|
browserPendingStaleSessions: Set<string>;
|
|
239
|
+
/** Engine kinds newly covered by an already-enabled browser setup. The
|
|
240
|
+
* aggregate setup result can then recycle only sessions for those kinds. */
|
|
241
|
+
browserConfigAffectedKinds: Set<string>;
|
|
231
242
|
/** One reconcile loop per channel. It covers ordinary missing agents and
|
|
232
243
|
* stale-session recycling without overlapping factory creates. */
|
|
233
244
|
reconcileReady: Promise<void> | null;
|
|
245
|
+
/** Forced removal closes begun by refreshChannel. ensureSessions joins this
|
|
246
|
+
* fence before spawning so one outbox consumer cannot overlap another. */
|
|
247
|
+
rosterRefreshReady: Promise<void> | null;
|
|
248
|
+
/** First-start cleanup of durable rows whose ids are absent from the fresh
|
|
249
|
+
* cloud roster (the host may have restarted after their removal). */
|
|
250
|
+
persistedRosterChecked: boolean;
|
|
251
|
+
persistedRosterReady: Promise<void> | null;
|
|
234
252
|
/** A reconcile request that arrived while the loop was in flight. */
|
|
235
253
|
reconcileAgain: boolean;
|
|
236
254
|
/** Set before teardown awaits anything. Every async continuation checks both
|
|
@@ -260,6 +278,14 @@ interface Channel {
|
|
|
260
278
|
* turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
|
|
261
279
|
* half-turn instead of delivering it to @-mentioned peers. */
|
|
262
280
|
interrupted: Set<string>;
|
|
281
|
+
/** Cloud turn boundaries whose runner generation was retired by recovery.
|
|
282
|
+
* Kept separately from activeTurns because remove/edit may overtake the
|
|
283
|
+
* async recovery after generation-local turn state has been cleared. */
|
|
284
|
+
retiredTurnBoundaries: Set<string>;
|
|
285
|
+
/** Async account recoveries fenced by roster removal/edit. A same-settings
|
|
286
|
+
* remove/re-add is invisible to launch-tuple comparison, so the recovery
|
|
287
|
+
* itself carries the supersession bit until it releases the spawn slot. */
|
|
288
|
+
recoveryByAgent: Map<string, RecoveryFence>;
|
|
263
289
|
/** Per-agent respawn counter — bounded so a broken agent can't
|
|
264
290
|
* loop forever rewriting its config. */
|
|
265
291
|
respawns: Map<string, number>;
|
|
@@ -288,6 +314,9 @@ interface Channel {
|
|
|
288
314
|
mcpConfigDirtyEpoch: number;
|
|
289
315
|
/** Dirty epoch covered by the last successful gateway publication. */
|
|
290
316
|
mcpConfigAppliedEpoch: number;
|
|
317
|
+
/** Engine kinds whose provisioned config directories dirtied the managed
|
|
318
|
+
* gateway files at the current unapplied epoch. */
|
|
319
|
+
mcpProvisionAffectedKinds: Set<string>;
|
|
291
320
|
/** Monotonic version of the managed config files agent processes read at
|
|
292
321
|
* startup. A factory create that straddles a rewrite is discarded and
|
|
293
322
|
* retried before it can be bound as the live generation. */
|
|
@@ -309,6 +338,9 @@ interface Channel {
|
|
|
309
338
|
* drains them so an old channel cannot resume against a reused container
|
|
310
339
|
* name and overwrite the replacement generation's durable transport. */
|
|
311
340
|
factoryOperations: Set<Promise<unknown>>;
|
|
341
|
+
/** The create currently owning an agent id, used to fence a removal until a
|
|
342
|
+
* stale in-flight process has observed the new roster and closed itself. */
|
|
343
|
+
factoryOperationByAgent: Map<string, Promise<AgentSession | null>>;
|
|
312
344
|
/** Terminal-event recovery/rotation can remove a session before awaiting
|
|
313
345
|
* its close. Track the enclosing handler so teardown still drains that
|
|
314
346
|
* external operation before the container name can be reused. */
|
|
@@ -328,6 +360,16 @@ interface AcceptedPrompt {
|
|
|
328
360
|
text: string;
|
|
329
361
|
}
|
|
330
362
|
|
|
363
|
+
interface RosterSessionChange {
|
|
364
|
+
reason: string;
|
|
365
|
+
state: "pending-close" | "restarting";
|
|
366
|
+
abortPublished: boolean;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
interface RecoveryFence {
|
|
370
|
+
superseded: boolean;
|
|
371
|
+
}
|
|
372
|
+
|
|
331
373
|
/** Hard cap on automatic respawns per agent per channel lifetime. */
|
|
332
374
|
const MAX_RESPAWNS_PER_AGENT = 5;
|
|
333
375
|
/** ADR-076: hard cap on account rotations per agent between successful turns. */
|
|
@@ -545,31 +587,100 @@ export class Orchestrator {
|
|
|
545
587
|
/** Fold a fresh channel spec into a live channel (ADR-049). */
|
|
546
588
|
private refreshChannel(channel: Channel, spec: ChannelEnsureInput): void {
|
|
547
589
|
const browserWasEnabled = channel.browserTesting;
|
|
590
|
+
const previousRoster = channel.roster;
|
|
548
591
|
const previousMcpShape = JSON.stringify({
|
|
549
592
|
connections: channel.mcpConnections,
|
|
550
|
-
kinds: [...new Set(
|
|
593
|
+
kinds: [...new Set(previousRoster.map((agent) => agent.kind))].sort(),
|
|
551
594
|
});
|
|
552
|
-
const
|
|
595
|
+
const previousById = new Map(previousRoster.map((agent) => [agent.id, agent]));
|
|
596
|
+
const nextById = new Map(spec.agents.map((agent) => [agent.id, agent]));
|
|
597
|
+
const previousKinds = new Set(previousRoster.map((agent) => agent.kind));
|
|
598
|
+
const nextKinds = new Set(spec.agents.map((agent) => agent.kind));
|
|
599
|
+
const removed = previousRoster.filter((agent) => !nextById.has(agent.id));
|
|
600
|
+
const added = spec.agents.filter((agent) => !previousById.has(agent.id));
|
|
601
|
+
const changed = spec.agents.flatMap((agent) => {
|
|
602
|
+
const previous = previousById.get(agent.id);
|
|
603
|
+
if (!previous) return [];
|
|
604
|
+
const reason = rosterLaunchChangeReason(previous, agent);
|
|
605
|
+
return reason ? [{ agent, reason }] : [];
|
|
606
|
+
});
|
|
607
|
+
|
|
553
608
|
// Replace EXISTING agents' data with the fresh spec — a mid-task roster edit
|
|
554
609
|
// can change permissions, model, brief, role or skills, and without this the
|
|
555
610
|
// stale snapshot persists (e.g. granting `uai` CLI permissions mid-task never
|
|
556
|
-
// took effect: writeAgentCli kept seeing the old empty list).
|
|
557
|
-
//
|
|
558
|
-
|
|
559
|
-
const
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
611
|
+
// took effect: writeAgentCli kept seeing the old empty list). Replacing the
|
|
612
|
+
// array exactly also makes absence authoritative for live removal.
|
|
613
|
+
channel.roster = [...spec.agents];
|
|
614
|
+
for (const agent of added) {
|
|
615
|
+
// Per-agent link/document skills, like getOrCreateChannel does at
|
|
616
|
+
// channel birth. Best-effort.
|
|
617
|
+
writeAgentSkills(channel.taskId, agent);
|
|
618
|
+
}
|
|
619
|
+
for (const { agent, reason } of changed) {
|
|
620
|
+
const recoveryAbortPublished = this.supersedeRecovery(
|
|
621
|
+
channel,
|
|
622
|
+
agent.id,
|
|
623
|
+
);
|
|
624
|
+
// Clear before any reconcile can construct a replacement. A session can
|
|
625
|
+
// already be absent from the map while its durable runner/thread row is
|
|
626
|
+
// still live, so this must not depend on in-memory session presence.
|
|
627
|
+
try {
|
|
628
|
+
clearPersistedCodexThread(channel.taskId, agent.id);
|
|
629
|
+
} catch (err) {
|
|
630
|
+
console.warn(
|
|
631
|
+
`[orchestrator] ${channel.taskId}/${agent.id}: could not clear persisted Codex thread for roster change: ${
|
|
632
|
+
err instanceof Error ? err.message : String(err)
|
|
633
|
+
}`,
|
|
634
|
+
);
|
|
567
635
|
}
|
|
636
|
+
channel.accountByAgent.delete(agent.id);
|
|
637
|
+
channel.rotations.delete(agent.id);
|
|
638
|
+
// Crash/recovery budgets belong to the retired launch generation. A
|
|
639
|
+
// newly selected engine/model/effort must get an immediate clean start,
|
|
640
|
+
// not inherit the predecessor's cooldown window.
|
|
641
|
+
channel.respawns.delete(agent.id);
|
|
642
|
+
channel.respawnLastAt.delete(agent.id);
|
|
643
|
+
const priorChange = channel.rosterSessionChanges.get(agent.id);
|
|
644
|
+
channel.rosterSessionChanges.set(agent.id, {
|
|
645
|
+
reason,
|
|
646
|
+
state:
|
|
647
|
+
priorChange?.state === "restarting"
|
|
648
|
+
? "restarting"
|
|
649
|
+
: "pending-close",
|
|
650
|
+
abortPublished:
|
|
651
|
+
priorChange?.abortPublished === true || recoveryAbortPublished,
|
|
652
|
+
});
|
|
653
|
+
channel.browserStaleSessions.add(agent.id);
|
|
654
|
+
}
|
|
655
|
+
if (removed.length > 0) {
|
|
656
|
+
const previousRefresh = channel.rosterRefreshReady ?? Promise.resolve();
|
|
657
|
+
const removals = removed.map((agent) =>
|
|
658
|
+
this.removeRosterAgent(channel, agent.id),
|
|
659
|
+
);
|
|
660
|
+
let refresh: Promise<void>;
|
|
661
|
+
refresh = Promise.allSettled([previousRefresh, ...removals])
|
|
662
|
+
.then(() => undefined)
|
|
663
|
+
.finally(() => {
|
|
664
|
+
if (channel.rosterRefreshReady === refresh) {
|
|
665
|
+
channel.rosterRefreshReady = null;
|
|
666
|
+
}
|
|
667
|
+
});
|
|
668
|
+
channel.rosterRefreshReady = refresh;
|
|
568
669
|
}
|
|
569
670
|
channel.humans = spec.humans ?? [];
|
|
570
671
|
channel.mode = spec.mode === "secretary" ? "secretary" : "open";
|
|
571
672
|
channel.secretaryAgentId = spec.secretaryAgentId;
|
|
572
673
|
channel.browserTesting = spec.browserTesting === true;
|
|
674
|
+
if (channel.browserTesting) {
|
|
675
|
+
const affectedKinds = browserWasEnabled
|
|
676
|
+
? [...nextKinds].filter((kind) => !previousKinds.has(kind))
|
|
677
|
+
: [...nextKinds];
|
|
678
|
+
for (const kind of affectedKinds) {
|
|
679
|
+
channel.browserConfigAffectedKinds.add(kind);
|
|
680
|
+
}
|
|
681
|
+
} else {
|
|
682
|
+
channel.browserConfigAffectedKinds.clear();
|
|
683
|
+
}
|
|
573
684
|
channel.routingShadow = spec.routing?.shadow === true;
|
|
574
685
|
channel.workspacePath = spec.workspacePath;
|
|
575
686
|
channel.sharedFiles = spec.sharedFiles ?? "ro";
|
|
@@ -626,6 +737,162 @@ export class Orchestrator {
|
|
|
626
737
|
}
|
|
627
738
|
}
|
|
628
739
|
|
|
740
|
+
/** Remove every live/in-flight trace of one agent without deleting its
|
|
741
|
+
* durable session directory. Removal is intentionally immediate mid-turn. */
|
|
742
|
+
private async removeRosterAgent(
|
|
743
|
+
channel: Channel,
|
|
744
|
+
agentId: string,
|
|
745
|
+
): Promise<void> {
|
|
746
|
+
const session = channel.sessions.get(agentId);
|
|
747
|
+
const inFlightCreate = channel.factoryOperationByAgent.get(agentId);
|
|
748
|
+
const lifecycleAtRemoval = channel.rosterSessionChanges.get(agentId);
|
|
749
|
+
channel.sessions.delete(agentId);
|
|
750
|
+
// Do not release another lifecycle owner's spawn slot. Recovery/reconcile
|
|
751
|
+
// paths clear it in their own finally blocks after rechecking the desired
|
|
752
|
+
// roster; releasing it here lets a rapid re-add start a second same-id
|
|
753
|
+
// factory while the first is still provisioning.
|
|
754
|
+
|
|
755
|
+
// Recovery may already have retired the runner-local turn state while an
|
|
756
|
+
// async provision/broker call was in flight. Consume that saved boundary
|
|
757
|
+
// before looking at the live generation's maps.
|
|
758
|
+
this.supersedeRecovery(channel, agentId);
|
|
759
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
760
|
+
// A removed partial turn can never receive its natural terminal event.
|
|
761
|
+
// Close that cloud boundary as aborted before the old session's identity
|
|
762
|
+
// guard begins suppressing its final adapter events.
|
|
763
|
+
if (
|
|
764
|
+
channel.activeTurns.has(agentId) ||
|
|
765
|
+
channel.openTurns.has(agentId) ||
|
|
766
|
+
channel.acceptedPrompts.has(agentId)
|
|
767
|
+
) {
|
|
768
|
+
channel.interrupted.add(agentId);
|
|
769
|
+
}
|
|
770
|
+
this.retireRunnerGeneration(channel, agentId);
|
|
771
|
+
channel.preambles.delete(agentId);
|
|
772
|
+
channel.firstTurns.delete(agentId);
|
|
773
|
+
channel.browserStaleSessions.delete(agentId);
|
|
774
|
+
channel.rosterSessionChanges.delete(agentId);
|
|
775
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
776
|
+
channel.pendingPrompts.delete(agentId);
|
|
777
|
+
channel.runtimeDeferredPrompts.delete(agentId);
|
|
778
|
+
channel.runtimeDeferredAgents.delete(agentId);
|
|
779
|
+
channel.interrupted.delete(agentId);
|
|
780
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
781
|
+
channel.respawns.delete(agentId);
|
|
782
|
+
channel.respawnLastAt.delete(agentId);
|
|
783
|
+
channel.everBound.delete(agentId);
|
|
784
|
+
channel.accountByAgent.delete(agentId);
|
|
785
|
+
channel.lastPrompt.delete(agentId);
|
|
786
|
+
channel.rotations.delete(agentId);
|
|
787
|
+
|
|
788
|
+
if (session) {
|
|
789
|
+
try {
|
|
790
|
+
// Delete-before-close above keeps the adapter's terminal event
|
|
791
|
+
// identity-guarded and prevents an outbox consumer overlap.
|
|
792
|
+
await session.close();
|
|
793
|
+
} catch {
|
|
794
|
+
// Already dead — close is idempotent for our adapters.
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
if (inFlightCreate) {
|
|
798
|
+
// createAndBindStableSessionCore rechecks the desired roster after the
|
|
799
|
+
// factory await and closes the stale process instead of binding it.
|
|
800
|
+
await inFlightCreate.catch(() => null);
|
|
801
|
+
}
|
|
802
|
+
if (!this.isActiveChannel(channel)) return;
|
|
803
|
+
// A newer ensure can re-add the same id while the old close/create drains.
|
|
804
|
+
// That spec owns the status now; do not stop its bound generation or emit
|
|
805
|
+
// a stale removed edge.
|
|
806
|
+
if (channel.roster.some((agent) => agent.id === agentId)) {
|
|
807
|
+
this.restoreReaddedRosterLifecycle(
|
|
808
|
+
channel,
|
|
809
|
+
agentId,
|
|
810
|
+
lifecycleAtRemoval,
|
|
811
|
+
);
|
|
812
|
+
void this.reconcileIfStarted(channel).catch((err: unknown) => {
|
|
813
|
+
console.warn(
|
|
814
|
+
`[orchestrator] ${channel.taskId}/${agentId}: re-added roster reconcile failed: ${
|
|
815
|
+
err instanceof Error ? err.message : String(err)
|
|
816
|
+
}`,
|
|
817
|
+
);
|
|
818
|
+
});
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
let durableCleanupFailed = false;
|
|
822
|
+
try {
|
|
823
|
+
await this.stopPersistedRunnerIfPresent(channel, agentId);
|
|
824
|
+
} catch (err) {
|
|
825
|
+
durableCleanupFailed = true;
|
|
826
|
+
// The same ensure that joined this removal fence will retry through the
|
|
827
|
+
// persisted-roster recovery pass instead of trusting a poisoned result.
|
|
828
|
+
channel.persistedRosterChecked = false;
|
|
829
|
+
console.warn(
|
|
830
|
+
`[orchestrator] ${channel.taskId}/${agentId}: removed durable session cleanup failed: ${
|
|
831
|
+
err instanceof Error ? err.message : String(err)
|
|
832
|
+
}`,
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
if (!this.isActiveChannel(channel)) return;
|
|
836
|
+
if (channel.roster.some((agent) => agent.id === agentId)) {
|
|
837
|
+
this.restoreReaddedRosterLifecycle(
|
|
838
|
+
channel,
|
|
839
|
+
agentId,
|
|
840
|
+
lifecycleAtRemoval,
|
|
841
|
+
);
|
|
842
|
+
void this.reconcileIfStarted(channel).catch((err: unknown) => {
|
|
843
|
+
console.warn(
|
|
844
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-stop re-add reconcile failed: ${
|
|
845
|
+
err instanceof Error ? err.message : String(err)
|
|
846
|
+
}`,
|
|
847
|
+
);
|
|
848
|
+
});
|
|
849
|
+
return;
|
|
850
|
+
}
|
|
851
|
+
// Slow rotation/account continuations captured before the roster swap may
|
|
852
|
+
// have reached their finally blocks while the close was in flight.
|
|
853
|
+
channel.accountByAgent.delete(agentId);
|
|
854
|
+
channel.rotations.delete(agentId);
|
|
855
|
+
channel.respawns.delete(agentId);
|
|
856
|
+
channel.respawnLastAt.delete(agentId);
|
|
857
|
+
channel.lastPrompt.delete(agentId);
|
|
858
|
+
if (!this.isActiveChannel(channel)) return;
|
|
859
|
+
if (durableCleanupFailed) return;
|
|
860
|
+
this.emitHost({
|
|
861
|
+
kind: "agent.session",
|
|
862
|
+
taskId: channel.taskId,
|
|
863
|
+
agentId,
|
|
864
|
+
state: "removed",
|
|
865
|
+
});
|
|
866
|
+
console.log(
|
|
867
|
+
`[orchestrator] ${channel.taskId}/${agentId}: stopped removed roster session`,
|
|
868
|
+
);
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/** A remove superseded while an earlier edit was already restarting must
|
|
872
|
+
* still close that visible lifecycle with ready for the re-added roster. */
|
|
873
|
+
private restoreReaddedRosterLifecycle(
|
|
874
|
+
channel: Channel,
|
|
875
|
+
agentId: string,
|
|
876
|
+
removedChange: RosterSessionChange | undefined,
|
|
877
|
+
): void {
|
|
878
|
+
if (
|
|
879
|
+
removedChange?.state !== "restarting" ||
|
|
880
|
+
channel.rosterSessionChanges.has(agentId)
|
|
881
|
+
) {
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
if (channel.sessions.has(agentId)) {
|
|
885
|
+
this.emitHost({
|
|
886
|
+
kind: "agent.session",
|
|
887
|
+
taskId: channel.taskId,
|
|
888
|
+
agentId,
|
|
889
|
+
state: "ready",
|
|
890
|
+
});
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
893
|
+
channel.rosterSessionChanges.set(agentId, removedChange);
|
|
894
|
+
}
|
|
895
|
+
|
|
629
896
|
/** Materialize a browser flag enabled after the channel already started. */
|
|
630
897
|
private async enableBrowserTesting(channel: Channel): Promise<void> {
|
|
631
898
|
try {
|
|
@@ -709,8 +976,13 @@ export class Orchestrator {
|
|
|
709
976
|
browserSetupCoverage: null,
|
|
710
977
|
browserRetryTimer: null,
|
|
711
978
|
browserStaleSessions: new Set(),
|
|
979
|
+
rosterSessionChanges: new Map(),
|
|
712
980
|
browserPendingStaleSessions: new Set(),
|
|
981
|
+
browserConfigAffectedKinds: new Set(),
|
|
713
982
|
reconcileReady: null,
|
|
983
|
+
rosterRefreshReady: null,
|
|
984
|
+
persistedRosterChecked: false,
|
|
985
|
+
persistedRosterReady: null,
|
|
714
986
|
reconcileAgain: false,
|
|
715
987
|
closed: false,
|
|
716
988
|
openTurns: new Set(),
|
|
@@ -720,6 +992,8 @@ export class Orchestrator {
|
|
|
720
992
|
runtimeDeferredPrompts: new Map(),
|
|
721
993
|
runtimeDeferredAgents: new Set(),
|
|
722
994
|
interrupted: new Set(),
|
|
995
|
+
retiredTurnBoundaries: new Set(),
|
|
996
|
+
recoveryByAgent: new Map(),
|
|
723
997
|
respawns: new Map(),
|
|
724
998
|
respawnLastAt: new Map(),
|
|
725
999
|
humans: spec.humans ?? [],
|
|
@@ -729,12 +1003,14 @@ export class Orchestrator {
|
|
|
729
1003
|
mcpConfigWrite: null,
|
|
730
1004
|
mcpConfigDirtyEpoch: 1,
|
|
731
1005
|
mcpConfigAppliedEpoch: 0,
|
|
1006
|
+
mcpProvisionAffectedKinds: new Set(),
|
|
732
1007
|
configGeneration: 0,
|
|
733
1008
|
spawning: new Set(),
|
|
734
1009
|
everBound: new Set(),
|
|
735
1010
|
routingShadow: spec.routing?.shadow === true,
|
|
736
1011
|
workspacePath: spec.workspacePath,
|
|
737
1012
|
factoryOperations: new Set(),
|
|
1013
|
+
factoryOperationByAgent: new Map(),
|
|
738
1014
|
eventOperations: new Set(),
|
|
739
1015
|
accountByAgent: new Map(),
|
|
740
1016
|
lastPrompt: new Map(),
|
|
@@ -761,6 +1037,58 @@ export class Orchestrator {
|
|
|
761
1037
|
// copy cannot land after browser/MCP materialization and erase config.
|
|
762
1038
|
await taskRecoveryComplete(channel.taskId);
|
|
763
1039
|
if (!this.isActiveChannel(channel)) return false;
|
|
1040
|
+
if (channel.rosterRefreshReady) {
|
|
1041
|
+
await channel.rosterRefreshReady;
|
|
1042
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1043
|
+
}
|
|
1044
|
+
if (!channel.persistedRosterChecked) {
|
|
1045
|
+
let removedIds: string[];
|
|
1046
|
+
try {
|
|
1047
|
+
const rosterIds = new Set(channel.roster.map((agent) => agent.id));
|
|
1048
|
+
removedIds = persistedAgentIds(channel.taskId).filter(
|
|
1049
|
+
(agentId) => !rosterIds.has(agentId),
|
|
1050
|
+
);
|
|
1051
|
+
} catch (err) {
|
|
1052
|
+
console.warn(
|
|
1053
|
+
`[orchestrator] ${channel.taskId}: persisted roster scan failed: ${
|
|
1054
|
+
err instanceof Error ? err.message : String(err)
|
|
1055
|
+
}`,
|
|
1056
|
+
);
|
|
1057
|
+
throw err;
|
|
1058
|
+
}
|
|
1059
|
+
channel.persistedRosterChecked = true;
|
|
1060
|
+
if (removedIds.length > 0) {
|
|
1061
|
+
const operation = this.stopPersistedRemovedAgents(
|
|
1062
|
+
channel,
|
|
1063
|
+
removedIds,
|
|
1064
|
+
);
|
|
1065
|
+
let tracked: Promise<void>;
|
|
1066
|
+
tracked = operation.then(
|
|
1067
|
+
() => {
|
|
1068
|
+
if (channel.persistedRosterReady === tracked) {
|
|
1069
|
+
channel.persistedRosterReady = null;
|
|
1070
|
+
}
|
|
1071
|
+
},
|
|
1072
|
+
(err: unknown) => {
|
|
1073
|
+
if (channel.persistedRosterReady === tracked) {
|
|
1074
|
+
channel.persistedRosterReady = null;
|
|
1075
|
+
channel.persistedRosterChecked = false;
|
|
1076
|
+
}
|
|
1077
|
+
console.warn(
|
|
1078
|
+
`[orchestrator] ${channel.taskId}: persisted roster cleanup failed: ${
|
|
1079
|
+
err instanceof Error ? err.message : String(err)
|
|
1080
|
+
}`,
|
|
1081
|
+
);
|
|
1082
|
+
throw err;
|
|
1083
|
+
},
|
|
1084
|
+
);
|
|
1085
|
+
channel.persistedRosterReady = tracked;
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
if (channel.persistedRosterReady) {
|
|
1089
|
+
await channel.persistedRosterReady;
|
|
1090
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1091
|
+
}
|
|
764
1092
|
if (!backgroundContainerWorkAllowed()) return false;
|
|
765
1093
|
// Memoized: every caller awaits the SAME in-flight start, so a concurrent
|
|
766
1094
|
// deliver() can't observe "ready" while the sessions map is still empty
|
|
@@ -825,6 +1153,48 @@ export class Orchestrator {
|
|
|
825
1153
|
return ready;
|
|
826
1154
|
}
|
|
827
1155
|
|
|
1156
|
+
/** A host restart loses the in-memory old roster, not its durable runners.
|
|
1157
|
+
* Stop persisted ids that the authoritative fresh spec no longer contains. */
|
|
1158
|
+
private async stopPersistedRemovedAgents(
|
|
1159
|
+
channel: Channel,
|
|
1160
|
+
removedIds: readonly string[],
|
|
1161
|
+
): Promise<void> {
|
|
1162
|
+
const environment = await this.agentEnvironment(channel);
|
|
1163
|
+
for (const agentId of removedIds) {
|
|
1164
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1165
|
+
// The roster may have advanced while environment reconstruction was in
|
|
1166
|
+
// flight. Last arrival wins; never stop an id the current spec wants.
|
|
1167
|
+
if (channel.roster.some((agent) => agent.id === agentId)) continue;
|
|
1168
|
+
const stoppedDir = await stopPersistedAgentSession(
|
|
1169
|
+
channel.taskId,
|
|
1170
|
+
agentId,
|
|
1171
|
+
environment,
|
|
1172
|
+
);
|
|
1173
|
+
if (channel.roster.some((agent) => agent.id === agentId)) continue;
|
|
1174
|
+
if (!stoppedDir || !this.isActiveChannel(channel)) continue;
|
|
1175
|
+
this.emitHost({
|
|
1176
|
+
kind: "agent.session",
|
|
1177
|
+
taskId: channel.taskId,
|
|
1178
|
+
agentId,
|
|
1179
|
+
state: "removed",
|
|
1180
|
+
});
|
|
1181
|
+
console.log(
|
|
1182
|
+
`[orchestrator] ${channel.taskId}/${agentId}: stopped removed durable session during roster recovery`,
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
/** Stop a durable predecessor even when its row is already marked closed.
|
|
1188
|
+
* Closed is not proof of runner death: markClosed precedes the stop write. */
|
|
1189
|
+
private async stopPersistedRunnerIfPresent(
|
|
1190
|
+
channel: Channel,
|
|
1191
|
+
agentId: string,
|
|
1192
|
+
): Promise<string | null> {
|
|
1193
|
+
if (!persistedAgentIds(channel.taskId).includes(agentId)) return null;
|
|
1194
|
+
const environment = await this.agentEnvironment(channel);
|
|
1195
|
+
return stopPersistedAgentSession(channel.taskId, agentId, environment);
|
|
1196
|
+
}
|
|
1197
|
+
|
|
828
1198
|
/**
|
|
829
1199
|
* Background browser work may recycle sessions only after the memoized
|
|
830
1200
|
* initial start has finished. Waiting here prevents live enable/retry from
|
|
@@ -873,11 +1243,17 @@ export class Orchestrator {
|
|
|
873
1243
|
private noteProvisionMutation(
|
|
874
1244
|
channel: Channel,
|
|
875
1245
|
waitForBrowserReadiness: boolean,
|
|
1246
|
+
affectedKinds: ReadonlySet<string>,
|
|
876
1247
|
): void {
|
|
877
1248
|
if (!this.isActiveChannel(channel)) return;
|
|
878
1249
|
channel.configGeneration += 1;
|
|
879
1250
|
channel.mcpConfigDirtyEpoch += 1;
|
|
880
|
-
for (const
|
|
1251
|
+
for (const kind of affectedKinds) {
|
|
1252
|
+
channel.mcpProvisionAffectedKinds.add(kind);
|
|
1253
|
+
}
|
|
1254
|
+
for (const [agentId] of channel.sessions) {
|
|
1255
|
+
const kind = channel.roster.find((agent) => agent.id === agentId)?.kind;
|
|
1256
|
+
if (!kind || !affectedKinds.has(kind)) continue;
|
|
881
1257
|
if (waitForBrowserReadiness) {
|
|
882
1258
|
channel.browserPendingStaleSessions.add(agentId);
|
|
883
1259
|
} else {
|
|
@@ -910,6 +1286,42 @@ export class Orchestrator {
|
|
|
910
1286
|
});
|
|
911
1287
|
}
|
|
912
1288
|
|
|
1289
|
+
/** Kinds whose account-copy inputs changed since the last attempted pass. */
|
|
1290
|
+
private changedProvisionKinds(
|
|
1291
|
+
previousFingerprint: string | undefined,
|
|
1292
|
+
nextFingerprint: string,
|
|
1293
|
+
requestedKinds: readonly string[],
|
|
1294
|
+
): string[] {
|
|
1295
|
+
if (!previousFingerprint) return [...requestedKinds];
|
|
1296
|
+
type Fingerprint = {
|
|
1297
|
+
kinds?: string[];
|
|
1298
|
+
accounts?: Array<{
|
|
1299
|
+
kind: string;
|
|
1300
|
+
id: string;
|
|
1301
|
+
hostDir: string;
|
|
1302
|
+
containerDir: string;
|
|
1303
|
+
}>;
|
|
1304
|
+
};
|
|
1305
|
+
try {
|
|
1306
|
+
const previous = JSON.parse(previousFingerprint) as Fingerprint;
|
|
1307
|
+
const next = JSON.parse(nextFingerprint) as Fingerprint;
|
|
1308
|
+
return requestedKinds.filter((kind) => {
|
|
1309
|
+
if (!previous.kinds?.includes(kind)) return true;
|
|
1310
|
+
const previousAccounts = (previous.accounts ?? []).filter(
|
|
1311
|
+
(account) => account.kind === kind,
|
|
1312
|
+
);
|
|
1313
|
+
const nextAccounts = (next.accounts ?? []).filter(
|
|
1314
|
+
(account) => account.kind === kind,
|
|
1315
|
+
);
|
|
1316
|
+
return JSON.stringify(previousAccounts) !== JSON.stringify(nextAccounts);
|
|
1317
|
+
});
|
|
1318
|
+
} catch {
|
|
1319
|
+
// A legacy/partial fingerprint is not trustworthy enough to narrow a
|
|
1320
|
+
// write; copy and invalidate the requested kinds conservatively.
|
|
1321
|
+
return [...requestedKinds];
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
|
|
913
1325
|
private async ensureBrowserSetup(
|
|
914
1326
|
channel: Channel,
|
|
915
1327
|
hasCodex: boolean,
|
|
@@ -936,11 +1348,20 @@ export class Orchestrator {
|
|
|
936
1348
|
];
|
|
937
1349
|
const requestedProvisionFingerprint =
|
|
938
1350
|
this.provisionFingerprint(requestedKinds, channel.engineAccountScope);
|
|
939
|
-
const
|
|
1351
|
+
const changedProvisionKinds = this.changedProvisionKinds(
|
|
1352
|
+
channel.provisionAttemptedFingerprint,
|
|
1353
|
+
requestedProvisionFingerprint,
|
|
1354
|
+
requestedKinds,
|
|
1355
|
+
);
|
|
1356
|
+
const provisionKinds = forceProvision
|
|
1357
|
+
? requestedKinds
|
|
1358
|
+
: changedProvisionKinds;
|
|
1359
|
+
const mutableProvisionKinds = provisionKinds.filter((kind) =>
|
|
940
1360
|
resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
|
|
941
1361
|
(account) => !account.isDefault && Boolean(account.configDir),
|
|
942
1362
|
),
|
|
943
1363
|
);
|
|
1364
|
+
const mayProvisionConfig = mutableProvisionKinds.length > 0;
|
|
944
1365
|
|
|
945
1366
|
const inFlight = channel.browserSetup;
|
|
946
1367
|
if (inFlight) {
|
|
@@ -1011,14 +1432,14 @@ export class Orchestrator {
|
|
|
1011
1432
|
const provisioned = channel.agentHandle
|
|
1012
1433
|
? await provisionEngineAccounts(
|
|
1013
1434
|
channel.containerName,
|
|
1014
|
-
|
|
1435
|
+
provisionKinds,
|
|
1015
1436
|
{},
|
|
1016
1437
|
channel.agentHandle,
|
|
1017
1438
|
channel.engineAccountScope,
|
|
1018
1439
|
)
|
|
1019
1440
|
: await provisionEngineAccounts(
|
|
1020
1441
|
channel.containerName,
|
|
1021
|
-
|
|
1442
|
+
provisionKinds,
|
|
1022
1443
|
{},
|
|
1023
1444
|
undefined,
|
|
1024
1445
|
channel.engineAccountScope,
|
|
@@ -1050,6 +1471,7 @@ export class Orchestrator {
|
|
|
1050
1471
|
this.noteProvisionMutation(
|
|
1051
1472
|
channel,
|
|
1052
1473
|
channel.browserTesting,
|
|
1474
|
+
new Set(mutableProvisionKinds),
|
|
1053
1475
|
);
|
|
1054
1476
|
} else {
|
|
1055
1477
|
this.fenceConfigGeneration(channel);
|
|
@@ -1107,10 +1529,23 @@ export class Orchestrator {
|
|
|
1107
1529
|
channel.browserReady = setup.ready && setup.configured;
|
|
1108
1530
|
if (setup.changed) {
|
|
1109
1531
|
channel.configGeneration += 1;
|
|
1110
|
-
|
|
1532
|
+
const affectedKinds = channel.browserConfigAffectedKinds;
|
|
1533
|
+
const affectedIds =
|
|
1534
|
+
affectedKinds.size > 0
|
|
1535
|
+
? channel.roster
|
|
1536
|
+
.filter((agent) => affectedKinds.has(agent.kind))
|
|
1537
|
+
.map((agent) => agent.id)
|
|
1538
|
+
: [...channel.sessions.keys()];
|
|
1539
|
+
for (const agentId of affectedIds) {
|
|
1540
|
+
if (!channel.sessions.has(agentId)) continue;
|
|
1111
1541
|
channel.browserPendingStaleSessions.add(agentId);
|
|
1112
1542
|
}
|
|
1113
1543
|
}
|
|
1544
|
+
if (setup.configured) {
|
|
1545
|
+
for (const kind of requestedKinds) {
|
|
1546
|
+
channel.browserConfigAffectedKinds.delete(kind);
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1114
1549
|
if (channel.browserReady) {
|
|
1115
1550
|
for (const agentId of channel.browserPendingStaleSessions) {
|
|
1116
1551
|
channel.browserStaleSessions.add(agentId);
|
|
@@ -1259,6 +1694,9 @@ export class Orchestrator {
|
|
|
1259
1694
|
...(grokHomes.length > 0 ? { grokHomes } : {}),
|
|
1260
1695
|
});
|
|
1261
1696
|
const dirtyEpoch = channel.mcpConfigDirtyEpoch;
|
|
1697
|
+
const provisionAffectedKinds = new Set(
|
|
1698
|
+
channel.mcpProvisionAffectedKinds,
|
|
1699
|
+
);
|
|
1262
1700
|
const repairingProvisionedConfig =
|
|
1263
1701
|
channel.mcpConfigAppliedEpoch !== dirtyEpoch &&
|
|
1264
1702
|
channel.mcpConfigFingerprint === fingerprint;
|
|
@@ -1321,6 +1759,9 @@ export class Orchestrator {
|
|
|
1321
1759
|
// the files while docker exec was in flight, the newer epoch remains
|
|
1322
1760
|
// uncovered and the serialized ensure immediately publishes another pass.
|
|
1323
1761
|
channel.mcpConfigAppliedEpoch = dirtyEpoch;
|
|
1762
|
+
if (channel.mcpConfigDirtyEpoch === dirtyEpoch) {
|
|
1763
|
+
channel.mcpProvisionAffectedKinds.clear();
|
|
1764
|
+
}
|
|
1324
1765
|
// Agent CLIs read MCP servers once, at process start — and durable
|
|
1325
1766
|
// sessions (ADR-061) make processes long-lived, so without this a
|
|
1326
1767
|
// connection added mid-task stays invisible indefinitely. Recycle IDLE
|
|
@@ -1338,7 +1779,10 @@ export class Orchestrator {
|
|
|
1338
1779
|
// Provisioning marked the sessions that existed around its copy. Mark
|
|
1339
1780
|
// again after the repair so a factory that bound during this write is
|
|
1340
1781
|
// covered too (Set de-duplicates the earlier marks).
|
|
1341
|
-
|
|
1782
|
+
const affected = channel.roster
|
|
1783
|
+
.filter((agent) => provisionAffectedKinds.has(agent.kind))
|
|
1784
|
+
.map((agent) => agent.id);
|
|
1785
|
+
this.markSessionsStale(channel, affected);
|
|
1342
1786
|
} else if (!hadPrevious && hadLiveSessions) {
|
|
1343
1787
|
this.markSessionsStale(channel);
|
|
1344
1788
|
} else if (hadPrevious && connectionsChanged) {
|
|
@@ -1372,7 +1816,16 @@ export class Orchestrator {
|
|
|
1372
1816
|
/** Close stale sessions that are idle, without recursively reconciling. */
|
|
1373
1817
|
private async closeIdleStaleSessions(channel: Channel): Promise<number> {
|
|
1374
1818
|
let recycled = 0;
|
|
1375
|
-
|
|
1819
|
+
let mcpRecycled = 0;
|
|
1820
|
+
// The roster-change tracker is authoritative. Recovery paths written for
|
|
1821
|
+
// ordinary config recycling may clear browserStaleSessions while an edit
|
|
1822
|
+
// is waiting on their spawn slot; never let a pending-close edit bind its
|
|
1823
|
+
// replacement without first publishing the actual restart boundary.
|
|
1824
|
+
const staleAgentIds = new Set([
|
|
1825
|
+
...channel.browserStaleSessions,
|
|
1826
|
+
...channel.rosterSessionChanges.keys(),
|
|
1827
|
+
]);
|
|
1828
|
+
for (const agentId of staleAgentIds) {
|
|
1376
1829
|
if (!this.isActiveChannel(channel)) return recycled;
|
|
1377
1830
|
if (
|
|
1378
1831
|
channel.activeTurns.has(agentId) ||
|
|
@@ -1383,31 +1836,155 @@ export class Orchestrator {
|
|
|
1383
1836
|
}
|
|
1384
1837
|
const session = channel.sessions.get(agentId);
|
|
1385
1838
|
// A session that exited on its own is already absent. Its eventual
|
|
1386
|
-
// replacement is fresh
|
|
1839
|
+
// replacement is fresh for MCP purposes. A roster change still has to
|
|
1840
|
+
// stop a possibly-live durable predecessor before that replacement can
|
|
1841
|
+
// bind under new launch settings.
|
|
1387
1842
|
if (!session) {
|
|
1388
|
-
channel.
|
|
1843
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
1844
|
+
if (rosterChange) {
|
|
1845
|
+
await this.closeMissingRosterChangedSession(
|
|
1846
|
+
channel,
|
|
1847
|
+
agentId,
|
|
1848
|
+
rosterChange,
|
|
1849
|
+
);
|
|
1850
|
+
recycled += 1;
|
|
1851
|
+
} else {
|
|
1852
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1853
|
+
}
|
|
1389
1854
|
continue;
|
|
1390
1855
|
}
|
|
1391
1856
|
channel.browserStaleSessions.delete(agentId);
|
|
1392
1857
|
channel.sessions.delete(agentId);
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1858
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
1859
|
+
if (rosterChange) {
|
|
1860
|
+
await this.closeRosterChangedSession(
|
|
1861
|
+
channel,
|
|
1862
|
+
agentId,
|
|
1863
|
+
session,
|
|
1864
|
+
rosterChange,
|
|
1865
|
+
);
|
|
1866
|
+
} else {
|
|
1867
|
+
try {
|
|
1868
|
+
// Delete before close. Event handlers are identity-guarded, so the
|
|
1869
|
+
// old session's synthetic exit cannot consume respawn budget or delete
|
|
1870
|
+
// the replacement that follows.
|
|
1871
|
+
await session.close();
|
|
1872
|
+
} catch {
|
|
1873
|
+
// Already dead — close is idempotent for our adapters.
|
|
1874
|
+
}
|
|
1875
|
+
mcpRecycled += 1;
|
|
1400
1876
|
}
|
|
1401
1877
|
recycled += 1;
|
|
1402
1878
|
}
|
|
1403
|
-
if (
|
|
1879
|
+
if (mcpRecycled > 0) {
|
|
1404
1880
|
console.log(
|
|
1405
|
-
`[orchestrator] ${channel.taskId}: recycled ${
|
|
1881
|
+
`[orchestrator] ${channel.taskId}: recycled ${mcpRecycled} idle session(s) for MCP config change`,
|
|
1406
1882
|
);
|
|
1407
1883
|
}
|
|
1408
1884
|
return recycled;
|
|
1409
1885
|
}
|
|
1410
1886
|
|
|
1887
|
+
/** Stop one generation for a live launch-setting edit and publish the
|
|
1888
|
+
* roster-specific lifecycle edge only if that edit still owns the id. */
|
|
1889
|
+
private async closeRosterChangedSession(
|
|
1890
|
+
channel: Channel,
|
|
1891
|
+
agentId: string,
|
|
1892
|
+
session: AgentSession,
|
|
1893
|
+
change: RosterSessionChange,
|
|
1894
|
+
): Promise<void> {
|
|
1895
|
+
// Durable attach cannot reconstruct whether a pre-restart runner is
|
|
1896
|
+
// already mid-turn. Publish the terminal boundary before every forced
|
|
1897
|
+
// launch close; it is a no-op for an actually idle cloud stream.
|
|
1898
|
+
this.publishRosterChangeAbort(channel, agentId, change);
|
|
1899
|
+
try {
|
|
1900
|
+
await session.close();
|
|
1901
|
+
} catch {
|
|
1902
|
+
// Already dead — close is idempotent for our adapters.
|
|
1903
|
+
}
|
|
1904
|
+
try {
|
|
1905
|
+
await this.stopPersistedRunnerIfPresent(channel, agentId);
|
|
1906
|
+
// Clear again after close. A thread/started notification already queued
|
|
1907
|
+
// by the old Codex process must not restore its resume id between the
|
|
1908
|
+
// eager roster clear and replacement construction.
|
|
1909
|
+
clearPersistedCodexThread(channel.taskId, agentId);
|
|
1910
|
+
} catch (err) {
|
|
1911
|
+
if (
|
|
1912
|
+
this.isActiveChannel(channel) &&
|
|
1913
|
+
channel.rosterSessionChanges.get(agentId) === change
|
|
1914
|
+
) {
|
|
1915
|
+
channel.browserStaleSessions.add(agentId);
|
|
1916
|
+
}
|
|
1917
|
+
throw err;
|
|
1918
|
+
}
|
|
1919
|
+
if (
|
|
1920
|
+
!this.isActiveChannel(channel) ||
|
|
1921
|
+
channel.rosterSessionChanges.get(agentId) !== change ||
|
|
1922
|
+
!channel.roster.some((agent) => agent.id === agentId)
|
|
1923
|
+
) {
|
|
1924
|
+
return;
|
|
1925
|
+
}
|
|
1926
|
+
const alreadyRestarting = change.state === "restarting";
|
|
1927
|
+
change.state = "restarting";
|
|
1928
|
+
if (!alreadyRestarting) {
|
|
1929
|
+
this.emitHost({
|
|
1930
|
+
kind: "agent.session",
|
|
1931
|
+
taskId: channel.taskId,
|
|
1932
|
+
agentId,
|
|
1933
|
+
state: "restarting",
|
|
1934
|
+
});
|
|
1935
|
+
}
|
|
1936
|
+
console.log(
|
|
1937
|
+
`[orchestrator] ${channel.taskId}/${agentId}: recycled session for live roster ${change.reason}`,
|
|
1938
|
+
);
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
/** Finish a launch edit whose adapter already disappeared, but whose
|
|
1942
|
+
* durable predecessor may still own the CLI process/outbox. */
|
|
1943
|
+
private async closeMissingRosterChangedSession(
|
|
1944
|
+
channel: Channel,
|
|
1945
|
+
agentId: string,
|
|
1946
|
+
change: RosterSessionChange,
|
|
1947
|
+
): Promise<void> {
|
|
1948
|
+
this.publishRosterChangeAbort(channel, agentId, change);
|
|
1949
|
+
const stoppedDir = await this.stopPersistedRunnerIfPresent(channel, agentId);
|
|
1950
|
+
clearPersistedCodexThread(channel.taskId, agentId);
|
|
1951
|
+
if (
|
|
1952
|
+
!this.isActiveChannel(channel) ||
|
|
1953
|
+
channel.rosterSessionChanges.get(agentId) !== change ||
|
|
1954
|
+
!channel.roster.some((agent) => agent.id === agentId)
|
|
1955
|
+
) {
|
|
1956
|
+
return;
|
|
1957
|
+
}
|
|
1958
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1959
|
+
if (!stoppedDir) {
|
|
1960
|
+
// No old generation exists to recycle (one-shot engines between turns,
|
|
1961
|
+
// crash cooldown, or an agent not spawned yet). The cloud set the row to
|
|
1962
|
+
// Restarting when it persisted the edit, so clear that status now even
|
|
1963
|
+
// though there is no close/restart edge to publish first.
|
|
1964
|
+
channel.rosterSessionChanges.delete(agentId);
|
|
1965
|
+
this.emitHost({
|
|
1966
|
+
kind: "agent.session",
|
|
1967
|
+
taskId: channel.taskId,
|
|
1968
|
+
agentId,
|
|
1969
|
+
state: "ready",
|
|
1970
|
+
});
|
|
1971
|
+
return;
|
|
1972
|
+
}
|
|
1973
|
+
const alreadyRestarting = change.state === "restarting";
|
|
1974
|
+
change.state = "restarting";
|
|
1975
|
+
if (!alreadyRestarting) {
|
|
1976
|
+
this.emitHost({
|
|
1977
|
+
kind: "agent.session",
|
|
1978
|
+
taskId: channel.taskId,
|
|
1979
|
+
agentId,
|
|
1980
|
+
state: "restarting",
|
|
1981
|
+
});
|
|
1982
|
+
}
|
|
1983
|
+
console.log(
|
|
1984
|
+
`[orchestrator] ${channel.taskId}/${agentId}: stopped detached durable session for live roster ${change.reason}`,
|
|
1985
|
+
);
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1411
1988
|
/** Serialize all missing-session and stale-session reconciliation. */
|
|
1412
1989
|
private async reconcileSessions(channel: Channel): Promise<void> {
|
|
1413
1990
|
if (!this.isActiveChannel(channel)) return;
|
|
@@ -1520,6 +2097,10 @@ export class Orchestrator {
|
|
|
1520
2097
|
// Another recovery path may have filled this slot while the browser
|
|
1521
2098
|
// setup was awaiting.
|
|
1522
2099
|
if (channel.sessions.has(agent.id)) continue;
|
|
2100
|
+
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2101
|
+
channel.reconcileAgain = true;
|
|
2102
|
+
continue;
|
|
2103
|
+
}
|
|
1523
2104
|
const session = await this.createAndBindStableSession(
|
|
1524
2105
|
channel,
|
|
1525
2106
|
agent.id,
|
|
@@ -1530,7 +2111,7 @@ export class Orchestrator {
|
|
|
1530
2111
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
1531
2112
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
1532
2113
|
channel,
|
|
1533
|
-
agent
|
|
2114
|
+
agent,
|
|
1534
2115
|
),
|
|
1535
2116
|
agentEnv: this.accountAgentEnv(
|
|
1536
2117
|
channel,
|
|
@@ -1571,16 +2152,30 @@ export class Orchestrator {
|
|
|
1571
2152
|
|
|
1572
2153
|
private attachCompatibilityKeyFor(
|
|
1573
2154
|
channel: Channel,
|
|
1574
|
-
|
|
2155
|
+
agent: RosterAgent,
|
|
1575
2156
|
): string | undefined {
|
|
1576
|
-
if (!this.isSecretary(channel, agentId)) return undefined;
|
|
1577
2157
|
// Key the durable runner to the policy actually passed to the adapter.
|
|
1578
2158
|
// The restricted profile is dormant today, but giving it the normal-tools
|
|
1579
2159
|
// generation would make a future opt-out able to attach a runner launched
|
|
1580
2160
|
// under tool denial.
|
|
1581
|
-
|
|
1582
|
-
?
|
|
1583
|
-
|
|
2161
|
+
const policy = this.isSecretary(channel, agent.id)
|
|
2162
|
+
? this.executionProfileFor(channel, agent.id) === "communicator"
|
|
2163
|
+
? RESTRICTED_SECRETARY_SESSION_ATTACH_KEY
|
|
2164
|
+
: SECRETARY_SESSION_ATTACH_KEY
|
|
2165
|
+
: undefined;
|
|
2166
|
+
const model = agent.model ?? "";
|
|
2167
|
+
const effort = agent.effort ?? "";
|
|
2168
|
+
// Preserve legacy null/policy-only attachment for agents that choose no
|
|
2169
|
+
// launch flags. Explicit model/effort gets a stable key so a host restart
|
|
2170
|
+
// cannot attach a runner created under different settings.
|
|
2171
|
+
if (!model && !effort) return policy;
|
|
2172
|
+
return JSON.stringify([
|
|
2173
|
+
"roster-launch-v1",
|
|
2174
|
+
agent.kind,
|
|
2175
|
+
model,
|
|
2176
|
+
effort,
|
|
2177
|
+
policy ?? "",
|
|
2178
|
+
]);
|
|
1584
2179
|
}
|
|
1585
2180
|
|
|
1586
2181
|
private isSecretary(channel: Channel, agentId: string): boolean {
|
|
@@ -1618,6 +2213,10 @@ export class Orchestrator {
|
|
|
1618
2213
|
* container env to carry it. The account's env and the uai base win. */
|
|
1619
2214
|
floor: Record<string, string> = {},
|
|
1620
2215
|
): Record<string, string> {
|
|
2216
|
+
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2217
|
+
channel.accountByAgent.delete(agent.id);
|
|
2218
|
+
return { ...floor, ...base };
|
|
2219
|
+
}
|
|
1621
2220
|
const account = pickEngineAccount(agent.kind, {
|
|
1622
2221
|
scope: channel.engineAccountScope,
|
|
1623
2222
|
});
|
|
@@ -1662,6 +2261,18 @@ export class Orchestrator {
|
|
|
1662
2261
|
): void {
|
|
1663
2262
|
channel.sessions.set(agentId, session);
|
|
1664
2263
|
channel.everBound.add(agentId);
|
|
2264
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
2265
|
+
if (rosterChange) {
|
|
2266
|
+
channel.rosterSessionChanges.delete(agentId);
|
|
2267
|
+
if (rosterChange.state === "restarting") {
|
|
2268
|
+
this.emitHost({
|
|
2269
|
+
kind: "agent.session",
|
|
2270
|
+
taskId: channel.taskId,
|
|
2271
|
+
agentId,
|
|
2272
|
+
state: "ready",
|
|
2273
|
+
});
|
|
2274
|
+
}
|
|
2275
|
+
}
|
|
1665
2276
|
// The launch screen's `ready` rides the bind, not the tail of the start
|
|
1666
2277
|
// loop: whichever path brings the last roster agent up — the initial
|
|
1667
2278
|
// start, a reconcile that won the race with it, a respawn after Resume —
|
|
@@ -1729,16 +2340,29 @@ export class Orchestrator {
|
|
|
1729
2340
|
channel: Channel,
|
|
1730
2341
|
agentId: string,
|
|
1731
2342
|
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
2343
|
+
recovery?: RecoveryFence,
|
|
1732
2344
|
): Promise<AgentSession | null> {
|
|
1733
2345
|
const operation = this.createAndBindStableSessionCore(
|
|
1734
2346
|
channel,
|
|
1735
2347
|
agentId,
|
|
1736
2348
|
args,
|
|
2349
|
+
recovery,
|
|
1737
2350
|
);
|
|
1738
2351
|
channel.factoryOperations.add(operation);
|
|
2352
|
+
channel.factoryOperationByAgent.set(agentId, operation);
|
|
1739
2353
|
operation.then(
|
|
1740
|
-
() =>
|
|
1741
|
-
|
|
2354
|
+
() => {
|
|
2355
|
+
channel.factoryOperations.delete(operation);
|
|
2356
|
+
if (channel.factoryOperationByAgent.get(agentId) === operation) {
|
|
2357
|
+
channel.factoryOperationByAgent.delete(agentId);
|
|
2358
|
+
}
|
|
2359
|
+
},
|
|
2360
|
+
() => {
|
|
2361
|
+
channel.factoryOperations.delete(operation);
|
|
2362
|
+
if (channel.factoryOperationByAgent.get(agentId) === operation) {
|
|
2363
|
+
channel.factoryOperationByAgent.delete(agentId);
|
|
2364
|
+
}
|
|
2365
|
+
},
|
|
1742
2366
|
);
|
|
1743
2367
|
return operation;
|
|
1744
2368
|
}
|
|
@@ -1791,8 +2415,27 @@ export class Orchestrator {
|
|
|
1791
2415
|
channel: Channel,
|
|
1792
2416
|
agentId: string,
|
|
1793
2417
|
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
2418
|
+
recovery?: RecoveryFence,
|
|
1794
2419
|
): Promise<AgentSession | null> {
|
|
2420
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2421
|
+
channel.reconcileAgain = true;
|
|
2422
|
+
return null;
|
|
2423
|
+
}
|
|
2424
|
+
if (
|
|
2425
|
+
channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
|
|
2426
|
+
) {
|
|
2427
|
+
channel.reconcileAgain = true;
|
|
2428
|
+
return null;
|
|
2429
|
+
}
|
|
2430
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, args.agent)) {
|
|
2431
|
+
channel.reconcileAgain = true;
|
|
2432
|
+
return null;
|
|
2433
|
+
}
|
|
1795
2434
|
const environment = await this.agentEnvironment(channel);
|
|
2435
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2436
|
+
channel.reconcileAgain = true;
|
|
2437
|
+
return null;
|
|
2438
|
+
}
|
|
1796
2439
|
// ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
|
|
1797
2440
|
// account share one CODEX_HOME, and codex's new sqlite state runtime
|
|
1798
2441
|
// cannot be opened by two concurrent processes — the second agent dies
|
|
@@ -1809,12 +2452,57 @@ export class Orchestrator {
|
|
|
1809
2452
|
? await this.isolateCodexHome(environment, agentId, args)
|
|
1810
2453
|
: args;
|
|
1811
2454
|
while (this.isActiveChannel(channel)) {
|
|
2455
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2456
|
+
channel.reconcileAgain = true;
|
|
2457
|
+
return null;
|
|
2458
|
+
}
|
|
2459
|
+
if (
|
|
2460
|
+
channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
|
|
2461
|
+
) {
|
|
2462
|
+
channel.reconcileAgain = true;
|
|
2463
|
+
return null;
|
|
2464
|
+
}
|
|
2465
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
2466
|
+
channel.reconcileAgain = true;
|
|
2467
|
+
return null;
|
|
2468
|
+
}
|
|
1812
2469
|
const generation = channel.configGeneration;
|
|
1813
2470
|
const session = await this.factory.create({ ...boundArgs, environment });
|
|
1814
2471
|
if (!this.isActiveChannel(channel)) {
|
|
1815
2472
|
await session.close().catch(() => {});
|
|
1816
2473
|
return null;
|
|
1817
2474
|
}
|
|
2475
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2476
|
+
await session.close().catch(() => {});
|
|
2477
|
+
channel.reconcileAgain = true;
|
|
2478
|
+
return null;
|
|
2479
|
+
}
|
|
2480
|
+
const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
|
|
2481
|
+
if (pendingRosterClose?.state === "pending-close") {
|
|
2482
|
+
await this.closeRosterChangedSession(
|
|
2483
|
+
channel,
|
|
2484
|
+
agentId,
|
|
2485
|
+
session,
|
|
2486
|
+
pendingRosterClose,
|
|
2487
|
+
);
|
|
2488
|
+
channel.reconcileAgain = true;
|
|
2489
|
+
return null;
|
|
2490
|
+
}
|
|
2491
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
2492
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
2493
|
+
if (change) {
|
|
2494
|
+
await this.closeRosterChangedSession(
|
|
2495
|
+
channel,
|
|
2496
|
+
agentId,
|
|
2497
|
+
session,
|
|
2498
|
+
change,
|
|
2499
|
+
);
|
|
2500
|
+
} else {
|
|
2501
|
+
await session.close().catch(() => {});
|
|
2502
|
+
}
|
|
2503
|
+
channel.reconcileAgain = true;
|
|
2504
|
+
return null;
|
|
2505
|
+
}
|
|
1818
2506
|
if (generation === channel.configGeneration) {
|
|
1819
2507
|
// Keep the final generation check, stale cleanup, and map bind in one
|
|
1820
2508
|
// synchronous continuation. A config writer that resumes immediately
|
|
@@ -1831,6 +2519,17 @@ export class Orchestrator {
|
|
|
1831
2519
|
return null;
|
|
1832
2520
|
}
|
|
1833
2521
|
|
|
2522
|
+
private hasDesiredLaunchSettings(
|
|
2523
|
+
channel: Channel,
|
|
2524
|
+
agentId: string,
|
|
2525
|
+
launched: RosterAgent,
|
|
2526
|
+
): boolean {
|
|
2527
|
+
const desired = channel.roster.find((agent) => agent.id === agentId);
|
|
2528
|
+
return Boolean(
|
|
2529
|
+
desired && rosterLaunchChangeReason(launched, desired) === null,
|
|
2530
|
+
);
|
|
2531
|
+
}
|
|
2532
|
+
|
|
1834
2533
|
private agentEnvironment(
|
|
1835
2534
|
channel: Channel,
|
|
1836
2535
|
): Promise<TaskEnvironmentHandle<unknown>> {
|
|
@@ -2029,6 +2728,10 @@ export class Orchestrator {
|
|
|
2029
2728
|
if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
|
|
2030
2729
|
continue;
|
|
2031
2730
|
}
|
|
2731
|
+
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2732
|
+
channel.reconcileAgain = true;
|
|
2733
|
+
continue;
|
|
2734
|
+
}
|
|
2032
2735
|
channel.spawning.add(agent.id);
|
|
2033
2736
|
try {
|
|
2034
2737
|
const session = await this.createAndBindStableSession(
|
|
@@ -2041,7 +2744,7 @@ export class Orchestrator {
|
|
|
2041
2744
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
2042
2745
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2043
2746
|
channel,
|
|
2044
|
-
agent
|
|
2747
|
+
agent,
|
|
2045
2748
|
),
|
|
2046
2749
|
agentEnv: this.accountAgentEnv(
|
|
2047
2750
|
channel,
|
|
@@ -2185,6 +2888,78 @@ export class Orchestrator {
|
|
|
2185
2888
|
return remaining;
|
|
2186
2889
|
}
|
|
2187
2890
|
|
|
2891
|
+
/** Whether a retired generation still owns a cloud turn boundary. An
|
|
2892
|
+
* explicit interrupt already publishes its own aborted completion. */
|
|
2893
|
+
private needsAbortedTurnBoundary(
|
|
2894
|
+
channel: Channel,
|
|
2895
|
+
agentId: string,
|
|
2896
|
+
): boolean {
|
|
2897
|
+
return (
|
|
2898
|
+
!channel.interrupted.has(agentId) &&
|
|
2899
|
+
(channel.activeTurns.has(agentId) ||
|
|
2900
|
+
channel.openTurns.has(agentId) ||
|
|
2901
|
+
(channel.acceptedPrompts.get(agentId)?.length ?? 0) > 0)
|
|
2902
|
+
);
|
|
2903
|
+
}
|
|
2904
|
+
|
|
2905
|
+
private retainRetiredTurnBoundary(
|
|
2906
|
+
channel: Channel,
|
|
2907
|
+
agentId: string,
|
|
2908
|
+
needed: boolean,
|
|
2909
|
+
): void {
|
|
2910
|
+
if (needed) channel.retiredTurnBoundaries.add(agentId);
|
|
2911
|
+
}
|
|
2912
|
+
|
|
2913
|
+
/** Publish a retained terminal edge at most once. Recovery can be overtaken
|
|
2914
|
+
* by remove/edit after its generation-local turn maps have been cleared. */
|
|
2915
|
+
private emitRetiredTurnAbort(channel: Channel, agentId: string): boolean {
|
|
2916
|
+
if (!channel.retiredTurnBoundaries.delete(agentId)) return false;
|
|
2917
|
+
this.emitHost({
|
|
2918
|
+
kind: "agent.turn_complete",
|
|
2919
|
+
taskId: channel.taskId,
|
|
2920
|
+
agentId,
|
|
2921
|
+
aborted: true,
|
|
2922
|
+
});
|
|
2923
|
+
return true;
|
|
2924
|
+
}
|
|
2925
|
+
|
|
2926
|
+
private supersedeRecovery(channel: Channel, agentId: string): boolean {
|
|
2927
|
+
const recovery = channel.recoveryByAgent.get(agentId);
|
|
2928
|
+
if (!recovery) return false;
|
|
2929
|
+
recovery.superseded = true;
|
|
2930
|
+
// A live edit/removal now owns the terminal edge. Publish it before the
|
|
2931
|
+
// roster lifecycle can emit restarting/removed or bind a replacement.
|
|
2932
|
+
return this.emitRetiredTurnAbort(channel, agentId);
|
|
2933
|
+
}
|
|
2934
|
+
|
|
2935
|
+
private publishRosterChangeAbort(
|
|
2936
|
+
channel: Channel,
|
|
2937
|
+
agentId: string,
|
|
2938
|
+
change: RosterSessionChange,
|
|
2939
|
+
): void {
|
|
2940
|
+
if (change.abortPublished) return;
|
|
2941
|
+
if (!this.emitRetiredTurnAbort(channel, agentId)) {
|
|
2942
|
+
this.emitHost({
|
|
2943
|
+
kind: "agent.turn_complete",
|
|
2944
|
+
taskId: channel.taskId,
|
|
2945
|
+
agentId,
|
|
2946
|
+
aborted: true,
|
|
2947
|
+
});
|
|
2948
|
+
}
|
|
2949
|
+
change.abortPublished = true;
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
private recoveryIsCurrent(
|
|
2953
|
+
channel: Channel,
|
|
2954
|
+
agentId: string,
|
|
2955
|
+
recovery: RecoveryFence,
|
|
2956
|
+
): boolean {
|
|
2957
|
+
return (
|
|
2958
|
+
channel.recoveryByAgent.get(agentId) === recovery &&
|
|
2959
|
+
!recovery.superseded
|
|
2960
|
+
);
|
|
2961
|
+
}
|
|
2962
|
+
|
|
2188
2963
|
/**
|
|
2189
2964
|
* Retire turn-local state owned by a runner generation before replacement.
|
|
2190
2965
|
*
|
|
@@ -2618,6 +3393,13 @@ export class Orchestrator {
|
|
|
2618
3393
|
if (channel.sessions.get(agentId) !== session) break;
|
|
2619
3394
|
// The turn died with the session — drop its turn-state flags so a
|
|
2620
3395
|
// respawned session starts clean.
|
|
3396
|
+
const abortBoundaryOnRosterChange =
|
|
3397
|
+
this.needsAbortedTurnBoundary(channel, agentId);
|
|
3398
|
+
this.retainRetiredTurnBoundary(
|
|
3399
|
+
channel,
|
|
3400
|
+
agentId,
|
|
3401
|
+
abortBoundaryOnRosterChange,
|
|
3402
|
+
);
|
|
2621
3403
|
this.retireRunnerGeneration(channel, agentId);
|
|
2622
3404
|
// Claude under load (especially Docker Desktop macOS) occasionally
|
|
2623
3405
|
// unlinks ~/.claude.json mid-write during atomic config rewrites,
|
|
@@ -2627,13 +3409,32 @@ export class Orchestrator {
|
|
|
2627
3409
|
// session so the user keeps working instead of staring at a dead
|
|
2628
3410
|
// chat.
|
|
2629
3411
|
const agent = channel.roster.find((a) => a.id === agentId);
|
|
3412
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3413
|
+
// The authoritative roster already wants a different launch. Do not
|
|
3414
|
+
// rotate/recover this obsolete generation with its captured account;
|
|
3415
|
+
// preserve the live adapter so the normal stale-session recycler
|
|
3416
|
+
// closes it (and fences late Codex notifications). The cloud still
|
|
3417
|
+
// needs a terminal edge for any partial turn before restarting.
|
|
3418
|
+
this.reconcileErroredRosterChange(
|
|
3419
|
+
channel,
|
|
3420
|
+
agentId,
|
|
3421
|
+
event.message,
|
|
3422
|
+
abortBoundaryOnRosterChange,
|
|
3423
|
+
);
|
|
3424
|
+
break;
|
|
3425
|
+
}
|
|
2630
3426
|
// ADR-076: a rate-limit/usage-cap error fails over to another account
|
|
2631
3427
|
// for this kind (cool the current one, recreate the session on the
|
|
2632
3428
|
// next, re-deliver the in-flight prompt). No-op with <2 accounts.
|
|
2633
3429
|
if (
|
|
2634
3430
|
event.retryable === "rate_limit" &&
|
|
2635
3431
|
agent &&
|
|
2636
|
-
(await this.rotateAccount(
|
|
3432
|
+
(await this.rotateAccount(
|
|
3433
|
+
channel,
|
|
3434
|
+
agentId,
|
|
3435
|
+
agent,
|
|
3436
|
+
abortBoundaryOnRosterChange,
|
|
3437
|
+
))
|
|
2637
3438
|
) {
|
|
2638
3439
|
break;
|
|
2639
3440
|
}
|
|
@@ -2650,13 +3451,71 @@ export class Orchestrator {
|
|
|
2650
3451
|
`[orchestrator] ${channel.taskId}/${agentId}: ${agent.kind} credential-invalid pattern matched: ${event.message.slice(0, 300)}`,
|
|
2651
3452
|
);
|
|
2652
3453
|
const accountId = channel.accountByAgent.get(agentId) ?? null;
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
3454
|
+
// The broker is a slow await between a failed turn and its recovery.
|
|
3455
|
+
// Own the spawn slot so a concurrent roster edit cannot publish
|
|
3456
|
+
// ready for a replacement before this old generation publishes its
|
|
3457
|
+
// aborted boundary (events carry no generation id in the cloud).
|
|
3458
|
+
const ownsBrokerSlot = !channel.spawning.has(agentId);
|
|
3459
|
+
if (ownsBrokerSlot) channel.spawning.add(agentId);
|
|
3460
|
+
let requested = false;
|
|
3461
|
+
try {
|
|
3462
|
+
requested = await requestCredentialRefresh({
|
|
3463
|
+
kind: agent.kind,
|
|
3464
|
+
accountId:
|
|
3465
|
+
accountId && !accountId.startsWith("default:")
|
|
3466
|
+
? accountId
|
|
3467
|
+
: null,
|
|
3468
|
+
scope: channel.engineAccountScope,
|
|
3469
|
+
});
|
|
3470
|
+
} catch (err) {
|
|
3471
|
+
console.warn(
|
|
3472
|
+
`[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
|
|
3473
|
+
err instanceof Error ? err.message : String(err)
|
|
3474
|
+
}`,
|
|
3475
|
+
);
|
|
3476
|
+
} finally {
|
|
3477
|
+
if (ownsBrokerSlot) channel.spawning.delete(agentId);
|
|
3478
|
+
}
|
|
3479
|
+
// Removal/edit/recovery may have retired this generation while the
|
|
3480
|
+
// broker request was in flight. Never let its continuation mutate a
|
|
3481
|
+
// replacement or repopulate state for an absent roster id.
|
|
3482
|
+
if (channel.sessions.get(agentId) !== session) {
|
|
3483
|
+
// An edit can complete its restart while the broker request is
|
|
3484
|
+
// still outstanding, consuming the roster-change marker at bind.
|
|
3485
|
+
// The captured launch tuple still proves that this failed turn
|
|
3486
|
+
// belongs to the retired generation and needs its aborted edge.
|
|
3487
|
+
const stillRostered = channel.roster.some(
|
|
3488
|
+
(candidate) => candidate.id === agentId,
|
|
3489
|
+
);
|
|
3490
|
+
if (
|
|
3491
|
+
stillRostered &&
|
|
3492
|
+
(!channel.sessions.has(agentId) ||
|
|
3493
|
+
!this.hasDesiredLaunchSettings(channel, agentId, agent))
|
|
3494
|
+
) {
|
|
3495
|
+
this.reconcileErroredRosterChange(
|
|
3496
|
+
channel,
|
|
3497
|
+
agentId,
|
|
3498
|
+
event.message,
|
|
3499
|
+
abortBoundaryOnRosterChange,
|
|
3500
|
+
);
|
|
3501
|
+
}
|
|
3502
|
+
break;
|
|
3503
|
+
}
|
|
3504
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3505
|
+
this.reconcileErroredRosterChange(
|
|
3506
|
+
channel,
|
|
3507
|
+
agentId,
|
|
3508
|
+
event.message,
|
|
3509
|
+
abortBoundaryOnRosterChange,
|
|
3510
|
+
);
|
|
3511
|
+
break;
|
|
3512
|
+
}
|
|
2658
3513
|
if (requested) {
|
|
2659
|
-
await this.refreshAgentToken(
|
|
3514
|
+
await this.refreshAgentToken(
|
|
3515
|
+
channel,
|
|
3516
|
+
agent,
|
|
3517
|
+
abortBoundaryOnRosterChange,
|
|
3518
|
+
);
|
|
2660
3519
|
break;
|
|
2661
3520
|
}
|
|
2662
3521
|
}
|
|
@@ -2670,7 +3529,11 @@ export class Orchestrator {
|
|
|
2670
3529
|
console.warn(
|
|
2671
3530
|
`[orchestrator] ${channel.taskId}/${agentId}: claude auth-revoked pattern matched: ${event.message.slice(0, 300)}`,
|
|
2672
3531
|
);
|
|
2673
|
-
await this.refreshAgentToken(
|
|
3532
|
+
await this.refreshAgentToken(
|
|
3533
|
+
channel,
|
|
3534
|
+
agent,
|
|
3535
|
+
abortBoundaryOnRosterChange,
|
|
3536
|
+
);
|
|
2674
3537
|
break;
|
|
2675
3538
|
}
|
|
2676
3539
|
const configMissing = CLAUDE_CONFIG_MISSING_PATTERNS.some((re) =>
|
|
@@ -2681,10 +3544,15 @@ export class Orchestrator {
|
|
|
2681
3544
|
// than replaying the failed one. Do not leave it looking in-flight to
|
|
2682
3545
|
// a later account refresh.
|
|
2683
3546
|
channel.lastPrompt.delete(agentId);
|
|
2684
|
-
await this.recoverClaudeAgent(
|
|
3547
|
+
await this.recoverClaudeAgent(
|
|
3548
|
+
channel,
|
|
3549
|
+
agentId,
|
|
3550
|
+
abortBoundaryOnRosterChange,
|
|
3551
|
+
);
|
|
2685
3552
|
break;
|
|
2686
3553
|
}
|
|
2687
3554
|
channel.lastPrompt.delete(agentId);
|
|
3555
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
2688
3556
|
this.emitHost({
|
|
2689
3557
|
kind: "agent.exit",
|
|
2690
3558
|
taskId: channel.taskId,
|
|
@@ -2730,7 +3598,11 @@ export class Orchestrator {
|
|
|
2730
3598
|
});
|
|
2731
3599
|
if (remainingTurns === 0) {
|
|
2732
3600
|
channel.lastPrompt.delete(agentId);
|
|
2733
|
-
|
|
3601
|
+
// A queued Secretary reply must run on the replacement settings,
|
|
3602
|
+
// not extend the obsolete generation past its first safe boundary.
|
|
3603
|
+
if (!channel.rosterSessionChanges.has(agentId)) {
|
|
3604
|
+
this.startNextPendingPrompt(channel, agentId);
|
|
3605
|
+
}
|
|
2734
3606
|
}
|
|
2735
3607
|
// A config repair never interrupts a turn. The stale id remains queued
|
|
2736
3608
|
// until this exact boundary, then reconcile closes and replaces it.
|
|
@@ -2761,18 +3633,73 @@ export class Orchestrator {
|
|
|
2761
3633
|
if (channel.sessions.get(agentId) !== session) break;
|
|
2762
3634
|
// Same zombie hazard as the error path — a session whose process
|
|
2763
3635
|
// ended (even cleanly) can never carry another turn.
|
|
3636
|
+
const abortBoundaryOnRosterChange =
|
|
3637
|
+
this.needsAbortedTurnBoundary(channel, agentId);
|
|
2764
3638
|
this.retireRunnerGeneration(channel, agentId);
|
|
2765
3639
|
channel.lastPrompt.delete(agentId);
|
|
2766
3640
|
channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
|
|
2767
3641
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
2768
3642
|
channel.sessions.delete(agentId);
|
|
2769
|
-
channel.
|
|
3643
|
+
const rosterChangePending = channel.rosterSessionChanges.has(agentId);
|
|
3644
|
+
if (rosterChangePending && abortBoundaryOnRosterChange) {
|
|
3645
|
+
this.publishRosterChangeAbort(
|
|
3646
|
+
channel,
|
|
3647
|
+
agentId,
|
|
3648
|
+
channel.rosterSessionChanges.get(agentId)!,
|
|
3649
|
+
);
|
|
3650
|
+
}
|
|
3651
|
+
if (!rosterChangePending) channel.browserStaleSessions.delete(agentId);
|
|
2770
3652
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
2771
3653
|
this.reconcilePendingSecretary(channel, agentId);
|
|
3654
|
+
if (rosterChangePending) {
|
|
3655
|
+
void this.reconcileSessions(channel).catch((err: unknown) => {
|
|
3656
|
+
console.warn(
|
|
3657
|
+
`[orchestrator] ${channel.taskId}/${agentId}: roster-change exit reconcile failed: ${
|
|
3658
|
+
err instanceof Error ? err.message : String(err)
|
|
3659
|
+
}`,
|
|
3660
|
+
);
|
|
3661
|
+
});
|
|
3662
|
+
}
|
|
2772
3663
|
break;
|
|
2773
3664
|
}
|
|
2774
3665
|
}
|
|
2775
3666
|
|
|
3667
|
+
/** An engine error can race a live launch edit after a partial turn. Keep
|
|
3668
|
+
* the obsolete adapter owned until the stale recycler closes it, but close
|
|
3669
|
+
* the cloud turn boundary immediately so restarting/ready cannot leave the
|
|
3670
|
+
* crew UI stuck in a working state. */
|
|
3671
|
+
private reconcileErroredRosterChange(
|
|
3672
|
+
channel: Channel,
|
|
3673
|
+
agentId: string,
|
|
3674
|
+
reason: string,
|
|
3675
|
+
abortBoundary: boolean,
|
|
3676
|
+
): void {
|
|
3677
|
+
channel.lastPrompt.delete(agentId);
|
|
3678
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
3679
|
+
if (abortBoundary) {
|
|
3680
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
3681
|
+
if (change) this.publishRosterChangeAbort(channel, agentId, change);
|
|
3682
|
+
else this.emitRetiredTurnAbort(channel, agentId);
|
|
3683
|
+
}
|
|
3684
|
+
void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
|
|
3685
|
+
console.warn(
|
|
3686
|
+
`[orchestrator] ${channel.taskId}/${agentId}: roster-change error (${reason}) reconcile failed: ${
|
|
3687
|
+
err instanceof Error ? err.message : String(err)
|
|
3688
|
+
}`,
|
|
3689
|
+
);
|
|
3690
|
+
});
|
|
3691
|
+
}
|
|
3692
|
+
|
|
3693
|
+
/** Removal owns an asynchronous close/durable-stop fence. Recovery paths
|
|
3694
|
+
* overtaken by remove/re-add must join it before they can spawn the re-added
|
|
3695
|
+
* id, otherwise removal can stop the replacement generation. */
|
|
3696
|
+
private async reconcileAfterRosterRefresh(channel: Channel): Promise<void> {
|
|
3697
|
+
const rosterRefresh = channel.rosterRefreshReady;
|
|
3698
|
+
if (rosterRefresh) await rosterRefresh;
|
|
3699
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3700
|
+
await this.reconcileSessions(channel);
|
|
3701
|
+
}
|
|
3702
|
+
|
|
2776
3703
|
/** A daemon loss is infrastructure recovery, not an agent crash. Preserve
|
|
2777
3704
|
* accepted work, drop the dead generation, and leave respawn/exit accounting
|
|
2778
3705
|
* untouched; the runtime-ready hook recreates it deterministically. */
|
|
@@ -2797,9 +3724,20 @@ export class Orchestrator {
|
|
|
2797
3724
|
channel.runtimeDeferredAgents.add(agentId);
|
|
2798
3725
|
}
|
|
2799
3726
|
channel.sessions.delete(agentId);
|
|
2800
|
-
channel.
|
|
3727
|
+
if (!channel.rosterSessionChanges.has(agentId)) {
|
|
3728
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3729
|
+
}
|
|
2801
3730
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
2802
3731
|
await session.close().catch(() => {});
|
|
3732
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
3733
|
+
if (rosterChange && this.isActiveChannel(channel)) {
|
|
3734
|
+
await this.closeRosterChangedSession(
|
|
3735
|
+
channel,
|
|
3736
|
+
agentId,
|
|
3737
|
+
session,
|
|
3738
|
+
rosterChange,
|
|
3739
|
+
);
|
|
3740
|
+
}
|
|
2803
3741
|
}
|
|
2804
3742
|
|
|
2805
3743
|
// -- session recovery -----------------------------------------------------
|
|
@@ -2812,6 +3750,7 @@ export class Orchestrator {
|
|
|
2812
3750
|
private async recoverClaudeAgent(
|
|
2813
3751
|
channel: Channel,
|
|
2814
3752
|
agentId: string,
|
|
3753
|
+
abortBoundaryOnRosterChange = false,
|
|
2815
3754
|
): Promise<void> {
|
|
2816
3755
|
if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
|
|
2817
3756
|
|
|
@@ -2823,6 +3762,7 @@ export class Orchestrator {
|
|
|
2823
3762
|
channel.sessions.delete(agentId);
|
|
2824
3763
|
channel.browserStaleSessions.delete(agentId);
|
|
2825
3764
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
3765
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
2826
3766
|
this.emitHost({
|
|
2827
3767
|
kind: "agent.exit",
|
|
2828
3768
|
taskId: channel.taskId,
|
|
@@ -2841,6 +3781,15 @@ export class Orchestrator {
|
|
|
2841
3781
|
channel.containerName,
|
|
2842
3782
|
channel.agentHandle ?? undefined,
|
|
2843
3783
|
);
|
|
3784
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3785
|
+
this.reconcileErroredRosterChange(
|
|
3786
|
+
channel,
|
|
3787
|
+
agentId,
|
|
3788
|
+
"Claude config recovery superseded by live roster edit",
|
|
3789
|
+
abortBoundaryOnRosterChange,
|
|
3790
|
+
);
|
|
3791
|
+
return;
|
|
3792
|
+
}
|
|
2844
3793
|
|
|
2845
3794
|
// Delete the old generation before awaiting close. Its adapter emits a
|
|
2846
3795
|
// terminal event during teardown; bindSession's identity guard must see
|
|
@@ -2848,7 +3797,9 @@ export class Orchestrator {
|
|
|
2848
3797
|
// a second replacement path.
|
|
2849
3798
|
const old = channel.sessions.get(agentId);
|
|
2850
3799
|
channel.sessions.delete(agentId);
|
|
2851
|
-
channel.
|
|
3800
|
+
if (!channel.rosterSessionChanges.has(agentId)) {
|
|
3801
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3802
|
+
}
|
|
2852
3803
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
2853
3804
|
this.retireRunnerGeneration(channel, agentId);
|
|
2854
3805
|
if (old) {
|
|
@@ -2859,6 +3810,27 @@ export class Orchestrator {
|
|
|
2859
3810
|
}
|
|
2860
3811
|
}
|
|
2861
3812
|
if (!this.isActiveChannel(channel)) return;
|
|
3813
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
3814
|
+
if (rosterChange) {
|
|
3815
|
+
// The generic recovery owned the close before the roster edit arrived.
|
|
3816
|
+
// close() is idempotent; route its completed boundary through the
|
|
3817
|
+
// roster lifecycle so the cloud still observes restarting -> ready.
|
|
3818
|
+
this.reconcileErroredRosterChange(
|
|
3819
|
+
channel,
|
|
3820
|
+
agentId,
|
|
3821
|
+
"Claude config recovery superseded by live roster edit",
|
|
3822
|
+
abortBoundaryOnRosterChange,
|
|
3823
|
+
);
|
|
3824
|
+
if (old) {
|
|
3825
|
+
await this.closeRosterChangedSession(
|
|
3826
|
+
channel,
|
|
3827
|
+
agentId,
|
|
3828
|
+
old,
|
|
3829
|
+
rosterChange,
|
|
3830
|
+
);
|
|
3831
|
+
}
|
|
3832
|
+
return;
|
|
3833
|
+
}
|
|
2862
3834
|
|
|
2863
3835
|
const agent = channel.roster.find((a) => a.id === agentId);
|
|
2864
3836
|
const task = getHostTask(channel.taskId);
|
|
@@ -2887,18 +3859,40 @@ export class Orchestrator {
|
|
|
2887
3859
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2888
3860
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2889
3861
|
channel,
|
|
2890
|
-
|
|
3862
|
+
agent,
|
|
2891
3863
|
),
|
|
2892
3864
|
agentEnv: boundAccount
|
|
2893
3865
|
? { ...boundAccount.execEnv, ...base }
|
|
2894
3866
|
: this.accountAgentEnv(channel, agent, base),
|
|
2895
3867
|
});
|
|
2896
|
-
if (!session)
|
|
3868
|
+
if (!session) {
|
|
3869
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3870
|
+
this.reconcileErroredRosterChange(
|
|
3871
|
+
channel,
|
|
3872
|
+
agentId,
|
|
3873
|
+
"Claude config recovery superseded by live roster edit",
|
|
3874
|
+
abortBoundaryOnRosterChange,
|
|
3875
|
+
);
|
|
3876
|
+
}
|
|
3877
|
+
return;
|
|
3878
|
+
}
|
|
3879
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
3880
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3881
|
+
this.reconcileErroredRosterChange(
|
|
3882
|
+
channel,
|
|
3883
|
+
agentId,
|
|
3884
|
+
"Claude config recovery superseded by live roster edit",
|
|
3885
|
+
abortBoundaryOnRosterChange,
|
|
3886
|
+
);
|
|
3887
|
+
}
|
|
3888
|
+
return;
|
|
3889
|
+
}
|
|
2897
3890
|
this.startNextPendingPrompt(channel, agentId);
|
|
2898
3891
|
|
|
2899
3892
|
const message = restored
|
|
2900
3893
|
? `${agentId} restarted — config file was missing in the container, restored from the host.`
|
|
2901
3894
|
: `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
|
|
3895
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
2902
3896
|
this.emitHost({
|
|
2903
3897
|
kind: "agent.exit",
|
|
2904
3898
|
taskId: channel.taskId,
|
|
@@ -2940,8 +3934,18 @@ export class Orchestrator {
|
|
|
2940
3934
|
channel: Channel,
|
|
2941
3935
|
agentId: string,
|
|
2942
3936
|
agent: RosterAgent,
|
|
3937
|
+
abortBoundaryOnRosterChange = false,
|
|
2943
3938
|
): Promise<boolean> {
|
|
2944
3939
|
if (!this.isActiveChannel(channel)) return true;
|
|
3940
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3941
|
+
this.reconcileErroredRosterChange(
|
|
3942
|
+
channel,
|
|
3943
|
+
agentId,
|
|
3944
|
+
"account rotation superseded by live roster edit",
|
|
3945
|
+
abortBoundaryOnRosterChange,
|
|
3946
|
+
);
|
|
3947
|
+
return true;
|
|
3948
|
+
}
|
|
2945
3949
|
// A concurrent terminal event for the same session is already being
|
|
2946
3950
|
// handled by the path that owns this spawn slot.
|
|
2947
3951
|
if (channel.spawning.has(agentId)) return true;
|
|
@@ -2964,6 +3968,8 @@ export class Orchestrator {
|
|
|
2964
3968
|
const task = getHostTask(channel.taskId);
|
|
2965
3969
|
if (!task || task.statusMirror !== "running") return false;
|
|
2966
3970
|
|
|
3971
|
+
const recovery: RecoveryFence = { superseded: false };
|
|
3972
|
+
channel.recoveryByAgent.set(agentId, recovery);
|
|
2967
3973
|
let replacement: AgentSession | null = null;
|
|
2968
3974
|
channel.spawning.add(agentId);
|
|
2969
3975
|
try {
|
|
@@ -2974,6 +3980,11 @@ export class Orchestrator {
|
|
|
2974
3980
|
channel.sessions.delete(agentId);
|
|
2975
3981
|
channel.browserStaleSessions.delete(agentId);
|
|
2976
3982
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
3983
|
+
this.retainRetiredTurnBoundary(
|
|
3984
|
+
channel,
|
|
3985
|
+
agentId,
|
|
3986
|
+
abortBoundaryOnRosterChange,
|
|
3987
|
+
);
|
|
2977
3988
|
this.retireRunnerGeneration(channel, agentId);
|
|
2978
3989
|
if (old) {
|
|
2979
3990
|
try {
|
|
@@ -2983,6 +3994,25 @@ export class Orchestrator {
|
|
|
2983
3994
|
}
|
|
2984
3995
|
}
|
|
2985
3996
|
if (!this.isActiveChannel(channel)) return true;
|
|
3997
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
3998
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
3999
|
+
if (rosterChange) {
|
|
4000
|
+
this.reconcileErroredRosterChange(
|
|
4001
|
+
channel,
|
|
4002
|
+
agentId,
|
|
4003
|
+
"account rotation superseded by live roster edit",
|
|
4004
|
+
abortBoundaryOnRosterChange,
|
|
4005
|
+
);
|
|
4006
|
+
if (old) {
|
|
4007
|
+
await this.closeRosterChangedSession(
|
|
4008
|
+
channel,
|
|
4009
|
+
agentId,
|
|
4010
|
+
old,
|
|
4011
|
+
rosterChange,
|
|
4012
|
+
);
|
|
4013
|
+
}
|
|
4014
|
+
return true;
|
|
4015
|
+
}
|
|
2986
4016
|
|
|
2987
4017
|
// The serialized materialization gate owns both account provisioning
|
|
2988
4018
|
// and, when enabled, browser migration.
|
|
@@ -2992,8 +4022,21 @@ export class Orchestrator {
|
|
|
2992
4022
|
true,
|
|
2993
4023
|
);
|
|
2994
4024
|
if (!this.isActiveChannel(channel)) return true;
|
|
4025
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
2995
4026
|
await this.ensureMcpConfig(channel);
|
|
2996
4027
|
if (!this.isActiveChannel(channel)) return true;
|
|
4028
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4029
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4030
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4031
|
+
this.reconcileErroredRosterChange(
|
|
4032
|
+
channel,
|
|
4033
|
+
agentId,
|
|
4034
|
+
"account rotation superseded by live roster edit",
|
|
4035
|
+
abortBoundaryOnRosterChange,
|
|
4036
|
+
);
|
|
4037
|
+
}
|
|
4038
|
+
return true;
|
|
4039
|
+
}
|
|
2997
4040
|
|
|
2998
4041
|
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
2999
4042
|
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
@@ -3018,12 +4061,34 @@ export class Orchestrator {
|
|
|
3018
4061
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
3019
4062
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
3020
4063
|
channel,
|
|
3021
|
-
|
|
4064
|
+
agent,
|
|
3022
4065
|
),
|
|
3023
4066
|
agentEnv: { ...next.execEnv, ...base },
|
|
3024
4067
|
},
|
|
4068
|
+
recovery,
|
|
3025
4069
|
);
|
|
3026
|
-
if (!replacement)
|
|
4070
|
+
if (!replacement) {
|
|
4071
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4072
|
+
this.reconcileErroredRosterChange(
|
|
4073
|
+
channel,
|
|
4074
|
+
agentId,
|
|
4075
|
+
"account rotation superseded by live roster edit",
|
|
4076
|
+
abortBoundaryOnRosterChange,
|
|
4077
|
+
);
|
|
4078
|
+
}
|
|
4079
|
+
return true;
|
|
4080
|
+
}
|
|
4081
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4082
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4083
|
+
this.reconcileErroredRosterChange(
|
|
4084
|
+
channel,
|
|
4085
|
+
agentId,
|
|
4086
|
+
"account rotation superseded by live roster edit",
|
|
4087
|
+
abortBoundaryOnRosterChange,
|
|
4088
|
+
);
|
|
4089
|
+
}
|
|
4090
|
+
return true;
|
|
4091
|
+
}
|
|
3027
4092
|
|
|
3028
4093
|
this.emitSystemNote(
|
|
3029
4094
|
channel.taskId,
|
|
@@ -3033,14 +4098,35 @@ export class Orchestrator {
|
|
|
3033
4098
|
// Re-deliver the in-flight prompt so the interrupted turn resumes.
|
|
3034
4099
|
const prompt = channel.lastPrompt.get(agentId);
|
|
3035
4100
|
if (prompt) {
|
|
4101
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
3036
4102
|
this.startPrompt(channel, agentId, replacement, prompt);
|
|
3037
4103
|
} else {
|
|
4104
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
3038
4105
|
this.startNextPendingPrompt(channel, agentId);
|
|
3039
4106
|
}
|
|
3040
4107
|
} finally {
|
|
3041
4108
|
channel.spawning.delete(agentId);
|
|
4109
|
+
if (channel.recoveryByAgent.get(agentId) === recovery) {
|
|
4110
|
+
channel.recoveryByAgent.delete(agentId);
|
|
4111
|
+
}
|
|
3042
4112
|
if (!replacement) {
|
|
4113
|
+
if (!channel.roster.some((candidate) => candidate.id === agentId)) {
|
|
4114
|
+
channel.accountByAgent.delete(agentId);
|
|
4115
|
+
channel.rotations.delete(agentId);
|
|
4116
|
+
}
|
|
3043
4117
|
this.reconcilePendingSecretary(channel, agentId);
|
|
4118
|
+
if (
|
|
4119
|
+
this.isActiveChannel(channel) &&
|
|
4120
|
+
(recovery.superseded || channel.rosterSessionChanges.has(agentId))
|
|
4121
|
+
) {
|
|
4122
|
+
void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
|
|
4123
|
+
console.warn(
|
|
4124
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-rotation roster reconcile failed: ${
|
|
4125
|
+
err instanceof Error ? err.message : String(err)
|
|
4126
|
+
}`,
|
|
4127
|
+
);
|
|
4128
|
+
});
|
|
4129
|
+
}
|
|
3044
4130
|
}
|
|
3045
4131
|
}
|
|
3046
4132
|
|
|
@@ -3077,10 +4163,23 @@ export class Orchestrator {
|
|
|
3077
4163
|
agent: RosterAgent,
|
|
3078
4164
|
account: ResolvedEngineAccount,
|
|
3079
4165
|
note: string,
|
|
4166
|
+
abortBoundaryOnRosterChange = false,
|
|
3080
4167
|
): Promise<boolean> {
|
|
4168
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4169
|
+
this.reconcileErroredRosterChange(
|
|
4170
|
+
channel,
|
|
4171
|
+
agentId,
|
|
4172
|
+
"credential respawn superseded by live roster edit",
|
|
4173
|
+
abortBoundaryOnRosterChange,
|
|
4174
|
+
);
|
|
4175
|
+
return true;
|
|
4176
|
+
}
|
|
4177
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return true;
|
|
3081
4178
|
const task = getHostTask(channel.taskId);
|
|
3082
4179
|
if (!task || task.statusMirror !== "running") return false;
|
|
3083
4180
|
|
|
4181
|
+
const recovery: RecoveryFence = { superseded: false };
|
|
4182
|
+
channel.recoveryByAgent.set(agentId, recovery);
|
|
3084
4183
|
let replacement: AgentSession | null = null;
|
|
3085
4184
|
channel.spawning.add(agentId);
|
|
3086
4185
|
try {
|
|
@@ -3102,6 +4201,25 @@ export class Orchestrator {
|
|
|
3102
4201
|
}
|
|
3103
4202
|
}
|
|
3104
4203
|
if (!this.isActiveChannel(channel)) return true;
|
|
4204
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4205
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
4206
|
+
if (rosterChange) {
|
|
4207
|
+
this.reconcileErroredRosterChange(
|
|
4208
|
+
channel,
|
|
4209
|
+
agentId,
|
|
4210
|
+
"credential respawn superseded by live roster edit",
|
|
4211
|
+
abortBoundaryOnRosterChange,
|
|
4212
|
+
);
|
|
4213
|
+
if (old) {
|
|
4214
|
+
await this.closeRosterChangedSession(
|
|
4215
|
+
channel,
|
|
4216
|
+
agentId,
|
|
4217
|
+
old,
|
|
4218
|
+
rosterChange,
|
|
4219
|
+
);
|
|
4220
|
+
}
|
|
4221
|
+
return true;
|
|
4222
|
+
}
|
|
3105
4223
|
|
|
3106
4224
|
await this.ensureBrowserSetup(
|
|
3107
4225
|
channel,
|
|
@@ -3109,8 +4227,21 @@ export class Orchestrator {
|
|
|
3109
4227
|
true,
|
|
3110
4228
|
);
|
|
3111
4229
|
if (!this.isActiveChannel(channel)) return true;
|
|
4230
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
3112
4231
|
await this.ensureMcpConfig(channel);
|
|
3113
4232
|
if (!this.isActiveChannel(channel)) return true;
|
|
4233
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4234
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4235
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4236
|
+
this.reconcileErroredRosterChange(
|
|
4237
|
+
channel,
|
|
4238
|
+
agentId,
|
|
4239
|
+
"credential respawn superseded by live roster edit",
|
|
4240
|
+
abortBoundaryOnRosterChange,
|
|
4241
|
+
);
|
|
4242
|
+
}
|
|
4243
|
+
return true;
|
|
4244
|
+
}
|
|
3114
4245
|
|
|
3115
4246
|
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
3116
4247
|
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
@@ -3125,32 +4256,79 @@ export class Orchestrator {
|
|
|
3125
4256
|
channel.accountByAgent.set(agentId, account.id);
|
|
3126
4257
|
noteEngineAccountUsed(account.id);
|
|
3127
4258
|
|
|
3128
|
-
replacement = await this.createAndBindStableSession(
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
channel,
|
|
3135
|
-
agentId,
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
4259
|
+
replacement = await this.createAndBindStableSession(
|
|
4260
|
+
channel,
|
|
4261
|
+
agentId,
|
|
4262
|
+
{
|
|
4263
|
+
taskId: channel.taskId,
|
|
4264
|
+
agent,
|
|
4265
|
+
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
4266
|
+
executionProfile: this.executionProfileFor(channel, agentId),
|
|
4267
|
+
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
4268
|
+
channel,
|
|
4269
|
+
agent,
|
|
4270
|
+
),
|
|
4271
|
+
agentEnv: { ...account.execEnv, ...base },
|
|
4272
|
+
},
|
|
4273
|
+
recovery,
|
|
4274
|
+
);
|
|
4275
|
+
if (!replacement) {
|
|
4276
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4277
|
+
this.reconcileErroredRosterChange(
|
|
4278
|
+
channel,
|
|
4279
|
+
agentId,
|
|
4280
|
+
"credential respawn superseded by live roster edit",
|
|
4281
|
+
abortBoundaryOnRosterChange,
|
|
4282
|
+
);
|
|
4283
|
+
}
|
|
4284
|
+
return true;
|
|
4285
|
+
}
|
|
4286
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4287
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4288
|
+
this.reconcileErroredRosterChange(
|
|
4289
|
+
channel,
|
|
4290
|
+
agentId,
|
|
4291
|
+
"credential respawn superseded by live roster edit",
|
|
4292
|
+
abortBoundaryOnRosterChange,
|
|
4293
|
+
);
|
|
4294
|
+
}
|
|
4295
|
+
return true;
|
|
4296
|
+
}
|
|
3140
4297
|
|
|
3141
4298
|
this.emitSystemNote(channel.taskId, note);
|
|
3142
4299
|
|
|
3143
4300
|
// Re-deliver the in-flight prompt so the interrupted turn resumes.
|
|
3144
4301
|
const prompt = channel.lastPrompt.get(agentId);
|
|
3145
4302
|
if (prompt) {
|
|
4303
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
3146
4304
|
this.startPrompt(channel, agentId, replacement, prompt);
|
|
3147
4305
|
} else {
|
|
4306
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
3148
4307
|
this.startNextPendingPrompt(channel, agentId);
|
|
3149
4308
|
}
|
|
3150
4309
|
} finally {
|
|
3151
4310
|
channel.spawning.delete(agentId);
|
|
4311
|
+
if (channel.recoveryByAgent.get(agentId) === recovery) {
|
|
4312
|
+
channel.recoveryByAgent.delete(agentId);
|
|
4313
|
+
}
|
|
3152
4314
|
if (!replacement) {
|
|
4315
|
+
if (!channel.roster.some((candidate) => candidate.id === agentId)) {
|
|
4316
|
+
channel.accountByAgent.delete(agentId);
|
|
4317
|
+
channel.rotations.delete(agentId);
|
|
4318
|
+
}
|
|
3153
4319
|
this.reconcilePendingSecretary(channel, agentId);
|
|
4320
|
+
if (
|
|
4321
|
+
this.isActiveChannel(channel) &&
|
|
4322
|
+
(recovery.superseded || channel.rosterSessionChanges.has(agentId))
|
|
4323
|
+
) {
|
|
4324
|
+
void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
|
|
4325
|
+
console.warn(
|
|
4326
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-respawn roster reconcile failed: ${
|
|
4327
|
+
err instanceof Error ? err.message : String(err)
|
|
4328
|
+
}`,
|
|
4329
|
+
);
|
|
4330
|
+
});
|
|
4331
|
+
}
|
|
3154
4332
|
}
|
|
3155
4333
|
}
|
|
3156
4334
|
|
|
@@ -3179,14 +4357,21 @@ export class Orchestrator {
|
|
|
3179
4357
|
private async refreshAgentToken(
|
|
3180
4358
|
channel: Channel,
|
|
3181
4359
|
agent: RosterAgent,
|
|
4360
|
+
abortBoundaryOnRosterChange?: boolean,
|
|
3182
4361
|
): Promise<void> {
|
|
3183
4362
|
const agentId = agent.id;
|
|
3184
4363
|
if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
|
|
4364
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return;
|
|
4365
|
+
const abortBoundary =
|
|
4366
|
+
abortBoundaryOnRosterChange ??
|
|
4367
|
+
this.needsAbortedTurnBoundary(channel, agentId);
|
|
4368
|
+
this.retainRetiredTurnBoundary(channel, agentId, abortBoundary);
|
|
3185
4369
|
|
|
3186
4370
|
const tries = (channel.respawns.get(agentId) ?? 0) + 1;
|
|
3187
4371
|
channel.respawns.set(agentId, tries);
|
|
3188
4372
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
3189
4373
|
if (tries > MAX_RESPAWNS_PER_AGENT) {
|
|
4374
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
3190
4375
|
this.emitHost({
|
|
3191
4376
|
kind: "agent.exit",
|
|
3192
4377
|
taskId: channel.taskId,
|
|
@@ -3203,7 +4388,10 @@ export class Orchestrator {
|
|
|
3203
4388
|
const account =
|
|
3204
4389
|
accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
|
|
3205
4390
|
accounts[0];
|
|
3206
|
-
if (!account)
|
|
4391
|
+
if (!account) {
|
|
4392
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
4393
|
+
return;
|
|
4394
|
+
}
|
|
3207
4395
|
|
|
3208
4396
|
await this.respawnAgentOnAccount(
|
|
3209
4397
|
channel,
|
|
@@ -3211,6 +4399,7 @@ export class Orchestrator {
|
|
|
3211
4399
|
agent,
|
|
3212
4400
|
account,
|
|
3213
4401
|
`${agentId}: ${agent.kind} credential refreshed — restarted the agent.`,
|
|
4402
|
+
abortBoundary,
|
|
3214
4403
|
);
|
|
3215
4404
|
}
|
|
3216
4405
|
|
|
@@ -3252,9 +4441,21 @@ export class Orchestrator {
|
|
|
3252
4441
|
async refreshEngineAccountAgents(kind: string): Promise<void> {
|
|
3253
4442
|
for (const channel of this.channels.values()) {
|
|
3254
4443
|
if (!this.isActiveChannel(channel)) continue;
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
4444
|
+
const agentIds = channel.roster
|
|
4445
|
+
.filter((agent) => agent.kind === kind)
|
|
4446
|
+
.map((agent) => agent.id);
|
|
4447
|
+
for (const agentId of agentIds) {
|
|
4448
|
+
const agent = channel.roster.find(
|
|
4449
|
+
(candidate) => candidate.id === agentId,
|
|
4450
|
+
);
|
|
4451
|
+
if (
|
|
4452
|
+
!agent ||
|
|
4453
|
+
agent.kind !== kind ||
|
|
4454
|
+
!channel.sessions.has(agentId)
|
|
4455
|
+
) {
|
|
4456
|
+
continue;
|
|
4457
|
+
}
|
|
4458
|
+
channel.respawns.set(agentId, 0);
|
|
3258
4459
|
try {
|
|
3259
4460
|
await this.refreshAgentToken(channel, agent);
|
|
3260
4461
|
} catch (err) {
|
|
@@ -3299,7 +4500,10 @@ export class Orchestrator {
|
|
|
3299
4500
|
this.channels.delete(taskId);
|
|
3300
4501
|
this.clearBrowserRetry(ch);
|
|
3301
4502
|
ch.browserStaleSessions.clear();
|
|
4503
|
+
ch.rosterSessionChanges.clear();
|
|
3302
4504
|
ch.browserPendingStaleSessions.clear();
|
|
4505
|
+
ch.browserConfigAffectedKinds.clear();
|
|
4506
|
+
ch.mcpProvisionAffectedKinds.clear();
|
|
3303
4507
|
ch.activeTurns.clear();
|
|
3304
4508
|
ch.acceptedPrompts.clear();
|
|
3305
4509
|
ch.pendingPrompts.clear();
|
|
@@ -3307,17 +4511,22 @@ export class Orchestrator {
|
|
|
3307
4511
|
ch.runtimeDeferredAgents.clear();
|
|
3308
4512
|
ch.openTurns.clear();
|
|
3309
4513
|
ch.interrupted.clear();
|
|
4514
|
+
ch.retiredTurnBoundaries.clear();
|
|
4515
|
+
ch.recoveryByAgent.clear();
|
|
3310
4516
|
ch.reconcileAgain = false;
|
|
3311
4517
|
const drains = [
|
|
3312
4518
|
ch.browserSetup,
|
|
3313
4519
|
ch.mcpConfigWrite,
|
|
3314
4520
|
ch.reconcileReady,
|
|
4521
|
+
ch.rosterRefreshReady,
|
|
4522
|
+
ch.persistedRosterReady,
|
|
3315
4523
|
ch.sessionsReady,
|
|
3316
4524
|
...ch.factoryOperations,
|
|
3317
4525
|
...ch.eventOperations,
|
|
3318
4526
|
].filter((promise): promise is Promise<unknown> => promise !== null);
|
|
3319
4527
|
const sessions = [...ch.sessions.values()];
|
|
3320
4528
|
ch.sessions.clear();
|
|
4529
|
+
ch.factoryOperationByAgent.clear();
|
|
3321
4530
|
const closes = sessions.map((session) =>
|
|
3322
4531
|
session.close().catch(() => {
|
|
3323
4532
|
// Teardown is best-effort; every adapter close is intended idempotent.
|
|
@@ -3441,6 +4650,18 @@ export class Orchestrator {
|
|
|
3441
4650
|
}
|
|
3442
4651
|
}
|
|
3443
4652
|
|
|
4653
|
+
/** Launch-time settings only. Wire-compatible absence spellings are equal. */
|
|
4654
|
+
function rosterLaunchChangeReason(
|
|
4655
|
+
previous: RosterAgent,
|
|
4656
|
+
next: RosterAgent,
|
|
4657
|
+
): string | null {
|
|
4658
|
+
const changed: string[] = [];
|
|
4659
|
+
if (previous.kind !== next.kind) changed.push("engine");
|
|
4660
|
+
if ((previous.model ?? "") !== (next.model ?? "")) changed.push("model");
|
|
4661
|
+
if ((previous.effort ?? "") !== (next.effort ?? "")) changed.push("effort");
|
|
4662
|
+
return changed.length > 0 ? `${changed.join("/")} change` : null;
|
|
4663
|
+
}
|
|
4664
|
+
|
|
3444
4665
|
/** Exact host-wire role validation; never trims or canonicalises agent ids. */
|
|
3445
4666
|
export function hasValidSecretarySelection(
|
|
3446
4667
|
spec: Pick<
|