@runuai/host 0.9.147 → 0.9.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0024_task_env_mode.sql +9 -0
- package/db/migrations/0025_project_env_authority.sql +147 -0
- package/db/migrations/0026_project_env_replica.sql +150 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +312 -2
- package/images/standard/Dockerfile +44 -6
- package/images/standard/container/uai-init +85 -2
- package/images/standard/container/uai-task-forwarder.mjs +303 -0
- package/lib/agent-cli.ts +18 -1
- package/lib/agent.ts +99 -34
- package/lib/agents/attach-compatibility.ts +51 -0
- package/lib/agents/codex.ts +137 -13
- package/lib/agents/cursor.ts +15 -1
- package/lib/agents/durable-proc.ts +2 -0
- package/lib/agents/grok.ts +8 -0
- package/lib/agents/kimi.ts +1 -0
- package/lib/agents/machine-durable.ts +17 -0
- package/lib/agents/opencode.ts +1 -0
- package/lib/agents/proc.ts +6 -1
- package/lib/agents/session-state.ts +107 -0
- package/lib/agents/transport.ts +125 -58
- package/lib/agents/types.ts +3 -3
- package/lib/browser-testing.ts +6 -0
- package/lib/codex-auth.ts +2 -0
- package/lib/credential-store.ts +141 -6
- package/lib/engine-accounts.ts +1 -0
- package/lib/env-file.ts +7 -0
- package/lib/env-schema.ts +841 -51
- package/lib/git-diff.ts +1 -0
- package/lib/git-identity.ts +2 -0
- package/lib/github-tokens.ts +1 -0
- package/lib/host-env.ts +98 -18
- package/lib/lease.ts +11 -3
- package/lib/machine-gateway-tunnel.ts +195 -17
- package/lib/machine-project-env-loader.ts +152 -0
- package/lib/managed-runtime.ts +270 -51
- package/lib/mcp-gateway.ts +777 -26
- package/lib/orchestrator.ts +1681 -138
- package/lib/project-env-authority.ts +1319 -0
- package/lib/project-env-feature.ts +74 -0
- package/lib/project-env-loader.ts +237 -0
- package/lib/project-env-replica.ts +2094 -0
- package/lib/project-env-schema-capture.ts +342 -0
- package/lib/project-env-staging.ts +148 -0
- package/lib/project-env-task-policy.ts +176 -0
- package/lib/runtime-archive.ts +79 -16
- package/lib/runtime-authority.ts +44 -0
- package/lib/runtime-state.ts +175 -10
- package/lib/secrets-proxy/address.ts +72 -0
- package/lib/secrets-proxy/audit.ts +30 -0
- package/lib/secrets-proxy/ca-store.ts +200 -0
- package/lib/secrets-proxy/ca.ts +470 -0
- package/lib/secrets-proxy/container-steps.ts +138 -0
- package/lib/secrets-proxy/der.ts +273 -0
- package/lib/secrets-proxy/engine.ts +1633 -0
- package/lib/secrets-proxy/host.ts +93 -0
- package/lib/secrets-proxy/hostname.ts +60 -0
- package/lib/secrets-proxy/placeholders.ts +196 -0
- package/lib/secrets-proxy/policy.ts +141 -0
- package/lib/secrets-proxy/readiness.ts +148 -0
- package/lib/secrets-proxy/runtime.ts +582 -0
- package/lib/secrets-proxy/scrub.ts +559 -0
- package/lib/secrets-proxy/sidecar-view.ts +139 -0
- package/lib/secrets-proxy/task-scripts.ts +262 -0
- package/lib/secrets-proxy/task-trust.ts +574 -0
- package/lib/secrets-proxy/trust-watch.ts +80 -0
- package/lib/shared-files-s3.ts +3 -0
- package/lib/sidecar-guard.ts +58 -0
- package/lib/sidecar-resume.ts +51 -3
- package/lib/standard-image.ts +50 -12
- package/lib/task-env-forwarder.ts +228 -0
- package/lib/task-env-source.ts +193 -0
- package/lib/task-environment/apple-container.ts +45 -11
- package/lib/task-environment/docker.ts +51 -12
- package/lib/task-environment/index.ts +6 -2
- package/lib/task-environment/machine-task-up.ts +258 -25
- package/lib/task-environment/machine.ts +80 -22
- package/lib/task-environment/registry.ts +3 -2
- package/lib/task-environment/self-identity.ts +20 -0
- package/lib/task-environment/self.ts +92 -16
- package/lib/task-environment/types.ts +63 -0
- package/lib/task-gateway-port.ts +22 -0
- package/lib/task-gateway-protocol.ts +190 -0
- package/lib/task-gateway-recovery.ts +44 -0
- package/lib/task-gateway-state.ts +497 -0
- package/lib/transcript.ts +1 -0
- package/lib/ulid.ts +10 -1
- package/package.json +13 -1
- package/scripts/agent/env-hook.sh +626 -0
- package/scripts/agent/task-up.sh +476 -27
- package/scripts/agent/uai-env-exec.sh +26 -0
- package/src/cli.ts +26 -3
- package/src/index.ts +273 -19
- package/src/main.ts +941 -48
- package/src/project-env-latest.ts +130 -0
- package/src/project-env-policy.ts +281 -0
- package/src/project-env-wording.ts +47 -0
- package/src/protocol.ts +2239 -15
package/lib/orchestrator.ts
CHANGED
|
@@ -34,6 +34,11 @@ import { getDb, schema } from "./db";
|
|
|
34
34
|
import { mockAgentFactory } from "./agents/mock";
|
|
35
35
|
import { realAgentFactory } from "./agents/factory";
|
|
36
36
|
import { resolveAgentMode } from "./agents/mode";
|
|
37
|
+
import {
|
|
38
|
+
clearPersistedCodexThread,
|
|
39
|
+
persistedAgentIds,
|
|
40
|
+
stopPersistedAgentSession,
|
|
41
|
+
} from "./agents/session-state";
|
|
37
42
|
import {
|
|
38
43
|
type AgentEvent,
|
|
39
44
|
type AgentSession,
|
|
@@ -45,7 +50,12 @@ import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
|
|
|
45
50
|
import { isSafeHostTaskId } from "./task-identity";
|
|
46
51
|
import { projectEnvForTask } from "./host-env";
|
|
47
52
|
import { shadowRoute } from "./routing/shadow";
|
|
48
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
getHostTask,
|
|
55
|
+
hostTaskEnvMode,
|
|
56
|
+
hostTaskProjectIds,
|
|
57
|
+
upsertHostTask,
|
|
58
|
+
} from "./runtime-state";
|
|
49
59
|
import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
|
|
50
60
|
import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
|
|
51
61
|
import { dockerCli } from "./docker-exec";
|
|
@@ -78,7 +88,20 @@ import {
|
|
|
78
88
|
appleTaskStoreVolumeName,
|
|
79
89
|
parseAppleTaskEnvironmentLocator,
|
|
80
90
|
} from "./task-environment/apple-container";
|
|
81
|
-
import {
|
|
91
|
+
import {
|
|
92
|
+
DOCKER_TASK_ENVIRONMENT_PROVIDER,
|
|
93
|
+
parseDockerTaskEnvironmentLocator,
|
|
94
|
+
} from "./task-environment/docker";
|
|
95
|
+
import { APPLE_TASK_ENVIRONMENT_PROVIDER } from "./task-environment/apple-container";
|
|
96
|
+
import { activeSecretsProxy } from "./secrets-proxy/host";
|
|
97
|
+
import { usableProxiedKeys } from "./secrets-proxy/policy";
|
|
98
|
+
import { forgetTaskProxyReadiness } from "./secrets-proxy/readiness";
|
|
99
|
+
import {
|
|
100
|
+
invalidateTaskWrites,
|
|
101
|
+
applyTaskRootSteps,
|
|
102
|
+
taskWritesQuiescing,
|
|
103
|
+
type RecoveryProxyFailure,
|
|
104
|
+
} from "./secrets-proxy/task-trust";
|
|
82
105
|
import type {
|
|
83
106
|
TaskEnvironmentDescriptor,
|
|
84
107
|
TaskEnvironmentHandle,
|
|
@@ -92,9 +115,15 @@ import {
|
|
|
92
115
|
type SkillExec,
|
|
93
116
|
writeAgentSkills,
|
|
94
117
|
} from "./skills";
|
|
95
|
-
import {
|
|
118
|
+
import {
|
|
119
|
+
ensureMachineGatewayTunnel,
|
|
120
|
+
ensureMachineGatewayTunnelReady,
|
|
121
|
+
MACHINE_GATEWAY_TUNNEL_PORT,
|
|
122
|
+
} from "./machine-gateway-tunnel";
|
|
123
|
+
import { ensureContainerTaskEnvForwarder } from "./task-env-forwarder";
|
|
96
124
|
import {
|
|
97
125
|
CONTAINER_CLI_PATH,
|
|
126
|
+
agentCliArgv,
|
|
98
127
|
agentCliEnv,
|
|
99
128
|
apiUrlFromCloudUrl,
|
|
100
129
|
loadTaskCliSecret,
|
|
@@ -121,6 +150,12 @@ import {
|
|
|
121
150
|
type ResolvedEngineAccount,
|
|
122
151
|
} from "./engine-accounts";
|
|
123
152
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
153
|
+
import {
|
|
154
|
+
reactivateTaskGatewayTask,
|
|
155
|
+
removeTaskGatewayTask,
|
|
156
|
+
suspendTaskGatewayTask,
|
|
157
|
+
type TaskGatewayRegistration,
|
|
158
|
+
} from "./task-gateway-state";
|
|
124
159
|
import {
|
|
125
160
|
PREVIEW_SIDECAR_TASK_LABEL,
|
|
126
161
|
stopPreviewSidecars,
|
|
@@ -145,6 +180,7 @@ import {
|
|
|
145
180
|
SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
|
|
146
181
|
} from "../src/protocol";
|
|
147
182
|
import { isEnvironmentTransportProvider } from "./task-environment/types";
|
|
183
|
+
import { forgetTaskEnvSnapshot } from "./task-env-source";
|
|
148
184
|
|
|
149
185
|
export type HostEventSubscriber = (event: HostEvent) => void;
|
|
150
186
|
|
|
@@ -223,14 +259,27 @@ interface Channel {
|
|
|
223
259
|
/** Sessions that loaded MCP config before the latest rewrite. Busy sessions
|
|
224
260
|
* stay here until their turn completes; each id is recycled exactly once. */
|
|
225
261
|
browserStaleSessions: Set<string>;
|
|
262
|
+
/** Launch-setting edits reuse the idle-safe stale-session path, but carry
|
|
263
|
+
* their own lifecycle so only those replacements emit agent.session. */
|
|
264
|
+
rosterSessionChanges: Map<string, RosterSessionChange>;
|
|
226
265
|
/** Browser-config rewrites whose install/config preconditions are not ready
|
|
227
266
|
* yet. These sessions must eventually reload the definition, but recycling
|
|
228
267
|
* them before the browser is usable only trades one broken session for
|
|
229
268
|
* another. */
|
|
230
269
|
browserPendingStaleSessions: Set<string>;
|
|
270
|
+
/** Engine kinds newly covered by an already-enabled browser setup. The
|
|
271
|
+
* aggregate setup result can then recycle only sessions for those kinds. */
|
|
272
|
+
browserConfigAffectedKinds: Set<string>;
|
|
231
273
|
/** One reconcile loop per channel. It covers ordinary missing agents and
|
|
232
274
|
* stale-session recycling without overlapping factory creates. */
|
|
233
275
|
reconcileReady: Promise<void> | null;
|
|
276
|
+
/** Forced removal closes begun by refreshChannel. ensureSessions joins this
|
|
277
|
+
* fence before spawning so one outbox consumer cannot overlap another. */
|
|
278
|
+
rosterRefreshReady: Promise<void> | null;
|
|
279
|
+
/** First-start cleanup of durable rows whose ids are absent from the fresh
|
|
280
|
+
* cloud roster (the host may have restarted after their removal). */
|
|
281
|
+
persistedRosterChecked: boolean;
|
|
282
|
+
persistedRosterReady: Promise<void> | null;
|
|
234
283
|
/** A reconcile request that arrived while the loop was in flight. */
|
|
235
284
|
reconcileAgain: boolean;
|
|
236
285
|
/** Set before teardown awaits anything. Every async continuation checks both
|
|
@@ -260,6 +309,14 @@ interface Channel {
|
|
|
260
309
|
* turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
|
|
261
310
|
* half-turn instead of delivering it to @-mentioned peers. */
|
|
262
311
|
interrupted: Set<string>;
|
|
312
|
+
/** Cloud turn boundaries whose runner generation was retired by recovery.
|
|
313
|
+
* Kept separately from activeTurns because remove/edit may overtake the
|
|
314
|
+
* async recovery after generation-local turn state has been cleared. */
|
|
315
|
+
retiredTurnBoundaries: Set<string>;
|
|
316
|
+
/** Async account recoveries fenced by roster removal/edit. A same-settings
|
|
317
|
+
* remove/re-add is invisible to launch-tuple comparison, so the recovery
|
|
318
|
+
* itself carries the supersession bit until it releases the spawn slot. */
|
|
319
|
+
recoveryByAgent: Map<string, RecoveryFence>;
|
|
263
320
|
/** Per-agent respawn counter — bounded so a broken agent can't
|
|
264
321
|
* loop forever rewriting its config. */
|
|
265
322
|
respawns: Map<string, number>;
|
|
@@ -288,6 +345,9 @@ interface Channel {
|
|
|
288
345
|
mcpConfigDirtyEpoch: number;
|
|
289
346
|
/** Dirty epoch covered by the last successful gateway publication. */
|
|
290
347
|
mcpConfigAppliedEpoch: number;
|
|
348
|
+
/** Engine kinds whose provisioned config directories dirtied the managed
|
|
349
|
+
* gateway files at the current unapplied epoch. */
|
|
350
|
+
mcpProvisionAffectedKinds: Set<string>;
|
|
291
351
|
/** Monotonic version of the managed config files agent processes read at
|
|
292
352
|
* startup. A factory create that straddles a rewrite is discarded and
|
|
293
353
|
* retried before it can be bound as the live generation. */
|
|
@@ -309,6 +369,9 @@ interface Channel {
|
|
|
309
369
|
* drains them so an old channel cannot resume against a reused container
|
|
310
370
|
* name and overwrite the replacement generation's durable transport. */
|
|
311
371
|
factoryOperations: Set<Promise<unknown>>;
|
|
372
|
+
/** The create currently owning an agent id, used to fence a removal until a
|
|
373
|
+
* stale in-flight process has observed the new roster and closed itself. */
|
|
374
|
+
factoryOperationByAgent: Map<string, Promise<AgentSession | null>>;
|
|
312
375
|
/** Terminal-event recovery/rotation can remove a session before awaiting
|
|
313
376
|
* its close. Track the enclosing handler so teardown still drains that
|
|
314
377
|
* external operation before the container name can be reused. */
|
|
@@ -328,6 +391,16 @@ interface AcceptedPrompt {
|
|
|
328
391
|
text: string;
|
|
329
392
|
}
|
|
330
393
|
|
|
394
|
+
interface RosterSessionChange {
|
|
395
|
+
reason: string;
|
|
396
|
+
state: "pending-close" | "restarting";
|
|
397
|
+
abortPublished: boolean;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
interface RecoveryFence {
|
|
401
|
+
superseded: boolean;
|
|
402
|
+
}
|
|
403
|
+
|
|
331
404
|
/** Hard cap on automatic respawns per agent per channel lifetime. */
|
|
332
405
|
const MAX_RESPAWNS_PER_AGENT = 5;
|
|
333
406
|
/** ADR-076: hard cap on account rotations per agent between successful turns. */
|
|
@@ -545,31 +618,100 @@ export class Orchestrator {
|
|
|
545
618
|
/** Fold a fresh channel spec into a live channel (ADR-049). */
|
|
546
619
|
private refreshChannel(channel: Channel, spec: ChannelEnsureInput): void {
|
|
547
620
|
const browserWasEnabled = channel.browserTesting;
|
|
621
|
+
const previousRoster = channel.roster;
|
|
548
622
|
const previousMcpShape = JSON.stringify({
|
|
549
623
|
connections: channel.mcpConnections,
|
|
550
|
-
kinds: [...new Set(
|
|
624
|
+
kinds: [...new Set(previousRoster.map((agent) => agent.kind))].sort(),
|
|
551
625
|
});
|
|
552
|
-
const
|
|
626
|
+
const previousById = new Map(previousRoster.map((agent) => [agent.id, agent]));
|
|
627
|
+
const nextById = new Map(spec.agents.map((agent) => [agent.id, agent]));
|
|
628
|
+
const previousKinds = new Set(previousRoster.map((agent) => agent.kind));
|
|
629
|
+
const nextKinds = new Set(spec.agents.map((agent) => agent.kind));
|
|
630
|
+
const removed = previousRoster.filter((agent) => !nextById.has(agent.id));
|
|
631
|
+
const added = spec.agents.filter((agent) => !previousById.has(agent.id));
|
|
632
|
+
const changed = spec.agents.flatMap((agent) => {
|
|
633
|
+
const previous = previousById.get(agent.id);
|
|
634
|
+
if (!previous) return [];
|
|
635
|
+
const reason = rosterLaunchChangeReason(previous, agent);
|
|
636
|
+
return reason ? [{ agent, reason }] : [];
|
|
637
|
+
});
|
|
638
|
+
|
|
553
639
|
// Replace EXISTING agents' data with the fresh spec — a mid-task roster edit
|
|
554
640
|
// can change permissions, model, brief, role or skills, and without this the
|
|
555
641
|
// stale snapshot persists (e.g. granting `uai` CLI permissions mid-task never
|
|
556
|
-
// took effect: writeAgentCli kept seeing the old empty list).
|
|
557
|
-
//
|
|
558
|
-
|
|
559
|
-
const
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
642
|
+
// took effect: writeAgentCli kept seeing the old empty list). Replacing the
|
|
643
|
+
// array exactly also makes absence authoritative for live removal.
|
|
644
|
+
channel.roster = [...spec.agents];
|
|
645
|
+
for (const agent of added) {
|
|
646
|
+
// Per-agent link/document skills, like getOrCreateChannel does at
|
|
647
|
+
// channel birth. Best-effort.
|
|
648
|
+
writeAgentSkills(channel.taskId, agent);
|
|
649
|
+
}
|
|
650
|
+
for (const { agent, reason } of changed) {
|
|
651
|
+
const recoveryAbortPublished = this.supersedeRecovery(
|
|
652
|
+
channel,
|
|
653
|
+
agent.id,
|
|
654
|
+
);
|
|
655
|
+
// Clear before any reconcile can construct a replacement. A session can
|
|
656
|
+
// already be absent from the map while its durable runner/thread row is
|
|
657
|
+
// still live, so this must not depend on in-memory session presence.
|
|
658
|
+
try {
|
|
659
|
+
clearPersistedCodexThread(channel.taskId, agent.id);
|
|
660
|
+
} catch (err) {
|
|
661
|
+
console.warn(
|
|
662
|
+
`[orchestrator] ${channel.taskId}/${agent.id}: could not clear persisted Codex thread for roster change: ${
|
|
663
|
+
err instanceof Error ? err.message : String(err)
|
|
664
|
+
}`,
|
|
665
|
+
);
|
|
567
666
|
}
|
|
667
|
+
channel.accountByAgent.delete(agent.id);
|
|
668
|
+
channel.rotations.delete(agent.id);
|
|
669
|
+
// Crash/recovery budgets belong to the retired launch generation. A
|
|
670
|
+
// newly selected engine/model/effort must get an immediate clean start,
|
|
671
|
+
// not inherit the predecessor's cooldown window.
|
|
672
|
+
channel.respawns.delete(agent.id);
|
|
673
|
+
channel.respawnLastAt.delete(agent.id);
|
|
674
|
+
const priorChange = channel.rosterSessionChanges.get(agent.id);
|
|
675
|
+
channel.rosterSessionChanges.set(agent.id, {
|
|
676
|
+
reason,
|
|
677
|
+
state:
|
|
678
|
+
priorChange?.state === "restarting"
|
|
679
|
+
? "restarting"
|
|
680
|
+
: "pending-close",
|
|
681
|
+
abortPublished:
|
|
682
|
+
priorChange?.abortPublished === true || recoveryAbortPublished,
|
|
683
|
+
});
|
|
684
|
+
channel.browserStaleSessions.add(agent.id);
|
|
685
|
+
}
|
|
686
|
+
if (removed.length > 0) {
|
|
687
|
+
const previousRefresh = channel.rosterRefreshReady ?? Promise.resolve();
|
|
688
|
+
const removals = removed.map((agent) =>
|
|
689
|
+
this.removeRosterAgent(channel, agent.id),
|
|
690
|
+
);
|
|
691
|
+
let refresh: Promise<void>;
|
|
692
|
+
refresh = Promise.allSettled([previousRefresh, ...removals])
|
|
693
|
+
.then(() => undefined)
|
|
694
|
+
.finally(() => {
|
|
695
|
+
if (channel.rosterRefreshReady === refresh) {
|
|
696
|
+
channel.rosterRefreshReady = null;
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
channel.rosterRefreshReady = refresh;
|
|
568
700
|
}
|
|
569
701
|
channel.humans = spec.humans ?? [];
|
|
570
702
|
channel.mode = spec.mode === "secretary" ? "secretary" : "open";
|
|
571
703
|
channel.secretaryAgentId = spec.secretaryAgentId;
|
|
572
704
|
channel.browserTesting = spec.browserTesting === true;
|
|
705
|
+
if (channel.browserTesting) {
|
|
706
|
+
const affectedKinds = browserWasEnabled
|
|
707
|
+
? [...nextKinds].filter((kind) => !previousKinds.has(kind))
|
|
708
|
+
: [...nextKinds];
|
|
709
|
+
for (const kind of affectedKinds) {
|
|
710
|
+
channel.browserConfigAffectedKinds.add(kind);
|
|
711
|
+
}
|
|
712
|
+
} else {
|
|
713
|
+
channel.browserConfigAffectedKinds.clear();
|
|
714
|
+
}
|
|
573
715
|
channel.routingShadow = spec.routing?.shadow === true;
|
|
574
716
|
channel.workspacePath = spec.workspacePath;
|
|
575
717
|
channel.sharedFiles = spec.sharedFiles ?? "ro";
|
|
@@ -597,6 +739,7 @@ export class Orchestrator {
|
|
|
597
739
|
channel.sharedFiles,
|
|
598
740
|
spec.mode,
|
|
599
741
|
spec.secretaryAgentId,
|
|
742
|
+
taskProxiedKeys(channel.taskId),
|
|
600
743
|
),
|
|
601
744
|
);
|
|
602
745
|
}
|
|
@@ -626,6 +769,162 @@ export class Orchestrator {
|
|
|
626
769
|
}
|
|
627
770
|
}
|
|
628
771
|
|
|
772
|
+
/** Remove every live/in-flight trace of one agent without deleting its
|
|
773
|
+
* durable session directory. Removal is intentionally immediate mid-turn. */
|
|
774
|
+
private async removeRosterAgent(
|
|
775
|
+
channel: Channel,
|
|
776
|
+
agentId: string,
|
|
777
|
+
): Promise<void> {
|
|
778
|
+
const session = channel.sessions.get(agentId);
|
|
779
|
+
const inFlightCreate = channel.factoryOperationByAgent.get(agentId);
|
|
780
|
+
const lifecycleAtRemoval = channel.rosterSessionChanges.get(agentId);
|
|
781
|
+
channel.sessions.delete(agentId);
|
|
782
|
+
// Do not release another lifecycle owner's spawn slot. Recovery/reconcile
|
|
783
|
+
// paths clear it in their own finally blocks after rechecking the desired
|
|
784
|
+
// roster; releasing it here lets a rapid re-add start a second same-id
|
|
785
|
+
// factory while the first is still provisioning.
|
|
786
|
+
|
|
787
|
+
// Recovery may already have retired the runner-local turn state while an
|
|
788
|
+
// async provision/broker call was in flight. Consume that saved boundary
|
|
789
|
+
// before looking at the live generation's maps.
|
|
790
|
+
this.supersedeRecovery(channel, agentId);
|
|
791
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
792
|
+
// A removed partial turn can never receive its natural terminal event.
|
|
793
|
+
// Close that cloud boundary as aborted before the old session's identity
|
|
794
|
+
// guard begins suppressing its final adapter events.
|
|
795
|
+
if (
|
|
796
|
+
channel.activeTurns.has(agentId) ||
|
|
797
|
+
channel.openTurns.has(agentId) ||
|
|
798
|
+
channel.acceptedPrompts.has(agentId)
|
|
799
|
+
) {
|
|
800
|
+
channel.interrupted.add(agentId);
|
|
801
|
+
}
|
|
802
|
+
this.retireRunnerGeneration(channel, agentId);
|
|
803
|
+
channel.preambles.delete(agentId);
|
|
804
|
+
channel.firstTurns.delete(agentId);
|
|
805
|
+
channel.browserStaleSessions.delete(agentId);
|
|
806
|
+
channel.rosterSessionChanges.delete(agentId);
|
|
807
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
808
|
+
channel.pendingPrompts.delete(agentId);
|
|
809
|
+
channel.runtimeDeferredPrompts.delete(agentId);
|
|
810
|
+
channel.runtimeDeferredAgents.delete(agentId);
|
|
811
|
+
channel.interrupted.delete(agentId);
|
|
812
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
813
|
+
channel.respawns.delete(agentId);
|
|
814
|
+
channel.respawnLastAt.delete(agentId);
|
|
815
|
+
channel.everBound.delete(agentId);
|
|
816
|
+
channel.accountByAgent.delete(agentId);
|
|
817
|
+
channel.lastPrompt.delete(agentId);
|
|
818
|
+
channel.rotations.delete(agentId);
|
|
819
|
+
|
|
820
|
+
if (session) {
|
|
821
|
+
try {
|
|
822
|
+
// Delete-before-close above keeps the adapter's terminal event
|
|
823
|
+
// identity-guarded and prevents an outbox consumer overlap.
|
|
824
|
+
await session.close();
|
|
825
|
+
} catch {
|
|
826
|
+
// Already dead — close is idempotent for our adapters.
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
if (inFlightCreate) {
|
|
830
|
+
// createAndBindStableSessionCore rechecks the desired roster after the
|
|
831
|
+
// factory await and closes the stale process instead of binding it.
|
|
832
|
+
await inFlightCreate.catch(() => null);
|
|
833
|
+
}
|
|
834
|
+
if (!this.isActiveChannel(channel)) return;
|
|
835
|
+
// A newer ensure can re-add the same id while the old close/create drains.
|
|
836
|
+
// That spec owns the status now; do not stop its bound generation or emit
|
|
837
|
+
// a stale removed edge.
|
|
838
|
+
if (channel.roster.some((agent) => agent.id === agentId)) {
|
|
839
|
+
this.restoreReaddedRosterLifecycle(
|
|
840
|
+
channel,
|
|
841
|
+
agentId,
|
|
842
|
+
lifecycleAtRemoval,
|
|
843
|
+
);
|
|
844
|
+
void this.reconcileIfStarted(channel).catch((err: unknown) => {
|
|
845
|
+
console.warn(
|
|
846
|
+
`[orchestrator] ${channel.taskId}/${agentId}: re-added roster reconcile failed: ${
|
|
847
|
+
err instanceof Error ? err.message : String(err)
|
|
848
|
+
}`,
|
|
849
|
+
);
|
|
850
|
+
});
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
853
|
+
let durableCleanupFailed = false;
|
|
854
|
+
try {
|
|
855
|
+
await this.stopPersistedRunnerIfPresent(channel, agentId);
|
|
856
|
+
} catch (err) {
|
|
857
|
+
durableCleanupFailed = true;
|
|
858
|
+
// The same ensure that joined this removal fence will retry through the
|
|
859
|
+
// persisted-roster recovery pass instead of trusting a poisoned result.
|
|
860
|
+
channel.persistedRosterChecked = false;
|
|
861
|
+
console.warn(
|
|
862
|
+
`[orchestrator] ${channel.taskId}/${agentId}: removed durable session cleanup failed: ${
|
|
863
|
+
err instanceof Error ? err.message : String(err)
|
|
864
|
+
}`,
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
if (!this.isActiveChannel(channel)) return;
|
|
868
|
+
if (channel.roster.some((agent) => agent.id === agentId)) {
|
|
869
|
+
this.restoreReaddedRosterLifecycle(
|
|
870
|
+
channel,
|
|
871
|
+
agentId,
|
|
872
|
+
lifecycleAtRemoval,
|
|
873
|
+
);
|
|
874
|
+
void this.reconcileIfStarted(channel).catch((err: unknown) => {
|
|
875
|
+
console.warn(
|
|
876
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-stop re-add reconcile failed: ${
|
|
877
|
+
err instanceof Error ? err.message : String(err)
|
|
878
|
+
}`,
|
|
879
|
+
);
|
|
880
|
+
});
|
|
881
|
+
return;
|
|
882
|
+
}
|
|
883
|
+
// Slow rotation/account continuations captured before the roster swap may
|
|
884
|
+
// have reached their finally blocks while the close was in flight.
|
|
885
|
+
channel.accountByAgent.delete(agentId);
|
|
886
|
+
channel.rotations.delete(agentId);
|
|
887
|
+
channel.respawns.delete(agentId);
|
|
888
|
+
channel.respawnLastAt.delete(agentId);
|
|
889
|
+
channel.lastPrompt.delete(agentId);
|
|
890
|
+
if (!this.isActiveChannel(channel)) return;
|
|
891
|
+
if (durableCleanupFailed) return;
|
|
892
|
+
this.emitHost({
|
|
893
|
+
kind: "agent.session",
|
|
894
|
+
taskId: channel.taskId,
|
|
895
|
+
agentId,
|
|
896
|
+
state: "removed",
|
|
897
|
+
});
|
|
898
|
+
console.log(
|
|
899
|
+
`[orchestrator] ${channel.taskId}/${agentId}: stopped removed roster session`,
|
|
900
|
+
);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/** A remove superseded while an earlier edit was already restarting must
|
|
904
|
+
* still close that visible lifecycle with ready for the re-added roster. */
|
|
905
|
+
private restoreReaddedRosterLifecycle(
|
|
906
|
+
channel: Channel,
|
|
907
|
+
agentId: string,
|
|
908
|
+
removedChange: RosterSessionChange | undefined,
|
|
909
|
+
): void {
|
|
910
|
+
if (
|
|
911
|
+
removedChange?.state !== "restarting" ||
|
|
912
|
+
channel.rosterSessionChanges.has(agentId)
|
|
913
|
+
) {
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
if (channel.sessions.has(agentId)) {
|
|
917
|
+
this.emitHost({
|
|
918
|
+
kind: "agent.session",
|
|
919
|
+
taskId: channel.taskId,
|
|
920
|
+
agentId,
|
|
921
|
+
state: "ready",
|
|
922
|
+
});
|
|
923
|
+
return;
|
|
924
|
+
}
|
|
925
|
+
channel.rosterSessionChanges.set(agentId, removedChange);
|
|
926
|
+
}
|
|
927
|
+
|
|
629
928
|
/** Materialize a browser flag enabled after the channel already started. */
|
|
630
929
|
private async enableBrowserTesting(channel: Channel): Promise<void> {
|
|
631
930
|
try {
|
|
@@ -670,6 +969,7 @@ export class Orchestrator {
|
|
|
670
969
|
spec.sharedFiles ?? "ro",
|
|
671
970
|
spec.mode,
|
|
672
971
|
spec.secretaryAgentId,
|
|
972
|
+
taskProxiedKeys(taskId),
|
|
673
973
|
),
|
|
674
974
|
);
|
|
675
975
|
// ADR-046: materialise this agent's skills to its per-agent SKILL.md in
|
|
@@ -709,8 +1009,13 @@ export class Orchestrator {
|
|
|
709
1009
|
browserSetupCoverage: null,
|
|
710
1010
|
browserRetryTimer: null,
|
|
711
1011
|
browserStaleSessions: new Set(),
|
|
1012
|
+
rosterSessionChanges: new Map(),
|
|
712
1013
|
browserPendingStaleSessions: new Set(),
|
|
1014
|
+
browserConfigAffectedKinds: new Set(),
|
|
713
1015
|
reconcileReady: null,
|
|
1016
|
+
rosterRefreshReady: null,
|
|
1017
|
+
persistedRosterChecked: false,
|
|
1018
|
+
persistedRosterReady: null,
|
|
714
1019
|
reconcileAgain: false,
|
|
715
1020
|
closed: false,
|
|
716
1021
|
openTurns: new Set(),
|
|
@@ -720,6 +1025,8 @@ export class Orchestrator {
|
|
|
720
1025
|
runtimeDeferredPrompts: new Map(),
|
|
721
1026
|
runtimeDeferredAgents: new Set(),
|
|
722
1027
|
interrupted: new Set(),
|
|
1028
|
+
retiredTurnBoundaries: new Set(),
|
|
1029
|
+
recoveryByAgent: new Map(),
|
|
723
1030
|
respawns: new Map(),
|
|
724
1031
|
respawnLastAt: new Map(),
|
|
725
1032
|
humans: spec.humans ?? [],
|
|
@@ -729,12 +1036,14 @@ export class Orchestrator {
|
|
|
729
1036
|
mcpConfigWrite: null,
|
|
730
1037
|
mcpConfigDirtyEpoch: 1,
|
|
731
1038
|
mcpConfigAppliedEpoch: 0,
|
|
1039
|
+
mcpProvisionAffectedKinds: new Set(),
|
|
732
1040
|
configGeneration: 0,
|
|
733
1041
|
spawning: new Set(),
|
|
734
1042
|
everBound: new Set(),
|
|
735
1043
|
routingShadow: spec.routing?.shadow === true,
|
|
736
1044
|
workspacePath: spec.workspacePath,
|
|
737
1045
|
factoryOperations: new Set(),
|
|
1046
|
+
factoryOperationByAgent: new Map(),
|
|
738
1047
|
eventOperations: new Set(),
|
|
739
1048
|
accountByAgent: new Map(),
|
|
740
1049
|
lastPrompt: new Map(),
|
|
@@ -761,6 +1070,58 @@ export class Orchestrator {
|
|
|
761
1070
|
// copy cannot land after browser/MCP materialization and erase config.
|
|
762
1071
|
await taskRecoveryComplete(channel.taskId);
|
|
763
1072
|
if (!this.isActiveChannel(channel)) return false;
|
|
1073
|
+
if (channel.rosterRefreshReady) {
|
|
1074
|
+
await channel.rosterRefreshReady;
|
|
1075
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1076
|
+
}
|
|
1077
|
+
if (!channel.persistedRosterChecked) {
|
|
1078
|
+
let removedIds: string[];
|
|
1079
|
+
try {
|
|
1080
|
+
const rosterIds = new Set(channel.roster.map((agent) => agent.id));
|
|
1081
|
+
removedIds = persistedAgentIds(channel.taskId).filter(
|
|
1082
|
+
(agentId) => !rosterIds.has(agentId),
|
|
1083
|
+
);
|
|
1084
|
+
} catch (err) {
|
|
1085
|
+
console.warn(
|
|
1086
|
+
`[orchestrator] ${channel.taskId}: persisted roster scan failed: ${
|
|
1087
|
+
err instanceof Error ? err.message : String(err)
|
|
1088
|
+
}`,
|
|
1089
|
+
);
|
|
1090
|
+
throw err;
|
|
1091
|
+
}
|
|
1092
|
+
channel.persistedRosterChecked = true;
|
|
1093
|
+
if (removedIds.length > 0) {
|
|
1094
|
+
const operation = this.stopPersistedRemovedAgents(
|
|
1095
|
+
channel,
|
|
1096
|
+
removedIds,
|
|
1097
|
+
);
|
|
1098
|
+
let tracked: Promise<void>;
|
|
1099
|
+
tracked = operation.then(
|
|
1100
|
+
() => {
|
|
1101
|
+
if (channel.persistedRosterReady === tracked) {
|
|
1102
|
+
channel.persistedRosterReady = null;
|
|
1103
|
+
}
|
|
1104
|
+
},
|
|
1105
|
+
(err: unknown) => {
|
|
1106
|
+
if (channel.persistedRosterReady === tracked) {
|
|
1107
|
+
channel.persistedRosterReady = null;
|
|
1108
|
+
channel.persistedRosterChecked = false;
|
|
1109
|
+
}
|
|
1110
|
+
console.warn(
|
|
1111
|
+
`[orchestrator] ${channel.taskId}: persisted roster cleanup failed: ${
|
|
1112
|
+
err instanceof Error ? err.message : String(err)
|
|
1113
|
+
}`,
|
|
1114
|
+
);
|
|
1115
|
+
throw err;
|
|
1116
|
+
},
|
|
1117
|
+
);
|
|
1118
|
+
channel.persistedRosterReady = tracked;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
if (channel.persistedRosterReady) {
|
|
1122
|
+
await channel.persistedRosterReady;
|
|
1123
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1124
|
+
}
|
|
764
1125
|
if (!backgroundContainerWorkAllowed()) return false;
|
|
765
1126
|
// Memoized: every caller awaits the SAME in-flight start, so a concurrent
|
|
766
1127
|
// deliver() can't observe "ready" while the sessions map is still empty
|
|
@@ -825,6 +1186,48 @@ export class Orchestrator {
|
|
|
825
1186
|
return ready;
|
|
826
1187
|
}
|
|
827
1188
|
|
|
1189
|
+
/** A host restart loses the in-memory old roster, not its durable runners.
|
|
1190
|
+
* Stop persisted ids that the authoritative fresh spec no longer contains. */
|
|
1191
|
+
private async stopPersistedRemovedAgents(
|
|
1192
|
+
channel: Channel,
|
|
1193
|
+
removedIds: readonly string[],
|
|
1194
|
+
): Promise<void> {
|
|
1195
|
+
const environment = await this.agentEnvironment(channel);
|
|
1196
|
+
for (const agentId of removedIds) {
|
|
1197
|
+
if (!this.isActiveChannel(channel)) return;
|
|
1198
|
+
// The roster may have advanced while environment reconstruction was in
|
|
1199
|
+
// flight. Last arrival wins; never stop an id the current spec wants.
|
|
1200
|
+
if (channel.roster.some((agent) => agent.id === agentId)) continue;
|
|
1201
|
+
const stoppedDir = await stopPersistedAgentSession(
|
|
1202
|
+
channel.taskId,
|
|
1203
|
+
agentId,
|
|
1204
|
+
environment,
|
|
1205
|
+
);
|
|
1206
|
+
if (channel.roster.some((agent) => agent.id === agentId)) continue;
|
|
1207
|
+
if (!stoppedDir || !this.isActiveChannel(channel)) continue;
|
|
1208
|
+
this.emitHost({
|
|
1209
|
+
kind: "agent.session",
|
|
1210
|
+
taskId: channel.taskId,
|
|
1211
|
+
agentId,
|
|
1212
|
+
state: "removed",
|
|
1213
|
+
});
|
|
1214
|
+
console.log(
|
|
1215
|
+
`[orchestrator] ${channel.taskId}/${agentId}: stopped removed durable session during roster recovery`,
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
/** Stop a durable predecessor even when its row is already marked closed.
|
|
1221
|
+
* Closed is not proof of runner death: markClosed precedes the stop write. */
|
|
1222
|
+
private async stopPersistedRunnerIfPresent(
|
|
1223
|
+
channel: Channel,
|
|
1224
|
+
agentId: string,
|
|
1225
|
+
): Promise<string | null> {
|
|
1226
|
+
if (!persistedAgentIds(channel.taskId).includes(agentId)) return null;
|
|
1227
|
+
const environment = await this.agentEnvironment(channel);
|
|
1228
|
+
return stopPersistedAgentSession(channel.taskId, agentId, environment);
|
|
1229
|
+
}
|
|
1230
|
+
|
|
828
1231
|
/**
|
|
829
1232
|
* Background browser work may recycle sessions only after the memoized
|
|
830
1233
|
* initial start has finished. Waiting here prevents live enable/retry from
|
|
@@ -873,11 +1276,17 @@ export class Orchestrator {
|
|
|
873
1276
|
private noteProvisionMutation(
|
|
874
1277
|
channel: Channel,
|
|
875
1278
|
waitForBrowserReadiness: boolean,
|
|
1279
|
+
affectedKinds: ReadonlySet<string>,
|
|
876
1280
|
): void {
|
|
877
1281
|
if (!this.isActiveChannel(channel)) return;
|
|
878
1282
|
channel.configGeneration += 1;
|
|
879
1283
|
channel.mcpConfigDirtyEpoch += 1;
|
|
880
|
-
for (const
|
|
1284
|
+
for (const kind of affectedKinds) {
|
|
1285
|
+
channel.mcpProvisionAffectedKinds.add(kind);
|
|
1286
|
+
}
|
|
1287
|
+
for (const [agentId] of channel.sessions) {
|
|
1288
|
+
const kind = channel.roster.find((agent) => agent.id === agentId)?.kind;
|
|
1289
|
+
if (!kind || !affectedKinds.has(kind)) continue;
|
|
881
1290
|
if (waitForBrowserReadiness) {
|
|
882
1291
|
channel.browserPendingStaleSessions.add(agentId);
|
|
883
1292
|
} else {
|
|
@@ -910,6 +1319,42 @@ export class Orchestrator {
|
|
|
910
1319
|
});
|
|
911
1320
|
}
|
|
912
1321
|
|
|
1322
|
+
/** Kinds whose account-copy inputs changed since the last attempted pass. */
|
|
1323
|
+
private changedProvisionKinds(
|
|
1324
|
+
previousFingerprint: string | undefined,
|
|
1325
|
+
nextFingerprint: string,
|
|
1326
|
+
requestedKinds: readonly string[],
|
|
1327
|
+
): string[] {
|
|
1328
|
+
if (!previousFingerprint) return [...requestedKinds];
|
|
1329
|
+
type Fingerprint = {
|
|
1330
|
+
kinds?: string[];
|
|
1331
|
+
accounts?: Array<{
|
|
1332
|
+
kind: string;
|
|
1333
|
+
id: string;
|
|
1334
|
+
hostDir: string;
|
|
1335
|
+
containerDir: string;
|
|
1336
|
+
}>;
|
|
1337
|
+
};
|
|
1338
|
+
try {
|
|
1339
|
+
const previous = JSON.parse(previousFingerprint) as Fingerprint;
|
|
1340
|
+
const next = JSON.parse(nextFingerprint) as Fingerprint;
|
|
1341
|
+
return requestedKinds.filter((kind) => {
|
|
1342
|
+
if (!previous.kinds?.includes(kind)) return true;
|
|
1343
|
+
const previousAccounts = (previous.accounts ?? []).filter(
|
|
1344
|
+
(account) => account.kind === kind,
|
|
1345
|
+
);
|
|
1346
|
+
const nextAccounts = (next.accounts ?? []).filter(
|
|
1347
|
+
(account) => account.kind === kind,
|
|
1348
|
+
);
|
|
1349
|
+
return JSON.stringify(previousAccounts) !== JSON.stringify(nextAccounts);
|
|
1350
|
+
});
|
|
1351
|
+
} catch {
|
|
1352
|
+
// A legacy/partial fingerprint is not trustworthy enough to narrow a
|
|
1353
|
+
// write; copy and invalidate the requested kinds conservatively.
|
|
1354
|
+
return [...requestedKinds];
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
|
|
913
1358
|
private async ensureBrowserSetup(
|
|
914
1359
|
channel: Channel,
|
|
915
1360
|
hasCodex: boolean,
|
|
@@ -936,11 +1381,20 @@ export class Orchestrator {
|
|
|
936
1381
|
];
|
|
937
1382
|
const requestedProvisionFingerprint =
|
|
938
1383
|
this.provisionFingerprint(requestedKinds, channel.engineAccountScope);
|
|
939
|
-
const
|
|
1384
|
+
const changedProvisionKinds = this.changedProvisionKinds(
|
|
1385
|
+
channel.provisionAttemptedFingerprint,
|
|
1386
|
+
requestedProvisionFingerprint,
|
|
1387
|
+
requestedKinds,
|
|
1388
|
+
);
|
|
1389
|
+
const provisionKinds = forceProvision
|
|
1390
|
+
? requestedKinds
|
|
1391
|
+
: changedProvisionKinds;
|
|
1392
|
+
const mutableProvisionKinds = provisionKinds.filter((kind) =>
|
|
940
1393
|
resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
|
|
941
1394
|
(account) => !account.isDefault && Boolean(account.configDir),
|
|
942
1395
|
),
|
|
943
1396
|
);
|
|
1397
|
+
const mayProvisionConfig = mutableProvisionKinds.length > 0;
|
|
944
1398
|
|
|
945
1399
|
const inFlight = channel.browserSetup;
|
|
946
1400
|
if (inFlight) {
|
|
@@ -1011,14 +1465,14 @@ export class Orchestrator {
|
|
|
1011
1465
|
const provisioned = channel.agentHandle
|
|
1012
1466
|
? await provisionEngineAccounts(
|
|
1013
1467
|
channel.containerName,
|
|
1014
|
-
|
|
1468
|
+
provisionKinds,
|
|
1015
1469
|
{},
|
|
1016
1470
|
channel.agentHandle,
|
|
1017
1471
|
channel.engineAccountScope,
|
|
1018
1472
|
)
|
|
1019
1473
|
: await provisionEngineAccounts(
|
|
1020
1474
|
channel.containerName,
|
|
1021
|
-
|
|
1475
|
+
provisionKinds,
|
|
1022
1476
|
{},
|
|
1023
1477
|
undefined,
|
|
1024
1478
|
channel.engineAccountScope,
|
|
@@ -1050,6 +1504,7 @@ export class Orchestrator {
|
|
|
1050
1504
|
this.noteProvisionMutation(
|
|
1051
1505
|
channel,
|
|
1052
1506
|
channel.browserTesting,
|
|
1507
|
+
new Set(mutableProvisionKinds),
|
|
1053
1508
|
);
|
|
1054
1509
|
} else {
|
|
1055
1510
|
this.fenceConfigGeneration(channel);
|
|
@@ -1107,10 +1562,23 @@ export class Orchestrator {
|
|
|
1107
1562
|
channel.browserReady = setup.ready && setup.configured;
|
|
1108
1563
|
if (setup.changed) {
|
|
1109
1564
|
channel.configGeneration += 1;
|
|
1110
|
-
|
|
1565
|
+
const affectedKinds = channel.browserConfigAffectedKinds;
|
|
1566
|
+
const affectedIds =
|
|
1567
|
+
affectedKinds.size > 0
|
|
1568
|
+
? channel.roster
|
|
1569
|
+
.filter((agent) => affectedKinds.has(agent.kind))
|
|
1570
|
+
.map((agent) => agent.id)
|
|
1571
|
+
: [...channel.sessions.keys()];
|
|
1572
|
+
for (const agentId of affectedIds) {
|
|
1573
|
+
if (!channel.sessions.has(agentId)) continue;
|
|
1111
1574
|
channel.browserPendingStaleSessions.add(agentId);
|
|
1112
1575
|
}
|
|
1113
1576
|
}
|
|
1577
|
+
if (setup.configured) {
|
|
1578
|
+
for (const kind of requestedKinds) {
|
|
1579
|
+
channel.browserConfigAffectedKinds.delete(kind);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1114
1582
|
if (channel.browserReady) {
|
|
1115
1583
|
for (const agentId of channel.browserPendingStaleSessions) {
|
|
1116
1584
|
channel.browserStaleSessions.add(agentId);
|
|
@@ -1259,6 +1727,9 @@ export class Orchestrator {
|
|
|
1259
1727
|
...(grokHomes.length > 0 ? { grokHomes } : {}),
|
|
1260
1728
|
});
|
|
1261
1729
|
const dirtyEpoch = channel.mcpConfigDirtyEpoch;
|
|
1730
|
+
const provisionAffectedKinds = new Set(
|
|
1731
|
+
channel.mcpProvisionAffectedKinds,
|
|
1732
|
+
);
|
|
1262
1733
|
const repairingProvisionedConfig =
|
|
1263
1734
|
channel.mcpConfigAppliedEpoch !== dirtyEpoch &&
|
|
1264
1735
|
channel.mcpConfigFingerprint === fingerprint;
|
|
@@ -1321,6 +1792,9 @@ export class Orchestrator {
|
|
|
1321
1792
|
// the files while docker exec was in flight, the newer epoch remains
|
|
1322
1793
|
// uncovered and the serialized ensure immediately publishes another pass.
|
|
1323
1794
|
channel.mcpConfigAppliedEpoch = dirtyEpoch;
|
|
1795
|
+
if (channel.mcpConfigDirtyEpoch === dirtyEpoch) {
|
|
1796
|
+
channel.mcpProvisionAffectedKinds.clear();
|
|
1797
|
+
}
|
|
1324
1798
|
// Agent CLIs read MCP servers once, at process start — and durable
|
|
1325
1799
|
// sessions (ADR-061) make processes long-lived, so without this a
|
|
1326
1800
|
// connection added mid-task stays invisible indefinitely. Recycle IDLE
|
|
@@ -1338,7 +1812,10 @@ export class Orchestrator {
|
|
|
1338
1812
|
// Provisioning marked the sessions that existed around its copy. Mark
|
|
1339
1813
|
// again after the repair so a factory that bound during this write is
|
|
1340
1814
|
// covered too (Set de-duplicates the earlier marks).
|
|
1341
|
-
|
|
1815
|
+
const affected = channel.roster
|
|
1816
|
+
.filter((agent) => provisionAffectedKinds.has(agent.kind))
|
|
1817
|
+
.map((agent) => agent.id);
|
|
1818
|
+
this.markSessionsStale(channel, affected);
|
|
1342
1819
|
} else if (!hadPrevious && hadLiveSessions) {
|
|
1343
1820
|
this.markSessionsStale(channel);
|
|
1344
1821
|
} else if (hadPrevious && connectionsChanged) {
|
|
@@ -1372,7 +1849,16 @@ export class Orchestrator {
|
|
|
1372
1849
|
/** Close stale sessions that are idle, without recursively reconciling. */
|
|
1373
1850
|
private async closeIdleStaleSessions(channel: Channel): Promise<number> {
|
|
1374
1851
|
let recycled = 0;
|
|
1375
|
-
|
|
1852
|
+
let mcpRecycled = 0;
|
|
1853
|
+
// The roster-change tracker is authoritative. Recovery paths written for
|
|
1854
|
+
// ordinary config recycling may clear browserStaleSessions while an edit
|
|
1855
|
+
// is waiting on their spawn slot; never let a pending-close edit bind its
|
|
1856
|
+
// replacement without first publishing the actual restart boundary.
|
|
1857
|
+
const staleAgentIds = new Set([
|
|
1858
|
+
...channel.browserStaleSessions,
|
|
1859
|
+
...channel.rosterSessionChanges.keys(),
|
|
1860
|
+
]);
|
|
1861
|
+
for (const agentId of staleAgentIds) {
|
|
1376
1862
|
if (!this.isActiveChannel(channel)) return recycled;
|
|
1377
1863
|
if (
|
|
1378
1864
|
channel.activeTurns.has(agentId) ||
|
|
@@ -1383,31 +1869,155 @@ export class Orchestrator {
|
|
|
1383
1869
|
}
|
|
1384
1870
|
const session = channel.sessions.get(agentId);
|
|
1385
1871
|
// A session that exited on its own is already absent. Its eventual
|
|
1386
|
-
// replacement is fresh
|
|
1872
|
+
// replacement is fresh for MCP purposes. A roster change still has to
|
|
1873
|
+
// stop a possibly-live durable predecessor before that replacement can
|
|
1874
|
+
// bind under new launch settings.
|
|
1387
1875
|
if (!session) {
|
|
1388
|
-
channel.
|
|
1876
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
1877
|
+
if (rosterChange) {
|
|
1878
|
+
await this.closeMissingRosterChangedSession(
|
|
1879
|
+
channel,
|
|
1880
|
+
agentId,
|
|
1881
|
+
rosterChange,
|
|
1882
|
+
);
|
|
1883
|
+
recycled += 1;
|
|
1884
|
+
} else {
|
|
1885
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1886
|
+
}
|
|
1389
1887
|
continue;
|
|
1390
1888
|
}
|
|
1391
1889
|
channel.browserStaleSessions.delete(agentId);
|
|
1392
1890
|
channel.sessions.delete(agentId);
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1891
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
1892
|
+
if (rosterChange) {
|
|
1893
|
+
await this.closeRosterChangedSession(
|
|
1894
|
+
channel,
|
|
1895
|
+
agentId,
|
|
1896
|
+
session,
|
|
1897
|
+
rosterChange,
|
|
1898
|
+
);
|
|
1899
|
+
} else {
|
|
1900
|
+
try {
|
|
1901
|
+
// Delete before close. Event handlers are identity-guarded, so the
|
|
1902
|
+
// old session's synthetic exit cannot consume respawn budget or delete
|
|
1903
|
+
// the replacement that follows.
|
|
1904
|
+
await session.close();
|
|
1905
|
+
} catch {
|
|
1906
|
+
// Already dead — close is idempotent for our adapters.
|
|
1907
|
+
}
|
|
1908
|
+
mcpRecycled += 1;
|
|
1400
1909
|
}
|
|
1401
1910
|
recycled += 1;
|
|
1402
1911
|
}
|
|
1403
|
-
if (
|
|
1912
|
+
if (mcpRecycled > 0) {
|
|
1404
1913
|
console.log(
|
|
1405
|
-
`[orchestrator] ${channel.taskId}: recycled ${
|
|
1914
|
+
`[orchestrator] ${channel.taskId}: recycled ${mcpRecycled} idle session(s) for MCP config change`,
|
|
1406
1915
|
);
|
|
1407
1916
|
}
|
|
1408
1917
|
return recycled;
|
|
1409
1918
|
}
|
|
1410
1919
|
|
|
1920
|
+
/** Stop one generation for a live launch-setting edit and publish the
|
|
1921
|
+
* roster-specific lifecycle edge only if that edit still owns the id. */
|
|
1922
|
+
private async closeRosterChangedSession(
|
|
1923
|
+
channel: Channel,
|
|
1924
|
+
agentId: string,
|
|
1925
|
+
session: AgentSession,
|
|
1926
|
+
change: RosterSessionChange,
|
|
1927
|
+
): Promise<void> {
|
|
1928
|
+
// Durable attach cannot reconstruct whether a pre-restart runner is
|
|
1929
|
+
// already mid-turn. Publish the terminal boundary before every forced
|
|
1930
|
+
// launch close; it is a no-op for an actually idle cloud stream.
|
|
1931
|
+
this.publishRosterChangeAbort(channel, agentId, change);
|
|
1932
|
+
try {
|
|
1933
|
+
await session.close();
|
|
1934
|
+
} catch {
|
|
1935
|
+
// Already dead — close is idempotent for our adapters.
|
|
1936
|
+
}
|
|
1937
|
+
try {
|
|
1938
|
+
await this.stopPersistedRunnerIfPresent(channel, agentId);
|
|
1939
|
+
// Clear again after close. A thread/started notification already queued
|
|
1940
|
+
// by the old Codex process must not restore its resume id between the
|
|
1941
|
+
// eager roster clear and replacement construction.
|
|
1942
|
+
clearPersistedCodexThread(channel.taskId, agentId);
|
|
1943
|
+
} catch (err) {
|
|
1944
|
+
if (
|
|
1945
|
+
this.isActiveChannel(channel) &&
|
|
1946
|
+
channel.rosterSessionChanges.get(agentId) === change
|
|
1947
|
+
) {
|
|
1948
|
+
channel.browserStaleSessions.add(agentId);
|
|
1949
|
+
}
|
|
1950
|
+
throw err;
|
|
1951
|
+
}
|
|
1952
|
+
if (
|
|
1953
|
+
!this.isActiveChannel(channel) ||
|
|
1954
|
+
channel.rosterSessionChanges.get(agentId) !== change ||
|
|
1955
|
+
!channel.roster.some((agent) => agent.id === agentId)
|
|
1956
|
+
) {
|
|
1957
|
+
return;
|
|
1958
|
+
}
|
|
1959
|
+
const alreadyRestarting = change.state === "restarting";
|
|
1960
|
+
change.state = "restarting";
|
|
1961
|
+
if (!alreadyRestarting) {
|
|
1962
|
+
this.emitHost({
|
|
1963
|
+
kind: "agent.session",
|
|
1964
|
+
taskId: channel.taskId,
|
|
1965
|
+
agentId,
|
|
1966
|
+
state: "restarting",
|
|
1967
|
+
});
|
|
1968
|
+
}
|
|
1969
|
+
console.log(
|
|
1970
|
+
`[orchestrator] ${channel.taskId}/${agentId}: recycled session for live roster ${change.reason}`,
|
|
1971
|
+
);
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
/** Finish a launch edit whose adapter already disappeared, but whose
|
|
1975
|
+
* durable predecessor may still own the CLI process/outbox. */
|
|
1976
|
+
private async closeMissingRosterChangedSession(
|
|
1977
|
+
channel: Channel,
|
|
1978
|
+
agentId: string,
|
|
1979
|
+
change: RosterSessionChange,
|
|
1980
|
+
): Promise<void> {
|
|
1981
|
+
this.publishRosterChangeAbort(channel, agentId, change);
|
|
1982
|
+
const stoppedDir = await this.stopPersistedRunnerIfPresent(channel, agentId);
|
|
1983
|
+
clearPersistedCodexThread(channel.taskId, agentId);
|
|
1984
|
+
if (
|
|
1985
|
+
!this.isActiveChannel(channel) ||
|
|
1986
|
+
channel.rosterSessionChanges.get(agentId) !== change ||
|
|
1987
|
+
!channel.roster.some((agent) => agent.id === agentId)
|
|
1988
|
+
) {
|
|
1989
|
+
return;
|
|
1990
|
+
}
|
|
1991
|
+
channel.browserStaleSessions.delete(agentId);
|
|
1992
|
+
if (!stoppedDir) {
|
|
1993
|
+
// No old generation exists to recycle (one-shot engines between turns,
|
|
1994
|
+
// crash cooldown, or an agent not spawned yet). The cloud set the row to
|
|
1995
|
+
// Restarting when it persisted the edit, so clear that status now even
|
|
1996
|
+
// though there is no close/restart edge to publish first.
|
|
1997
|
+
channel.rosterSessionChanges.delete(agentId);
|
|
1998
|
+
this.emitHost({
|
|
1999
|
+
kind: "agent.session",
|
|
2000
|
+
taskId: channel.taskId,
|
|
2001
|
+
agentId,
|
|
2002
|
+
state: "ready",
|
|
2003
|
+
});
|
|
2004
|
+
return;
|
|
2005
|
+
}
|
|
2006
|
+
const alreadyRestarting = change.state === "restarting";
|
|
2007
|
+
change.state = "restarting";
|
|
2008
|
+
if (!alreadyRestarting) {
|
|
2009
|
+
this.emitHost({
|
|
2010
|
+
kind: "agent.session",
|
|
2011
|
+
taskId: channel.taskId,
|
|
2012
|
+
agentId,
|
|
2013
|
+
state: "restarting",
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
console.log(
|
|
2017
|
+
`[orchestrator] ${channel.taskId}/${agentId}: stopped detached durable session for live roster ${change.reason}`,
|
|
2018
|
+
);
|
|
2019
|
+
}
|
|
2020
|
+
|
|
1411
2021
|
/** Serialize all missing-session and stale-session reconciliation. */
|
|
1412
2022
|
private async reconcileSessions(channel: Channel): Promise<void> {
|
|
1413
2023
|
if (!this.isActiveChannel(channel)) return;
|
|
@@ -1520,6 +2130,10 @@ export class Orchestrator {
|
|
|
1520
2130
|
// Another recovery path may have filled this slot while the browser
|
|
1521
2131
|
// setup was awaiting.
|
|
1522
2132
|
if (channel.sessions.has(agent.id)) continue;
|
|
2133
|
+
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2134
|
+
channel.reconcileAgain = true;
|
|
2135
|
+
continue;
|
|
2136
|
+
}
|
|
1523
2137
|
const session = await this.createAndBindStableSession(
|
|
1524
2138
|
channel,
|
|
1525
2139
|
agent.id,
|
|
@@ -1530,7 +2144,8 @@ export class Orchestrator {
|
|
|
1530
2144
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
1531
2145
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
1532
2146
|
channel,
|
|
1533
|
-
agent
|
|
2147
|
+
agent,
|
|
2148
|
+
task,
|
|
1534
2149
|
),
|
|
1535
2150
|
agentEnv: this.accountAgentEnv(
|
|
1536
2151
|
channel,
|
|
@@ -1543,7 +2158,7 @@ export class Orchestrator {
|
|
|
1543
2158
|
cliSecret,
|
|
1544
2159
|
this.isPromptLevelSecretary(channel, agent.id),
|
|
1545
2160
|
),
|
|
1546
|
-
|
|
2161
|
+
task,
|
|
1547
2162
|
),
|
|
1548
2163
|
},
|
|
1549
2164
|
);
|
|
@@ -1571,16 +2186,33 @@ export class Orchestrator {
|
|
|
1571
2186
|
|
|
1572
2187
|
private attachCompatibilityKeyFor(
|
|
1573
2188
|
channel: Channel,
|
|
1574
|
-
|
|
2189
|
+
agent: RosterAgent,
|
|
2190
|
+
task: { envMode?: string | null },
|
|
1575
2191
|
): string | undefined {
|
|
1576
|
-
if (!this.isSecretary(channel, agentId)) return undefined;
|
|
1577
2192
|
// Key the durable runner to the policy actually passed to the adapter.
|
|
1578
2193
|
// The restricted profile is dormant today, but giving it the normal-tools
|
|
1579
2194
|
// generation would make a future opt-out able to attach a runner launched
|
|
1580
2195
|
// under tool denial.
|
|
1581
|
-
|
|
1582
|
-
?
|
|
1583
|
-
|
|
2196
|
+
const policy = this.isSecretary(channel, agent.id)
|
|
2197
|
+
? this.executionProfileFor(channel, agent.id) === "communicator"
|
|
2198
|
+
? RESTRICTED_SECRETARY_SESSION_ATTACH_KEY
|
|
2199
|
+
: SECRETARY_SESSION_ATTACH_KEY
|
|
2200
|
+
: undefined;
|
|
2201
|
+
const model = agent.model ?? "";
|
|
2202
|
+
const effort = agent.effort ?? "";
|
|
2203
|
+
// Preserve legacy null/policy-only attachment for agents that choose no
|
|
2204
|
+
// launch flags. Explicit model/effort gets a stable key so a host restart
|
|
2205
|
+
// cannot attach a runner created under different settings.
|
|
2206
|
+
const envMode = hostTaskEnvMode(task);
|
|
2207
|
+
if (envMode === "baked" && !model && !effort) return policy;
|
|
2208
|
+
return JSON.stringify([
|
|
2209
|
+
envMode === "baked" ? "roster-launch-v1" : "roster-launch-v2",
|
|
2210
|
+
...(envMode === "loader" ? [envMode] : []),
|
|
2211
|
+
agent.kind,
|
|
2212
|
+
model,
|
|
2213
|
+
effort,
|
|
2214
|
+
policy ?? "",
|
|
2215
|
+
]);
|
|
1584
2216
|
}
|
|
1585
2217
|
|
|
1586
2218
|
private isSecretary(channel: Channel, agentId: string): boolean {
|
|
@@ -1614,10 +2246,17 @@ export class Orchestrator {
|
|
|
1614
2246
|
channel: Channel,
|
|
1615
2247
|
agent: RosterAgent,
|
|
1616
2248
|
base: Record<string, string>,
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
2249
|
+
task: {
|
|
2250
|
+
environmentProvider: string | null;
|
|
2251
|
+
projectIds?: string | null;
|
|
2252
|
+
envMode?: string | null;
|
|
2253
|
+
},
|
|
1620
2254
|
): Record<string, string> {
|
|
2255
|
+
const floor = this.machineProjectEnv(task);
|
|
2256
|
+
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2257
|
+
channel.accountByAgent.delete(agent.id);
|
|
2258
|
+
return { ...floor, ...base };
|
|
2259
|
+
}
|
|
1621
2260
|
const account = pickEngineAccount(agent.kind, {
|
|
1622
2261
|
scope: channel.engineAccountScope,
|
|
1623
2262
|
});
|
|
@@ -1631,18 +2270,22 @@ export class Orchestrator {
|
|
|
1631
2270
|
}
|
|
1632
2271
|
|
|
1633
2272
|
/**
|
|
1634
|
-
*
|
|
1635
|
-
*
|
|
1636
|
-
*
|
|
1637
|
-
*
|
|
1638
|
-
* tasks — their env already carries it and must not be doubled into argv.
|
|
2273
|
+
* Rollout compatibility for a machine/self task created before ADR-153.
|
|
2274
|
+
* Those runtimes had no container env: project values rode every agent
|
|
2275
|
+
* spawn. Loader tasks must never take this path; they fetch through the
|
|
2276
|
+
* gateway, and adding values here would freeze them and bypass declarations.
|
|
1639
2277
|
*/
|
|
1640
2278
|
private machineProjectEnv(task: {
|
|
1641
2279
|
environmentProvider: string | null;
|
|
1642
2280
|
projectIds?: string | null;
|
|
2281
|
+
envMode?: string | null;
|
|
1643
2282
|
}): Record<string, string> {
|
|
1644
|
-
|
|
1645
|
-
|
|
2283
|
+
if (
|
|
2284
|
+
hostTaskEnvMode(task) !== "baked" ||
|
|
2285
|
+
!isEnvironmentTransportProvider(task.environmentProvider)
|
|
2286
|
+
) {
|
|
2287
|
+
return {};
|
|
2288
|
+
}
|
|
1646
2289
|
return projectEnvForTask(hostTaskProjectIds(task));
|
|
1647
2290
|
}
|
|
1648
2291
|
|
|
@@ -1662,6 +2305,18 @@ export class Orchestrator {
|
|
|
1662
2305
|
): void {
|
|
1663
2306
|
channel.sessions.set(agentId, session);
|
|
1664
2307
|
channel.everBound.add(agentId);
|
|
2308
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
2309
|
+
if (rosterChange) {
|
|
2310
|
+
channel.rosterSessionChanges.delete(agentId);
|
|
2311
|
+
if (rosterChange.state === "restarting") {
|
|
2312
|
+
this.emitHost({
|
|
2313
|
+
kind: "agent.session",
|
|
2314
|
+
taskId: channel.taskId,
|
|
2315
|
+
agentId,
|
|
2316
|
+
state: "ready",
|
|
2317
|
+
});
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
1665
2320
|
// The launch screen's `ready` rides the bind, not the tail of the start
|
|
1666
2321
|
// loop: whichever path brings the last roster agent up — the initial
|
|
1667
2322
|
// start, a reconcile that won the race with it, a respawn after Resume —
|
|
@@ -1729,16 +2384,29 @@ export class Orchestrator {
|
|
|
1729
2384
|
channel: Channel,
|
|
1730
2385
|
agentId: string,
|
|
1731
2386
|
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
2387
|
+
recovery?: RecoveryFence,
|
|
1732
2388
|
): Promise<AgentSession | null> {
|
|
1733
2389
|
const operation = this.createAndBindStableSessionCore(
|
|
1734
2390
|
channel,
|
|
1735
2391
|
agentId,
|
|
1736
2392
|
args,
|
|
2393
|
+
recovery,
|
|
1737
2394
|
);
|
|
1738
2395
|
channel.factoryOperations.add(operation);
|
|
2396
|
+
channel.factoryOperationByAgent.set(agentId, operation);
|
|
1739
2397
|
operation.then(
|
|
1740
|
-
() =>
|
|
1741
|
-
|
|
2398
|
+
() => {
|
|
2399
|
+
channel.factoryOperations.delete(operation);
|
|
2400
|
+
if (channel.factoryOperationByAgent.get(agentId) === operation) {
|
|
2401
|
+
channel.factoryOperationByAgent.delete(agentId);
|
|
2402
|
+
}
|
|
2403
|
+
},
|
|
2404
|
+
() => {
|
|
2405
|
+
channel.factoryOperations.delete(operation);
|
|
2406
|
+
if (channel.factoryOperationByAgent.get(agentId) === operation) {
|
|
2407
|
+
channel.factoryOperationByAgent.delete(agentId);
|
|
2408
|
+
}
|
|
2409
|
+
},
|
|
1742
2410
|
);
|
|
1743
2411
|
return operation;
|
|
1744
2412
|
}
|
|
@@ -1762,6 +2430,7 @@ export class Orchestrator {
|
|
|
1762
2430
|
try {
|
|
1763
2431
|
const res = await environment.exec({
|
|
1764
2432
|
argv: ["/bin/sh", "-c", codexHomeSeedCommand(home, perAgent)],
|
|
2433
|
+
projectEnvProfile: "maintenance",
|
|
1765
2434
|
user: "root",
|
|
1766
2435
|
env: RUNTIME_AUTHORITY_ENV,
|
|
1767
2436
|
timeoutMs: 30_000,
|
|
@@ -1791,8 +2460,27 @@ export class Orchestrator {
|
|
|
1791
2460
|
channel: Channel,
|
|
1792
2461
|
agentId: string,
|
|
1793
2462
|
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
2463
|
+
recovery?: RecoveryFence,
|
|
1794
2464
|
): Promise<AgentSession | null> {
|
|
2465
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2466
|
+
channel.reconcileAgain = true;
|
|
2467
|
+
return null;
|
|
2468
|
+
}
|
|
2469
|
+
if (
|
|
2470
|
+
channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
|
|
2471
|
+
) {
|
|
2472
|
+
channel.reconcileAgain = true;
|
|
2473
|
+
return null;
|
|
2474
|
+
}
|
|
2475
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, args.agent)) {
|
|
2476
|
+
channel.reconcileAgain = true;
|
|
2477
|
+
return null;
|
|
2478
|
+
}
|
|
1795
2479
|
const environment = await this.agentEnvironment(channel);
|
|
2480
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2481
|
+
channel.reconcileAgain = true;
|
|
2482
|
+
return null;
|
|
2483
|
+
}
|
|
1796
2484
|
// ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
|
|
1797
2485
|
// account share one CODEX_HOME, and codex's new sqlite state runtime
|
|
1798
2486
|
// cannot be opened by two concurrent processes — the second agent dies
|
|
@@ -1809,12 +2497,57 @@ export class Orchestrator {
|
|
|
1809
2497
|
? await this.isolateCodexHome(environment, agentId, args)
|
|
1810
2498
|
: args;
|
|
1811
2499
|
while (this.isActiveChannel(channel)) {
|
|
2500
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2501
|
+
channel.reconcileAgain = true;
|
|
2502
|
+
return null;
|
|
2503
|
+
}
|
|
2504
|
+
if (
|
|
2505
|
+
channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
|
|
2506
|
+
) {
|
|
2507
|
+
channel.reconcileAgain = true;
|
|
2508
|
+
return null;
|
|
2509
|
+
}
|
|
2510
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
2511
|
+
channel.reconcileAgain = true;
|
|
2512
|
+
return null;
|
|
2513
|
+
}
|
|
1812
2514
|
const generation = channel.configGeneration;
|
|
1813
2515
|
const session = await this.factory.create({ ...boundArgs, environment });
|
|
1814
2516
|
if (!this.isActiveChannel(channel)) {
|
|
1815
2517
|
await session.close().catch(() => {});
|
|
1816
2518
|
return null;
|
|
1817
2519
|
}
|
|
2520
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
2521
|
+
await session.close().catch(() => {});
|
|
2522
|
+
channel.reconcileAgain = true;
|
|
2523
|
+
return null;
|
|
2524
|
+
}
|
|
2525
|
+
const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
|
|
2526
|
+
if (pendingRosterClose?.state === "pending-close") {
|
|
2527
|
+
await this.closeRosterChangedSession(
|
|
2528
|
+
channel,
|
|
2529
|
+
agentId,
|
|
2530
|
+
session,
|
|
2531
|
+
pendingRosterClose,
|
|
2532
|
+
);
|
|
2533
|
+
channel.reconcileAgain = true;
|
|
2534
|
+
return null;
|
|
2535
|
+
}
|
|
2536
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
2537
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
2538
|
+
if (change) {
|
|
2539
|
+
await this.closeRosterChangedSession(
|
|
2540
|
+
channel,
|
|
2541
|
+
agentId,
|
|
2542
|
+
session,
|
|
2543
|
+
change,
|
|
2544
|
+
);
|
|
2545
|
+
} else {
|
|
2546
|
+
await session.close().catch(() => {});
|
|
2547
|
+
}
|
|
2548
|
+
channel.reconcileAgain = true;
|
|
2549
|
+
return null;
|
|
2550
|
+
}
|
|
1818
2551
|
if (generation === channel.configGeneration) {
|
|
1819
2552
|
// Keep the final generation check, stale cleanup, and map bind in one
|
|
1820
2553
|
// synchronous continuation. A config writer that resumes immediately
|
|
@@ -1831,6 +2564,17 @@ export class Orchestrator {
|
|
|
1831
2564
|
return null;
|
|
1832
2565
|
}
|
|
1833
2566
|
|
|
2567
|
+
private hasDesiredLaunchSettings(
|
|
2568
|
+
channel: Channel,
|
|
2569
|
+
agentId: string,
|
|
2570
|
+
launched: RosterAgent,
|
|
2571
|
+
): boolean {
|
|
2572
|
+
const desired = channel.roster.find((agent) => agent.id === agentId);
|
|
2573
|
+
return Boolean(
|
|
2574
|
+
desired && rosterLaunchChangeReason(launched, desired) === null,
|
|
2575
|
+
);
|
|
2576
|
+
}
|
|
2577
|
+
|
|
1834
2578
|
private agentEnvironment(
|
|
1835
2579
|
channel: Channel,
|
|
1836
2580
|
): Promise<TaskEnvironmentHandle<unknown>> {
|
|
@@ -1878,6 +2622,7 @@ export class Orchestrator {
|
|
|
1878
2622
|
const argv = cmd as [string, ...string[]];
|
|
1879
2623
|
const result = await environment.exec({
|
|
1880
2624
|
argv,
|
|
2625
|
+
projectEnvProfile: "maintenance",
|
|
1881
2626
|
cwd,
|
|
1882
2627
|
user: "node",
|
|
1883
2628
|
env: RUNTIME_AUTHORITY_ENV,
|
|
@@ -2029,6 +2774,10 @@ export class Orchestrator {
|
|
|
2029
2774
|
if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
|
|
2030
2775
|
continue;
|
|
2031
2776
|
}
|
|
2777
|
+
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2778
|
+
channel.reconcileAgain = true;
|
|
2779
|
+
continue;
|
|
2780
|
+
}
|
|
2032
2781
|
channel.spawning.add(agent.id);
|
|
2033
2782
|
try {
|
|
2034
2783
|
const session = await this.createAndBindStableSession(
|
|
@@ -2037,13 +2786,14 @@ export class Orchestrator {
|
|
|
2037
2786
|
{
|
|
2038
2787
|
taskId: channel.taskId,
|
|
2039
2788
|
agent,
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2789
|
+
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
2790
|
+
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
2791
|
+
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2792
|
+
channel,
|
|
2793
|
+
agent,
|
|
2794
|
+
task,
|
|
2795
|
+
),
|
|
2796
|
+
agentEnv: this.accountAgentEnv(
|
|
2047
2797
|
channel,
|
|
2048
2798
|
agent,
|
|
2049
2799
|
agentCliEnv(
|
|
@@ -2054,7 +2804,7 @@ export class Orchestrator {
|
|
|
2054
2804
|
cliSecret,
|
|
2055
2805
|
this.isPromptLevelSecretary(channel, agent.id),
|
|
2056
2806
|
),
|
|
2057
|
-
|
|
2807
|
+
task,
|
|
2058
2808
|
),
|
|
2059
2809
|
},
|
|
2060
2810
|
);
|
|
@@ -2185,6 +2935,78 @@ export class Orchestrator {
|
|
|
2185
2935
|
return remaining;
|
|
2186
2936
|
}
|
|
2187
2937
|
|
|
2938
|
+
/** Whether a retired generation still owns a cloud turn boundary. An
|
|
2939
|
+
* explicit interrupt already publishes its own aborted completion. */
|
|
2940
|
+
private needsAbortedTurnBoundary(
|
|
2941
|
+
channel: Channel,
|
|
2942
|
+
agentId: string,
|
|
2943
|
+
): boolean {
|
|
2944
|
+
return (
|
|
2945
|
+
!channel.interrupted.has(agentId) &&
|
|
2946
|
+
(channel.activeTurns.has(agentId) ||
|
|
2947
|
+
channel.openTurns.has(agentId) ||
|
|
2948
|
+
(channel.acceptedPrompts.get(agentId)?.length ?? 0) > 0)
|
|
2949
|
+
);
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
private retainRetiredTurnBoundary(
|
|
2953
|
+
channel: Channel,
|
|
2954
|
+
agentId: string,
|
|
2955
|
+
needed: boolean,
|
|
2956
|
+
): void {
|
|
2957
|
+
if (needed) channel.retiredTurnBoundaries.add(agentId);
|
|
2958
|
+
}
|
|
2959
|
+
|
|
2960
|
+
/** Publish a retained terminal edge at most once. Recovery can be overtaken
|
|
2961
|
+
* by remove/edit after its generation-local turn maps have been cleared. */
|
|
2962
|
+
private emitRetiredTurnAbort(channel: Channel, agentId: string): boolean {
|
|
2963
|
+
if (!channel.retiredTurnBoundaries.delete(agentId)) return false;
|
|
2964
|
+
this.emitHost({
|
|
2965
|
+
kind: "agent.turn_complete",
|
|
2966
|
+
taskId: channel.taskId,
|
|
2967
|
+
agentId,
|
|
2968
|
+
aborted: true,
|
|
2969
|
+
});
|
|
2970
|
+
return true;
|
|
2971
|
+
}
|
|
2972
|
+
|
|
2973
|
+
private supersedeRecovery(channel: Channel, agentId: string): boolean {
|
|
2974
|
+
const recovery = channel.recoveryByAgent.get(agentId);
|
|
2975
|
+
if (!recovery) return false;
|
|
2976
|
+
recovery.superseded = true;
|
|
2977
|
+
// A live edit/removal now owns the terminal edge. Publish it before the
|
|
2978
|
+
// roster lifecycle can emit restarting/removed or bind a replacement.
|
|
2979
|
+
return this.emitRetiredTurnAbort(channel, agentId);
|
|
2980
|
+
}
|
|
2981
|
+
|
|
2982
|
+
private publishRosterChangeAbort(
|
|
2983
|
+
channel: Channel,
|
|
2984
|
+
agentId: string,
|
|
2985
|
+
change: RosterSessionChange,
|
|
2986
|
+
): void {
|
|
2987
|
+
if (change.abortPublished) return;
|
|
2988
|
+
if (!this.emitRetiredTurnAbort(channel, agentId)) {
|
|
2989
|
+
this.emitHost({
|
|
2990
|
+
kind: "agent.turn_complete",
|
|
2991
|
+
taskId: channel.taskId,
|
|
2992
|
+
agentId,
|
|
2993
|
+
aborted: true,
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
change.abortPublished = true;
|
|
2997
|
+
}
|
|
2998
|
+
|
|
2999
|
+
private recoveryIsCurrent(
|
|
3000
|
+
channel: Channel,
|
|
3001
|
+
agentId: string,
|
|
3002
|
+
recovery: RecoveryFence,
|
|
3003
|
+
): boolean {
|
|
3004
|
+
return (
|
|
3005
|
+
channel.recoveryByAgent.get(agentId) === recovery &&
|
|
3006
|
+
!recovery.superseded
|
|
3007
|
+
);
|
|
3008
|
+
}
|
|
3009
|
+
|
|
2188
3010
|
/**
|
|
2189
3011
|
* Retire turn-local state owned by a runner generation before replacement.
|
|
2190
3012
|
*
|
|
@@ -2429,7 +3251,8 @@ export class Orchestrator {
|
|
|
2429
3251
|
exec: async (argv) => {
|
|
2430
3252
|
if (!handle || !canExec) return null;
|
|
2431
3253
|
const r = await handle.exec({
|
|
2432
|
-
argv,
|
|
3254
|
+
argv: agentCliArgv(handle, argv),
|
|
3255
|
+
projectEnvProfile: "task",
|
|
2433
3256
|
cwd: scratchDir,
|
|
2434
3257
|
user: "node",
|
|
2435
3258
|
env,
|
|
@@ -2618,6 +3441,13 @@ export class Orchestrator {
|
|
|
2618
3441
|
if (channel.sessions.get(agentId) !== session) break;
|
|
2619
3442
|
// The turn died with the session — drop its turn-state flags so a
|
|
2620
3443
|
// respawned session starts clean.
|
|
3444
|
+
const abortBoundaryOnRosterChange =
|
|
3445
|
+
this.needsAbortedTurnBoundary(channel, agentId);
|
|
3446
|
+
this.retainRetiredTurnBoundary(
|
|
3447
|
+
channel,
|
|
3448
|
+
agentId,
|
|
3449
|
+
abortBoundaryOnRosterChange,
|
|
3450
|
+
);
|
|
2621
3451
|
this.retireRunnerGeneration(channel, agentId);
|
|
2622
3452
|
// Claude under load (especially Docker Desktop macOS) occasionally
|
|
2623
3453
|
// unlinks ~/.claude.json mid-write during atomic config rewrites,
|
|
@@ -2627,13 +3457,32 @@ export class Orchestrator {
|
|
|
2627
3457
|
// session so the user keeps working instead of staring at a dead
|
|
2628
3458
|
// chat.
|
|
2629
3459
|
const agent = channel.roster.find((a) => a.id === agentId);
|
|
3460
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3461
|
+
// The authoritative roster already wants a different launch. Do not
|
|
3462
|
+
// rotate/recover this obsolete generation with its captured account;
|
|
3463
|
+
// preserve the live adapter so the normal stale-session recycler
|
|
3464
|
+
// closes it (and fences late Codex notifications). The cloud still
|
|
3465
|
+
// needs a terminal edge for any partial turn before restarting.
|
|
3466
|
+
this.reconcileErroredRosterChange(
|
|
3467
|
+
channel,
|
|
3468
|
+
agentId,
|
|
3469
|
+
event.message,
|
|
3470
|
+
abortBoundaryOnRosterChange,
|
|
3471
|
+
);
|
|
3472
|
+
break;
|
|
3473
|
+
}
|
|
2630
3474
|
// ADR-076: a rate-limit/usage-cap error fails over to another account
|
|
2631
3475
|
// for this kind (cool the current one, recreate the session on the
|
|
2632
3476
|
// next, re-deliver the in-flight prompt). No-op with <2 accounts.
|
|
2633
3477
|
if (
|
|
2634
3478
|
event.retryable === "rate_limit" &&
|
|
2635
3479
|
agent &&
|
|
2636
|
-
(await this.rotateAccount(
|
|
3480
|
+
(await this.rotateAccount(
|
|
3481
|
+
channel,
|
|
3482
|
+
agentId,
|
|
3483
|
+
agent,
|
|
3484
|
+
abortBoundaryOnRosterChange,
|
|
3485
|
+
))
|
|
2637
3486
|
) {
|
|
2638
3487
|
break;
|
|
2639
3488
|
}
|
|
@@ -2650,13 +3499,71 @@ export class Orchestrator {
|
|
|
2650
3499
|
`[orchestrator] ${channel.taskId}/${agentId}: ${agent.kind} credential-invalid pattern matched: ${event.message.slice(0, 300)}`,
|
|
2651
3500
|
);
|
|
2652
3501
|
const accountId = channel.accountByAgent.get(agentId) ?? null;
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
3502
|
+
// The broker is a slow await between a failed turn and its recovery.
|
|
3503
|
+
// Own the spawn slot so a concurrent roster edit cannot publish
|
|
3504
|
+
// ready for a replacement before this old generation publishes its
|
|
3505
|
+
// aborted boundary (events carry no generation id in the cloud).
|
|
3506
|
+
const ownsBrokerSlot = !channel.spawning.has(agentId);
|
|
3507
|
+
if (ownsBrokerSlot) channel.spawning.add(agentId);
|
|
3508
|
+
let requested = false;
|
|
3509
|
+
try {
|
|
3510
|
+
requested = await requestCredentialRefresh({
|
|
3511
|
+
kind: agent.kind,
|
|
3512
|
+
accountId:
|
|
3513
|
+
accountId && !accountId.startsWith("default:")
|
|
3514
|
+
? accountId
|
|
3515
|
+
: null,
|
|
3516
|
+
scope: channel.engineAccountScope,
|
|
3517
|
+
});
|
|
3518
|
+
} catch (err) {
|
|
3519
|
+
console.warn(
|
|
3520
|
+
`[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
|
|
3521
|
+
err instanceof Error ? err.message : String(err)
|
|
3522
|
+
}`,
|
|
3523
|
+
);
|
|
3524
|
+
} finally {
|
|
3525
|
+
if (ownsBrokerSlot) channel.spawning.delete(agentId);
|
|
3526
|
+
}
|
|
3527
|
+
// Removal/edit/recovery may have retired this generation while the
|
|
3528
|
+
// broker request was in flight. Never let its continuation mutate a
|
|
3529
|
+
// replacement or repopulate state for an absent roster id.
|
|
3530
|
+
if (channel.sessions.get(agentId) !== session) {
|
|
3531
|
+
// An edit can complete its restart while the broker request is
|
|
3532
|
+
// still outstanding, consuming the roster-change marker at bind.
|
|
3533
|
+
// The captured launch tuple still proves that this failed turn
|
|
3534
|
+
// belongs to the retired generation and needs its aborted edge.
|
|
3535
|
+
const stillRostered = channel.roster.some(
|
|
3536
|
+
(candidate) => candidate.id === agentId,
|
|
3537
|
+
);
|
|
3538
|
+
if (
|
|
3539
|
+
stillRostered &&
|
|
3540
|
+
(!channel.sessions.has(agentId) ||
|
|
3541
|
+
!this.hasDesiredLaunchSettings(channel, agentId, agent))
|
|
3542
|
+
) {
|
|
3543
|
+
this.reconcileErroredRosterChange(
|
|
3544
|
+
channel,
|
|
3545
|
+
agentId,
|
|
3546
|
+
event.message,
|
|
3547
|
+
abortBoundaryOnRosterChange,
|
|
3548
|
+
);
|
|
3549
|
+
}
|
|
3550
|
+
break;
|
|
3551
|
+
}
|
|
3552
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3553
|
+
this.reconcileErroredRosterChange(
|
|
3554
|
+
channel,
|
|
3555
|
+
agentId,
|
|
3556
|
+
event.message,
|
|
3557
|
+
abortBoundaryOnRosterChange,
|
|
3558
|
+
);
|
|
3559
|
+
break;
|
|
3560
|
+
}
|
|
2658
3561
|
if (requested) {
|
|
2659
|
-
await this.refreshAgentToken(
|
|
3562
|
+
await this.refreshAgentToken(
|
|
3563
|
+
channel,
|
|
3564
|
+
agent,
|
|
3565
|
+
abortBoundaryOnRosterChange,
|
|
3566
|
+
);
|
|
2660
3567
|
break;
|
|
2661
3568
|
}
|
|
2662
3569
|
}
|
|
@@ -2670,7 +3577,11 @@ export class Orchestrator {
|
|
|
2670
3577
|
console.warn(
|
|
2671
3578
|
`[orchestrator] ${channel.taskId}/${agentId}: claude auth-revoked pattern matched: ${event.message.slice(0, 300)}`,
|
|
2672
3579
|
);
|
|
2673
|
-
await this.refreshAgentToken(
|
|
3580
|
+
await this.refreshAgentToken(
|
|
3581
|
+
channel,
|
|
3582
|
+
agent,
|
|
3583
|
+
abortBoundaryOnRosterChange,
|
|
3584
|
+
);
|
|
2674
3585
|
break;
|
|
2675
3586
|
}
|
|
2676
3587
|
const configMissing = CLAUDE_CONFIG_MISSING_PATTERNS.some((re) =>
|
|
@@ -2681,10 +3592,15 @@ export class Orchestrator {
|
|
|
2681
3592
|
// than replaying the failed one. Do not leave it looking in-flight to
|
|
2682
3593
|
// a later account refresh.
|
|
2683
3594
|
channel.lastPrompt.delete(agentId);
|
|
2684
|
-
await this.recoverClaudeAgent(
|
|
3595
|
+
await this.recoverClaudeAgent(
|
|
3596
|
+
channel,
|
|
3597
|
+
agentId,
|
|
3598
|
+
abortBoundaryOnRosterChange,
|
|
3599
|
+
);
|
|
2685
3600
|
break;
|
|
2686
3601
|
}
|
|
2687
3602
|
channel.lastPrompt.delete(agentId);
|
|
3603
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
2688
3604
|
this.emitHost({
|
|
2689
3605
|
kind: "agent.exit",
|
|
2690
3606
|
taskId: channel.taskId,
|
|
@@ -2730,7 +3646,11 @@ export class Orchestrator {
|
|
|
2730
3646
|
});
|
|
2731
3647
|
if (remainingTurns === 0) {
|
|
2732
3648
|
channel.lastPrompt.delete(agentId);
|
|
2733
|
-
|
|
3649
|
+
// A queued Secretary reply must run on the replacement settings,
|
|
3650
|
+
// not extend the obsolete generation past its first safe boundary.
|
|
3651
|
+
if (!channel.rosterSessionChanges.has(agentId)) {
|
|
3652
|
+
this.startNextPendingPrompt(channel, agentId);
|
|
3653
|
+
}
|
|
2734
3654
|
}
|
|
2735
3655
|
// A config repair never interrupts a turn. The stale id remains queued
|
|
2736
3656
|
// until this exact boundary, then reconcile closes and replaces it.
|
|
@@ -2761,18 +3681,73 @@ export class Orchestrator {
|
|
|
2761
3681
|
if (channel.sessions.get(agentId) !== session) break;
|
|
2762
3682
|
// Same zombie hazard as the error path — a session whose process
|
|
2763
3683
|
// ended (even cleanly) can never carry another turn.
|
|
3684
|
+
const abortBoundaryOnRosterChange =
|
|
3685
|
+
this.needsAbortedTurnBoundary(channel, agentId);
|
|
2764
3686
|
this.retireRunnerGeneration(channel, agentId);
|
|
2765
3687
|
channel.lastPrompt.delete(agentId);
|
|
2766
3688
|
channel.respawns.set(agentId, (channel.respawns.get(agentId) ?? 0) + 1);
|
|
2767
3689
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
2768
3690
|
channel.sessions.delete(agentId);
|
|
2769
|
-
channel.
|
|
3691
|
+
const rosterChangePending = channel.rosterSessionChanges.has(agentId);
|
|
3692
|
+
if (rosterChangePending && abortBoundaryOnRosterChange) {
|
|
3693
|
+
this.publishRosterChangeAbort(
|
|
3694
|
+
channel,
|
|
3695
|
+
agentId,
|
|
3696
|
+
channel.rosterSessionChanges.get(agentId)!,
|
|
3697
|
+
);
|
|
3698
|
+
}
|
|
3699
|
+
if (!rosterChangePending) channel.browserStaleSessions.delete(agentId);
|
|
2770
3700
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
2771
3701
|
this.reconcilePendingSecretary(channel, agentId);
|
|
3702
|
+
if (rosterChangePending) {
|
|
3703
|
+
void this.reconcileSessions(channel).catch((err: unknown) => {
|
|
3704
|
+
console.warn(
|
|
3705
|
+
`[orchestrator] ${channel.taskId}/${agentId}: roster-change exit reconcile failed: ${
|
|
3706
|
+
err instanceof Error ? err.message : String(err)
|
|
3707
|
+
}`,
|
|
3708
|
+
);
|
|
3709
|
+
});
|
|
3710
|
+
}
|
|
2772
3711
|
break;
|
|
2773
3712
|
}
|
|
2774
3713
|
}
|
|
2775
3714
|
|
|
3715
|
+
/** An engine error can race a live launch edit after a partial turn. Keep
|
|
3716
|
+
* the obsolete adapter owned until the stale recycler closes it, but close
|
|
3717
|
+
* the cloud turn boundary immediately so restarting/ready cannot leave the
|
|
3718
|
+
* crew UI stuck in a working state. */
|
|
3719
|
+
private reconcileErroredRosterChange(
|
|
3720
|
+
channel: Channel,
|
|
3721
|
+
agentId: string,
|
|
3722
|
+
reason: string,
|
|
3723
|
+
abortBoundary: boolean,
|
|
3724
|
+
): void {
|
|
3725
|
+
channel.lastPrompt.delete(agentId);
|
|
3726
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
3727
|
+
if (abortBoundary) {
|
|
3728
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
3729
|
+
if (change) this.publishRosterChangeAbort(channel, agentId, change);
|
|
3730
|
+
else this.emitRetiredTurnAbort(channel, agentId);
|
|
3731
|
+
}
|
|
3732
|
+
void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
|
|
3733
|
+
console.warn(
|
|
3734
|
+
`[orchestrator] ${channel.taskId}/${agentId}: roster-change error (${reason}) reconcile failed: ${
|
|
3735
|
+
err instanceof Error ? err.message : String(err)
|
|
3736
|
+
}`,
|
|
3737
|
+
);
|
|
3738
|
+
});
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
/** Removal owns an asynchronous close/durable-stop fence. Recovery paths
|
|
3742
|
+
* overtaken by remove/re-add must join it before they can spawn the re-added
|
|
3743
|
+
* id, otherwise removal can stop the replacement generation. */
|
|
3744
|
+
private async reconcileAfterRosterRefresh(channel: Channel): Promise<void> {
|
|
3745
|
+
const rosterRefresh = channel.rosterRefreshReady;
|
|
3746
|
+
if (rosterRefresh) await rosterRefresh;
|
|
3747
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3748
|
+
await this.reconcileSessions(channel);
|
|
3749
|
+
}
|
|
3750
|
+
|
|
2776
3751
|
/** A daemon loss is infrastructure recovery, not an agent crash. Preserve
|
|
2777
3752
|
* accepted work, drop the dead generation, and leave respawn/exit accounting
|
|
2778
3753
|
* untouched; the runtime-ready hook recreates it deterministically. */
|
|
@@ -2797,9 +3772,20 @@ export class Orchestrator {
|
|
|
2797
3772
|
channel.runtimeDeferredAgents.add(agentId);
|
|
2798
3773
|
}
|
|
2799
3774
|
channel.sessions.delete(agentId);
|
|
2800
|
-
channel.
|
|
3775
|
+
if (!channel.rosterSessionChanges.has(agentId)) {
|
|
3776
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3777
|
+
}
|
|
2801
3778
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
2802
3779
|
await session.close().catch(() => {});
|
|
3780
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
3781
|
+
if (rosterChange && this.isActiveChannel(channel)) {
|
|
3782
|
+
await this.closeRosterChangedSession(
|
|
3783
|
+
channel,
|
|
3784
|
+
agentId,
|
|
3785
|
+
session,
|
|
3786
|
+
rosterChange,
|
|
3787
|
+
);
|
|
3788
|
+
}
|
|
2803
3789
|
}
|
|
2804
3790
|
|
|
2805
3791
|
// -- session recovery -----------------------------------------------------
|
|
@@ -2812,6 +3798,7 @@ export class Orchestrator {
|
|
|
2812
3798
|
private async recoverClaudeAgent(
|
|
2813
3799
|
channel: Channel,
|
|
2814
3800
|
agentId: string,
|
|
3801
|
+
abortBoundaryOnRosterChange = false,
|
|
2815
3802
|
): Promise<void> {
|
|
2816
3803
|
if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
|
|
2817
3804
|
|
|
@@ -2823,6 +3810,7 @@ export class Orchestrator {
|
|
|
2823
3810
|
channel.sessions.delete(agentId);
|
|
2824
3811
|
channel.browserStaleSessions.delete(agentId);
|
|
2825
3812
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
3813
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
2826
3814
|
this.emitHost({
|
|
2827
3815
|
kind: "agent.exit",
|
|
2828
3816
|
taskId: channel.taskId,
|
|
@@ -2841,6 +3829,15 @@ export class Orchestrator {
|
|
|
2841
3829
|
channel.containerName,
|
|
2842
3830
|
channel.agentHandle ?? undefined,
|
|
2843
3831
|
);
|
|
3832
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3833
|
+
this.reconcileErroredRosterChange(
|
|
3834
|
+
channel,
|
|
3835
|
+
agentId,
|
|
3836
|
+
"Claude config recovery superseded by live roster edit",
|
|
3837
|
+
abortBoundaryOnRosterChange,
|
|
3838
|
+
);
|
|
3839
|
+
return;
|
|
3840
|
+
}
|
|
2844
3841
|
|
|
2845
3842
|
// Delete the old generation before awaiting close. Its adapter emits a
|
|
2846
3843
|
// terminal event during teardown; bindSession's identity guard must see
|
|
@@ -2848,7 +3845,9 @@ export class Orchestrator {
|
|
|
2848
3845
|
// a second replacement path.
|
|
2849
3846
|
const old = channel.sessions.get(agentId);
|
|
2850
3847
|
channel.sessions.delete(agentId);
|
|
2851
|
-
channel.
|
|
3848
|
+
if (!channel.rosterSessionChanges.has(agentId)) {
|
|
3849
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3850
|
+
}
|
|
2852
3851
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
2853
3852
|
this.retireRunnerGeneration(channel, agentId);
|
|
2854
3853
|
if (old) {
|
|
@@ -2859,6 +3858,27 @@ export class Orchestrator {
|
|
|
2859
3858
|
}
|
|
2860
3859
|
}
|
|
2861
3860
|
if (!this.isActiveChannel(channel)) return;
|
|
3861
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
3862
|
+
if (rosterChange) {
|
|
3863
|
+
// The generic recovery owned the close before the roster edit arrived.
|
|
3864
|
+
// close() is idempotent; route its completed boundary through the
|
|
3865
|
+
// roster lifecycle so the cloud still observes restarting -> ready.
|
|
3866
|
+
this.reconcileErroredRosterChange(
|
|
3867
|
+
channel,
|
|
3868
|
+
agentId,
|
|
3869
|
+
"Claude config recovery superseded by live roster edit",
|
|
3870
|
+
abortBoundaryOnRosterChange,
|
|
3871
|
+
);
|
|
3872
|
+
if (old) {
|
|
3873
|
+
await this.closeRosterChangedSession(
|
|
3874
|
+
channel,
|
|
3875
|
+
agentId,
|
|
3876
|
+
old,
|
|
3877
|
+
rosterChange,
|
|
3878
|
+
);
|
|
3879
|
+
}
|
|
3880
|
+
return;
|
|
3881
|
+
}
|
|
2862
3882
|
|
|
2863
3883
|
const agent = channel.roster.find((a) => a.id === agentId);
|
|
2864
3884
|
const task = getHostTask(channel.taskId);
|
|
@@ -2887,18 +3907,45 @@ export class Orchestrator {
|
|
|
2887
3907
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2888
3908
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2889
3909
|
channel,
|
|
2890
|
-
|
|
3910
|
+
agent,
|
|
3911
|
+
task,
|
|
2891
3912
|
),
|
|
2892
3913
|
agentEnv: boundAccount
|
|
2893
|
-
? {
|
|
2894
|
-
|
|
3914
|
+
? {
|
|
3915
|
+
...this.machineProjectEnv(task),
|
|
3916
|
+
...boundAccount.execEnv,
|
|
3917
|
+
...base,
|
|
3918
|
+
}
|
|
3919
|
+
: this.accountAgentEnv(channel, agent, base, task),
|
|
2895
3920
|
});
|
|
2896
|
-
if (!session)
|
|
3921
|
+
if (!session) {
|
|
3922
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3923
|
+
this.reconcileErroredRosterChange(
|
|
3924
|
+
channel,
|
|
3925
|
+
agentId,
|
|
3926
|
+
"Claude config recovery superseded by live roster edit",
|
|
3927
|
+
abortBoundaryOnRosterChange,
|
|
3928
|
+
);
|
|
3929
|
+
}
|
|
3930
|
+
return;
|
|
3931
|
+
}
|
|
3932
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
3933
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3934
|
+
this.reconcileErroredRosterChange(
|
|
3935
|
+
channel,
|
|
3936
|
+
agentId,
|
|
3937
|
+
"Claude config recovery superseded by live roster edit",
|
|
3938
|
+
abortBoundaryOnRosterChange,
|
|
3939
|
+
);
|
|
3940
|
+
}
|
|
3941
|
+
return;
|
|
3942
|
+
}
|
|
2897
3943
|
this.startNextPendingPrompt(channel, agentId);
|
|
2898
3944
|
|
|
2899
3945
|
const message = restored
|
|
2900
3946
|
? `${agentId} restarted — config file was missing in the container, restored from the host.`
|
|
2901
3947
|
: `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
|
|
3948
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
2902
3949
|
this.emitHost({
|
|
2903
3950
|
kind: "agent.exit",
|
|
2904
3951
|
taskId: channel.taskId,
|
|
@@ -2940,8 +3987,18 @@ export class Orchestrator {
|
|
|
2940
3987
|
channel: Channel,
|
|
2941
3988
|
agentId: string,
|
|
2942
3989
|
agent: RosterAgent,
|
|
3990
|
+
abortBoundaryOnRosterChange = false,
|
|
2943
3991
|
): Promise<boolean> {
|
|
2944
3992
|
if (!this.isActiveChannel(channel)) return true;
|
|
3993
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3994
|
+
this.reconcileErroredRosterChange(
|
|
3995
|
+
channel,
|
|
3996
|
+
agentId,
|
|
3997
|
+
"account rotation superseded by live roster edit",
|
|
3998
|
+
abortBoundaryOnRosterChange,
|
|
3999
|
+
);
|
|
4000
|
+
return true;
|
|
4001
|
+
}
|
|
2945
4002
|
// A concurrent terminal event for the same session is already being
|
|
2946
4003
|
// handled by the path that owns this spawn slot.
|
|
2947
4004
|
if (channel.spawning.has(agentId)) return true;
|
|
@@ -2964,6 +4021,8 @@ export class Orchestrator {
|
|
|
2964
4021
|
const task = getHostTask(channel.taskId);
|
|
2965
4022
|
if (!task || task.statusMirror !== "running") return false;
|
|
2966
4023
|
|
|
4024
|
+
const recovery: RecoveryFence = { superseded: false };
|
|
4025
|
+
channel.recoveryByAgent.set(agentId, recovery);
|
|
2967
4026
|
let replacement: AgentSession | null = null;
|
|
2968
4027
|
channel.spawning.add(agentId);
|
|
2969
4028
|
try {
|
|
@@ -2974,6 +4033,11 @@ export class Orchestrator {
|
|
|
2974
4033
|
channel.sessions.delete(agentId);
|
|
2975
4034
|
channel.browserStaleSessions.delete(agentId);
|
|
2976
4035
|
channel.browserPendingStaleSessions.delete(agentId);
|
|
4036
|
+
this.retainRetiredTurnBoundary(
|
|
4037
|
+
channel,
|
|
4038
|
+
agentId,
|
|
4039
|
+
abortBoundaryOnRosterChange,
|
|
4040
|
+
);
|
|
2977
4041
|
this.retireRunnerGeneration(channel, agentId);
|
|
2978
4042
|
if (old) {
|
|
2979
4043
|
try {
|
|
@@ -2983,6 +4047,25 @@ export class Orchestrator {
|
|
|
2983
4047
|
}
|
|
2984
4048
|
}
|
|
2985
4049
|
if (!this.isActiveChannel(channel)) return true;
|
|
4050
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4051
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
4052
|
+
if (rosterChange) {
|
|
4053
|
+
this.reconcileErroredRosterChange(
|
|
4054
|
+
channel,
|
|
4055
|
+
agentId,
|
|
4056
|
+
"account rotation superseded by live roster edit",
|
|
4057
|
+
abortBoundaryOnRosterChange,
|
|
4058
|
+
);
|
|
4059
|
+
if (old) {
|
|
4060
|
+
await this.closeRosterChangedSession(
|
|
4061
|
+
channel,
|
|
4062
|
+
agentId,
|
|
4063
|
+
old,
|
|
4064
|
+
rosterChange,
|
|
4065
|
+
);
|
|
4066
|
+
}
|
|
4067
|
+
return true;
|
|
4068
|
+
}
|
|
2986
4069
|
|
|
2987
4070
|
// The serialized materialization gate owns both account provisioning
|
|
2988
4071
|
// and, when enabled, browser migration.
|
|
@@ -2992,8 +4075,21 @@ export class Orchestrator {
|
|
|
2992
4075
|
true,
|
|
2993
4076
|
);
|
|
2994
4077
|
if (!this.isActiveChannel(channel)) return true;
|
|
4078
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
2995
4079
|
await this.ensureMcpConfig(channel);
|
|
2996
4080
|
if (!this.isActiveChannel(channel)) return true;
|
|
4081
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4082
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4083
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4084
|
+
this.reconcileErroredRosterChange(
|
|
4085
|
+
channel,
|
|
4086
|
+
agentId,
|
|
4087
|
+
"account rotation superseded by live roster edit",
|
|
4088
|
+
abortBoundaryOnRosterChange,
|
|
4089
|
+
);
|
|
4090
|
+
}
|
|
4091
|
+
return true;
|
|
4092
|
+
}
|
|
2997
4093
|
|
|
2998
4094
|
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
2999
4095
|
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
@@ -3018,12 +4114,39 @@ export class Orchestrator {
|
|
|
3018
4114
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
3019
4115
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
3020
4116
|
channel,
|
|
3021
|
-
|
|
4117
|
+
agent,
|
|
4118
|
+
task,
|
|
3022
4119
|
),
|
|
3023
|
-
agentEnv: {
|
|
4120
|
+
agentEnv: {
|
|
4121
|
+
...this.machineProjectEnv(task),
|
|
4122
|
+
...next.execEnv,
|
|
4123
|
+
...base,
|
|
4124
|
+
},
|
|
3024
4125
|
},
|
|
4126
|
+
recovery,
|
|
3025
4127
|
);
|
|
3026
|
-
if (!replacement)
|
|
4128
|
+
if (!replacement) {
|
|
4129
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4130
|
+
this.reconcileErroredRosterChange(
|
|
4131
|
+
channel,
|
|
4132
|
+
agentId,
|
|
4133
|
+
"account rotation superseded by live roster edit",
|
|
4134
|
+
abortBoundaryOnRosterChange,
|
|
4135
|
+
);
|
|
4136
|
+
}
|
|
4137
|
+
return true;
|
|
4138
|
+
}
|
|
4139
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4140
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4141
|
+
this.reconcileErroredRosterChange(
|
|
4142
|
+
channel,
|
|
4143
|
+
agentId,
|
|
4144
|
+
"account rotation superseded by live roster edit",
|
|
4145
|
+
abortBoundaryOnRosterChange,
|
|
4146
|
+
);
|
|
4147
|
+
}
|
|
4148
|
+
return true;
|
|
4149
|
+
}
|
|
3027
4150
|
|
|
3028
4151
|
this.emitSystemNote(
|
|
3029
4152
|
channel.taskId,
|
|
@@ -3033,14 +4156,35 @@ export class Orchestrator {
|
|
|
3033
4156
|
// Re-deliver the in-flight prompt so the interrupted turn resumes.
|
|
3034
4157
|
const prompt = channel.lastPrompt.get(agentId);
|
|
3035
4158
|
if (prompt) {
|
|
4159
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
3036
4160
|
this.startPrompt(channel, agentId, replacement, prompt);
|
|
3037
4161
|
} else {
|
|
4162
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
3038
4163
|
this.startNextPendingPrompt(channel, agentId);
|
|
3039
4164
|
}
|
|
3040
4165
|
} finally {
|
|
3041
4166
|
channel.spawning.delete(agentId);
|
|
4167
|
+
if (channel.recoveryByAgent.get(agentId) === recovery) {
|
|
4168
|
+
channel.recoveryByAgent.delete(agentId);
|
|
4169
|
+
}
|
|
3042
4170
|
if (!replacement) {
|
|
4171
|
+
if (!channel.roster.some((candidate) => candidate.id === agentId)) {
|
|
4172
|
+
channel.accountByAgent.delete(agentId);
|
|
4173
|
+
channel.rotations.delete(agentId);
|
|
4174
|
+
}
|
|
3043
4175
|
this.reconcilePendingSecretary(channel, agentId);
|
|
4176
|
+
if (
|
|
4177
|
+
this.isActiveChannel(channel) &&
|
|
4178
|
+
(recovery.superseded || channel.rosterSessionChanges.has(agentId))
|
|
4179
|
+
) {
|
|
4180
|
+
void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
|
|
4181
|
+
console.warn(
|
|
4182
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-rotation roster reconcile failed: ${
|
|
4183
|
+
err instanceof Error ? err.message : String(err)
|
|
4184
|
+
}`,
|
|
4185
|
+
);
|
|
4186
|
+
});
|
|
4187
|
+
}
|
|
3044
4188
|
}
|
|
3045
4189
|
}
|
|
3046
4190
|
|
|
@@ -3077,10 +4221,23 @@ export class Orchestrator {
|
|
|
3077
4221
|
agent: RosterAgent,
|
|
3078
4222
|
account: ResolvedEngineAccount,
|
|
3079
4223
|
note: string,
|
|
4224
|
+
abortBoundaryOnRosterChange = false,
|
|
3080
4225
|
): Promise<boolean> {
|
|
4226
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4227
|
+
this.reconcileErroredRosterChange(
|
|
4228
|
+
channel,
|
|
4229
|
+
agentId,
|
|
4230
|
+
"credential respawn superseded by live roster edit",
|
|
4231
|
+
abortBoundaryOnRosterChange,
|
|
4232
|
+
);
|
|
4233
|
+
return true;
|
|
4234
|
+
}
|
|
4235
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return true;
|
|
3081
4236
|
const task = getHostTask(channel.taskId);
|
|
3082
4237
|
if (!task || task.statusMirror !== "running") return false;
|
|
3083
4238
|
|
|
4239
|
+
const recovery: RecoveryFence = { superseded: false };
|
|
4240
|
+
channel.recoveryByAgent.set(agentId, recovery);
|
|
3084
4241
|
let replacement: AgentSession | null = null;
|
|
3085
4242
|
channel.spawning.add(agentId);
|
|
3086
4243
|
try {
|
|
@@ -3102,6 +4259,25 @@ export class Orchestrator {
|
|
|
3102
4259
|
}
|
|
3103
4260
|
}
|
|
3104
4261
|
if (!this.isActiveChannel(channel)) return true;
|
|
4262
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4263
|
+
const rosterChange = channel.rosterSessionChanges.get(agentId);
|
|
4264
|
+
if (rosterChange) {
|
|
4265
|
+
this.reconcileErroredRosterChange(
|
|
4266
|
+
channel,
|
|
4267
|
+
agentId,
|
|
4268
|
+
"credential respawn superseded by live roster edit",
|
|
4269
|
+
abortBoundaryOnRosterChange,
|
|
4270
|
+
);
|
|
4271
|
+
if (old) {
|
|
4272
|
+
await this.closeRosterChangedSession(
|
|
4273
|
+
channel,
|
|
4274
|
+
agentId,
|
|
4275
|
+
old,
|
|
4276
|
+
rosterChange,
|
|
4277
|
+
);
|
|
4278
|
+
}
|
|
4279
|
+
return true;
|
|
4280
|
+
}
|
|
3105
4281
|
|
|
3106
4282
|
await this.ensureBrowserSetup(
|
|
3107
4283
|
channel,
|
|
@@ -3109,8 +4285,21 @@ export class Orchestrator {
|
|
|
3109
4285
|
true,
|
|
3110
4286
|
);
|
|
3111
4287
|
if (!this.isActiveChannel(channel)) return true;
|
|
4288
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
3112
4289
|
await this.ensureMcpConfig(channel);
|
|
3113
4290
|
if (!this.isActiveChannel(channel)) return true;
|
|
4291
|
+
if (!this.recoveryIsCurrent(channel, agentId, recovery)) return true;
|
|
4292
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4293
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4294
|
+
this.reconcileErroredRosterChange(
|
|
4295
|
+
channel,
|
|
4296
|
+
agentId,
|
|
4297
|
+
"credential respawn superseded by live roster edit",
|
|
4298
|
+
abortBoundaryOnRosterChange,
|
|
4299
|
+
);
|
|
4300
|
+
}
|
|
4301
|
+
return true;
|
|
4302
|
+
}
|
|
3114
4303
|
|
|
3115
4304
|
const apiUrl = apiUrlFromCloudUrl(env.UAI_CLOUD_URL);
|
|
3116
4305
|
const cliSecret = loadTaskCliSecret(channel.taskId);
|
|
@@ -3125,32 +4314,84 @@ export class Orchestrator {
|
|
|
3125
4314
|
channel.accountByAgent.set(agentId, account.id);
|
|
3126
4315
|
noteEngineAccountUsed(account.id);
|
|
3127
4316
|
|
|
3128
|
-
replacement = await this.createAndBindStableSession(
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
channel,
|
|
3135
|
-
agentId,
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
4317
|
+
replacement = await this.createAndBindStableSession(
|
|
4318
|
+
channel,
|
|
4319
|
+
agentId,
|
|
4320
|
+
{
|
|
4321
|
+
taskId: channel.taskId,
|
|
4322
|
+
agent,
|
|
4323
|
+
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
4324
|
+
executionProfile: this.executionProfileFor(channel, agentId),
|
|
4325
|
+
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
4326
|
+
channel,
|
|
4327
|
+
agent,
|
|
4328
|
+
task,
|
|
4329
|
+
),
|
|
4330
|
+
agentEnv: {
|
|
4331
|
+
...this.machineProjectEnv(task),
|
|
4332
|
+
...account.execEnv,
|
|
4333
|
+
...base,
|
|
4334
|
+
},
|
|
4335
|
+
},
|
|
4336
|
+
recovery,
|
|
4337
|
+
);
|
|
4338
|
+
if (!replacement) {
|
|
4339
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4340
|
+
this.reconcileErroredRosterChange(
|
|
4341
|
+
channel,
|
|
4342
|
+
agentId,
|
|
4343
|
+
"credential respawn superseded by live roster edit",
|
|
4344
|
+
abortBoundaryOnRosterChange,
|
|
4345
|
+
);
|
|
4346
|
+
}
|
|
4347
|
+
return true;
|
|
4348
|
+
}
|
|
4349
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) {
|
|
4350
|
+
if (channel.rosterSessionChanges.has(agentId)) {
|
|
4351
|
+
this.reconcileErroredRosterChange(
|
|
4352
|
+
channel,
|
|
4353
|
+
agentId,
|
|
4354
|
+
"credential respawn superseded by live roster edit",
|
|
4355
|
+
abortBoundaryOnRosterChange,
|
|
4356
|
+
);
|
|
4357
|
+
}
|
|
4358
|
+
return true;
|
|
4359
|
+
}
|
|
3140
4360
|
|
|
3141
4361
|
this.emitSystemNote(channel.taskId, note);
|
|
3142
4362
|
|
|
3143
4363
|
// Re-deliver the in-flight prompt so the interrupted turn resumes.
|
|
3144
4364
|
const prompt = channel.lastPrompt.get(agentId);
|
|
3145
4365
|
if (prompt) {
|
|
4366
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
3146
4367
|
this.startPrompt(channel, agentId, replacement, prompt);
|
|
3147
4368
|
} else {
|
|
4369
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
3148
4370
|
this.startNextPendingPrompt(channel, agentId);
|
|
3149
4371
|
}
|
|
3150
4372
|
} finally {
|
|
3151
4373
|
channel.spawning.delete(agentId);
|
|
4374
|
+
if (channel.recoveryByAgent.get(agentId) === recovery) {
|
|
4375
|
+
channel.recoveryByAgent.delete(agentId);
|
|
4376
|
+
}
|
|
3152
4377
|
if (!replacement) {
|
|
4378
|
+
if (!channel.roster.some((candidate) => candidate.id === agentId)) {
|
|
4379
|
+
channel.accountByAgent.delete(agentId);
|
|
4380
|
+
channel.rotations.delete(agentId);
|
|
4381
|
+
}
|
|
3153
4382
|
this.reconcilePendingSecretary(channel, agentId);
|
|
4383
|
+
if (
|
|
4384
|
+
this.isActiveChannel(channel) &&
|
|
4385
|
+
(recovery.superseded || channel.rosterSessionChanges.has(agentId))
|
|
4386
|
+
) {
|
|
4387
|
+
void this.reconcileAfterRosterRefresh(channel).catch((err: unknown) => {
|
|
4388
|
+
console.warn(
|
|
4389
|
+
`[orchestrator] ${channel.taskId}/${agentId}: post-respawn roster reconcile failed: ${
|
|
4390
|
+
err instanceof Error ? err.message : String(err)
|
|
4391
|
+
}`,
|
|
4392
|
+
);
|
|
4393
|
+
});
|
|
4394
|
+
}
|
|
3154
4395
|
}
|
|
3155
4396
|
}
|
|
3156
4397
|
|
|
@@ -3179,14 +4420,21 @@ export class Orchestrator {
|
|
|
3179
4420
|
private async refreshAgentToken(
|
|
3180
4421
|
channel: Channel,
|
|
3181
4422
|
agent: RosterAgent,
|
|
4423
|
+
abortBoundaryOnRosterChange?: boolean,
|
|
3182
4424
|
): Promise<void> {
|
|
3183
4425
|
const agentId = agent.id;
|
|
3184
4426
|
if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
|
|
4427
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return;
|
|
4428
|
+
const abortBoundary =
|
|
4429
|
+
abortBoundaryOnRosterChange ??
|
|
4430
|
+
this.needsAbortedTurnBoundary(channel, agentId);
|
|
4431
|
+
this.retainRetiredTurnBoundary(channel, agentId, abortBoundary);
|
|
3185
4432
|
|
|
3186
4433
|
const tries = (channel.respawns.get(agentId) ?? 0) + 1;
|
|
3187
4434
|
channel.respawns.set(agentId, tries);
|
|
3188
4435
|
channel.respawnLastAt.set(agentId, Date.now());
|
|
3189
4436
|
if (tries > MAX_RESPAWNS_PER_AGENT) {
|
|
4437
|
+
channel.retiredTurnBoundaries.delete(agentId);
|
|
3190
4438
|
this.emitHost({
|
|
3191
4439
|
kind: "agent.exit",
|
|
3192
4440
|
taskId: channel.taskId,
|
|
@@ -3203,7 +4451,10 @@ export class Orchestrator {
|
|
|
3203
4451
|
const account =
|
|
3204
4452
|
accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
|
|
3205
4453
|
accounts[0];
|
|
3206
|
-
if (!account)
|
|
4454
|
+
if (!account) {
|
|
4455
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
4456
|
+
return;
|
|
4457
|
+
}
|
|
3207
4458
|
|
|
3208
4459
|
await this.respawnAgentOnAccount(
|
|
3209
4460
|
channel,
|
|
@@ -3211,6 +4462,7 @@ export class Orchestrator {
|
|
|
3211
4462
|
agent,
|
|
3212
4463
|
account,
|
|
3213
4464
|
`${agentId}: ${agent.kind} credential refreshed — restarted the agent.`,
|
|
4465
|
+
abortBoundary,
|
|
3214
4466
|
);
|
|
3215
4467
|
}
|
|
3216
4468
|
|
|
@@ -3252,9 +4504,21 @@ export class Orchestrator {
|
|
|
3252
4504
|
async refreshEngineAccountAgents(kind: string): Promise<void> {
|
|
3253
4505
|
for (const channel of this.channels.values()) {
|
|
3254
4506
|
if (!this.isActiveChannel(channel)) continue;
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
4507
|
+
const agentIds = channel.roster
|
|
4508
|
+
.filter((agent) => agent.kind === kind)
|
|
4509
|
+
.map((agent) => agent.id);
|
|
4510
|
+
for (const agentId of agentIds) {
|
|
4511
|
+
const agent = channel.roster.find(
|
|
4512
|
+
(candidate) => candidate.id === agentId,
|
|
4513
|
+
);
|
|
4514
|
+
if (
|
|
4515
|
+
!agent ||
|
|
4516
|
+
agent.kind !== kind ||
|
|
4517
|
+
!channel.sessions.has(agentId)
|
|
4518
|
+
) {
|
|
4519
|
+
continue;
|
|
4520
|
+
}
|
|
4521
|
+
channel.respawns.set(agentId, 0);
|
|
3258
4522
|
try {
|
|
3259
4523
|
await this.refreshAgentToken(channel, agent);
|
|
3260
4524
|
} catch (err) {
|
|
@@ -3299,7 +4563,10 @@ export class Orchestrator {
|
|
|
3299
4563
|
this.channels.delete(taskId);
|
|
3300
4564
|
this.clearBrowserRetry(ch);
|
|
3301
4565
|
ch.browserStaleSessions.clear();
|
|
4566
|
+
ch.rosterSessionChanges.clear();
|
|
3302
4567
|
ch.browserPendingStaleSessions.clear();
|
|
4568
|
+
ch.browserConfigAffectedKinds.clear();
|
|
4569
|
+
ch.mcpProvisionAffectedKinds.clear();
|
|
3303
4570
|
ch.activeTurns.clear();
|
|
3304
4571
|
ch.acceptedPrompts.clear();
|
|
3305
4572
|
ch.pendingPrompts.clear();
|
|
@@ -3307,17 +4574,22 @@ export class Orchestrator {
|
|
|
3307
4574
|
ch.runtimeDeferredAgents.clear();
|
|
3308
4575
|
ch.openTurns.clear();
|
|
3309
4576
|
ch.interrupted.clear();
|
|
4577
|
+
ch.retiredTurnBoundaries.clear();
|
|
4578
|
+
ch.recoveryByAgent.clear();
|
|
3310
4579
|
ch.reconcileAgain = false;
|
|
3311
4580
|
const drains = [
|
|
3312
4581
|
ch.browserSetup,
|
|
3313
4582
|
ch.mcpConfigWrite,
|
|
3314
4583
|
ch.reconcileReady,
|
|
4584
|
+
ch.rosterRefreshReady,
|
|
4585
|
+
ch.persistedRosterReady,
|
|
3315
4586
|
ch.sessionsReady,
|
|
3316
4587
|
...ch.factoryOperations,
|
|
3317
4588
|
...ch.eventOperations,
|
|
3318
4589
|
].filter((promise): promise is Promise<unknown> => promise !== null);
|
|
3319
4590
|
const sessions = [...ch.sessions.values()];
|
|
3320
4591
|
ch.sessions.clear();
|
|
4592
|
+
ch.factoryOperationByAgent.clear();
|
|
3321
4593
|
const closes = sessions.map((session) =>
|
|
3322
4594
|
session.close().catch(() => {
|
|
3323
4595
|
// Teardown is best-effort; every adapter close is intended idempotent.
|
|
@@ -3350,18 +4622,38 @@ export class Orchestrator {
|
|
|
3350
4622
|
* tombstone prevents any agent process from respawning during maintenance. */
|
|
3351
4623
|
async quarantineChannelForRecovery(taskId: string): Promise<void> {
|
|
3352
4624
|
const currentSpec = this.channelSpecs.get(taskId);
|
|
4625
|
+
const suspended = suspendTaskGatewayTask(taskId);
|
|
4626
|
+
if (suspended) this.suspendedTaskGateways.set(taskId, suspended);
|
|
3353
4627
|
await this.closeChannel(taskId);
|
|
3354
4628
|
if (currentSpec && !this.channelSpecs.has(taskId)) {
|
|
3355
4629
|
this.channelSpecs.set(taskId, currentSpec);
|
|
3356
4630
|
}
|
|
3357
4631
|
}
|
|
3358
4632
|
|
|
4633
|
+
/** Reopen only the authenticated task gateway for sanctioned recovery
|
|
4634
|
+
* work. The channel tombstone remains in place until allowChannel(), so
|
|
4635
|
+
* agents cannot respawn while uai-init's dependency worker refreshes env. */
|
|
4636
|
+
reactivateTaskGatewayForRecovery(taskId: string): void {
|
|
4637
|
+
const suspended = this.suspendedTaskGateways.get(taskId);
|
|
4638
|
+
if (!suspended) return;
|
|
4639
|
+
if (!reactivateTaskGatewayTask(suspended)) {
|
|
4640
|
+
throw new Error(
|
|
4641
|
+
`task ${taskId} gateway generation could not be reactivated for recovery`,
|
|
4642
|
+
);
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
|
|
3359
4646
|
/** A successful task-up is the only transition that reopens a tombstoned
|
|
3360
4647
|
* task id after stop/teardown. */
|
|
3361
4648
|
allowChannel(taskId: string): void {
|
|
3362
4649
|
// A caller that forgot the task-up fence cannot reopen the id while old
|
|
3363
4650
|
// external operations are still draining.
|
|
3364
4651
|
if (this.channelClosures.has(taskId)) return;
|
|
4652
|
+
const suspended = this.suspendedTaskGateways.get(taskId);
|
|
4653
|
+
if (suspended) {
|
|
4654
|
+
reactivateTaskGatewayTask(suspended);
|
|
4655
|
+
this.suspendedTaskGateways.delete(taskId);
|
|
4656
|
+
}
|
|
3365
4657
|
this.blockedTasks.delete(taskId);
|
|
3366
4658
|
}
|
|
3367
4659
|
|
|
@@ -3378,6 +4670,10 @@ export class Orchestrator {
|
|
|
3378
4670
|
* (ADR-125) reads this — the cloud decides idleness from what it can see,
|
|
3379
4671
|
* but only the host can see a turn that is mid-flight RIGHT NOW. */
|
|
3380
4672
|
readonly #channelActivity = new Map<string, number>();
|
|
4673
|
+
private readonly suspendedTaskGateways = new Map<
|
|
4674
|
+
string,
|
|
4675
|
+
TaskGatewayRegistration
|
|
4676
|
+
>();
|
|
3381
4677
|
|
|
3382
4678
|
markChannelActivity(taskId: string): void {
|
|
3383
4679
|
this.#channelActivity.set(taskId, Date.now());
|
|
@@ -3403,7 +4699,11 @@ export class Orchestrator {
|
|
|
3403
4699
|
if (!task) return { ok: false, error: "unknown task" };
|
|
3404
4700
|
const project = task.composeProject;
|
|
3405
4701
|
if (!project) return { ok: false, error: "task has no running stack" };
|
|
3406
|
-
//
|
|
4702
|
+
// Revoke env/proxy access before a potentially long session drain, then
|
|
4703
|
+
// close sessions so nothing races to respawn them mid-stop.
|
|
4704
|
+
const suspendedGateway = suspendTaskGatewayTask(taskId);
|
|
4705
|
+
// ADR-153: no longer served; end its open proxy connections too.
|
|
4706
|
+
activeSecretsProxy()?.runtime.stopTask(taskId);
|
|
3407
4707
|
await this.closeChannel(taskId);
|
|
3408
4708
|
try {
|
|
3409
4709
|
await stopPreviewSidecars(taskId);
|
|
@@ -3424,11 +4724,13 @@ export class Orchestrator {
|
|
|
3424
4724
|
try {
|
|
3425
4725
|
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
3426
4726
|
if (!environment) {
|
|
4727
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
3427
4728
|
this.allowChannel(taskId);
|
|
3428
4729
|
return { ok: false, error: "task has no persisted environment" };
|
|
3429
4730
|
}
|
|
3430
4731
|
await environment.stop();
|
|
3431
4732
|
} catch (err) {
|
|
4733
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
3432
4734
|
this.allowChannel(taskId);
|
|
3433
4735
|
return {
|
|
3434
4736
|
ok: false,
|
|
@@ -3436,11 +4738,41 @@ export class Orchestrator {
|
|
|
3436
4738
|
};
|
|
3437
4739
|
}
|
|
3438
4740
|
upsertHostTask(taskId, { statusMirror: "stopped" });
|
|
4741
|
+
// The next start installs the CA again before anything reads the env;
|
|
4742
|
+
// no write still queued for the stopped container may land or record.
|
|
4743
|
+
invalidateTaskWrites(taskId);
|
|
4744
|
+
forgetTaskProxyReadiness(taskId);
|
|
4745
|
+
try {
|
|
4746
|
+
removeTaskGatewayTask(taskId);
|
|
4747
|
+
} catch (error) {
|
|
4748
|
+
// The runtime is stopped and the in-memory generation is already
|
|
4749
|
+
// inactive. A leftover durable file also loads inactive; explicit
|
|
4750
|
+
// Resume rotates it through task-up.
|
|
4751
|
+
console.warn(
|
|
4752
|
+
`[task-gateway] ${taskId}: paused with stale inactive gateway state: ${
|
|
4753
|
+
error instanceof Error ? error.message : String(error)
|
|
4754
|
+
}`,
|
|
4755
|
+
);
|
|
4756
|
+
}
|
|
4757
|
+
// Nothing serves the task's env now: no value of it stays in memory.
|
|
4758
|
+
forgetTaskEnvSnapshot(taskId);
|
|
3439
4759
|
clearTaskGatewayAcl(taskId);
|
|
3440
4760
|
return { ok: true };
|
|
3441
4761
|
}
|
|
3442
4762
|
}
|
|
3443
4763
|
|
|
4764
|
+
/** Launch-time settings only. Wire-compatible absence spellings are equal. */
|
|
4765
|
+
function rosterLaunchChangeReason(
|
|
4766
|
+
previous: RosterAgent,
|
|
4767
|
+
next: RosterAgent,
|
|
4768
|
+
): string | null {
|
|
4769
|
+
const changed: string[] = [];
|
|
4770
|
+
if (previous.kind !== next.kind) changed.push("engine");
|
|
4771
|
+
if ((previous.model ?? "") !== (next.model ?? "")) changed.push("model");
|
|
4772
|
+
if ((previous.effort ?? "") !== (next.effort ?? "")) changed.push("effort");
|
|
4773
|
+
return changed.length > 0 ? `${changed.join("/")} change` : null;
|
|
4774
|
+
}
|
|
4775
|
+
|
|
3444
4776
|
/** Exact host-wire role validation; never trims or canonicalises agent ids. */
|
|
3445
4777
|
export function hasValidSecretarySelection(
|
|
3446
4778
|
spec: Pick<
|
|
@@ -3581,6 +4913,7 @@ async function repairClaudeConfigInContainer(
|
|
|
3581
4913
|
if (environment) {
|
|
3582
4914
|
const write = await environment.exec({
|
|
3583
4915
|
argv: ["/bin/sh", "-c", repairScript],
|
|
4916
|
+
projectEnvProfile: "maintenance",
|
|
3584
4917
|
env: RUNTIME_AUTHORITY_ENV,
|
|
3585
4918
|
stdin: content,
|
|
3586
4919
|
timeoutMs: 30_000,
|
|
@@ -3677,6 +5010,28 @@ export function assembleFirstTurnPrompt(
|
|
|
3677
5010
|
* to every turn; only `initialPrompt` is delivered as the first user
|
|
3678
5011
|
* turn (see `assembleFirstTurnPrompt`).
|
|
3679
5012
|
*/
|
|
5013
|
+
/** The task's usable proxied keys right now (ADR-153 §5): its agents are
|
|
5014
|
+
* told which env vars hold placeholders, and for which domains; a withheld
|
|
5015
|
+
* key is absent from the task, so it is not mentioned. */
|
|
5016
|
+
function taskProxiedKeys(taskId: string): { key: string; domains: string[] }[] {
|
|
5017
|
+
// The same gate as the placeholder and the injection.
|
|
5018
|
+
return usableProxiedKeys(taskId).map((key) => ({ key: key.key, domains: [...key.domains] }));
|
|
5019
|
+
}
|
|
5020
|
+
|
|
5021
|
+
/** ADR-153 §5's line, with the exact key-to-domain mapping. */
|
|
5022
|
+
export function proxiedEnvLine(keys: readonly { readonly key: string; readonly domains: readonly string[] }[]): string {
|
|
5023
|
+
const listed = [...keys]
|
|
5024
|
+
.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
|
|
5025
|
+
.map((entry) => `\`${entry.key}\` (${entry.domains.join(", ")})`)
|
|
5026
|
+
.join(", ");
|
|
5027
|
+
const domainCount = new Set(keys.flatMap((entry) => entry.domains)).size;
|
|
5028
|
+
return (
|
|
5029
|
+
`These env vars hold placeholders, not the real secrets: ${listed}. Uai's proxy puts the real value in on ` +
|
|
5030
|
+
`HTTPS requests to ${domainCount === 1 ? "that domain" : "those domains"}, so use them as they are — don't ` +
|
|
5031
|
+
`replace, re-fetch or "fix" them. They won't work anywhere else.`
|
|
5032
|
+
);
|
|
5033
|
+
}
|
|
5034
|
+
|
|
3680
5035
|
export function buildSystemPreamble(
|
|
3681
5036
|
roster: Roster,
|
|
3682
5037
|
agent: RosterAgent,
|
|
@@ -3689,6 +5044,7 @@ export function buildSystemPreamble(
|
|
|
3689
5044
|
sharedFiles?: string,
|
|
3690
5045
|
mode?: "open" | "secretary",
|
|
3691
5046
|
secretaryAgentId?: string,
|
|
5047
|
+
proxiedKeys?: readonly { readonly key: string; readonly domains: readonly string[] }[],
|
|
3692
5048
|
): string {
|
|
3693
5049
|
const channelList = roster
|
|
3694
5050
|
.map((a) =>
|
|
@@ -4134,6 +5490,10 @@ export function buildSystemPreamble(
|
|
|
4134
5490
|
"",
|
|
4135
5491
|
]
|
|
4136
5492
|
: []),
|
|
5493
|
+
// ADR-153 §5: proxied keys reach the task as placeholders.
|
|
5494
|
+
...(proxiedKeys !== undefined && proxiedKeys.length > 0
|
|
5495
|
+
? ["## Proxied env vars", "", proxiedEnvLine(proxiedKeys), ""]
|
|
5496
|
+
: []),
|
|
4137
5497
|
// ADR-048: tell every agent with persona-granted permissions about its
|
|
4138
5498
|
// `uai` CLI. Secretary mode adds dispatch but does not invent any other
|
|
4139
5499
|
// permission: an empty persona permission set remains empty.
|
|
@@ -4329,7 +5689,15 @@ function beginRuntimeRecovery(
|
|
|
4329
5689
|
const predecessor = recoveryInFlight;
|
|
4330
5690
|
const sweep = async (): Promise<boolean> => {
|
|
4331
5691
|
try {
|
|
4332
|
-
return await recoverRunningTasks({
|
|
5692
|
+
return await recoverRunningTasks({
|
|
5693
|
+
lifecycle,
|
|
5694
|
+
// Boot recovery deliberately keeps its per-task barriers instead of
|
|
5695
|
+
// entering lifecycle slots, but still needs the live orchestrator to
|
|
5696
|
+
// suspend/reactivate gateway generations around uai-init.
|
|
5697
|
+
consumerLifecycle:
|
|
5698
|
+
lifecycle ?? globalForOrchestrator.__uaiOrchestrator,
|
|
5699
|
+
isCurrent,
|
|
5700
|
+
});
|
|
4333
5701
|
} catch (error) {
|
|
4334
5702
|
console.error(
|
|
4335
5703
|
"[orchestrator] recovery: unhandled failure",
|
|
@@ -5750,6 +7118,9 @@ export async function recoverRunningTasks(opts?: {
|
|
|
5750
7118
|
/** Post-boot recovery must queue behind live taskUp/taskDown operations.
|
|
5751
7119
|
* Boot recovery omits this and uses its earlier per-task barriers instead. */
|
|
5752
7120
|
lifecycle?: Orchestrator;
|
|
7121
|
+
/** Consumer quarantine/admission hooks without changing boot recovery's
|
|
7122
|
+
* barrier-based sequencing. Production boot supplies the live singleton. */
|
|
7123
|
+
consumerLifecycle?: Orchestrator;
|
|
5753
7124
|
/** Test seam; production waits for the shared image/CLI-volume reconcile. */
|
|
5754
7125
|
maintenanceReady?: Promise<void>;
|
|
5755
7126
|
/** A daemon-generation fence. False cancels every remaining side effect. */
|
|
@@ -5769,6 +7140,7 @@ export async function recoverRunningTasks(opts?: {
|
|
|
5769
7140
|
const result = await recoveryPass(
|
|
5770
7141
|
pendingTaskIds,
|
|
5771
7142
|
opts?.lifecycle,
|
|
7143
|
+
opts?.consumerLifecycle,
|
|
5772
7144
|
opts?.maintenanceReady ?? agentClisReady,
|
|
5773
7145
|
opts?.isCurrent ?? (() => true),
|
|
5774
7146
|
);
|
|
@@ -5813,6 +7185,7 @@ interface RecoveryPassResult {
|
|
|
5813
7185
|
async function recoveryPass(
|
|
5814
7186
|
onlyTaskIds?: ReadonlySet<string>,
|
|
5815
7187
|
lifecycle?: Orchestrator,
|
|
7188
|
+
consumerLifecycle?: Orchestrator,
|
|
5816
7189
|
maintenanceReady: Promise<void> = agentClisReady,
|
|
5817
7190
|
isCurrent: () => boolean = () => true,
|
|
5818
7191
|
): Promise<RecoveryPassResult> {
|
|
@@ -5870,6 +7243,7 @@ async function recoveryPass(
|
|
|
5870
7243
|
task,
|
|
5871
7244
|
maintenanceReady,
|
|
5872
7245
|
isCurrent,
|
|
7246
|
+
consumerLifecycle,
|
|
5873
7247
|
);
|
|
5874
7248
|
if (recovered) {
|
|
5875
7249
|
if (!lifecycle) finishTaskRecovery(task.taskId);
|
|
@@ -6064,15 +7438,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
6064
7438
|
/** Keep the MCP gateway reverse tunnel alive for a machine channel. The
|
|
6065
7439
|
* handle exposes tunnel argv only on the machine provider; anything else is
|
|
6066
7440
|
* a quiet no-op. */
|
|
6067
|
-
function ensureMachineGatewayTunnelFor(
|
|
7441
|
+
async function ensureMachineGatewayTunnelFor(
|
|
6068
7442
|
taskId: string,
|
|
6069
7443
|
handle: TaskEnvironmentHandle<unknown> | null,
|
|
6070
|
-
|
|
7444
|
+
waitUntilReady = false,
|
|
7445
|
+
): Promise<void> {
|
|
6071
7446
|
const argv = handle?.gatewayTunnelSshArgs;
|
|
6072
7447
|
if (!argv) return;
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
7448
|
+
const source = { gatewayTunnelSshArgs: argv.bind(handle) };
|
|
7449
|
+
const loaderPort =
|
|
7450
|
+
handle.descriptor.envMode === "loader"
|
|
7451
|
+
? MACHINE_GATEWAY_TUNNEL_PORT
|
|
7452
|
+
: undefined;
|
|
7453
|
+
if (waitUntilReady && loaderPort !== undefined) {
|
|
7454
|
+
await ensureMachineGatewayTunnelReady(taskId, source, loaderPort);
|
|
7455
|
+
return;
|
|
7456
|
+
}
|
|
7457
|
+
ensureMachineGatewayTunnel(taskId, source, loaderPort);
|
|
6076
7458
|
}
|
|
6077
7459
|
|
|
6078
7460
|
async function materializeAgentCli(
|
|
@@ -6107,25 +7489,105 @@ async function recoverPersistedTaskEnvironment(
|
|
|
6107
7489
|
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
6108
7490
|
if (environment === null) return true;
|
|
6109
7491
|
if (!isCurrent()) return true;
|
|
6110
|
-
|
|
6111
|
-
|
|
6112
|
-
|
|
6113
|
-
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6120
|
-
|
|
6121
|
-
|
|
6122
|
-
|
|
6123
|
-
|
|
6124
|
-
|
|
7492
|
+
// ADR-153: a recovery may restart the task's container or VM, which takes
|
|
7493
|
+
// the proxy's CA, chain and names with it, and its forwarder. Before the
|
|
7494
|
+
// provider touches it, nothing recorded counts any more and no write queued
|
|
7495
|
+
// or under way may land: a reconcile in flight ends with its exec under
|
|
7496
|
+
// way, and none pins again until this recovery's own slot has the
|
|
7497
|
+
// forwarder running.
|
|
7498
|
+
invalidateTaskWrites(task.taskId);
|
|
7499
|
+
forgetTaskProxyReadiness(task.taskId);
|
|
7500
|
+
// A running machine's editor can submit a command before any agent channel
|
|
7501
|
+
// is reconstructed. Re-arm the authenticated gateway tunnel at recovery,
|
|
7502
|
+
// not only at the first session spawn, so that command can refresh env.
|
|
7503
|
+
void ensureMachineGatewayTunnelFor(task.taskId, environment);
|
|
7504
|
+
let consumersPrepared = false;
|
|
7505
|
+
let tunnelReadinessFailed = false;
|
|
7506
|
+
const prepareConsumers = async (): Promise<void> => {
|
|
7507
|
+
if (!isCurrent()) return;
|
|
7508
|
+
try {
|
|
7509
|
+
await ensureMachineGatewayTunnelFor(task.taskId, environment, true);
|
|
7510
|
+
} catch (error) {
|
|
7511
|
+
tunnelReadinessFailed = true;
|
|
7512
|
+
console.warn(
|
|
7513
|
+
`[machine-tunnel] ${task.taskId}: recovery deferred until the gateway tunnel is ready (${error instanceof Error ? error.message : String(error)})`,
|
|
7514
|
+
);
|
|
7515
|
+
throw error;
|
|
7516
|
+
}
|
|
7517
|
+
// ADR-153 "Order", in one slot of the task's write queue: the CA (or,
|
|
7518
|
+
// with the proxy off, the pins out), then the forwarder, then the pins,
|
|
7519
|
+
// so no name is pinned with nothing listening behind it. The forwarder
|
|
7520
|
+
// carries the generation's secret into the task as root, and the
|
|
7521
|
+
// reactivation opens the gateway to it: neither once this recovery is
|
|
7522
|
+
// stale or the runner is shutting down. Every exec is fenced, so
|
|
7523
|
+
// shutdown waits for one under way rather than exiting under it, and
|
|
7524
|
+
// refuses (throws) any that would start after.
|
|
7525
|
+
if (!isCurrent()) return;
|
|
7526
|
+
const secretsProxy = activeSecretsProxy();
|
|
7527
|
+
let proxyFailure: RecoveryProxyFailure | null;
|
|
7528
|
+
try {
|
|
7529
|
+
proxyFailure = await applyTaskRootSteps(
|
|
7530
|
+
environment,
|
|
7531
|
+
environment.descriptor.envMode === "loader" ? { runtime: secretsProxy?.runtime ?? null } : null,
|
|
7532
|
+
(fenced) => ensureContainerTaskEnvForwarder(fenced),
|
|
7533
|
+
isCurrent,
|
|
7534
|
+
);
|
|
7535
|
+
} catch (error) {
|
|
7536
|
+
// Made stale meanwhile, its exec refused: recover() handles a stale
|
|
7537
|
+
// recovery itself.
|
|
7538
|
+
if (!isCurrent()) return;
|
|
7539
|
+
throw error;
|
|
7540
|
+
}
|
|
7541
|
+
if (proxyFailure !== null) {
|
|
7542
|
+
console.warn(
|
|
7543
|
+
`[secrets-proxy] ${task.taskId}: recovery could not reconcile the proxy's pins and trust: ${
|
|
7544
|
+
proxyFailure.error instanceof Error ? proxyFailure.error.message : String(proxyFailure.error)
|
|
7545
|
+
}`,
|
|
7546
|
+
);
|
|
7547
|
+
// A failure leaves the task up without the proxy: the trust watcher
|
|
7548
|
+
// retries, and its proxied keys stay withheld until the CA and the
|
|
7549
|
+
// pins are in.
|
|
7550
|
+
secretsProxy?.runtime.requestTrustDelivery();
|
|
7551
|
+
}
|
|
7552
|
+
if (!isCurrent()) return;
|
|
7553
|
+
// Shutdown began during the launch: not prepared, so the consumers stay
|
|
7554
|
+
// unadmitted, as for any failed preparation.
|
|
7555
|
+
if (taskWritesQuiescing()) throw new Error("the runner is shutting down");
|
|
7556
|
+
lifecycle?.reactivateTaskGatewayForRecovery(task.taskId);
|
|
7557
|
+
consumersPrepared = true;
|
|
7558
|
+
};
|
|
7559
|
+
try {
|
|
7560
|
+
const result = await environment.recover({
|
|
7561
|
+
maintenanceReady,
|
|
7562
|
+
isCurrent,
|
|
7563
|
+
...(lifecycle
|
|
7564
|
+
? {
|
|
7565
|
+
quarantineConsumers: () =>
|
|
7566
|
+
lifecycle.quarantineChannelForRecovery(task.taskId),
|
|
7567
|
+
prepareConsumers,
|
|
7568
|
+
allowConsumers: () => lifecycle.allowChannel(task.taskId),
|
|
7569
|
+
ensureConsumersStarted: async () => {
|
|
7570
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
7571
|
+
},
|
|
7572
|
+
}
|
|
7573
|
+
: {}),
|
|
7574
|
+
});
|
|
7575
|
+
if (result.outcome === "recovered" && !consumersPrepared && isCurrent()) {
|
|
7576
|
+
const current = getHostTask(task.taskId);
|
|
7577
|
+
if (current?.statusMirror && isActive(current.statusMirror)) {
|
|
7578
|
+
await prepareConsumers();
|
|
7579
|
+
}
|
|
7580
|
+
}
|
|
7581
|
+
return result.outcome === "recovered";
|
|
7582
|
+
} catch (error) {
|
|
7583
|
+
if (tunnelReadinessFailed) return false;
|
|
7584
|
+
throw error;
|
|
7585
|
+
}
|
|
6125
7586
|
}
|
|
6126
7587
|
|
|
6127
7588
|
interface RecoveryTaskConsumers {
|
|
6128
7589
|
quarantineChannelForRecovery(taskId: string): Promise<void>;
|
|
7590
|
+
prepareChannelForRecovery(taskId: string): Promise<void>;
|
|
6129
7591
|
allowChannel(taskId: string): void;
|
|
6130
7592
|
ensureStarted(taskId: string): Promise<boolean | void>;
|
|
6131
7593
|
}
|
|
@@ -6588,6 +8050,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
6588
8050
|
'[ "$(/usr/bin/stat -c %U /opt/uai/store/pnpm)" = "node" ] || { echo "store dir is not node-owned" >&2; exit 1; }; ' +
|
|
6589
8051
|
"fi",
|
|
6590
8052
|
],
|
|
8053
|
+
projectEnvProfile: "maintenance",
|
|
6591
8054
|
user: "root",
|
|
6592
8055
|
env: {},
|
|
6593
8056
|
timeoutMs: 30_000,
|
|
@@ -6614,23 +8077,47 @@ export async function recoverAppleTaskEnvironment(
|
|
|
6614
8077
|
);
|
|
6615
8078
|
return settleFromStatus(afterOverlay);
|
|
6616
8079
|
}
|
|
8080
|
+
// The dependency worker inside uai-init performs its one loader sync. Its
|
|
8081
|
+
// root forwarder and authenticated generation must be live first, while
|
|
8082
|
+
// public agent admission remains quarantined until initialization ends.
|
|
8083
|
+
let consumersPreparedForInit = false;
|
|
8084
|
+
const reclosePreparedConsumers = async (): Promise<void> => {
|
|
8085
|
+
if (!consumersPreparedForInit) return;
|
|
8086
|
+
consumersPreparedForInit = false;
|
|
8087
|
+
await context.quarantineConsumers?.();
|
|
8088
|
+
};
|
|
8089
|
+
if (context.prepareConsumers) {
|
|
8090
|
+
await context.prepareConsumers();
|
|
8091
|
+
consumersPreparedForInit = true;
|
|
8092
|
+
}
|
|
8093
|
+
if (!context.isCurrent()) {
|
|
8094
|
+
await reclosePreparedConsumers();
|
|
8095
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
8096
|
+
}
|
|
6617
8097
|
// Dependencies + code-server. The in-container deadline (TERM then KILL)
|
|
6618
8098
|
// means a host-side expiry can never leave the initializer mutating the
|
|
6619
8099
|
// workspace while agents use it; exit 78 is a runtime-authority violation
|
|
6620
8100
|
// and quarantines exactly like at task-up.
|
|
6621
|
-
const init = await environment
|
|
6622
|
-
|
|
6623
|
-
|
|
6624
|
-
|
|
6625
|
-
|
|
6626
|
-
|
|
6627
|
-
|
|
6628
|
-
|
|
6629
|
-
|
|
6630
|
-
|
|
6631
|
-
|
|
6632
|
-
|
|
8101
|
+
const init = await environment
|
|
8102
|
+
.exec({
|
|
8103
|
+
argv: [
|
|
8104
|
+
"/usr/bin/timeout",
|
|
8105
|
+
"--signal=TERM",
|
|
8106
|
+
"--kill-after=15s",
|
|
8107
|
+
"570s",
|
|
8108
|
+
"/usr/local/bin/uai-init",
|
|
8109
|
+
],
|
|
8110
|
+
projectEnvProfile: "maintenance",
|
|
8111
|
+
env: {},
|
|
8112
|
+
timeoutMs: 10 * 60_000,
|
|
8113
|
+
maxOutputBytes: 1024 * 1024,
|
|
8114
|
+
})
|
|
8115
|
+
.catch(async (error: unknown) => {
|
|
8116
|
+
await reclosePreparedConsumers();
|
|
8117
|
+
throw error;
|
|
8118
|
+
});
|
|
6633
8119
|
if (!context.isCurrent()) {
|
|
8120
|
+
await reclosePreparedConsumers();
|
|
6634
8121
|
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6635
8122
|
}
|
|
6636
8123
|
if (
|
|
@@ -6639,6 +8126,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
6639
8126
|
init.stdoutTruncated ||
|
|
6640
8127
|
init.stderrTruncated
|
|
6641
8128
|
) {
|
|
8129
|
+
await reclosePreparedConsumers();
|
|
6642
8130
|
// The host-side client died (deadline, overflow, forced settlement)
|
|
6643
8131
|
// without a real exit code, so the in-container initializer may STILL be
|
|
6644
8132
|
// mutating dependencies for several minutes. Never expose the task on an
|
|
@@ -6661,6 +8149,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
6661
8149
|
return settleFromStatus(after);
|
|
6662
8150
|
}
|
|
6663
8151
|
if (init.exitCode === 78) {
|
|
8152
|
+
await reclosePreparedConsumers();
|
|
6664
8153
|
await environment.stop();
|
|
6665
8154
|
const after = await environment.status();
|
|
6666
8155
|
if (after.state === "running" || after.state === "unknown") {
|
|
@@ -6679,6 +8168,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
6679
8168
|
return settleFromStatus(after);
|
|
6680
8169
|
}
|
|
6681
8170
|
if (init.exitCode === 127) {
|
|
8171
|
+
await reclosePreparedConsumers();
|
|
6682
8172
|
// Command-not-found from a read-only bind mount the guest cannot touch:
|
|
6683
8173
|
// the initializer itself is unrunnable — live 2026-08-26, containers
|
|
6684
8174
|
// whose versioned asset tree a host update had pruned looped here as
|
|
@@ -6709,6 +8199,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
6709
8199
|
);
|
|
6710
8200
|
}
|
|
6711
8201
|
if (!context.isCurrent()) {
|
|
8202
|
+
await reclosePreparedConsumers();
|
|
6712
8203
|
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6713
8204
|
}
|
|
6714
8205
|
db_setRuntime(descriptor.taskId, { codeServerPort: null });
|
|
@@ -6762,6 +8253,7 @@ async function recoverDockerTaskEnvironment(
|
|
|
6762
8253
|
|
|
6763
8254
|
const lifecycleCallbacks = [
|
|
6764
8255
|
context.quarantineConsumers,
|
|
8256
|
+
context.prepareConsumers,
|
|
6765
8257
|
context.allowConsumers,
|
|
6766
8258
|
context.ensureConsumersStarted,
|
|
6767
8259
|
];
|
|
@@ -6777,6 +8269,12 @@ async function recoverDockerTaskEnvironment(
|
|
|
6777
8269
|
}
|
|
6778
8270
|
await context.quarantineConsumers!();
|
|
6779
8271
|
},
|
|
8272
|
+
prepareChannelForRecovery: async (taskId) => {
|
|
8273
|
+
if (taskId !== descriptor.taskId) {
|
|
8274
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
8275
|
+
}
|
|
8276
|
+
await context.prepareConsumers!();
|
|
8277
|
+
},
|
|
6780
8278
|
allowChannel: (taskId) => {
|
|
6781
8279
|
if (taskId !== descriptor.taskId) {
|
|
6782
8280
|
throw new Error("task environment recovery consumer identity mismatch");
|
|
@@ -6984,6 +8482,13 @@ async function recoverOneDockerTask(
|
|
|
6984
8482
|
);
|
|
6985
8483
|
if (!isCurrent()) return true;
|
|
6986
8484
|
if (lifecycle) {
|
|
8485
|
+
await lifecycle.prepareChannelForRecovery(task.taskId);
|
|
8486
|
+
if (!isCurrent()) {
|
|
8487
|
+
// Preparation may have reactivated this task's gateway generation.
|
|
8488
|
+
// A superseded pass must close that authority again before returning.
|
|
8489
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
8490
|
+
return true;
|
|
8491
|
+
}
|
|
6987
8492
|
lifecycle.allowChannel(task.taskId);
|
|
6988
8493
|
await lifecycle.ensureStarted(task.taskId);
|
|
6989
8494
|
if (!isCurrent()) {
|
|
@@ -7119,6 +8624,23 @@ async function recoverOneDockerTask(
|
|
|
7119
8624
|
}
|
|
7120
8625
|
if (!isCurrent()) return true;
|
|
7121
8626
|
}
|
|
8627
|
+
// uai-init's dependency worker loads the task env once. Recreate the
|
|
8628
|
+
// forwarder and reactivate only its authenticated gateway before that
|
|
8629
|
+
// worker runs; the channel itself stays tombstoned until recovery finishes.
|
|
8630
|
+
let consumersPreparedForInit = false;
|
|
8631
|
+
const reclosePreparedConsumers = async (): Promise<void> => {
|
|
8632
|
+
if (!consumersPreparedForInit || !lifecycle) return;
|
|
8633
|
+
consumersPreparedForInit = false;
|
|
8634
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
8635
|
+
};
|
|
8636
|
+
if (lifecycle) {
|
|
8637
|
+
await lifecycle.prepareChannelForRecovery(task.taskId);
|
|
8638
|
+
consumersPreparedForInit = true;
|
|
8639
|
+
if (!isCurrent()) {
|
|
8640
|
+
await reclosePreparedConsumers();
|
|
8641
|
+
return true;
|
|
8642
|
+
}
|
|
8643
|
+
}
|
|
7122
8644
|
// uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
|
|
7123
8645
|
// repo. dockerCli's 30s default would SIGKILL it mid-install.
|
|
7124
8646
|
const initResult = await dockerCli(
|
|
@@ -7210,9 +8732,16 @@ async function recoverOneDockerTask(
|
|
|
7210
8732
|
RECOVERY_INIT_PATH,
|
|
7211
8733
|
],
|
|
7212
8734
|
{ timeoutMs: 10 * 60_000 },
|
|
7213
|
-
)
|
|
7214
|
-
|
|
8735
|
+
).catch(async (error: unknown) => {
|
|
8736
|
+
await reclosePreparedConsumers();
|
|
8737
|
+
throw error;
|
|
8738
|
+
});
|
|
8739
|
+
if (!isCurrent()) {
|
|
8740
|
+
await reclosePreparedConsumers();
|
|
8741
|
+
return true;
|
|
8742
|
+
}
|
|
7215
8743
|
if (initResult.status === 78) {
|
|
8744
|
+
await reclosePreparedConsumers();
|
|
7216
8745
|
const quarantined = await quarantineRunningContainer(containerName);
|
|
7217
8746
|
if (!isCurrent()) return true;
|
|
7218
8747
|
if (quarantined === "unreachable") return false;
|
|
@@ -7235,9 +8764,20 @@ async function recoverOneDockerTask(
|
|
|
7235
8764
|
`[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
|
|
7236
8765
|
);
|
|
7237
8766
|
}
|
|
7238
|
-
const portResult = await dockerPort(containerName, 8080)
|
|
7239
|
-
|
|
7240
|
-
|
|
8767
|
+
const portResult = await dockerPort(containerName, 8080).catch(
|
|
8768
|
+
async (error: unknown) => {
|
|
8769
|
+
await reclosePreparedConsumers();
|
|
8770
|
+
throw error;
|
|
8771
|
+
},
|
|
8772
|
+
);
|
|
8773
|
+
if (!isCurrent()) {
|
|
8774
|
+
await reclosePreparedConsumers();
|
|
8775
|
+
return true;
|
|
8776
|
+
}
|
|
8777
|
+
if (!portResult.reachable) {
|
|
8778
|
+
await reclosePreparedConsumers();
|
|
8779
|
+
return false;
|
|
8780
|
+
}
|
|
7241
8781
|
const port = portResult.port;
|
|
7242
8782
|
db_setRuntime(task.taskId, {
|
|
7243
8783
|
codeServerPort: port,
|
|
@@ -7254,7 +8794,10 @@ async function recoverOneDockerTask(
|
|
|
7254
8794
|
console.log(
|
|
7255
8795
|
`[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
|
|
7256
8796
|
);
|
|
7257
|
-
if (!isCurrent())
|
|
8797
|
+
if (!isCurrent()) {
|
|
8798
|
+
await reclosePreparedConsumers();
|
|
8799
|
+
return true;
|
|
8800
|
+
}
|
|
7258
8801
|
if (lifecycle) {
|
|
7259
8802
|
lifecycle.allowChannel(task.taskId);
|
|
7260
8803
|
await lifecycle.ensureStarted(task.taskId);
|