@runuai/host 0.9.170 → 0.9.172

package/lib/orchestrator.ts

@@ -1,4 +1,10 @@
+import { isOwnedInternalConnectionRuntime } from "./internal-connections";
+import { registerConnectionSecret } from "./connection-secret-redaction";
+import { buildCustomConnectionLaunch, tomlString, connectionNativeBlanks, connectionHomePreparation, newConnectionHome, connectionDnsLookupArgv, validateConnectionAddresses } from "./custom-connection-launch";
+import { connectionFailureReason, ConnectionEventSanitizer } from "./custom-connection-runtime";
+import { customConnectionEnabled, type CustomConnectionMaterial, type CustomConnectionLease, type CustomConnectionReason } from "../src/custom-connections";
 import { randomUUID } from "node:crypto";
+import { rememberCredentialCleanupPath } from "./credential-cleanup-inventory";
 import { AssignmentEvidenceJournal } from "./assignment-evidence";
 import { durableAgentSessionId } from "./agents/transport";
 import { agentOutputSource } from "./agents/output-source";
@@ -46,6 +52,7 @@ import { realAgentFactory } from "./agents/factory";
 import { resolveAgentMode } from "./agents/mode";
 import {
   clearPersistedCodexThread,
+  purgeInternalAgentSessionState,
   persistedAgentIds,
   stopPersistedAgentSession,
 } from "./agents/session-state";
@@ -633,7 +640,7 @@ function isEngineUsageKind(kind: string): kind is EngineUsageFrame["engine"] {
  *  or, for the Managed Engine, the task's open run alone (its agent's task
  *  token is its credential). */
 type LeasedIdentity =
-  | { subject: "account"; runEpoch: number; credentialId: string; revision: string; version: number }
+  | { subject: "account"; runEpoch: number; credentialId: string; revision: string; version: number; connection?: boolean; settingsHash?: string; internal?: boolean }
   | { subject: "task-token"; runEpoch: number };
 
 type SpawnArgs = Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">;
@@ -691,7 +698,7 @@ function sameLeasedIdentity(a: LeasedIdentity, b: LeasedIdentity): boolean {
       a.runEpoch === b.runEpoch &&
       a.credentialId === b.credentialId &&
       a.revision === b.revision &&
-      a.version === b.version
+      a.version === b.version && a.settingsHash === b.settingsHash && a.internal === b.internal
     );
   }
   return a.subject === b.subject && a.runEpoch === b.runEpoch;
@@ -815,6 +822,15 @@ export interface OrchestratorTaskStopResult {
 }
 
 export class Orchestrator {
+  private readonly connectionSessions = new WeakMap<AgentSession, { material: CustomConnectionMaterial; metadata: CustomConnectionLease; identity: Extract<LeasedIdentity, { subject: "account" }> }>();
+  private readonly connectionSecretReleases = new WeakMap<AgentSession, () => void>();
+  private readonly connectionRetries = new WeakMap<Channel, Map<string, {
+    timer: ReturnType<typeof setTimeout> | null;
+    identity: LeasedIdentity;
+    agent: RosterAgent;
+  }>>();
+  private readonly connectionExpiryTimers = new WeakMap<AgentSession, ReturnType<typeof setTimeout>>();
+  private readonly sessionIdentities = new WeakMap<AgentSession, LeasedIdentity>();
   private readonly sessionEventHandlers = new WeakMap<AgentSession, AgentEventHandler>();
   private readonly channels = new Map<string, Channel>();
   private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
@@ -1073,6 +1089,7 @@ export class Orchestrator {
     // took effect: writeAgentCli kept seeing the old empty list). Replacing the
     // array exactly also makes absence authoritative for live removal.
     channel.roster = [...spec.agents];
+    this.cancelStaleConnectionRetries(channel);
     for (const agent of added) {
       // Per-agent link/document skills, like getOrCreateChannel does at
       // channel birth. Best-effort.
@@ -1213,6 +1230,7 @@ export class Orchestrator {
     channel: Channel,
     agentId: string,
   ): Promise<void> {
+    this.cancelConnectionRetries(channel, agentId);
     const session = channel.sessions.get(agentId);
     const inFlightCreate = channel.factoryOperationByAgent.get(agentId);
     const lifecycleAtRemoval = channel.rosterSessionChanges.get(agentId);
@@ -1775,7 +1793,7 @@ export class Orchestrator {
     return lease.accounts
       .filter(
         (account): account is LeasedEngineAccount =>
-          account.engineKind === kind && isLeasedEngineAccount(account),
+          account.engineKind === kind && account.authKind !== "custom-connection" && isLeasedEngineAccount(account),
       )
       .map((account) => ({
         id: account.credentialId,
@@ -1783,7 +1801,7 @@ export class Orchestrator {
         label: account.label,
         isDefault: false,
         scope: account.scope,
-        authKind: account.authKind,
+        authKind: account.authKind as "env" | "config-dir",
         execEnv: {},
       }));
   }
@@ -2886,12 +2904,41 @@ export class Orchestrator {
     );
   }
 
+  private armConnectionExpiry(channel: Channel, agentId: string, session: AgentSession): void {
+    const expiresAt = this.connectionSessions.get(session)?.material.expiresAt;
+    if (expiresAt == null) return;
+    const check = (): void => {
+      this.connectionExpiryTimers.delete(session);
+      if (!this.isActiveChannel(channel) || channel.sessions.get(agentId) !== session) return;
+      const remaining = expiresAt - Date.now();
+      if (remaining <= 0) { this.expireConnectionSession(channel, agentId, session); return; }
+      const timer = setTimeout(check, Math.min(remaining, 2_147_483_647));
+      timer.unref?.();
+      this.connectionExpiryTimers.set(session, timer);
+    };
+    check();
+  }
+
+  private expireConnectionSession(channel: Channel, agentId: string, session: AgentSession): void {
+    if (channel.sessions.get(agentId) !== session) return;
+    const timer = this.connectionExpiryTimers.get(session);
+    if (timer) clearTimeout(timer);
+    this.connectionExpiryTimers.delete(session);
+    this.retainRetiredTurnBoundary(channel, agentId, this.needsAbortedTurnBoundary(channel, agentId));
+    this.endLeasedLaunches(channel, agentId, session);
+    this.emitRetiredTurnAbort(channel, agentId);
+    channel.sessions.delete(agentId);
+    this.leasedUnavailable(channel, agentId, "expired");
+    this.trackEventOperation(channel, session.close().catch(() => {}));
+  }
+
   private bindSession(
     channel: Channel,
     agentId: string,
     session: AgentSession,
   ): void {
     channel.sessions.set(agentId, session);
+    this.armConnectionExpiry(channel, agentId, session);
     channel.everBound.add(agentId);
     const rosterChange = channel.rosterSessionChanges.get(agentId);
     if (rosterChange) {
@@ -2922,7 +2969,9 @@ export class Orchestrator {
     // transport emission order across that await; otherwise the later boundary
     // can consume/reorder prompt state while the earlier error still owns it.
     let eventTail: Promise<void> = Promise.resolve();
-    const onEvent: AgentEventHandler = (event) => {
+    const connection = this.connectionSessions.get(session);
+    const sanitizer = connection ? new ConnectionEventSanitizer(connection.material.apiKey) : null;
+    const onSanitizedEvent: AgentEventHandler = (event) => {
       this.markChannelActivity(channel.taskId);
       // Recycle/teardown deletes the old session before calling close().
       // Adapters deliberately emit a final exit event; without this identity
@@ -2936,6 +2985,9 @@ export class Orchestrator {
       // ADR-154 §6: an engine that ended launches nothing more from now,
       // whatever its terminal handling still awaits (the runtime's proof).
       if (event.type === "error" || event.type === "exit") {
+        const timer = this.connectionExpiryTimers.get(session);
+        if (timer) clearTimeout(timer);
+        this.connectionExpiryTimers.delete(session);
         this.endLeasedLaunches(channel, agentId, session);
       }
       const operation = eventTail
@@ -2950,11 +3002,12 @@ export class Orchestrator {
             return;
           }
           await this.handleAgentEvent(channel, agentId, session, event);
+          if (connection && event.type === "error") await session.close().catch(() => {});
         })
         .catch((err: unknown) => {
           console.warn(
             `[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
-              err instanceof Error ? (err.stack ?? err.message) : String(err)
+              connection ? connectionFailureReason(err) : err instanceof Error ? (err.stack ?? err.message) : String(err)
             }`,
           );
         });
@@ -2965,6 +3018,9 @@ export class Orchestrator {
         () => channel.eventOperations.delete(operation),
       );
     };
+    const onEvent: AgentEventHandler = (event) => {
+      for (const safe of sanitizer ? sanitizer.push(event) : [event]) onSanitizedEvent(safe);
+    };
     this.sessionEventHandlers.set(session, onEvent);
     session.onEvent(onEvent);
   }
@@ -3047,12 +3103,64 @@ export class Orchestrator {
         if (!this.isActiveChannel(channel)) return null;
         const account = this.leasedAccountFor(channel, agentId, kind, open);
         if (!account) {
-          reason = `no ${kind} account this task can use`;
+          reason = args.agent.accountBinding === "required"
+            ? args.agent.accountId && open.connectionsUnsupported?.includes(args.agent.accountId) ? "unsupported" : "connection removed"
+            : `no ${kind} account this task can use`;
           break;
         }
         if (account.authKind === "env") {
           return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch);
         }
+        if (account.authKind === "custom-connection") {
+          try {
+            if (!account.connection || !account.connectionMaterial || args.agent.accountBinding !== "required") throw new Error("unsupported");
+            const home = newConnectionHome(account.credentialId, account.revision, agentId, account.connection.settingsHash);
+            const launch = buildCustomConnectionLaunch({
+              material: account.connectionMaterial, connection: account.connection,
+              model: args.agent.model, enabled: open.connectionsEnabled ?? [], home,
+              runtimeProvider: environment.descriptor.locator.provider,
+            });
+            for (const name of ["", "auth.json", "config.toml", "aws-config", "aws-credentials"]) {
+              rememberCredentialCleanupPath({
+                taskId: channel.taskId, runEpoch: open.assignment.runEpoch,
+                runtime: environment.descriptor.locator, lane: "engine-auth",
+                path: name ? `${home}/${name}` : home, credentialId: account.credentialId,
+              });
+            }
+            const prepared = await environment.exec({
+              argv: connectionHomePreparation(home), projectEnvProfile: "maintenance",
+              user: "node", timeoutMs: 20_000, maxOutputBytes: 1024,
+              admit: () => {
+                const current = leasedEnginesFor(channel.taskId);
+                const held = current?.open && current.assignment.runEpoch === open.assignment.runEpoch &&
+                  current.accounts.some(a => isLeasedEngineAccount(a) &&
+                    a.credentialId === account.credentialId && a.revision === account.revision &&
+                    a.version === account.version && a.connection?.settingsHash === account.connection?.settingsHash);
+                if (!this.isActiveChannel(channel) || !held || (account.connection?.internal === true && !isOwnedInternalConnectionRuntime(environment.descriptor.locator.provider))) throw new Error("unsupported");
+              },
+            });
+            if (prepared.exitCode !== 0) throw new Error("unsupported");
+            const lookup = await environment.exec({
+              argv: agentCliArgv(environment, connectionDnsLookupArgv(account.connectionMaterial.baseUrl)),
+              projectEnvProfile: "maintenance", user: "node", env: RUNTIME_AUTHORITY_ENV,
+              timeoutMs: 10_000, maxOutputBytes: 8192,
+            });
+            if (lookup.exitCode !== 0) throw new Error("unreachable");
+            validateConnectionAddresses(Buffer.from(lookup.stdout).toString("utf8"), account.connection.internal === true);
+            if (!this.isActiveChannel(channel)) return null;
+            const bound = this.withLeasedAccount(channel, agentId, {
+              ...args,
+              agentEnv: { ...(args.agentEnv ?? {}), ...launch.agentEnv, UAI_CUSTOM_CONNECTION: "1" },
+              cliConfigOverrides: [...launch.cliConfigOverrides,
+                ...(account.engineKind === "codex" ? [`model=${tomlString(launch.model)}`] : [])],
+              ...(launch.claudeSettings ? { claudeSettings: launch.claudeSettings } : {}),
+            }, { ...account, authKind: "env", env: launch.agentEnv }, open.assignment.runEpoch);
+            // The launch compiler, not the env-account path, owns this material.
+            const identity: LeasedIdentity = { subject: "account", runEpoch: open.assignment.runEpoch, credentialId: account.credentialId, revision: account.revision, version: account.version, connection: true, settingsHash: account.connection.settingsHash, internal: account.connection.internal === true };
+            channel.leasedIdentityByAgent.set(agentId, identity);
+            return { args: bound.args, identity };
+          } catch (error) { reason = connectionFailureReason(error); break; }
+        }
         // The account's home gets the task's managed config before anything
         // can start in it, whenever the gate's opening first reveals it.
         await this.ensureBrowserSetup(channel, kind === "codex");
@@ -3238,6 +3346,8 @@ export class Orchestrator {
    *  launch from this moment, before its terminal handling awaits anything.
    *  Every such handling retires or replaces the session. */
   private endLeasedLaunches(channel: Channel, agentId: string, session: AgentSession): void {
+    this.connectionSecretReleases.get(session)?.();
+    this.connectionSecretReleases.delete(session);
     const slot = channel.leasedLaunches.get(agentId);
     if (slot?.session === session) void this.retireLeasedLaunch(channel, agentId, slot);
   }
@@ -3255,6 +3365,7 @@ export class Orchestrator {
   private retireLeasedLaunch(channel: Channel, agentId: string, slot: LeasedLaunchSlot | undefined): Promise<void> {
     if (!slot || slot.retired) return Promise.resolve();
     slot.retired = true;
+    if (slot.session) { this.connectionSecretReleases.get(slot.session)?.(); this.connectionSecretReleases.delete(slot.session); }
     if (channel.leasedLaunches.get(agentId) === slot) channel.leasedLaunches.delete(agentId);
     // It never wrote an admission: nothing of it is in its runtime.
     if (!slot.admitted) return Promise.resolve();
@@ -3406,6 +3517,10 @@ export class Orchestrator {
       isLeasedEngineAccount(account) &&
       account.revision === identity.revision &&
       account.version === identity.version &&
+      (account.connection?.settingsHash === identity.settingsHash) &&
+      ((account.connection?.internal === true) === (identity.internal === true)) &&
+      (account.connection?.internal !== true || isOwnedInternalConnectionRuntime(channel.agentHandle?.descriptor.locator.provider)) &&
+      (account.authKind !== "custom-connection" || ((account.expiresAt === null || account.expiresAt > Date.now()) && !!account.connectionMaterial && customConnectionEnabled(account.connectionMaterial, lease.connectionsEnabled ?? []))) &&
       !engineVersionHeldBack(lease.assignment, identity)
     );
   }
@@ -3436,9 +3551,14 @@ export class Orchestrator {
     kind: string,
     lease: TaskEngineLease,
   ): LeasedEngineAccount | undefined {
+    const agent = channel.roster.find(candidate => candidate.id === agentId);
+    if (agent?.accountBinding === "required") {
+      return lease.accounts.find((account): account is LeasedEngineAccount => account.credentialId === agent.accountId && account.engineKind === kind && account.authKind === "custom-connection" && isLeasedEngineAccount(account));
+    }
+    if (lease.accounts.some(account => account.credentialId === agent?.accountId && account.authKind === "custom-connection")) return undefined;
     const accounts = lease.accounts.filter(
       (account): account is LeasedEngineAccount =>
-        account.engineKind === kind && isLeasedEngineAccount(account),
+        account.engineKind === kind && account.authKind !== "custom-connection" && isLeasedEngineAccount(account),
     );
     const bound = channel.accountByAgent.get(agentId);
     const preferredId = channel.roster.find(
@@ -3456,7 +3576,7 @@ export class Orchestrator {
         label: account.label,
         isDefault: false,
         scope: account.scope,
-        authKind: account.authKind,
+        authKind: account.authKind as "env" | "config-dir",
         execEnv: {},
       })),
       { preferredId },
@@ -3481,6 +3601,8 @@ export class Orchestrator {
       credentialId: account.credentialId,
       revision: account.revision,
       version: account.version,
+      connection: account.authKind === "custom-connection",
+      ...(account.connection ? { settingsHash: account.connection.settingsHash, internal: account.connection.internal === true } : {}),
     };
     channel.accountByAgent.set(agentId, account.credentialId);
     channel.leasedIdentityByAgent.set(agentId, identity);
@@ -3498,6 +3620,8 @@ export class Orchestrator {
           identity.credentialId,
           identity.revision,
           identity.version,
+          ...(account.connection ? [account.connection.settingsHash] : []),
+          ...(account.connection?.internal === true ? ["custom-internal"] : []),
         ]),
         agentEnv: {
           ...(args.agentEnv ?? {}),
@@ -3526,6 +3650,7 @@ export class Orchestrator {
     account: LeasedEngineAccount,
     environment: TaskEnvironmentHandle<unknown>,
   ): Promise<Record<string, string> | null> {
+    if (account.authKind === "custom-connection") return null;
     const marker = {
       [LEASED_RUN_ENV]: String(lease.assignment.runEpoch),
       [LEASED_ACCOUNT_ENV]: account.credentialId,
@@ -3562,6 +3687,7 @@ export class Orchestrator {
   private leasedEnginesChanged(taskId: string): void {
     const channel = this.channels.get(taskId);
     if (!channel || !this.isActiveChannel(channel) || !this.isLeasedChannel(channel)) return;
+    this.cancelStaleConnectionRetries(channel);
     if (!leasedEnginesFor(taskId)?.open) return;
     if (channel.leasedSyncRetry) {
       clearTimeout(channel.leasedSyncRetry);
@@ -3746,6 +3872,7 @@ export class Orchestrator {
         }
       }
     }
+    await purgeInternalAgentSessionState(request, [...runtimes.values()]);
     // Every admission of these runs not yet used is revoked in every runtime
     // they were written in, or that runtime is stopped or gone and its gates
     // with it: no revocation or slot retirement retained for them is left to
@@ -3763,6 +3890,7 @@ export class Orchestrator {
    *  so nothing launches on it again, its session is asked to stop, and it
    *  starts again on what the task holds once the gate opens. */
   private quiesceLeasedAgent(channel: Channel, agentId: string): void {
+    this.cancelConnectionRetries(channel, agentId);
     channel.leasedIdentityByAgent.delete(agentId);
     const session = channel.sessions.get(agentId);
     if (this.isActiveChannel(channel)) channel.leasedRespawn.add(agentId);
@@ -3932,6 +4060,8 @@ export class Orchestrator {
     }
     const onOwn = ownHold !== undefined;
     try {
+      if (onOwn && args.agent.accountBinding === "required") { this.leasedUnavailable(channel, agentId, "unsupported"); return null; }
+      args = { ...args, agentEnv: { ...connectionNativeBlanks(), UAI_CUSTOM_CONNECTION: "", ...(args.agentEnv ?? {}) } };
       // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
       // account share one CODEX_HOME, and codex's new sqlite state runtime
       // cannot be opened by two concurrent processes — the second agent dies
@@ -4036,7 +4166,12 @@ export class Orchestrator {
           void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
           channel.leasedLaunches.set(agentId, slot);
         }
-        const creating = this.factory.create({
+        const releaseConnectionSecret = identity?.subject === "account" && identity.connection
+          ? registerConnectionSecret(boundArgs.agentEnv?.UAI_CUSTOM_API_KEY || boundArgs.agentEnv?.AWS_BEARER_TOKEN_BEDROCK || boundArgs.agentEnv?.ANTHROPIC_AUTH_TOKEN || boundArgs.agentEnv?.ANTHROPIC_API_KEY || "")
+          : () => {};
+        let creating: Promise<AgentSession>;
+        try {
+          creating = this.factory.create({
           ...boundArgs,
           environment:
             identity && slot
@@ -4044,7 +4179,11 @@ export class Orchestrator {
               : onOwn
                 ? onOwnCredentialsEnvironment(environment)
                 : environment,
-        });
+          });
+        } catch (error) {
+          releaseConnectionSecret();
+          throw identity?.subject === "account" && identity.connection ? new Error(connectionFailureReason(error)) : error;
+        }
         // Settled before this spawn's own continuation: from here until it
         // binds, nothing launches on it.
         if (slot) {
@@ -4074,14 +4213,22 @@ export class Orchestrator {
         // after it launched, or its session is closed unbound) is retired at
         // once, before anything else: a launch it handed over that has not
         // started in its runtime never does.
-        const unbound = (): Promise<void> =>
-          slot ? this.retireLeasedLaunch(channel, agentId, slot) : Promise.resolve();
+        const unbound = (): Promise<void> => {
+          releaseConnectionSecret();
+          return slot ? this.retireLeasedLaunch(channel, agentId, slot) : Promise.resolve();
+        };
         let session: AgentSession;
         try {
           session = await creating;
         } catch (error) {
           await unbound();
-          throw error;
+          throw identity?.subject === "account" && identity.connection ? new Error(connectionFailureReason(error)) : error;
+        }
+        if (identity?.subject === "account" && identity.connection) this.connectionSecretReleases.set(session, releaseConnectionSecret);
+        if (identity) this.sessionIdentities.set(session, { ...identity });
+        if (identity?.subject === "account" && identity.connection) {
+          const account = leasedEnginesFor(channel.taskId)?.accounts.find((a): a is LeasedEngineAccount => a.credentialId === identity.credentialId && a.revision === identity.revision && isLeasedEngineAccount(a));
+          if (account?.connectionMaterial && account.connection) this.connectionSessions.set(session, { material: account.connectionMaterial, metadata: account.connection, identity: { ...identity } });
         }
         if (!this.isActiveChannel(channel)) {
           await Promise.all([unbound(), session.close().catch(() => {})]);
@@ -4178,6 +4325,7 @@ export class Orchestrator {
     scratchDir: string,
     timeoutMs: number,
   ): Promise<{ exitCode: number | null; stdout: string } | null> {
+    if (channel.roster.find(a => a.id === agentId)?.accountBinding === "required") return null;
     const request = (env: Record<string, string>, admit: () => void) => ({
       argv: agentCliArgv(handle, argv),
       projectEnvProfile: "task" as const,
@@ -4205,7 +4353,7 @@ export class Orchestrator {
     const lease = leasedEnginesFor(channel.taskId);
     if (!lease?.open) return null;
     const account = this.leasedAccountFor(channel, agentId, kind, lease);
-    if (!account) return null;
+    if (!account || account.authKind === "custom-connection") return null;
     const identity: LeasedIdentity = {
       subject: "account",
       runEpoch: lease.assignment.runEpoch,
@@ -4740,6 +4888,8 @@ export class Orchestrator {
     prompt: string | AcceptedPrompt,
     assignment?: AssignmentDelivery,
   ): void {
+    const connection = this.connectionSessions.get(session);
+    if (connection?.material.expiresAt != null && connection.material.expiresAt <= Date.now()) { this.expireConnectionSession(channel, agentId, session); return; }
     const replay = typeof prompt !== "string";
     const text = typeof prompt === "string" ? prompt : prompt.text;
     const leadRunId = replay ? (prompt as AcceptedPrompt).leadRunId ?? null : this.isSecretary(channel, agentId) ? channel.leadRunId ?? null : null;
@@ -4753,7 +4903,8 @@ export class Orchestrator {
       const begun = journal.begin({ sessionId, engine: session.kind,
         runEpoch: readTaskRunEpochState(channel.taskId)?.highestRunEpoch ?? 0,
         recipientGeneration: channel.roster.find(a => a.id === agentId)?.tokenGeneration ?? null,
-        runtimeVersion: evidenceRuntimeVersion, assignment: originalAssignment, replay, leadRunId });
+        runtimeVersion: evidenceRuntimeVersion, assignment: originalAssignment, replay, leadRunId,
+        actualAccount: (() => { const identity = this.sessionIdentities.get(session); return identity?.subject === "account" ? { credentialId: identity.credentialId, credentialRevision: identity.revision, connection: identity.connection === true } : null; })() });
       if (!begun) return; // stable assignment replay, already accepted
       turn = begun;
       retainedAssignment = begun.assignment;
@@ -4768,7 +4919,7 @@ export class Orchestrator {
     this.incrementActiveTurns(channel, agentId);
     void session.send(text).catch((err: unknown) => {
       if (turn) journal?.failed(sessionId, turn.turnId);
-      const diagnosticMessage = `Could not send the message to ${agentId}: ${err instanceof Error ? err.message : String(err)}`;
+      const diagnosticMessage = this.connectionSessions.has(session) ? connectionFailureReason(err) : `Could not send the message to ${agentId}: ${err instanceof Error ? err.message : String(err)}`;
       reportAgentError(diagnosticMessage, { taskId: channel.taskId, agentId, engine: session.kind },
         { capture: !(err instanceof ContainerRuntimeUnavailableError) });
       if (channel.sessions.get(agentId) === session) {
@@ -5091,6 +5242,8 @@ export class Orchestrator {
     if (!this.isActiveChannel(channel)) return;
     switch (event.type) {
       case "account_usage": {
+        // Connection limits come from explicit provider budgets, never CLI login probes.
+        if (this.connectionSessions.has(session)) break;
         // Only the exact live session's cloud-leased ACCOUNT may be named on
         // the wire. A runner default slot and the Managed Engine's task-token
         // identity are deliberately unreportable.
@@ -5115,7 +5268,7 @@ export class Orchestrator {
           credentialId: identity.credentialId,
           revision: identity.revision,
           usage: event.usage,
-          leasedAccount,
+          leasedAccount: leasedAccount?.authKind === "custom-connection" ? undefined : leasedAccount ? { ...leasedAccount, authKind: leasedAccount.authKind as "env" | "config-dir" } : undefined,
         });
 
         const coverage = event.usage.coverage;
@@ -5328,6 +5481,10 @@ export class Orchestrator {
           );
           break;
         }
+        if (agent?.accountBinding === "required") {
+          await this.recoverConnectionAgent(channel, agent, connectionFailureReason(diagnosticMessage), abortBoundaryOnRosterChange, event.resetsAt);
+          break;
+        }
         // ADR-076: a rate-limit/usage-cap error fails over to another account
         // for this kind (cool the current one, recreate the session on the
         // next, re-deliver the in-flight prompt). No-op with <2 accounts.
@@ -5508,14 +5665,16 @@ export class Orchestrator {
         // ledger can roll up observed spend per credential (host 0.9.167+).
         // Only a cloud-leased account identity is reportable; runner-default
         // and Managed Engine identities stay unnamed (same rule as account_usage).
-        const leasedIdentity = channel.leasedIdentityByAgent.get(agentId);
+        const leasedIdentity = this.sessionIdentities.get(session) ?? channel.leasedIdentityByAgent.get(agentId);
+        const connection = this.connectionSessions.get(session);
+        const reportedUsage = connection && event.usage ? Object.fromEntries(Object.entries(event.usage).filter(([k]) => k !== "costUsd")) : event.usage;
         const usage =
-          event.usage === undefined
+          reportedUsage === undefined
             ? undefined
             : {
-                ...event.usage,
+                ...reportedUsage,
                 ...(leasedIdentity?.subject === "account"
-                  ? { credentialId: leasedIdentity.credentialId }
+                  ? { credentialId: leasedIdentity.credentialId, credentialRevision: leasedIdentity.revision, connection: leasedIdentity.connection === true }
                   : {}),
               };
         this.emitHost({
@@ -5702,6 +5861,11 @@ export class Orchestrator {
   ): Promise<void> {
     if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
 
+    const pinned = channel.roster.find(a => a.id === agentId);
+    if (pinned?.accountBinding === "required") {
+      await this.recoverConnectionAgent(channel, pinned, "upstream-error", abortBoundaryOnRosterChange);
+      return;
+    }
     const tries = (channel.respawns.get(agentId) ?? 0) + 1;
     channel.respawns.set(agentId, tries);
     channel.respawnLastAt.set(agentId, Date.now());
@@ -5911,6 +6075,7 @@ export class Orchestrator {
     // A concurrent terminal event for the same session is already being
     // handled by the path that owns this spawn slot.
     if (channel.spawning.has(agentId)) return true;
+    if (agent.accountBinding === "required") return false;
     if (this.engineAccounts(channel, agent.kind).length < 2) {
       return false;
     }
@@ -6133,10 +6298,11 @@ export class Orchestrator {
     channel: Channel,
     agentId: string,
     agent: RosterAgent,
-    account: ResolvedEngineAccount,
+    account: ResolvedEngineAccount | null,
     note: string,
     abortBoundaryOnRosterChange = false,
   ): Promise<boolean> {
+    if (!account && agent.accountBinding !== "required") return false;
     if (channel.rosterSessionChanges.has(agentId)) {
       this.reconcileErroredRosterChange(
         channel,
@@ -6225,8 +6391,10 @@ export class Orchestrator {
         cliSecret,
         this.isSecretary(channel, agent.id) ? channel.leadRunId : null,
       );
-      channel.accountByAgent.set(agentId, account.id);
-      noteEngineAccountUsed(account.id);
+      if (account) {
+        channel.accountByAgent.set(agentId, account.id);
+        noteEngineAccountUsed(account.id);
+      }
 
       replacement = await this.createAndBindStableSession(
         channel,
@@ -6243,7 +6411,7 @@ export class Orchestrator {
           ),
           agentEnv: {
             ...this.machineProjectEnv(task),
-            ...account.execEnv,
+            ...(account?.execEnv ?? {}),
             ...base,
           },
         },
@@ -6325,6 +6493,113 @@ export class Orchestrator {
     return true;
   }
 
+  private cancelConnectionRetries(channel: Channel, agentId?: string): void {
+    const pending = this.connectionRetries.get(channel);
+    if (!pending) return;
+    for (const [id, entry] of pending) {
+      if (agentId !== undefined && id !== agentId) continue;
+      if (entry.timer) clearTimeout(entry.timer);
+      pending.delete(id);
+      channel.spawning.delete(id); // This timer owns the reservation.
+    }
+  }
+
+  private cancelStaleConnectionRetries(channel: Channel): void {
+    const pending = this.connectionRetries.get(channel);
+    if (!pending) return;
+    for (const [id, entry] of pending) {
+      if (this.hasDesiredLaunchSettings(channel, id, entry.agent) && this.leasedMaterialServed(channel, entry.identity)) continue;
+      this.cancelConnectionRetries(channel, id);
+      if (channel.roster.some(agent => agent.id === id)) channel.leasedRespawn.add(id);
+    }
+  }
+
+  private async deferConnectionRecovery(channel: Channel, agent: RosterAgent, dueAt: number): Promise<void> {
+    const id = agent.id;
+    const identity = channel.leasedIdentityByAgent.get(id);
+    if (!identity) { this.leasedUnavailable(channel, id, "unsupported"); return; }
+    channel.spawning.add(id);
+    let scheduled = false;
+    try {
+      const old = channel.sessions.get(id);
+      channel.sessions.delete(id);
+      if (old) this.endLeasedLaunches(channel, id, old);
+      this.retainRetiredTurnBoundary(channel, id, this.needsAbortedTurnBoundary(channel, id));
+      this.retireRunnerGeneration(channel, id);
+      this.emitRetiredTurnAbort(channel, id);
+      this.emitSystemNote(channel.taskId, "rate-limited");
+      if (old) await old.close().catch(() => {});
+      if (!this.isActiveChannel(channel) || !this.hasDesiredLaunchSettings(channel, id, agent) || !this.leasedMaterialServed(channel, identity)) return;
+      let pending = this.connectionRetries.get(channel);
+      if (!pending) { pending = new Map(); this.connectionRetries.set(channel, pending); }
+      const entry = { timer: null as ReturnType<typeof setTimeout> | null, agent, identity };
+      pending.set(id, entry);
+      const arm = (): void => {
+        if (pending.get(id) !== entry) return;
+        const remaining = dueAt - Date.now();
+        if (remaining > 0) {
+          entry.timer = setTimeout(arm, Math.min(remaining, 2_147_483_647));
+          entry.timer.unref?.();
+          return;
+        }
+        this.cancelConnectionRetries(channel, id);
+        if (!this.isActiveChannel(channel) || !this.hasDesiredLaunchSettings(channel, id, agent) || !this.leasedMaterialServed(channel, identity)) return;
+        this.trackEventOperation(channel, this.respawnAgentOnAccount(channel, id, agent, null, "rate-limited")
+          .then(() => {})
+          .catch(error => {
+            if (this.isActiveChannel(channel) && this.hasDesiredLaunchSettings(channel, id, agent) &&
+                this.leasedMaterialServed(channel, identity) && !channel.sessions.has(id)) {
+              this.leasedUnavailable(channel, id, connectionFailureReason(error));
+            }
+          }));
+      };
+      scheduled = true;
+      arm();
+    } finally {
+      if (!scheduled) {
+        channel.spawning.delete(id);
+        if (this.isActiveChannel(channel)) this.retryLeasedStart(channel, id);
+      }
+    }
+  }
+
+  /** Retry a transport/runtime failure on exactly the same connection. Auth
+   * failures stop; no native refresh, rotation or runner login is consulted. */
+  private async recoverConnectionAgent(
+    channel: Channel, agent: RosterAgent, reason: CustomConnectionReason,
+    abortBoundaryOnRosterChange?: boolean, retryAt?: number,
+  ): Promise<void> {
+    const agentId = agent.id;
+    if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
+    if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return;
+    if (agent.kind !== "claude" && agent.kind !== "codex") reason = "unsupported";
+    const terminal = ["unauthorized", "forbidden", "expired", "unsupported", "private-address", "model-not-found", "wrong-protocol"].includes(reason);
+    const tries = (channel.respawns.get(agentId) ?? 0) + 1;
+    channel.respawns.set(agentId, tries);
+    channel.respawnLastAt.set(agentId, Date.now());
+    const providerRetryTooLate = reason === "rate-limited" &&
+      typeof retryAt === "number" && Number.isSafeInteger(retryAt) && retryAt - Date.now() > 60 * 60_000;
+    if (terminal || providerRetryTooLate || tries > MAX_RESPAWNS_PER_AGENT) {
+      const old = channel.sessions.get(agentId);
+      if (old) this.endLeasedLaunches(channel, agentId, old);
+      this.retainRetiredTurnBoundary(channel, agentId, this.needsAbortedTurnBoundary(channel, agentId));
+      this.retireRunnerGeneration(channel, agentId);
+      channel.sessions.delete(agentId);
+      this.emitRetiredTurnAbort(channel, agentId);
+      this.leasedUnavailable(channel, agentId, reason);
+      if (old) await old.close().catch(() => {});
+      return;
+    }
+    if (reason === "rate-limited") {
+      const delay = Math.max(Math.min(60_000, 3000 * 2 ** (tries - 1)),
+        typeof retryAt === "number" && Number.isSafeInteger(retryAt) ? retryAt - Date.now() : 0);
+      await this.deferConnectionRecovery(channel, agent, Date.now() + delay);
+      return;
+    }
+    await this.respawnAgentOnAccount(channel, agentId, agent, null, reason,
+      abortBoundaryOnRosterChange ?? this.needsAbortedTurnBoundary(channel, agentId));
+  }
+
   /**
    * Refresh ONE agent's credential by re-spawning it on its current
    * (re-resolved) account — env accounts read process.env at spawn, so a
@@ -6335,8 +6610,13 @@ export class Orchestrator {
     channel: Channel,
     agent: RosterAgent,
     abortBoundaryOnRosterChange?: boolean,
+    reason: CustomConnectionReason = "upstream-error",
   ): Promise<void> {
     const agentId = agent.id;
+    if (agent.accountBinding === "required") {
+      await this.recoverConnectionAgent(channel, agent, reason, abortBoundaryOnRosterChange);
+      return;
+    }
     if (!this.isActiveChannel(channel) || channel.spawning.has(agentId)) return;
     if (!this.hasDesiredLaunchSettings(channel, agentId, agent)) return;
     const abortBoundary =
@@ -6481,6 +6761,7 @@ export class Orchestrator {
     // start/reconcile factory continuations all check this identity, so none
     // can repopulate an orphan channel while its sessions close.
     ch.closed = true;
+    this.cancelConnectionRetries(ch);
     this.channels.delete(taskId);
     // ADR-154 §6: nothing its agents' spawns were admitted for starts after:
     // the close ends only once every retirement of the task under way, its
@@ -6798,6 +7079,7 @@ function rosterLaunchChangeReason(
   if (previous.kind !== next.kind) changed.push("engine");
   if ((previous.model ?? "") !== (next.model ?? "")) changed.push("model");
   if ((previous.effort ?? "") !== (next.effort ?? "")) changed.push("effort");
+  if ((previous.accountBinding ?? "preferred") !== (next.accountBinding ?? "preferred")) changed.push("binding");
   if ((previous.accountId ?? "") !== (next.accountId ?? "")) {
     changed.push("account");
   }