@runuai/host 0.9.149 → 0.9.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
  2. package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
  3. package/db/migrations/meta/_journal.json +14 -0
  4. package/db/schema.ts +188 -0
  5. package/lib/agent-cli.ts +11 -13
  6. package/lib/agent.ts +177 -7
  7. package/lib/agents/durable-proc.ts +56 -44
  8. package/lib/agents/registry.ts +19 -3
  9. package/lib/agents/session-state.ts +23 -10
  10. package/lib/agents/transport.ts +101 -69
  11. package/lib/apple-one-off.ts +277 -0
  12. package/lib/apple-runtime-recycle.ts +4 -3
  13. package/lib/attachments.ts +19 -17
  14. package/lib/capture-flow.ts +319 -0
  15. package/lib/cloud-credential-cleanup-adapter.ts +123 -0
  16. package/lib/cloud-credential-host.ts +512 -0
  17. package/lib/cloud-credential-leases.ts +2544 -0
  18. package/lib/cloud-credential-runtime.ts +1035 -0
  19. package/lib/codex-auth.ts +21 -1
  20. package/lib/container-runtime.ts +76 -0
  21. package/lib/credential-cleanup-inventory.ts +405 -0
  22. package/lib/credential-consumers/engine-anchor.ts +1876 -0
  23. package/lib/credential-consumers/engines.ts +773 -0
  24. package/lib/credential-consumers/github.ts +594 -0
  25. package/lib/credential-consumers/mcp.ts +384 -0
  26. package/lib/credential-consumers/task-files.ts +127 -0
  27. package/lib/credential-store-pickup.ts +15 -2
  28. package/lib/credential-store.ts +361 -33
  29. package/lib/credentials-activation-authority.ts +546 -0
  30. package/lib/credentials-purge.ts +1006 -0
  31. package/lib/db.ts +11 -0
  32. package/lib/docker-exec.ts +3 -1
  33. package/lib/engine-accounts.ts +42 -32
  34. package/lib/engine-connect.ts +47 -2
  35. package/lib/engine-credential-env.ts +50 -0
  36. package/lib/engine-login.ts +83 -1
  37. package/lib/env-file.ts +62 -8
  38. package/lib/env.ts +5 -0
  39. package/lib/fs-aliases.ts +49 -0
  40. package/lib/git-clone-credentials.ts +334 -0
  41. package/lib/git-diff.ts +219 -177
  42. package/lib/git-identity.ts +59 -14
  43. package/lib/github-origin.ts +28 -0
  44. package/lib/github-tokens.ts +56 -21
  45. package/lib/host-env.ts +17 -4
  46. package/lib/launch-mode.ts +25 -0
  47. package/lib/legacy-runtime-retirement.ts +388 -0
  48. package/lib/machine-exec.ts +14 -3
  49. package/lib/machine-provider-aws.ts +50 -8
  50. package/lib/machine-provider-local.ts +4 -0
  51. package/lib/machine-provider.ts +4 -0
  52. package/lib/managed-operation-drain.ts +8 -0
  53. package/lib/managed-runtime.ts +371 -7
  54. package/lib/mcp-capture.ts +860 -0
  55. package/lib/mcp-connections.ts +8 -1
  56. package/lib/mcp-gateway.ts +168 -10
  57. package/lib/obs.ts +2 -6
  58. package/lib/orchestrator.ts +1847 -236
  59. package/lib/person-capture.ts +327 -0
  60. package/lib/project-env-legacy-tags.ts +130 -0
  61. package/lib/project-env-task-replica.ts +1752 -0
  62. package/lib/runner-boot.ts +10 -0
  63. package/lib/runtime-activation-tail.ts +50 -3
  64. package/lib/runtime-archive.ts +68 -1
  65. package/lib/runtime-reconnect.ts +86 -0
  66. package/lib/runtime-state.ts +429 -13
  67. package/lib/safe-http.ts +364 -0
  68. package/lib/secrets-proxy/task-trust.ts +34 -3
  69. package/lib/sidecar-resume.ts +18 -5
  70. package/lib/skills.ts +47 -8
  71. package/lib/standard-image.ts +157 -103
  72. package/lib/stopped-task-status-queue.ts +68 -10
  73. package/lib/task-credential-cleanup.ts +525 -0
  74. package/lib/task-diff-sandbox.ts +464 -0
  75. package/lib/task-diff.ts +66 -163
  76. package/lib/task-env-source.ts +68 -14
  77. package/lib/task-environment/anchor.ts +31 -0
  78. package/lib/task-environment/apple-container.ts +69 -28
  79. package/lib/task-environment/docker.ts +101 -20
  80. package/lib/task-environment/machine-environment.ts +61 -0
  81. package/lib/task-environment/machine-task-up.ts +459 -96
  82. package/lib/task-environment/machine.ts +101 -11
  83. package/lib/task-environment/self.ts +108 -50
  84. package/lib/task-environment/types.ts +89 -5
  85. package/lib/task-environment/workspace-files.ts +623 -25
  86. package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
  87. package/lib/task-gateway-recovery.ts +4 -1
  88. package/lib/task-gateway-state.ts +15 -0
  89. package/lib/task-machine-swap-safety.ts +729 -0
  90. package/lib/token-broker-adopt.ts +344 -0
  91. package/lib/token-broker-lambda.ts +56 -2
  92. package/lib/token-broker-local.ts +28 -4
  93. package/lib/token-broker-revision.ts +310 -0
  94. package/lib/token-broker-ssm.ts +368 -28
  95. package/lib/token-lanes/openai-oauth.ts +7 -3
  96. package/lib/token-lanes/xai-oauth.ts +7 -3
  97. package/lib/transcript.ts +9 -5
  98. package/package.json +6 -2
  99. package/scripts/agent/task-up.sh +242 -100
  100. package/src/cloud-credentials.ts +2399 -0
  101. package/src/event-outbox.ts +8 -6
  102. package/src/index.ts +1386 -875
  103. package/src/main.ts +1062 -140
  104. package/src/outbound-frame.ts +31 -0
  105. package/src/project-env-policy.ts +125 -10
  106. package/src/protocol.ts +699 -23
  107. package/src/ui/server.ts +48 -10
@@ -51,6 +51,7 @@ import { isSafeHostTaskId } from "./task-identity";
51
51
  import { projectEnvForTask } from "./host-env";
52
52
  import { shadowRoute } from "./routing/shadow";
53
53
  import {
54
+ closeCurrentTaskRunAssignmentForStoppedRuntime,
54
55
  getHostTask,
55
56
  hostTaskEnvMode,
56
57
  hostTaskProjectIds,
@@ -79,6 +80,7 @@ import {
79
80
  } from "./container-runtime";
80
81
  import {
81
82
  reconstructPersistedTaskEnvironment,
83
+ reconstructTaskEnvironment,
82
84
  setAppleTaskEnvironmentRecoveryDriver,
83
85
  setDockerTaskEnvironmentRecoveryDriver,
84
86
  } from "./task-environment";
@@ -137,6 +139,14 @@ import {
137
139
  type BrowserSetup,
138
140
  } from "./browser-testing";
139
141
  import { injectCodexIntoContainer } from "./codex-auth";
142
+ import { runtimeNamespaceOf, type RuntimeNamespace } from "./task-environment/workspace-files";
143
+ import {
144
+ holdOwnCredentials,
145
+ LeftOwnCredentialsError,
146
+ onOwnCredentials,
147
+ onOwnCredentialsEnvironment,
148
+ whileOnOwnCredentials,
149
+ } from "./cloud-credential-host";
140
150
  import { isCredentialInvalidSignal, requestCredentialRefresh } from "./token-broker-local";
141
151
  import {
142
152
  codexHomeIsolatable,
@@ -147,8 +157,51 @@ import {
147
157
  pickEngineAccount,
148
158
  provisionEngineAccounts,
149
159
  resolveEngineAccounts,
160
+ selectAccount,
150
161
  type ResolvedEngineAccount,
151
162
  } from "./engine-accounts";
163
+ import {
164
+ ENGINE_ACCOUNT_HOMES,
165
+ EngineCredentialWriteRefused,
166
+ engineSpawnEnv,
167
+ engineVersionHeldBack,
168
+ ensureEngineHome,
169
+ escalateEngineLaunchFence,
170
+ isLeasedEngineAccount,
171
+ leasedEnginesFor,
172
+ onLeasedEnginesChanged,
173
+ refreshLeasedEngineAccount,
174
+ setEngineAdmissionFloorRaiser,
175
+ syncEngineHomes,
176
+ waitForEngineGate,
177
+ type LeasedEngineAccount,
178
+ type TaskEngineLease,
179
+ } from "./credential-consumers/engines";
180
+ import {
181
+ admitEngineLaunch,
182
+ anchoredEnvironment,
183
+ dischargeEngineRevocations,
184
+ ENGINE_ACCOUNT_ENV,
185
+ ENGINE_RUN_ENV,
186
+ EngineAnchorLost,
187
+ EngineLaunchRefused,
188
+ newEngineLaunchSlotId,
189
+ raiseEngineAdmissionFloors,
190
+ retireEngineLaunchSlot,
191
+ sameRuntime,
192
+ stopAnchoredEngines,
193
+ type EngineAdmissionFloor,
194
+ type EngineLaunchIdentity,
195
+ } from "./credential-consumers/engine-anchor";
196
+ import { CredentialLeaseUnavailableError } from "./cloud-credential-leases";
197
+ import type { TaskCredentialCleanupRequest } from "./task-credential-cleanup";
198
+ import {
199
+ forgetLeasedEngineRuntimes,
200
+ listCredentialCleanupPaths,
201
+ listLeasedEngineRuntimes,
202
+ rememberLeasedEngineRuntime,
203
+ } from "./credential-cleanup-inventory";
204
+ import { ENGINE_CREDENTIAL_FILES, ENGINE_ENV_CREDENTIAL_KEYS } from "../src/cloud-credentials";
152
205
  import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
153
206
  import {
154
207
  reactivateTaskGatewayTask,
@@ -179,7 +232,11 @@ import {
179
232
  MAX_SECRETARY_DISPATCH_ID_CHARS,
180
233
  SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
181
234
  } from "../src/protocol";
182
- import { isEnvironmentTransportProvider } from "./task-environment/types";
235
+ import {
236
+ isEnvironmentTransportProvider,
237
+ parseTaskEnvironmentLocator,
238
+ serializeTaskEnvironmentLocator,
239
+ } from "./task-environment/types";
183
240
  import { forgetTaskEnvSnapshot } from "./task-env-source";
184
241
 
185
242
  export type HostEventSubscriber = (event: HostEvent) => void;
@@ -385,6 +442,41 @@ interface Channel {
385
442
  /** ADR-076: per-agent account-rotation budget (reset on a successful turn) —
386
443
  * bounds a rate-limit storm across accounts, mirroring the respawn budget. */
387
444
  rotations: Map<string, number>;
445
+ /** ADR-154: the task's AI accounts come from its lease. Sticky for the
446
+ * channel: once on cloud credentials, never the runner's own accounts. */
447
+ leased: boolean;
448
+ /** ADR-154 §8: rewrites of the task's account homes after a change, one
449
+ * after another. */
450
+ leasedSync: Promise<void> | null;
451
+ /** ADR-154 §6: agents a cleanup stopped, started again on their next
452
+ * account once the task's gate opens. */
453
+ leasedRespawn: Set<string>;
454
+ /** ADR-154 §6: the exact lease material each agent's session was picked
455
+ * on, so a spawn starts only on what the task still holds and a cleanup
456
+ * of an older run never stops a newer run's agent. */
457
+ leasedIdentityByAgent: Map<string, LeasedIdentity>;
458
+ /** ADR-154 §5: agents down for want of a usable account; the gate's next
459
+ * opening starts them again. */
460
+ leasedBlocked: Set<string>;
461
+ /** ADR-154 §8: a failed rewrite of the account homes, retried. */
462
+ leasedSyncRetry: ReturnType<typeof setTimeout> | null;
463
+ /** ADR-154 §5, §6: agents wait to be started again (stopped by a cleanup,
464
+ * down for want of an account, or their spawn's material moved); kept
465
+ * until a start pass actually ran, so a failed pass does not lose it. */
466
+ leasedWakePending: boolean;
467
+ /** ADR-154 §6: engines being created on a leased account (null: on an
468
+ * agent's task token alone, the Managed Engine) right now, each until its
469
+ * create settles. Every cleanup attempt on the account (on every one, for
470
+ * a whole run's) waits for them before it proves the engines gone,
471
+ * whatever has become of the agent's binding meanwhile. */
472
+ leasedCreating: Set<{
473
+ agentId: string;
474
+ accountId: string | null;
475
+ runEpoch: number;
476
+ settled: Promise<void>;
477
+ }>;
478
+ /** ADR-154 §6: each agent's current spawn of a leased engine. */
479
+ leasedLaunches: Map<string, LeasedLaunchSlot>;
388
480
  }
389
481
 
390
482
  interface AcceptedPrompt {
@@ -405,6 +497,119 @@ interface RecoveryFence {
405
497
  const MAX_RESPAWNS_PER_AGENT = 5;
406
498
  /** ADR-076: hard cap on account rotations per agent between successful turns. */
407
499
  const MAX_ROTATIONS_PER_AGENT = 6;
500
+ /** ADR-154 §5: how long a spawn waits for the task's leased AI accounts
501
+ * before the agent fails with "credentials unavailable". */
502
+ const LEASED_ACCOUNT_WAIT_MS = 60_000;
503
+ /** ADR-154 §6: how long the cleanup's quiesce step waits for a spawn
504
+ * already creating an engine on a removed account. */
505
+ const LEASED_QUIESCE_EXIT_MS = 20_000;
506
+ /** ADR-141: the shadow router reads the tail of the task's chat (it keeps
507
+ * 1,200 characters) and its classifier's answer, never more. */
508
+ const SHADOW_CONTEXT_BYTES = 8 * 1024;
509
+ const SHADOW_OUTPUT_BYTES = 64 * 1024;
510
+ /** ADR-154 §6: the markers every process of a leased engine inherits: its
511
+ * run and its account. A launch carrying them is an engine launch, started
512
+ * only under the engine supervisor. */
513
+ const LEASED_RUN_ENV = ENGINE_RUN_ENV;
514
+ const LEASED_ACCOUNT_ENV = ENGINE_ACCOUNT_ENV;
515
+ /**
516
+ * ADR-154 §6: every variable an env-token account may set, blank. A leased
517
+ * spawn carries these under its account's own values: an explicit blank
518
+ * wins over an adapter's inheritance of the runner's variable (Claude's
519
+ * passEnv, Grok's XAI_API_KEY, Cursor's fallback) and over the container's
520
+ * env, so no engine ever runs on the runner's own credential beside, or
521
+ * instead of, the task's.
522
+ */
523
+ const LEASED_CREDENTIAL_BLANKS: Readonly<Record<string, string>> = Object.freeze(
524
+ Object.fromEntries(
525
+ [...new Set(Object.values(ENGINE_ENV_CREDENTIAL_KEYS).flat())].map((name) => [name, ""]),
526
+ ),
527
+ );
528
+ /** ADR-128: the Managed Engine. It needs no AI account (it runs on Uai's
529
+ * gateway), but its agent's task token is its bearer: on a leased task its
530
+ * launches are admitted as that token (ADR-154 §6). */
531
+ const MANAGED_ENGINE_KIND = "uai";
532
+ /** ADR-154 §10: kinds whose agents carry no credential at all; every other
533
+ * kind outside the leased lanes (Kimi) stays down on a leased task until
534
+ * its lane exists. */
535
+ const CREDENTIAL_FREE_KINDS: ReadonlySet<string> = new Set(["mock"]);
536
+ /** ADR-154 §8: how soon a failed rewrite of a task's account homes is tried
537
+ * again. */
538
+ const LEASED_SYNC_RETRY_MS = 30_000;
539
+
540
+ /** ADR-154 §6: the exact lease material a spawn starts on: an AI account's,
541
+ * or, for the Managed Engine, the task's open run alone (its agent's task
542
+ * token is its credential). */
543
+ type LeasedIdentity =
544
+ | { subject: "account"; runEpoch: number; credentialId: string; revision: string; version: number }
545
+ | { subject: "task-token"; runEpoch: number };
546
+
547
+ type SpawnArgs = Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">;
548
+
549
+ /**
550
+ * ADR-154 §6: one spawn of an agent's leased engine. Its launches are
551
+ * admitted only while it is the agent's current spawn and, once its create
552
+ * settled, only while the session it bound is the agent's live one: a spawn
553
+ * replaced (on the same material too), retired, removed, never bound, or
554
+ * whose session reported its end, launches nothing more, and once retired,
555
+ * nothing it was admitted for starts in its runtime (`retireLeasedLaunch`).
556
+ */
557
+ interface LeasedLaunchSlot {
558
+ created: boolean;
559
+ session: AgentSession | null;
560
+ /** What its launches are admitted as, its slot's id included, and where. */
561
+ readonly identity: EngineLaunchIdentity & { slotId: string };
562
+ readonly environment: TaskEnvironmentHandle<unknown>;
563
+ /** It began writing an admission: its retirement reaches its runtime. */
564
+ admitted: boolean;
565
+ retired: boolean;
566
+ }
567
+
568
+ /** ADR-154 §6: a retired slot not yet retired in its runtime. */
569
+ interface RetiredLaunchSlot {
570
+ readonly agentId: string;
571
+ readonly slot: LeasedLaunchSlot;
572
+ /** The attempt under way, if one is. */
573
+ attempt: Promise<void> | null;
574
+ /** A cleanup of its run that covers it proves what it fences: a failure
575
+ * after is not escalated. */
576
+ covered: boolean;
577
+ /** Its run's full cleanup was asked for once an attempt failed. */
578
+ escalated: boolean;
579
+ }
580
+
581
+ function newLeasedLaunchSlot(
582
+ taskId: string,
583
+ identity: LeasedIdentity,
584
+ environment: TaskEnvironmentHandle<unknown>,
585
+ ): LeasedLaunchSlot {
586
+ return {
587
+ created: false,
588
+ session: null,
589
+ identity: { taskId, slotId: newEngineLaunchSlotId(), ...identity },
590
+ environment,
591
+ admitted: false,
592
+ retired: false,
593
+ };
594
+ }
595
+
596
+ function sameLeasedIdentity(a: LeasedIdentity, b: LeasedIdentity): boolean {
597
+ if (a.subject === "account" && b.subject === "account") {
598
+ return (
599
+ a.runEpoch === b.runEpoch &&
600
+ a.credentialId === b.credentialId &&
601
+ a.revision === b.revision &&
602
+ a.version === b.version
603
+ );
604
+ }
605
+ return a.subject === b.subject && a.runEpoch === b.runEpoch;
606
+ }
607
+
608
+ /** The engine kinds whose agents run on an AI account from the lease. */
609
+ const LEASED_ENGINE_KINDS: ReadonlySet<string> = new Set([
610
+ ...Object.keys(ENGINE_ENV_CREDENTIAL_KEYS),
611
+ ...Object.keys(ENGINE_CREDENTIAL_FILES),
612
+ ]);
408
613
  /** A burned respawn budget resets after this quiet period (see reconcile). */
409
614
  const RESPAWN_COOLDOWN_MS = 10 * 60_000;
410
615
  /** ADR-053: gap between browser-install retries. Ensures fire every couple of
@@ -504,6 +709,20 @@ async function sshIdentityDepsFor(task: {
504
709
  return { environment: () => environment };
505
710
  }
506
711
 
712
+ export type TaskStopAssignmentAdmission =
713
+ | { kind: "legacy" }
714
+ | { kind: "closed"; runEpoch: number }
715
+ | { kind: "stale" };
716
+
717
+ export interface OrchestratorTaskStopResult {
718
+ ok: boolean;
719
+ error?: string;
720
+ assignmentStale?: boolean;
721
+ vetoed?: boolean;
722
+ /** Exact v1 run stopped under the lifecycle lock. Absent for legacy. */
723
+ stoppedRunEpoch?: number;
724
+ }
725
+
507
726
  export class Orchestrator {
508
727
  private readonly channels = new Map<string, Channel>();
509
728
  private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
@@ -517,8 +736,15 @@ export class Orchestrator {
517
736
  * later command cannot overtake an earlier one after channel close. */
518
737
  private readonly taskLifecycleTails = new Map<string, Promise<void>>();
519
738
  private readonly hostSubscribers = new Set<HostEventSubscriber>();
739
+ /** ADR-154 §6: launch slots retired whose retirement has not yet reached
740
+ * their runtime, per task, whatever channel they were in: an agent's next
741
+ * spawn launches there only once they have (`retireLeasedLaunch`). */
742
+ private readonly leasedRetired = new Map<string, Set<RetiredLaunchSlot>>();
520
743
 
521
- constructor(private readonly factory: AgentSessionFactory) {}
744
+ constructor(private readonly factory: AgentSessionFactory) {
745
+ onLeasedEnginesChanged((taskId) => this.leasedEnginesChanged(taskId));
746
+ setEngineAdmissionFloorRaiser((assignment, floors) => this.raiseLeasedAdmissionFloors(assignment, floors));
747
+ }
522
748
 
523
749
  // -- subscriptions --------------------------------------------------------
524
750
 
@@ -645,7 +871,7 @@ export class Orchestrator {
645
871
  for (const agent of added) {
646
872
  // Per-agent link/document skills, like getOrCreateChannel does at
647
873
  // channel birth. Best-effort.
648
- writeAgentSkills(channel.taskId, agent);
874
+ writeAgentSkills(channel.taskId, agent, () => this.workspaceRuntime(channel.taskId));
649
875
  }
650
876
  for (const { agent, reason } of changed) {
651
877
  const recoveryAbortPublished = this.supersedeRecovery(
@@ -665,6 +891,9 @@ export class Orchestrator {
665
891
  );
666
892
  }
667
893
  channel.accountByAgent.delete(agent.id);
894
+ // ADR-154 §6: the retired spawn launches nothing more on its material.
895
+ channel.leasedIdentityByAgent.delete(agent.id);
896
+ void this.retireLeasedLaunch(channel, agent.id, channel.leasedLaunches.get(agent.id));
668
897
  channel.rotations.delete(agent.id);
669
898
  // Crash/recovery budgets belong to the retired launch generation. A
670
899
  // newly selected engine/model/effort must get an immediate clean start,
@@ -814,6 +1043,9 @@ export class Orchestrator {
814
1043
  channel.respawnLastAt.delete(agentId);
815
1044
  channel.everBound.delete(agentId);
816
1045
  channel.accountByAgent.delete(agentId);
1046
+ channel.leasedIdentityByAgent.delete(agentId);
1047
+ // ADR-154 §6: its spawn admits nothing from here (see below for the end).
1048
+ void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
817
1049
  channel.lastPrompt.delete(agentId);
818
1050
  channel.rotations.delete(agentId);
819
1051
 
@@ -831,6 +1063,10 @@ export class Orchestrator {
831
1063
  // factory await and closes the stale process instead of binding it.
832
1064
  await inFlightCreate.catch(() => null);
833
1065
  }
1066
+ // The removal ends only once its spawns are retired in their runtime, or
1067
+ // their run's cleanup is asked for (retireInRuntime): the current one, and
1068
+ // any whose retirement an earlier event, or this removal's drain, began.
1069
+ await Promise.all(this.leasedRetirementsUnderWay(channel.taskId, agentId));
834
1070
  if (!this.isActiveChannel(channel)) return;
835
1071
  // A newer ensure can re-add the same id while the old close/create drains.
836
1072
  // That spec owns the status now; do not stop its bound generation or emit
@@ -972,9 +1208,6 @@ export class Orchestrator {
972
1208
  taskProxiedKeys(taskId),
973
1209
  ),
974
1210
  );
975
- // ADR-046: materialise this agent's skills to its per-agent SKILL.md in
976
- // the workspace (bind-mounted into the container). Best-effort.
977
- writeAgentSkills(taskId, agent);
978
1211
  // ADR-022: any agent with a non-empty initialPrompt opens a first
979
1212
  // turn at container-ready. The rest stay silent until addressed.
980
1213
  const firstTurn = assembleFirstTurnPrompt(
@@ -1048,8 +1281,24 @@ export class Orchestrator {
1048
1281
  accountByAgent: new Map(),
1049
1282
  lastPrompt: new Map(),
1050
1283
  rotations: new Map(),
1284
+ leased: false,
1285
+ leasedSync: null,
1286
+ leasedRespawn: new Set(),
1287
+ leasedIdentityByAgent: new Map(),
1288
+ leasedBlocked: new Set(),
1289
+ leasedSyncRetry: null,
1290
+ leasedWakePending: false,
1291
+ leasedCreating: new Set(),
1292
+ leasedLaunches: new Map(),
1051
1293
  };
1052
1294
  this.channels.set(taskId, channel);
1295
+ // ADR-046: materialise each agent's skills to its per-agent SKILL.md in
1296
+ // the workspace (bind-mounted into the container), best-effort, once the
1297
+ // channel can give the task's runtime: where only it can make the
1298
+ // agent's directory (macOS, #cb3j0z), it is asked for then.
1299
+ for (const agent of roster) {
1300
+ writeAgentSkills(taskId, agent, () => this.workspaceRuntime(taskId));
1301
+ }
1053
1302
  return channel;
1054
1303
  }
1055
1304
 
@@ -1255,22 +1504,90 @@ export class Orchestrator {
1255
1504
  * copies use this gate: a false→true toggle queues migration after the older
1256
1505
  * copy, so that copy cannot land last and erase the managed browser block.
1257
1506
  */
1258
- private codexHomes(hasCodex: boolean, scope: string): string[] {
1507
+ private codexHomes(hasCodex: boolean, channel: Channel): string[] {
1259
1508
  if (!hasCodex) return [];
1260
- const homes = resolveEngineAccounts("codex", {}, scope)
1261
- .map((account) => account.configDir?.containerDir)
1262
- .filter((home): home is string => Boolean(home));
1263
- return [...new Set([DEFAULT_CODEX_HOME, ...homes])];
1509
+ return [...new Set([DEFAULT_CODEX_HOME, ...this.accountHomes(channel, "codex")])];
1264
1510
  }
1265
1511
 
1266
- /** Every GROK_HOME a task in `scope` may run under: the default plus each
1267
- * sign-in account's isolated home (API-key accounts use the default). */
1268
- private grokHomes(hasGrok: boolean, scope: string): string[] {
1512
+ /** Every GROK_HOME a task may run under: the default plus each sign-in
1513
+ * account's isolated home (API-key accounts use the default). */
1514
+ private grokHomes(hasGrok: boolean, channel: Channel): string[] {
1269
1515
  if (!hasGrok) return [];
1270
- const homes = resolveEngineAccounts("grok", {}, scope)
1271
- .map((account) => account.configDir?.containerDir)
1272
- .filter((home): home is string => Boolean(home));
1273
- return [...new Set([DEFAULT_GROK_HOME, ...homes])];
1516
+ return [...new Set([DEFAULT_GROK_HOME, ...this.accountHomes(channel, "grok")])];
1517
+ }
1518
+
1519
+ /** The homes `kind`'s config-directory accounts live in: the task's leased
1520
+ * ones, or the runner's own for a task not on cloud credentials. A leased
1521
+ * Codex account's home is listed with every Codex agent's own home on it,
1522
+ * so the task's managed config (browser, MCP) reaches the homes the
1523
+ * agents run in, now and whenever it changes. */
1524
+ private accountHomes(channel: Channel, kind: string): string[] {
1525
+ if (!this.isLeasedChannel(channel)) {
1526
+ return resolveEngineAccounts(kind, {}, channel.engineAccountScope)
1527
+ .map((account) => account.configDir?.containerDir)
1528
+ .filter((home): home is string => Boolean(home));
1529
+ }
1530
+ const homes = (leasedEnginesFor(channel.taskId)?.accounts ?? [])
1531
+ .filter((account) => account.engineKind === kind && account.home !== null)
1532
+ .map((account) => account.home!);
1533
+ if (kind !== "codex") return homes;
1534
+ const agents = channel.roster.filter((agent) => agent.kind === "codex");
1535
+ return homes.flatMap((home) => [home, ...agents.map((agent) => codexPerAgentHome(home, agent.id))]);
1536
+ }
1537
+
1538
+ /** ADR-154: whether this channel's AI accounts come from the task's lease
1539
+ * (the engine consumer holds a record for the task, or the runner is no
1540
+ * longer on its own credentials at all: no default slot, so an agent
1541
+ * reattached before its grant waits for it). Sticky: a channel that was
1542
+ * on cloud credentials never falls back to the runner's own accounts,
1543
+ * not even after the record is gone. */
1544
+ private isLeasedChannel(channel: Channel): boolean {
1545
+ if (!channel.leased && (leasedEnginesFor(channel.taskId) !== undefined || !onOwnCredentials())) {
1546
+ channel.leased = true;
1547
+ }
1548
+ return channel.leased;
1549
+ }
1550
+
1551
+ /**
1552
+ * The accounts `kind` may run on, in try order: the task's leased ones
1553
+ * that are readable now (none while its gate is closed), or the runner's
1554
+ * own for a task not on cloud credentials. A leased account carries no env
1555
+ * and no directory here: the spawn boundary applies it (leasedSpawnArgs),
1556
+ * and no runner-side copy ever names it.
1557
+ */
1558
+ private engineAccounts(channel: Channel, kind: string): ResolvedEngineAccount[] {
1559
+ if (!this.isLeasedChannel(channel)) {
1560
+ return resolveEngineAccounts(kind, {}, channel.engineAccountScope);
1561
+ }
1562
+ const lease = leasedEnginesFor(channel.taskId);
1563
+ if (!lease?.open) return [];
1564
+ return lease.accounts
1565
+ .filter(
1566
+ (account): account is LeasedEngineAccount =>
1567
+ account.engineKind === kind && isLeasedEngineAccount(account),
1568
+ )
1569
+ .map((account) => ({
1570
+ id: account.credentialId,
1571
+ kind,
1572
+ label: account.label,
1573
+ isDefault: false,
1574
+ scope: account.scope,
1575
+ authKind: account.authKind,
1576
+ execEnv: {},
1577
+ }));
1578
+ }
1579
+
1580
+ /** ADR-076 selection over `engineAccounts`: the first healthy account in
1581
+ * try order, skipping `exclude`. */
1582
+ private pickAccount(
1583
+ channel: Channel,
1584
+ kind: string,
1585
+ exclude?: Set<string>,
1586
+ ): ResolvedEngineAccount | null {
1587
+ if (!this.isLeasedChannel(channel)) {
1588
+ return pickEngineAccount(kind, { scope: channel.engineAccountScope, exclude });
1589
+ }
1590
+ return selectAccount(this.engineAccounts(channel, kind), { exclude });
1274
1591
  }
1275
1592
 
1276
1593
  private noteProvisionMutation(
@@ -1299,9 +1616,12 @@ export class Orchestrator {
1299
1616
  if (this.isActiveChannel(channel)) channel.configGeneration += 1;
1300
1617
  }
1301
1618
 
1302
- private provisionFingerprint(kinds: readonly string[], scope: string): string {
1619
+ private provisionFingerprint(kinds: readonly string[], channel: Channel): string {
1620
+ // ADR-154: a leased task's accounts are never copied from the runner;
1621
+ // the spawn boundary writes each one's files through the engine consumer.
1622
+ if (this.isLeasedChannel(channel)) return "leased";
1303
1623
  const accounts = kinds.flatMap((kind) =>
1304
- resolveEngineAccounts(kind, {}, scope)
1624
+ resolveEngineAccounts(kind, {}, channel.engineAccountScope)
1305
1625
  .filter((account) => !account.isDefault && account.configDir)
1306
1626
  .map((account) => ({
1307
1627
  kind,
@@ -1374,13 +1694,13 @@ export class Orchestrator {
1374
1694
  const requestedHasCodex =
1375
1695
  hasCodex || channel.roster.some((agent) => agent.kind === "codex");
1376
1696
  const requestedHomes = requestedBrowser
1377
- ? this.codexHomes(requestedHasCodex, channel.engineAccountScope)
1697
+ ? this.codexHomes(requestedHasCodex, channel)
1378
1698
  : [];
1379
1699
  const requestedKinds = [
1380
1700
  ...new Set(channel.roster.map((agent) => agent.kind)),
1381
1701
  ];
1382
1702
  const requestedProvisionFingerprint =
1383
- this.provisionFingerprint(requestedKinds, channel.engineAccountScope);
1703
+ this.provisionFingerprint(requestedKinds, channel);
1384
1704
  const changedProvisionKinds = this.changedProvisionKinds(
1385
1705
  channel.provisionAttemptedFingerprint,
1386
1706
  requestedProvisionFingerprint,
@@ -1389,11 +1709,14 @@ export class Orchestrator {
1389
1709
  const provisionKinds = forceProvision
1390
1710
  ? requestedKinds
1391
1711
  : changedProvisionKinds;
1392
- const mutableProvisionKinds = provisionKinds.filter((kind) =>
1393
- resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
1394
- (account) => !account.isDefault && Boolean(account.configDir),
1395
- ),
1396
- );
1712
+ const leased = this.isLeasedChannel(channel);
1713
+ const mutableProvisionKinds = leased
1714
+ ? []
1715
+ : provisionKinds.filter((kind) =>
1716
+ resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
1717
+ (account) => !account.isDefault && Boolean(account.configDir),
1718
+ ),
1719
+ );
1397
1720
  const mayProvisionConfig = mutableProvisionKinds.length > 0;
1398
1721
 
1399
1722
  const inFlight = channel.browserSetup;
@@ -1410,13 +1733,13 @@ export class Orchestrator {
1410
1733
  requestedHasCodex ||
1411
1734
  channel.roster.some((agent) => agent.kind === "codex");
1412
1735
  const latestHomes = latestBrowser
1413
- ? this.codexHomes(latestHasCodex, channel.engineAccountScope)
1736
+ ? this.codexHomes(latestHasCodex, channel)
1414
1737
  : [];
1415
1738
  const latestKinds = [
1416
1739
  ...new Set(channel.roster.map((agent) => agent.kind)),
1417
1740
  ];
1418
1741
  const latestProvisionFingerprint =
1419
- this.provisionFingerprint(latestKinds, channel.engineAccountScope);
1742
+ this.provisionFingerprint(latestKinds, channel);
1420
1743
  if (
1421
1744
  (latestBrowser &&
1422
1745
  coverage?.browserAttempted !== true &&
@@ -1452,7 +1775,12 @@ export class Orchestrator {
1452
1775
  forceProvision ||
1453
1776
  channel.provisionAttemptedFingerprint !==
1454
1777
  requestedProvisionFingerprint;
1455
- if (shouldProvision) {
1778
+ if (shouldProvision && leased) {
1779
+ // ADR-154: nothing is copied from the runner for a leased task.
1780
+ coverage.provisionAttempted = true;
1781
+ channel.provisionAttemptedFingerprint = requestedProvisionFingerprint;
1782
+ channel.provisionRetryFingerprint = undefined;
1783
+ } else if (shouldProvision) {
1456
1784
  coverage.provisionAttempted = true;
1457
1785
  if (mayProvisionConfig) {
1458
1786
  // Fence a factory that started before the copy without declaring a
@@ -1462,21 +1790,27 @@ export class Orchestrator {
1462
1790
  }
1463
1791
  let copiedOrUnknown = false;
1464
1792
  try {
1465
- const provisioned = channel.agentHandle
1466
- ? await provisionEngineAccounts(
1467
- channel.containerName,
1468
- provisionKinds,
1469
- {},
1470
- channel.agentHandle,
1471
- channel.engineAccountScope,
1472
- )
1473
- : await provisionEngineAccounts(
1474
- channel.containerName,
1475
- provisionKinds,
1476
- {},
1477
- undefined,
1478
- channel.engineAccountScope,
1479
- );
1793
+ // ADR-154: the runner's own accounts, copied only while it is on
1794
+ // its own credentials, counted (leaving waits for the copy);
1795
+ // once leaving has begun, nothing is copied.
1796
+ const provisioning = whileOnOwnCredentials(() =>
1797
+ channel.agentHandle
1798
+ ? provisionEngineAccounts(
1799
+ channel.containerName,
1800
+ provisionKinds,
1801
+ {},
1802
+ channel.agentHandle,
1803
+ channel.engineAccountScope,
1804
+ )
1805
+ : provisionEngineAccounts(
1806
+ channel.containerName,
1807
+ provisionKinds,
1808
+ {},
1809
+ undefined,
1810
+ channel.engineAccountScope,
1811
+ ),
1812
+ );
1813
+ const provisioned = (await provisioning) ?? { copied: 0, failed: 0, mayHaveMutated: false };
1480
1814
  copiedOrUnknown =
1481
1815
  provisioned.copied > 0 || provisioned.mayHaveMutated;
1482
1816
  if (provisioned.failed > 0) {
@@ -1537,7 +1871,7 @@ export class Orchestrator {
1537
1871
  requestedHomes,
1538
1872
  channel.agentHandle,
1539
1873
  requestedKinds,
1540
- this.grokHomes(requestedKinds.includes("grok"), channel.engineAccountScope),
1874
+ this.grokHomes(requestedKinds.includes("grok"), channel),
1541
1875
  )
1542
1876
  : await setupBrowserTesting(
1543
1877
  channel.taskId,
@@ -1546,7 +1880,7 @@ export class Orchestrator {
1546
1880
  requestedHomes,
1547
1881
  undefined,
1548
1882
  requestedKinds,
1549
- this.grokHomes(requestedKinds.includes("grok"), channel.engineAccountScope),
1883
+ this.grokHomes(requestedKinds.includes("grok"), channel),
1550
1884
  );
1551
1885
  } catch (err) {
1552
1886
  console.warn(
@@ -1617,15 +1951,12 @@ export class Orchestrator {
1617
1951
  requestedHasCodex ||
1618
1952
  channel.roster.some((agent) => agent.kind === "codex");
1619
1953
  const latestHomes = latestBrowser
1620
- ? this.codexHomes(latestHasCodex, channel.engineAccountScope)
1954
+ ? this.codexHomes(latestHasCodex, channel)
1621
1955
  : [];
1622
1956
  const latestKinds = [
1623
1957
  ...new Set(channel.roster.map((agent) => agent.kind)),
1624
1958
  ];
1625
- const latestProvisionFingerprint = this.provisionFingerprint(
1626
- latestKinds,
1627
- channel.engineAccountScope,
1628
- );
1959
+ const latestProvisionFingerprint = this.provisionFingerprint(latestKinds, channel);
1629
1960
  if (
1630
1961
  (latestBrowser &&
1631
1962
  !coverage.browserAttempted &&
@@ -1714,12 +2045,9 @@ export class Orchestrator {
1714
2045
  ...new Set(channel.roster.map((agent) => agent.kind)),
1715
2046
  ].sort();
1716
2047
  const codexHomes = engineKinds.includes("codex")
1717
- ? this.codexHomes(true, channel.engineAccountScope)
2048
+ ? this.codexHomes(true, channel)
1718
2049
  : [];
1719
- const grokHomes = this.grokHomes(
1720
- engineKinds.includes("grok"),
1721
- channel.engineAccountScope,
1722
- );
2050
+ const grokHomes = this.grokHomes(engineKinds.includes("grok"), channel);
1723
2051
  const fingerprint = JSON.stringify({
1724
2052
  connections,
1725
2053
  engineKinds,
@@ -2053,6 +2381,9 @@ export class Orchestrator {
2053
2381
  if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
2054
2382
  return false;
2055
2383
  }
2384
+ // ADR-154 §5: down for want of a usable account until the gate next
2385
+ // opens (which starts it again), not retried on every pass.
2386
+ if (channel.leasedBlocked.has(agent.id)) return false;
2056
2387
  // Crash-loop budget: an agent whose session keeps dying stops being
2057
2388
  // respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
2058
2389
  // but a cold budget resets so the task can heal after the cause clears.
@@ -2162,7 +2493,12 @@ export class Orchestrator {
2162
2493
  ),
2163
2494
  },
2164
2495
  );
2165
- if (!session) return;
2496
+ if (!session) {
2497
+ // Credentials unavailable is this agent's alone: its siblings
2498
+ // still start.
2499
+ if (channel.leasedBlocked.has(agent.id)) continue;
2500
+ return;
2501
+ }
2166
2502
  this.startNextPendingPrompt(channel, agent.id);
2167
2503
  }
2168
2504
  } finally {
@@ -2257,6 +2593,9 @@ export class Orchestrator {
2257
2593
  channel.accountByAgent.delete(agent.id);
2258
2594
  return { ...floor, ...base };
2259
2595
  }
2596
+ // ADR-154: a leased task's account is picked and applied at the spawn
2597
+ // boundary (leasedSpawnArgs), which can wait for the task's gate.
2598
+ if (this.isLeasedChannel(channel)) return { ...floor, ...base };
2260
2599
  const account = pickEngineAccount(agent.kind, {
2261
2600
  scope: channel.engineAccountScope,
2262
2601
  });
@@ -2345,6 +2684,11 @@ export class Orchestrator {
2345
2684
  ) {
2346
2685
  return;
2347
2686
  }
2687
+ // ADR-154 §6: an engine that ended launches nothing more from now,
2688
+ // whatever its terminal handling still awaits (the runtime's proof).
2689
+ if (event.type === "error" || event.type === "exit") {
2690
+ this.endLeasedLaunches(channel, agentId, session);
2691
+ }
2348
2692
  const operation = eventTail
2349
2693
  .catch(() => {})
2350
2694
  .then(async () => {
@@ -2411,6 +2755,835 @@ export class Orchestrator {
2411
2755
  return operation;
2412
2756
  }
2413
2757
 
2758
+ /**
2759
+ * ADR-154 §5, §6, §10: what a spawn on the task's leased AI account takes,
2760
+ * and the exact lease material it starts on. The agent runs on the account
2761
+ * it is bound to while that account is still served readable, else on the
2762
+ * first healthy one in try order (the owner's accounts, then the
2763
+ * workspace's). An env-token account's values ride the spawn, synchronously
2764
+ * when nothing has to be waited for. A config-directory account's home gets
2765
+ * the task's managed config first (browser, MCP), then its files, through
2766
+ * the engine consumer, in the home the agent runs in (Codex's own copy,
2767
+ * seeded from the account's home without its credential files). While the
2768
+ * task's gate is closed it waits, bounded; null when no account can be
2769
+ * used, and the agent stays down with "credentials unavailable", never on
2770
+ * the runner's own accounts, until the gate next opens.
2771
+ */
2772
+ private leasedSpawnArgs(
2773
+ channel: Channel,
2774
+ agentId: string,
2775
+ args: SpawnArgs,
2776
+ environment: TaskEnvironmentHandle<unknown>,
2777
+ ):
2778
+ | { args: SpawnArgs; identity: LeasedIdentity }
2779
+ | null
2780
+ | Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
2781
+ const kind = args.agent?.kind ?? "";
2782
+ const lease = leasedEnginesFor(channel.taskId);
2783
+ if (lease?.open) {
2784
+ const account = this.leasedAccountFor(channel, agentId, kind, lease);
2785
+ if (account?.authKind === "env") {
2786
+ return this.withLeasedAccount(channel, agentId, args, account, lease.assignment.runEpoch);
2787
+ }
2788
+ }
2789
+ return (async () => {
2790
+ const deadline = Date.now() + LEASED_ACCOUNT_WAIT_MS;
2791
+ let reason = "the task's AI accounts are not available";
2792
+ let moved = 0;
2793
+ while (this.isActiveChannel(channel)) {
2794
+ const open = await this.leasedGateOpen(channel, deadline);
2795
+ if (!open) break;
2796
+ if (!this.isActiveChannel(channel)) return null;
2797
+ const account = this.leasedAccountFor(channel, agentId, kind, open);
2798
+ if (!account) {
2799
+ reason = `no ${kind} account this task can use`;
2800
+ break;
2801
+ }
2802
+ if (account.authKind === "env") {
2803
+ return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch);
2804
+ }
2805
+ // The account's home gets the task's managed config before anything
2806
+ // can start in it, whenever the gate's opening first reveals it.
2807
+ await this.ensureBrowserSetup(channel, kind === "codex");
2808
+ if (!this.isActiveChannel(channel)) return null;
2809
+ await this.ensureMcpConfig(channel);
2810
+ if (!this.isActiveChannel(channel)) return null;
2811
+ // Codex runs in a home of its own (two agents on one account never
2812
+ // share its state runtime); the config writers above reached it.
2813
+ const home = kind === "codex" ? codexPerAgentHome(account.home!, agentId) : account.home!;
2814
+ try {
2815
+ await ensureEngineHome(environment, open.assignment, account.credentialId, home);
2816
+ } catch (error) {
2817
+ // The account moved while it was written (a newer version, a
2818
+ // cleanup, no longer readable, the gate closing): pick again on
2819
+ // what the gate serves, or will once it reopens, a few times while
2820
+ // time remains. Anything else ends here.
2821
+ if (
2822
+ (error instanceof EngineCredentialWriteRefused || error instanceof CredentialLeaseUnavailableError) &&
2823
+ moved++ < 3 &&
2824
+ Date.now() < deadline
2825
+ ) {
2826
+ continue;
2827
+ }
2828
+ reason = error instanceof CredentialLeaseUnavailableError ? error.message : "its files could not be written";
2829
+ break;
2830
+ }
2831
+ if (!this.isActiveChannel(channel)) return null;
2832
+ return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch, home);
2833
+ }
2834
+ if (!this.isActiveChannel(channel)) return null;
2835
+ this.leasedUnavailable(channel, agentId, reason);
2836
+ return null;
2837
+ })();
2838
+ }
2839
+
2840
+ /** The task's open engine lease, waited for while time remains, a newer
2841
+ * run's when the task moves on to one; null when none opens in time. */
2842
+ private async leasedGateOpen(channel: Channel, deadline: number): Promise<TaskEngineLease | null> {
2843
+ while (this.isActiveChannel(channel)) {
2844
+ const record = leasedEnginesFor(channel.taskId);
2845
+ if (!record) return null;
2846
+ try {
2847
+ return await waitForEngineGate(record.assignment, Math.max(0, deadline - Date.now()));
2848
+ } catch {
2849
+ // The task moved on to another run: wait for that run's gate while
2850
+ // time remains.
2851
+ if (leasedEnginesFor(channel.taskId)?.assignment.runEpoch !== record.assignment.runEpoch && Date.now() < deadline) {
2852
+ continue;
2853
+ }
2854
+ return null;
2855
+ }
2856
+ }
2857
+ return null;
2858
+ }
2859
+
2860
+ /**
2861
+ * ADR-154 §6, ADR-128: what a Managed Engine spawn on a leased task takes.
2862
+ * It runs on no AI account, but its agent's task token is its bearer: it
2863
+ * is bound to the task's open run, its launches admitted as that token on
2864
+ * this binding, under the engine supervisor like any leased engine's.
2865
+ * While the task's gate is closed it waits, bounded; null when it stays
2866
+ * closed, and the agent stays down with "credentials unavailable" until
2867
+ * the gate next opens.
2868
+ */
2869
+ private leasedTaskTokenSpawnArgs(
2870
+ channel: Channel,
2871
+ agentId: string,
2872
+ args: SpawnArgs,
2873
+ ):
2874
+ | { args: SpawnArgs; identity: LeasedIdentity }
2875
+ | null
2876
+ | Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
2877
+ const lease = leasedEnginesFor(channel.taskId);
2878
+ if (lease?.open) return this.withLeasedTaskToken(channel, agentId, args, lease.assignment.runEpoch);
2879
+ return (async () => {
2880
+ const open = await this.leasedGateOpen(channel, Date.now() + LEASED_ACCOUNT_WAIT_MS);
2881
+ if (!this.isActiveChannel(channel)) return null;
2882
+ if (!open) {
2883
+ this.leasedUnavailable(channel, agentId, "the task's credentials are not available");
2884
+ return null;
2885
+ }
2886
+ return this.withLeasedTaskToken(channel, agentId, args, open.assignment.runEpoch);
2887
+ })();
2888
+ }
2889
+
2890
+ /** Bind the agent to the task's run `runEpoch` on its task token alone,
2891
+ * and give its spawn the run's marker every process it starts inherits
2892
+ * (its launches are known by it, and admitted on this binding). */
2893
+ private withLeasedTaskToken(
2894
+ channel: Channel,
2895
+ agentId: string,
2896
+ args: SpawnArgs,
2897
+ runEpoch: number,
2898
+ ): { args: SpawnArgs; identity: LeasedIdentity } {
2899
+ const identity = { subject: "task-token" as const, runEpoch };
2900
+ channel.leasedIdentityByAgent.set(agentId, identity);
2901
+ return {
2902
+ identity,
2903
+ args: {
2904
+ ...args,
2905
+ attachCompatibilityKey: JSON.stringify([args.attachCompatibilityKey ?? null, "lease-v1", runEpoch, "task-token"]),
2906
+ agentEnv: {
2907
+ ...(args.agentEnv ?? {}),
2908
+ // The gateway's bearer is the agent's own token; no engine's
2909
+ // credential of the runner's rides beside it.
2910
+ ...LEASED_CREDENTIAL_BLANKS,
2911
+ [LEASED_RUN_ENV]: String(runEpoch),
2912
+ },
2913
+ },
2914
+ };
2915
+ }
2916
+
2917
+ /**
2918
+ * ADR-154 §6: the environment a leased engine's adapter launches through.
2919
+ * Each engine launch it makes (the session, each one-shot turn, a durable
2920
+ * runner once deployed, a classifier's exec), at whatever moment its
2921
+ * transport gets there, is admitted then: for an agent's spawn (`owner`),
2922
+ * only while that spawn is the agent's current one and, once its create
2923
+ * settled, the session it bound is the agent's live one, and `identity`
2924
+ * is still what the agent runs on and the task's open gate serves (for a
2925
+ * one-shot with no agent, only the latter). The admission is written into
2926
+ * the runtime first (its exec admitted at the provider's last moment) and
2927
+ * the check made again just before the launch is handed over and at the
2928
+ * provider's last moment. The launch then starts under the engine
2929
+ * supervisor, which consumes that admission. A spawn's admissions name its
2930
+ * slot: before its first, the agent's earlier spawns are retired in the
2931
+ * runtime, and once the slot is retired, nothing it was admitted for
2932
+ * starts there (`retireLeasedLaunch`). A cleanup unbinds first, waits for
2933
+ * the admissions under way (registered before their first await), and
2934
+ * revokes in the runtime what they wrote before it kills what the
2935
+ * supervisors own.
2936
+ */
2937
+ private leasedLaunchEnvironment(
2938
+ channel: Channel,
2939
+ owner: { agentId: string; slot: LeasedLaunchSlot } | null,
2940
+ identity: LeasedIdentity,
2941
+ environment: TaskEnvironmentHandle<unknown>,
2942
+ ): TaskEnvironmentHandle<unknown> {
2943
+ const agentId = owner?.agentId ?? null;
2944
+ const held = (): boolean =>
2945
+ this.isActiveChannel(channel) &&
2946
+ (owner === null
2947
+ ? this.leasedMaterialServed(channel, identity)
2948
+ : channel.leasedLaunches.get(owner.agentId) === owner.slot &&
2949
+ (owner.slot.session === null
2950
+ ? !owner.slot.created
2951
+ : channel.sessions.get(owner.agentId) === owner.slot.session) &&
2952
+ this.leasedIdentityHeld(channel, owner.agentId, identity));
2953
+ const admittedAs: EngineLaunchIdentity = owner?.slot.identity ?? { taskId: channel.taskId, slotId: null, ...identity };
2954
+ return anchoredEnvironment(environment, {
2955
+ held,
2956
+ admit: (carried) => {
2957
+ if (environment.anchors !== true || !held()) return Promise.reject(new EngineLaunchRefused());
2958
+ if (owner) owner.slot.admitted = true;
2959
+ const admitted = (async () => {
2960
+ if (owner) await this.leasedPredecessorsRetired(owner.agentId, owner.slot);
2961
+ return admitEngineLaunch(
2962
+ environment,
2963
+ admittedAs,
2964
+ () => {
2965
+ if (!held()) throw new EngineLaunchRefused();
2966
+ },
2967
+ carried,
2968
+ );
2969
+ })();
2970
+ const entry = {
2971
+ agentId: agentId ?? "",
2972
+ accountId: identity.subject === "account" ? identity.credentialId : null,
2973
+ runEpoch: identity.runEpoch,
2974
+ settled: admitted.then(
2975
+ () => undefined,
2976
+ () => undefined,
2977
+ ),
2978
+ };
2979
+ channel.leasedCreating.add(entry);
2980
+ void entry.settled.then(() => channel.leasedCreating.delete(entry));
2981
+ return admitted;
2982
+ },
2983
+ });
2984
+ }
2985
+
2986
+ /** `session` ended (its engine's error or exit): its spawn admits no
2987
+ * launch from this moment, before its terminal handling awaits anything.
2988
+ * Every such handling retires or replaces the session. */
2989
+ private endLeasedLaunches(channel: Channel, agentId: string, session: AgentSession): void {
2990
+ const slot = channel.leasedLaunches.get(agentId);
2991
+ if (slot?.session === session) void this.retireLeasedLaunch(channel, agentId, slot);
2992
+ }
2993
+
2994
+ /**
2995
+ * ADR-154 §6: `slot` admits nothing from this moment. Then, in its
2996
+ * runtime, it is retired: every admission it wrote that no launch has used
2997
+ * is revoked, and none of its own is written or used after, so a launch
2998
+ * handed to its provider before now that has not started there never
2999
+ * does, with the agent's old task token or anything else. The agent's
3000
+ * next spawn launches in that runtime only once that is done
3001
+ * (`leasedPredecessorsRetired`). Resolves once that is done or escalated
3002
+ * (see `retireInRuntime`); never rejects.
3003
+ */
3004
+ private retireLeasedLaunch(channel: Channel, agentId: string, slot: LeasedLaunchSlot | undefined): Promise<void> {
3005
+ if (!slot || slot.retired) return Promise.resolve();
3006
+ slot.retired = true;
3007
+ if (channel.leasedLaunches.get(agentId) === slot) channel.leasedLaunches.delete(agentId);
3008
+ // It never wrote an admission: nothing of it is in its runtime.
3009
+ if (!slot.admitted) return Promise.resolve();
3010
+ const retired: RetiredLaunchSlot = { agentId, slot, attempt: null, covered: false, escalated: false };
3011
+ let pending = this.leasedRetired.get(channel.taskId);
3012
+ if (!pending) {
3013
+ pending = new Set();
3014
+ this.leasedRetired.set(channel.taskId, pending);
3015
+ }
3016
+ pending.add(retired);
3017
+ return this.retireInRuntime(retired).catch(() => undefined);
3018
+ }
3019
+
3020
+ /**
3021
+ * One attempt to retire `retired` in its runtime, joined while one is
3022
+ * under way; once done, it is no longer pending. Should it fail, the order
3023
+ * is lost: a launch handed over before the retirement may start before
3024
+ * any later one lands, and nothing a retry does stops it. So, unless a
3025
+ * cleanup that covers it is under way or done, the run's full credential
3026
+ * cleanup is asked for at once (`escalateEngineLaunchFence`): its gate
3027
+ * closed first, it proves no engine of the run runs (or stops the
3028
+ * runtime), and keeps failing closed until it can. Rejects when this
3029
+ * attempt failed (it is retained and retried meanwhile).
3030
+ */
3031
+ private retireInRuntime(retired: RetiredLaunchSlot): Promise<void> {
3032
+ if (retired.attempt) return retired.attempt;
3033
+ const { taskId, runEpoch } = retired.slot.identity;
3034
+ const attempt = retireEngineLaunchSlot(retired.slot.environment, retired.slot.identity)
3035
+ .then(
3036
+ () => this.forgetRetired(taskId, retired),
3037
+ (error: unknown) => {
3038
+ if (!retired.covered && !retired.escalated) {
3039
+ retired.escalated = true;
3040
+ console.warn(
3041
+ `[orchestrator] ${taskId}/${retired.agentId}: a retired spawn could not be retired in its runtime; cleaning up its run's credentials (${
3042
+ error instanceof Error ? error.message : String(error)
3043
+ })`,
3044
+ );
3045
+ escalateEngineLaunchFence({ taskId, runEpoch });
3046
+ }
3047
+ throw error;
3048
+ },
3049
+ )
3050
+ .finally(() => {
3051
+ if (retired.attempt === attempt) retired.attempt = null;
3052
+ });
3053
+ retired.attempt = attempt;
3054
+ return attempt;
3055
+ }
3056
+
3057
+ /** The task's retirements still under way (of `agentId`'s spawns alone,
3058
+ * when named), whichever event began them: each resolves once its slot is
3059
+ * retired in its runtime, or its run's cleanup has been asked for. What a
3060
+ * removal or a channel's close waits for. */
3061
+ private leasedRetirementsUnderWay(taskId: string, agentId?: string): Array<Promise<void>> {
3062
+ return [...(this.leasedRetired.get(taskId) ?? [])]
3063
+ .filter((retired) => agentId === undefined || retired.agentId === agentId)
3064
+ .flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
3065
+ }
3066
+
3067
+ private forgetRetired(taskId: string, retired: RetiredLaunchSlot): void {
3068
+ const pending = this.leasedRetired.get(taskId);
3069
+ if (!pending) return;
3070
+ pending.delete(retired);
3071
+ if (pending.size === 0) this.leasedRetired.delete(taskId);
3072
+ }
3073
+
3074
+ /**
3075
+ * ADR-154 §6: before `slot` writes an admission, every earlier slot of the
3076
+ * same agent retired for the same run in the same runtime is retired
3077
+ * there: nothing the agent's earlier spawns were admitted for can start
3078
+ * after this one's launch. Rejects with `EngineLaunchRefused` when one
3079
+ * could not be now.
3080
+ */
3081
+ private async leasedPredecessorsRetired(agentId: string, slot: LeasedLaunchSlot): Promise<void> {
3082
+ const earlier = [...(this.leasedRetired.get(slot.identity.taskId) ?? [])].filter(
3083
+ (retired) =>
3084
+ retired.agentId === agentId &&
3085
+ retired.slot.identity.runEpoch === slot.identity.runEpoch &&
3086
+ sameRuntime(retired.slot.environment, slot.environment),
3087
+ );
3088
+ try {
3089
+ await Promise.all(earlier.map((retired) => this.retireInRuntime(retired)));
3090
+ } catch {
3091
+ throw new EngineLaunchRefused("an earlier start of this agent could not be retired in its runtime yet");
3092
+ }
3093
+ }
3094
+
3095
+ /** The retirements still pending that a cleanup of these runs and
3096
+ * accounts (of every subject, for a whole run's) covers: its stop revokes
3097
+ * everything they fence, wherever it was written, and kills what any
3098
+ * launch of theirs started. Each is marked covered: one that fails from
3099
+ * now on is not escalated. */
3100
+ private coveredLeasedRetirements(
3101
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3102
+ ): RetiredLaunchSlot[] {
3103
+ const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
3104
+ const covered = [...(this.leasedRetired.get(request.taskId) ?? [])].filter((retired) => {
3105
+ const { identity } = retired.slot;
3106
+ return (
3107
+ identity.runEpoch <= request.runEpoch &&
3108
+ (!named || (identity.subject === "account" && named.has(identity.credentialId)))
3109
+ );
3110
+ });
3111
+ for (const retired of covered) retired.covered = true;
3112
+ return covered;
3113
+ }
3114
+
3115
+ /** A cleanup proved every admission of these runs and accounts (of every
3116
+ * subject, for a whole run's) revoked wherever it was written: the
3117
+ * retirements still pending for them are done. */
3118
+ private dischargeLeasedRetirements(
3119
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3120
+ ): void {
3121
+ for (const retired of this.coveredLeasedRetirements(request)) this.forgetRetired(request.taskId, retired);
3122
+ }
3123
+
3124
+ /** The agent stays down for want of a usable account, says why, and is
3125
+ * started again when the task's gate next opens. */
3126
+ private leasedUnavailable(channel: Channel, agentId: string, reason: string): void {
3127
+ channel.accountByAgent.delete(agentId);
3128
+ channel.leasedIdentityByAgent.delete(agentId);
3129
+ channel.leasedBlocked.add(agentId);
3130
+ this.emitHost({
3131
+ kind: "agent.exit",
3132
+ taskId: channel.taskId,
3133
+ agentId,
3134
+ reason: `${agentId}: credentials unavailable (${reason}) — the agent starts again when they are.`,
3135
+ });
3136
+ }
3137
+
3138
+ /** Whether the task's open gate still serves `identity`'s account at its
3139
+ * revision and version, for its run, and that version is not being held
3140
+ * back for a newer one: what a one-shot may start on. An agent's task
3141
+ * token alone needs only its run's gate open. */
3142
+ private leasedMaterialServed(channel: Channel, identity: LeasedIdentity): boolean {
3143
+ const lease = leasedEnginesFor(channel.taskId);
3144
+ if (!lease?.open || lease.assignment.runEpoch !== identity.runEpoch) return false;
3145
+ if (identity.subject === "task-token") return true;
3146
+ const account = lease.accounts.find((candidate) => candidate.credentialId === identity.credentialId);
3147
+ return (
3148
+ isLeasedEngineAccount(account) &&
3149
+ account.revision === identity.revision &&
3150
+ account.version === identity.version &&
3151
+ !engineVersionHeldBack(lease.assignment, identity)
3152
+ );
3153
+ }
3154
+
3155
+ /** A spawn found its lease material moved (a newer version, a cleanup):
3156
+ * start the agent again on what the task holds now, once its gate is
3157
+ * open (now, or at its next opening). */
3158
+ private retryLeasedStart(channel: Channel, agentId: string): void {
3159
+ if (!this.isActiveChannel(channel)) return;
3160
+ channel.leasedRespawn.add(agentId);
3161
+ channel.leasedWakePending = true;
3162
+ if (leasedEnginesFor(channel.taskId)?.open) this.leasedEnginesChanged(channel.taskId);
3163
+ }
3164
+
3165
+ /** Whether the agent may start on `identity` right now: still bound to it,
3166
+ * and the task's open gate still serves that account at that revision and
3167
+ * version, for that run. */
3168
+ private leasedIdentityHeld(channel: Channel, agentId: string, identity: LeasedIdentity): boolean {
3169
+ const bound = channel.leasedIdentityByAgent.get(agentId);
3170
+ return bound !== undefined && sameLeasedIdentity(bound, identity) && this.leasedMaterialServed(channel, identity);
3171
+ }
3172
+
3173
+ /** The leased account an agent runs on: the one it is bound to while that
3174
+ * is served readable, else the first healthy one in try order. */
3175
+ private leasedAccountFor(
3176
+ channel: Channel,
3177
+ agentId: string,
3178
+ kind: string,
3179
+ lease: TaskEngineLease,
3180
+ ): LeasedEngineAccount | undefined {
3181
+ const accounts = lease.accounts.filter(
3182
+ (account): account is LeasedEngineAccount =>
3183
+ account.engineKind === kind && isLeasedEngineAccount(account),
3184
+ );
3185
+ const bound = channel.accountByAgent.get(agentId);
3186
+ const kept = accounts.find((account) => account.credentialId === bound);
3187
+ if (kept) return kept;
3188
+ const picked = selectAccount(
3189
+ accounts.map((account) => ({
3190
+ id: account.credentialId,
3191
+ kind,
3192
+ label: account.label,
3193
+ isDefault: false,
3194
+ scope: account.scope,
3195
+ authKind: account.authKind,
3196
+ execEnv: {},
3197
+ })),
3198
+ );
3199
+ return accounts.find((account) => account.credentialId === picked?.id);
3200
+ }
3201
+
3202
+ /** Bind the agent to `account` for `runEpoch` and give its spawn the
3203
+ * account, and the markers every process it starts inherits (its launches
3204
+ * are known by them, and admitted on this binding). */
3205
+ private withLeasedAccount(
3206
+ channel: Channel,
3207
+ agentId: string,
3208
+ args: SpawnArgs,
3209
+ account: LeasedEngineAccount,
3210
+ runEpoch: number,
3211
+ home?: string,
3212
+ ): { args: SpawnArgs; identity: LeasedIdentity } {
3213
+ const identity = {
3214
+ subject: "account" as const,
3215
+ runEpoch,
3216
+ credentialId: account.credentialId,
3217
+ revision: account.revision,
3218
+ version: account.version,
3219
+ };
3220
+ channel.accountByAgent.set(agentId, account.credentialId);
3221
+ channel.leasedIdentityByAgent.set(agentId, identity);
3222
+ noteEngineAccountUsed(account.credentialId);
3223
+ return {
3224
+ identity,
3225
+ args: {
3226
+ ...args,
3227
+ // A durable runner from before a restart is attached to only when it
3228
+ // was launched on this exact material.
3229
+ attachCompatibilityKey: JSON.stringify([
3230
+ args.attachCompatibilityKey ?? null,
3231
+ "lease-v1",
3232
+ identity.runEpoch,
3233
+ identity.credentialId,
3234
+ identity.revision,
3235
+ identity.version,
3236
+ ]),
3237
+ agentEnv: {
3238
+ ...(args.agentEnv ?? {}),
3239
+ ...LEASED_CREDENTIAL_BLANKS,
3240
+ ...engineSpawnEnv(account, home),
3241
+ [LEASED_RUN_ENV]: String(runEpoch),
3242
+ [LEASED_ACCOUNT_ENV]: account.credentialId,
3243
+ },
3244
+ },
3245
+ };
3246
+ }
3247
+
3248
+ /**
3249
+ * The env of a one-shot run on the task's leased account for `kind` (the
3250
+ * shadow router's classifier): the agent's own account when it runs on
3251
+ * one of that kind, else the first healthy one; a config-directory
3252
+ * account's home is written first. Null when none can be used now: the
3253
+ * classifier does not run and the pass routes by rules.
3254
+ */
3255
+ /** ADR-154 §6: a one-shot's env on the leased `account` chosen for it: its
3256
+ * home written first for a config-directory account. Null when it cannot
3257
+ * be given. */
3258
+ private async leasedOneShotEnv(
3259
+ channel: Channel,
3260
+ lease: TaskEngineLease,
3261
+ account: LeasedEngineAccount,
3262
+ environment: TaskEnvironmentHandle<unknown>,
3263
+ ): Promise<Record<string, string> | null> {
3264
+ const marker = {
3265
+ [LEASED_RUN_ENV]: String(lease.assignment.runEpoch),
3266
+ [LEASED_ACCOUNT_ENV]: account.credentialId,
3267
+ };
3268
+ try {
3269
+ rememberLeasedEngineRuntime({
3270
+ taskId: channel.taskId,
3271
+ runEpoch: lease.assignment.runEpoch,
3272
+ runtime: environment.descriptor.locator,
3273
+ });
3274
+ } catch {
3275
+ return null;
3276
+ }
3277
+ if (account.authKind === "env") {
3278
+ return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
3279
+ }
3280
+ try {
3281
+ await ensureEngineHome(environment, lease.assignment, account.credentialId);
3282
+ return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
3283
+ } catch {
3284
+ return null;
3285
+ }
3286
+ }
3287
+
3288
+ /**
3289
+ * ADR-154 §6, §8: what the task's AI accounts are, or whether its gate is
3290
+ * open, changed. Rewrite what moved in the homes this run wrote, then let
3291
+ * each long-lived engine running on a rewritten account (Codex's
3292
+ * app-server) respawn at its first idle moment; one-shot engines read the
3293
+ * file again at their next turn, and env-token engines take a new value at
3294
+ * their next spawn. Called synchronously by the engine consumer: the work
3295
+ * runs after, one pass at a time per channel.
3296
+ */
3297
+ private leasedEnginesChanged(taskId: string): void {
3298
+ const channel = this.channels.get(taskId);
3299
+ if (!channel || !this.isActiveChannel(channel) || !this.isLeasedChannel(channel)) return;
3300
+ if (!leasedEnginesFor(taskId)?.open) return;
3301
+ if (channel.leasedSyncRetry) {
3302
+ clearTimeout(channel.leasedSyncRetry);
3303
+ channel.leasedSyncRetry = null;
3304
+ }
3305
+ // Agents a cleanup stopped, or that found no usable account, start again
3306
+ // on this view, each on its next account: one attempt per opening. The
3307
+ // intent outlives a pass that fails, for its retry.
3308
+ if (channel.leasedRespawn.size > 0 || channel.leasedBlocked.size > 0) {
3309
+ channel.leasedWakePending = true;
3310
+ }
3311
+ const run = (channel.leasedSync ?? Promise.resolve())
3312
+ .then(async () => {
3313
+ await this.syncLeasedHomes(channel);
3314
+ if (!channel.leasedWakePending || !this.isActiveChannel(channel)) return;
3315
+ channel.leasedWakePending = false;
3316
+ channel.leasedRespawn.clear();
3317
+ channel.leasedBlocked.clear();
3318
+ try {
3319
+ await this.ensureSessions(channel);
3320
+ } catch (err) {
3321
+ channel.leasedWakePending = true;
3322
+ throw err;
3323
+ }
3324
+ })
3325
+ .catch((err: unknown) => {
3326
+ console.warn(
3327
+ `[orchestrator] ${taskId}: rewriting AI account files failed: ${err instanceof Error ? err.message : String(err)}`,
3328
+ );
3329
+ // A home left on an old token is written again, not left so.
3330
+ if (this.isActiveChannel(channel) && !channel.leasedSyncRetry) {
3331
+ const retry = setTimeout(() => {
3332
+ if (channel.leasedSyncRetry !== retry) return;
3333
+ channel.leasedSyncRetry = null;
3334
+ this.leasedEnginesChanged(taskId);
3335
+ }, LEASED_SYNC_RETRY_MS);
3336
+ retry.unref?.();
3337
+ channel.leasedSyncRetry = retry;
3338
+ }
3339
+ });
3340
+ channel.leasedSync = run;
3341
+ }
3342
+
3343
+ private async syncLeasedHomes(channel: Channel): Promise<void> {
3344
+ if (!this.isActiveChannel(channel)) return;
3345
+ const lease = leasedEnginesFor(channel.taskId);
3346
+ if (!lease?.open) return;
3347
+ // Nothing was written into a runtime this channel has not settled.
3348
+ const environment = channel.agentHandle;
3349
+ if (environment) await syncEngineHomes(environment, lease.assignment);
3350
+ if (!this.isActiveChannel(channel)) return;
3351
+ // Every launch is admitted on the exact material its agent is bound to
3352
+ // (§6): an agent whose account moved since starts again on what the
3353
+ // task holds now. A long-lived engine holds the old token or file; a
3354
+ // one-shot one, env-token or not, would have its next turn refused.
3355
+ let stale = false;
3356
+ for (const agent of channel.roster) {
3357
+ const identity = channel.leasedIdentityByAgent.get(agent.id);
3358
+ if (!identity || this.leasedMaterialServed(channel, identity)) continue;
3359
+ if (!channel.sessions.has(agent.id)) continue;
3360
+ channel.browserStaleSessions.add(agent.id);
3361
+ stale = true;
3362
+ }
3363
+ // A stale idle session is recycled now; one mid-turn finishes its turn
3364
+ // on the material it has and is recycled at its next idle moment.
3365
+ if (stale) await this.reconcileIfStarted(channel);
3366
+ }
3367
+
3368
+ /**
3369
+ * ADR-154 §6: the cleanup coordinator's quiesce step for engines (its
3370
+ * `quiesceProcesses` hook takes this request as it is), for the run it
3371
+ * names and every older one, never a newer run's:
3372
+ * - every agent on one of the request's accounts (on any leased account,
3373
+ * or on its task token alone, for a whole-task cleanup) is unbound at
3374
+ * once, so nothing launches on them again (a launch is admitted only
3375
+ * while its agent is bound to its exact material), and its session is
3376
+ * asked to stop; a spawn creating an engine on one, or a launch being
3377
+ * admitted, is waited for, bounded;
3378
+ * - then, in every runtime those runs' engines ran in, every admission not
3379
+ * used yet is revoked, and everything each engine supervisor of those
3380
+ * runs and accounts owns is killed until the supervisor ends on its own,
3381
+ * having proved nothing it owned is left (`stopAnchoredEngines`). That
3382
+ * holds no state here, so a retried cleanup proves it again even after
3383
+ * this channel let the agents go. A supervisor lost without proving its
3384
+ * subtree empty leaves one proof: the runtime is stopped. An env-token
3385
+ * engine holds no credential file: stopping it moves it to its next
3386
+ * account.
3387
+ * Rejects when anything could not be proven stopped: the cleanup then
3388
+ * fails and is retried, never deleting under a live engine. Stopped agents
3389
+ * start again on their next account once the gate opens.
3390
+ */
3391
+ async quiesceLeasedEngineAgents(
3392
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3393
+ ): Promise<void> {
3394
+ const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
3395
+ const channel = this.channels.get(request.taskId);
3396
+ // Every engine being created on one of the accounts (on any subject, a
3397
+ // task token included, for a whole run's), this attempt's or left over
3398
+ // from one that timed out: never only the bound agents'.
3399
+ const creating = [...(channel?.leasedCreating ?? [])]
3400
+ .filter(
3401
+ (entry) =>
3402
+ (!named || (entry.accountId !== null && named.has(entry.accountId))) && entry.runEpoch <= request.runEpoch,
3403
+ )
3404
+ .map((entry) => entry.settled);
3405
+ // Every retirement still pending that this cleanup covers: its stop
3406
+ // proves what they fence, so one that fails from now on is not escalated.
3407
+ const covered = this.coveredLeasedRetirements(request);
3408
+ if (channel) {
3409
+ for (const [agentId, accountId] of [...channel.accountByAgent]) {
3410
+ if (named && !named.has(accountId)) continue;
3411
+ if ((channel.leasedIdentityByAgent.get(agentId)?.runEpoch ?? 0) > request.runEpoch) continue;
3412
+ channel.accountByAgent.delete(agentId);
3413
+ this.quiesceLeasedAgent(channel, agentId);
3414
+ }
3415
+ // A whole run's cleanup takes every agent on its task token alone too
3416
+ // (the Managed Engine); a cleanup of named accounts never does.
3417
+ if (!named) {
3418
+ for (const [agentId, identity] of [...channel.leasedIdentityByAgent]) {
3419
+ if (identity.subject !== "task-token" || identity.runEpoch > request.runEpoch) continue;
3420
+ this.quiesceLeasedAgent(channel, agentId);
3421
+ }
3422
+ }
3423
+ }
3424
+ if (creating.length > 0) {
3425
+ let timer: ReturnType<typeof setTimeout> | undefined;
3426
+ const bounded = new Promise<"late">((resolve) => {
3427
+ timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
3428
+ });
3429
+ const outcome = await Promise.race([Promise.all(creating), bounded]);
3430
+ clearTimeout(timer);
3431
+ if (outcome === "late") {
3432
+ throw new Error("an engine being started on a removed AI account did not finish starting");
3433
+ }
3434
+ }
3435
+ // One of those under way is waited for, bounded, so nothing of it lands
3436
+ // after this cleanup's stop.
3437
+ const retiring = covered.flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
3438
+ if (retiring.length > 0) {
3439
+ let timer: ReturnType<typeof setTimeout> | undefined;
3440
+ const bounded = new Promise<"late">((resolve) => {
3441
+ timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
3442
+ });
3443
+ const outcome = await Promise.race([Promise.all(retiring), bounded]);
3444
+ clearTimeout(timer);
3445
+ if (outcome === "late") {
3446
+ throw new Error("an engine launch slot being retired in its runtime did not finish");
3447
+ }
3448
+ }
3449
+ const runtimes = await this.leasedCleanupRuntimes(request, channel);
3450
+ for (const environment of runtimes.values()) {
3451
+ // A runtime that is not running holds no process, unless it is frozen
3452
+ // (paused or restarting: its processes run again without a start);
3453
+ // one that is frozen or cannot say cannot be proven.
3454
+ const status = await environment.status();
3455
+ if (status.state !== "unknown" && status.frozen) {
3456
+ throw new Error("a task runtime's engines could not be proven stopped: it is paused");
3457
+ }
3458
+ if (status.state === "stopped" || status.state === "absent") continue;
3459
+ if (status.state !== "running") {
3460
+ throw new Error("a task runtime's engines could not be proven stopped: its state is unknown");
3461
+ }
3462
+ try {
3463
+ await stopAnchoredEngines(environment, {
3464
+ taskId: request.taskId,
3465
+ runEpoch: request.runEpoch,
3466
+ ...(request.credentialIds === undefined ? {} : { credentialIds: request.credentialIds }),
3467
+ });
3468
+ } catch (error) {
3469
+ if (!(error instanceof EngineAnchorLost)) throw error;
3470
+ // What a lost supervisor owned may have been reparented away from
3471
+ // it: only a stopped runtime proves none of it still runs.
3472
+ console.warn(`[orchestrator] ${request.taskId}: ${error.message}; stopping the runtime`);
3473
+ await environment.stop();
3474
+ const stopped = await environment.status();
3475
+ if (
3476
+ stopped.state === "unknown" ||
3477
+ stopped.frozen ||
3478
+ (stopped.state !== "stopped" && stopped.state !== "absent")
3479
+ ) {
3480
+ throw error;
3481
+ }
3482
+ }
3483
+ }
3484
+ // Every admission of these runs not yet used is revoked in every runtime
3485
+ // they were written in, or that runtime is stopped or gone and its gates
3486
+ // with it: no revocation or slot retirement retained for them is left to
3487
+ // do.
3488
+ dischargeEngineRevocations(request);
3489
+ this.dischargeLeasedRetirements(request);
3490
+ // Every engine of these runs is proven stopped in every runtime they
3491
+ // ran in: a whole-task cleanup lets the record of those runtimes go.
3492
+ if (request.credentialIds === undefined) {
3493
+ forgetLeasedEngineRuntimes(request.taskId, request.runEpoch, [...runtimes.keys()]);
3494
+ }
3495
+ }
3496
+
3497
+ /** A cleanup took the material `agentId` runs on: it is unbound at once,
3498
+ * so nothing launches on it again, its session is asked to stop, and it
3499
+ * starts again on what the task holds once the gate opens. */
3500
+ private quiesceLeasedAgent(channel: Channel, agentId: string): void {
3501
+ channel.leasedIdentityByAgent.delete(agentId);
3502
+ const session = channel.sessions.get(agentId);
3503
+ if (this.isActiveChannel(channel)) channel.leasedRespawn.add(agentId);
3504
+ if (!session) return;
3505
+ // Delete first: the old session's last events are generation-stale.
3506
+ channel.sessions.delete(agentId);
3507
+ channel.browserStaleSessions.delete(agentId);
3508
+ channel.browserPendingStaleSessions.delete(agentId);
3509
+ this.retainRetiredTurnBoundary(
3510
+ channel,
3511
+ agentId,
3512
+ this.needsAbortedTurnBoundary(channel, agentId),
3513
+ );
3514
+ this.retireRunnerGeneration(channel, agentId);
3515
+ void session.close().catch(() => {});
3516
+ }
3517
+
3518
+ /**
3519
+ * ADR-154 §6: an account is about to be served at a newer version. In
3520
+ * every runtime a launch of this run could start in, its older versions
3521
+ * are held back first: a launch admitted on one that has not started yet
3522
+ * never will. Rejects, so the install fails and none of the newer version
3523
+ * is served, when a runtime cannot be reached or proven not running.
3524
+ */
3525
+ private async raiseLeasedAdmissionFloors(
3526
+ assignment: { taskId: string; runEpoch: number },
3527
+ floors: readonly EngineAdmissionFloor[],
3528
+ ): Promise<void> {
3529
+ const channel = this.channels.get(assignment.taskId);
3530
+ const runtimes = await this.leasedCleanupRuntimes(
3531
+ { taskId: assignment.taskId, runEpoch: assignment.runEpoch },
3532
+ channel,
3533
+ );
3534
+ for (const environment of runtimes.values()) {
3535
+ const status = await environment.status();
3536
+ if (status.state !== "unknown" && status.frozen) {
3537
+ throw new Error("an AI account's older version could not be held back: a task runtime is paused");
3538
+ }
3539
+ if (status.state === "stopped" || status.state === "absent") continue;
3540
+ if (status.state !== "running") {
3541
+ throw new Error("an AI account's older version could not be held back: a task runtime's state is unknown");
3542
+ }
3543
+ await raiseEngineAdmissionFloors(environment, {
3544
+ taskId: assignment.taskId,
3545
+ runEpoch: assignment.runEpoch,
3546
+ floors,
3547
+ });
3548
+ }
3549
+ }
3550
+
3551
+ /**
3552
+ * Every runtime a cleanup reaches, by locator: the task's current one,
3553
+ * every one a leased engine of these runs was started in (recorded before
3554
+ * it started, files or none), and every one the inventory names for these
3555
+ * runs (an older runtime's files are deleted from that exact runtime).
3556
+ */
3557
+ private async leasedCleanupRuntimes(
3558
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3559
+ channel: Channel | undefined,
3560
+ ): Promise<Map<string, TaskEnvironmentHandle<unknown>>> {
3561
+ const runtimes = new Map<string, TaskEnvironmentHandle<unknown>>();
3562
+ const current =
3563
+ channel?.agentHandle ??
3564
+ (await (async () => {
3565
+ const task = getHostTask(request.taskId);
3566
+ return task ? reconstructPersistedTaskEnvironment(task) : null;
3567
+ })());
3568
+ if (current) runtimes.set(serializeTaskEnvironmentLocator(current.descriptor.locator), current);
3569
+ const recorded = [
3570
+ ...listLeasedEngineRuntimes(request.taskId),
3571
+ ...listCredentialCleanupPaths(request.taskId),
3572
+ ];
3573
+ for (const row of recorded) {
3574
+ if (row.runEpoch > request.runEpoch || runtimes.has(row.runtimeLocator)) continue;
3575
+ const environment = await reconstructTaskEnvironment(
3576
+ parseTaskEnvironmentLocator(row.runtimeLocator),
3577
+ "loader",
3578
+ );
3579
+ if (environment.descriptor.taskId !== request.taskId) {
3580
+ throw new Error("a cleanup runtime belongs to a different task");
3581
+ }
3582
+ runtimes.set(row.runtimeLocator, environment);
3583
+ }
3584
+ return runtimes;
3585
+ }
3586
+
2414
3587
  /**
2415
3588
  * Per-agent CODEX_HOME (see the call site). Returns `args` unchanged unless
2416
3589
  * this is a codex agent with a config-dir CODEX_HOME; then it seeds
@@ -2481,87 +3654,347 @@ export class Orchestrator {
2481
3654
  channel.reconcileAgain = true;
2482
3655
  return null;
2483
3656
  }
2484
- // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
2485
- // account share one CODEX_HOME, and codex's new sqlite state runtime
2486
- // cannot be opened by two concurrent processes — the second agent dies
2487
- // with "failed to initialize sqlite state runtime under <CODEX_HOME>"
2488
- // (live 2026-09-04, a task with two codex agents on one account). Give
2489
- // each codex agent its OWN CODEX_HOME, seeded once from the resolved
2490
- // account's dir, so their state runtimes never collide. Single-codex
2491
- // tasks are unaffected beyond a one-time in-container copy.
2492
- // Only codex agents take the isolation path; everything else uses args
2493
- // directly, WITHOUT an extra await — a stray microtask here reordered the
2494
- // Claude re-spawn's prompt re-delivery.
2495
- const boundArgs =
2496
- args.agent?.kind === "codex"
2497
- ? await this.isolateCodexHome(environment, agentId, args)
2498
- : args;
2499
- while (this.isActiveChannel(channel)) {
2500
- if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2501
- channel.reconcileAgain = true;
2502
- return null;
2503
- }
2504
- if (
2505
- channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
2506
- ) {
2507
- channel.reconcileAgain = true;
2508
- return null;
2509
- }
2510
- if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2511
- channel.reconcileAgain = true;
2512
- return null;
2513
- }
2514
- const generation = channel.configGeneration;
2515
- const session = await this.factory.create({ ...boundArgs, environment });
2516
- if (!this.isActiveChannel(channel)) {
2517
- await session.close().catch(() => {});
2518
- return null;
2519
- }
2520
- if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2521
- await session.close().catch(() => {});
2522
- channel.reconcileAgain = true;
2523
- return null;
2524
- }
2525
- const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
2526
- if (pendingRosterClose?.state === "pending-close") {
2527
- await this.closeRosterChangedSession(
2528
- channel,
2529
- agentId,
2530
- session,
2531
- pendingRosterClose,
2532
- );
2533
- channel.reconcileAgain = true;
3657
+ // ADR-154: an agent on the runner's own accounts holds them from here
3658
+ // until it is bound or given up. Leaving them waits for it; every process
3659
+ // it starts, now or at a later turn, is refused at its provider's last
3660
+ // moment once leaving has begun; and one created across that moment is
3661
+ // closed, never bound. Its next start is on the task's lease. The hold
3662
+ // ends in a `finally`, which adds no await (see below).
3663
+ const ownHold = this.isLeasedChannel(channel) ? undefined : holdOwnCredentials();
3664
+ if (ownHold === null) {
3665
+ channel.reconcileAgain = true;
3666
+ return null;
3667
+ }
3668
+ const onOwn = ownHold !== undefined;
3669
+ try {
3670
+ // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
3671
+ // account share one CODEX_HOME, and codex's new sqlite state runtime
3672
+ // cannot be opened by two concurrent processes — the second agent dies
3673
+ // with "failed to initialize sqlite state runtime under <CODEX_HOME>"
3674
+ // (live 2026-09-04, a task with two codex agents on one account). Give
3675
+ // each codex agent its OWN CODEX_HOME, seeded once from the resolved
3676
+ // account's dir, so their state runtimes never collide. Single-codex
3677
+ // tasks are unaffected beyond a one-time in-container copy.
3678
+ // Only codex agents take the isolation path; everything else uses args
3679
+ // directly, WITHOUT an extra await — a stray microtask here reordered the
3680
+ // Claude re-spawn's prompt re-delivery. A leased account is applied here
3681
+ // too, synchronously when nothing has to be waited for or written.
3682
+ let boundArgs = args;
3683
+ let identity: LeasedIdentity | undefined;
3684
+ const kind = args.agent?.kind ?? "";
3685
+ if (!onOwn && LEASED_ENGINE_KINDS.has(kind)) {
3686
+ // §6: a runtime that cannot start a launch under the engine supervisor
3687
+ // never runs a leased engine.
3688
+ if (environment.anchors !== true) {
3689
+ this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
3690
+ return null;
3691
+ }
3692
+ const leased = this.leasedSpawnArgs(channel, agentId, args, environment);
3693
+ const applied = leased instanceof Promise ? await leased : leased;
3694
+ // Down until the gate next opens: never a loop.
3695
+ if (!applied) return null;
3696
+ boundArgs = applied.args;
3697
+ identity = applied.identity;
3698
+ } else if (!onOwn && kind === MANAGED_ENGINE_KIND) {
3699
+ // ADR-128, ADR-154 §6: no AI account, but the agent's task token is
3700
+ // the engine's bearer: started under the supervisor, admitted as
3701
+ // that token on the task's open run, or not at all.
3702
+ if (environment.anchors !== true) {
3703
+ this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
3704
+ return null;
3705
+ }
3706
+ const leased = this.leasedTaskTokenSpawnArgs(channel, agentId, args);
3707
+ const applied = leased instanceof Promise ? await leased : leased;
3708
+ if (!applied) return null;
3709
+ boundArgs = applied.args;
3710
+ identity = applied.identity;
3711
+ } else if (!onOwn && !CREDENTIAL_FREE_KINDS.has(kind)) {
3712
+ // ADR-154 §10: a kind with no lane (Kimi) never runs on the runner's
3713
+ // own sign-in for a leased task.
3714
+ this.leasedUnavailable(channel, agentId, `${kind} has no AI account lane on cloud credentials`);
2534
3715
  return null;
3716
+ } else if (kind === "codex") {
3717
+ boundArgs = await this.isolateCodexHome(environment, agentId, args);
2535
3718
  }
2536
- if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2537
- const change = channel.rosterSessionChanges.get(agentId);
2538
- if (change) {
2539
- await this.closeRosterChangedSession(
2540
- channel,
2541
- agentId,
2542
- session,
2543
- change,
3719
+ while (this.isActiveChannel(channel)) {
3720
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
3721
+ channel.reconcileAgain = true;
3722
+ return null;
3723
+ }
3724
+ if (
3725
+ channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
3726
+ ) {
3727
+ channel.reconcileAgain = true;
3728
+ return null;
3729
+ }
3730
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
3731
+ channel.reconcileAgain = true;
3732
+ return null;
3733
+ }
3734
+ if (onOwn && !onOwnCredentials()) {
3735
+ channel.reconcileAgain = true;
3736
+ return null;
3737
+ }
3738
+ const generation = channel.configGeneration;
3739
+ // ADR-154 §6: never start an engine on lease material the task does not
3740
+ // hold right now (its run, account, revision and version). A cleanup
3741
+ // drops the binding synchronously, so a spawn whose account it took
3742
+ // stops here; one already creating is registered, and every cleanup
3743
+ // attempt waits for it.
3744
+ if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
3745
+ this.retryLeasedStart(channel, agentId);
3746
+ return null;
3747
+ }
3748
+ if (identity) {
3749
+ // Where this engine runs, before it can run there (§6): a cleanup
3750
+ // proves it stopped in this runtime even if it leaves no file.
3751
+ try {
3752
+ rememberLeasedEngineRuntime({
3753
+ taskId: channel.taskId,
3754
+ runEpoch: identity.runEpoch,
3755
+ runtime: environment.descriptor.locator,
3756
+ });
3757
+ } catch (err) {
3758
+ console.warn(
3759
+ `[orchestrator] ${channel.taskId}/${agentId}: not starting on its AI account, its runtime could not be recorded: ${err instanceof Error ? err.message : String(err)}`,
3760
+ );
3761
+ return null;
3762
+ }
3763
+ }
3764
+ // Every process the adapter launches for this engine (now, at each
3765
+ // turn, or once its durable runner is deployed) is admitted then, on
3766
+ // this exact material, for this spawn while it is the agent's current
3767
+ // one, and started under the supervisor (§6). The agent's spawn this
3768
+ // one replaces is retired.
3769
+ const slot = identity ? newLeasedLaunchSlot(channel.taskId, identity, environment) : null;
3770
+ if (slot) {
3771
+ void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
3772
+ channel.leasedLaunches.set(agentId, slot);
3773
+ }
3774
+ const creating = this.factory.create({
3775
+ ...boundArgs,
3776
+ environment:
3777
+ identity && slot
3778
+ ? this.leasedLaunchEnvironment(channel, { agentId, slot }, identity, environment)
3779
+ : onOwn
3780
+ ? onOwnCredentialsEnvironment(environment)
3781
+ : environment,
3782
+ });
3783
+ // Settled before this spawn's own continuation: from here until it
3784
+ // binds, nothing launches on it.
3785
+ if (slot) {
3786
+ void creating.then(
3787
+ () => {
3788
+ slot.created = true;
3789
+ },
3790
+ () => {
3791
+ slot.created = true;
3792
+ },
2544
3793
  );
2545
- } else {
2546
- await session.close().catch(() => {});
2547
3794
  }
2548
- channel.reconcileAgain = true;
2549
- return null;
2550
- }
2551
- if (generation === channel.configGeneration) {
2552
- // Keep the final generation check, stale cleanup, and map bind in one
2553
- // synchronous continuation. A config writer that resumes immediately
2554
- // afterward now sees this session and marks it stale; it can no longer
2555
- // change the generation in the await→bind gap while the process is
2556
- // invisible.
2557
- channel.browserStaleSessions.delete(agentId);
2558
- channel.browserPendingStaleSessions.delete(agentId);
2559
- this.bindSession(channel, agentId, session);
2560
- return session;
3795
+ if (identity) {
3796
+ const entry = {
3797
+ agentId,
3798
+ accountId: identity.subject === "account" ? identity.credentialId : null,
3799
+ runEpoch: identity.runEpoch,
3800
+ settled: creating.then(
3801
+ () => undefined,
3802
+ () => undefined,
3803
+ ),
3804
+ };
3805
+ channel.leasedCreating.add(entry);
3806
+ void entry.settled.then(() => channel.leasedCreating.delete(entry));
3807
+ }
3808
+ // ADR-154 §6: a spawn that does not bind (its create failed, maybe
3809
+ // after it launched, or its session is closed unbound) is retired at
3810
+ // once, before anything else: a launch it handed over that has not
3811
+ // started in its runtime never does.
3812
+ const unbound = (): Promise<void> =>
3813
+ slot ? this.retireLeasedLaunch(channel, agentId, slot) : Promise.resolve();
3814
+ let session: AgentSession;
3815
+ try {
3816
+ session = await creating;
3817
+ } catch (error) {
3818
+ await unbound();
3819
+ throw error;
3820
+ }
3821
+ if (!this.isActiveChannel(channel)) {
3822
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3823
+ return null;
3824
+ }
3825
+ // ADR-154: created on the runner's own accounts across the moment
3826
+ // leaving them began: closed, never bound.
3827
+ if (onOwn && !onOwnCredentials()) {
3828
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3829
+ channel.reconcileAgain = true;
3830
+ return null;
3831
+ }
3832
+ // ADR-154 §6: a cleanup that took this account while the session was
3833
+ // being created quiesced the agent already; never keep a session on
3834
+ // material the task no longer holds.
3835
+ if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
3836
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3837
+ this.retryLeasedStart(channel, agentId);
3838
+ return null;
3839
+ }
3840
+ if (identity) channel.leasedBlocked.delete(agentId);
3841
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
3842
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3843
+ channel.reconcileAgain = true;
3844
+ return null;
3845
+ }
3846
+ const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
3847
+ if (pendingRosterClose?.state === "pending-close") {
3848
+ await Promise.all([
3849
+ unbound(),
3850
+ this.closeRosterChangedSession(channel, agentId, session, pendingRosterClose),
3851
+ ]);
3852
+ channel.reconcileAgain = true;
3853
+ return null;
3854
+ }
3855
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
3856
+ const change = channel.rosterSessionChanges.get(agentId);
3857
+ const retiring = unbound();
3858
+ if (change) {
3859
+ await this.closeRosterChangedSession(
3860
+ channel,
3861
+ agentId,
3862
+ session,
3863
+ change,
3864
+ );
3865
+ } else {
3866
+ await session.close().catch(() => {});
3867
+ }
3868
+ await retiring;
3869
+ channel.reconcileAgain = true;
3870
+ return null;
3871
+ }
3872
+ if (generation === channel.configGeneration) {
3873
+ // Keep the final generation check, stale cleanup, and map bind in one
3874
+ // synchronous continuation. A config writer that resumes immediately
3875
+ // afterward now sees this session and marks it stale; it can no longer
3876
+ // change the generation in the await→bind gap while the process is
3877
+ // invisible.
3878
+ channel.browserStaleSessions.delete(agentId);
3879
+ channel.browserPendingStaleSessions.delete(agentId);
3880
+ if (slot) slot.session = session;
3881
+ this.bindSession(channel, agentId, session);
3882
+ return session;
3883
+ }
3884
+ await Promise.all([unbound(), session.close().catch(() => {})]);
2561
3885
  }
2562
- await session.close().catch(() => {});
3886
+ return null;
3887
+ } finally {
3888
+ ownHold?.();
2563
3889
  }
2564
- return null;
3890
+ }
3891
+
3892
+ /**
3893
+ * ADR-154, the shadow router's one exec: the only use it makes of an AI
3894
+ * account, and the only place that account's material exists for it.
3895
+ * Null when it cannot run, and the pass routes by rules.
3896
+ * - A leased task's: its account is chosen, and the exec registered where
3897
+ * a cleanup of that account waits, in one synchronous step before any of
3898
+ * its material is read; the registration settles when the exec does,
3899
+ * and nothing the material is in outlives this call (§6). A cleanup that
3900
+ * took the account first leaves it unchosen. The exec starts under the
3901
+ * engine supervisor, only on material the task still holds at the
3902
+ * provider's last moment.
3903
+ * - On the runner's own credentials: they are held from the account's
3904
+ * choice to the exec's end, and the exec is refused at the provider's
3905
+ * last moment once leaving them has begun or the channel has closed.
3906
+ */
3907
+ private async shadowClassify(
3908
+ channel: Channel,
3909
+ agentId: string,
3910
+ kind: string,
3911
+ handle: TaskEnvironmentHandle<unknown>,
3912
+ argv: [string, ...string[]],
3913
+ scratchDir: string,
3914
+ timeoutMs: number,
3915
+ ): Promise<{ exitCode: number | null; stdout: string } | null> {
3916
+ const request = (env: Record<string, string>, admit: () => void) => ({
3917
+ argv: agentCliArgv(handle, argv),
3918
+ projectEnvProfile: "task" as const,
3919
+ cwd: scratchDir,
3920
+ user: "node",
3921
+ env,
3922
+ timeoutMs,
3923
+ maxOutputBytes: 64 * 1024,
3924
+ admit,
3925
+ });
3926
+ if (!this.isLeasedChannel(channel)) {
3927
+ const running = whileOnOwnCredentials(async () => {
3928
+ const env = { ...(pickEngineAccount(kind, { scope: channel.engineAccountScope })?.execEnv ?? {}) };
3929
+ const r = await handle.exec(
3930
+ request(env, () => {
3931
+ if (!this.isActiveChannel(channel) || !onOwnCredentials()) throw new LeftOwnCredentialsError();
3932
+ }),
3933
+ );
3934
+ return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
3935
+ });
3936
+ if (running === null) return null;
3937
+ this.trackEventOperation(channel, running);
3938
+ return running;
3939
+ }
3940
+ const lease = leasedEnginesFor(channel.taskId);
3941
+ if (!lease?.open) return null;
3942
+ const account = this.leasedAccountFor(channel, agentId, kind, lease);
3943
+ if (!account) return null;
3944
+ const identity: LeasedIdentity = {
3945
+ subject: "account",
3946
+ runEpoch: lease.assignment.runEpoch,
3947
+ credentialId: account.credentialId,
3948
+ revision: account.revision,
3949
+ version: account.version,
3950
+ };
3951
+ let settle!: () => void;
3952
+ const entry = {
3953
+ agentId: `shadow:${agentId}`,
3954
+ accountId: account.credentialId,
3955
+ runEpoch: identity.runEpoch,
3956
+ settled: new Promise<void>((resolve) => (settle = resolve)),
3957
+ };
3958
+ channel.leasedCreating.add(entry);
3959
+ void entry.settled.then(() => channel.leasedCreating.delete(entry));
3960
+ this.trackEventOperation(channel, entry.settled);
3961
+ try {
3962
+ const env = await this.leasedOneShotEnv(channel, lease, account, handle);
3963
+ if (!env || !this.leasedMaterialServed(channel, identity)) return null;
3964
+ const r = await this.leasedLaunchEnvironment(channel, null, identity, handle).exec(
3965
+ request(env, () => {
3966
+ if (!this.isActiveChannel(channel) || !this.leasedMaterialServed(channel, identity)) {
3967
+ throw new CredentialLeaseUnavailableError("the shadow classifier's account moved");
3968
+ }
3969
+ }),
3970
+ );
3971
+ return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
3972
+ } finally {
3973
+ settle();
3974
+ }
3975
+ }
3976
+
3977
+ /** #cb3j0z: the task's runtime, for what only it can do in the task's
3978
+ * workspace without following its links (macOS); null without one. */
3979
+ private async workspaceRuntime(taskId: string): Promise<RuntimeNamespace | null> {
3980
+ const channel = this.channels.get(taskId);
3981
+ if (!channel || !this.isActiveChannel(channel)) return null;
3982
+ try {
3983
+ return runtimeNamespaceOf(await this.agentEnvironment(channel));
3984
+ } catch {
3985
+ return null;
3986
+ }
3987
+ }
3988
+
3989
+ /** Teardown drains `operation` before the task's runtime name can be
3990
+ * reused. */
3991
+ private trackEventOperation(channel: Channel, operation: Promise<unknown>): void {
3992
+ const settled = operation.then(
3993
+ () => undefined,
3994
+ () => undefined,
3995
+ );
3996
+ channel.eventOperations.add(settled);
3997
+ void settled.then(() => channel.eventOperations.delete(settled));
2565
3998
  }
2566
3999
 
2567
4000
  private hasDesiredLaunchSettings(
@@ -2808,6 +4241,9 @@ export class Orchestrator {
2808
4241
  ),
2809
4242
  },
2810
4243
  );
4244
+ // Credentials unavailable is this agent's alone: the gate's next
4245
+ // opening starts it; its siblings start now.
4246
+ if (!session && channel.leasedBlocked.has(agent.id)) continue;
2811
4247
  if (!session) {
2812
4248
  // The agents that did come up are usable; the reconcile pass keeps
2813
4249
  // retrying this one. Do not leave the page covered for a session
@@ -3203,13 +4639,16 @@ export class Orchestrator {
3203
4639
  // through this same command, and classifying them would poison the very
3204
4640
  // data the shadow pass exists to collect.
3205
4641
  if (channel.routingShadow && origin === "human") {
3206
- void this.shadowRouteDelivery(channel, agentId, text).catch((err) => {
4642
+ // Its exec runs in the task's runtime: teardown drains it like any
4643
+ // other external operation before the runtime's name can be reused.
4644
+ const pass = this.shadowRouteDelivery(channel, agentId, text).catch((err) => {
3207
4645
  console.warn(
3208
4646
  `[routing] shadow pass failed for ${channel.taskId}/${agentId}: ${
3209
4647
  err instanceof Error ? err.message : String(err)
3210
4648
  }`,
3211
4649
  );
3212
4650
  });
4651
+ this.trackEventOperation(channel, pass);
3213
4652
  }
3214
4653
  return { ok: true };
3215
4654
  }
@@ -3219,7 +4658,10 @@ export class Orchestrator {
3219
4658
  * one-shot executes INSIDE the task environment as the task user, with the
3220
4659
  * same engine account the agent runs on (so the user's own subscription
3221
4660
  * pays for routing, never Uai), then the result rides a host event. Bounded
3222
- * by a hard timeout so a hung CLI cannot pile up passes.
4661
+ * by a hard timeout, the task's own files read included, so a hung CLI or
4662
+ * read cannot pile up passes. ADR-154: the pass itself holds no credential
4663
+ * (its reads are the task's own files, which take as long as the task
4664
+ * makes them); only its one exec does (`shadowClassify`).
3223
4665
  */
3224
4666
  private async shadowRouteDelivery(channel: Channel, agentId: string, text: string): Promise<void> {
3225
4667
  const agent = channel.roster.find((a) => a.id === agentId);
@@ -3236,39 +4678,44 @@ export class Orchestrator {
3236
4678
  // Rung 1 hosts the classifier on the agent's own engine; the Managed
3237
4679
  // Engine delegates to claude/codex, so borrow the delegate's account.
3238
4680
  const hostEngine = agent.kind === "uai" ? (agent.uaiProvider === "openai" ? "codex" : "claude") : agent.kind;
3239
- const account = pickEngineAccount(hostEngine, { scope: channel.engineAccountScope });
3240
- const env: Record<string, string> = { ...(account?.execEnv ?? {}) };
3241
- const context =
3242
- handle && canRead
3243
- ? Buffer.from((await handle.readWorkspaceFile(`${scratchDir}/chat.md`).catch(() => null)) ?? new Uint8Array())
3244
- .toString("utf8")
3245
- .slice(-1200)
3246
- : "";
3247
4681
  const timeoutMs = 45_000;
4682
+ let timer: ReturnType<typeof setTimeout> | undefined;
4683
+ // Past its bound the pass is given up, and what is left of it (a read
4684
+ // the task held) launches nothing.
4685
+ let abandoned = false;
3248
4686
  const result = await Promise.race([
3249
- shadowRoute(text, context, hostEngine, {
3250
- scratchDir,
3251
- exec: async (argv) => {
3252
- if (!handle || !canExec) return null;
3253
- const r = await handle.exec({
3254
- argv: agentCliArgv(handle, argv),
3255
- projectEnvProfile: "task",
3256
- cwd: scratchDir,
3257
- user: "node",
3258
- env,
3259
- timeoutMs,
3260
- maxOutputBytes: 64 * 1024,
3261
- });
3262
- return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
3263
- },
3264
- readFile: async (path) => {
3265
- if (!handle || !canRead) return null;
3266
- const bytes = await handle.readWorkspaceFile(path).catch(() => null);
3267
- return bytes ? Buffer.from(bytes).toString("utf8") : null;
3268
- },
4687
+ (async () => {
4688
+ const context =
4689
+ handle && canRead
4690
+ ? Buffer.from(
4691
+ (await handle
4692
+ .readWorkspaceFile(`${scratchDir}/chat.md`, { maxBytes: SHADOW_CONTEXT_BYTES, from: "end" })
4693
+ .catch(() => null)) ?? new Uint8Array(),
4694
+ )
4695
+ .toString("utf8")
4696
+ .slice(-1200)
4697
+ : "";
4698
+ return shadowRoute(text, context, hostEngine, {
4699
+ scratchDir,
4700
+ exec: (argv) =>
4701
+ handle && canExec && !abandoned
4702
+ ? this.shadowClassify(channel, agentId, hostEngine, handle, argv, scratchDir, timeoutMs)
4703
+ : Promise.resolve(null),
4704
+ readFile: async (path) => {
4705
+ if (!handle || !canRead) return null;
4706
+ const bytes = await handle.readWorkspaceFile(path, { maxBytes: SHADOW_OUTPUT_BYTES }).catch(() => null);
4707
+ return bytes ? Buffer.from(bytes).toString("utf8") : null;
4708
+ },
4709
+ });
4710
+ })(),
4711
+ new Promise<null>((resolve) => {
4712
+ timer = setTimeout(() => {
4713
+ abandoned = true;
4714
+ resolve(null);
4715
+ }, timeoutMs + 5_000);
3269
4716
  }),
3270
- new Promise<null>((resolve) => setTimeout(() => resolve(null), timeoutMs + 5_000)),
3271
4717
  ]);
4718
+ clearTimeout(timer);
3272
4719
  if (!result) {
3273
4720
  console.warn(`[routing] shadow pass timed out for ${channel.taskId}/${agentId}`);
3274
4721
  return;
@@ -3507,14 +4954,21 @@ export class Orchestrator {
3507
4954
  if (ownsBrokerSlot) channel.spawning.add(agentId);
3508
4955
  let requested = false;
3509
4956
  try {
3510
- requested = await requestCredentialRefresh({
3511
- kind: agent.kind,
3512
- accountId:
3513
- accountId && !accountId.startsWith("default:")
3514
- ? accountId
3515
- : null,
3516
- scope: channel.engineAccountScope,
3517
- });
4957
+ // ADR-154 §7: a leased account is refreshed by the cloud's
4958
+ // broker, through the task's lease, never by this runner.
4959
+ const lease = this.isLeasedChannel(channel) ? leasedEnginesFor(channel.taskId) : undefined;
4960
+ requested = this.isLeasedChannel(channel)
4961
+ ? lease !== undefined &&
4962
+ accountId !== null &&
4963
+ (await refreshLeasedEngineAccount(lease.assignment, accountId)) === "refreshed"
4964
+ : await requestCredentialRefresh({
4965
+ kind: agent.kind,
4966
+ accountId:
4967
+ accountId && !accountId.startsWith("default:")
4968
+ ? accountId
4969
+ : null,
4970
+ scope: channel.engineAccountScope,
4971
+ });
3518
4972
  } catch (err) {
3519
4973
  console.warn(
3520
4974
  `[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
@@ -3825,10 +5279,16 @@ export class Orchestrator {
3825
5279
 
3826
5280
  channel.spawning.add(agentId);
3827
5281
  try {
3828
- const restored = await repairClaudeConfigInContainer(
3829
- channel.containerName,
3830
- channel.agentHandle ?? undefined,
3831
- );
5282
+ // ADR-154 §6: never the runner owner's own ~/.claude.json into a task on
5283
+ // cloud credentials; Claude writes its own again when it starts. A
5284
+ // repair on the runner's own credentials is counted: leaving them waits
5285
+ // for it, and none starts once leaving has begun.
5286
+ const repairing = this.isLeasedChannel(channel)
5287
+ ? null
5288
+ : whileOnOwnCredentials(() =>
5289
+ repairClaudeConfigInContainer(channel.containerName, channel.agentHandle ?? undefined),
5290
+ );
5291
+ const restored = repairing === null ? false : await repairing;
3832
5292
  if (channel.rosterSessionChanges.has(agentId)) {
3833
5293
  this.reconcileErroredRosterChange(
3834
5294
  channel,
@@ -3896,7 +5356,7 @@ export class Orchestrator {
3896
5356
  );
3897
5357
  const boundAccountId = channel.accountByAgent.get(agentId);
3898
5358
  const boundAccount = boundAccountId
3899
- ? resolveEngineAccounts(agent.kind, {}, channel.engineAccountScope).find(
5359
+ ? this.engineAccounts(channel, agent.kind).find(
3900
5360
  (account) => account.id === boundAccountId,
3901
5361
  )
3902
5362
  : undefined;
@@ -3942,9 +5402,11 @@ export class Orchestrator {
3942
5402
  }
3943
5403
  this.startNextPendingPrompt(channel, agentId);
3944
5404
 
3945
- const message = restored
3946
- ? `${agentId} restarted — config file was missing in the container, restored from the host.`
3947
- : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
5405
+ const message = this.isLeasedChannel(channel)
5406
+ ? `${agentId} restarted — config file was missing in the container; Claude writes its own again.`
5407
+ : restored
5408
+ ? `${agentId} restarted — config file was missing in the container, restored from the host.`
5409
+ : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
3948
5410
  channel.retiredTurnBoundaries.delete(agentId);
3949
5411
  this.emitHost({
3950
5412
  kind: "agent.exit",
@@ -4002,7 +5464,7 @@ export class Orchestrator {
4002
5464
  // A concurrent terminal event for the same session is already being
4003
5465
  // handled by the path that owns this spawn slot.
4004
5466
  if (channel.spawning.has(agentId)) return true;
4005
- if (resolveEngineAccounts(agent.kind, {}, channel.engineAccountScope).length < 2) {
5467
+ if (this.engineAccounts(channel, agent.kind).length < 2) {
4006
5468
  return false;
4007
5469
  }
4008
5470
 
@@ -4012,10 +5474,11 @@ export class Orchestrator {
4012
5474
 
4013
5475
  const current = channel.accountByAgent.get(agentId);
4014
5476
  if (current) cooldownEngineAccount(current);
4015
- const next = pickEngineAccount(agent.kind, {
4016
- scope: channel.engineAccountScope,
4017
- exclude: current ? new Set([current]) : undefined,
4018
- });
5477
+ const next = this.pickAccount(
5478
+ channel,
5479
+ agent.kind,
5480
+ current ? new Set([current]) : undefined,
5481
+ );
4019
5482
  if (!next) return false;
4020
5483
 
4021
5484
  const task = getHostTask(channel.taskId);
@@ -4447,7 +5910,7 @@ export class Orchestrator {
4447
5910
  return;
4448
5911
  }
4449
5912
 
4450
- const accounts = resolveEngineAccounts(agent.kind, {}, channel.engineAccountScope);
5913
+ const accounts = this.engineAccounts(channel, agent.kind);
4451
5914
  const account =
4452
5915
  accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
4453
5916
  accounts[0];
@@ -4482,14 +5945,19 @@ export class Orchestrator {
4482
5945
  async refreshEngineAccountCopies(kind: string): Promise<void> {
4483
5946
  for (const channel of this.channels.values()) {
4484
5947
  if (!this.isActiveChannel(channel)) continue;
5948
+ // ADR-154: a leased task's accounts change through its lease only.
5949
+ if (this.isLeasedChannel(channel)) continue;
4485
5950
  if (!channel.roster.some((agent) => agent.kind === kind)) continue;
4486
5951
  try {
4487
- await provisionEngineAccounts(
4488
- channel.containerName,
4489
- [kind],
4490
- {},
4491
- channel.agentHandle ?? undefined,
4492
- channel.engineAccountScope,
5952
+ // Only while the runner is on its own credentials, counted.
5953
+ await whileOnOwnCredentials(() =>
5954
+ provisionEngineAccounts(
5955
+ channel.containerName,
5956
+ [kind],
5957
+ {},
5958
+ channel.agentHandle ?? undefined,
5959
+ channel.engineAccountScope,
5960
+ ),
4493
5961
  );
4494
5962
  } catch (err) {
4495
5963
  console.warn(
@@ -4504,6 +5972,7 @@ export class Orchestrator {
4504
5972
  async refreshEngineAccountAgents(kind: string): Promise<void> {
4505
5973
  for (const channel of this.channels.values()) {
4506
5974
  if (!this.isActiveChannel(channel)) continue;
5975
+ if (this.isLeasedChannel(channel)) continue;
4507
5976
  const agentIds = channel.roster
4508
5977
  .filter((agent) => agent.kind === kind)
4509
5978
  .map((agent) => agent.id);
@@ -4561,7 +6030,16 @@ export class Orchestrator {
4561
6030
  // can repopulate an orphan channel while its sessions close.
4562
6031
  ch.closed = true;
4563
6032
  this.channels.delete(taskId);
6033
+ // ADR-154 §6: nothing its agents' spawns were admitted for starts after:
6034
+ // the close ends only once every retirement of the task under way, its
6035
+ // own and any an earlier event began, is done or escalated.
6036
+ for (const [agentId, slot] of [...ch.leasedLaunches]) void this.retireLeasedLaunch(ch, agentId, slot);
6037
+ const retirements = this.leasedRetirementsUnderWay(taskId);
4564
6038
  this.clearBrowserRetry(ch);
6039
+ if (ch.leasedSyncRetry) {
6040
+ clearTimeout(ch.leasedSyncRetry);
6041
+ ch.leasedSyncRetry = null;
6042
+ }
4565
6043
  ch.browserStaleSessions.clear();
4566
6044
  ch.rosterSessionChanges.clear();
4567
6045
  ch.browserPendingStaleSessions.clear();
@@ -4586,6 +6064,7 @@ export class Orchestrator {
4586
6064
  ch.sessionsReady,
4587
6065
  ...ch.factoryOperations,
4588
6066
  ...ch.eventOperations,
6067
+ ...retirements,
4589
6068
  ].filter((promise): promise is Promise<unknown> => promise !== null);
4590
6069
  const sessions = [...ch.sessions.values()];
4591
6070
  ch.sessions.clear();
@@ -4683,28 +6162,103 @@ export class Orchestrator {
4683
6162
  return this.#channelActivity.get(taskId) ?? null;
4684
6163
  }
4685
6164
 
4686
- async stopTask(taskId: string): Promise<{ ok: boolean; error?: string }> {
6165
+ async stopTask(
6166
+ taskId: string,
6167
+ options: {
6168
+ admitAssignment?: () => TaskStopAssignmentAdmission;
6169
+ shouldVeto?: () => boolean;
6170
+ /** ADR-154 §6: the closed run's credentials cleaned up, its sessions
6171
+ * closed and its runtime still up for the cleanup to reach, before
6172
+ * the runtime stops and Stop is acknowledged. */
6173
+ closeCredentials?: (runEpoch: number) => Promise<void>;
6174
+ } = {},
6175
+ ): Promise<OrchestratorTaskStopResult> {
4687
6176
  return this.runTaskLifecycle(taskId, () =>
4688
- this.stopTaskWithinLifecycle(taskId),
6177
+ this.stopTaskWithinLifecycle(taskId, options),
4689
6178
  );
4690
6179
  }
4691
6180
 
6181
+ /** `stopTask` for a caller that already holds this task's lifecycle slot
6182
+ * (runTaskLifecycle), so that what it checks before the stop and does
6183
+ * after it share the slot: ADR-154 §13's retirement re-proves the engine
6184
+ * first, and removes the runtime after. */
6185
+ async stopTaskInHeldLifecycle(
6186
+ taskId: string,
6187
+ options: {
6188
+ admitAssignment?: () => TaskStopAssignmentAdmission;
6189
+ shouldVeto?: () => boolean;
6190
+ closeCredentials?: (runEpoch: number) => Promise<void>;
6191
+ /** After the channel drained and before anything touches the
6192
+ * container runtime (previews, the runtime's stop): why the caller's
6193
+ * proof of the daemon no longer holds, or null. A reason stops the
6194
+ * Stop there, mutating nothing. */
6195
+ beforeRuntimeMutation?: () => Promise<string | null>;
6196
+ } = {},
6197
+ ): Promise<OrchestratorTaskStopResult> {
6198
+ return this.stopTaskWithinLifecycle(taskId, options);
6199
+ }
6200
+
4692
6201
  private async stopTaskWithinLifecycle(
4693
6202
  taskId: string,
4694
- ): Promise<{ ok: boolean; error?: string }> {
6203
+ options: {
6204
+ admitAssignment?: () => TaskStopAssignmentAdmission;
6205
+ shouldVeto?: () => boolean;
6206
+ closeCredentials?: (runEpoch: number) => Promise<void>;
6207
+ beforeRuntimeMutation?: () => Promise<string | null>;
6208
+ },
6209
+ ): Promise<OrchestratorTaskStopResult> {
6210
+ // The activity check belongs in the same lifecycle slot as assignment
6211
+ // closure and runtime stop. A turn may become active while Stop waits for
6212
+ // earlier lifecycle work; veto it before closing any v1 authority.
6213
+ if (options.shouldVeto?.()) return { ok: true, vetoed: true };
6214
+ let assignmentAdmission: TaskStopAssignmentAdmission = { kind: "legacy" };
6215
+ try {
6216
+ assignmentAdmission =
6217
+ options.admitAssignment?.() ?? ({ kind: "legacy" } as const);
6218
+ } catch (error) {
6219
+ return {
6220
+ ok: false,
6221
+ error: (error instanceof Error ? error.message : String(error)).slice(
6222
+ 0,
6223
+ 200,
6224
+ ),
6225
+ };
6226
+ }
6227
+ if (assignmentAdmission.kind === "stale") {
6228
+ return {
6229
+ ok: false,
6230
+ error: "task assignment is no longer current",
6231
+ assignmentStale: true,
6232
+ };
6233
+ }
6234
+ const assignmentClosed = assignmentAdmission.kind === "closed";
4695
6235
  if (!backgroundContainerWorkAllowed()) {
4696
6236
  return { ok: false, error: "container runtime is not operational" };
4697
6237
  }
4698
6238
  const task = getHostTask(taskId);
4699
6239
  if (!task) return { ok: false, error: "unknown task" };
4700
- const project = task.composeProject;
4701
- if (!project) return { ok: false, error: "task has no running stack" };
6240
+ // A runtime is named by its persisted locator, a legacy row by its
6241
+ // Compose project. ADR-154: a start interrupted after recording its
6242
+ // runtime's locator keeps it, with no Compose project, for Stop to reach.
6243
+ if (!task.environmentLocator && !task.composeProject) {
6244
+ return { ok: false, error: "task has no running stack" };
6245
+ }
4702
6246
  // Revoke env/proxy access before a potentially long session drain, then
4703
6247
  // close sessions so nothing races to respawn them mid-stop.
4704
6248
  const suspendedGateway = suspendTaskGatewayTask(taskId);
4705
6249
  // ADR-153: no longer served; end its open proxy connections too.
4706
6250
  activeSecretsProxy()?.runtime.stopTask(taskId);
4707
6251
  await this.closeChannel(taskId);
6252
+ // ADR-154 §13: a proof of the daemon taken before the drain may have
6253
+ // expired during it; nothing below may reach a replaced one.
6254
+ const stale = await options.beforeRuntimeMutation?.();
6255
+ if (stale !== undefined && stale !== null) {
6256
+ if (!assignmentClosed) {
6257
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
6258
+ this.allowChannel(taskId);
6259
+ }
6260
+ return { ok: false, error: stale.slice(0, 200) };
6261
+ }
4708
6262
  try {
4709
6263
  await stopPreviewSidecars(taskId);
4710
6264
  } catch (err) {
@@ -4714,6 +6268,22 @@ export class Orchestrator {
4714
6268
  }`,
4715
6269
  );
4716
6270
  }
6271
+ // ADR-154 §6: Stop is acknowledged only once the closed run's
6272
+ // credentials are cleaned up, while its runtime still runs for the
6273
+ // cleanup to reach. A failed cleanup fails the stop; the cloud retries,
6274
+ // and its closed run stays closed meanwhile.
6275
+ if (assignmentAdmission.kind === "closed" && options.closeCredentials) {
6276
+ try {
6277
+ await options.closeCredentials(assignmentAdmission.runEpoch);
6278
+ } catch (err) {
6279
+ return {
6280
+ ok: false,
6281
+ error: `the task's credentials could not be cleaned up: ${
6282
+ (err instanceof Error ? err.message : String(err)).slice(0, 160)
6283
+ }`,
6284
+ };
6285
+ }
6286
+ }
4717
6287
  // Provider-neutral resumable stop. A row that reaches this point with
4718
6288
  // runtime resources always carries a proven locator (legacy rows cross
4719
6289
  // the adoption fence before the runtime becomes operational), so a
@@ -4724,14 +6294,18 @@ export class Orchestrator {
4724
6294
  try {
4725
6295
  const environment = await reconstructPersistedTaskEnvironment(task);
4726
6296
  if (!environment) {
4727
- if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
4728
- this.allowChannel(taskId);
6297
+ if (!assignmentClosed) {
6298
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
6299
+ this.allowChannel(taskId);
6300
+ }
4729
6301
  return { ok: false, error: "task has no persisted environment" };
4730
6302
  }
4731
6303
  await environment.stop();
4732
6304
  } catch (err) {
4733
- if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
4734
- this.allowChannel(taskId);
6305
+ if (!assignmentClosed) {
6306
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
6307
+ this.allowChannel(taskId);
6308
+ }
4735
6309
  return {
4736
6310
  ok: false,
4737
6311
  error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
@@ -4757,7 +6331,9 @@ export class Orchestrator {
4757
6331
  // Nothing serves the task's env now: no value of it stays in memory.
4758
6332
  forgetTaskEnvSnapshot(taskId);
4759
6333
  clearTaskGatewayAcl(taskId);
4760
- return { ok: true };
6334
+ return assignmentAdmission.kind === "closed"
6335
+ ? { ok: true, stoppedRunEpoch: assignmentAdmission.runEpoch }
6336
+ : { ok: true };
4761
6337
  }
4762
6338
  }
4763
6339
 
@@ -6001,8 +7577,26 @@ export type TaskRuntimeContainerInspection =
6001
7577
 
6002
7578
  export interface WritableRuntimeQuarantineResult {
6003
7579
  verdict: "ready" | "retry" | "stale";
6004
- /** Known rows whose local `stopped` verdict must be published or replayed. */
6005
- stoppedTaskIds: string[];
7580
+ /**
7581
+ * Known rows whose local `stopped` verdict must be published or replayed.
7582
+ * The epoch is captured while the task lifecycle slot is still held and is
7583
+ * deliberately absent for a legacy assignment.
7584
+ */
7585
+ stoppedTasks: StoppedTaskStatusTarget[];
7586
+ }
7587
+
7588
+ export interface StoppedTaskStatusTarget {
7589
+ taskId: string;
7590
+ runEpoch?: number;
7591
+ }
7592
+
7593
+ function captureStoppedTaskStatusTarget(
7594
+ taskId: string,
7595
+ ): StoppedTaskStatusTarget {
7596
+ const assignment = closeCurrentTaskRunAssignmentForStoppedRuntime(taskId);
7597
+ return assignment.kind === "legacy"
7598
+ ? { taskId }
7599
+ : { taskId, runEpoch: assignment.runEpoch };
6006
7600
  }
6007
7601
 
6008
7602
  function runtimeWriterCandidate(
@@ -6390,19 +7984,19 @@ export async function quarantineWritableRuntimeContainers(options: {
6390
7984
  return quarantineWritableAppleRuntimeContainers(options);
6391
7985
  }
6392
7986
  const isCurrent = options.isCurrent ?? (() => true);
6393
- const stoppedTaskIds = new Set<string>();
6394
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
7987
+ const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
7988
+ if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
6395
7989
 
6396
7990
  const listed = await listRuntimeWriterCandidates();
6397
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
7991
+ if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
6398
7992
  if (listed.kind === "unreachable") {
6399
- return { verdict: "retry", stoppedTaskIds: [] };
7993
+ return { verdict: "retry", stoppedTasks: [] };
6400
7994
  }
6401
7995
  let resolved = !listed.ambiguous;
6402
7996
 
6403
7997
  for (const candidate of listed.candidates) {
6404
7998
  if (!isCurrent()) {
6405
- return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
7999
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
6406
8000
  }
6407
8001
  await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
6408
8002
  if (!isCurrent()) return;
@@ -6459,7 +8053,10 @@ export async function quarantineWritableRuntimeContainers(options: {
6459
8053
  // Always re-publish the stopped verdict. A prior process may have
6460
8054
  // committed SQLite and crashed before its websocket frame left; the
6461
8055
  // cloud could otherwise remain ghost-running indefinitely.
6462
- stoppedTaskIds.add(candidate.taskId);
8056
+ stoppedTasks.set(
8057
+ candidate.taskId,
8058
+ captureStoppedTaskStatusTarget(candidate.taskId),
8059
+ );
6463
8060
  db_setStatus(candidate.taskId, "stopped", {
6464
8061
  codeServerPort: null,
6465
8062
  previewPorts: "[]",
@@ -6488,11 +8085,13 @@ export async function quarantineWritableRuntimeContainers(options: {
6488
8085
  }
6489
8086
 
6490
8087
  if (!isCurrent()) {
6491
- return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
8088
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
6492
8089
  }
6493
8090
  return {
6494
8091
  verdict: resolved ? "ready" : "retry",
6495
- stoppedTaskIds: [...stoppedTaskIds].sort(),
8092
+ stoppedTasks: [...stoppedTasks.values()].sort((a, b) =>
8093
+ a.taskId.localeCompare(b.taskId),
8094
+ ),
6496
8095
  };
6497
8096
  }
6498
8097
 
@@ -7282,25 +8881,25 @@ async function quarantineWritableAppleRuntimeContainers(options: {
7282
8881
  scope?: "contract" | "volume-holders";
7283
8882
  }): Promise<WritableRuntimeQuarantineResult> {
7284
8883
  const isCurrent = options.isCurrent ?? (() => true);
7285
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
8884
+ if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
7286
8885
  const binding = appleContainerRuntimeBinding();
7287
- if (binding === null) return { verdict: "retry", stoppedTaskIds: [] };
8886
+ if (binding === null) return { verdict: "retry", stoppedTasks: [] };
7288
8887
  const cli = appleCliRunner(() => binding.containerCliPath);
7289
8888
  const listed = await cli(["list", "--all", "--format", "json"], {
7290
8889
  timeoutMs: 30_000,
7291
8890
  maxOutputBytes: 8 * 1024 * 1024,
7292
8891
  });
7293
8892
  if (listed.status !== 0 || listed.truncated === true) {
7294
- return { verdict: "retry", stoppedTaskIds: [] };
8893
+ return { verdict: "retry", stoppedTasks: [] };
7295
8894
  }
7296
8895
  let parsed: unknown;
7297
8896
  try {
7298
8897
  parsed = JSON.parse(listed.stdout);
7299
8898
  } catch {
7300
- return { verdict: "retry", stoppedTaskIds: [] };
8899
+ return { verdict: "retry", stoppedTasks: [] };
7301
8900
  }
7302
- if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTaskIds: [] };
7303
- const stoppedTaskIds = new Set<string>();
8901
+ if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTasks: [] };
8902
+ const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
7304
8903
  let resolved = true;
7305
8904
 
7306
8905
  const stopAndProve = async (id: string): Promise<boolean> => {
@@ -7357,7 +8956,9 @@ async function quarantineWritableAppleRuntimeContainers(options: {
7357
8956
  }>;
7358
8957
  };
7359
8958
  }>) {
7360
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
8959
+ if (!isCurrent()) {
8960
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
8961
+ }
7361
8962
  const id = item?.configuration?.id;
7362
8963
  if (typeof id !== "string" || id.length === 0) {
7363
8964
  resolved = false;
@@ -7410,20 +9011,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
7410
9011
  }
7411
9012
  const task = getHostTask(taskId);
7412
9013
  if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
9014
+ const stoppedTask = captureStoppedTaskStatusTarget(taskId);
7413
9015
  db_setStatus(taskId, "stopped", {
7414
9016
  codeServerPort: null,
7415
9017
  previewPorts: "[]",
7416
9018
  });
7417
- stoppedTaskIds.add(taskId);
9019
+ stoppedTasks.set(taskId, stoppedTask);
7418
9020
  }
7419
9021
  // Revoke live access with the environment, like the Docker sweep.
7420
9022
  clearTaskGatewayAcl(taskId);
7421
9023
  });
7422
9024
  }
7423
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
9025
+ if (!isCurrent()) {
9026
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
9027
+ }
7424
9028
  return {
7425
9029
  verdict: resolved ? "ready" : "retry",
7426
- stoppedTaskIds: [...stoppedTaskIds],
9030
+ stoppedTasks: [...stoppedTasks.values()],
7427
9031
  };
7428
9032
  }
7429
9033
 
@@ -7477,7 +9081,7 @@ async function materializeAgentCli(
7477
9081
  return false;
7478
9082
  }
7479
9083
  }
7480
- return writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
9084
+ return await writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
7481
9085
  }
7482
9086
 
7483
9087
  async function recoverPersistedTaskEnvironment(
@@ -8314,6 +9918,17 @@ async function recoverOneDockerTask(
8314
9918
  lifecycle?: RecoveryTaskConsumers,
8315
9919
  ): Promise<boolean> {
8316
9920
  if (!isCurrent()) return true;
9921
+ // Recovery is sanctioned private maintenance while public admission is
9922
+ // deliberately `checking`. Its container work is fenced by this daemon
9923
+ // epoch instead of the public operational bit: work it starts keeps going
9924
+ // while the generation is current, and a superseded one stops.
9925
+ const admitRecoveryContainerWork = (): void => {
9926
+ if (!isCurrent()) {
9927
+ throw new ContainerRuntimeUnavailableError(
9928
+ "container runtime recovery generation was superseded",
9929
+ );
9930
+ }
9931
+ };
8317
9932
  const composeProject = task.composeProject;
8318
9933
  if (!composeProject) {
8319
9934
  // We never wrote a compose project name for this row — must be
@@ -8463,7 +10078,9 @@ async function recoverOneDockerTask(
8463
10078
  // ~8h later. Cheap with the per-user access-token cache — one exchange
8464
10079
  // per user, every task re-injects from it. Best-effort.
8465
10080
  if (task.ownerUserId) {
8466
- void reconcileTaskGitAuth(task.taskId, task.ownerUserId).catch((err) =>
10081
+ void reconcileTaskGitAuth(task.taskId, task.ownerUserId, {
10082
+ admitContainerWork: admitRecoveryContainerWork,
10083
+ }).catch((err) =>
8467
10084
  console.warn(
8468
10085
  `[github] task ${task.taskId}: running recovery reconciliation failed: ${
8469
10086
  err instanceof Error ? err.message : String(err)
@@ -8591,7 +10208,9 @@ async function recoverOneDockerTask(
8591
10208
  // reinject sweep only targets containers already running, so a container
8592
10209
  // recovered here would otherwise keep whatever it held when it exited —
8593
10210
  // possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
8594
- // by the inject itself and must not abort the task's recovery.
10211
+ // by the inject itself and must not abort the task's recovery. ADR-154:
10212
+ // the runner owner's own sign-in, so only while the runner is on its own
10213
+ // credentials (the inject fences itself).
8595
10214
  await injectCodexIntoContainer(containerName);
8596
10215
  if (!isCurrent()) return true;
8597
10216
  // Establish the task owner's GitHub transport BEFORE uai-init performs any
@@ -8602,17 +10221,9 @@ async function recoverOneDockerTask(
8602
10221
  if (task.ownerUserId) {
8603
10222
  try {
8604
10223
  await reconcileTaskGitAuth(task.taskId, task.ownerUserId, {
8605
- // Recovery is sanctioned private maintenance while public admission is
8606
- // deliberately `checking`. Fence it by this daemon epoch instead of
8607
- // the public operational bit, preserving private-dependency setup
8608
- // before uai-init without letting stale recovery keep mutating.
8609
- admitContainerWork: () => {
8610
- if (!isCurrent()) {
8611
- throw new ContainerRuntimeUnavailableError(
8612
- "container runtime recovery generation was superseded",
8613
- );
8614
- }
8615
- },
10224
+ // Before uai-init, so private dependencies resolve; fenced by this
10225
+ // recovery's epoch, not the public operational bit.
10226
+ admitContainerWork: admitRecoveryContainerWork,
8616
10227
  });
8617
10228
  if (!isCurrent()) return true;
8618
10229
  } catch (err) {