@runuai/host 0.9.150 → 0.9.152

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
  2. package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
  3. package/db/migrations/meta/_journal.json +14 -0
  4. package/db/schema.ts +188 -0
  5. package/lib/agent-cli.ts +11 -13
  6. package/lib/agent.ts +177 -7
  7. package/lib/agents/durable-proc.ts +56 -44
  8. package/lib/agents/registry.ts +19 -3
  9. package/lib/agents/session-state.ts +23 -10
  10. package/lib/agents/transport.ts +101 -69
  11. package/lib/apple-one-off.ts +277 -0
  12. package/lib/apple-runtime-recycle.ts +4 -3
  13. package/lib/attachments.ts +19 -17
  14. package/lib/capture-flow.ts +319 -0
  15. package/lib/cloud-credential-cleanup-adapter.ts +123 -0
  16. package/lib/cloud-credential-host.ts +512 -0
  17. package/lib/cloud-credential-leases.ts +2544 -0
  18. package/lib/cloud-credential-runtime.ts +1035 -0
  19. package/lib/codex-auth.ts +21 -1
  20. package/lib/container-runtime.ts +76 -0
  21. package/lib/credential-cleanup-inventory.ts +405 -0
  22. package/lib/credential-consumers/engine-anchor.ts +1880 -0
  23. package/lib/credential-consumers/engines.ts +773 -0
  24. package/lib/credential-consumers/github.ts +594 -0
  25. package/lib/credential-consumers/mcp.ts +384 -0
  26. package/lib/credential-consumers/task-files.ts +127 -0
  27. package/lib/credential-store-pickup.ts +15 -2
  28. package/lib/credential-store.ts +361 -33
  29. package/lib/credentials-activation-authority.ts +546 -0
  30. package/lib/credentials-purge.ts +1006 -0
  31. package/lib/db.ts +11 -0
  32. package/lib/docker-exec.ts +3 -1
  33. package/lib/engine-accounts.ts +42 -32
  34. package/lib/engine-connect.ts +47 -2
  35. package/lib/engine-credential-env.ts +50 -0
  36. package/lib/engine-login.ts +83 -1
  37. package/lib/env-file.ts +62 -8
  38. package/lib/env.ts +5 -0
  39. package/lib/fs-aliases.ts +49 -0
  40. package/lib/git-clone-credentials.ts +334 -0
  41. package/lib/git-diff.ts +219 -177
  42. package/lib/git-identity.ts +59 -14
  43. package/lib/github-origin.ts +28 -0
  44. package/lib/github-tokens.ts +56 -21
  45. package/lib/host-env.ts +17 -4
  46. package/lib/launch-mode.ts +25 -0
  47. package/lib/legacy-runtime-retirement.ts +388 -0
  48. package/lib/machine-exec.ts +14 -3
  49. package/lib/machine-provider-aws.ts +50 -8
  50. package/lib/machine-provider-local.ts +4 -0
  51. package/lib/machine-provider.ts +4 -0
  52. package/lib/managed-operation-drain.ts +8 -0
  53. package/lib/managed-runtime.ts +371 -7
  54. package/lib/mcp-capture.ts +860 -0
  55. package/lib/mcp-connections.ts +8 -1
  56. package/lib/mcp-gateway.ts +168 -10
  57. package/lib/obs.ts +2 -6
  58. package/lib/orchestrator.ts +1837 -224
  59. package/lib/person-capture.ts +327 -0
  60. package/lib/project-env-legacy-tags.ts +130 -0
  61. package/lib/project-env-task-replica.ts +1752 -0
  62. package/lib/runner-boot.ts +10 -0
  63. package/lib/runtime-activation-tail.ts +50 -3
  64. package/lib/runtime-archive.ts +68 -1
  65. package/lib/runtime-reconnect.ts +86 -0
  66. package/lib/runtime-state.ts +429 -13
  67. package/lib/safe-http.ts +364 -0
  68. package/lib/secrets-proxy/task-trust.ts +34 -3
  69. package/lib/sidecar-resume.ts +18 -5
  70. package/lib/skills.ts +47 -8
  71. package/lib/standard-image.ts +157 -103
  72. package/lib/stopped-task-status-queue.ts +68 -10
  73. package/lib/task-credential-cleanup.ts +525 -0
  74. package/lib/task-diff-sandbox.ts +464 -0
  75. package/lib/task-diff.ts +66 -163
  76. package/lib/task-env-source.ts +68 -14
  77. package/lib/task-environment/anchor.ts +31 -0
  78. package/lib/task-environment/apple-container.ts +69 -28
  79. package/lib/task-environment/docker.ts +101 -20
  80. package/lib/task-environment/machine-environment.ts +61 -0
  81. package/lib/task-environment/machine-task-up.ts +459 -96
  82. package/lib/task-environment/machine.ts +101 -11
  83. package/lib/task-environment/self.ts +108 -50
  84. package/lib/task-environment/types.ts +89 -5
  85. package/lib/task-environment/workspace-files.ts +650 -25
  86. package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
  87. package/lib/task-gateway-recovery.ts +4 -1
  88. package/lib/task-gateway-state.ts +15 -0
  89. package/lib/task-machine-swap-safety.ts +729 -0
  90. package/lib/token-broker-adopt.ts +344 -0
  91. package/lib/token-broker-lambda.ts +56 -2
  92. package/lib/token-broker-local.ts +28 -4
  93. package/lib/token-broker-revision.ts +310 -0
  94. package/lib/token-broker-ssm.ts +368 -28
  95. package/lib/token-lanes/openai-oauth.ts +7 -3
  96. package/lib/token-lanes/xai-oauth.ts +7 -3
  97. package/lib/transcript.ts +9 -5
  98. package/package.json +6 -2
  99. package/scripts/agent/task-up.sh +242 -100
  100. package/src/cloud-credentials.ts +2399 -0
  101. package/src/event-outbox.ts +8 -6
  102. package/src/index.ts +1386 -875
  103. package/src/main.ts +1061 -139
  104. package/src/outbound-frame.ts +31 -0
  105. package/src/project-env-policy.ts +125 -10
  106. package/src/protocol.ts +699 -23
  107. package/src/ui/server.ts +48 -10
@@ -51,6 +51,7 @@ import { isSafeHostTaskId } from "./task-identity";
51
51
  import { projectEnvForTask } from "./host-env";
52
52
  import { shadowRoute } from "./routing/shadow";
53
53
  import {
54
+ closeCurrentTaskRunAssignmentForStoppedRuntime,
54
55
  getHostTask,
55
56
  hostTaskEnvMode,
56
57
  hostTaskProjectIds,
@@ -79,6 +80,7 @@ import {
79
80
  } from "./container-runtime";
80
81
  import {
81
82
  reconstructPersistedTaskEnvironment,
83
+ reconstructTaskEnvironment,
82
84
  setAppleTaskEnvironmentRecoveryDriver,
83
85
  setDockerTaskEnvironmentRecoveryDriver,
84
86
  } from "./task-environment";
@@ -137,6 +139,14 @@ import {
137
139
  type BrowserSetup,
138
140
  } from "./browser-testing";
139
141
  import { injectCodexIntoContainer } from "./codex-auth";
142
+ import { runtimeNamespaceOf, type RuntimeNamespace } from "./task-environment/workspace-files";
143
+ import {
144
+ holdOwnCredentials,
145
+ LeftOwnCredentialsError,
146
+ onOwnCredentials,
147
+ onOwnCredentialsEnvironment,
148
+ whileOnOwnCredentials,
149
+ } from "./cloud-credential-host";
140
150
  import { isCredentialInvalidSignal, requestCredentialRefresh } from "./token-broker-local";
141
151
  import {
142
152
  codexHomeIsolatable,
@@ -147,8 +157,51 @@ import {
147
157
  pickEngineAccount,
148
158
  provisionEngineAccounts,
149
159
  resolveEngineAccounts,
160
+ selectAccount,
150
161
  type ResolvedEngineAccount,
151
162
  } from "./engine-accounts";
163
+ import {
164
+ ENGINE_ACCOUNT_HOMES,
165
+ EngineCredentialWriteRefused,
166
+ engineSpawnEnv,
167
+ engineVersionHeldBack,
168
+ ensureEngineHome,
169
+ escalateEngineLaunchFence,
170
+ isLeasedEngineAccount,
171
+ leasedEnginesFor,
172
+ onLeasedEnginesChanged,
173
+ refreshLeasedEngineAccount,
174
+ setEngineAdmissionFloorRaiser,
175
+ syncEngineHomes,
176
+ waitForEngineGate,
177
+ type LeasedEngineAccount,
178
+ type TaskEngineLease,
179
+ } from "./credential-consumers/engines";
180
+ import {
181
+ admitEngineLaunch,
182
+ anchoredEnvironment,
183
+ dischargeEngineRevocations,
184
+ ENGINE_ACCOUNT_ENV,
185
+ ENGINE_RUN_ENV,
186
+ EngineAnchorLost,
187
+ EngineLaunchRefused,
188
+ newEngineLaunchSlotId,
189
+ raiseEngineAdmissionFloors,
190
+ retireEngineLaunchSlot,
191
+ sameRuntime,
192
+ stopAnchoredEngines,
193
+ type EngineAdmissionFloor,
194
+ type EngineLaunchIdentity,
195
+ } from "./credential-consumers/engine-anchor";
196
+ import { CredentialLeaseUnavailableError } from "./cloud-credential-leases";
197
+ import type { TaskCredentialCleanupRequest } from "./task-credential-cleanup";
198
+ import {
199
+ forgetLeasedEngineRuntimes,
200
+ listCredentialCleanupPaths,
201
+ listLeasedEngineRuntimes,
202
+ rememberLeasedEngineRuntime,
203
+ } from "./credential-cleanup-inventory";
204
+ import { ENGINE_CREDENTIAL_FILES, ENGINE_ENV_CREDENTIAL_KEYS } from "../src/cloud-credentials";
152
205
  import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
153
206
  import {
154
207
  reactivateTaskGatewayTask,
@@ -179,7 +232,11 @@ import {
179
232
  MAX_SECRETARY_DISPATCH_ID_CHARS,
180
233
  SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
181
234
  } from "../src/protocol";
182
- import { isEnvironmentTransportProvider } from "./task-environment/types";
235
+ import {
236
+ isEnvironmentTransportProvider,
237
+ parseTaskEnvironmentLocator,
238
+ serializeTaskEnvironmentLocator,
239
+ } from "./task-environment/types";
183
240
  import { forgetTaskEnvSnapshot } from "./task-env-source";
184
241
 
185
242
  export type HostEventSubscriber = (event: HostEvent) => void;
@@ -385,6 +442,41 @@ interface Channel {
385
442
  /** ADR-076: per-agent account-rotation budget (reset on a successful turn) —
386
443
  * bounds a rate-limit storm across accounts, mirroring the respawn budget. */
387
444
  rotations: Map<string, number>;
445
+ /** ADR-154: the task's AI accounts come from its lease. Sticky for the
446
+ * channel: once on cloud credentials, never the runner's own accounts. */
447
+ leased: boolean;
448
+ /** ADR-154 §8: rewrites of the task's account homes after a change, one
449
+ * after another. */
450
+ leasedSync: Promise<void> | null;
451
+ /** ADR-154 §6: agents a cleanup stopped, started again on their next
452
+ * account once the task's gate opens. */
453
+ leasedRespawn: Set<string>;
454
+ /** ADR-154 §6: the exact lease material each agent's session was picked
455
+ * on, so a spawn starts only on what the task still holds and a cleanup
456
+ * of an older run never stops a newer run's agent. */
457
+ leasedIdentityByAgent: Map<string, LeasedIdentity>;
458
+ /** ADR-154 §5: agents down for want of a usable account; the gate's next
459
+ * opening starts them again. */
460
+ leasedBlocked: Set<string>;
461
+ /** ADR-154 §8: a failed rewrite of the account homes, retried. */
462
+ leasedSyncRetry: ReturnType<typeof setTimeout> | null;
463
+ /** ADR-154 §5, §6: agents wait to be started again (stopped by a cleanup,
464
+ * down for want of an account, or their spawn's material moved); kept
465
+ * until a start pass actually ran, so a failed pass does not lose it. */
466
+ leasedWakePending: boolean;
467
+ /** ADR-154 §6: engines being created on a leased account (null: on an
468
+ * agent's task token alone, the Managed Engine) right now, each until its
469
+ * create settles. Every cleanup attempt on the account (on every one, for
470
+ * a whole run's) waits for them before it proves the engines gone,
471
+ * whatever has become of the agent's binding meanwhile. */
472
+ leasedCreating: Set<{
473
+ agentId: string;
474
+ accountId: string | null;
475
+ runEpoch: number;
476
+ settled: Promise<void>;
477
+ }>;
478
+ /** ADR-154 §6: each agent's current spawn of a leased engine. */
479
+ leasedLaunches: Map<string, LeasedLaunchSlot>;
388
480
  }
389
481
 
390
482
  interface AcceptedPrompt {
@@ -405,6 +497,119 @@ interface RecoveryFence {
405
497
  const MAX_RESPAWNS_PER_AGENT = 5;
406
498
  /** ADR-076: hard cap on account rotations per agent between successful turns. */
407
499
  const MAX_ROTATIONS_PER_AGENT = 6;
500
+ /** ADR-154 §5: how long a spawn waits for the task's leased AI accounts
501
+ * before the agent fails with "credentials unavailable". */
502
+ const LEASED_ACCOUNT_WAIT_MS = 60_000;
503
+ /** ADR-154 §6: how long the cleanup's quiesce step waits for a spawn
504
+ * already creating an engine on a removed account. */
505
+ const LEASED_QUIESCE_EXIT_MS = 20_000;
506
+ /** ADR-141: the shadow router reads the tail of the task's chat (it keeps
507
+ * 1,200 characters) and its classifier's answer, never more. */
508
+ const SHADOW_CONTEXT_BYTES = 8 * 1024;
509
+ const SHADOW_OUTPUT_BYTES = 64 * 1024;
510
+ /** ADR-154 §6: the markers every process of a leased engine inherits: its
511
+ * run and its account. A launch carrying them is an engine launch, started
512
+ * only under the engine supervisor. */
513
+ const LEASED_RUN_ENV = ENGINE_RUN_ENV;
514
+ const LEASED_ACCOUNT_ENV = ENGINE_ACCOUNT_ENV;
515
+ /**
516
+ * ADR-154 §6: every variable an env-token account may set, blank. A leased
517
+ * spawn carries these under its account's own values: an explicit blank
518
+ * wins over an adapter's inheritance of the runner's variable (Claude's
519
+ * passEnv, Grok's XAI_API_KEY, Cursor's fallback) and over the container's
520
+ * env, so no engine ever runs on the runner's own credential beside, or
521
+ * instead of, the task's.
522
+ */
523
+ const LEASED_CREDENTIAL_BLANKS: Readonly<Record<string, string>> = Object.freeze(
524
+ Object.fromEntries(
525
+ [...new Set(Object.values(ENGINE_ENV_CREDENTIAL_KEYS).flat())].map((name) => [name, ""]),
526
+ ),
527
+ );
528
+ /** ADR-128: the Managed Engine. It needs no AI account (it runs on Uai's
529
+ * gateway), but its agent's task token is its bearer: on a leased task its
530
+ * launches are admitted as that token (ADR-154 §6). */
531
+ const MANAGED_ENGINE_KIND = "uai";
532
+ /** ADR-154 §10: kinds whose agents carry no credential at all; every other
533
+ * kind outside the leased lanes (Kimi) stays down on a leased task until
534
+ * its lane exists. */
535
+ const CREDENTIAL_FREE_KINDS: ReadonlySet<string> = new Set(["mock"]);
536
+ /** ADR-154 §8: how soon a failed rewrite of a task's account homes is tried
537
+ * again. */
538
+ const LEASED_SYNC_RETRY_MS = 30_000;
539
+
540
+ /** ADR-154 §6: the exact lease material a spawn starts on: an AI account's,
541
+ * or, for the Managed Engine, the task's open run alone (its agent's task
542
+ * token is its credential). */
543
+ type LeasedIdentity =
544
+ | { subject: "account"; runEpoch: number; credentialId: string; revision: string; version: number }
545
+ | { subject: "task-token"; runEpoch: number };
546
+
547
+ type SpawnArgs = Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">;
548
+
549
+ /**
550
+ * ADR-154 §6: one spawn of an agent's leased engine. Its launches are
551
+ * admitted only while it is the agent's current spawn and, once its create
552
+ * settled, only while the session it bound is the agent's live one: a spawn
553
+ * replaced (on the same material too), retired, removed, never bound, or
554
+ * whose session reported its end, launches nothing more, and once retired,
555
+ * nothing it was admitted for starts in its runtime (`retireLeasedLaunch`).
556
+ */
557
+ interface LeasedLaunchSlot {
558
+ created: boolean;
559
+ session: AgentSession | null;
560
+ /** What its launches are admitted as, its slot's id included, and where. */
561
+ readonly identity: EngineLaunchIdentity & { slotId: string };
562
+ readonly environment: TaskEnvironmentHandle<unknown>;
563
+ /** It began writing an admission: its retirement reaches its runtime. */
564
+ admitted: boolean;
565
+ retired: boolean;
566
+ }
567
+
568
+ /** ADR-154 §6: a retired slot not yet retired in its runtime. */
569
+ interface RetiredLaunchSlot {
570
+ readonly agentId: string;
571
+ readonly slot: LeasedLaunchSlot;
572
+ /** The attempt under way, if one is. */
573
+ attempt: Promise<void> | null;
574
+ /** A cleanup of its run that covers it proves what it fences: a failure
575
+ * after is not escalated. */
576
+ covered: boolean;
577
+ /** Its run's full cleanup was asked for once an attempt failed. */
578
+ escalated: boolean;
579
+ }
580
+
581
+ function newLeasedLaunchSlot(
582
+ taskId: string,
583
+ identity: LeasedIdentity,
584
+ environment: TaskEnvironmentHandle<unknown>,
585
+ ): LeasedLaunchSlot {
586
+ return {
587
+ created: false,
588
+ session: null,
589
+ identity: { taskId, slotId: newEngineLaunchSlotId(), ...identity },
590
+ environment,
591
+ admitted: false,
592
+ retired: false,
593
+ };
594
+ }
595
+
596
+ function sameLeasedIdentity(a: LeasedIdentity, b: LeasedIdentity): boolean {
597
+ if (a.subject === "account" && b.subject === "account") {
598
+ return (
599
+ a.runEpoch === b.runEpoch &&
600
+ a.credentialId === b.credentialId &&
601
+ a.revision === b.revision &&
602
+ a.version === b.version
603
+ );
604
+ }
605
+ return a.subject === b.subject && a.runEpoch === b.runEpoch;
606
+ }
607
+
608
+ /** The engine kinds whose agents run on an AI account from the lease. */
609
+ const LEASED_ENGINE_KINDS: ReadonlySet<string> = new Set([
610
+ ...Object.keys(ENGINE_ENV_CREDENTIAL_KEYS),
611
+ ...Object.keys(ENGINE_CREDENTIAL_FILES),
612
+ ]);
408
613
  /** A burned respawn budget resets after this quiet period (see reconcile). */
409
614
  const RESPAWN_COOLDOWN_MS = 10 * 60_000;
410
615
  /** ADR-053: gap between browser-install retries. Ensures fire every couple of
@@ -504,6 +709,20 @@ async function sshIdentityDepsFor(task: {
504
709
  return { environment: () => environment };
505
710
  }
506
711
 
712
+ export type TaskStopAssignmentAdmission =
713
+ | { kind: "legacy" }
714
+ | { kind: "closed"; runEpoch: number }
715
+ | { kind: "stale" };
716
+
717
+ export interface OrchestratorTaskStopResult {
718
+ ok: boolean;
719
+ error?: string;
720
+ assignmentStale?: boolean;
721
+ vetoed?: boolean;
722
+ /** Exact v1 run stopped under the lifecycle lock. Absent for legacy. */
723
+ stoppedRunEpoch?: number;
724
+ }
725
+
507
726
  export class Orchestrator {
508
727
  private readonly channels = new Map<string, Channel>();
509
728
  private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
@@ -517,8 +736,15 @@ export class Orchestrator {
517
736
  * later command cannot overtake an earlier one after channel close. */
518
737
  private readonly taskLifecycleTails = new Map<string, Promise<void>>();
519
738
  private readonly hostSubscribers = new Set<HostEventSubscriber>();
739
+ /** ADR-154 §6: launch slots retired whose retirement has not yet reached
740
+ * their runtime, per task, whatever channel they were in: an agent's next
741
+ * spawn launches there only once they have (`retireLeasedLaunch`). */
742
+ private readonly leasedRetired = new Map<string, Set<RetiredLaunchSlot>>();
520
743
 
521
- constructor(private readonly factory: AgentSessionFactory) {}
744
+ constructor(private readonly factory: AgentSessionFactory) {
745
+ onLeasedEnginesChanged((taskId) => this.leasedEnginesChanged(taskId));
746
+ setEngineAdmissionFloorRaiser((assignment, floors) => this.raiseLeasedAdmissionFloors(assignment, floors));
747
+ }
522
748
 
523
749
  // -- subscriptions --------------------------------------------------------
524
750
 
@@ -645,7 +871,7 @@ export class Orchestrator {
645
871
  for (const agent of added) {
646
872
  // Per-agent link/document skills, like getOrCreateChannel does at
647
873
  // channel birth. Best-effort.
648
- writeAgentSkills(channel.taskId, agent);
874
+ writeAgentSkills(channel.taskId, agent, () => this.workspaceRuntime(channel.taskId));
649
875
  }
650
876
  for (const { agent, reason } of changed) {
651
877
  const recoveryAbortPublished = this.supersedeRecovery(
@@ -665,6 +891,9 @@ export class Orchestrator {
665
891
  );
666
892
  }
667
893
  channel.accountByAgent.delete(agent.id);
894
+ // ADR-154 §6: the retired spawn launches nothing more on its material.
895
+ channel.leasedIdentityByAgent.delete(agent.id);
896
+ void this.retireLeasedLaunch(channel, agent.id, channel.leasedLaunches.get(agent.id));
668
897
  channel.rotations.delete(agent.id);
669
898
  // Crash/recovery budgets belong to the retired launch generation. A
670
899
  // newly selected engine/model/effort must get an immediate clean start,
@@ -814,6 +1043,9 @@ export class Orchestrator {
814
1043
  channel.respawnLastAt.delete(agentId);
815
1044
  channel.everBound.delete(agentId);
816
1045
  channel.accountByAgent.delete(agentId);
1046
+ channel.leasedIdentityByAgent.delete(agentId);
1047
+ // ADR-154 §6: its spawn admits nothing from here (see below for the end).
1048
+ void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
817
1049
  channel.lastPrompt.delete(agentId);
818
1050
  channel.rotations.delete(agentId);
819
1051
 
@@ -831,6 +1063,10 @@ export class Orchestrator {
831
1063
  // factory await and closes the stale process instead of binding it.
832
1064
  await inFlightCreate.catch(() => null);
833
1065
  }
1066
+ // The removal ends only once its spawns are retired in their runtime, or
1067
+ // their run's cleanup is asked for (retireInRuntime): the current one, and
1068
+ // any whose retirement an earlier event, or this removal's drain, began.
1069
+ await Promise.all(this.leasedRetirementsUnderWay(channel.taskId, agentId));
834
1070
  if (!this.isActiveChannel(channel)) return;
835
1071
  // A newer ensure can re-add the same id while the old close/create drains.
836
1072
  // That spec owns the status now; do not stop its bound generation or emit
@@ -972,9 +1208,6 @@ export class Orchestrator {
972
1208
  taskProxiedKeys(taskId),
973
1209
  ),
974
1210
  );
975
- // ADR-046: materialise this agent's skills to its per-agent SKILL.md in
976
- // the workspace (bind-mounted into the container). Best-effort.
977
- writeAgentSkills(taskId, agent);
978
1211
  // ADR-022: any agent with a non-empty initialPrompt opens a first
979
1212
  // turn at container-ready. The rest stay silent until addressed.
980
1213
  const firstTurn = assembleFirstTurnPrompt(
@@ -1048,8 +1281,24 @@ export class Orchestrator {
1048
1281
  accountByAgent: new Map(),
1049
1282
  lastPrompt: new Map(),
1050
1283
  rotations: new Map(),
1284
+ leased: false,
1285
+ leasedSync: null,
1286
+ leasedRespawn: new Set(),
1287
+ leasedIdentityByAgent: new Map(),
1288
+ leasedBlocked: new Set(),
1289
+ leasedSyncRetry: null,
1290
+ leasedWakePending: false,
1291
+ leasedCreating: new Set(),
1292
+ leasedLaunches: new Map(),
1051
1293
  };
1052
1294
  this.channels.set(taskId, channel);
1295
+ // ADR-046: materialise each agent's skills to its per-agent SKILL.md in
1296
+ // the workspace (bind-mounted into the container), best-effort, once the
1297
+ // channel can give the task's runtime: where only it can make the
1298
+ // agent's directory (macOS, #cb3j0z), it is asked for then.
1299
+ for (const agent of roster) {
1300
+ writeAgentSkills(taskId, agent, () => this.workspaceRuntime(taskId));
1301
+ }
1053
1302
  return channel;
1054
1303
  }
1055
1304
 
@@ -1255,22 +1504,90 @@ export class Orchestrator {
1255
1504
  * copies use this gate: a false→true toggle queues migration after the older
1256
1505
  * copy, so that copy cannot land last and erase the managed browser block.
1257
1506
  */
1258
- private codexHomes(hasCodex: boolean, scope: string): string[] {
1507
+ private codexHomes(hasCodex: boolean, channel: Channel): string[] {
1259
1508
  if (!hasCodex) return [];
1260
- const homes = resolveEngineAccounts("codex", {}, scope)
1261
- .map((account) => account.configDir?.containerDir)
1262
- .filter((home): home is string => Boolean(home));
1263
- return [...new Set([DEFAULT_CODEX_HOME, ...homes])];
1509
+ return [...new Set([DEFAULT_CODEX_HOME, ...this.accountHomes(channel, "codex")])];
1264
1510
  }
1265
1511
 
1266
- /** Every GROK_HOME a task in `scope` may run under: the default plus each
1267
- * sign-in account's isolated home (API-key accounts use the default). */
1268
- private grokHomes(hasGrok: boolean, scope: string): string[] {
1512
+ /** Every GROK_HOME a task may run under: the default plus each sign-in
1513
+ * account's isolated home (API-key accounts use the default). */
1514
+ private grokHomes(hasGrok: boolean, channel: Channel): string[] {
1269
1515
  if (!hasGrok) return [];
1270
- const homes = resolveEngineAccounts("grok", {}, scope)
1271
- .map((account) => account.configDir?.containerDir)
1272
- .filter((home): home is string => Boolean(home));
1273
- return [...new Set([DEFAULT_GROK_HOME, ...homes])];
1516
+ return [...new Set([DEFAULT_GROK_HOME, ...this.accountHomes(channel, "grok")])];
1517
+ }
1518
+
1519
+ /** The homes `kind`'s config-directory accounts live in: the task's leased
1520
+ * ones, or the runner's own for a task not on cloud credentials. A leased
1521
+ * Codex account's home is listed with every Codex agent's own home on it,
1522
+ * so the task's managed config (browser, MCP) reaches the homes the
1523
+ * agents run in, now and whenever it changes. */
1524
+ private accountHomes(channel: Channel, kind: string): string[] {
1525
+ if (!this.isLeasedChannel(channel)) {
1526
+ return resolveEngineAccounts(kind, {}, channel.engineAccountScope)
1527
+ .map((account) => account.configDir?.containerDir)
1528
+ .filter((home): home is string => Boolean(home));
1529
+ }
1530
+ const homes = (leasedEnginesFor(channel.taskId)?.accounts ?? [])
1531
+ .filter((account) => account.engineKind === kind && account.home !== null)
1532
+ .map((account) => account.home!);
1533
+ if (kind !== "codex") return homes;
1534
+ const agents = channel.roster.filter((agent) => agent.kind === "codex");
1535
+ return homes.flatMap((home) => [home, ...agents.map((agent) => codexPerAgentHome(home, agent.id))]);
1536
+ }
1537
+
1538
+ /** ADR-154: whether this channel's AI accounts come from the task's lease
1539
+ * (the engine consumer holds a record for the task, or the runner is no
1540
+ * longer on its own credentials at all: no default slot, so an agent
1541
+ * reattached before its grant waits for it). Sticky: a channel that was
1542
+ * on cloud credentials never falls back to the runner's own accounts,
1543
+ * not even after the record is gone. */
1544
+ private isLeasedChannel(channel: Channel): boolean {
1545
+ if (!channel.leased && (leasedEnginesFor(channel.taskId) !== undefined || !onOwnCredentials())) {
1546
+ channel.leased = true;
1547
+ }
1548
+ return channel.leased;
1549
+ }
1550
+
1551
+ /**
1552
+ * The accounts `kind` may run on, in try order: the task's leased ones
1553
+ * that are readable now (none while its gate is closed), or the runner's
1554
+ * own for a task not on cloud credentials. A leased account carries no env
1555
+ * and no directory here: the spawn boundary applies it (leasedSpawnArgs),
1556
+ * and no runner-side copy ever names it.
1557
+ */
1558
+ private engineAccounts(channel: Channel, kind: string): ResolvedEngineAccount[] {
1559
+ if (!this.isLeasedChannel(channel)) {
1560
+ return resolveEngineAccounts(kind, {}, channel.engineAccountScope);
1561
+ }
1562
+ const lease = leasedEnginesFor(channel.taskId);
1563
+ if (!lease?.open) return [];
1564
+ return lease.accounts
1565
+ .filter(
1566
+ (account): account is LeasedEngineAccount =>
1567
+ account.engineKind === kind && isLeasedEngineAccount(account),
1568
+ )
1569
+ .map((account) => ({
1570
+ id: account.credentialId,
1571
+ kind,
1572
+ label: account.label,
1573
+ isDefault: false,
1574
+ scope: account.scope,
1575
+ authKind: account.authKind,
1576
+ execEnv: {},
1577
+ }));
1578
+ }
1579
+
1580
+ /** ADR-076 selection over `engineAccounts`: the first healthy account in
1581
+ * try order, skipping `exclude`. */
1582
+ private pickAccount(
1583
+ channel: Channel,
1584
+ kind: string,
1585
+ exclude?: Set<string>,
1586
+ ): ResolvedEngineAccount | null {
1587
+ if (!this.isLeasedChannel(channel)) {
1588
+ return pickEngineAccount(kind, { scope: channel.engineAccountScope, exclude });
1589
+ }
1590
+ return selectAccount(this.engineAccounts(channel, kind), { exclude });
1274
1591
  }
1275
1592
 
1276
1593
  private noteProvisionMutation(
@@ -1299,9 +1616,12 @@ export class Orchestrator {
1299
1616
  if (this.isActiveChannel(channel)) channel.configGeneration += 1;
1300
1617
  }
1301
1618
 
1302
- private provisionFingerprint(kinds: readonly string[], scope: string): string {
1619
+ private provisionFingerprint(kinds: readonly string[], channel: Channel): string {
1620
+ // ADR-154: a leased task's accounts are never copied from the runner;
1621
+ // the spawn boundary writes each one's files through the engine consumer.
1622
+ if (this.isLeasedChannel(channel)) return "leased";
1303
1623
  const accounts = kinds.flatMap((kind) =>
1304
- resolveEngineAccounts(kind, {}, scope)
1624
+ resolveEngineAccounts(kind, {}, channel.engineAccountScope)
1305
1625
  .filter((account) => !account.isDefault && account.configDir)
1306
1626
  .map((account) => ({
1307
1627
  kind,
@@ -1374,13 +1694,13 @@ export class Orchestrator {
1374
1694
  const requestedHasCodex =
1375
1695
  hasCodex || channel.roster.some((agent) => agent.kind === "codex");
1376
1696
  const requestedHomes = requestedBrowser
1377
- ? this.codexHomes(requestedHasCodex, channel.engineAccountScope)
1697
+ ? this.codexHomes(requestedHasCodex, channel)
1378
1698
  : [];
1379
1699
  const requestedKinds = [
1380
1700
  ...new Set(channel.roster.map((agent) => agent.kind)),
1381
1701
  ];
1382
1702
  const requestedProvisionFingerprint =
1383
- this.provisionFingerprint(requestedKinds, channel.engineAccountScope);
1703
+ this.provisionFingerprint(requestedKinds, channel);
1384
1704
  const changedProvisionKinds = this.changedProvisionKinds(
1385
1705
  channel.provisionAttemptedFingerprint,
1386
1706
  requestedProvisionFingerprint,
@@ -1389,11 +1709,14 @@ export class Orchestrator {
1389
1709
  const provisionKinds = forceProvision
1390
1710
  ? requestedKinds
1391
1711
  : changedProvisionKinds;
1392
- const mutableProvisionKinds = provisionKinds.filter((kind) =>
1393
- resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
1394
- (account) => !account.isDefault && Boolean(account.configDir),
1395
- ),
1396
- );
1712
+ const leased = this.isLeasedChannel(channel);
1713
+ const mutableProvisionKinds = leased
1714
+ ? []
1715
+ : provisionKinds.filter((kind) =>
1716
+ resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
1717
+ (account) => !account.isDefault && Boolean(account.configDir),
1718
+ ),
1719
+ );
1397
1720
  const mayProvisionConfig = mutableProvisionKinds.length > 0;
1398
1721
 
1399
1722
  const inFlight = channel.browserSetup;
@@ -1410,13 +1733,13 @@ export class Orchestrator {
1410
1733
  requestedHasCodex ||
1411
1734
  channel.roster.some((agent) => agent.kind === "codex");
1412
1735
  const latestHomes = latestBrowser
1413
- ? this.codexHomes(latestHasCodex, channel.engineAccountScope)
1736
+ ? this.codexHomes(latestHasCodex, channel)
1414
1737
  : [];
1415
1738
  const latestKinds = [
1416
1739
  ...new Set(channel.roster.map((agent) => agent.kind)),
1417
1740
  ];
1418
1741
  const latestProvisionFingerprint =
1419
- this.provisionFingerprint(latestKinds, channel.engineAccountScope);
1742
+ this.provisionFingerprint(latestKinds, channel);
1420
1743
  if (
1421
1744
  (latestBrowser &&
1422
1745
  coverage?.browserAttempted !== true &&
@@ -1452,7 +1775,12 @@ export class Orchestrator {
1452
1775
  forceProvision ||
1453
1776
  channel.provisionAttemptedFingerprint !==
1454
1777
  requestedProvisionFingerprint;
1455
- if (shouldProvision) {
1778
+ if (shouldProvision && leased) {
1779
+ // ADR-154: nothing is copied from the runner for a leased task.
1780
+ coverage.provisionAttempted = true;
1781
+ channel.provisionAttemptedFingerprint = requestedProvisionFingerprint;
1782
+ channel.provisionRetryFingerprint = undefined;
1783
+ } else if (shouldProvision) {
1456
1784
  coverage.provisionAttempted = true;
1457
1785
  if (mayProvisionConfig) {
1458
1786
  // Fence a factory that started before the copy without declaring a
@@ -1462,21 +1790,27 @@ export class Orchestrator {
1462
1790
  }
1463
1791
  let copiedOrUnknown = false;
1464
1792
  try {
1465
- const provisioned = channel.agentHandle
1466
- ? await provisionEngineAccounts(
1467
- channel.containerName,
1468
- provisionKinds,
1469
- {},
1470
- channel.agentHandle,
1471
- channel.engineAccountScope,
1472
- )
1473
- : await provisionEngineAccounts(
1474
- channel.containerName,
1475
- provisionKinds,
1476
- {},
1477
- undefined,
1478
- channel.engineAccountScope,
1479
- );
1793
+ // ADR-154: the runner's own accounts, copied only while it is on
1794
+ // its own credentials, counted (leaving waits for the copy);
1795
+ // once leaving has begun, nothing is copied.
1796
+ const provisioning = whileOnOwnCredentials(() =>
1797
+ channel.agentHandle
1798
+ ? provisionEngineAccounts(
1799
+ channel.containerName,
1800
+ provisionKinds,
1801
+ {},
1802
+ channel.agentHandle,
1803
+ channel.engineAccountScope,
1804
+ )
1805
+ : provisionEngineAccounts(
1806
+ channel.containerName,
1807
+ provisionKinds,
1808
+ {},
1809
+ undefined,
1810
+ channel.engineAccountScope,
1811
+ ),
1812
+ );
1813
+ const provisioned = (await provisioning) ?? { copied: 0, failed: 0, mayHaveMutated: false };
1480
1814
  copiedOrUnknown =
1481
1815
  provisioned.copied > 0 || provisioned.mayHaveMutated;
1482
1816
  if (provisioned.failed > 0) {
@@ -1537,7 +1871,7 @@ export class Orchestrator {
1537
1871
  requestedHomes,
1538
1872
  channel.agentHandle,
1539
1873
  requestedKinds,
1540
- this.grokHomes(requestedKinds.includes("grok"), channel.engineAccountScope),
1874
+ this.grokHomes(requestedKinds.includes("grok"), channel),
1541
1875
  )
1542
1876
  : await setupBrowserTesting(
1543
1877
  channel.taskId,
@@ -1546,7 +1880,7 @@ export class Orchestrator {
1546
1880
  requestedHomes,
1547
1881
  undefined,
1548
1882
  requestedKinds,
1549
- this.grokHomes(requestedKinds.includes("grok"), channel.engineAccountScope),
1883
+ this.grokHomes(requestedKinds.includes("grok"), channel),
1550
1884
  );
1551
1885
  } catch (err) {
1552
1886
  console.warn(
@@ -1617,15 +1951,12 @@ export class Orchestrator {
1617
1951
  requestedHasCodex ||
1618
1952
  channel.roster.some((agent) => agent.kind === "codex");
1619
1953
  const latestHomes = latestBrowser
1620
- ? this.codexHomes(latestHasCodex, channel.engineAccountScope)
1954
+ ? this.codexHomes(latestHasCodex, channel)
1621
1955
  : [];
1622
1956
  const latestKinds = [
1623
1957
  ...new Set(channel.roster.map((agent) => agent.kind)),
1624
1958
  ];
1625
- const latestProvisionFingerprint = this.provisionFingerprint(
1626
- latestKinds,
1627
- channel.engineAccountScope,
1628
- );
1959
+ const latestProvisionFingerprint = this.provisionFingerprint(latestKinds, channel);
1629
1960
  if (
1630
1961
  (latestBrowser &&
1631
1962
  !coverage.browserAttempted &&
@@ -1714,12 +2045,9 @@ export class Orchestrator {
1714
2045
  ...new Set(channel.roster.map((agent) => agent.kind)),
1715
2046
  ].sort();
1716
2047
  const codexHomes = engineKinds.includes("codex")
1717
- ? this.codexHomes(true, channel.engineAccountScope)
2048
+ ? this.codexHomes(true, channel)
1718
2049
  : [];
1719
- const grokHomes = this.grokHomes(
1720
- engineKinds.includes("grok"),
1721
- channel.engineAccountScope,
1722
- );
2050
+ const grokHomes = this.grokHomes(engineKinds.includes("grok"), channel);
1723
2051
  const fingerprint = JSON.stringify({
1724
2052
  connections,
1725
2053
  engineKinds,
@@ -2053,6 +2381,9 @@ export class Orchestrator {
2053
2381
  if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
2054
2382
  return false;
2055
2383
  }
2384
+ // ADR-154 §5: down for want of a usable account until the gate next
2385
+ // opens (which starts it again), not retried on every pass.
2386
+ if (channel.leasedBlocked.has(agent.id)) return false;
2056
2387
  // Crash-loop budget: an agent whose session keeps dying stops being
2057
2388
  // respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
2058
2389
  // but a cold budget resets so the task can heal after the cause clears.
@@ -2162,7 +2493,12 @@ export class Orchestrator {
2162
2493
  ),
2163
2494
  },
2164
2495
  );
2165
- if (!session) return;
2496
+ if (!session) {
2497
+ // Credentials unavailable is this agent's alone: its siblings
2498
+ // still start.
2499
+ if (channel.leasedBlocked.has(agent.id)) continue;
2500
+ return;
2501
+ }
2166
2502
  this.startNextPendingPrompt(channel, agent.id);
2167
2503
  }
2168
2504
  } finally {
@@ -2257,6 +2593,9 @@ export class Orchestrator {
2257
2593
  channel.accountByAgent.delete(agent.id);
2258
2594
  return { ...floor, ...base };
2259
2595
  }
2596
+ // ADR-154: a leased task's account is picked and applied at the spawn
2597
+ // boundary (leasedSpawnArgs), which can wait for the task's gate.
2598
+ if (this.isLeasedChannel(channel)) return { ...floor, ...base };
2260
2599
  const account = pickEngineAccount(agent.kind, {
2261
2600
  scope: channel.engineAccountScope,
2262
2601
  });
@@ -2345,6 +2684,11 @@ export class Orchestrator {
2345
2684
  ) {
2346
2685
  return;
2347
2686
  }
2687
+ // ADR-154 §6: an engine that ended launches nothing more from now,
2688
+ // whatever its terminal handling still awaits (the runtime's proof).
2689
+ if (event.type === "error" || event.type === "exit") {
2690
+ this.endLeasedLaunches(channel, agentId, session);
2691
+ }
2348
2692
  const operation = eventTail
2349
2693
  .catch(() => {})
2350
2694
  .then(async () => {
@@ -2411,6 +2755,842 @@ export class Orchestrator {
2411
2755
  return operation;
2412
2756
  }
2413
2757
 
2758
+ /**
2759
+ * ADR-154 §5, §6, §10: what a spawn on the task's leased AI account takes,
2760
+ * and the exact lease material it starts on. The agent runs on the account
2761
+ * it is bound to while that account is still served readable, else on the
2762
+ * first healthy one in try order (the owner's accounts, then the
2763
+ * workspace's). An env-token account's values ride the spawn, synchronously
2764
+ * when nothing has to be waited for. A config-directory account's home gets
2765
+ * the task's managed config first (browser, MCP), then its files, through
2766
+ * the engine consumer, in the home the agent runs in (Codex's own copy,
2767
+ * seeded from the account's home without its credential files). While the
2768
+ * task's gate is closed it waits, bounded; null when no account can be
2769
+ * used, and the agent stays down with "credentials unavailable", never on
2770
+ * the runner's own accounts, until the gate next opens.
2771
+ */
2772
+ private leasedSpawnArgs(
2773
+ channel: Channel,
2774
+ agentId: string,
2775
+ args: SpawnArgs,
2776
+ environment: TaskEnvironmentHandle<unknown>,
2777
+ ):
2778
+ | { args: SpawnArgs; identity: LeasedIdentity }
2779
+ | null
2780
+ | Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
2781
+ const kind = args.agent?.kind ?? "";
2782
+ const lease = leasedEnginesFor(channel.taskId);
2783
+ if (lease?.open) {
2784
+ const account = this.leasedAccountFor(channel, agentId, kind, lease);
2785
+ if (account?.authKind === "env") {
2786
+ return this.withLeasedAccount(channel, agentId, args, account, lease.assignment.runEpoch);
2787
+ }
2788
+ }
2789
+ return (async () => {
2790
+ const deadline = Date.now() + LEASED_ACCOUNT_WAIT_MS;
2791
+ let reason = "the task's AI accounts are not available";
2792
+ let moved = 0;
2793
+ while (this.isActiveChannel(channel)) {
2794
+ const open = await this.leasedGateOpen(channel, deadline);
2795
+ if (!open) break;
2796
+ if (!this.isActiveChannel(channel)) return null;
2797
+ const account = this.leasedAccountFor(channel, agentId, kind, open);
2798
+ if (!account) {
2799
+ reason = `no ${kind} account this task can use`;
2800
+ break;
2801
+ }
2802
+ if (account.authKind === "env") {
2803
+ return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch);
2804
+ }
2805
+ // The account's home gets the task's managed config before anything
2806
+ // can start in it, whenever the gate's opening first reveals it.
2807
+ await this.ensureBrowserSetup(channel, kind === "codex");
2808
+ if (!this.isActiveChannel(channel)) return null;
2809
+ await this.ensureMcpConfig(channel);
2810
+ if (!this.isActiveChannel(channel)) return null;
2811
+ // Codex runs in a home of its own (two agents on one account never
2812
+ // share its state runtime); the config writers above reached it.
2813
+ const home = kind === "codex" ? codexPerAgentHome(account.home!, agentId) : account.home!;
2814
+ try {
2815
+ await ensureEngineHome(environment, open.assignment, account.credentialId, home);
2816
+ } catch (error) {
2817
+ // The account moved while it was written (a newer version, a
2818
+ // cleanup, no longer readable, the gate closing): pick again on
2819
+ // what the gate serves, or will once it reopens, a few times while
2820
+ // time remains. Anything else ends here.
2821
+ if (
2822
+ (error instanceof EngineCredentialWriteRefused || error instanceof CredentialLeaseUnavailableError) &&
2823
+ moved++ < 3 &&
2824
+ Date.now() < deadline
2825
+ ) {
2826
+ continue;
2827
+ }
2828
+ reason = error instanceof CredentialLeaseUnavailableError ? error.message : "its files could not be written";
2829
+ break;
2830
+ }
2831
+ if (!this.isActiveChannel(channel)) return null;
2832
+ return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch, home);
2833
+ }
2834
+ if (!this.isActiveChannel(channel)) return null;
2835
+ this.leasedUnavailable(channel, agentId, reason);
2836
+ return null;
2837
+ })();
2838
+ }
2839
+
2840
+ /** The task's open engine lease, waited for while time remains, a newer
2841
+ * run's when the task moves on to one; null when none opens in time. */
2842
+ private async leasedGateOpen(channel: Channel, deadline: number): Promise<TaskEngineLease | null> {
2843
+ while (this.isActiveChannel(channel)) {
2844
+ const record = leasedEnginesFor(channel.taskId);
2845
+ if (!record) return null;
2846
+ try {
2847
+ return await waitForEngineGate(record.assignment, Math.max(0, deadline - Date.now()));
2848
+ } catch {
2849
+ // The task moved on to another run: wait for that run's gate while
2850
+ // time remains.
2851
+ if (leasedEnginesFor(channel.taskId)?.assignment.runEpoch !== record.assignment.runEpoch && Date.now() < deadline) {
2852
+ continue;
2853
+ }
2854
+ return null;
2855
+ }
2856
+ }
2857
+ return null;
2858
+ }
2859
+
2860
+ /**
2861
+ * ADR-154 §6, ADR-128: what a Managed Engine spawn on a leased task takes.
2862
+ * It runs on no AI account, but its agent's task token is its bearer: it
2863
+ * is bound to the task's open run, its launches admitted as that token on
2864
+ * this binding, under the engine supervisor like any leased engine's.
2865
+ * While the task's gate is closed it waits, bounded; null when it stays
2866
+ * closed, and the agent stays down with "credentials unavailable" until
2867
+ * the gate next opens.
2868
+ */
2869
+ private leasedTaskTokenSpawnArgs(
2870
+ channel: Channel,
2871
+ agentId: string,
2872
+ args: SpawnArgs,
2873
+ ):
2874
+ | { args: SpawnArgs; identity: LeasedIdentity }
2875
+ | null
2876
+ | Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
2877
+ const lease = leasedEnginesFor(channel.taskId);
2878
+ if (lease?.open) return this.withLeasedTaskToken(channel, agentId, args, lease.assignment.runEpoch);
2879
+ return (async () => {
2880
+ const open = await this.leasedGateOpen(channel, Date.now() + LEASED_ACCOUNT_WAIT_MS);
2881
+ if (!this.isActiveChannel(channel)) return null;
2882
+ if (!open) {
2883
+ this.leasedUnavailable(channel, agentId, "the task's credentials are not available");
2884
+ return null;
2885
+ }
2886
+ return this.withLeasedTaskToken(channel, agentId, args, open.assignment.runEpoch);
2887
+ })();
2888
+ }
2889
+
2890
+ /** Bind the agent to the task's run `runEpoch` on its task token alone,
2891
+ * and give its spawn the run's marker every process it starts inherits
2892
+ * (its launches are known by it, and admitted on this binding). */
2893
+ private withLeasedTaskToken(
2894
+ channel: Channel,
2895
+ agentId: string,
2896
+ args: SpawnArgs,
2897
+ runEpoch: number,
2898
+ ): { args: SpawnArgs; identity: LeasedIdentity } {
2899
+ const identity = { subject: "task-token" as const, runEpoch };
2900
+ channel.leasedIdentityByAgent.set(agentId, identity);
2901
+ return {
2902
+ identity,
2903
+ args: {
2904
+ ...args,
2905
+ attachCompatibilityKey: JSON.stringify([args.attachCompatibilityKey ?? null, "lease-v1", runEpoch, "task-token"]),
2906
+ agentEnv: {
2907
+ ...(args.agentEnv ?? {}),
2908
+ // The gateway's bearer is the agent's own token; no engine's
2909
+ // credential of the runner's rides beside it.
2910
+ ...LEASED_CREDENTIAL_BLANKS,
2911
+ [LEASED_RUN_ENV]: String(runEpoch),
2912
+ },
2913
+ },
2914
+ };
2915
+ }
2916
+
2917
+ /**
2918
+ * ADR-154 §6: the environment a leased engine's adapter launches through.
2919
+ * Each engine launch it makes (the session, each one-shot turn, a durable
2920
+ * runner once deployed, a classifier's exec), at whatever moment its
2921
+ * transport gets there, is admitted then: for an agent's spawn (`owner`),
2922
+ * only while that spawn is the agent's current one and, once its create
2923
+ * settled, the session it bound is the agent's live one, and `identity`
2924
+ * is still what the agent runs on and the task's open gate serves (for a
2925
+ * one-shot with no agent, only the latter). The admission is written into
2926
+ * the runtime first (its exec admitted at the provider's last moment) and
2927
+ * the check made again just before the launch is handed over and at the
2928
+ * provider's last moment. The launch then starts under the engine
2929
+ * supervisor, which consumes that admission. A spawn's admissions name its
2930
+ * slot: before its first, the agent's earlier spawns are retired in the
2931
+ * runtime, and once the slot is retired, nothing it was admitted for
2932
+ * starts there (`retireLeasedLaunch`). A cleanup unbinds first, waits for
2933
+ * the admissions under way (registered before their first await), and
2934
+ * revokes in the runtime what they wrote before it kills what the
2935
+ * supervisors own.
2936
+ */
2937
+ private leasedLaunchEnvironment(
2938
+ channel: Channel,
2939
+ owner: { agentId: string; slot: LeasedLaunchSlot } | null,
2940
+ identity: LeasedIdentity,
2941
+ environment: TaskEnvironmentHandle<unknown>,
2942
+ ): TaskEnvironmentHandle<unknown> {
2943
+ const agentId = owner?.agentId ?? null;
2944
+ const held = (): boolean =>
2945
+ this.isActiveChannel(channel) &&
2946
+ (owner === null
2947
+ ? this.leasedMaterialServed(channel, identity)
2948
+ : channel.leasedLaunches.get(owner.agentId) === owner.slot &&
2949
+ (owner.slot.session === null
2950
+ ? !owner.slot.created
2951
+ : channel.sessions.get(owner.agentId) === owner.slot.session) &&
2952
+ this.leasedIdentityHeld(channel, owner.agentId, identity));
2953
+ const admittedAs: EngineLaunchIdentity = owner?.slot.identity ?? { taskId: channel.taskId, slotId: null, ...identity };
2954
+ return anchoredEnvironment(environment, {
2955
+ held,
2956
+ admit: (carried) => {
2957
+ if (environment.anchors !== true || !held()) return Promise.reject(new EngineLaunchRefused());
2958
+ if (owner) owner.slot.admitted = true;
2959
+ const admitted = (async () => {
2960
+ if (owner) await this.leasedPredecessorsRetired(owner.agentId, owner.slot);
2961
+ return admitEngineLaunch(
2962
+ environment,
2963
+ admittedAs,
2964
+ () => {
2965
+ if (!held()) throw new EngineLaunchRefused();
2966
+ },
2967
+ carried,
2968
+ );
2969
+ })();
2970
+ const entry = {
2971
+ agentId: agentId ?? "",
2972
+ accountId: identity.subject === "account" ? identity.credentialId : null,
2973
+ runEpoch: identity.runEpoch,
2974
+ settled: admitted.then(
2975
+ () => undefined,
2976
+ () => undefined,
2977
+ ),
2978
+ };
2979
+ channel.leasedCreating.add(entry);
2980
+ void entry.settled.then(() => channel.leasedCreating.delete(entry));
2981
+ return admitted;
2982
+ },
2983
+ });
2984
+ }
2985
+
2986
+ /** `session` ended (its engine's error or exit): its spawn admits no
2987
+ * launch from this moment, before its terminal handling awaits anything.
2988
+ * Every such handling retires or replaces the session. */
2989
+ private endLeasedLaunches(channel: Channel, agentId: string, session: AgentSession): void {
2990
+ const slot = channel.leasedLaunches.get(agentId);
2991
+ if (slot?.session === session) void this.retireLeasedLaunch(channel, agentId, slot);
2992
+ }
2993
+
2994
+ /**
2995
+ * ADR-154 §6: `slot` admits nothing from this moment. Then, in its
2996
+ * runtime, it is retired: every admission it wrote that no launch has used
2997
+ * is revoked, and none of its own is written or used after, so a launch
2998
+ * handed to its provider before now that has not started there never
2999
+ * does, with the agent's old task token or anything else. The agent's
3000
+ * next spawn launches in that runtime only once that is done
3001
+ * (`leasedPredecessorsRetired`). Resolves once that is done or escalated
3002
+ * (see `retireInRuntime`); never rejects.
3003
+ */
3004
+ private retireLeasedLaunch(channel: Channel, agentId: string, slot: LeasedLaunchSlot | undefined): Promise<void> {
3005
+ if (!slot || slot.retired) return Promise.resolve();
3006
+ slot.retired = true;
3007
+ if (channel.leasedLaunches.get(agentId) === slot) channel.leasedLaunches.delete(agentId);
3008
+ // It never wrote an admission: nothing of it is in its runtime.
3009
+ if (!slot.admitted) return Promise.resolve();
3010
+ const retired: RetiredLaunchSlot = { agentId, slot, attempt: null, covered: false, escalated: false };
3011
+ let pending = this.leasedRetired.get(channel.taskId);
3012
+ if (!pending) {
3013
+ pending = new Set();
3014
+ this.leasedRetired.set(channel.taskId, pending);
3015
+ }
3016
+ pending.add(retired);
3017
+ return this.retireInRuntime(retired).catch(() => undefined);
3018
+ }
3019
+
3020
+ /**
3021
+ * One attempt to retire `retired` in its runtime, joined while one is
3022
+ * under way; once done, it is no longer pending. Should it fail, the order
3023
+ * is lost: a launch handed over before the retirement may start before
3024
+ * any later one lands, and nothing a retry does stops it. So, unless a
3025
+ * cleanup that covers it is under way or done, the run's full credential
3026
+ * cleanup is asked for at once (`escalateEngineLaunchFence`): its gate
3027
+ * closed first, it proves no engine of the run runs (or stops the
3028
+ * runtime), and keeps failing closed until it can. Rejects when this
3029
+ * attempt failed (it is retained and retried meanwhile).
3030
+ */
3031
+ private retireInRuntime(retired: RetiredLaunchSlot): Promise<void> {
3032
+ if (retired.attempt) return retired.attempt;
3033
+ const { taskId, runEpoch } = retired.slot.identity;
3034
+ const attempt = retireEngineLaunchSlot(retired.slot.environment, retired.slot.identity)
3035
+ .then(
3036
+ () => this.forgetRetired(taskId, retired),
3037
+ (error: unknown) => {
3038
+ if (!retired.covered && !retired.escalated) {
3039
+ retired.escalated = true;
3040
+ console.warn(
3041
+ `[orchestrator] ${taskId}/${retired.agentId}: a retired spawn could not be retired in its runtime; cleaning up its run's credentials (${
3042
+ error instanceof Error ? error.message : String(error)
3043
+ })`,
3044
+ );
3045
+ escalateEngineLaunchFence({ taskId, runEpoch });
3046
+ }
3047
+ throw error;
3048
+ },
3049
+ )
3050
+ .finally(() => {
3051
+ if (retired.attempt === attempt) retired.attempt = null;
3052
+ });
3053
+ retired.attempt = attempt;
3054
+ return attempt;
3055
+ }
3056
+
3057
+ /** The task's retirements still under way (of `agentId`'s spawns alone,
3058
+ * when named), whichever event began them: each resolves once its slot is
3059
+ * retired in its runtime, or its run's cleanup has been asked for. What a
3060
+ * removal or a channel's close waits for. */
3061
+ private leasedRetirementsUnderWay(taskId: string, agentId?: string): Array<Promise<void>> {
3062
+ return [...(this.leasedRetired.get(taskId) ?? [])]
3063
+ .filter((retired) => agentId === undefined || retired.agentId === agentId)
3064
+ .flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
3065
+ }
3066
+
3067
+ private forgetRetired(taskId: string, retired: RetiredLaunchSlot): void {
3068
+ const pending = this.leasedRetired.get(taskId);
3069
+ if (!pending) return;
3070
+ pending.delete(retired);
3071
+ if (pending.size === 0) this.leasedRetired.delete(taskId);
3072
+ }
3073
+
3074
+ /**
3075
+ * ADR-154 §6: before `slot` writes an admission, every earlier slot of the
3076
+ * same agent retired for the same run in the same runtime is retired
3077
+ * there: nothing the agent's earlier spawns were admitted for can start
3078
+ * after this one's launch. Rejects with `EngineLaunchRefused` when one
3079
+ * could not be now.
3080
+ */
3081
+ private async leasedPredecessorsRetired(agentId: string, slot: LeasedLaunchSlot): Promise<void> {
3082
+ const earlier = [...(this.leasedRetired.get(slot.identity.taskId) ?? [])].filter(
3083
+ (retired) =>
3084
+ retired.agentId === agentId &&
3085
+ retired.slot.identity.runEpoch === slot.identity.runEpoch &&
3086
+ sameRuntime(retired.slot.environment, slot.environment),
3087
+ );
3088
+ try {
3089
+ await Promise.all(earlier.map((retired) => this.retireInRuntime(retired)));
3090
+ } catch (error) {
3091
+ // Live 2026-10-11: this refusal was silent, so agents that never started
3092
+ // left nothing in the log to say why.
3093
+ console.warn(
3094
+ `[orchestrator] ${slot.identity.taskId}/${agentId}: not starting the agent, an earlier start could not be retired in its runtime (${
3095
+ error instanceof Error ? error.message : String(error)
3096
+ })`,
3097
+ );
3098
+ throw new EngineLaunchRefused("an earlier start of this agent could not be retired in its runtime yet");
3099
+ }
3100
+ }
3101
+
3102
+ /** The retirements still pending that a cleanup of these runs and
3103
+ * accounts (of every subject, for a whole run's) covers: its stop revokes
3104
+ * everything they fence, wherever it was written, and kills what any
3105
+ * launch of theirs started. Each is marked covered: one that fails from
3106
+ * now on is not escalated. */
3107
+ private coveredLeasedRetirements(
3108
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3109
+ ): RetiredLaunchSlot[] {
3110
+ const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
3111
+ const covered = [...(this.leasedRetired.get(request.taskId) ?? [])].filter((retired) => {
3112
+ const { identity } = retired.slot;
3113
+ return (
3114
+ identity.runEpoch <= request.runEpoch &&
3115
+ (!named || (identity.subject === "account" && named.has(identity.credentialId)))
3116
+ );
3117
+ });
3118
+ for (const retired of covered) retired.covered = true;
3119
+ return covered;
3120
+ }
3121
+
3122
+ /** A cleanup proved every admission of these runs and accounts (of every
3123
+ * subject, for a whole run's) revoked wherever it was written: the
3124
+ * retirements still pending for them are done. */
3125
+ private dischargeLeasedRetirements(
3126
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3127
+ ): void {
3128
+ for (const retired of this.coveredLeasedRetirements(request)) this.forgetRetired(request.taskId, retired);
3129
+ }
3130
+
3131
+ /** The agent stays down for want of a usable account, says why, and is
3132
+ * started again when the task's gate next opens. */
3133
+ private leasedUnavailable(channel: Channel, agentId: string, reason: string): void {
3134
+ channel.accountByAgent.delete(agentId);
3135
+ channel.leasedIdentityByAgent.delete(agentId);
3136
+ channel.leasedBlocked.add(agentId);
3137
+ this.emitHost({
3138
+ kind: "agent.exit",
3139
+ taskId: channel.taskId,
3140
+ agentId,
3141
+ reason: `${agentId}: credentials unavailable (${reason}) — the agent starts again when they are.`,
3142
+ });
3143
+ }
3144
+
3145
+ /** Whether the task's open gate still serves `identity`'s account at its
3146
+ * revision and version, for its run, and that version is not being held
3147
+ * back for a newer one: what a one-shot may start on. An agent's task
3148
+ * token alone needs only its run's gate open. */
3149
+ private leasedMaterialServed(channel: Channel, identity: LeasedIdentity): boolean {
3150
+ const lease = leasedEnginesFor(channel.taskId);
3151
+ if (!lease?.open || lease.assignment.runEpoch !== identity.runEpoch) return false;
3152
+ if (identity.subject === "task-token") return true;
3153
+ const account = lease.accounts.find((candidate) => candidate.credentialId === identity.credentialId);
3154
+ return (
3155
+ isLeasedEngineAccount(account) &&
3156
+ account.revision === identity.revision &&
3157
+ account.version === identity.version &&
3158
+ !engineVersionHeldBack(lease.assignment, identity)
3159
+ );
3160
+ }
3161
+
3162
+ /** A spawn found its lease material moved (a newer version, a cleanup):
3163
+ * start the agent again on what the task holds now, once its gate is
3164
+ * open (now, or at its next opening). */
3165
+ private retryLeasedStart(channel: Channel, agentId: string): void {
3166
+ if (!this.isActiveChannel(channel)) return;
3167
+ channel.leasedRespawn.add(agentId);
3168
+ channel.leasedWakePending = true;
3169
+ if (leasedEnginesFor(channel.taskId)?.open) this.leasedEnginesChanged(channel.taskId);
3170
+ }
3171
+
3172
+ /** Whether the agent may start on `identity` right now: still bound to it,
3173
+ * and the task's open gate still serves that account at that revision and
3174
+ * version, for that run. */
3175
+ private leasedIdentityHeld(channel: Channel, agentId: string, identity: LeasedIdentity): boolean {
3176
+ const bound = channel.leasedIdentityByAgent.get(agentId);
3177
+ return bound !== undefined && sameLeasedIdentity(bound, identity) && this.leasedMaterialServed(channel, identity);
3178
+ }
3179
+
3180
+ /** The leased account an agent runs on: the one it is bound to while that
3181
+ * is served readable, else the first healthy one in try order. */
3182
+ private leasedAccountFor(
3183
+ channel: Channel,
3184
+ agentId: string,
3185
+ kind: string,
3186
+ lease: TaskEngineLease,
3187
+ ): LeasedEngineAccount | undefined {
3188
+ const accounts = lease.accounts.filter(
3189
+ (account): account is LeasedEngineAccount =>
3190
+ account.engineKind === kind && isLeasedEngineAccount(account),
3191
+ );
3192
+ const bound = channel.accountByAgent.get(agentId);
3193
+ const kept = accounts.find((account) => account.credentialId === bound);
3194
+ if (kept) return kept;
3195
+ const picked = selectAccount(
3196
+ accounts.map((account) => ({
3197
+ id: account.credentialId,
3198
+ kind,
3199
+ label: account.label,
3200
+ isDefault: false,
3201
+ scope: account.scope,
3202
+ authKind: account.authKind,
3203
+ execEnv: {},
3204
+ })),
3205
+ );
3206
+ return accounts.find((account) => account.credentialId === picked?.id);
3207
+ }
3208
+
3209
+ /** Bind the agent to `account` for `runEpoch` and give its spawn the
3210
+ * account, and the markers every process it starts inherits (its launches
3211
+ * are known by them, and admitted on this binding). */
3212
+ private withLeasedAccount(
3213
+ channel: Channel,
3214
+ agentId: string,
3215
+ args: SpawnArgs,
3216
+ account: LeasedEngineAccount,
3217
+ runEpoch: number,
3218
+ home?: string,
3219
+ ): { args: SpawnArgs; identity: LeasedIdentity } {
3220
+ const identity = {
3221
+ subject: "account" as const,
3222
+ runEpoch,
3223
+ credentialId: account.credentialId,
3224
+ revision: account.revision,
3225
+ version: account.version,
3226
+ };
3227
+ channel.accountByAgent.set(agentId, account.credentialId);
3228
+ channel.leasedIdentityByAgent.set(agentId, identity);
3229
+ noteEngineAccountUsed(account.credentialId);
3230
+ return {
3231
+ identity,
3232
+ args: {
3233
+ ...args,
3234
+ // A durable runner from before a restart is attached to only when it
3235
+ // was launched on this exact material.
3236
+ attachCompatibilityKey: JSON.stringify([
3237
+ args.attachCompatibilityKey ?? null,
3238
+ "lease-v1",
3239
+ identity.runEpoch,
3240
+ identity.credentialId,
3241
+ identity.revision,
3242
+ identity.version,
3243
+ ]),
3244
+ agentEnv: {
3245
+ ...(args.agentEnv ?? {}),
3246
+ ...LEASED_CREDENTIAL_BLANKS,
3247
+ ...engineSpawnEnv(account, home),
3248
+ [LEASED_RUN_ENV]: String(runEpoch),
3249
+ [LEASED_ACCOUNT_ENV]: account.credentialId,
3250
+ },
3251
+ },
3252
+ };
3253
+ }
3254
+
3255
+ /**
3256
+ * The env of a one-shot run on the task's leased account for `kind` (the
3257
+ * shadow router's classifier): the agent's own account when it runs on
3258
+ * one of that kind, else the first healthy one; a config-directory
3259
+ * account's home is written first. Null when none can be used now: the
3260
+ * classifier does not run and the pass routes by rules.
3261
+ */
3262
+ /** ADR-154 §6: a one-shot's env on the leased `account` chosen for it: its
3263
+ * home written first for a config-directory account. Null when it cannot
3264
+ * be given. */
3265
+ private async leasedOneShotEnv(
3266
+ channel: Channel,
3267
+ lease: TaskEngineLease,
3268
+ account: LeasedEngineAccount,
3269
+ environment: TaskEnvironmentHandle<unknown>,
3270
+ ): Promise<Record<string, string> | null> {
3271
+ const marker = {
3272
+ [LEASED_RUN_ENV]: String(lease.assignment.runEpoch),
3273
+ [LEASED_ACCOUNT_ENV]: account.credentialId,
3274
+ };
3275
+ try {
3276
+ rememberLeasedEngineRuntime({
3277
+ taskId: channel.taskId,
3278
+ runEpoch: lease.assignment.runEpoch,
3279
+ runtime: environment.descriptor.locator,
3280
+ });
3281
+ } catch {
3282
+ return null;
3283
+ }
3284
+ if (account.authKind === "env") {
3285
+ return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
3286
+ }
3287
+ try {
3288
+ await ensureEngineHome(environment, lease.assignment, account.credentialId);
3289
+ return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
3290
+ } catch {
3291
+ return null;
3292
+ }
3293
+ }
3294
+
3295
+ /**
3296
+ * ADR-154 §6, §8: what the task's AI accounts are, or whether its gate is
3297
+ * open, changed. Rewrite what moved in the homes this run wrote, then let
3298
+ * each long-lived engine running on a rewritten account (Codex's
3299
+ * app-server) respawn at its first idle moment; one-shot engines read the
3300
+ * file again at their next turn, and env-token engines take a new value at
3301
+ * their next spawn. Called synchronously by the engine consumer: the work
3302
+ * runs after, one pass at a time per channel.
3303
+ */
3304
+ private leasedEnginesChanged(taskId: string): void {
3305
+ const channel = this.channels.get(taskId);
3306
+ if (!channel || !this.isActiveChannel(channel) || !this.isLeasedChannel(channel)) return;
3307
+ if (!leasedEnginesFor(taskId)?.open) return;
3308
+ if (channel.leasedSyncRetry) {
3309
+ clearTimeout(channel.leasedSyncRetry);
3310
+ channel.leasedSyncRetry = null;
3311
+ }
3312
+ // Agents a cleanup stopped, or that found no usable account, start again
3313
+ // on this view, each on its next account: one attempt per opening. The
3314
+ // intent outlives a pass that fails, for its retry.
3315
+ if (channel.leasedRespawn.size > 0 || channel.leasedBlocked.size > 0) {
3316
+ channel.leasedWakePending = true;
3317
+ }
3318
+ const run = (channel.leasedSync ?? Promise.resolve())
3319
+ .then(async () => {
3320
+ await this.syncLeasedHomes(channel);
3321
+ if (!channel.leasedWakePending || !this.isActiveChannel(channel)) return;
3322
+ channel.leasedWakePending = false;
3323
+ channel.leasedRespawn.clear();
3324
+ channel.leasedBlocked.clear();
3325
+ try {
3326
+ await this.ensureSessions(channel);
3327
+ } catch (err) {
3328
+ channel.leasedWakePending = true;
3329
+ throw err;
3330
+ }
3331
+ })
3332
+ .catch((err: unknown) => {
3333
+ console.warn(
3334
+ `[orchestrator] ${taskId}: rewriting AI account files failed: ${err instanceof Error ? err.message : String(err)}`,
3335
+ );
3336
+ // A home left on an old token is written again, not left so.
3337
+ if (this.isActiveChannel(channel) && !channel.leasedSyncRetry) {
3338
+ const retry = setTimeout(() => {
3339
+ if (channel.leasedSyncRetry !== retry) return;
3340
+ channel.leasedSyncRetry = null;
3341
+ this.leasedEnginesChanged(taskId);
3342
+ }, LEASED_SYNC_RETRY_MS);
3343
+ retry.unref?.();
3344
+ channel.leasedSyncRetry = retry;
3345
+ }
3346
+ });
3347
+ channel.leasedSync = run;
3348
+ }
3349
+
3350
+ private async syncLeasedHomes(channel: Channel): Promise<void> {
3351
+ if (!this.isActiveChannel(channel)) return;
3352
+ const lease = leasedEnginesFor(channel.taskId);
3353
+ if (!lease?.open) return;
3354
+ // Nothing was written into a runtime this channel has not settled.
3355
+ const environment = channel.agentHandle;
3356
+ if (environment) await syncEngineHomes(environment, lease.assignment);
3357
+ if (!this.isActiveChannel(channel)) return;
3358
+ // Every launch is admitted on the exact material its agent is bound to
3359
+ // (§6): an agent whose account moved since starts again on what the
3360
+ // task holds now. A long-lived engine holds the old token or file; a
3361
+ // one-shot one, env-token or not, would have its next turn refused.
3362
+ let stale = false;
3363
+ for (const agent of channel.roster) {
3364
+ const identity = channel.leasedIdentityByAgent.get(agent.id);
3365
+ if (!identity || this.leasedMaterialServed(channel, identity)) continue;
3366
+ if (!channel.sessions.has(agent.id)) continue;
3367
+ channel.browserStaleSessions.add(agent.id);
3368
+ stale = true;
3369
+ }
3370
+ // A stale idle session is recycled now; one mid-turn finishes its turn
3371
+ // on the material it has and is recycled at its next idle moment.
3372
+ if (stale) await this.reconcileIfStarted(channel);
3373
+ }
3374
+
3375
+ /**
3376
+ * ADR-154 §6: the cleanup coordinator's quiesce step for engines (its
3377
+ * `quiesceProcesses` hook takes this request as it is), for the run it
3378
+ * names and every older one, never a newer run's:
3379
+ * - every agent on one of the request's accounts (on any leased account,
3380
+ * or on its task token alone, for a whole-task cleanup) is unbound at
3381
+ * once, so nothing launches on them again (a launch is admitted only
3382
+ * while its agent is bound to its exact material), and its session is
3383
+ * asked to stop; a spawn creating an engine on one, or a launch being
3384
+ * admitted, is waited for, bounded;
3385
+ * - then, in every runtime those runs' engines ran in, every admission not
3386
+ * used yet is revoked, and everything each engine supervisor of those
3387
+ * runs and accounts owns is killed until the supervisor ends on its own,
3388
+ * having proved nothing it owned is left (`stopAnchoredEngines`). That
3389
+ * holds no state here, so a retried cleanup proves it again even after
3390
+ * this channel let the agents go. A supervisor lost without proving its
3391
+ * subtree empty leaves one proof: the runtime is stopped. An env-token
3392
+ * engine holds no credential file: stopping it moves it to its next
3393
+ * account.
3394
+ * Rejects when anything could not be proven stopped: the cleanup then
3395
+ * fails and is retried, never deleting under a live engine. Stopped agents
3396
+ * start again on their next account once the gate opens.
3397
+ */
3398
+ async quiesceLeasedEngineAgents(
3399
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3400
+ ): Promise<void> {
3401
+ const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
3402
+ const channel = this.channels.get(request.taskId);
3403
+ // Every engine being created on one of the accounts (on any subject, a
3404
+ // task token included, for a whole run's), this attempt's or left over
3405
+ // from one that timed out: never only the bound agents'.
3406
+ const creating = [...(channel?.leasedCreating ?? [])]
3407
+ .filter(
3408
+ (entry) =>
3409
+ (!named || (entry.accountId !== null && named.has(entry.accountId))) && entry.runEpoch <= request.runEpoch,
3410
+ )
3411
+ .map((entry) => entry.settled);
3412
+ // Every retirement still pending that this cleanup covers: its stop
3413
+ // proves what they fence, so one that fails from now on is not escalated.
3414
+ const covered = this.coveredLeasedRetirements(request);
3415
+ if (channel) {
3416
+ for (const [agentId, accountId] of [...channel.accountByAgent]) {
3417
+ if (named && !named.has(accountId)) continue;
3418
+ if ((channel.leasedIdentityByAgent.get(agentId)?.runEpoch ?? 0) > request.runEpoch) continue;
3419
+ channel.accountByAgent.delete(agentId);
3420
+ this.quiesceLeasedAgent(channel, agentId);
3421
+ }
3422
+ // A whole run's cleanup takes every agent on its task token alone too
3423
+ // (the Managed Engine); a cleanup of named accounts never does.
3424
+ if (!named) {
3425
+ for (const [agentId, identity] of [...channel.leasedIdentityByAgent]) {
3426
+ if (identity.subject !== "task-token" || identity.runEpoch > request.runEpoch) continue;
3427
+ this.quiesceLeasedAgent(channel, agentId);
3428
+ }
3429
+ }
3430
+ }
3431
+ if (creating.length > 0) {
3432
+ let timer: ReturnType<typeof setTimeout> | undefined;
3433
+ const bounded = new Promise<"late">((resolve) => {
3434
+ timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
3435
+ });
3436
+ const outcome = await Promise.race([Promise.all(creating), bounded]);
3437
+ clearTimeout(timer);
3438
+ if (outcome === "late") {
3439
+ throw new Error("an engine being started on a removed AI account did not finish starting");
3440
+ }
3441
+ }
3442
+ // One of those under way is waited for, bounded, so nothing of it lands
3443
+ // after this cleanup's stop.
3444
+ const retiring = covered.flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
3445
+ if (retiring.length > 0) {
3446
+ let timer: ReturnType<typeof setTimeout> | undefined;
3447
+ const bounded = new Promise<"late">((resolve) => {
3448
+ timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
3449
+ });
3450
+ const outcome = await Promise.race([Promise.all(retiring), bounded]);
3451
+ clearTimeout(timer);
3452
+ if (outcome === "late") {
3453
+ throw new Error("an engine launch slot being retired in its runtime did not finish");
3454
+ }
3455
+ }
3456
+ const runtimes = await this.leasedCleanupRuntimes(request, channel);
3457
+ for (const environment of runtimes.values()) {
3458
+ // A runtime that is not running holds no process, unless it is frozen
3459
+ // (paused or restarting: its processes run again without a start);
3460
+ // one that is frozen or cannot say cannot be proven.
3461
+ const status = await environment.status();
3462
+ if (status.state !== "unknown" && status.frozen) {
3463
+ throw new Error("a task runtime's engines could not be proven stopped: it is paused");
3464
+ }
3465
+ if (status.state === "stopped" || status.state === "absent") continue;
3466
+ if (status.state !== "running") {
3467
+ throw new Error("a task runtime's engines could not be proven stopped: its state is unknown");
3468
+ }
3469
+ try {
3470
+ await stopAnchoredEngines(environment, {
3471
+ taskId: request.taskId,
3472
+ runEpoch: request.runEpoch,
3473
+ ...(request.credentialIds === undefined ? {} : { credentialIds: request.credentialIds }),
3474
+ });
3475
+ } catch (error) {
3476
+ if (!(error instanceof EngineAnchorLost)) throw error;
3477
+ // What a lost supervisor owned may have been reparented away from
3478
+ // it: only a stopped runtime proves none of it still runs.
3479
+ console.warn(`[orchestrator] ${request.taskId}: ${error.message}; stopping the runtime`);
3480
+ await environment.stop();
3481
+ const stopped = await environment.status();
3482
+ if (
3483
+ stopped.state === "unknown" ||
3484
+ stopped.frozen ||
3485
+ (stopped.state !== "stopped" && stopped.state !== "absent")
3486
+ ) {
3487
+ throw error;
3488
+ }
3489
+ }
3490
+ }
3491
+ // Every admission of these runs not yet used is revoked in every runtime
3492
+ // they were written in, or that runtime is stopped or gone and its gates
3493
+ // with it: no revocation or slot retirement retained for them is left to
3494
+ // do.
3495
+ dischargeEngineRevocations(request);
3496
+ this.dischargeLeasedRetirements(request);
3497
+ // Every engine of these runs is proven stopped in every runtime they
3498
+ // ran in: a whole-task cleanup lets the record of those runtimes go.
3499
+ if (request.credentialIds === undefined) {
3500
+ forgetLeasedEngineRuntimes(request.taskId, request.runEpoch, [...runtimes.keys()]);
3501
+ }
3502
+ }
3503
+
3504
+ /** A cleanup took the material `agentId` runs on: it is unbound at once,
3505
+ * so nothing launches on it again, its session is asked to stop, and it
3506
+ * starts again on what the task holds once the gate opens. */
3507
+ private quiesceLeasedAgent(channel: Channel, agentId: string): void {
3508
+ channel.leasedIdentityByAgent.delete(agentId);
3509
+ const session = channel.sessions.get(agentId);
3510
+ if (this.isActiveChannel(channel)) channel.leasedRespawn.add(agentId);
3511
+ if (!session) return;
3512
+ // Delete first: the old session's last events are generation-stale.
3513
+ channel.sessions.delete(agentId);
3514
+ channel.browserStaleSessions.delete(agentId);
3515
+ channel.browserPendingStaleSessions.delete(agentId);
3516
+ this.retainRetiredTurnBoundary(
3517
+ channel,
3518
+ agentId,
3519
+ this.needsAbortedTurnBoundary(channel, agentId),
3520
+ );
3521
+ this.retireRunnerGeneration(channel, agentId);
3522
+ void session.close().catch(() => {});
3523
+ }
3524
+
3525
+ /**
3526
+ * ADR-154 §6: an account is about to be served at a newer version. In
3527
+ * every runtime a launch of this run could start in, its older versions
3528
+ * are held back first: a launch admitted on one that has not started yet
3529
+ * never will. Rejects, so the install fails and none of the newer version
3530
+ * is served, when a runtime cannot be reached or proven not running.
3531
+ */
3532
+ private async raiseLeasedAdmissionFloors(
3533
+ assignment: { taskId: string; runEpoch: number },
3534
+ floors: readonly EngineAdmissionFloor[],
3535
+ ): Promise<void> {
3536
+ const channel = this.channels.get(assignment.taskId);
3537
+ const runtimes = await this.leasedCleanupRuntimes(
3538
+ { taskId: assignment.taskId, runEpoch: assignment.runEpoch },
3539
+ channel,
3540
+ );
3541
+ for (const environment of runtimes.values()) {
3542
+ const status = await environment.status();
3543
+ if (status.state !== "unknown" && status.frozen) {
3544
+ throw new Error("an AI account's older version could not be held back: a task runtime is paused");
3545
+ }
3546
+ if (status.state === "stopped" || status.state === "absent") continue;
3547
+ if (status.state !== "running") {
3548
+ throw new Error("an AI account's older version could not be held back: a task runtime's state is unknown");
3549
+ }
3550
+ await raiseEngineAdmissionFloors(environment, {
3551
+ taskId: assignment.taskId,
3552
+ runEpoch: assignment.runEpoch,
3553
+ floors,
3554
+ });
3555
+ }
3556
+ }
3557
+
3558
+ /**
3559
+ * Every runtime a cleanup reaches, by locator: the task's current one,
3560
+ * every one a leased engine of these runs was started in (recorded before
3561
+ * it started, files or none), and every one the inventory names for these
3562
+ * runs (an older runtime's files are deleted from that exact runtime).
3563
+ */
3564
+ private async leasedCleanupRuntimes(
3565
+ request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
3566
+ channel: Channel | undefined,
3567
+ ): Promise<Map<string, TaskEnvironmentHandle<unknown>>> {
3568
+ const runtimes = new Map<string, TaskEnvironmentHandle<unknown>>();
3569
+ const current =
3570
+ channel?.agentHandle ??
3571
+ (await (async () => {
3572
+ const task = getHostTask(request.taskId);
3573
+ return task ? reconstructPersistedTaskEnvironment(task) : null;
3574
+ })());
3575
+ if (current) runtimes.set(serializeTaskEnvironmentLocator(current.descriptor.locator), current);
3576
+ const recorded = [
3577
+ ...listLeasedEngineRuntimes(request.taskId),
3578
+ ...listCredentialCleanupPaths(request.taskId),
3579
+ ];
3580
+ for (const row of recorded) {
3581
+ if (row.runEpoch > request.runEpoch || runtimes.has(row.runtimeLocator)) continue;
3582
+ const environment = await reconstructTaskEnvironment(
3583
+ parseTaskEnvironmentLocator(row.runtimeLocator),
3584
+ "loader",
3585
+ );
3586
+ if (environment.descriptor.taskId !== request.taskId) {
3587
+ throw new Error("a cleanup runtime belongs to a different task");
3588
+ }
3589
+ runtimes.set(row.runtimeLocator, environment);
3590
+ }
3591
+ return runtimes;
3592
+ }
3593
+
2414
3594
  /**
2415
3595
  * Per-agent CODEX_HOME (see the call site). Returns `args` unchanged unless
2416
3596
  * this is a codex agent with a config-dir CODEX_HOME; then it seeds
@@ -2481,87 +3661,347 @@ export class Orchestrator {
2481
3661
  channel.reconcileAgain = true;
2482
3662
  return null;
2483
3663
  }
2484
- // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
2485
- // account share one CODEX_HOME, and codex's new sqlite state runtime
2486
- // cannot be opened by two concurrent processes — the second agent dies
2487
- // with "failed to initialize sqlite state runtime under <CODEX_HOME>"
2488
- // (live 2026-09-04, a task with two codex agents on one account). Give
2489
- // each codex agent its OWN CODEX_HOME, seeded once from the resolved
2490
- // account's dir, so their state runtimes never collide. Single-codex
2491
- // tasks are unaffected beyond a one-time in-container copy.
2492
- // Only codex agents take the isolation path; everything else uses args
2493
- // directly, WITHOUT an extra await — a stray microtask here reordered the
2494
- // Claude re-spawn's prompt re-delivery.
2495
- const boundArgs =
2496
- args.agent?.kind === "codex"
2497
- ? await this.isolateCodexHome(environment, agentId, args)
2498
- : args;
2499
- while (this.isActiveChannel(channel)) {
2500
- if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2501
- channel.reconcileAgain = true;
2502
- return null;
2503
- }
2504
- if (
2505
- channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
2506
- ) {
2507
- channel.reconcileAgain = true;
2508
- return null;
2509
- }
2510
- if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2511
- channel.reconcileAgain = true;
2512
- return null;
2513
- }
2514
- const generation = channel.configGeneration;
2515
- const session = await this.factory.create({ ...boundArgs, environment });
2516
- if (!this.isActiveChannel(channel)) {
2517
- await session.close().catch(() => {});
2518
- return null;
2519
- }
2520
- if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
2521
- await session.close().catch(() => {});
2522
- channel.reconcileAgain = true;
2523
- return null;
2524
- }
2525
- const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
2526
- if (pendingRosterClose?.state === "pending-close") {
2527
- await this.closeRosterChangedSession(
2528
- channel,
2529
- agentId,
2530
- session,
2531
- pendingRosterClose,
2532
- );
2533
- channel.reconcileAgain = true;
3664
+ // ADR-154: an agent on the runner's own accounts holds them from here
3665
+ // until it is bound or given up. Leaving them waits for it; every process
3666
+ // it starts, now or at a later turn, is refused at its provider's last
3667
+ // moment once leaving has begun; and one created across that moment is
3668
+ // closed, never bound. Its next start is on the task's lease. The hold
3669
+ // ends in a `finally`, which adds no await (see below).
3670
+ const ownHold = this.isLeasedChannel(channel) ? undefined : holdOwnCredentials();
3671
+ if (ownHold === null) {
3672
+ channel.reconcileAgain = true;
3673
+ return null;
3674
+ }
3675
+ const onOwn = ownHold !== undefined;
3676
+ try {
3677
+ // ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
3678
+ // account share one CODEX_HOME, and codex's new sqlite state runtime
3679
+ // cannot be opened by two concurrent processes — the second agent dies
3680
+ // with "failed to initialize sqlite state runtime under <CODEX_HOME>"
3681
+ // (live 2026-09-04, a task with two codex agents on one account). Give
3682
+ // each codex agent its OWN CODEX_HOME, seeded once from the resolved
3683
+ // account's dir, so their state runtimes never collide. Single-codex
3684
+ // tasks are unaffected beyond a one-time in-container copy.
3685
+ // Only codex agents take the isolation path; everything else uses args
3686
+ // directly, WITHOUT an extra await — a stray microtask here reordered the
3687
+ // Claude re-spawn's prompt re-delivery. A leased account is applied here
3688
+ // too, synchronously when nothing has to be waited for or written.
3689
+ let boundArgs = args;
3690
+ let identity: LeasedIdentity | undefined;
3691
+ const kind = args.agent?.kind ?? "";
3692
+ if (!onOwn && LEASED_ENGINE_KINDS.has(kind)) {
3693
+ // §6: a runtime that cannot start a launch under the engine supervisor
3694
+ // never runs a leased engine.
3695
+ if (environment.anchors !== true) {
3696
+ this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
3697
+ return null;
3698
+ }
3699
+ const leased = this.leasedSpawnArgs(channel, agentId, args, environment);
3700
+ const applied = leased instanceof Promise ? await leased : leased;
3701
+ // Down until the gate next opens: never a loop.
3702
+ if (!applied) return null;
3703
+ boundArgs = applied.args;
3704
+ identity = applied.identity;
3705
+ } else if (!onOwn && kind === MANAGED_ENGINE_KIND) {
3706
+ // ADR-128, ADR-154 §6: no AI account, but the agent's task token is
3707
+ // the engine's bearer: started under the supervisor, admitted as
3708
+ // that token on the task's open run, or not at all.
3709
+ if (environment.anchors !== true) {
3710
+ this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
3711
+ return null;
3712
+ }
3713
+ const leased = this.leasedTaskTokenSpawnArgs(channel, agentId, args);
3714
+ const applied = leased instanceof Promise ? await leased : leased;
3715
+ if (!applied) return null;
3716
+ boundArgs = applied.args;
3717
+ identity = applied.identity;
3718
+ } else if (!onOwn && !CREDENTIAL_FREE_KINDS.has(kind)) {
3719
+ // ADR-154 §10: a kind with no lane (Kimi) never runs on the runner's
3720
+ // own sign-in for a leased task.
3721
+ this.leasedUnavailable(channel, agentId, `${kind} has no AI account lane on cloud credentials`);
2534
3722
  return null;
3723
+ } else if (kind === "codex") {
3724
+ boundArgs = await this.isolateCodexHome(environment, agentId, args);
2535
3725
  }
2536
- if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
2537
- const change = channel.rosterSessionChanges.get(agentId);
2538
- if (change) {
2539
- await this.closeRosterChangedSession(
2540
- channel,
2541
- agentId,
2542
- session,
2543
- change,
3726
+ while (this.isActiveChannel(channel)) {
3727
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
3728
+ channel.reconcileAgain = true;
3729
+ return null;
3730
+ }
3731
+ if (
3732
+ channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
3733
+ ) {
3734
+ channel.reconcileAgain = true;
3735
+ return null;
3736
+ }
3737
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
3738
+ channel.reconcileAgain = true;
3739
+ return null;
3740
+ }
3741
+ if (onOwn && !onOwnCredentials()) {
3742
+ channel.reconcileAgain = true;
3743
+ return null;
3744
+ }
3745
+ const generation = channel.configGeneration;
3746
+ // ADR-154 §6: never start an engine on lease material the task does not
3747
+ // hold right now (its run, account, revision and version). A cleanup
3748
+ // drops the binding synchronously, so a spawn whose account it took
3749
+ // stops here; one already creating is registered, and every cleanup
3750
+ // attempt waits for it.
3751
+ if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
3752
+ this.retryLeasedStart(channel, agentId);
3753
+ return null;
3754
+ }
3755
+ if (identity) {
3756
+ // Where this engine runs, before it can run there (§6): a cleanup
3757
+ // proves it stopped in this runtime even if it leaves no file.
3758
+ try {
3759
+ rememberLeasedEngineRuntime({
3760
+ taskId: channel.taskId,
3761
+ runEpoch: identity.runEpoch,
3762
+ runtime: environment.descriptor.locator,
3763
+ });
3764
+ } catch (err) {
3765
+ console.warn(
3766
+ `[orchestrator] ${channel.taskId}/${agentId}: not starting on its AI account, its runtime could not be recorded: ${err instanceof Error ? err.message : String(err)}`,
3767
+ );
3768
+ return null;
3769
+ }
3770
+ }
3771
+ // Every process the adapter launches for this engine (now, at each
3772
+ // turn, or once its durable runner is deployed) is admitted then, on
3773
+ // this exact material, for this spawn while it is the agent's current
3774
+ // one, and started under the supervisor (§6). The agent's spawn this
3775
+ // one replaces is retired.
3776
+ const slot = identity ? newLeasedLaunchSlot(channel.taskId, identity, environment) : null;
3777
+ if (slot) {
3778
+ void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
3779
+ channel.leasedLaunches.set(agentId, slot);
3780
+ }
3781
+ const creating = this.factory.create({
3782
+ ...boundArgs,
3783
+ environment:
3784
+ identity && slot
3785
+ ? this.leasedLaunchEnvironment(channel, { agentId, slot }, identity, environment)
3786
+ : onOwn
3787
+ ? onOwnCredentialsEnvironment(environment)
3788
+ : environment,
3789
+ });
3790
+ // Settled before this spawn's own continuation: from here until it
3791
+ // binds, nothing launches on it.
3792
+ if (slot) {
3793
+ void creating.then(
3794
+ () => {
3795
+ slot.created = true;
3796
+ },
3797
+ () => {
3798
+ slot.created = true;
3799
+ },
2544
3800
  );
2545
- } else {
2546
- await session.close().catch(() => {});
2547
3801
  }
2548
- channel.reconcileAgain = true;
2549
- return null;
2550
- }
2551
- if (generation === channel.configGeneration) {
2552
- // Keep the final generation check, stale cleanup, and map bind in one
2553
- // synchronous continuation. A config writer that resumes immediately
2554
- // afterward now sees this session and marks it stale; it can no longer
2555
- // change the generation in the await→bind gap while the process is
2556
- // invisible.
2557
- channel.browserStaleSessions.delete(agentId);
2558
- channel.browserPendingStaleSessions.delete(agentId);
2559
- this.bindSession(channel, agentId, session);
2560
- return session;
3802
+ if (identity) {
3803
+ const entry = {
3804
+ agentId,
3805
+ accountId: identity.subject === "account" ? identity.credentialId : null,
3806
+ runEpoch: identity.runEpoch,
3807
+ settled: creating.then(
3808
+ () => undefined,
3809
+ () => undefined,
3810
+ ),
3811
+ };
3812
+ channel.leasedCreating.add(entry);
3813
+ void entry.settled.then(() => channel.leasedCreating.delete(entry));
3814
+ }
3815
+ // ADR-154 §6: a spawn that does not bind (its create failed, maybe
3816
+ // after it launched, or its session is closed unbound) is retired at
3817
+ // once, before anything else: a launch it handed over that has not
3818
+ // started in its runtime never does.
3819
+ const unbound = (): Promise<void> =>
3820
+ slot ? this.retireLeasedLaunch(channel, agentId, slot) : Promise.resolve();
3821
+ let session: AgentSession;
3822
+ try {
3823
+ session = await creating;
3824
+ } catch (error) {
3825
+ await unbound();
3826
+ throw error;
3827
+ }
3828
+ if (!this.isActiveChannel(channel)) {
3829
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3830
+ return null;
3831
+ }
3832
+ // ADR-154: created on the runner's own accounts across the moment
3833
+ // leaving them began: closed, never bound.
3834
+ if (onOwn && !onOwnCredentials()) {
3835
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3836
+ channel.reconcileAgain = true;
3837
+ return null;
3838
+ }
3839
+ // ADR-154 §6: a cleanup that took this account while the session was
3840
+ // being created quiesced the agent already; never keep a session on
3841
+ // material the task no longer holds.
3842
+ if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
3843
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3844
+ this.retryLeasedStart(channel, agentId);
3845
+ return null;
3846
+ }
3847
+ if (identity) channel.leasedBlocked.delete(agentId);
3848
+ if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
3849
+ await Promise.all([unbound(), session.close().catch(() => {})]);
3850
+ channel.reconcileAgain = true;
3851
+ return null;
3852
+ }
3853
+ const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
3854
+ if (pendingRosterClose?.state === "pending-close") {
3855
+ await Promise.all([
3856
+ unbound(),
3857
+ this.closeRosterChangedSession(channel, agentId, session, pendingRosterClose),
3858
+ ]);
3859
+ channel.reconcileAgain = true;
3860
+ return null;
3861
+ }
3862
+ if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
3863
+ const change = channel.rosterSessionChanges.get(agentId);
3864
+ const retiring = unbound();
3865
+ if (change) {
3866
+ await this.closeRosterChangedSession(
3867
+ channel,
3868
+ agentId,
3869
+ session,
3870
+ change,
3871
+ );
3872
+ } else {
3873
+ await session.close().catch(() => {});
3874
+ }
3875
+ await retiring;
3876
+ channel.reconcileAgain = true;
3877
+ return null;
3878
+ }
3879
+ if (generation === channel.configGeneration) {
3880
+ // Keep the final generation check, stale cleanup, and map bind in one
3881
+ // synchronous continuation. A config writer that resumes immediately
3882
+ // afterward now sees this session and marks it stale; it can no longer
3883
+ // change the generation in the await→bind gap while the process is
3884
+ // invisible.
3885
+ channel.browserStaleSessions.delete(agentId);
3886
+ channel.browserPendingStaleSessions.delete(agentId);
3887
+ if (slot) slot.session = session;
3888
+ this.bindSession(channel, agentId, session);
3889
+ return session;
3890
+ }
3891
+ await Promise.all([unbound(), session.close().catch(() => {})]);
2561
3892
  }
2562
- await session.close().catch(() => {});
3893
+ return null;
3894
+ } finally {
3895
+ ownHold?.();
2563
3896
  }
2564
- return null;
3897
+ }
3898
+
3899
+ /**
3900
+ * ADR-154, the shadow router's one exec: the only use it makes of an AI
3901
+ * account, and the only place that account's material exists for it.
3902
+ * Null when it cannot run, and the pass routes by rules.
3903
+ * - A leased task's: its account is chosen, and the exec registered where
3904
+ * a cleanup of that account waits, in one synchronous step before any of
3905
+ * its material is read; the registration settles when the exec does,
3906
+ * and nothing the material is in outlives this call (§6). A cleanup that
3907
+ * took the account first leaves it unchosen. The exec starts under the
3908
+ * engine supervisor, only on material the task still holds at the
3909
+ * provider's last moment.
3910
+ * - On the runner's own credentials: they are held from the account's
3911
+ * choice to the exec's end, and the exec is refused at the provider's
3912
+ * last moment once leaving them has begun or the channel has closed.
3913
+ */
3914
+ private async shadowClassify(
3915
+ channel: Channel,
3916
+ agentId: string,
3917
+ kind: string,
3918
+ handle: TaskEnvironmentHandle<unknown>,
3919
+ argv: [string, ...string[]],
3920
+ scratchDir: string,
3921
+ timeoutMs: number,
3922
+ ): Promise<{ exitCode: number | null; stdout: string } | null> {
3923
+ const request = (env: Record<string, string>, admit: () => void) => ({
3924
+ argv: agentCliArgv(handle, argv),
3925
+ projectEnvProfile: "task" as const,
3926
+ cwd: scratchDir,
3927
+ user: "node",
3928
+ env,
3929
+ timeoutMs,
3930
+ maxOutputBytes: 64 * 1024,
3931
+ admit,
3932
+ });
3933
+ if (!this.isLeasedChannel(channel)) {
3934
+ const running = whileOnOwnCredentials(async () => {
3935
+ const env = { ...(pickEngineAccount(kind, { scope: channel.engineAccountScope })?.execEnv ?? {}) };
3936
+ const r = await handle.exec(
3937
+ request(env, () => {
3938
+ if (!this.isActiveChannel(channel) || !onOwnCredentials()) throw new LeftOwnCredentialsError();
3939
+ }),
3940
+ );
3941
+ return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
3942
+ });
3943
+ if (running === null) return null;
3944
+ this.trackEventOperation(channel, running);
3945
+ return running;
3946
+ }
3947
+ const lease = leasedEnginesFor(channel.taskId);
3948
+ if (!lease?.open) return null;
3949
+ const account = this.leasedAccountFor(channel, agentId, kind, lease);
3950
+ if (!account) return null;
3951
+ const identity: LeasedIdentity = {
3952
+ subject: "account",
3953
+ runEpoch: lease.assignment.runEpoch,
3954
+ credentialId: account.credentialId,
3955
+ revision: account.revision,
3956
+ version: account.version,
3957
+ };
3958
+ let settle!: () => void;
3959
+ const entry = {
3960
+ agentId: `shadow:${agentId}`,
3961
+ accountId: account.credentialId,
3962
+ runEpoch: identity.runEpoch,
3963
+ settled: new Promise<void>((resolve) => (settle = resolve)),
3964
+ };
3965
+ channel.leasedCreating.add(entry);
3966
+ void entry.settled.then(() => channel.leasedCreating.delete(entry));
3967
+ this.trackEventOperation(channel, entry.settled);
3968
+ try {
3969
+ const env = await this.leasedOneShotEnv(channel, lease, account, handle);
3970
+ if (!env || !this.leasedMaterialServed(channel, identity)) return null;
3971
+ const r = await this.leasedLaunchEnvironment(channel, null, identity, handle).exec(
3972
+ request(env, () => {
3973
+ if (!this.isActiveChannel(channel) || !this.leasedMaterialServed(channel, identity)) {
3974
+ throw new CredentialLeaseUnavailableError("the shadow classifier's account moved");
3975
+ }
3976
+ }),
3977
+ );
3978
+ return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
3979
+ } finally {
3980
+ settle();
3981
+ }
3982
+ }
3983
+
3984
+ /** #cb3j0z: the task's runtime, for what only it can do in the task's
3985
+ * workspace without following its links (macOS); null without one. */
3986
+ private async workspaceRuntime(taskId: string): Promise<RuntimeNamespace | null> {
3987
+ const channel = this.channels.get(taskId);
3988
+ if (!channel || !this.isActiveChannel(channel)) return null;
3989
+ try {
3990
+ return runtimeNamespaceOf(await this.agentEnvironment(channel));
3991
+ } catch {
3992
+ return null;
3993
+ }
3994
+ }
3995
+
3996
+ /** Teardown drains `operation` before the task's runtime name can be
3997
+ * reused. */
3998
+ private trackEventOperation(channel: Channel, operation: Promise<unknown>): void {
3999
+ const settled = operation.then(
4000
+ () => undefined,
4001
+ () => undefined,
4002
+ );
4003
+ channel.eventOperations.add(settled);
4004
+ void settled.then(() => channel.eventOperations.delete(settled));
2565
4005
  }
2566
4006
 
2567
4007
  private hasDesiredLaunchSettings(
@@ -2808,6 +4248,9 @@ export class Orchestrator {
2808
4248
  ),
2809
4249
  },
2810
4250
  );
4251
+ // Credentials unavailable is this agent's alone: the gate's next
4252
+ // opening starts it; its siblings start now.
4253
+ if (!session && channel.leasedBlocked.has(agent.id)) continue;
2811
4254
  if (!session) {
2812
4255
  // The agents that did come up are usable; the reconcile pass keeps
2813
4256
  // retrying this one. Do not leave the page covered for a session
@@ -3203,13 +4646,16 @@ export class Orchestrator {
3203
4646
  // through this same command, and classifying them would poison the very
3204
4647
  // data the shadow pass exists to collect.
3205
4648
  if (channel.routingShadow && origin === "human") {
3206
- void this.shadowRouteDelivery(channel, agentId, text).catch((err) => {
4649
+ // Its exec runs in the task's runtime: teardown drains it like any
4650
+ // other external operation before the runtime's name can be reused.
4651
+ const pass = this.shadowRouteDelivery(channel, agentId, text).catch((err) => {
3207
4652
  console.warn(
3208
4653
  `[routing] shadow pass failed for ${channel.taskId}/${agentId}: ${
3209
4654
  err instanceof Error ? err.message : String(err)
3210
4655
  }`,
3211
4656
  );
3212
4657
  });
4658
+ this.trackEventOperation(channel, pass);
3213
4659
  }
3214
4660
  return { ok: true };
3215
4661
  }
@@ -3219,7 +4665,10 @@ export class Orchestrator {
3219
4665
  * one-shot executes INSIDE the task environment as the task user, with the
3220
4666
  * same engine account the agent runs on (so the user's own subscription
3221
4667
  * pays for routing, never Uai), then the result rides a host event. Bounded
3222
- * by a hard timeout so a hung CLI cannot pile up passes.
4668
+ * by a hard timeout, the task's own files read included, so a hung CLI or
4669
+ * read cannot pile up passes. ADR-154: the pass itself holds no credential
4670
+ * (its reads are the task's own files, which take as long as the task
4671
+ * makes them); only its one exec does (`shadowClassify`).
3223
4672
  */
3224
4673
  private async shadowRouteDelivery(channel: Channel, agentId: string, text: string): Promise<void> {
3225
4674
  const agent = channel.roster.find((a) => a.id === agentId);
@@ -3236,39 +4685,44 @@ export class Orchestrator {
3236
4685
  // Rung 1 hosts the classifier on the agent's own engine; the Managed
3237
4686
  // Engine delegates to claude/codex, so borrow the delegate's account.
3238
4687
  const hostEngine = agent.kind === "uai" ? (agent.uaiProvider === "openai" ? "codex" : "claude") : agent.kind;
3239
- const account = pickEngineAccount(hostEngine, { scope: channel.engineAccountScope });
3240
- const env: Record<string, string> = { ...(account?.execEnv ?? {}) };
3241
- const context =
3242
- handle && canRead
3243
- ? Buffer.from((await handle.readWorkspaceFile(`${scratchDir}/chat.md`).catch(() => null)) ?? new Uint8Array())
3244
- .toString("utf8")
3245
- .slice(-1200)
3246
- : "";
3247
4688
  const timeoutMs = 45_000;
4689
+ let timer: ReturnType<typeof setTimeout> | undefined;
4690
+ // Past its bound the pass is given up, and what is left of it (a read
4691
+ // the task held) launches nothing.
4692
+ let abandoned = false;
3248
4693
  const result = await Promise.race([
3249
- shadowRoute(text, context, hostEngine, {
3250
- scratchDir,
3251
- exec: async (argv) => {
3252
- if (!handle || !canExec) return null;
3253
- const r = await handle.exec({
3254
- argv: agentCliArgv(handle, argv),
3255
- projectEnvProfile: "task",
3256
- cwd: scratchDir,
3257
- user: "node",
3258
- env,
3259
- timeoutMs,
3260
- maxOutputBytes: 64 * 1024,
3261
- });
3262
- return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
3263
- },
3264
- readFile: async (path) => {
3265
- if (!handle || !canRead) return null;
3266
- const bytes = await handle.readWorkspaceFile(path).catch(() => null);
3267
- return bytes ? Buffer.from(bytes).toString("utf8") : null;
3268
- },
4694
+ (async () => {
4695
+ const context =
4696
+ handle && canRead
4697
+ ? Buffer.from(
4698
+ (await handle
4699
+ .readWorkspaceFile(`${scratchDir}/chat.md`, { maxBytes: SHADOW_CONTEXT_BYTES, from: "end" })
4700
+ .catch(() => null)) ?? new Uint8Array(),
4701
+ )
4702
+ .toString("utf8")
4703
+ .slice(-1200)
4704
+ : "";
4705
+ return shadowRoute(text, context, hostEngine, {
4706
+ scratchDir,
4707
+ exec: (argv) =>
4708
+ handle && canExec && !abandoned
4709
+ ? this.shadowClassify(channel, agentId, hostEngine, handle, argv, scratchDir, timeoutMs)
4710
+ : Promise.resolve(null),
4711
+ readFile: async (path) => {
4712
+ if (!handle || !canRead) return null;
4713
+ const bytes = await handle.readWorkspaceFile(path, { maxBytes: SHADOW_OUTPUT_BYTES }).catch(() => null);
4714
+ return bytes ? Buffer.from(bytes).toString("utf8") : null;
4715
+ },
4716
+ });
4717
+ })(),
4718
+ new Promise<null>((resolve) => {
4719
+ timer = setTimeout(() => {
4720
+ abandoned = true;
4721
+ resolve(null);
4722
+ }, timeoutMs + 5_000);
3269
4723
  }),
3270
- new Promise<null>((resolve) => setTimeout(() => resolve(null), timeoutMs + 5_000)),
3271
4724
  ]);
4725
+ clearTimeout(timer);
3272
4726
  if (!result) {
3273
4727
  console.warn(`[routing] shadow pass timed out for ${channel.taskId}/${agentId}`);
3274
4728
  return;
@@ -3507,14 +4961,21 @@ export class Orchestrator {
3507
4961
  if (ownsBrokerSlot) channel.spawning.add(agentId);
3508
4962
  let requested = false;
3509
4963
  try {
3510
- requested = await requestCredentialRefresh({
3511
- kind: agent.kind,
3512
- accountId:
3513
- accountId && !accountId.startsWith("default:")
3514
- ? accountId
3515
- : null,
3516
- scope: channel.engineAccountScope,
3517
- });
4964
+ // ADR-154 §7: a leased account is refreshed by the cloud's
4965
+ // broker, through the task's lease, never by this runner.
4966
+ const lease = this.isLeasedChannel(channel) ? leasedEnginesFor(channel.taskId) : undefined;
4967
+ requested = this.isLeasedChannel(channel)
4968
+ ? lease !== undefined &&
4969
+ accountId !== null &&
4970
+ (await refreshLeasedEngineAccount(lease.assignment, accountId)) === "refreshed"
4971
+ : await requestCredentialRefresh({
4972
+ kind: agent.kind,
4973
+ accountId:
4974
+ accountId && !accountId.startsWith("default:")
4975
+ ? accountId
4976
+ : null,
4977
+ scope: channel.engineAccountScope,
4978
+ });
3518
4979
  } catch (err) {
3519
4980
  console.warn(
3520
4981
  `[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
@@ -3825,10 +5286,16 @@ export class Orchestrator {
3825
5286
 
3826
5287
  channel.spawning.add(agentId);
3827
5288
  try {
3828
- const restored = await repairClaudeConfigInContainer(
3829
- channel.containerName,
3830
- channel.agentHandle ?? undefined,
3831
- );
5289
+ // ADR-154 §6: never the runner owner's own ~/.claude.json into a task on
5290
+ // cloud credentials; Claude writes its own again when it starts. A
5291
+ // repair on the runner's own credentials is counted: leaving them waits
5292
+ // for it, and none starts once leaving has begun.
5293
+ const repairing = this.isLeasedChannel(channel)
5294
+ ? null
5295
+ : whileOnOwnCredentials(() =>
5296
+ repairClaudeConfigInContainer(channel.containerName, channel.agentHandle ?? undefined),
5297
+ );
5298
+ const restored = repairing === null ? false : await repairing;
3832
5299
  if (channel.rosterSessionChanges.has(agentId)) {
3833
5300
  this.reconcileErroredRosterChange(
3834
5301
  channel,
@@ -3896,7 +5363,7 @@ export class Orchestrator {
3896
5363
  );
3897
5364
  const boundAccountId = channel.accountByAgent.get(agentId);
3898
5365
  const boundAccount = boundAccountId
3899
- ? resolveEngineAccounts(agent.kind, {}, channel.engineAccountScope).find(
5366
+ ? this.engineAccounts(channel, agent.kind).find(
3900
5367
  (account) => account.id === boundAccountId,
3901
5368
  )
3902
5369
  : undefined;
@@ -3942,9 +5409,11 @@ export class Orchestrator {
3942
5409
  }
3943
5410
  this.startNextPendingPrompt(channel, agentId);
3944
5411
 
3945
- const message = restored
3946
- ? `${agentId} restarted — config file was missing in the container, restored from the host.`
3947
- : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
5412
+ const message = this.isLeasedChannel(channel)
5413
+ ? `${agentId} restarted — config file was missing in the container; Claude writes its own again.`
5414
+ : restored
5415
+ ? `${agentId} restarted — config file was missing in the container, restored from the host.`
5416
+ : `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
3948
5417
  channel.retiredTurnBoundaries.delete(agentId);
3949
5418
  this.emitHost({
3950
5419
  kind: "agent.exit",
@@ -4002,7 +5471,7 @@ export class Orchestrator {
4002
5471
  // A concurrent terminal event for the same session is already being
4003
5472
  // handled by the path that owns this spawn slot.
4004
5473
  if (channel.spawning.has(agentId)) return true;
4005
- if (resolveEngineAccounts(agent.kind, {}, channel.engineAccountScope).length < 2) {
5474
+ if (this.engineAccounts(channel, agent.kind).length < 2) {
4006
5475
  return false;
4007
5476
  }
4008
5477
 
@@ -4012,10 +5481,11 @@ export class Orchestrator {
4012
5481
 
4013
5482
  const current = channel.accountByAgent.get(agentId);
4014
5483
  if (current) cooldownEngineAccount(current);
4015
- const next = pickEngineAccount(agent.kind, {
4016
- scope: channel.engineAccountScope,
4017
- exclude: current ? new Set([current]) : undefined,
4018
- });
5484
+ const next = this.pickAccount(
5485
+ channel,
5486
+ agent.kind,
5487
+ current ? new Set([current]) : undefined,
5488
+ );
4019
5489
  if (!next) return false;
4020
5490
 
4021
5491
  const task = getHostTask(channel.taskId);
@@ -4447,7 +5917,7 @@ export class Orchestrator {
4447
5917
  return;
4448
5918
  }
4449
5919
 
4450
- const accounts = resolveEngineAccounts(agent.kind, {}, channel.engineAccountScope);
5920
+ const accounts = this.engineAccounts(channel, agent.kind);
4451
5921
  const account =
4452
5922
  accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
4453
5923
  accounts[0];
@@ -4482,14 +5952,19 @@ export class Orchestrator {
4482
5952
  async refreshEngineAccountCopies(kind: string): Promise<void> {
4483
5953
  for (const channel of this.channels.values()) {
4484
5954
  if (!this.isActiveChannel(channel)) continue;
5955
+ // ADR-154: a leased task's accounts change through its lease only.
5956
+ if (this.isLeasedChannel(channel)) continue;
4485
5957
  if (!channel.roster.some((agent) => agent.kind === kind)) continue;
4486
5958
  try {
4487
- await provisionEngineAccounts(
4488
- channel.containerName,
4489
- [kind],
4490
- {},
4491
- channel.agentHandle ?? undefined,
4492
- channel.engineAccountScope,
5959
+ // Only while the runner is on its own credentials, counted.
5960
+ await whileOnOwnCredentials(() =>
5961
+ provisionEngineAccounts(
5962
+ channel.containerName,
5963
+ [kind],
5964
+ {},
5965
+ channel.agentHandle ?? undefined,
5966
+ channel.engineAccountScope,
5967
+ ),
4493
5968
  );
4494
5969
  } catch (err) {
4495
5970
  console.warn(
@@ -4504,6 +5979,7 @@ export class Orchestrator {
4504
5979
  async refreshEngineAccountAgents(kind: string): Promise<void> {
4505
5980
  for (const channel of this.channels.values()) {
4506
5981
  if (!this.isActiveChannel(channel)) continue;
5982
+ if (this.isLeasedChannel(channel)) continue;
4507
5983
  const agentIds = channel.roster
4508
5984
  .filter((agent) => agent.kind === kind)
4509
5985
  .map((agent) => agent.id);
@@ -4561,7 +6037,16 @@ export class Orchestrator {
4561
6037
  // can repopulate an orphan channel while its sessions close.
4562
6038
  ch.closed = true;
4563
6039
  this.channels.delete(taskId);
6040
+ // ADR-154 §6: nothing its agents' spawns were admitted for starts after:
6041
+ // the close ends only once every retirement of the task under way, its
6042
+ // own and any an earlier event began, is done or escalated.
6043
+ for (const [agentId, slot] of [...ch.leasedLaunches]) void this.retireLeasedLaunch(ch, agentId, slot);
6044
+ const retirements = this.leasedRetirementsUnderWay(taskId);
4564
6045
  this.clearBrowserRetry(ch);
6046
+ if (ch.leasedSyncRetry) {
6047
+ clearTimeout(ch.leasedSyncRetry);
6048
+ ch.leasedSyncRetry = null;
6049
+ }
4565
6050
  ch.browserStaleSessions.clear();
4566
6051
  ch.rosterSessionChanges.clear();
4567
6052
  ch.browserPendingStaleSessions.clear();
@@ -4586,6 +6071,7 @@ export class Orchestrator {
4586
6071
  ch.sessionsReady,
4587
6072
  ...ch.factoryOperations,
4588
6073
  ...ch.eventOperations,
6074
+ ...retirements,
4589
6075
  ].filter((promise): promise is Promise<unknown> => promise !== null);
4590
6076
  const sessions = [...ch.sessions.values()];
4591
6077
  ch.sessions.clear();
@@ -4683,28 +6169,103 @@ export class Orchestrator {
4683
6169
  return this.#channelActivity.get(taskId) ?? null;
4684
6170
  }
4685
6171
 
4686
- async stopTask(taskId: string): Promise<{ ok: boolean; error?: string }> {
6172
+ async stopTask(
6173
+ taskId: string,
6174
+ options: {
6175
+ admitAssignment?: () => TaskStopAssignmentAdmission;
6176
+ shouldVeto?: () => boolean;
6177
+ /** ADR-154 §6: the closed run's credentials cleaned up, its sessions
6178
+ * closed and its runtime still up for the cleanup to reach, before
6179
+ * the runtime stops and Stop is acknowledged. */
6180
+ closeCredentials?: (runEpoch: number) => Promise<void>;
6181
+ } = {},
6182
+ ): Promise<OrchestratorTaskStopResult> {
4687
6183
  return this.runTaskLifecycle(taskId, () =>
4688
- this.stopTaskWithinLifecycle(taskId),
6184
+ this.stopTaskWithinLifecycle(taskId, options),
4689
6185
  );
4690
6186
  }
4691
6187
 
6188
+ /** `stopTask` for a caller that already holds this task's lifecycle slot
6189
+ * (runTaskLifecycle), so that what it checks before the stop and does
6190
+ * after it share the slot: ADR-154 §13's retirement re-proves the engine
6191
+ * first, and removes the runtime after. */
6192
+ async stopTaskInHeldLifecycle(
6193
+ taskId: string,
6194
+ options: {
6195
+ admitAssignment?: () => TaskStopAssignmentAdmission;
6196
+ shouldVeto?: () => boolean;
6197
+ closeCredentials?: (runEpoch: number) => Promise<void>;
6198
+ /** After the channel drained and before anything touches the
6199
+ * container runtime (previews, the runtime's stop): why the caller's
6200
+ * proof of the daemon no longer holds, or null. A reason stops the
6201
+ * Stop there, mutating nothing. */
6202
+ beforeRuntimeMutation?: () => Promise<string | null>;
6203
+ } = {},
6204
+ ): Promise<OrchestratorTaskStopResult> {
6205
+ return this.stopTaskWithinLifecycle(taskId, options);
6206
+ }
6207
+
4692
6208
  private async stopTaskWithinLifecycle(
4693
6209
  taskId: string,
4694
- ): Promise<{ ok: boolean; error?: string }> {
6210
+ options: {
6211
+ admitAssignment?: () => TaskStopAssignmentAdmission;
6212
+ shouldVeto?: () => boolean;
6213
+ closeCredentials?: (runEpoch: number) => Promise<void>;
6214
+ beforeRuntimeMutation?: () => Promise<string | null>;
6215
+ },
6216
+ ): Promise<OrchestratorTaskStopResult> {
6217
+ // The activity check belongs in the same lifecycle slot as assignment
6218
+ // closure and runtime stop. A turn may become active while Stop waits for
6219
+ // earlier lifecycle work; veto it before closing any v1 authority.
6220
+ if (options.shouldVeto?.()) return { ok: true, vetoed: true };
6221
+ let assignmentAdmission: TaskStopAssignmentAdmission = { kind: "legacy" };
6222
+ try {
6223
+ assignmentAdmission =
6224
+ options.admitAssignment?.() ?? ({ kind: "legacy" } as const);
6225
+ } catch (error) {
6226
+ return {
6227
+ ok: false,
6228
+ error: (error instanceof Error ? error.message : String(error)).slice(
6229
+ 0,
6230
+ 200,
6231
+ ),
6232
+ };
6233
+ }
6234
+ if (assignmentAdmission.kind === "stale") {
6235
+ return {
6236
+ ok: false,
6237
+ error: "task assignment is no longer current",
6238
+ assignmentStale: true,
6239
+ };
6240
+ }
6241
+ const assignmentClosed = assignmentAdmission.kind === "closed";
4695
6242
  if (!backgroundContainerWorkAllowed()) {
4696
6243
  return { ok: false, error: "container runtime is not operational" };
4697
6244
  }
4698
6245
  const task = getHostTask(taskId);
4699
6246
  if (!task) return { ok: false, error: "unknown task" };
4700
- const project = task.composeProject;
4701
- if (!project) return { ok: false, error: "task has no running stack" };
6247
+ // A runtime is named by its persisted locator, a legacy row by its
6248
+ // Compose project. ADR-154: a start interrupted after recording its
6249
+ // runtime's locator keeps it, with no Compose project, for Stop to reach.
6250
+ if (!task.environmentLocator && !task.composeProject) {
6251
+ return { ok: false, error: "task has no running stack" };
6252
+ }
4702
6253
  // Revoke env/proxy access before a potentially long session drain, then
4703
6254
  // close sessions so nothing races to respawn them mid-stop.
4704
6255
  const suspendedGateway = suspendTaskGatewayTask(taskId);
4705
6256
  // ADR-153: no longer served; end its open proxy connections too.
4706
6257
  activeSecretsProxy()?.runtime.stopTask(taskId);
4707
6258
  await this.closeChannel(taskId);
6259
+ // ADR-154 §13: a proof of the daemon taken before the drain may have
6260
+ // expired during it; nothing below may reach a replaced one.
6261
+ const stale = await options.beforeRuntimeMutation?.();
6262
+ if (stale !== undefined && stale !== null) {
6263
+ if (!assignmentClosed) {
6264
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
6265
+ this.allowChannel(taskId);
6266
+ }
6267
+ return { ok: false, error: stale.slice(0, 200) };
6268
+ }
4708
6269
  try {
4709
6270
  await stopPreviewSidecars(taskId);
4710
6271
  } catch (err) {
@@ -4714,6 +6275,22 @@ export class Orchestrator {
4714
6275
  }`,
4715
6276
  );
4716
6277
  }
6278
+ // ADR-154 §6: Stop is acknowledged only once the closed run's
6279
+ // credentials are cleaned up, while its runtime still runs for the
6280
+ // cleanup to reach. A failed cleanup fails the stop; the cloud retries,
6281
+ // and its closed run stays closed meanwhile.
6282
+ if (assignmentAdmission.kind === "closed" && options.closeCredentials) {
6283
+ try {
6284
+ await options.closeCredentials(assignmentAdmission.runEpoch);
6285
+ } catch (err) {
6286
+ return {
6287
+ ok: false,
6288
+ error: `the task's credentials could not be cleaned up: ${
6289
+ (err instanceof Error ? err.message : String(err)).slice(0, 160)
6290
+ }`,
6291
+ };
6292
+ }
6293
+ }
4717
6294
  // Provider-neutral resumable stop. A row that reaches this point with
4718
6295
  // runtime resources always carries a proven locator (legacy rows cross
4719
6296
  // the adoption fence before the runtime becomes operational), so a
@@ -4724,14 +6301,18 @@ export class Orchestrator {
4724
6301
  try {
4725
6302
  const environment = await reconstructPersistedTaskEnvironment(task);
4726
6303
  if (!environment) {
4727
- if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
4728
- this.allowChannel(taskId);
6304
+ if (!assignmentClosed) {
6305
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
6306
+ this.allowChannel(taskId);
6307
+ }
4729
6308
  return { ok: false, error: "task has no persisted environment" };
4730
6309
  }
4731
6310
  await environment.stop();
4732
6311
  } catch (err) {
4733
- if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
4734
- this.allowChannel(taskId);
6312
+ if (!assignmentClosed) {
6313
+ if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
6314
+ this.allowChannel(taskId);
6315
+ }
4735
6316
  return {
4736
6317
  ok: false,
4737
6318
  error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
@@ -4757,7 +6338,9 @@ export class Orchestrator {
4757
6338
  // Nothing serves the task's env now: no value of it stays in memory.
4758
6339
  forgetTaskEnvSnapshot(taskId);
4759
6340
  clearTaskGatewayAcl(taskId);
4760
- return { ok: true };
6341
+ return assignmentAdmission.kind === "closed"
6342
+ ? { ok: true, stoppedRunEpoch: assignmentAdmission.runEpoch }
6343
+ : { ok: true };
4761
6344
  }
4762
6345
  }
4763
6346
 
@@ -6001,8 +7584,26 @@ export type TaskRuntimeContainerInspection =
6001
7584
 
6002
7585
  export interface WritableRuntimeQuarantineResult {
6003
7586
  verdict: "ready" | "retry" | "stale";
6004
- /** Known rows whose local `stopped` verdict must be published or replayed. */
6005
- stoppedTaskIds: string[];
7587
+ /**
7588
+ * Known rows whose local `stopped` verdict must be published or replayed.
7589
+ * The epoch is captured while the task lifecycle slot is still held and is
7590
+ * deliberately absent for a legacy assignment.
7591
+ */
7592
+ stoppedTasks: StoppedTaskStatusTarget[];
7593
+ }
7594
+
7595
+ export interface StoppedTaskStatusTarget {
7596
+ taskId: string;
7597
+ runEpoch?: number;
7598
+ }
7599
+
7600
+ function captureStoppedTaskStatusTarget(
7601
+ taskId: string,
7602
+ ): StoppedTaskStatusTarget {
7603
+ const assignment = closeCurrentTaskRunAssignmentForStoppedRuntime(taskId);
7604
+ return assignment.kind === "legacy"
7605
+ ? { taskId }
7606
+ : { taskId, runEpoch: assignment.runEpoch };
6006
7607
  }
6007
7608
 
6008
7609
  function runtimeWriterCandidate(
@@ -6390,19 +7991,19 @@ export async function quarantineWritableRuntimeContainers(options: {
6390
7991
  return quarantineWritableAppleRuntimeContainers(options);
6391
7992
  }
6392
7993
  const isCurrent = options.isCurrent ?? (() => true);
6393
- const stoppedTaskIds = new Set<string>();
6394
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
7994
+ const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
7995
+ if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
6395
7996
 
6396
7997
  const listed = await listRuntimeWriterCandidates();
6397
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
7998
+ if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
6398
7999
  if (listed.kind === "unreachable") {
6399
- return { verdict: "retry", stoppedTaskIds: [] };
8000
+ return { verdict: "retry", stoppedTasks: [] };
6400
8001
  }
6401
8002
  let resolved = !listed.ambiguous;
6402
8003
 
6403
8004
  for (const candidate of listed.candidates) {
6404
8005
  if (!isCurrent()) {
6405
- return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
8006
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
6406
8007
  }
6407
8008
  await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
6408
8009
  if (!isCurrent()) return;
@@ -6459,7 +8060,10 @@ export async function quarantineWritableRuntimeContainers(options: {
6459
8060
  // Always re-publish the stopped verdict. A prior process may have
6460
8061
  // committed SQLite and crashed before its websocket frame left; the
6461
8062
  // cloud could otherwise remain ghost-running indefinitely.
6462
- stoppedTaskIds.add(candidate.taskId);
8063
+ stoppedTasks.set(
8064
+ candidate.taskId,
8065
+ captureStoppedTaskStatusTarget(candidate.taskId),
8066
+ );
6463
8067
  db_setStatus(candidate.taskId, "stopped", {
6464
8068
  codeServerPort: null,
6465
8069
  previewPorts: "[]",
@@ -6488,11 +8092,13 @@ export async function quarantineWritableRuntimeContainers(options: {
6488
8092
  }
6489
8093
 
6490
8094
  if (!isCurrent()) {
6491
- return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
8095
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
6492
8096
  }
6493
8097
  return {
6494
8098
  verdict: resolved ? "ready" : "retry",
6495
- stoppedTaskIds: [...stoppedTaskIds].sort(),
8099
+ stoppedTasks: [...stoppedTasks.values()].sort((a, b) =>
8100
+ a.taskId.localeCompare(b.taskId),
8101
+ ),
6496
8102
  };
6497
8103
  }
6498
8104
 
@@ -7282,25 +8888,25 @@ async function quarantineWritableAppleRuntimeContainers(options: {
7282
8888
  scope?: "contract" | "volume-holders";
7283
8889
  }): Promise<WritableRuntimeQuarantineResult> {
7284
8890
  const isCurrent = options.isCurrent ?? (() => true);
7285
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
8891
+ if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
7286
8892
  const binding = appleContainerRuntimeBinding();
7287
- if (binding === null) return { verdict: "retry", stoppedTaskIds: [] };
8893
+ if (binding === null) return { verdict: "retry", stoppedTasks: [] };
7288
8894
  const cli = appleCliRunner(() => binding.containerCliPath);
7289
8895
  const listed = await cli(["list", "--all", "--format", "json"], {
7290
8896
  timeoutMs: 30_000,
7291
8897
  maxOutputBytes: 8 * 1024 * 1024,
7292
8898
  });
7293
8899
  if (listed.status !== 0 || listed.truncated === true) {
7294
- return { verdict: "retry", stoppedTaskIds: [] };
8900
+ return { verdict: "retry", stoppedTasks: [] };
7295
8901
  }
7296
8902
  let parsed: unknown;
7297
8903
  try {
7298
8904
  parsed = JSON.parse(listed.stdout);
7299
8905
  } catch {
7300
- return { verdict: "retry", stoppedTaskIds: [] };
8906
+ return { verdict: "retry", stoppedTasks: [] };
7301
8907
  }
7302
- if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTaskIds: [] };
7303
- const stoppedTaskIds = new Set<string>();
8908
+ if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTasks: [] };
8909
+ const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
7304
8910
  let resolved = true;
7305
8911
 
7306
8912
  const stopAndProve = async (id: string): Promise<boolean> => {
@@ -7357,7 +8963,9 @@ async function quarantineWritableAppleRuntimeContainers(options: {
7357
8963
  }>;
7358
8964
  };
7359
8965
  }>) {
7360
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
8966
+ if (!isCurrent()) {
8967
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
8968
+ }
7361
8969
  const id = item?.configuration?.id;
7362
8970
  if (typeof id !== "string" || id.length === 0) {
7363
8971
  resolved = false;
@@ -7410,20 +9018,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
7410
9018
  }
7411
9019
  const task = getHostTask(taskId);
7412
9020
  if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
9021
+ const stoppedTask = captureStoppedTaskStatusTarget(taskId);
7413
9022
  db_setStatus(taskId, "stopped", {
7414
9023
  codeServerPort: null,
7415
9024
  previewPorts: "[]",
7416
9025
  });
7417
- stoppedTaskIds.add(taskId);
9026
+ stoppedTasks.set(taskId, stoppedTask);
7418
9027
  }
7419
9028
  // Revoke live access with the environment, like the Docker sweep.
7420
9029
  clearTaskGatewayAcl(taskId);
7421
9030
  });
7422
9031
  }
7423
- if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
9032
+ if (!isCurrent()) {
9033
+ return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
9034
+ }
7424
9035
  return {
7425
9036
  verdict: resolved ? "ready" : "retry",
7426
- stoppedTaskIds: [...stoppedTaskIds],
9037
+ stoppedTasks: [...stoppedTasks.values()],
7427
9038
  };
7428
9039
  }
7429
9040
 
@@ -7477,7 +9088,7 @@ async function materializeAgentCli(
7477
9088
  return false;
7478
9089
  }
7479
9090
  }
7480
- return writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
9091
+ return await writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
7481
9092
  }
7482
9093
 
7483
9094
  async function recoverPersistedTaskEnvironment(
@@ -8604,7 +10215,9 @@ async function recoverOneDockerTask(
8604
10215
  // reinject sweep only targets containers already running, so a container
8605
10216
  // recovered here would otherwise keep whatever it held when it exited —
8606
10217
  // possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
8607
- // by the inject itself and must not abort the task's recovery.
10218
+ // by the inject itself and must not abort the task's recovery. ADR-154:
10219
+ // the runner owner's own sign-in, so only while the runner is on its own
10220
+ // credentials (the inject fences itself).
8608
10221
  await injectCodexIntoContainer(containerName);
8609
10222
  if (!isCurrent()) return true;
8610
10223
  // Establish the task owner's GitHub transport BEFORE uai-init performs any