@runuai/host 0.9.13 → 0.9.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +22 -5
  2. package/db/migrations/0014_host_inventory_event_index.sql +1 -0
  3. package/db/migrations/0015_host_settings.sql +9 -0
  4. package/db/migrations/0016_task_environment.sql +2 -0
  5. package/db/migrations/meta/_journal.json +21 -0
  6. package/db/schema.ts +80 -30
  7. package/images/standard/Dockerfile +36 -10
  8. package/images/standard/README.md +63 -18
  9. package/images/standard/container/corepack-version +1 -0
  10. package/images/standard/container/uai-init +308 -38
  11. package/images/standard/container/uai-materialize-runtimes +1527 -0
  12. package/lib/agent-cli.ts +69 -4
  13. package/lib/agent.ts +46 -7
  14. package/lib/agents/claude.ts +13 -8
  15. package/lib/agents/codex.ts +11 -6
  16. package/lib/agents/cursor.ts +39 -29
  17. package/lib/agents/durable-proc.ts +20 -27
  18. package/lib/agents/factory.ts +9 -25
  19. package/lib/agents/grok.ts +43 -30
  20. package/lib/agents/kimi.ts +44 -29
  21. package/lib/agents/opencode.ts +43 -31
  22. package/lib/agents/proc.ts +149 -114
  23. package/lib/agents/transport.ts +62 -50
  24. package/lib/agents/types.ts +6 -4
  25. package/lib/apple-runtime-recycle.ts +236 -0
  26. package/lib/apple-uninstall-teardown.ts +224 -0
  27. package/lib/browser-testing.ts +233 -93
  28. package/lib/codex-auth.ts +40 -6
  29. package/lib/command-db.ts +20 -0
  30. package/lib/container-runtime.ts +1338 -0
  31. package/lib/db.ts +1 -0
  32. package/lib/docker-exec.ts +87 -5
  33. package/lib/engine-accounts.ts +68 -5
  34. package/lib/engine-login.ts +1952 -0
  35. package/lib/enrollment-state.ts +251 -0
  36. package/lib/env-file.ts +155 -0
  37. package/lib/env.ts +4 -0
  38. package/lib/git-diff.ts +98 -32
  39. package/lib/git-identity.ts +199 -87
  40. package/lib/github-tokens.ts +202 -91
  41. package/lib/host-cloud-url.ts +62 -0
  42. package/lib/host-config.ts +279 -0
  43. package/lib/host-logs.ts +962 -0
  44. package/lib/keyed-promise-tail.ts +23 -0
  45. package/lib/legacy-runtime-v1.fixture.ts +627 -0
  46. package/lib/managed-activation-watcher.ts +72 -0
  47. package/lib/managed-install-owner-watcher.ts +55 -0
  48. package/lib/managed-operation-drain.ts +49 -0
  49. package/lib/managed-runtime.ts +3644 -0
  50. package/lib/managed-update-scheduler.ts +125 -0
  51. package/lib/mcp-gateway.ts +450 -23
  52. package/lib/orchestrator.ts +3070 -223
  53. package/lib/preview-sidecar.ts +57 -13
  54. package/lib/release-manifest.ts +708 -0
  55. package/lib/release-trust.ts +28 -0
  56. package/lib/runtime-activation-tail.ts +232 -0
  57. package/lib/runtime-archive.ts +1086 -0
  58. package/lib/runtime-authority.ts +79 -0
  59. package/lib/runtime-guard.ts +36 -0
  60. package/lib/runtime-provider-state.ts +169 -0
  61. package/lib/runtime-state.ts +232 -12
  62. package/lib/skills.ts +24 -3
  63. package/lib/ssh.ts +18 -0
  64. package/lib/standard-image.ts +1104 -141
  65. package/lib/stopped-task-status-queue.ts +44 -0
  66. package/lib/task-container-cli.ts +269 -0
  67. package/lib/task-diff.ts +66 -46
  68. package/lib/task-environment/apple-container.ts +757 -0
  69. package/lib/task-environment/docker.ts +945 -0
  70. package/lib/task-environment/index.ts +364 -0
  71. package/lib/task-environment/legacy-adoption.ts +443 -0
  72. package/lib/task-environment/registry.ts +58 -0
  73. package/lib/task-environment/types.ts +408 -0
  74. package/lib/task-identity.ts +19 -0
  75. package/lib/task-inventory.ts +585 -0
  76. package/lib/tunnel-registry.ts +135 -19
  77. package/lib/tunnel-runtime.ts +235 -0
  78. package/package.json +1 -1
  79. package/scripts/agent/_common.sh +123 -3
  80. package/scripts/agent/task-down.sh +146 -38
  81. package/scripts/agent/task-status.sh +19 -3
  82. package/scripts/agent/task-up.sh +1463 -109
  83. package/scripts/install/darwin.ts +848 -50
  84. package/scripts/install/linux.ts +838 -35
  85. package/scripts/install/types.ts +43 -0
  86. package/scripts/install/util.ts +215 -8
  87. package/scripts/install/win.ts +12 -0
  88. package/src/apple-tunnel-route.ts +104 -0
  89. package/src/cli.ts +1464 -72
  90. package/src/event-outbox.ts +83 -4
  91. package/src/index.ts +871 -50
  92. package/src/main.ts +1398 -255
  93. package/src/paths.ts +17 -1
  94. package/src/protocol.ts +695 -1
  95. package/src/runtime-bootstrap.ts +165 -0
  96. package/src/ui/server.ts +46 -10
  97. package/src/ui/types.ts +37 -0
@@ -15,10 +15,18 @@
15
15
  * instead of leaking a second orchestrator.
16
16
  */
17
17
 
18
- import { spawnSync } from "node:child_process";
19
- import { existsSync, readFileSync } from "node:fs";
18
+ import {
19
+ existsSync,
20
+ lstatSync,
21
+ mkdtempSync,
22
+ readFileSync,
23
+ realpathSync,
24
+ renameSync,
25
+ rmSync,
26
+ writeFileSync,
27
+ } from "node:fs";
20
28
  import { homedir } from "node:os";
21
- import { join } from "node:path";
29
+ import { dirname, join } from "node:path";
22
30
 
23
31
  import { inArray } from "drizzle-orm";
24
32
 
@@ -33,15 +41,52 @@ import {
33
41
  type Roster,
34
42
  type RosterAgent,
35
43
  } from "./agents/types";
36
- import { ACTIVE_STATUSES } from "./task-status";
44
+ import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
45
+ import { isSafeHostTaskId } from "./task-identity";
37
46
  import { getHostTask, upsertHostTask } from "./runtime-state";
38
47
  import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
39
48
  import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
40
49
  import { dockerCli } from "./docker-exec";
50
+ import {
51
+ taskContainerBackend,
52
+ taskContainerCli,
53
+ } from "./task-container-cli";
54
+ import {
55
+ RUNTIME_AUTHORITY_ENV,
56
+ runtimeAuthorityDockerExecArgs,
57
+ } from "./runtime-authority";
58
+ import {
59
+ backgroundContainerWorkAllowed,
60
+ ContainerRuntimeUnavailableError,
61
+ } from "./runtime-guard";
62
+ import {
63
+ appleContainerRuntimeBinding,
64
+ initializeContainerRuntime,
65
+ pinnedContainerRuntimeProvider,
66
+ suspendContainerRuntime,
67
+ } from "./container-runtime";
68
+ import {
69
+ reconstructPersistedTaskEnvironment,
70
+ setAppleTaskEnvironmentRecoveryDriver,
71
+ setDockerTaskEnvironmentRecoveryDriver,
72
+ } from "./task-environment";
73
+ import {
74
+ appleCliRunner,
75
+ appleTaskContainerName,
76
+ parseAppleTaskEnvironmentLocator,
77
+ } from "./task-environment/apple-container";
78
+ import { parseDockerTaskEnvironmentLocator } from "./task-environment/docker";
79
+ import type {
80
+ TaskEnvironmentDescriptor,
81
+ TaskEnvironmentHandle,
82
+ TaskEnvironmentRecoveryContext,
83
+ TaskEnvironmentRecoveryResult,
84
+ } from "./task-environment/types";
41
85
  import {
42
86
  containerSkillFile,
43
87
  installedSkillPath,
44
88
  installPackageSkills,
89
+ type SkillExec,
45
90
  writeAgentSkills,
46
91
  } from "./skills";
47
92
  import {
@@ -67,6 +112,11 @@ import {
67
112
  } from "./engine-accounts";
68
113
  import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
69
114
  import { stopPreviewSidecars } from "./preview-sidecar";
115
+ import {
116
+ agentClisReady,
117
+ standardImageCorepackVersionPath,
118
+ standardImageRuntimeMaterializerPath,
119
+ } from "./standard-image";
70
120
  import { env } from "./env";
71
121
  import type {
72
122
  ChannelEnsureInput,
@@ -102,6 +152,11 @@ interface Channel {
102
152
  sessions: Map<string, AgentSession>;
103
153
  /** Session-spawn inputs, kept so sessions can be started lazily. */
104
154
  containerName: string;
155
+ /** Opaque provider-owned boundary used only for agent process creation. */
156
+ agentEnvironment: Promise<TaskEnvironmentHandle<unknown>> | null;
157
+ /** Settled handle for later config/setup passes; avoids a new await on
158
+ * legacy/bootstrap paths. */
159
+ agentHandle: TaskEnvironmentHandle<unknown> | null;
105
160
  /** Per-agent system preamble (channel briefing + assembled persona). */
106
161
  preambles: Map<string, string>;
107
162
  /** Per-agent first-turn message — only agents whose `initialPrompt` is
@@ -167,11 +222,20 @@ interface Channel {
167
222
  /** Agents with a delivered turn that has not reached a terminal event yet.
168
223
  * Unlike openTurns, this covers thinking/tool/permission time before text. */
169
224
  activeTurns: Map<string, number>;
225
+ /** Exact accepted prompts still owned by each runner generation, in send
226
+ * order. Open mode deliberately permits more than one; a process-level
227
+ * daemon-loss event must replay all of them rather than only `lastPrompt`. */
228
+ acceptedPrompts: Map<string, AcceptedPrompt[]>;
170
229
  /** Crew replies waiting for the designated Secretary's current turn
171
230
  * boundary. Adapters do not share one mid-turn input contract, so only this
172
231
  * cross-lane role is serialized at the host boundary; open mode keeps its
173
232
  * legacy immediate-concurrent delivery behavior. */
174
233
  pendingPrompts: Map<string, string[]>;
234
+ /** Prompts accepted by channelDeliver whose final session boundary rejected
235
+ * specifically because the container runtime became unavailable. */
236
+ runtimeDeferredPrompts: Map<string, string[]>;
237
+ /** Missing/blocked sessions to reconcile exactly once after runtime ready. */
238
+ runtimeDeferredAgents: Set<string>;
175
239
  /** Agents whose current turn was interrupted (ESC) — their next
176
240
  * turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
177
241
  * half-turn instead of delivering it to @-mentioned peers. */
@@ -230,6 +294,10 @@ interface Channel {
230
294
  rotations: Map<string, number>;
231
295
  }
232
296
 
297
+ interface AcceptedPrompt {
298
+ text: string;
299
+ }
300
+
233
301
  /** Hard cap on automatic respawns per agent per channel lifetime. */
234
302
  const MAX_RESPAWNS_PER_AGENT = 5;
235
303
  /** ADR-076: hard cap on account rotations per agent between successful turns. */
@@ -241,6 +309,43 @@ const RESPAWN_COOLDOWN_MS = 10 * 60_000;
241
309
  * retry would spin npx continuously for the life of a broken task. */
242
310
  const BROWSER_RETRY_COOLDOWN_MS = 60_000;
243
311
 
312
+ /**
313
+ * A runner can be the first observable casualty of a daemon loss, before any
314
+ * command/tunnel path invalidates the cached ready verdict. Prove the daemon
315
+ * before classifying a terminal agent event as an agent crash. A failed proof
316
+ * poisons the runtime generation synchronously; normal detection/recovery owns
317
+ * the eventual ready edge and deferred-prompt replay.
318
+ */
319
+ async function agentTerminationLostContainerRuntime(): Promise<boolean> {
320
+ if (!backgroundContainerWorkAllowed()) return true;
321
+ let reachable = false;
322
+ try {
323
+ // Backend-aware liveness: `docker` does not exist on an apple host, and
324
+ // an ENOENT here would poison the generation on EVERY abnormal agent
325
+ // termination (found in the 2026-08-18 spawn-docker-ENOENT sweep).
326
+ const proof = taskContainerBackend().apple
327
+ ? await taskContainerCli(["system", "status"], { timeoutMs: 10_000 })
328
+ : await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
329
+ timeoutMs: 10_000,
330
+ });
331
+ reachable = proof.status === 0;
332
+ } catch {
333
+ reachable = false;
334
+ }
335
+ if (!backgroundContainerWorkAllowed()) return true;
336
+ if (reachable) return false;
337
+
338
+ suspendContainerRuntime(true);
339
+ void initializeContainerRuntime().catch((err) =>
340
+ console.warn(
341
+ `[orchestrator] container runtime recheck after agent termination failed: ${
342
+ err instanceof Error ? err.message : String(err)
343
+ }`,
344
+ ),
345
+ );
346
+ return true;
347
+ }
348
+
244
349
  /** Substrings in an agent's error output that mean "config was
245
350
  * unlinked between runs" — repair-and-respawn covers the common
246
351
  * Docker-Desktop-macOS race where Claude's atomic writes briefly
@@ -530,6 +635,8 @@ export class Orchestrator {
530
635
  secretaryAgentId: spec.secretaryAgentId,
531
636
  sessions: new Map(),
532
637
  containerName: `task-${taskId.toLowerCase()}-app-1`,
638
+ agentEnvironment: null,
639
+ agentHandle: null,
533
640
  preambles,
534
641
  firstTurns,
535
642
  sessionsReady: null,
@@ -547,7 +654,10 @@ export class Orchestrator {
547
654
  closed: false,
548
655
  openTurns: new Set(),
549
656
  activeTurns: new Map(),
657
+ acceptedPrompts: new Map(),
550
658
  pendingPrompts: new Map(),
659
+ runtimeDeferredPrompts: new Map(),
660
+ runtimeDeferredAgents: new Set(),
551
661
  interrupted: new Set(),
552
662
  respawns: new Map(),
553
663
  respawnLastAt: new Map(),
@@ -587,6 +697,7 @@ export class Orchestrator {
587
697
  // copy cannot land after browser/MCP materialization and erase config.
588
698
  await taskRecoveryComplete(channel.taskId);
589
699
  if (!this.isActiveChannel(channel)) return false;
700
+ if (!backgroundContainerWorkAllowed()) return false;
590
701
  // Memoized: every caller awaits the SAME in-flight start, so a concurrent
591
702
  // deliver() can't observe "ready" while the sessions map is still empty
592
703
  // (startSessions awaits docker work before populating it — the old boolean
@@ -632,6 +743,7 @@ export class Orchestrator {
632
743
  await this.ensureMcpConfig(channel);
633
744
  if (!this.isActiveChannel(channel)) return false;
634
745
  await this.reconcileSessions(channel);
746
+ this.drainRuntimeDeferredAgentWork(channel);
635
747
  }
636
748
  // The initial-start promise stays memoized after a healthy boot. A later
637
749
  // fatal exit can therefore leave the designated Secretary missing even
@@ -735,6 +847,10 @@ export class Orchestrator {
735
847
  changed: false,
736
848
  };
737
849
  if (!this.isActiveChannel(channel)) return unavailable;
850
+ if (!backgroundContainerWorkAllowed()) {
851
+ this.scheduleBrowserRetry(channel);
852
+ return unavailable;
853
+ }
738
854
  const requestedBrowser = channel.browserTesting;
739
855
  const requestedHasCodex =
740
856
  hasCodex || channel.roster.some((agent) => agent.kind === "codex");
@@ -818,10 +934,17 @@ export class Orchestrator {
818
934
  }
819
935
  let copiedOrUnknown = false;
820
936
  try {
821
- const provisioned = await provisionEngineAccounts(
822
- channel.containerName,
823
- requestedKinds,
824
- );
937
+ const provisioned = channel.agentHandle
938
+ ? await provisionEngineAccounts(
939
+ channel.containerName,
940
+ requestedKinds,
941
+ {},
942
+ channel.agentHandle,
943
+ )
944
+ : await provisionEngineAccounts(
945
+ channel.containerName,
946
+ requestedKinds,
947
+ );
825
948
  copiedOrUnknown =
826
949
  provisioned.copied > 0 || provisioned.mayHaveMutated;
827
950
  if (provisioned.failed > 0) {
@@ -873,12 +996,20 @@ export class Orchestrator {
873
996
  return unavailable;
874
997
  }
875
998
  coverage.browserAttempted = true;
876
- setup = await setupBrowserTesting(
877
- channel.taskId,
878
- channel.containerName,
879
- requestedHasCodex,
880
- requestedHomes,
881
- );
999
+ setup = channel.agentHandle
1000
+ ? await setupBrowserTesting(
1001
+ channel.taskId,
1002
+ channel.containerName,
1003
+ requestedHasCodex,
1004
+ requestedHomes,
1005
+ channel.agentHandle,
1006
+ )
1007
+ : await setupBrowserTesting(
1008
+ channel.taskId,
1009
+ channel.containerName,
1010
+ requestedHasCodex,
1011
+ requestedHomes,
1012
+ );
882
1013
  } catch (err) {
883
1014
  console.warn(
884
1015
  `[browser] task ${channel.taskId}: setup failed: ${
@@ -977,6 +1108,10 @@ export class Orchestrator {
977
1108
  channel.browserRetryTimer = null;
978
1109
  }
979
1110
  if (!this.isActiveChannel(channel)) return;
1111
+ if (!backgroundContainerWorkAllowed()) {
1112
+ this.scheduleBrowserRetry(channel);
1113
+ return;
1114
+ }
980
1115
  const forceProvision = Boolean(channel.provisionRetryFingerprint);
981
1116
  void this.ensureBrowserSetup(
982
1117
  channel,
@@ -1063,13 +1198,28 @@ export class Orchestrator {
1063
1198
  engineKinds.filter((kind) => !previous?.engineKinds?.includes(kind)),
1064
1199
  );
1065
1200
  const hadLiveSessions = channel.sessions.size > 0;
1066
- const configured = await setupMcpTaskConfig(
1067
- channel.taskId,
1068
- channel.containerName,
1069
- connections,
1070
- engineKinds,
1071
- codexHomes,
1072
- );
1201
+ // Once a provider-neutral handle has already been reconstructed for this
1202
+ // channel, keep subsequent config writes on that same environment. The
1203
+ // first legacy/bootstrap pass intentionally retains its Docker fallback;
1204
+ // it must not introduce an eager reconstruction dependency before the
1205
+ // channel has a durable provider locator.
1206
+ const providerEnvironment = channel.agentHandle ?? undefined;
1207
+ const configured = providerEnvironment
1208
+ ? await setupMcpTaskConfig(
1209
+ channel.taskId,
1210
+ channel.containerName,
1211
+ connections,
1212
+ engineKinds,
1213
+ codexHomes,
1214
+ providerEnvironment,
1215
+ )
1216
+ : await setupMcpTaskConfig(
1217
+ channel.taskId,
1218
+ channel.containerName,
1219
+ connections,
1220
+ engineKinds,
1221
+ codexHomes,
1222
+ );
1073
1223
  if (!configured || !this.isActiveChannel(channel)) return;
1074
1224
  channel.configGeneration += 1;
1075
1225
  channel.mcpConfigFingerprint = fingerprint;
@@ -1219,7 +1369,12 @@ export class Orchestrator {
1219
1369
  // Same per-agent materialisation the initial start does. Browser setup
1220
1370
  // is awaited before EVERY missing-session spawn: that reasserts configs
1221
1371
  // clobbered by resume/auth injection and covers roster generation.
1222
- await installPackageSkills(channel.taskId, missing);
1372
+ const skillExec = this.providerSkillExec(channel);
1373
+ if (skillExec) {
1374
+ await installPackageSkills(channel.taskId, missing, skillExec);
1375
+ } else {
1376
+ await installPackageSkills(channel.taskId, missing);
1377
+ }
1223
1378
  if (!this.isActiveChannel(channel)) return;
1224
1379
 
1225
1380
  // One materialization gate owns provisioning and (when enabled)
@@ -1270,7 +1425,6 @@ export class Orchestrator {
1270
1425
  {
1271
1426
  taskId: channel.taskId,
1272
1427
  agent,
1273
- containerName: channel.containerName,
1274
1428
  systemPreamble: channel.preambles.get(agent.id) ?? "",
1275
1429
  executionProfile: this.executionProfileFor(channel, agent.id),
1276
1430
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
@@ -1366,7 +1520,7 @@ export class Orchestrator {
1366
1520
  }
1367
1521
  channel.accountByAgent.set(agent.id, account.id);
1368
1522
  noteEngineAccountUsed(account.id);
1369
- return { ...base, ...account.execEnv };
1523
+ return { ...account.execEnv, ...base };
1370
1524
  }
1371
1525
 
1372
1526
  /** Install a session with an event handler scoped to this exact generation. */
@@ -1376,6 +1530,11 @@ export class Orchestrator {
1376
1530
  session: AgentSession,
1377
1531
  ): void {
1378
1532
  channel.sessions.set(agentId, session);
1533
+ // Adapters may synchronously emit `error` followed by `turn_complete` (or
1534
+ // `exit`). Terminal classification now awaits a daemon proof, so preserve
1535
+ // transport emission order across that await; otherwise the later boundary
1536
+ // can consume/reorder prompt state while the earlier error still owns it.
1537
+ let eventTail: Promise<void> = Promise.resolve();
1379
1538
  session.onEvent((event) => {
1380
1539
  // Recycle/teardown deletes the old session before calling close().
1381
1540
  // Adapters deliberately emit a final exit event; without this identity
@@ -1386,15 +1545,27 @@ export class Orchestrator {
1386
1545
  ) {
1387
1546
  return;
1388
1547
  }
1389
- const operation = this.handleAgentEvent(channel, agentId, event).catch(
1390
- (err: unknown) => {
1548
+ const operation = eventTail
1549
+ .catch(() => {})
1550
+ .then(async () => {
1551
+ // The session may have been retired while an earlier event awaited.
1552
+ // Recheck identity at execution time, not only emission time.
1553
+ if (
1554
+ !this.isActiveChannel(channel) ||
1555
+ channel.sessions.get(agentId) !== session
1556
+ ) {
1557
+ return;
1558
+ }
1559
+ await this.handleAgentEvent(channel, agentId, session, event);
1560
+ })
1561
+ .catch((err: unknown) => {
1391
1562
  console.warn(
1392
1563
  `[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
1393
1564
  err instanceof Error ? (err.stack ?? err.message) : String(err)
1394
1565
  }`,
1395
1566
  );
1396
- },
1397
- );
1567
+ });
1568
+ eventTail = operation;
1398
1569
  channel.eventOperations.add(operation);
1399
1570
  operation.then(
1400
1571
  () => channel.eventOperations.delete(operation),
@@ -1412,7 +1583,7 @@ export class Orchestrator {
1412
1583
  private createAndBindStableSession(
1413
1584
  channel: Channel,
1414
1585
  agentId: string,
1415
- args: Parameters<AgentSessionFactory["create"]>[0],
1586
+ args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
1416
1587
  ): Promise<AgentSession | null> {
1417
1588
  const operation = this.createAndBindStableSessionCore(
1418
1589
  channel,
@@ -1430,11 +1601,12 @@ export class Orchestrator {
1430
1601
  private async createAndBindStableSessionCore(
1431
1602
  channel: Channel,
1432
1603
  agentId: string,
1433
- args: Parameters<AgentSessionFactory["create"]>[0],
1604
+ args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
1434
1605
  ): Promise<AgentSession | null> {
1606
+ const environment = await this.agentEnvironment(channel);
1435
1607
  while (this.isActiveChannel(channel)) {
1436
1608
  const generation = channel.configGeneration;
1437
- const session = await this.factory.create(args);
1609
+ const session = await this.factory.create({ ...args, environment });
1438
1610
  if (!this.isActiveChannel(channel)) {
1439
1611
  await session.close().catch(() => {});
1440
1612
  return null;
@@ -1455,6 +1627,59 @@ export class Orchestrator {
1455
1627
  return null;
1456
1628
  }
1457
1629
 
1630
+ private agentEnvironment(
1631
+ channel: Channel,
1632
+ ): Promise<TaskEnvironmentHandle<unknown>> {
1633
+ const existing = channel.agentEnvironment;
1634
+ if (existing) return existing;
1635
+ const created = (async () => {
1636
+ const task = getHostTask(channel.taskId);
1637
+ if (!task) throw new Error("task environment is not persisted");
1638
+ const environment = await reconstructPersistedTaskEnvironment(task);
1639
+ if (!environment) throw new Error("task environment is not available");
1640
+ if (environment.descriptor.taskId !== channel.taskId) {
1641
+ throw new Error("task environment belongs to a different task");
1642
+ }
1643
+ return environment;
1644
+ })();
1645
+ const tracked = created.then((environment) => {
1646
+ if (this.isActiveChannel(channel)) channel.agentHandle = environment;
1647
+ return environment;
1648
+ });
1649
+ channel.agentEnvironment = tracked;
1650
+ tracked.catch(() => {
1651
+ if (this.isActiveChannel(channel)) {
1652
+ channel.agentEnvironment = null;
1653
+ channel.agentHandle = null;
1654
+ }
1655
+ });
1656
+ return tracked;
1657
+ }
1658
+
1659
+ /** Provider-neutral skill installation for a settled task environment. The
1660
+ * legacy path remains the default until the first environment handle has
1661
+ * actually been reconstructed, avoiding a new await during cold admission.
1662
+ */
1663
+ private providerSkillExec(channel: Channel): SkillExec | undefined {
1664
+ const environment = channel.agentHandle;
1665
+ if (!environment) return undefined;
1666
+ return async (_container, cmd, cwd) => {
1667
+ const argv = cmd as [string, ...string[]];
1668
+ const result = await environment.exec({
1669
+ argv,
1670
+ cwd,
1671
+ user: "node",
1672
+ env: RUNTIME_AUTHORITY_ENV,
1673
+ timeoutMs: 120_000,
1674
+ maxOutputBytes: 256 * 1024,
1675
+ });
1676
+ return {
1677
+ status: result.exitCode,
1678
+ stderr: Buffer.from(result.stderr).toString("utf8"),
1679
+ };
1680
+ };
1681
+ }
1682
+
1458
1683
  private async startSessions(channel: Channel): Promise<boolean> {
1459
1684
  const task = getHostTask(channel.taskId);
1460
1685
  if (
@@ -1464,6 +1689,11 @@ export class Orchestrator {
1464
1689
  ) {
1465
1690
  return false;
1466
1691
  }
1692
+ // Every initial in-container materialization step shares the same boot
1693
+ // fence as the real agent factory. Gate before identity, auth, skills,
1694
+ // browser/MCP configuration, or runner creation can execute in Docker.
1695
+ await agentClisReady;
1696
+ if (!this.isActiveChannel(channel)) return false;
1467
1697
  // Freeze the generation before any slow docker work. A roster add while
1468
1698
  // setup awaits must reconcile under its own engine-aware browser pass,
1469
1699
  // not slip into this factory loop under the old generation's config.
@@ -1474,7 +1704,13 @@ export class Orchestrator {
1474
1704
  // task-up.sh using the creator's per-user key. Best-effort. Awaited but
1475
1705
  // async (docker exec via dockerCli) so it doesn't block the event loop —
1476
1706
  // this runs on every channel ensure, including each post-restart reconnect.
1477
- await setupTaskGitIdentity(channel.taskId, task.ownerName, task.ownerEmail);
1707
+ await setupTaskGitIdentity(
1708
+ channel.taskId,
1709
+ task.ownerName,
1710
+ task.ownerEmail,
1711
+ undefined,
1712
+ channel.agentHandle ?? undefined,
1713
+ );
1478
1714
  if (!this.isActiveChannel(channel)) return false;
1479
1715
 
1480
1716
  // GitHub auth for both `gh` and HTTPS Git (ADR-027): inject the token,
@@ -1482,7 +1718,11 @@ export class Orchestrator {
1482
1718
  // serialized reconcile so agents cannot race their first fetch/push after
1483
1719
  // task creation, reconnect, or host recovery.
1484
1720
  if (task.ownerUserId) {
1485
- await reconcileTaskGitAuth(channel.taskId, task.ownerUserId);
1721
+ await reconcileTaskGitAuth(channel.taskId, task.ownerUserId, {
1722
+ setupDeps: {
1723
+ environment: () => channel.agentHandle,
1724
+ },
1725
+ });
1486
1726
  }
1487
1727
 
1488
1728
  // SSH signing identity, re-asserted independently of transport. Connected
@@ -1501,7 +1741,12 @@ export class Orchestrator {
1501
1741
  // turn. Idempotent + best-effort (returns fast when there are none); a slow
1502
1742
  // clone/install briefly delays start, which is acceptable for skill-bearing
1503
1743
  // tasks. Never throws.
1504
- await installPackageSkills(channel.taskId, initialRoster);
1744
+ const skillExec = this.providerSkillExec(channel);
1745
+ if (skillExec) {
1746
+ await installPackageSkills(channel.taskId, initialRoster, skillExec);
1747
+ } else {
1748
+ await installPackageSkills(channel.taskId, initialRoster);
1749
+ }
1505
1750
  if (!this.isActiveChannel(channel)) return false;
1506
1751
 
1507
1752
  // ADR-053: wire the Playwright MCP browser before agents spawn — configs
@@ -1559,7 +1804,6 @@ export class Orchestrator {
1559
1804
  {
1560
1805
  taskId: channel.taskId,
1561
1806
  agent,
1562
- containerName: channel.containerName,
1563
1807
  systemPreamble: channel.preambles.get(agent.id) ?? "",
1564
1808
  executionProfile: this.executionProfileFor(channel, agent.id),
1565
1809
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
@@ -1608,6 +1852,61 @@ export class Orchestrator {
1608
1852
  return channel ? this.ensureSessions(channel) : false;
1609
1853
  }
1610
1854
 
1855
+ /** Recreate sessions lost with the daemon and replay only prompts whose
1856
+ * adapter boundary explicitly rejected for runtime unavailability. */
1857
+ async resumeRuntimeDeferredAgentWork(): Promise<void> {
1858
+ if (!backgroundContainerWorkAllowed()) return;
1859
+ await Promise.all(
1860
+ [...this.channels.values()].map(async (channel) => {
1861
+ if (
1862
+ !this.isActiveChannel(channel) ||
1863
+ channel.runtimeDeferredAgents.size === 0
1864
+ ) {
1865
+ return;
1866
+ }
1867
+ try {
1868
+ await this.reconcileSessions(channel);
1869
+ this.drainRuntimeDeferredAgentWork(channel);
1870
+ } catch (err) {
1871
+ // A single ready edge fans out across channels. One broken task must
1872
+ // neither abort the others nor consume its own retryable prompts;
1873
+ // the next ordinary ensure/poll retries and drains this channel.
1874
+ console.warn(
1875
+ `[orchestrator] ${channel.taskId}: deferred runtime work remains queued: ${
1876
+ err instanceof Error ? err.message : String(err)
1877
+ }`,
1878
+ );
1879
+ }
1880
+ }),
1881
+ );
1882
+ }
1883
+
1884
+ private drainRuntimeDeferredAgentWork(channel: Channel): void {
1885
+ if (
1886
+ !this.isActiveChannel(channel) ||
1887
+ !backgroundContainerWorkAllowed()
1888
+ ) {
1889
+ return;
1890
+ }
1891
+ for (const agentId of [...channel.runtimeDeferredAgents]) {
1892
+ const session = channel.sessions.get(agentId);
1893
+ if (!session) continue;
1894
+
1895
+ // Clear ownership before sending. If runtime drops at the same last hop,
1896
+ // startPrompt's typed catch atomically re-adds the exact work.
1897
+ channel.runtimeDeferredAgents.delete(agentId);
1898
+ if (this.isSecretary(channel, agentId)) {
1899
+ this.startNextPendingPrompt(channel, agentId);
1900
+ continue;
1901
+ }
1902
+ const prompts = channel.runtimeDeferredPrompts.get(agentId) ?? [];
1903
+ channel.runtimeDeferredPrompts.delete(agentId);
1904
+ for (const prompt of prompts) {
1905
+ this.startPrompt(channel, agentId, session, prompt);
1906
+ }
1907
+ }
1908
+ }
1909
+
1611
1910
  private incrementActiveTurns(channel: Channel, agentId: string): void {
1612
1911
  channel.activeTurns.set(
1613
1912
  agentId,
@@ -1645,6 +1944,7 @@ export class Orchestrator {
1645
1944
  private retireRunnerGeneration(channel: Channel, agentId: string): void {
1646
1945
  const aborted = channel.interrupted.delete(agentId);
1647
1946
  channel.activeTurns.delete(agentId);
1947
+ channel.acceptedPrompts.delete(agentId);
1648
1948
  channel.openTurns.delete(agentId);
1649
1949
  if (aborted) {
1650
1950
  channel.lastPrompt.delete(agentId);
@@ -1666,9 +1966,14 @@ export class Orchestrator {
1666
1966
  // ADR-076: this is the prompt actually in flight. A queued follow-up must
1667
1967
  // not replace it or account rotation would replay the wrong turn.
1668
1968
  channel.lastPrompt.set(agentId, prompt);
1969
+ const accepted = { text: prompt };
1970
+ const acceptedForAgent = channel.acceptedPrompts.get(agentId) ?? [];
1971
+ acceptedForAgent.push(accepted);
1972
+ channel.acceptedPrompts.set(agentId, acceptedForAgent);
1669
1973
  this.incrementActiveTurns(channel, agentId);
1670
1974
  void session.send(prompt).catch((err: unknown) => {
1671
1975
  if (channel.sessions.get(agentId) === session) {
1976
+ this.removeAcceptedPrompt(channel, agentId, accepted);
1672
1977
  const remaining = this.decrementActiveTurns(channel, agentId);
1673
1978
  if (
1674
1979
  remaining === 0 &&
@@ -1676,7 +1981,11 @@ export class Orchestrator {
1676
1981
  ) {
1677
1982
  channel.lastPrompt.delete(agentId);
1678
1983
  }
1679
- this.startNextPendingPrompt(channel, agentId);
1984
+ if (err instanceof ContainerRuntimeUnavailableError) {
1985
+ this.deferPromptUntilRuntimeReady(channel, agentId, prompt);
1986
+ } else {
1987
+ this.startNextPendingPrompt(channel, agentId);
1988
+ }
1680
1989
  }
1681
1990
  console.warn(
1682
1991
  `[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
@@ -1686,6 +1995,49 @@ export class Orchestrator {
1686
1995
  });
1687
1996
  }
1688
1997
 
1998
+ private removeAcceptedPrompt(
1999
+ channel: Channel,
2000
+ agentId: string,
2001
+ accepted: AcceptedPrompt,
2002
+ ): void {
2003
+ const prompts = channel.acceptedPrompts.get(agentId);
2004
+ if (!prompts) return;
2005
+ const index = prompts.indexOf(accepted);
2006
+ if (index >= 0) prompts.splice(index, 1);
2007
+ if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
2008
+ }
2009
+
2010
+ private completeNextAcceptedPrompt(
2011
+ channel: Channel,
2012
+ agentId: string,
2013
+ ): void {
2014
+ const prompts = channel.acceptedPrompts.get(agentId);
2015
+ if (!prompts) return;
2016
+ prompts.shift();
2017
+ if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
2018
+ }
2019
+
2020
+ private deferPromptUntilRuntimeReady(
2021
+ channel: Channel,
2022
+ agentId: string,
2023
+ prompt: string,
2024
+ ): void {
2025
+ channel.runtimeDeferredAgents.add(agentId);
2026
+ if (this.isSecretary(channel, agentId)) {
2027
+ // Secretary delivery is serialized. Put the failed in-flight prompt in
2028
+ // front of crew replies that were accepted behind it.
2029
+ const pending = channel.pendingPrompts.get(agentId) ?? [];
2030
+ pending.unshift(prompt);
2031
+ channel.pendingPrompts.set(agentId, pending);
2032
+ return;
2033
+ }
2034
+ // Open-mode sends are intentionally concurrent. Promise rejection handlers
2035
+ // run in submission order, so append preserves that accepted order.
2036
+ const deferred = channel.runtimeDeferredPrompts.get(agentId) ?? [];
2037
+ deferred.push(prompt);
2038
+ channel.runtimeDeferredPrompts.set(agentId, deferred);
2039
+ }
2040
+
1689
2041
  /** Start at most one queued prompt after the previous turn is terminal. */
1690
2042
  private startNextPendingPrompt(channel: Channel, agentId: string): boolean {
1691
2043
  if (
@@ -1804,7 +2156,11 @@ export class Orchestrator {
1804
2156
  // Cancel any armed auto-retry chain so this manual attempt runs fresh
1805
2157
  // (setup's duplicate-chain guard would otherwise no-op it).
1806
2158
  clearRefresh(taskId);
1807
- const ok = await reconcileTaskGitAuth(taskId, owner);
2159
+ const ok = await reconcileTaskGitAuth(taskId, owner, {
2160
+ setupDeps: {
2161
+ environment: () => this.channels.get(taskId)?.agentHandle ?? null,
2162
+ },
2163
+ });
1808
2164
  this.emitSystemNote(
1809
2165
  taskId,
1810
2166
  ok
@@ -1818,6 +2174,7 @@ export class Orchestrator {
1818
2174
  private async handleAgentEvent(
1819
2175
  channel: Channel,
1820
2176
  agentId: string,
2177
+ session: AgentSession,
1821
2178
  event: AgentEvent,
1822
2179
  ): Promise<void> {
1823
2180
  if (!this.isActiveChannel(channel)) return;
@@ -1901,6 +2258,18 @@ export class Orchestrator {
1901
2258
  break;
1902
2259
  }
1903
2260
  case "error": {
2261
+ if (await agentTerminationLostContainerRuntime()) {
2262
+ // `error` and `exit` can arrive back-to-back while the daemon probe
2263
+ // is pending. Only the first terminal event still owns this session.
2264
+ if (channel.sessions.get(agentId) !== session) break;
2265
+ await this.retireSessionForRuntimeLoss(
2266
+ channel,
2267
+ agentId,
2268
+ session,
2269
+ );
2270
+ break;
2271
+ }
2272
+ if (channel.sessions.get(agentId) !== session) break;
1904
2273
  // The turn died with the session — drop its turn-state flags so a
1905
2274
  // respawned session starts clean.
1906
2275
  this.retireRunnerGeneration(channel, agentId);
@@ -1972,6 +2341,7 @@ export class Orchestrator {
1972
2341
  // An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
1973
2342
  // the cloud discards the buffer instead of handing it to peers.
1974
2343
  const aborted = channel.interrupted.delete(agentId);
2344
+ this.completeNextAcceptedPrompt(channel, agentId);
1975
2345
  const remainingTurns = this.decrementActiveTurns(channel, agentId);
1976
2346
  channel.openTurns.delete(agentId);
1977
2347
  // ADR-076: a turn that completed means the current account is healthy —
@@ -2005,6 +2375,16 @@ export class Orchestrator {
2005
2375
  break;
2006
2376
  }
2007
2377
  case "exit":
2378
+ if (await agentTerminationLostContainerRuntime()) {
2379
+ if (channel.sessions.get(agentId) !== session) break;
2380
+ await this.retireSessionForRuntimeLoss(
2381
+ channel,
2382
+ agentId,
2383
+ session,
2384
+ );
2385
+ break;
2386
+ }
2387
+ if (channel.sessions.get(agentId) !== session) break;
2008
2388
  // Same zombie hazard as the error path — a session whose process
2009
2389
  // ended (even cleanly) can never carry another turn.
2010
2390
  this.retireRunnerGeneration(channel, agentId);
@@ -2019,6 +2399,35 @@ export class Orchestrator {
2019
2399
  }
2020
2400
  }
2021
2401
 
2402
+ /** A daemon loss is infrastructure recovery, not an agent crash. Preserve
2403
+ * accepted work, drop the dead generation, and leave respawn/exit accounting
2404
+ * untouched; the runtime-ready hook recreates it deterministically. */
2405
+ private async retireSessionForRuntimeLoss(
2406
+ channel: Channel,
2407
+ agentId: string,
2408
+ session: AgentSession,
2409
+ ): Promise<void> {
2410
+ const accepted = [
2411
+ ...(channel.acceptedPrompts.get(agentId) ?? []),
2412
+ ];
2413
+ const interrupted = channel.interrupted.has(agentId);
2414
+ this.retireRunnerGeneration(channel, agentId);
2415
+ channel.lastPrompt.delete(agentId);
2416
+ if (accepted.length > 0 && !interrupted) {
2417
+ for (const prompt of accepted) {
2418
+ this.deferPromptUntilRuntimeReady(channel, agentId, prompt.text);
2419
+ }
2420
+ } else {
2421
+ // Recreate even an idle session after readiness, without fabricating a
2422
+ // prompt or requiring another human delivery to discover the dead slot.
2423
+ channel.runtimeDeferredAgents.add(agentId);
2424
+ }
2425
+ channel.sessions.delete(agentId);
2426
+ channel.browserStaleSessions.delete(agentId);
2427
+ channel.browserPendingStaleSessions.delete(agentId);
2428
+ await session.close().catch(() => {});
2429
+ }
2430
+
2022
2431
  // -- session recovery -----------------------------------------------------
2023
2432
 
2024
2433
  /**
@@ -2054,7 +2463,10 @@ export class Orchestrator {
2054
2463
 
2055
2464
  channel.spawning.add(agentId);
2056
2465
  try {
2057
- const restored = repairClaudeConfigInContainer(channel.containerName);
2466
+ const restored = await repairClaudeConfigInContainer(
2467
+ channel.containerName,
2468
+ channel.agentHandle ?? undefined,
2469
+ );
2058
2470
 
2059
2471
  // Delete the old generation before awaiting close. Its adapter emits a
2060
2472
  // terminal event during teardown; bindSession's identity guard must see
@@ -2097,7 +2509,6 @@ export class Orchestrator {
2097
2509
  const session = await this.createAndBindStableSession(channel, agentId, {
2098
2510
  taskId: channel.taskId,
2099
2511
  agent,
2100
- containerName: channel.containerName,
2101
2512
  systemPreamble: channel.preambles.get(agentId) ?? "",
2102
2513
  executionProfile: this.executionProfileFor(channel, agentId),
2103
2514
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
@@ -2105,7 +2516,7 @@ export class Orchestrator {
2105
2516
  agentId,
2106
2517
  ),
2107
2518
  agentEnv: boundAccount
2108
- ? { ...base, ...boundAccount.execEnv }
2519
+ ? { ...boundAccount.execEnv, ...base }
2109
2520
  : this.accountAgentEnv(channel, agent, base),
2110
2521
  });
2111
2522
  if (!session) return;
@@ -2226,14 +2637,13 @@ export class Orchestrator {
2226
2637
  {
2227
2638
  taskId: channel.taskId,
2228
2639
  agent,
2229
- containerName: channel.containerName,
2230
2640
  systemPreamble: channel.preambles.get(agentId) ?? "",
2231
2641
  executionProfile: this.executionProfileFor(channel, agentId),
2232
2642
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2233
2643
  channel,
2234
2644
  agentId,
2235
2645
  ),
2236
- agentEnv: { ...base, ...next.execEnv },
2646
+ agentEnv: { ...next.execEnv, ...base },
2237
2647
  },
2238
2648
  );
2239
2649
  if (!replacement) return true;
@@ -2341,14 +2751,13 @@ export class Orchestrator {
2341
2751
  replacement = await this.createAndBindStableSession(channel, agentId, {
2342
2752
  taskId: channel.taskId,
2343
2753
  agent,
2344
- containerName: channel.containerName,
2345
2754
  systemPreamble: channel.preambles.get(agentId) ?? "",
2346
2755
  executionProfile: this.executionProfileFor(channel, agentId),
2347
2756
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2348
2757
  channel,
2349
2758
  agentId,
2350
2759
  ),
2351
- agentEnv: { ...base, ...account.execEnv },
2760
+ agentEnv: { ...account.execEnv, ...base },
2352
2761
  });
2353
2762
  if (!replacement) return true;
2354
2763
 
@@ -2487,7 +2896,10 @@ export class Orchestrator {
2487
2896
  ch.browserStaleSessions.clear();
2488
2897
  ch.browserPendingStaleSessions.clear();
2489
2898
  ch.activeTurns.clear();
2899
+ ch.acceptedPrompts.clear();
2490
2900
  ch.pendingPrompts.clear();
2901
+ ch.runtimeDeferredPrompts.clear();
2902
+ ch.runtimeDeferredAgents.clear();
2491
2903
  ch.openTurns.clear();
2492
2904
  ch.interrupted.clear();
2493
2905
  ch.reconcileAgain = false;
@@ -2527,6 +2939,18 @@ export class Orchestrator {
2527
2939
  return this.channelClosures.get(taskId) ?? Promise.resolve();
2528
2940
  }
2529
2941
 
2942
+ /** Quarantine a live channel before recovery stops its app container.
2943
+ * Preserve the latest channel spec so a successful recovery can reopen the
2944
+ * task without waiting for another cloud message, while the ordinary close
2945
+ * tombstone prevents any agent process from respawning during maintenance. */
2946
+ async quarantineChannelForRecovery(taskId: string): Promise<void> {
2947
+ const currentSpec = this.channelSpecs.get(taskId);
2948
+ await this.closeChannel(taskId);
2949
+ if (currentSpec && !this.channelSpecs.has(taskId)) {
2950
+ this.channelSpecs.set(taskId, currentSpec);
2951
+ }
2952
+ }
2953
+
2530
2954
  /** A successful task-up is the only transition that reopens a tombstoned
2531
2955
  * task id after stop/teardown. */
2532
2956
  allowChannel(taskId: string): void {
@@ -2553,6 +2977,9 @@ export class Orchestrator {
2553
2977
  private async stopTaskWithinLifecycle(
2554
2978
  taskId: string,
2555
2979
  ): Promise<{ ok: boolean; error?: string }> {
2980
+ if (!backgroundContainerWorkAllowed()) {
2981
+ return { ok: false, error: "container runtime is not operational" };
2982
+ }
2556
2983
  const task = getHostTask(taskId);
2557
2984
  if (!task) return { ok: false, error: "unknown task" };
2558
2985
  const project = task.composeProject;
@@ -2568,33 +2995,27 @@ export class Orchestrator {
2568
2995
  }`,
2569
2996
  );
2570
2997
  }
2571
- const ps = await dockerCli([
2572
- "ps",
2573
- "-q",
2574
- "--filter",
2575
- `label=com.docker.compose.project=${project}`,
2576
- ]);
2577
- if (ps.status !== 0) {
2578
- // An empty stdout is not evidence that the stack is gone when docker
2579
- // itself failed. Reopen the channel and leave the runtime mirror alone
2580
- // so cloud reconciliation cannot turn a live container into "stopped".
2998
+ // Provider-neutral resumable stop. A row that reaches this point with
2999
+ // runtime resources always carries a proven locator (legacy rows cross
3000
+ // the adoption fence before the runtime becomes operational), so a
3001
+ // reconstruction failure is a stop failure, never a fallback hint. An
3002
+ // environment failure reopens the channel and leaves the runtime mirror
3003
+ // alone so cloud reconciliation cannot turn a live container into
3004
+ // "stopped".
3005
+ try {
3006
+ const environment = await reconstructPersistedTaskEnvironment(task);
3007
+ if (!environment) {
3008
+ this.allowChannel(taskId);
3009
+ return { ok: false, error: "task has no persisted environment" };
3010
+ }
3011
+ await environment.stop();
3012
+ } catch (err) {
2581
3013
  this.allowChannel(taskId);
2582
3014
  return {
2583
3015
  ok: false,
2584
- error: ps.stderr.trim().slice(0, 200) || "docker ps failed",
3016
+ error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
2585
3017
  };
2586
3018
  }
2587
- const ids = ps.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
2588
- if (ids.length > 0) {
2589
- const stopped = await dockerCli(["stop", ...ids], { timeoutMs: 60_000 });
2590
- if (stopped.status !== 0) {
2591
- this.allowChannel(taskId);
2592
- return {
2593
- ok: false,
2594
- error: stopped.stderr.trim().slice(0, 200) || "docker stop failed",
2595
- };
2596
- }
2597
- }
2598
3019
  upsertHostTask(taskId, { statusMirror: "stopped" });
2599
3020
  clearTaskGatewayAcl(taskId);
2600
3021
  return { ok: true };
@@ -2688,7 +3109,10 @@ export function resolveAddressing(
2688
3109
  *
2689
3110
  * Returns true when the file actually made it into the container.
2690
3111
  */
2691
- function repairClaudeConfigInContainer(containerName: string): boolean {
3112
+ async function repairClaudeConfigInContainer(
3113
+ containerName: string,
3114
+ environment?: Pick<TaskEnvironmentHandle, "exec">,
3115
+ ): Promise<boolean> {
2692
3116
  // Operator can override the source path via UAI_OWNER_HOME so the
2693
3117
  // server doesn't have to guess. Useful when Next.js's dev process
2694
3118
  // env diverges from `os.homedir()` for any reason.
@@ -2733,25 +3157,44 @@ function repairClaudeConfigInContainer(containerName: string): boolean {
2733
3157
  // bounces. `rm -f` punches through it (or no-ops on a genuine
2734
3158
  // missing file). Then `cat >` creates the fresh inode. Both steps
2735
3159
  // run in one `sh -c` so a partial failure leaves a clean state.
2736
- const write = spawnSync(
2737
- "docker",
3160
+ const repairScript =
3161
+ "rm -f /home/node/.claude.json && cat > /home/node/.claude.json";
3162
+ if (environment) {
3163
+ const write = await environment.exec({
3164
+ argv: ["/bin/sh", "-c", repairScript],
3165
+ env: RUNTIME_AUTHORITY_ENV,
3166
+ stdin: content,
3167
+ timeoutMs: 30_000,
3168
+ maxOutputBytes: 64 * 1024,
3169
+ });
3170
+ if (write.exitCode !== 0) {
3171
+ console.error(
3172
+ "[orchestrator] repair: in-place write to .claude.json failed: " +
3173
+ Buffer.from(write.stderr).toString("utf8").trim(),
3174
+ );
3175
+ return false;
3176
+ }
3177
+ return true;
3178
+ }
3179
+ const write = await taskContainerCli(
2738
3180
  [
2739
3181
  "exec",
2740
3182
  "-i",
2741
- containerName,
2742
- "sh",
3183
+ ...runtimeAuthorityDockerExecArgs(),
3184
+ // The channel carries the Docker replica name; the apple container
3185
+ // drops the replica suffix (2026-08-18 spawn-docker-ENOENT sweep).
3186
+ taskContainerBackend().apple
3187
+ ? containerName.replace(/-app-1$/, "-app")
3188
+ : containerName,
3189
+ "/bin/sh",
2743
3190
  "-c",
2744
- "rm -f /home/node/.claude.json && cat > /home/node/.claude.json",
3191
+ repairScript,
2745
3192
  ],
2746
- { input: content, encoding: "buffer" },
3193
+ { input: content.toString("utf8"), timeoutMs: 30_000 },
2747
3194
  );
2748
3195
  if (write.status !== 0) {
2749
- const stderr =
2750
- write.stderr instanceof Buffer
2751
- ? write.stderr.toString("utf8")
2752
- : String(write.stderr ?? "");
2753
3196
  console.error(
2754
- `[orchestrator] repair: in-place write to .claude.json failed: ${stderr.trim()}`,
3197
+ `[orchestrator] repair: in-place write to .claude.json failed: ${write.stderr.trim()}`,
2755
3198
  );
2756
3199
  return false;
2757
3200
  }
@@ -3083,7 +3526,9 @@ export function buildSystemPreamble(
3083
3526
  "You are one agent in a uai task chat channel, shared with the human",
3084
3527
  "and the other agents. To hand work to or ask another agent, mention",
3085
3528
  "it by id at the start of a line — e.g. `@codex please review the",
3086
- "diff`. uai routes that message into that agent's input.",
3529
+ "diff`. uai buffers the handoff and routes your complete turn into that",
3530
+ "agent's input only after your current turn ends. The mention does NOT",
3531
+ "wake the peer while you continue using tools in the same turn.",
3087
3532
  ];
3088
3533
  const collaborationBrief = isSecretary
3089
3534
  ? [
@@ -3107,8 +3552,9 @@ export function buildSystemPreamble(
3107
3552
  "Collaborate with your peers — divide up the work, review each other's",
3108
3553
  "changes, share concrete ideas, and debate approach decisions by",
3109
3554
  "@-mentioning them. That is how the team gets things done, and you",
3110
- "should do it freely whenever it moves the work forward. But mentioning",
3111
- "a peer WAKES it and costs a turn, so make each one count: a message to",
3555
+ "should do it freely whenever it moves the work forward. But a peer",
3556
+ "@mention WAKES it after your turn ends and costs a turn, so make each",
3557
+ "one count: a message to",
3112
3558
  "a peer should ADVANCE the work — a real proposal, a question you need",
3113
3559
  "answered, a hand-off, or a review with specific findings. Do NOT",
3114
3560
  "@-mention a peer just to greet, thank, agree, acknowledge, or say",
@@ -3267,7 +3713,7 @@ export function buildSystemPreamble(
3267
3713
  ? ", `memory search <query>`"
3268
3714
  : "") +
3269
3715
  (agent.permissions?.includes("todo.write")
3270
- ? ", `todo list|add|claim|done`"
3716
+ ? ", `todo list|add|edit|remove|claim|done`"
3271
3717
  : "") +
3272
3718
  (agent.permissions?.includes("previews.write")
3273
3719
  ? ", `preview add <name> <port>`"
@@ -3327,7 +3773,9 @@ export function buildSystemPreamble(
3327
3773
  "**`todo claim <#id>` BEFORE you start** an item so two agents",
3328
3774
  "don't do the same work — and `todo done <#id>` when it's",
3329
3775
  "complete. Check `todo list` before picking up work. The human",
3330
- "watches the same list in the task UI.",
3776
+ "watches the same list in the task UI. Use `todo edit <#id>",
3777
+ "<text>` to correct an item and `todo remove <#id>` to delete",
3778
+ "one; both refuse an item another agent is actively claiming.",
3331
3779
  ]
3332
3780
  : []),
3333
3781
  ...(agent.permissions?.includes("tasks.history")
@@ -3402,19 +3850,60 @@ export function getOrchestrator(): Orchestrator {
3402
3850
  const factory = mode === "real" ? realAgentFactory : mockAgentFactory;
3403
3851
  globalForOrchestrator.__uaiOrchestrator = new Orchestrator(factory);
3404
3852
  }
3405
- if (!globalForOrchestrator.__uaiRecoverRan) {
3406
- globalForOrchestrator.__uaiRecoverRan = true;
3407
- // Fire-and-forget — the orchestrator is usable while recovery runs.
3408
- // The promise is kept so boot steps that must NOT race recovery (the
3409
- // Codex credential reinject) can sequence behind it via
3410
- // recoveryComplete(). recoverRunningTasks never rejects.
3411
- recoveryPromise = recoverRunningTasks();
3412
- void recoveryPromise;
3413
- }
3414
3853
  return globalForOrchestrator.__uaiOrchestrator;
3415
3854
  }
3416
3855
 
3417
- let recoveryPromise: Promise<void> = Promise.resolve();
3856
+ /** Runtime activation publication hook. Do not construct an orchestrator just
3857
+ * to run it: deferred work can only exist on the already-live singleton. */
3858
+ export function resumeRuntimeDeferredAgentWork(): void {
3859
+ const orchestrator = globalForOrchestrator.__uaiOrchestrator;
3860
+ if (!orchestrator) return;
3861
+ void orchestrator.resumeRuntimeDeferredAgentWork().catch((err) =>
3862
+ console.warn(
3863
+ `[orchestrator] deferred agent work resume failed: ${
3864
+ err instanceof Error ? err.message : String(err)
3865
+ }`,
3866
+ ),
3867
+ );
3868
+ }
3869
+
3870
+ let recoveryPromise: Promise<boolean> = Promise.resolve(true);
3871
+ let recoveryInFlight: Promise<boolean> | null = null;
3872
+
3873
+ /** Start or join one recovery sweep. A runtime-ready recheck can call this
3874
+ * after the original boot window gave up, without racing an active sweep. */
3875
+ function beginRuntimeRecovery(
3876
+ lifecycle?: Orchestrator,
3877
+ isCurrent: () => boolean = () => true,
3878
+ ): Promise<boolean> {
3879
+ // A caller that only wants the boot barrier joins it. A live-runtime request
3880
+ // is stronger: chain a fresh lifecycle-sequenced pass behind any active
3881
+ // scan. Promise-tail sequencing has no "last queue read vs finally" gap, so
3882
+ // a readiness request arriving in that microtask window cannot be lost.
3883
+ if (recoveryInFlight && !lifecycle) return recoveryInFlight;
3884
+ const predecessor = recoveryInFlight;
3885
+ const sweep = async (): Promise<boolean> => {
3886
+ try {
3887
+ return await recoverRunningTasks({ lifecycle, isCurrent });
3888
+ } catch (error) {
3889
+ console.error(
3890
+ "[orchestrator] recovery: unhandled failure",
3891
+ error instanceof Error ? error.message : error,
3892
+ );
3893
+ return false;
3894
+ }
3895
+ };
3896
+ let running!: Promise<boolean>;
3897
+ running = (predecessor
3898
+ ? predecessor.catch(() => false).then(sweep)
3899
+ : sweep()
3900
+ ).finally(() => {
3901
+ if (recoveryInFlight === running) recoveryInFlight = null;
3902
+ });
3903
+ recoveryInFlight = running;
3904
+ recoveryPromise = running;
3905
+ return running;
3906
+ }
3418
3907
 
3419
3908
  interface TaskRecoveryBarrier {
3420
3909
  promise: Promise<void>;
@@ -3451,13 +3940,73 @@ export function taskRecoveryComplete(taskId: string): Promise<void> {
3451
3940
 
3452
3941
  /**
3453
3942
  * Resolves once boot-time recovery has finished (starting the orchestrator —
3454
- * and with it recovery — if that hasn't happened yet). Never rejects.
3943
+ * and with it recovery — if that hasn't happened yet). Never rejects; false
3944
+ * means one or more active tasks remain unresolved.
3455
3945
  */
3456
- export function recoveryComplete(): Promise<void> {
3946
+ export function recoveryComplete(
3947
+ isCurrent: () => boolean = () => true,
3948
+ ): Promise<boolean> {
3457
3949
  getOrchestrator();
3950
+ if (!globalForOrchestrator.__uaiRecoverRan) {
3951
+ globalForOrchestrator.__uaiRecoverRan = true;
3952
+ // Fire-and-forget — the orchestrator is usable while recovery runs. Main
3953
+ // calls this only after the selected container runtime is ready.
3954
+ void beginRuntimeRecovery(undefined, isCurrent);
3955
+ }
3458
3956
  return recoveryPromise;
3459
3957
  }
3460
3958
 
3959
+ /**
3960
+ * Re-run boot recovery after a container runtime transitions to ready.
3961
+ *
3962
+ * `recoveryComplete()` is a one-boot barrier; this is the explicit recovery
3963
+ * trigger for ADR-101 runtime rechecks. It joins an active boot sweep, or
3964
+ * starts a new sweep after the prior bounded retry window completed.
3965
+ */
3966
+ export function requestRuntimeRecovery(
3967
+ isCurrent: () => boolean = () => true,
3968
+ ): Promise<boolean> {
3969
+ const orchestrator = getOrchestrator();
3970
+ globalForOrchestrator.__uaiRecoverRan = true;
3971
+ return beginRuntimeRecovery(orchestrator, isCurrent);
3972
+ }
3973
+
3974
+ /** Join only work that is already mutating/reconciling runtime state. This is
3975
+ * used before image maintenance so an explicit recheck cannot overlap a
3976
+ * reconnect sweep on the shared daemon/CLI volume. */
3977
+ export function waitForRuntimeRecovery(): Promise<void> {
3978
+ return recoveryInFlight?.then(() => undefined) ?? Promise.resolve();
3979
+ }
3980
+
3981
+ export type RuntimeRecoveryVerdict = "clean" | "needed" | "unknown";
3982
+
3983
+ /** Cheap reconnect proof. Each active environment provider reports whether
3984
+ * its durable workload is still running; ordinary cloud/network flaps avoid
3985
+ * the heavier repair path, while an unseen runtime restart still forces full
3986
+ * lifecycle-sequenced recovery before commands reopen. */
3987
+ export async function runtimeRecoveryVerdict(): Promise<RuntimeRecoveryVerdict> {
3988
+ const activeRows = getDb()
3989
+ .select()
3990
+ .from(schema.hostTasks)
3991
+ .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
3992
+ .all();
3993
+ for (const task of activeRows) {
3994
+ try {
3995
+ const environment = await reconstructPersistedTaskEnvironment(task);
3996
+ // A pre-provision row has no provider-owned runtime to inventory.
3997
+ if (environment === null) continue;
3998
+ const status = await environment.status();
3999
+ if (status.state === "unknown") return "unknown";
4000
+ if (status.state !== "running") return "needed";
4001
+ } catch {
4002
+ // An unavailable provider, malformed locator, or backend mismatch is
4003
+ // unknown—not evidence that the selected runtime is clean or absent.
4004
+ return "unknown";
4005
+ }
4006
+ }
4007
+ return "clean";
4008
+ }
4009
+
3461
4010
  // ---------------------------------------------------------------------------
3462
4011
  // Boot-time recovery
3463
4012
  //
@@ -3476,8 +4025,10 @@ export function recoveryComplete(): Promise<void> {
3476
4025
  // id (task-up is idempotent).
3477
4026
  // container gone, worktree gone → mark `error`. Data loss.
3478
4027
  //
3479
- // Idempotent per process — guarded by `__uaiRecoverRan` on globalThis
3480
- // so a Next.js HMR rebuild doesn't trigger it twice.
4028
+ // Idempotent per process — recoveryComplete/requestRuntimeRecovery mark
4029
+ // `__uaiRecoverRan` on globalThis so a Next.js HMR rebuild cannot trigger the
4030
+ // boot sweep twice. Merely constructing the orchestrator is deliberately not
4031
+ // enough: a degraded host must not probe an unrelated ambient daemon.
3481
4032
  // ---------------------------------------------------------------------------
3482
4033
 
3483
4034
  /**
@@ -3524,63 +4075,1207 @@ async function dockerListContainersByLabel(
3524
4075
  .filter((x): x is DockerPs => x !== null);
3525
4076
  }
3526
4077
 
3527
- async function dockerStart(containerName: string): Promise<boolean> {
4078
+ type DockerStartResult = "started" | "failed" | "unreachable";
4079
+
4080
+ async function dockerDaemonReachable(): Promise<boolean> {
4081
+ const probe = await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
4082
+ timeoutMs: 10_000,
4083
+ });
4084
+ return probe.status === 0 && probe.stdout.trim().length > 0;
4085
+ }
4086
+
4087
+ async function dockerStart(containerName: string): Promise<DockerStartResult> {
3528
4088
  // Starting a big dev container can legitimately take a while — give it far
3529
4089
  // more than dockerCli's 30s default before declaring the task stopped.
3530
4090
  const res = await dockerCli(["start", containerName], { timeoutMs: 120_000 });
3531
- if (res.status !== 0) {
3532
- console.error(
3533
- `[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
3534
- );
3535
- return false;
3536
- }
3537
- return true;
3538
- }
3539
-
3540
- /**
3541
- * Repair the shared node data root before any recovered-container init.
3542
- * Older task-up versions created the OpenCode leaf as root and accidentally
3543
- * left this parent root-owned, which prevented code-server from creating its
3544
- * managed User profile. `install -d` creates or repairs the exact directory
3545
- * without recursively changing unrelated application state beneath it.
3546
- */
3547
- async function repairNodeDataRoot(containerName: string): Promise<boolean> {
3548
- const res = await dockerCli([
3549
- "exec",
3550
- "-u",
3551
- "root",
4091
+ if (res.status === 0) return "started";
4092
+ // A timeout is ambiguous: Docker may have completed the start after our CLI
4093
+ // was killed. Inspect the exact container, then prove the daemon still
4094
+ // exists before deciding a non-inspectable container is a terminal failure.
4095
+ const inspected = await dockerCli([
4096
+ "inspect",
4097
+ "--format",
4098
+ "{{.State.Running}}",
3552
4099
  containerName,
3553
- "/usr/bin/install",
3554
- "-d",
3555
- "-o",
3556
- "node",
3557
- "-g",
3558
- "node",
3559
- "-m",
3560
- "0755",
3561
- "/home/node/.local/share",
3562
4100
  ]);
3563
- if (res.status !== 0) {
3564
- console.error(
3565
- `[orchestrator] recovery: ${containerName} could not repair /home/node/.local/share ownership: ${res.stderr.trim() || `docker exec exited ${String(res.status)}`}`,
3566
- );
3567
- return false;
4101
+ if (inspected.status === 0 && inspected.stdout.trim() === "true") {
4102
+ return "started";
3568
4103
  }
3569
- return true;
4104
+ if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
4105
+ return "unreachable";
4106
+ }
4107
+ console.error(
4108
+ `[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
4109
+ );
4110
+ return "failed";
4111
+ }
4112
+
4113
+ const RUNTIME_WRITER_LIST_FORMAT =
4114
+ '{{json .Names}}\t{{.Label "com.docker.compose.project"}}\t{{.Label "com.docker.compose.service"}}';
4115
+ const RUNTIME_WRITER_INSPECT_FORMAT = [
4116
+ "{{json .Id}}",
4117
+ "{{json .Name}}",
4118
+ "{{json .State.Running}}",
4119
+ "{{json .State.Restarting}}",
4120
+ "{{json .State.Paused}}",
4121
+ "{{json .HostConfig.RestartPolicy.Name}}",
4122
+ '{{json (index .Config.Labels "com.docker.compose.project")}}',
4123
+ '{{json (index .Config.Labels "com.docker.compose.service")}}',
4124
+ "{{json .Mounts}}",
4125
+ "{{json .Config.Env}}",
4126
+ ].join("\n");
4127
+ const RUNTIME_WRITER_CONTAINER_ID_RE = /^[0-9a-f]{64}$/;
4128
+ const RUNTIME_WRITER_SCAN_OUTPUT_BYTES = 2 * 1024 * 1024;
4129
+ const TASK_RUNTIME_PATH = RUNTIME_AUTHORITY_ENV.PATH;
4130
+ const TASK_OWNED_ENV = new Map<string, string>(
4131
+ Object.entries(RUNTIME_AUTHORITY_ENV),
4132
+ );
4133
+ const NPM_CONFIG_ENV_PREFIX = "npm_config_";
4134
+ function normalizeNpmConfigEnvKey(key: string): string | undefined {
4135
+ const folded = key.toLowerCase();
4136
+ if (!folded.startsWith(NPM_CONFIG_ENV_PREFIX)) return undefined;
4137
+ // npm converts underscores in the option suffix to hyphens. Docker can
4138
+ // preserve spellings a shell cannot assign (for example LOGS-DIR), so both
4139
+ // forms must be treated as the same owned option during immutable-env proof.
4140
+ return (
4141
+ NPM_CONFIG_ENV_PREFIX +
4142
+ folded.slice(NPM_CONFIG_ENV_PREFIX.length).replaceAll("_", "-")
4143
+ );
4144
+ }
4145
+ const TASK_OWNED_NORMALIZED_NPM_ENV_KEYS = new Set(
4146
+ [...TASK_OWNED_ENV.keys()]
4147
+ .map(normalizeNpmConfigEnvKey)
4148
+ .filter((key): key is string => key !== undefined),
4149
+ );
4150
+
4151
+ interface RuntimeWriterCandidate {
4152
+ taskId: string;
4153
+ composeProject: string;
4154
+ containerName: string;
4155
+ }
4156
+
4157
+ interface RuntimeWriterMetadata extends RuntimeWriterCandidate {
4158
+ containerId: string;
4159
+ running: boolean;
4160
+ restarting: boolean;
4161
+ paused: boolean;
4162
+ restartPolicy: string;
4163
+ sharedMount: "absent" | "read-only" | "writable" | "invalid";
4164
+ taskEnvironment: "current" | "invalid";
4165
+ }
4166
+
4167
+ type RuntimeWriterInspection =
4168
+ | { kind: "ready"; metadata: RuntimeWriterMetadata }
4169
+ | { kind: "failed"; detail: string }
4170
+ | { kind: "unreachable" };
4171
+
4172
+ export type TaskRuntimeContainerInspection =
4173
+ | {
4174
+ kind: "ready";
4175
+ containerId: string;
4176
+ running: boolean;
4177
+ restarting: boolean;
4178
+ paused: boolean;
4179
+ sharedMount: RuntimeWriterMetadata["sharedMount"];
4180
+ taskEnvironment: RuntimeWriterMetadata["taskEnvironment"];
4181
+ }
4182
+ | { kind: "failed"; detail: string }
4183
+ | { kind: "unreachable" };
4184
+
4185
+ export interface WritableRuntimeQuarantineResult {
4186
+ verdict: "ready" | "retry" | "stale";
4187
+ /** Known rows whose local `stopped` verdict must be published or replayed. */
4188
+ stoppedTaskIds: string[];
4189
+ }
4190
+
4191
+ function runtimeWriterCandidate(
4192
+ rawName: unknown,
4193
+ rawProject: string,
4194
+ rawService: string,
4195
+ ): RuntimeWriterCandidate | "unrelated" | "invalid" {
4196
+ if (!rawProject.startsWith("task-")) return "unrelated";
4197
+ const taskId = rawProject.slice("task-".length);
4198
+ if (!isSafeHostTaskId(taskId)) return "invalid";
4199
+ if (rawService !== "app" || typeof rawName !== "string") return "invalid";
4200
+ const containerName = `${rawProject}-app-1`;
4201
+ if (rawName !== containerName) return "invalid";
4202
+ return { taskId, composeProject: rawProject, containerName };
4203
+ }
4204
+
4205
+ /**
4206
+ * Enumerate the Docker authority, not only SQLite. A crashed task-down or an
4207
+ * older host can leave a Uai app container behind after its row changes, and
4208
+ * that orphan can still mutate the host-wide runtime volume.
4209
+ */
4210
+ async function listRuntimeWriterCandidates(): Promise<
4211
+ | { kind: "ready"; candidates: RuntimeWriterCandidate[]; ambiguous: boolean }
4212
+ | { kind: "unreachable" }
4213
+ > {
4214
+ const listed = await dockerCli(
4215
+ [
4216
+ "ps",
4217
+ "--all",
4218
+ "--filter",
4219
+ "label=com.docker.compose.project",
4220
+ "--filter",
4221
+ "label=com.docker.compose.service=app",
4222
+ "--format",
4223
+ RUNTIME_WRITER_LIST_FORMAT,
4224
+ ],
4225
+ { timeoutMs: 30_000, maxOutputBytes: RUNTIME_WRITER_SCAN_OUTPUT_BYTES },
4226
+ );
4227
+ if (listed.status !== 0 || listed.outputTruncated) {
4228
+ return { kind: "unreachable" };
4229
+ }
4230
+
4231
+ const candidates = new Map<string, RuntimeWriterCandidate>();
4232
+ let ambiguous = false;
4233
+ for (const rawLine of listed.stdout.split(/\r?\n/)) {
4234
+ if (!rawLine) continue;
4235
+ const fields = rawLine.split("\t");
4236
+ if (fields.length !== 3) {
4237
+ ambiguous = true;
4238
+ continue;
4239
+ }
4240
+ let name: unknown;
4241
+ try {
4242
+ name = JSON.parse(fields[0]!);
4243
+ } catch {
4244
+ ambiguous = true;
4245
+ continue;
4246
+ }
4247
+ const candidate = runtimeWriterCandidate(name, fields[1]!, fields[2]!);
4248
+ if (candidate === "unrelated") continue;
4249
+ if (candidate === "invalid") {
4250
+ ambiguous = true;
4251
+ continue;
4252
+ }
4253
+ if (candidates.has(candidate.containerName)) {
4254
+ ambiguous = true;
4255
+ continue;
4256
+ }
4257
+ candidates.set(candidate.containerName, candidate);
4258
+ }
4259
+ return { kind: "ready", candidates: [...candidates.values()], ambiguous };
4260
+ }
4261
+
4262
+ async function inspectRuntimeWriterContainer(
4263
+ candidate: RuntimeWriterCandidate,
4264
+ target: string = candidate.containerName,
4265
+ ): Promise<RuntimeWriterInspection> {
4266
+ const inspected = await dockerCli(
4267
+ ["inspect", "--format", RUNTIME_WRITER_INSPECT_FORMAT, target],
4268
+ { timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
4269
+ );
4270
+ if (inspected.status !== 0 || inspected.outputTruncated) {
4271
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
4272
+ return {
4273
+ kind: "failed",
4274
+ detail:
4275
+ inspected.stderr.trim() ||
4276
+ `could not inspect Uai app container ${candidate.containerName}`,
4277
+ };
4278
+ }
4279
+
4280
+ const fields = inspected.stdout.trimEnd().split(/\r?\n/);
4281
+ if (fields.length !== 10) {
4282
+ return { kind: "failed", detail: "Docker returned malformed app metadata" };
4283
+ }
4284
+ let parsed: unknown[];
4285
+ try {
4286
+ parsed = fields.map((field) => JSON.parse(field));
4287
+ } catch {
4288
+ return { kind: "failed", detail: "Docker returned malformed app metadata" };
4289
+ }
4290
+ const [
4291
+ containerId,
4292
+ containerName,
4293
+ running,
4294
+ restarting,
4295
+ paused,
4296
+ restartPolicy,
4297
+ composeProject,
4298
+ composeService,
4299
+ mounts,
4300
+ environment,
4301
+ ] = parsed;
4302
+ if (
4303
+ typeof containerId !== "string" ||
4304
+ !RUNTIME_WRITER_CONTAINER_ID_RE.test(containerId) ||
4305
+ containerName !== `/${candidate.containerName}` ||
4306
+ typeof running !== "boolean" ||
4307
+ typeof restarting !== "boolean" ||
4308
+ typeof paused !== "boolean" ||
4309
+ typeof restartPolicy !== "string" ||
4310
+ composeProject !== candidate.composeProject ||
4311
+ composeService !== "app" ||
4312
+ !Array.isArray(mounts)
4313
+ ) {
4314
+ return {
4315
+ kind: "failed",
4316
+ detail: `Docker returned ambiguous identity/state metadata for ${candidate.containerName}`,
4317
+ };
4318
+ }
4319
+
4320
+ if (
4321
+ !mounts.every(
4322
+ (mount) => mount && typeof mount === "object" && !Array.isArray(mount),
4323
+ )
4324
+ ) {
4325
+ return {
4326
+ kind: "failed",
4327
+ detail: `Docker returned malformed mount metadata for ${candidate.containerName}`,
4328
+ };
4329
+ }
4330
+ const mountRecords = mounts as Array<Record<string, unknown>>;
4331
+ const asdfMounts = mountRecords.filter(
4332
+ (mount) => mount.Destination === "/opt/asdf-data",
4333
+ );
4334
+ const namedSharedMounts = mountRecords.filter(
4335
+ (mount) => mount.Name === "uai-asdf-data",
4336
+ );
4337
+ let sharedMount: RuntimeWriterMetadata["sharedMount"];
4338
+ if (
4339
+ namedSharedMounts.some(
4340
+ (mount) => mount.Type === "volume" && mount.RW === true,
4341
+ )
4342
+ ) {
4343
+ // The volume name, not its destination, is the shared writer authority.
4344
+ // An orphan can mount it at an arbitrary path and still race maintenance.
4345
+ sharedMount = "writable";
4346
+ } else if (
4347
+ namedSharedMounts.some(
4348
+ (mount) => mount.Type !== "volume" || typeof mount.RW !== "boolean",
4349
+ )
4350
+ ) {
4351
+ sharedMount = "invalid";
4352
+ } else if (asdfMounts.length === 0) {
4353
+ sharedMount = "absent";
4354
+ } else if (
4355
+ asdfMounts.length === 1 &&
4356
+ namedSharedMounts.length === 1 &&
4357
+ asdfMounts[0]!.Type === "volume" &&
4358
+ asdfMounts[0]!.Name === "uai-asdf-data" &&
4359
+ asdfMounts[0]!.RW === false
4360
+ ) {
4361
+ sharedMount = "read-only";
4362
+ } else {
4363
+ sharedMount = "invalid";
4364
+ }
4365
+ const environmentIsStringArray =
4366
+ Array.isArray(environment) &&
4367
+ environment.every((entry) => typeof entry === "string");
4368
+ const environmentEntries = environmentIsStringArray
4369
+ ? (environment as string[])
4370
+ : [];
4371
+ const exactValues = new Map<string, string[]>();
4372
+ let taskEnvironment: RuntimeWriterMetadata["taskEnvironment"] =
4373
+ environmentIsStringArray ? "current" : "invalid";
4374
+ for (const entry of environmentEntries) {
4375
+ const divider = entry.indexOf("=");
4376
+ const key = divider < 0 ? entry : entry.slice(0, divider);
4377
+ const value = divider < 0 ? "" : entry.slice(divider + 1);
4378
+ if (divider < 1) taskEnvironment = "invalid";
4379
+ const normalizedNpmKey = normalizeNpmConfigEnvKey(key);
4380
+ if (
4381
+ normalizedNpmKey !== undefined &&
4382
+ TASK_OWNED_NORMALIZED_NPM_ENV_KEYS.has(normalizedNpmKey) &&
4383
+ !TASK_OWNED_ENV.has(key)
4384
+ ) {
4385
+ taskEnvironment = "invalid";
4386
+ }
4387
+ const values = exactValues.get(key) ?? [];
4388
+ values.push(value);
4389
+ exactValues.set(key, values);
4390
+ }
4391
+ for (const [key, expected] of TASK_OWNED_ENV) {
4392
+ const values = exactValues.get(key);
4393
+ if (values?.length !== 1 || values[0] !== expected) {
4394
+ taskEnvironment = "invalid";
4395
+ }
4396
+ }
4397
+ // Keep an explicit assertion for the trust-precedence property even though
4398
+ // PATH is also part of TASK_OWNED_ENV above.
4399
+ if (exactValues.get("PATH")?.[0] !== TASK_RUNTIME_PATH) taskEnvironment = "invalid";
4400
+ return {
4401
+ kind: "ready",
4402
+ metadata: {
4403
+ ...candidate,
4404
+ containerId,
4405
+ running,
4406
+ restarting,
4407
+ paused,
4408
+ restartPolicy,
4409
+ sharedMount,
4410
+ taskEnvironment,
4411
+ },
4412
+ };
4413
+ }
4414
+
4415
+ /** Exact single-task form used by duplicate taskUp admission. */
4416
+ export async function inspectTaskRuntimeContainer(
4417
+ taskId: string,
4418
+ composeProject: string,
4419
+ ): Promise<TaskRuntimeContainerInspection> {
4420
+ if (!isSafeHostTaskId(taskId) || composeProject !== `task-${taskId}`) {
4421
+ return { kind: "failed", detail: "invalid task/Compose runtime identity" };
4422
+ }
4423
+ const inspected = await inspectRuntimeWriterContainer({
4424
+ taskId,
4425
+ composeProject,
4426
+ containerName: `${composeProject}-app-1`,
4427
+ });
4428
+ if (inspected.kind !== "ready") return inspected;
4429
+ return {
4430
+ kind: "ready",
4431
+ containerId: inspected.metadata.containerId,
4432
+ running: inspected.metadata.running,
4433
+ restarting: inspected.metadata.restarting,
4434
+ paused: inspected.metadata.paused,
4435
+ sharedMount: inspected.metadata.sharedMount,
4436
+ taskEnvironment: inspected.metadata.taskEnvironment,
4437
+ };
4438
+ }
4439
+
4440
+ function restartPolicyDisabled(name: string): boolean {
4441
+ return name === "" || name === "no";
4442
+ }
4443
+
4444
+ async function stopRuntimeWriter(
4445
+ metadata: RuntimeWriterMetadata,
4446
+ ): Promise<"stopped" | "failed" | "unreachable"> {
4447
+ // A preserved legacy container must not spring back up under Docker's own
4448
+ // restart policy after the host has declared it stopped. Mount mode is
4449
+ // immutable, so only explicit taskUp may recreate it with the L5 contract.
4450
+ await dockerCli(["update", "--restart=no", metadata.containerId], {
4451
+ timeoutMs: 30_000,
4452
+ maxOutputBytes: 1024 * 1024,
4453
+ });
4454
+ let current = await inspectRuntimeWriterContainer(
4455
+ metadata,
4456
+ metadata.containerId,
4457
+ );
4458
+ if (current.kind !== "ready") return current.kind;
4459
+ if (
4460
+ current.metadata.containerId !== metadata.containerId ||
4461
+ !restartPolicyDisabled(current.metadata.restartPolicy)
4462
+ ) {
4463
+ return "failed";
4464
+ }
4465
+
4466
+ if (current.metadata.paused) {
4467
+ // Docker refuses ordinary stop/kill operations for some paused states.
4468
+ // The restart policy is already disabled; unpause the exact immutable id,
4469
+ // re-prove identity/policy, then immediately enter the bounded stop path.
4470
+ await dockerCli(["unpause", metadata.containerId], {
4471
+ timeoutMs: 15_000,
4472
+ maxOutputBytes: 1024 * 1024,
4473
+ });
4474
+ current = await inspectRuntimeWriterContainer(
4475
+ metadata,
4476
+ metadata.containerId,
4477
+ );
4478
+ if (current.kind !== "ready") return current.kind;
4479
+ if (
4480
+ current.metadata.containerId !== metadata.containerId ||
4481
+ !restartPolicyDisabled(current.metadata.restartPolicy) ||
4482
+ current.metadata.paused
4483
+ ) {
4484
+ return "failed";
4485
+ }
4486
+ }
4487
+
4488
+ if (!current.metadata.running && !current.metadata.restarting) {
4489
+ return "stopped";
4490
+ }
4491
+ await dockerCli(["stop", "--time", "10", metadata.containerId], {
4492
+ timeoutMs: 30_000,
4493
+ maxOutputBytes: 1024 * 1024,
4494
+ });
4495
+ current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
4496
+ if (current.kind !== "ready") return current.kind;
4497
+ if (
4498
+ current.metadata.containerId === metadata.containerId &&
4499
+ restartPolicyDisabled(current.metadata.restartPolicy) &&
4500
+ !current.metadata.running &&
4501
+ !current.metadata.restarting
4502
+ ) {
4503
+ return "stopped";
4504
+ }
4505
+
4506
+ await dockerCli(["kill", metadata.containerId], {
4507
+ timeoutMs: 15_000,
4508
+ maxOutputBytes: 1024 * 1024,
4509
+ });
4510
+ current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
4511
+ if (current.kind !== "ready") return current.kind;
4512
+ return current.metadata.containerId === metadata.containerId &&
4513
+ restartPolicyDisabled(current.metadata.restartPolicy) &&
4514
+ !current.metadata.running &&
4515
+ !current.metadata.restarting
4516
+ ? "stopped"
4517
+ : "failed";
4518
+ }
4519
+
4520
+ /** Stop the exact container previously inspected by taskUp. Re-inspection by
4521
+ * immutable id closes the name-reuse gap before any shared-volume writer. */
4522
+ export async function quarantineTaskRuntimeContainer(options: {
4523
+ taskId: string;
4524
+ composeProject: string;
4525
+ expectedContainerId: string;
4526
+ }): Promise<"stopped" | "failed" | "unreachable"> {
4527
+ if (
4528
+ !isSafeHostTaskId(options.taskId) ||
4529
+ options.composeProject !== `task-${options.taskId}` ||
4530
+ !RUNTIME_WRITER_CONTAINER_ID_RE.test(options.expectedContainerId)
4531
+ ) {
4532
+ return "failed";
4533
+ }
4534
+ const candidate: RuntimeWriterCandidate = {
4535
+ taskId: options.taskId,
4536
+ composeProject: options.composeProject,
4537
+ containerName: `${options.composeProject}-app-1`,
4538
+ };
4539
+ const inspected = await inspectRuntimeWriterContainer(
4540
+ candidate,
4541
+ options.expectedContainerId,
4542
+ );
4543
+ if (inspected.kind !== "ready") return inspected.kind;
4544
+ if (inspected.metadata.containerId !== options.expectedContainerId) {
4545
+ return "failed";
4546
+ }
4547
+ return stopRuntimeWriter(inspected.metadata);
4548
+ }
4549
+
4550
+ /**
4551
+ * L5 rolling fence. It must complete before the first standard-image/runtime
4552
+ * volume writer of an activation generation. New task containers mount the
4553
+ * shared volume read-only; every preserved legacy writer is tombstoned and
4554
+ * stopped, while an exact RO container remains eligible for recovery.
4555
+ */
4556
+ export async function quarantineWritableRuntimeContainers(options: {
4557
+ lifecycle: Orchestrator;
4558
+ isCurrent?: () => boolean;
4559
+ /** `contract` (default) sweeps only containers whose shared-volume
4560
+ * attachment violates the current contract. `volume-holders` sweeps EVERY
4561
+ * task container attached to the shared volume — the activation driver's
4562
+ * non-writer escape when the volume needs a repair the holders block
4563
+ * (ADR-108; apple only, a no-op on Docker). */
4564
+ scope?: "contract" | "volume-holders";
4565
+ }): Promise<WritableRuntimeQuarantineResult> {
4566
+ // Dispatch on the PIN: this sweep runs inside activation's
4567
+ // beforeMaintenance, where the public state is still `checking` and the
4568
+ // ready-gated identity is null — the docker fallback then retries forever
4569
+ // against a daemon that does not exist on a Docker-free Mac.
4570
+ if (pinnedContainerRuntimeProvider() === "apple-container") {
4571
+ return quarantineWritableAppleRuntimeContainers(options);
4572
+ }
4573
+ const isCurrent = options.isCurrent ?? (() => true);
4574
+ const stoppedTaskIds = new Set<string>();
4575
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
4576
+
4577
+ const listed = await listRuntimeWriterCandidates();
4578
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
4579
+ if (listed.kind === "unreachable") {
4580
+ return { verdict: "retry", stoppedTaskIds: [] };
4581
+ }
4582
+ let resolved = !listed.ambiguous;
4583
+
4584
+ for (const candidate of listed.candidates) {
4585
+ if (!isCurrent()) {
4586
+ return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
4587
+ }
4588
+ await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
4589
+ if (!isCurrent()) return;
4590
+ const inspection = await inspectRuntimeWriterContainer(candidate);
4591
+ if (inspection.kind !== "ready") {
4592
+ resolved = false;
4593
+ console.error(
4594
+ `[orchestrator] runtime isolation: ${candidate.containerName} inspection did not prove a safe mount (${inspection.kind === "failed" ? inspection.detail : "Docker unreachable"})`,
4595
+ );
4596
+ return;
4597
+ }
4598
+ const task = getHostTask(candidate.taskId);
4599
+ const knownPreservedTask =
4600
+ task?.composeProject === candidate.composeProject &&
4601
+ typeof task.statusMirror === "string" &&
4602
+ isOnDashboard(task.statusMirror);
4603
+ if (
4604
+ inspection.metadata.sharedMount === "read-only" &&
4605
+ inspection.metadata.taskEnvironment === "current"
4606
+ ) {
4607
+ return;
4608
+ }
4609
+ if (!knownPreservedTask) {
4610
+ // A task-shaped orphan with a proven RO shared mount cannot race
4611
+ // maintenance. Without a host row, its non-current task-local npm
4612
+ // environment is not enough ownership evidence to mutate it.
4613
+ if (inspection.metadata.sharedMount === "read-only") return;
4614
+ if (inspection.metadata.sharedMount === "absent") return;
4615
+ if (inspection.metadata.sharedMount === "invalid") {
4616
+ // `task-*` is Uai's namespace, but it is not by itself permission to
4617
+ // stop an unrelated/stale user stack. Only the exact named writable
4618
+ // Uai volume is positive ownership evidence without a host row.
4619
+ resolved = false;
4620
+ console.error(
4621
+ `[orchestrator] runtime isolation: orphan ${candidate.containerName} has ambiguous shared-runtime mount metadata; maintenance remains fenced`,
4622
+ );
4623
+ return;
4624
+ }
4625
+ }
4626
+
4627
+ // Drain before the first stop/update. A running agent continuation may
4628
+ // otherwise race the quarantine and respawn into the legacy container.
4629
+ await options.lifecycle.quarantineChannelForRecovery(candidate.taskId);
4630
+ const stopped = await stopRuntimeWriter(inspection.metadata);
4631
+ if (stopped !== "stopped") {
4632
+ resolved = false;
4633
+ console.error(
4634
+ `[orchestrator] runtime isolation: could not prove ${candidate.containerName} stopped (${stopped})`,
4635
+ );
4636
+ return;
4637
+ }
4638
+
4639
+ if (knownPreservedTask) {
4640
+ // Always re-publish the stopped verdict. A prior process may have
4641
+ // committed SQLite and crashed before its websocket frame left; the
4642
+ // cloud could otherwise remain ghost-running indefinitely.
4643
+ stoppedTaskIds.add(candidate.taskId);
4644
+ db_setStatus(candidate.taskId, "stopped", {
4645
+ codeServerPort: null,
4646
+ previewPorts: "[]",
4647
+ });
4648
+ }
4649
+ clearTaskGatewayAcl(candidate.taskId);
4650
+ // We deliberately reflect a proven physical stop even if the daemon
4651
+ // epoch changed while stopping: leaving the row active would invite a
4652
+ // later recovery to restart an immutable legacy writer. The next
4653
+ // activation generation independently re-enumerates Docker before
4654
+ // maintenance. Non-safety cleanup below may wait for that generation.
4655
+ if (!isCurrent()) return;
4656
+ try {
4657
+ await stopPreviewSidecars(candidate.taskId);
4658
+ } catch (error) {
4659
+ console.warn(
4660
+ `[orchestrator] runtime isolation: ${candidate.taskId} preview cleanup failed: ${
4661
+ error instanceof Error ? error.message : String(error)
4662
+ }`,
4663
+ );
4664
+ }
4665
+ console.warn(
4666
+ `[orchestrator] runtime isolation: ${candidate.containerName} used a writable/non-current shared runtime mount and remains stopped until explicit Resume recreates it`,
4667
+ );
4668
+ });
4669
+ }
4670
+
4671
+ if (!isCurrent()) {
4672
+ return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
4673
+ }
4674
+ return {
4675
+ verdict: resolved ? "ready" : "retry",
4676
+ stoppedTaskIds: [...stoppedTaskIds].sort(),
4677
+ };
4678
+ }
4679
+
4680
+ const RECOVERY_RUNTIME_INSPECT_FORMAT =
4681
+ "{{json .Image}}|{{json .State.Running}}|{{json .HostConfig.GroupAdd}}|{{json .Mounts}}";
4682
+ const RECOVERY_RUNTIME_IMAGE_RE = /^sha256:[0-9a-f]{64}$/;
4683
+ const RECOVERY_RUNTIME_SCRIPT_PATH =
4684
+ "/usr/local/bin/uai-materialize-runtimes";
4685
+ const RECOVERY_COREPACK_VERSION_PATH =
4686
+ "/usr/local/share/uai/corepack-version";
4687
+ const RECOVERY_INIT_PATH = "/uai-init-current";
4688
+ const RECOVERY_INIT_PROJECTS_PATH = "/uai-runtime-projects-current";
4689
+ const RECOVERY_RUNTIME_TIMEOUT_MS = 630_000;
4690
+ const RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES = 1024 * 1024;
4691
+
4692
+ interface RecoveryDockerMount {
4693
+ Type?: unknown;
4694
+ Name?: unknown;
4695
+ Source?: unknown;
4696
+ Destination?: unknown;
4697
+ RW?: unknown;
4698
+ }
4699
+
4700
+ interface RecoveryRuntimeSpec {
4701
+ imageId: string;
4702
+ workspaceSource: string;
4703
+ groupAdds: string[];
4704
+ projectCount: number;
4705
+ projectAllowlistSource: string;
4706
+ }
4707
+
4708
+ type RecoveryRuntimeInspection =
4709
+ | { kind: "ready"; spec: RecoveryRuntimeSpec }
4710
+ | { kind: "failed"; detail: string }
4711
+ | { kind: "unreachable" };
4712
+
4713
+ type RecoveryRuntimeResult =
4714
+ | { kind: "ready" }
4715
+ | { kind: "failed"; detail: string }
4716
+ | { kind: "unreachable" };
4717
+
4718
+ function recoveryRuntimeAssets():
4719
+ | { materializer: string; corepackVersion: string; uaiInit: string }
4720
+ | { error: string } {
4721
+ const materializer = standardImageRuntimeMaterializerPath();
4722
+ const corepackVersion = standardImageCorepackVersionPath();
4723
+ const uaiInit = join(dirname(materializer), "uai-init");
4724
+ try {
4725
+ const materializerStat = lstatSync(materializer);
4726
+ if (!materializerStat.isFile() || materializerStat.isSymbolicLink() ||
4727
+ (materializerStat.mode & 0o111) === 0) {
4728
+ return { error: `packaged materializer is not a regular executable: ${materializer}` };
4729
+ }
4730
+ const versionStat = lstatSync(corepackVersion);
4731
+ if (!versionStat.isFile() || versionStat.isSymbolicLink()) {
4732
+ return { error: `packaged Corepack pin is not a regular file: ${corepackVersion}` };
4733
+ }
4734
+ const version = readFileSync(corepackVersion, "utf8");
4735
+ if (!/^\d+\.\d+\.\d+\n?$/.test(version)) {
4736
+ return { error: `packaged Corepack pin is malformed: ${corepackVersion}` };
4737
+ }
4738
+ const initStat = lstatSync(uaiInit);
4739
+ if (
4740
+ !initStat.isFile() ||
4741
+ initStat.isSymbolicLink() ||
4742
+ (initStat.mode & 0o111) === 0
4743
+ ) {
4744
+ return { error: `packaged task initializer is not a regular executable: ${uaiInit}` };
4745
+ }
4746
+ } catch (err) {
4747
+ return {
4748
+ error: `packaged runtime assets are unavailable: ${
4749
+ err instanceof Error ? err.message : String(err)
4750
+ }`,
4751
+ };
4752
+ }
4753
+ return { materializer, corepackVersion, uaiInit };
4754
+ }
4755
+
4756
+ function expectedRecoveryWorkspaceSources(
4757
+ task: typeof schema.hostTasks.$inferSelect,
4758
+ ): Set<string> {
4759
+ const expected = new Set<string>();
4760
+ if (!task.worktreePath) return expected;
4761
+ const lexical = join(task.worktreePath, "workspace");
4762
+ try {
4763
+ const stat = lstatSync(lexical);
4764
+ if (!stat.isDirectory() || stat.isSymbolicLink()) return expected;
4765
+ expected.add(lexical);
4766
+ expected.add(realpathSync(lexical));
4767
+ } catch {
4768
+ // Missing, unreadable, and unresolvable workspaces are never safe input
4769
+ // to `docker run`: a legacy `-v` bind would recreate a missing source as
4770
+ // an empty directory and falsely prove that the task had no projects.
4771
+ }
4772
+ return expected;
4773
+ }
4774
+
4775
+ function validRecoveryGroupAdds(value: unknown): string[] | null {
4776
+ if (value === null || value === undefined) return [];
4777
+ if (!Array.isArray(value)) return null;
4778
+ const groups: string[] = [];
4779
+ for (const candidate of value) {
4780
+ if (typeof candidate !== "string" || !/^[1-9]\d{0,9}$/.test(candidate)) {
4781
+ return null;
4782
+ }
4783
+ const numeric = Number(candidate);
4784
+ if (!Number.isSafeInteger(numeric) || numeric > 4_294_967_294) {
4785
+ return null;
4786
+ }
4787
+ if (!groups.includes(candidate)) groups.push(candidate);
4788
+ }
4789
+ return groups;
4790
+ }
4791
+
4792
+ function safeRecoveryProjectSlug(slug: unknown): slug is string {
4793
+ return typeof slug === "string" && /^[a-z0-9-]{1,64}$/.test(slug);
4794
+ }
4795
+
4796
+ function validateRecoveryProjects(
4797
+ task: typeof schema.hostTasks.$inferSelect,
4798
+ workspaceSource: string,
4799
+ ): { slugs: string[] } | { error: string } {
4800
+ let parsed: unknown;
4801
+ try {
4802
+ parsed = JSON.parse(task.projectSlugs);
4803
+ } catch {
4804
+ return { error: "the persisted project slug list is invalid JSON" };
4805
+ }
4806
+ if (!Array.isArray(parsed) || !parsed.every(safeRecoveryProjectSlug)) {
4807
+ return { error: "the persisted project slug list contains an unsafe entry" };
4808
+ }
4809
+ const slugs = parsed as string[];
4810
+ const expected = new Set(slugs);
4811
+ if (expected.size !== slugs.length) {
4812
+ return { error: "the persisted project slug list contains duplicates" };
4813
+ }
4814
+
4815
+ for (const slug of expected) {
4816
+ const project = join(workspaceSource, slug);
4817
+ let projectStat: ReturnType<typeof lstatSync>;
4818
+ let markerStat: ReturnType<typeof lstatSync>;
4819
+ try {
4820
+ projectStat = lstatSync(project);
4821
+ markerStat = lstatSync(join(project, ".git"));
4822
+ } catch {
4823
+ return { error: `expected project ${slug} or its .git marker is missing` };
4824
+ }
4825
+ if (
4826
+ projectStat.isSymbolicLink() ||
4827
+ !projectStat.isDirectory() ||
4828
+ markerStat.isSymbolicLink() ||
4829
+ (!markerStat.isFile() && !markerStat.isDirectory())
4830
+ ) {
4831
+ return { error: `expected project ${slug} is not a safe Git worktree` };
4832
+ }
4833
+ }
4834
+ return { slugs };
4835
+ }
4836
+
4837
+ function validateRecoveryWorkspaceRuntimeRoot(
4838
+ workspaceSource: string,
4839
+ ): string | null {
4840
+ const shadow = join(workspaceSource, ".tool-versions");
4841
+ try {
4842
+ // Any type is forbidden, including regular files and dangling symlinks.
4843
+ // lstat is metadata-only, so FIFO/device/socket entries cannot block the
4844
+ // host recovery loop.
4845
+ lstatSync(shadow);
4846
+ return "the workspace root contains a forbidden .tool-versions runtime override";
4847
+ } catch (err) {
4848
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
4849
+ return `the workspace-root runtime authority could not be verified: ${
4850
+ err instanceof Error ? err.message : String(err)
4851
+ }`;
4852
+ }
4853
+ }
4854
+
4855
+ function recoveryRuntimeProjectAllowlist(
4856
+ task: typeof schema.hostTasks.$inferSelect,
4857
+ slugs: string[],
4858
+ ): { path: string } | { error: string } {
4859
+ if (!task.worktreePath) {
4860
+ return { error: "the task has no persisted runtime directory" };
4861
+ }
4862
+ const runtimeDir = join(task.worktreePath, ".uai");
4863
+ const target = join(runtimeDir, "runtime-projects");
4864
+ const expected = slugs.length > 0 ? `${slugs.join("\n")}\n` : "";
4865
+ try {
4866
+ const runtimeDirStat = lstatSync(runtimeDir);
4867
+ if (!runtimeDirStat.isDirectory() || runtimeDirStat.isSymbolicLink()) {
4868
+ return { error: "the task runtime directory is not a safe directory" };
4869
+ }
4870
+ try {
4871
+ const targetStat = lstatSync(target);
4872
+ if (!targetStat.isFile() || targetStat.isSymbolicLink()) {
4873
+ return { error: "the task runtime-project allowlist is not a regular file" };
4874
+ }
4875
+ if (readFileSync(target, "utf8") !== expected) {
4876
+ return { error: "the task runtime-project allowlist does not match persisted projects" };
4877
+ }
4878
+ return { path: target };
4879
+ } catch (err) {
4880
+ if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
4881
+ }
4882
+
4883
+ if (slugs.length === 0) {
4884
+ return {
4885
+ error: "zero-project recovery lacks an existing authoritative empty runtime-project allowlist",
4886
+ };
4887
+ }
4888
+
4889
+ const stagingDir = mkdtempSync(join(runtimeDir, ".runtime-projects-"));
4890
+ const staged = join(stagingDir, "runtime-projects");
4891
+ try {
4892
+ writeFileSync(staged, expected, { encoding: "utf8", flag: "wx", mode: 0o644 });
4893
+ renameSync(staged, target);
4894
+ } finally {
4895
+ rmSync(stagingDir, { recursive: true, force: true });
4896
+ }
4897
+ return { path: target };
4898
+ } catch (err) {
4899
+ return {
4900
+ error: `the task runtime-project allowlist is unavailable: ${
4901
+ err instanceof Error ? err.message : String(err)
4902
+ }`,
4903
+ };
4904
+ }
4905
+ }
4906
+
4907
+ async function inspectRecoveryRuntime(
4908
+ task: typeof schema.hostTasks.$inferSelect,
4909
+ containerName: string,
4910
+ ): Promise<RecoveryRuntimeInspection> {
4911
+ const inspected = await dockerCli(
4912
+ [
4913
+ "inspect",
4914
+ "--format",
4915
+ RECOVERY_RUNTIME_INSPECT_FORMAT,
4916
+ containerName,
4917
+ ],
4918
+ { timeoutMs: 30_000, maxOutputBytes: 128 * 1024 },
4919
+ );
4920
+ if (inspected.status !== 0) {
4921
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
4922
+ return {
4923
+ kind: "failed",
4924
+ detail:
4925
+ inspected.stderr.trim() ||
4926
+ `could not inspect stopped app container ${containerName}`,
4927
+ };
4928
+ }
4929
+
4930
+ const firstDivider = inspected.stdout.indexOf("|");
4931
+ const secondDivider = inspected.stdout.indexOf("|", firstDivider + 1);
4932
+ const thirdDivider = inspected.stdout.indexOf("|", secondDivider + 1);
4933
+ if (firstDivider < 0 || secondDivider < 0 || thirdDivider < 0) {
4934
+ return { kind: "failed", detail: "Docker returned malformed runtime metadata" };
4935
+ }
4936
+
4937
+ let imageId: unknown;
4938
+ let running: unknown;
4939
+ let rawGroupAdds: unknown;
4940
+ let mounts: unknown;
4941
+ try {
4942
+ imageId = JSON.parse(inspected.stdout.slice(0, firstDivider));
4943
+ running = JSON.parse(
4944
+ inspected.stdout.slice(firstDivider + 1, secondDivider),
4945
+ );
4946
+ rawGroupAdds = JSON.parse(
4947
+ inspected.stdout.slice(secondDivider + 1, thirdDivider),
4948
+ );
4949
+ mounts = JSON.parse(inspected.stdout.slice(thirdDivider + 1));
4950
+ } catch {
4951
+ return { kind: "failed", detail: "Docker returned invalid runtime metadata JSON" };
4952
+ }
4953
+ if (running !== false) {
4954
+ return {
4955
+ kind: "failed",
4956
+ detail: "the app container was not proven stopped before runtime maintenance",
4957
+ };
4958
+ }
4959
+ if (typeof imageId !== "string" || !RECOVERY_RUNTIME_IMAGE_RE.test(imageId)) {
4960
+ return { kind: "failed", detail: "the app container has an invalid immutable image ID" };
4961
+ }
4962
+ if (!Array.isArray(mounts)) {
4963
+ return { kind: "failed", detail: "the app container has invalid mount metadata" };
4964
+ }
4965
+ const groupAdds = validRecoveryGroupAdds(rawGroupAdds);
4966
+ if (groupAdds === null) {
4967
+ return {
4968
+ kind: "failed",
4969
+ detail: "the app container has malformed or unsafe supplemental groups",
4970
+ };
4971
+ }
4972
+ const typedMounts = mounts as RecoveryDockerMount[];
4973
+ const workspaceMounts = typedMounts.filter(
4974
+ (mount) => mount.Destination === "/workspace",
4975
+ );
4976
+ const expectedWorkspaceSources = expectedRecoveryWorkspaceSources(task);
4977
+ if (
4978
+ workspaceMounts.length !== 1 ||
4979
+ workspaceMounts[0]?.Type !== "bind" ||
4980
+ typeof workspaceMounts[0]?.Source !== "string" ||
4981
+ !expectedWorkspaceSources.has(workspaceMounts[0].Source)
4982
+ ) {
4983
+ return {
4984
+ kind: "failed",
4985
+ detail: "the app container does not have the exact expected /workspace bind",
4986
+ };
4987
+ }
4988
+ try {
4989
+ const workspaceStat = lstatSync(workspaceMounts[0].Source);
4990
+ if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
4991
+ return {
4992
+ kind: "failed",
4993
+ detail: "the app container's /workspace source is not a real directory",
4994
+ };
4995
+ }
4996
+ } catch {
4997
+ return {
4998
+ kind: "failed",
4999
+ detail: "the app container's /workspace source no longer exists",
5000
+ };
5001
+ }
5002
+ const workspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
5003
+ workspaceMounts[0].Source,
5004
+ );
5005
+ if (workspaceRuntimeError) {
5006
+ return { kind: "failed", detail: workspaceRuntimeError };
5007
+ }
5008
+ const projects = validateRecoveryProjects(
5009
+ task,
5010
+ workspaceMounts[0].Source,
5011
+ );
5012
+ if ("error" in projects) {
5013
+ return { kind: "failed", detail: projects.error };
5014
+ }
5015
+ const projectAllowlist = recoveryRuntimeProjectAllowlist(
5016
+ task,
5017
+ projects.slugs,
5018
+ );
5019
+ if ("error" in projectAllowlist) {
5020
+ return { kind: "failed", detail: projectAllowlist.error };
5021
+ }
5022
+ const asdfMounts = typedMounts.filter(
5023
+ (mount) => mount.Destination === "/opt/asdf-data",
5024
+ );
5025
+ if (
5026
+ asdfMounts.length !== 1 ||
5027
+ asdfMounts[0]?.Type !== "volume" ||
5028
+ asdfMounts[0]?.Name !== "uai-asdf-data" ||
5029
+ asdfMounts[0]?.RW !== false
5030
+ ) {
5031
+ return {
5032
+ kind: "failed",
5033
+ detail:
5034
+ "the app container does not use the exact read-only uai-asdf-data volume; explicit Resume must recreate legacy writable containers",
5035
+ };
5036
+ }
5037
+ return {
5038
+ kind: "ready",
5039
+ spec: {
5040
+ imageId,
5041
+ workspaceSource: workspaceMounts[0].Source,
5042
+ groupAdds,
5043
+ projectCount: projects.slugs.length,
5044
+ projectAllowlistSource: projectAllowlist.path,
5045
+ },
5046
+ };
5047
+ }
5048
+
5049
+ async function materializeRecoveryRuntimes(
5050
+ spec: RecoveryRuntimeSpec,
5051
+ ): Promise<RecoveryRuntimeResult> {
5052
+ if (spec.projectCount === 0) return { kind: "ready" };
5053
+ const assets = recoveryRuntimeAssets();
5054
+ if ("error" in assets) return { kind: "failed", detail: assets.error };
5055
+ try {
5056
+ const workspaceStat = lstatSync(spec.workspaceSource);
5057
+ if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
5058
+ return {
5059
+ kind: "failed",
5060
+ detail: "the app container's /workspace source is no longer a real directory",
5061
+ };
5062
+ }
5063
+ } catch {
5064
+ return {
5065
+ kind: "failed",
5066
+ detail: "the app container's /workspace source disappeared before runtime proof",
5067
+ };
5068
+ }
5069
+ try {
5070
+ const allowlistStat = lstatSync(spec.projectAllowlistSource);
5071
+ if (!allowlistStat.isFile() || allowlistStat.isSymbolicLink()) {
5072
+ return {
5073
+ kind: "failed",
5074
+ detail: "the runtime-project allowlist is no longer a regular file",
5075
+ };
5076
+ }
5077
+ } catch {
5078
+ return {
5079
+ kind: "failed",
5080
+ detail: "the runtime-project allowlist disappeared before runtime proof",
5081
+ };
5082
+ }
5083
+ const supplementalGroups = spec.groupAdds.flatMap((group) => [
5084
+ "--group-add",
5085
+ group,
5086
+ ]);
5087
+ const result = await dockerCli(
5088
+ [
5089
+ "run",
5090
+ "--rm",
5091
+ "--user",
5092
+ "node",
5093
+ ...supplementalGroups,
5094
+ "--workdir",
5095
+ "/workspace",
5096
+ "--mount",
5097
+ `type=bind,src=${spec.workspaceSource},dst=/workspace,readonly`,
5098
+ "-v",
5099
+ "uai-asdf-data:/opt/asdf-data:rw",
5100
+ "-v",
5101
+ `${assets.materializer}:${RECOVERY_RUNTIME_SCRIPT_PATH}:ro`,
5102
+ "-v",
5103
+ `${assets.corepackVersion}:${RECOVERY_COREPACK_VERSION_PATH}:ro`,
5104
+ "-v",
5105
+ `${spec.projectAllowlistSource}:/run/uai/runtime-projects:ro`,
5106
+ "--env",
5107
+ "LD_PRELOAD=",
5108
+ "--env",
5109
+ "LD_AUDIT=",
5110
+ "--env",
5111
+ "LD_LIBRARY_PATH=",
5112
+ "--env",
5113
+ "BASH_ENV=",
5114
+ "--env",
5115
+ "ENV=",
5116
+ "--entrypoint",
5117
+ "/usr/bin/env",
5118
+ spec.imageId,
5119
+ "-i",
5120
+ "HOME=/home/node",
5121
+ "ASDF_DIR=/opt/asdf",
5122
+ "ASDF_DATA_DIR=/opt/asdf-data",
5123
+ "PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
5124
+ "/usr/bin/timeout",
5125
+ "--signal=TERM",
5126
+ "--kill-after=10s",
5127
+ "600s",
5128
+ RECOVERY_RUNTIME_SCRIPT_PATH,
5129
+ ],
5130
+ {
5131
+ timeoutMs: RECOVERY_RUNTIME_TIMEOUT_MS,
5132
+ tailOutputBytes: RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES,
5133
+ },
5134
+ );
5135
+ if (result.status === 0) return { kind: "ready" };
5136
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
5137
+ return {
5138
+ kind: "failed",
5139
+ detail: `${result.outputTruncated ? "earlier runtime output was truncated; " : ""}${
5140
+ result.stderr.trim() ||
5141
+ `runtime materializer exited ${String(result.status)}`
5142
+ }`,
5143
+ };
5144
+ }
5145
+
5146
+ async function stageRecoveryInit(
5147
+ containerName: string,
5148
+ spec: RecoveryRuntimeSpec,
5149
+ ): Promise<RecoveryRuntimeResult> {
5150
+ const assets = recoveryRuntimeAssets();
5151
+ if ("error" in assets) return { kind: "failed", detail: assets.error };
5152
+ const copies: Array<[string, string]> = [
5153
+ [assets.uaiInit, RECOVERY_INIT_PATH],
5154
+ [spec.projectAllowlistSource, RECOVERY_INIT_PROJECTS_PATH],
5155
+ ];
5156
+ for (const [source, destination] of copies) {
5157
+ const copied = await dockerCli(
5158
+ ["cp", source, `${containerName}:${destination}`],
5159
+ { timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
5160
+ );
5161
+ if (copied.status === 0) continue;
5162
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
5163
+ return {
5164
+ kind: "failed",
5165
+ detail:
5166
+ copied.stderr.trim() ||
5167
+ `could not stage ${destination} in the stopped app container`,
5168
+ };
5169
+ }
5170
+ return { kind: "ready" };
5171
+ }
5172
+
5173
+ type DockerQuarantineResult = "stopped" | "failed" | "unreachable";
5174
+
5175
+ async function quarantineRunningContainer(
5176
+ containerName: string,
5177
+ ): Promise<DockerQuarantineResult> {
5178
+ const stopped = await dockerCli(
5179
+ ["stop", "--time", "10", containerName],
5180
+ { timeoutMs: 30_000 },
5181
+ );
5182
+ const inspected = await dockerCli(
5183
+ ["inspect", "--format", "{{.State.Running}}", containerName],
5184
+ { timeoutMs: 10_000 },
5185
+ );
5186
+ if (inspected.status === 0 && inspected.stdout.trim() === "false") {
5187
+ return "stopped";
5188
+ }
5189
+ if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
5190
+ return "unreachable";
5191
+ }
5192
+ const killed = await dockerCli(["kill", containerName], {
5193
+ timeoutMs: 15_000,
5194
+ });
5195
+ const killInspection = await dockerCli(
5196
+ ["inspect", "--format", "{{.State.Running}}", containerName],
5197
+ { timeoutMs: 10_000 },
5198
+ );
5199
+ if (
5200
+ killInspection.status === 0 &&
5201
+ killInspection.stdout.trim() === "false"
5202
+ ) {
5203
+ return "stopped";
5204
+ }
5205
+ if (killInspection.status !== 0 && !(await dockerDaemonReachable())) {
5206
+ return "unreachable";
5207
+ }
5208
+ console.error(
5209
+ `[orchestrator] recovery: could not quarantine ${containerName} before runtime proof: ${
5210
+ stopped.stderr.trim() ||
5211
+ inspected.stderr.trim() ||
5212
+ killed.stderr.trim() ||
5213
+ killInspection.stderr.trim() ||
5214
+ "container is still running after stop and kill"
5215
+ }`,
5216
+ );
5217
+ return "failed";
5218
+ }
5219
+
5220
+ /**
5221
+ * Repair the shared node data root before any recovered-container init.
5222
+ * Older task-up versions created the OpenCode leaf as root and accidentally
5223
+ * left this parent root-owned, which prevented code-server from creating its
5224
+ * managed User profile. `install -d` creates or repairs the exact directory
5225
+ * without recursively changing unrelated application state beneath it.
5226
+ */
5227
+ async function repairNodeDataRoot(containerName: string): Promise<boolean> {
5228
+ const res = await dockerCli([
5229
+ "exec",
5230
+ "-u",
5231
+ "root",
5232
+ ...runtimeAuthorityDockerExecArgs(),
5233
+ containerName,
5234
+ "/usr/bin/install",
5235
+ "-d",
5236
+ "-o",
5237
+ "node",
5238
+ "-g",
5239
+ "node",
5240
+ "-m",
5241
+ "0755",
5242
+ "/home/node/.local/share",
5243
+ ]);
5244
+ if (res.status !== 0) {
5245
+ console.error(
5246
+ `[orchestrator] recovery: ${containerName} could not repair /home/node/.local/share ownership: ${res.stderr.trim() || `docker exec exited ${String(res.status)}`}`,
5247
+ );
5248
+ return false;
5249
+ }
5250
+ return true;
5251
+ }
5252
+
5253
+ interface DockerPortResult {
5254
+ reachable: boolean;
5255
+ port: number | null;
3570
5256
  }
3571
5257
 
3572
5258
  async function dockerPort(
3573
5259
  containerName: string,
3574
5260
  containerPort: number,
3575
- ): Promise<number | null> {
5261
+ ): Promise<DockerPortResult> {
3576
5262
  const res = await dockerCli(["port", containerName, String(containerPort)]);
3577
- if (res.status !== 0) return null;
5263
+ if (res.status === null) return { reachable: false, port: null };
5264
+ if (res.status !== 0) {
5265
+ return {
5266
+ reachable: await dockerDaemonReachable(),
5267
+ port: null,
5268
+ };
5269
+ }
3578
5270
  // Output: "127.0.0.1:32785\n"
3579
5271
  const firstLine = res.stdout.split("\n")[0]?.trim() ?? "";
3580
5272
  const colonIdx = firstLine.lastIndexOf(":");
3581
- if (colonIdx === -1) return null;
5273
+ if (colonIdx === -1) return { reachable: true, port: null };
3582
5274
  const port = Number(firstLine.slice(colonIdx + 1));
3583
- return Number.isFinite(port) ? port : null;
5275
+ return {
5276
+ reachable: true,
5277
+ port: Number.isFinite(port) ? port : null,
5278
+ };
3584
5279
  }
3585
5280
 
3586
5281
  async function dockerExec(
@@ -3589,102 +5284,952 @@ async function dockerExec(
3589
5284
  timeoutMs?: number,
3590
5285
  ): Promise<boolean> {
3591
5286
  const res = await dockerCli(
3592
- ["exec", containerName, ...cmd],
5287
+ ["exec", ...runtimeAuthorityDockerExecArgs(), containerName, ...cmd],
3593
5288
  timeoutMs === undefined ? {} : { timeoutMs },
3594
5289
  );
3595
5290
  return res.status === 0;
3596
5291
  }
3597
5292
 
3598
- /** Exported for tests; production entry is the getOrchestrator() boot guard. */
3599
- export async function recoverRunningTasks(opts?: {
3600
- /** Delay between passes while docker is unreachable (tests shrink it). */
3601
- retryMs?: number;
3602
- /** Bound on passes; ~15 min at the default cadence covers slow boots. */
3603
- maxPasses?: number;
3604
- }): Promise<void> {
3605
- const retryMs = opts?.retryMs ?? 20_000;
3606
- const maxPasses = opts?.maxPasses ?? 45;
3607
- let pendingTaskIds: ReadonlySet<string> | undefined;
3608
- // After a machine reboot the host service and the Docker daemon start
3609
- // concurrently; a pass that finds docker unreachable resolves nothing and
3610
- // must retry once the daemon is up — the whole point of recovery is to
3611
- // restart the exited containers that reboot left behind.
3612
- for (let pass = 1; ; pass++) {
3613
- const deferred = await recoveryPass(pendingTaskIds);
3614
- if (deferred.size === 0) {
3615
- finishAllTaskRecovery();
3616
- return;
5293
+ /** Exported for tests; production entry is the getOrchestrator() boot guard. */
5294
+ export async function recoverRunningTasks(opts?: {
5295
+ /** Delay between passes while docker is unreachable (tests shrink it). */
5296
+ retryMs?: number;
5297
+ /** Bound on passes; ~15 min at the default cadence covers slow boots. */
5298
+ maxPasses?: number;
5299
+ /** Post-boot recovery must queue behind live taskUp/taskDown operations.
5300
+ * Boot recovery omits this and uses its earlier per-task barriers instead. */
5301
+ lifecycle?: Orchestrator;
5302
+ /** Test seam; production waits for the shared image/CLI-volume reconcile. */
5303
+ maintenanceReady?: Promise<void>;
5304
+ /** A daemon-generation fence. False cancels every remaining side effect. */
5305
+ isCurrent?: () => boolean;
5306
+ }): Promise<boolean> {
5307
+ const retryMs = opts?.retryMs ?? 20_000;
5308
+ const maxPasses = opts?.maxPasses ?? 45;
5309
+ let pendingTaskIds: ReadonlySet<string> | undefined;
5310
+ // After a machine reboot the host service and its selected environment
5311
+ // provider start concurrently. Unknown provider state resolves nothing and
5312
+ // must retry once the runtime answers.
5313
+ for (let pass = 1; ; pass++) {
5314
+ if (opts?.isCurrent?.() === false) {
5315
+ if (!opts.lifecycle) finishAllTaskRecovery();
5316
+ return false;
5317
+ }
5318
+ const result = await recoveryPass(
5319
+ pendingTaskIds,
5320
+ opts?.lifecycle,
5321
+ opts?.maintenanceReady ?? agentClisReady,
5322
+ opts?.isCurrent ?? (() => true),
5323
+ );
5324
+ if (opts?.isCurrent?.() === false) {
5325
+ if (!opts.lifecycle) finishAllTaskRecovery();
5326
+ return false;
5327
+ }
5328
+ const { deferred } = result;
5329
+ if (!result.resolved) {
5330
+ if (!opts?.lifecycle) finishAllTaskRecovery();
5331
+ return false;
5332
+ }
5333
+ if (deferred.size === 0) {
5334
+ if (!opts?.lifecycle) finishAllTaskRecovery();
5335
+ return true;
5336
+ }
5337
+ pendingTaskIds = deferred;
5338
+ if (pass >= maxPasses) {
5339
+ console.warn(
5340
+ `[orchestrator] recovery: task environment runtime still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
5341
+ );
5342
+ if (!opts?.lifecycle) finishAllTaskRecovery();
5343
+ return false;
5344
+ }
5345
+ if (pass === 1) {
5346
+ console.log(
5347
+ `[orchestrator] recovery: task environment runtime not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
5348
+ );
5349
+ }
5350
+ await new Promise((resolve) => setTimeout(resolve, retryMs));
5351
+ }
5352
+ }
5353
+
5354
+ interface RecoveryPassResult {
5355
+ deferred: Set<string>;
5356
+ resolved: boolean;
5357
+ }
5358
+
5359
+ /** One recovery sweep. Returns only task ids deferred by an unreachable provider;
5360
+ * successful ids are never revisited by a later retry pass. `resolved` is
5361
+ * false for an unexpected failure whose runtime state could not be proved. */
5362
+ async function recoveryPass(
5363
+ onlyTaskIds?: ReadonlySet<string>,
5364
+ lifecycle?: Orchestrator,
5365
+ maintenanceReady: Promise<void> = agentClisReady,
5366
+ isCurrent: () => boolean = () => true,
5367
+ ): Promise<RecoveryPassResult> {
5368
+ const deferred = new Set<string>();
5369
+ let resolved = true;
5370
+ try {
5371
+ if (!isCurrent()) return { deferred, resolved: false };
5372
+ const db = getDb();
5373
+ const activeRows = db
5374
+ .select()
5375
+ .from(schema.hostTasks)
5376
+ .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
5377
+ .all();
5378
+ const rows = onlyTaskIds
5379
+ ? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
5380
+ : activeRows;
5381
+ if (!lifecycle) {
5382
+ const activeTaskIds = new Set(rows.map((task) => task.taskId));
5383
+ const expectedTaskIds =
5384
+ onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
5385
+ for (const taskId of expectedTaskIds) {
5386
+ if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
5387
+ }
5388
+ for (const task of rows) openTaskRecoveryBarrier(task.taskId);
5389
+ }
5390
+ if (rows.length === 0) return { deferred, resolved };
5391
+ console.log(
5392
+ `[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
5393
+ );
5394
+ for (const task of rows) {
5395
+ if (!isCurrent()) break;
5396
+ try {
5397
+ const recovered = lifecycle
5398
+ ? await lifecycle.runTaskLifecycle(task.taskId, async () => {
5399
+ // The row was selected before this lifecycle slot became ours.
5400
+ // A queued taskDown/taskUp may have completed meanwhile, so act
5401
+ // only on the current local authority—not the stale snapshot.
5402
+ const current = getHostTask(task.taskId);
5403
+ if (
5404
+ !current ||
5405
+ !isCurrent() ||
5406
+ !current.statusMirror ||
5407
+ !isActive(current.statusMirror)
5408
+ ) {
5409
+ return true;
5410
+ }
5411
+ return recoverPersistedTaskEnvironment(
5412
+ current,
5413
+ maintenanceReady,
5414
+ isCurrent,
5415
+ lifecycle,
5416
+ );
5417
+ })
5418
+ : await recoverPersistedTaskEnvironment(
5419
+ task,
5420
+ maintenanceReady,
5421
+ isCurrent,
5422
+ );
5423
+ if (recovered) {
5424
+ if (!lifecycle) finishTaskRecovery(task.taskId);
5425
+ } else {
5426
+ deferred.add(task.taskId);
5427
+ }
5428
+ } catch (err) {
5429
+ resolved = false;
5430
+ if (!lifecycle) finishTaskRecovery(task.taskId);
5431
+ console.error(
5432
+ `[orchestrator] recovery: ${task.taskId} failed:`,
5433
+ err instanceof Error ? err.message : err,
5434
+ );
5435
+ }
5436
+ }
5437
+ } catch (err) {
5438
+ resolved = false;
5439
+ if (!lifecycle) finishAllTaskRecovery();
5440
+ console.error(
5441
+ "[orchestrator] recovery: top-level failure",
5442
+ err instanceof Error ? err.message : err,
5443
+ );
5444
+ }
5445
+ return { deferred, resolved };
5446
+ }
5447
+
5448
+ /** Apple-backend variant of the writable-runtime quarantine. Task containers
5449
+ * prove their shared-cache mount read-only at creation, so a writable mount
5450
+ * here is either a live materializer that outlived its bound or tampering —
5451
+ * both are stopped (task-labeled rows report `stopped`). A CLI that cannot
5452
+ * answer yields `retry`, exactly like an unreachable Docker daemon; it is
5453
+ * never treated as an empty inventory. */
5454
+ async function quarantineWritableAppleRuntimeContainers(options: {
5455
+ lifecycle: Orchestrator;
5456
+ isCurrent?: () => boolean;
5457
+ scope?: "contract" | "volume-holders";
5458
+ }): Promise<WritableRuntimeQuarantineResult> {
5459
+ const isCurrent = options.isCurrent ?? (() => true);
5460
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
5461
+ const binding = appleContainerRuntimeBinding();
5462
+ if (binding === null) return { verdict: "retry", stoppedTaskIds: [] };
5463
+ const cli = appleCliRunner(() => binding.containerCliPath);
5464
+ const listed = await cli(["list", "--all", "--format", "json"], {
5465
+ timeoutMs: 30_000,
5466
+ maxOutputBytes: 8 * 1024 * 1024,
5467
+ });
5468
+ if (listed.status !== 0 || listed.truncated === true) {
5469
+ return { verdict: "retry", stoppedTaskIds: [] };
5470
+ }
5471
+ let parsed: unknown;
5472
+ try {
5473
+ parsed = JSON.parse(listed.stdout);
5474
+ } catch {
5475
+ return { verdict: "retry", stoppedTaskIds: [] };
5476
+ }
5477
+ if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTaskIds: [] };
5478
+ const stoppedTaskIds = new Set<string>();
5479
+ let resolved = true;
5480
+
5481
+ const stopAndProve = async (id: string): Promise<boolean> => {
5482
+ const stopped = await cli(["stop", "--time", "30", id], {
5483
+ timeoutMs: 90_000,
5484
+ maxOutputBytes: 256 * 1024,
5485
+ });
5486
+ const after = await cli(["inspect", id], {
5487
+ timeoutMs: 15_000,
5488
+ maxOutputBytes: 1024 * 1024,
5489
+ });
5490
+ if (after.status !== 0) {
5491
+ return (
5492
+ after.stdout.trim() === "" &&
5493
+ after.stderr
5494
+ .split("\n")
5495
+ .some((line) => line.trim() === `Error: container not found: ${id}`)
5496
+ );
5497
+ }
5498
+ try {
5499
+ const parsedAfter = JSON.parse(after.stdout) as Array<{
5500
+ id?: unknown;
5501
+ status?: { state?: unknown };
5502
+ }>;
5503
+ // Only one exact-identity record in the runtime's terminal stopped
5504
+ // state settles the proof. A wrong-identity record, a transitional
5505
+ // state ("starting"), or an ambiguous shape must never release
5506
+ // shared-runtime maintenance while the writer may still be active.
5507
+ if (
5508
+ Array.isArray(parsedAfter) &&
5509
+ parsedAfter.length === 1 &&
5510
+ parsedAfter[0]?.id === id &&
5511
+ parsedAfter[0]?.status?.state === "stopped"
5512
+ ) {
5513
+ return true;
5514
+ }
5515
+ } catch {
5516
+ // fall through to the failure log below
5517
+ }
5518
+ console.error(
5519
+ `[orchestrator] runtime isolation: apple container ${id} holds the shared runtime cache writable and could not be proven stopped (stop exit ${stopped.status})`,
5520
+ );
5521
+ return false;
5522
+ };
5523
+
5524
+ for (const item of parsed as Array<{
5525
+ configuration?: {
5526
+ id?: unknown;
5527
+ labels?: Record<string, unknown>;
5528
+ mounts?: Array<{
5529
+ destination?: unknown;
5530
+ options?: unknown;
5531
+ type?: { volume?: { name?: unknown } };
5532
+ }>;
5533
+ };
5534
+ }>) {
5535
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
5536
+ const id = item?.configuration?.id;
5537
+ if (typeof id !== "string" || id.length === 0) {
5538
+ resolved = false;
5539
+ continue;
5540
+ }
5541
+ const mounts = item.configuration?.mounts;
5542
+ // Sweep every container whose shared-volume attachment violates the
5543
+ // CURRENT contract: writable (missing ro, or contradictory ro+rw), or at
5544
+ // any destination other than the ADR-108 staging path — which includes
5545
+ // every pre-ADR-108 container (read-only at /opt/asdf-data). Stopping
5546
+ // legacy containers HERE, before writer maintenance, is what lets a
5547
+ // legacy host migrate at all: a running legacy container would otherwise
5548
+ // hold the volume against the very reconcile whose completion recovery
5549
+ // needs (review 2026-08-18: maintenance retried forever and recovery —
5550
+ // the step that settles legacy containers — never ran).
5551
+ const writableShared = (Array.isArray(mounts) ? mounts : []).some(
5552
+ (mount) =>
5553
+ mount?.type?.volume?.name === "uai-asdf-data" &&
5554
+ (options.scope === "volume-holders" ||
5555
+ !(Array.isArray(mount.options) && mount.options.includes("ro")) ||
5556
+ (Array.isArray(mount.options) && mount.options.includes("rw")) ||
5557
+ mount?.destination !== "/run/uai/asdf-lower"),
5558
+ );
5559
+ if (!writableShared) continue;
5560
+ const label = item.configuration?.labels?.["com.uai.task"];
5561
+ // The label is container-influenced data. Only a syntactically safe task
5562
+ // id whose derived container name matches this exact container may drive
5563
+ // task-scoped mutations (lifecycle slot, DB row, ACL removal); anything
5564
+ // else is stopped as an anonymous writer, nothing task-scoped touched.
5565
+ const taskId =
5566
+ typeof label === "string" &&
5567
+ isSafeHostTaskId(label) &&
5568
+ id === appleTaskContainerName(label)
5569
+ ? label
5570
+ : null;
5571
+ if (taskId === null) {
5572
+ if (!(await stopAndProve(id))) resolved = false;
5573
+ continue;
5574
+ }
5575
+ // Task-labeled writers stop inside the task's lifecycle slot so a
5576
+ // concurrent taskUp/taskDown cannot interleave, and a known row commits
5577
+ // its durable `stopped` verdict before the frame is queued — a crashed
5578
+ // publisher must never leave SQLite saying running.
5579
+ await options.lifecycle.runTaskLifecycle(taskId, async () => {
5580
+ if (!isCurrent()) return;
5581
+ await options.lifecycle.quarantineChannelForRecovery(taskId);
5582
+ if (!(await stopAndProve(id))) {
5583
+ resolved = false;
5584
+ return;
5585
+ }
5586
+ const task = getHostTask(taskId);
5587
+ if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
5588
+ db_setStatus(taskId, "stopped", {
5589
+ codeServerPort: null,
5590
+ previewPorts: "[]",
5591
+ });
5592
+ stoppedTaskIds.add(taskId);
5593
+ }
5594
+ // Revoke live access with the environment, like the Docker sweep.
5595
+ clearTaskGatewayAcl(taskId);
5596
+ });
5597
+ }
5598
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
5599
+ return {
5600
+ verdict: resolved ? "ready" : "retry",
5601
+ stoppedTaskIds: [...stoppedTaskIds],
5602
+ };
5603
+ }
5604
+
5605
+ /** Dispatch one durable row through its provider. Generic boot orchestration
5606
+ * owns DB/lifecycle sequencing but never derives a Compose/container identity. */
5607
+ async function recoverPersistedTaskEnvironment(
5608
+ task: typeof schema.hostTasks.$inferSelect,
5609
+ maintenanceReady: Promise<void>,
5610
+ isCurrent: () => boolean,
5611
+ lifecycle?: Orchestrator,
5612
+ ): Promise<boolean> {
5613
+ const environment = await reconstructPersistedTaskEnvironment(task);
5614
+ if (environment === null) return true;
5615
+ if (!isCurrent()) return true;
5616
+ const result = await environment.recover({
5617
+ maintenanceReady,
5618
+ isCurrent,
5619
+ ...(lifecycle
5620
+ ? {
5621
+ quarantineConsumers: () =>
5622
+ lifecycle.quarantineChannelForRecovery(task.taskId),
5623
+ allowConsumers: () => lifecycle.allowChannel(task.taskId),
5624
+ ensureConsumersStarted: async () => {
5625
+ await lifecycle.ensureStarted(task.taskId);
5626
+ },
5627
+ }
5628
+ : {}),
5629
+ });
5630
+ return result.outcome === "recovered";
5631
+ }
5632
+
5633
+ interface RecoveryTaskConsumers {
5634
+ quarantineChannelForRecovery(taskId: string): Promise<void>;
5635
+ allowChannel(taskId: string): void;
5636
+ ensureStarted(taskId: string): Promise<boolean | void>;
5637
+ }
5638
+
5639
+ /** Apple provider driver. Recovery must end in a durable truth: a running
5640
+ * container is re-initialized (dependencies + code-server) under an
5641
+ * in-container deadline; a stopped one is started and initialized; a
5642
+ * container that is provenly absent — or cannot be started — marks the task
5643
+ * `stopped` so the host never reports operational with a phantom running
5644
+ * task. Unknown CLI answers stay deferred and retryable. */
5645
+ /** Exact TypeScript port of task-up.sh's container environment-authority
5646
+ * proof (the jq filter behind "verify task runtime isolation"): each required
5647
+ * canonical entry appears exactly once, every owned-npm-config spelling
5648
+ * (case-folded, underscores treated as hyphens) must be one of the canonical
5649
+ * eight, and PATH plus the asdf/corepack controls hold their exact values
5650
+ * exactly once. */
5651
+ export function appleTaskEnvironmentAuthorityHolds(value: unknown): boolean {
5652
+ if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) {
5653
+ return false;
5654
+ }
5655
+ const environment = value as string[];
5656
+ const npmPrefix = "/home/node/.local/share/uai-npm";
5657
+ const npmCache = "/home/node/.cache/uai-npm";
5658
+ const runtimePath =
5659
+ "/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:" +
5660
+ `/usr/sbin:/usr/bin:/sbin:/bin:${npmPrefix}/bin`;
5661
+ const required = [
5662
+ `NPM_CONFIG_PREFIX=${npmPrefix}`,
5663
+ `npm_config_prefix=${npmPrefix}`,
5664
+ `NPM_CONFIG_CACHE=${npmCache}`,
5665
+ `npm_config_cache=${npmCache}`,
5666
+ `NPM_CONFIG_LOGS_DIR=${npmCache}/_logs`,
5667
+ `npm_config_logs_dir=${npmCache}/_logs`,
5668
+ "NPM_CONFIG_UMASK=0002",
5669
+ "npm_config_umask=0002",
5670
+ ];
5671
+ const owned = ["prefix", "cache", "logs-dir", "umask"];
5672
+ const countOf = (exact: string): number =>
5673
+ environment.filter((entry) => entry === exact).length;
5674
+ const isOwnedNpmConfig = (key: string): boolean => {
5675
+ const folded = key.toLowerCase();
5676
+ return (
5677
+ folded.startsWith("npm_config_") &&
5678
+ owned.includes(folded.slice("npm_config_".length).replace(/_/g, "-"))
5679
+ );
5680
+ };
5681
+ const ownedEntries = environment.filter((entry) =>
5682
+ isOwnedNpmConfig(entry.split("=")[0]!),
5683
+ );
5684
+ return (
5685
+ required.every((entry) => countOf(entry) === 1) &&
5686
+ ownedEntries.length === required.length &&
5687
+ countOf(`PATH=${runtimePath}`) === 1 &&
5688
+ countOf("ASDF_DATA_DIR=/opt/asdf-data") === 1 &&
5689
+ countOf("ASDF_SKIP_RESHIM=1") === 1 &&
5690
+ countOf("COREPACK_HOME=/opt/asdf-data/corepack") === 1 &&
5691
+ countOf("COREPACK_ENABLE_NETWORK=0") === 1
5692
+ );
5693
+ }
5694
+
5695
+ export type AppleTaskRuntimeContractVerdict =
5696
+ | { kind: "holds" }
5697
+ | { kind: "violated" }
5698
+ | { kind: "unreachable"; detail: string };
5699
+
5700
+ /**
5701
+ * The apple runtime-authority contract, in one inspection: this exact name
5702
+ * carries this task's label, is running, mounts the shared cache read-only
5703
+ * at exactly the ADR-108 staging path (and nowhere else, with no
5704
+ * contradictory options), binds /workspace as exactly one writable virtiofs
5705
+ * share of this task's real worktree leaf, and carries the same immutable
5706
+ * npm/runtime environment authority task-up proved at creation. Factored
5707
+ * from recovery so the taskUp lost-response fast path proves the SAME
5708
+ * invariants (review 2026-08-18 round 4). `unreachable` and `violated` are
5709
+ * distinct so the fast path can retry without stopping a container it could
5710
+ * not observe, while recovery keeps treating both as quarantine grounds.
5711
+ */
5712
+ export async function proveAppleTaskRuntimeContract(options: {
5713
+ taskId: string;
5714
+ containerName: string;
5715
+ hostWorktreePath: string;
5716
+ cli?: ReturnType<typeof appleCliRunner>;
5717
+ }): Promise<AppleTaskRuntimeContractVerdict> {
5718
+ let cli = options.cli;
5719
+ if (!cli) {
5720
+ const binding = appleContainerRuntimeBinding();
5721
+ if (binding === null) {
5722
+ return { kind: "unreachable", detail: "apple runtime binding unavailable" };
5723
+ }
5724
+ cli = appleCliRunner(() => binding.containerCliPath);
5725
+ }
5726
+ const contract = await cli(["inspect", options.containerName], {
5727
+ timeoutMs: 15_000,
5728
+ maxOutputBytes: 4 * 1024 * 1024,
5729
+ });
5730
+ if (contract.status !== 0) {
5731
+ return {
5732
+ kind: "unreachable",
5733
+ detail:
5734
+ contract.stderr.trim().slice(0, 200) ||
5735
+ `inspect exited ${contract.status ?? "killed"}`,
5736
+ };
5737
+ }
5738
+ try {
5739
+ const records = JSON.parse(contract.stdout) as Array<{
5740
+ id?: unknown;
5741
+ status?: { state?: unknown };
5742
+ configuration?: {
5743
+ labels?: Record<string, unknown>;
5744
+ mounts?: Array<{
5745
+ destination?: unknown;
5746
+ options?: unknown;
5747
+ type?: { volume?: { name?: unknown } };
5748
+ }>;
5749
+ initProcess?: { environment?: unknown };
5750
+ };
5751
+ }>;
5752
+ // Same strictness as the provider's inspection: exactly one record and
5753
+ // it must identify itself as the requested container, or a malformed
5754
+ // response describing another container could authorize execution.
5755
+ if (!Array.isArray(records) || records.length !== 1) {
5756
+ throw new Error("contract inspection shape is invalid");
5757
+ }
5758
+ const record = records[0]!;
5759
+ if (record.id !== options.containerName) {
5760
+ throw new Error("contract inspection identity mismatch");
5761
+ }
5762
+ const mounts = record?.configuration?.mounts ?? [];
5763
+ // ADR-108: the shared volume attaches read-only at the overlay staging
5764
+ // path and /opt/asdf-data must NOT be a host mount at all — the
5765
+ // writable view there is the in-guest overlay.
5766
+ const runtimeMounts = mounts.filter(
5767
+ (mount) => mount?.destination === "/run/uai/asdf-lower",
5768
+ );
5769
+ const sharedMounts = mounts.filter(
5770
+ (mount) => mount?.type?.volume?.name === "uai-asdf-data",
5771
+ );
5772
+ const asdfHostMounts = mounts.filter(
5773
+ (mount) => mount?.destination === "/opt/asdf-data",
5774
+ );
5775
+ // The /workspace authority: exactly one bind (never a volume) whose
5776
+ // source is this task's exact worktree leaf, still a real non-symlink
5777
+ // directory on disk — the same proof Docker recovery performs.
5778
+ const workspaceMounts = mounts.filter(
5779
+ (mount) => mount?.destination === "/workspace",
5780
+ );
5781
+ const expectedWorkspaceSource = join(
5782
+ options.hostWorktreePath,
5783
+ "workspace",
5784
+ );
5785
+ // The verified Apple dialect for a host-directory bind is exactly
5786
+ // type:{virtiofs:{}} — a closed shape.
5787
+ const workspaceType = workspaceMounts[0]?.type as
5788
+ | Record<string, unknown>
5789
+ | undefined;
5790
+ const workspaceTypeIsVirtiofs =
5791
+ workspaceType !== undefined &&
5792
+ typeof workspaceType === "object" &&
5793
+ !Array.isArray(workspaceType) &&
5794
+ Object.keys(workspaceType).length === 1 &&
5795
+ typeof workspaceType.virtiofs === "object" &&
5796
+ workspaceType.virtiofs !== null &&
5797
+ !Array.isArray(workspaceType.virtiofs) &&
5798
+ Object.keys(workspaceType.virtiofs as object).length === 0;
5799
+ // Writable is part of the authority: a read-only /workspace would
5800
+ // expose a task whose agents cannot work.
5801
+ const workspaceOptions = workspaceMounts[0]?.options;
5802
+ let workspaceAuthorityHolds =
5803
+ workspaceMounts.length === 1 &&
5804
+ workspaceTypeIsVirtiofs &&
5805
+ Array.isArray(workspaceOptions) &&
5806
+ !(workspaceOptions as unknown[]).includes("ro") &&
5807
+ (workspaceMounts[0] as { source?: unknown } | undefined)?.source ===
5808
+ expectedWorkspaceSource;
5809
+ if (workspaceAuthorityHolds) {
5810
+ try {
5811
+ const workspaceStat = lstatSync(expectedWorkspaceSource);
5812
+ workspaceAuthorityHolds =
5813
+ workspaceStat.isDirectory() && !workspaceStat.isSymbolicLink();
5814
+ } catch {
5815
+ workspaceAuthorityHolds = false;
5816
+ }
5817
+ }
5818
+ const rawEnvironment = record?.configuration?.initProcess?.environment;
5819
+ const holds =
5820
+ record?.status?.state === "running" &&
5821
+ record?.configuration?.labels?.["com.uai.task"] === options.taskId &&
5822
+ runtimeMounts.length === 1 &&
5823
+ sharedMounts.length === 1 &&
5824
+ sharedMounts[0] === runtimeMounts[0] &&
5825
+ asdfHostMounts.length === 0 &&
5826
+ Array.isArray(runtimeMounts[0]?.options) &&
5827
+ (runtimeMounts[0]!.options as unknown[]).includes("ro") &&
5828
+ // Closed options contract: an array carrying BOTH ro and rw is
5829
+ // contradictory and proves nothing.
5830
+ !(runtimeMounts[0]!.options as unknown[]).includes("rw") &&
5831
+ workspaceAuthorityHolds &&
5832
+ appleTaskEnvironmentAuthorityHolds(rawEnvironment);
5833
+ return holds ? { kind: "holds" } : { kind: "violated" };
5834
+ } catch {
5835
+ return { kind: "violated" };
5836
+ }
5837
+ }
5838
+
5839
+ export async function recoverAppleTaskEnvironment(
5840
+ descriptor: TaskEnvironmentDescriptor,
5841
+ context: TaskEnvironmentRecoveryContext,
5842
+ ): Promise<TaskEnvironmentRecoveryResult> {
5843
+ const locator = parseAppleTaskEnvironmentLocator(descriptor.locator);
5844
+ const task = getHostTask(descriptor.taskId);
5845
+ if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
5846
+ return { outcome: "recovered" };
5847
+ }
5848
+ // The boot maintenance latch (agentClisReady) is awaited ONLY before the
5849
+ // first in-container exec, exactly like Docker recovery. Awaiting it here
5850
+ // deadlocked the first fresh Apple host (live 2026-08-15): the on-connect
5851
+ // recovery sweep held this driver on the latch, activation joins in-flight
5852
+ // recovery before its maintenance, and the maintenance that settles the
5853
+ // latch could therefore never run — the host sat silently in `checking`
5854
+ // forever. Status verdicts, quarantines, and workspace settlement need no
5855
+ // maintenance and must never wait on it.
5856
+ if (!context.isCurrent()) {
5857
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5858
+ }
5859
+ const binding = appleContainerRuntimeBinding();
5860
+ if (binding === null) {
5861
+ return { outcome: "deferred", detail: "apple runtime binding unavailable" };
5862
+ }
5863
+ const cli = appleCliRunner(() => binding.containerCliPath);
5864
+ const environment = await reconstructPersistedTaskEnvironment(task);
5865
+ if (environment === null) return { outcome: "recovered" };
5866
+
5867
+ const markStopped = (): TaskEnvironmentRecoveryResult => {
5868
+ db_setStatus(descriptor.taskId, "stopped", {
5869
+ codeServerPort: null,
5870
+ previewPorts: "[]",
5871
+ });
5872
+ // A stopped verdict must also revoke live access paths, exactly like the
5873
+ // Docker quarantine: the MCP gateway token dies with the environment.
5874
+ clearTaskGatewayAcl(descriptor.taskId);
5875
+ return { outcome: "recovered" };
5876
+ };
5877
+ // Without its workspace the task cannot resume — stopped would advertise a
5878
+ // restart that must fail. Docker recovery reconciles this as error.
5879
+ const markError = (): TaskEnvironmentRecoveryResult => {
5880
+ db_setStatus(descriptor.taskId, "error", {
5881
+ codeServerPort: null,
5882
+ previewPorts: "[]",
5883
+ composeProject: null,
5884
+ });
5885
+ clearTaskGatewayAcl(descriptor.taskId);
5886
+ return { outcome: "recovered" };
5887
+ };
5888
+ // Every settled verdict consults the freshest workspace observation: a
5889
+ // workspace that vanished mid-recovery must land as error exactly like one
5890
+ // that was already gone at the first look.
5891
+ const settleFromStatus = (settled: {
5892
+ workspacePresent: boolean;
5893
+ }): TaskEnvironmentRecoveryResult =>
5894
+ settled.workspacePresent ? markStopped() : markError();
5895
+
5896
+ let status = await environment.status();
5897
+ if (status.state === "unknown") {
5898
+ return { outcome: "deferred", detail: status.detail };
5899
+ }
5900
+ // Every awaited runtime observation can outlive this recovery's readiness
5901
+ // generation; a superseded pass must observe only, never mutate. Recheck
5902
+ // after each await and before each mutation from here on.
5903
+ if (!context.isCurrent()) {
5904
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5905
+ }
5906
+ // Drain consumers before ANY durable verdict or mutation — including the
5907
+ // absent path, where a live channel could otherwise keep serving a task
5908
+ // whose row is about to say stopped.
5909
+ await context.quarantineConsumers?.();
5910
+ if (!context.isCurrent()) {
5911
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5912
+ }
5913
+ if (status.state === "absent") {
5914
+ // Name-anchored absence came from the provider's inspect. The disk truth
5915
+ // decides the verdict: an intact workspace resumes via re-provision
5916
+ // (stopped); a vanished one cannot, and stopped would advertise a
5917
+ // restart that must fail (error).
5918
+ return status.workspacePresent ? markStopped() : markError();
5919
+ }
5920
+ if (status.state === "stopped" && !status.workspacePresent) {
5921
+ // Never start a container whose workspace is gone: dependency recovery
5922
+ // would mutate a recreated or missing tree.
5923
+ return markError();
5924
+ }
5925
+ if (status.state === "stopped") {
5926
+ const started = await cli(["start", locator.containerName], {
5927
+ timeoutMs: 120_000,
5928
+ maxOutputBytes: 256 * 1024,
5929
+ });
5930
+ if (!context.isCurrent()) {
5931
+ return { outcome: "deferred", detail: "runtime generation superseded" };
3617
5932
  }
3618
- pendingTaskIds = deferred;
3619
- if (pass >= maxPasses) {
3620
- console.warn(
3621
- `[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
3622
- );
3623
- finishAllTaskRecovery();
3624
- return;
5933
+ // A failed start command or an ambiguous follow-up is NOT proof of a
5934
+ // stopped container: only an exact stopped/absent answer may settle the
5935
+ // durable row, and unknown always stays deferred and retryable.
5936
+ status = await environment.status();
5937
+ if (status.state === "unknown") {
5938
+ return { outcome: "deferred", detail: status.detail };
3625
5939
  }
3626
- if (pass === 1) {
3627
- console.log(
3628
- `[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
5940
+ if (!context.isCurrent()) {
5941
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5942
+ }
5943
+ if (status.state === "stopped" || status.state === "absent") {
5944
+ return settleFromStatus(status);
5945
+ }
5946
+ if (started.status !== 0) {
5947
+ // The CLI reported failure yet the container runs — trust the state
5948
+ // probe over the command's exit and continue into revalidation.
5949
+ console.warn(
5950
+ `[orchestrator] recovery: ${descriptor.taskId} container start exited ${started.status} but the container is running; continuing`,
3629
5951
  );
3630
5952
  }
3631
- await new Promise((resolve) => setTimeout(resolve, retryMs));
3632
5953
  }
3633
- }
3634
5954
 
3635
- /** One recovery sweep. Returns only task ids deferred by unreachable Docker;
3636
- * successful ids are never revisited by a later retry pass. */
3637
- async function recoveryPass(
3638
- onlyTaskIds?: ReadonlySet<string>,
3639
- ): Promise<Set<string>> {
3640
- const deferred = new Set<string>();
3641
- try {
3642
- const db = getDb();
3643
- const activeRows = db
3644
- .select()
3645
- .from(schema.hostTasks)
3646
- .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
3647
- .all();
3648
- const rows = onlyTaskIds
3649
- ? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
3650
- : activeRows;
3651
- const activeTaskIds = new Set(rows.map((task) => task.taskId));
3652
- const expectedTaskIds =
3653
- onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
3654
- for (const taskId of expectedTaskIds) {
3655
- if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
3656
- }
3657
- for (const task of rows) openTaskRecoveryBarrier(task.taskId);
3658
- if (rows.length === 0) return deferred;
3659
- console.log(
3660
- `[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
5955
+ // One inspection proves the whole trust decision factored into
5956
+ // proveAppleTaskRuntimeContract so the taskUp lost-response fast path can
5957
+ // prove the SAME invariants (review 2026-08-18 round 4). Recovery keeps
5958
+ // its historical mapping: anything short of `holds` — including an
5959
+ // unreachable inspect — quarantines below.
5960
+ const contractVerdict = await proveAppleTaskRuntimeContract({
5961
+ taskId: descriptor.taskId,
5962
+ containerName: locator.containerName,
5963
+ hostWorktreePath: locator.hostWorktreePath,
5964
+ cli,
5965
+ });
5966
+ const contractHolds = contractVerdict.kind === "holds";
5967
+ if (!context.isCurrent()) {
5968
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5969
+ }
5970
+ if (!contractHolds) {
5971
+ await environment.stop();
5972
+ const after = await environment.status();
5973
+ if (after.state === "running" || after.state === "unknown") {
5974
+ return {
5975
+ outcome: "deferred",
5976
+ detail:
5977
+ "task container failed its runtime-authority contract and could not be proven stopped",
5978
+ };
5979
+ }
5980
+ if (!context.isCurrent()) {
5981
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5982
+ }
5983
+ console.error(
5984
+ `[orchestrator] recovery: ${descriptor.taskId} container failed its runtime-authority contract; task was stopped and remains quarantined`,
3661
5985
  );
3662
- for (const task of rows) {
3663
- try {
3664
- if (await recoverOneTask(task)) finishTaskRecovery(task.taskId);
3665
- else deferred.add(task.taskId);
3666
- } catch (err) {
3667
- finishTaskRecovery(task.taskId);
3668
- console.error(
3669
- `[orchestrator] recovery: ${task.taskId} failed:`,
3670
- err instanceof Error ? err.message : err,
3671
- );
3672
- }
5986
+ return settleFromStatus(after);
5987
+ }
5988
+ // Even an already-running container's first exec resolves through shared
5989
+ // asdf shims; fence it behind the one-way boot maintenance latch, exactly
5990
+ // like Docker recovery fences its workbench probe.
5991
+ await context.maintenanceReady;
5992
+ if (!context.isCurrent()) {
5993
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5994
+ }
5995
+ // ADR-108: guest mounts do not survive a VM restart, so the runtime
5996
+ // overlay must be re-established (idempotently) before ANY exec resolves
5997
+ // through /opt/asdf-data shims — including uai-init below. The upper layer
5998
+ // persists in the container filesystem, so the task's own installs return
5999
+ // with the mount. Proven from the guest mount table, exactly like task-up.
6000
+ const overlay = await environment.exec({
6001
+ argv: [
6002
+ "/bin/sh",
6003
+ "-c",
6004
+ // Same establishment + token-exact proof as task-up §6a: exactly one
6005
+ // effective mount at the merged path and the lower, comma-delimited
6006
+ // option tokens (substring matching accepted a -evil suffix), a
6007
+ // read-only lower, and a real node-owned upper.
6008
+ "set -e; /bin/mkdir -p /var/lib/uai/asdf-upper /var/lib/uai/asdf-work /opt/asdf-data; " +
6009
+ "/bin/chown node:node /var/lib/uai/asdf-upper; " +
6010
+ 'if ! /bin/grep -q "^overlay /opt/asdf-data overlay " /proc/mounts; then ' +
6011
+ "/bin/mount -t overlay overlay -o lowerdir=/run/uai/asdf-lower,upperdir=/var/lib/uai/asdf-upper,workdir=/var/lib/uai/asdf-work /opt/asdf-data; fi; " +
6012
+ 'overlay_lines=$(/bin/grep -c "^overlay /opt/asdf-data overlay " /proc/mounts || true); ' +
6013
+ '[ "$overlay_lines" = "1" ] || { echo "expected exactly one /opt/asdf-data overlay, found $overlay_lines" >&2; exit 1; }; ' +
6014
+ 'any_lines=$(/bin/grep -c " /opt/asdf-data " /proc/mounts || true); ' +
6015
+ '[ "$any_lines" = "1" ] || { echo "unexpected extra mounts at /opt/asdf-data" >&2; exit 1; }; ' +
6016
+ 'opts=$(/bin/grep "^overlay /opt/asdf-data overlay " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
6017
+ 'case ",$opts," in *",lowerdir=/run/uai/asdf-lower,"*) ;; *) echo "overlay lower is foreign" >&2; exit 1 ;; esac; ' +
6018
+ 'case ",$opts," in *",upperdir=/var/lib/uai/asdf-upper,"*) ;; *) echo "overlay upper is foreign" >&2; exit 1 ;; esac; ' +
6019
+ 'case ",$opts," in *",workdir=/var/lib/uai/asdf-work,"*) ;; *) echo "overlay workdir is foreign" >&2; exit 1 ;; esac; ' +
6020
+ 'case ",$opts," in *",ro,"*) echo "overlay is unexpectedly read-only" >&2; exit 1 ;; esac; ' +
6021
+ 'lower_lines=$(/bin/grep -c " /run/uai/asdf-lower " /proc/mounts || true); ' +
6022
+ '[ "$lower_lines" = "1" ] || { echo "expected exactly one lower attachment, found $lower_lines" >&2; exit 1; }; ' +
6023
+ 'lopts=$(/bin/grep " /run/uai/asdf-lower " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
6024
+ 'case ",$lopts," in *",ro,"*) ;; *) echo "lower attachment is not read-only" >&2; exit 1 ;; esac; ' +
6025
+ '[ ! -L /var/lib/uai/asdf-upper ] || { echo "upper is a symlink" >&2; exit 1; }; ' +
6026
+ '[ -d /var/lib/uai/asdf-upper ] || { echo "upper is not a directory" >&2; exit 1; }; ' +
6027
+ '[ "$(/usr/bin/stat -c %U /var/lib/uai/asdf-upper)" = "node" ] || { echo "upper is not node-owned" >&2; exit 1; }; ' +
6028
+ "/usr/bin/test -d /opt/asdf-data/plugins",
6029
+ ],
6030
+ user: "root",
6031
+ env: {},
6032
+ timeoutMs: 30_000,
6033
+ maxOutputBytes: 64 * 1024,
6034
+ });
6035
+ if (!context.isCurrent()) {
6036
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6037
+ }
6038
+ if (overlay.exitCode !== 0 || overlay.signal !== null) {
6039
+ await environment.stop();
6040
+ const afterOverlay = await environment.status();
6041
+ if (afterOverlay.state === "running" || afterOverlay.state === "unknown") {
6042
+ return {
6043
+ outcome: "deferred",
6044
+ detail:
6045
+ "task runtime overlay could not be established and the container could not be proven stopped",
6046
+ };
6047
+ }
6048
+ if (!context.isCurrent()) {
6049
+ return { outcome: "deferred", detail: "runtime generation superseded" };
3673
6050
  }
3674
- } catch (err) {
3675
- finishAllTaskRecovery();
3676
6051
  console.error(
3677
- "[orchestrator] recovery: top-level failure",
3678
- err instanceof Error ? err.message : err,
6052
+ `[orchestrator] recovery: ${descriptor.taskId} runtime overlay could not be established (exit ${overlay.exitCode}, signal ${overlay.signal}); task was stopped`,
6053
+ );
6054
+ return settleFromStatus(afterOverlay);
6055
+ }
6056
+ // Dependencies + code-server. The in-container deadline (TERM then KILL)
6057
+ // means a host-side expiry can never leave the initializer mutating the
6058
+ // workspace while agents use it; exit 78 is a runtime-authority violation
6059
+ // and quarantines exactly like at task-up.
6060
+ const init = await environment.exec({
6061
+ argv: [
6062
+ "/usr/bin/timeout",
6063
+ "--signal=TERM",
6064
+ "--kill-after=15s",
6065
+ "570s",
6066
+ "/usr/local/bin/uai-init",
6067
+ ],
6068
+ env: {},
6069
+ timeoutMs: 10 * 60_000,
6070
+ maxOutputBytes: 1024 * 1024,
6071
+ });
6072
+ if (!context.isCurrent()) {
6073
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6074
+ }
6075
+ if (
6076
+ init.exitCode === null ||
6077
+ init.signal !== null ||
6078
+ init.stdoutTruncated ||
6079
+ init.stderrTruncated
6080
+ ) {
6081
+ // The host-side client died (deadline, overflow, forced settlement)
6082
+ // without a real exit code, so the in-container initializer may STILL be
6083
+ // mutating dependencies for several minutes. Never expose the task on an
6084
+ // unknowable outcome: stop and prove, or stay deferred.
6085
+ await environment.stop();
6086
+ const after = await environment.status();
6087
+ if (after.state === "running" || after.state === "unknown") {
6088
+ return {
6089
+ outcome: "deferred",
6090
+ detail:
6091
+ "task initializer outcome was unknowable and the container could not be proven stopped",
6092
+ };
6093
+ }
6094
+ if (!context.isCurrent()) {
6095
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6096
+ }
6097
+ console.error(
6098
+ `[orchestrator] recovery: ${descriptor.taskId} initializer outcome was unknowable (exit ${init.exitCode}, signal ${init.signal}); task was stopped pending a clean re-run`,
6099
+ );
6100
+ return settleFromStatus(after);
6101
+ }
6102
+ if (init.exitCode === 78) {
6103
+ await environment.stop();
6104
+ const after = await environment.status();
6105
+ if (after.state === "running" || after.state === "unknown") {
6106
+ return {
6107
+ outcome: "deferred",
6108
+ detail:
6109
+ "runtime-authority violation detected and the container could not be proven stopped",
6110
+ };
6111
+ }
6112
+ if (!context.isCurrent()) {
6113
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6114
+ }
6115
+ console.error(
6116
+ `[orchestrator] recovery: ${descriptor.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
6117
+ );
6118
+ return settleFromStatus(after);
6119
+ }
6120
+ if (init.exitCode !== 0) {
6121
+ console.warn(
6122
+ `[orchestrator] recovery: ${descriptor.taskId} uai-init failed (exit ${init.exitCode}); container is up but Editor may be down`,
6123
+ );
6124
+ }
6125
+ if (!context.isCurrent()) {
6126
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6127
+ }
6128
+ db_setRuntime(descriptor.taskId, { codeServerPort: null });
6129
+ context.allowConsumers?.();
6130
+ await context.ensureConsumersStarted?.();
6131
+ if (!context.isCurrent()) {
6132
+ // Admission reopened against a superseded generation: re-quarantine so
6133
+ // the successor generation re-proves the environment before consumers
6134
+ // touch it — the same closing move Docker recovery makes.
6135
+ await context.quarantineConsumers?.();
6136
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6137
+ }
6138
+ return { outcome: "recovered" };
6139
+ }
6140
+
6141
+ /** Docker provider driver. The legacy repair implementation remains local to
6142
+ * this module for now, but it is reachable only through the provider handle;
6143
+ * the generic recovery sweep above never consumes its locator payload. */
6144
+ async function recoverDockerTaskEnvironment(
6145
+ descriptor: TaskEnvironmentDescriptor,
6146
+ context: TaskEnvironmentRecoveryContext,
6147
+ ): Promise<TaskEnvironmentRecoveryResult> {
6148
+ const locator = parseDockerTaskEnvironmentLocator(descriptor.locator);
6149
+ const task = getHostTask(descriptor.taskId);
6150
+ if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
6151
+ return { outcome: "recovered" };
6152
+ }
6153
+ const isPreparedRow =
6154
+ (task.statusMirror === "starting" || task.statusMirror === "error") &&
6155
+ task.composeProject === null &&
6156
+ task.worktreePath === null;
6157
+ if (
6158
+ !isPreparedRow &&
6159
+ (task.composeProject !== locator.composeProject ||
6160
+ task.worktreePath !== locator.hostWorktreePath)
6161
+ ) {
6162
+ throw new Error(
6163
+ `task ${descriptor.taskId} environment locator no longer matches its durable row`,
3679
6164
  );
3680
6165
  }
3681
- return deferred;
6166
+ // Provider-owned provisional metadata is authoritative after a crash. Feed
6167
+ // the already-validated deterministic values to the Docker-private repair
6168
+ // helpers without teaching the generic sweep how to derive them.
6169
+ const recoveryTask = isPreparedRow
6170
+ ? {
6171
+ ...task,
6172
+ composeProject: locator.composeProject,
6173
+ worktreePath: locator.hostWorktreePath,
6174
+ }
6175
+ : task;
6176
+
6177
+ const lifecycleCallbacks = [
6178
+ context.quarantineConsumers,
6179
+ context.allowConsumers,
6180
+ context.ensureConsumersStarted,
6181
+ ];
6182
+ const hasLifecycle = lifecycleCallbacks.some(Boolean);
6183
+ if (hasLifecycle && !lifecycleCallbacks.every(Boolean)) {
6184
+ throw new Error("task environment recovery lifecycle callbacks are incomplete");
6185
+ }
6186
+ const consumers: RecoveryTaskConsumers | undefined = hasLifecycle
6187
+ ? {
6188
+ quarantineChannelForRecovery: async (taskId) => {
6189
+ if (taskId !== descriptor.taskId) {
6190
+ throw new Error("task environment recovery consumer identity mismatch");
6191
+ }
6192
+ await context.quarantineConsumers!();
6193
+ },
6194
+ allowChannel: (taskId) => {
6195
+ if (taskId !== descriptor.taskId) {
6196
+ throw new Error("task environment recovery consumer identity mismatch");
6197
+ }
6198
+ context.allowConsumers!();
6199
+ },
6200
+ ensureStarted: async (taskId) => {
6201
+ if (taskId !== descriptor.taskId) {
6202
+ throw new Error("task environment recovery consumer identity mismatch");
6203
+ }
6204
+ await context.ensureConsumersStarted!();
6205
+ },
6206
+ }
6207
+ : undefined;
6208
+ const recovered = await recoverOneDockerTask(
6209
+ recoveryTask,
6210
+ context.maintenanceReady,
6211
+ context.isCurrent,
6212
+ consumers,
6213
+ );
6214
+ return recovered
6215
+ ? { outcome: "recovered" }
6216
+ : {
6217
+ outcome: "deferred",
6218
+ detail: "container runtime did not answer during task recovery",
6219
+ };
3682
6220
  }
3683
6221
 
6222
+ setDockerTaskEnvironmentRecoveryDriver(recoverDockerTaskEnvironment);
6223
+ setAppleTaskEnvironmentRecoveryDriver(recoverAppleTaskEnvironment);
6224
+
3684
6225
  /** Returns false when docker was unreachable (caller retries the pass). */
3685
- async function recoverOneTask(
6226
+ async function recoverOneDockerTask(
3686
6227
  task: typeof schema.hostTasks.$inferSelect,
6228
+ maintenanceReady: Promise<void>,
6229
+ isCurrent: () => boolean = () => true,
6230
+ lifecycle?: RecoveryTaskConsumers,
3687
6231
  ): Promise<boolean> {
6232
+ if (!isCurrent()) return true;
3688
6233
  const composeProject = task.composeProject;
3689
6234
  if (!composeProject) {
3690
6235
  // We never wrote a compose project name for this row — must be
@@ -3696,6 +6241,7 @@ async function recoverOneTask(
3696
6241
  const containers = await dockerListContainersByLabel(
3697
6242
  `com.docker.compose.project=${composeProject}`,
3698
6243
  );
6244
+ if (!isCurrent()) return true;
3699
6245
  if (containers === null) {
3700
6246
  // Docker unreachable / timed out — actual state unknown. Leave the row
3701
6247
  // alone; the retrying caller re-scans once docker answers.
@@ -3722,12 +6268,106 @@ async function recoverOneTask(
3722
6268
  return true;
3723
6269
  }
3724
6270
 
3725
- const running = containers.some((c) => c.State === "running");
6271
+ // Defense in depth for every recovery entry point: the maintained
6272
+ // activation normally quarantines legacy writers before its first shared
6273
+ // volume mutation, but recovery itself must never fast-path or `docker
6274
+ // start` a container whose immutable mount is not the exact L5 RO contract.
6275
+ const mountContract = await inspectRuntimeWriterContainer({
6276
+ taskId: task.taskId,
6277
+ composeProject,
6278
+ containerName,
6279
+ });
6280
+ if (!isCurrent()) return true;
6281
+ if (mountContract.kind === "unreachable") return false;
6282
+ if (mountContract.kind === "failed") {
6283
+ console.error(
6284
+ `[orchestrator] recovery: ${task.taskId} mount identity is ambiguous; deferring without starting it: ${mountContract.detail}`,
6285
+ );
6286
+ return false;
6287
+ }
6288
+ if (
6289
+ mountContract.metadata.sharedMount !== "read-only" ||
6290
+ mountContract.metadata.taskEnvironment !== "current"
6291
+ ) {
6292
+ if (lifecycle) {
6293
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6294
+ if (!isCurrent()) return true;
6295
+ }
6296
+ const stopped = await stopRuntimeWriter(mountContract.metadata);
6297
+ if (!isCurrent()) return true;
6298
+ if (stopped === "unreachable") return false;
6299
+ if (stopped === "failed") {
6300
+ throw new Error(
6301
+ `${task.taskId} has a legacy/non-current runtime mount and could not be proven stopped`,
6302
+ );
6303
+ }
6304
+ db_setStatus(task.taskId, "stopped", {
6305
+ codeServerPort: null,
6306
+ previewPorts: "[]",
6307
+ });
6308
+ clearTaskGatewayAcl(task.taskId);
6309
+ console.warn(
6310
+ `[orchestrator] recovery: ${task.taskId} uses a legacy/non-current runtime mount; it remains stopped until explicit Resume recreates it`,
6311
+ );
6312
+ return true;
6313
+ }
6314
+
6315
+ // Even a container already marked running needs an in-container workbench
6316
+ // probe. Fence that first exec, as well as the exited-container repair path,
6317
+ // behind the same one-way boot maintenance latch.
6318
+ await maintenanceReady;
6319
+ if (!isCurrent()) return true;
6320
+
6321
+ let running = containers.some(
6322
+ (container) =>
6323
+ container.Names === containerName && container.State === "running",
6324
+ );
3726
6325
  if (running) {
6326
+ // A stale recovery generation may have successfully issued `docker start`
6327
+ // and then been cancelled before uai-init. Do not bless every running
6328
+ // container solely from Docker state: code-server is the cheap proof that
6329
+ // the workbench init completed. A fresh sweep repairs a merely-started
6330
+ // container through the shared init path below.
6331
+ const workbenchReady = await dockerExec(
6332
+ containerName,
6333
+ ["/usr/bin/pgrep", "-f", "code-server.*--bind-addr"],
6334
+ 10_000,
6335
+ );
6336
+ if (!isCurrent()) return true;
6337
+ if (!workbenchReady && !(await dockerDaemonReachable())) return false;
6338
+ if (!isCurrent()) return true;
6339
+ if (!workbenchReady) {
6340
+ console.log(
6341
+ `[orchestrator] recovery: ${task.taskId} running without its workbench; quarantining before runtime proof`,
6342
+ );
6343
+ // Post-boot recovery can race live agent sessions. Drain their external
6344
+ // work and tombstone the channel before stopping the shared-volume
6345
+ // writer; successful recovery reopens the preserved channel spec below.
6346
+ if (lifecycle) {
6347
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6348
+ if (!isCurrent()) return true;
6349
+ }
6350
+ const quarantined = await quarantineRunningContainer(containerName);
6351
+ if (!isCurrent()) return true;
6352
+ if (quarantined === "unreachable") return false;
6353
+ if (quarantined === "failed") {
6354
+ db_setStatus(task.taskId, "stopped", {
6355
+ codeServerPort: null,
6356
+ previewPorts: "[]",
6357
+ });
6358
+ throw new Error(
6359
+ `${task.taskId} could not prove its running app stopped after stop and kill; task remains quarantined`,
6360
+ );
6361
+ }
6362
+ running = false;
6363
+ } else {
3727
6364
  // Container is up. Re-discover the host port in case Docker
3728
6365
  // remapped it across restarts (it usually does for ephemeral
3729
6366
  // bindings).
3730
- const port = await dockerPort(containerName, 8080);
6367
+ const portResult = await dockerPort(containerName, 8080);
6368
+ if (!isCurrent()) return true;
6369
+ if (!portResult.reachable) return false;
6370
+ const port = portResult.port;
3731
6371
  if (port && port !== task.codeServerPort) {
3732
6372
  db_setRuntime(task.taskId, {
3733
6373
  codeServerPort: port,
@@ -3754,30 +6394,113 @@ async function recoverOneTask(
3754
6394
  }`,
3755
6395
  ),
3756
6396
  );
6397
+ if (!isCurrent()) return true;
6398
+ if (lifecycle) {
6399
+ lifecycle.allowChannel(task.taskId);
6400
+ await lifecycle.ensureStarted(task.taskId);
6401
+ if (!isCurrent()) {
6402
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6403
+ }
6404
+ }
3757
6405
  return true;
6406
+ }
3758
6407
  }
3759
6408
 
3760
- // Container exists but exited. Bring it back up + re-launch the
3761
- // in-container init (deps + code-server).
3762
- console.log(
3763
- `[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
6409
+ // The exact app container is now stopped. Prove every project runtime from
6410
+ // its immutable image and mounts before bringing it back up. Rolling old
6411
+ // images receive the current packaged materializer/Corepack pin as read-only
6412
+ // binds, so recovery cannot bypass a newly-shipped proof contract.
6413
+ const runtimeInspection = await inspectRecoveryRuntime(task, containerName);
6414
+ if (!isCurrent()) return true;
6415
+ if (runtimeInspection.kind === "unreachable") return false;
6416
+ if (runtimeInspection.kind === "failed") {
6417
+ console.error(
6418
+ `[orchestrator] recovery: ${task.taskId} runtime inspection failed; keeping task stopped: ${runtimeInspection.detail}`,
6419
+ );
6420
+ db_setStatus(task.taskId, "stopped", {
6421
+ codeServerPort: null,
6422
+ previewPorts: "[]",
6423
+ });
6424
+ return true;
6425
+ }
6426
+ const runtimeMaterialization = await materializeRecoveryRuntimes(
6427
+ runtimeInspection.spec,
6428
+ );
6429
+ if (!isCurrent()) return true;
6430
+ if (runtimeMaterialization.kind === "unreachable") return false;
6431
+ if (runtimeMaterialization.kind === "failed") {
6432
+ console.error(
6433
+ `[orchestrator] recovery: ${task.taskId} runtime materialization failed; keeping task stopped and requiring a complete retry: ${runtimeMaterialization.detail}`,
6434
+ );
6435
+ db_setStatus(task.taskId, "stopped", {
6436
+ codeServerPort: null,
6437
+ previewPorts: "[]",
6438
+ });
6439
+ return true;
6440
+ }
6441
+ const stagedInit = await stageRecoveryInit(
6442
+ containerName,
6443
+ runtimeInspection.spec,
6444
+ );
6445
+ if (!isCurrent()) return true;
6446
+ if (stagedInit.kind === "unreachable") return false;
6447
+ if (stagedInit.kind === "failed") {
6448
+ console.error(
6449
+ `[orchestrator] recovery: ${task.taskId} could not stage the current task initializer; keeping task stopped: ${stagedInit.detail}`,
6450
+ );
6451
+ db_setStatus(task.taskId, "stopped", {
6452
+ codeServerPort: null,
6453
+ previewPorts: "[]",
6454
+ });
6455
+ return true;
6456
+ }
6457
+ const preStartWorkspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
6458
+ runtimeInspection.spec.workspaceSource,
3764
6459
  );
3765
- if (!(await dockerStart(containerName))) {
6460
+ if (preStartWorkspaceRuntimeError) {
6461
+ console.error(
6462
+ `[orchestrator] recovery: ${task.taskId} workspace runtime authority changed after proof; keeping task stopped: ${preStartWorkspaceRuntimeError}`,
6463
+ );
3766
6464
  db_setStatus(task.taskId, "stopped", {
3767
6465
  codeServerPort: null,
3768
6466
  previewPorts: "[]",
3769
6467
  });
3770
6468
  return true;
3771
6469
  }
6470
+
6471
+ // Runtime proof succeeded. Bring the preserved app back up + re-launch the
6472
+ // task-local dependency/editor init.
6473
+ // If Docker appeared after the host's initial no-runtime verdict, the
6474
+ // standard-image repair starts concurrently with this boot sweep. uai-init
6475
+ // is now task-local (dependencies + editor), but it still resolves through
6476
+ // shared asdf shims; never run it until the boot owner has finished
6477
+ // reconciling the image and agent CLI volume.
6478
+ if (!running) {
6479
+ console.log(
6480
+ `[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
6481
+ );
6482
+ const started = await dockerStart(containerName);
6483
+ if (!isCurrent()) return true;
6484
+ if (started === "unreachable") return false;
6485
+ if (started === "failed") {
6486
+ db_setStatus(task.taskId, "stopped", {
6487
+ codeServerPort: null,
6488
+ previewPorts: "[]",
6489
+ });
6490
+ return true;
6491
+ }
6492
+ }
3772
6493
  // uai-init seeds code-server's profile beneath this shared data root. Heal
3773
6494
  // containers created by older hosts before running it as node.
3774
6495
  await repairNodeDataRoot(containerName);
6496
+ if (!isCurrent()) return true;
3775
6497
  // Correctly-owned Codex creds BEFORE uai-init / any agent respawn: the boot
3776
6498
  // reinject sweep only targets containers already running, so a container
3777
6499
  // recovered here would otherwise keep whatever it held when it exited —
3778
6500
  // possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
3779
6501
  // by the inject itself and must not abort the task's recovery.
3780
6502
  await injectCodexIntoContainer(containerName);
6503
+ if (!isCurrent()) return true;
3781
6504
  // Establish the task owner's GitHub transport BEFORE uai-init performs any
3782
6505
  // dependency-network work. This matters for package manifests that refer to
3783
6506
  // private GitHub repositories: gh's credential helper must already be live
@@ -3785,7 +6508,20 @@ async function recoverOneTask(
3785
6508
  // like the prior post-init call, but it is now ordered rather than raced.
3786
6509
  if (task.ownerUserId) {
3787
6510
  try {
3788
- await reconcileTaskGitAuth(task.taskId, task.ownerUserId);
6511
+ await reconcileTaskGitAuth(task.taskId, task.ownerUserId, {
6512
+ // Recovery is sanctioned private maintenance while public admission is
6513
+ // deliberately `checking`. Fence it by this daemon epoch instead of
6514
+ // the public operational bit, preserving private-dependency setup
6515
+ // before uai-init without letting stale recovery keep mutating.
6516
+ admitContainerWork: () => {
6517
+ if (!isCurrent()) {
6518
+ throw new ContainerRuntimeUnavailableError(
6519
+ "container runtime recovery generation was superseded",
6520
+ );
6521
+ }
6522
+ },
6523
+ });
6524
+ if (!isCurrent()) return true;
3789
6525
  } catch (err) {
3790
6526
  console.warn(
3791
6527
  `[orchestrator] recovery: ${task.taskId} GitHub auth setup failed before uai-init: ${
@@ -3793,6 +6529,7 @@ async function recoverOneTask(
3793
6529
  }`,
3794
6530
  );
3795
6531
  }
6532
+ if (!isCurrent()) return true;
3796
6533
  }
3797
6534
  // uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
3798
6535
  // repo. dockerCli's 30s default would SIGKILL it mid-install.
@@ -3801,17 +6538,119 @@ async function recoverOneTask(
3801
6538
  "exec",
3802
6539
  "-e",
3803
6540
  "UAI_SKIP_GIT_TRANSPORT=1",
6541
+ "-e",
6542
+ "UAI_WORKSPACE=/workspace",
6543
+ "-e",
6544
+ `UAI_RUNTIME_PROJECTS=${RECOVERY_INIT_PROJECTS_PATH}`,
6545
+ "-e",
6546
+ "PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
6547
+ "-e",
6548
+ "HOME=/home/node",
6549
+ "-e",
6550
+ "XDG_CONFIG_HOME=/home/node/.config",
6551
+ "-e",
6552
+ "XDG_DATA_HOME=/home/node/.local/share",
6553
+ "-e",
6554
+ "ASDF_DIR=/opt/asdf",
6555
+ "-e",
6556
+ "ASDF_DATA_DIR=/opt/asdf-data",
6557
+ "-e",
6558
+ "ASDF_CONFIG_FILE=/dev/null",
6559
+ "-e",
6560
+ "ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions",
6561
+ "-e",
6562
+ "ASDF_NODEJS_VERSION=",
6563
+ "-e",
6564
+ "ASDF_PYTHON_VERSION=",
6565
+ "-e",
6566
+ "ASDF_GOLANG_VERSION=",
6567
+ "-e",
6568
+ "ASDF_RUBY_VERSION=",
6569
+ "-e",
6570
+ "ASDF_RUST_VERSION=",
6571
+ "-e",
6572
+ "COREPACK_HOME=/opt/asdf-data/corepack",
6573
+ "-e",
6574
+ "COREPACK_DEFAULT_TO_LATEST=0",
6575
+ "-e",
6576
+ "COREPACK_ENV_FILE=0",
6577
+ "-e",
6578
+ "COREPACK_NPM_REGISTRY=https://registry.npmjs.org",
6579
+ "-e",
6580
+ "COREPACK_ENABLE_NETWORK=0",
6581
+ "-e",
6582
+ "COREPACK_ENABLE_AUTO_PIN=0",
6583
+ "-e",
6584
+ "COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
6585
+ "-e",
6586
+ "COREPACK_ENABLE_PROJECT_SPEC=1",
6587
+ "-e",
6588
+ "COREPACK_ENABLE_STRICT=1",
6589
+ "-e",
6590
+ "COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0",
6591
+ "-e",
6592
+ "COREPACK_INTEGRITY_CHECK=1",
6593
+ "-e",
6594
+ "LD_PRELOAD=",
6595
+ "-e",
6596
+ "LD_AUDIT=",
6597
+ "-e",
6598
+ "LD_LIBRARY_PATH=",
6599
+ "-e",
6600
+ "NODE_OPTIONS=",
6601
+ "-e",
6602
+ "NODE_PATH=",
6603
+ // SHELLOPTS/BASHOPTS deliberately absent: present-in-env (even empty)
6604
+ // makes bash export its live options into children.
6605
+ "-e",
6606
+ "PS4=",
6607
+ "-e",
6608
+ "BASH_ENV=",
6609
+ "-e",
6610
+ "ENV=",
3804
6611
  containerName,
3805
- "/usr/local/bin/uai-init",
6612
+ // The host-side deadline below only kills the local docker client; the
6613
+ // server-side exec would keep mutating workspace dependencies while the
6614
+ // recovered task is already exposed to agents. This in-container
6615
+ // deadline kills the actual initializer first (TERM, then KILL), and
6616
+ // the host allowance is deliberately wider so a timed-out run settles
6617
+ // with exit 124 instead of racing the client kill.
6618
+ "/usr/bin/timeout",
6619
+ "--signal=TERM",
6620
+ "--kill-after=15s",
6621
+ "570s",
6622
+ RECOVERY_INIT_PATH,
3806
6623
  ],
3807
6624
  { timeoutMs: 10 * 60_000 },
3808
6625
  );
6626
+ if (!isCurrent()) return true;
6627
+ if (initResult.status === 78) {
6628
+ const quarantined = await quarantineRunningContainer(containerName);
6629
+ if (!isCurrent()) return true;
6630
+ if (quarantined === "unreachable") return false;
6631
+ db_setStatus(task.taskId, "stopped", {
6632
+ codeServerPort: null,
6633
+ previewPorts: "[]",
6634
+ });
6635
+ if (quarantined === "failed") {
6636
+ throw new Error(
6637
+ `${task.taskId} reported a runtime-authority violation and could not be proven stopped`,
6638
+ );
6639
+ }
6640
+ console.error(
6641
+ `[orchestrator] recovery: ${task.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
6642
+ );
6643
+ return true;
6644
+ }
3809
6645
  if (initResult.status !== 0) {
3810
6646
  console.warn(
3811
6647
  `[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
3812
6648
  );
3813
6649
  }
3814
- const port = (await dockerPort(containerName, 8080)) ?? null;
6650
+ const portResult = await dockerPort(containerName, 8080);
6651
+ if (!isCurrent()) return true;
6652
+ if (!portResult.reachable) return false;
6653
+ const port = portResult.port;
3815
6654
  db_setRuntime(task.taskId, {
3816
6655
  codeServerPort: port,
3817
6656
  });
@@ -3825,6 +6664,14 @@ async function recoverOneTask(
3825
6664
  console.log(
3826
6665
  `[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
3827
6666
  );
6667
+ if (!isCurrent()) return true;
6668
+ if (lifecycle) {
6669
+ lifecycle.allowChannel(task.taskId);
6670
+ await lifecycle.ensureStarted(task.taskId);
6671
+ if (!isCurrent()) {
6672
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6673
+ }
6674
+ }
3828
6675
  return true;
3829
6676
  }
3830
6677