@runuai/host 0.9.14 → 0.9.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +22 -5
  2. package/db/migrations/0014_host_inventory_event_index.sql +1 -0
  3. package/db/migrations/0015_host_settings.sql +9 -0
  4. package/db/migrations/0016_task_environment.sql +2 -0
  5. package/db/migrations/meta/_journal.json +21 -0
  6. package/db/schema.ts +80 -30
  7. package/images/standard/Dockerfile +36 -10
  8. package/images/standard/README.md +63 -18
  9. package/images/standard/container/corepack-version +1 -0
  10. package/images/standard/container/uai-init +308 -38
  11. package/images/standard/container/uai-materialize-runtimes +1527 -0
  12. package/lib/agent-cli.ts +33 -2
  13. package/lib/agent.ts +46 -7
  14. package/lib/agents/claude.ts +13 -8
  15. package/lib/agents/codex.ts +11 -6
  16. package/lib/agents/cursor.ts +39 -29
  17. package/lib/agents/durable-proc.ts +20 -27
  18. package/lib/agents/factory.ts +9 -25
  19. package/lib/agents/grok.ts +43 -30
  20. package/lib/agents/kimi.ts +44 -29
  21. package/lib/agents/opencode.ts +43 -31
  22. package/lib/agents/proc.ts +149 -114
  23. package/lib/agents/transport.ts +62 -50
  24. package/lib/agents/types.ts +6 -4
  25. package/lib/apple-runtime-recycle.ts +236 -0
  26. package/lib/apple-uninstall-teardown.ts +224 -0
  27. package/lib/browser-testing.ts +233 -93
  28. package/lib/codex-auth.ts +40 -6
  29. package/lib/command-db.ts +20 -0
  30. package/lib/container-runtime.ts +1338 -0
  31. package/lib/db.ts +1 -0
  32. package/lib/docker-exec.ts +87 -5
  33. package/lib/engine-accounts.ts +68 -5
  34. package/lib/engine-login.ts +1952 -0
  35. package/lib/enrollment-state.ts +251 -0
  36. package/lib/env-file.ts +155 -0
  37. package/lib/env.ts +4 -0
  38. package/lib/git-diff.ts +98 -32
  39. package/lib/git-identity.ts +199 -87
  40. package/lib/github-tokens.ts +202 -91
  41. package/lib/host-cloud-url.ts +62 -0
  42. package/lib/host-config.ts +279 -0
  43. package/lib/host-logs.ts +962 -0
  44. package/lib/keyed-promise-tail.ts +23 -0
  45. package/lib/legacy-runtime-v1.fixture.ts +627 -0
  46. package/lib/managed-activation-watcher.ts +72 -0
  47. package/lib/managed-install-owner-watcher.ts +55 -0
  48. package/lib/managed-operation-drain.ts +49 -0
  49. package/lib/managed-runtime.ts +3644 -0
  50. package/lib/managed-update-scheduler.ts +125 -0
  51. package/lib/mcp-gateway.ts +450 -23
  52. package/lib/orchestrator.ts +3051 -200
  53. package/lib/preview-sidecar.ts +68 -14
  54. package/lib/release-manifest.ts +708 -0
  55. package/lib/release-trust.ts +28 -0
  56. package/lib/runtime-activation-tail.ts +232 -0
  57. package/lib/runtime-archive.ts +1086 -0
  58. package/lib/runtime-authority.ts +79 -0
  59. package/lib/runtime-guard.ts +36 -0
  60. package/lib/runtime-provider-state.ts +169 -0
  61. package/lib/runtime-state.ts +232 -12
  62. package/lib/skills.ts +24 -3
  63. package/lib/ssh.ts +18 -0
  64. package/lib/standard-image.ts +1104 -141
  65. package/lib/stopped-task-status-queue.ts +44 -0
  66. package/lib/task-container-cli.ts +269 -0
  67. package/lib/task-diff.ts +66 -46
  68. package/lib/task-environment/apple-container.ts +757 -0
  69. package/lib/task-environment/docker.ts +956 -0
  70. package/lib/task-environment/index.ts +364 -0
  71. package/lib/task-environment/legacy-adoption.ts +459 -0
  72. package/lib/task-environment/registry.ts +58 -0
  73. package/lib/task-environment/types.ts +408 -0
  74. package/lib/task-identity.ts +19 -0
  75. package/lib/task-inventory.ts +585 -0
  76. package/lib/tunnel-registry.ts +135 -19
  77. package/lib/tunnel-runtime.ts +235 -0
  78. package/package.json +1 -1
  79. package/scripts/agent/_common.sh +123 -3
  80. package/scripts/agent/task-down.sh +146 -38
  81. package/scripts/agent/task-status.sh +19 -3
  82. package/scripts/agent/task-up.sh +1405 -107
  83. package/scripts/install/darwin.ts +848 -50
  84. package/scripts/install/linux.ts +838 -35
  85. package/scripts/install/types.ts +43 -0
  86. package/scripts/install/util.ts +215 -8
  87. package/scripts/install/win.ts +12 -0
  88. package/src/apple-tunnel-route.ts +104 -0
  89. package/src/cli.ts +1464 -72
  90. package/src/event-outbox.ts +83 -4
  91. package/src/index.ts +766 -42
  92. package/src/main.ts +1398 -255
  93. package/src/paths.ts +17 -1
  94. package/src/protocol.ts +695 -1
  95. package/src/runtime-bootstrap.ts +165 -0
  96. package/src/ui/server.ts +46 -10
  97. package/src/ui/types.ts +37 -0
@@ -15,10 +15,18 @@
15
15
  * instead of leaking a second orchestrator.
16
16
  */
17
17
 
18
- import { spawnSync } from "node:child_process";
19
- import { existsSync, readFileSync } from "node:fs";
18
+ import {
19
+ existsSync,
20
+ lstatSync,
21
+ mkdtempSync,
22
+ readFileSync,
23
+ realpathSync,
24
+ renameSync,
25
+ rmSync,
26
+ writeFileSync,
27
+ } from "node:fs";
20
28
  import { homedir } from "node:os";
21
- import { join } from "node:path";
29
+ import { dirname, join } from "node:path";
22
30
 
23
31
  import { inArray } from "drizzle-orm";
24
32
 
@@ -33,15 +41,52 @@ import {
33
41
  type Roster,
34
42
  type RosterAgent,
35
43
  } from "./agents/types";
36
- import { ACTIVE_STATUSES } from "./task-status";
44
+ import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
45
+ import { isSafeHostTaskId } from "./task-identity";
37
46
  import { getHostTask, upsertHostTask } from "./runtime-state";
38
47
  import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
39
48
  import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
40
49
  import { dockerCli } from "./docker-exec";
50
+ import {
51
+ taskContainerBackend,
52
+ taskContainerCli,
53
+ } from "./task-container-cli";
54
+ import {
55
+ RUNTIME_AUTHORITY_ENV,
56
+ runtimeAuthorityDockerExecArgs,
57
+ } from "./runtime-authority";
58
+ import {
59
+ backgroundContainerWorkAllowed,
60
+ ContainerRuntimeUnavailableError,
61
+ } from "./runtime-guard";
62
+ import {
63
+ appleContainerRuntimeBinding,
64
+ initializeContainerRuntime,
65
+ pinnedContainerRuntimeProvider,
66
+ suspendContainerRuntime,
67
+ } from "./container-runtime";
68
+ import {
69
+ reconstructPersistedTaskEnvironment,
70
+ setAppleTaskEnvironmentRecoveryDriver,
71
+ setDockerTaskEnvironmentRecoveryDriver,
72
+ } from "./task-environment";
73
+ import {
74
+ appleCliRunner,
75
+ appleTaskContainerName,
76
+ parseAppleTaskEnvironmentLocator,
77
+ } from "./task-environment/apple-container";
78
+ import { parseDockerTaskEnvironmentLocator } from "./task-environment/docker";
79
+ import type {
80
+ TaskEnvironmentDescriptor,
81
+ TaskEnvironmentHandle,
82
+ TaskEnvironmentRecoveryContext,
83
+ TaskEnvironmentRecoveryResult,
84
+ } from "./task-environment/types";
41
85
  import {
42
86
  containerSkillFile,
43
87
  installedSkillPath,
44
88
  installPackageSkills,
89
+ type SkillExec,
45
90
  writeAgentSkills,
46
91
  } from "./skills";
47
92
  import {
@@ -66,7 +111,15 @@ import {
66
111
  type ResolvedEngineAccount,
67
112
  } from "./engine-accounts";
68
113
  import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
69
- import { stopPreviewSidecars } from "./preview-sidecar";
114
+ import {
115
+ PREVIEW_SIDECAR_TASK_LABEL,
116
+ stopPreviewSidecars,
117
+ } from "./preview-sidecar";
118
+ import {
119
+ agentClisReady,
120
+ standardImageCorepackVersionPath,
121
+ standardImageRuntimeMaterializerPath,
122
+ } from "./standard-image";
70
123
  import { env } from "./env";
71
124
  import type {
72
125
  ChannelEnsureInput,
@@ -102,6 +155,11 @@ interface Channel {
102
155
  sessions: Map<string, AgentSession>;
103
156
  /** Session-spawn inputs, kept so sessions can be started lazily. */
104
157
  containerName: string;
158
+ /** Opaque provider-owned boundary used only for agent process creation. */
159
+ agentEnvironment: Promise<TaskEnvironmentHandle<unknown>> | null;
160
+ /** Settled handle for later config/setup passes; avoids a new await on
161
+ * legacy/bootstrap paths. */
162
+ agentHandle: TaskEnvironmentHandle<unknown> | null;
105
163
  /** Per-agent system preamble (channel briefing + assembled persona). */
106
164
  preambles: Map<string, string>;
107
165
  /** Per-agent first-turn message — only agents whose `initialPrompt` is
@@ -167,11 +225,20 @@ interface Channel {
167
225
  /** Agents with a delivered turn that has not reached a terminal event yet.
168
226
  * Unlike openTurns, this covers thinking/tool/permission time before text. */
169
227
  activeTurns: Map<string, number>;
228
+ /** Exact accepted prompts still owned by each runner generation, in send
229
+ * order. Open mode deliberately permits more than one; a process-level
230
+ * daemon-loss event must replay all of them rather than only `lastPrompt`. */
231
+ acceptedPrompts: Map<string, AcceptedPrompt[]>;
170
232
  /** Crew replies waiting for the designated Secretary's current turn
171
233
  * boundary. Adapters do not share one mid-turn input contract, so only this
172
234
  * cross-lane role is serialized at the host boundary; open mode keeps its
173
235
  * legacy immediate-concurrent delivery behavior. */
174
236
  pendingPrompts: Map<string, string[]>;
237
+ /** Prompts accepted by channelDeliver whose final session boundary rejected
238
+ * specifically because the container runtime became unavailable. */
239
+ runtimeDeferredPrompts: Map<string, string[]>;
240
+ /** Missing/blocked sessions to reconcile exactly once after runtime ready. */
241
+ runtimeDeferredAgents: Set<string>;
175
242
  /** Agents whose current turn was interrupted (ESC) — their next
176
243
  * turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
177
244
  * half-turn instead of delivering it to @-mentioned peers. */
@@ -230,6 +297,10 @@ interface Channel {
230
297
  rotations: Map<string, number>;
231
298
  }
232
299
 
300
+ interface AcceptedPrompt {
301
+ text: string;
302
+ }
303
+
233
304
  /** Hard cap on automatic respawns per agent per channel lifetime. */
234
305
  const MAX_RESPAWNS_PER_AGENT = 5;
235
306
  /** ADR-076: hard cap on account rotations per agent between successful turns. */
@@ -241,6 +312,43 @@ const RESPAWN_COOLDOWN_MS = 10 * 60_000;
241
312
  * retry would spin npx continuously for the life of a broken task. */
242
313
  const BROWSER_RETRY_COOLDOWN_MS = 60_000;
243
314
 
315
+ /**
316
+ * A runner can be the first observable casualty of a daemon loss, before any
317
+ * command/tunnel path invalidates the cached ready verdict. Prove the daemon
318
+ * before classifying a terminal agent event as an agent crash. A failed proof
319
+ * poisons the runtime generation synchronously; normal detection/recovery owns
320
+ * the eventual ready edge and deferred-prompt replay.
321
+ */
322
+ async function agentTerminationLostContainerRuntime(): Promise<boolean> {
323
+ if (!backgroundContainerWorkAllowed()) return true;
324
+ let reachable = false;
325
+ try {
326
+ // Backend-aware liveness: `docker` does not exist on an apple host, and
327
+ // an ENOENT here would poison the generation on EVERY abnormal agent
328
+ // termination (found in the 2026-08-18 spawn-docker-ENOENT sweep).
329
+ const proof = taskContainerBackend().apple
330
+ ? await taskContainerCli(["system", "status"], { timeoutMs: 10_000 })
331
+ : await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
332
+ timeoutMs: 10_000,
333
+ });
334
+ reachable = proof.status === 0;
335
+ } catch {
336
+ reachable = false;
337
+ }
338
+ if (!backgroundContainerWorkAllowed()) return true;
339
+ if (reachable) return false;
340
+
341
+ suspendContainerRuntime(true);
342
+ void initializeContainerRuntime().catch((err) =>
343
+ console.warn(
344
+ `[orchestrator] container runtime recheck after agent termination failed: ${
345
+ err instanceof Error ? err.message : String(err)
346
+ }`,
347
+ ),
348
+ );
349
+ return true;
350
+ }
351
+
244
352
  /** Substrings in an agent's error output that mean "config was
245
353
  * unlinked between runs" — repair-and-respawn covers the common
246
354
  * Docker-Desktop-macOS race where Claude's atomic writes briefly
@@ -530,6 +638,8 @@ export class Orchestrator {
530
638
  secretaryAgentId: spec.secretaryAgentId,
531
639
  sessions: new Map(),
532
640
  containerName: `task-${taskId.toLowerCase()}-app-1`,
641
+ agentEnvironment: null,
642
+ agentHandle: null,
533
643
  preambles,
534
644
  firstTurns,
535
645
  sessionsReady: null,
@@ -547,7 +657,10 @@ export class Orchestrator {
547
657
  closed: false,
548
658
  openTurns: new Set(),
549
659
  activeTurns: new Map(),
660
+ acceptedPrompts: new Map(),
550
661
  pendingPrompts: new Map(),
662
+ runtimeDeferredPrompts: new Map(),
663
+ runtimeDeferredAgents: new Set(),
551
664
  interrupted: new Set(),
552
665
  respawns: new Map(),
553
666
  respawnLastAt: new Map(),
@@ -587,6 +700,7 @@ export class Orchestrator {
587
700
  // copy cannot land after browser/MCP materialization and erase config.
588
701
  await taskRecoveryComplete(channel.taskId);
589
702
  if (!this.isActiveChannel(channel)) return false;
703
+ if (!backgroundContainerWorkAllowed()) return false;
590
704
  // Memoized: every caller awaits the SAME in-flight start, so a concurrent
591
705
  // deliver() can't observe "ready" while the sessions map is still empty
592
706
  // (startSessions awaits docker work before populating it — the old boolean
@@ -632,6 +746,7 @@ export class Orchestrator {
632
746
  await this.ensureMcpConfig(channel);
633
747
  if (!this.isActiveChannel(channel)) return false;
634
748
  await this.reconcileSessions(channel);
749
+ this.drainRuntimeDeferredAgentWork(channel);
635
750
  }
636
751
  // The initial-start promise stays memoized after a healthy boot. A later
637
752
  // fatal exit can therefore leave the designated Secretary missing even
@@ -735,6 +850,10 @@ export class Orchestrator {
735
850
  changed: false,
736
851
  };
737
852
  if (!this.isActiveChannel(channel)) return unavailable;
853
+ if (!backgroundContainerWorkAllowed()) {
854
+ this.scheduleBrowserRetry(channel);
855
+ return unavailable;
856
+ }
738
857
  const requestedBrowser = channel.browserTesting;
739
858
  const requestedHasCodex =
740
859
  hasCodex || channel.roster.some((agent) => agent.kind === "codex");
@@ -818,10 +937,17 @@ export class Orchestrator {
818
937
  }
819
938
  let copiedOrUnknown = false;
820
939
  try {
821
- const provisioned = await provisionEngineAccounts(
822
- channel.containerName,
823
- requestedKinds,
824
- );
940
+ const provisioned = channel.agentHandle
941
+ ? await provisionEngineAccounts(
942
+ channel.containerName,
943
+ requestedKinds,
944
+ {},
945
+ channel.agentHandle,
946
+ )
947
+ : await provisionEngineAccounts(
948
+ channel.containerName,
949
+ requestedKinds,
950
+ );
825
951
  copiedOrUnknown =
826
952
  provisioned.copied > 0 || provisioned.mayHaveMutated;
827
953
  if (provisioned.failed > 0) {
@@ -873,12 +999,20 @@ export class Orchestrator {
873
999
  return unavailable;
874
1000
  }
875
1001
  coverage.browserAttempted = true;
876
- setup = await setupBrowserTesting(
877
- channel.taskId,
878
- channel.containerName,
879
- requestedHasCodex,
880
- requestedHomes,
881
- );
1002
+ setup = channel.agentHandle
1003
+ ? await setupBrowserTesting(
1004
+ channel.taskId,
1005
+ channel.containerName,
1006
+ requestedHasCodex,
1007
+ requestedHomes,
1008
+ channel.agentHandle,
1009
+ )
1010
+ : await setupBrowserTesting(
1011
+ channel.taskId,
1012
+ channel.containerName,
1013
+ requestedHasCodex,
1014
+ requestedHomes,
1015
+ );
882
1016
  } catch (err) {
883
1017
  console.warn(
884
1018
  `[browser] task ${channel.taskId}: setup failed: ${
@@ -977,6 +1111,10 @@ export class Orchestrator {
977
1111
  channel.browserRetryTimer = null;
978
1112
  }
979
1113
  if (!this.isActiveChannel(channel)) return;
1114
+ if (!backgroundContainerWorkAllowed()) {
1115
+ this.scheduleBrowserRetry(channel);
1116
+ return;
1117
+ }
980
1118
  const forceProvision = Boolean(channel.provisionRetryFingerprint);
981
1119
  void this.ensureBrowserSetup(
982
1120
  channel,
@@ -1063,13 +1201,28 @@ export class Orchestrator {
1063
1201
  engineKinds.filter((kind) => !previous?.engineKinds?.includes(kind)),
1064
1202
  );
1065
1203
  const hadLiveSessions = channel.sessions.size > 0;
1066
- const configured = await setupMcpTaskConfig(
1067
- channel.taskId,
1068
- channel.containerName,
1069
- connections,
1070
- engineKinds,
1071
- codexHomes,
1072
- );
1204
+ // Once a provider-neutral handle has already been reconstructed for this
1205
+ // channel, keep subsequent config writes on that same environment. The
1206
+ // first legacy/bootstrap pass intentionally retains its Docker fallback;
1207
+ // it must not introduce an eager reconstruction dependency before the
1208
+ // channel has a durable provider locator.
1209
+ const providerEnvironment = channel.agentHandle ?? undefined;
1210
+ const configured = providerEnvironment
1211
+ ? await setupMcpTaskConfig(
1212
+ channel.taskId,
1213
+ channel.containerName,
1214
+ connections,
1215
+ engineKinds,
1216
+ codexHomes,
1217
+ providerEnvironment,
1218
+ )
1219
+ : await setupMcpTaskConfig(
1220
+ channel.taskId,
1221
+ channel.containerName,
1222
+ connections,
1223
+ engineKinds,
1224
+ codexHomes,
1225
+ );
1073
1226
  if (!configured || !this.isActiveChannel(channel)) return;
1074
1227
  channel.configGeneration += 1;
1075
1228
  channel.mcpConfigFingerprint = fingerprint;
@@ -1219,7 +1372,12 @@ export class Orchestrator {
1219
1372
  // Same per-agent materialisation the initial start does. Browser setup
1220
1373
  // is awaited before EVERY missing-session spawn: that reasserts configs
1221
1374
  // clobbered by resume/auth injection and covers roster generation.
1222
- await installPackageSkills(channel.taskId, missing);
1375
+ const skillExec = this.providerSkillExec(channel);
1376
+ if (skillExec) {
1377
+ await installPackageSkills(channel.taskId, missing, skillExec);
1378
+ } else {
1379
+ await installPackageSkills(channel.taskId, missing);
1380
+ }
1223
1381
  if (!this.isActiveChannel(channel)) return;
1224
1382
 
1225
1383
  // One materialization gate owns provisioning and (when enabled)
@@ -1270,7 +1428,6 @@ export class Orchestrator {
1270
1428
  {
1271
1429
  taskId: channel.taskId,
1272
1430
  agent,
1273
- containerName: channel.containerName,
1274
1431
  systemPreamble: channel.preambles.get(agent.id) ?? "",
1275
1432
  executionProfile: this.executionProfileFor(channel, agent.id),
1276
1433
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
@@ -1366,7 +1523,7 @@ export class Orchestrator {
1366
1523
  }
1367
1524
  channel.accountByAgent.set(agent.id, account.id);
1368
1525
  noteEngineAccountUsed(account.id);
1369
- return { ...base, ...account.execEnv };
1526
+ return { ...account.execEnv, ...base };
1370
1527
  }
1371
1528
 
1372
1529
  /** Install a session with an event handler scoped to this exact generation. */
@@ -1376,6 +1533,11 @@ export class Orchestrator {
1376
1533
  session: AgentSession,
1377
1534
  ): void {
1378
1535
  channel.sessions.set(agentId, session);
1536
+ // Adapters may synchronously emit `error` followed by `turn_complete` (or
1537
+ // `exit`). Terminal classification now awaits a daemon proof, so preserve
1538
+ // transport emission order across that await; otherwise the later boundary
1539
+ // can consume/reorder prompt state while the earlier error still owns it.
1540
+ let eventTail: Promise<void> = Promise.resolve();
1379
1541
  session.onEvent((event) => {
1380
1542
  // Recycle/teardown deletes the old session before calling close().
1381
1543
  // Adapters deliberately emit a final exit event; without this identity
@@ -1386,15 +1548,27 @@ export class Orchestrator {
1386
1548
  ) {
1387
1549
  return;
1388
1550
  }
1389
- const operation = this.handleAgentEvent(channel, agentId, event).catch(
1390
- (err: unknown) => {
1551
+ const operation = eventTail
1552
+ .catch(() => {})
1553
+ .then(async () => {
1554
+ // The session may have been retired while an earlier event awaited.
1555
+ // Recheck identity at execution time, not only emission time.
1556
+ if (
1557
+ !this.isActiveChannel(channel) ||
1558
+ channel.sessions.get(agentId) !== session
1559
+ ) {
1560
+ return;
1561
+ }
1562
+ await this.handleAgentEvent(channel, agentId, session, event);
1563
+ })
1564
+ .catch((err: unknown) => {
1391
1565
  console.warn(
1392
1566
  `[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
1393
1567
  err instanceof Error ? (err.stack ?? err.message) : String(err)
1394
1568
  }`,
1395
1569
  );
1396
- },
1397
- );
1570
+ });
1571
+ eventTail = operation;
1398
1572
  channel.eventOperations.add(operation);
1399
1573
  operation.then(
1400
1574
  () => channel.eventOperations.delete(operation),
@@ -1412,7 +1586,7 @@ export class Orchestrator {
1412
1586
  private createAndBindStableSession(
1413
1587
  channel: Channel,
1414
1588
  agentId: string,
1415
- args: Parameters<AgentSessionFactory["create"]>[0],
1589
+ args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
1416
1590
  ): Promise<AgentSession | null> {
1417
1591
  const operation = this.createAndBindStableSessionCore(
1418
1592
  channel,
@@ -1430,11 +1604,12 @@ export class Orchestrator {
1430
1604
  private async createAndBindStableSessionCore(
1431
1605
  channel: Channel,
1432
1606
  agentId: string,
1433
- args: Parameters<AgentSessionFactory["create"]>[0],
1607
+ args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
1434
1608
  ): Promise<AgentSession | null> {
1609
+ const environment = await this.agentEnvironment(channel);
1435
1610
  while (this.isActiveChannel(channel)) {
1436
1611
  const generation = channel.configGeneration;
1437
- const session = await this.factory.create(args);
1612
+ const session = await this.factory.create({ ...args, environment });
1438
1613
  if (!this.isActiveChannel(channel)) {
1439
1614
  await session.close().catch(() => {});
1440
1615
  return null;
@@ -1455,6 +1630,59 @@ export class Orchestrator {
1455
1630
  return null;
1456
1631
  }
1457
1632
 
1633
+ private agentEnvironment(
1634
+ channel: Channel,
1635
+ ): Promise<TaskEnvironmentHandle<unknown>> {
1636
+ const existing = channel.agentEnvironment;
1637
+ if (existing) return existing;
1638
+ const created = (async () => {
1639
+ const task = getHostTask(channel.taskId);
1640
+ if (!task) throw new Error("task environment is not persisted");
1641
+ const environment = await reconstructPersistedTaskEnvironment(task);
1642
+ if (!environment) throw new Error("task environment is not available");
1643
+ if (environment.descriptor.taskId !== channel.taskId) {
1644
+ throw new Error("task environment belongs to a different task");
1645
+ }
1646
+ return environment;
1647
+ })();
1648
+ const tracked = created.then((environment) => {
1649
+ if (this.isActiveChannel(channel)) channel.agentHandle = environment;
1650
+ return environment;
1651
+ });
1652
+ channel.agentEnvironment = tracked;
1653
+ tracked.catch(() => {
1654
+ if (this.isActiveChannel(channel)) {
1655
+ channel.agentEnvironment = null;
1656
+ channel.agentHandle = null;
1657
+ }
1658
+ });
1659
+ return tracked;
1660
+ }
1661
+
1662
+ /** Provider-neutral skill installation for a settled task environment. The
1663
+ * legacy path remains the default until the first environment handle has
1664
+ * actually been reconstructed, avoiding a new await during cold admission.
1665
+ */
1666
+ private providerSkillExec(channel: Channel): SkillExec | undefined {
1667
+ const environment = channel.agentHandle;
1668
+ if (!environment) return undefined;
1669
+ return async (_container, cmd, cwd) => {
1670
+ const argv = cmd as [string, ...string[]];
1671
+ const result = await environment.exec({
1672
+ argv,
1673
+ cwd,
1674
+ user: "node",
1675
+ env: RUNTIME_AUTHORITY_ENV,
1676
+ timeoutMs: 120_000,
1677
+ maxOutputBytes: 256 * 1024,
1678
+ });
1679
+ return {
1680
+ status: result.exitCode,
1681
+ stderr: Buffer.from(result.stderr).toString("utf8"),
1682
+ };
1683
+ };
1684
+ }
1685
+
1458
1686
  private async startSessions(channel: Channel): Promise<boolean> {
1459
1687
  const task = getHostTask(channel.taskId);
1460
1688
  if (
@@ -1464,6 +1692,11 @@ export class Orchestrator {
1464
1692
  ) {
1465
1693
  return false;
1466
1694
  }
1695
+ // Every initial in-container materialization step shares the same boot
1696
+ // fence as the real agent factory. Gate before identity, auth, skills,
1697
+ // browser/MCP configuration, or runner creation can execute in Docker.
1698
+ await agentClisReady;
1699
+ if (!this.isActiveChannel(channel)) return false;
1467
1700
  // Freeze the generation before any slow docker work. A roster add while
1468
1701
  // setup awaits must reconcile under its own engine-aware browser pass,
1469
1702
  // not slip into this factory loop under the old generation's config.
@@ -1474,7 +1707,13 @@ export class Orchestrator {
1474
1707
  // task-up.sh using the creator's per-user key. Best-effort. Awaited but
1475
1708
  // async (docker exec via dockerCli) so it doesn't block the event loop —
1476
1709
  // this runs on every channel ensure, including each post-restart reconnect.
1477
- await setupTaskGitIdentity(channel.taskId, task.ownerName, task.ownerEmail);
1710
+ await setupTaskGitIdentity(
1711
+ channel.taskId,
1712
+ task.ownerName,
1713
+ task.ownerEmail,
1714
+ undefined,
1715
+ channel.agentHandle ?? undefined,
1716
+ );
1478
1717
  if (!this.isActiveChannel(channel)) return false;
1479
1718
 
1480
1719
  // GitHub auth for both `gh` and HTTPS Git (ADR-027): inject the token,
@@ -1482,7 +1721,11 @@ export class Orchestrator {
1482
1721
  // serialized reconcile so agents cannot race their first fetch/push after
1483
1722
  // task creation, reconnect, or host recovery.
1484
1723
  if (task.ownerUserId) {
1485
- await reconcileTaskGitAuth(channel.taskId, task.ownerUserId);
1724
+ await reconcileTaskGitAuth(channel.taskId, task.ownerUserId, {
1725
+ setupDeps: {
1726
+ environment: () => channel.agentHandle,
1727
+ },
1728
+ });
1486
1729
  }
1487
1730
 
1488
1731
  // SSH signing identity, re-asserted independently of transport. Connected
@@ -1501,7 +1744,12 @@ export class Orchestrator {
1501
1744
  // turn. Idempotent + best-effort (returns fast when there are none); a slow
1502
1745
  // clone/install briefly delays start, which is acceptable for skill-bearing
1503
1746
  // tasks. Never throws.
1504
- await installPackageSkills(channel.taskId, initialRoster);
1747
+ const skillExec = this.providerSkillExec(channel);
1748
+ if (skillExec) {
1749
+ await installPackageSkills(channel.taskId, initialRoster, skillExec);
1750
+ } else {
1751
+ await installPackageSkills(channel.taskId, initialRoster);
1752
+ }
1505
1753
  if (!this.isActiveChannel(channel)) return false;
1506
1754
 
1507
1755
  // ADR-053: wire the Playwright MCP browser before agents spawn — configs
@@ -1559,7 +1807,6 @@ export class Orchestrator {
1559
1807
  {
1560
1808
  taskId: channel.taskId,
1561
1809
  agent,
1562
- containerName: channel.containerName,
1563
1810
  systemPreamble: channel.preambles.get(agent.id) ?? "",
1564
1811
  executionProfile: this.executionProfileFor(channel, agent.id),
1565
1812
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
@@ -1608,6 +1855,61 @@ export class Orchestrator {
1608
1855
  return channel ? this.ensureSessions(channel) : false;
1609
1856
  }
1610
1857
 
1858
+ /** Recreate sessions lost with the daemon and replay only prompts whose
1859
+ * adapter boundary explicitly rejected for runtime unavailability. */
1860
+ async resumeRuntimeDeferredAgentWork(): Promise<void> {
1861
+ if (!backgroundContainerWorkAllowed()) return;
1862
+ await Promise.all(
1863
+ [...this.channels.values()].map(async (channel) => {
1864
+ if (
1865
+ !this.isActiveChannel(channel) ||
1866
+ channel.runtimeDeferredAgents.size === 0
1867
+ ) {
1868
+ return;
1869
+ }
1870
+ try {
1871
+ await this.reconcileSessions(channel);
1872
+ this.drainRuntimeDeferredAgentWork(channel);
1873
+ } catch (err) {
1874
+ // A single ready edge fans out across channels. One broken task must
1875
+ // neither abort the others nor consume its own retryable prompts;
1876
+ // the next ordinary ensure/poll retries and drains this channel.
1877
+ console.warn(
1878
+ `[orchestrator] ${channel.taskId}: deferred runtime work remains queued: ${
1879
+ err instanceof Error ? err.message : String(err)
1880
+ }`,
1881
+ );
1882
+ }
1883
+ }),
1884
+ );
1885
+ }
1886
+
1887
+ private drainRuntimeDeferredAgentWork(channel: Channel): void {
1888
+ if (
1889
+ !this.isActiveChannel(channel) ||
1890
+ !backgroundContainerWorkAllowed()
1891
+ ) {
1892
+ return;
1893
+ }
1894
+ for (const agentId of [...channel.runtimeDeferredAgents]) {
1895
+ const session = channel.sessions.get(agentId);
1896
+ if (!session) continue;
1897
+
1898
+ // Clear ownership before sending. If runtime drops at the same last hop,
1899
+ // startPrompt's typed catch atomically re-adds the exact work.
1900
+ channel.runtimeDeferredAgents.delete(agentId);
1901
+ if (this.isSecretary(channel, agentId)) {
1902
+ this.startNextPendingPrompt(channel, agentId);
1903
+ continue;
1904
+ }
1905
+ const prompts = channel.runtimeDeferredPrompts.get(agentId) ?? [];
1906
+ channel.runtimeDeferredPrompts.delete(agentId);
1907
+ for (const prompt of prompts) {
1908
+ this.startPrompt(channel, agentId, session, prompt);
1909
+ }
1910
+ }
1911
+ }
1912
+
1611
1913
  private incrementActiveTurns(channel: Channel, agentId: string): void {
1612
1914
  channel.activeTurns.set(
1613
1915
  agentId,
@@ -1645,6 +1947,7 @@ export class Orchestrator {
1645
1947
  private retireRunnerGeneration(channel: Channel, agentId: string): void {
1646
1948
  const aborted = channel.interrupted.delete(agentId);
1647
1949
  channel.activeTurns.delete(agentId);
1950
+ channel.acceptedPrompts.delete(agentId);
1648
1951
  channel.openTurns.delete(agentId);
1649
1952
  if (aborted) {
1650
1953
  channel.lastPrompt.delete(agentId);
@@ -1666,9 +1969,14 @@ export class Orchestrator {
1666
1969
  // ADR-076: this is the prompt actually in flight. A queued follow-up must
1667
1970
  // not replace it or account rotation would replay the wrong turn.
1668
1971
  channel.lastPrompt.set(agentId, prompt);
1972
+ const accepted = { text: prompt };
1973
+ const acceptedForAgent = channel.acceptedPrompts.get(agentId) ?? [];
1974
+ acceptedForAgent.push(accepted);
1975
+ channel.acceptedPrompts.set(agentId, acceptedForAgent);
1669
1976
  this.incrementActiveTurns(channel, agentId);
1670
1977
  void session.send(prompt).catch((err: unknown) => {
1671
1978
  if (channel.sessions.get(agentId) === session) {
1979
+ this.removeAcceptedPrompt(channel, agentId, accepted);
1672
1980
  const remaining = this.decrementActiveTurns(channel, agentId);
1673
1981
  if (
1674
1982
  remaining === 0 &&
@@ -1676,7 +1984,11 @@ export class Orchestrator {
1676
1984
  ) {
1677
1985
  channel.lastPrompt.delete(agentId);
1678
1986
  }
1679
- this.startNextPendingPrompt(channel, agentId);
1987
+ if (err instanceof ContainerRuntimeUnavailableError) {
1988
+ this.deferPromptUntilRuntimeReady(channel, agentId, prompt);
1989
+ } else {
1990
+ this.startNextPendingPrompt(channel, agentId);
1991
+ }
1680
1992
  }
1681
1993
  console.warn(
1682
1994
  `[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
@@ -1686,6 +1998,49 @@ export class Orchestrator {
1686
1998
  });
1687
1999
  }
1688
2000
 
2001
+ private removeAcceptedPrompt(
2002
+ channel: Channel,
2003
+ agentId: string,
2004
+ accepted: AcceptedPrompt,
2005
+ ): void {
2006
+ const prompts = channel.acceptedPrompts.get(agentId);
2007
+ if (!prompts) return;
2008
+ const index = prompts.indexOf(accepted);
2009
+ if (index >= 0) prompts.splice(index, 1);
2010
+ if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
2011
+ }
2012
+
2013
+ private completeNextAcceptedPrompt(
2014
+ channel: Channel,
2015
+ agentId: string,
2016
+ ): void {
2017
+ const prompts = channel.acceptedPrompts.get(agentId);
2018
+ if (!prompts) return;
2019
+ prompts.shift();
2020
+ if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
2021
+ }
2022
+
2023
+ private deferPromptUntilRuntimeReady(
2024
+ channel: Channel,
2025
+ agentId: string,
2026
+ prompt: string,
2027
+ ): void {
2028
+ channel.runtimeDeferredAgents.add(agentId);
2029
+ if (this.isSecretary(channel, agentId)) {
2030
+ // Secretary delivery is serialized. Put the failed in-flight prompt in
2031
+ // front of crew replies that were accepted behind it.
2032
+ const pending = channel.pendingPrompts.get(agentId) ?? [];
2033
+ pending.unshift(prompt);
2034
+ channel.pendingPrompts.set(agentId, pending);
2035
+ return;
2036
+ }
2037
+ // Open-mode sends are intentionally concurrent. Promise rejection handlers
2038
+ // run in submission order, so append preserves that accepted order.
2039
+ const deferred = channel.runtimeDeferredPrompts.get(agentId) ?? [];
2040
+ deferred.push(prompt);
2041
+ channel.runtimeDeferredPrompts.set(agentId, deferred);
2042
+ }
2043
+
1689
2044
  /** Start at most one queued prompt after the previous turn is terminal. */
1690
2045
  private startNextPendingPrompt(channel: Channel, agentId: string): boolean {
1691
2046
  if (
@@ -1804,7 +2159,11 @@ export class Orchestrator {
1804
2159
  // Cancel any armed auto-retry chain so this manual attempt runs fresh
1805
2160
  // (setup's duplicate-chain guard would otherwise no-op it).
1806
2161
  clearRefresh(taskId);
1807
- const ok = await reconcileTaskGitAuth(taskId, owner);
2162
+ const ok = await reconcileTaskGitAuth(taskId, owner, {
2163
+ setupDeps: {
2164
+ environment: () => this.channels.get(taskId)?.agentHandle ?? null,
2165
+ },
2166
+ });
1808
2167
  this.emitSystemNote(
1809
2168
  taskId,
1810
2169
  ok
@@ -1818,6 +2177,7 @@ export class Orchestrator {
1818
2177
  private async handleAgentEvent(
1819
2178
  channel: Channel,
1820
2179
  agentId: string,
2180
+ session: AgentSession,
1821
2181
  event: AgentEvent,
1822
2182
  ): Promise<void> {
1823
2183
  if (!this.isActiveChannel(channel)) return;
@@ -1901,6 +2261,18 @@ export class Orchestrator {
1901
2261
  break;
1902
2262
  }
1903
2263
  case "error": {
2264
+ if (await agentTerminationLostContainerRuntime()) {
2265
+ // `error` and `exit` can arrive back-to-back while the daemon probe
2266
+ // is pending. Only the first terminal event still owns this session.
2267
+ if (channel.sessions.get(agentId) !== session) break;
2268
+ await this.retireSessionForRuntimeLoss(
2269
+ channel,
2270
+ agentId,
2271
+ session,
2272
+ );
2273
+ break;
2274
+ }
2275
+ if (channel.sessions.get(agentId) !== session) break;
1904
2276
  // The turn died with the session — drop its turn-state flags so a
1905
2277
  // respawned session starts clean.
1906
2278
  this.retireRunnerGeneration(channel, agentId);
@@ -1972,6 +2344,7 @@ export class Orchestrator {
1972
2344
  // An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
1973
2345
  // the cloud discards the buffer instead of handing it to peers.
1974
2346
  const aborted = channel.interrupted.delete(agentId);
2347
+ this.completeNextAcceptedPrompt(channel, agentId);
1975
2348
  const remainingTurns = this.decrementActiveTurns(channel, agentId);
1976
2349
  channel.openTurns.delete(agentId);
1977
2350
  // ADR-076: a turn that completed means the current account is healthy —
@@ -2005,6 +2378,16 @@ export class Orchestrator {
2005
2378
  break;
2006
2379
  }
2007
2380
  case "exit":
2381
+ if (await agentTerminationLostContainerRuntime()) {
2382
+ if (channel.sessions.get(agentId) !== session) break;
2383
+ await this.retireSessionForRuntimeLoss(
2384
+ channel,
2385
+ agentId,
2386
+ session,
2387
+ );
2388
+ break;
2389
+ }
2390
+ if (channel.sessions.get(agentId) !== session) break;
2008
2391
  // Same zombie hazard as the error path — a session whose process
2009
2392
  // ended (even cleanly) can never carry another turn.
2010
2393
  this.retireRunnerGeneration(channel, agentId);
@@ -2019,6 +2402,35 @@ export class Orchestrator {
2019
2402
  }
2020
2403
  }
2021
2404
 
2405
+ /** A daemon loss is infrastructure recovery, not an agent crash. Preserve
2406
+ * accepted work, drop the dead generation, and leave respawn/exit accounting
2407
+ * untouched; the runtime-ready hook recreates it deterministically. */
2408
+ private async retireSessionForRuntimeLoss(
2409
+ channel: Channel,
2410
+ agentId: string,
2411
+ session: AgentSession,
2412
+ ): Promise<void> {
2413
+ const accepted = [
2414
+ ...(channel.acceptedPrompts.get(agentId) ?? []),
2415
+ ];
2416
+ const interrupted = channel.interrupted.has(agentId);
2417
+ this.retireRunnerGeneration(channel, agentId);
2418
+ channel.lastPrompt.delete(agentId);
2419
+ if (accepted.length > 0 && !interrupted) {
2420
+ for (const prompt of accepted) {
2421
+ this.deferPromptUntilRuntimeReady(channel, agentId, prompt.text);
2422
+ }
2423
+ } else {
2424
+ // Recreate even an idle session after readiness, without fabricating a
2425
+ // prompt or requiring another human delivery to discover the dead slot.
2426
+ channel.runtimeDeferredAgents.add(agentId);
2427
+ }
2428
+ channel.sessions.delete(agentId);
2429
+ channel.browserStaleSessions.delete(agentId);
2430
+ channel.browserPendingStaleSessions.delete(agentId);
2431
+ await session.close().catch(() => {});
2432
+ }
2433
+
2022
2434
  // -- session recovery -----------------------------------------------------
2023
2435
 
2024
2436
  /**
@@ -2054,7 +2466,10 @@ export class Orchestrator {
2054
2466
 
2055
2467
  channel.spawning.add(agentId);
2056
2468
  try {
2057
- const restored = repairClaudeConfigInContainer(channel.containerName);
2469
+ const restored = await repairClaudeConfigInContainer(
2470
+ channel.containerName,
2471
+ channel.agentHandle ?? undefined,
2472
+ );
2058
2473
 
2059
2474
  // Delete the old generation before awaiting close. Its adapter emits a
2060
2475
  // terminal event during teardown; bindSession's identity guard must see
@@ -2097,7 +2512,6 @@ export class Orchestrator {
2097
2512
  const session = await this.createAndBindStableSession(channel, agentId, {
2098
2513
  taskId: channel.taskId,
2099
2514
  agent,
2100
- containerName: channel.containerName,
2101
2515
  systemPreamble: channel.preambles.get(agentId) ?? "",
2102
2516
  executionProfile: this.executionProfileFor(channel, agentId),
2103
2517
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
@@ -2105,7 +2519,7 @@ export class Orchestrator {
2105
2519
  agentId,
2106
2520
  ),
2107
2521
  agentEnv: boundAccount
2108
- ? { ...base, ...boundAccount.execEnv }
2522
+ ? { ...boundAccount.execEnv, ...base }
2109
2523
  : this.accountAgentEnv(channel, agent, base),
2110
2524
  });
2111
2525
  if (!session) return;
@@ -2226,14 +2640,13 @@ export class Orchestrator {
2226
2640
  {
2227
2641
  taskId: channel.taskId,
2228
2642
  agent,
2229
- containerName: channel.containerName,
2230
2643
  systemPreamble: channel.preambles.get(agentId) ?? "",
2231
2644
  executionProfile: this.executionProfileFor(channel, agentId),
2232
2645
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2233
2646
  channel,
2234
2647
  agentId,
2235
2648
  ),
2236
- agentEnv: { ...base, ...next.execEnv },
2649
+ agentEnv: { ...next.execEnv, ...base },
2237
2650
  },
2238
2651
  );
2239
2652
  if (!replacement) return true;
@@ -2341,14 +2754,13 @@ export class Orchestrator {
2341
2754
  replacement = await this.createAndBindStableSession(channel, agentId, {
2342
2755
  taskId: channel.taskId,
2343
2756
  agent,
2344
- containerName: channel.containerName,
2345
2757
  systemPreamble: channel.preambles.get(agentId) ?? "",
2346
2758
  executionProfile: this.executionProfileFor(channel, agentId),
2347
2759
  attachCompatibilityKey: this.attachCompatibilityKeyFor(
2348
2760
  channel,
2349
2761
  agentId,
2350
2762
  ),
2351
- agentEnv: { ...base, ...account.execEnv },
2763
+ agentEnv: { ...account.execEnv, ...base },
2352
2764
  });
2353
2765
  if (!replacement) return true;
2354
2766
 
@@ -2487,7 +2899,10 @@ export class Orchestrator {
2487
2899
  ch.browserStaleSessions.clear();
2488
2900
  ch.browserPendingStaleSessions.clear();
2489
2901
  ch.activeTurns.clear();
2902
+ ch.acceptedPrompts.clear();
2490
2903
  ch.pendingPrompts.clear();
2904
+ ch.runtimeDeferredPrompts.clear();
2905
+ ch.runtimeDeferredAgents.clear();
2491
2906
  ch.openTurns.clear();
2492
2907
  ch.interrupted.clear();
2493
2908
  ch.reconcileAgain = false;
@@ -2527,6 +2942,18 @@ export class Orchestrator {
2527
2942
  return this.channelClosures.get(taskId) ?? Promise.resolve();
2528
2943
  }
2529
2944
 
2945
+ /** Quarantine a live channel before recovery stops its app container.
2946
+ * Preserve the latest channel spec so a successful recovery can reopen the
2947
+ * task without waiting for another cloud message, while the ordinary close
2948
+ * tombstone prevents any agent process from respawning during maintenance. */
2949
+ async quarantineChannelForRecovery(taskId: string): Promise<void> {
2950
+ const currentSpec = this.channelSpecs.get(taskId);
2951
+ await this.closeChannel(taskId);
2952
+ if (currentSpec && !this.channelSpecs.has(taskId)) {
2953
+ this.channelSpecs.set(taskId, currentSpec);
2954
+ }
2955
+ }
2956
+
2530
2957
  /** A successful task-up is the only transition that reopens a tombstoned
2531
2958
  * task id after stop/teardown. */
2532
2959
  allowChannel(taskId: string): void {
@@ -2553,6 +2980,9 @@ export class Orchestrator {
2553
2980
  private async stopTaskWithinLifecycle(
2554
2981
  taskId: string,
2555
2982
  ): Promise<{ ok: boolean; error?: string }> {
2983
+ if (!backgroundContainerWorkAllowed()) {
2984
+ return { ok: false, error: "container runtime is not operational" };
2985
+ }
2556
2986
  const task = getHostTask(taskId);
2557
2987
  if (!task) return { ok: false, error: "unknown task" };
2558
2988
  const project = task.composeProject;
@@ -2568,33 +2998,27 @@ export class Orchestrator {
2568
2998
  }`,
2569
2999
  );
2570
3000
  }
2571
- const ps = await dockerCli([
2572
- "ps",
2573
- "-q",
2574
- "--filter",
2575
- `label=com.docker.compose.project=${project}`,
2576
- ]);
2577
- if (ps.status !== 0) {
2578
- // An empty stdout is not evidence that the stack is gone when docker
2579
- // itself failed. Reopen the channel and leave the runtime mirror alone
2580
- // so cloud reconciliation cannot turn a live container into "stopped".
3001
+ // Provider-neutral resumable stop. A row that reaches this point with
3002
+ // runtime resources always carries a proven locator (legacy rows cross
3003
+ // the adoption fence before the runtime becomes operational), so a
3004
+ // reconstruction failure is a stop failure, never a fallback hint. An
3005
+ // environment failure reopens the channel and leaves the runtime mirror
3006
+ // alone so cloud reconciliation cannot turn a live container into
3007
+ // "stopped".
3008
+ try {
3009
+ const environment = await reconstructPersistedTaskEnvironment(task);
3010
+ if (!environment) {
3011
+ this.allowChannel(taskId);
3012
+ return { ok: false, error: "task has no persisted environment" };
3013
+ }
3014
+ await environment.stop();
3015
+ } catch (err) {
2581
3016
  this.allowChannel(taskId);
2582
3017
  return {
2583
3018
  ok: false,
2584
- error: ps.stderr.trim().slice(0, 200) || "docker ps failed",
3019
+ error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
2585
3020
  };
2586
3021
  }
2587
- const ids = ps.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
2588
- if (ids.length > 0) {
2589
- const stopped = await dockerCli(["stop", ...ids], { timeoutMs: 60_000 });
2590
- if (stopped.status !== 0) {
2591
- this.allowChannel(taskId);
2592
- return {
2593
- ok: false,
2594
- error: stopped.stderr.trim().slice(0, 200) || "docker stop failed",
2595
- };
2596
- }
2597
- }
2598
3022
  upsertHostTask(taskId, { statusMirror: "stopped" });
2599
3023
  clearTaskGatewayAcl(taskId);
2600
3024
  return { ok: true };
@@ -2688,7 +3112,10 @@ export function resolveAddressing(
2688
3112
  *
2689
3113
  * Returns true when the file actually made it into the container.
2690
3114
  */
2691
- function repairClaudeConfigInContainer(containerName: string): boolean {
3115
+ async function repairClaudeConfigInContainer(
3116
+ containerName: string,
3117
+ environment?: Pick<TaskEnvironmentHandle, "exec">,
3118
+ ): Promise<boolean> {
2692
3119
  // Operator can override the source path via UAI_OWNER_HOME so the
2693
3120
  // server doesn't have to guess. Useful when Next.js's dev process
2694
3121
  // env diverges from `os.homedir()` for any reason.
@@ -2733,25 +3160,44 @@ function repairClaudeConfigInContainer(containerName: string): boolean {
2733
3160
  // bounces. `rm -f` punches through it (or no-ops on a genuine
2734
3161
  // missing file). Then `cat >` creates the fresh inode. Both steps
2735
3162
  // run in one `sh -c` so a partial failure leaves a clean state.
2736
- const write = spawnSync(
2737
- "docker",
3163
+ const repairScript =
3164
+ "rm -f /home/node/.claude.json && cat > /home/node/.claude.json";
3165
+ if (environment) {
3166
+ const write = await environment.exec({
3167
+ argv: ["/bin/sh", "-c", repairScript],
3168
+ env: RUNTIME_AUTHORITY_ENV,
3169
+ stdin: content,
3170
+ timeoutMs: 30_000,
3171
+ maxOutputBytes: 64 * 1024,
3172
+ });
3173
+ if (write.exitCode !== 0) {
3174
+ console.error(
3175
+ "[orchestrator] repair: in-place write to .claude.json failed: " +
3176
+ Buffer.from(write.stderr).toString("utf8").trim(),
3177
+ );
3178
+ return false;
3179
+ }
3180
+ return true;
3181
+ }
3182
+ const write = await taskContainerCli(
2738
3183
  [
2739
3184
  "exec",
2740
3185
  "-i",
2741
- containerName,
2742
- "sh",
3186
+ ...runtimeAuthorityDockerExecArgs(),
3187
+ // The channel carries the Docker replica name; the apple container
3188
+ // drops the replica suffix (2026-08-18 spawn-docker-ENOENT sweep).
3189
+ taskContainerBackend().apple
3190
+ ? containerName.replace(/-app-1$/, "-app")
3191
+ : containerName,
3192
+ "/bin/sh",
2743
3193
  "-c",
2744
- "rm -f /home/node/.claude.json && cat > /home/node/.claude.json",
3194
+ repairScript,
2745
3195
  ],
2746
- { input: content, encoding: "buffer" },
3196
+ { input: content.toString("utf8"), timeoutMs: 30_000 },
2747
3197
  );
2748
3198
  if (write.status !== 0) {
2749
- const stderr =
2750
- write.stderr instanceof Buffer
2751
- ? write.stderr.toString("utf8")
2752
- : String(write.stderr ?? "");
2753
3199
  console.error(
2754
- `[orchestrator] repair: in-place write to .claude.json failed: ${stderr.trim()}`,
3200
+ `[orchestrator] repair: in-place write to .claude.json failed: ${write.stderr.trim()}`,
2755
3201
  );
2756
3202
  return false;
2757
3203
  }
@@ -3407,19 +3853,60 @@ export function getOrchestrator(): Orchestrator {
3407
3853
  const factory = mode === "real" ? realAgentFactory : mockAgentFactory;
3408
3854
  globalForOrchestrator.__uaiOrchestrator = new Orchestrator(factory);
3409
3855
  }
3410
- if (!globalForOrchestrator.__uaiRecoverRan) {
3411
- globalForOrchestrator.__uaiRecoverRan = true;
3412
- // Fire-and-forget — the orchestrator is usable while recovery runs.
3413
- // The promise is kept so boot steps that must NOT race recovery (the
3414
- // Codex credential reinject) can sequence behind it via
3415
- // recoveryComplete(). recoverRunningTasks never rejects.
3416
- recoveryPromise = recoverRunningTasks();
3417
- void recoveryPromise;
3418
- }
3419
3856
  return globalForOrchestrator.__uaiOrchestrator;
3420
3857
  }
3421
3858
 
3422
- let recoveryPromise: Promise<void> = Promise.resolve();
3859
+ /** Runtime activation publication hook. Do not construct an orchestrator just
3860
+ * to run it: deferred work can only exist on the already-live singleton. */
3861
+ export function resumeRuntimeDeferredAgentWork(): void {
3862
+ const orchestrator = globalForOrchestrator.__uaiOrchestrator;
3863
+ if (!orchestrator) return;
3864
+ void orchestrator.resumeRuntimeDeferredAgentWork().catch((err) =>
3865
+ console.warn(
3866
+ `[orchestrator] deferred agent work resume failed: ${
3867
+ err instanceof Error ? err.message : String(err)
3868
+ }`,
3869
+ ),
3870
+ );
3871
+ }
3872
+
3873
+ let recoveryPromise: Promise<boolean> = Promise.resolve(true);
3874
+ let recoveryInFlight: Promise<boolean> | null = null;
3875
+
3876
+ /** Start or join one recovery sweep. A runtime-ready recheck can call this
3877
+ * after the original boot window gave up, without racing an active sweep. */
3878
+ function beginRuntimeRecovery(
3879
+ lifecycle?: Orchestrator,
3880
+ isCurrent: () => boolean = () => true,
3881
+ ): Promise<boolean> {
3882
+ // A caller that only wants the boot barrier joins it. A live-runtime request
3883
+ // is stronger: chain a fresh lifecycle-sequenced pass behind any active
3884
+ // scan. Promise-tail sequencing has no "last queue read vs finally" gap, so
3885
+ // a readiness request arriving in that microtask window cannot be lost.
3886
+ if (recoveryInFlight && !lifecycle) return recoveryInFlight;
3887
+ const predecessor = recoveryInFlight;
3888
+ const sweep = async (): Promise<boolean> => {
3889
+ try {
3890
+ return await recoverRunningTasks({ lifecycle, isCurrent });
3891
+ } catch (error) {
3892
+ console.error(
3893
+ "[orchestrator] recovery: unhandled failure",
3894
+ error instanceof Error ? error.message : error,
3895
+ );
3896
+ return false;
3897
+ }
3898
+ };
3899
+ let running!: Promise<boolean>;
3900
+ running = (predecessor
3901
+ ? predecessor.catch(() => false).then(sweep)
3902
+ : sweep()
3903
+ ).finally(() => {
3904
+ if (recoveryInFlight === running) recoveryInFlight = null;
3905
+ });
3906
+ recoveryInFlight = running;
3907
+ recoveryPromise = running;
3908
+ return running;
3909
+ }
3423
3910
 
3424
3911
  interface TaskRecoveryBarrier {
3425
3912
  promise: Promise<void>;
@@ -3456,13 +3943,73 @@ export function taskRecoveryComplete(taskId: string): Promise<void> {
3456
3943
 
3457
3944
  /**
3458
3945
  * Resolves once boot-time recovery has finished (starting the orchestrator —
3459
- * and with it recovery — if that hasn't happened yet). Never rejects.
3946
+ * and with it recovery — if that hasn't happened yet). Never rejects; false
3947
+ * means one or more active tasks remain unresolved.
3460
3948
  */
3461
- export function recoveryComplete(): Promise<void> {
3949
+ export function recoveryComplete(
3950
+ isCurrent: () => boolean = () => true,
3951
+ ): Promise<boolean> {
3462
3952
  getOrchestrator();
3953
+ if (!globalForOrchestrator.__uaiRecoverRan) {
3954
+ globalForOrchestrator.__uaiRecoverRan = true;
3955
+ // Fire-and-forget — the orchestrator is usable while recovery runs. Main
3956
+ // calls this only after the selected container runtime is ready.
3957
+ void beginRuntimeRecovery(undefined, isCurrent);
3958
+ }
3463
3959
  return recoveryPromise;
3464
3960
  }
3465
3961
 
3962
+ /**
3963
+ * Re-run boot recovery after a container runtime transitions to ready.
3964
+ *
3965
+ * `recoveryComplete()` is a one-boot barrier; this is the explicit recovery
3966
+ * trigger for ADR-101 runtime rechecks. It joins an active boot sweep, or
3967
+ * starts a new sweep after the prior bounded retry window completed.
3968
+ */
3969
+ export function requestRuntimeRecovery(
3970
+ isCurrent: () => boolean = () => true,
3971
+ ): Promise<boolean> {
3972
+ const orchestrator = getOrchestrator();
3973
+ globalForOrchestrator.__uaiRecoverRan = true;
3974
+ return beginRuntimeRecovery(orchestrator, isCurrent);
3975
+ }
3976
+
3977
+ /** Join only work that is already mutating/reconciling runtime state. This is
3978
+ * used before image maintenance so an explicit recheck cannot overlap a
3979
+ * reconnect sweep on the shared daemon/CLI volume. */
3980
+ export function waitForRuntimeRecovery(): Promise<void> {
3981
+ return recoveryInFlight?.then(() => undefined) ?? Promise.resolve();
3982
+ }
3983
+
3984
+ export type RuntimeRecoveryVerdict = "clean" | "needed" | "unknown";
3985
+
3986
+ /** Cheap reconnect proof. Each active environment provider reports whether
3987
+ * its durable workload is still running; ordinary cloud/network flaps avoid
3988
+ * the heavier repair path, while an unseen runtime restart still forces full
3989
+ * lifecycle-sequenced recovery before commands reopen. */
3990
+ export async function runtimeRecoveryVerdict(): Promise<RuntimeRecoveryVerdict> {
3991
+ const activeRows = getDb()
3992
+ .select()
3993
+ .from(schema.hostTasks)
3994
+ .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
3995
+ .all();
3996
+ for (const task of activeRows) {
3997
+ try {
3998
+ const environment = await reconstructPersistedTaskEnvironment(task);
3999
+ // A pre-provision row has no provider-owned runtime to inventory.
4000
+ if (environment === null) continue;
4001
+ const status = await environment.status();
4002
+ if (status.state === "unknown") return "unknown";
4003
+ if (status.state !== "running") return "needed";
4004
+ } catch {
4005
+ // An unavailable provider, malformed locator, or backend mismatch is
4006
+ // unknown—not evidence that the selected runtime is clean or absent.
4007
+ return "unknown";
4008
+ }
4009
+ }
4010
+ return "clean";
4011
+ }
4012
+
3466
4013
  // ---------------------------------------------------------------------------
3467
4014
  // Boot-time recovery
3468
4015
  //
@@ -3481,8 +4028,10 @@ export function recoveryComplete(): Promise<void> {
3481
4028
  // id (task-up is idempotent).
3482
4029
  // container gone, worktree gone → mark `error`. Data loss.
3483
4030
  //
3484
- // Idempotent per process — guarded by `__uaiRecoverRan` on globalThis
3485
- // so a Next.js HMR rebuild doesn't trigger it twice.
4031
+ // Idempotent per process — recoveryComplete/requestRuntimeRecovery mark
4032
+ // `__uaiRecoverRan` on globalThis so a Next.js HMR rebuild cannot trigger the
4033
+ // boot sweep twice. Merely constructing the orchestrator is deliberately not
4034
+ // enough: a degraded host must not probe an unrelated ambient daemon.
3486
4035
  // ---------------------------------------------------------------------------
3487
4036
 
3488
4037
  /**
@@ -3529,32 +4078,1168 @@ async function dockerListContainersByLabel(
3529
4078
  .filter((x): x is DockerPs => x !== null);
3530
4079
  }
3531
4080
 
3532
- async function dockerStart(containerName: string): Promise<boolean> {
4081
+ type DockerStartResult = "started" | "failed" | "unreachable";
4082
+
4083
+ async function dockerDaemonReachable(): Promise<boolean> {
4084
+ const probe = await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
4085
+ timeoutMs: 10_000,
4086
+ });
4087
+ return probe.status === 0 && probe.stdout.trim().length > 0;
4088
+ }
4089
+
4090
+ async function dockerStart(containerName: string): Promise<DockerStartResult> {
3533
4091
  // Starting a big dev container can legitimately take a while — give it far
3534
4092
  // more than dockerCli's 30s default before declaring the task stopped.
3535
4093
  const res = await dockerCli(["start", containerName], { timeoutMs: 120_000 });
3536
- if (res.status !== 0) {
3537
- console.error(
3538
- `[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
3539
- );
3540
- return false;
4094
+ if (res.status === 0) return "started";
4095
+ // A timeout is ambiguous: Docker may have completed the start after our CLI
4096
+ // was killed. Inspect the exact container, then prove the daemon still
4097
+ // exists before deciding a non-inspectable container is a terminal failure.
4098
+ const inspected = await dockerCli([
4099
+ "inspect",
4100
+ "--format",
4101
+ "{{.State.Running}}",
4102
+ containerName,
4103
+ ]);
4104
+ if (inspected.status === 0 && inspected.stdout.trim() === "true") {
4105
+ return "started";
3541
4106
  }
3542
- return true;
4107
+ if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
4108
+ return "unreachable";
4109
+ }
4110
+ console.error(
4111
+ `[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
4112
+ );
4113
+ return "failed";
4114
+ }
4115
+
4116
+ // The fourth column is the preview-sidecar marker: a sidecar can wear the
4117
+ // task's compose labels, and a row that is neither app-shaped nor a known
4118
+ // sidecar reads as AMBIGUOUS here — which fenced maintenance on a healthy
4119
+ // host (live 2026-08-18).
4120
+ const RUNTIME_WRITER_LIST_FORMAT =
4121
+ `{{json .Names}}\t{{.Label "com.docker.compose.project"}}\t{{.Label "com.docker.compose.service"}}\t{{.Label "${PREVIEW_SIDECAR_TASK_LABEL}"}}`;
4122
+ const RUNTIME_WRITER_INSPECT_FORMAT = [
4123
+ "{{json .Id}}",
4124
+ "{{json .Name}}",
4125
+ "{{json .State.Running}}",
4126
+ "{{json .State.Restarting}}",
4127
+ "{{json .State.Paused}}",
4128
+ "{{json .HostConfig.RestartPolicy.Name}}",
4129
+ '{{json (index .Config.Labels "com.docker.compose.project")}}',
4130
+ '{{json (index .Config.Labels "com.docker.compose.service")}}',
4131
+ "{{json .Mounts}}",
4132
+ "{{json .Config.Env}}",
4133
+ ].join("\n");
4134
+ const RUNTIME_WRITER_CONTAINER_ID_RE = /^[0-9a-f]{64}$/;
4135
+ const RUNTIME_WRITER_SCAN_OUTPUT_BYTES = 2 * 1024 * 1024;
4136
+ const TASK_RUNTIME_PATH = RUNTIME_AUTHORITY_ENV.PATH;
4137
+ const TASK_OWNED_ENV = new Map<string, string>(
4138
+ Object.entries(RUNTIME_AUTHORITY_ENV),
4139
+ );
4140
+ const NPM_CONFIG_ENV_PREFIX = "npm_config_";
4141
+ function normalizeNpmConfigEnvKey(key: string): string | undefined {
4142
+ const folded = key.toLowerCase();
4143
+ if (!folded.startsWith(NPM_CONFIG_ENV_PREFIX)) return undefined;
4144
+ // npm converts underscores in the option suffix to hyphens. Docker can
4145
+ // preserve spellings a shell cannot assign (for example LOGS-DIR), so both
4146
+ // forms must be treated as the same owned option during immutable-env proof.
4147
+ return (
4148
+ NPM_CONFIG_ENV_PREFIX +
4149
+ folded.slice(NPM_CONFIG_ENV_PREFIX.length).replaceAll("_", "-")
4150
+ );
4151
+ }
4152
+ const TASK_OWNED_NORMALIZED_NPM_ENV_KEYS = new Set(
4153
+ [...TASK_OWNED_ENV.keys()]
4154
+ .map(normalizeNpmConfigEnvKey)
4155
+ .filter((key): key is string => key !== undefined),
4156
+ );
4157
+
4158
+ interface RuntimeWriterCandidate {
4159
+ taskId: string;
4160
+ composeProject: string;
4161
+ containerName: string;
4162
+ }
4163
+
4164
+ interface RuntimeWriterMetadata extends RuntimeWriterCandidate {
4165
+ containerId: string;
4166
+ running: boolean;
4167
+ restarting: boolean;
4168
+ paused: boolean;
4169
+ restartPolicy: string;
4170
+ sharedMount: "absent" | "read-only" | "writable" | "invalid";
4171
+ taskEnvironment: "current" | "invalid";
4172
+ }
4173
+
4174
+ type RuntimeWriterInspection =
4175
+ | { kind: "ready"; metadata: RuntimeWriterMetadata }
4176
+ | { kind: "failed"; detail: string }
4177
+ | { kind: "unreachable" };
4178
+
4179
+ export type TaskRuntimeContainerInspection =
4180
+ | {
4181
+ kind: "ready";
4182
+ containerId: string;
4183
+ running: boolean;
4184
+ restarting: boolean;
4185
+ paused: boolean;
4186
+ sharedMount: RuntimeWriterMetadata["sharedMount"];
4187
+ taskEnvironment: RuntimeWriterMetadata["taskEnvironment"];
4188
+ }
4189
+ | { kind: "failed"; detail: string }
4190
+ | { kind: "unreachable" };
4191
+
4192
+ export interface WritableRuntimeQuarantineResult {
4193
+ verdict: "ready" | "retry" | "stale";
4194
+ /** Known rows whose local `stopped` verdict must be published or replayed. */
4195
+ stoppedTaskIds: string[];
4196
+ }
4197
+
4198
+ function runtimeWriterCandidate(
4199
+ rawName: unknown,
4200
+ rawProject: string,
4201
+ rawService: string,
4202
+ ): RuntimeWriterCandidate | "unrelated" | "invalid" {
4203
+ if (!rawProject.startsWith("task-")) return "unrelated";
4204
+ const taskId = rawProject.slice("task-".length);
4205
+ if (!isSafeHostTaskId(taskId)) return "invalid";
4206
+ if (rawService !== "app" || typeof rawName !== "string") return "invalid";
4207
+ const containerName = `${rawProject}-app-1`;
4208
+ if (rawName !== containerName) return "invalid";
4209
+ return { taskId, composeProject: rawProject, containerName };
3543
4210
  }
3544
4211
 
3545
4212
  /**
3546
- * Repair the shared node data root before any recovered-container init.
3547
- * Older task-up versions created the OpenCode leaf as root and accidentally
3548
- * left this parent root-owned, which prevented code-server from creating its
3549
- * managed User profile. `install -d` creates or repairs the exact directory
3550
- * without recursively changing unrelated application state beneath it.
4213
+ * Enumerate the Docker authority, not only SQLite. A crashed task-down or an
4214
+ * older host can leave a Uai app container behind after its row changes, and
4215
+ * that orphan can still mutate the host-wide runtime volume.
3551
4216
  */
3552
- async function repairNodeDataRoot(containerName: string): Promise<boolean> {
3553
- const res = await dockerCli([
3554
- "exec",
3555
- "-u",
3556
- "root",
3557
- containerName,
4217
+ async function listRuntimeWriterCandidates(): Promise<
4218
+ | { kind: "ready"; candidates: RuntimeWriterCandidate[]; ambiguous: boolean }
4219
+ | { kind: "unreachable" }
4220
+ > {
4221
+ const listed = await dockerCli(
4222
+ [
4223
+ "ps",
4224
+ "--all",
4225
+ "--filter",
4226
+ "label=com.docker.compose.project",
4227
+ "--filter",
4228
+ "label=com.docker.compose.service=app",
4229
+ "--format",
4230
+ RUNTIME_WRITER_LIST_FORMAT,
4231
+ ],
4232
+ { timeoutMs: 30_000, maxOutputBytes: RUNTIME_WRITER_SCAN_OUTPUT_BYTES },
4233
+ );
4234
+ if (listed.status !== 0 || listed.outputTruncated) {
4235
+ return { kind: "unreachable" };
4236
+ }
4237
+
4238
+ const candidates = new Map<string, RuntimeWriterCandidate>();
4239
+ let ambiguous = false;
4240
+ for (const rawLine of listed.stdout.split(/\r?\n/)) {
4241
+ if (!rawLine) continue;
4242
+ const fields = rawLine.split("\t");
4243
+ if (fields.length !== 4) {
4244
+ ambiguous = true;
4245
+ continue;
4246
+ }
4247
+ // A preview sidecar is a known non-writer, not an ambiguity.
4248
+ if (fields[3] !== "") continue;
4249
+ let name: unknown;
4250
+ try {
4251
+ name = JSON.parse(fields[0]!);
4252
+ } catch {
4253
+ ambiguous = true;
4254
+ continue;
4255
+ }
4256
+ const candidate = runtimeWriterCandidate(name, fields[1]!, fields[2]!);
4257
+ if (candidate === "unrelated") continue;
4258
+ if (candidate === "invalid") {
4259
+ ambiguous = true;
4260
+ continue;
4261
+ }
4262
+ if (candidates.has(candidate.containerName)) {
4263
+ ambiguous = true;
4264
+ continue;
4265
+ }
4266
+ candidates.set(candidate.containerName, candidate);
4267
+ }
4268
+ return { kind: "ready", candidates: [...candidates.values()], ambiguous };
4269
+ }
4270
+
4271
+ async function inspectRuntimeWriterContainer(
4272
+ candidate: RuntimeWriterCandidate,
4273
+ target: string = candidate.containerName,
4274
+ ): Promise<RuntimeWriterInspection> {
4275
+ const inspected = await dockerCli(
4276
+ ["inspect", "--format", RUNTIME_WRITER_INSPECT_FORMAT, target],
4277
+ { timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
4278
+ );
4279
+ if (inspected.status !== 0 || inspected.outputTruncated) {
4280
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
4281
+ return {
4282
+ kind: "failed",
4283
+ detail:
4284
+ inspected.stderr.trim() ||
4285
+ `could not inspect Uai app container ${candidate.containerName}`,
4286
+ };
4287
+ }
4288
+
4289
+ const fields = inspected.stdout.trimEnd().split(/\r?\n/);
4290
+ if (fields.length !== 10) {
4291
+ return { kind: "failed", detail: "Docker returned malformed app metadata" };
4292
+ }
4293
+ let parsed: unknown[];
4294
+ try {
4295
+ parsed = fields.map((field) => JSON.parse(field));
4296
+ } catch {
4297
+ return { kind: "failed", detail: "Docker returned malformed app metadata" };
4298
+ }
4299
+ const [
4300
+ containerId,
4301
+ containerName,
4302
+ running,
4303
+ restarting,
4304
+ paused,
4305
+ restartPolicy,
4306
+ composeProject,
4307
+ composeService,
4308
+ mounts,
4309
+ environment,
4310
+ ] = parsed;
4311
+ if (
4312
+ typeof containerId !== "string" ||
4313
+ !RUNTIME_WRITER_CONTAINER_ID_RE.test(containerId) ||
4314
+ containerName !== `/${candidate.containerName}` ||
4315
+ typeof running !== "boolean" ||
4316
+ typeof restarting !== "boolean" ||
4317
+ typeof paused !== "boolean" ||
4318
+ typeof restartPolicy !== "string" ||
4319
+ composeProject !== candidate.composeProject ||
4320
+ composeService !== "app" ||
4321
+ !Array.isArray(mounts)
4322
+ ) {
4323
+ return {
4324
+ kind: "failed",
4325
+ detail: `Docker returned ambiguous identity/state metadata for ${candidate.containerName}`,
4326
+ };
4327
+ }
4328
+
4329
+ if (
4330
+ !mounts.every(
4331
+ (mount) => mount && typeof mount === "object" && !Array.isArray(mount),
4332
+ )
4333
+ ) {
4334
+ return {
4335
+ kind: "failed",
4336
+ detail: `Docker returned malformed mount metadata for ${candidate.containerName}`,
4337
+ };
4338
+ }
4339
+ const mountRecords = mounts as Array<Record<string, unknown>>;
4340
+ const asdfMounts = mountRecords.filter(
4341
+ (mount) => mount.Destination === "/opt/asdf-data",
4342
+ );
4343
+ const namedSharedMounts = mountRecords.filter(
4344
+ (mount) => mount.Name === "uai-asdf-data",
4345
+ );
4346
+ let sharedMount: RuntimeWriterMetadata["sharedMount"];
4347
+ if (
4348
+ namedSharedMounts.some(
4349
+ (mount) => mount.Type === "volume" && mount.RW === true,
4350
+ )
4351
+ ) {
4352
+ // The volume name, not its destination, is the shared writer authority.
4353
+ // An orphan can mount it at an arbitrary path and still race maintenance.
4354
+ sharedMount = "writable";
4355
+ } else if (
4356
+ namedSharedMounts.some(
4357
+ (mount) => mount.Type !== "volume" || typeof mount.RW !== "boolean",
4358
+ )
4359
+ ) {
4360
+ sharedMount = "invalid";
4361
+ } else if (asdfMounts.length === 0) {
4362
+ sharedMount = "absent";
4363
+ } else if (
4364
+ asdfMounts.length === 1 &&
4365
+ namedSharedMounts.length === 1 &&
4366
+ asdfMounts[0]!.Type === "volume" &&
4367
+ asdfMounts[0]!.Name === "uai-asdf-data" &&
4368
+ asdfMounts[0]!.RW === false
4369
+ ) {
4370
+ sharedMount = "read-only";
4371
+ } else {
4372
+ sharedMount = "invalid";
4373
+ }
4374
+ const environmentIsStringArray =
4375
+ Array.isArray(environment) &&
4376
+ environment.every((entry) => typeof entry === "string");
4377
+ const environmentEntries = environmentIsStringArray
4378
+ ? (environment as string[])
4379
+ : [];
4380
+ const exactValues = new Map<string, string[]>();
4381
+ let taskEnvironment: RuntimeWriterMetadata["taskEnvironment"] =
4382
+ environmentIsStringArray ? "current" : "invalid";
4383
+ for (const entry of environmentEntries) {
4384
+ const divider = entry.indexOf("=");
4385
+ const key = divider < 0 ? entry : entry.slice(0, divider);
4386
+ const value = divider < 0 ? "" : entry.slice(divider + 1);
4387
+ if (divider < 1) taskEnvironment = "invalid";
4388
+ const normalizedNpmKey = normalizeNpmConfigEnvKey(key);
4389
+ if (
4390
+ normalizedNpmKey !== undefined &&
4391
+ TASK_OWNED_NORMALIZED_NPM_ENV_KEYS.has(normalizedNpmKey) &&
4392
+ !TASK_OWNED_ENV.has(key)
4393
+ ) {
4394
+ taskEnvironment = "invalid";
4395
+ }
4396
+ const values = exactValues.get(key) ?? [];
4397
+ values.push(value);
4398
+ exactValues.set(key, values);
4399
+ }
4400
+ for (const [key, expected] of TASK_OWNED_ENV) {
4401
+ const values = exactValues.get(key);
4402
+ if (values?.length !== 1 || values[0] !== expected) {
4403
+ taskEnvironment = "invalid";
4404
+ }
4405
+ }
4406
+ // Keep an explicit assertion for the trust-precedence property even though
4407
+ // PATH is also part of TASK_OWNED_ENV above.
4408
+ if (exactValues.get("PATH")?.[0] !== TASK_RUNTIME_PATH) taskEnvironment = "invalid";
4409
+ return {
4410
+ kind: "ready",
4411
+ metadata: {
4412
+ ...candidate,
4413
+ containerId,
4414
+ running,
4415
+ restarting,
4416
+ paused,
4417
+ restartPolicy,
4418
+ sharedMount,
4419
+ taskEnvironment,
4420
+ },
4421
+ };
4422
+ }
4423
+
4424
+ /** Exact single-task form used by duplicate taskUp admission. */
4425
+ export async function inspectTaskRuntimeContainer(
4426
+ taskId: string,
4427
+ composeProject: string,
4428
+ ): Promise<TaskRuntimeContainerInspection> {
4429
+ if (!isSafeHostTaskId(taskId) || composeProject !== `task-${taskId}`) {
4430
+ return { kind: "failed", detail: "invalid task/Compose runtime identity" };
4431
+ }
4432
+ const inspected = await inspectRuntimeWriterContainer({
4433
+ taskId,
4434
+ composeProject,
4435
+ containerName: `${composeProject}-app-1`,
4436
+ });
4437
+ if (inspected.kind !== "ready") return inspected;
4438
+ return {
4439
+ kind: "ready",
4440
+ containerId: inspected.metadata.containerId,
4441
+ running: inspected.metadata.running,
4442
+ restarting: inspected.metadata.restarting,
4443
+ paused: inspected.metadata.paused,
4444
+ sharedMount: inspected.metadata.sharedMount,
4445
+ taskEnvironment: inspected.metadata.taskEnvironment,
4446
+ };
4447
+ }
4448
+
4449
+ function restartPolicyDisabled(name: string): boolean {
4450
+ return name === "" || name === "no";
4451
+ }
4452
+
4453
+ async function stopRuntimeWriter(
4454
+ metadata: RuntimeWriterMetadata,
4455
+ ): Promise<"stopped" | "failed" | "unreachable"> {
4456
+ // A preserved legacy container must not spring back up under Docker's own
4457
+ // restart policy after the host has declared it stopped. Mount mode is
4458
+ // immutable, so only explicit taskUp may recreate it with the L5 contract.
4459
+ await dockerCli(["update", "--restart=no", metadata.containerId], {
4460
+ timeoutMs: 30_000,
4461
+ maxOutputBytes: 1024 * 1024,
4462
+ });
4463
+ let current = await inspectRuntimeWriterContainer(
4464
+ metadata,
4465
+ metadata.containerId,
4466
+ );
4467
+ if (current.kind !== "ready") return current.kind;
4468
+ if (
4469
+ current.metadata.containerId !== metadata.containerId ||
4470
+ !restartPolicyDisabled(current.metadata.restartPolicy)
4471
+ ) {
4472
+ return "failed";
4473
+ }
4474
+
4475
+ if (current.metadata.paused) {
4476
+ // Docker refuses ordinary stop/kill operations for some paused states.
4477
+ // The restart policy is already disabled; unpause the exact immutable id,
4478
+ // re-prove identity/policy, then immediately enter the bounded stop path.
4479
+ await dockerCli(["unpause", metadata.containerId], {
4480
+ timeoutMs: 15_000,
4481
+ maxOutputBytes: 1024 * 1024,
4482
+ });
4483
+ current = await inspectRuntimeWriterContainer(
4484
+ metadata,
4485
+ metadata.containerId,
4486
+ );
4487
+ if (current.kind !== "ready") return current.kind;
4488
+ if (
4489
+ current.metadata.containerId !== metadata.containerId ||
4490
+ !restartPolicyDisabled(current.metadata.restartPolicy) ||
4491
+ current.metadata.paused
4492
+ ) {
4493
+ return "failed";
4494
+ }
4495
+ }
4496
+
4497
+ if (!current.metadata.running && !current.metadata.restarting) {
4498
+ return "stopped";
4499
+ }
4500
+ await dockerCli(["stop", "--time", "10", metadata.containerId], {
4501
+ timeoutMs: 30_000,
4502
+ maxOutputBytes: 1024 * 1024,
4503
+ });
4504
+ current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
4505
+ if (current.kind !== "ready") return current.kind;
4506
+ if (
4507
+ current.metadata.containerId === metadata.containerId &&
4508
+ restartPolicyDisabled(current.metadata.restartPolicy) &&
4509
+ !current.metadata.running &&
4510
+ !current.metadata.restarting
4511
+ ) {
4512
+ return "stopped";
4513
+ }
4514
+
4515
+ await dockerCli(["kill", metadata.containerId], {
4516
+ timeoutMs: 15_000,
4517
+ maxOutputBytes: 1024 * 1024,
4518
+ });
4519
+ current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
4520
+ if (current.kind !== "ready") return current.kind;
4521
+ return current.metadata.containerId === metadata.containerId &&
4522
+ restartPolicyDisabled(current.metadata.restartPolicy) &&
4523
+ !current.metadata.running &&
4524
+ !current.metadata.restarting
4525
+ ? "stopped"
4526
+ : "failed";
4527
+ }
4528
+
4529
+ /** Stop the exact container previously inspected by taskUp. Re-inspection by
4530
+ * immutable id closes the name-reuse gap before any shared-volume writer. */
4531
+ export async function quarantineTaskRuntimeContainer(options: {
4532
+ taskId: string;
4533
+ composeProject: string;
4534
+ expectedContainerId: string;
4535
+ }): Promise<"stopped" | "failed" | "unreachable"> {
4536
+ if (
4537
+ !isSafeHostTaskId(options.taskId) ||
4538
+ options.composeProject !== `task-${options.taskId}` ||
4539
+ !RUNTIME_WRITER_CONTAINER_ID_RE.test(options.expectedContainerId)
4540
+ ) {
4541
+ return "failed";
4542
+ }
4543
+ const candidate: RuntimeWriterCandidate = {
4544
+ taskId: options.taskId,
4545
+ composeProject: options.composeProject,
4546
+ containerName: `${options.composeProject}-app-1`,
4547
+ };
4548
+ const inspected = await inspectRuntimeWriterContainer(
4549
+ candidate,
4550
+ options.expectedContainerId,
4551
+ );
4552
+ if (inspected.kind !== "ready") return inspected.kind;
4553
+ if (inspected.metadata.containerId !== options.expectedContainerId) {
4554
+ return "failed";
4555
+ }
4556
+ return stopRuntimeWriter(inspected.metadata);
4557
+ }
4558
+
4559
+ /**
4560
+ * L5 rolling fence. It must complete before the first standard-image/runtime
4561
+ * volume writer of an activation generation. New task containers mount the
4562
+ * shared volume read-only; every preserved legacy writer is tombstoned and
4563
+ * stopped, while an exact RO container remains eligible for recovery.
4564
+ */
4565
+ export async function quarantineWritableRuntimeContainers(options: {
4566
+ lifecycle: Orchestrator;
4567
+ isCurrent?: () => boolean;
4568
+ /** `contract` (default) sweeps only containers whose shared-volume
4569
+ * attachment violates the current contract. `volume-holders` sweeps EVERY
4570
+ * task container attached to the shared volume — the activation driver's
4571
+ * non-writer escape when the volume needs a repair the holders block
4572
+ * (ADR-108; apple only, a no-op on Docker). */
4573
+ scope?: "contract" | "volume-holders";
4574
+ }): Promise<WritableRuntimeQuarantineResult> {
4575
+ // Dispatch on the PIN: this sweep runs inside activation's
4576
+ // beforeMaintenance, where the public state is still `checking` and the
4577
+ // ready-gated identity is null — the docker fallback then retries forever
4578
+ // against a daemon that does not exist on a Docker-free Mac.
4579
+ if (pinnedContainerRuntimeProvider() === "apple-container") {
4580
+ return quarantineWritableAppleRuntimeContainers(options);
4581
+ }
4582
+ const isCurrent = options.isCurrent ?? (() => true);
4583
+ const stoppedTaskIds = new Set<string>();
4584
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
4585
+
4586
+ const listed = await listRuntimeWriterCandidates();
4587
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
4588
+ if (listed.kind === "unreachable") {
4589
+ return { verdict: "retry", stoppedTaskIds: [] };
4590
+ }
4591
+ let resolved = !listed.ambiguous;
4592
+
4593
+ for (const candidate of listed.candidates) {
4594
+ if (!isCurrent()) {
4595
+ return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
4596
+ }
4597
+ await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
4598
+ if (!isCurrent()) return;
4599
+ const inspection = await inspectRuntimeWriterContainer(candidate);
4600
+ if (inspection.kind !== "ready") {
4601
+ resolved = false;
4602
+ console.error(
4603
+ `[orchestrator] runtime isolation: ${candidate.containerName} inspection did not prove a safe mount (${inspection.kind === "failed" ? inspection.detail : "Docker unreachable"})`,
4604
+ );
4605
+ return;
4606
+ }
4607
+ const task = getHostTask(candidate.taskId);
4608
+ const knownPreservedTask =
4609
+ task?.composeProject === candidate.composeProject &&
4610
+ typeof task.statusMirror === "string" &&
4611
+ isOnDashboard(task.statusMirror);
4612
+ if (
4613
+ inspection.metadata.sharedMount === "read-only" &&
4614
+ inspection.metadata.taskEnvironment === "current"
4615
+ ) {
4616
+ return;
4617
+ }
4618
+ if (!knownPreservedTask) {
4619
+ // A task-shaped orphan with a proven RO shared mount cannot race
4620
+ // maintenance. Without a host row, its non-current task-local npm
4621
+ // environment is not enough ownership evidence to mutate it.
4622
+ if (inspection.metadata.sharedMount === "read-only") return;
4623
+ if (inspection.metadata.sharedMount === "absent") return;
4624
+ if (inspection.metadata.sharedMount === "invalid") {
4625
+ // `task-*` is Uai's namespace, but it is not by itself permission to
4626
+ // stop an unrelated/stale user stack. Only the exact named writable
4627
+ // Uai volume is positive ownership evidence without a host row.
4628
+ resolved = false;
4629
+ console.error(
4630
+ `[orchestrator] runtime isolation: orphan ${candidate.containerName} has ambiguous shared-runtime mount metadata; maintenance remains fenced`,
4631
+ );
4632
+ return;
4633
+ }
4634
+ }
4635
+
4636
+ // Drain before the first stop/update. A running agent continuation may
4637
+ // otherwise race the quarantine and respawn into the legacy container.
4638
+ await options.lifecycle.quarantineChannelForRecovery(candidate.taskId);
4639
+ const stopped = await stopRuntimeWriter(inspection.metadata);
4640
+ if (stopped !== "stopped") {
4641
+ resolved = false;
4642
+ console.error(
4643
+ `[orchestrator] runtime isolation: could not prove ${candidate.containerName} stopped (${stopped})`,
4644
+ );
4645
+ return;
4646
+ }
4647
+
4648
+ if (knownPreservedTask) {
4649
+ // Always re-publish the stopped verdict. A prior process may have
4650
+ // committed SQLite and crashed before its websocket frame left; the
4651
+ // cloud could otherwise remain ghost-running indefinitely.
4652
+ stoppedTaskIds.add(candidate.taskId);
4653
+ db_setStatus(candidate.taskId, "stopped", {
4654
+ codeServerPort: null,
4655
+ previewPorts: "[]",
4656
+ });
4657
+ }
4658
+ clearTaskGatewayAcl(candidate.taskId);
4659
+ // We deliberately reflect a proven physical stop even if the daemon
4660
+ // epoch changed while stopping: leaving the row active would invite a
4661
+ // later recovery to restart an immutable legacy writer. The next
4662
+ // activation generation independently re-enumerates Docker before
4663
+ // maintenance. Non-safety cleanup below may wait for that generation.
4664
+ if (!isCurrent()) return;
4665
+ try {
4666
+ await stopPreviewSidecars(candidate.taskId);
4667
+ } catch (error) {
4668
+ console.warn(
4669
+ `[orchestrator] runtime isolation: ${candidate.taskId} preview cleanup failed: ${
4670
+ error instanceof Error ? error.message : String(error)
4671
+ }`,
4672
+ );
4673
+ }
4674
+ console.warn(
4675
+ `[orchestrator] runtime isolation: ${candidate.containerName} used a writable/non-current shared runtime mount and remains stopped until explicit Resume recreates it`,
4676
+ );
4677
+ });
4678
+ }
4679
+
4680
+ if (!isCurrent()) {
4681
+ return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
4682
+ }
4683
+ return {
4684
+ verdict: resolved ? "ready" : "retry",
4685
+ stoppedTaskIds: [...stoppedTaskIds].sort(),
4686
+ };
4687
+ }
4688
+
4689
+ const RECOVERY_RUNTIME_INSPECT_FORMAT =
4690
+ "{{json .Image}}|{{json .State.Running}}|{{json .HostConfig.GroupAdd}}|{{json .Mounts}}";
4691
+ const RECOVERY_RUNTIME_IMAGE_RE = /^sha256:[0-9a-f]{64}$/;
4692
+ const RECOVERY_RUNTIME_SCRIPT_PATH =
4693
+ "/usr/local/bin/uai-materialize-runtimes";
4694
+ const RECOVERY_COREPACK_VERSION_PATH =
4695
+ "/usr/local/share/uai/corepack-version";
4696
+ const RECOVERY_INIT_PATH = "/uai-init-current";
4697
+ const RECOVERY_INIT_PROJECTS_PATH = "/uai-runtime-projects-current";
4698
+ const RECOVERY_RUNTIME_TIMEOUT_MS = 630_000;
4699
+ const RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES = 1024 * 1024;
4700
+
4701
+ interface RecoveryDockerMount {
4702
+ Type?: unknown;
4703
+ Name?: unknown;
4704
+ Source?: unknown;
4705
+ Destination?: unknown;
4706
+ RW?: unknown;
4707
+ }
4708
+
4709
+ interface RecoveryRuntimeSpec {
4710
+ imageId: string;
4711
+ workspaceSource: string;
4712
+ groupAdds: string[];
4713
+ projectCount: number;
4714
+ projectAllowlistSource: string;
4715
+ }
4716
+
4717
+ type RecoveryRuntimeInspection =
4718
+ | { kind: "ready"; spec: RecoveryRuntimeSpec }
4719
+ | { kind: "failed"; detail: string }
4720
+ | { kind: "unreachable" };
4721
+
4722
+ type RecoveryRuntimeResult =
4723
+ | { kind: "ready" }
4724
+ | { kind: "failed"; detail: string }
4725
+ | { kind: "unreachable" };
4726
+
4727
+ function recoveryRuntimeAssets():
4728
+ | { materializer: string; corepackVersion: string; uaiInit: string }
4729
+ | { error: string } {
4730
+ const materializer = standardImageRuntimeMaterializerPath();
4731
+ const corepackVersion = standardImageCorepackVersionPath();
4732
+ const uaiInit = join(dirname(materializer), "uai-init");
4733
+ try {
4734
+ const materializerStat = lstatSync(materializer);
4735
+ if (!materializerStat.isFile() || materializerStat.isSymbolicLink() ||
4736
+ (materializerStat.mode & 0o111) === 0) {
4737
+ return { error: `packaged materializer is not a regular executable: ${materializer}` };
4738
+ }
4739
+ const versionStat = lstatSync(corepackVersion);
4740
+ if (!versionStat.isFile() || versionStat.isSymbolicLink()) {
4741
+ return { error: `packaged Corepack pin is not a regular file: ${corepackVersion}` };
4742
+ }
4743
+ const version = readFileSync(corepackVersion, "utf8");
4744
+ if (!/^\d+\.\d+\.\d+\n?$/.test(version)) {
4745
+ return { error: `packaged Corepack pin is malformed: ${corepackVersion}` };
4746
+ }
4747
+ const initStat = lstatSync(uaiInit);
4748
+ if (
4749
+ !initStat.isFile() ||
4750
+ initStat.isSymbolicLink() ||
4751
+ (initStat.mode & 0o111) === 0
4752
+ ) {
4753
+ return { error: `packaged task initializer is not a regular executable: ${uaiInit}` };
4754
+ }
4755
+ } catch (err) {
4756
+ return {
4757
+ error: `packaged runtime assets are unavailable: ${
4758
+ err instanceof Error ? err.message : String(err)
4759
+ }`,
4760
+ };
4761
+ }
4762
+ return { materializer, corepackVersion, uaiInit };
4763
+ }
4764
+
4765
+ function expectedRecoveryWorkspaceSources(
4766
+ task: typeof schema.hostTasks.$inferSelect,
4767
+ ): Set<string> {
4768
+ const expected = new Set<string>();
4769
+ if (!task.worktreePath) return expected;
4770
+ const lexical = join(task.worktreePath, "workspace");
4771
+ try {
4772
+ const stat = lstatSync(lexical);
4773
+ if (!stat.isDirectory() || stat.isSymbolicLink()) return expected;
4774
+ expected.add(lexical);
4775
+ expected.add(realpathSync(lexical));
4776
+ } catch {
4777
+ // Missing, unreadable, and unresolvable workspaces are never safe input
4778
+ // to `docker run`: a legacy `-v` bind would recreate a missing source as
4779
+ // an empty directory and falsely prove that the task had no projects.
4780
+ }
4781
+ return expected;
4782
+ }
4783
+
4784
+ function validRecoveryGroupAdds(value: unknown): string[] | null {
4785
+ if (value === null || value === undefined) return [];
4786
+ if (!Array.isArray(value)) return null;
4787
+ const groups: string[] = [];
4788
+ for (const candidate of value) {
4789
+ if (typeof candidate !== "string" || !/^[1-9]\d{0,9}$/.test(candidate)) {
4790
+ return null;
4791
+ }
4792
+ const numeric = Number(candidate);
4793
+ if (!Number.isSafeInteger(numeric) || numeric > 4_294_967_294) {
4794
+ return null;
4795
+ }
4796
+ if (!groups.includes(candidate)) groups.push(candidate);
4797
+ }
4798
+ return groups;
4799
+ }
4800
+
4801
+ function safeRecoveryProjectSlug(slug: unknown): slug is string {
4802
+ return typeof slug === "string" && /^[a-z0-9-]{1,64}$/.test(slug);
4803
+ }
4804
+
4805
+ function validateRecoveryProjects(
4806
+ task: typeof schema.hostTasks.$inferSelect,
4807
+ workspaceSource: string,
4808
+ ): { slugs: string[] } | { error: string } {
4809
+ let parsed: unknown;
4810
+ try {
4811
+ parsed = JSON.parse(task.projectSlugs);
4812
+ } catch {
4813
+ return { error: "the persisted project slug list is invalid JSON" };
4814
+ }
4815
+ if (!Array.isArray(parsed) || !parsed.every(safeRecoveryProjectSlug)) {
4816
+ return { error: "the persisted project slug list contains an unsafe entry" };
4817
+ }
4818
+ const slugs = parsed as string[];
4819
+ const expected = new Set(slugs);
4820
+ if (expected.size !== slugs.length) {
4821
+ return { error: "the persisted project slug list contains duplicates" };
4822
+ }
4823
+
4824
+ for (const slug of expected) {
4825
+ const project = join(workspaceSource, slug);
4826
+ let projectStat: ReturnType<typeof lstatSync>;
4827
+ let markerStat: ReturnType<typeof lstatSync>;
4828
+ try {
4829
+ projectStat = lstatSync(project);
4830
+ markerStat = lstatSync(join(project, ".git"));
4831
+ } catch {
4832
+ return { error: `expected project ${slug} or its .git marker is missing` };
4833
+ }
4834
+ if (
4835
+ projectStat.isSymbolicLink() ||
4836
+ !projectStat.isDirectory() ||
4837
+ markerStat.isSymbolicLink() ||
4838
+ (!markerStat.isFile() && !markerStat.isDirectory())
4839
+ ) {
4840
+ return { error: `expected project ${slug} is not a safe Git worktree` };
4841
+ }
4842
+ }
4843
+ return { slugs };
4844
+ }
4845
+
4846
+ function validateRecoveryWorkspaceRuntimeRoot(
4847
+ workspaceSource: string,
4848
+ ): string | null {
4849
+ const shadow = join(workspaceSource, ".tool-versions");
4850
+ try {
4851
+ // Any type is forbidden, including regular files and dangling symlinks.
4852
+ // lstat is metadata-only, so FIFO/device/socket entries cannot block the
4853
+ // host recovery loop.
4854
+ lstatSync(shadow);
4855
+ return "the workspace root contains a forbidden .tool-versions runtime override";
4856
+ } catch (err) {
4857
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
4858
+ return `the workspace-root runtime authority could not be verified: ${
4859
+ err instanceof Error ? err.message : String(err)
4860
+ }`;
4861
+ }
4862
+ }
4863
+
4864
+ function recoveryRuntimeProjectAllowlist(
4865
+ task: typeof schema.hostTasks.$inferSelect,
4866
+ slugs: string[],
4867
+ ): { path: string } | { error: string } {
4868
+ if (!task.worktreePath) {
4869
+ return { error: "the task has no persisted runtime directory" };
4870
+ }
4871
+ const runtimeDir = join(task.worktreePath, ".uai");
4872
+ const target = join(runtimeDir, "runtime-projects");
4873
+ const expected = slugs.length > 0 ? `${slugs.join("\n")}\n` : "";
4874
+ try {
4875
+ const runtimeDirStat = lstatSync(runtimeDir);
4876
+ if (!runtimeDirStat.isDirectory() || runtimeDirStat.isSymbolicLink()) {
4877
+ return { error: "the task runtime directory is not a safe directory" };
4878
+ }
4879
+ try {
4880
+ const targetStat = lstatSync(target);
4881
+ if (!targetStat.isFile() || targetStat.isSymbolicLink()) {
4882
+ return { error: "the task runtime-project allowlist is not a regular file" };
4883
+ }
4884
+ if (readFileSync(target, "utf8") !== expected) {
4885
+ return { error: "the task runtime-project allowlist does not match persisted projects" };
4886
+ }
4887
+ return { path: target };
4888
+ } catch (err) {
4889
+ if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
4890
+ }
4891
+
4892
+ if (slugs.length === 0) {
4893
+ return {
4894
+ error: "zero-project recovery lacks an existing authoritative empty runtime-project allowlist",
4895
+ };
4896
+ }
4897
+
4898
+ const stagingDir = mkdtempSync(join(runtimeDir, ".runtime-projects-"));
4899
+ const staged = join(stagingDir, "runtime-projects");
4900
+ try {
4901
+ writeFileSync(staged, expected, { encoding: "utf8", flag: "wx", mode: 0o644 });
4902
+ renameSync(staged, target);
4903
+ } finally {
4904
+ rmSync(stagingDir, { recursive: true, force: true });
4905
+ }
4906
+ return { path: target };
4907
+ } catch (err) {
4908
+ return {
4909
+ error: `the task runtime-project allowlist is unavailable: ${
4910
+ err instanceof Error ? err.message : String(err)
4911
+ }`,
4912
+ };
4913
+ }
4914
+ }
4915
+
4916
+ async function inspectRecoveryRuntime(
4917
+ task: typeof schema.hostTasks.$inferSelect,
4918
+ containerName: string,
4919
+ ): Promise<RecoveryRuntimeInspection> {
4920
+ const inspected = await dockerCli(
4921
+ [
4922
+ "inspect",
4923
+ "--format",
4924
+ RECOVERY_RUNTIME_INSPECT_FORMAT,
4925
+ containerName,
4926
+ ],
4927
+ { timeoutMs: 30_000, maxOutputBytes: 128 * 1024 },
4928
+ );
4929
+ if (inspected.status !== 0) {
4930
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
4931
+ return {
4932
+ kind: "failed",
4933
+ detail:
4934
+ inspected.stderr.trim() ||
4935
+ `could not inspect stopped app container ${containerName}`,
4936
+ };
4937
+ }
4938
+
4939
+ const firstDivider = inspected.stdout.indexOf("|");
4940
+ const secondDivider = inspected.stdout.indexOf("|", firstDivider + 1);
4941
+ const thirdDivider = inspected.stdout.indexOf("|", secondDivider + 1);
4942
+ if (firstDivider < 0 || secondDivider < 0 || thirdDivider < 0) {
4943
+ return { kind: "failed", detail: "Docker returned malformed runtime metadata" };
4944
+ }
4945
+
4946
+ let imageId: unknown;
4947
+ let running: unknown;
4948
+ let rawGroupAdds: unknown;
4949
+ let mounts: unknown;
4950
+ try {
4951
+ imageId = JSON.parse(inspected.stdout.slice(0, firstDivider));
4952
+ running = JSON.parse(
4953
+ inspected.stdout.slice(firstDivider + 1, secondDivider),
4954
+ );
4955
+ rawGroupAdds = JSON.parse(
4956
+ inspected.stdout.slice(secondDivider + 1, thirdDivider),
4957
+ );
4958
+ mounts = JSON.parse(inspected.stdout.slice(thirdDivider + 1));
4959
+ } catch {
4960
+ return { kind: "failed", detail: "Docker returned invalid runtime metadata JSON" };
4961
+ }
4962
+ if (running !== false) {
4963
+ return {
4964
+ kind: "failed",
4965
+ detail: "the app container was not proven stopped before runtime maintenance",
4966
+ };
4967
+ }
4968
+ if (typeof imageId !== "string" || !RECOVERY_RUNTIME_IMAGE_RE.test(imageId)) {
4969
+ return { kind: "failed", detail: "the app container has an invalid immutable image ID" };
4970
+ }
4971
+ if (!Array.isArray(mounts)) {
4972
+ return { kind: "failed", detail: "the app container has invalid mount metadata" };
4973
+ }
4974
+ const groupAdds = validRecoveryGroupAdds(rawGroupAdds);
4975
+ if (groupAdds === null) {
4976
+ return {
4977
+ kind: "failed",
4978
+ detail: "the app container has malformed or unsafe supplemental groups",
4979
+ };
4980
+ }
4981
+ const typedMounts = mounts as RecoveryDockerMount[];
4982
+ const workspaceMounts = typedMounts.filter(
4983
+ (mount) => mount.Destination === "/workspace",
4984
+ );
4985
+ const expectedWorkspaceSources = expectedRecoveryWorkspaceSources(task);
4986
+ if (
4987
+ workspaceMounts.length !== 1 ||
4988
+ workspaceMounts[0]?.Type !== "bind" ||
4989
+ typeof workspaceMounts[0]?.Source !== "string" ||
4990
+ !expectedWorkspaceSources.has(workspaceMounts[0].Source)
4991
+ ) {
4992
+ return {
4993
+ kind: "failed",
4994
+ detail: "the app container does not have the exact expected /workspace bind",
4995
+ };
4996
+ }
4997
+ try {
4998
+ const workspaceStat = lstatSync(workspaceMounts[0].Source);
4999
+ if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
5000
+ return {
5001
+ kind: "failed",
5002
+ detail: "the app container's /workspace source is not a real directory",
5003
+ };
5004
+ }
5005
+ } catch {
5006
+ return {
5007
+ kind: "failed",
5008
+ detail: "the app container's /workspace source no longer exists",
5009
+ };
5010
+ }
5011
+ const workspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
5012
+ workspaceMounts[0].Source,
5013
+ );
5014
+ if (workspaceRuntimeError) {
5015
+ return { kind: "failed", detail: workspaceRuntimeError };
5016
+ }
5017
+ const projects = validateRecoveryProjects(
5018
+ task,
5019
+ workspaceMounts[0].Source,
5020
+ );
5021
+ if ("error" in projects) {
5022
+ return { kind: "failed", detail: projects.error };
5023
+ }
5024
+ const projectAllowlist = recoveryRuntimeProjectAllowlist(
5025
+ task,
5026
+ projects.slugs,
5027
+ );
5028
+ if ("error" in projectAllowlist) {
5029
+ return { kind: "failed", detail: projectAllowlist.error };
5030
+ }
5031
+ const asdfMounts = typedMounts.filter(
5032
+ (mount) => mount.Destination === "/opt/asdf-data",
5033
+ );
5034
+ if (
5035
+ asdfMounts.length !== 1 ||
5036
+ asdfMounts[0]?.Type !== "volume" ||
5037
+ asdfMounts[0]?.Name !== "uai-asdf-data" ||
5038
+ asdfMounts[0]?.RW !== false
5039
+ ) {
5040
+ return {
5041
+ kind: "failed",
5042
+ detail:
5043
+ "the app container does not use the exact read-only uai-asdf-data volume; explicit Resume must recreate legacy writable containers",
5044
+ };
5045
+ }
5046
+ return {
5047
+ kind: "ready",
5048
+ spec: {
5049
+ imageId,
5050
+ workspaceSource: workspaceMounts[0].Source,
5051
+ groupAdds,
5052
+ projectCount: projects.slugs.length,
5053
+ projectAllowlistSource: projectAllowlist.path,
5054
+ },
5055
+ };
5056
+ }
5057
+
5058
+ async function materializeRecoveryRuntimes(
5059
+ spec: RecoveryRuntimeSpec,
5060
+ ): Promise<RecoveryRuntimeResult> {
5061
+ if (spec.projectCount === 0) return { kind: "ready" };
5062
+ const assets = recoveryRuntimeAssets();
5063
+ if ("error" in assets) return { kind: "failed", detail: assets.error };
5064
+ try {
5065
+ const workspaceStat = lstatSync(spec.workspaceSource);
5066
+ if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
5067
+ return {
5068
+ kind: "failed",
5069
+ detail: "the app container's /workspace source is no longer a real directory",
5070
+ };
5071
+ }
5072
+ } catch {
5073
+ return {
5074
+ kind: "failed",
5075
+ detail: "the app container's /workspace source disappeared before runtime proof",
5076
+ };
5077
+ }
5078
+ try {
5079
+ const allowlistStat = lstatSync(spec.projectAllowlistSource);
5080
+ if (!allowlistStat.isFile() || allowlistStat.isSymbolicLink()) {
5081
+ return {
5082
+ kind: "failed",
5083
+ detail: "the runtime-project allowlist is no longer a regular file",
5084
+ };
5085
+ }
5086
+ } catch {
5087
+ return {
5088
+ kind: "failed",
5089
+ detail: "the runtime-project allowlist disappeared before runtime proof",
5090
+ };
5091
+ }
5092
+ const supplementalGroups = spec.groupAdds.flatMap((group) => [
5093
+ "--group-add",
5094
+ group,
5095
+ ]);
5096
+ const result = await dockerCli(
5097
+ [
5098
+ "run",
5099
+ "--rm",
5100
+ "--user",
5101
+ "node",
5102
+ ...supplementalGroups,
5103
+ "--workdir",
5104
+ "/workspace",
5105
+ "--mount",
5106
+ `type=bind,src=${spec.workspaceSource},dst=/workspace,readonly`,
5107
+ "-v",
5108
+ "uai-asdf-data:/opt/asdf-data:rw",
5109
+ "-v",
5110
+ `${assets.materializer}:${RECOVERY_RUNTIME_SCRIPT_PATH}:ro`,
5111
+ "-v",
5112
+ `${assets.corepackVersion}:${RECOVERY_COREPACK_VERSION_PATH}:ro`,
5113
+ "-v",
5114
+ `${spec.projectAllowlistSource}:/run/uai/runtime-projects:ro`,
5115
+ "--env",
5116
+ "LD_PRELOAD=",
5117
+ "--env",
5118
+ "LD_AUDIT=",
5119
+ "--env",
5120
+ "LD_LIBRARY_PATH=",
5121
+ "--env",
5122
+ "BASH_ENV=",
5123
+ "--env",
5124
+ "ENV=",
5125
+ "--entrypoint",
5126
+ "/usr/bin/env",
5127
+ spec.imageId,
5128
+ "-i",
5129
+ "HOME=/home/node",
5130
+ "ASDF_DIR=/opt/asdf",
5131
+ "ASDF_DATA_DIR=/opt/asdf-data",
5132
+ "PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
5133
+ "/usr/bin/timeout",
5134
+ "--signal=TERM",
5135
+ "--kill-after=10s",
5136
+ "600s",
5137
+ RECOVERY_RUNTIME_SCRIPT_PATH,
5138
+ ],
5139
+ {
5140
+ timeoutMs: RECOVERY_RUNTIME_TIMEOUT_MS,
5141
+ tailOutputBytes: RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES,
5142
+ },
5143
+ );
5144
+ if (result.status === 0) return { kind: "ready" };
5145
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
5146
+ return {
5147
+ kind: "failed",
5148
+ detail: `${result.outputTruncated ? "earlier runtime output was truncated; " : ""}${
5149
+ result.stderr.trim() ||
5150
+ `runtime materializer exited ${String(result.status)}`
5151
+ }`,
5152
+ };
5153
+ }
5154
+
5155
+ async function stageRecoveryInit(
5156
+ containerName: string,
5157
+ spec: RecoveryRuntimeSpec,
5158
+ ): Promise<RecoveryRuntimeResult> {
5159
+ const assets = recoveryRuntimeAssets();
5160
+ if ("error" in assets) return { kind: "failed", detail: assets.error };
5161
+ const copies: Array<[string, string]> = [
5162
+ [assets.uaiInit, RECOVERY_INIT_PATH],
5163
+ [spec.projectAllowlistSource, RECOVERY_INIT_PROJECTS_PATH],
5164
+ ];
5165
+ for (const [source, destination] of copies) {
5166
+ const copied = await dockerCli(
5167
+ ["cp", source, `${containerName}:${destination}`],
5168
+ { timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
5169
+ );
5170
+ if (copied.status === 0) continue;
5171
+ if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
5172
+ return {
5173
+ kind: "failed",
5174
+ detail:
5175
+ copied.stderr.trim() ||
5176
+ `could not stage ${destination} in the stopped app container`,
5177
+ };
5178
+ }
5179
+ return { kind: "ready" };
5180
+ }
5181
+
5182
+ type DockerQuarantineResult = "stopped" | "failed" | "unreachable";
5183
+
5184
+ async function quarantineRunningContainer(
5185
+ containerName: string,
5186
+ ): Promise<DockerQuarantineResult> {
5187
+ const stopped = await dockerCli(
5188
+ ["stop", "--time", "10", containerName],
5189
+ { timeoutMs: 30_000 },
5190
+ );
5191
+ const inspected = await dockerCli(
5192
+ ["inspect", "--format", "{{.State.Running}}", containerName],
5193
+ { timeoutMs: 10_000 },
5194
+ );
5195
+ if (inspected.status === 0 && inspected.stdout.trim() === "false") {
5196
+ return "stopped";
5197
+ }
5198
+ if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
5199
+ return "unreachable";
5200
+ }
5201
+ const killed = await dockerCli(["kill", containerName], {
5202
+ timeoutMs: 15_000,
5203
+ });
5204
+ const killInspection = await dockerCli(
5205
+ ["inspect", "--format", "{{.State.Running}}", containerName],
5206
+ { timeoutMs: 10_000 },
5207
+ );
5208
+ if (
5209
+ killInspection.status === 0 &&
5210
+ killInspection.stdout.trim() === "false"
5211
+ ) {
5212
+ return "stopped";
5213
+ }
5214
+ if (killInspection.status !== 0 && !(await dockerDaemonReachable())) {
5215
+ return "unreachable";
5216
+ }
5217
+ console.error(
5218
+ `[orchestrator] recovery: could not quarantine ${containerName} before runtime proof: ${
5219
+ stopped.stderr.trim() ||
5220
+ inspected.stderr.trim() ||
5221
+ killed.stderr.trim() ||
5222
+ killInspection.stderr.trim() ||
5223
+ "container is still running after stop and kill"
5224
+ }`,
5225
+ );
5226
+ return "failed";
5227
+ }
5228
+
5229
+ /**
5230
+ * Repair the shared node data root before any recovered-container init.
5231
+ * Older task-up versions created the OpenCode leaf as root and accidentally
5232
+ * left this parent root-owned, which prevented code-server from creating its
5233
+ * managed User profile. `install -d` creates or repairs the exact directory
5234
+ * without recursively changing unrelated application state beneath it.
5235
+ */
5236
+ async function repairNodeDataRoot(containerName: string): Promise<boolean> {
5237
+ const res = await dockerCli([
5238
+ "exec",
5239
+ "-u",
5240
+ "root",
5241
+ ...runtimeAuthorityDockerExecArgs(),
5242
+ containerName,
3558
5243
  "/usr/bin/install",
3559
5244
  "-d",
3560
5245
  "-o",
@@ -3574,18 +5259,32 @@ async function repairNodeDataRoot(containerName: string): Promise<boolean> {
3574
5259
  return true;
3575
5260
  }
3576
5261
 
5262
+ interface DockerPortResult {
5263
+ reachable: boolean;
5264
+ port: number | null;
5265
+ }
5266
+
3577
5267
  async function dockerPort(
3578
5268
  containerName: string,
3579
5269
  containerPort: number,
3580
- ): Promise<number | null> {
5270
+ ): Promise<DockerPortResult> {
3581
5271
  const res = await dockerCli(["port", containerName, String(containerPort)]);
3582
- if (res.status !== 0) return null;
5272
+ if (res.status === null) return { reachable: false, port: null };
5273
+ if (res.status !== 0) {
5274
+ return {
5275
+ reachable: await dockerDaemonReachable(),
5276
+ port: null,
5277
+ };
5278
+ }
3583
5279
  // Output: "127.0.0.1:32785\n"
3584
5280
  const firstLine = res.stdout.split("\n")[0]?.trim() ?? "";
3585
5281
  const colonIdx = firstLine.lastIndexOf(":");
3586
- if (colonIdx === -1) return null;
5282
+ if (colonIdx === -1) return { reachable: true, port: null };
3587
5283
  const port = Number(firstLine.slice(colonIdx + 1));
3588
- return Number.isFinite(port) ? port : null;
5284
+ return {
5285
+ reachable: true,
5286
+ port: Number.isFinite(port) ? port : null,
5287
+ };
3589
5288
  }
3590
5289
 
3591
5290
  async function dockerExec(
@@ -3594,102 +5293,952 @@ async function dockerExec(
3594
5293
  timeoutMs?: number,
3595
5294
  ): Promise<boolean> {
3596
5295
  const res = await dockerCli(
3597
- ["exec", containerName, ...cmd],
5296
+ ["exec", ...runtimeAuthorityDockerExecArgs(), containerName, ...cmd],
3598
5297
  timeoutMs === undefined ? {} : { timeoutMs },
3599
5298
  );
3600
5299
  return res.status === 0;
3601
5300
  }
3602
5301
 
3603
- /** Exported for tests; production entry is the getOrchestrator() boot guard. */
3604
- export async function recoverRunningTasks(opts?: {
3605
- /** Delay between passes while docker is unreachable (tests shrink it). */
3606
- retryMs?: number;
3607
- /** Bound on passes; ~15 min at the default cadence covers slow boots. */
3608
- maxPasses?: number;
3609
- }): Promise<void> {
3610
- const retryMs = opts?.retryMs ?? 20_000;
3611
- const maxPasses = opts?.maxPasses ?? 45;
3612
- let pendingTaskIds: ReadonlySet<string> | undefined;
3613
- // After a machine reboot the host service and the Docker daemon start
3614
- // concurrently; a pass that finds docker unreachable resolves nothing and
3615
- // must retry once the daemon is up — the whole point of recovery is to
3616
- // restart the exited containers that reboot left behind.
3617
- for (let pass = 1; ; pass++) {
3618
- const deferred = await recoveryPass(pendingTaskIds);
3619
- if (deferred.size === 0) {
3620
- finishAllTaskRecovery();
3621
- return;
5302
+ /** Exported for tests; production entry is the getOrchestrator() boot guard. */
5303
+ export async function recoverRunningTasks(opts?: {
5304
+ /** Delay between passes while docker is unreachable (tests shrink it). */
5305
+ retryMs?: number;
5306
+ /** Bound on passes; ~15 min at the default cadence covers slow boots. */
5307
+ maxPasses?: number;
5308
+ /** Post-boot recovery must queue behind live taskUp/taskDown operations.
5309
+ * Boot recovery omits this and uses its earlier per-task barriers instead. */
5310
+ lifecycle?: Orchestrator;
5311
+ /** Test seam; production waits for the shared image/CLI-volume reconcile. */
5312
+ maintenanceReady?: Promise<void>;
5313
+ /** A daemon-generation fence. False cancels every remaining side effect. */
5314
+ isCurrent?: () => boolean;
5315
+ }): Promise<boolean> {
5316
+ const retryMs = opts?.retryMs ?? 20_000;
5317
+ const maxPasses = opts?.maxPasses ?? 45;
5318
+ let pendingTaskIds: ReadonlySet<string> | undefined;
5319
+ // After a machine reboot the host service and its selected environment
5320
+ // provider start concurrently. Unknown provider state resolves nothing and
5321
+ // must retry once the runtime answers.
5322
+ for (let pass = 1; ; pass++) {
5323
+ if (opts?.isCurrent?.() === false) {
5324
+ if (!opts.lifecycle) finishAllTaskRecovery();
5325
+ return false;
5326
+ }
5327
+ const result = await recoveryPass(
5328
+ pendingTaskIds,
5329
+ opts?.lifecycle,
5330
+ opts?.maintenanceReady ?? agentClisReady,
5331
+ opts?.isCurrent ?? (() => true),
5332
+ );
5333
+ if (opts?.isCurrent?.() === false) {
5334
+ if (!opts.lifecycle) finishAllTaskRecovery();
5335
+ return false;
5336
+ }
5337
+ const { deferred } = result;
5338
+ if (!result.resolved) {
5339
+ if (!opts?.lifecycle) finishAllTaskRecovery();
5340
+ return false;
5341
+ }
5342
+ if (deferred.size === 0) {
5343
+ if (!opts?.lifecycle) finishAllTaskRecovery();
5344
+ return true;
5345
+ }
5346
+ pendingTaskIds = deferred;
5347
+ if (pass >= maxPasses) {
5348
+ console.warn(
5349
+ `[orchestrator] recovery: task environment runtime still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
5350
+ );
5351
+ if (!opts?.lifecycle) finishAllTaskRecovery();
5352
+ return false;
5353
+ }
5354
+ if (pass === 1) {
5355
+ console.log(
5356
+ `[orchestrator] recovery: task environment runtime not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
5357
+ );
5358
+ }
5359
+ await new Promise((resolve) => setTimeout(resolve, retryMs));
5360
+ }
5361
+ }
5362
+
5363
+ interface RecoveryPassResult {
5364
+ deferred: Set<string>;
5365
+ resolved: boolean;
5366
+ }
5367
+
5368
+ /** One recovery sweep. Returns only task ids deferred by an unreachable provider;
5369
+ * successful ids are never revisited by a later retry pass. `resolved` is
5370
+ * false for an unexpected failure whose runtime state could not be proved. */
5371
+ async function recoveryPass(
5372
+ onlyTaskIds?: ReadonlySet<string>,
5373
+ lifecycle?: Orchestrator,
5374
+ maintenanceReady: Promise<void> = agentClisReady,
5375
+ isCurrent: () => boolean = () => true,
5376
+ ): Promise<RecoveryPassResult> {
5377
+ const deferred = new Set<string>();
5378
+ let resolved = true;
5379
+ try {
5380
+ if (!isCurrent()) return { deferred, resolved: false };
5381
+ const db = getDb();
5382
+ const activeRows = db
5383
+ .select()
5384
+ .from(schema.hostTasks)
5385
+ .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
5386
+ .all();
5387
+ const rows = onlyTaskIds
5388
+ ? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
5389
+ : activeRows;
5390
+ if (!lifecycle) {
5391
+ const activeTaskIds = new Set(rows.map((task) => task.taskId));
5392
+ const expectedTaskIds =
5393
+ onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
5394
+ for (const taskId of expectedTaskIds) {
5395
+ if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
5396
+ }
5397
+ for (const task of rows) openTaskRecoveryBarrier(task.taskId);
5398
+ }
5399
+ if (rows.length === 0) return { deferred, resolved };
5400
+ console.log(
5401
+ `[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
5402
+ );
5403
+ for (const task of rows) {
5404
+ if (!isCurrent()) break;
5405
+ try {
5406
+ const recovered = lifecycle
5407
+ ? await lifecycle.runTaskLifecycle(task.taskId, async () => {
5408
+ // The row was selected before this lifecycle slot became ours.
5409
+ // A queued taskDown/taskUp may have completed meanwhile, so act
5410
+ // only on the current local authority—not the stale snapshot.
5411
+ const current = getHostTask(task.taskId);
5412
+ if (
5413
+ !current ||
5414
+ !isCurrent() ||
5415
+ !current.statusMirror ||
5416
+ !isActive(current.statusMirror)
5417
+ ) {
5418
+ return true;
5419
+ }
5420
+ return recoverPersistedTaskEnvironment(
5421
+ current,
5422
+ maintenanceReady,
5423
+ isCurrent,
5424
+ lifecycle,
5425
+ );
5426
+ })
5427
+ : await recoverPersistedTaskEnvironment(
5428
+ task,
5429
+ maintenanceReady,
5430
+ isCurrent,
5431
+ );
5432
+ if (recovered) {
5433
+ if (!lifecycle) finishTaskRecovery(task.taskId);
5434
+ } else {
5435
+ deferred.add(task.taskId);
5436
+ }
5437
+ } catch (err) {
5438
+ resolved = false;
5439
+ if (!lifecycle) finishTaskRecovery(task.taskId);
5440
+ console.error(
5441
+ `[orchestrator] recovery: ${task.taskId} failed:`,
5442
+ err instanceof Error ? err.message : err,
5443
+ );
5444
+ }
5445
+ }
5446
+ } catch (err) {
5447
+ resolved = false;
5448
+ if (!lifecycle) finishAllTaskRecovery();
5449
+ console.error(
5450
+ "[orchestrator] recovery: top-level failure",
5451
+ err instanceof Error ? err.message : err,
5452
+ );
5453
+ }
5454
+ return { deferred, resolved };
5455
+ }
5456
+
5457
+ /** Apple-backend variant of the writable-runtime quarantine. Task containers
5458
+ * prove their shared-cache mount read-only at creation, so a writable mount
5459
+ * here is either a live materializer that outlived its bound or tampering —
5460
+ * both are stopped (task-labeled rows report `stopped`). A CLI that cannot
5461
+ * answer yields `retry`, exactly like an unreachable Docker daemon; it is
5462
+ * never treated as an empty inventory. */
5463
+ async function quarantineWritableAppleRuntimeContainers(options: {
5464
+ lifecycle: Orchestrator;
5465
+ isCurrent?: () => boolean;
5466
+ scope?: "contract" | "volume-holders";
5467
+ }): Promise<WritableRuntimeQuarantineResult> {
5468
+ const isCurrent = options.isCurrent ?? (() => true);
5469
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
5470
+ const binding = appleContainerRuntimeBinding();
5471
+ if (binding === null) return { verdict: "retry", stoppedTaskIds: [] };
5472
+ const cli = appleCliRunner(() => binding.containerCliPath);
5473
+ const listed = await cli(["list", "--all", "--format", "json"], {
5474
+ timeoutMs: 30_000,
5475
+ maxOutputBytes: 8 * 1024 * 1024,
5476
+ });
5477
+ if (listed.status !== 0 || listed.truncated === true) {
5478
+ return { verdict: "retry", stoppedTaskIds: [] };
5479
+ }
5480
+ let parsed: unknown;
5481
+ try {
5482
+ parsed = JSON.parse(listed.stdout);
5483
+ } catch {
5484
+ return { verdict: "retry", stoppedTaskIds: [] };
5485
+ }
5486
+ if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTaskIds: [] };
5487
+ const stoppedTaskIds = new Set<string>();
5488
+ let resolved = true;
5489
+
5490
+ const stopAndProve = async (id: string): Promise<boolean> => {
5491
+ const stopped = await cli(["stop", "--time", "30", id], {
5492
+ timeoutMs: 90_000,
5493
+ maxOutputBytes: 256 * 1024,
5494
+ });
5495
+ const after = await cli(["inspect", id], {
5496
+ timeoutMs: 15_000,
5497
+ maxOutputBytes: 1024 * 1024,
5498
+ });
5499
+ if (after.status !== 0) {
5500
+ return (
5501
+ after.stdout.trim() === "" &&
5502
+ after.stderr
5503
+ .split("\n")
5504
+ .some((line) => line.trim() === `Error: container not found: ${id}`)
5505
+ );
5506
+ }
5507
+ try {
5508
+ const parsedAfter = JSON.parse(after.stdout) as Array<{
5509
+ id?: unknown;
5510
+ status?: { state?: unknown };
5511
+ }>;
5512
+ // Only one exact-identity record in the runtime's terminal stopped
5513
+ // state settles the proof. A wrong-identity record, a transitional
5514
+ // state ("starting"), or an ambiguous shape must never release
5515
+ // shared-runtime maintenance while the writer may still be active.
5516
+ if (
5517
+ Array.isArray(parsedAfter) &&
5518
+ parsedAfter.length === 1 &&
5519
+ parsedAfter[0]?.id === id &&
5520
+ parsedAfter[0]?.status?.state === "stopped"
5521
+ ) {
5522
+ return true;
5523
+ }
5524
+ } catch {
5525
+ // fall through to the failure log below
5526
+ }
5527
+ console.error(
5528
+ `[orchestrator] runtime isolation: apple container ${id} holds the shared runtime cache writable and could not be proven stopped (stop exit ${stopped.status})`,
5529
+ );
5530
+ return false;
5531
+ };
5532
+
5533
+ for (const item of parsed as Array<{
5534
+ configuration?: {
5535
+ id?: unknown;
5536
+ labels?: Record<string, unknown>;
5537
+ mounts?: Array<{
5538
+ destination?: unknown;
5539
+ options?: unknown;
5540
+ type?: { volume?: { name?: unknown } };
5541
+ }>;
5542
+ };
5543
+ }>) {
5544
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
5545
+ const id = item?.configuration?.id;
5546
+ if (typeof id !== "string" || id.length === 0) {
5547
+ resolved = false;
5548
+ continue;
5549
+ }
5550
+ const mounts = item.configuration?.mounts;
5551
+ // Sweep every container whose shared-volume attachment violates the
5552
+ // CURRENT contract: writable (missing ro, or contradictory ro+rw), or at
5553
+ // any destination other than the ADR-108 staging path — which includes
5554
+ // every pre-ADR-108 container (read-only at /opt/asdf-data). Stopping
5555
+ // legacy containers HERE, before writer maintenance, is what lets a
5556
+ // legacy host migrate at all: a running legacy container would otherwise
5557
+ // hold the volume against the very reconcile whose completion recovery
5558
+ // needs (review 2026-08-18: maintenance retried forever and recovery —
5559
+ // the step that settles legacy containers — never ran).
5560
+ const writableShared = (Array.isArray(mounts) ? mounts : []).some(
5561
+ (mount) =>
5562
+ mount?.type?.volume?.name === "uai-asdf-data" &&
5563
+ (options.scope === "volume-holders" ||
5564
+ !(Array.isArray(mount.options) && mount.options.includes("ro")) ||
5565
+ (Array.isArray(mount.options) && mount.options.includes("rw")) ||
5566
+ mount?.destination !== "/run/uai/asdf-lower"),
5567
+ );
5568
+ if (!writableShared) continue;
5569
+ const label = item.configuration?.labels?.["com.uai.task"];
5570
+ // The label is container-influenced data. Only a syntactically safe task
5571
+ // id whose derived container name matches this exact container may drive
5572
+ // task-scoped mutations (lifecycle slot, DB row, ACL removal); anything
5573
+ // else is stopped as an anonymous writer, nothing task-scoped touched.
5574
+ const taskId =
5575
+ typeof label === "string" &&
5576
+ isSafeHostTaskId(label) &&
5577
+ id === appleTaskContainerName(label)
5578
+ ? label
5579
+ : null;
5580
+ if (taskId === null) {
5581
+ if (!(await stopAndProve(id))) resolved = false;
5582
+ continue;
5583
+ }
5584
+ // Task-labeled writers stop inside the task's lifecycle slot so a
5585
+ // concurrent taskUp/taskDown cannot interleave, and a known row commits
5586
+ // its durable `stopped` verdict before the frame is queued — a crashed
5587
+ // publisher must never leave SQLite saying running.
5588
+ await options.lifecycle.runTaskLifecycle(taskId, async () => {
5589
+ if (!isCurrent()) return;
5590
+ await options.lifecycle.quarantineChannelForRecovery(taskId);
5591
+ if (!(await stopAndProve(id))) {
5592
+ resolved = false;
5593
+ return;
5594
+ }
5595
+ const task = getHostTask(taskId);
5596
+ if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
5597
+ db_setStatus(taskId, "stopped", {
5598
+ codeServerPort: null,
5599
+ previewPorts: "[]",
5600
+ });
5601
+ stoppedTaskIds.add(taskId);
5602
+ }
5603
+ // Revoke live access with the environment, like the Docker sweep.
5604
+ clearTaskGatewayAcl(taskId);
5605
+ });
5606
+ }
5607
+ if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
5608
+ return {
5609
+ verdict: resolved ? "ready" : "retry",
5610
+ stoppedTaskIds: [...stoppedTaskIds],
5611
+ };
5612
+ }
5613
+
5614
+ /** Dispatch one durable row through its provider. Generic boot orchestration
5615
+ * owns DB/lifecycle sequencing but never derives a Compose/container identity. */
5616
+ async function recoverPersistedTaskEnvironment(
5617
+ task: typeof schema.hostTasks.$inferSelect,
5618
+ maintenanceReady: Promise<void>,
5619
+ isCurrent: () => boolean,
5620
+ lifecycle?: Orchestrator,
5621
+ ): Promise<boolean> {
5622
+ const environment = await reconstructPersistedTaskEnvironment(task);
5623
+ if (environment === null) return true;
5624
+ if (!isCurrent()) return true;
5625
+ const result = await environment.recover({
5626
+ maintenanceReady,
5627
+ isCurrent,
5628
+ ...(lifecycle
5629
+ ? {
5630
+ quarantineConsumers: () =>
5631
+ lifecycle.quarantineChannelForRecovery(task.taskId),
5632
+ allowConsumers: () => lifecycle.allowChannel(task.taskId),
5633
+ ensureConsumersStarted: async () => {
5634
+ await lifecycle.ensureStarted(task.taskId);
5635
+ },
5636
+ }
5637
+ : {}),
5638
+ });
5639
+ return result.outcome === "recovered";
5640
+ }
5641
+
5642
+ interface RecoveryTaskConsumers {
5643
+ quarantineChannelForRecovery(taskId: string): Promise<void>;
5644
+ allowChannel(taskId: string): void;
5645
+ ensureStarted(taskId: string): Promise<boolean | void>;
5646
+ }
5647
+
5648
+ /** Apple provider driver. Recovery must end in a durable truth: a running
5649
+ * container is re-initialized (dependencies + code-server) under an
5650
+ * in-container deadline; a stopped one is started and initialized; a
5651
+ * container that is provenly absent — or cannot be started — marks the task
5652
+ * `stopped` so the host never reports operational with a phantom running
5653
+ * task. Unknown CLI answers stay deferred and retryable. */
5654
+ /** Exact TypeScript port of task-up.sh's container environment-authority
5655
+ * proof (the jq filter behind "verify task runtime isolation"): each required
5656
+ * canonical entry appears exactly once, every owned-npm-config spelling
5657
+ * (case-folded, underscores treated as hyphens) must be one of the canonical
5658
+ * eight, and PATH plus the asdf/corepack controls hold their exact values
5659
+ * exactly once. */
5660
+ export function appleTaskEnvironmentAuthorityHolds(value: unknown): boolean {
5661
+ if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) {
5662
+ return false;
5663
+ }
5664
+ const environment = value as string[];
5665
+ const npmPrefix = "/home/node/.local/share/uai-npm";
5666
+ const npmCache = "/home/node/.cache/uai-npm";
5667
+ const runtimePath =
5668
+ "/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:" +
5669
+ `/usr/sbin:/usr/bin:/sbin:/bin:${npmPrefix}/bin`;
5670
+ const required = [
5671
+ `NPM_CONFIG_PREFIX=${npmPrefix}`,
5672
+ `npm_config_prefix=${npmPrefix}`,
5673
+ `NPM_CONFIG_CACHE=${npmCache}`,
5674
+ `npm_config_cache=${npmCache}`,
5675
+ `NPM_CONFIG_LOGS_DIR=${npmCache}/_logs`,
5676
+ `npm_config_logs_dir=${npmCache}/_logs`,
5677
+ "NPM_CONFIG_UMASK=0002",
5678
+ "npm_config_umask=0002",
5679
+ ];
5680
+ const owned = ["prefix", "cache", "logs-dir", "umask"];
5681
+ const countOf = (exact: string): number =>
5682
+ environment.filter((entry) => entry === exact).length;
5683
+ const isOwnedNpmConfig = (key: string): boolean => {
5684
+ const folded = key.toLowerCase();
5685
+ return (
5686
+ folded.startsWith("npm_config_") &&
5687
+ owned.includes(folded.slice("npm_config_".length).replace(/_/g, "-"))
5688
+ );
5689
+ };
5690
+ const ownedEntries = environment.filter((entry) =>
5691
+ isOwnedNpmConfig(entry.split("=")[0]!),
5692
+ );
5693
+ return (
5694
+ required.every((entry) => countOf(entry) === 1) &&
5695
+ ownedEntries.length === required.length &&
5696
+ countOf(`PATH=${runtimePath}`) === 1 &&
5697
+ countOf("ASDF_DATA_DIR=/opt/asdf-data") === 1 &&
5698
+ countOf("ASDF_SKIP_RESHIM=1") === 1 &&
5699
+ countOf("COREPACK_HOME=/opt/asdf-data/corepack") === 1 &&
5700
+ countOf("COREPACK_ENABLE_NETWORK=0") === 1
5701
+ );
5702
+ }
5703
+
5704
+ export type AppleTaskRuntimeContractVerdict =
5705
+ | { kind: "holds" }
5706
+ | { kind: "violated" }
5707
+ | { kind: "unreachable"; detail: string };
5708
+
5709
+ /**
5710
+ * The apple runtime-authority contract, in one inspection: this exact name
5711
+ * carries this task's label, is running, mounts the shared cache read-only
5712
+ * at exactly the ADR-108 staging path (and nowhere else, with no
5713
+ * contradictory options), binds /workspace as exactly one writable virtiofs
5714
+ * share of this task's real worktree leaf, and carries the same immutable
5715
+ * npm/runtime environment authority task-up proved at creation. Factored
5716
+ * from recovery so the taskUp lost-response fast path proves the SAME
5717
+ * invariants (review 2026-08-18 round 4). `unreachable` and `violated` are
5718
+ * distinct so the fast path can retry without stopping a container it could
5719
+ * not observe, while recovery keeps treating both as quarantine grounds.
5720
+ */
5721
+ export async function proveAppleTaskRuntimeContract(options: {
5722
+ taskId: string;
5723
+ containerName: string;
5724
+ hostWorktreePath: string;
5725
+ cli?: ReturnType<typeof appleCliRunner>;
5726
+ }): Promise<AppleTaskRuntimeContractVerdict> {
5727
+ let cli = options.cli;
5728
+ if (!cli) {
5729
+ const binding = appleContainerRuntimeBinding();
5730
+ if (binding === null) {
5731
+ return { kind: "unreachable", detail: "apple runtime binding unavailable" };
5732
+ }
5733
+ cli = appleCliRunner(() => binding.containerCliPath);
5734
+ }
5735
+ const contract = await cli(["inspect", options.containerName], {
5736
+ timeoutMs: 15_000,
5737
+ maxOutputBytes: 4 * 1024 * 1024,
5738
+ });
5739
+ if (contract.status !== 0) {
5740
+ return {
5741
+ kind: "unreachable",
5742
+ detail:
5743
+ contract.stderr.trim().slice(0, 200) ||
5744
+ `inspect exited ${contract.status ?? "killed"}`,
5745
+ };
5746
+ }
5747
+ try {
5748
+ const records = JSON.parse(contract.stdout) as Array<{
5749
+ id?: unknown;
5750
+ status?: { state?: unknown };
5751
+ configuration?: {
5752
+ labels?: Record<string, unknown>;
5753
+ mounts?: Array<{
5754
+ destination?: unknown;
5755
+ options?: unknown;
5756
+ type?: { volume?: { name?: unknown } };
5757
+ }>;
5758
+ initProcess?: { environment?: unknown };
5759
+ };
5760
+ }>;
5761
+ // Same strictness as the provider's inspection: exactly one record and
5762
+ // it must identify itself as the requested container, or a malformed
5763
+ // response describing another container could authorize execution.
5764
+ if (!Array.isArray(records) || records.length !== 1) {
5765
+ throw new Error("contract inspection shape is invalid");
5766
+ }
5767
+ const record = records[0]!;
5768
+ if (record.id !== options.containerName) {
5769
+ throw new Error("contract inspection identity mismatch");
5770
+ }
5771
+ const mounts = record?.configuration?.mounts ?? [];
5772
+ // ADR-108: the shared volume attaches read-only at the overlay staging
5773
+ // path and /opt/asdf-data must NOT be a host mount at all — the
5774
+ // writable view there is the in-guest overlay.
5775
+ const runtimeMounts = mounts.filter(
5776
+ (mount) => mount?.destination === "/run/uai/asdf-lower",
5777
+ );
5778
+ const sharedMounts = mounts.filter(
5779
+ (mount) => mount?.type?.volume?.name === "uai-asdf-data",
5780
+ );
5781
+ const asdfHostMounts = mounts.filter(
5782
+ (mount) => mount?.destination === "/opt/asdf-data",
5783
+ );
5784
+ // The /workspace authority: exactly one bind (never a volume) whose
5785
+ // source is this task's exact worktree leaf, still a real non-symlink
5786
+ // directory on disk — the same proof Docker recovery performs.
5787
+ const workspaceMounts = mounts.filter(
5788
+ (mount) => mount?.destination === "/workspace",
5789
+ );
5790
+ const expectedWorkspaceSource = join(
5791
+ options.hostWorktreePath,
5792
+ "workspace",
5793
+ );
5794
+ // The verified Apple dialect for a host-directory bind is exactly
5795
+ // type:{virtiofs:{}} — a closed shape.
5796
+ const workspaceType = workspaceMounts[0]?.type as
5797
+ | Record<string, unknown>
5798
+ | undefined;
5799
+ const workspaceTypeIsVirtiofs =
5800
+ workspaceType !== undefined &&
5801
+ typeof workspaceType === "object" &&
5802
+ !Array.isArray(workspaceType) &&
5803
+ Object.keys(workspaceType).length === 1 &&
5804
+ typeof workspaceType.virtiofs === "object" &&
5805
+ workspaceType.virtiofs !== null &&
5806
+ !Array.isArray(workspaceType.virtiofs) &&
5807
+ Object.keys(workspaceType.virtiofs as object).length === 0;
5808
+ // Writable is part of the authority: a read-only /workspace would
5809
+ // expose a task whose agents cannot work.
5810
+ const workspaceOptions = workspaceMounts[0]?.options;
5811
+ let workspaceAuthorityHolds =
5812
+ workspaceMounts.length === 1 &&
5813
+ workspaceTypeIsVirtiofs &&
5814
+ Array.isArray(workspaceOptions) &&
5815
+ !(workspaceOptions as unknown[]).includes("ro") &&
5816
+ (workspaceMounts[0] as { source?: unknown } | undefined)?.source ===
5817
+ expectedWorkspaceSource;
5818
+ if (workspaceAuthorityHolds) {
5819
+ try {
5820
+ const workspaceStat = lstatSync(expectedWorkspaceSource);
5821
+ workspaceAuthorityHolds =
5822
+ workspaceStat.isDirectory() && !workspaceStat.isSymbolicLink();
5823
+ } catch {
5824
+ workspaceAuthorityHolds = false;
5825
+ }
5826
+ }
5827
+ const rawEnvironment = record?.configuration?.initProcess?.environment;
5828
+ const holds =
5829
+ record?.status?.state === "running" &&
5830
+ record?.configuration?.labels?.["com.uai.task"] === options.taskId &&
5831
+ runtimeMounts.length === 1 &&
5832
+ sharedMounts.length === 1 &&
5833
+ sharedMounts[0] === runtimeMounts[0] &&
5834
+ asdfHostMounts.length === 0 &&
5835
+ Array.isArray(runtimeMounts[0]?.options) &&
5836
+ (runtimeMounts[0]!.options as unknown[]).includes("ro") &&
5837
+ // Closed options contract: an array carrying BOTH ro and rw is
5838
+ // contradictory and proves nothing.
5839
+ !(runtimeMounts[0]!.options as unknown[]).includes("rw") &&
5840
+ workspaceAuthorityHolds &&
5841
+ appleTaskEnvironmentAuthorityHolds(rawEnvironment);
5842
+ return holds ? { kind: "holds" } : { kind: "violated" };
5843
+ } catch {
5844
+ return { kind: "violated" };
5845
+ }
5846
+ }
5847
+
5848
+ export async function recoverAppleTaskEnvironment(
5849
+ descriptor: TaskEnvironmentDescriptor,
5850
+ context: TaskEnvironmentRecoveryContext,
5851
+ ): Promise<TaskEnvironmentRecoveryResult> {
5852
+ const locator = parseAppleTaskEnvironmentLocator(descriptor.locator);
5853
+ const task = getHostTask(descriptor.taskId);
5854
+ if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
5855
+ return { outcome: "recovered" };
5856
+ }
5857
+ // The boot maintenance latch (agentClisReady) is awaited ONLY before the
5858
+ // first in-container exec, exactly like Docker recovery. Awaiting it here
5859
+ // deadlocked the first fresh Apple host (live 2026-08-15): the on-connect
5860
+ // recovery sweep held this driver on the latch, activation joins in-flight
5861
+ // recovery before its maintenance, and the maintenance that settles the
5862
+ // latch could therefore never run — the host sat silently in `checking`
5863
+ // forever. Status verdicts, quarantines, and workspace settlement need no
5864
+ // maintenance and must never wait on it.
5865
+ if (!context.isCurrent()) {
5866
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5867
+ }
5868
+ const binding = appleContainerRuntimeBinding();
5869
+ if (binding === null) {
5870
+ return { outcome: "deferred", detail: "apple runtime binding unavailable" };
5871
+ }
5872
+ const cli = appleCliRunner(() => binding.containerCliPath);
5873
+ const environment = await reconstructPersistedTaskEnvironment(task);
5874
+ if (environment === null) return { outcome: "recovered" };
5875
+
5876
+ const markStopped = (): TaskEnvironmentRecoveryResult => {
5877
+ db_setStatus(descriptor.taskId, "stopped", {
5878
+ codeServerPort: null,
5879
+ previewPorts: "[]",
5880
+ });
5881
+ // A stopped verdict must also revoke live access paths, exactly like the
5882
+ // Docker quarantine: the MCP gateway token dies with the environment.
5883
+ clearTaskGatewayAcl(descriptor.taskId);
5884
+ return { outcome: "recovered" };
5885
+ };
5886
+ // Without its workspace the task cannot resume — stopped would advertise a
5887
+ // restart that must fail. Docker recovery reconciles this as error.
5888
+ const markError = (): TaskEnvironmentRecoveryResult => {
5889
+ db_setStatus(descriptor.taskId, "error", {
5890
+ codeServerPort: null,
5891
+ previewPorts: "[]",
5892
+ composeProject: null,
5893
+ });
5894
+ clearTaskGatewayAcl(descriptor.taskId);
5895
+ return { outcome: "recovered" };
5896
+ };
5897
+ // Every settled verdict consults the freshest workspace observation: a
5898
+ // workspace that vanished mid-recovery must land as error exactly like one
5899
+ // that was already gone at the first look.
5900
+ const settleFromStatus = (settled: {
5901
+ workspacePresent: boolean;
5902
+ }): TaskEnvironmentRecoveryResult =>
5903
+ settled.workspacePresent ? markStopped() : markError();
5904
+
5905
+ let status = await environment.status();
5906
+ if (status.state === "unknown") {
5907
+ return { outcome: "deferred", detail: status.detail };
5908
+ }
5909
+ // Every awaited runtime observation can outlive this recovery's readiness
5910
+ // generation; a superseded pass must observe only, never mutate. Recheck
5911
+ // after each await and before each mutation from here on.
5912
+ if (!context.isCurrent()) {
5913
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5914
+ }
5915
+ // Drain consumers before ANY durable verdict or mutation — including the
5916
+ // absent path, where a live channel could otherwise keep serving a task
5917
+ // whose row is about to say stopped.
5918
+ await context.quarantineConsumers?.();
5919
+ if (!context.isCurrent()) {
5920
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5921
+ }
5922
+ if (status.state === "absent") {
5923
+ // Name-anchored absence came from the provider's inspect. The disk truth
5924
+ // decides the verdict: an intact workspace resumes via re-provision
5925
+ // (stopped); a vanished one cannot, and stopped would advertise a
5926
+ // restart that must fail (error).
5927
+ return status.workspacePresent ? markStopped() : markError();
5928
+ }
5929
+ if (status.state === "stopped" && !status.workspacePresent) {
5930
+ // Never start a container whose workspace is gone: dependency recovery
5931
+ // would mutate a recreated or missing tree.
5932
+ return markError();
5933
+ }
5934
+ if (status.state === "stopped") {
5935
+ const started = await cli(["start", locator.containerName], {
5936
+ timeoutMs: 120_000,
5937
+ maxOutputBytes: 256 * 1024,
5938
+ });
5939
+ if (!context.isCurrent()) {
5940
+ return { outcome: "deferred", detail: "runtime generation superseded" };
3622
5941
  }
3623
- pendingTaskIds = deferred;
3624
- if (pass >= maxPasses) {
3625
- console.warn(
3626
- `[orchestrator] recovery: docker still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
3627
- );
3628
- finishAllTaskRecovery();
3629
- return;
5942
+ // A failed start command or an ambiguous follow-up is NOT proof of a
5943
+ // stopped container: only an exact stopped/absent answer may settle the
5944
+ // durable row, and unknown always stays deferred and retryable.
5945
+ status = await environment.status();
5946
+ if (status.state === "unknown") {
5947
+ return { outcome: "deferred", detail: status.detail };
3630
5948
  }
3631
- if (pass === 1) {
3632
- console.log(
3633
- `[orchestrator] recovery: docker not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
5949
+ if (!context.isCurrent()) {
5950
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5951
+ }
5952
+ if (status.state === "stopped" || status.state === "absent") {
5953
+ return settleFromStatus(status);
5954
+ }
5955
+ if (started.status !== 0) {
5956
+ // The CLI reported failure yet the container runs — trust the state
5957
+ // probe over the command's exit and continue into revalidation.
5958
+ console.warn(
5959
+ `[orchestrator] recovery: ${descriptor.taskId} container start exited ${started.status} but the container is running; continuing`,
3634
5960
  );
3635
5961
  }
3636
- await new Promise((resolve) => setTimeout(resolve, retryMs));
3637
5962
  }
3638
- }
3639
5963
 
3640
- /** One recovery sweep. Returns only task ids deferred by unreachable Docker;
3641
- * successful ids are never revisited by a later retry pass. */
3642
- async function recoveryPass(
3643
- onlyTaskIds?: ReadonlySet<string>,
3644
- ): Promise<Set<string>> {
3645
- const deferred = new Set<string>();
3646
- try {
3647
- const db = getDb();
3648
- const activeRows = db
3649
- .select()
3650
- .from(schema.hostTasks)
3651
- .where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
3652
- .all();
3653
- const rows = onlyTaskIds
3654
- ? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
3655
- : activeRows;
3656
- const activeTaskIds = new Set(rows.map((task) => task.taskId));
3657
- const expectedTaskIds =
3658
- onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
3659
- for (const taskId of expectedTaskIds) {
3660
- if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
3661
- }
3662
- for (const task of rows) openTaskRecoveryBarrier(task.taskId);
3663
- if (rows.length === 0) return deferred;
3664
- console.log(
3665
- `[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
5964
+ // One inspection proves the whole trust decision factored into
5965
+ // proveAppleTaskRuntimeContract so the taskUp lost-response fast path can
5966
+ // prove the SAME invariants (review 2026-08-18 round 4). Recovery keeps
5967
+ // its historical mapping: anything short of `holds` — including an
5968
+ // unreachable inspect — quarantines below.
5969
+ const contractVerdict = await proveAppleTaskRuntimeContract({
5970
+ taskId: descriptor.taskId,
5971
+ containerName: locator.containerName,
5972
+ hostWorktreePath: locator.hostWorktreePath,
5973
+ cli,
5974
+ });
5975
+ const contractHolds = contractVerdict.kind === "holds";
5976
+ if (!context.isCurrent()) {
5977
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5978
+ }
5979
+ if (!contractHolds) {
5980
+ await environment.stop();
5981
+ const after = await environment.status();
5982
+ if (after.state === "running" || after.state === "unknown") {
5983
+ return {
5984
+ outcome: "deferred",
5985
+ detail:
5986
+ "task container failed its runtime-authority contract and could not be proven stopped",
5987
+ };
5988
+ }
5989
+ if (!context.isCurrent()) {
5990
+ return { outcome: "deferred", detail: "runtime generation superseded" };
5991
+ }
5992
+ console.error(
5993
+ `[orchestrator] recovery: ${descriptor.taskId} container failed its runtime-authority contract; task was stopped and remains quarantined`,
3666
5994
  );
3667
- for (const task of rows) {
3668
- try {
3669
- if (await recoverOneTask(task)) finishTaskRecovery(task.taskId);
3670
- else deferred.add(task.taskId);
3671
- } catch (err) {
3672
- finishTaskRecovery(task.taskId);
3673
- console.error(
3674
- `[orchestrator] recovery: ${task.taskId} failed:`,
3675
- err instanceof Error ? err.message : err,
3676
- );
3677
- }
5995
+ return settleFromStatus(after);
5996
+ }
5997
+ // Even an already-running container's first exec resolves through shared
5998
+ // asdf shims; fence it behind the one-way boot maintenance latch, exactly
5999
+ // like Docker recovery fences its workbench probe.
6000
+ await context.maintenanceReady;
6001
+ if (!context.isCurrent()) {
6002
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6003
+ }
6004
+ // ADR-108: guest mounts do not survive a VM restart, so the runtime
6005
+ // overlay must be re-established (idempotently) before ANY exec resolves
6006
+ // through /opt/asdf-data shims — including uai-init below. The upper layer
6007
+ // persists in the container filesystem, so the task's own installs return
6008
+ // with the mount. Proven from the guest mount table, exactly like task-up.
6009
+ const overlay = await environment.exec({
6010
+ argv: [
6011
+ "/bin/sh",
6012
+ "-c",
6013
+ // Same establishment + token-exact proof as task-up §6a: exactly one
6014
+ // effective mount at the merged path and the lower, comma-delimited
6015
+ // option tokens (substring matching accepted a -evil suffix), a
6016
+ // read-only lower, and a real node-owned upper.
6017
+ "set -e; /bin/mkdir -p /var/lib/uai/asdf-upper /var/lib/uai/asdf-work /opt/asdf-data; " +
6018
+ "/bin/chown node:node /var/lib/uai/asdf-upper; " +
6019
+ 'if ! /bin/grep -q "^overlay /opt/asdf-data overlay " /proc/mounts; then ' +
6020
+ "/bin/mount -t overlay overlay -o lowerdir=/run/uai/asdf-lower,upperdir=/var/lib/uai/asdf-upper,workdir=/var/lib/uai/asdf-work /opt/asdf-data; fi; " +
6021
+ 'overlay_lines=$(/bin/grep -c "^overlay /opt/asdf-data overlay " /proc/mounts || true); ' +
6022
+ '[ "$overlay_lines" = "1" ] || { echo "expected exactly one /opt/asdf-data overlay, found $overlay_lines" >&2; exit 1; }; ' +
6023
+ 'any_lines=$(/bin/grep -c " /opt/asdf-data " /proc/mounts || true); ' +
6024
+ '[ "$any_lines" = "1" ] || { echo "unexpected extra mounts at /opt/asdf-data" >&2; exit 1; }; ' +
6025
+ 'opts=$(/bin/grep "^overlay /opt/asdf-data overlay " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
6026
+ 'case ",$opts," in *",lowerdir=/run/uai/asdf-lower,"*) ;; *) echo "overlay lower is foreign" >&2; exit 1 ;; esac; ' +
6027
+ 'case ",$opts," in *",upperdir=/var/lib/uai/asdf-upper,"*) ;; *) echo "overlay upper is foreign" >&2; exit 1 ;; esac; ' +
6028
+ 'case ",$opts," in *",workdir=/var/lib/uai/asdf-work,"*) ;; *) echo "overlay workdir is foreign" >&2; exit 1 ;; esac; ' +
6029
+ 'case ",$opts," in *",ro,"*) echo "overlay is unexpectedly read-only" >&2; exit 1 ;; esac; ' +
6030
+ 'lower_lines=$(/bin/grep -c " /run/uai/asdf-lower " /proc/mounts || true); ' +
6031
+ '[ "$lower_lines" = "1" ] || { echo "expected exactly one lower attachment, found $lower_lines" >&2; exit 1; }; ' +
6032
+ 'lopts=$(/bin/grep " /run/uai/asdf-lower " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
6033
+ 'case ",$lopts," in *",ro,"*) ;; *) echo "lower attachment is not read-only" >&2; exit 1 ;; esac; ' +
6034
+ '[ ! -L /var/lib/uai/asdf-upper ] || { echo "upper is a symlink" >&2; exit 1; }; ' +
6035
+ '[ -d /var/lib/uai/asdf-upper ] || { echo "upper is not a directory" >&2; exit 1; }; ' +
6036
+ '[ "$(/usr/bin/stat -c %U /var/lib/uai/asdf-upper)" = "node" ] || { echo "upper is not node-owned" >&2; exit 1; }; ' +
6037
+ "/usr/bin/test -d /opt/asdf-data/plugins",
6038
+ ],
6039
+ user: "root",
6040
+ env: {},
6041
+ timeoutMs: 30_000,
6042
+ maxOutputBytes: 64 * 1024,
6043
+ });
6044
+ if (!context.isCurrent()) {
6045
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6046
+ }
6047
+ if (overlay.exitCode !== 0 || overlay.signal !== null) {
6048
+ await environment.stop();
6049
+ const afterOverlay = await environment.status();
6050
+ if (afterOverlay.state === "running" || afterOverlay.state === "unknown") {
6051
+ return {
6052
+ outcome: "deferred",
6053
+ detail:
6054
+ "task runtime overlay could not be established and the container could not be proven stopped",
6055
+ };
6056
+ }
6057
+ if (!context.isCurrent()) {
6058
+ return { outcome: "deferred", detail: "runtime generation superseded" };
3678
6059
  }
3679
- } catch (err) {
3680
- finishAllTaskRecovery();
3681
6060
  console.error(
3682
- "[orchestrator] recovery: top-level failure",
3683
- err instanceof Error ? err.message : err,
6061
+ `[orchestrator] recovery: ${descriptor.taskId} runtime overlay could not be established (exit ${overlay.exitCode}, signal ${overlay.signal}); task was stopped`,
6062
+ );
6063
+ return settleFromStatus(afterOverlay);
6064
+ }
6065
+ // Dependencies + code-server. The in-container deadline (TERM then KILL)
6066
+ // means a host-side expiry can never leave the initializer mutating the
6067
+ // workspace while agents use it; exit 78 is a runtime-authority violation
6068
+ // and quarantines exactly like at task-up.
6069
+ const init = await environment.exec({
6070
+ argv: [
6071
+ "/usr/bin/timeout",
6072
+ "--signal=TERM",
6073
+ "--kill-after=15s",
6074
+ "570s",
6075
+ "/usr/local/bin/uai-init",
6076
+ ],
6077
+ env: {},
6078
+ timeoutMs: 10 * 60_000,
6079
+ maxOutputBytes: 1024 * 1024,
6080
+ });
6081
+ if (!context.isCurrent()) {
6082
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6083
+ }
6084
+ if (
6085
+ init.exitCode === null ||
6086
+ init.signal !== null ||
6087
+ init.stdoutTruncated ||
6088
+ init.stderrTruncated
6089
+ ) {
6090
+ // The host-side client died (deadline, overflow, forced settlement)
6091
+ // without a real exit code, so the in-container initializer may STILL be
6092
+ // mutating dependencies for several minutes. Never expose the task on an
6093
+ // unknowable outcome: stop and prove, or stay deferred.
6094
+ await environment.stop();
6095
+ const after = await environment.status();
6096
+ if (after.state === "running" || after.state === "unknown") {
6097
+ return {
6098
+ outcome: "deferred",
6099
+ detail:
6100
+ "task initializer outcome was unknowable and the container could not be proven stopped",
6101
+ };
6102
+ }
6103
+ if (!context.isCurrent()) {
6104
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6105
+ }
6106
+ console.error(
6107
+ `[orchestrator] recovery: ${descriptor.taskId} initializer outcome was unknowable (exit ${init.exitCode}, signal ${init.signal}); task was stopped pending a clean re-run`,
6108
+ );
6109
+ return settleFromStatus(after);
6110
+ }
6111
+ if (init.exitCode === 78) {
6112
+ await environment.stop();
6113
+ const after = await environment.status();
6114
+ if (after.state === "running" || after.state === "unknown") {
6115
+ return {
6116
+ outcome: "deferred",
6117
+ detail:
6118
+ "runtime-authority violation detected and the container could not be proven stopped",
6119
+ };
6120
+ }
6121
+ if (!context.isCurrent()) {
6122
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6123
+ }
6124
+ console.error(
6125
+ `[orchestrator] recovery: ${descriptor.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
6126
+ );
6127
+ return settleFromStatus(after);
6128
+ }
6129
+ if (init.exitCode !== 0) {
6130
+ console.warn(
6131
+ `[orchestrator] recovery: ${descriptor.taskId} uai-init failed (exit ${init.exitCode}); container is up but Editor may be down`,
6132
+ );
6133
+ }
6134
+ if (!context.isCurrent()) {
6135
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6136
+ }
6137
+ db_setRuntime(descriptor.taskId, { codeServerPort: null });
6138
+ context.allowConsumers?.();
6139
+ await context.ensureConsumersStarted?.();
6140
+ if (!context.isCurrent()) {
6141
+ // Admission reopened against a superseded generation: re-quarantine so
6142
+ // the successor generation re-proves the environment before consumers
6143
+ // touch it — the same closing move Docker recovery makes.
6144
+ await context.quarantineConsumers?.();
6145
+ return { outcome: "deferred", detail: "runtime generation superseded" };
6146
+ }
6147
+ return { outcome: "recovered" };
6148
+ }
6149
+
6150
+ /** Docker provider driver. The legacy repair implementation remains local to
6151
+ * this module for now, but it is reachable only through the provider handle;
6152
+ * the generic recovery sweep above never consumes its locator payload. */
6153
+ async function recoverDockerTaskEnvironment(
6154
+ descriptor: TaskEnvironmentDescriptor,
6155
+ context: TaskEnvironmentRecoveryContext,
6156
+ ): Promise<TaskEnvironmentRecoveryResult> {
6157
+ const locator = parseDockerTaskEnvironmentLocator(descriptor.locator);
6158
+ const task = getHostTask(descriptor.taskId);
6159
+ if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
6160
+ return { outcome: "recovered" };
6161
+ }
6162
+ const isPreparedRow =
6163
+ (task.statusMirror === "starting" || task.statusMirror === "error") &&
6164
+ task.composeProject === null &&
6165
+ task.worktreePath === null;
6166
+ if (
6167
+ !isPreparedRow &&
6168
+ (task.composeProject !== locator.composeProject ||
6169
+ task.worktreePath !== locator.hostWorktreePath)
6170
+ ) {
6171
+ throw new Error(
6172
+ `task ${descriptor.taskId} environment locator no longer matches its durable row`,
3684
6173
  );
3685
6174
  }
3686
- return deferred;
6175
+ // Provider-owned provisional metadata is authoritative after a crash. Feed
6176
+ // the already-validated deterministic values to the Docker-private repair
6177
+ // helpers without teaching the generic sweep how to derive them.
6178
+ const recoveryTask = isPreparedRow
6179
+ ? {
6180
+ ...task,
6181
+ composeProject: locator.composeProject,
6182
+ worktreePath: locator.hostWorktreePath,
6183
+ }
6184
+ : task;
6185
+
6186
+ const lifecycleCallbacks = [
6187
+ context.quarantineConsumers,
6188
+ context.allowConsumers,
6189
+ context.ensureConsumersStarted,
6190
+ ];
6191
+ const hasLifecycle = lifecycleCallbacks.some(Boolean);
6192
+ if (hasLifecycle && !lifecycleCallbacks.every(Boolean)) {
6193
+ throw new Error("task environment recovery lifecycle callbacks are incomplete");
6194
+ }
6195
+ const consumers: RecoveryTaskConsumers | undefined = hasLifecycle
6196
+ ? {
6197
+ quarantineChannelForRecovery: async (taskId) => {
6198
+ if (taskId !== descriptor.taskId) {
6199
+ throw new Error("task environment recovery consumer identity mismatch");
6200
+ }
6201
+ await context.quarantineConsumers!();
6202
+ },
6203
+ allowChannel: (taskId) => {
6204
+ if (taskId !== descriptor.taskId) {
6205
+ throw new Error("task environment recovery consumer identity mismatch");
6206
+ }
6207
+ context.allowConsumers!();
6208
+ },
6209
+ ensureStarted: async (taskId) => {
6210
+ if (taskId !== descriptor.taskId) {
6211
+ throw new Error("task environment recovery consumer identity mismatch");
6212
+ }
6213
+ await context.ensureConsumersStarted!();
6214
+ },
6215
+ }
6216
+ : undefined;
6217
+ const recovered = await recoverOneDockerTask(
6218
+ recoveryTask,
6219
+ context.maintenanceReady,
6220
+ context.isCurrent,
6221
+ consumers,
6222
+ );
6223
+ return recovered
6224
+ ? { outcome: "recovered" }
6225
+ : {
6226
+ outcome: "deferred",
6227
+ detail: "container runtime did not answer during task recovery",
6228
+ };
3687
6229
  }
3688
6230
 
6231
+ setDockerTaskEnvironmentRecoveryDriver(recoverDockerTaskEnvironment);
6232
+ setAppleTaskEnvironmentRecoveryDriver(recoverAppleTaskEnvironment);
6233
+
3689
6234
  /** Returns false when docker was unreachable (caller retries the pass). */
3690
- async function recoverOneTask(
6235
+ async function recoverOneDockerTask(
3691
6236
  task: typeof schema.hostTasks.$inferSelect,
6237
+ maintenanceReady: Promise<void>,
6238
+ isCurrent: () => boolean = () => true,
6239
+ lifecycle?: RecoveryTaskConsumers,
3692
6240
  ): Promise<boolean> {
6241
+ if (!isCurrent()) return true;
3693
6242
  const composeProject = task.composeProject;
3694
6243
  if (!composeProject) {
3695
6244
  // We never wrote a compose project name for this row — must be
@@ -3701,6 +6250,7 @@ async function recoverOneTask(
3701
6250
  const containers = await dockerListContainersByLabel(
3702
6251
  `com.docker.compose.project=${composeProject}`,
3703
6252
  );
6253
+ if (!isCurrent()) return true;
3704
6254
  if (containers === null) {
3705
6255
  // Docker unreachable / timed out — actual state unknown. Leave the row
3706
6256
  // alone; the retrying caller re-scans once docker answers.
@@ -3727,12 +6277,106 @@ async function recoverOneTask(
3727
6277
  return true;
3728
6278
  }
3729
6279
 
3730
- const running = containers.some((c) => c.State === "running");
6280
+ // Defense in depth for every recovery entry point: the maintained
6281
+ // activation normally quarantines legacy writers before its first shared
6282
+ // volume mutation, but recovery itself must never fast-path or `docker
6283
+ // start` a container whose immutable mount is not the exact L5 RO contract.
6284
+ const mountContract = await inspectRuntimeWriterContainer({
6285
+ taskId: task.taskId,
6286
+ composeProject,
6287
+ containerName,
6288
+ });
6289
+ if (!isCurrent()) return true;
6290
+ if (mountContract.kind === "unreachable") return false;
6291
+ if (mountContract.kind === "failed") {
6292
+ console.error(
6293
+ `[orchestrator] recovery: ${task.taskId} mount identity is ambiguous; deferring without starting it: ${mountContract.detail}`,
6294
+ );
6295
+ return false;
6296
+ }
6297
+ if (
6298
+ mountContract.metadata.sharedMount !== "read-only" ||
6299
+ mountContract.metadata.taskEnvironment !== "current"
6300
+ ) {
6301
+ if (lifecycle) {
6302
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6303
+ if (!isCurrent()) return true;
6304
+ }
6305
+ const stopped = await stopRuntimeWriter(mountContract.metadata);
6306
+ if (!isCurrent()) return true;
6307
+ if (stopped === "unreachable") return false;
6308
+ if (stopped === "failed") {
6309
+ throw new Error(
6310
+ `${task.taskId} has a legacy/non-current runtime mount and could not be proven stopped`,
6311
+ );
6312
+ }
6313
+ db_setStatus(task.taskId, "stopped", {
6314
+ codeServerPort: null,
6315
+ previewPorts: "[]",
6316
+ });
6317
+ clearTaskGatewayAcl(task.taskId);
6318
+ console.warn(
6319
+ `[orchestrator] recovery: ${task.taskId} uses a legacy/non-current runtime mount; it remains stopped until explicit Resume recreates it`,
6320
+ );
6321
+ return true;
6322
+ }
6323
+
6324
+ // Even a container already marked running needs an in-container workbench
6325
+ // probe. Fence that first exec, as well as the exited-container repair path,
6326
+ // behind the same one-way boot maintenance latch.
6327
+ await maintenanceReady;
6328
+ if (!isCurrent()) return true;
6329
+
6330
+ let running = containers.some(
6331
+ (container) =>
6332
+ container.Names === containerName && container.State === "running",
6333
+ );
3731
6334
  if (running) {
6335
+ // A stale recovery generation may have successfully issued `docker start`
6336
+ // and then been cancelled before uai-init. Do not bless every running
6337
+ // container solely from Docker state: code-server is the cheap proof that
6338
+ // the workbench init completed. A fresh sweep repairs a merely-started
6339
+ // container through the shared init path below.
6340
+ const workbenchReady = await dockerExec(
6341
+ containerName,
6342
+ ["/usr/bin/pgrep", "-f", "code-server.*--bind-addr"],
6343
+ 10_000,
6344
+ );
6345
+ if (!isCurrent()) return true;
6346
+ if (!workbenchReady && !(await dockerDaemonReachable())) return false;
6347
+ if (!isCurrent()) return true;
6348
+ if (!workbenchReady) {
6349
+ console.log(
6350
+ `[orchestrator] recovery: ${task.taskId} running without its workbench; quarantining before runtime proof`,
6351
+ );
6352
+ // Post-boot recovery can race live agent sessions. Drain their external
6353
+ // work and tombstone the channel before stopping the shared-volume
6354
+ // writer; successful recovery reopens the preserved channel spec below.
6355
+ if (lifecycle) {
6356
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6357
+ if (!isCurrent()) return true;
6358
+ }
6359
+ const quarantined = await quarantineRunningContainer(containerName);
6360
+ if (!isCurrent()) return true;
6361
+ if (quarantined === "unreachable") return false;
6362
+ if (quarantined === "failed") {
6363
+ db_setStatus(task.taskId, "stopped", {
6364
+ codeServerPort: null,
6365
+ previewPorts: "[]",
6366
+ });
6367
+ throw new Error(
6368
+ `${task.taskId} could not prove its running app stopped after stop and kill; task remains quarantined`,
6369
+ );
6370
+ }
6371
+ running = false;
6372
+ } else {
3732
6373
  // Container is up. Re-discover the host port in case Docker
3733
6374
  // remapped it across restarts (it usually does for ephemeral
3734
6375
  // bindings).
3735
- const port = await dockerPort(containerName, 8080);
6376
+ const portResult = await dockerPort(containerName, 8080);
6377
+ if (!isCurrent()) return true;
6378
+ if (!portResult.reachable) return false;
6379
+ const port = portResult.port;
3736
6380
  if (port && port !== task.codeServerPort) {
3737
6381
  db_setRuntime(task.taskId, {
3738
6382
  codeServerPort: port,
@@ -3759,30 +6403,113 @@ async function recoverOneTask(
3759
6403
  }`,
3760
6404
  ),
3761
6405
  );
6406
+ if (!isCurrent()) return true;
6407
+ if (lifecycle) {
6408
+ lifecycle.allowChannel(task.taskId);
6409
+ await lifecycle.ensureStarted(task.taskId);
6410
+ if (!isCurrent()) {
6411
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6412
+ }
6413
+ }
3762
6414
  return true;
6415
+ }
3763
6416
  }
3764
6417
 
3765
- // Container exists but exited. Bring it back up + re-launch the
3766
- // in-container init (deps + code-server).
3767
- console.log(
3768
- `[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
6418
+ // The exact app container is now stopped. Prove every project runtime from
6419
+ // its immutable image and mounts before bringing it back up. Rolling old
6420
+ // images receive the current packaged materializer/Corepack pin as read-only
6421
+ // binds, so recovery cannot bypass a newly-shipped proof contract.
6422
+ const runtimeInspection = await inspectRecoveryRuntime(task, containerName);
6423
+ if (!isCurrent()) return true;
6424
+ if (runtimeInspection.kind === "unreachable") return false;
6425
+ if (runtimeInspection.kind === "failed") {
6426
+ console.error(
6427
+ `[orchestrator] recovery: ${task.taskId} runtime inspection failed; keeping task stopped: ${runtimeInspection.detail}`,
6428
+ );
6429
+ db_setStatus(task.taskId, "stopped", {
6430
+ codeServerPort: null,
6431
+ previewPorts: "[]",
6432
+ });
6433
+ return true;
6434
+ }
6435
+ const runtimeMaterialization = await materializeRecoveryRuntimes(
6436
+ runtimeInspection.spec,
6437
+ );
6438
+ if (!isCurrent()) return true;
6439
+ if (runtimeMaterialization.kind === "unreachable") return false;
6440
+ if (runtimeMaterialization.kind === "failed") {
6441
+ console.error(
6442
+ `[orchestrator] recovery: ${task.taskId} runtime materialization failed; keeping task stopped and requiring a complete retry: ${runtimeMaterialization.detail}`,
6443
+ );
6444
+ db_setStatus(task.taskId, "stopped", {
6445
+ codeServerPort: null,
6446
+ previewPorts: "[]",
6447
+ });
6448
+ return true;
6449
+ }
6450
+ const stagedInit = await stageRecoveryInit(
6451
+ containerName,
6452
+ runtimeInspection.spec,
6453
+ );
6454
+ if (!isCurrent()) return true;
6455
+ if (stagedInit.kind === "unreachable") return false;
6456
+ if (stagedInit.kind === "failed") {
6457
+ console.error(
6458
+ `[orchestrator] recovery: ${task.taskId} could not stage the current task initializer; keeping task stopped: ${stagedInit.detail}`,
6459
+ );
6460
+ db_setStatus(task.taskId, "stopped", {
6461
+ codeServerPort: null,
6462
+ previewPorts: "[]",
6463
+ });
6464
+ return true;
6465
+ }
6466
+ const preStartWorkspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
6467
+ runtimeInspection.spec.workspaceSource,
3769
6468
  );
3770
- if (!(await dockerStart(containerName))) {
6469
+ if (preStartWorkspaceRuntimeError) {
6470
+ console.error(
6471
+ `[orchestrator] recovery: ${task.taskId} workspace runtime authority changed after proof; keeping task stopped: ${preStartWorkspaceRuntimeError}`,
6472
+ );
3771
6473
  db_setStatus(task.taskId, "stopped", {
3772
6474
  codeServerPort: null,
3773
6475
  previewPorts: "[]",
3774
6476
  });
3775
6477
  return true;
3776
6478
  }
6479
+
6480
+ // Runtime proof succeeded. Bring the preserved app back up + re-launch the
6481
+ // task-local dependency/editor init.
6482
+ // If Docker appeared after the host's initial no-runtime verdict, the
6483
+ // standard-image repair starts concurrently with this boot sweep. uai-init
6484
+ // is now task-local (dependencies + editor), but it still resolves through
6485
+ // shared asdf shims; never run it until the boot owner has finished
6486
+ // reconciling the image and agent CLI volume.
6487
+ if (!running) {
6488
+ console.log(
6489
+ `[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
6490
+ );
6491
+ const started = await dockerStart(containerName);
6492
+ if (!isCurrent()) return true;
6493
+ if (started === "unreachable") return false;
6494
+ if (started === "failed") {
6495
+ db_setStatus(task.taskId, "stopped", {
6496
+ codeServerPort: null,
6497
+ previewPorts: "[]",
6498
+ });
6499
+ return true;
6500
+ }
6501
+ }
3777
6502
  // uai-init seeds code-server's profile beneath this shared data root. Heal
3778
6503
  // containers created by older hosts before running it as node.
3779
6504
  await repairNodeDataRoot(containerName);
6505
+ if (!isCurrent()) return true;
3780
6506
  // Correctly-owned Codex creds BEFORE uai-init / any agent respawn: the boot
3781
6507
  // reinject sweep only targets containers already running, so a container
3782
6508
  // recovered here would otherwise keep whatever it held when it exited —
3783
6509
  // possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
3784
6510
  // by the inject itself and must not abort the task's recovery.
3785
6511
  await injectCodexIntoContainer(containerName);
6512
+ if (!isCurrent()) return true;
3786
6513
  // Establish the task owner's GitHub transport BEFORE uai-init performs any
3787
6514
  // dependency-network work. This matters for package manifests that refer to
3788
6515
  // private GitHub repositories: gh's credential helper must already be live
@@ -3790,7 +6517,20 @@ async function recoverOneTask(
3790
6517
  // like the prior post-init call, but it is now ordered rather than raced.
3791
6518
  if (task.ownerUserId) {
3792
6519
  try {
3793
- await reconcileTaskGitAuth(task.taskId, task.ownerUserId);
6520
+ await reconcileTaskGitAuth(task.taskId, task.ownerUserId, {
6521
+ // Recovery is sanctioned private maintenance while public admission is
6522
+ // deliberately `checking`. Fence it by this daemon epoch instead of
6523
+ // the public operational bit, preserving private-dependency setup
6524
+ // before uai-init without letting stale recovery keep mutating.
6525
+ admitContainerWork: () => {
6526
+ if (!isCurrent()) {
6527
+ throw new ContainerRuntimeUnavailableError(
6528
+ "container runtime recovery generation was superseded",
6529
+ );
6530
+ }
6531
+ },
6532
+ });
6533
+ if (!isCurrent()) return true;
3794
6534
  } catch (err) {
3795
6535
  console.warn(
3796
6536
  `[orchestrator] recovery: ${task.taskId} GitHub auth setup failed before uai-init: ${
@@ -3798,6 +6538,7 @@ async function recoverOneTask(
3798
6538
  }`,
3799
6539
  );
3800
6540
  }
6541
+ if (!isCurrent()) return true;
3801
6542
  }
3802
6543
  // uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
3803
6544
  // repo. dockerCli's 30s default would SIGKILL it mid-install.
@@ -3806,17 +6547,119 @@ async function recoverOneTask(
3806
6547
  "exec",
3807
6548
  "-e",
3808
6549
  "UAI_SKIP_GIT_TRANSPORT=1",
6550
+ "-e",
6551
+ "UAI_WORKSPACE=/workspace",
6552
+ "-e",
6553
+ `UAI_RUNTIME_PROJECTS=${RECOVERY_INIT_PROJECTS_PATH}`,
6554
+ "-e",
6555
+ "PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
6556
+ "-e",
6557
+ "HOME=/home/node",
6558
+ "-e",
6559
+ "XDG_CONFIG_HOME=/home/node/.config",
6560
+ "-e",
6561
+ "XDG_DATA_HOME=/home/node/.local/share",
6562
+ "-e",
6563
+ "ASDF_DIR=/opt/asdf",
6564
+ "-e",
6565
+ "ASDF_DATA_DIR=/opt/asdf-data",
6566
+ "-e",
6567
+ "ASDF_CONFIG_FILE=/dev/null",
6568
+ "-e",
6569
+ "ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions",
6570
+ "-e",
6571
+ "ASDF_NODEJS_VERSION=",
6572
+ "-e",
6573
+ "ASDF_PYTHON_VERSION=",
6574
+ "-e",
6575
+ "ASDF_GOLANG_VERSION=",
6576
+ "-e",
6577
+ "ASDF_RUBY_VERSION=",
6578
+ "-e",
6579
+ "ASDF_RUST_VERSION=",
6580
+ "-e",
6581
+ "COREPACK_HOME=/opt/asdf-data/corepack",
6582
+ "-e",
6583
+ "COREPACK_DEFAULT_TO_LATEST=0",
6584
+ "-e",
6585
+ "COREPACK_ENV_FILE=0",
6586
+ "-e",
6587
+ "COREPACK_NPM_REGISTRY=https://registry.npmjs.org",
6588
+ "-e",
6589
+ "COREPACK_ENABLE_NETWORK=0",
6590
+ "-e",
6591
+ "COREPACK_ENABLE_AUTO_PIN=0",
6592
+ "-e",
6593
+ "COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
6594
+ "-e",
6595
+ "COREPACK_ENABLE_PROJECT_SPEC=1",
6596
+ "-e",
6597
+ "COREPACK_ENABLE_STRICT=1",
6598
+ "-e",
6599
+ "COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0",
6600
+ "-e",
6601
+ "COREPACK_INTEGRITY_CHECK=1",
6602
+ "-e",
6603
+ "LD_PRELOAD=",
6604
+ "-e",
6605
+ "LD_AUDIT=",
6606
+ "-e",
6607
+ "LD_LIBRARY_PATH=",
6608
+ "-e",
6609
+ "NODE_OPTIONS=",
6610
+ "-e",
6611
+ "NODE_PATH=",
6612
+ // SHELLOPTS/BASHOPTS deliberately absent: present-in-env (even empty)
6613
+ // makes bash export its live options into children.
6614
+ "-e",
6615
+ "PS4=",
6616
+ "-e",
6617
+ "BASH_ENV=",
6618
+ "-e",
6619
+ "ENV=",
3809
6620
  containerName,
3810
- "/usr/local/bin/uai-init",
6621
+ // The host-side deadline below only kills the local docker client; the
6622
+ // server-side exec would keep mutating workspace dependencies while the
6623
+ // recovered task is already exposed to agents. This in-container
6624
+ // deadline kills the actual initializer first (TERM, then KILL), and
6625
+ // the host allowance is deliberately wider so a timed-out run settles
6626
+ // with exit 124 instead of racing the client kill.
6627
+ "/usr/bin/timeout",
6628
+ "--signal=TERM",
6629
+ "--kill-after=15s",
6630
+ "570s",
6631
+ RECOVERY_INIT_PATH,
3811
6632
  ],
3812
6633
  { timeoutMs: 10 * 60_000 },
3813
6634
  );
6635
+ if (!isCurrent()) return true;
6636
+ if (initResult.status === 78) {
6637
+ const quarantined = await quarantineRunningContainer(containerName);
6638
+ if (!isCurrent()) return true;
6639
+ if (quarantined === "unreachable") return false;
6640
+ db_setStatus(task.taskId, "stopped", {
6641
+ codeServerPort: null,
6642
+ previewPorts: "[]",
6643
+ });
6644
+ if (quarantined === "failed") {
6645
+ throw new Error(
6646
+ `${task.taskId} reported a runtime-authority violation and could not be proven stopped`,
6647
+ );
6648
+ }
6649
+ console.error(
6650
+ `[orchestrator] recovery: ${task.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
6651
+ );
6652
+ return true;
6653
+ }
3814
6654
  if (initResult.status !== 0) {
3815
6655
  console.warn(
3816
6656
  `[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
3817
6657
  );
3818
6658
  }
3819
- const port = (await dockerPort(containerName, 8080)) ?? null;
6659
+ const portResult = await dockerPort(containerName, 8080);
6660
+ if (!isCurrent()) return true;
6661
+ if (!portResult.reachable) return false;
6662
+ const port = portResult.port;
3820
6663
  db_setRuntime(task.taskId, {
3821
6664
  codeServerPort: port,
3822
6665
  });
@@ -3830,6 +6673,14 @@ async function recoverOneTask(
3830
6673
  console.log(
3831
6674
  `[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
3832
6675
  );
6676
+ if (!isCurrent()) return true;
6677
+ if (lifecycle) {
6678
+ lifecycle.allowChannel(task.taskId);
6679
+ await lifecycle.ensureStarted(task.taskId);
6680
+ if (!isCurrent()) {
6681
+ await lifecycle.quarantineChannelForRecovery(task.taskId);
6682
+ }
6683
+ }
3833
6684
  return true;
3834
6685
  }
3835
6686