@runuai/host 0.9.14 → 0.9.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +22 -5
  2. package/db/migrations/0014_host_inventory_event_index.sql +1 -0
  3. package/db/migrations/0015_host_settings.sql +9 -0
  4. package/db/migrations/0016_task_environment.sql +2 -0
  5. package/db/migrations/meta/_journal.json +21 -0
  6. package/db/schema.ts +80 -30
  7. package/images/standard/Dockerfile +36 -10
  8. package/images/standard/README.md +63 -18
  9. package/images/standard/container/corepack-version +1 -0
  10. package/images/standard/container/uai-init +308 -38
  11. package/images/standard/container/uai-materialize-runtimes +1527 -0
  12. package/lib/agent-cli.ts +33 -2
  13. package/lib/agent.ts +46 -7
  14. package/lib/agents/claude.ts +13 -8
  15. package/lib/agents/codex.ts +11 -6
  16. package/lib/agents/cursor.ts +39 -29
  17. package/lib/agents/durable-proc.ts +20 -27
  18. package/lib/agents/factory.ts +9 -25
  19. package/lib/agents/grok.ts +43 -30
  20. package/lib/agents/kimi.ts +44 -29
  21. package/lib/agents/opencode.ts +43 -31
  22. package/lib/agents/proc.ts +149 -114
  23. package/lib/agents/transport.ts +62 -50
  24. package/lib/agents/types.ts +6 -4
  25. package/lib/apple-runtime-recycle.ts +236 -0
  26. package/lib/apple-uninstall-teardown.ts +224 -0
  27. package/lib/browser-testing.ts +233 -93
  28. package/lib/codex-auth.ts +40 -6
  29. package/lib/command-db.ts +20 -0
  30. package/lib/container-runtime.ts +1338 -0
  31. package/lib/db.ts +1 -0
  32. package/lib/docker-exec.ts +87 -5
  33. package/lib/engine-accounts.ts +68 -5
  34. package/lib/engine-login.ts +1952 -0
  35. package/lib/enrollment-state.ts +251 -0
  36. package/lib/env-file.ts +155 -0
  37. package/lib/env.ts +4 -0
  38. package/lib/git-diff.ts +98 -32
  39. package/lib/git-identity.ts +199 -87
  40. package/lib/github-tokens.ts +202 -91
  41. package/lib/host-cloud-url.ts +62 -0
  42. package/lib/host-config.ts +279 -0
  43. package/lib/host-logs.ts +962 -0
  44. package/lib/keyed-promise-tail.ts +23 -0
  45. package/lib/legacy-runtime-v1.fixture.ts +627 -0
  46. package/lib/managed-activation-watcher.ts +72 -0
  47. package/lib/managed-install-owner-watcher.ts +55 -0
  48. package/lib/managed-operation-drain.ts +49 -0
  49. package/lib/managed-runtime.ts +3644 -0
  50. package/lib/managed-update-scheduler.ts +125 -0
  51. package/lib/mcp-gateway.ts +450 -23
  52. package/lib/orchestrator.ts +3051 -200
  53. package/lib/preview-sidecar.ts +68 -14
  54. package/lib/release-manifest.ts +708 -0
  55. package/lib/release-trust.ts +28 -0
  56. package/lib/runtime-activation-tail.ts +232 -0
  57. package/lib/runtime-archive.ts +1086 -0
  58. package/lib/runtime-authority.ts +79 -0
  59. package/lib/runtime-guard.ts +36 -0
  60. package/lib/runtime-provider-state.ts +169 -0
  61. package/lib/runtime-state.ts +232 -12
  62. package/lib/skills.ts +24 -3
  63. package/lib/ssh.ts +18 -0
  64. package/lib/standard-image.ts +1104 -141
  65. package/lib/stopped-task-status-queue.ts +44 -0
  66. package/lib/task-container-cli.ts +269 -0
  67. package/lib/task-diff.ts +66 -46
  68. package/lib/task-environment/apple-container.ts +757 -0
  69. package/lib/task-environment/docker.ts +956 -0
  70. package/lib/task-environment/index.ts +364 -0
  71. package/lib/task-environment/legacy-adoption.ts +459 -0
  72. package/lib/task-environment/registry.ts +58 -0
  73. package/lib/task-environment/types.ts +408 -0
  74. package/lib/task-identity.ts +19 -0
  75. package/lib/task-inventory.ts +585 -0
  76. package/lib/tunnel-registry.ts +135 -19
  77. package/lib/tunnel-runtime.ts +235 -0
  78. package/package.json +1 -1
  79. package/scripts/agent/_common.sh +123 -3
  80. package/scripts/agent/task-down.sh +146 -38
  81. package/scripts/agent/task-status.sh +19 -3
  82. package/scripts/agent/task-up.sh +1405 -107
  83. package/scripts/install/darwin.ts +848 -50
  84. package/scripts/install/linux.ts +838 -35
  85. package/scripts/install/types.ts +43 -0
  86. package/scripts/install/util.ts +215 -8
  87. package/scripts/install/win.ts +12 -0
  88. package/src/apple-tunnel-route.ts +104 -0
  89. package/src/cli.ts +1464 -72
  90. package/src/event-outbox.ts +83 -4
  91. package/src/index.ts +766 -42
  92. package/src/main.ts +1398 -255
  93. package/src/paths.ts +17 -1
  94. package/src/protocol.ts +695 -1
  95. package/src/runtime-bootstrap.ts +165 -0
  96. package/src/ui/server.ts +46 -10
  97. package/src/ui/types.ts +37 -0
@@ -18,16 +18,38 @@
18
18
 
19
19
  import { spawn } from "node:child_process";
20
20
  import { createHash } from "node:crypto";
21
+ import { readFileSync, writeFileSync } from "node:fs";
21
22
  import { readdir, readFile } from "node:fs/promises";
22
23
  import { dirname, join, resolve } from "node:path";
23
24
  import { fileURLToPath } from "node:url";
24
25
 
26
+ import {
27
+ MAINTENANCE_CONTAINER_LABEL,
28
+ recycleAppleRuntimeServices,
29
+ } from "./apple-runtime-recycle";
25
30
  import { detectEngine } from "./engines";
31
+ import {
32
+ appleContainerRuntimeBinding,
33
+ claimContainerRuntimeForMaintenance,
34
+ containerRuntimeStandardAssetsRoot,
35
+ containerRuntimeStandardImageRoot,
36
+ pinnedContainerRuntimeProvider,
37
+ } from "./container-runtime";
38
+ import { KeyedPromiseTail } from "./keyed-promise-tail";
26
39
 
27
40
  /** Pinned, host-wide constants (must match task-up.sh and the compose gen). */
28
41
  export const STANDARD_IMAGE_TAG = "uai-standard:dev";
29
42
  export const ASDF_DATA_VOLUME = "uai-asdf-data";
30
43
 
44
+ /**
45
+ * The shared-volume agent CLIs belong to this exact HOME-default Node. A
46
+ * future default bump cannot transparently roll stopped containers: their
47
+ * baked /home/node/.tool-versions still names the old CLI-owning candidate.
48
+ * Changing this value therefore requires an explicit volume migration or
49
+ * container-recreation release slice, not an ordinary image refresh.
50
+ */
51
+ export const STANDARD_DEFAULT_NODE_VERSION = "22.11.0";
52
+
31
53
  /**
32
54
  * Runtimes the standard image's asdf plugins can satisfy, with a sensible set
33
55
  * of advertised versions per kind (ADR-021 `runtimes`). These are the
@@ -43,7 +65,14 @@ export const STANDARD_RUNTIMES: ReadonlyArray<{
43
65
  kind: string;
44
66
  availableVersions: string[];
45
67
  }> = [
46
- { kind: "node", availableVersions: ["22.11.0", "20.18.0", "18.20.4"] },
68
+ {
69
+ kind: "node",
70
+ availableVersions: [
71
+ STANDARD_DEFAULT_NODE_VERSION,
72
+ "20.18.0",
73
+ "18.20.4",
74
+ ],
75
+ },
47
76
  { kind: "python", availableVersions: ["3.12.4", "3.11.9"] },
48
77
  { kind: "go", availableVersions: ["1.23.2", "1.22.8"] },
49
78
  { kind: "ruby", availableVersions: ["3.3.5", "3.2.5"] },
@@ -91,6 +120,16 @@ export function configuredOptionalEngines(): {
91
120
  };
92
121
  }
93
122
 
123
+ type OptionalEngineConfiguration = ReturnType<
124
+ typeof configuredOptionalEngines
125
+ >;
126
+
127
+ function optionalEngineConfigurationKey(
128
+ engines: OptionalEngineConfiguration,
129
+ ): string {
130
+ return JSON.stringify(engines);
131
+ }
132
+
94
133
  async function hashBuildContext(extra = ""): Promise<string | null> {
95
134
  try {
96
135
  const root = standardImageDir();
@@ -123,23 +162,103 @@ async function hashBuildContext(extra = ""): Promise<string | null> {
123
162
 
124
163
  /** Absolute path to the standard image build context. */
125
164
  function standardImageDir(): string {
165
+ const providerRoot = containerRuntimeStandardImageRoot();
166
+ if (providerRoot !== null) return providerRoot;
126
167
  const here = dirname(fileURLToPath(import.meta.url));
127
168
  // lib/standard-image.ts -> host-agent/ -> images/standard
128
169
  return resolve(here, "..", "images", "standard");
129
170
  }
130
171
 
172
+ /** Absolute packaged path used to bind the materializer over rolling images
173
+ * that predate the baked executable. Resolution is deterministic; callers get
174
+ * the ordinary Docker bind/open error if a package install is corrupt. */
175
+ export function standardImageRuntimeMaterializerPath(): string {
176
+ return join(
177
+ standardContainerAssetsDir(),
178
+ "uai-materialize-runtimes",
179
+ );
180
+ }
181
+
182
+ /** Absolute packaged Corepack pin bound over both rolling and derived images. */
183
+ export function standardImageCorepackVersionPath(): string {
184
+ return join(standardContainerAssetsDir(), "corepack-version");
185
+ }
186
+
187
+ /** ADR-106: the retired bundled-VM fallback exposed a provider-specific
188
+ * asset root; every current backend uses the package's ordinary
189
+ * build-context assets, so the runtime override below is always null. */
190
+ function standardContainerAssetsDir(): string {
191
+ return (
192
+ containerRuntimeStandardAssetsRoot() ?? join(standardImageDir(), "container")
193
+ );
194
+ }
195
+
131
196
  interface RunResult {
132
197
  code: number | null;
133
198
  stdout: string;
134
199
  stderr: string;
135
200
  }
136
201
 
137
- /** Run a command to completion, capturing stdio. Never rejects. */
138
- function run(command: string, args: string[]): Promise<RunResult> {
202
+ const SHORT_DOCKER_TIMEOUT_MS = 30_000;
203
+ const CLI_MAINTENANCE_TIMEOUT_MS = 10 * 60_000;
204
+ const IMAGE_BUILD_TIMEOUT_MS = 15 * 60_000;
205
+ const TERMINATE_GRACE_MS = 5_000;
206
+ export const ASDF_MAINTENANCE_LOCK_PATH =
207
+ "/opt/asdf-data/.uai-maintenance.lock";
208
+ const ASDF_MAINTENANCE_LOCK_WAIT_SECONDS = 300;
209
+ const ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS = 570;
210
+
211
+ /** The selected container engine for image/volume maintenance (ADR-106).
212
+ * Resolved per call: the runtime pin settles before ensureStandardImage runs,
213
+ * but an HMR reload must not strand a stale CLI path. The docker fallback
214
+ * covers the pre-selection window, where docker-only behavior is historical. */
215
+ function selectedEngine(): { command: string; apple: boolean } {
216
+ // The PIN, not the readiness-gated identity: this runs during activation
217
+ // while the public state is still `checking`, and falling back to docker
218
+ // there deadlocks a Docker-free Mac.
219
+ if (pinnedContainerRuntimeProvider() === "apple-container") {
220
+ const binding = appleContainerRuntimeBinding();
221
+ if (binding !== null) {
222
+ return { command: binding.containerCliPath, apple: true };
223
+ }
224
+ }
225
+ return { command: "docker", apple: false };
226
+ }
227
+
228
+ /** Run a command to completion, capturing stdio. Never rejects and never
229
+ * waits forever: standard-image maintenance is an activation prerequisite,
230
+ * so an unbounded Docker/npm child would strand the entire host in checking. */
231
+ function run(
232
+ command: string,
233
+ args: string[],
234
+ timeoutMs = SHORT_DOCKER_TIMEOUT_MS,
235
+ ): Promise<RunResult> {
139
236
  return new Promise<RunResult>((resolveRun) => {
140
237
  let stdout = "";
141
238
  let stderr = "";
239
+ let settled = false;
240
+ let killTimer: NodeJS.Timeout | null = null;
142
241
  const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
242
+ const settle = (code: number | null): void => {
243
+ if (settled) return;
244
+ settled = true;
245
+ clearTimeout(timeoutTimer);
246
+ if (killTimer) clearTimeout(killTimer);
247
+ resolveRun({ code, stdout, stderr });
248
+ };
249
+ const timeoutTimer = setTimeout(() => {
250
+ stderr += `${stderr && !stderr.endsWith("\n") ? "\n" : ""}${command} timed out after ${timeoutMs}ms`;
251
+ child.kill("SIGTERM");
252
+ killTimer = setTimeout(() => {
253
+ child.kill("SIGKILL");
254
+ // SIGKILL should always produce close, but do not let a broken child
255
+ // handle defeat the host-wide readiness bound.
256
+ const settleTimer = setTimeout(() => settle(null), 1_000);
257
+ settleTimer.unref();
258
+ }, TERMINATE_GRACE_MS);
259
+ killTimer.unref();
260
+ }, timeoutMs);
261
+ timeoutTimer.unref();
143
262
  child.stdout?.setEncoding("utf8");
144
263
  child.stderr?.setEncoding("utf8");
145
264
  child.stdout?.on("data", (chunk: string) => {
@@ -149,14 +268,220 @@ function run(command: string, args: string[]): Promise<RunResult> {
149
268
  stderr += chunk;
150
269
  });
151
270
  child.on("error", (err: Error) => {
152
- resolveRun({ code: null, stdout, stderr: stderr + err.message });
271
+ stderr += err.message;
272
+ settle(null);
153
273
  });
154
274
  child.on("close", (code) => {
155
- resolveRun({ code, stdout, stderr });
275
+ settle(code);
156
276
  });
157
277
  });
158
278
  }
159
279
 
280
+ /** Narrow process-level seam for the timeout regression test. */
281
+ export const runStandardImageCommandForTest = run;
282
+
283
+ let maintenanceContainerSeq = 0;
284
+
285
+ /** How long a maintenance run may sit without its container reaching
286
+ * `running` before the wedge sensor passes judgment. Long HEALTHY operations
287
+ * (reconcile's npm installs, the seed copy) are past `running` within
288
+ * seconds, so this discriminates cleanly. Env-overridable for tests. */
289
+ function wedgeProbeDelayMs(): number {
290
+ const raw = Number(process.env.UAI_APPLE_WEDGE_PROBE_MS);
291
+ return Number.isFinite(raw) && raw > 0 ? raw : 60_000;
292
+ }
293
+
294
+ /** Post-recycle grace for the hung CLI client before a synthesized result. */
295
+ function wedgeGraceMs(): number {
296
+ const raw = Number(process.env.UAI_APPLE_WEDGE_GRACE_MS);
297
+ return Number.isFinite(raw) && raw > 0 ? raw : 10_000;
298
+ }
299
+
300
+ function delayMs(ms: number): Promise<void> {
301
+ return new Promise((resolveDelay) => {
302
+ const timer = setTimeout(resolveDelay, ms);
303
+ timer.unref();
304
+ });
305
+ }
306
+
307
+ /**
308
+ * One DEFINITIVE observation of the named container: a successful, parseable
309
+ * inspect reporting a non-running state. Anything else — inspect failure,
310
+ * unparseable output, missing state, or `running` — is NOT evidence of a
311
+ * wedge and must stand the sensor down (the old fall-through treated
312
+ * uncertainty as wedge evidence, which authorizes a service recycle on a
313
+ * guess).
314
+ */
315
+ async function appleContainerDefinitivelyNotRunning(
316
+ engine: { command: string },
317
+ name: string,
318
+ ): Promise<boolean> {
319
+ const inspected = await run(engine.command, ["inspect", name], 15_000);
320
+ if (inspected.code !== 0) return false;
321
+ try {
322
+ const parsed = JSON.parse(inspected.stdout) as Array<{
323
+ id?: unknown;
324
+ status?: { state?: unknown };
325
+ }>;
326
+ if (!Array.isArray(parsed) || parsed.length !== 1) return false;
327
+ if (parsed[0]?.id !== name) return false;
328
+ const state = parsed[0]?.status?.state;
329
+ return typeof state === "string" && state !== "running";
330
+ } catch {
331
+ return false;
332
+ }
333
+ }
334
+
335
+ /**
336
+ * The wedge signature, judged live: the named container has NOT reached
337
+ * `running` (definitively — see above) while the control plane still answers
338
+ * a `list`, observed STABLY: a second exact inspection after the list (and a
339
+ * short settle) must repeat the non-running verdict immediately before any
340
+ * recycle is authorized, closing the window where the container starts while
341
+ * the list is in flight. A dead apiserver fails the `list` too (that is an
342
+ * outage, not a wedge, and recycling on it would be a guess).
343
+ */
344
+ async function appleWedgeVerdict(
345
+ engine: { command: string },
346
+ name: string,
347
+ ): Promise<boolean> {
348
+ if (!(await appleContainerDefinitivelyNotRunning(engine, name))) {
349
+ return false;
350
+ }
351
+ const listed = await run(engine.command, ["list", "--format", "json"], 15_000);
352
+ if (listed.code !== 0) return false;
353
+ await delayMs(Math.min(5_000, wedgeProbeDelayMs()));
354
+ return appleContainerDefinitivelyNotRunning(engine, name);
355
+ }
356
+
357
+ /**
358
+ * Run a maintenance container with corpse hygiene on the Apple runtime.
359
+ *
360
+ * `container run --rm` only removes a container that STARTED. A run refused
361
+ * at start time (the exclusive-volume attach, most commonly) leaves a
362
+ * created-never-started container behind, and those corpses wedge the
363
+ * runtime's apiserver (observed live 2026-08-16, twice: run/delete hang
364
+ * while list still answers). Naming the container makes the corpse
365
+ * addressable, so a failed run is followed by a bounded best-effort
366
+ * `delete --force` of that exact name. Docker's `--rm` is reliable and its
367
+ * containers are not corpse-prone, so the Docker path passes through.
368
+ *
369
+ * Every apple run doubles as the WEDGE SENSOR (the owner's "this needs to
370
+ * be automatic"): if the container has not reached `running` after the
371
+ * probe delay while `list` still answers, the runtime services are recycled
372
+ * in place (see apple-runtime-recycle.ts) and the run reports a distinctive
373
+ * retryable failure — the activation retry loop then re-runs maintenance
374
+ * against the fresh runtime.
375
+ */
376
+ async function runMaintenanceContainer(
377
+ engine: { command: string; apple: boolean },
378
+ purpose: string,
379
+ argsAfterRun: string[],
380
+ timeoutMs: number,
381
+ ): Promise<RunResult> {
382
+ if (!engine.apple) {
383
+ return run(engine.command, ["run", ...argsAfterRun], timeoutMs);
384
+ }
385
+ maintenanceContainerSeq += 1;
386
+ const name = `uai-maint-${purpose}-${process.pid}-${maintenanceContainerSeq}`;
387
+ let runSettled = false;
388
+ const running = run(
389
+ engine.command,
390
+ // The ownership label is what authorizes the recycle sweep (and only the
391
+ // sweep) to delete this container later — a name alone is forgeable.
392
+ ["run", "--name", name, "-l", `${MAINTENANCE_CONTAINER_LABEL}=1`, ...argsAfterRun],
393
+ timeoutMs,
394
+ ).then((outcome) => {
395
+ runSettled = true;
396
+ return outcome;
397
+ });
398
+ const sensed = (async (): Promise<RunResult | null> => {
399
+ await delayMs(wedgeProbeDelayMs());
400
+ if (runSettled) return null;
401
+ if (!(await appleWedgeVerdict(engine, name))) return null;
402
+ if (runSettled) return null;
403
+ const outcome = await recycleAppleRuntimeServices({
404
+ cli: (args, cliTimeoutMs) => run(engine.command, args, cliTimeoutMs),
405
+ });
406
+ // The hung CLI client may exit once its apiserver dies; give it a short
407
+ // grace so a real result wins, then synthesize the retryable failure.
408
+ const graced = await Promise.race([
409
+ running,
410
+ delayMs(wedgeGraceMs()).then(() => null),
411
+ ]);
412
+ if (graced) return graced;
413
+ return {
414
+ code: null,
415
+ stdout: "",
416
+ stderr: outcome.recycled
417
+ ? "apple container runtime was wedged; runtime services were recycled — retry this operation"
418
+ : `apple container runtime is wedged and the automatic recycle did not recover it: ${outcome.reason}`,
419
+ };
420
+ })();
421
+ const result = await Promise.race([
422
+ running,
423
+ sensed.then((synthesized) => synthesized ?? running),
424
+ ]);
425
+ if (result.code !== 0) {
426
+ // The run may have failed BECAUSE this generated name already existed —
427
+ // then the container is not ours and the ownership label was never
428
+ // applied (review 2026-08-18 round 3). Deletion requires a positive
429
+ // label proof; an absent container is done; anything uncertain is left
430
+ // untouched and reported.
431
+ const inspected = await run(engine.command, ["inspect", name], 15_000);
432
+ let verdict: "ours" | "absent" | "foreign" | "unknown" = "unknown";
433
+ if (
434
+ inspected.code !== 0 &&
435
+ inspected.stdout.trim() === "" &&
436
+ inspected.stderr
437
+ .split("\n")
438
+ .some((line) => line.trim() === `Error: container not found: ${name}`)
439
+ ) {
440
+ verdict = "absent";
441
+ } else if (inspected.code === 0) {
442
+ try {
443
+ const parsed = JSON.parse(inspected.stdout) as Array<{
444
+ id?: unknown;
445
+ configuration?: { labels?: Record<string, unknown> };
446
+ }>;
447
+ if (Array.isArray(parsed) && parsed.length === 1 && parsed[0]?.id === name) {
448
+ verdict =
449
+ parsed[0]?.configuration?.labels?.[MAINTENANCE_CONTAINER_LABEL] === "1"
450
+ ? "ours"
451
+ : "foreign";
452
+ }
453
+ } catch {
454
+ verdict = "unknown";
455
+ }
456
+ }
457
+ if (verdict === "ours") {
458
+ const removed = await run(
459
+ engine.command,
460
+ ["delete", "--force", name],
461
+ 30_000,
462
+ );
463
+ if (removed.code !== 0) {
464
+ console.warn(
465
+ `[host-agent] could not remove maintenance container ${name} after a ` +
466
+ `failed run (exit ${removed.code ?? "spawn"}); a created-never-started ` +
467
+ `container can wedge the Apple runtime. ${removed.stderr.trim().slice(0, 200)}`,
468
+ );
469
+ }
470
+ } else if (verdict === "foreign") {
471
+ console.warn(
472
+ `[host-agent] a container named ${name} exists WITHOUT Uai's maintenance ` +
473
+ "ownership label — the failed run likely collided with it; leaving it untouched",
474
+ );
475
+ } else if (verdict === "unknown") {
476
+ console.warn(
477
+ `[host-agent] could not prove ownership of maintenance container ${name} ` +
478
+ `after a failed run; leaving it untouched (inspect exit ${inspected.code ?? "spawn"})`,
479
+ );
480
+ }
481
+ }
482
+ return result;
483
+ }
484
+
160
485
  /**
161
486
  * Agent CLIs that MUST exist inside the shared asdf volume. The volume mounts
162
487
  * over /opt/asdf-data and shadows the image's baked-in shims, so a volume
@@ -170,13 +495,11 @@ const VOLUME_AGENT_CLIS: { bin: string; pkg: string }[] = [
170
495
  ];
171
496
 
172
497
  /**
173
- * Self-heal the shared asdf volume: ensure each agent CLI resolves inside it,
174
- * installing + reshimming any that are missing. Cheap when healthy (one
175
- * container that just lists absent bins); only pays the npm cost on repair.
176
- * A fresh/empty volume is seeded from the image by Docker on first mount, so
177
- * this is a no-op there — it only fixes pre-existing stale volumes.
498
+ * Cheap, non-mutating diagnostic probe. This is never the authoritative final
499
+ * proof: another process can begin maintenance after a read-only check. The
500
+ * locked reconcile below always performs its own proof before releasing.
178
501
  */
179
- async function ensureVolumeAgentClis(): Promise<void> {
502
+ async function probeVolumeAgentClis(): Promise<string[] | null> {
180
503
  const bins = VOLUME_AGENT_CLIS.map((c) => c.bin).join(" ");
181
504
  // Probe by EXECUTING each bin (`--version`), not `command -v`. A stale asdf
182
505
  // shim — the shim file survives in the volume but the reshim DB no longer
@@ -189,154 +512,534 @@ async function ensureVolumeAgentClis(): Promise<void> {
189
512
  // version from /home/node/.tool-versions — the same one the repair installs
190
513
  // against — and the sentinel prefix keeps login-shell noise from being
191
514
  // misread as a missing bin.
192
- const check = await run("docker", [
193
- "run",
515
+ const check = await runMaintenanceContainer(selectedEngine(), "cli-probe", [
194
516
  "--rm",
195
517
  "-w",
196
518
  "/home/node",
197
519
  "-v",
198
- `${ASDF_DATA_VOLUME}:/opt/asdf-data`,
520
+ `${ASDF_DATA_VOLUME}:/opt/asdf-data:ro`,
199
521
  STANDARD_IMAGE_TAG,
200
- "bash",
201
- "-lc",
522
+ "/bin/bash",
523
+ "-c",
202
524
  `for b in ${bins}; do "$b" --version >/dev/null 2>&1 || echo "UAI_MISSING:$b"; done`,
203
- ]);
525
+ ], SHORT_DOCKER_TIMEOUT_MS);
204
526
  if (check.code !== 0) {
205
527
  console.warn(
206
528
  `[host-agent] could not check agent CLIs in ${ASDF_DATA_VOLUME} (exit ` +
207
529
  `${check.code ?? "spawn"}); continuing. ${check.stderr.trim()}`,
208
530
  );
209
- return;
531
+ return null;
210
532
  }
211
- const missing = check.stdout
533
+ return check.stdout
212
534
  .split("\n")
213
535
  .map((l) => l.trim())
214
536
  .filter((l) => l.startsWith("UAI_MISSING:"))
215
537
  .map((l) => l.slice("UAI_MISSING:".length));
216
- if (missing.length === 0) return;
538
+ }
217
539
 
218
- const pkgs = VOLUME_AGENT_CLIS.filter((c) => missing.includes(c.bin))
219
- .map((c) => c.pkg)
220
- .join(" ");
221
- console.log(
222
- `[host-agent] repairing ${ASDF_DATA_VOLUME}: agent CLI(s) [${missing.join(", ")}] ` +
223
- `broken/missing in the shared volume — reshim + install ${pkgs}`,
224
- );
225
- const repair = await run("docker", [
226
- "run",
540
+ /**
541
+ * Create/repair the shared lock as a regular node-writable file. The lock is
542
+ * seeded into fresh volumes by the image. Legacy volumes can contain a 0644
543
+ * root-created file; opening it read-only permits flock, then metadata repair
544
+ * occurs while holding that same inode (never replace it under a waiter).
545
+ */
546
+ async function prepareAsdfMaintenanceLock(): Promise<boolean> {
547
+ const command =
548
+ `set -eu; lock=${ASDF_MAINTENANCE_LOCK_PATH}; ` +
549
+ `if [ -L "$lock" ]; then echo "unsafe symlink lock: $lock" >&2; exit 1; fi; ` +
550
+ `if [ ! -e "$lock" ]; then umask 002; : > "$lock"; fi; ` +
551
+ `if [ ! -f "$lock" ]; then echo "lock is not regular: $lock" >&2; exit 1; fi; ` +
552
+ `exec 9<"$lock"; /usr/bin/flock --exclusive --timeout ${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS} 9; ` +
553
+ `/usr/bin/chown node:node "$lock"; /usr/bin/chmod 0664 "$lock"`;
554
+ const result = await runMaintenanceContainer(selectedEngine(), "lock", [
227
555
  "--rm",
228
556
  "-u",
229
557
  "root",
230
- // Run from /home/node so asdf resolves the node version from
231
- // /home/node/.tool-versions. As root HOME is /root (no .tool-versions),
232
- // and without a version asdf's npm shim aborts ("No version is set for
233
- // command npm", exit 126). root can still write the volume from here.
234
- "-w",
235
- "/home/node",
236
558
  "-v",
237
559
  `${ASDF_DATA_VOLUME}:/opt/asdf-data`,
238
560
  STANDARD_IMAGE_TAG,
239
- "bash",
240
- "-lc",
241
- // Reshim FIRST and with `;` (not `&&`): the common failure is a stale shim
242
- // over an already-installed package, which a bare reshim fixes offline —
243
- // gating it behind a network `npm install` (which may be down) would leave
244
- // it broken. Then install to cover a genuinely-missing package, and reshim
245
- // again for the freshly-installed one. Every step runs regardless of the
246
- // previous one's exit. Final loop re-probes by execution so the exit code
247
- // (and the log below) reflect whether the bins ACTUALLY run now, not just
248
- // the trailing reshim's own exit.
249
- `asdf reshim nodejs; npm install -g ${pkgs}; asdf reshim nodejs; ` +
250
- `for b in ${bins}; do "$b" --version >/dev/null 2>&1 || ` +
251
- `{ echo "UAI_STILL_BROKEN:$b" >&2; exit 1; }; done`,
252
- ]);
253
- if (repair.code === 0) {
254
- console.log(
255
- `[host-agent] ${ASDF_DATA_VOLUME} repaired — installed [${missing.join(", ")}]`,
256
- );
257
- } else {
258
- console.warn(
259
- `[host-agent] failed to repair ${ASDF_DATA_VOLUME} (exit ${repair.code ?? "spawn"}); ` +
260
- `tasks may be missing [${missing.join(", ")}].\n${repair.stderr.trim()}`,
261
- );
262
- }
561
+ "/usr/bin/timeout",
562
+ "--signal=TERM",
563
+ "--kill-after=10s",
564
+ `${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
565
+ "/bin/bash",
566
+ "-c",
567
+ command,
568
+ ], CLI_MAINTENANCE_TIMEOUT_MS);
569
+ if (result.code === 0) return true;
570
+ lastMaintenanceStderr = result.stderr;
571
+ console.warn(
572
+ `[host-agent] could not prepare ${ASDF_MAINTENANCE_LOCK_PATH} (exit ` +
573
+ `${result.code ?? "spawn"}): ${result.stderr.trim()}`,
574
+ );
575
+ return false;
263
576
  }
264
577
 
265
578
  /**
266
- * Upgrade the shared volume's agent CLIs to latest — once per HOST START, not
267
- * per task. Per-task installs would add npm latency to every task-up, race
268
- * concurrent starts on the shared volume, and give a broken vendor release a
269
- * fleet-wide blast radius with no rollback; host restarts are the deliberate
270
- * update moment (sessions respawn then anyway, and a bad release is one
271
- * `UAI_AGENT_CLI_AUTOUPDATE=0` + manual pin away from contained). Running
272
- * sessions keep their already-loaded process; new spawns pick up the new bin.
273
- * Best-effort: an offline registry just logs and keeps the current versions.
579
+ * One cross-process transaction owns upgrade, repair, Corepack repair, reshim,
580
+ * and the authoritative execution proof. A waiter always acquires the lock and
581
+ * performs its own full operation; a concurrent holder is never treated as
582
+ * success. The in-container timeout survives a disconnected Docker client and
583
+ * remains strictly inside the host-side child timeout.
274
584
  */
275
- async function upgradeVolumeAgentClis(): Promise<void> {
276
- if (process.env.UAI_AGENT_CLI_AUTOUPDATE === "0") {
277
- console.log("[host-agent] agent CLI auto-update disabled (UAI_AGENT_CLI_AUTOUPDATE=0)");
278
- return;
279
- }
585
+ /** Build the single locked maintenance program. Exported only so focused
586
+ * tests can assert the crash-repair and proof contract without duplicating
587
+ * this security-sensitive shell in a fixture. */
588
+ export function standardImageVolumeMaintenanceCommandForTest(
589
+ autoUpdate: boolean,
590
+ ): string {
280
591
  const pkgs = VOLUME_AGENT_CLIS.map((c) => `${c.pkg}@latest`).join(" ");
281
- const bins = VOLUME_AGENT_CLIS.map((c) => c.bin);
282
- // npm's in-place upgrade renames the old package dir, which fails with
283
- // ENOTEMPTY while (or right after) live sessions held it open through the
284
- // volume. Fallback: remove the scoped dirs and install fresh — safe at host
285
- // start, when the previous sessions' execs are gone.
286
- const scopeDirs = "/opt/asdf-data/installs/nodejs/*/lib/node_modules/@anthropic-ai /opt/asdf-data/installs/nodejs/*/lib/node_modules/@openai";
287
- const upgrade = await run("docker", [
288
- "run",
592
+ const trustedNpmFlags =
593
+ "--registry=https://registry.npmjs.org/ --userconfig=/dev/null " +
594
+ "--globalconfig=/dev/uai-empty-npmrc";
595
+ return (
596
+ `set -u; upgrade=${autoUpdate ? 1 : 0}; ` +
597
+ `fail() { echo "UAI_STILL_BROKEN:$1" >&2; exit 1; }; ` +
598
+ `export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; ` +
599
+ `export HOME=/home/node ASDF_DIR=/opt/asdf ASDF_DATA_DIR=/opt/asdf-data ` +
600
+ `ASDF_CONFIG_FILE=/dev/null ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
601
+ `COREPACK_HOME=/opt/asdf-data/corepack COREPACK_DEFAULT_TO_LATEST=0 ` +
602
+ `COREPACK_ENABLE_AUTO_PIN=0 COREPACK_ENABLE_DOWNLOAD_PROMPT=0 ` +
603
+ `COREPACK_ENV_FILE=0 COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
604
+ `COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
605
+ `COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org; ` +
606
+ `default_node="$(/usr/bin/awk '$1 == "nodejs" { print $2; exit }' /home/node/.tool-versions)"; ` +
607
+ `if ! printf '%s' "$default_node" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$'; then ` +
608
+ `fail default-node; fi; ` +
609
+ `[ "$default_node" = "${STANDARD_DEFAULT_NODE_VERSION}" ] || ` +
610
+ `fail default-node-release-migration-required; ` +
611
+ `export ASDF_NODEJS_VERSION="$default_node"; ` +
612
+ `if [ -e /dev/uai-empty-npmrc ] || [ -L /dev/uai-empty-npmrc ]; then ` +
613
+ `fail unsafe-npm-config; fi; ` +
614
+ `installs_root="$ASDF_DATA_DIR/installs"; node_root="$installs_root/nodejs"; ` +
615
+ `candidate="$node_root/$default_node"; shims_root="$ASDF_DATA_DIR/shims"; ` +
616
+ `corepack_root="$COREPACK_HOME"; ` +
617
+ `ensure_real_dir() { path="$1"; label="$2"; ` +
618
+ `if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then ` +
619
+ `fail "unsafe-$label"; fi; ` +
620
+ `if [ ! -d "$path" ]; then /usr/bin/install -d -o node -g node -m 0775 "$path" ` +
621
+ `|| fail "create-$label"; fi; ` +
622
+ `lexical="$(/usr/bin/realpath -m -s -- "$path")"; ` +
623
+ `resolved="$(/usr/bin/realpath -e -- "$path")" ` +
624
+ `|| fail "resolve-$label"; [ "$lexical" = "$resolved" ] ` +
625
+ `|| fail "escaped-$label"; /usr/bin/chown node:node "$path" ` +
626
+ `|| fail "owner-$label"; /usr/bin/chmod u+rwx,g+rwx "$path" ` +
627
+ `|| fail "mode-$label"; }; ` +
628
+ `ensure_real_dir "$ASDF_DATA_DIR" asdf-data; ` +
629
+ `ensure_real_dir "$installs_root" installs-root; ` +
630
+ `ensure_real_dir "$node_root" node-root; ` +
631
+ `ensure_real_dir "$shims_root" shims-root; ` +
632
+ `ensure_real_dir "$corepack_root" corepack-root; ` +
633
+ `candidate_is_safe() { ` +
634
+ `lexical_candidate="$(/usr/bin/realpath -m -s -- "$candidate")" ` +
635
+ `|| return 1; [ "$lexical_candidate" = "$node_root/$default_node" ] ` +
636
+ `|| return 1; if [ -e "$candidate" ] || [ -L "$candidate" ]; then ` +
637
+ `[ ! -L "$candidate" ] && [ -d "$candidate" ] || return 1; ` +
638
+ `resolved_candidate="$(/usr/bin/realpath -e -- "$candidate")" ` +
639
+ `|| return 1; ` +
640
+ `[ "$resolved_candidate" = "$lexical_candidate" ] || return 1; fi; }; ` +
641
+ `contained_executable() { root="$1"; executable="$2"; ` +
642
+ `[ -f "$executable" ] && [ -x "$executable" ] || return 1; ` +
643
+ `real_executable="$(/usr/bin/realpath -e -- "$executable")" ` +
644
+ `|| return 1; ` +
645
+ `case "$real_executable" in "$root"/*) return 0 ;; *) return 1 ;; esac; }; ` +
646
+ `contained_real_dir() { root="$1"; path="$2"; ` +
647
+ `[ ! -L "$path" ] && [ -d "$path" ] || return 1; ` +
648
+ `lexical_path="$(/usr/bin/realpath -m -s -- "$path")" || return 1; ` +
649
+ `real_path="$(/usr/bin/realpath -e -- "$path")" || return 1; ` +
650
+ `case "$lexical_path" in "$root"/*) ;; *) return 1 ;; esac; ` +
651
+ `[ "$real_path" = "$lexical_path" ]; }; ` +
652
+ `npm_prefix_is_safe() { contained_real_dir "$candidate" "$candidate/bin" ` +
653
+ `&& contained_real_dir "$candidate" "$candidate/lib" ` +
654
+ `&& contained_real_dir "$candidate" "$candidate/lib/node_modules"; }; ` +
655
+ `scope_is_safe_or_absent() { scope="$1"; ` +
656
+ `if [ -e "$scope" ] || [ -L "$scope" ]; then ` +
657
+ `contained_real_dir "$candidate" "$scope"; fi; }; ` +
658
+ `as_node() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
659
+ `/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
660
+ `ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
661
+ `ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
662
+ `ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
663
+ `COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
664
+ `COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
665
+ `COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
666
+ `COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
667
+ `COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
668
+ `NPM_CONFIG_PREFIX="$candidate" ` +
669
+ `PATH="$candidate/bin:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
670
+ `"$@"; }; ` +
671
+ `as_node_control() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
672
+ `/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
673
+ `ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
674
+ `ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
675
+ `ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
676
+ `COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
677
+ `COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
678
+ `COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
679
+ `COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
680
+ `COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
681
+ `PATH="/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
682
+ `"$@"; }; ` +
683
+ `clear_forbidden_shared_bash_shim() { fail_if_found="$1"; ` +
684
+ `bash_shim="$shims_root/bash"; bash_shim_found=0; ` +
685
+ `if [ -e "$bash_shim" ] || [ -L "$bash_shim" ]; then ` +
686
+ `bash_shim_found=1; ` +
687
+ `[ ! -d "$bash_shim" ] || fail unsafe-runtime-bash-shim-directory; ` +
688
+ `as_node /usr/bin/rm -f -- "$bash_shim" ` +
689
+ `|| fail remove-runtime-bash-shim; fi; ` +
690
+ `[ ! -e "$bash_shim" ] && [ ! -L "$bash_shim" ] ` +
691
+ `|| fail unsafe-runtime-bash-shim; ` +
692
+ `if [ "$fail_if_found" -ne 0 ] && [ "$bash_shim_found" -ne 0 ]; then ` +
693
+ `return 1; fi; }; ` +
694
+ `clear_forbidden_candidate_bash() { fail_if_found="$1"; ` +
695
+ `candidate_bash="$candidate/bin/bash"; candidate_bash_found=''; ` +
696
+ `if [ -e "$candidate_bash" ] || [ -L "$candidate_bash" ]; then ` +
697
+ `candidate_bash_found=1; [ ! -d "$candidate_bash" ] ` +
698
+ `|| fail unsafe-default-bash-directory; ` +
699
+ `as_node /usr/bin/rm -f -- "$candidate_bash" ` +
700
+ `|| fail remove-default-bash; fi; ` +
701
+ `[ ! -e "$candidate_bash" ] && [ ! -L "$candidate_bash" ] ` +
702
+ `|| fail unsafe-default-bash; ` +
703
+ `if [ "$fail_if_found" -ne 0 ] && [ -n "$candidate_bash_found" ]; then ` +
704
+ `return 1; fi; }; ` +
705
+ `run_asdf_install() { asdf_status=0; shim_status=0; ` +
706
+ `as_node_control /opt/asdf/bin/asdf install "$@" || asdf_status=$?; ` +
707
+ `clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
708
+ `[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
709
+ `run_asdf_reshim() { asdf_status=0; shim_status=0; ` +
710
+ `as_node_control /opt/asdf/bin/asdf reshim "$@" || asdf_status=$?; ` +
711
+ `clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
712
+ `[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
713
+ `reject_runtime_bash_shims() { ` +
714
+ `bash_cleanup_status=0; clear_forbidden_candidate_bash 1 ` +
715
+ `|| bash_cleanup_status=1; ` +
716
+ `clear_forbidden_shared_bash_shim 1 || bash_cleanup_status=1; ` +
717
+ `[ "$bash_cleanup_status" -eq 0 ] || fail unsafe-runtime-bash-shim; }; ` +
718
+ `node_write_probe() { writable="$1"; ` +
719
+ `probe="$(as_node /usr/bin/mktemp -d "$writable/.uai-write-probe.XXXXXX")" ` +
720
+ `|| return 1; case "$probe" in "$writable"/.uai-write-probe.*) ;; ` +
721
+ `*) return 1 ;; esac; as_node /usr/bin/rmdir "$probe"; }; ` +
722
+ `candidate_owned_by_node() { ` +
723
+ `[ -z "$(/usr/bin/find "$candidate" -xdev ! -user node -print -quit)" ] ` +
724
+ `&& [ -z "$(/usr/bin/find "$candidate" -xdev -type d ! -perm -u+w -print -quit)" ]; }; ` +
725
+ `prove_default_node() { candidate_is_safe && [ -d "$candidate" ] ` +
726
+ `&& [ ! -e "$candidate/bin/bash" ] && [ ! -L "$candidate/bin/bash" ] ` +
727
+ `&& contained_executable "$candidate" "$candidate/bin/node" ` +
728
+ `&& [ "$(as_node "$candidate/bin/node" --version 2>/dev/null)" = "v$default_node" ] ` +
729
+ `&& contained_executable "$candidate" "$candidate/bin/npm" ` +
730
+ `&& [ -n "$(as_node "$candidate/bin/npm" --version 2>/dev/null)" ] ` +
731
+ `&& npm_prefix_is_safe && candidate_owned_by_node ` +
732
+ `&& node_write_probe "$candidate/bin"; }; ` +
733
+ `clear_forbidden_shared_bash_shim 0; ` +
734
+ `if ! prove_default_node; then candidate_is_safe || fail unsafe-default-candidate; ` +
735
+ `/usr/bin/rm -rf --one-file-system -- "$candidate" ` +
736
+ `|| fail remove-default-candidate; ` +
737
+ `run_asdf_install nodejs "$default_node" ` +
738
+ `|| fail install-default-node; prove_default_node || fail repaired-default-node; fi; ` +
739
+ `npm_bin="$candidate/bin/npm"; cd / || fail chdir; ` +
740
+ `if [ "$upgrade" = 1 ]; then ` +
741
+ `npm_prefix_is_safe || fail unsafe-npm-prefix; ` +
742
+ `if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
743
+ `>/tmp/uai-cli-upgrade.out 2>&1; then ` +
744
+ `if /usr/bin/grep -qi 'ENOTEMPTY' /tmp/uai-cli-upgrade.out; then ` +
745
+ `scope_is_safe_or_absent "$candidate/lib/node_modules/@anthropic-ai" ` +
746
+ `&& scope_is_safe_or_absent "$candidate/lib/node_modules/@openai" ` +
747
+ `|| fail unsafe-cli-scope; ` +
748
+ `as_node /usr/bin/rm -rf --one-file-system -- ` +
749
+ `"$candidate/lib/node_modules/@anthropic-ai" ` +
750
+ `"$candidate/lib/node_modules/@openai"; ` +
751
+ `if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
752
+ `>/tmp/uai-cli-upgrade-retry.out 2>&1; then ` +
753
+ `echo 'UAI_CLI_UPDATE_DEFERRED:bulk-retry' >&2; fi; ` +
754
+ `else echo 'UAI_CLI_UPDATE_DEFERRED:bulk' >&2; fi; fi; fi; ` +
755
+ // Reshim first so an installed package behind a stale shim can recover
756
+ // without network access. Only bins that still fail are reinstalled.
757
+ `if ! run_asdf_reshim nodejs "$default_node" ` +
758
+ `>/dev/null 2>&1; then echo 'UAI_CLI_RESHIM_DEFERRED:initial' >&2; fi; ` +
759
+ `cli_ready_once() { b="$1"; ` +
760
+ `contained_executable "$candidate" "$candidate/bin/$b" || return 1; ` +
761
+ `direct_version="$(as_node "$candidate/bin/$b" --version 2>/dev/null ` +
762
+ `| /usr/bin/head -1)"; [ -n "$direct_version" ] || return 1; ` +
763
+ `shim="$shims_root/$b"; [ ! -L "$shim" ] && [ -f "$shim" ] ` +
764
+ `&& [ -x "$shim" ] || return 1; ` +
765
+ `shim_version="$(as_node "$shim" --version 2>/dev/null ` +
766
+ `| /usr/bin/head -1)"; [ "$shim_version" = "$direct_version" ]; }; ` +
767
+ `repair_cli_once() { b="$1"; case "$b" in ` +
768
+ `claude) pkg='@anthropic-ai/claude-code'; scope='@anthropic-ai'; ` +
769
+ `package_name='claude-code';; ` +
770
+ `codex) pkg='@openai/codex'; scope='@openai'; package_name='codex';; ` +
771
+ `*) return 1;; esac; ` +
772
+ `package_path="$candidate/lib/node_modules/$scope/$package_name"; ` +
773
+ `npm_prefix_is_safe || return 1; ` +
774
+ `scope_is_safe_or_absent "$candidate/lib/node_modules/$scope" ` +
775
+ `|| return 1; ` +
776
+ `as_node /usr/bin/rm -rf --one-file-system -- ` +
777
+ `"$candidate/bin/$b" "$package_path" || return 1; ` +
778
+ `as_node "$npm_bin" install -g --force "$pkg@latest" ${trustedNpmFlags} ` +
779
+ `>/dev/null 2>&1 || return 1; ` +
780
+ `run_asdf_reshim nodejs "$default_node" ` +
781
+ `>/dev/null 2>&1 || return 1; cli_ready_once "$b"; }; ` +
782
+ `for b in claude codex; do if ! cli_ready_once "$b"; then ` +
783
+ `repair_cli_once "$b" || fail "$b-repair"; fi; done; ` +
784
+ // Node <=22's bundled Corepack has obsolete registry signing keys. The
785
+ // host-packaged pin is compatible with every advertised Node default.
786
+ `required_corepack="$(/usr/bin/sed -n '1p' /usr/local/share/uai/corepack-version)"; ` +
787
+ `printf '%s' "$required_corepack" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$' || { ` +
788
+ `echo 'UAI_STILL_BROKEN:corepack-pin' >&2; exit 1; }; ` +
789
+ `corepack_ready_once() { ` +
790
+ `contained_executable "$candidate" "$candidate/bin/corepack" || return 1; ` +
791
+ `[ "$(as_node "$candidate/bin/corepack" --version 2>/dev/null)" = "$required_corepack" ] ` +
792
+ `|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
793
+ `|| return 1; run_asdf_reshim nodejs "$default_node" ` +
794
+ `>/dev/null 2>&1 || return 1; ` +
795
+ `corepack_shim="$shims_root/corepack"; [ ! -L "$corepack_shim" ] ` +
796
+ `&& [ -f "$corepack_shim" ] && [ -x "$corepack_shim" ] || return 1; ` +
797
+ `[ "$(as_node "$corepack_shim" --version 2>/dev/null)" = "$required_corepack" ]; }; ` +
798
+ `repair_corepack_once() { ` +
799
+ `npm_prefix_is_safe || return 1; ` +
800
+ `as_node /usr/bin/rm -rf --one-file-system -- ` +
801
+ `"$candidate/lib/node_modules/corepack" "$candidate/bin/corepack" ` +
802
+ `"$candidate/bin/pnpm" "$candidate/bin/pnpx" "$candidate/bin/yarn" ` +
803
+ `"$candidate/bin/yarnpkg" || return 1; ` +
804
+ `as_node "$npm_bin" install -g --force "corepack@$required_corepack" ` +
805
+ `--no-audit --no-fund ${trustedNpmFlags} >/dev/null 2>&1 || return 1; ` +
806
+ `run_asdf_reshim nodejs "$default_node" >/dev/null 2>&1 ` +
807
+ `|| return 1; contained_executable "$candidate" "$candidate/bin/corepack" ` +
808
+ `|| return 1; as_node "$candidate/bin/corepack" disable >/dev/null 2>&1 ` +
809
+ `|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
810
+ `|| return 1; run_asdf_reshim nodejs "$default_node" ` +
811
+ `>/dev/null 2>&1 || return 1; corepack_ready_once; }; ` +
812
+ `if ! corepack_ready_once; then repair_corepack_once ` +
813
+ `|| fail corepack-repair; fi; reject_runtime_bash_shims; failed=0; ` +
814
+ `prove_default_node || { echo "UAI_STILL_BROKEN:default-node-final" >&2; failed=1; }; ` +
815
+ `for b in claude codex; do ` +
816
+ `if ! contained_executable "$candidate" "$candidate/bin/$b"; then ` +
817
+ `echo "UAI_STILL_BROKEN:$b-contained" >&2; failed=1; continue; fi; ` +
818
+ `version="$(as_node "$candidate/bin/$b" --version 2>/dev/null | /usr/bin/head -1)"; ` +
819
+ `shim="$shims_root/$b"; if [ -L "$shim" ] || [ ! -f "$shim" ] ` +
820
+ `|| [ ! -x "$shim" ]; then echo "UAI_STILL_BROKEN:$b-shim" >&2; ` +
821
+ `failed=1; continue; fi; ` +
822
+ `shim_version="$(as_node "$shim" --version 2>/dev/null | /usr/bin/head -1)"; ` +
823
+ `if [ -z "$version" ] || [ "$shim_version" != "$version" ]; then ` +
824
+ `echo "UAI_STILL_BROKEN:$b" >&2; failed=1; ` +
825
+ `else printf '%s ' "$version"; fi; done; ` +
826
+ `node_shim="$shims_root/node"; ` +
827
+ `[ ! -L "$node_shim" ] && [ -f "$node_shim" ] && [ -x "$node_shim" ] ` +
828
+ `|| { echo "UAI_STILL_BROKEN:node-shim-file" >&2; failed=1; }; ` +
829
+ `shim_node="$(as_node "$node_shim" --version 2>/dev/null || true)"; ` +
830
+ `[ "$shim_node" = "v$default_node" ] || { ` +
831
+ `echo "UAI_STILL_BROKEN:node-shim" >&2; failed=1; }; ` +
832
+ `npm_shim="$shims_root/npm"; ` +
833
+ `[ ! -L "$npm_shim" ] && [ -f "$npm_shim" ] && [ -x "$npm_shim" ] ` +
834
+ `|| { echo "UAI_STILL_BROKEN:npm-shim-file" >&2; failed=1; }; ` +
835
+ `shim_npm="$(as_node "$npm_shim" --version 2>/dev/null || true)"; ` +
836
+ `[ -n "$shim_npm" ] || { echo "UAI_STILL_BROKEN:npm-shim" >&2; failed=1; }; ` +
837
+ `if ! corepack_ready_once; then echo "UAI_STILL_BROKEN:corepack-final" >&2; ` +
838
+ `failed=1; else printf 'corepack %s' "$required_corepack"; fi; ` +
839
+ `node_write_probe "$candidate/bin" || { ` +
840
+ `echo "UAI_STILL_BROKEN:default-node-owner" >&2; failed=1; }; ` +
841
+ `node_write_probe "$shims_root" || { ` +
842
+ `echo "UAI_STILL_BROKEN:shims-owner" >&2; failed=1; }; ` +
843
+ `node_write_probe "$corepack_root" || { ` +
844
+ `echo "UAI_STILL_BROKEN:corepack-owner" >&2; failed=1; }; ` +
845
+ `exit "$failed"`
846
+ );
847
+ }
848
+
849
+ async function reconcileVolumeAgentClis(): Promise<boolean> {
850
+ const missingBefore = await probeVolumeAgentClis();
851
+ if (missingBefore?.length) {
852
+ console.log(
853
+ `[host-agent] repairing ${ASDF_DATA_VOLUME}: agent CLI(s) ` +
854
+ `[${missingBefore.join(", ")}] failed their execution probe`,
855
+ );
856
+ }
857
+
858
+ const autoUpdate = process.env.UAI_AGENT_CLI_AUTOUPDATE !== "0";
859
+ if (!autoUpdate) {
860
+ console.log(
861
+ "[host-agent] agent CLI auto-update disabled (UAI_AGENT_CLI_AUTOUPDATE=0)",
862
+ );
863
+ }
864
+ const command = standardImageVolumeMaintenanceCommandForTest(autoUpdate);
865
+ const reconcile = await runMaintenanceContainer(selectedEngine(), "reconcile", [
289
866
  "--rm",
290
- // The container NAME is the mutex: overlapping boots (or a crash-looping
291
- // service) must never race two npm installs on the shared volume — that
292
- // once left it with no `claude` at all (2026-07-13). Docker rejects the
293
- // duplicate name; we treat that as "already upgrading, skip".
294
- "--name",
295
- "uai-cli-upgrade",
296
867
  "-u",
297
868
  "root",
298
869
  "-w",
299
870
  "/home/node",
300
871
  "-v",
301
872
  `${ASDF_DATA_VOLUME}:/opt/asdf-data`,
873
+ "-v",
874
+ `${standardImageCorepackVersionPath()}:/usr/local/share/uai/corepack-version:ro`,
302
875
  STANDARD_IMAGE_TAG,
303
- "bash",
304
- "-lc",
305
- // Reshim is its OWN statement (`;`, not `&&`) so it ALWAYS runs after the
306
- // install attempts — with `&& reshim`, the `|| { rm -rf; reinstall; }`
307
- // fallback's exit code could skip the reshim, leaving a stale `codex`/
308
- // `claude` shim ("No <cli> executable found for nodejs …") even though the
309
- // bin installed fine (hit codex live 2026-07-16). Reshim is idempotent and
310
- // cheap, so running it unconditionally is strictly safer.
311
- `npm install -g ${pkgs} >/dev/null 2>&1 || { rm -rf ${scopeDirs}; npm install -g ${pkgs} >/dev/null 2>&1; }; ` +
312
- `asdf reshim nodejs >/dev/null 2>&1; ` +
313
- bins.map((b) => `printf '%s ' "$(${b} --version 2>/dev/null | head -1)"`).join("; "),
314
- ]);
315
- if (upgrade.code !== 0 && /already in use/i.test(upgrade.stderr)) {
316
- console.log("[host-agent] agent CLI upgrade already running — skipped");
317
- return;
318
- }
319
- if (upgrade.code === 0) {
876
+ "/usr/bin/timeout",
877
+ "--signal=TERM",
878
+ "--kill-after=10s",
879
+ `${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
880
+ "/usr/bin/flock",
881
+ "--exclusive",
882
+ "--timeout",
883
+ `${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS}`,
884
+ ASDF_MAINTENANCE_LOCK_PATH,
885
+ "/bin/bash",
886
+ "-c",
887
+ command,
888
+ ], CLI_MAINTENANCE_TIMEOUT_MS);
889
+ if (reconcile.code === 0) {
320
890
  console.log(
321
- `[host-agent] agent CLIs current on ${ASDF_DATA_VOLUME}: ${upgrade.stdout.trim()}`,
891
+ `[host-agent] agent CLIs and Corepack current on ${ASDF_DATA_VOLUME}: ` +
892
+ reconcile.stdout.trim(),
322
893
  );
894
+ await writeVolumeReconcileProof();
895
+ return true;
323
896
  } else {
897
+ lastMaintenanceStderr = reconcile.stderr;
324
898
  console.warn(
325
- `[host-agent] agent CLI upgrade skipped (exit ${upgrade.code ?? "spawn"}) ` +
326
- `tasks keep the volume's current versions. ${upgrade.stderr.trim().slice(0, 200)}`,
899
+ `[host-agent] failed to reconcile ${ASDF_DATA_VOLUME} (exit ` +
900
+ `${reconcile.code ?? "spawn"}); task runtimes are not ready. ` +
901
+ reconcile.stderr.trim().slice(0, 500),
327
902
  );
903
+ return false;
328
904
  }
329
905
  }
330
906
 
907
+ /** Durable evidence that the shared volume completed a full reconcile at
908
+ * least once on this host. Apple volumes are block devices with EXCLUSIVE
909
+ * writable attachments, so boot maintenance cannot re-prove the volume while
910
+ * running task containers hold it read-only — and blocking activation on
911
+ * that (the 2026-08-16 ring) froze hosts and blocked the very teardowns that
912
+ * would free the volume. With this proof on disk, a busy volume defers
913
+ * maintenance instead of failing it. */
914
+ let lastMaintenanceStderr = "";
915
+
916
+ async function volumeReconcileProofPath(): Promise<string> {
917
+ // Lazy env resolve: module import must stay side-effect free (see
918
+ // defaultRuntimeProviderStateStore). Dynamic import, not require — a bare
919
+ // require cannot resolve the TypeScript module outside the tsx runtime,
920
+ // which silently read as "no proof" and disabled deferral.
921
+ const { env } = await import("./env");
922
+ return resolve(env.dataDir, "runtime-volume-proof.json");
923
+ }
924
+
925
+ async function writeVolumeReconcileProof(): Promise<void> {
926
+ try {
927
+ writeFileSync(
928
+ await volumeReconcileProofPath(),
929
+ `${JSON.stringify({ schemaVersion: 1, provedAt: Date.now() })}\n`,
930
+ { mode: 0o600 },
931
+ );
932
+ } catch (error) {
933
+ console.warn(
934
+ `[host-agent] could not persist the shared-volume reconcile proof: ${
935
+ error instanceof Error ? error.message : String(error)
936
+ }`,
937
+ );
938
+ }
939
+ }
940
+
941
+ async function volumeReconcileProofExists(): Promise<boolean> {
942
+ try {
943
+ const raw = JSON.parse(
944
+ readFileSync(await volumeReconcileProofPath(), "utf8"),
945
+ ) as { schemaVersion?: unknown };
946
+ return raw?.schemaVersion === 1;
947
+ } catch {
948
+ return false;
949
+ }
950
+ }
951
+
952
+ const VOLUME_ATTACH_BUSY_SIGNATURE = "storage device attachment is invalid";
953
+
954
+ /**
955
+ * Whether task containers hold the shared volume — tri-state per container,
956
+ * never fail-open: `running` holds, exact `stopped` is released (a stopped
957
+ * VM releases its block attachments — proven live 2026-08-17, when a full
958
+ * writable reconcile succeeded next to a stopped task container), and EVERY
959
+ * other observation (starting, stopping, creating, a missing or foreign
960
+ * state string, an unreadable inventory) is `unknown` — a task acquiring or
961
+ * releasing the volume mid-transition must never read as permission for a
962
+ * writer (review 2026-08-18 round 2). `held` dominates `unknown` dominates
963
+ * `free`; writers require a positive `free`.
964
+ */
965
+ async function sharedVolumeHolders(engine: {
966
+ command: string;
967
+ apple: boolean;
968
+ }): Promise<"held" | "free" | "unknown"> {
969
+ const listed = await run(
970
+ engine.command,
971
+ ["list", "--all", "--format", "json"],
972
+ 30_000,
973
+ );
974
+ if (listed.code !== 0) return "unknown";
975
+ try {
976
+ const parsed = JSON.parse(listed.stdout) as Array<{
977
+ configuration?: {
978
+ id?: unknown;
979
+ mounts?: Array<{ type?: { volume?: { name?: unknown } } }>;
980
+ };
981
+ status?: { state?: unknown };
982
+ }>;
983
+ if (!Array.isArray(parsed)) return "unknown";
984
+ let verdict: "held" | "free" | "unknown" = "free";
985
+ for (const item of parsed) {
986
+ // A holder is a container whose CONFIGURED MOUNTS include the shared
987
+ // volume — physical attachment, not a name shape (review 2026-08-18
988
+ // round 3: an unrelated task-*-app container with no attachment
989
+ // fenced activation forever, and the holder-remedy sweep — which
990
+ // correctly matches by mount — could not clear it). A row whose
991
+ // mounts cannot be read might hold the volume, so it is `unknown`.
992
+ const mounts = item?.configuration?.mounts;
993
+ if (!Array.isArray(mounts)) {
994
+ verdict = "unknown";
995
+ continue;
996
+ }
997
+ const holds = mounts.some(
998
+ (mount) => mount?.type?.volume?.name === ASDF_DATA_VOLUME,
999
+ );
1000
+ if (!holds) continue;
1001
+ const state = item?.status?.state;
1002
+ if (state === "running") return "held";
1003
+ if (state !== "stopped") verdict = "unknown";
1004
+ }
1005
+ return verdict;
1006
+ } catch {
1007
+ return "unknown";
1008
+ }
1009
+ }
1010
+
1011
+ /** Backstop for the race the proof-first check cannot close: a task container
1012
+ * appearing between the check and the attach. Admits deferral only on the
1013
+ * exact busy signature, with the same proof requirement. */
1014
+ async function volumeBusyDeferral(
1015
+ engine: { command: string; apple: boolean },
1016
+ stderr: string,
1017
+ ): Promise<boolean> {
1018
+ if (!engine.apple) return false;
1019
+ if (!stderr.includes(VOLUME_ATTACH_BUSY_SIGNATURE)) return false;
1020
+ if (!(await volumeReconcileProofExists())) return false;
1021
+ return (await sharedVolumeHolders(engine)) === "held";
1022
+ }
1023
+
331
1024
  /** Outcome of ensureStandardImage — `error` carries WHY when not ready, so
332
1025
  * the taskUp path can surface a build failure to the cloud (not just the host
333
1026
  * log). Callers that only fire-and-forget at boot ignore the return. */
334
1027
  export interface StandardImageResult {
335
1028
  ok: boolean;
336
1029
  error?: string;
1030
+ /** A non-writer escape the activation driver can perform before retrying:
1031
+ * lifecycle-stop the running task containers holding the shared volume
1032
+ * (resumable), because the volume needs a repair they block and cannot
1033
+ * safely use. */
1034
+ remedy?: "stop-volume-holders";
337
1035
  }
338
1036
 
1037
+ type AgentClisReadyListener = () => void;
1038
+
1039
+ let agentClisReadyLatched = false;
339
1040
  let signalClisReady: (() => void) | null = null;
1041
+ const agentClisReadyListeners = new Set<AgentClisReadyListener>();
1042
+ const ensureQueue = new KeyedPromiseTail<StandardImageResult>();
340
1043
  /**
341
1044
  * Resolves once ensureStandardImage has reconciled the shared-volume agent CLIs
342
1045
  * at least once (or bailed because docker is down / the build failed). The agent
@@ -349,6 +1052,55 @@ export const agentClisReady: Promise<void> = new Promise<void>((resolve) => {
349
1052
  signalClisReady = resolve;
350
1053
  });
351
1054
 
1055
+ /** Whether this process has completed its one boot-time CLI maintenance pass.
1056
+ * This is intentionally a one-way boot latch, not a claim that no later
1057
+ * operator-triggered image work is in flight. */
1058
+ export function areAgentClisReady(): boolean {
1059
+ return agentClisReadyLatched;
1060
+ }
1061
+
1062
+ /** Subscribe to the one readiness edge. A late subscriber is notified in a
1063
+ * microtask so registration never re-enters its caller. */
1064
+ export function onAgentClisReady(
1065
+ listener: AgentClisReadyListener,
1066
+ ): () => void {
1067
+ if (agentClisReadyLatched) {
1068
+ let cancelled = false;
1069
+ queueMicrotask(() => {
1070
+ if (!cancelled) notifyAgentClisReadyListener(listener);
1071
+ });
1072
+ return () => {
1073
+ cancelled = true;
1074
+ };
1075
+ }
1076
+ agentClisReadyListeners.add(listener);
1077
+ return () => agentClisReadyListeners.delete(listener);
1078
+ }
1079
+
1080
+ function notifyAgentClisReadyListener(listener: AgentClisReadyListener): void {
1081
+ try {
1082
+ listener();
1083
+ } catch (error) {
1084
+ console.warn(
1085
+ `[host-agent] agent CLI readiness listener failed: ${error instanceof Error ? error.message : String(error)}`,
1086
+ );
1087
+ }
1088
+ }
1089
+
1090
+ function markAgentClisReady(): void {
1091
+ if (agentClisReadyLatched) return;
1092
+ // Publish the synchronous capability state before resolving the promise:
1093
+ // every waiter and listener must observe `true` after the readiness edge.
1094
+ agentClisReadyLatched = true;
1095
+ signalClisReady?.();
1096
+ signalClisReady = null;
1097
+ const listeners = [...agentClisReadyListeners];
1098
+ agentClisReadyListeners.clear();
1099
+ for (const listener of listeners) {
1100
+ notifyAgentClisReadyListener(listener);
1101
+ }
1102
+ }
1103
+
352
1104
  /**
353
1105
  * Ensure the standard image and the shared asdf data volume exist, and that the
354
1106
  * agent CLIs are present inside the volume. Builds the image only when `docker
@@ -356,21 +1108,52 @@ export const agentClisReady: Promise<void> = new Promise<void>((resolve) => {
356
1108
  * `{ ok, error? }` so the caller decides (boot: fire-and-forget; taskUp:
357
1109
  * surface the reason). Resolves `agentClisReady` on every exit path.
358
1110
  */
359
- export async function ensureStandardImage(): Promise<StandardImageResult> {
360
- try {
361
- return await ensureStandardImageInner();
362
- } finally {
363
- // Always signal even a docker-down bail so agent spawns never hang
364
- // waiting on a reconcile that will not happen this boot.
365
- signalClisReady?.();
366
- signalClisReady = null;
367
- }
1111
+ export function ensureStandardImage(): Promise<StandardImageResult> {
1112
+ // Startup, runtime recovery, and a newly-arrived task can all discover the
1113
+ // same daemon transition. One shared volume repair/build must own that
1114
+ // window; concurrent maintenance containers can otherwise replace the same
1115
+ // CLI shims underneath each other and defeat agentClisReady's fence.
1116
+ const engines = configuredOptionalEngines();
1117
+ return ensureQueue.run(optionalEngineConfigurationKey(engines), () =>
1118
+ ensureStandardImageInner(engines).finally(() => {
1119
+ // Always signal — even a docker-down bail — so agent spawns never hang
1120
+ // waiting on a reconcile that will not happen this boot.
1121
+ markAgentClisReady();
1122
+ }),
1123
+ );
368
1124
  }
369
1125
 
370
- async function ensureStandardImageInner(): Promise<StandardImageResult> {
371
- // 1. Shared asdf data volume — idempotent.
372
- const vol = await run("docker", ["volume", "create", ASDF_DATA_VOLUME]);
373
- if (vol.code !== 0) {
1126
+ async function ensureStandardImageInner(
1127
+ engines: OptionalEngineConfiguration,
1128
+ ): Promise<StandardImageResult> {
1129
+ // 1. Shared asdf data volume — idempotent under Docker; the Apple CLI
1130
+ // errors on an existing name, which is the same healthy outcome.
1131
+ const engine = selectedEngine();
1132
+ console.log(
1133
+ `[host-agent] standard-image maintenance starting (engine: ${engine.apple ? "apple-container" : "docker"})`,
1134
+ );
1135
+ if (engine.apple) {
1136
+ // The volume create below is the FIRST apple mutation on a host — before
1137
+ // any task exists — and uninstall reads an absent provider-claim marker
1138
+ // as "nothing to tear down". Claim durably before mutating, or a
1139
+ // zero-task install→activate→uninstall skips Apple cleanup (review
1140
+ // 2026-08-18 round 4).
1141
+ try {
1142
+ await claimContainerRuntimeForMaintenance();
1143
+ } catch (error) {
1144
+ return {
1145
+ ok: false,
1146
+ error:
1147
+ "Could not durably record the Apple container runtime claim " +
1148
+ `before provisioning: ${error instanceof Error ? error.message : String(error)}`,
1149
+ };
1150
+ }
1151
+ }
1152
+ const vol = await run(engine.command, ["volume", "create", ASDF_DATA_VOLUME]);
1153
+ if (
1154
+ vol.code !== 0 &&
1155
+ !(engine.apple && vol.stderr.includes("already exists"))
1156
+ ) {
374
1157
  console.warn(
375
1158
  `[host-agent] could not create asdf data volume (${ASDF_DATA_VOLUME}); ` +
376
1159
  `continuing. ${vol.stderr.trim()}`,
@@ -391,7 +1174,6 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
391
1174
  let buildError: string | null = null;
392
1175
  // Install only the optional engines the operator has configured; the flags
393
1176
  // are build args AND part of the content hash (so a new login rebuilds).
394
- const engines = configuredOptionalEngines();
395
1177
  const engineArgs = [
396
1178
  "--build-arg",
397
1179
  `INSTALL_KIMI=${engines.kimi ? 1 : 0}`,
@@ -405,21 +1187,52 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
405
1187
  const contextHash = await hashBuildContext(
406
1188
  `kimi=${engines.kimi ? 1 : 0};grok=${engines.grok ? 1 : 0};cursor=${engines.cursor ? 1 : 0};opencode=${engines.opencode ? 1 : 0}`,
407
1189
  );
408
- const inspect = await run("docker", [
409
- "image",
410
- "inspect",
411
- "-f",
412
- `{{index .Config.Labels "${CONTEXT_HASH_LABEL}"}}`,
413
- STANDARD_IMAGE_TAG,
414
- ]);
1190
+ let inspect: RunResult;
1191
+ let labeledHash: string | null = null;
1192
+ if (engine.apple) {
1193
+ // The Apple CLI has no --format; labels live in the single variant's OCI
1194
+ // config. A malformed answer counts as "absent" and triggers a rebuild.
1195
+ inspect = await run(engine.command, [
1196
+ "image",
1197
+ "inspect",
1198
+ STANDARD_IMAGE_TAG,
1199
+ ]);
1200
+ if (inspect.code === 0) {
1201
+ try {
1202
+ const parsed = JSON.parse(inspect.stdout) as Array<{
1203
+ variants?: Array<{
1204
+ config?: { config?: { Labels?: Record<string, string> } };
1205
+ }>;
1206
+ }>;
1207
+ labeledHash =
1208
+ parsed[0]?.variants?.[0]?.config?.config?.Labels?.[
1209
+ CONTEXT_HASH_LABEL
1210
+ ] ?? "";
1211
+ } catch {
1212
+ inspect = { ...inspect, code: 1 };
1213
+ }
1214
+ }
1215
+ } else {
1216
+ inspect = await run("docker", [
1217
+ "image",
1218
+ "inspect",
1219
+ "-f",
1220
+ `{{index .Config.Labels "${CONTEXT_HASH_LABEL}"}}`,
1221
+ STANDARD_IMAGE_TAG,
1222
+ ]);
1223
+ labeledHash = inspect.code === 0 ? inspect.stdout.trim() : null;
1224
+ }
415
1225
  if (inspect.code === null) {
416
1226
  console.warn(
417
- "[host-agent] docker unavailable; skipping standard image build. " +
418
- "Tasks will fail until docker is running.",
1227
+ "[host-agent] container runtime unavailable; skipping standard image " +
1228
+ "build. Tasks will fail until it is running.",
419
1229
  );
420
- return { ok: false, error: "Docker is not available on this host." };
1230
+ return {
1231
+ ok: false,
1232
+ error: "The container runtime is not available on this host.",
1233
+ };
421
1234
  }
422
- const labeledHash = inspect.code === 0 ? inspect.stdout.trim() : null;
1235
+ if (inspect.code !== 0) labeledHash = null;
423
1236
  if (inspect.code === 0 && contextHash !== null && labeledHash === contextHash) {
424
1237
  console.log(`[host-agent] standard image ${STANDARD_IMAGE_TAG} current`);
425
1238
  imageReady = true;
@@ -434,7 +1247,7 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
434
1247
  ? `[host-agent] standard image ${STANDARD_IMAGE_TAG} stale (context changed) — rebuilding from ${context}`
435
1248
  : `[host-agent] building standard image ${STANDARD_IMAGE_TAG} from ${context}`,
436
1249
  );
437
- const build = await run("docker", [
1250
+ const build = await run(engine.command, [
438
1251
  "build",
439
1252
  "-t",
440
1253
  STANDARD_IMAGE_TAG,
@@ -443,7 +1256,7 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
443
1256
  ? ["--label", `${CONTEXT_HASH_LABEL}=${contextHash}`]
444
1257
  : []),
445
1258
  context,
446
- ]);
1259
+ ], IMAGE_BUILD_TIMEOUT_MS);
447
1260
  if (build.code === 0) {
448
1261
  console.log(`[host-agent] built standard image ${STANDARD_IMAGE_TAG}`);
449
1262
  imageReady = true;
@@ -461,15 +1274,165 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
461
1274
  }
462
1275
  }
463
1276
 
464
- // 3. Reconcile the agent CLIs in the shared volume (it shadows the image's
465
- // shims, so a stale volume can be missing one — the claude-127 bug).
466
- // Upgrade FIRST, self-heal LAST: the upgrade's `rm -rf`+reinstall fallback
467
- // can leave a stale shim (a failed reinstall over a removed package), so
468
- // ensureVolumeAgentClis must be the final word — it verifies every bin
469
- // actually runs and repairs whatever the upgrade left broken.
1277
+ // 3. Reconcile every Uai-owned writer under the volume-global flock. Lock
1278
+ // bootstrap/permission repair precedes the single upgrade→repair→proof
1279
+ // transaction so node-run task materializers can acquire legacy volumes.
470
1280
  if (imageReady) {
471
- await upgradeVolumeAgentClis();
472
- await ensureVolumeAgentClis();
1281
+ // Docker populates a fresh named volume from the image's directory
1282
+ // content on first mount; the Apple runtime mounts it empty (ADR-106,
1283
+ // found live 2026-08-15: every asdf plugin/shim the image bakes into
1284
+ // /opt/asdf-data was missing, so CLI reconcile looped on "No such
1285
+ // plugin: nodejs"). Replicate the Docker semantics explicitly: when the
1286
+ // volume lacks the plugins tree, copy the image's /opt/asdf-data into it
1287
+ // once, ownership preserved. Idempotent — the copied tree is the marker.
1288
+ if (engine.apple) {
1289
+ // Probe at a path the image does NOT contain: mounted over
1290
+ // /opt/asdf-data, a silently-failed volume mount would expose the
1291
+ // image's own baked tree and fake a "seeded" answer. At a neutral
1292
+ // mountpoint the plugins tree can only come from the volume itself,
1293
+ // and any mount failure lands on the safe side (reseeding is
1294
+ // idempotent).
1295
+ const seeded = await runMaintenanceContainer(engine, "seed-probe", [
1296
+ "--rm",
1297
+ "-v",
1298
+ `${ASDF_DATA_VOLUME}:/uai-volume-probe:ro`,
1299
+ STANDARD_IMAGE_TAG,
1300
+ "/usr/bin/test",
1301
+ "-d",
1302
+ "/uai-volume-probe/plugins",
1303
+ ], CLI_MAINTENANCE_TIMEOUT_MS);
1304
+ // Only `/usr/bin/test`'s own exit 1 means "not seeded". Everything else
1305
+ // — a host-side deadline kill (null), an in-container SIGTERM/SIGKILL
1306
+ // (143/137: live 2026-08-17, a wedged apiserver let the probe run ten
1307
+ // minutes before dying 143), or any CLI failure — answers neither way,
1308
+ // and must never trigger a reseed of a volume that may be full and in
1309
+ // use.
1310
+ if (seeded.code !== 0 && seeded.code !== 1) {
1311
+ return {
1312
+ ok: false,
1313
+ error:
1314
+ "The container runtime did not answer while checking the shared " +
1315
+ `runtime volume (probe exit ${seeded.code ?? "killed"}).`,
1316
+ };
1317
+ }
1318
+ if (seeded.code === 0) {
1319
+ console.log(
1320
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is seeded`,
1321
+ );
1322
+ }
1323
+ // The positive no-holder proof comes BEFORE every writable attachment
1324
+ // — including the seeder. A doomed writable attach both fails the
1325
+ // operation and can mint the corpse that wedges the apiserver, so a
1326
+ // writer requires a positive `free`; `unknown` (unreadable inventory)
1327
+ // must never read as permission.
1328
+ const holders = await sharedVolumeHolders(engine);
1329
+ if (seeded.code !== 0) {
1330
+ if (holders !== "free") {
1331
+ return {
1332
+ ok: false,
1333
+ error:
1334
+ holders === "held"
1335
+ ? `The shared runtime volume ${ASDF_DATA_VOLUME} is unseeded while task containers are running; it cannot be seeded safely until they stop.`
1336
+ : "Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
1337
+ };
1338
+ }
1339
+ console.log(
1340
+ `[host-agent] seeding ${ASDF_DATA_VOLUME} from the standard image (probe exit ${seeded.code}; Apple volumes mount empty)`,
1341
+ );
1342
+ const seed = await runMaintenanceContainer(engine, "seed", [
1343
+ "--rm",
1344
+ "--user",
1345
+ "root",
1346
+ "-v",
1347
+ `${ASDF_DATA_VOLUME}:/uai-seed-target`,
1348
+ STANDARD_IMAGE_TAG,
1349
+ "/bin/sh",
1350
+ "-c",
1351
+ "cp -a /opt/asdf-data/. /uai-seed-target/ && chown -R node:node /uai-seed-target",
1352
+ ], CLI_MAINTENANCE_TIMEOUT_MS);
1353
+ if (seed.code !== 0) {
1354
+ return {
1355
+ ok: false,
1356
+ error: `Could not seed the shared runtime volume from the standard image (exit ${seed.code ?? "spawn"}).`,
1357
+ };
1358
+ }
1359
+ } else if (holders === "held") {
1360
+ // Proof-first deferral: while running task containers hold the
1361
+ // volume, a writable attach can only fail (and mint a corpse) —
1362
+ // don't attempt it. The stderr-signature deferrals below remain as
1363
+ // the backstop for a task appearing between this check and the
1364
+ // attach.
1365
+ if (await volumeReconcileProofExists()) {
1366
+ console.log(
1367
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by task containers; deferring maintenance (prior reconcile proof present)`,
1368
+ );
1369
+ return { ok: true };
1370
+ }
1371
+ // First-proof bootstrap. A host with no persisted proof (fresh file,
1372
+ // or upgraded from a version whose proof write failed silently —
1373
+ // live 2026-08-17 on 0.9.26) would otherwise be stuck in a ring: the
1374
+ // proof needs a full reconcile, the reconcile needs the writable
1375
+ // volume, and the volume stays held for as long as tasks run. The
1376
+ // read-only CLI probe attaches fine next to tasks, and EXECUTING
1377
+ // every agent CLI is the same positive evidence the full reconcile
1378
+ // ends with — enough to defer on, and worth persisting as the proof.
1379
+ const missing = await probeVolumeAgentClis();
1380
+ if (missing !== null && missing.length === 0) {
1381
+ await writeVolumeReconcileProof();
1382
+ console.log(
1383
+ `[host-agent] read-only agent-CLI probe passed while task containers hold ${ASDF_DATA_VOLUME}; recording the reconcile proof and deferring maintenance`,
1384
+ );
1385
+ return { ok: true };
1386
+ }
1387
+ // No proof and a failed probe, while tasks hold the volume: a
1388
+ // writable attach here can only fail and mint a corpse, and simply
1389
+ // retrying strands activation forever (review 2026-08-18 round 2 —
1390
+ // recovery, which could settle the holders, runs only after
1391
+ // maintenance succeeds). Ask the activation driver to lifecycle-stop
1392
+ // the holder tasks (resumable) so the next attempt can repair the
1393
+ // volume they cannot safely use anyway.
1394
+ return {
1395
+ ok: false,
1396
+ error:
1397
+ `The shared runtime volume ${ASDF_DATA_VOLUME} needs repair ` +
1398
+ `(agent CLIs failed their probe${missing === null ? "" : `: ${missing.join(", ")} missing`}) ` +
1399
+ "while running tasks hold it; stopping the holder tasks so maintenance can repair it.",
1400
+ remedy: "stop-volume-holders",
1401
+ };
1402
+ } else if (holders === "unknown") {
1403
+ return {
1404
+ ok: false,
1405
+ error:
1406
+ "Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
1407
+ };
1408
+ }
1409
+ }
1410
+ const lockPrepared = await prepareAsdfMaintenanceLock();
1411
+ if (!lockPrepared) {
1412
+ if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
1413
+ console.log(
1414
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
1415
+ );
1416
+ return { ok: true };
1417
+ }
1418
+ return {
1419
+ ok: false,
1420
+ error: `Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
1421
+ };
1422
+ }
1423
+ if (!(await reconcileVolumeAgentClis())) {
1424
+ if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
1425
+ console.log(
1426
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
1427
+ );
1428
+ return { ok: true };
1429
+ }
1430
+ return {
1431
+ ok: false,
1432
+ error:
1433
+ "Shared runtime maintenance failed; agent CLIs or Corepack did not pass execution proof.",
1434
+ };
1435
+ }
473
1436
  return { ok: true };
474
1437
  }
475
1438
  return {