@moondeck/station 0.0.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (326) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +149 -2
  3. package/bin/uai-host.mjs +130 -0
  4. package/db/migrations/0000_host_tasks.sql +12 -0
  5. package/db/migrations/0001_host_ui.sql +11 -0
  6. package/db/migrations/0002_host_github_tokens.sql +8 -0
  7. package/db/migrations/0003_host_ssh_keys.sql +8 -0
  8. package/db/migrations/0004_host_owner_name.sql +1 -0
  9. package/db/migrations/0005_host_env_vars.sql +5 -0
  10. package/db/migrations/0006_host_project_env.sql +9 -0
  11. package/db/migrations/0007_host_github_token_kind.sql +7 -0
  12. package/db/migrations/0008_host_mcp_connections.sql +18 -0
  13. package/db/migrations/0009_host_agent_sessions.sql +15 -0
  14. package/db/migrations/0010_host_engine_accounts.sql +13 -0
  15. package/db/migrations/0011_codex_thread_id.sql +5 -0
  16. package/db/migrations/0012_agent_session_attach_key.sql +4 -0
  17. package/db/migrations/0013_github_credential_generations.sql +11 -0
  18. package/db/migrations/0014_host_inventory_event_index.sql +1 -0
  19. package/db/migrations/0015_host_settings.sql +9 -0
  20. package/db/migrations/0016_task_environment.sql +2 -0
  21. package/db/migrations/0017_engine_account_scope.sql +7 -0
  22. package/db/migrations/0018_engine_account_position.sql +3 -0
  23. package/db/migrations/0019_engine_account_store_ref.sql +3 -0
  24. package/db/migrations/0020_user_credential_store_ref.sql +6 -0
  25. package/db/migrations/0021_project_env_store_ref.sql +3 -0
  26. package/db/migrations/0022_task_project_ids.sql +5 -0
  27. package/db/migrations/0023_project_env_org.sql +4 -0
  28. package/db/migrations/0024_task_env_mode.sql +9 -0
  29. package/db/migrations/0025_project_env_authority.sql +147 -0
  30. package/db/migrations/0026_project_env_replica.sql +150 -0
  31. package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
  32. package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
  33. package/db/migrations/0029_task_shared_files_mode.sql +5 -0
  34. package/db/migrations/0030_credential_cleanup_restops.sql +12 -0
  35. package/db/migrations/meta/_journal.json +223 -0
  36. package/db/schema.ts +850 -0
  37. package/images/standard/Dockerfile +347 -0
  38. package/images/standard/README.md +167 -0
  39. package/images/standard/container/code-server-settings.json +63 -0
  40. package/images/standard/container/corepack-version +1 -0
  41. package/images/standard/container/uai-init +704 -0
  42. package/images/standard/container/uai-materialize-runtimes +1527 -0
  43. package/images/standard/container/uai-task-forwarder.mjs +303 -0
  44. package/images/standard/tool-versions +2 -0
  45. package/lib/__fixtures__/skill-index/SKILL.md +10 -0
  46. package/lib/__fixtures__/skill-index/skills/accessibility-checklist.md +7 -0
  47. package/lib/__fixtures__/skill-index/skills/brand-voice.md +9 -0
  48. package/lib/__fixtures__/skill-index/skills/design-tokens.md +9 -0
  49. package/lib/agent-cli.ts +1507 -0
  50. package/lib/agent-errors.ts +22 -0
  51. package/lib/agent.ts +672 -0
  52. package/lib/agents/attach-compatibility.ts +51 -0
  53. package/lib/agents/claude.ts +918 -0
  54. package/lib/agents/codex-token-usage.fixture.json +130 -0
  55. package/lib/agents/codex.ts +1392 -0
  56. package/lib/agents/cursor-period-usage.ts +253 -0
  57. package/lib/agents/cursor.ts +562 -0
  58. package/lib/agents/dispatch.ts +78 -0
  59. package/lib/agents/durable-proc.ts +334 -0
  60. package/lib/agents/factory.ts +57 -0
  61. package/lib/agents/grok.ts +381 -0
  62. package/lib/agents/kimi.ts +329 -0
  63. package/lib/agents/machine-durable.ts +604 -0
  64. package/lib/agents/mock.ts +133 -0
  65. package/lib/agents/mode.ts +64 -0
  66. package/lib/agents/opencode.ts +508 -0
  67. package/lib/agents/output-source.ts +3 -0
  68. package/lib/agents/proc.ts +221 -0
  69. package/lib/agents/rate-limit.ts +337 -0
  70. package/lib/agents/registry.ts +182 -0
  71. package/lib/agents/session-state.ts +197 -0
  72. package/lib/agents/skill-index-preamble.fixture.ts +26 -0
  73. package/lib/agents/transport.ts +741 -0
  74. package/lib/agents/types.ts +254 -0
  75. package/lib/agents/uai.ts +106 -0
  76. package/lib/agents/usage.ts +44 -0
  77. package/lib/apple-one-off.ts +277 -0
  78. package/lib/apple-runtime-recycle.ts +237 -0
  79. package/lib/apple-uninstall-teardown.ts +224 -0
  80. package/lib/assignment-evidence.ts +94 -0
  81. package/lib/attachments.ts +48 -0
  82. package/lib/browser-testing.ts +1537 -0
  83. package/lib/capture-flow.ts +332 -0
  84. package/lib/cloud-credential-cleanup-adapter.ts +123 -0
  85. package/lib/cloud-credential-host.ts +512 -0
  86. package/lib/cloud-credential-leases.ts +2587 -0
  87. package/lib/cloud-credential-runtime.ts +1357 -0
  88. package/lib/cloud-state.ts +56 -0
  89. package/lib/codex-auth.ts +456 -0
  90. package/lib/command-db.ts +310 -0
  91. package/lib/compose-network-recovery.ts +283 -0
  92. package/lib/connection-secret-redaction.ts +25 -0
  93. package/lib/container-runtime.ts +1453 -0
  94. package/lib/credential-cleanup-inventory.ts +490 -0
  95. package/lib/credential-consumers/engine-anchor.ts +1880 -0
  96. package/lib/credential-consumers/engines.ts +797 -0
  97. package/lib/credential-consumers/files.ts +934 -0
  98. package/lib/credential-consumers/github.ts +702 -0
  99. package/lib/credential-consumers/mcp.ts +389 -0
  100. package/lib/credential-consumers/task-files.ts +127 -0
  101. package/lib/credential-consumers/task-secrets.ts +10 -0
  102. package/lib/credential-store-pickup.ts +149 -0
  103. package/lib/credential-store-scope.ts +112 -0
  104. package/lib/credential-store.ts +932 -0
  105. package/lib/credentials-activation-authority.ts +709 -0
  106. package/lib/credentials-purge.ts +1006 -0
  107. package/lib/custom-connection-capture.ts +395 -0
  108. package/lib/custom-connection-launch.ts +384 -0
  109. package/lib/custom-connection-runtime.ts +141 -0
  110. package/lib/db.ts +80 -0
  111. package/lib/disk-pressure.ts +42 -0
  112. package/lib/docker-exec.ts +156 -0
  113. package/lib/engine-accounts.ts +1970 -0
  114. package/lib/engine-connect.ts +321 -0
  115. package/lib/engine-credential-env.ts +52 -0
  116. package/lib/engine-login.ts +2933 -0
  117. package/lib/engines.ts +1342 -0
  118. package/lib/enrollment-state.ts +251 -0
  119. package/lib/env-file.ts +246 -0
  120. package/lib/env-schema.ts +891 -0
  121. package/lib/env.ts +195 -0
  122. package/lib/fs-aliases.ts +49 -0
  123. package/lib/git-clone-credentials.ts +334 -0
  124. package/lib/git-diff.ts +530 -0
  125. package/lib/git-identity.ts +715 -0
  126. package/lib/github-git-auth.ts +207 -0
  127. package/lib/github-origin.ts +28 -0
  128. package/lib/github-tokens.ts +2439 -0
  129. package/lib/host-cloud-url.ts +62 -0
  130. package/lib/host-config.ts +279 -0
  131. package/lib/host-env.ts +424 -0
  132. package/lib/host-logs.ts +965 -0
  133. package/lib/internal-connections.ts +5 -0
  134. package/lib/keyed-promise-tail.ts +23 -0
  135. package/lib/launch-mode.ts +25 -0
  136. package/lib/lease.ts +239 -0
  137. package/lib/legacy-host.ts +769 -0
  138. package/lib/legacy-runtime-retirement.ts +424 -0
  139. package/lib/legacy-runtime-v1.fixture.ts +627 -0
  140. package/lib/machine-exec.ts +238 -0
  141. package/lib/machine-gateway-tunnel.ts +344 -0
  142. package/lib/machine-keys.ts +65 -0
  143. package/lib/machine-project-env-loader.ts +152 -0
  144. package/lib/machine-provider-aws.ts +923 -0
  145. package/lib/machine-provider-local.ts +338 -0
  146. package/lib/machine-provider.ts +90 -0
  147. package/lib/managed-activation-watcher.ts +72 -0
  148. package/lib/managed-install-owner-watcher.ts +55 -0
  149. package/lib/managed-operation-drain.ts +57 -0
  150. package/lib/managed-runtime.ts +4296 -0
  151. package/lib/managed-update-scheduler.ts +125 -0
  152. package/lib/mcp-capture.ts +865 -0
  153. package/lib/mcp-config-lock.ts +2 -0
  154. package/lib/mcp-connections.ts +959 -0
  155. package/lib/mcp-gateway.ts +2594 -0
  156. package/lib/obs.ts +512 -0
  157. package/lib/openrouter-budget.ts +247 -0
  158. package/lib/orchestrator.ts +11409 -0
  159. package/lib/person-capture.ts +327 -0
  160. package/lib/preview-ports.ts +85 -0
  161. package/lib/preview-sidecar.ts +331 -0
  162. package/lib/project-env-authority.ts +1319 -0
  163. package/lib/project-env-feature.ts +74 -0
  164. package/lib/project-env-legacy-tags.ts +130 -0
  165. package/lib/project-env-loader.ts +242 -0
  166. package/lib/project-env-replica.ts +2094 -0
  167. package/lib/project-env-schema-capture.ts +342 -0
  168. package/lib/project-env-staging.ts +148 -0
  169. package/lib/project-env-task-policy.ts +176 -0
  170. package/lib/project-env-task-replica.ts +1752 -0
  171. package/lib/release-feed.ts +61 -0
  172. package/lib/release-manifest.ts +710 -0
  173. package/lib/release-trust.ts +46 -0
  174. package/lib/repo-clone.ts +38 -0
  175. package/lib/routing/shadow.ts +261 -0
  176. package/lib/runner-boot.ts +10 -0
  177. package/lib/runtime-activation-tail.ts +279 -0
  178. package/lib/runtime-archive.ts +1216 -0
  179. package/lib/runtime-authority.ts +166 -0
  180. package/lib/runtime-guard.ts +36 -0
  181. package/lib/runtime-provider-state.ts +169 -0
  182. package/lib/runtime-reconnect.ts +86 -0
  183. package/lib/runtime-state.ts +1033 -0
  184. package/lib/safe-http.ts +364 -0
  185. package/lib/secrets-proxy/address.ts +95 -0
  186. package/lib/secrets-proxy/audit.ts +30 -0
  187. package/lib/secrets-proxy/ca-store.ts +200 -0
  188. package/lib/secrets-proxy/ca.ts +470 -0
  189. package/lib/secrets-proxy/container-steps.ts +138 -0
  190. package/lib/secrets-proxy/der.ts +273 -0
  191. package/lib/secrets-proxy/engine.ts +1633 -0
  192. package/lib/secrets-proxy/host.ts +93 -0
  193. package/lib/secrets-proxy/hostname.ts +57 -0
  194. package/lib/secrets-proxy/placeholders.ts +196 -0
  195. package/lib/secrets-proxy/policy.ts +150 -0
  196. package/lib/secrets-proxy/readiness.ts +148 -0
  197. package/lib/secrets-proxy/runtime.ts +582 -0
  198. package/lib/secrets-proxy/scrub.ts +559 -0
  199. package/lib/secrets-proxy/sidecar-view.ts +139 -0
  200. package/lib/secrets-proxy/task-scripts.ts +262 -0
  201. package/lib/secrets-proxy/task-trust.ts +605 -0
  202. package/lib/secrets-proxy/trust-watch.ts +80 -0
  203. package/lib/secrets.ts +71 -0
  204. package/lib/shared-files-s3.ts +976 -0
  205. package/lib/shared-files.ts +156 -0
  206. package/lib/sidecar-assign.ts +49 -0
  207. package/lib/sidecar-guard.ts +58 -0
  208. package/lib/sidecar-resume.ts +178 -0
  209. package/lib/sidecar.ts +69 -0
  210. package/lib/signin-box-credentials.ts +465 -0
  211. package/lib/signin-box.ts +114 -0
  212. package/lib/skill-index-example.fixture.ts +56 -0
  213. package/lib/skill-layout.ts +417 -0
  214. package/lib/skills.ts +489 -0
  215. package/lib/ssh.ts +401 -0
  216. package/lib/standard-image-cleanup.ts +117 -0
  217. package/lib/standard-image-pin.ts +207 -0
  218. package/lib/standard-image.ts +1697 -0
  219. package/lib/stopped-task-status-queue.ts +120 -0
  220. package/lib/task-container-cli.ts +269 -0
  221. package/lib/task-credential-cleanup.ts +752 -0
  222. package/lib/task-diff-sandbox.ts +464 -0
  223. package/lib/task-diff.ts +91 -0
  224. package/lib/task-env-forwarder.ts +228 -0
  225. package/lib/task-env-source.ts +249 -0
  226. package/lib/task-environment/anchor.ts +31 -0
  227. package/lib/task-environment/apple-container.ts +860 -0
  228. package/lib/task-environment/docker.ts +1344 -0
  229. package/lib/task-environment/index.ts +466 -0
  230. package/lib/task-environment/legacy-adoption.ts +467 -0
  231. package/lib/task-environment/machine-environment.ts +61 -0
  232. package/lib/task-environment/machine-task-up.ts +1227 -0
  233. package/lib/task-environment/machine.ts +974 -0
  234. package/lib/task-environment/registry.ts +59 -0
  235. package/lib/task-environment/self-identity.ts +20 -0
  236. package/lib/task-environment/self.ts +566 -0
  237. package/lib/task-environment/types.ts +642 -0
  238. package/lib/task-environment/workspace-files.ts +685 -0
  239. package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
  240. package/lib/task-files-machine-retirement-outbox.ts +592 -0
  241. package/lib/task-files-machine-retirement.ts +116 -0
  242. package/lib/task-files-readiness.ts +127 -0
  243. package/lib/task-gateway-port.ts +22 -0
  244. package/lib/task-gateway-protocol.ts +190 -0
  245. package/lib/task-gateway-recovery.ts +47 -0
  246. package/lib/task-gateway-state.ts +512 -0
  247. package/lib/task-identity.ts +19 -0
  248. package/lib/task-inventory.ts +682 -0
  249. package/lib/task-machine-credentials-activation.ts +95 -0
  250. package/lib/task-machine-swap-safety.ts +729 -0
  251. package/lib/task-person-credentials.ts +76 -0
  252. package/lib/task-project-ops-host.ts +147 -0
  253. package/lib/task-project-ops.ts +1062 -0
  254. package/lib/task-secret-staging.ts +16 -0
  255. package/lib/task-status.ts +46 -0
  256. package/lib/task-token.ts +46 -0
  257. package/lib/token-broker-adopt.ts +344 -0
  258. package/lib/token-broker-lambda.ts +105 -0
  259. package/lib/token-broker-local.ts +392 -0
  260. package/lib/token-broker-revision.ts +310 -0
  261. package/lib/token-broker-ssm.ts +525 -0
  262. package/lib/token-broker.ts +218 -0
  263. package/lib/token-lanes/openai-oauth.ts +123 -0
  264. package/lib/token-lanes/registry.ts +45 -0
  265. package/lib/token-lanes/types.ts +163 -0
  266. package/lib/token-lanes/xai-oauth.ts +165 -0
  267. package/lib/transcript.ts +128 -0
  268. package/lib/tunnel-registry.ts +220 -0
  269. package/lib/tunnel-runtime.ts +235 -0
  270. package/lib/ulid.ts +16 -0
  271. package/lib/user-bus-env.ts +116 -0
  272. package/lib/user-credentials-store.ts +435 -0
  273. package/lib/vnode-pressure.ts +142 -0
  274. package/package.json +115 -8
  275. package/runner/runner.mjs +442 -0
  276. package/scripts/agent/_common.sh +633 -0
  277. package/scripts/agent/env-hook.sh +626 -0
  278. package/scripts/agent/moondeck-cli.sh +5 -0
  279. package/scripts/agent/task-down.sh +291 -0
  280. package/scripts/agent/task-status.sh +70 -0
  281. package/scripts/agent/task-up.sh +3761 -0
  282. package/scripts/agent/uai-env-exec.sh +26 -0
  283. package/scripts/install/darwin.ts +1048 -0
  284. package/scripts/install/linux.ts +958 -0
  285. package/scripts/install/types.ts +78 -0
  286. package/scripts/install/util.ts +254 -0
  287. package/scripts/install/win.ts +143 -0
  288. package/src/activation-respawn.ts +155 -0
  289. package/src/apple-tunnel-route.ts +104 -0
  290. package/src/capture-wire.ts +102 -0
  291. package/src/cli.ts +2583 -0
  292. package/src/cloud-credentials.ts +2144 -0
  293. package/src/cloud-origins.ts +23 -0
  294. package/src/credential-location.ts +248 -0
  295. package/src/credential-wire-validation.ts +25 -0
  296. package/src/custom-connections.ts +237 -0
  297. package/src/engine-account-paths.ts +11 -0
  298. package/src/engine-models.ts +39 -0
  299. package/src/engine-usage.ts +104 -0
  300. package/src/event-outbox.ts +185 -0
  301. package/src/index.ts +2899 -0
  302. package/src/load-env.ts +97 -0
  303. package/src/machine-tunnel-route.ts +156 -0
  304. package/src/main.ts +6349 -0
  305. package/src/never-proxied-domains.ts +31 -0
  306. package/src/outbound-frame.ts +31 -0
  307. package/src/paths.ts +81 -0
  308. package/src/project-env-latest.ts +130 -0
  309. package/src/project-env-policy.ts +396 -0
  310. package/src/project-env-wording.ts +53 -0
  311. package/src/protocol.ts +5545 -0
  312. package/src/runtime-bootstrap.ts +165 -0
  313. package/src/station-identity.ts +96 -0
  314. package/src/task-env-names.ts +28 -0
  315. package/src/task-secrets.ts +75 -0
  316. package/src/tunnel-dial.ts +102 -0
  317. package/src/ui/server.ts +852 -0
  318. package/src/ui/types.ts +173 -0
  319. package/src/workspace-path-policy.ts +41 -0
  320. package/ui/app.js +1102 -0
  321. package/ui/index.html +90 -0
  322. package/ui/style.css +685 -0
  323. package/ui/uai-favicon.svg +21 -0
  324. package/ui/uai-logo-black.svg +9 -0
  325. package/ui/uai-wheel.svg +9 -0
  326. package/bin/moondeck-station.mjs +0 -3
@@ -0,0 +1,1697 @@
1
+ /**
2
+ * Standard-image build pipeline (ADR-022 / docs/runtime.md).
3
+ *
4
+ * Every host builds exactly one standard image (`moondeck-standard:dev`) and
5
+ * caches it. `ensureStandardImage()` runs at startup and on the owned-host
6
+ * hourly maintenance schedule (local build failures back off):
7
+ *
8
+ * 1. `docker volume create moondeck-asdf-data` — the host-wide asdf data volume
9
+ * that caches lazily-installed runtime versions across tasks (idempotent;
10
+ * `docker volume create` is a no-op if the volume already exists).
11
+ * 2. `docker image inspect moondeck-standard:dev` — if it resolves, the image is
12
+ * present and we skip the build. Otherwise `docker build` it from
13
+ * `host-agent/images/standard/`.
14
+ *
15
+ * Best-effort: any failure (docker missing, daemon down, build error) is
16
+ * logged and swallowed so the host process still boots and serves tunnels.
17
+ * task-up surfaces the real error to the user if a build is genuinely needed.
18
+ */
19
+
20
+ import { spawn } from "node:child_process";
21
+ import { readFileSync, writeFileSync } from "node:fs";
22
+ import { readFile } from "node:fs/promises";
23
+ import { dirname, join, resolve } from "node:path";
24
+ import { fileURLToPath } from "node:url";
25
+
26
+ import {
27
+ appleOneOffNeedsWatch,
28
+ appleOneOffRunArgs,
29
+ askRuntimeOwnerOnce,
30
+ newAppleOneOff,
31
+ settleAppleOneOff,
32
+ watchAppleOneOff,
33
+ } from "./apple-one-off";
34
+ import { recycleAppleRuntimeServices, type AppleRecycleOutcome } from "./apple-runtime-recycle";
35
+ import { detectEngine } from "./engines";
36
+ import { hasStoredEngineAccounts } from "./engine-accounts";
37
+ import {
38
+ appleContainerRuntimeBinding,
39
+ claimContainerRuntimeForMaintenance,
40
+ containerRuntimeStandardAssetsRoot,
41
+ containerRuntimeStandardImageRoot,
42
+ pinnedContainerRuntimeProvider,
43
+ requestContainerRuntimeRecheck,
44
+ } from "./container-runtime";
45
+ import { KeyedPromiseTail } from "./keyed-promise-tail";
46
+ import {
47
+ CONTEXT_HASH_LABEL,
48
+ hashStandardImageContext,
49
+ parseStandardImagePin,
50
+ STANDARD_IMAGE_PIN_BASENAME,
51
+ type StandardImagePin,
52
+ standardImagePullCommands,
53
+ standardImagePullDigest,
54
+ } from "./standard-image-pin";
55
+ import { sidecarMode } from "./sidecar";
56
+ import { StandardImageCleanup, StandardImageBuildBackoff } from "./standard-image-cleanup";
57
+
58
+ /** Pinned, host-wide constants (must match task-up.sh and the compose gen). */
59
+ export const STANDARD_IMAGE_TAG = "moondeck-standard:dev";
60
+ export const ASDF_DATA_VOLUME = "moondeck-asdf-data";
61
+
62
+ /**
63
+ * The shared-volume agent CLIs belong to this exact HOME-default Node. A
64
+ * future default bump cannot transparently roll stopped containers: their
65
+ * baked /home/node/.tool-versions still names the old CLI-owning candidate.
66
+ * Changing this value therefore requires an explicit volume migration or
67
+ * container-recreation release slice, not an ordinary image refresh.
68
+ */
69
+ export const STANDARD_DEFAULT_NODE_VERSION = "22.11.0";
70
+
71
+ /** Exact Codex CLI release exposed to task runtimes through the shared volume. */
72
+ export const STANDARD_CODEX_CLI_VERSION = "0.164.0";
73
+
74
+ /**
75
+ * Runtimes the standard image's asdf plugins can satisfy, with a sensible set
76
+ * of advertised versions per kind (ADR-021 `runtimes`). These are the
77
+ * versions the cloud's task-creation picker hints; asdf will lazily install
78
+ * any of them on first use (docs/runtime.md). The first entry of each list is
79
+ * the root `/etc/.tool-versions` default baked into the image.
80
+ *
81
+ * KEEP IN SYNC with host-agent/images/standard/ (the asdf plugins and the
82
+ * root tool-versions). Treat that directory as authoritative for the numbers;
83
+ * this list is the advertisement surface.
84
+ */
85
+ export const STANDARD_RUNTIMES: ReadonlyArray<{
86
+ kind: string;
87
+ availableVersions: string[];
88
+ }> = [
89
+ {
90
+ kind: "node",
91
+ availableVersions: [
92
+ STANDARD_DEFAULT_NODE_VERSION,
93
+ "20.18.0",
94
+ "18.20.4",
95
+ ],
96
+ },
97
+ { kind: "python", availableVersions: ["3.12.4", "3.11.9"] },
98
+ { kind: "go", availableVersions: ["1.23.2", "1.22.8"] },
99
+ { kind: "ruby", availableVersions: ["3.3.5", "3.2.5"] },
100
+ { kind: "rust", availableVersions: ["1.81.0"] },
101
+ ];
102
+
103
+ /** A fresh, mutable copy of the advertised runtimes for the capability frame. */
104
+ export function standardRuntimes(): Array<{
105
+ kind: string;
106
+ availableVersions: string[];
107
+ }> {
108
+ return STANDARD_RUNTIMES.map((r) => ({
109
+ kind: r.kind,
110
+ availableVersions: [...r.availableVersions],
111
+ }));
112
+ }
113
+
114
+ /**
115
+ * Which OPTIONAL agent CLIs the operator has actually configured — so the
116
+ * image installs only those, not every engine on every host. Delegates to
117
+ * engines.ts detection so every connect flavor counts: kimi's copied cred
118
+ * file, grok's login file OR a pasted XAI_API_KEY, cursor's CURSOR_API_KEY
119
+ * (claude/codex are always baked). Folded into the build hash below, so
120
+ * connecting a new engine triggers a rebuild.
121
+ */
122
+ export function configuredOptionalEngines(): {
123
+ kimi: boolean;
124
+ grok: boolean;
125
+ cursor: boolean;
126
+ opencode: boolean;
127
+ } {
128
+ // A stored account (a workspace's, on a shared host; a labeled extra on an
129
+ // owned one) needs the CLI in the image just like the host's own sign-in
130
+ // does — without this a grok/cursor account existed but its tasks exited
131
+ // 127 for a missing binary.
132
+ const on = (kind: "kimi" | "grok" | "cursor" | "opencode"): boolean =>
133
+ detectEngine(kind) || hasStoredEngineAccounts(kind);
134
+ return {
135
+ kimi: on("kimi"),
136
+ grok: on("grok"),
137
+ cursor: on("cursor"),
138
+ opencode: on("opencode"),
139
+ };
140
+ }
141
+
142
+ type OptionalEngineConfiguration = ReturnType<
143
+ typeof configuredOptionalEngines
144
+ >;
145
+
146
+ function optionalEngineConfigurationKey(
147
+ engines: OptionalEngineConfiguration,
148
+ ): string {
149
+ return JSON.stringify(engines);
150
+ }
151
+
152
+ // Build args (which optional engines are installed) are part of the image
153
+ // identity — a config change must invalidate the label so it rebuilds. The
154
+ // walk-and-hash itself lives in standard-image-pin.ts so CI computes the
155
+ // identical identity (ADR-114).
156
+ function hashBuildContext(extra = ""): Promise<string | null> {
157
+ return hashStandardImageContext(standardImageDir(), extra);
158
+ }
159
+
160
+ /** The packaged digest pin (ADR-114) at `agent/images/standard-image.pin.json`,
161
+ * one level above the build context so it never perturbs the context hash.
162
+ * Absent in dev checkouts and npm installs — those keep the local build.
163
+ * `UAI_IMAGE_PULL=0` is the operator kill switch back to local semantics. */
164
+ async function readStandardImagePin(): Promise<StandardImagePin | null> {
165
+ if (process.env.UAI_IMAGE_PULL === "0") return null;
166
+ let raw: string;
167
+ try {
168
+ raw = await readFile(
169
+ resolve(standardImageDir(), "..", STANDARD_IMAGE_PIN_BASENAME),
170
+ "utf8",
171
+ );
172
+ } catch {
173
+ return null;
174
+ }
175
+ const pin = parseStandardImagePin(raw);
176
+ if (pin === null) {
177
+ console.warn(
178
+ "[host-agent] packaged standard-image pin is malformed; using the local build path",
179
+ );
180
+ }
181
+ return pin;
182
+ }
183
+
184
+ /** Absolute path to the standard image build context. */
185
+ function standardImageDir(): string {
186
+ const providerRoot = containerRuntimeStandardImageRoot();
187
+ if (providerRoot !== null) return providerRoot;
188
+ const here = dirname(fileURLToPath(import.meta.url));
189
+ // lib/standard-image.ts -> host-agent/ -> images/standard
190
+ return resolve(here, "..", "images", "standard");
191
+ }
192
+
193
+ /** Absolute packaged path used to bind the materializer over rolling images
194
+ * that predate the baked executable. Resolution is deterministic; callers get
195
+ * the ordinary Docker bind/open error if a package install is corrupt. */
196
+ export function standardImageRuntimeMaterializerPath(): string {
197
+ return join(
198
+ standardContainerAssetsDir(),
199
+ "uai-materialize-runtimes",
200
+ );
201
+ }
202
+
203
+ /** Absolute packaged Corepack pin bound over both rolling and derived images. */
204
+ export function standardImageCorepackVersionPath(): string {
205
+ return join(standardContainerAssetsDir(), "corepack-version");
206
+ }
207
+
208
+ /** ADR-106: the retired bundled-VM fallback exposed a provider-specific
209
+ * asset root; every current backend uses the package's ordinary
210
+ * build-context assets, so the runtime override below is always null. */
211
+ function standardContainerAssetsDir(): string {
212
+ return (
213
+ containerRuntimeStandardAssetsRoot() ?? join(standardImageDir(), "container")
214
+ );
215
+ }
216
+
217
+ interface RunResult {
218
+ code: number | null;
219
+ stdout: string;
220
+ stderr: string;
221
+ }
222
+
223
+ const SHORT_DOCKER_TIMEOUT_MS = 30_000;
224
+ const CLI_MAINTENANCE_TIMEOUT_MS = 10 * 60_000;
225
+ // Overridable ceiling: the hard-coded 15 minutes looped a fanless Air forever
226
+ // (live 2026-08-19 — every attempt died at 15:00 and the activation driver
227
+ // silently restarted it). Slow hardware needs a bigger window, not a loop.
228
+ const IMAGE_BUILD_TIMEOUT_MS = imageBuildTimeoutMs();
229
+ function imageBuildTimeoutMs(): number {
230
+ const minutes = Number(process.env.UAI_IMAGE_BUILD_TIMEOUT_MINUTES ?? "");
231
+ if (Number.isFinite(minutes) && minutes >= 1) {
232
+ return Math.min(minutes, 240) * 60_000;
233
+ }
234
+ return 15 * 60_000;
235
+ }
236
+ const IMAGE_PULL_TIMEOUT_MS = 15 * 60_000;
237
+ const TERMINATE_GRACE_MS = 5_000;
238
+ export const ASDF_MAINTENANCE_LOCK_PATH =
239
+ "/opt/asdf-data/.uai-maintenance.lock";
240
+ const ASDF_MAINTENANCE_LOCK_WAIT_SECONDS = 300;
241
+ const ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS = 570;
242
+
243
+ /** The selected container engine for image/volume maintenance (ADR-106).
244
+ * Resolved per call: the runtime pin settles before ensureStandardImage runs,
245
+ * but an HMR reload must not strand a stale CLI path. The docker fallback
246
+ * covers the pre-selection window, where docker-only behavior is historical. */
247
+ function selectedEngine(): { command: string; apple: boolean } {
248
+ // The PIN, not the readiness-gated identity: this runs during activation
249
+ // while the public state is still `checking`, and falling back to docker
250
+ // there deadlocks a Docker-free Mac.
251
+ if (pinnedContainerRuntimeProvider() === "apple-container") {
252
+ const binding = appleContainerRuntimeBinding();
253
+ if (binding !== null) {
254
+ return { command: binding.containerCliPath, apple: true };
255
+ }
256
+ }
257
+ return { command: "docker", apple: false };
258
+ }
259
+
260
+ /** The most bytes of each stream a standard-image command keeps; the rest is
261
+ * read and dropped, so a child that writes without end holds no more of the
262
+ * runner's memory than this. */
263
+ const RUN_CAPTURE_BYTES = 8 * 1024 * 1024;
264
+
265
+ /** One stream, kept to at most RUN_CAPTURE_BYTES of its encoded bytes; the
266
+ * runner's own diagnostics go through the same bound. */
267
+ class BoundedCapture {
268
+ #chunks: Buffer[] = [];
269
+ #bytes = 0;
270
+ #lastByte: number | null = null;
271
+
272
+ add(chunk: Buffer | string): void {
273
+ const bytes = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk;
274
+ if (bytes.length === 0 || this.#bytes >= RUN_CAPTURE_BYTES) return;
275
+ const kept = bytes.subarray(0, RUN_CAPTURE_BYTES - this.#bytes);
276
+ this.#chunks.push(kept);
277
+ this.#bytes += kept.length;
278
+ this.#lastByte = kept[kept.length - 1]!;
279
+ }
280
+
281
+ /** A line of the runner's own, on a line of its own. */
282
+ addLine(line: string): void {
283
+ this.add(`${this.#lastByte !== null && this.#lastByte !== 0x0a ? "\n" : ""}${line}`);
284
+ }
285
+
286
+ /** What it kept, as text that keeps the bound too: a character the
287
+ * bound cut, or bytes that are not UTF-8, decode to a three-byte
288
+ * replacement character, so past the bound the text ends at its last
289
+ * whole character. */
290
+ text(): string {
291
+ const text = Buffer.concat(this.#chunks).toString("utf8");
292
+ if (Buffer.byteLength(text, "utf8") <= RUN_CAPTURE_BYTES) return text;
293
+ const encoded = Buffer.from(text, "utf8");
294
+ let end = RUN_CAPTURE_BYTES;
295
+ // Back to the first byte of the character the bound cuts.
296
+ while (end > 0 && (encoded[end]! & 0xc0) === 0x80) end -= 1;
297
+ return encoded.subarray(0, end).toString("utf8");
298
+ }
299
+ }
300
+
301
+ /** Run a command to completion, capturing stdio, at most RUN_CAPTURE_BYTES
302
+ * of each. Never rejects and never waits forever: standard-image maintenance
303
+ * is an activation prerequisite, so an unbounded Docker/npm child would
304
+ * strand the entire host in checking. */
305
+ function run(
306
+ command: string,
307
+ args: string[],
308
+ timeoutMs = SHORT_DOCKER_TIMEOUT_MS,
309
+ ): Promise<RunResult> {
310
+ return new Promise<RunResult>((resolveRun) => {
311
+ const stdout = new BoundedCapture();
312
+ const stderr = new BoundedCapture();
313
+ let settled = false;
314
+ let killTimer: NodeJS.Timeout | null = null;
315
+ const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
316
+ const settle = (code: number | null): void => {
317
+ if (settled) return;
318
+ settled = true;
319
+ clearTimeout(timeoutTimer);
320
+ if (killTimer) clearTimeout(killTimer);
321
+ resolveRun({ code, stdout: stdout.text(), stderr: stderr.text() });
322
+ };
323
+ const timeoutTimer = setTimeout(() => {
324
+ stderr.addLine(`${command} timed out after ${timeoutMs}ms`);
325
+ child.kill("SIGTERM");
326
+ killTimer = setTimeout(() => {
327
+ child.kill("SIGKILL");
328
+ // SIGKILL should always produce close, but do not let a broken child
329
+ // handle defeat the host-wide readiness bound.
330
+ const settleTimer = setTimeout(() => settle(null), 1_000);
331
+ settleTimer.unref();
332
+ }, TERMINATE_GRACE_MS);
333
+ killTimer.unref();
334
+ }, timeoutMs);
335
+ timeoutTimer.unref();
336
+ child.stdout?.on("data", (chunk: Buffer) => {
337
+ stdout.add(chunk);
338
+ });
339
+ child.stderr?.on("data", (chunk: Buffer) => {
340
+ stderr.add(chunk);
341
+ });
342
+ child.on("error", (err: Error) => {
343
+ stderr.add(err.message);
344
+ settle(null);
345
+ });
346
+ child.on("close", (code) => {
347
+ settle(code);
348
+ });
349
+ });
350
+ }
351
+
352
+ /** Narrow process-level seam for the timeout regression test. */
353
+ export const runStandardImageCommandForTest = run;
354
+
355
+ /** How long a maintenance run may sit without its container reaching
356
+ * `running` before the wedge sensor passes judgment. Long HEALTHY operations
357
+ * (reconcile's npm installs, the seed copy) are past `running` within
358
+ * seconds, so this discriminates cleanly. Env-overridable for tests. */
359
+ function wedgeProbeDelayMs(): number {
360
+ const raw = Number(process.env.UAI_APPLE_WEDGE_PROBE_MS);
361
+ return Number.isFinite(raw) && raw > 0 ? raw : 60_000;
362
+ }
363
+
364
+ /** Post-recycle grace for the hung CLI client before a synthesized result. */
365
+ function wedgeGraceMs(): number {
366
+ const raw = Number(process.env.UAI_APPLE_WEDGE_GRACE_MS);
367
+ return Number.isFinite(raw) && raw > 0 ? raw : 10_000;
368
+ }
369
+
370
+ function delayMs(ms: number): Promise<void> {
371
+ return new Promise((resolveDelay) => {
372
+ const timer = setTimeout(resolveDelay, ms);
373
+ timer.unref();
374
+ });
375
+ }
376
+
377
+ /**
378
+ * One DEFINITIVE observation of the named container: a successful, parseable
379
+ * inspect reporting a non-running state. Anything else — inspect failure,
380
+ * unparseable output, missing state, or `running` — is NOT evidence of a
381
+ * wedge and must stand the sensor down (the old fall-through treated
382
+ * uncertainty as wedge evidence, which authorizes a service recycle on a
383
+ * guess).
384
+ */
385
+ async function appleContainerDefinitivelyNotRunning(
386
+ engine: { command: string },
387
+ name: string,
388
+ ): Promise<boolean> {
389
+ const inspected = await run(engine.command, ["inspect", name], 15_000);
390
+ if (inspected.code !== 0) return false;
391
+ try {
392
+ const parsed = JSON.parse(inspected.stdout) as Array<{
393
+ id?: unknown;
394
+ status?: { state?: unknown };
395
+ }>;
396
+ if (!Array.isArray(parsed) || parsed.length !== 1) return false;
397
+ if (parsed[0]?.id !== name) return false;
398
+ const state = parsed[0]?.status?.state;
399
+ return typeof state === "string" && state !== "running";
400
+ } catch {
401
+ return false;
402
+ }
403
+ }
404
+
405
+ /**
406
+ * The wedge signature, judged live: the named container has NOT reached
407
+ * `running` (definitively — see above) while the control plane still answers
408
+ * a `list`, observed STABLY: a second exact inspection after the list (and a
409
+ * short settle) must repeat the non-running verdict immediately before any
410
+ * recycle is authorized, closing the window where the container starts while
411
+ * the list is in flight. A dead apiserver fails the `list` too (that is an
412
+ * outage, not a wedge, and recycling on it would be a guess).
413
+ */
414
+ async function appleWedgeVerdict(
415
+ engine: { command: string },
416
+ name: string,
417
+ ): Promise<boolean> {
418
+ if (!(await appleContainerDefinitivelyNotRunning(engine, name))) {
419
+ return false;
420
+ }
421
+ const listed = await run(engine.command, ["list", "--format", "json"], 15_000);
422
+ if (listed.code !== 0) return false;
423
+ await delayMs(Math.min(5_000, wedgeProbeDelayMs()));
424
+ return appleContainerDefinitivelyNotRunning(engine, name);
425
+ }
426
+
427
+ /**
428
+ * Run a maintenance container with corpse hygiene on the Apple runtime.
429
+ *
430
+ * `container run --rm` only removes a container that STARTED. A run refused
431
+ * at start time (the exclusive-volume attach, most commonly) leaves a
432
+ * created-never-started container behind, and those corpses wedge the
433
+ * runtime's apiserver (observed live 2026-08-16, twice: run/delete hang
434
+ * while list still answers). So the container is named and marked Uai's
435
+ * (apple-one-off.ts), and a failed run is settled: its container proven
436
+ * gone, deleted until it is, the runtime's services recycled when it will
437
+ * not go; a run whose client never reported its end is watched a while
438
+ * longer for a container created late. Docker's `--rm` is reliable and its
439
+ * containers are not corpse-prone, so the Docker path passes through.
440
+ *
441
+ * Every apple run doubles as the WEDGE SENSOR (the owner's "this needs to
442
+ * be automatic"): if the container has not reached `running` after the
443
+ * probe delay while `list` still answers, the run reports a distinctive
444
+ * retryable failure. Within the runtime's current activation, the runtime
445
+ * services are first recycled in place (see apple-runtime-recycle.ts), and
446
+ * the activation recovers every task after, its retry loop re-running
447
+ * maintenance against the fresh runtime. Within an activation that is no
448
+ * longer current, nothing restarts the runtime: the current one is coming,
449
+ * with its own maintenance and recovery. Outside any (the hourly pass, a
450
+ * task's start, an engine's sign-in), nothing restarts the runtime either:
451
+ * its owner is asked to recheck it, whose activation does (#p2jdax).
452
+ */
453
+ async function runMaintenanceContainer(
454
+ engine: { command: string; apple: boolean },
455
+ purpose: string,
456
+ argsAfterRun: string[],
457
+ timeoutMs: number,
458
+ activation: ActivationIsCurrent | null,
459
+ ): Promise<RunResult> {
460
+ if (!engine.apple) {
461
+ return run(engine.command, ["run", ...argsAfterRun], timeoutMs);
462
+ }
463
+ const oneOff = newAppleOneOff(purpose);
464
+ const { name } = oneOff;
465
+ const cli = (args: string[], cliTimeoutMs: number) => run(engine.command, args, cliTimeoutMs);
466
+ // The remedy outside activation, and for a watch (which outlives any
467
+ // activation): ask the runtime's owner, once for this run.
468
+ const askOwner = askRuntimeOwnerOnce(
469
+ requestContainerRuntimeRecheck,
470
+ `maintenance container ${name} found the Apple runtime wedged, or could not be removed`,
471
+ );
472
+ // The remedy within an activation: the runtime's services restarted, only
473
+ // while that activation is current.
474
+ const notCurrent = "its activation is no longer the current one";
475
+ const restart = async (): Promise<AppleRecycleOutcome> =>
476
+ activation?.() ? recycleAppleRuntimeServices({ cli }) : { recycled: false, reason: notCurrent };
477
+ let wedgedUnremedied = false;
478
+ let runSettled = false;
479
+ const running = run(
480
+ engine.command,
481
+ // Its labels are what authorize deleting this container later (here, by
482
+ // its run's token; in the recycle sweep, by Uai's) — a name alone is
483
+ // forgeable.
484
+ ["run", ...appleOneOffRunArgs(oneOff), ...argsAfterRun],
485
+ timeoutMs,
486
+ ).then((outcome) => {
487
+ runSettled = true;
488
+ return outcome;
489
+ });
490
+ const sensed = (async (): Promise<RunResult | null> => {
491
+ await delayMs(wedgeProbeDelayMs());
492
+ if (runSettled) return null;
493
+ if (!(await appleWedgeVerdict(engine, name))) return null;
494
+ if (runSettled) return null;
495
+ if (activation === null || !activation()) {
496
+ wedgedUnremedied = true;
497
+ const reason = activation === null ? (await askOwner()).reason : notCurrent;
498
+ return {
499
+ code: null,
500
+ stdout: "",
501
+ stderr: `apple container runtime is wedged; ${reason} — retry this operation`,
502
+ };
503
+ }
504
+ const outcome = await restart();
505
+ // The hung CLI client may exit once its apiserver dies; give it a short
506
+ // grace so a real result wins, then synthesize the retryable failure.
507
+ const graced = await Promise.race([
508
+ running,
509
+ delayMs(wedgeGraceMs()).then(() => null),
510
+ ]);
511
+ if (graced) return graced;
512
+ return {
513
+ code: null,
514
+ stdout: "",
515
+ stderr: outcome.recycled
516
+ ? "apple container runtime was wedged; runtime services were recycled — retry this operation"
517
+ : `apple container runtime is wedged and the automatic recycle did not recover it: ${outcome.reason}`,
518
+ };
519
+ })();
520
+ const result = await Promise.race([
521
+ running,
522
+ sensed.then((synthesized) => synthesized ?? running),
523
+ ]);
524
+ if (result.code !== 0) {
525
+ const watched = { cli, recycle: askOwner };
526
+ if (wedgedUnremedied) {
527
+ // Nothing can be deleted until a current activation has restarted the
528
+ // runtime: its name is only watched, and this pass (which that
529
+ // activation's own may wait behind) ends now.
530
+ void watchAppleOneOff(watched, oneOff, { lateCreation: true });
531
+ return result;
532
+ }
533
+ // The run may have failed BECAUSE this generated name already existed —
534
+ // then the container is not ours and the ownership label was never
535
+ // applied (review 2026-08-18 round 3): only a positive label proof
536
+ // authorizes deleting it, and an unobservable one is never recycled for.
537
+ const settled = await settleAppleOneOff(
538
+ { cli, recycle: activation ? restart : askOwner },
539
+ oneOff,
540
+ );
541
+ // Unproven, or its client never reported its end (the container may yet
542
+ // be created): watched until it is settled.
543
+ const lateCreation = result.code === null;
544
+ if (appleOneOffNeedsWatch(settled, lateCreation)) void watchAppleOneOff(watched, oneOff, { lateCreation });
545
+ }
546
+ return result;
547
+ }
548
+
549
+ /**
550
+ * Agent CLIs that MUST exist inside the shared asdf volume. The volume mounts
551
+ * over /opt/asdf-data and shadows the image's baked-in shims, so a volume
552
+ * seeded before a CLI was added (or before a version bump) can be missing it —
553
+ * e.g. a stale volume with `codex` but no `claude`, which makes `claude` exit
554
+ * 127 in every task container. Reconciled on each host start.
555
+ */
556
+ const VOLUME_AGENT_CLIS: {
557
+ bin: string;
558
+ pkg: string;
559
+ version: string;
560
+ expectedVersionOutput?: string;
561
+ }[] = [
562
+ { bin: "claude", pkg: "@anthropic-ai/claude-code", version: "latest" },
563
+ {
564
+ bin: "codex",
565
+ pkg: "@openai/codex",
566
+ version: STANDARD_CODEX_CLI_VERSION,
567
+ expectedVersionOutput: `codex-cli ${STANDARD_CODEX_CLI_VERSION}`,
568
+ },
569
+ ];
570
+
571
+ /**
572
+ * Cheap, non-mutating diagnostic probe. This is never the authoritative final
573
+ * proof: another process can begin maintenance after a read-only check. The
574
+ * locked reconcile below always performs its own proof before releasing.
575
+ */
576
+ async function probeVolumeAgentClis(activation: ActivationIsCurrent | null): Promise<string[] | null> {
577
+ const checks = VOLUME_AGENT_CLIS.map((cli) => {
578
+ const missing = `echo "UAI_MISSING:${cli.bin}"`;
579
+ if (cli.expectedVersionOutput === undefined) {
580
+ return `"${cli.bin}" --version >/dev/null 2>&1 || ${missing}`;
581
+ }
582
+ return (
583
+ `version="$("${cli.bin}" --version 2>/dev/null)" && ` +
584
+ `[ "$version" = "${cli.expectedVersionOutput}" ] || ${missing}`
585
+ );
586
+ }).join("; ");
587
+ // Probe by EXECUTING each bin (`--version`), not `command -v`. A stale asdf
588
+ // shim — the shim file survives in the volume but the reshim DB no longer
589
+ // maps it to an installed exe — passes `command -v` (the file is present and
590
+ // executable) yet aborts at runtime with "unknown command: codex. Perhaps
591
+ // you have to reshim?" (exit 1). That made a broken codex/claude read as
592
+ // "present," so the self-heal skipped it and every task/resume spawn failed,
593
+ // restart after restart (live 2026-07-22). Running the bin is the only probe
594
+ // that catches a stale shim. `-w /home/node` so asdf resolves the node
595
+ // version from /home/node/.tool-versions — the same one the repair installs
596
+ // against — and the sentinel prefix keeps login-shell noise from being
597
+ // misread as a missing bin.
598
+ const check = await runMaintenanceContainer(selectedEngine(), "cli-probe", [
599
+ "--rm",
600
+ "-w",
601
+ "/home/node",
602
+ "-v",
603
+ `${ASDF_DATA_VOLUME}:/opt/asdf-data:ro`,
604
+ STANDARD_IMAGE_TAG,
605
+ "/bin/bash",
606
+ "-c",
607
+ checks,
608
+ ], SHORT_DOCKER_TIMEOUT_MS, activation);
609
+ if (check.code !== 0) {
610
+ console.warn(
611
+ `[host-agent] could not check agent CLIs in ${ASDF_DATA_VOLUME} (exit ` +
612
+ `${check.code ?? "spawn"}); continuing. ${check.stderr.trim()}`,
613
+ );
614
+ return null;
615
+ }
616
+ return check.stdout
617
+ .split("\n")
618
+ .map((l) => l.trim())
619
+ .filter((l) => l.startsWith("UAI_MISSING:"))
620
+ .map((l) => l.slice("UAI_MISSING:".length));
621
+ }
622
+
623
+ /**
624
+ * Create/repair the shared lock as a regular node-writable file. The lock is
625
+ * seeded into fresh volumes by the image. Legacy volumes can contain a 0644
626
+ * root-created file; opening it read-only permits flock, then metadata repair
627
+ * occurs while holding that same inode (never replace it under a waiter).
628
+ */
629
+ async function prepareAsdfMaintenanceLock(activation: ActivationIsCurrent | null): Promise<boolean> {
630
+ const command =
631
+ `set -eu; lock=${ASDF_MAINTENANCE_LOCK_PATH}; ` +
632
+ `if [ -L "$lock" ]; then echo "unsafe symlink lock: $lock" >&2; exit 1; fi; ` +
633
+ `if [ ! -e "$lock" ]; then umask 002; : > "$lock"; fi; ` +
634
+ `if [ ! -f "$lock" ]; then echo "lock is not regular: $lock" >&2; exit 1; fi; ` +
635
+ `exec 9<"$lock"; /usr/bin/flock --exclusive --timeout ${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS} 9; ` +
636
+ `/usr/bin/chown node:node "$lock"; /usr/bin/chmod 0664 "$lock"`;
637
+ const result = await runMaintenanceContainer(selectedEngine(), "lock", [
638
+ "--rm",
639
+ "-u",
640
+ "root",
641
+ "-v",
642
+ `${ASDF_DATA_VOLUME}:/opt/asdf-data`,
643
+ STANDARD_IMAGE_TAG,
644
+ "/usr/bin/timeout",
645
+ "--signal=TERM",
646
+ "--kill-after=10s",
647
+ `${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
648
+ "/bin/bash",
649
+ "-c",
650
+ command,
651
+ ], CLI_MAINTENANCE_TIMEOUT_MS, activation);
652
+ if (result.code === 0) return true;
653
+ lastMaintenanceStderr = result.stderr;
654
+ console.warn(
655
+ `[host-agent] could not prepare ${ASDF_MAINTENANCE_LOCK_PATH} (exit ` +
656
+ `${result.code ?? "spawn"}): ${result.stderr.trim()}`,
657
+ );
658
+ return false;
659
+ }
660
+
661
+ /**
662
+ * One cross-process transaction owns upgrade, repair, Corepack repair, reshim,
663
+ * and the authoritative execution proof. A waiter always acquires the lock and
664
+ * performs its own full operation; a concurrent holder is never treated as
665
+ * success. The in-container timeout survives a disconnected Docker client and
666
+ * remains strictly inside the host-side child timeout.
667
+ */
668
+ /** Build the single locked maintenance program. Exported only so focused
669
+ * tests can assert the crash-repair and proof contract without duplicating
670
+ * this security-sensitive shell in a fixture. */
671
+ export function standardImageVolumeMaintenanceCommandForTest(
672
+ autoUpdate: boolean,
673
+ ): string {
674
+ const pkgs = VOLUME_AGENT_CLIS.map(
675
+ (cli) => `${cli.pkg}@${cli.version}`,
676
+ ).join(" ");
677
+ const trustedNpmFlags =
678
+ "--registry=https://registry.npmjs.org/ --userconfig=/dev/null " +
679
+ "--globalconfig=/dev/uai-empty-npmrc";
680
+ return (
681
+ `set -u; upgrade=${autoUpdate ? 1 : 0}; ` +
682
+ `fail() { echo "UAI_STILL_BROKEN:$1" >&2; exit 1; }; ` +
683
+ `export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; ` +
684
+ `export HOME=/home/node ASDF_DIR=/opt/asdf ASDF_DATA_DIR=/opt/asdf-data ` +
685
+ `ASDF_CONFIG_FILE=/dev/null ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
686
+ `COREPACK_HOME=/opt/asdf-data/corepack COREPACK_DEFAULT_TO_LATEST=0 ` +
687
+ `COREPACK_ENABLE_AUTO_PIN=0 COREPACK_ENABLE_DOWNLOAD_PROMPT=0 ` +
688
+ `COREPACK_ENV_FILE=0 COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
689
+ `COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
690
+ `COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org; ` +
691
+ `default_node="$(/usr/bin/awk '$1 == "nodejs" { print $2; exit }' /home/node/.tool-versions)"; ` +
692
+ `required_codex="${STANDARD_CODEX_CLI_VERSION}"; ` +
693
+ `if ! printf '%s' "$default_node" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$'; then ` +
694
+ `fail default-node; fi; ` +
695
+ `[ "$default_node" = "${STANDARD_DEFAULT_NODE_VERSION}" ] || ` +
696
+ `fail default-node-release-migration-required; ` +
697
+ `export ASDF_NODEJS_VERSION="$default_node"; ` +
698
+ `if [ -e /dev/uai-empty-npmrc ] || [ -L /dev/uai-empty-npmrc ]; then ` +
699
+ `fail unsafe-npm-config; fi; ` +
700
+ `installs_root="$ASDF_DATA_DIR/installs"; node_root="$installs_root/nodejs"; ` +
701
+ `candidate="$node_root/$default_node"; shims_root="$ASDF_DATA_DIR/shims"; ` +
702
+ `corepack_root="$COREPACK_HOME"; ` +
703
+ `ensure_real_dir() { path="$1"; label="$2"; ` +
704
+ `if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then ` +
705
+ `fail "unsafe-$label"; fi; ` +
706
+ `if [ ! -d "$path" ]; then /usr/bin/install -d -o node -g node -m 0775 "$path" ` +
707
+ `|| fail "create-$label"; fi; ` +
708
+ `lexical="$(/usr/bin/realpath -m -s -- "$path")"; ` +
709
+ `resolved="$(/usr/bin/realpath -e -- "$path")" ` +
710
+ `|| fail "resolve-$label"; [ "$lexical" = "$resolved" ] ` +
711
+ `|| fail "escaped-$label"; /usr/bin/chown node:node "$path" ` +
712
+ `|| fail "owner-$label"; /usr/bin/chmod u+rwx,g+rwx "$path" ` +
713
+ `|| fail "mode-$label"; }; ` +
714
+ `ensure_real_dir "$ASDF_DATA_DIR" asdf-data; ` +
715
+ `ensure_real_dir "$installs_root" installs-root; ` +
716
+ `ensure_real_dir "$node_root" node-root; ` +
717
+ `ensure_real_dir "$shims_root" shims-root; ` +
718
+ `ensure_real_dir "$corepack_root" corepack-root; ` +
719
+ // Legacy images could seed individual shims as root even though the
720
+ // directory itself was later repaired. asdf 0.14 then prints permission
721
+ // errors while reshim quietly leaves the old executable behind; task
722
+ // materialization can appear successful but the next CLI launch is stale
723
+ // or non-executable. The trusted root maintenance transaction repairs only
724
+ // the flat, regular-file shim set before dropping to node for every asdf
725
+ // mutation. Anything else is ambiguous volume content and fails closed.
726
+ `repair_shim_metadata() { ` +
727
+ `unsafe_shim_entry="$(/usr/bin/find "$shims_root" -xdev -mindepth 1 ` +
728
+ `-maxdepth 1 ! -type f -print -quit)"; ` +
729
+ `[ -z "$unsafe_shim_entry" ] || fail unsafe-shim-entry; ` +
730
+ `/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
731
+ `-exec /usr/bin/chown node:node -- {} + ` +
732
+ `|| fail repair-shim-owner; ` +
733
+ `/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
734
+ `-exec /usr/bin/chmod 0755 -- {} + ` +
735
+ `|| fail repair-shim-mode; }; ` +
736
+ `repair_shim_metadata; ` +
737
+ `candidate_is_safe() { ` +
738
+ `lexical_candidate="$(/usr/bin/realpath -m -s -- "$candidate")" ` +
739
+ `|| return 1; [ "$lexical_candidate" = "$node_root/$default_node" ] ` +
740
+ `|| return 1; if [ -e "$candidate" ] || [ -L "$candidate" ]; then ` +
741
+ `[ ! -L "$candidate" ] && [ -d "$candidate" ] || return 1; ` +
742
+ `resolved_candidate="$(/usr/bin/realpath -e -- "$candidate")" ` +
743
+ `|| return 1; ` +
744
+ `[ "$resolved_candidate" = "$lexical_candidate" ] || return 1; fi; }; ` +
745
+ `contained_executable() { root="$1"; executable="$2"; ` +
746
+ `[ -f "$executable" ] && [ -x "$executable" ] || return 1; ` +
747
+ `real_executable="$(/usr/bin/realpath -e -- "$executable")" ` +
748
+ `|| return 1; ` +
749
+ `case "$real_executable" in "$root"/*) return 0 ;; *) return 1 ;; esac; }; ` +
750
+ `contained_real_dir() { root="$1"; path="$2"; ` +
751
+ `[ ! -L "$path" ] && [ -d "$path" ] || return 1; ` +
752
+ `lexical_path="$(/usr/bin/realpath -m -s -- "$path")" || return 1; ` +
753
+ `real_path="$(/usr/bin/realpath -e -- "$path")" || return 1; ` +
754
+ `case "$lexical_path" in "$root"/*) ;; *) return 1 ;; esac; ` +
755
+ `[ "$real_path" = "$lexical_path" ]; }; ` +
756
+ `npm_prefix_is_safe() { contained_real_dir "$candidate" "$candidate/bin" ` +
757
+ `&& contained_real_dir "$candidate" "$candidate/lib" ` +
758
+ `&& contained_real_dir "$candidate" "$candidate/lib/node_modules"; }; ` +
759
+ `scope_is_safe_or_absent() { scope="$1"; ` +
760
+ `if [ -e "$scope" ] || [ -L "$scope" ]; then ` +
761
+ `contained_real_dir "$candidate" "$scope"; fi; }; ` +
762
+ `as_node() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
763
+ `/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
764
+ `ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
765
+ `ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
766
+ `ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
767
+ `COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
768
+ `COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
769
+ `COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
770
+ `COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
771
+ `COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
772
+ `NPM_CONFIG_PREFIX="$candidate" ` +
773
+ `PATH="$candidate/bin:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
774
+ `"$@"; }; ` +
775
+ `as_node_control() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
776
+ `/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
777
+ `ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
778
+ `ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
779
+ `ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
780
+ `COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
781
+ `COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
782
+ `COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
783
+ `COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
784
+ `COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
785
+ `PATH="/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
786
+ `"$@"; }; ` +
787
+ `clear_forbidden_shared_bash_shim() { fail_if_found="$1"; ` +
788
+ `bash_shim="$shims_root/bash"; bash_shim_found=0; ` +
789
+ `if [ -e "$bash_shim" ] || [ -L "$bash_shim" ]; then ` +
790
+ `bash_shim_found=1; ` +
791
+ `[ ! -d "$bash_shim" ] || fail unsafe-runtime-bash-shim-directory; ` +
792
+ `as_node /usr/bin/rm -f -- "$bash_shim" ` +
793
+ `|| fail remove-runtime-bash-shim; fi; ` +
794
+ `[ ! -e "$bash_shim" ] && [ ! -L "$bash_shim" ] ` +
795
+ `|| fail unsafe-runtime-bash-shim; ` +
796
+ `if [ "$fail_if_found" -ne 0 ] && [ "$bash_shim_found" -ne 0 ]; then ` +
797
+ `return 1; fi; }; ` +
798
+ `clear_forbidden_candidate_bash() { fail_if_found="$1"; ` +
799
+ `candidate_bash="$candidate/bin/bash"; candidate_bash_found=''; ` +
800
+ `if [ -e "$candidate_bash" ] || [ -L "$candidate_bash" ]; then ` +
801
+ `candidate_bash_found=1; [ ! -d "$candidate_bash" ] ` +
802
+ `|| fail unsafe-default-bash-directory; ` +
803
+ `as_node /usr/bin/rm -f -- "$candidate_bash" ` +
804
+ `|| fail remove-default-bash; fi; ` +
805
+ `[ ! -e "$candidate_bash" ] && [ ! -L "$candidate_bash" ] ` +
806
+ `|| fail unsafe-default-bash; ` +
807
+ `if [ "$fail_if_found" -ne 0 ] && [ -n "$candidate_bash_found" ]; then ` +
808
+ `return 1; fi; }; ` +
809
+ `run_asdf_install() { asdf_status=0; shim_status=0; ` +
810
+ `as_node_control /opt/asdf/bin/asdf install "$@" || asdf_status=$?; ` +
811
+ `clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
812
+ `[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
813
+ `run_asdf_reshim() { asdf_status=0; shim_status=0; ` +
814
+ `as_node_control /opt/asdf/bin/asdf reshim "$@" || asdf_status=$?; ` +
815
+ `clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
816
+ `[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
817
+ `reject_runtime_bash_shims() { ` +
818
+ `bash_cleanup_status=0; clear_forbidden_candidate_bash 1 ` +
819
+ `|| bash_cleanup_status=1; ` +
820
+ `clear_forbidden_shared_bash_shim 1 || bash_cleanup_status=1; ` +
821
+ `[ "$bash_cleanup_status" -eq 0 ] || fail unsafe-runtime-bash-shim; }; ` +
822
+ `node_write_probe() { writable="$1"; ` +
823
+ `probe="$(as_node /usr/bin/mktemp -d "$writable/.uai-write-probe.XXXXXX")" ` +
824
+ `|| return 1; case "$probe" in "$writable"/.uai-write-probe.*) ;; ` +
825
+ `*) return 1 ;; esac; as_node /usr/bin/rmdir "$probe"; }; ` +
826
+ `candidate_owned_by_node() { ` +
827
+ `[ -z "$(/usr/bin/find "$candidate" -xdev ! -user node -print -quit)" ] ` +
828
+ `&& [ -z "$(/usr/bin/find "$candidate" -xdev -type d ! -perm -u+w -print -quit)" ]; }; ` +
829
+ `prove_default_node() { candidate_is_safe && [ -d "$candidate" ] ` +
830
+ `&& [ ! -e "$candidate/bin/bash" ] && [ ! -L "$candidate/bin/bash" ] ` +
831
+ `&& contained_executable "$candidate" "$candidate/bin/node" ` +
832
+ `&& [ "$(as_node "$candidate/bin/node" --version 2>/dev/null)" = "v$default_node" ] ` +
833
+ `&& contained_executable "$candidate" "$candidate/bin/npm" ` +
834
+ `&& [ -n "$(as_node "$candidate/bin/npm" --version 2>/dev/null)" ] ` +
835
+ `&& npm_prefix_is_safe && candidate_owned_by_node ` +
836
+ `&& node_write_probe "$candidate/bin"; }; ` +
837
+ `clear_forbidden_shared_bash_shim 0; ` +
838
+ `if ! prove_default_node; then candidate_is_safe || fail unsafe-default-candidate; ` +
839
+ `/usr/bin/rm -rf --one-file-system -- "$candidate" ` +
840
+ `|| fail remove-default-candidate; ` +
841
+ `run_asdf_install nodejs "$default_node" ` +
842
+ `|| fail install-default-node; prove_default_node || fail repaired-default-node; fi; ` +
843
+ `npm_bin="$candidate/bin/npm"; cd / || fail chdir; ` +
844
+ `if [ "$upgrade" = 1 ]; then ` +
845
+ `npm_prefix_is_safe || fail unsafe-npm-prefix; ` +
846
+ `if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
847
+ `>/tmp/uai-cli-upgrade.out 2>&1; then ` +
848
+ `if /usr/bin/grep -qi 'ENOTEMPTY' /tmp/uai-cli-upgrade.out; then ` +
849
+ `scope_is_safe_or_absent "$candidate/lib/node_modules/@anthropic-ai" ` +
850
+ `&& scope_is_safe_or_absent "$candidate/lib/node_modules/@openai" ` +
851
+ `|| fail unsafe-cli-scope; ` +
852
+ `as_node /usr/bin/rm -rf --one-file-system -- ` +
853
+ `"$candidate/lib/node_modules/@anthropic-ai" ` +
854
+ `"$candidate/lib/node_modules/@openai"; ` +
855
+ `if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
856
+ `>/tmp/uai-cli-upgrade-retry.out 2>&1; then ` +
857
+ `echo 'UAI_CLI_UPDATE_DEFERRED:bulk-retry' >&2; fi; ` +
858
+ `else echo 'UAI_CLI_UPDATE_DEFERRED:bulk' >&2; fi; fi; fi; ` +
859
+ // Reshim first so an installed package behind a stale shim can recover
860
+ // without network access. Only bins that still fail are reinstalled.
861
+ `if ! run_asdf_reshim nodejs "$default_node" ` +
862
+ `>/dev/null 2>&1; then echo 'UAI_CLI_RESHIM_DEFERRED:initial' >&2; fi; ` +
863
+ `cli_ready_once() { b="$1"; ` +
864
+ `contained_executable "$candidate" "$candidate/bin/$b" || return 1; ` +
865
+ `direct_version="$(as_node "$candidate/bin/$b" --version 2>/dev/null ` +
866
+ `| /usr/bin/head -1)"; [ -n "$direct_version" ] || return 1; ` +
867
+ `if [ "$b" = codex ] && ` +
868
+ `[ "$direct_version" != "codex-cli $required_codex" ]; then return 1; fi; ` +
869
+ `shim="$shims_root/$b"; [ ! -L "$shim" ] && [ -f "$shim" ] ` +
870
+ `&& [ -x "$shim" ] || return 1; ` +
871
+ `shim_version="$(as_node "$shim" --version 2>/dev/null ` +
872
+ `| /usr/bin/head -1)"; [ "$shim_version" = "$direct_version" ]; }; ` +
873
+ `repair_cli_once() { b="$1"; case "$b" in ` +
874
+ `claude) pkg='@anthropic-ai/claude-code'; scope='@anthropic-ai'; ` +
875
+ `package_name='claude-code'; version='latest';; ` +
876
+ `codex) pkg='@openai/codex'; scope='@openai'; package_name='codex'; ` +
877
+ `version="$required_codex";; ` +
878
+ `*) return 1;; esac; ` +
879
+ `package_path="$candidate/lib/node_modules/$scope/$package_name"; ` +
880
+ `npm_prefix_is_safe || return 1; ` +
881
+ `scope_is_safe_or_absent "$candidate/lib/node_modules/$scope" ` +
882
+ `|| return 1; ` +
883
+ `as_node /usr/bin/rm -rf --one-file-system -- ` +
884
+ `"$candidate/bin/$b" "$package_path" || return 1; ` +
885
+ `as_node "$npm_bin" install -g --force "$pkg@$version" ${trustedNpmFlags} ` +
886
+ `>/dev/null 2>&1 || return 1; ` +
887
+ `run_asdf_reshim nodejs "$default_node" ` +
888
+ `>/dev/null 2>&1 || return 1; cli_ready_once "$b"; }; ` +
889
+ `for b in claude codex; do if ! cli_ready_once "$b"; then ` +
890
+ `repair_cli_once "$b" || fail "$b-repair"; fi; done; ` +
891
+ // Node <=22's bundled Corepack has obsolete registry signing keys. The
892
+ // host-packaged pin is compatible with every advertised Node default.
893
+ `required_corepack="$(/usr/bin/sed -n '1p' /usr/local/share/uai/corepack-version)"; ` +
894
+ `printf '%s' "$required_corepack" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$' || { ` +
895
+ `echo 'UAI_STILL_BROKEN:corepack-pin' >&2; exit 1; }; ` +
896
+ `corepack_ready_once() { ` +
897
+ `contained_executable "$candidate" "$candidate/bin/corepack" || return 1; ` +
898
+ `[ "$(as_node "$candidate/bin/corepack" --version 2>/dev/null)" = "$required_corepack" ] ` +
899
+ `|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
900
+ `|| return 1; run_asdf_reshim nodejs "$default_node" ` +
901
+ `>/dev/null 2>&1 || return 1; ` +
902
+ `corepack_shim="$shims_root/corepack"; [ ! -L "$corepack_shim" ] ` +
903
+ `&& [ -f "$corepack_shim" ] && [ -x "$corepack_shim" ] || return 1; ` +
904
+ `[ "$(as_node "$corepack_shim" --version 2>/dev/null)" = "$required_corepack" ]; }; ` +
905
+ `repair_corepack_once() { ` +
906
+ `npm_prefix_is_safe || return 1; ` +
907
+ `as_node /usr/bin/rm -rf --one-file-system -- ` +
908
+ `"$candidate/lib/node_modules/corepack" "$candidate/bin/corepack" ` +
909
+ `"$candidate/bin/pnpm" "$candidate/bin/pnpx" "$candidate/bin/yarn" ` +
910
+ `"$candidate/bin/yarnpkg" || return 1; ` +
911
+ `as_node "$npm_bin" install -g --force "corepack@$required_corepack" ` +
912
+ `--no-audit --no-fund ${trustedNpmFlags} >/dev/null 2>&1 || return 1; ` +
913
+ `run_asdf_reshim nodejs "$default_node" >/dev/null 2>&1 ` +
914
+ `|| return 1; contained_executable "$candidate" "$candidate/bin/corepack" ` +
915
+ `|| return 1; as_node "$candidate/bin/corepack" disable >/dev/null 2>&1 ` +
916
+ `|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
917
+ `|| return 1; run_asdf_reshim nodejs "$default_node" ` +
918
+ `>/dev/null 2>&1 || return 1; corepack_ready_once; }; ` +
919
+ `if ! corepack_ready_once; then repair_corepack_once ` +
920
+ `|| fail corepack-repair; fi; reject_runtime_bash_shims; failed=0; ` +
921
+ `prove_default_node || { echo "UAI_STILL_BROKEN:default-node-final" >&2; failed=1; }; ` +
922
+ `for b in claude codex; do ` +
923
+ `if ! contained_executable "$candidate" "$candidate/bin/$b"; then ` +
924
+ `echo "UAI_STILL_BROKEN:$b-contained" >&2; failed=1; continue; fi; ` +
925
+ `version="$(as_node "$candidate/bin/$b" --version 2>/dev/null | /usr/bin/head -1)"; ` +
926
+ `shim="$shims_root/$b"; if [ -L "$shim" ] || [ ! -f "$shim" ] ` +
927
+ `|| [ ! -x "$shim" ]; then echo "UAI_STILL_BROKEN:$b-shim" >&2; ` +
928
+ `failed=1; continue; fi; ` +
929
+ `shim_version="$(as_node "$shim" --version 2>/dev/null | /usr/bin/head -1)"; ` +
930
+ `if [ "$b" = codex ] && ` +
931
+ `[ "$version" != "codex-cli $required_codex" ]; then ` +
932
+ `echo "UAI_STILL_BROKEN:$b-version" >&2; failed=1; continue; fi; ` +
933
+ `if [ -z "$version" ] || [ "$shim_version" != "$version" ]; then ` +
934
+ `echo "UAI_STILL_BROKEN:$b" >&2; failed=1; ` +
935
+ `else printf '%s ' "$version"; fi; done; ` +
936
+ `node_shim="$shims_root/node"; ` +
937
+ `[ ! -L "$node_shim" ] && [ -f "$node_shim" ] && [ -x "$node_shim" ] ` +
938
+ `|| { echo "UAI_STILL_BROKEN:node-shim-file" >&2; failed=1; }; ` +
939
+ `shim_node="$(as_node "$node_shim" --version 2>/dev/null || true)"; ` +
940
+ `[ "$shim_node" = "v$default_node" ] || { ` +
941
+ `echo "UAI_STILL_BROKEN:node-shim" >&2; failed=1; }; ` +
942
+ `npm_shim="$shims_root/npm"; ` +
943
+ `[ ! -L "$npm_shim" ] && [ -f "$npm_shim" ] && [ -x "$npm_shim" ] ` +
944
+ `|| { echo "UAI_STILL_BROKEN:npm-shim-file" >&2; failed=1; }; ` +
945
+ `shim_npm="$(as_node "$npm_shim" --version 2>/dev/null || true)"; ` +
946
+ `[ -n "$shim_npm" ] || { echo "UAI_STILL_BROKEN:npm-shim" >&2; failed=1; }; ` +
947
+ `if ! corepack_ready_once; then echo "UAI_STILL_BROKEN:corepack-final" >&2; ` +
948
+ `failed=1; else printf 'corepack %s' "$required_corepack"; fi; ` +
949
+ `node_write_probe "$candidate/bin" || { ` +
950
+ `echo "UAI_STILL_BROKEN:default-node-owner" >&2; failed=1; }; ` +
951
+ `node_write_probe "$shims_root" || { ` +
952
+ `echo "UAI_STILL_BROKEN:shims-owner" >&2; failed=1; }; ` +
953
+ `node_write_probe "$corepack_root" || { ` +
954
+ `echo "UAI_STILL_BROKEN:corepack-owner" >&2; failed=1; }; ` +
955
+ `exit "$failed"`
956
+ );
957
+ }
958
+
959
+ async function reconcileVolumeAgentClis(activation: ActivationIsCurrent | null): Promise<boolean> {
960
+ const missingBefore = await probeVolumeAgentClis(activation);
961
+ if (missingBefore?.length) {
962
+ console.log(
963
+ `[host-agent] repairing ${ASDF_DATA_VOLUME}: agent CLI(s) ` +
964
+ `[${missingBefore.join(", ")}] failed their execution probe`,
965
+ );
966
+ }
967
+
968
+ const autoUpdate = process.env.UAI_AGENT_CLI_AUTOUPDATE !== "0";
969
+ if (!autoUpdate) {
970
+ console.log(
971
+ "[host-agent] agent CLI auto-update disabled (UAI_AGENT_CLI_AUTOUPDATE=0)",
972
+ );
973
+ }
974
+ const command = standardImageVolumeMaintenanceCommandForTest(autoUpdate);
975
+ const reconcile = await runMaintenanceContainer(selectedEngine(), "reconcile", [
976
+ "--rm",
977
+ "-u",
978
+ "root",
979
+ "-w",
980
+ "/home/node",
981
+ "-v",
982
+ `${ASDF_DATA_VOLUME}:/opt/asdf-data`,
983
+ "-v",
984
+ `${standardImageCorepackVersionPath()}:/usr/local/share/uai/corepack-version:ro`,
985
+ STANDARD_IMAGE_TAG,
986
+ "/usr/bin/timeout",
987
+ "--signal=TERM",
988
+ "--kill-after=10s",
989
+ `${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
990
+ "/usr/bin/flock",
991
+ "--exclusive",
992
+ "--timeout",
993
+ `${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS}`,
994
+ ASDF_MAINTENANCE_LOCK_PATH,
995
+ "/bin/bash",
996
+ "-c",
997
+ command,
998
+ ], CLI_MAINTENANCE_TIMEOUT_MS, activation);
999
+ if (reconcile.code === 0) {
1000
+ console.log(
1001
+ `[host-agent] agent CLIs and Corepack current on ${ASDF_DATA_VOLUME}: ` +
1002
+ reconcile.stdout.trim(),
1003
+ );
1004
+ await writeVolumeReconcileProof();
1005
+ return true;
1006
+ } else {
1007
+ lastMaintenanceStderr = reconcile.stderr;
1008
+ console.warn(
1009
+ `[host-agent] failed to reconcile ${ASDF_DATA_VOLUME} (exit ` +
1010
+ `${reconcile.code ?? "spawn"}); task runtimes are not ready. ` +
1011
+ reconcile.stderr.trim().slice(0, 500),
1012
+ );
1013
+ return false;
1014
+ }
1015
+ }
1016
+
1017
+ /** Durable evidence that the shared volume completed a full reconcile at
1018
+ * least once on this host. Apple volumes are block devices with EXCLUSIVE
1019
+ * writable attachments, so boot maintenance cannot re-prove the volume while
1020
+ * running task containers hold it read-only — and blocking activation on
1021
+ * that (the 2026-08-16 ring) froze hosts and blocked the very teardowns that
1022
+ * would free the volume. With this proof on disk, a busy volume defers
1023
+ * maintenance instead of failing it. */
1024
+ let lastMaintenanceStderr = "";
1025
+
1026
+ async function volumeReconcileProofPath(): Promise<string> {
1027
+ // Lazy env resolve: module import must stay side-effect free (see
1028
+ // defaultRuntimeProviderStateStore). Dynamic import, not require — a bare
1029
+ // require cannot resolve the TypeScript module outside the tsx runtime,
1030
+ // which silently read as "no proof" and disabled deferral.
1031
+ const { env } = await import("./env");
1032
+ return resolve(env.dataDir, "runtime-volume-proof.json");
1033
+ }
1034
+
1035
+ async function writeVolumeReconcileProof(): Promise<void> {
1036
+ try {
1037
+ writeFileSync(
1038
+ await volumeReconcileProofPath(),
1039
+ `${JSON.stringify({
1040
+ schemaVersion: 1,
1041
+ codexVersion: STANDARD_CODEX_CLI_VERSION,
1042
+ provedAt: Date.now(),
1043
+ })}\n`,
1044
+ { mode: 0o600 },
1045
+ );
1046
+ } catch (error) {
1047
+ console.warn(
1048
+ `[host-agent] could not persist the shared-volume reconcile proof: ${
1049
+ error instanceof Error ? error.message : String(error)
1050
+ }`,
1051
+ );
1052
+ }
1053
+ }
1054
+
1055
+ async function volumeReconcileProofExists(): Promise<boolean> {
1056
+ try {
1057
+ const raw = JSON.parse(
1058
+ readFileSync(await volumeReconcileProofPath(), "utf8"),
1059
+ ) as { schemaVersion?: unknown; codexVersion?: unknown };
1060
+ return (
1061
+ raw?.schemaVersion === 1 &&
1062
+ raw.codexVersion === STANDARD_CODEX_CLI_VERSION
1063
+ );
1064
+ } catch {
1065
+ return false;
1066
+ }
1067
+ }
1068
+
1069
+ const VOLUME_ATTACH_BUSY_SIGNATURE = "storage device attachment is invalid";
1070
+
1071
+ /**
1072
+ * Whether task containers hold the shared volume — tri-state per container,
1073
+ * never fail-open: `running` holds, exact `stopped` is released (a stopped
1074
+ * VM releases its block attachments — proven live 2026-08-17, when a full
1075
+ * writable reconcile succeeded next to a stopped task container), and EVERY
1076
+ * other observation (starting, stopping, creating, a missing or foreign
1077
+ * state string, an unreadable inventory) is `unknown` — a task acquiring or
1078
+ * releasing the volume mid-transition must never read as permission for a
1079
+ * writer (review 2026-08-18 round 2). `held` dominates `unknown` dominates
1080
+ * `free`; writers require a positive `free`.
1081
+ */
1082
+ async function sharedVolumeHolders(engine: {
1083
+ command: string;
1084
+ apple: boolean;
1085
+ }): Promise<"held" | "free" | "unknown"> {
1086
+ const listed = await run(
1087
+ engine.command,
1088
+ ["list", "--all", "--format", "json"],
1089
+ 30_000,
1090
+ );
1091
+ if (listed.code !== 0) return "unknown";
1092
+ try {
1093
+ const parsed = JSON.parse(listed.stdout) as Array<{
1094
+ configuration?: {
1095
+ id?: unknown;
1096
+ mounts?: Array<{ type?: { volume?: { name?: unknown } } }>;
1097
+ };
1098
+ status?: { state?: unknown };
1099
+ }>;
1100
+ if (!Array.isArray(parsed)) return "unknown";
1101
+ let verdict: "held" | "free" | "unknown" = "free";
1102
+ for (const item of parsed) {
1103
+ // A holder is a container whose CONFIGURED MOUNTS include the shared
1104
+ // volume — physical attachment, not a name shape (review 2026-08-18
1105
+ // round 3: an unrelated task-*-app container with no attachment
1106
+ // fenced activation forever, and the holder-remedy sweep — which
1107
+ // correctly matches by mount — could not clear it). A row whose
1108
+ // mounts cannot be read might hold the volume, so it is `unknown`.
1109
+ const mounts = item?.configuration?.mounts;
1110
+ if (!Array.isArray(mounts)) {
1111
+ verdict = "unknown";
1112
+ continue;
1113
+ }
1114
+ const holds = mounts.some(
1115
+ (mount) => mount?.type?.volume?.name === ASDF_DATA_VOLUME,
1116
+ );
1117
+ if (!holds) continue;
1118
+ const state = item?.status?.state;
1119
+ if (state === "running") return "held";
1120
+ if (state !== "stopped") verdict = "unknown";
1121
+ }
1122
+ return verdict;
1123
+ } catch {
1124
+ return "unknown";
1125
+ }
1126
+ }
1127
+
1128
+ /** Backstop for the race the proof-first check cannot close: a task container
1129
+ * appearing between the check and the attach. Admits deferral only on the
1130
+ * exact busy signature, with the same proof requirement. */
1131
+ async function volumeBusyDeferral(
1132
+ engine: { command: string; apple: boolean },
1133
+ stderr: string,
1134
+ ): Promise<boolean> {
1135
+ if (!engine.apple) return false;
1136
+ if (!stderr.includes(VOLUME_ATTACH_BUSY_SIGNATURE)) return false;
1137
+ if (!(await volumeReconcileProofExists())) return false;
1138
+ return (await sharedVolumeHolders(engine)) === "held";
1139
+ }
1140
+
1141
+ /** Outcome of ensureStandardImage — `error` carries WHY when not ready, so
1142
+ * the taskUp path can surface a build failure to the cloud (not just the host
1143
+ * log). Callers that only fire-and-forget at boot ignore the return. */
1144
+ export interface StandardImageResult {
1145
+ ok: boolean;
1146
+ error?: string;
1147
+ /** A non-writer escape the activation driver can perform before retrying:
1148
+ * lifecycle-stop the running task containers holding the shared volume
1149
+ * (resumable), because the volume needs a repair they block and cannot
1150
+ * safely use. */
1151
+ remedy?: "stop-volume-holders";
1152
+ }
1153
+
1154
+ type AgentClisReadyListener = () => void;
1155
+
1156
+ let agentClisReadyLatched = false;
1157
+ let signalClisReady: (() => void) | null = null;
1158
+ const agentClisReadyListeners = new Set<AgentClisReadyListener>();
1159
+ const ensureQueue = new KeyedPromiseTail<StandardImageResult>();
1160
+ /**
1161
+ * Resolves once ensureStandardImage has reconciled the shared-volume agent CLIs
1162
+ * at least once (or bailed because docker is down / the build failed). The agent
1163
+ * factory awaits this before spawning a CLI, so a boot-time resume never races
1164
+ * the CLI auto-upgrade's brief remove-then-reinstall window — the "No codex
1165
+ * executable found for nodejs X" (exit 2) resume failure. Post-boot it is
1166
+ * already resolved, so the await is a no-op.
1167
+ */
1168
+ export const agentClisReady: Promise<void> = new Promise<void>((resolve) => {
1169
+ signalClisReady = resolve;
1170
+ });
1171
+
1172
+ /** Whether this process has completed its one boot-time CLI maintenance pass.
1173
+ * This is intentionally a one-way boot latch, not a claim that no later
1174
+ * operator-triggered image work is in flight. */
1175
+ export function areAgentClisReady(): boolean {
1176
+ return agentClisReadyLatched;
1177
+ }
1178
+
1179
+ /** Subscribe to the one readiness edge. A late subscriber is notified in a
1180
+ * microtask so registration never re-enters its caller. */
1181
+ export function onAgentClisReady(
1182
+ listener: AgentClisReadyListener,
1183
+ ): () => void {
1184
+ if (agentClisReadyLatched) {
1185
+ let cancelled = false;
1186
+ queueMicrotask(() => {
1187
+ if (!cancelled) notifyAgentClisReadyListener(listener);
1188
+ });
1189
+ return () => {
1190
+ cancelled = true;
1191
+ };
1192
+ }
1193
+ agentClisReadyListeners.add(listener);
1194
+ return () => agentClisReadyListeners.delete(listener);
1195
+ }
1196
+
1197
+ function notifyAgentClisReadyListener(listener: AgentClisReadyListener): void {
1198
+ try {
1199
+ listener();
1200
+ } catch (error) {
1201
+ console.warn(
1202
+ `[host-agent] agent CLI readiness listener failed: ${error instanceof Error ? error.message : String(error)}`,
1203
+ );
1204
+ }
1205
+ }
1206
+
1207
+ /** ADR-133 sidecar mode: the agent CLIs are baked into the task machine's
1208
+ * image, so there is no standard-image pass to wait on — boot declares them
1209
+ * ready. Never called on a host that owns a container runtime. */
1210
+ export function declareAgentClisReady(): void {
1211
+ markAgentClisReady();
1212
+ }
1213
+
1214
+ function markAgentClisReady(): void {
1215
+ if (agentClisReadyLatched) return;
1216
+ // Publish the synchronous capability state before resolving the promise:
1217
+ // every waiter and listener must observe `true` after the readiness edge.
1218
+ agentClisReadyLatched = true;
1219
+ signalClisReady?.();
1220
+ signalClisReady = null;
1221
+ const listeners = [...agentClisReadyListeners];
1222
+ agentClisReadyListeners.clear();
1223
+ for (const listener of listeners) {
1224
+ notifyAgentClisReadyListener(listener);
1225
+ }
1226
+ }
1227
+
1228
+ /**
1229
+ * Ensure the standard image and the shared asdf data volume exist, and that the
1230
+ * agent CLIs are present inside the volume. Builds the image only when `docker
1231
+ * image inspect` fails or the content hash is stale. Never throws — returns
1232
+ * `{ ok, error? }` so the caller decides (boot: fire-and-forget; taskUp:
1233
+ * surface the reason). Resolves `agentClisReady` on every exit path.
1234
+ */
1235
+ export function ensureStandardImage(
1236
+ opts: {
1237
+ maxAgeMs?: number;
1238
+ /** Only the runtime's activation passes this, answering whether it is
1239
+ * still the current activation: within it alone, and only while it is
1240
+ * current, may a wedged Apple runtime's services be restarted, since
1241
+ * its recovery follows its maintenance. Every other pass asks the
1242
+ * runtime's owner for that activation instead (#p2jdax). */
1243
+ withinActivation?: ActivationIsCurrent;
1244
+ } = {},
1245
+ ): Promise<StandardImageResult> {
1246
+ // ADR-133 sidecar mode: a task machine has no container image to keep —
1247
+ // its agent CLIs are baked into the machine image. Live 2026-09-10
1248
+ // (hostless smoke 4): task-up refused with "could not prepare the host
1249
+ // base image moondeck-standard:dev … Docker is not available".
1250
+ if (sidecarMode()) {
1251
+ markAgentClisReady();
1252
+ return Promise.resolve({ ok: true });
1253
+ }
1254
+ // A task-up passes maxAgeMs: a pass that succeeded recently for the same
1255
+ // engine configuration is reused instead of spending ~15 s spinning a
1256
+ // maintenance container per task (live 2026-09-09: 16 s of every task
1257
+ // start). The hourly schedule (main.ts) and the boot pass keep it fresh.
1258
+ const engines = configuredOptionalEngines();
1259
+ const key = optionalEngineConfigurationKey(engines);
1260
+ const recent = lastOkByKey.get(key);
1261
+ if (
1262
+ opts.maxAgeMs !== undefined &&
1263
+ recent !== undefined &&
1264
+ Date.now() - recent.at <= opts.maxAgeMs
1265
+ ) {
1266
+ markAgentClisReady();
1267
+ return Promise.resolve(recent.result);
1268
+ }
1269
+ // Startup, runtime recovery, and a newly-arrived task can all discover the
1270
+ // same daemon transition. One shared volume repair/build must own that
1271
+ // window; concurrent maintenance containers can otherwise replace the same
1272
+ // CLI shims underneath each other and defeat agentClisReady's fence. The
1273
+ // activation's pass never joins one begun outside it (which may not
1274
+ // restart the runtime): it runs after it, as its own.
1275
+ const activation = opts.withinActivation ?? null;
1276
+ return ensureQueue.run(activation ? `activation:${key}` : key, () =>
1277
+ ensureStandardImageInner(engines, activation)
1278
+ .then((result) => {
1279
+ if (result.ok) lastOkByKey.set(key, { at: Date.now(), result });
1280
+ else lastOkByKey.delete(key);
1281
+ return result;
1282
+ })
1283
+ .finally(() => {
1284
+ // Always signal — even a docker-down bail — so agent spawns never hang
1285
+ // waiting on a reconcile that will not happen this boot.
1286
+ markAgentClisReady();
1287
+ }),
1288
+ );
1289
+ }
1290
+
1291
+ /** Last successful pass per engine configuration (in-process). */
1292
+ const lastOkByKey = new Map<string, { at: number; result: StandardImageResult }>();
1293
+
1294
+ /** The runtime's activation, asked whether it is still the current one: what
1295
+ * alone may restart a wedged Apple runtime's services (#p2jdax). */
1296
+ export type ActivationIsCurrent = () => boolean;
1297
+
1298
+ /** Tests: forget every remembered pass. */
1299
+ export function resetStandardImageMemo(): void {
1300
+ lastOkByKey.clear();
1301
+ buildBackoff.clear();
1302
+ }
1303
+
1304
+ const buildBackoff = new StandardImageBuildBackoff();
1305
+ const imageCleanup = new StandardImageCleanup(async () => {
1306
+ const { env } = await import("./env");
1307
+ return resolve(env.dataDir, "standard-image-seen.json");
1308
+ });
1309
+
1310
+ /** The owned-host schedule (docs/agent-notes.md, "image maintenance"). */
1311
+ export const MAINTENANCE_INTERVAL_MS = 60 * 60_000;
1312
+
1313
+ async function ensureStandardImageInner(
1314
+ engines: OptionalEngineConfiguration,
1315
+ activation: ActivationIsCurrent | null,
1316
+ ): Promise<StandardImageResult> {
1317
+ // 1. Shared asdf data volume — idempotent under Docker; the Apple CLI
1318
+ // errors on an existing name, which is the same healthy outcome.
1319
+ const engine = selectedEngine();
1320
+ console.log(
1321
+ `[host-agent] standard-image maintenance starting (engine: ${engine.apple ? "apple-container" : "docker"})`,
1322
+ );
1323
+ if (engine.apple) {
1324
+ // The volume create below is the FIRST apple mutation on a host — before
1325
+ // any task exists — and uninstall reads an absent provider-claim marker
1326
+ // as "nothing to tear down". Claim durably before mutating, or a
1327
+ // zero-task install→activate→uninstall skips Apple cleanup (review
1328
+ // 2026-08-18 round 4).
1329
+ try {
1330
+ await claimContainerRuntimeForMaintenance();
1331
+ } catch (error) {
1332
+ return {
1333
+ ok: false,
1334
+ error:
1335
+ "Could not durably record the Apple container runtime claim " +
1336
+ `before provisioning: ${error instanceof Error ? error.message : String(error)}`,
1337
+ };
1338
+ }
1339
+ }
1340
+ const vol = await run(engine.command, ["volume", "create", ASDF_DATA_VOLUME]);
1341
+ if (
1342
+ vol.code !== 0 &&
1343
+ !(engine.apple && vol.stderr.includes("already exists"))
1344
+ ) {
1345
+ console.warn(
1346
+ `[host-agent] could not create asdf data volume (${ASDF_DATA_VOLUME}); ` +
1347
+ `continuing. ${vol.stderr.trim()}`,
1348
+ );
1349
+ // If docker itself is unavailable, the image step will also fail; bail
1350
+ // early so we don't double-log a confusing build error.
1351
+ if (vol.code === null) {
1352
+ return { ok: false, error: "Docker is not available on this host." };
1353
+ }
1354
+ }
1355
+
1356
+ // 2. Ensure the image is built AND current. "Present" is not enough — a
1357
+ // Dockerfile change used to no-op forever because we only built when
1358
+ // inspect failed (found live 2026-07-08: ADR-053's baked packages never
1359
+ // landed). The build context is content-hashed into an image label;
1360
+ // a mismatch triggers a rebuild (layer cache keeps it cheap).
1361
+ let imageReady = false;
1362
+ let buildError: string | null = null;
1363
+ // Install only the optional engines the operator has configured; the flags
1364
+ // are build args AND part of the content hash (so a new login rebuilds).
1365
+ const engineArgs = [
1366
+ "--build-arg",
1367
+ `INSTALL_KIMI=${engines.kimi ? 1 : 0}`,
1368
+ "--build-arg",
1369
+ `INSTALL_GROK=${engines.grok ? 1 : 0}`,
1370
+ "--build-arg",
1371
+ `INSTALL_CURSOR=${engines.cursor ? 1 : 0}`,
1372
+ "--build-arg",
1373
+ `INSTALL_OPENCODE=${engines.opencode ? 1 : 0}`,
1374
+ ];
1375
+ const contextHash = await hashBuildContext(
1376
+ `kimi=${engines.kimi ? 1 : 0};grok=${engines.grok ? 1 : 0};cursor=${engines.cursor ? 1 : 0};opencode=${engines.opencode ? 1 : 0}`,
1377
+ );
1378
+ let inspect: RunResult;
1379
+ let labeledHash: string | null = null;
1380
+ if (engine.apple) {
1381
+ // The Apple CLI has no --format; labels live in the single variant's OCI
1382
+ // config. A malformed answer counts as "absent" and triggers a rebuild.
1383
+ inspect = await run(engine.command, [
1384
+ "image",
1385
+ "inspect",
1386
+ STANDARD_IMAGE_TAG,
1387
+ ]);
1388
+ if (inspect.code === 0) {
1389
+ try {
1390
+ const parsed = JSON.parse(inspect.stdout) as Array<{
1391
+ variants?: Array<{
1392
+ config?: { config?: { Labels?: Record<string, string> } };
1393
+ }>;
1394
+ }>;
1395
+ labeledHash =
1396
+ parsed[0]?.variants?.[0]?.config?.config?.Labels?.[
1397
+ CONTEXT_HASH_LABEL
1398
+ ] ?? "";
1399
+ } catch {
1400
+ inspect = { ...inspect, code: 1 };
1401
+ }
1402
+ }
1403
+ } else {
1404
+ inspect = await run("docker", [
1405
+ "image",
1406
+ "inspect",
1407
+ "-f",
1408
+ `{{index .Config.Labels "${CONTEXT_HASH_LABEL}"}}`,
1409
+ STANDARD_IMAGE_TAG,
1410
+ ]);
1411
+ labeledHash = inspect.code === 0 ? inspect.stdout.trim() : null;
1412
+ }
1413
+ if (inspect.code === null) {
1414
+ console.warn(
1415
+ "[host-agent] container runtime unavailable; skipping standard image " +
1416
+ "build. Tasks will fail until it is running.",
1417
+ );
1418
+ return {
1419
+ ok: false,
1420
+ error: "The container runtime is not available on this host.",
1421
+ };
1422
+ }
1423
+ if (inspect.code !== 0) labeledHash = null;
1424
+
1425
+ // ADR-114: when the payload pins a prebuilt digest, the pin decides image
1426
+ // identity — the label must equal the PIN's context hash (the CI superset
1427
+ // hash, not this host's per-config hash, which would disagree by design
1428
+ // and rebuild forever). Any pull-path failure falls through to the local
1429
+ // build below, so offline and registry-down hosts behave exactly as today.
1430
+ const previousImageId = engine.apple ? null : await imageCleanup.observe(run);
1431
+ const pin = await readStandardImagePin();
1432
+ if (pin !== null) {
1433
+ if (inspect.code === 0 && labeledHash === pin.contextHash) {
1434
+ console.log(
1435
+ `[host-agent] standard image ${STANDARD_IMAGE_TAG} current (pinned)`,
1436
+ );
1437
+ imageReady = true;
1438
+ } else {
1439
+ const pullDigest = standardImagePullDigest(pin);
1440
+ console.log(
1441
+ `[host-agent] pulling standard image ${pin.ref}@${pullDigest}`,
1442
+ );
1443
+ const commands = standardImagePullCommands(
1444
+ pin,
1445
+ STANDARD_IMAGE_TAG,
1446
+ engine.apple,
1447
+ );
1448
+ const pulled = await run(
1449
+ engine.command,
1450
+ commands.pull,
1451
+ IMAGE_PULL_TIMEOUT_MS,
1452
+ );
1453
+ if (pulled.code === 0) {
1454
+ const tagged = await run(engine.command, commands.tag);
1455
+ if (tagged.code === 0) {
1456
+ console.log(
1457
+ `[host-agent] pulled standard image ${STANDARD_IMAGE_TAG} (${pullDigest.slice(0, 19)}…)`,
1458
+ );
1459
+ imageReady = true;
1460
+ } else {
1461
+ console.warn(
1462
+ "[host-agent] pulled standard image could not be tagged; " +
1463
+ `falling back to a local build. ${tagged.stderr.trim()}`,
1464
+ );
1465
+ }
1466
+ } else {
1467
+ console.warn(
1468
+ "[host-agent] standard image pull failed; falling back to a " +
1469
+ `local build. ${pulled.stderr.trim()}`,
1470
+ );
1471
+ }
1472
+ }
1473
+ }
1474
+
1475
+ if (imageReady) {
1476
+ // Pinned image adopted — skip the local-hash decision entirely.
1477
+ } else if (inspect.code === 0 && contextHash !== null && labeledHash === contextHash) {
1478
+ console.log(`[host-agent] standard image ${STANDARD_IMAGE_TAG} current`);
1479
+ imageReady = true;
1480
+ } else if (inspect.code === 0 && contextHash === null) {
1481
+ // Can't hash (packaged install without the context?) — keep the image.
1482
+ console.log(`[host-agent] standard image ${STANDARD_IMAGE_TAG} present`);
1483
+ imageReady = true;
1484
+ } else {
1485
+ const context = standardImageDir();
1486
+ const buildKey = `${contextHash ?? "unavailable"}:${JSON.stringify(engines)}:${pin?.contextHash ?? "local"}`;
1487
+ const deferred = engine.apple || labeledHash === null ? null : buildBackoff.remaining(buildKey);
1488
+ if (!deferred) {
1489
+ console.log(
1490
+ inspect.code === 0
1491
+ ? `[host-agent] standard image ${STANDARD_IMAGE_TAG} stale (context changed) — rebuilding from ${context}`
1492
+ : `[host-agent] building standard image ${STANDARD_IMAGE_TAG} from ${context}`,
1493
+ );
1494
+ }
1495
+ const build = deferred ? { code: 1, stdout: "", stderr: `${deferred.error}\nStandard image build deferred until ${new Date(deferred.retryAt).toISOString()}.` } : await run(engine.command, [
1496
+ "build",
1497
+ ...(engine.apple ? [] : ["--force-rm"]),
1498
+ "-t",
1499
+ STANDARD_IMAGE_TAG,
1500
+ ...engineArgs,
1501
+ ...(contextHash !== null
1502
+ ? ["--label", `${CONTEXT_HASH_LABEL}=${contextHash}`]
1503
+ : []),
1504
+ context,
1505
+ ], IMAGE_BUILD_TIMEOUT_MS);
1506
+ if (build.code === 0) {
1507
+ console.log(`[host-agent] built standard image ${STANDARD_IMAGE_TAG}`);
1508
+ imageReady = true;
1509
+ buildBackoff.clear();
1510
+ } else {
1511
+ buildError =
1512
+ build.stderr.trim() ||
1513
+ `docker build exited ${build.code ?? "spawn error"}`;
1514
+ if (!deferred && !engine.apple) buildBackoff.failed(buildKey, buildError);
1515
+ console.warn(
1516
+ deferred ? `[host-agent] ${buildError}` :
1517
+ `[host-agent] standard image build failed (exit ${build.code ?? "spawn"}); ` +
1518
+ "continuing. Tasks needing the image will surface this error.\n" +
1519
+ buildError,
1520
+ );
1521
+ // A stale-but-working image is better than none.
1522
+ imageReady = labeledHash !== null;
1523
+ }
1524
+ }
1525
+
1526
+ // Revisit ids durably observed under the standard tag across restarts.
1527
+ // Inherited labels never authorize sweeping task/compose images.
1528
+ if (imageReady && !engine.apple) await imageCleanup.sweep(run, previousImageId);
1529
+
1530
+ // 3. Reconcile every Uai-owned writer under the volume-global flock. Lock
1531
+ // bootstrap/permission repair precedes the single upgrade→repair→proof
1532
+ // transaction so node-run task materializers can acquire legacy volumes.
1533
+ if (imageReady) {
1534
+ // Docker populates a fresh named volume from the image's directory
1535
+ // content on first mount; the Apple runtime mounts it empty (ADR-106,
1536
+ // found live 2026-08-15: every asdf plugin/shim the image bakes into
1537
+ // /opt/asdf-data was missing, so CLI reconcile looped on "No such
1538
+ // plugin: nodejs"). Replicate the Docker semantics explicitly: when the
1539
+ // volume lacks the plugins tree, copy the image's /opt/asdf-data into it
1540
+ // once, ownership preserved. Idempotent — the copied tree is the marker.
1541
+ if (engine.apple) {
1542
+ // Probe at a path the image does NOT contain: mounted over
1543
+ // /opt/asdf-data, a silently-failed volume mount would expose the
1544
+ // image's own baked tree and fake a "seeded" answer. At a neutral
1545
+ // mountpoint the plugins tree can only come from the volume itself,
1546
+ // and any mount failure lands on the safe side (reseeding is
1547
+ // idempotent).
1548
+ const seeded = await runMaintenanceContainer(engine, "seed-probe", [
1549
+ "--rm",
1550
+ "-v",
1551
+ `${ASDF_DATA_VOLUME}:/uai-volume-probe:ro`,
1552
+ STANDARD_IMAGE_TAG,
1553
+ "/usr/bin/test",
1554
+ "-d",
1555
+ "/uai-volume-probe/plugins",
1556
+ ], CLI_MAINTENANCE_TIMEOUT_MS, activation);
1557
+ // Only `/usr/bin/test`'s own exit 1 means "not seeded". Everything else
1558
+ // — a host-side deadline kill (null), an in-container SIGTERM/SIGKILL
1559
+ // (143/137: live 2026-08-17, a wedged apiserver let the probe run ten
1560
+ // minutes before dying 143), or any CLI failure — answers neither way,
1561
+ // and must never trigger a reseed of a volume that may be full and in
1562
+ // use.
1563
+ if (seeded.code !== 0 && seeded.code !== 1) {
1564
+ return {
1565
+ ok: false,
1566
+ error:
1567
+ "The container runtime did not answer while checking the shared " +
1568
+ `runtime volume (probe exit ${seeded.code ?? "killed"}).`,
1569
+ };
1570
+ }
1571
+ if (seeded.code === 0) {
1572
+ console.log(
1573
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is seeded`,
1574
+ );
1575
+ }
1576
+ // The positive no-holder proof comes BEFORE every writable attachment
1577
+ // — including the seeder. A doomed writable attach both fails the
1578
+ // operation and can mint the corpse that wedges the apiserver, so a
1579
+ // writer requires a positive `free`; `unknown` (unreadable inventory)
1580
+ // must never read as permission.
1581
+ const holders = await sharedVolumeHolders(engine);
1582
+ if (seeded.code !== 0) {
1583
+ if (holders !== "free") {
1584
+ return {
1585
+ ok: false,
1586
+ error:
1587
+ holders === "held"
1588
+ ? `The shared runtime volume ${ASDF_DATA_VOLUME} is unseeded while task containers are running; it cannot be seeded safely until they stop.`
1589
+ : "Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
1590
+ };
1591
+ }
1592
+ console.log(
1593
+ `[host-agent] seeding ${ASDF_DATA_VOLUME} from the standard image (probe exit ${seeded.code}; Apple volumes mount empty)`,
1594
+ );
1595
+ const seed = await runMaintenanceContainer(engine, "seed", [
1596
+ "--rm",
1597
+ "--user",
1598
+ "root",
1599
+ "-v",
1600
+ `${ASDF_DATA_VOLUME}:/uai-seed-target`,
1601
+ STANDARD_IMAGE_TAG,
1602
+ "/bin/sh",
1603
+ "-c",
1604
+ "cp -a /opt/asdf-data/. /uai-seed-target/ && chown -R node:node /uai-seed-target",
1605
+ ], CLI_MAINTENANCE_TIMEOUT_MS, activation);
1606
+ if (seed.code !== 0) {
1607
+ return {
1608
+ ok: false,
1609
+ error: `Could not seed the shared runtime volume from the standard image (exit ${seed.code ?? "spawn"}).`,
1610
+ };
1611
+ }
1612
+ } else if (holders === "held") {
1613
+ // Proof-first deferral: while running task containers hold the
1614
+ // volume, a writable attach can only fail (and mint a corpse) —
1615
+ // don't attempt it. The stderr-signature deferrals below remain as
1616
+ // the backstop for a task appearing between this check and the
1617
+ // attach.
1618
+ if (await volumeReconcileProofExists()) {
1619
+ console.log(
1620
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by task containers; deferring maintenance (prior reconcile proof present)`,
1621
+ );
1622
+ return { ok: true };
1623
+ }
1624
+ // First-proof bootstrap. A host with no persisted proof (fresh file,
1625
+ // or upgraded from a version whose proof write failed silently —
1626
+ // live 2026-08-17 on 0.9.26) would otherwise be stuck in a ring: the
1627
+ // proof needs a full reconcile, the reconcile needs the writable
1628
+ // volume, and the volume stays held for as long as tasks run. The
1629
+ // read-only CLI probe attaches fine next to tasks, and EXECUTING
1630
+ // every agent CLI is the same positive evidence the full reconcile
1631
+ // ends with — enough to defer on, and worth persisting as the proof.
1632
+ const missing = await probeVolumeAgentClis(activation);
1633
+ if (missing !== null && missing.length === 0) {
1634
+ await writeVolumeReconcileProof();
1635
+ console.log(
1636
+ `[host-agent] read-only agent-CLI probe passed while task containers hold ${ASDF_DATA_VOLUME}; recording the reconcile proof and deferring maintenance`,
1637
+ );
1638
+ return { ok: true };
1639
+ }
1640
+ // No proof and a failed probe, while tasks hold the volume: a
1641
+ // writable attach here can only fail and mint a corpse, and simply
1642
+ // retrying strands activation forever (review 2026-08-18 round 2 —
1643
+ // recovery, which could settle the holders, runs only after
1644
+ // maintenance succeeds). Ask the activation driver to lifecycle-stop
1645
+ // the holder tasks (resumable) so the next attempt can repair the
1646
+ // volume they cannot safely use anyway.
1647
+ return {
1648
+ ok: false,
1649
+ error:
1650
+ `The shared runtime volume ${ASDF_DATA_VOLUME} needs repair ` +
1651
+ `(agent CLIs failed their probe${missing === null ? "" : `: ${missing.join(", ")} missing`}) ` +
1652
+ "while running tasks hold it; stopping the holder tasks so maintenance can repair it.",
1653
+ remedy: "stop-volume-holders",
1654
+ };
1655
+ } else if (holders === "unknown") {
1656
+ return {
1657
+ ok: false,
1658
+ error:
1659
+ "Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
1660
+ };
1661
+ }
1662
+ }
1663
+ const lockPrepared = await prepareAsdfMaintenanceLock(activation);
1664
+ if (!lockPrepared) {
1665
+ if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
1666
+ console.log(
1667
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
1668
+ );
1669
+ return { ok: true };
1670
+ }
1671
+ return {
1672
+ ok: false,
1673
+ error: `${buildError ? `${buildError} ` : ""}Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
1674
+ };
1675
+ }
1676
+ if (!(await reconcileVolumeAgentClis(activation))) {
1677
+ if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
1678
+ console.log(
1679
+ `[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
1680
+ );
1681
+ return { ok: true };
1682
+ }
1683
+ return {
1684
+ ok: false,
1685
+ error:
1686
+ `${buildError ? `${buildError} ` : ""}Shared runtime maintenance failed; agent CLIs or Corepack did not pass execution proof.`,
1687
+ };
1688
+ }
1689
+ return { ok: true };
1690
+ }
1691
+ return {
1692
+ ok: false,
1693
+ error:
1694
+ buildError ??
1695
+ "the standard base image (moondeck-standard:dev) is not built and no build error was captured.",
1696
+ };
1697
+ }