@moondeck/station 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +149 -2
- package/bin/uai-host.mjs +130 -0
- package/db/migrations/0000_host_tasks.sql +12 -0
- package/db/migrations/0001_host_ui.sql +11 -0
- package/db/migrations/0002_host_github_tokens.sql +8 -0
- package/db/migrations/0003_host_ssh_keys.sql +8 -0
- package/db/migrations/0004_host_owner_name.sql +1 -0
- package/db/migrations/0005_host_env_vars.sql +5 -0
- package/db/migrations/0006_host_project_env.sql +9 -0
- package/db/migrations/0007_host_github_token_kind.sql +7 -0
- package/db/migrations/0008_host_mcp_connections.sql +18 -0
- package/db/migrations/0009_host_agent_sessions.sql +15 -0
- package/db/migrations/0010_host_engine_accounts.sql +13 -0
- package/db/migrations/0011_codex_thread_id.sql +5 -0
- package/db/migrations/0012_agent_session_attach_key.sql +4 -0
- package/db/migrations/0013_github_credential_generations.sql +11 -0
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/0017_engine_account_scope.sql +7 -0
- package/db/migrations/0018_engine_account_position.sql +3 -0
- package/db/migrations/0019_engine_account_store_ref.sql +3 -0
- package/db/migrations/0020_user_credential_store_ref.sql +6 -0
- package/db/migrations/0021_project_env_store_ref.sql +3 -0
- package/db/migrations/0022_task_project_ids.sql +5 -0
- package/db/migrations/0023_project_env_org.sql +4 -0
- package/db/migrations/0024_task_env_mode.sql +9 -0
- package/db/migrations/0025_project_env_authority.sql +147 -0
- package/db/migrations/0026_project_env_replica.sql +150 -0
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/0029_task_shared_files_mode.sql +5 -0
- package/db/migrations/0030_credential_cleanup_restops.sql +12 -0
- package/db/migrations/meta/_journal.json +223 -0
- package/db/schema.ts +850 -0
- package/images/standard/Dockerfile +347 -0
- package/images/standard/README.md +167 -0
- package/images/standard/container/code-server-settings.json +63 -0
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +704 -0
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/images/standard/container/uai-task-forwarder.mjs +303 -0
- package/images/standard/tool-versions +2 -0
- package/lib/__fixtures__/skill-index/SKILL.md +10 -0
- package/lib/__fixtures__/skill-index/skills/accessibility-checklist.md +7 -0
- package/lib/__fixtures__/skill-index/skills/brand-voice.md +9 -0
- package/lib/__fixtures__/skill-index/skills/design-tokens.md +9 -0
- package/lib/agent-cli.ts +1507 -0
- package/lib/agent-errors.ts +22 -0
- package/lib/agent.ts +672 -0
- package/lib/agents/attach-compatibility.ts +51 -0
- package/lib/agents/claude.ts +918 -0
- package/lib/agents/codex-token-usage.fixture.json +130 -0
- package/lib/agents/codex.ts +1392 -0
- package/lib/agents/cursor-period-usage.ts +253 -0
- package/lib/agents/cursor.ts +562 -0
- package/lib/agents/dispatch.ts +78 -0
- package/lib/agents/durable-proc.ts +334 -0
- package/lib/agents/factory.ts +57 -0
- package/lib/agents/grok.ts +381 -0
- package/lib/agents/kimi.ts +329 -0
- package/lib/agents/machine-durable.ts +604 -0
- package/lib/agents/mock.ts +133 -0
- package/lib/agents/mode.ts +64 -0
- package/lib/agents/opencode.ts +508 -0
- package/lib/agents/output-source.ts +3 -0
- package/lib/agents/proc.ts +221 -0
- package/lib/agents/rate-limit.ts +337 -0
- package/lib/agents/registry.ts +182 -0
- package/lib/agents/session-state.ts +197 -0
- package/lib/agents/skill-index-preamble.fixture.ts +26 -0
- package/lib/agents/transport.ts +741 -0
- package/lib/agents/types.ts +254 -0
- package/lib/agents/uai.ts +106 -0
- package/lib/agents/usage.ts +44 -0
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +237 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/assignment-evidence.ts +94 -0
- package/lib/attachments.ts +48 -0
- package/lib/browser-testing.ts +1537 -0
- package/lib/capture-flow.ts +332 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2587 -0
- package/lib/cloud-credential-runtime.ts +1357 -0
- package/lib/cloud-state.ts +56 -0
- package/lib/codex-auth.ts +456 -0
- package/lib/command-db.ts +310 -0
- package/lib/compose-network-recovery.ts +283 -0
- package/lib/connection-secret-redaction.ts +25 -0
- package/lib/container-runtime.ts +1453 -0
- package/lib/credential-cleanup-inventory.ts +490 -0
- package/lib/credential-consumers/engine-anchor.ts +1880 -0
- package/lib/credential-consumers/engines.ts +797 -0
- package/lib/credential-consumers/files.ts +934 -0
- package/lib/credential-consumers/github.ts +702 -0
- package/lib/credential-consumers/mcp.ts +389 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-consumers/task-secrets.ts +10 -0
- package/lib/credential-store-pickup.ts +149 -0
- package/lib/credential-store-scope.ts +112 -0
- package/lib/credential-store.ts +932 -0
- package/lib/credentials-activation-authority.ts +709 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/custom-connection-capture.ts +395 -0
- package/lib/custom-connection-launch.ts +384 -0
- package/lib/custom-connection-runtime.ts +141 -0
- package/lib/db.ts +80 -0
- package/lib/disk-pressure.ts +42 -0
- package/lib/docker-exec.ts +156 -0
- package/lib/engine-accounts.ts +1970 -0
- package/lib/engine-connect.ts +321 -0
- package/lib/engine-credential-env.ts +52 -0
- package/lib/engine-login.ts +2933 -0
- package/lib/engines.ts +1342 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +246 -0
- package/lib/env-schema.ts +891 -0
- package/lib/env.ts +195 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +530 -0
- package/lib/git-identity.ts +715 -0
- package/lib/github-git-auth.ts +207 -0
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +2439 -0
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-env.ts +424 -0
- package/lib/host-logs.ts +965 -0
- package/lib/internal-connections.ts +5 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/launch-mode.ts +25 -0
- package/lib/lease.ts +239 -0
- package/lib/legacy-host.ts +769 -0
- package/lib/legacy-runtime-retirement.ts +424 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/machine-exec.ts +238 -0
- package/lib/machine-gateway-tunnel.ts +344 -0
- package/lib/machine-keys.ts +65 -0
- package/lib/machine-project-env-loader.ts +152 -0
- package/lib/machine-provider-aws.ts +923 -0
- package/lib/machine-provider-local.ts +338 -0
- package/lib/machine-provider.ts +90 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +57 -0
- package/lib/managed-runtime.ts +4296 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-capture.ts +865 -0
- package/lib/mcp-config-lock.ts +2 -0
- package/lib/mcp-connections.ts +959 -0
- package/lib/mcp-gateway.ts +2594 -0
- package/lib/obs.ts +512 -0
- package/lib/openrouter-budget.ts +247 -0
- package/lib/orchestrator.ts +11409 -0
- package/lib/person-capture.ts +327 -0
- package/lib/preview-ports.ts +85 -0
- package/lib/preview-sidecar.ts +331 -0
- package/lib/project-env-authority.ts +1319 -0
- package/lib/project-env-feature.ts +74 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-loader.ts +242 -0
- package/lib/project-env-replica.ts +2094 -0
- package/lib/project-env-schema-capture.ts +342 -0
- package/lib/project-env-staging.ts +148 -0
- package/lib/project-env-task-policy.ts +176 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/release-feed.ts +61 -0
- package/lib/release-manifest.ts +710 -0
- package/lib/release-trust.ts +46 -0
- package/lib/repo-clone.ts +38 -0
- package/lib/routing/shadow.ts +261 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +279 -0
- package/lib/runtime-archive.ts +1216 -0
- package/lib/runtime-authority.ts +166 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +1033 -0
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/address.ts +95 -0
- package/lib/secrets-proxy/audit.ts +30 -0
- package/lib/secrets-proxy/ca-store.ts +200 -0
- package/lib/secrets-proxy/ca.ts +470 -0
- package/lib/secrets-proxy/container-steps.ts +138 -0
- package/lib/secrets-proxy/der.ts +273 -0
- package/lib/secrets-proxy/engine.ts +1633 -0
- package/lib/secrets-proxy/host.ts +93 -0
- package/lib/secrets-proxy/hostname.ts +57 -0
- package/lib/secrets-proxy/placeholders.ts +196 -0
- package/lib/secrets-proxy/policy.ts +150 -0
- package/lib/secrets-proxy/readiness.ts +148 -0
- package/lib/secrets-proxy/runtime.ts +582 -0
- package/lib/secrets-proxy/scrub.ts +559 -0
- package/lib/secrets-proxy/sidecar-view.ts +139 -0
- package/lib/secrets-proxy/task-scripts.ts +262 -0
- package/lib/secrets-proxy/task-trust.ts +605 -0
- package/lib/secrets-proxy/trust-watch.ts +80 -0
- package/lib/secrets.ts +71 -0
- package/lib/shared-files-s3.ts +976 -0
- package/lib/shared-files.ts +156 -0
- package/lib/sidecar-assign.ts +49 -0
- package/lib/sidecar-guard.ts +58 -0
- package/lib/sidecar-resume.ts +178 -0
- package/lib/sidecar.ts +69 -0
- package/lib/signin-box-credentials.ts +465 -0
- package/lib/signin-box.ts +114 -0
- package/lib/skill-index-example.fixture.ts +56 -0
- package/lib/skill-layout.ts +417 -0
- package/lib/skills.ts +489 -0
- package/lib/ssh.ts +401 -0
- package/lib/standard-image-cleanup.ts +117 -0
- package/lib/standard-image-pin.ts +207 -0
- package/lib/standard-image.ts +1697 -0
- package/lib/stopped-task-status-queue.ts +120 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-credential-cleanup.ts +752 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +91 -0
- package/lib/task-env-forwarder.ts +228 -0
- package/lib/task-env-source.ts +249 -0
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +860 -0
- package/lib/task-environment/docker.ts +1344 -0
- package/lib/task-environment/index.ts +466 -0
- package/lib/task-environment/legacy-adoption.ts +467 -0
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +1227 -0
- package/lib/task-environment/machine.ts +974 -0
- package/lib/task-environment/registry.ts +59 -0
- package/lib/task-environment/self-identity.ts +20 -0
- package/lib/task-environment/self.ts +566 -0
- package/lib/task-environment/types.ts +642 -0
- package/lib/task-environment/workspace-files.ts +685 -0
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-files-machine-retirement-outbox.ts +592 -0
- package/lib/task-files-machine-retirement.ts +116 -0
- package/lib/task-files-readiness.ts +127 -0
- package/lib/task-gateway-port.ts +22 -0
- package/lib/task-gateway-protocol.ts +190 -0
- package/lib/task-gateway-recovery.ts +47 -0
- package/lib/task-gateway-state.ts +512 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +682 -0
- package/lib/task-machine-credentials-activation.ts +95 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/task-person-credentials.ts +76 -0
- package/lib/task-project-ops-host.ts +147 -0
- package/lib/task-project-ops.ts +1062 -0
- package/lib/task-secret-staging.ts +16 -0
- package/lib/task-status.ts +46 -0
- package/lib/task-token.ts +46 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +105 -0
- package/lib/token-broker-local.ts +392 -0
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +525 -0
- package/lib/token-broker.ts +218 -0
- package/lib/token-lanes/openai-oauth.ts +123 -0
- package/lib/token-lanes/registry.ts +45 -0
- package/lib/token-lanes/types.ts +163 -0
- package/lib/token-lanes/xai-oauth.ts +165 -0
- package/lib/transcript.ts +128 -0
- package/lib/tunnel-registry.ts +220 -0
- package/lib/tunnel-runtime.ts +235 -0
- package/lib/ulid.ts +16 -0
- package/lib/user-bus-env.ts +116 -0
- package/lib/user-credentials-store.ts +435 -0
- package/lib/vnode-pressure.ts +142 -0
- package/package.json +131 -4
- package/runner/runner.mjs +442 -0
- package/scripts/agent/_common.sh +633 -0
- package/scripts/agent/env-hook.sh +626 -0
- package/scripts/agent/moondeck-cli.sh +5 -0
- package/scripts/agent/task-down.sh +291 -0
- package/scripts/agent/task-status.sh +70 -0
- package/scripts/agent/task-up.sh +3761 -0
- package/scripts/agent/uai-env-exec.sh +26 -0
- package/scripts/install/darwin.ts +1048 -0
- package/scripts/install/linux.ts +958 -0
- package/scripts/install/types.ts +78 -0
- package/scripts/install/util.ts +254 -0
- package/scripts/install/win.ts +143 -0
- package/src/activation-respawn.ts +155 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/capture-wire.ts +102 -0
- package/src/cli.ts +2583 -0
- package/src/cloud-credentials.ts +2144 -0
- package/src/cloud-origins.ts +23 -0
- package/src/credential-location.ts +248 -0
- package/src/credential-wire-validation.ts +25 -0
- package/src/custom-connections.ts +237 -0
- package/src/engine-account-paths.ts +11 -0
- package/src/engine-models.ts +39 -0
- package/src/engine-usage.ts +104 -0
- package/src/event-outbox.ts +185 -0
- package/src/index.ts +2899 -0
- package/src/load-env.ts +97 -0
- package/src/machine-tunnel-route.ts +156 -0
- package/src/main.ts +6349 -0
- package/src/never-proxied-domains.ts +31 -0
- package/src/outbound-frame.ts +31 -0
- package/src/paths.ts +81 -0
- package/src/project-env-latest.ts +130 -0
- package/src/project-env-policy.ts +396 -0
- package/src/project-env-wording.ts +53 -0
- package/src/protocol.ts +5545 -0
- package/src/runtime-bootstrap.ts +165 -0
- package/src/station-identity.ts +96 -0
- package/src/task-env-names.ts +28 -0
- package/src/task-secrets.ts +75 -0
- package/src/tunnel-dial.ts +102 -0
- package/src/ui/server.ts +852 -0
- package/src/ui/types.ts +173 -0
- package/src/workspace-path-policy.ts +41 -0
- package/ui/app.js +1102 -0
- package/ui/index.html +90 -0
- package/ui/style.css +685 -0
- package/ui/uai-favicon.svg +21 -0
- package/ui/uai-logo-black.svg +9 -0
- package/ui/uai-wheel.svg +9 -0
|
@@ -0,0 +1,1697 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Standard-image build pipeline (ADR-022 / docs/runtime.md).
|
|
3
|
+
*
|
|
4
|
+
* Every host builds exactly one standard image (`moondeck-standard:dev`) and
|
|
5
|
+
* caches it. `ensureStandardImage()` runs at startup and on the owned-host
|
|
6
|
+
* hourly maintenance schedule (local build failures back off):
|
|
7
|
+
*
|
|
8
|
+
* 1. `docker volume create moondeck-asdf-data` — the host-wide asdf data volume
|
|
9
|
+
* that caches lazily-installed runtime versions across tasks (idempotent;
|
|
10
|
+
* `docker volume create` is a no-op if the volume already exists).
|
|
11
|
+
* 2. `docker image inspect moondeck-standard:dev` — if it resolves, the image is
|
|
12
|
+
* present and we skip the build. Otherwise `docker build` it from
|
|
13
|
+
* `host-agent/images/standard/`.
|
|
14
|
+
*
|
|
15
|
+
* Best-effort: any failure (docker missing, daemon down, build error) is
|
|
16
|
+
* logged and swallowed so the host process still boots and serves tunnels.
|
|
17
|
+
* task-up surfaces the real error to the user if a build is genuinely needed.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { spawn } from "node:child_process";
|
|
21
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
22
|
+
import { readFile } from "node:fs/promises";
|
|
23
|
+
import { dirname, join, resolve } from "node:path";
|
|
24
|
+
import { fileURLToPath } from "node:url";
|
|
25
|
+
|
|
26
|
+
import {
|
|
27
|
+
appleOneOffNeedsWatch,
|
|
28
|
+
appleOneOffRunArgs,
|
|
29
|
+
askRuntimeOwnerOnce,
|
|
30
|
+
newAppleOneOff,
|
|
31
|
+
settleAppleOneOff,
|
|
32
|
+
watchAppleOneOff,
|
|
33
|
+
} from "./apple-one-off";
|
|
34
|
+
import { recycleAppleRuntimeServices, type AppleRecycleOutcome } from "./apple-runtime-recycle";
|
|
35
|
+
import { detectEngine } from "./engines";
|
|
36
|
+
import { hasStoredEngineAccounts } from "./engine-accounts";
|
|
37
|
+
import {
|
|
38
|
+
appleContainerRuntimeBinding,
|
|
39
|
+
claimContainerRuntimeForMaintenance,
|
|
40
|
+
containerRuntimeStandardAssetsRoot,
|
|
41
|
+
containerRuntimeStandardImageRoot,
|
|
42
|
+
pinnedContainerRuntimeProvider,
|
|
43
|
+
requestContainerRuntimeRecheck,
|
|
44
|
+
} from "./container-runtime";
|
|
45
|
+
import { KeyedPromiseTail } from "./keyed-promise-tail";
|
|
46
|
+
import {
|
|
47
|
+
CONTEXT_HASH_LABEL,
|
|
48
|
+
hashStandardImageContext,
|
|
49
|
+
parseStandardImagePin,
|
|
50
|
+
STANDARD_IMAGE_PIN_BASENAME,
|
|
51
|
+
type StandardImagePin,
|
|
52
|
+
standardImagePullCommands,
|
|
53
|
+
standardImagePullDigest,
|
|
54
|
+
} from "./standard-image-pin";
|
|
55
|
+
import { sidecarMode } from "./sidecar";
|
|
56
|
+
import { StandardImageCleanup, StandardImageBuildBackoff } from "./standard-image-cleanup";
|
|
57
|
+
|
|
58
|
+
/** Pinned, host-wide constants (must match task-up.sh and the compose gen). */
|
|
59
|
+
export const STANDARD_IMAGE_TAG = "moondeck-standard:dev";
|
|
60
|
+
export const ASDF_DATA_VOLUME = "moondeck-asdf-data";
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The shared-volume agent CLIs belong to this exact HOME-default Node. A
|
|
64
|
+
* future default bump cannot transparently roll stopped containers: their
|
|
65
|
+
* baked /home/node/.tool-versions still names the old CLI-owning candidate.
|
|
66
|
+
* Changing this value therefore requires an explicit volume migration or
|
|
67
|
+
* container-recreation release slice, not an ordinary image refresh.
|
|
68
|
+
*/
|
|
69
|
+
export const STANDARD_DEFAULT_NODE_VERSION = "22.11.0";
|
|
70
|
+
|
|
71
|
+
/** Exact Codex CLI release exposed to task runtimes through the shared volume. */
|
|
72
|
+
export const STANDARD_CODEX_CLI_VERSION = "0.164.0";
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Runtimes the standard image's asdf plugins can satisfy, with a sensible set
|
|
76
|
+
* of advertised versions per kind (ADR-021 `runtimes`). These are the
|
|
77
|
+
* versions the cloud's task-creation picker hints; asdf will lazily install
|
|
78
|
+
* any of them on first use (docs/runtime.md). The first entry of each list is
|
|
79
|
+
* the root `/etc/.tool-versions` default baked into the image.
|
|
80
|
+
*
|
|
81
|
+
* KEEP IN SYNC with host-agent/images/standard/ (the asdf plugins and the
|
|
82
|
+
* root tool-versions). Treat that directory as authoritative for the numbers;
|
|
83
|
+
* this list is the advertisement surface.
|
|
84
|
+
*/
|
|
85
|
+
export const STANDARD_RUNTIMES: ReadonlyArray<{
|
|
86
|
+
kind: string;
|
|
87
|
+
availableVersions: string[];
|
|
88
|
+
}> = [
|
|
89
|
+
{
|
|
90
|
+
kind: "node",
|
|
91
|
+
availableVersions: [
|
|
92
|
+
STANDARD_DEFAULT_NODE_VERSION,
|
|
93
|
+
"20.18.0",
|
|
94
|
+
"18.20.4",
|
|
95
|
+
],
|
|
96
|
+
},
|
|
97
|
+
{ kind: "python", availableVersions: ["3.12.4", "3.11.9"] },
|
|
98
|
+
{ kind: "go", availableVersions: ["1.23.2", "1.22.8"] },
|
|
99
|
+
{ kind: "ruby", availableVersions: ["3.3.5", "3.2.5"] },
|
|
100
|
+
{ kind: "rust", availableVersions: ["1.81.0"] },
|
|
101
|
+
];
|
|
102
|
+
|
|
103
|
+
/** A fresh, mutable copy of the advertised runtimes for the capability frame. */
|
|
104
|
+
export function standardRuntimes(): Array<{
|
|
105
|
+
kind: string;
|
|
106
|
+
availableVersions: string[];
|
|
107
|
+
}> {
|
|
108
|
+
return STANDARD_RUNTIMES.map((r) => ({
|
|
109
|
+
kind: r.kind,
|
|
110
|
+
availableVersions: [...r.availableVersions],
|
|
111
|
+
}));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Which OPTIONAL agent CLIs the operator has actually configured — so the
|
|
116
|
+
* image installs only those, not every engine on every host. Delegates to
|
|
117
|
+
* engines.ts detection so every connect flavor counts: kimi's copied cred
|
|
118
|
+
* file, grok's login file OR a pasted XAI_API_KEY, cursor's CURSOR_API_KEY
|
|
119
|
+
* (claude/codex are always baked). Folded into the build hash below, so
|
|
120
|
+
* connecting a new engine triggers a rebuild.
|
|
121
|
+
*/
|
|
122
|
+
export function configuredOptionalEngines(): {
|
|
123
|
+
kimi: boolean;
|
|
124
|
+
grok: boolean;
|
|
125
|
+
cursor: boolean;
|
|
126
|
+
opencode: boolean;
|
|
127
|
+
} {
|
|
128
|
+
// A stored account (a workspace's, on a shared host; a labeled extra on an
|
|
129
|
+
// owned one) needs the CLI in the image just like the host's own sign-in
|
|
130
|
+
// does — without this a grok/cursor account existed but its tasks exited
|
|
131
|
+
// 127 for a missing binary.
|
|
132
|
+
const on = (kind: "kimi" | "grok" | "cursor" | "opencode"): boolean =>
|
|
133
|
+
detectEngine(kind) || hasStoredEngineAccounts(kind);
|
|
134
|
+
return {
|
|
135
|
+
kimi: on("kimi"),
|
|
136
|
+
grok: on("grok"),
|
|
137
|
+
cursor: on("cursor"),
|
|
138
|
+
opencode: on("opencode"),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
type OptionalEngineConfiguration = ReturnType<
|
|
143
|
+
typeof configuredOptionalEngines
|
|
144
|
+
>;
|
|
145
|
+
|
|
146
|
+
function optionalEngineConfigurationKey(
|
|
147
|
+
engines: OptionalEngineConfiguration,
|
|
148
|
+
): string {
|
|
149
|
+
return JSON.stringify(engines);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Build args (which optional engines are installed) are part of the image
|
|
153
|
+
// identity — a config change must invalidate the label so it rebuilds. The
|
|
154
|
+
// walk-and-hash itself lives in standard-image-pin.ts so CI computes the
|
|
155
|
+
// identical identity (ADR-114).
|
|
156
|
+
function hashBuildContext(extra = ""): Promise<string | null> {
|
|
157
|
+
return hashStandardImageContext(standardImageDir(), extra);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The packaged digest pin (ADR-114) at `agent/images/standard-image.pin.json`,
|
|
161
|
+
* one level above the build context so it never perturbs the context hash.
|
|
162
|
+
* Absent in dev checkouts and npm installs — those keep the local build.
|
|
163
|
+
* `UAI_IMAGE_PULL=0` is the operator kill switch back to local semantics. */
|
|
164
|
+
async function readStandardImagePin(): Promise<StandardImagePin | null> {
|
|
165
|
+
if (process.env.UAI_IMAGE_PULL === "0") return null;
|
|
166
|
+
let raw: string;
|
|
167
|
+
try {
|
|
168
|
+
raw = await readFile(
|
|
169
|
+
resolve(standardImageDir(), "..", STANDARD_IMAGE_PIN_BASENAME),
|
|
170
|
+
"utf8",
|
|
171
|
+
);
|
|
172
|
+
} catch {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
const pin = parseStandardImagePin(raw);
|
|
176
|
+
if (pin === null) {
|
|
177
|
+
console.warn(
|
|
178
|
+
"[host-agent] packaged standard-image pin is malformed; using the local build path",
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
return pin;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Absolute path to the standard image build context. */
|
|
185
|
+
function standardImageDir(): string {
|
|
186
|
+
const providerRoot = containerRuntimeStandardImageRoot();
|
|
187
|
+
if (providerRoot !== null) return providerRoot;
|
|
188
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
189
|
+
// lib/standard-image.ts -> host-agent/ -> images/standard
|
|
190
|
+
return resolve(here, "..", "images", "standard");
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Absolute packaged path used to bind the materializer over rolling images
|
|
194
|
+
* that predate the baked executable. Resolution is deterministic; callers get
|
|
195
|
+
* the ordinary Docker bind/open error if a package install is corrupt. */
|
|
196
|
+
export function standardImageRuntimeMaterializerPath(): string {
|
|
197
|
+
return join(
|
|
198
|
+
standardContainerAssetsDir(),
|
|
199
|
+
"uai-materialize-runtimes",
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Absolute packaged Corepack pin bound over both rolling and derived images. */
|
|
204
|
+
export function standardImageCorepackVersionPath(): string {
|
|
205
|
+
return join(standardContainerAssetsDir(), "corepack-version");
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** ADR-106: the retired bundled-VM fallback exposed a provider-specific
|
|
209
|
+
* asset root; every current backend uses the package's ordinary
|
|
210
|
+
* build-context assets, so the runtime override below is always null. */
|
|
211
|
+
function standardContainerAssetsDir(): string {
|
|
212
|
+
return (
|
|
213
|
+
containerRuntimeStandardAssetsRoot() ?? join(standardImageDir(), "container")
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
interface RunResult {
|
|
218
|
+
code: number | null;
|
|
219
|
+
stdout: string;
|
|
220
|
+
stderr: string;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const SHORT_DOCKER_TIMEOUT_MS = 30_000;
|
|
224
|
+
const CLI_MAINTENANCE_TIMEOUT_MS = 10 * 60_000;
|
|
225
|
+
// Overridable ceiling: the hard-coded 15 minutes looped a fanless Air forever
|
|
226
|
+
// (live 2026-08-19 — every attempt died at 15:00 and the activation driver
|
|
227
|
+
// silently restarted it). Slow hardware needs a bigger window, not a loop.
|
|
228
|
+
const IMAGE_BUILD_TIMEOUT_MS = imageBuildTimeoutMs();
|
|
229
|
+
function imageBuildTimeoutMs(): number {
|
|
230
|
+
const minutes = Number(process.env.UAI_IMAGE_BUILD_TIMEOUT_MINUTES ?? "");
|
|
231
|
+
if (Number.isFinite(minutes) && minutes >= 1) {
|
|
232
|
+
return Math.min(minutes, 240) * 60_000;
|
|
233
|
+
}
|
|
234
|
+
return 15 * 60_000;
|
|
235
|
+
}
|
|
236
|
+
const IMAGE_PULL_TIMEOUT_MS = 15 * 60_000;
|
|
237
|
+
const TERMINATE_GRACE_MS = 5_000;
|
|
238
|
+
export const ASDF_MAINTENANCE_LOCK_PATH =
|
|
239
|
+
"/opt/asdf-data/.uai-maintenance.lock";
|
|
240
|
+
const ASDF_MAINTENANCE_LOCK_WAIT_SECONDS = 300;
|
|
241
|
+
const ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS = 570;
|
|
242
|
+
|
|
243
|
+
/** The selected container engine for image/volume maintenance (ADR-106).
|
|
244
|
+
* Resolved per call: the runtime pin settles before ensureStandardImage runs,
|
|
245
|
+
* but an HMR reload must not strand a stale CLI path. The docker fallback
|
|
246
|
+
* covers the pre-selection window, where docker-only behavior is historical. */
|
|
247
|
+
function selectedEngine(): { command: string; apple: boolean } {
|
|
248
|
+
// The PIN, not the readiness-gated identity: this runs during activation
|
|
249
|
+
// while the public state is still `checking`, and falling back to docker
|
|
250
|
+
// there deadlocks a Docker-free Mac.
|
|
251
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
252
|
+
const binding = appleContainerRuntimeBinding();
|
|
253
|
+
if (binding !== null) {
|
|
254
|
+
return { command: binding.containerCliPath, apple: true };
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return { command: "docker", apple: false };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** The most bytes of each stream a standard-image command keeps; the rest is
|
|
261
|
+
* read and dropped, so a child that writes without end holds no more of the
|
|
262
|
+
* runner's memory than this. */
|
|
263
|
+
const RUN_CAPTURE_BYTES = 8 * 1024 * 1024;
|
|
264
|
+
|
|
265
|
+
/** One stream, kept to at most RUN_CAPTURE_BYTES of its encoded bytes; the
|
|
266
|
+
* runner's own diagnostics go through the same bound. */
|
|
267
|
+
class BoundedCapture {
|
|
268
|
+
#chunks: Buffer[] = [];
|
|
269
|
+
#bytes = 0;
|
|
270
|
+
#lastByte: number | null = null;
|
|
271
|
+
|
|
272
|
+
add(chunk: Buffer | string): void {
|
|
273
|
+
const bytes = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk;
|
|
274
|
+
if (bytes.length === 0 || this.#bytes >= RUN_CAPTURE_BYTES) return;
|
|
275
|
+
const kept = bytes.subarray(0, RUN_CAPTURE_BYTES - this.#bytes);
|
|
276
|
+
this.#chunks.push(kept);
|
|
277
|
+
this.#bytes += kept.length;
|
|
278
|
+
this.#lastByte = kept[kept.length - 1]!;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** A line of the runner's own, on a line of its own. */
|
|
282
|
+
addLine(line: string): void {
|
|
283
|
+
this.add(`${this.#lastByte !== null && this.#lastByte !== 0x0a ? "\n" : ""}${line}`);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** What it kept, as text that keeps the bound too: a character the
|
|
287
|
+
* bound cut, or bytes that are not UTF-8, decode to a three-byte
|
|
288
|
+
* replacement character, so past the bound the text ends at its last
|
|
289
|
+
* whole character. */
|
|
290
|
+
text(): string {
|
|
291
|
+
const text = Buffer.concat(this.#chunks).toString("utf8");
|
|
292
|
+
if (Buffer.byteLength(text, "utf8") <= RUN_CAPTURE_BYTES) return text;
|
|
293
|
+
const encoded = Buffer.from(text, "utf8");
|
|
294
|
+
let end = RUN_CAPTURE_BYTES;
|
|
295
|
+
// Back to the first byte of the character the bound cuts.
|
|
296
|
+
while (end > 0 && (encoded[end]! & 0xc0) === 0x80) end -= 1;
|
|
297
|
+
return encoded.subarray(0, end).toString("utf8");
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** Run a command to completion, capturing stdio, at most RUN_CAPTURE_BYTES
|
|
302
|
+
* of each. Never rejects and never waits forever: standard-image maintenance
|
|
303
|
+
* is an activation prerequisite, so an unbounded Docker/npm child would
|
|
304
|
+
* strand the entire host in checking. */
|
|
305
|
+
function run(
|
|
306
|
+
command: string,
|
|
307
|
+
args: string[],
|
|
308
|
+
timeoutMs = SHORT_DOCKER_TIMEOUT_MS,
|
|
309
|
+
): Promise<RunResult> {
|
|
310
|
+
return new Promise<RunResult>((resolveRun) => {
|
|
311
|
+
const stdout = new BoundedCapture();
|
|
312
|
+
const stderr = new BoundedCapture();
|
|
313
|
+
let settled = false;
|
|
314
|
+
let killTimer: NodeJS.Timeout | null = null;
|
|
315
|
+
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
316
|
+
const settle = (code: number | null): void => {
|
|
317
|
+
if (settled) return;
|
|
318
|
+
settled = true;
|
|
319
|
+
clearTimeout(timeoutTimer);
|
|
320
|
+
if (killTimer) clearTimeout(killTimer);
|
|
321
|
+
resolveRun({ code, stdout: stdout.text(), stderr: stderr.text() });
|
|
322
|
+
};
|
|
323
|
+
const timeoutTimer = setTimeout(() => {
|
|
324
|
+
stderr.addLine(`${command} timed out after ${timeoutMs}ms`);
|
|
325
|
+
child.kill("SIGTERM");
|
|
326
|
+
killTimer = setTimeout(() => {
|
|
327
|
+
child.kill("SIGKILL");
|
|
328
|
+
// SIGKILL should always produce close, but do not let a broken child
|
|
329
|
+
// handle defeat the host-wide readiness bound.
|
|
330
|
+
const settleTimer = setTimeout(() => settle(null), 1_000);
|
|
331
|
+
settleTimer.unref();
|
|
332
|
+
}, TERMINATE_GRACE_MS);
|
|
333
|
+
killTimer.unref();
|
|
334
|
+
}, timeoutMs);
|
|
335
|
+
timeoutTimer.unref();
|
|
336
|
+
child.stdout?.on("data", (chunk: Buffer) => {
|
|
337
|
+
stdout.add(chunk);
|
|
338
|
+
});
|
|
339
|
+
child.stderr?.on("data", (chunk: Buffer) => {
|
|
340
|
+
stderr.add(chunk);
|
|
341
|
+
});
|
|
342
|
+
child.on("error", (err: Error) => {
|
|
343
|
+
stderr.add(err.message);
|
|
344
|
+
settle(null);
|
|
345
|
+
});
|
|
346
|
+
child.on("close", (code) => {
|
|
347
|
+
settle(code);
|
|
348
|
+
});
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** Narrow process-level seam for the timeout regression test. */
|
|
353
|
+
export const runStandardImageCommandForTest = run;
|
|
354
|
+
|
|
355
|
+
/** How long a maintenance run may sit without its container reaching
|
|
356
|
+
* `running` before the wedge sensor passes judgment. Long HEALTHY operations
|
|
357
|
+
* (reconcile's npm installs, the seed copy) are past `running` within
|
|
358
|
+
* seconds, so this discriminates cleanly. Env-overridable for tests. */
|
|
359
|
+
function wedgeProbeDelayMs(): number {
|
|
360
|
+
const raw = Number(process.env.UAI_APPLE_WEDGE_PROBE_MS);
|
|
361
|
+
return Number.isFinite(raw) && raw > 0 ? raw : 60_000;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/** Post-recycle grace for the hung CLI client before a synthesized result. */
|
|
365
|
+
function wedgeGraceMs(): number {
|
|
366
|
+
const raw = Number(process.env.UAI_APPLE_WEDGE_GRACE_MS);
|
|
367
|
+
return Number.isFinite(raw) && raw > 0 ? raw : 10_000;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function delayMs(ms: number): Promise<void> {
|
|
371
|
+
return new Promise((resolveDelay) => {
|
|
372
|
+
const timer = setTimeout(resolveDelay, ms);
|
|
373
|
+
timer.unref();
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* One DEFINITIVE observation of the named container: a successful, parseable
|
|
379
|
+
* inspect reporting a non-running state. Anything else — inspect failure,
|
|
380
|
+
* unparseable output, missing state, or `running` — is NOT evidence of a
|
|
381
|
+
* wedge and must stand the sensor down (the old fall-through treated
|
|
382
|
+
* uncertainty as wedge evidence, which authorizes a service recycle on a
|
|
383
|
+
* guess).
|
|
384
|
+
*/
|
|
385
|
+
async function appleContainerDefinitivelyNotRunning(
|
|
386
|
+
engine: { command: string },
|
|
387
|
+
name: string,
|
|
388
|
+
): Promise<boolean> {
|
|
389
|
+
const inspected = await run(engine.command, ["inspect", name], 15_000);
|
|
390
|
+
if (inspected.code !== 0) return false;
|
|
391
|
+
try {
|
|
392
|
+
const parsed = JSON.parse(inspected.stdout) as Array<{
|
|
393
|
+
id?: unknown;
|
|
394
|
+
status?: { state?: unknown };
|
|
395
|
+
}>;
|
|
396
|
+
if (!Array.isArray(parsed) || parsed.length !== 1) return false;
|
|
397
|
+
if (parsed[0]?.id !== name) return false;
|
|
398
|
+
const state = parsed[0]?.status?.state;
|
|
399
|
+
return typeof state === "string" && state !== "running";
|
|
400
|
+
} catch {
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* The wedge signature, judged live: the named container has NOT reached
|
|
407
|
+
* `running` (definitively — see above) while the control plane still answers
|
|
408
|
+
* a `list`, observed STABLY: a second exact inspection after the list (and a
|
|
409
|
+
* short settle) must repeat the non-running verdict immediately before any
|
|
410
|
+
* recycle is authorized, closing the window where the container starts while
|
|
411
|
+
* the list is in flight. A dead apiserver fails the `list` too (that is an
|
|
412
|
+
* outage, not a wedge, and recycling on it would be a guess).
|
|
413
|
+
*/
|
|
414
|
+
async function appleWedgeVerdict(
|
|
415
|
+
engine: { command: string },
|
|
416
|
+
name: string,
|
|
417
|
+
): Promise<boolean> {
|
|
418
|
+
if (!(await appleContainerDefinitivelyNotRunning(engine, name))) {
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
const listed = await run(engine.command, ["list", "--format", "json"], 15_000);
|
|
422
|
+
if (listed.code !== 0) return false;
|
|
423
|
+
await delayMs(Math.min(5_000, wedgeProbeDelayMs()));
|
|
424
|
+
return appleContainerDefinitivelyNotRunning(engine, name);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Run a maintenance container with corpse hygiene on the Apple runtime.
|
|
429
|
+
*
|
|
430
|
+
* `container run --rm` only removes a container that STARTED. A run refused
|
|
431
|
+
* at start time (the exclusive-volume attach, most commonly) leaves a
|
|
432
|
+
* created-never-started container behind, and those corpses wedge the
|
|
433
|
+
* runtime's apiserver (observed live 2026-08-16, twice: run/delete hang
|
|
434
|
+
* while list still answers). So the container is named and marked Uai's
|
|
435
|
+
* (apple-one-off.ts), and a failed run is settled: its container proven
|
|
436
|
+
* gone, deleted until it is, the runtime's services recycled when it will
|
|
437
|
+
* not go; a run whose client never reported its end is watched a while
|
|
438
|
+
* longer for a container created late. Docker's `--rm` is reliable and its
|
|
439
|
+
* containers are not corpse-prone, so the Docker path passes through.
|
|
440
|
+
*
|
|
441
|
+
* Every apple run doubles as the WEDGE SENSOR (the owner's "this needs to
|
|
442
|
+
* be automatic"): if the container has not reached `running` after the
|
|
443
|
+
* probe delay while `list` still answers, the run reports a distinctive
|
|
444
|
+
* retryable failure. Within the runtime's current activation, the runtime
|
|
445
|
+
* services are first recycled in place (see apple-runtime-recycle.ts), and
|
|
446
|
+
* the activation recovers every task after, its retry loop re-running
|
|
447
|
+
* maintenance against the fresh runtime. Within an activation that is no
|
|
448
|
+
* longer current, nothing restarts the runtime: the current one is coming,
|
|
449
|
+
* with its own maintenance and recovery. Outside any (the hourly pass, a
|
|
450
|
+
* task's start, an engine's sign-in), nothing restarts the runtime either:
|
|
451
|
+
* its owner is asked to recheck it, whose activation does (#p2jdax).
|
|
452
|
+
*/
|
|
453
|
+
async function runMaintenanceContainer(
|
|
454
|
+
engine: { command: string; apple: boolean },
|
|
455
|
+
purpose: string,
|
|
456
|
+
argsAfterRun: string[],
|
|
457
|
+
timeoutMs: number,
|
|
458
|
+
activation: ActivationIsCurrent | null,
|
|
459
|
+
): Promise<RunResult> {
|
|
460
|
+
if (!engine.apple) {
|
|
461
|
+
return run(engine.command, ["run", ...argsAfterRun], timeoutMs);
|
|
462
|
+
}
|
|
463
|
+
const oneOff = newAppleOneOff(purpose);
|
|
464
|
+
const { name } = oneOff;
|
|
465
|
+
const cli = (args: string[], cliTimeoutMs: number) => run(engine.command, args, cliTimeoutMs);
|
|
466
|
+
// The remedy outside activation, and for a watch (which outlives any
|
|
467
|
+
// activation): ask the runtime's owner, once for this run.
|
|
468
|
+
const askOwner = askRuntimeOwnerOnce(
|
|
469
|
+
requestContainerRuntimeRecheck,
|
|
470
|
+
`maintenance container ${name} found the Apple runtime wedged, or could not be removed`,
|
|
471
|
+
);
|
|
472
|
+
// The remedy within an activation: the runtime's services restarted, only
|
|
473
|
+
// while that activation is current.
|
|
474
|
+
const notCurrent = "its activation is no longer the current one";
|
|
475
|
+
const restart = async (): Promise<AppleRecycleOutcome> =>
|
|
476
|
+
activation?.() ? recycleAppleRuntimeServices({ cli }) : { recycled: false, reason: notCurrent };
|
|
477
|
+
let wedgedUnremedied = false;
|
|
478
|
+
let runSettled = false;
|
|
479
|
+
const running = run(
|
|
480
|
+
engine.command,
|
|
481
|
+
// Its labels are what authorize deleting this container later (here, by
|
|
482
|
+
// its run's token; in the recycle sweep, by Uai's) — a name alone is
|
|
483
|
+
// forgeable.
|
|
484
|
+
["run", ...appleOneOffRunArgs(oneOff), ...argsAfterRun],
|
|
485
|
+
timeoutMs,
|
|
486
|
+
).then((outcome) => {
|
|
487
|
+
runSettled = true;
|
|
488
|
+
return outcome;
|
|
489
|
+
});
|
|
490
|
+
const sensed = (async (): Promise<RunResult | null> => {
|
|
491
|
+
await delayMs(wedgeProbeDelayMs());
|
|
492
|
+
if (runSettled) return null;
|
|
493
|
+
if (!(await appleWedgeVerdict(engine, name))) return null;
|
|
494
|
+
if (runSettled) return null;
|
|
495
|
+
if (activation === null || !activation()) {
|
|
496
|
+
wedgedUnremedied = true;
|
|
497
|
+
const reason = activation === null ? (await askOwner()).reason : notCurrent;
|
|
498
|
+
return {
|
|
499
|
+
code: null,
|
|
500
|
+
stdout: "",
|
|
501
|
+
stderr: `apple container runtime is wedged; ${reason} — retry this operation`,
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
const outcome = await restart();
|
|
505
|
+
// The hung CLI client may exit once its apiserver dies; give it a short
|
|
506
|
+
// grace so a real result wins, then synthesize the retryable failure.
|
|
507
|
+
const graced = await Promise.race([
|
|
508
|
+
running,
|
|
509
|
+
delayMs(wedgeGraceMs()).then(() => null),
|
|
510
|
+
]);
|
|
511
|
+
if (graced) return graced;
|
|
512
|
+
return {
|
|
513
|
+
code: null,
|
|
514
|
+
stdout: "",
|
|
515
|
+
stderr: outcome.recycled
|
|
516
|
+
? "apple container runtime was wedged; runtime services were recycled — retry this operation"
|
|
517
|
+
: `apple container runtime is wedged and the automatic recycle did not recover it: ${outcome.reason}`,
|
|
518
|
+
};
|
|
519
|
+
})();
|
|
520
|
+
const result = await Promise.race([
|
|
521
|
+
running,
|
|
522
|
+
sensed.then((synthesized) => synthesized ?? running),
|
|
523
|
+
]);
|
|
524
|
+
if (result.code !== 0) {
|
|
525
|
+
const watched = { cli, recycle: askOwner };
|
|
526
|
+
if (wedgedUnremedied) {
|
|
527
|
+
// Nothing can be deleted until a current activation has restarted the
|
|
528
|
+
// runtime: its name is only watched, and this pass (which that
|
|
529
|
+
// activation's own may wait behind) ends now.
|
|
530
|
+
void watchAppleOneOff(watched, oneOff, { lateCreation: true });
|
|
531
|
+
return result;
|
|
532
|
+
}
|
|
533
|
+
// The run may have failed BECAUSE this generated name already existed —
|
|
534
|
+
// then the container is not ours and the ownership label was never
|
|
535
|
+
// applied (review 2026-08-18 round 3): only a positive label proof
|
|
536
|
+
// authorizes deleting it, and an unobservable one is never recycled for.
|
|
537
|
+
const settled = await settleAppleOneOff(
|
|
538
|
+
{ cli, recycle: activation ? restart : askOwner },
|
|
539
|
+
oneOff,
|
|
540
|
+
);
|
|
541
|
+
// Unproven, or its client never reported its end (the container may yet
|
|
542
|
+
// be created): watched until it is settled.
|
|
543
|
+
const lateCreation = result.code === null;
|
|
544
|
+
if (appleOneOffNeedsWatch(settled, lateCreation)) void watchAppleOneOff(watched, oneOff, { lateCreation });
|
|
545
|
+
}
|
|
546
|
+
return result;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Agent CLIs that MUST exist inside the shared asdf volume. The volume mounts
|
|
551
|
+
* over /opt/asdf-data and shadows the image's baked-in shims, so a volume
|
|
552
|
+
* seeded before a CLI was added (or before a version bump) can be missing it —
|
|
553
|
+
* e.g. a stale volume with `codex` but no `claude`, which makes `claude` exit
|
|
554
|
+
* 127 in every task container. Reconciled on each host start.
|
|
555
|
+
*/
|
|
556
|
+
const VOLUME_AGENT_CLIS: {
|
|
557
|
+
bin: string;
|
|
558
|
+
pkg: string;
|
|
559
|
+
version: string;
|
|
560
|
+
expectedVersionOutput?: string;
|
|
561
|
+
}[] = [
|
|
562
|
+
{ bin: "claude", pkg: "@anthropic-ai/claude-code", version: "latest" },
|
|
563
|
+
{
|
|
564
|
+
bin: "codex",
|
|
565
|
+
pkg: "@openai/codex",
|
|
566
|
+
version: STANDARD_CODEX_CLI_VERSION,
|
|
567
|
+
expectedVersionOutput: `codex-cli ${STANDARD_CODEX_CLI_VERSION}`,
|
|
568
|
+
},
|
|
569
|
+
];
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Cheap, non-mutating diagnostic probe. This is never the authoritative final
|
|
573
|
+
* proof: another process can begin maintenance after a read-only check. The
|
|
574
|
+
* locked reconcile below always performs its own proof before releasing.
|
|
575
|
+
*/
|
|
576
|
+
async function probeVolumeAgentClis(activation: ActivationIsCurrent | null): Promise<string[] | null> {
|
|
577
|
+
const checks = VOLUME_AGENT_CLIS.map((cli) => {
|
|
578
|
+
const missing = `echo "UAI_MISSING:${cli.bin}"`;
|
|
579
|
+
if (cli.expectedVersionOutput === undefined) {
|
|
580
|
+
return `"${cli.bin}" --version >/dev/null 2>&1 || ${missing}`;
|
|
581
|
+
}
|
|
582
|
+
return (
|
|
583
|
+
`version="$("${cli.bin}" --version 2>/dev/null)" && ` +
|
|
584
|
+
`[ "$version" = "${cli.expectedVersionOutput}" ] || ${missing}`
|
|
585
|
+
);
|
|
586
|
+
}).join("; ");
|
|
587
|
+
// Probe by EXECUTING each bin (`--version`), not `command -v`. A stale asdf
|
|
588
|
+
// shim — the shim file survives in the volume but the reshim DB no longer
|
|
589
|
+
// maps it to an installed exe — passes `command -v` (the file is present and
|
|
590
|
+
// executable) yet aborts at runtime with "unknown command: codex. Perhaps
|
|
591
|
+
// you have to reshim?" (exit 1). That made a broken codex/claude read as
|
|
592
|
+
// "present," so the self-heal skipped it and every task/resume spawn failed,
|
|
593
|
+
// restart after restart (live 2026-07-22). Running the bin is the only probe
|
|
594
|
+
// that catches a stale shim. `-w /home/node` so asdf resolves the node
|
|
595
|
+
// version from /home/node/.tool-versions — the same one the repair installs
|
|
596
|
+
// against — and the sentinel prefix keeps login-shell noise from being
|
|
597
|
+
// misread as a missing bin.
|
|
598
|
+
const check = await runMaintenanceContainer(selectedEngine(), "cli-probe", [
|
|
599
|
+
"--rm",
|
|
600
|
+
"-w",
|
|
601
|
+
"/home/node",
|
|
602
|
+
"-v",
|
|
603
|
+
`${ASDF_DATA_VOLUME}:/opt/asdf-data:ro`,
|
|
604
|
+
STANDARD_IMAGE_TAG,
|
|
605
|
+
"/bin/bash",
|
|
606
|
+
"-c",
|
|
607
|
+
checks,
|
|
608
|
+
], SHORT_DOCKER_TIMEOUT_MS, activation);
|
|
609
|
+
if (check.code !== 0) {
|
|
610
|
+
console.warn(
|
|
611
|
+
`[host-agent] could not check agent CLIs in ${ASDF_DATA_VOLUME} (exit ` +
|
|
612
|
+
`${check.code ?? "spawn"}); continuing. ${check.stderr.trim()}`,
|
|
613
|
+
);
|
|
614
|
+
return null;
|
|
615
|
+
}
|
|
616
|
+
return check.stdout
|
|
617
|
+
.split("\n")
|
|
618
|
+
.map((l) => l.trim())
|
|
619
|
+
.filter((l) => l.startsWith("UAI_MISSING:"))
|
|
620
|
+
.map((l) => l.slice("UAI_MISSING:".length));
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Create/repair the shared lock as a regular node-writable file. The lock is
|
|
625
|
+
* seeded into fresh volumes by the image. Legacy volumes can contain a 0644
|
|
626
|
+
* root-created file; opening it read-only permits flock, then metadata repair
|
|
627
|
+
* occurs while holding that same inode (never replace it under a waiter).
|
|
628
|
+
*/
|
|
629
|
+
async function prepareAsdfMaintenanceLock(activation: ActivationIsCurrent | null): Promise<boolean> {
|
|
630
|
+
const command =
|
|
631
|
+
`set -eu; lock=${ASDF_MAINTENANCE_LOCK_PATH}; ` +
|
|
632
|
+
`if [ -L "$lock" ]; then echo "unsafe symlink lock: $lock" >&2; exit 1; fi; ` +
|
|
633
|
+
`if [ ! -e "$lock" ]; then umask 002; : > "$lock"; fi; ` +
|
|
634
|
+
`if [ ! -f "$lock" ]; then echo "lock is not regular: $lock" >&2; exit 1; fi; ` +
|
|
635
|
+
`exec 9<"$lock"; /usr/bin/flock --exclusive --timeout ${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS} 9; ` +
|
|
636
|
+
`/usr/bin/chown node:node "$lock"; /usr/bin/chmod 0664 "$lock"`;
|
|
637
|
+
const result = await runMaintenanceContainer(selectedEngine(), "lock", [
|
|
638
|
+
"--rm",
|
|
639
|
+
"-u",
|
|
640
|
+
"root",
|
|
641
|
+
"-v",
|
|
642
|
+
`${ASDF_DATA_VOLUME}:/opt/asdf-data`,
|
|
643
|
+
STANDARD_IMAGE_TAG,
|
|
644
|
+
"/usr/bin/timeout",
|
|
645
|
+
"--signal=TERM",
|
|
646
|
+
"--kill-after=10s",
|
|
647
|
+
`${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
|
|
648
|
+
"/bin/bash",
|
|
649
|
+
"-c",
|
|
650
|
+
command,
|
|
651
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
652
|
+
if (result.code === 0) return true;
|
|
653
|
+
lastMaintenanceStderr = result.stderr;
|
|
654
|
+
console.warn(
|
|
655
|
+
`[host-agent] could not prepare ${ASDF_MAINTENANCE_LOCK_PATH} (exit ` +
|
|
656
|
+
`${result.code ?? "spawn"}): ${result.stderr.trim()}`,
|
|
657
|
+
);
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* One cross-process transaction owns upgrade, repair, Corepack repair, reshim,
|
|
663
|
+
* and the authoritative execution proof. A waiter always acquires the lock and
|
|
664
|
+
* performs its own full operation; a concurrent holder is never treated as
|
|
665
|
+
* success. The in-container timeout survives a disconnected Docker client and
|
|
666
|
+
* remains strictly inside the host-side child timeout.
|
|
667
|
+
*/
|
|
668
|
+
/** Build the single locked maintenance program. Exported only so focused
|
|
669
|
+
* tests can assert the crash-repair and proof contract without duplicating
|
|
670
|
+
* this security-sensitive shell in a fixture. */
|
|
671
|
+
export function standardImageVolumeMaintenanceCommandForTest(
|
|
672
|
+
autoUpdate: boolean,
|
|
673
|
+
): string {
|
|
674
|
+
const pkgs = VOLUME_AGENT_CLIS.map(
|
|
675
|
+
(cli) => `${cli.pkg}@${cli.version}`,
|
|
676
|
+
).join(" ");
|
|
677
|
+
const trustedNpmFlags =
|
|
678
|
+
"--registry=https://registry.npmjs.org/ --userconfig=/dev/null " +
|
|
679
|
+
"--globalconfig=/dev/uai-empty-npmrc";
|
|
680
|
+
return (
|
|
681
|
+
`set -u; upgrade=${autoUpdate ? 1 : 0}; ` +
|
|
682
|
+
`fail() { echo "UAI_STILL_BROKEN:$1" >&2; exit 1; }; ` +
|
|
683
|
+
`export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; ` +
|
|
684
|
+
`export HOME=/home/node ASDF_DIR=/opt/asdf ASDF_DATA_DIR=/opt/asdf-data ` +
|
|
685
|
+
`ASDF_CONFIG_FILE=/dev/null ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
|
|
686
|
+
`COREPACK_HOME=/opt/asdf-data/corepack COREPACK_DEFAULT_TO_LATEST=0 ` +
|
|
687
|
+
`COREPACK_ENABLE_AUTO_PIN=0 COREPACK_ENABLE_DOWNLOAD_PROMPT=0 ` +
|
|
688
|
+
`COREPACK_ENV_FILE=0 COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
|
|
689
|
+
`COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
|
|
690
|
+
`COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org; ` +
|
|
691
|
+
`default_node="$(/usr/bin/awk '$1 == "nodejs" { print $2; exit }' /home/node/.tool-versions)"; ` +
|
|
692
|
+
`required_codex="${STANDARD_CODEX_CLI_VERSION}"; ` +
|
|
693
|
+
`if ! printf '%s' "$default_node" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$'; then ` +
|
|
694
|
+
`fail default-node; fi; ` +
|
|
695
|
+
`[ "$default_node" = "${STANDARD_DEFAULT_NODE_VERSION}" ] || ` +
|
|
696
|
+
`fail default-node-release-migration-required; ` +
|
|
697
|
+
`export ASDF_NODEJS_VERSION="$default_node"; ` +
|
|
698
|
+
`if [ -e /dev/uai-empty-npmrc ] || [ -L /dev/uai-empty-npmrc ]; then ` +
|
|
699
|
+
`fail unsafe-npm-config; fi; ` +
|
|
700
|
+
`installs_root="$ASDF_DATA_DIR/installs"; node_root="$installs_root/nodejs"; ` +
|
|
701
|
+
`candidate="$node_root/$default_node"; shims_root="$ASDF_DATA_DIR/shims"; ` +
|
|
702
|
+
`corepack_root="$COREPACK_HOME"; ` +
|
|
703
|
+
`ensure_real_dir() { path="$1"; label="$2"; ` +
|
|
704
|
+
`if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then ` +
|
|
705
|
+
`fail "unsafe-$label"; fi; ` +
|
|
706
|
+
`if [ ! -d "$path" ]; then /usr/bin/install -d -o node -g node -m 0775 "$path" ` +
|
|
707
|
+
`|| fail "create-$label"; fi; ` +
|
|
708
|
+
`lexical="$(/usr/bin/realpath -m -s -- "$path")"; ` +
|
|
709
|
+
`resolved="$(/usr/bin/realpath -e -- "$path")" ` +
|
|
710
|
+
`|| fail "resolve-$label"; [ "$lexical" = "$resolved" ] ` +
|
|
711
|
+
`|| fail "escaped-$label"; /usr/bin/chown node:node "$path" ` +
|
|
712
|
+
`|| fail "owner-$label"; /usr/bin/chmod u+rwx,g+rwx "$path" ` +
|
|
713
|
+
`|| fail "mode-$label"; }; ` +
|
|
714
|
+
`ensure_real_dir "$ASDF_DATA_DIR" asdf-data; ` +
|
|
715
|
+
`ensure_real_dir "$installs_root" installs-root; ` +
|
|
716
|
+
`ensure_real_dir "$node_root" node-root; ` +
|
|
717
|
+
`ensure_real_dir "$shims_root" shims-root; ` +
|
|
718
|
+
`ensure_real_dir "$corepack_root" corepack-root; ` +
|
|
719
|
+
// Legacy images could seed individual shims as root even though the
|
|
720
|
+
// directory itself was later repaired. asdf 0.14 then prints permission
|
|
721
|
+
// errors while reshim quietly leaves the old executable behind; task
|
|
722
|
+
// materialization can appear successful but the next CLI launch is stale
|
|
723
|
+
// or non-executable. The trusted root maintenance transaction repairs only
|
|
724
|
+
// the flat, regular-file shim set before dropping to node for every asdf
|
|
725
|
+
// mutation. Anything else is ambiguous volume content and fails closed.
|
|
726
|
+
`repair_shim_metadata() { ` +
|
|
727
|
+
`unsafe_shim_entry="$(/usr/bin/find "$shims_root" -xdev -mindepth 1 ` +
|
|
728
|
+
`-maxdepth 1 ! -type f -print -quit)"; ` +
|
|
729
|
+
`[ -z "$unsafe_shim_entry" ] || fail unsafe-shim-entry; ` +
|
|
730
|
+
`/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
|
|
731
|
+
`-exec /usr/bin/chown node:node -- {} + ` +
|
|
732
|
+
`|| fail repair-shim-owner; ` +
|
|
733
|
+
`/usr/bin/find "$shims_root" -xdev -mindepth 1 -maxdepth 1 -type f ` +
|
|
734
|
+
`-exec /usr/bin/chmod 0755 -- {} + ` +
|
|
735
|
+
`|| fail repair-shim-mode; }; ` +
|
|
736
|
+
`repair_shim_metadata; ` +
|
|
737
|
+
`candidate_is_safe() { ` +
|
|
738
|
+
`lexical_candidate="$(/usr/bin/realpath -m -s -- "$candidate")" ` +
|
|
739
|
+
`|| return 1; [ "$lexical_candidate" = "$node_root/$default_node" ] ` +
|
|
740
|
+
`|| return 1; if [ -e "$candidate" ] || [ -L "$candidate" ]; then ` +
|
|
741
|
+
`[ ! -L "$candidate" ] && [ -d "$candidate" ] || return 1; ` +
|
|
742
|
+
`resolved_candidate="$(/usr/bin/realpath -e -- "$candidate")" ` +
|
|
743
|
+
`|| return 1; ` +
|
|
744
|
+
`[ "$resolved_candidate" = "$lexical_candidate" ] || return 1; fi; }; ` +
|
|
745
|
+
`contained_executable() { root="$1"; executable="$2"; ` +
|
|
746
|
+
`[ -f "$executable" ] && [ -x "$executable" ] || return 1; ` +
|
|
747
|
+
`real_executable="$(/usr/bin/realpath -e -- "$executable")" ` +
|
|
748
|
+
`|| return 1; ` +
|
|
749
|
+
`case "$real_executable" in "$root"/*) return 0 ;; *) return 1 ;; esac; }; ` +
|
|
750
|
+
`contained_real_dir() { root="$1"; path="$2"; ` +
|
|
751
|
+
`[ ! -L "$path" ] && [ -d "$path" ] || return 1; ` +
|
|
752
|
+
`lexical_path="$(/usr/bin/realpath -m -s -- "$path")" || return 1; ` +
|
|
753
|
+
`real_path="$(/usr/bin/realpath -e -- "$path")" || return 1; ` +
|
|
754
|
+
`case "$lexical_path" in "$root"/*) ;; *) return 1 ;; esac; ` +
|
|
755
|
+
`[ "$real_path" = "$lexical_path" ]; }; ` +
|
|
756
|
+
`npm_prefix_is_safe() { contained_real_dir "$candidate" "$candidate/bin" ` +
|
|
757
|
+
`&& contained_real_dir "$candidate" "$candidate/lib" ` +
|
|
758
|
+
`&& contained_real_dir "$candidate" "$candidate/lib/node_modules"; }; ` +
|
|
759
|
+
`scope_is_safe_or_absent() { scope="$1"; ` +
|
|
760
|
+
`if [ -e "$scope" ] || [ -L "$scope" ]; then ` +
|
|
761
|
+
`contained_real_dir "$candidate" "$scope"; fi; }; ` +
|
|
762
|
+
`as_node() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
|
|
763
|
+
`/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
|
|
764
|
+
`ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
|
|
765
|
+
`ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
|
|
766
|
+
`ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
|
|
767
|
+
`COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
|
|
768
|
+
`COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
|
|
769
|
+
`COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
|
|
770
|
+
`COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
|
|
771
|
+
`COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
|
|
772
|
+
`NPM_CONFIG_PREFIX="$candidate" ` +
|
|
773
|
+
`PATH="$candidate/bin:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
|
|
774
|
+
`"$@"; }; ` +
|
|
775
|
+
`as_node_control() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
|
|
776
|
+
`/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
|
|
777
|
+
`ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
|
|
778
|
+
`ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
|
|
779
|
+
`ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
|
|
780
|
+
`COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
|
|
781
|
+
`COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
|
|
782
|
+
`COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
|
|
783
|
+
`COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
|
|
784
|
+
`COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
|
|
785
|
+
`PATH="/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
|
|
786
|
+
`"$@"; }; ` +
|
|
787
|
+
`clear_forbidden_shared_bash_shim() { fail_if_found="$1"; ` +
|
|
788
|
+
`bash_shim="$shims_root/bash"; bash_shim_found=0; ` +
|
|
789
|
+
`if [ -e "$bash_shim" ] || [ -L "$bash_shim" ]; then ` +
|
|
790
|
+
`bash_shim_found=1; ` +
|
|
791
|
+
`[ ! -d "$bash_shim" ] || fail unsafe-runtime-bash-shim-directory; ` +
|
|
792
|
+
`as_node /usr/bin/rm -f -- "$bash_shim" ` +
|
|
793
|
+
`|| fail remove-runtime-bash-shim; fi; ` +
|
|
794
|
+
`[ ! -e "$bash_shim" ] && [ ! -L "$bash_shim" ] ` +
|
|
795
|
+
`|| fail unsafe-runtime-bash-shim; ` +
|
|
796
|
+
`if [ "$fail_if_found" -ne 0 ] && [ "$bash_shim_found" -ne 0 ]; then ` +
|
|
797
|
+
`return 1; fi; }; ` +
|
|
798
|
+
`clear_forbidden_candidate_bash() { fail_if_found="$1"; ` +
|
|
799
|
+
`candidate_bash="$candidate/bin/bash"; candidate_bash_found=''; ` +
|
|
800
|
+
`if [ -e "$candidate_bash" ] || [ -L "$candidate_bash" ]; then ` +
|
|
801
|
+
`candidate_bash_found=1; [ ! -d "$candidate_bash" ] ` +
|
|
802
|
+
`|| fail unsafe-default-bash-directory; ` +
|
|
803
|
+
`as_node /usr/bin/rm -f -- "$candidate_bash" ` +
|
|
804
|
+
`|| fail remove-default-bash; fi; ` +
|
|
805
|
+
`[ ! -e "$candidate_bash" ] && [ ! -L "$candidate_bash" ] ` +
|
|
806
|
+
`|| fail unsafe-default-bash; ` +
|
|
807
|
+
`if [ "$fail_if_found" -ne 0 ] && [ -n "$candidate_bash_found" ]; then ` +
|
|
808
|
+
`return 1; fi; }; ` +
|
|
809
|
+
`run_asdf_install() { asdf_status=0; shim_status=0; ` +
|
|
810
|
+
`as_node_control /opt/asdf/bin/asdf install "$@" || asdf_status=$?; ` +
|
|
811
|
+
`clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
|
|
812
|
+
`[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
|
|
813
|
+
`run_asdf_reshim() { asdf_status=0; shim_status=0; ` +
|
|
814
|
+
`as_node_control /opt/asdf/bin/asdf reshim "$@" || asdf_status=$?; ` +
|
|
815
|
+
`clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
|
|
816
|
+
`[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
|
|
817
|
+
`reject_runtime_bash_shims() { ` +
|
|
818
|
+
`bash_cleanup_status=0; clear_forbidden_candidate_bash 1 ` +
|
|
819
|
+
`|| bash_cleanup_status=1; ` +
|
|
820
|
+
`clear_forbidden_shared_bash_shim 1 || bash_cleanup_status=1; ` +
|
|
821
|
+
`[ "$bash_cleanup_status" -eq 0 ] || fail unsafe-runtime-bash-shim; }; ` +
|
|
822
|
+
`node_write_probe() { writable="$1"; ` +
|
|
823
|
+
`probe="$(as_node /usr/bin/mktemp -d "$writable/.uai-write-probe.XXXXXX")" ` +
|
|
824
|
+
`|| return 1; case "$probe" in "$writable"/.uai-write-probe.*) ;; ` +
|
|
825
|
+
`*) return 1 ;; esac; as_node /usr/bin/rmdir "$probe"; }; ` +
|
|
826
|
+
`candidate_owned_by_node() { ` +
|
|
827
|
+
`[ -z "$(/usr/bin/find "$candidate" -xdev ! -user node -print -quit)" ] ` +
|
|
828
|
+
`&& [ -z "$(/usr/bin/find "$candidate" -xdev -type d ! -perm -u+w -print -quit)" ]; }; ` +
|
|
829
|
+
`prove_default_node() { candidate_is_safe && [ -d "$candidate" ] ` +
|
|
830
|
+
`&& [ ! -e "$candidate/bin/bash" ] && [ ! -L "$candidate/bin/bash" ] ` +
|
|
831
|
+
`&& contained_executable "$candidate" "$candidate/bin/node" ` +
|
|
832
|
+
`&& [ "$(as_node "$candidate/bin/node" --version 2>/dev/null)" = "v$default_node" ] ` +
|
|
833
|
+
`&& contained_executable "$candidate" "$candidate/bin/npm" ` +
|
|
834
|
+
`&& [ -n "$(as_node "$candidate/bin/npm" --version 2>/dev/null)" ] ` +
|
|
835
|
+
`&& npm_prefix_is_safe && candidate_owned_by_node ` +
|
|
836
|
+
`&& node_write_probe "$candidate/bin"; }; ` +
|
|
837
|
+
`clear_forbidden_shared_bash_shim 0; ` +
|
|
838
|
+
`if ! prove_default_node; then candidate_is_safe || fail unsafe-default-candidate; ` +
|
|
839
|
+
`/usr/bin/rm -rf --one-file-system -- "$candidate" ` +
|
|
840
|
+
`|| fail remove-default-candidate; ` +
|
|
841
|
+
`run_asdf_install nodejs "$default_node" ` +
|
|
842
|
+
`|| fail install-default-node; prove_default_node || fail repaired-default-node; fi; ` +
|
|
843
|
+
`npm_bin="$candidate/bin/npm"; cd / || fail chdir; ` +
|
|
844
|
+
`if [ "$upgrade" = 1 ]; then ` +
|
|
845
|
+
`npm_prefix_is_safe || fail unsafe-npm-prefix; ` +
|
|
846
|
+
`if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
|
|
847
|
+
`>/tmp/uai-cli-upgrade.out 2>&1; then ` +
|
|
848
|
+
`if /usr/bin/grep -qi 'ENOTEMPTY' /tmp/uai-cli-upgrade.out; then ` +
|
|
849
|
+
`scope_is_safe_or_absent "$candidate/lib/node_modules/@anthropic-ai" ` +
|
|
850
|
+
`&& scope_is_safe_or_absent "$candidate/lib/node_modules/@openai" ` +
|
|
851
|
+
`|| fail unsafe-cli-scope; ` +
|
|
852
|
+
`as_node /usr/bin/rm -rf --one-file-system -- ` +
|
|
853
|
+
`"$candidate/lib/node_modules/@anthropic-ai" ` +
|
|
854
|
+
`"$candidate/lib/node_modules/@openai"; ` +
|
|
855
|
+
`if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
|
|
856
|
+
`>/tmp/uai-cli-upgrade-retry.out 2>&1; then ` +
|
|
857
|
+
`echo 'UAI_CLI_UPDATE_DEFERRED:bulk-retry' >&2; fi; ` +
|
|
858
|
+
`else echo 'UAI_CLI_UPDATE_DEFERRED:bulk' >&2; fi; fi; fi; ` +
|
|
859
|
+
// Reshim first so an installed package behind a stale shim can recover
|
|
860
|
+
// without network access. Only bins that still fail are reinstalled.
|
|
861
|
+
`if ! run_asdf_reshim nodejs "$default_node" ` +
|
|
862
|
+
`>/dev/null 2>&1; then echo 'UAI_CLI_RESHIM_DEFERRED:initial' >&2; fi; ` +
|
|
863
|
+
`cli_ready_once() { b="$1"; ` +
|
|
864
|
+
`contained_executable "$candidate" "$candidate/bin/$b" || return 1; ` +
|
|
865
|
+
`direct_version="$(as_node "$candidate/bin/$b" --version 2>/dev/null ` +
|
|
866
|
+
`| /usr/bin/head -1)"; [ -n "$direct_version" ] || return 1; ` +
|
|
867
|
+
`if [ "$b" = codex ] && ` +
|
|
868
|
+
`[ "$direct_version" != "codex-cli $required_codex" ]; then return 1; fi; ` +
|
|
869
|
+
`shim="$shims_root/$b"; [ ! -L "$shim" ] && [ -f "$shim" ] ` +
|
|
870
|
+
`&& [ -x "$shim" ] || return 1; ` +
|
|
871
|
+
`shim_version="$(as_node "$shim" --version 2>/dev/null ` +
|
|
872
|
+
`| /usr/bin/head -1)"; [ "$shim_version" = "$direct_version" ]; }; ` +
|
|
873
|
+
`repair_cli_once() { b="$1"; case "$b" in ` +
|
|
874
|
+
`claude) pkg='@anthropic-ai/claude-code'; scope='@anthropic-ai'; ` +
|
|
875
|
+
`package_name='claude-code'; version='latest';; ` +
|
|
876
|
+
`codex) pkg='@openai/codex'; scope='@openai'; package_name='codex'; ` +
|
|
877
|
+
`version="$required_codex";; ` +
|
|
878
|
+
`*) return 1;; esac; ` +
|
|
879
|
+
`package_path="$candidate/lib/node_modules/$scope/$package_name"; ` +
|
|
880
|
+
`npm_prefix_is_safe || return 1; ` +
|
|
881
|
+
`scope_is_safe_or_absent "$candidate/lib/node_modules/$scope" ` +
|
|
882
|
+
`|| return 1; ` +
|
|
883
|
+
`as_node /usr/bin/rm -rf --one-file-system -- ` +
|
|
884
|
+
`"$candidate/bin/$b" "$package_path" || return 1; ` +
|
|
885
|
+
`as_node "$npm_bin" install -g --force "$pkg@$version" ${trustedNpmFlags} ` +
|
|
886
|
+
`>/dev/null 2>&1 || return 1; ` +
|
|
887
|
+
`run_asdf_reshim nodejs "$default_node" ` +
|
|
888
|
+
`>/dev/null 2>&1 || return 1; cli_ready_once "$b"; }; ` +
|
|
889
|
+
`for b in claude codex; do if ! cli_ready_once "$b"; then ` +
|
|
890
|
+
`repair_cli_once "$b" || fail "$b-repair"; fi; done; ` +
|
|
891
|
+
// Node <=22's bundled Corepack has obsolete registry signing keys. The
|
|
892
|
+
// host-packaged pin is compatible with every advertised Node default.
|
|
893
|
+
`required_corepack="$(/usr/bin/sed -n '1p' /usr/local/share/uai/corepack-version)"; ` +
|
|
894
|
+
`printf '%s' "$required_corepack" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$' || { ` +
|
|
895
|
+
`echo 'UAI_STILL_BROKEN:corepack-pin' >&2; exit 1; }; ` +
|
|
896
|
+
`corepack_ready_once() { ` +
|
|
897
|
+
`contained_executable "$candidate" "$candidate/bin/corepack" || return 1; ` +
|
|
898
|
+
`[ "$(as_node "$candidate/bin/corepack" --version 2>/dev/null)" = "$required_corepack" ] ` +
|
|
899
|
+
`|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
|
|
900
|
+
`|| return 1; run_asdf_reshim nodejs "$default_node" ` +
|
|
901
|
+
`>/dev/null 2>&1 || return 1; ` +
|
|
902
|
+
`corepack_shim="$shims_root/corepack"; [ ! -L "$corepack_shim" ] ` +
|
|
903
|
+
`&& [ -f "$corepack_shim" ] && [ -x "$corepack_shim" ] || return 1; ` +
|
|
904
|
+
`[ "$(as_node "$corepack_shim" --version 2>/dev/null)" = "$required_corepack" ]; }; ` +
|
|
905
|
+
`repair_corepack_once() { ` +
|
|
906
|
+
`npm_prefix_is_safe || return 1; ` +
|
|
907
|
+
`as_node /usr/bin/rm -rf --one-file-system -- ` +
|
|
908
|
+
`"$candidate/lib/node_modules/corepack" "$candidate/bin/corepack" ` +
|
|
909
|
+
`"$candidate/bin/pnpm" "$candidate/bin/pnpx" "$candidate/bin/yarn" ` +
|
|
910
|
+
`"$candidate/bin/yarnpkg" || return 1; ` +
|
|
911
|
+
`as_node "$npm_bin" install -g --force "corepack@$required_corepack" ` +
|
|
912
|
+
`--no-audit --no-fund ${trustedNpmFlags} >/dev/null 2>&1 || return 1; ` +
|
|
913
|
+
`run_asdf_reshim nodejs "$default_node" >/dev/null 2>&1 ` +
|
|
914
|
+
`|| return 1; contained_executable "$candidate" "$candidate/bin/corepack" ` +
|
|
915
|
+
`|| return 1; as_node "$candidate/bin/corepack" disable >/dev/null 2>&1 ` +
|
|
916
|
+
`|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
|
|
917
|
+
`|| return 1; run_asdf_reshim nodejs "$default_node" ` +
|
|
918
|
+
`>/dev/null 2>&1 || return 1; corepack_ready_once; }; ` +
|
|
919
|
+
`if ! corepack_ready_once; then repair_corepack_once ` +
|
|
920
|
+
`|| fail corepack-repair; fi; reject_runtime_bash_shims; failed=0; ` +
|
|
921
|
+
`prove_default_node || { echo "UAI_STILL_BROKEN:default-node-final" >&2; failed=1; }; ` +
|
|
922
|
+
`for b in claude codex; do ` +
|
|
923
|
+
`if ! contained_executable "$candidate" "$candidate/bin/$b"; then ` +
|
|
924
|
+
`echo "UAI_STILL_BROKEN:$b-contained" >&2; failed=1; continue; fi; ` +
|
|
925
|
+
`version="$(as_node "$candidate/bin/$b" --version 2>/dev/null | /usr/bin/head -1)"; ` +
|
|
926
|
+
`shim="$shims_root/$b"; if [ -L "$shim" ] || [ ! -f "$shim" ] ` +
|
|
927
|
+
`|| [ ! -x "$shim" ]; then echo "UAI_STILL_BROKEN:$b-shim" >&2; ` +
|
|
928
|
+
`failed=1; continue; fi; ` +
|
|
929
|
+
`shim_version="$(as_node "$shim" --version 2>/dev/null | /usr/bin/head -1)"; ` +
|
|
930
|
+
`if [ "$b" = codex ] && ` +
|
|
931
|
+
`[ "$version" != "codex-cli $required_codex" ]; then ` +
|
|
932
|
+
`echo "UAI_STILL_BROKEN:$b-version" >&2; failed=1; continue; fi; ` +
|
|
933
|
+
`if [ -z "$version" ] || [ "$shim_version" != "$version" ]; then ` +
|
|
934
|
+
`echo "UAI_STILL_BROKEN:$b" >&2; failed=1; ` +
|
|
935
|
+
`else printf '%s ' "$version"; fi; done; ` +
|
|
936
|
+
`node_shim="$shims_root/node"; ` +
|
|
937
|
+
`[ ! -L "$node_shim" ] && [ -f "$node_shim" ] && [ -x "$node_shim" ] ` +
|
|
938
|
+
`|| { echo "UAI_STILL_BROKEN:node-shim-file" >&2; failed=1; }; ` +
|
|
939
|
+
`shim_node="$(as_node "$node_shim" --version 2>/dev/null || true)"; ` +
|
|
940
|
+
`[ "$shim_node" = "v$default_node" ] || { ` +
|
|
941
|
+
`echo "UAI_STILL_BROKEN:node-shim" >&2; failed=1; }; ` +
|
|
942
|
+
`npm_shim="$shims_root/npm"; ` +
|
|
943
|
+
`[ ! -L "$npm_shim" ] && [ -f "$npm_shim" ] && [ -x "$npm_shim" ] ` +
|
|
944
|
+
`|| { echo "UAI_STILL_BROKEN:npm-shim-file" >&2; failed=1; }; ` +
|
|
945
|
+
`shim_npm="$(as_node "$npm_shim" --version 2>/dev/null || true)"; ` +
|
|
946
|
+
`[ -n "$shim_npm" ] || { echo "UAI_STILL_BROKEN:npm-shim" >&2; failed=1; }; ` +
|
|
947
|
+
`if ! corepack_ready_once; then echo "UAI_STILL_BROKEN:corepack-final" >&2; ` +
|
|
948
|
+
`failed=1; else printf 'corepack %s' "$required_corepack"; fi; ` +
|
|
949
|
+
`node_write_probe "$candidate/bin" || { ` +
|
|
950
|
+
`echo "UAI_STILL_BROKEN:default-node-owner" >&2; failed=1; }; ` +
|
|
951
|
+
`node_write_probe "$shims_root" || { ` +
|
|
952
|
+
`echo "UAI_STILL_BROKEN:shims-owner" >&2; failed=1; }; ` +
|
|
953
|
+
`node_write_probe "$corepack_root" || { ` +
|
|
954
|
+
`echo "UAI_STILL_BROKEN:corepack-owner" >&2; failed=1; }; ` +
|
|
955
|
+
`exit "$failed"`
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
async function reconcileVolumeAgentClis(activation: ActivationIsCurrent | null): Promise<boolean> {
|
|
960
|
+
const missingBefore = await probeVolumeAgentClis(activation);
|
|
961
|
+
if (missingBefore?.length) {
|
|
962
|
+
console.log(
|
|
963
|
+
`[host-agent] repairing ${ASDF_DATA_VOLUME}: agent CLI(s) ` +
|
|
964
|
+
`[${missingBefore.join(", ")}] failed their execution probe`,
|
|
965
|
+
);
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
const autoUpdate = process.env.UAI_AGENT_CLI_AUTOUPDATE !== "0";
|
|
969
|
+
if (!autoUpdate) {
|
|
970
|
+
console.log(
|
|
971
|
+
"[host-agent] agent CLI auto-update disabled (UAI_AGENT_CLI_AUTOUPDATE=0)",
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
const command = standardImageVolumeMaintenanceCommandForTest(autoUpdate);
|
|
975
|
+
const reconcile = await runMaintenanceContainer(selectedEngine(), "reconcile", [
|
|
976
|
+
"--rm",
|
|
977
|
+
"-u",
|
|
978
|
+
"root",
|
|
979
|
+
"-w",
|
|
980
|
+
"/home/node",
|
|
981
|
+
"-v",
|
|
982
|
+
`${ASDF_DATA_VOLUME}:/opt/asdf-data`,
|
|
983
|
+
"-v",
|
|
984
|
+
`${standardImageCorepackVersionPath()}:/usr/local/share/uai/corepack-version:ro`,
|
|
985
|
+
STANDARD_IMAGE_TAG,
|
|
986
|
+
"/usr/bin/timeout",
|
|
987
|
+
"--signal=TERM",
|
|
988
|
+
"--kill-after=10s",
|
|
989
|
+
`${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
|
|
990
|
+
"/usr/bin/flock",
|
|
991
|
+
"--exclusive",
|
|
992
|
+
"--timeout",
|
|
993
|
+
`${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS}`,
|
|
994
|
+
ASDF_MAINTENANCE_LOCK_PATH,
|
|
995
|
+
"/bin/bash",
|
|
996
|
+
"-c",
|
|
997
|
+
command,
|
|
998
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
999
|
+
if (reconcile.code === 0) {
|
|
1000
|
+
console.log(
|
|
1001
|
+
`[host-agent] agent CLIs and Corepack current on ${ASDF_DATA_VOLUME}: ` +
|
|
1002
|
+
reconcile.stdout.trim(),
|
|
1003
|
+
);
|
|
1004
|
+
await writeVolumeReconcileProof();
|
|
1005
|
+
return true;
|
|
1006
|
+
} else {
|
|
1007
|
+
lastMaintenanceStderr = reconcile.stderr;
|
|
1008
|
+
console.warn(
|
|
1009
|
+
`[host-agent] failed to reconcile ${ASDF_DATA_VOLUME} (exit ` +
|
|
1010
|
+
`${reconcile.code ?? "spawn"}); task runtimes are not ready. ` +
|
|
1011
|
+
reconcile.stderr.trim().slice(0, 500),
|
|
1012
|
+
);
|
|
1013
|
+
return false;
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
/** Durable evidence that the shared volume completed a full reconcile at
|
|
1018
|
+
* least once on this host. Apple volumes are block devices with EXCLUSIVE
|
|
1019
|
+
* writable attachments, so boot maintenance cannot re-prove the volume while
|
|
1020
|
+
* running task containers hold it read-only — and blocking activation on
|
|
1021
|
+
* that (the 2026-08-16 ring) froze hosts and blocked the very teardowns that
|
|
1022
|
+
* would free the volume. With this proof on disk, a busy volume defers
|
|
1023
|
+
* maintenance instead of failing it. */
|
|
1024
|
+
let lastMaintenanceStderr = "";
|
|
1025
|
+
|
|
1026
|
+
async function volumeReconcileProofPath(): Promise<string> {
|
|
1027
|
+
// Lazy env resolve: module import must stay side-effect free (see
|
|
1028
|
+
// defaultRuntimeProviderStateStore). Dynamic import, not require — a bare
|
|
1029
|
+
// require cannot resolve the TypeScript module outside the tsx runtime,
|
|
1030
|
+
// which silently read as "no proof" and disabled deferral.
|
|
1031
|
+
const { env } = await import("./env");
|
|
1032
|
+
return resolve(env.dataDir, "runtime-volume-proof.json");
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
async function writeVolumeReconcileProof(): Promise<void> {
|
|
1036
|
+
try {
|
|
1037
|
+
writeFileSync(
|
|
1038
|
+
await volumeReconcileProofPath(),
|
|
1039
|
+
`${JSON.stringify({
|
|
1040
|
+
schemaVersion: 1,
|
|
1041
|
+
codexVersion: STANDARD_CODEX_CLI_VERSION,
|
|
1042
|
+
provedAt: Date.now(),
|
|
1043
|
+
})}\n`,
|
|
1044
|
+
{ mode: 0o600 },
|
|
1045
|
+
);
|
|
1046
|
+
} catch (error) {
|
|
1047
|
+
console.warn(
|
|
1048
|
+
`[host-agent] could not persist the shared-volume reconcile proof: ${
|
|
1049
|
+
error instanceof Error ? error.message : String(error)
|
|
1050
|
+
}`,
|
|
1051
|
+
);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
async function volumeReconcileProofExists(): Promise<boolean> {
|
|
1056
|
+
try {
|
|
1057
|
+
const raw = JSON.parse(
|
|
1058
|
+
readFileSync(await volumeReconcileProofPath(), "utf8"),
|
|
1059
|
+
) as { schemaVersion?: unknown; codexVersion?: unknown };
|
|
1060
|
+
return (
|
|
1061
|
+
raw?.schemaVersion === 1 &&
|
|
1062
|
+
raw.codexVersion === STANDARD_CODEX_CLI_VERSION
|
|
1063
|
+
);
|
|
1064
|
+
} catch {
|
|
1065
|
+
return false;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
const VOLUME_ATTACH_BUSY_SIGNATURE = "storage device attachment is invalid";
|
|
1070
|
+
|
|
1071
|
+
/**
|
|
1072
|
+
* Whether task containers hold the shared volume — tri-state per container,
|
|
1073
|
+
* never fail-open: `running` holds, exact `stopped` is released (a stopped
|
|
1074
|
+
* VM releases its block attachments — proven live 2026-08-17, when a full
|
|
1075
|
+
* writable reconcile succeeded next to a stopped task container), and EVERY
|
|
1076
|
+
* other observation (starting, stopping, creating, a missing or foreign
|
|
1077
|
+
* state string, an unreadable inventory) is `unknown` — a task acquiring or
|
|
1078
|
+
* releasing the volume mid-transition must never read as permission for a
|
|
1079
|
+
* writer (review 2026-08-18 round 2). `held` dominates `unknown` dominates
|
|
1080
|
+
* `free`; writers require a positive `free`.
|
|
1081
|
+
*/
|
|
1082
|
+
async function sharedVolumeHolders(engine: {
|
|
1083
|
+
command: string;
|
|
1084
|
+
apple: boolean;
|
|
1085
|
+
}): Promise<"held" | "free" | "unknown"> {
|
|
1086
|
+
const listed = await run(
|
|
1087
|
+
engine.command,
|
|
1088
|
+
["list", "--all", "--format", "json"],
|
|
1089
|
+
30_000,
|
|
1090
|
+
);
|
|
1091
|
+
if (listed.code !== 0) return "unknown";
|
|
1092
|
+
try {
|
|
1093
|
+
const parsed = JSON.parse(listed.stdout) as Array<{
|
|
1094
|
+
configuration?: {
|
|
1095
|
+
id?: unknown;
|
|
1096
|
+
mounts?: Array<{ type?: { volume?: { name?: unknown } } }>;
|
|
1097
|
+
};
|
|
1098
|
+
status?: { state?: unknown };
|
|
1099
|
+
}>;
|
|
1100
|
+
if (!Array.isArray(parsed)) return "unknown";
|
|
1101
|
+
let verdict: "held" | "free" | "unknown" = "free";
|
|
1102
|
+
for (const item of parsed) {
|
|
1103
|
+
// A holder is a container whose CONFIGURED MOUNTS include the shared
|
|
1104
|
+
// volume — physical attachment, not a name shape (review 2026-08-18
|
|
1105
|
+
// round 3: an unrelated task-*-app container with no attachment
|
|
1106
|
+
// fenced activation forever, and the holder-remedy sweep — which
|
|
1107
|
+
// correctly matches by mount — could not clear it). A row whose
|
|
1108
|
+
// mounts cannot be read might hold the volume, so it is `unknown`.
|
|
1109
|
+
const mounts = item?.configuration?.mounts;
|
|
1110
|
+
if (!Array.isArray(mounts)) {
|
|
1111
|
+
verdict = "unknown";
|
|
1112
|
+
continue;
|
|
1113
|
+
}
|
|
1114
|
+
const holds = mounts.some(
|
|
1115
|
+
(mount) => mount?.type?.volume?.name === ASDF_DATA_VOLUME,
|
|
1116
|
+
);
|
|
1117
|
+
if (!holds) continue;
|
|
1118
|
+
const state = item?.status?.state;
|
|
1119
|
+
if (state === "running") return "held";
|
|
1120
|
+
if (state !== "stopped") verdict = "unknown";
|
|
1121
|
+
}
|
|
1122
|
+
return verdict;
|
|
1123
|
+
} catch {
|
|
1124
|
+
return "unknown";
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/** Backstop for the race the proof-first check cannot close: a task container
|
|
1129
|
+
* appearing between the check and the attach. Admits deferral only on the
|
|
1130
|
+
* exact busy signature, with the same proof requirement. */
|
|
1131
|
+
async function volumeBusyDeferral(
|
|
1132
|
+
engine: { command: string; apple: boolean },
|
|
1133
|
+
stderr: string,
|
|
1134
|
+
): Promise<boolean> {
|
|
1135
|
+
if (!engine.apple) return false;
|
|
1136
|
+
if (!stderr.includes(VOLUME_ATTACH_BUSY_SIGNATURE)) return false;
|
|
1137
|
+
if (!(await volumeReconcileProofExists())) return false;
|
|
1138
|
+
return (await sharedVolumeHolders(engine)) === "held";
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
/** Outcome of ensureStandardImage — `error` carries WHY when not ready, so
|
|
1142
|
+
* the taskUp path can surface a build failure to the cloud (not just the host
|
|
1143
|
+
* log). Callers that only fire-and-forget at boot ignore the return. */
|
|
1144
|
+
export interface StandardImageResult {
|
|
1145
|
+
ok: boolean;
|
|
1146
|
+
error?: string;
|
|
1147
|
+
/** A non-writer escape the activation driver can perform before retrying:
|
|
1148
|
+
* lifecycle-stop the running task containers holding the shared volume
|
|
1149
|
+
* (resumable), because the volume needs a repair they block and cannot
|
|
1150
|
+
* safely use. */
|
|
1151
|
+
remedy?: "stop-volume-holders";
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
type AgentClisReadyListener = () => void;
|
|
1155
|
+
|
|
1156
|
+
let agentClisReadyLatched = false;
|
|
1157
|
+
let signalClisReady: (() => void) | null = null;
|
|
1158
|
+
const agentClisReadyListeners = new Set<AgentClisReadyListener>();
|
|
1159
|
+
const ensureQueue = new KeyedPromiseTail<StandardImageResult>();
|
|
1160
|
+
/**
|
|
1161
|
+
* Resolves once ensureStandardImage has reconciled the shared-volume agent CLIs
|
|
1162
|
+
* at least once (or bailed because docker is down / the build failed). The agent
|
|
1163
|
+
* factory awaits this before spawning a CLI, so a boot-time resume never races
|
|
1164
|
+
* the CLI auto-upgrade's brief remove-then-reinstall window — the "No codex
|
|
1165
|
+
* executable found for nodejs X" (exit 2) resume failure. Post-boot it is
|
|
1166
|
+
* already resolved, so the await is a no-op.
|
|
1167
|
+
*/
|
|
1168
|
+
export const agentClisReady: Promise<void> = new Promise<void>((resolve) => {
|
|
1169
|
+
signalClisReady = resolve;
|
|
1170
|
+
});
|
|
1171
|
+
|
|
1172
|
+
/** Whether this process has completed its one boot-time CLI maintenance pass.
|
|
1173
|
+
* This is intentionally a one-way boot latch, not a claim that no later
|
|
1174
|
+
* operator-triggered image work is in flight. */
|
|
1175
|
+
export function areAgentClisReady(): boolean {
|
|
1176
|
+
return agentClisReadyLatched;
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
/** Subscribe to the one readiness edge. A late subscriber is notified in a
|
|
1180
|
+
* microtask so registration never re-enters its caller. */
|
|
1181
|
+
export function onAgentClisReady(
|
|
1182
|
+
listener: AgentClisReadyListener,
|
|
1183
|
+
): () => void {
|
|
1184
|
+
if (agentClisReadyLatched) {
|
|
1185
|
+
let cancelled = false;
|
|
1186
|
+
queueMicrotask(() => {
|
|
1187
|
+
if (!cancelled) notifyAgentClisReadyListener(listener);
|
|
1188
|
+
});
|
|
1189
|
+
return () => {
|
|
1190
|
+
cancelled = true;
|
|
1191
|
+
};
|
|
1192
|
+
}
|
|
1193
|
+
agentClisReadyListeners.add(listener);
|
|
1194
|
+
return () => agentClisReadyListeners.delete(listener);
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
function notifyAgentClisReadyListener(listener: AgentClisReadyListener): void {
|
|
1198
|
+
try {
|
|
1199
|
+
listener();
|
|
1200
|
+
} catch (error) {
|
|
1201
|
+
console.warn(
|
|
1202
|
+
`[host-agent] agent CLI readiness listener failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1203
|
+
);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/** ADR-133 sidecar mode: the agent CLIs are baked into the task machine's
|
|
1208
|
+
* image, so there is no standard-image pass to wait on — boot declares them
|
|
1209
|
+
* ready. Never called on a host that owns a container runtime. */
|
|
1210
|
+
export function declareAgentClisReady(): void {
|
|
1211
|
+
markAgentClisReady();
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
function markAgentClisReady(): void {
|
|
1215
|
+
if (agentClisReadyLatched) return;
|
|
1216
|
+
// Publish the synchronous capability state before resolving the promise:
|
|
1217
|
+
// every waiter and listener must observe `true` after the readiness edge.
|
|
1218
|
+
agentClisReadyLatched = true;
|
|
1219
|
+
signalClisReady?.();
|
|
1220
|
+
signalClisReady = null;
|
|
1221
|
+
const listeners = [...agentClisReadyListeners];
|
|
1222
|
+
agentClisReadyListeners.clear();
|
|
1223
|
+
for (const listener of listeners) {
|
|
1224
|
+
notifyAgentClisReadyListener(listener);
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
/**
|
|
1229
|
+
* Ensure the standard image and the shared asdf data volume exist, and that the
|
|
1230
|
+
* agent CLIs are present inside the volume. Builds the image only when `docker
|
|
1231
|
+
* image inspect` fails or the content hash is stale. Never throws — returns
|
|
1232
|
+
* `{ ok, error? }` so the caller decides (boot: fire-and-forget; taskUp:
|
|
1233
|
+
* surface the reason). Resolves `agentClisReady` on every exit path.
|
|
1234
|
+
*/
|
|
1235
|
+
export function ensureStandardImage(
|
|
1236
|
+
opts: {
|
|
1237
|
+
maxAgeMs?: number;
|
|
1238
|
+
/** Only the runtime's activation passes this, answering whether it is
|
|
1239
|
+
* still the current activation: within it alone, and only while it is
|
|
1240
|
+
* current, may a wedged Apple runtime's services be restarted, since
|
|
1241
|
+
* its recovery follows its maintenance. Every other pass asks the
|
|
1242
|
+
* runtime's owner for that activation instead (#p2jdax). */
|
|
1243
|
+
withinActivation?: ActivationIsCurrent;
|
|
1244
|
+
} = {},
|
|
1245
|
+
): Promise<StandardImageResult> {
|
|
1246
|
+
// ADR-133 sidecar mode: a task machine has no container image to keep —
|
|
1247
|
+
// its agent CLIs are baked into the machine image. Live 2026-09-10
|
|
1248
|
+
// (hostless smoke 4): task-up refused with "could not prepare the host
|
|
1249
|
+
// base image moondeck-standard:dev … Docker is not available".
|
|
1250
|
+
if (sidecarMode()) {
|
|
1251
|
+
markAgentClisReady();
|
|
1252
|
+
return Promise.resolve({ ok: true });
|
|
1253
|
+
}
|
|
1254
|
+
// A task-up passes maxAgeMs: a pass that succeeded recently for the same
|
|
1255
|
+
// engine configuration is reused instead of spending ~15 s spinning a
|
|
1256
|
+
// maintenance container per task (live 2026-09-09: 16 s of every task
|
|
1257
|
+
// start). The hourly schedule (main.ts) and the boot pass keep it fresh.
|
|
1258
|
+
const engines = configuredOptionalEngines();
|
|
1259
|
+
const key = optionalEngineConfigurationKey(engines);
|
|
1260
|
+
const recent = lastOkByKey.get(key);
|
|
1261
|
+
if (
|
|
1262
|
+
opts.maxAgeMs !== undefined &&
|
|
1263
|
+
recent !== undefined &&
|
|
1264
|
+
Date.now() - recent.at <= opts.maxAgeMs
|
|
1265
|
+
) {
|
|
1266
|
+
markAgentClisReady();
|
|
1267
|
+
return Promise.resolve(recent.result);
|
|
1268
|
+
}
|
|
1269
|
+
// Startup, runtime recovery, and a newly-arrived task can all discover the
|
|
1270
|
+
// same daemon transition. One shared volume repair/build must own that
|
|
1271
|
+
// window; concurrent maintenance containers can otherwise replace the same
|
|
1272
|
+
// CLI shims underneath each other and defeat agentClisReady's fence. The
|
|
1273
|
+
// activation's pass never joins one begun outside it (which may not
|
|
1274
|
+
// restart the runtime): it runs after it, as its own.
|
|
1275
|
+
const activation = opts.withinActivation ?? null;
|
|
1276
|
+
return ensureQueue.run(activation ? `activation:${key}` : key, () =>
|
|
1277
|
+
ensureStandardImageInner(engines, activation)
|
|
1278
|
+
.then((result) => {
|
|
1279
|
+
if (result.ok) lastOkByKey.set(key, { at: Date.now(), result });
|
|
1280
|
+
else lastOkByKey.delete(key);
|
|
1281
|
+
return result;
|
|
1282
|
+
})
|
|
1283
|
+
.finally(() => {
|
|
1284
|
+
// Always signal — even a docker-down bail — so agent spawns never hang
|
|
1285
|
+
// waiting on a reconcile that will not happen this boot.
|
|
1286
|
+
markAgentClisReady();
|
|
1287
|
+
}),
|
|
1288
|
+
);
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
/** Last successful pass per engine configuration (in-process). */
|
|
1292
|
+
const lastOkByKey = new Map<string, { at: number; result: StandardImageResult }>();
|
|
1293
|
+
|
|
1294
|
+
/** The runtime's activation, asked whether it is still the current one: what
|
|
1295
|
+
* alone may restart a wedged Apple runtime's services (#p2jdax). */
|
|
1296
|
+
export type ActivationIsCurrent = () => boolean;
|
|
1297
|
+
|
|
1298
|
+
/** Tests: forget every remembered pass. */
|
|
1299
|
+
export function resetStandardImageMemo(): void {
|
|
1300
|
+
lastOkByKey.clear();
|
|
1301
|
+
buildBackoff.clear();
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
const buildBackoff = new StandardImageBuildBackoff();
|
|
1305
|
+
const imageCleanup = new StandardImageCleanup(async () => {
|
|
1306
|
+
const { env } = await import("./env");
|
|
1307
|
+
return resolve(env.dataDir, "standard-image-seen.json");
|
|
1308
|
+
});
|
|
1309
|
+
|
|
1310
|
+
/** The owned-host schedule (docs/agent-notes.md, "image maintenance"). */
|
|
1311
|
+
export const MAINTENANCE_INTERVAL_MS = 60 * 60_000;
|
|
1312
|
+
|
|
1313
|
+
async function ensureStandardImageInner(
|
|
1314
|
+
engines: OptionalEngineConfiguration,
|
|
1315
|
+
activation: ActivationIsCurrent | null,
|
|
1316
|
+
): Promise<StandardImageResult> {
|
|
1317
|
+
// 1. Shared asdf data volume — idempotent under Docker; the Apple CLI
|
|
1318
|
+
// errors on an existing name, which is the same healthy outcome.
|
|
1319
|
+
const engine = selectedEngine();
|
|
1320
|
+
console.log(
|
|
1321
|
+
`[host-agent] standard-image maintenance starting (engine: ${engine.apple ? "apple-container" : "docker"})`,
|
|
1322
|
+
);
|
|
1323
|
+
if (engine.apple) {
|
|
1324
|
+
// The volume create below is the FIRST apple mutation on a host — before
|
|
1325
|
+
// any task exists — and uninstall reads an absent provider-claim marker
|
|
1326
|
+
// as "nothing to tear down". Claim durably before mutating, or a
|
|
1327
|
+
// zero-task install→activate→uninstall skips Apple cleanup (review
|
|
1328
|
+
// 2026-08-18 round 4).
|
|
1329
|
+
try {
|
|
1330
|
+
await claimContainerRuntimeForMaintenance();
|
|
1331
|
+
} catch (error) {
|
|
1332
|
+
return {
|
|
1333
|
+
ok: false,
|
|
1334
|
+
error:
|
|
1335
|
+
"Could not durably record the Apple container runtime claim " +
|
|
1336
|
+
`before provisioning: ${error instanceof Error ? error.message : String(error)}`,
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
const vol = await run(engine.command, ["volume", "create", ASDF_DATA_VOLUME]);
|
|
1341
|
+
if (
|
|
1342
|
+
vol.code !== 0 &&
|
|
1343
|
+
!(engine.apple && vol.stderr.includes("already exists"))
|
|
1344
|
+
) {
|
|
1345
|
+
console.warn(
|
|
1346
|
+
`[host-agent] could not create asdf data volume (${ASDF_DATA_VOLUME}); ` +
|
|
1347
|
+
`continuing. ${vol.stderr.trim()}`,
|
|
1348
|
+
);
|
|
1349
|
+
// If docker itself is unavailable, the image step will also fail; bail
|
|
1350
|
+
// early so we don't double-log a confusing build error.
|
|
1351
|
+
if (vol.code === null) {
|
|
1352
|
+
return { ok: false, error: "Docker is not available on this host." };
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
// 2. Ensure the image is built AND current. "Present" is not enough — a
|
|
1357
|
+
// Dockerfile change used to no-op forever because we only built when
|
|
1358
|
+
// inspect failed (found live 2026-07-08: ADR-053's baked packages never
|
|
1359
|
+
// landed). The build context is content-hashed into an image label;
|
|
1360
|
+
// a mismatch triggers a rebuild (layer cache keeps it cheap).
|
|
1361
|
+
let imageReady = false;
|
|
1362
|
+
let buildError: string | null = null;
|
|
1363
|
+
// Install only the optional engines the operator has configured; the flags
|
|
1364
|
+
// are build args AND part of the content hash (so a new login rebuilds).
|
|
1365
|
+
const engineArgs = [
|
|
1366
|
+
"--build-arg",
|
|
1367
|
+
`INSTALL_KIMI=${engines.kimi ? 1 : 0}`,
|
|
1368
|
+
"--build-arg",
|
|
1369
|
+
`INSTALL_GROK=${engines.grok ? 1 : 0}`,
|
|
1370
|
+
"--build-arg",
|
|
1371
|
+
`INSTALL_CURSOR=${engines.cursor ? 1 : 0}`,
|
|
1372
|
+
"--build-arg",
|
|
1373
|
+
`INSTALL_OPENCODE=${engines.opencode ? 1 : 0}`,
|
|
1374
|
+
];
|
|
1375
|
+
const contextHash = await hashBuildContext(
|
|
1376
|
+
`kimi=${engines.kimi ? 1 : 0};grok=${engines.grok ? 1 : 0};cursor=${engines.cursor ? 1 : 0};opencode=${engines.opencode ? 1 : 0}`,
|
|
1377
|
+
);
|
|
1378
|
+
let inspect: RunResult;
|
|
1379
|
+
let labeledHash: string | null = null;
|
|
1380
|
+
if (engine.apple) {
|
|
1381
|
+
// The Apple CLI has no --format; labels live in the single variant's OCI
|
|
1382
|
+
// config. A malformed answer counts as "absent" and triggers a rebuild.
|
|
1383
|
+
inspect = await run(engine.command, [
|
|
1384
|
+
"image",
|
|
1385
|
+
"inspect",
|
|
1386
|
+
STANDARD_IMAGE_TAG,
|
|
1387
|
+
]);
|
|
1388
|
+
if (inspect.code === 0) {
|
|
1389
|
+
try {
|
|
1390
|
+
const parsed = JSON.parse(inspect.stdout) as Array<{
|
|
1391
|
+
variants?: Array<{
|
|
1392
|
+
config?: { config?: { Labels?: Record<string, string> } };
|
|
1393
|
+
}>;
|
|
1394
|
+
}>;
|
|
1395
|
+
labeledHash =
|
|
1396
|
+
parsed[0]?.variants?.[0]?.config?.config?.Labels?.[
|
|
1397
|
+
CONTEXT_HASH_LABEL
|
|
1398
|
+
] ?? "";
|
|
1399
|
+
} catch {
|
|
1400
|
+
inspect = { ...inspect, code: 1 };
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
} else {
|
|
1404
|
+
inspect = await run("docker", [
|
|
1405
|
+
"image",
|
|
1406
|
+
"inspect",
|
|
1407
|
+
"-f",
|
|
1408
|
+
`{{index .Config.Labels "${CONTEXT_HASH_LABEL}"}}`,
|
|
1409
|
+
STANDARD_IMAGE_TAG,
|
|
1410
|
+
]);
|
|
1411
|
+
labeledHash = inspect.code === 0 ? inspect.stdout.trim() : null;
|
|
1412
|
+
}
|
|
1413
|
+
if (inspect.code === null) {
|
|
1414
|
+
console.warn(
|
|
1415
|
+
"[host-agent] container runtime unavailable; skipping standard image " +
|
|
1416
|
+
"build. Tasks will fail until it is running.",
|
|
1417
|
+
);
|
|
1418
|
+
return {
|
|
1419
|
+
ok: false,
|
|
1420
|
+
error: "The container runtime is not available on this host.",
|
|
1421
|
+
};
|
|
1422
|
+
}
|
|
1423
|
+
if (inspect.code !== 0) labeledHash = null;
|
|
1424
|
+
|
|
1425
|
+
// ADR-114: when the payload pins a prebuilt digest, the pin decides image
|
|
1426
|
+
// identity — the label must equal the PIN's context hash (the CI superset
|
|
1427
|
+
// hash, not this host's per-config hash, which would disagree by design
|
|
1428
|
+
// and rebuild forever). Any pull-path failure falls through to the local
|
|
1429
|
+
// build below, so offline and registry-down hosts behave exactly as today.
|
|
1430
|
+
const previousImageId = engine.apple ? null : await imageCleanup.observe(run);
|
|
1431
|
+
const pin = await readStandardImagePin();
|
|
1432
|
+
if (pin !== null) {
|
|
1433
|
+
if (inspect.code === 0 && labeledHash === pin.contextHash) {
|
|
1434
|
+
console.log(
|
|
1435
|
+
`[host-agent] standard image ${STANDARD_IMAGE_TAG} current (pinned)`,
|
|
1436
|
+
);
|
|
1437
|
+
imageReady = true;
|
|
1438
|
+
} else {
|
|
1439
|
+
const pullDigest = standardImagePullDigest(pin);
|
|
1440
|
+
console.log(
|
|
1441
|
+
`[host-agent] pulling standard image ${pin.ref}@${pullDigest}`,
|
|
1442
|
+
);
|
|
1443
|
+
const commands = standardImagePullCommands(
|
|
1444
|
+
pin,
|
|
1445
|
+
STANDARD_IMAGE_TAG,
|
|
1446
|
+
engine.apple,
|
|
1447
|
+
);
|
|
1448
|
+
const pulled = await run(
|
|
1449
|
+
engine.command,
|
|
1450
|
+
commands.pull,
|
|
1451
|
+
IMAGE_PULL_TIMEOUT_MS,
|
|
1452
|
+
);
|
|
1453
|
+
if (pulled.code === 0) {
|
|
1454
|
+
const tagged = await run(engine.command, commands.tag);
|
|
1455
|
+
if (tagged.code === 0) {
|
|
1456
|
+
console.log(
|
|
1457
|
+
`[host-agent] pulled standard image ${STANDARD_IMAGE_TAG} (${pullDigest.slice(0, 19)}…)`,
|
|
1458
|
+
);
|
|
1459
|
+
imageReady = true;
|
|
1460
|
+
} else {
|
|
1461
|
+
console.warn(
|
|
1462
|
+
"[host-agent] pulled standard image could not be tagged; " +
|
|
1463
|
+
`falling back to a local build. ${tagged.stderr.trim()}`,
|
|
1464
|
+
);
|
|
1465
|
+
}
|
|
1466
|
+
} else {
|
|
1467
|
+
console.warn(
|
|
1468
|
+
"[host-agent] standard image pull failed; falling back to a " +
|
|
1469
|
+
`local build. ${pulled.stderr.trim()}`,
|
|
1470
|
+
);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
if (imageReady) {
|
|
1476
|
+
// Pinned image adopted — skip the local-hash decision entirely.
|
|
1477
|
+
} else if (inspect.code === 0 && contextHash !== null && labeledHash === contextHash) {
|
|
1478
|
+
console.log(`[host-agent] standard image ${STANDARD_IMAGE_TAG} current`);
|
|
1479
|
+
imageReady = true;
|
|
1480
|
+
} else if (inspect.code === 0 && contextHash === null) {
|
|
1481
|
+
// Can't hash (packaged install without the context?) — keep the image.
|
|
1482
|
+
console.log(`[host-agent] standard image ${STANDARD_IMAGE_TAG} present`);
|
|
1483
|
+
imageReady = true;
|
|
1484
|
+
} else {
|
|
1485
|
+
const context = standardImageDir();
|
|
1486
|
+
const buildKey = `${contextHash ?? "unavailable"}:${JSON.stringify(engines)}:${pin?.contextHash ?? "local"}`;
|
|
1487
|
+
const deferred = engine.apple || labeledHash === null ? null : buildBackoff.remaining(buildKey);
|
|
1488
|
+
if (!deferred) {
|
|
1489
|
+
console.log(
|
|
1490
|
+
inspect.code === 0
|
|
1491
|
+
? `[host-agent] standard image ${STANDARD_IMAGE_TAG} stale (context changed) — rebuilding from ${context}`
|
|
1492
|
+
: `[host-agent] building standard image ${STANDARD_IMAGE_TAG} from ${context}`,
|
|
1493
|
+
);
|
|
1494
|
+
}
|
|
1495
|
+
const build = deferred ? { code: 1, stdout: "", stderr: `${deferred.error}\nStandard image build deferred until ${new Date(deferred.retryAt).toISOString()}.` } : await run(engine.command, [
|
|
1496
|
+
"build",
|
|
1497
|
+
...(engine.apple ? [] : ["--force-rm"]),
|
|
1498
|
+
"-t",
|
|
1499
|
+
STANDARD_IMAGE_TAG,
|
|
1500
|
+
...engineArgs,
|
|
1501
|
+
...(contextHash !== null
|
|
1502
|
+
? ["--label", `${CONTEXT_HASH_LABEL}=${contextHash}`]
|
|
1503
|
+
: []),
|
|
1504
|
+
context,
|
|
1505
|
+
], IMAGE_BUILD_TIMEOUT_MS);
|
|
1506
|
+
if (build.code === 0) {
|
|
1507
|
+
console.log(`[host-agent] built standard image ${STANDARD_IMAGE_TAG}`);
|
|
1508
|
+
imageReady = true;
|
|
1509
|
+
buildBackoff.clear();
|
|
1510
|
+
} else {
|
|
1511
|
+
buildError =
|
|
1512
|
+
build.stderr.trim() ||
|
|
1513
|
+
`docker build exited ${build.code ?? "spawn error"}`;
|
|
1514
|
+
if (!deferred && !engine.apple) buildBackoff.failed(buildKey, buildError);
|
|
1515
|
+
console.warn(
|
|
1516
|
+
deferred ? `[host-agent] ${buildError}` :
|
|
1517
|
+
`[host-agent] standard image build failed (exit ${build.code ?? "spawn"}); ` +
|
|
1518
|
+
"continuing. Tasks needing the image will surface this error.\n" +
|
|
1519
|
+
buildError,
|
|
1520
|
+
);
|
|
1521
|
+
// A stale-but-working image is better than none.
|
|
1522
|
+
imageReady = labeledHash !== null;
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
// Revisit ids durably observed under the standard tag across restarts.
|
|
1527
|
+
// Inherited labels never authorize sweeping task/compose images.
|
|
1528
|
+
if (imageReady && !engine.apple) await imageCleanup.sweep(run, previousImageId);
|
|
1529
|
+
|
|
1530
|
+
// 3. Reconcile every Uai-owned writer under the volume-global flock. Lock
|
|
1531
|
+
// bootstrap/permission repair precedes the single upgrade→repair→proof
|
|
1532
|
+
// transaction so node-run task materializers can acquire legacy volumes.
|
|
1533
|
+
if (imageReady) {
|
|
1534
|
+
// Docker populates a fresh named volume from the image's directory
|
|
1535
|
+
// content on first mount; the Apple runtime mounts it empty (ADR-106,
|
|
1536
|
+
// found live 2026-08-15: every asdf plugin/shim the image bakes into
|
|
1537
|
+
// /opt/asdf-data was missing, so CLI reconcile looped on "No such
|
|
1538
|
+
// plugin: nodejs"). Replicate the Docker semantics explicitly: when the
|
|
1539
|
+
// volume lacks the plugins tree, copy the image's /opt/asdf-data into it
|
|
1540
|
+
// once, ownership preserved. Idempotent — the copied tree is the marker.
|
|
1541
|
+
if (engine.apple) {
|
|
1542
|
+
// Probe at a path the image does NOT contain: mounted over
|
|
1543
|
+
// /opt/asdf-data, a silently-failed volume mount would expose the
|
|
1544
|
+
// image's own baked tree and fake a "seeded" answer. At a neutral
|
|
1545
|
+
// mountpoint the plugins tree can only come from the volume itself,
|
|
1546
|
+
// and any mount failure lands on the safe side (reseeding is
|
|
1547
|
+
// idempotent).
|
|
1548
|
+
const seeded = await runMaintenanceContainer(engine, "seed-probe", [
|
|
1549
|
+
"--rm",
|
|
1550
|
+
"-v",
|
|
1551
|
+
`${ASDF_DATA_VOLUME}:/uai-volume-probe:ro`,
|
|
1552
|
+
STANDARD_IMAGE_TAG,
|
|
1553
|
+
"/usr/bin/test",
|
|
1554
|
+
"-d",
|
|
1555
|
+
"/uai-volume-probe/plugins",
|
|
1556
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
1557
|
+
// Only `/usr/bin/test`'s own exit 1 means "not seeded". Everything else
|
|
1558
|
+
// — a host-side deadline kill (null), an in-container SIGTERM/SIGKILL
|
|
1559
|
+
// (143/137: live 2026-08-17, a wedged apiserver let the probe run ten
|
|
1560
|
+
// minutes before dying 143), or any CLI failure — answers neither way,
|
|
1561
|
+
// and must never trigger a reseed of a volume that may be full and in
|
|
1562
|
+
// use.
|
|
1563
|
+
if (seeded.code !== 0 && seeded.code !== 1) {
|
|
1564
|
+
return {
|
|
1565
|
+
ok: false,
|
|
1566
|
+
error:
|
|
1567
|
+
"The container runtime did not answer while checking the shared " +
|
|
1568
|
+
`runtime volume (probe exit ${seeded.code ?? "killed"}).`,
|
|
1569
|
+
};
|
|
1570
|
+
}
|
|
1571
|
+
if (seeded.code === 0) {
|
|
1572
|
+
console.log(
|
|
1573
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is seeded`,
|
|
1574
|
+
);
|
|
1575
|
+
}
|
|
1576
|
+
// The positive no-holder proof comes BEFORE every writable attachment
|
|
1577
|
+
// — including the seeder. A doomed writable attach both fails the
|
|
1578
|
+
// operation and can mint the corpse that wedges the apiserver, so a
|
|
1579
|
+
// writer requires a positive `free`; `unknown` (unreadable inventory)
|
|
1580
|
+
// must never read as permission.
|
|
1581
|
+
const holders = await sharedVolumeHolders(engine);
|
|
1582
|
+
if (seeded.code !== 0) {
|
|
1583
|
+
if (holders !== "free") {
|
|
1584
|
+
return {
|
|
1585
|
+
ok: false,
|
|
1586
|
+
error:
|
|
1587
|
+
holders === "held"
|
|
1588
|
+
? `The shared runtime volume ${ASDF_DATA_VOLUME} is unseeded while task containers are running; it cannot be seeded safely until they stop.`
|
|
1589
|
+
: "Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
|
|
1590
|
+
};
|
|
1591
|
+
}
|
|
1592
|
+
console.log(
|
|
1593
|
+
`[host-agent] seeding ${ASDF_DATA_VOLUME} from the standard image (probe exit ${seeded.code}; Apple volumes mount empty)`,
|
|
1594
|
+
);
|
|
1595
|
+
const seed = await runMaintenanceContainer(engine, "seed", [
|
|
1596
|
+
"--rm",
|
|
1597
|
+
"--user",
|
|
1598
|
+
"root",
|
|
1599
|
+
"-v",
|
|
1600
|
+
`${ASDF_DATA_VOLUME}:/uai-seed-target`,
|
|
1601
|
+
STANDARD_IMAGE_TAG,
|
|
1602
|
+
"/bin/sh",
|
|
1603
|
+
"-c",
|
|
1604
|
+
"cp -a /opt/asdf-data/. /uai-seed-target/ && chown -R node:node /uai-seed-target",
|
|
1605
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
1606
|
+
if (seed.code !== 0) {
|
|
1607
|
+
return {
|
|
1608
|
+
ok: false,
|
|
1609
|
+
error: `Could not seed the shared runtime volume from the standard image (exit ${seed.code ?? "spawn"}).`,
|
|
1610
|
+
};
|
|
1611
|
+
}
|
|
1612
|
+
} else if (holders === "held") {
|
|
1613
|
+
// Proof-first deferral: while running task containers hold the
|
|
1614
|
+
// volume, a writable attach can only fail (and mint a corpse) —
|
|
1615
|
+
// don't attempt it. The stderr-signature deferrals below remain as
|
|
1616
|
+
// the backstop for a task appearing between this check and the
|
|
1617
|
+
// attach.
|
|
1618
|
+
if (await volumeReconcileProofExists()) {
|
|
1619
|
+
console.log(
|
|
1620
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by task containers; deferring maintenance (prior reconcile proof present)`,
|
|
1621
|
+
);
|
|
1622
|
+
return { ok: true };
|
|
1623
|
+
}
|
|
1624
|
+
// First-proof bootstrap. A host with no persisted proof (fresh file,
|
|
1625
|
+
// or upgraded from a version whose proof write failed silently —
|
|
1626
|
+
// live 2026-08-17 on 0.9.26) would otherwise be stuck in a ring: the
|
|
1627
|
+
// proof needs a full reconcile, the reconcile needs the writable
|
|
1628
|
+
// volume, and the volume stays held for as long as tasks run. The
|
|
1629
|
+
// read-only CLI probe attaches fine next to tasks, and EXECUTING
|
|
1630
|
+
// every agent CLI is the same positive evidence the full reconcile
|
|
1631
|
+
// ends with — enough to defer on, and worth persisting as the proof.
|
|
1632
|
+
const missing = await probeVolumeAgentClis(activation);
|
|
1633
|
+
if (missing !== null && missing.length === 0) {
|
|
1634
|
+
await writeVolumeReconcileProof();
|
|
1635
|
+
console.log(
|
|
1636
|
+
`[host-agent] read-only agent-CLI probe passed while task containers hold ${ASDF_DATA_VOLUME}; recording the reconcile proof and deferring maintenance`,
|
|
1637
|
+
);
|
|
1638
|
+
return { ok: true };
|
|
1639
|
+
}
|
|
1640
|
+
// No proof and a failed probe, while tasks hold the volume: a
|
|
1641
|
+
// writable attach here can only fail and mint a corpse, and simply
|
|
1642
|
+
// retrying strands activation forever (review 2026-08-18 round 2 —
|
|
1643
|
+
// recovery, which could settle the holders, runs only after
|
|
1644
|
+
// maintenance succeeds). Ask the activation driver to lifecycle-stop
|
|
1645
|
+
// the holder tasks (resumable) so the next attempt can repair the
|
|
1646
|
+
// volume they cannot safely use anyway.
|
|
1647
|
+
return {
|
|
1648
|
+
ok: false,
|
|
1649
|
+
error:
|
|
1650
|
+
`The shared runtime volume ${ASDF_DATA_VOLUME} needs repair ` +
|
|
1651
|
+
`(agent CLIs failed their probe${missing === null ? "" : `: ${missing.join(", ")} missing`}) ` +
|
|
1652
|
+
"while running tasks hold it; stopping the holder tasks so maintenance can repair it.",
|
|
1653
|
+
remedy: "stop-volume-holders",
|
|
1654
|
+
};
|
|
1655
|
+
} else if (holders === "unknown") {
|
|
1656
|
+
return {
|
|
1657
|
+
ok: false,
|
|
1658
|
+
error:
|
|
1659
|
+
"Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
|
|
1660
|
+
};
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
const lockPrepared = await prepareAsdfMaintenanceLock(activation);
|
|
1664
|
+
if (!lockPrepared) {
|
|
1665
|
+
if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
|
|
1666
|
+
console.log(
|
|
1667
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
|
|
1668
|
+
);
|
|
1669
|
+
return { ok: true };
|
|
1670
|
+
}
|
|
1671
|
+
return {
|
|
1672
|
+
ok: false,
|
|
1673
|
+
error: `${buildError ? `${buildError} ` : ""}Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
|
|
1674
|
+
};
|
|
1675
|
+
}
|
|
1676
|
+
if (!(await reconcileVolumeAgentClis(activation))) {
|
|
1677
|
+
if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
|
|
1678
|
+
console.log(
|
|
1679
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
|
|
1680
|
+
);
|
|
1681
|
+
return { ok: true };
|
|
1682
|
+
}
|
|
1683
|
+
return {
|
|
1684
|
+
ok: false,
|
|
1685
|
+
error:
|
|
1686
|
+
`${buildError ? `${buildError} ` : ""}Shared runtime maintenance failed; agent CLIs or Corepack did not pass execution proof.`,
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
return { ok: true };
|
|
1690
|
+
}
|
|
1691
|
+
return {
|
|
1692
|
+
ok: false,
|
|
1693
|
+
error:
|
|
1694
|
+
buildError ??
|
|
1695
|
+
"the standard base image (moondeck-standard:dev) is not built and no build error was captured.",
|
|
1696
|
+
};
|
|
1697
|
+
}
|