@runuai/host 0.9.14 → 0.9.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +80 -30
- package/images/standard/Dockerfile +36 -10
- package/images/standard/README.md +63 -18
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +308 -38
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/lib/agent-cli.ts +33 -2
- package/lib/agent.ts +46 -7
- package/lib/agents/claude.ts +13 -8
- package/lib/agents/codex.ts +11 -6
- package/lib/agents/cursor.ts +39 -29
- package/lib/agents/durable-proc.ts +20 -27
- package/lib/agents/factory.ts +9 -25
- package/lib/agents/grok.ts +43 -30
- package/lib/agents/kimi.ts +44 -29
- package/lib/agents/opencode.ts +43 -31
- package/lib/agents/proc.ts +149 -114
- package/lib/agents/transport.ts +62 -50
- package/lib/agents/types.ts +6 -4
- package/lib/apple-runtime-recycle.ts +236 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/browser-testing.ts +233 -93
- package/lib/codex-auth.ts +40 -6
- package/lib/command-db.ts +20 -0
- package/lib/container-runtime.ts +1338 -0
- package/lib/db.ts +1 -0
- package/lib/docker-exec.ts +87 -5
- package/lib/engine-accounts.ts +68 -5
- package/lib/engine-login.ts +1952 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +155 -0
- package/lib/env.ts +4 -0
- package/lib/git-diff.ts +98 -32
- package/lib/git-identity.ts +199 -87
- package/lib/github-tokens.ts +202 -91
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-logs.ts +962 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +49 -0
- package/lib/managed-runtime.ts +3644 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-gateway.ts +450 -23
- package/lib/orchestrator.ts +3051 -200
- package/lib/preview-sidecar.ts +68 -14
- package/lib/release-manifest.ts +708 -0
- package/lib/release-trust.ts +28 -0
- package/lib/runtime-activation-tail.ts +232 -0
- package/lib/runtime-archive.ts +1086 -0
- package/lib/runtime-authority.ts +79 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-state.ts +232 -12
- package/lib/skills.ts +24 -3
- package/lib/ssh.ts +18 -0
- package/lib/standard-image.ts +1104 -141
- package/lib/stopped-task-status-queue.ts +44 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-diff.ts +66 -46
- package/lib/task-environment/apple-container.ts +757 -0
- package/lib/task-environment/docker.ts +956 -0
- package/lib/task-environment/index.ts +364 -0
- package/lib/task-environment/legacy-adoption.ts +459 -0
- package/lib/task-environment/registry.ts +58 -0
- package/lib/task-environment/types.ts +408 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +585 -0
- package/lib/tunnel-registry.ts +135 -19
- package/lib/tunnel-runtime.ts +235 -0
- package/package.json +1 -1
- package/scripts/agent/_common.sh +123 -3
- package/scripts/agent/task-down.sh +146 -38
- package/scripts/agent/task-status.sh +19 -3
- package/scripts/agent/task-up.sh +1405 -107
- package/scripts/install/darwin.ts +848 -50
- package/scripts/install/linux.ts +838 -35
- package/scripts/install/types.ts +43 -0
- package/scripts/install/util.ts +215 -8
- package/scripts/install/win.ts +12 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/cli.ts +1464 -72
- package/src/event-outbox.ts +83 -4
- package/src/index.ts +766 -42
- package/src/main.ts +1398 -255
- package/src/paths.ts +17 -1
- package/src/protocol.ts +695 -1
- package/src/runtime-bootstrap.ts +165 -0
- package/src/ui/server.ts +46 -10
- package/src/ui/types.ts +37 -0
package/lib/standard-image.ts
CHANGED
|
@@ -18,16 +18,38 @@
|
|
|
18
18
|
|
|
19
19
|
import { spawn } from "node:child_process";
|
|
20
20
|
import { createHash } from "node:crypto";
|
|
21
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
21
22
|
import { readdir, readFile } from "node:fs/promises";
|
|
22
23
|
import { dirname, join, resolve } from "node:path";
|
|
23
24
|
import { fileURLToPath } from "node:url";
|
|
24
25
|
|
|
26
|
+
import {
|
|
27
|
+
MAINTENANCE_CONTAINER_LABEL,
|
|
28
|
+
recycleAppleRuntimeServices,
|
|
29
|
+
} from "./apple-runtime-recycle";
|
|
25
30
|
import { detectEngine } from "./engines";
|
|
31
|
+
import {
|
|
32
|
+
appleContainerRuntimeBinding,
|
|
33
|
+
claimContainerRuntimeForMaintenance,
|
|
34
|
+
containerRuntimeStandardAssetsRoot,
|
|
35
|
+
containerRuntimeStandardImageRoot,
|
|
36
|
+
pinnedContainerRuntimeProvider,
|
|
37
|
+
} from "./container-runtime";
|
|
38
|
+
import { KeyedPromiseTail } from "./keyed-promise-tail";
|
|
26
39
|
|
|
27
40
|
/** Pinned, host-wide constants (must match task-up.sh and the compose gen). */
|
|
28
41
|
export const STANDARD_IMAGE_TAG = "uai-standard:dev";
|
|
29
42
|
export const ASDF_DATA_VOLUME = "uai-asdf-data";
|
|
30
43
|
|
|
44
|
+
/**
|
|
45
|
+
* The shared-volume agent CLIs belong to this exact HOME-default Node. A
|
|
46
|
+
* future default bump cannot transparently roll stopped containers: their
|
|
47
|
+
* baked /home/node/.tool-versions still names the old CLI-owning candidate.
|
|
48
|
+
* Changing this value therefore requires an explicit volume migration or
|
|
49
|
+
* container-recreation release slice, not an ordinary image refresh.
|
|
50
|
+
*/
|
|
51
|
+
export const STANDARD_DEFAULT_NODE_VERSION = "22.11.0";
|
|
52
|
+
|
|
31
53
|
/**
|
|
32
54
|
* Runtimes the standard image's asdf plugins can satisfy, with a sensible set
|
|
33
55
|
* of advertised versions per kind (ADR-021 `runtimes`). These are the
|
|
@@ -43,7 +65,14 @@ export const STANDARD_RUNTIMES: ReadonlyArray<{
|
|
|
43
65
|
kind: string;
|
|
44
66
|
availableVersions: string[];
|
|
45
67
|
}> = [
|
|
46
|
-
{
|
|
68
|
+
{
|
|
69
|
+
kind: "node",
|
|
70
|
+
availableVersions: [
|
|
71
|
+
STANDARD_DEFAULT_NODE_VERSION,
|
|
72
|
+
"20.18.0",
|
|
73
|
+
"18.20.4",
|
|
74
|
+
],
|
|
75
|
+
},
|
|
47
76
|
{ kind: "python", availableVersions: ["3.12.4", "3.11.9"] },
|
|
48
77
|
{ kind: "go", availableVersions: ["1.23.2", "1.22.8"] },
|
|
49
78
|
{ kind: "ruby", availableVersions: ["3.3.5", "3.2.5"] },
|
|
@@ -91,6 +120,16 @@ export function configuredOptionalEngines(): {
|
|
|
91
120
|
};
|
|
92
121
|
}
|
|
93
122
|
|
|
123
|
+
type OptionalEngineConfiguration = ReturnType<
|
|
124
|
+
typeof configuredOptionalEngines
|
|
125
|
+
>;
|
|
126
|
+
|
|
127
|
+
function optionalEngineConfigurationKey(
|
|
128
|
+
engines: OptionalEngineConfiguration,
|
|
129
|
+
): string {
|
|
130
|
+
return JSON.stringify(engines);
|
|
131
|
+
}
|
|
132
|
+
|
|
94
133
|
async function hashBuildContext(extra = ""): Promise<string | null> {
|
|
95
134
|
try {
|
|
96
135
|
const root = standardImageDir();
|
|
@@ -123,23 +162,103 @@ async function hashBuildContext(extra = ""): Promise<string | null> {
|
|
|
123
162
|
|
|
124
163
|
/** Absolute path to the standard image build context. */
|
|
125
164
|
function standardImageDir(): string {
|
|
165
|
+
const providerRoot = containerRuntimeStandardImageRoot();
|
|
166
|
+
if (providerRoot !== null) return providerRoot;
|
|
126
167
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
127
168
|
// lib/standard-image.ts -> host-agent/ -> images/standard
|
|
128
169
|
return resolve(here, "..", "images", "standard");
|
|
129
170
|
}
|
|
130
171
|
|
|
172
|
+
/** Absolute packaged path used to bind the materializer over rolling images
|
|
173
|
+
* that predate the baked executable. Resolution is deterministic; callers get
|
|
174
|
+
* the ordinary Docker bind/open error if a package install is corrupt. */
|
|
175
|
+
export function standardImageRuntimeMaterializerPath(): string {
|
|
176
|
+
return join(
|
|
177
|
+
standardContainerAssetsDir(),
|
|
178
|
+
"uai-materialize-runtimes",
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Absolute packaged Corepack pin bound over both rolling and derived images. */
|
|
183
|
+
export function standardImageCorepackVersionPath(): string {
|
|
184
|
+
return join(standardContainerAssetsDir(), "corepack-version");
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** ADR-106: the retired bundled-VM fallback exposed a provider-specific
|
|
188
|
+
* asset root; every current backend uses the package's ordinary
|
|
189
|
+
* build-context assets, so the runtime override below is always null. */
|
|
190
|
+
function standardContainerAssetsDir(): string {
|
|
191
|
+
return (
|
|
192
|
+
containerRuntimeStandardAssetsRoot() ?? join(standardImageDir(), "container")
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
131
196
|
interface RunResult {
|
|
132
197
|
code: number | null;
|
|
133
198
|
stdout: string;
|
|
134
199
|
stderr: string;
|
|
135
200
|
}
|
|
136
201
|
|
|
137
|
-
|
|
138
|
-
|
|
202
|
+
const SHORT_DOCKER_TIMEOUT_MS = 30_000;
|
|
203
|
+
const CLI_MAINTENANCE_TIMEOUT_MS = 10 * 60_000;
|
|
204
|
+
const IMAGE_BUILD_TIMEOUT_MS = 15 * 60_000;
|
|
205
|
+
const TERMINATE_GRACE_MS = 5_000;
|
|
206
|
+
export const ASDF_MAINTENANCE_LOCK_PATH =
|
|
207
|
+
"/opt/asdf-data/.uai-maintenance.lock";
|
|
208
|
+
const ASDF_MAINTENANCE_LOCK_WAIT_SECONDS = 300;
|
|
209
|
+
const ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS = 570;
|
|
210
|
+
|
|
211
|
+
/** The selected container engine for image/volume maintenance (ADR-106).
|
|
212
|
+
* Resolved per call: the runtime pin settles before ensureStandardImage runs,
|
|
213
|
+
* but an HMR reload must not strand a stale CLI path. The docker fallback
|
|
214
|
+
* covers the pre-selection window, where docker-only behavior is historical. */
|
|
215
|
+
function selectedEngine(): { command: string; apple: boolean } {
|
|
216
|
+
// The PIN, not the readiness-gated identity: this runs during activation
|
|
217
|
+
// while the public state is still `checking`, and falling back to docker
|
|
218
|
+
// there deadlocks a Docker-free Mac.
|
|
219
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
220
|
+
const binding = appleContainerRuntimeBinding();
|
|
221
|
+
if (binding !== null) {
|
|
222
|
+
return { command: binding.containerCliPath, apple: true };
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return { command: "docker", apple: false };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Run a command to completion, capturing stdio. Never rejects and never
|
|
229
|
+
* waits forever: standard-image maintenance is an activation prerequisite,
|
|
230
|
+
* so an unbounded Docker/npm child would strand the entire host in checking. */
|
|
231
|
+
function run(
|
|
232
|
+
command: string,
|
|
233
|
+
args: string[],
|
|
234
|
+
timeoutMs = SHORT_DOCKER_TIMEOUT_MS,
|
|
235
|
+
): Promise<RunResult> {
|
|
139
236
|
return new Promise<RunResult>((resolveRun) => {
|
|
140
237
|
let stdout = "";
|
|
141
238
|
let stderr = "";
|
|
239
|
+
let settled = false;
|
|
240
|
+
let killTimer: NodeJS.Timeout | null = null;
|
|
142
241
|
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
242
|
+
const settle = (code: number | null): void => {
|
|
243
|
+
if (settled) return;
|
|
244
|
+
settled = true;
|
|
245
|
+
clearTimeout(timeoutTimer);
|
|
246
|
+
if (killTimer) clearTimeout(killTimer);
|
|
247
|
+
resolveRun({ code, stdout, stderr });
|
|
248
|
+
};
|
|
249
|
+
const timeoutTimer = setTimeout(() => {
|
|
250
|
+
stderr += `${stderr && !stderr.endsWith("\n") ? "\n" : ""}${command} timed out after ${timeoutMs}ms`;
|
|
251
|
+
child.kill("SIGTERM");
|
|
252
|
+
killTimer = setTimeout(() => {
|
|
253
|
+
child.kill("SIGKILL");
|
|
254
|
+
// SIGKILL should always produce close, but do not let a broken child
|
|
255
|
+
// handle defeat the host-wide readiness bound.
|
|
256
|
+
const settleTimer = setTimeout(() => settle(null), 1_000);
|
|
257
|
+
settleTimer.unref();
|
|
258
|
+
}, TERMINATE_GRACE_MS);
|
|
259
|
+
killTimer.unref();
|
|
260
|
+
}, timeoutMs);
|
|
261
|
+
timeoutTimer.unref();
|
|
143
262
|
child.stdout?.setEncoding("utf8");
|
|
144
263
|
child.stderr?.setEncoding("utf8");
|
|
145
264
|
child.stdout?.on("data", (chunk: string) => {
|
|
@@ -149,14 +268,220 @@ function run(command: string, args: string[]): Promise<RunResult> {
|
|
|
149
268
|
stderr += chunk;
|
|
150
269
|
});
|
|
151
270
|
child.on("error", (err: Error) => {
|
|
152
|
-
|
|
271
|
+
stderr += err.message;
|
|
272
|
+
settle(null);
|
|
153
273
|
});
|
|
154
274
|
child.on("close", (code) => {
|
|
155
|
-
|
|
275
|
+
settle(code);
|
|
156
276
|
});
|
|
157
277
|
});
|
|
158
278
|
}
|
|
159
279
|
|
|
280
|
+
/** Narrow process-level seam for the timeout regression test. */
|
|
281
|
+
export const runStandardImageCommandForTest = run;
|
|
282
|
+
|
|
283
|
+
let maintenanceContainerSeq = 0;
|
|
284
|
+
|
|
285
|
+
/** How long a maintenance run may sit without its container reaching
|
|
286
|
+
* `running` before the wedge sensor passes judgment. Long HEALTHY operations
|
|
287
|
+
* (reconcile's npm installs, the seed copy) are past `running` within
|
|
288
|
+
* seconds, so this discriminates cleanly. Env-overridable for tests. */
|
|
289
|
+
function wedgeProbeDelayMs(): number {
|
|
290
|
+
const raw = Number(process.env.UAI_APPLE_WEDGE_PROBE_MS);
|
|
291
|
+
return Number.isFinite(raw) && raw > 0 ? raw : 60_000;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Post-recycle grace for the hung CLI client before a synthesized result. */
|
|
295
|
+
function wedgeGraceMs(): number {
|
|
296
|
+
const raw = Number(process.env.UAI_APPLE_WEDGE_GRACE_MS);
|
|
297
|
+
return Number.isFinite(raw) && raw > 0 ? raw : 10_000;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function delayMs(ms: number): Promise<void> {
|
|
301
|
+
return new Promise((resolveDelay) => {
|
|
302
|
+
const timer = setTimeout(resolveDelay, ms);
|
|
303
|
+
timer.unref();
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* One DEFINITIVE observation of the named container: a successful, parseable
|
|
309
|
+
* inspect reporting a non-running state. Anything else — inspect failure,
|
|
310
|
+
* unparseable output, missing state, or `running` — is NOT evidence of a
|
|
311
|
+
* wedge and must stand the sensor down (the old fall-through treated
|
|
312
|
+
* uncertainty as wedge evidence, which authorizes a service recycle on a
|
|
313
|
+
* guess).
|
|
314
|
+
*/
|
|
315
|
+
async function appleContainerDefinitivelyNotRunning(
|
|
316
|
+
engine: { command: string },
|
|
317
|
+
name: string,
|
|
318
|
+
): Promise<boolean> {
|
|
319
|
+
const inspected = await run(engine.command, ["inspect", name], 15_000);
|
|
320
|
+
if (inspected.code !== 0) return false;
|
|
321
|
+
try {
|
|
322
|
+
const parsed = JSON.parse(inspected.stdout) as Array<{
|
|
323
|
+
id?: unknown;
|
|
324
|
+
status?: { state?: unknown };
|
|
325
|
+
}>;
|
|
326
|
+
if (!Array.isArray(parsed) || parsed.length !== 1) return false;
|
|
327
|
+
if (parsed[0]?.id !== name) return false;
|
|
328
|
+
const state = parsed[0]?.status?.state;
|
|
329
|
+
return typeof state === "string" && state !== "running";
|
|
330
|
+
} catch {
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* The wedge signature, judged live: the named container has NOT reached
|
|
337
|
+
* `running` (definitively — see above) while the control plane still answers
|
|
338
|
+
* a `list`, observed STABLY: a second exact inspection after the list (and a
|
|
339
|
+
* short settle) must repeat the non-running verdict immediately before any
|
|
340
|
+
* recycle is authorized, closing the window where the container starts while
|
|
341
|
+
* the list is in flight. A dead apiserver fails the `list` too (that is an
|
|
342
|
+
* outage, not a wedge, and recycling on it would be a guess).
|
|
343
|
+
*/
|
|
344
|
+
async function appleWedgeVerdict(
|
|
345
|
+
engine: { command: string },
|
|
346
|
+
name: string,
|
|
347
|
+
): Promise<boolean> {
|
|
348
|
+
if (!(await appleContainerDefinitivelyNotRunning(engine, name))) {
|
|
349
|
+
return false;
|
|
350
|
+
}
|
|
351
|
+
const listed = await run(engine.command, ["list", "--format", "json"], 15_000);
|
|
352
|
+
if (listed.code !== 0) return false;
|
|
353
|
+
await delayMs(Math.min(5_000, wedgeProbeDelayMs()));
|
|
354
|
+
return appleContainerDefinitivelyNotRunning(engine, name);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Run a maintenance container with corpse hygiene on the Apple runtime.
|
|
359
|
+
*
|
|
360
|
+
* `container run --rm` only removes a container that STARTED. A run refused
|
|
361
|
+
* at start time (the exclusive-volume attach, most commonly) leaves a
|
|
362
|
+
* created-never-started container behind, and those corpses wedge the
|
|
363
|
+
* runtime's apiserver (observed live 2026-08-16, twice: run/delete hang
|
|
364
|
+
* while list still answers). Naming the container makes the corpse
|
|
365
|
+
* addressable, so a failed run is followed by a bounded best-effort
|
|
366
|
+
* `delete --force` of that exact name. Docker's `--rm` is reliable and its
|
|
367
|
+
* containers are not corpse-prone, so the Docker path passes through.
|
|
368
|
+
*
|
|
369
|
+
* Every apple run doubles as the WEDGE SENSOR (the owner's "this needs to
|
|
370
|
+
* be automatic"): if the container has not reached `running` after the
|
|
371
|
+
* probe delay while `list` still answers, the runtime services are recycled
|
|
372
|
+
* in place (see apple-runtime-recycle.ts) and the run reports a distinctive
|
|
373
|
+
* retryable failure — the activation retry loop then re-runs maintenance
|
|
374
|
+
* against the fresh runtime.
|
|
375
|
+
*/
|
|
376
|
+
async function runMaintenanceContainer(
|
|
377
|
+
engine: { command: string; apple: boolean },
|
|
378
|
+
purpose: string,
|
|
379
|
+
argsAfterRun: string[],
|
|
380
|
+
timeoutMs: number,
|
|
381
|
+
): Promise<RunResult> {
|
|
382
|
+
if (!engine.apple) {
|
|
383
|
+
return run(engine.command, ["run", ...argsAfterRun], timeoutMs);
|
|
384
|
+
}
|
|
385
|
+
maintenanceContainerSeq += 1;
|
|
386
|
+
const name = `uai-maint-${purpose}-${process.pid}-${maintenanceContainerSeq}`;
|
|
387
|
+
let runSettled = false;
|
|
388
|
+
const running = run(
|
|
389
|
+
engine.command,
|
|
390
|
+
// The ownership label is what authorizes the recycle sweep (and only the
|
|
391
|
+
// sweep) to delete this container later — a name alone is forgeable.
|
|
392
|
+
["run", "--name", name, "-l", `${MAINTENANCE_CONTAINER_LABEL}=1`, ...argsAfterRun],
|
|
393
|
+
timeoutMs,
|
|
394
|
+
).then((outcome) => {
|
|
395
|
+
runSettled = true;
|
|
396
|
+
return outcome;
|
|
397
|
+
});
|
|
398
|
+
const sensed = (async (): Promise<RunResult | null> => {
|
|
399
|
+
await delayMs(wedgeProbeDelayMs());
|
|
400
|
+
if (runSettled) return null;
|
|
401
|
+
if (!(await appleWedgeVerdict(engine, name))) return null;
|
|
402
|
+
if (runSettled) return null;
|
|
403
|
+
const outcome = await recycleAppleRuntimeServices({
|
|
404
|
+
cli: (args, cliTimeoutMs) => run(engine.command, args, cliTimeoutMs),
|
|
405
|
+
});
|
|
406
|
+
// The hung CLI client may exit once its apiserver dies; give it a short
|
|
407
|
+
// grace so a real result wins, then synthesize the retryable failure.
|
|
408
|
+
const graced = await Promise.race([
|
|
409
|
+
running,
|
|
410
|
+
delayMs(wedgeGraceMs()).then(() => null),
|
|
411
|
+
]);
|
|
412
|
+
if (graced) return graced;
|
|
413
|
+
return {
|
|
414
|
+
code: null,
|
|
415
|
+
stdout: "",
|
|
416
|
+
stderr: outcome.recycled
|
|
417
|
+
? "apple container runtime was wedged; runtime services were recycled — retry this operation"
|
|
418
|
+
: `apple container runtime is wedged and the automatic recycle did not recover it: ${outcome.reason}`,
|
|
419
|
+
};
|
|
420
|
+
})();
|
|
421
|
+
const result = await Promise.race([
|
|
422
|
+
running,
|
|
423
|
+
sensed.then((synthesized) => synthesized ?? running),
|
|
424
|
+
]);
|
|
425
|
+
if (result.code !== 0) {
|
|
426
|
+
// The run may have failed BECAUSE this generated name already existed —
|
|
427
|
+
// then the container is not ours and the ownership label was never
|
|
428
|
+
// applied (review 2026-08-18 round 3). Deletion requires a positive
|
|
429
|
+
// label proof; an absent container is done; anything uncertain is left
|
|
430
|
+
// untouched and reported.
|
|
431
|
+
const inspected = await run(engine.command, ["inspect", name], 15_000);
|
|
432
|
+
let verdict: "ours" | "absent" | "foreign" | "unknown" = "unknown";
|
|
433
|
+
if (
|
|
434
|
+
inspected.code !== 0 &&
|
|
435
|
+
inspected.stdout.trim() === "" &&
|
|
436
|
+
inspected.stderr
|
|
437
|
+
.split("\n")
|
|
438
|
+
.some((line) => line.trim() === `Error: container not found: ${name}`)
|
|
439
|
+
) {
|
|
440
|
+
verdict = "absent";
|
|
441
|
+
} else if (inspected.code === 0) {
|
|
442
|
+
try {
|
|
443
|
+
const parsed = JSON.parse(inspected.stdout) as Array<{
|
|
444
|
+
id?: unknown;
|
|
445
|
+
configuration?: { labels?: Record<string, unknown> };
|
|
446
|
+
}>;
|
|
447
|
+
if (Array.isArray(parsed) && parsed.length === 1 && parsed[0]?.id === name) {
|
|
448
|
+
verdict =
|
|
449
|
+
parsed[0]?.configuration?.labels?.[MAINTENANCE_CONTAINER_LABEL] === "1"
|
|
450
|
+
? "ours"
|
|
451
|
+
: "foreign";
|
|
452
|
+
}
|
|
453
|
+
} catch {
|
|
454
|
+
verdict = "unknown";
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
if (verdict === "ours") {
|
|
458
|
+
const removed = await run(
|
|
459
|
+
engine.command,
|
|
460
|
+
["delete", "--force", name],
|
|
461
|
+
30_000,
|
|
462
|
+
);
|
|
463
|
+
if (removed.code !== 0) {
|
|
464
|
+
console.warn(
|
|
465
|
+
`[host-agent] could not remove maintenance container ${name} after a ` +
|
|
466
|
+
`failed run (exit ${removed.code ?? "spawn"}); a created-never-started ` +
|
|
467
|
+
`container can wedge the Apple runtime. ${removed.stderr.trim().slice(0, 200)}`,
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
} else if (verdict === "foreign") {
|
|
471
|
+
console.warn(
|
|
472
|
+
`[host-agent] a container named ${name} exists WITHOUT Uai's maintenance ` +
|
|
473
|
+
"ownership label — the failed run likely collided with it; leaving it untouched",
|
|
474
|
+
);
|
|
475
|
+
} else if (verdict === "unknown") {
|
|
476
|
+
console.warn(
|
|
477
|
+
`[host-agent] could not prove ownership of maintenance container ${name} ` +
|
|
478
|
+
`after a failed run; leaving it untouched (inspect exit ${inspected.code ?? "spawn"})`,
|
|
479
|
+
);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
return result;
|
|
483
|
+
}
|
|
484
|
+
|
|
160
485
|
/**
|
|
161
486
|
* Agent CLIs that MUST exist inside the shared asdf volume. The volume mounts
|
|
162
487
|
* over /opt/asdf-data and shadows the image's baked-in shims, so a volume
|
|
@@ -170,13 +495,11 @@ const VOLUME_AGENT_CLIS: { bin: string; pkg: string }[] = [
|
|
|
170
495
|
];
|
|
171
496
|
|
|
172
497
|
/**
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
* A fresh/empty volume is seeded from the image by Docker on first mount, so
|
|
177
|
-
* this is a no-op there — it only fixes pre-existing stale volumes.
|
|
498
|
+
* Cheap, non-mutating diagnostic probe. This is never the authoritative final
|
|
499
|
+
* proof: another process can begin maintenance after a read-only check. The
|
|
500
|
+
* locked reconcile below always performs its own proof before releasing.
|
|
178
501
|
*/
|
|
179
|
-
async function
|
|
502
|
+
async function probeVolumeAgentClis(): Promise<string[] | null> {
|
|
180
503
|
const bins = VOLUME_AGENT_CLIS.map((c) => c.bin).join(" ");
|
|
181
504
|
// Probe by EXECUTING each bin (`--version`), not `command -v`. A stale asdf
|
|
182
505
|
// shim — the shim file survives in the volume but the reshim DB no longer
|
|
@@ -189,154 +512,534 @@ async function ensureVolumeAgentClis(): Promise<void> {
|
|
|
189
512
|
// version from /home/node/.tool-versions — the same one the repair installs
|
|
190
513
|
// against — and the sentinel prefix keeps login-shell noise from being
|
|
191
514
|
// misread as a missing bin.
|
|
192
|
-
const check = await
|
|
193
|
-
"run",
|
|
515
|
+
const check = await runMaintenanceContainer(selectedEngine(), "cli-probe", [
|
|
194
516
|
"--rm",
|
|
195
517
|
"-w",
|
|
196
518
|
"/home/node",
|
|
197
519
|
"-v",
|
|
198
|
-
`${ASDF_DATA_VOLUME}:/opt/asdf-data`,
|
|
520
|
+
`${ASDF_DATA_VOLUME}:/opt/asdf-data:ro`,
|
|
199
521
|
STANDARD_IMAGE_TAG,
|
|
200
|
-
"bash",
|
|
201
|
-
"-
|
|
522
|
+
"/bin/bash",
|
|
523
|
+
"-c",
|
|
202
524
|
`for b in ${bins}; do "$b" --version >/dev/null 2>&1 || echo "UAI_MISSING:$b"; done`,
|
|
203
|
-
]);
|
|
525
|
+
], SHORT_DOCKER_TIMEOUT_MS);
|
|
204
526
|
if (check.code !== 0) {
|
|
205
527
|
console.warn(
|
|
206
528
|
`[host-agent] could not check agent CLIs in ${ASDF_DATA_VOLUME} (exit ` +
|
|
207
529
|
`${check.code ?? "spawn"}); continuing. ${check.stderr.trim()}`,
|
|
208
530
|
);
|
|
209
|
-
return;
|
|
531
|
+
return null;
|
|
210
532
|
}
|
|
211
|
-
|
|
533
|
+
return check.stdout
|
|
212
534
|
.split("\n")
|
|
213
535
|
.map((l) => l.trim())
|
|
214
536
|
.filter((l) => l.startsWith("UAI_MISSING:"))
|
|
215
537
|
.map((l) => l.slice("UAI_MISSING:".length));
|
|
216
|
-
|
|
538
|
+
}
|
|
217
539
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
const
|
|
226
|
-
|
|
540
|
+
/**
|
|
541
|
+
* Create/repair the shared lock as a regular node-writable file. The lock is
|
|
542
|
+
* seeded into fresh volumes by the image. Legacy volumes can contain a 0644
|
|
543
|
+
* root-created file; opening it read-only permits flock, then metadata repair
|
|
544
|
+
* occurs while holding that same inode (never replace it under a waiter).
|
|
545
|
+
*/
|
|
546
|
+
async function prepareAsdfMaintenanceLock(): Promise<boolean> {
|
|
547
|
+
const command =
|
|
548
|
+
`set -eu; lock=${ASDF_MAINTENANCE_LOCK_PATH}; ` +
|
|
549
|
+
`if [ -L "$lock" ]; then echo "unsafe symlink lock: $lock" >&2; exit 1; fi; ` +
|
|
550
|
+
`if [ ! -e "$lock" ]; then umask 002; : > "$lock"; fi; ` +
|
|
551
|
+
`if [ ! -f "$lock" ]; then echo "lock is not regular: $lock" >&2; exit 1; fi; ` +
|
|
552
|
+
`exec 9<"$lock"; /usr/bin/flock --exclusive --timeout ${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS} 9; ` +
|
|
553
|
+
`/usr/bin/chown node:node "$lock"; /usr/bin/chmod 0664 "$lock"`;
|
|
554
|
+
const result = await runMaintenanceContainer(selectedEngine(), "lock", [
|
|
227
555
|
"--rm",
|
|
228
556
|
"-u",
|
|
229
557
|
"root",
|
|
230
|
-
// Run from /home/node so asdf resolves the node version from
|
|
231
|
-
// /home/node/.tool-versions. As root HOME is /root (no .tool-versions),
|
|
232
|
-
// and without a version asdf's npm shim aborts ("No version is set for
|
|
233
|
-
// command npm", exit 126). root can still write the volume from here.
|
|
234
|
-
"-w",
|
|
235
|
-
"/home/node",
|
|
236
558
|
"-v",
|
|
237
559
|
`${ASDF_DATA_VOLUME}:/opt/asdf-data`,
|
|
238
560
|
STANDARD_IMAGE_TAG,
|
|
239
|
-
"
|
|
240
|
-
"
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
console.log(
|
|
255
|
-
`[host-agent] ${ASDF_DATA_VOLUME} repaired — installed [${missing.join(", ")}]`,
|
|
256
|
-
);
|
|
257
|
-
} else {
|
|
258
|
-
console.warn(
|
|
259
|
-
`[host-agent] failed to repair ${ASDF_DATA_VOLUME} (exit ${repair.code ?? "spawn"}); ` +
|
|
260
|
-
`tasks may be missing [${missing.join(", ")}].\n${repair.stderr.trim()}`,
|
|
261
|
-
);
|
|
262
|
-
}
|
|
561
|
+
"/usr/bin/timeout",
|
|
562
|
+
"--signal=TERM",
|
|
563
|
+
"--kill-after=10s",
|
|
564
|
+
`${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
|
|
565
|
+
"/bin/bash",
|
|
566
|
+
"-c",
|
|
567
|
+
command,
|
|
568
|
+
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
569
|
+
if (result.code === 0) return true;
|
|
570
|
+
lastMaintenanceStderr = result.stderr;
|
|
571
|
+
console.warn(
|
|
572
|
+
`[host-agent] could not prepare ${ASDF_MAINTENANCE_LOCK_PATH} (exit ` +
|
|
573
|
+
`${result.code ?? "spawn"}): ${result.stderr.trim()}`,
|
|
574
|
+
);
|
|
575
|
+
return false;
|
|
263
576
|
}
|
|
264
577
|
|
|
265
578
|
/**
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
270
|
-
*
|
|
271
|
-
* `UAI_AGENT_CLI_AUTOUPDATE=0` + manual pin away from contained). Running
|
|
272
|
-
* sessions keep their already-loaded process; new spawns pick up the new bin.
|
|
273
|
-
* Best-effort: an offline registry just logs and keeps the current versions.
|
|
579
|
+
* One cross-process transaction owns upgrade, repair, Corepack repair, reshim,
|
|
580
|
+
* and the authoritative execution proof. A waiter always acquires the lock and
|
|
581
|
+
* performs its own full operation; a concurrent holder is never treated as
|
|
582
|
+
* success. The in-container timeout survives a disconnected Docker client and
|
|
583
|
+
* remains strictly inside the host-side child timeout.
|
|
274
584
|
*/
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
585
|
+
/** Build the single locked maintenance program. Exported only so focused
|
|
586
|
+
* tests can assert the crash-repair and proof contract without duplicating
|
|
587
|
+
* this security-sensitive shell in a fixture. */
|
|
588
|
+
export function standardImageVolumeMaintenanceCommandForTest(
|
|
589
|
+
autoUpdate: boolean,
|
|
590
|
+
): string {
|
|
280
591
|
const pkgs = VOLUME_AGENT_CLIS.map((c) => `${c.pkg}@latest`).join(" ");
|
|
281
|
-
const
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
592
|
+
const trustedNpmFlags =
|
|
593
|
+
"--registry=https://registry.npmjs.org/ --userconfig=/dev/null " +
|
|
594
|
+
"--globalconfig=/dev/uai-empty-npmrc";
|
|
595
|
+
return (
|
|
596
|
+
`set -u; upgrade=${autoUpdate ? 1 : 0}; ` +
|
|
597
|
+
`fail() { echo "UAI_STILL_BROKEN:$1" >&2; exit 1; }; ` +
|
|
598
|
+
`export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; ` +
|
|
599
|
+
`export HOME=/home/node ASDF_DIR=/opt/asdf ASDF_DATA_DIR=/opt/asdf-data ` +
|
|
600
|
+
`ASDF_CONFIG_FILE=/dev/null ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
|
|
601
|
+
`COREPACK_HOME=/opt/asdf-data/corepack COREPACK_DEFAULT_TO_LATEST=0 ` +
|
|
602
|
+
`COREPACK_ENABLE_AUTO_PIN=0 COREPACK_ENABLE_DOWNLOAD_PROMPT=0 ` +
|
|
603
|
+
`COREPACK_ENV_FILE=0 COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
|
|
604
|
+
`COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
|
|
605
|
+
`COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org; ` +
|
|
606
|
+
`default_node="$(/usr/bin/awk '$1 == "nodejs" { print $2; exit }' /home/node/.tool-versions)"; ` +
|
|
607
|
+
`if ! printf '%s' "$default_node" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$'; then ` +
|
|
608
|
+
`fail default-node; fi; ` +
|
|
609
|
+
`[ "$default_node" = "${STANDARD_DEFAULT_NODE_VERSION}" ] || ` +
|
|
610
|
+
`fail default-node-release-migration-required; ` +
|
|
611
|
+
`export ASDF_NODEJS_VERSION="$default_node"; ` +
|
|
612
|
+
`if [ -e /dev/uai-empty-npmrc ] || [ -L /dev/uai-empty-npmrc ]; then ` +
|
|
613
|
+
`fail unsafe-npm-config; fi; ` +
|
|
614
|
+
`installs_root="$ASDF_DATA_DIR/installs"; node_root="$installs_root/nodejs"; ` +
|
|
615
|
+
`candidate="$node_root/$default_node"; shims_root="$ASDF_DATA_DIR/shims"; ` +
|
|
616
|
+
`corepack_root="$COREPACK_HOME"; ` +
|
|
617
|
+
`ensure_real_dir() { path="$1"; label="$2"; ` +
|
|
618
|
+
`if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then ` +
|
|
619
|
+
`fail "unsafe-$label"; fi; ` +
|
|
620
|
+
`if [ ! -d "$path" ]; then /usr/bin/install -d -o node -g node -m 0775 "$path" ` +
|
|
621
|
+
`|| fail "create-$label"; fi; ` +
|
|
622
|
+
`lexical="$(/usr/bin/realpath -m -s -- "$path")"; ` +
|
|
623
|
+
`resolved="$(/usr/bin/realpath -e -- "$path")" ` +
|
|
624
|
+
`|| fail "resolve-$label"; [ "$lexical" = "$resolved" ] ` +
|
|
625
|
+
`|| fail "escaped-$label"; /usr/bin/chown node:node "$path" ` +
|
|
626
|
+
`|| fail "owner-$label"; /usr/bin/chmod u+rwx,g+rwx "$path" ` +
|
|
627
|
+
`|| fail "mode-$label"; }; ` +
|
|
628
|
+
`ensure_real_dir "$ASDF_DATA_DIR" asdf-data; ` +
|
|
629
|
+
`ensure_real_dir "$installs_root" installs-root; ` +
|
|
630
|
+
`ensure_real_dir "$node_root" node-root; ` +
|
|
631
|
+
`ensure_real_dir "$shims_root" shims-root; ` +
|
|
632
|
+
`ensure_real_dir "$corepack_root" corepack-root; ` +
|
|
633
|
+
`candidate_is_safe() { ` +
|
|
634
|
+
`lexical_candidate="$(/usr/bin/realpath -m -s -- "$candidate")" ` +
|
|
635
|
+
`|| return 1; [ "$lexical_candidate" = "$node_root/$default_node" ] ` +
|
|
636
|
+
`|| return 1; if [ -e "$candidate" ] || [ -L "$candidate" ]; then ` +
|
|
637
|
+
`[ ! -L "$candidate" ] && [ -d "$candidate" ] || return 1; ` +
|
|
638
|
+
`resolved_candidate="$(/usr/bin/realpath -e -- "$candidate")" ` +
|
|
639
|
+
`|| return 1; ` +
|
|
640
|
+
`[ "$resolved_candidate" = "$lexical_candidate" ] || return 1; fi; }; ` +
|
|
641
|
+
`contained_executable() { root="$1"; executable="$2"; ` +
|
|
642
|
+
`[ -f "$executable" ] && [ -x "$executable" ] || return 1; ` +
|
|
643
|
+
`real_executable="$(/usr/bin/realpath -e -- "$executable")" ` +
|
|
644
|
+
`|| return 1; ` +
|
|
645
|
+
`case "$real_executable" in "$root"/*) return 0 ;; *) return 1 ;; esac; }; ` +
|
|
646
|
+
`contained_real_dir() { root="$1"; path="$2"; ` +
|
|
647
|
+
`[ ! -L "$path" ] && [ -d "$path" ] || return 1; ` +
|
|
648
|
+
`lexical_path="$(/usr/bin/realpath -m -s -- "$path")" || return 1; ` +
|
|
649
|
+
`real_path="$(/usr/bin/realpath -e -- "$path")" || return 1; ` +
|
|
650
|
+
`case "$lexical_path" in "$root"/*) ;; *) return 1 ;; esac; ` +
|
|
651
|
+
`[ "$real_path" = "$lexical_path" ]; }; ` +
|
|
652
|
+
`npm_prefix_is_safe() { contained_real_dir "$candidate" "$candidate/bin" ` +
|
|
653
|
+
`&& contained_real_dir "$candidate" "$candidate/lib" ` +
|
|
654
|
+
`&& contained_real_dir "$candidate" "$candidate/lib/node_modules"; }; ` +
|
|
655
|
+
`scope_is_safe_or_absent() { scope="$1"; ` +
|
|
656
|
+
`if [ -e "$scope" ] || [ -L "$scope" ]; then ` +
|
|
657
|
+
`contained_real_dir "$candidate" "$scope"; fi; }; ` +
|
|
658
|
+
`as_node() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
|
|
659
|
+
`/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
|
|
660
|
+
`ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
|
|
661
|
+
`ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
|
|
662
|
+
`ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
|
|
663
|
+
`COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
|
|
664
|
+
`COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
|
|
665
|
+
`COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
|
|
666
|
+
`COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
|
|
667
|
+
`COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
|
|
668
|
+
`NPM_CONFIG_PREFIX="$candidate" ` +
|
|
669
|
+
`PATH="$candidate/bin:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
|
|
670
|
+
`"$@"; }; ` +
|
|
671
|
+
`as_node_control() { /usr/bin/setpriv --reuid=node --regid=node --init-groups ` +
|
|
672
|
+
`/usr/bin/env -i HOME="$HOME" ASDF_DIR="$ASDF_DIR" ` +
|
|
673
|
+
`ASDF_DATA_DIR="$ASDF_DATA_DIR" ASDF_CONFIG_FILE=/dev/null ` +
|
|
674
|
+
`ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions ` +
|
|
675
|
+
`ASDF_NODEJS_VERSION="$default_node" COREPACK_HOME="$COREPACK_HOME" ` +
|
|
676
|
+
`COREPACK_DEFAULT_TO_LATEST=0 COREPACK_ENABLE_AUTO_PIN=0 ` +
|
|
677
|
+
`COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENV_FILE=0 ` +
|
|
678
|
+
`COREPACK_ENABLE_PROJECT_SPEC=1 COREPACK_ENABLE_STRICT=1 ` +
|
|
679
|
+
`COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0 COREPACK_INTEGRITY_CHECK=1 ` +
|
|
680
|
+
`COREPACK_ENABLE_NETWORK=0 COREPACK_NPM_REGISTRY=https://registry.npmjs.org ` +
|
|
681
|
+
`PATH="/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ` +
|
|
682
|
+
`"$@"; }; ` +
|
|
683
|
+
`clear_forbidden_shared_bash_shim() { fail_if_found="$1"; ` +
|
|
684
|
+
`bash_shim="$shims_root/bash"; bash_shim_found=0; ` +
|
|
685
|
+
`if [ -e "$bash_shim" ] || [ -L "$bash_shim" ]; then ` +
|
|
686
|
+
`bash_shim_found=1; ` +
|
|
687
|
+
`[ ! -d "$bash_shim" ] || fail unsafe-runtime-bash-shim-directory; ` +
|
|
688
|
+
`as_node /usr/bin/rm -f -- "$bash_shim" ` +
|
|
689
|
+
`|| fail remove-runtime-bash-shim; fi; ` +
|
|
690
|
+
`[ ! -e "$bash_shim" ] && [ ! -L "$bash_shim" ] ` +
|
|
691
|
+
`|| fail unsafe-runtime-bash-shim; ` +
|
|
692
|
+
`if [ "$fail_if_found" -ne 0 ] && [ "$bash_shim_found" -ne 0 ]; then ` +
|
|
693
|
+
`return 1; fi; }; ` +
|
|
694
|
+
`clear_forbidden_candidate_bash() { fail_if_found="$1"; ` +
|
|
695
|
+
`candidate_bash="$candidate/bin/bash"; candidate_bash_found=''; ` +
|
|
696
|
+
`if [ -e "$candidate_bash" ] || [ -L "$candidate_bash" ]; then ` +
|
|
697
|
+
`candidate_bash_found=1; [ ! -d "$candidate_bash" ] ` +
|
|
698
|
+
`|| fail unsafe-default-bash-directory; ` +
|
|
699
|
+
`as_node /usr/bin/rm -f -- "$candidate_bash" ` +
|
|
700
|
+
`|| fail remove-default-bash; fi; ` +
|
|
701
|
+
`[ ! -e "$candidate_bash" ] && [ ! -L "$candidate_bash" ] ` +
|
|
702
|
+
`|| fail unsafe-default-bash; ` +
|
|
703
|
+
`if [ "$fail_if_found" -ne 0 ] && [ -n "$candidate_bash_found" ]; then ` +
|
|
704
|
+
`return 1; fi; }; ` +
|
|
705
|
+
`run_asdf_install() { asdf_status=0; shim_status=0; ` +
|
|
706
|
+
`as_node_control /opt/asdf/bin/asdf install "$@" || asdf_status=$?; ` +
|
|
707
|
+
`clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
|
|
708
|
+
`[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
|
|
709
|
+
`run_asdf_reshim() { asdf_status=0; shim_status=0; ` +
|
|
710
|
+
`as_node_control /opt/asdf/bin/asdf reshim "$@" || asdf_status=$?; ` +
|
|
711
|
+
`clear_forbidden_shared_bash_shim 1 || shim_status=$?; ` +
|
|
712
|
+
`[ "$shim_status" -eq 0 ] || return 1; return "$asdf_status"; }; ` +
|
|
713
|
+
`reject_runtime_bash_shims() { ` +
|
|
714
|
+
`bash_cleanup_status=0; clear_forbidden_candidate_bash 1 ` +
|
|
715
|
+
`|| bash_cleanup_status=1; ` +
|
|
716
|
+
`clear_forbidden_shared_bash_shim 1 || bash_cleanup_status=1; ` +
|
|
717
|
+
`[ "$bash_cleanup_status" -eq 0 ] || fail unsafe-runtime-bash-shim; }; ` +
|
|
718
|
+
`node_write_probe() { writable="$1"; ` +
|
|
719
|
+
`probe="$(as_node /usr/bin/mktemp -d "$writable/.uai-write-probe.XXXXXX")" ` +
|
|
720
|
+
`|| return 1; case "$probe" in "$writable"/.uai-write-probe.*) ;; ` +
|
|
721
|
+
`*) return 1 ;; esac; as_node /usr/bin/rmdir "$probe"; }; ` +
|
|
722
|
+
`candidate_owned_by_node() { ` +
|
|
723
|
+
`[ -z "$(/usr/bin/find "$candidate" -xdev ! -user node -print -quit)" ] ` +
|
|
724
|
+
`&& [ -z "$(/usr/bin/find "$candidate" -xdev -type d ! -perm -u+w -print -quit)" ]; }; ` +
|
|
725
|
+
`prove_default_node() { candidate_is_safe && [ -d "$candidate" ] ` +
|
|
726
|
+
`&& [ ! -e "$candidate/bin/bash" ] && [ ! -L "$candidate/bin/bash" ] ` +
|
|
727
|
+
`&& contained_executable "$candidate" "$candidate/bin/node" ` +
|
|
728
|
+
`&& [ "$(as_node "$candidate/bin/node" --version 2>/dev/null)" = "v$default_node" ] ` +
|
|
729
|
+
`&& contained_executable "$candidate" "$candidate/bin/npm" ` +
|
|
730
|
+
`&& [ -n "$(as_node "$candidate/bin/npm" --version 2>/dev/null)" ] ` +
|
|
731
|
+
`&& npm_prefix_is_safe && candidate_owned_by_node ` +
|
|
732
|
+
`&& node_write_probe "$candidate/bin"; }; ` +
|
|
733
|
+
`clear_forbidden_shared_bash_shim 0; ` +
|
|
734
|
+
`if ! prove_default_node; then candidate_is_safe || fail unsafe-default-candidate; ` +
|
|
735
|
+
`/usr/bin/rm -rf --one-file-system -- "$candidate" ` +
|
|
736
|
+
`|| fail remove-default-candidate; ` +
|
|
737
|
+
`run_asdf_install nodejs "$default_node" ` +
|
|
738
|
+
`|| fail install-default-node; prove_default_node || fail repaired-default-node; fi; ` +
|
|
739
|
+
`npm_bin="$candidate/bin/npm"; cd / || fail chdir; ` +
|
|
740
|
+
`if [ "$upgrade" = 1 ]; then ` +
|
|
741
|
+
`npm_prefix_is_safe || fail unsafe-npm-prefix; ` +
|
|
742
|
+
`if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
|
|
743
|
+
`>/tmp/uai-cli-upgrade.out 2>&1; then ` +
|
|
744
|
+
`if /usr/bin/grep -qi 'ENOTEMPTY' /tmp/uai-cli-upgrade.out; then ` +
|
|
745
|
+
`scope_is_safe_or_absent "$candidate/lib/node_modules/@anthropic-ai" ` +
|
|
746
|
+
`&& scope_is_safe_or_absent "$candidate/lib/node_modules/@openai" ` +
|
|
747
|
+
`|| fail unsafe-cli-scope; ` +
|
|
748
|
+
`as_node /usr/bin/rm -rf --one-file-system -- ` +
|
|
749
|
+
`"$candidate/lib/node_modules/@anthropic-ai" ` +
|
|
750
|
+
`"$candidate/lib/node_modules/@openai"; ` +
|
|
751
|
+
`if ! as_node "$npm_bin" install -g ${pkgs} ${trustedNpmFlags} ` +
|
|
752
|
+
`>/tmp/uai-cli-upgrade-retry.out 2>&1; then ` +
|
|
753
|
+
`echo 'UAI_CLI_UPDATE_DEFERRED:bulk-retry' >&2; fi; ` +
|
|
754
|
+
`else echo 'UAI_CLI_UPDATE_DEFERRED:bulk' >&2; fi; fi; fi; ` +
|
|
755
|
+
// Reshim first so an installed package behind a stale shim can recover
|
|
756
|
+
// without network access. Only bins that still fail are reinstalled.
|
|
757
|
+
`if ! run_asdf_reshim nodejs "$default_node" ` +
|
|
758
|
+
`>/dev/null 2>&1; then echo 'UAI_CLI_RESHIM_DEFERRED:initial' >&2; fi; ` +
|
|
759
|
+
`cli_ready_once() { b="$1"; ` +
|
|
760
|
+
`contained_executable "$candidate" "$candidate/bin/$b" || return 1; ` +
|
|
761
|
+
`direct_version="$(as_node "$candidate/bin/$b" --version 2>/dev/null ` +
|
|
762
|
+
`| /usr/bin/head -1)"; [ -n "$direct_version" ] || return 1; ` +
|
|
763
|
+
`shim="$shims_root/$b"; [ ! -L "$shim" ] && [ -f "$shim" ] ` +
|
|
764
|
+
`&& [ -x "$shim" ] || return 1; ` +
|
|
765
|
+
`shim_version="$(as_node "$shim" --version 2>/dev/null ` +
|
|
766
|
+
`| /usr/bin/head -1)"; [ "$shim_version" = "$direct_version" ]; }; ` +
|
|
767
|
+
`repair_cli_once() { b="$1"; case "$b" in ` +
|
|
768
|
+
`claude) pkg='@anthropic-ai/claude-code'; scope='@anthropic-ai'; ` +
|
|
769
|
+
`package_name='claude-code';; ` +
|
|
770
|
+
`codex) pkg='@openai/codex'; scope='@openai'; package_name='codex';; ` +
|
|
771
|
+
`*) return 1;; esac; ` +
|
|
772
|
+
`package_path="$candidate/lib/node_modules/$scope/$package_name"; ` +
|
|
773
|
+
`npm_prefix_is_safe || return 1; ` +
|
|
774
|
+
`scope_is_safe_or_absent "$candidate/lib/node_modules/$scope" ` +
|
|
775
|
+
`|| return 1; ` +
|
|
776
|
+
`as_node /usr/bin/rm -rf --one-file-system -- ` +
|
|
777
|
+
`"$candidate/bin/$b" "$package_path" || return 1; ` +
|
|
778
|
+
`as_node "$npm_bin" install -g --force "$pkg@latest" ${trustedNpmFlags} ` +
|
|
779
|
+
`>/dev/null 2>&1 || return 1; ` +
|
|
780
|
+
`run_asdf_reshim nodejs "$default_node" ` +
|
|
781
|
+
`>/dev/null 2>&1 || return 1; cli_ready_once "$b"; }; ` +
|
|
782
|
+
`for b in claude codex; do if ! cli_ready_once "$b"; then ` +
|
|
783
|
+
`repair_cli_once "$b" || fail "$b-repair"; fi; done; ` +
|
|
784
|
+
// Node <=22's bundled Corepack has obsolete registry signing keys. The
|
|
785
|
+
// host-packaged pin is compatible with every advertised Node default.
|
|
786
|
+
`required_corepack="$(/usr/bin/sed -n '1p' /usr/local/share/uai/corepack-version)"; ` +
|
|
787
|
+
`printf '%s' "$required_corepack" | /usr/bin/grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+$' || { ` +
|
|
788
|
+
`echo 'UAI_STILL_BROKEN:corepack-pin' >&2; exit 1; }; ` +
|
|
789
|
+
`corepack_ready_once() { ` +
|
|
790
|
+
`contained_executable "$candidate" "$candidate/bin/corepack" || return 1; ` +
|
|
791
|
+
`[ "$(as_node "$candidate/bin/corepack" --version 2>/dev/null)" = "$required_corepack" ] ` +
|
|
792
|
+
`|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
|
|
793
|
+
`|| return 1; run_asdf_reshim nodejs "$default_node" ` +
|
|
794
|
+
`>/dev/null 2>&1 || return 1; ` +
|
|
795
|
+
`corepack_shim="$shims_root/corepack"; [ ! -L "$corepack_shim" ] ` +
|
|
796
|
+
`&& [ -f "$corepack_shim" ] && [ -x "$corepack_shim" ] || return 1; ` +
|
|
797
|
+
`[ "$(as_node "$corepack_shim" --version 2>/dev/null)" = "$required_corepack" ]; }; ` +
|
|
798
|
+
`repair_corepack_once() { ` +
|
|
799
|
+
`npm_prefix_is_safe || return 1; ` +
|
|
800
|
+
`as_node /usr/bin/rm -rf --one-file-system -- ` +
|
|
801
|
+
`"$candidate/lib/node_modules/corepack" "$candidate/bin/corepack" ` +
|
|
802
|
+
`"$candidate/bin/pnpm" "$candidate/bin/pnpx" "$candidate/bin/yarn" ` +
|
|
803
|
+
`"$candidate/bin/yarnpkg" || return 1; ` +
|
|
804
|
+
`as_node "$npm_bin" install -g --force "corepack@$required_corepack" ` +
|
|
805
|
+
`--no-audit --no-fund ${trustedNpmFlags} >/dev/null 2>&1 || return 1; ` +
|
|
806
|
+
`run_asdf_reshim nodejs "$default_node" >/dev/null 2>&1 ` +
|
|
807
|
+
`|| return 1; contained_executable "$candidate" "$candidate/bin/corepack" ` +
|
|
808
|
+
`|| return 1; as_node "$candidate/bin/corepack" disable >/dev/null 2>&1 ` +
|
|
809
|
+
`|| return 1; as_node "$candidate/bin/corepack" enable >/dev/null 2>&1 ` +
|
|
810
|
+
`|| return 1; run_asdf_reshim nodejs "$default_node" ` +
|
|
811
|
+
`>/dev/null 2>&1 || return 1; corepack_ready_once; }; ` +
|
|
812
|
+
`if ! corepack_ready_once; then repair_corepack_once ` +
|
|
813
|
+
`|| fail corepack-repair; fi; reject_runtime_bash_shims; failed=0; ` +
|
|
814
|
+
`prove_default_node || { echo "UAI_STILL_BROKEN:default-node-final" >&2; failed=1; }; ` +
|
|
815
|
+
`for b in claude codex; do ` +
|
|
816
|
+
`if ! contained_executable "$candidate" "$candidate/bin/$b"; then ` +
|
|
817
|
+
`echo "UAI_STILL_BROKEN:$b-contained" >&2; failed=1; continue; fi; ` +
|
|
818
|
+
`version="$(as_node "$candidate/bin/$b" --version 2>/dev/null | /usr/bin/head -1)"; ` +
|
|
819
|
+
`shim="$shims_root/$b"; if [ -L "$shim" ] || [ ! -f "$shim" ] ` +
|
|
820
|
+
`|| [ ! -x "$shim" ]; then echo "UAI_STILL_BROKEN:$b-shim" >&2; ` +
|
|
821
|
+
`failed=1; continue; fi; ` +
|
|
822
|
+
`shim_version="$(as_node "$shim" --version 2>/dev/null | /usr/bin/head -1)"; ` +
|
|
823
|
+
`if [ -z "$version" ] || [ "$shim_version" != "$version" ]; then ` +
|
|
824
|
+
`echo "UAI_STILL_BROKEN:$b" >&2; failed=1; ` +
|
|
825
|
+
`else printf '%s ' "$version"; fi; done; ` +
|
|
826
|
+
`node_shim="$shims_root/node"; ` +
|
|
827
|
+
`[ ! -L "$node_shim" ] && [ -f "$node_shim" ] && [ -x "$node_shim" ] ` +
|
|
828
|
+
`|| { echo "UAI_STILL_BROKEN:node-shim-file" >&2; failed=1; }; ` +
|
|
829
|
+
`shim_node="$(as_node "$node_shim" --version 2>/dev/null || true)"; ` +
|
|
830
|
+
`[ "$shim_node" = "v$default_node" ] || { ` +
|
|
831
|
+
`echo "UAI_STILL_BROKEN:node-shim" >&2; failed=1; }; ` +
|
|
832
|
+
`npm_shim="$shims_root/npm"; ` +
|
|
833
|
+
`[ ! -L "$npm_shim" ] && [ -f "$npm_shim" ] && [ -x "$npm_shim" ] ` +
|
|
834
|
+
`|| { echo "UAI_STILL_BROKEN:npm-shim-file" >&2; failed=1; }; ` +
|
|
835
|
+
`shim_npm="$(as_node "$npm_shim" --version 2>/dev/null || true)"; ` +
|
|
836
|
+
`[ -n "$shim_npm" ] || { echo "UAI_STILL_BROKEN:npm-shim" >&2; failed=1; }; ` +
|
|
837
|
+
`if ! corepack_ready_once; then echo "UAI_STILL_BROKEN:corepack-final" >&2; ` +
|
|
838
|
+
`failed=1; else printf 'corepack %s' "$required_corepack"; fi; ` +
|
|
839
|
+
`node_write_probe "$candidate/bin" || { ` +
|
|
840
|
+
`echo "UAI_STILL_BROKEN:default-node-owner" >&2; failed=1; }; ` +
|
|
841
|
+
`node_write_probe "$shims_root" || { ` +
|
|
842
|
+
`echo "UAI_STILL_BROKEN:shims-owner" >&2; failed=1; }; ` +
|
|
843
|
+
`node_write_probe "$corepack_root" || { ` +
|
|
844
|
+
`echo "UAI_STILL_BROKEN:corepack-owner" >&2; failed=1; }; ` +
|
|
845
|
+
`exit "$failed"`
|
|
846
|
+
);
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
async function reconcileVolumeAgentClis(): Promise<boolean> {
|
|
850
|
+
const missingBefore = await probeVolumeAgentClis();
|
|
851
|
+
if (missingBefore?.length) {
|
|
852
|
+
console.log(
|
|
853
|
+
`[host-agent] repairing ${ASDF_DATA_VOLUME}: agent CLI(s) ` +
|
|
854
|
+
`[${missingBefore.join(", ")}] failed their execution probe`,
|
|
855
|
+
);
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
const autoUpdate = process.env.UAI_AGENT_CLI_AUTOUPDATE !== "0";
|
|
859
|
+
if (!autoUpdate) {
|
|
860
|
+
console.log(
|
|
861
|
+
"[host-agent] agent CLI auto-update disabled (UAI_AGENT_CLI_AUTOUPDATE=0)",
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
const command = standardImageVolumeMaintenanceCommandForTest(autoUpdate);
|
|
865
|
+
const reconcile = await runMaintenanceContainer(selectedEngine(), "reconcile", [
|
|
289
866
|
"--rm",
|
|
290
|
-
// The container NAME is the mutex: overlapping boots (or a crash-looping
|
|
291
|
-
// service) must never race two npm installs on the shared volume — that
|
|
292
|
-
// once left it with no `claude` at all (2026-07-13). Docker rejects the
|
|
293
|
-
// duplicate name; we treat that as "already upgrading, skip".
|
|
294
|
-
"--name",
|
|
295
|
-
"uai-cli-upgrade",
|
|
296
867
|
"-u",
|
|
297
868
|
"root",
|
|
298
869
|
"-w",
|
|
299
870
|
"/home/node",
|
|
300
871
|
"-v",
|
|
301
872
|
`${ASDF_DATA_VOLUME}:/opt/asdf-data`,
|
|
873
|
+
"-v",
|
|
874
|
+
`${standardImageCorepackVersionPath()}:/usr/local/share/uai/corepack-version:ro`,
|
|
302
875
|
STANDARD_IMAGE_TAG,
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
return;
|
|
318
|
-
}
|
|
319
|
-
if (upgrade.code === 0) {
|
|
876
|
+
"/usr/bin/timeout",
|
|
877
|
+
"--signal=TERM",
|
|
878
|
+
"--kill-after=10s",
|
|
879
|
+
`${ASDF_MAINTENANCE_INTERNAL_TIMEOUT_SECONDS}s`,
|
|
880
|
+
"/usr/bin/flock",
|
|
881
|
+
"--exclusive",
|
|
882
|
+
"--timeout",
|
|
883
|
+
`${ASDF_MAINTENANCE_LOCK_WAIT_SECONDS}`,
|
|
884
|
+
ASDF_MAINTENANCE_LOCK_PATH,
|
|
885
|
+
"/bin/bash",
|
|
886
|
+
"-c",
|
|
887
|
+
command,
|
|
888
|
+
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
889
|
+
if (reconcile.code === 0) {
|
|
320
890
|
console.log(
|
|
321
|
-
`[host-agent] agent CLIs current on ${ASDF_DATA_VOLUME}:
|
|
891
|
+
`[host-agent] agent CLIs and Corepack current on ${ASDF_DATA_VOLUME}: ` +
|
|
892
|
+
reconcile.stdout.trim(),
|
|
322
893
|
);
|
|
894
|
+
await writeVolumeReconcileProof();
|
|
895
|
+
return true;
|
|
323
896
|
} else {
|
|
897
|
+
lastMaintenanceStderr = reconcile.stderr;
|
|
324
898
|
console.warn(
|
|
325
|
-
`[host-agent]
|
|
326
|
-
|
|
899
|
+
`[host-agent] failed to reconcile ${ASDF_DATA_VOLUME} (exit ` +
|
|
900
|
+
`${reconcile.code ?? "spawn"}); task runtimes are not ready. ` +
|
|
901
|
+
reconcile.stderr.trim().slice(0, 500),
|
|
327
902
|
);
|
|
903
|
+
return false;
|
|
328
904
|
}
|
|
329
905
|
}
|
|
330
906
|
|
|
907
|
+
/** Durable evidence that the shared volume completed a full reconcile at
|
|
908
|
+
* least once on this host. Apple volumes are block devices with EXCLUSIVE
|
|
909
|
+
* writable attachments, so boot maintenance cannot re-prove the volume while
|
|
910
|
+
* running task containers hold it read-only — and blocking activation on
|
|
911
|
+
* that (the 2026-08-16 ring) froze hosts and blocked the very teardowns that
|
|
912
|
+
* would free the volume. With this proof on disk, a busy volume defers
|
|
913
|
+
* maintenance instead of failing it. */
|
|
914
|
+
let lastMaintenanceStderr = "";
|
|
915
|
+
|
|
916
|
+
async function volumeReconcileProofPath(): Promise<string> {
|
|
917
|
+
// Lazy env resolve: module import must stay side-effect free (see
|
|
918
|
+
// defaultRuntimeProviderStateStore). Dynamic import, not require — a bare
|
|
919
|
+
// require cannot resolve the TypeScript module outside the tsx runtime,
|
|
920
|
+
// which silently read as "no proof" and disabled deferral.
|
|
921
|
+
const { env } = await import("./env");
|
|
922
|
+
return resolve(env.dataDir, "runtime-volume-proof.json");
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
async function writeVolumeReconcileProof(): Promise<void> {
|
|
926
|
+
try {
|
|
927
|
+
writeFileSync(
|
|
928
|
+
await volumeReconcileProofPath(),
|
|
929
|
+
`${JSON.stringify({ schemaVersion: 1, provedAt: Date.now() })}\n`,
|
|
930
|
+
{ mode: 0o600 },
|
|
931
|
+
);
|
|
932
|
+
} catch (error) {
|
|
933
|
+
console.warn(
|
|
934
|
+
`[host-agent] could not persist the shared-volume reconcile proof: ${
|
|
935
|
+
error instanceof Error ? error.message : String(error)
|
|
936
|
+
}`,
|
|
937
|
+
);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
async function volumeReconcileProofExists(): Promise<boolean> {
|
|
942
|
+
try {
|
|
943
|
+
const raw = JSON.parse(
|
|
944
|
+
readFileSync(await volumeReconcileProofPath(), "utf8"),
|
|
945
|
+
) as { schemaVersion?: unknown };
|
|
946
|
+
return raw?.schemaVersion === 1;
|
|
947
|
+
} catch {
|
|
948
|
+
return false;
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
const VOLUME_ATTACH_BUSY_SIGNATURE = "storage device attachment is invalid";
|
|
953
|
+
|
|
954
|
+
/**
|
|
955
|
+
* Whether task containers hold the shared volume — tri-state per container,
|
|
956
|
+
* never fail-open: `running` holds, exact `stopped` is released (a stopped
|
|
957
|
+
* VM releases its block attachments — proven live 2026-08-17, when a full
|
|
958
|
+
* writable reconcile succeeded next to a stopped task container), and EVERY
|
|
959
|
+
* other observation (starting, stopping, creating, a missing or foreign
|
|
960
|
+
* state string, an unreadable inventory) is `unknown` — a task acquiring or
|
|
961
|
+
* releasing the volume mid-transition must never read as permission for a
|
|
962
|
+
* writer (review 2026-08-18 round 2). `held` dominates `unknown` dominates
|
|
963
|
+
* `free`; writers require a positive `free`.
|
|
964
|
+
*/
|
|
965
|
+
async function sharedVolumeHolders(engine: {
|
|
966
|
+
command: string;
|
|
967
|
+
apple: boolean;
|
|
968
|
+
}): Promise<"held" | "free" | "unknown"> {
|
|
969
|
+
const listed = await run(
|
|
970
|
+
engine.command,
|
|
971
|
+
["list", "--all", "--format", "json"],
|
|
972
|
+
30_000,
|
|
973
|
+
);
|
|
974
|
+
if (listed.code !== 0) return "unknown";
|
|
975
|
+
try {
|
|
976
|
+
const parsed = JSON.parse(listed.stdout) as Array<{
|
|
977
|
+
configuration?: {
|
|
978
|
+
id?: unknown;
|
|
979
|
+
mounts?: Array<{ type?: { volume?: { name?: unknown } } }>;
|
|
980
|
+
};
|
|
981
|
+
status?: { state?: unknown };
|
|
982
|
+
}>;
|
|
983
|
+
if (!Array.isArray(parsed)) return "unknown";
|
|
984
|
+
let verdict: "held" | "free" | "unknown" = "free";
|
|
985
|
+
for (const item of parsed) {
|
|
986
|
+
// A holder is a container whose CONFIGURED MOUNTS include the shared
|
|
987
|
+
// volume — physical attachment, not a name shape (review 2026-08-18
|
|
988
|
+
// round 3: an unrelated task-*-app container with no attachment
|
|
989
|
+
// fenced activation forever, and the holder-remedy sweep — which
|
|
990
|
+
// correctly matches by mount — could not clear it). A row whose
|
|
991
|
+
// mounts cannot be read might hold the volume, so it is `unknown`.
|
|
992
|
+
const mounts = item?.configuration?.mounts;
|
|
993
|
+
if (!Array.isArray(mounts)) {
|
|
994
|
+
verdict = "unknown";
|
|
995
|
+
continue;
|
|
996
|
+
}
|
|
997
|
+
const holds = mounts.some(
|
|
998
|
+
(mount) => mount?.type?.volume?.name === ASDF_DATA_VOLUME,
|
|
999
|
+
);
|
|
1000
|
+
if (!holds) continue;
|
|
1001
|
+
const state = item?.status?.state;
|
|
1002
|
+
if (state === "running") return "held";
|
|
1003
|
+
if (state !== "stopped") verdict = "unknown";
|
|
1004
|
+
}
|
|
1005
|
+
return verdict;
|
|
1006
|
+
} catch {
|
|
1007
|
+
return "unknown";
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
/** Backstop for the race the proof-first check cannot close: a task container
|
|
1012
|
+
* appearing between the check and the attach. Admits deferral only on the
|
|
1013
|
+
* exact busy signature, with the same proof requirement. */
|
|
1014
|
+
async function volumeBusyDeferral(
|
|
1015
|
+
engine: { command: string; apple: boolean },
|
|
1016
|
+
stderr: string,
|
|
1017
|
+
): Promise<boolean> {
|
|
1018
|
+
if (!engine.apple) return false;
|
|
1019
|
+
if (!stderr.includes(VOLUME_ATTACH_BUSY_SIGNATURE)) return false;
|
|
1020
|
+
if (!(await volumeReconcileProofExists())) return false;
|
|
1021
|
+
return (await sharedVolumeHolders(engine)) === "held";
|
|
1022
|
+
}
|
|
1023
|
+
|
|
331
1024
|
/** Outcome of ensureStandardImage — `error` carries WHY when not ready, so
|
|
332
1025
|
* the taskUp path can surface a build failure to the cloud (not just the host
|
|
333
1026
|
* log). Callers that only fire-and-forget at boot ignore the return. */
|
|
334
1027
|
export interface StandardImageResult {
|
|
335
1028
|
ok: boolean;
|
|
336
1029
|
error?: string;
|
|
1030
|
+
/** A non-writer escape the activation driver can perform before retrying:
|
|
1031
|
+
* lifecycle-stop the running task containers holding the shared volume
|
|
1032
|
+
* (resumable), because the volume needs a repair they block and cannot
|
|
1033
|
+
* safely use. */
|
|
1034
|
+
remedy?: "stop-volume-holders";
|
|
337
1035
|
}
|
|
338
1036
|
|
|
1037
|
+
type AgentClisReadyListener = () => void;
|
|
1038
|
+
|
|
1039
|
+
let agentClisReadyLatched = false;
|
|
339
1040
|
let signalClisReady: (() => void) | null = null;
|
|
1041
|
+
const agentClisReadyListeners = new Set<AgentClisReadyListener>();
|
|
1042
|
+
const ensureQueue = new KeyedPromiseTail<StandardImageResult>();
|
|
340
1043
|
/**
|
|
341
1044
|
* Resolves once ensureStandardImage has reconciled the shared-volume agent CLIs
|
|
342
1045
|
* at least once (or bailed because docker is down / the build failed). The agent
|
|
@@ -349,6 +1052,55 @@ export const agentClisReady: Promise<void> = new Promise<void>((resolve) => {
|
|
|
349
1052
|
signalClisReady = resolve;
|
|
350
1053
|
});
|
|
351
1054
|
|
|
1055
|
+
/** Whether this process has completed its one boot-time CLI maintenance pass.
|
|
1056
|
+
* This is intentionally a one-way boot latch, not a claim that no later
|
|
1057
|
+
* operator-triggered image work is in flight. */
|
|
1058
|
+
export function areAgentClisReady(): boolean {
|
|
1059
|
+
return agentClisReadyLatched;
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
/** Subscribe to the one readiness edge. A late subscriber is notified in a
|
|
1063
|
+
* microtask so registration never re-enters its caller. */
|
|
1064
|
+
export function onAgentClisReady(
|
|
1065
|
+
listener: AgentClisReadyListener,
|
|
1066
|
+
): () => void {
|
|
1067
|
+
if (agentClisReadyLatched) {
|
|
1068
|
+
let cancelled = false;
|
|
1069
|
+
queueMicrotask(() => {
|
|
1070
|
+
if (!cancelled) notifyAgentClisReadyListener(listener);
|
|
1071
|
+
});
|
|
1072
|
+
return () => {
|
|
1073
|
+
cancelled = true;
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
agentClisReadyListeners.add(listener);
|
|
1077
|
+
return () => agentClisReadyListeners.delete(listener);
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
function notifyAgentClisReadyListener(listener: AgentClisReadyListener): void {
|
|
1081
|
+
try {
|
|
1082
|
+
listener();
|
|
1083
|
+
} catch (error) {
|
|
1084
|
+
console.warn(
|
|
1085
|
+
`[host-agent] agent CLI readiness listener failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1086
|
+
);
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
function markAgentClisReady(): void {
|
|
1091
|
+
if (agentClisReadyLatched) return;
|
|
1092
|
+
// Publish the synchronous capability state before resolving the promise:
|
|
1093
|
+
// every waiter and listener must observe `true` after the readiness edge.
|
|
1094
|
+
agentClisReadyLatched = true;
|
|
1095
|
+
signalClisReady?.();
|
|
1096
|
+
signalClisReady = null;
|
|
1097
|
+
const listeners = [...agentClisReadyListeners];
|
|
1098
|
+
agentClisReadyListeners.clear();
|
|
1099
|
+
for (const listener of listeners) {
|
|
1100
|
+
notifyAgentClisReadyListener(listener);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
|
|
352
1104
|
/**
|
|
353
1105
|
* Ensure the standard image and the shared asdf data volume exist, and that the
|
|
354
1106
|
* agent CLIs are present inside the volume. Builds the image only when `docker
|
|
@@ -356,21 +1108,52 @@ export const agentClisReady: Promise<void> = new Promise<void>((resolve) => {
|
|
|
356
1108
|
* `{ ok, error? }` so the caller decides (boot: fire-and-forget; taskUp:
|
|
357
1109
|
* surface the reason). Resolves `agentClisReady` on every exit path.
|
|
358
1110
|
*/
|
|
359
|
-
export
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
1111
|
+
export function ensureStandardImage(): Promise<StandardImageResult> {
|
|
1112
|
+
// Startup, runtime recovery, and a newly-arrived task can all discover the
|
|
1113
|
+
// same daemon transition. One shared volume repair/build must own that
|
|
1114
|
+
// window; concurrent maintenance containers can otherwise replace the same
|
|
1115
|
+
// CLI shims underneath each other and defeat agentClisReady's fence.
|
|
1116
|
+
const engines = configuredOptionalEngines();
|
|
1117
|
+
return ensureQueue.run(optionalEngineConfigurationKey(engines), () =>
|
|
1118
|
+
ensureStandardImageInner(engines).finally(() => {
|
|
1119
|
+
// Always signal — even a docker-down bail — so agent spawns never hang
|
|
1120
|
+
// waiting on a reconcile that will not happen this boot.
|
|
1121
|
+
markAgentClisReady();
|
|
1122
|
+
}),
|
|
1123
|
+
);
|
|
368
1124
|
}
|
|
369
1125
|
|
|
370
|
-
async function ensureStandardImageInner(
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
1126
|
+
async function ensureStandardImageInner(
|
|
1127
|
+
engines: OptionalEngineConfiguration,
|
|
1128
|
+
): Promise<StandardImageResult> {
|
|
1129
|
+
// 1. Shared asdf data volume — idempotent under Docker; the Apple CLI
|
|
1130
|
+
// errors on an existing name, which is the same healthy outcome.
|
|
1131
|
+
const engine = selectedEngine();
|
|
1132
|
+
console.log(
|
|
1133
|
+
`[host-agent] standard-image maintenance starting (engine: ${engine.apple ? "apple-container" : "docker"})`,
|
|
1134
|
+
);
|
|
1135
|
+
if (engine.apple) {
|
|
1136
|
+
// The volume create below is the FIRST apple mutation on a host — before
|
|
1137
|
+
// any task exists — and uninstall reads an absent provider-claim marker
|
|
1138
|
+
// as "nothing to tear down". Claim durably before mutating, or a
|
|
1139
|
+
// zero-task install→activate→uninstall skips Apple cleanup (review
|
|
1140
|
+
// 2026-08-18 round 4).
|
|
1141
|
+
try {
|
|
1142
|
+
await claimContainerRuntimeForMaintenance();
|
|
1143
|
+
} catch (error) {
|
|
1144
|
+
return {
|
|
1145
|
+
ok: false,
|
|
1146
|
+
error:
|
|
1147
|
+
"Could not durably record the Apple container runtime claim " +
|
|
1148
|
+
`before provisioning: ${error instanceof Error ? error.message : String(error)}`,
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
const vol = await run(engine.command, ["volume", "create", ASDF_DATA_VOLUME]);
|
|
1153
|
+
if (
|
|
1154
|
+
vol.code !== 0 &&
|
|
1155
|
+
!(engine.apple && vol.stderr.includes("already exists"))
|
|
1156
|
+
) {
|
|
374
1157
|
console.warn(
|
|
375
1158
|
`[host-agent] could not create asdf data volume (${ASDF_DATA_VOLUME}); ` +
|
|
376
1159
|
`continuing. ${vol.stderr.trim()}`,
|
|
@@ -391,7 +1174,6 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
|
|
|
391
1174
|
let buildError: string | null = null;
|
|
392
1175
|
// Install only the optional engines the operator has configured; the flags
|
|
393
1176
|
// are build args AND part of the content hash (so a new login rebuilds).
|
|
394
|
-
const engines = configuredOptionalEngines();
|
|
395
1177
|
const engineArgs = [
|
|
396
1178
|
"--build-arg",
|
|
397
1179
|
`INSTALL_KIMI=${engines.kimi ? 1 : 0}`,
|
|
@@ -405,21 +1187,52 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
|
|
|
405
1187
|
const contextHash = await hashBuildContext(
|
|
406
1188
|
`kimi=${engines.kimi ? 1 : 0};grok=${engines.grok ? 1 : 0};cursor=${engines.cursor ? 1 : 0};opencode=${engines.opencode ? 1 : 0}`,
|
|
407
1189
|
);
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
1190
|
+
let inspect: RunResult;
|
|
1191
|
+
let labeledHash: string | null = null;
|
|
1192
|
+
if (engine.apple) {
|
|
1193
|
+
// The Apple CLI has no --format; labels live in the single variant's OCI
|
|
1194
|
+
// config. A malformed answer counts as "absent" and triggers a rebuild.
|
|
1195
|
+
inspect = await run(engine.command, [
|
|
1196
|
+
"image",
|
|
1197
|
+
"inspect",
|
|
1198
|
+
STANDARD_IMAGE_TAG,
|
|
1199
|
+
]);
|
|
1200
|
+
if (inspect.code === 0) {
|
|
1201
|
+
try {
|
|
1202
|
+
const parsed = JSON.parse(inspect.stdout) as Array<{
|
|
1203
|
+
variants?: Array<{
|
|
1204
|
+
config?: { config?: { Labels?: Record<string, string> } };
|
|
1205
|
+
}>;
|
|
1206
|
+
}>;
|
|
1207
|
+
labeledHash =
|
|
1208
|
+
parsed[0]?.variants?.[0]?.config?.config?.Labels?.[
|
|
1209
|
+
CONTEXT_HASH_LABEL
|
|
1210
|
+
] ?? "";
|
|
1211
|
+
} catch {
|
|
1212
|
+
inspect = { ...inspect, code: 1 };
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
} else {
|
|
1216
|
+
inspect = await run("docker", [
|
|
1217
|
+
"image",
|
|
1218
|
+
"inspect",
|
|
1219
|
+
"-f",
|
|
1220
|
+
`{{index .Config.Labels "${CONTEXT_HASH_LABEL}"}}`,
|
|
1221
|
+
STANDARD_IMAGE_TAG,
|
|
1222
|
+
]);
|
|
1223
|
+
labeledHash = inspect.code === 0 ? inspect.stdout.trim() : null;
|
|
1224
|
+
}
|
|
415
1225
|
if (inspect.code === null) {
|
|
416
1226
|
console.warn(
|
|
417
|
-
"[host-agent]
|
|
418
|
-
"Tasks will fail until
|
|
1227
|
+
"[host-agent] container runtime unavailable; skipping standard image " +
|
|
1228
|
+
"build. Tasks will fail until it is running.",
|
|
419
1229
|
);
|
|
420
|
-
return {
|
|
1230
|
+
return {
|
|
1231
|
+
ok: false,
|
|
1232
|
+
error: "The container runtime is not available on this host.",
|
|
1233
|
+
};
|
|
421
1234
|
}
|
|
422
|
-
|
|
1235
|
+
if (inspect.code !== 0) labeledHash = null;
|
|
423
1236
|
if (inspect.code === 0 && contextHash !== null && labeledHash === contextHash) {
|
|
424
1237
|
console.log(`[host-agent] standard image ${STANDARD_IMAGE_TAG} current`);
|
|
425
1238
|
imageReady = true;
|
|
@@ -434,7 +1247,7 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
|
|
|
434
1247
|
? `[host-agent] standard image ${STANDARD_IMAGE_TAG} stale (context changed) — rebuilding from ${context}`
|
|
435
1248
|
: `[host-agent] building standard image ${STANDARD_IMAGE_TAG} from ${context}`,
|
|
436
1249
|
);
|
|
437
|
-
const build = await run(
|
|
1250
|
+
const build = await run(engine.command, [
|
|
438
1251
|
"build",
|
|
439
1252
|
"-t",
|
|
440
1253
|
STANDARD_IMAGE_TAG,
|
|
@@ -443,7 +1256,7 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
|
|
|
443
1256
|
? ["--label", `${CONTEXT_HASH_LABEL}=${contextHash}`]
|
|
444
1257
|
: []),
|
|
445
1258
|
context,
|
|
446
|
-
]);
|
|
1259
|
+
], IMAGE_BUILD_TIMEOUT_MS);
|
|
447
1260
|
if (build.code === 0) {
|
|
448
1261
|
console.log(`[host-agent] built standard image ${STANDARD_IMAGE_TAG}`);
|
|
449
1262
|
imageReady = true;
|
|
@@ -461,15 +1274,165 @@ async function ensureStandardImageInner(): Promise<StandardImageResult> {
|
|
|
461
1274
|
}
|
|
462
1275
|
}
|
|
463
1276
|
|
|
464
|
-
// 3. Reconcile
|
|
465
|
-
//
|
|
466
|
-
//
|
|
467
|
-
// can leave a stale shim (a failed reinstall over a removed package), so
|
|
468
|
-
// ensureVolumeAgentClis must be the final word — it verifies every bin
|
|
469
|
-
// actually runs and repairs whatever the upgrade left broken.
|
|
1277
|
+
// 3. Reconcile every Uai-owned writer under the volume-global flock. Lock
|
|
1278
|
+
// bootstrap/permission repair precedes the single upgrade→repair→proof
|
|
1279
|
+
// transaction so node-run task materializers can acquire legacy volumes.
|
|
470
1280
|
if (imageReady) {
|
|
471
|
-
|
|
472
|
-
|
|
1281
|
+
// Docker populates a fresh named volume from the image's directory
|
|
1282
|
+
// content on first mount; the Apple runtime mounts it empty (ADR-106,
|
|
1283
|
+
// found live 2026-08-15: every asdf plugin/shim the image bakes into
|
|
1284
|
+
// /opt/asdf-data was missing, so CLI reconcile looped on "No such
|
|
1285
|
+
// plugin: nodejs"). Replicate the Docker semantics explicitly: when the
|
|
1286
|
+
// volume lacks the plugins tree, copy the image's /opt/asdf-data into it
|
|
1287
|
+
// once, ownership preserved. Idempotent — the copied tree is the marker.
|
|
1288
|
+
if (engine.apple) {
|
|
1289
|
+
// Probe at a path the image does NOT contain: mounted over
|
|
1290
|
+
// /opt/asdf-data, a silently-failed volume mount would expose the
|
|
1291
|
+
// image's own baked tree and fake a "seeded" answer. At a neutral
|
|
1292
|
+
// mountpoint the plugins tree can only come from the volume itself,
|
|
1293
|
+
// and any mount failure lands on the safe side (reseeding is
|
|
1294
|
+
// idempotent).
|
|
1295
|
+
const seeded = await runMaintenanceContainer(engine, "seed-probe", [
|
|
1296
|
+
"--rm",
|
|
1297
|
+
"-v",
|
|
1298
|
+
`${ASDF_DATA_VOLUME}:/uai-volume-probe:ro`,
|
|
1299
|
+
STANDARD_IMAGE_TAG,
|
|
1300
|
+
"/usr/bin/test",
|
|
1301
|
+
"-d",
|
|
1302
|
+
"/uai-volume-probe/plugins",
|
|
1303
|
+
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
1304
|
+
// Only `/usr/bin/test`'s own exit 1 means "not seeded". Everything else
|
|
1305
|
+
// — a host-side deadline kill (null), an in-container SIGTERM/SIGKILL
|
|
1306
|
+
// (143/137: live 2026-08-17, a wedged apiserver let the probe run ten
|
|
1307
|
+
// minutes before dying 143), or any CLI failure — answers neither way,
|
|
1308
|
+
// and must never trigger a reseed of a volume that may be full and in
|
|
1309
|
+
// use.
|
|
1310
|
+
if (seeded.code !== 0 && seeded.code !== 1) {
|
|
1311
|
+
return {
|
|
1312
|
+
ok: false,
|
|
1313
|
+
error:
|
|
1314
|
+
"The container runtime did not answer while checking the shared " +
|
|
1315
|
+
`runtime volume (probe exit ${seeded.code ?? "killed"}).`,
|
|
1316
|
+
};
|
|
1317
|
+
}
|
|
1318
|
+
if (seeded.code === 0) {
|
|
1319
|
+
console.log(
|
|
1320
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is seeded`,
|
|
1321
|
+
);
|
|
1322
|
+
}
|
|
1323
|
+
// The positive no-holder proof comes BEFORE every writable attachment
|
|
1324
|
+
// — including the seeder. A doomed writable attach both fails the
|
|
1325
|
+
// operation and can mint the corpse that wedges the apiserver, so a
|
|
1326
|
+
// writer requires a positive `free`; `unknown` (unreadable inventory)
|
|
1327
|
+
// must never read as permission.
|
|
1328
|
+
const holders = await sharedVolumeHolders(engine);
|
|
1329
|
+
if (seeded.code !== 0) {
|
|
1330
|
+
if (holders !== "free") {
|
|
1331
|
+
return {
|
|
1332
|
+
ok: false,
|
|
1333
|
+
error:
|
|
1334
|
+
holders === "held"
|
|
1335
|
+
? `The shared runtime volume ${ASDF_DATA_VOLUME} is unseeded while task containers are running; it cannot be seeded safely until they stop.`
|
|
1336
|
+
: "Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
console.log(
|
|
1340
|
+
`[host-agent] seeding ${ASDF_DATA_VOLUME} from the standard image (probe exit ${seeded.code}; Apple volumes mount empty)`,
|
|
1341
|
+
);
|
|
1342
|
+
const seed = await runMaintenanceContainer(engine, "seed", [
|
|
1343
|
+
"--rm",
|
|
1344
|
+
"--user",
|
|
1345
|
+
"root",
|
|
1346
|
+
"-v",
|
|
1347
|
+
`${ASDF_DATA_VOLUME}:/uai-seed-target`,
|
|
1348
|
+
STANDARD_IMAGE_TAG,
|
|
1349
|
+
"/bin/sh",
|
|
1350
|
+
"-c",
|
|
1351
|
+
"cp -a /opt/asdf-data/. /uai-seed-target/ && chown -R node:node /uai-seed-target",
|
|
1352
|
+
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
1353
|
+
if (seed.code !== 0) {
|
|
1354
|
+
return {
|
|
1355
|
+
ok: false,
|
|
1356
|
+
error: `Could not seed the shared runtime volume from the standard image (exit ${seed.code ?? "spawn"}).`,
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
} else if (holders === "held") {
|
|
1360
|
+
// Proof-first deferral: while running task containers hold the
|
|
1361
|
+
// volume, a writable attach can only fail (and mint a corpse) —
|
|
1362
|
+
// don't attempt it. The stderr-signature deferrals below remain as
|
|
1363
|
+
// the backstop for a task appearing between this check and the
|
|
1364
|
+
// attach.
|
|
1365
|
+
if (await volumeReconcileProofExists()) {
|
|
1366
|
+
console.log(
|
|
1367
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by task containers; deferring maintenance (prior reconcile proof present)`,
|
|
1368
|
+
);
|
|
1369
|
+
return { ok: true };
|
|
1370
|
+
}
|
|
1371
|
+
// First-proof bootstrap. A host with no persisted proof (fresh file,
|
|
1372
|
+
// or upgraded from a version whose proof write failed silently —
|
|
1373
|
+
// live 2026-08-17 on 0.9.26) would otherwise be stuck in a ring: the
|
|
1374
|
+
// proof needs a full reconcile, the reconcile needs the writable
|
|
1375
|
+
// volume, and the volume stays held for as long as tasks run. The
|
|
1376
|
+
// read-only CLI probe attaches fine next to tasks, and EXECUTING
|
|
1377
|
+
// every agent CLI is the same positive evidence the full reconcile
|
|
1378
|
+
// ends with — enough to defer on, and worth persisting as the proof.
|
|
1379
|
+
const missing = await probeVolumeAgentClis();
|
|
1380
|
+
if (missing !== null && missing.length === 0) {
|
|
1381
|
+
await writeVolumeReconcileProof();
|
|
1382
|
+
console.log(
|
|
1383
|
+
`[host-agent] read-only agent-CLI probe passed while task containers hold ${ASDF_DATA_VOLUME}; recording the reconcile proof and deferring maintenance`,
|
|
1384
|
+
);
|
|
1385
|
+
return { ok: true };
|
|
1386
|
+
}
|
|
1387
|
+
// No proof and a failed probe, while tasks hold the volume: a
|
|
1388
|
+
// writable attach here can only fail and mint a corpse, and simply
|
|
1389
|
+
// retrying strands activation forever (review 2026-08-18 round 2 —
|
|
1390
|
+
// recovery, which could settle the holders, runs only after
|
|
1391
|
+
// maintenance succeeds). Ask the activation driver to lifecycle-stop
|
|
1392
|
+
// the holder tasks (resumable) so the next attempt can repair the
|
|
1393
|
+
// volume they cannot safely use anyway.
|
|
1394
|
+
return {
|
|
1395
|
+
ok: false,
|
|
1396
|
+
error:
|
|
1397
|
+
`The shared runtime volume ${ASDF_DATA_VOLUME} needs repair ` +
|
|
1398
|
+
`(agent CLIs failed their probe${missing === null ? "" : `: ${missing.join(", ")} missing`}) ` +
|
|
1399
|
+
"while running tasks hold it; stopping the holder tasks so maintenance can repair it.",
|
|
1400
|
+
remedy: "stop-volume-holders",
|
|
1401
|
+
};
|
|
1402
|
+
} else if (holders === "unknown") {
|
|
1403
|
+
return {
|
|
1404
|
+
ok: false,
|
|
1405
|
+
error:
|
|
1406
|
+
"Could not determine whether task containers hold the shared runtime volume; retrying before any writable attachment.",
|
|
1407
|
+
};
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
const lockPrepared = await prepareAsdfMaintenanceLock();
|
|
1411
|
+
if (!lockPrepared) {
|
|
1412
|
+
if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
|
|
1413
|
+
console.log(
|
|
1414
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
|
|
1415
|
+
);
|
|
1416
|
+
return { ok: true };
|
|
1417
|
+
}
|
|
1418
|
+
return {
|
|
1419
|
+
ok: false,
|
|
1420
|
+
error: `Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
|
|
1421
|
+
};
|
|
1422
|
+
}
|
|
1423
|
+
if (!(await reconcileVolumeAgentClis())) {
|
|
1424
|
+
if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
|
|
1425
|
+
console.log(
|
|
1426
|
+
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
|
|
1427
|
+
);
|
|
1428
|
+
return { ok: true };
|
|
1429
|
+
}
|
|
1430
|
+
return {
|
|
1431
|
+
ok: false,
|
|
1432
|
+
error:
|
|
1433
|
+
"Shared runtime maintenance failed; agent CLIs or Corepack did not pass execution proof.",
|
|
1434
|
+
};
|
|
1435
|
+
}
|
|
473
1436
|
return { ok: true };
|
|
474
1437
|
}
|
|
475
1438
|
return {
|