@runuai/host 0.9.150 → 0.9.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1880 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1837 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +650 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
package/lib/standard-image.ts
CHANGED
|
@@ -23,9 +23,14 @@ import { dirname, join, resolve } from "node:path";
|
|
|
23
23
|
import { fileURLToPath } from "node:url";
|
|
24
24
|
|
|
25
25
|
import {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
appleOneOffNeedsWatch,
|
|
27
|
+
appleOneOffRunArgs,
|
|
28
|
+
askRuntimeOwnerOnce,
|
|
29
|
+
newAppleOneOff,
|
|
30
|
+
settleAppleOneOff,
|
|
31
|
+
watchAppleOneOff,
|
|
32
|
+
} from "./apple-one-off";
|
|
33
|
+
import { recycleAppleRuntimeServices, type AppleRecycleOutcome } from "./apple-runtime-recycle";
|
|
29
34
|
import { detectEngine } from "./engines";
|
|
30
35
|
import { hasStoredEngineAccounts } from "./engine-accounts";
|
|
31
36
|
import {
|
|
@@ -34,6 +39,7 @@ import {
|
|
|
34
39
|
containerRuntimeStandardAssetsRoot,
|
|
35
40
|
containerRuntimeStandardImageRoot,
|
|
36
41
|
pinnedContainerRuntimeProvider,
|
|
42
|
+
requestContainerRuntimeRecheck,
|
|
37
43
|
} from "./container-runtime";
|
|
38
44
|
import { KeyedPromiseTail } from "./keyed-promise-tail";
|
|
39
45
|
import {
|
|
@@ -249,17 +255,59 @@ function selectedEngine(): { command: string; apple: boolean } {
|
|
|
249
255
|
return { command: "docker", apple: false };
|
|
250
256
|
}
|
|
251
257
|
|
|
252
|
-
/**
|
|
253
|
-
*
|
|
254
|
-
*
|
|
258
|
+
/** The most bytes of each stream a standard-image command keeps; the rest is
|
|
259
|
+
* read and dropped, so a child that writes without end holds no more of the
|
|
260
|
+
* runner's memory than this. */
|
|
261
|
+
const RUN_CAPTURE_BYTES = 8 * 1024 * 1024;
|
|
262
|
+
|
|
263
|
+
/** One stream, kept to at most RUN_CAPTURE_BYTES of its encoded bytes; the
|
|
264
|
+
* runner's own diagnostics go through the same bound. */
|
|
265
|
+
class BoundedCapture {
|
|
266
|
+
#chunks: Buffer[] = [];
|
|
267
|
+
#bytes = 0;
|
|
268
|
+
#lastByte: number | null = null;
|
|
269
|
+
|
|
270
|
+
add(chunk: Buffer | string): void {
|
|
271
|
+
const bytes = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk;
|
|
272
|
+
if (bytes.length === 0 || this.#bytes >= RUN_CAPTURE_BYTES) return;
|
|
273
|
+
const kept = bytes.subarray(0, RUN_CAPTURE_BYTES - this.#bytes);
|
|
274
|
+
this.#chunks.push(kept);
|
|
275
|
+
this.#bytes += kept.length;
|
|
276
|
+
this.#lastByte = kept[kept.length - 1]!;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** A line of the runner's own, on a line of its own. */
|
|
280
|
+
addLine(line: string): void {
|
|
281
|
+
this.add(`${this.#lastByte !== null && this.#lastByte !== 0x0a ? "\n" : ""}${line}`);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** What it kept, as text that keeps the bound too: a character the
|
|
285
|
+
* bound cut, or bytes that are not UTF-8, decode to a three-byte
|
|
286
|
+
* replacement character, so past the bound the text ends at its last
|
|
287
|
+
* whole character. */
|
|
288
|
+
text(): string {
|
|
289
|
+
const text = Buffer.concat(this.#chunks).toString("utf8");
|
|
290
|
+
if (Buffer.byteLength(text, "utf8") <= RUN_CAPTURE_BYTES) return text;
|
|
291
|
+
const encoded = Buffer.from(text, "utf8");
|
|
292
|
+
let end = RUN_CAPTURE_BYTES;
|
|
293
|
+
// Back to the first byte of the character the bound cuts.
|
|
294
|
+
while (end > 0 && (encoded[end]! & 0xc0) === 0x80) end -= 1;
|
|
295
|
+
return encoded.subarray(0, end).toString("utf8");
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Run a command to completion, capturing stdio, at most RUN_CAPTURE_BYTES
|
|
300
|
+
* of each. Never rejects and never waits forever: standard-image maintenance
|
|
301
|
+
* is an activation prerequisite, so an unbounded Docker/npm child would
|
|
302
|
+
* strand the entire host in checking. */
|
|
255
303
|
function run(
|
|
256
304
|
command: string,
|
|
257
305
|
args: string[],
|
|
258
306
|
timeoutMs = SHORT_DOCKER_TIMEOUT_MS,
|
|
259
307
|
): Promise<RunResult> {
|
|
260
308
|
return new Promise<RunResult>((resolveRun) => {
|
|
261
|
-
|
|
262
|
-
|
|
309
|
+
const stdout = new BoundedCapture();
|
|
310
|
+
const stderr = new BoundedCapture();
|
|
263
311
|
let settled = false;
|
|
264
312
|
let killTimer: NodeJS.Timeout | null = null;
|
|
265
313
|
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
@@ -268,10 +316,10 @@ function run(
|
|
|
268
316
|
settled = true;
|
|
269
317
|
clearTimeout(timeoutTimer);
|
|
270
318
|
if (killTimer) clearTimeout(killTimer);
|
|
271
|
-
resolveRun({ code, stdout, stderr });
|
|
319
|
+
resolveRun({ code, stdout: stdout.text(), stderr: stderr.text() });
|
|
272
320
|
};
|
|
273
321
|
const timeoutTimer = setTimeout(() => {
|
|
274
|
-
stderr
|
|
322
|
+
stderr.addLine(`${command} timed out after ${timeoutMs}ms`);
|
|
275
323
|
child.kill("SIGTERM");
|
|
276
324
|
killTimer = setTimeout(() => {
|
|
277
325
|
child.kill("SIGKILL");
|
|
@@ -283,16 +331,14 @@ function run(
|
|
|
283
331
|
killTimer.unref();
|
|
284
332
|
}, timeoutMs);
|
|
285
333
|
timeoutTimer.unref();
|
|
286
|
-
child.stdout?.
|
|
287
|
-
|
|
288
|
-
child.stdout?.on("data", (chunk: string) => {
|
|
289
|
-
stdout += chunk;
|
|
334
|
+
child.stdout?.on("data", (chunk: Buffer) => {
|
|
335
|
+
stdout.add(chunk);
|
|
290
336
|
});
|
|
291
|
-
child.stderr?.on("data", (chunk:
|
|
292
|
-
stderr
|
|
337
|
+
child.stderr?.on("data", (chunk: Buffer) => {
|
|
338
|
+
stderr.add(chunk);
|
|
293
339
|
});
|
|
294
340
|
child.on("error", (err: Error) => {
|
|
295
|
-
stderr
|
|
341
|
+
stderr.add(err.message);
|
|
296
342
|
settle(null);
|
|
297
343
|
});
|
|
298
344
|
child.on("close", (code) => {
|
|
@@ -304,8 +350,6 @@ function run(
|
|
|
304
350
|
/** Narrow process-level seam for the timeout regression test. */
|
|
305
351
|
export const runStandardImageCommandForTest = run;
|
|
306
352
|
|
|
307
|
-
let maintenanceContainerSeq = 0;
|
|
308
|
-
|
|
309
353
|
/** How long a maintenance run may sit without its container reaching
|
|
310
354
|
* `running` before the wedge sensor passes judgment. Long HEALTHY operations
|
|
311
355
|
* (reconcile's npm installs, the seed copy) are past `running` within
|
|
@@ -385,35 +429,57 @@ async function appleWedgeVerdict(
|
|
|
385
429
|
* at start time (the exclusive-volume attach, most commonly) leaves a
|
|
386
430
|
* created-never-started container behind, and those corpses wedge the
|
|
387
431
|
* runtime's apiserver (observed live 2026-08-16, twice: run/delete hang
|
|
388
|
-
* while list still answers).
|
|
389
|
-
*
|
|
390
|
-
*
|
|
432
|
+
* while list still answers). So the container is named and marked Uai's
|
|
433
|
+
* (apple-one-off.ts), and a failed run is settled: its container proven
|
|
434
|
+
* gone, deleted until it is, the runtime's services recycled when it will
|
|
435
|
+
* not go; a run whose client never reported its end is watched a while
|
|
436
|
+
* longer for a container created late. Docker's `--rm` is reliable and its
|
|
391
437
|
* containers are not corpse-prone, so the Docker path passes through.
|
|
392
438
|
*
|
|
393
439
|
* Every apple run doubles as the WEDGE SENSOR (the owner's "this needs to
|
|
394
440
|
* be automatic"): if the container has not reached `running` after the
|
|
395
|
-
* probe delay while `list` still answers, the
|
|
396
|
-
*
|
|
397
|
-
*
|
|
398
|
-
*
|
|
441
|
+
* probe delay while `list` still answers, the run reports a distinctive
|
|
442
|
+
* retryable failure. Within the runtime's current activation, the runtime
|
|
443
|
+
* services are first recycled in place (see apple-runtime-recycle.ts), and
|
|
444
|
+
* the activation recovers every task after, its retry loop re-running
|
|
445
|
+
* maintenance against the fresh runtime. Within an activation that is no
|
|
446
|
+
* longer current, nothing restarts the runtime: the current one is coming,
|
|
447
|
+
* with its own maintenance and recovery. Outside any (the hourly pass, a
|
|
448
|
+
* task's start, an engine's sign-in), nothing restarts the runtime either:
|
|
449
|
+
* its owner is asked to recheck it, whose activation does (#p2jdax).
|
|
399
450
|
*/
|
|
400
451
|
async function runMaintenanceContainer(
|
|
401
452
|
engine: { command: string; apple: boolean },
|
|
402
453
|
purpose: string,
|
|
403
454
|
argsAfterRun: string[],
|
|
404
455
|
timeoutMs: number,
|
|
456
|
+
activation: ActivationIsCurrent | null,
|
|
405
457
|
): Promise<RunResult> {
|
|
406
458
|
if (!engine.apple) {
|
|
407
459
|
return run(engine.command, ["run", ...argsAfterRun], timeoutMs);
|
|
408
460
|
}
|
|
409
|
-
|
|
410
|
-
const name =
|
|
461
|
+
const oneOff = newAppleOneOff(purpose);
|
|
462
|
+
const { name } = oneOff;
|
|
463
|
+
const cli = (args: string[], cliTimeoutMs: number) => run(engine.command, args, cliTimeoutMs);
|
|
464
|
+
// The remedy outside activation, and for a watch (which outlives any
|
|
465
|
+
// activation): ask the runtime's owner, once for this run.
|
|
466
|
+
const askOwner = askRuntimeOwnerOnce(
|
|
467
|
+
requestContainerRuntimeRecheck,
|
|
468
|
+
`maintenance container ${name} found the Apple runtime wedged, or could not be removed`,
|
|
469
|
+
);
|
|
470
|
+
// The remedy within an activation: the runtime's services restarted, only
|
|
471
|
+
// while that activation is current.
|
|
472
|
+
const notCurrent = "its activation is no longer the current one";
|
|
473
|
+
const restart = async (): Promise<AppleRecycleOutcome> =>
|
|
474
|
+
activation?.() ? recycleAppleRuntimeServices({ cli }) : { recycled: false, reason: notCurrent };
|
|
475
|
+
let wedgedUnremedied = false;
|
|
411
476
|
let runSettled = false;
|
|
412
477
|
const running = run(
|
|
413
478
|
engine.command,
|
|
414
|
-
//
|
|
415
|
-
//
|
|
416
|
-
|
|
479
|
+
// Its labels are what authorize deleting this container later (here, by
|
|
480
|
+
// its run's token; in the recycle sweep, by Uai's) — a name alone is
|
|
481
|
+
// forgeable.
|
|
482
|
+
["run", ...appleOneOffRunArgs(oneOff), ...argsAfterRun],
|
|
417
483
|
timeoutMs,
|
|
418
484
|
).then((outcome) => {
|
|
419
485
|
runSettled = true;
|
|
@@ -424,9 +490,16 @@ async function runMaintenanceContainer(
|
|
|
424
490
|
if (runSettled) return null;
|
|
425
491
|
if (!(await appleWedgeVerdict(engine, name))) return null;
|
|
426
492
|
if (runSettled) return null;
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
493
|
+
if (activation === null || !activation()) {
|
|
494
|
+
wedgedUnremedied = true;
|
|
495
|
+
const reason = activation === null ? (await askOwner()).reason : notCurrent;
|
|
496
|
+
return {
|
|
497
|
+
code: null,
|
|
498
|
+
stdout: "",
|
|
499
|
+
stderr: `apple container runtime is wedged; ${reason} — retry this operation`,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
const outcome = await restart();
|
|
430
503
|
// The hung CLI client may exit once its apiserver dies; give it a short
|
|
431
504
|
// grace so a real result wins, then synthesize the retryable failure.
|
|
432
505
|
const graced = await Promise.race([
|
|
@@ -447,61 +520,26 @@ async function runMaintenanceContainer(
|
|
|
447
520
|
sensed.then((synthesized) => synthesized ?? running),
|
|
448
521
|
]);
|
|
449
522
|
if (result.code !== 0) {
|
|
523
|
+
const watched = { cli, recycle: askOwner };
|
|
524
|
+
if (wedgedUnremedied) {
|
|
525
|
+
// Nothing can be deleted until a current activation has restarted the
|
|
526
|
+
// runtime: its name is only watched, and this pass (which that
|
|
527
|
+
// activation's own may wait behind) ends now.
|
|
528
|
+
void watchAppleOneOff(watched, oneOff, { lateCreation: true });
|
|
529
|
+
return result;
|
|
530
|
+
}
|
|
450
531
|
// The run may have failed BECAUSE this generated name already existed —
|
|
451
532
|
// then the container is not ours and the ownership label was never
|
|
452
|
-
// applied (review 2026-08-18 round 3)
|
|
453
|
-
//
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
.some((line) => line.trim() === `Error: container not found: ${name}`)
|
|
463
|
-
) {
|
|
464
|
-
verdict = "absent";
|
|
465
|
-
} else if (inspected.code === 0) {
|
|
466
|
-
try {
|
|
467
|
-
const parsed = JSON.parse(inspected.stdout) as Array<{
|
|
468
|
-
id?: unknown;
|
|
469
|
-
configuration?: { labels?: Record<string, unknown> };
|
|
470
|
-
}>;
|
|
471
|
-
if (Array.isArray(parsed) && parsed.length === 1 && parsed[0]?.id === name) {
|
|
472
|
-
verdict =
|
|
473
|
-
parsed[0]?.configuration?.labels?.[MAINTENANCE_CONTAINER_LABEL] === "1"
|
|
474
|
-
? "ours"
|
|
475
|
-
: "foreign";
|
|
476
|
-
}
|
|
477
|
-
} catch {
|
|
478
|
-
verdict = "unknown";
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
if (verdict === "ours") {
|
|
482
|
-
const removed = await run(
|
|
483
|
-
engine.command,
|
|
484
|
-
["delete", "--force", name],
|
|
485
|
-
30_000,
|
|
486
|
-
);
|
|
487
|
-
if (removed.code !== 0) {
|
|
488
|
-
console.warn(
|
|
489
|
-
`[host-agent] could not remove maintenance container ${name} after a ` +
|
|
490
|
-
`failed run (exit ${removed.code ?? "spawn"}); a created-never-started ` +
|
|
491
|
-
`container can wedge the Apple runtime. ${removed.stderr.trim().slice(0, 200)}`,
|
|
492
|
-
);
|
|
493
|
-
}
|
|
494
|
-
} else if (verdict === "foreign") {
|
|
495
|
-
console.warn(
|
|
496
|
-
`[host-agent] a container named ${name} exists WITHOUT Uai's maintenance ` +
|
|
497
|
-
"ownership label — the failed run likely collided with it; leaving it untouched",
|
|
498
|
-
);
|
|
499
|
-
} else if (verdict === "unknown") {
|
|
500
|
-
console.warn(
|
|
501
|
-
`[host-agent] could not prove ownership of maintenance container ${name} ` +
|
|
502
|
-
`after a failed run; leaving it untouched (inspect exit ${inspected.code ?? "spawn"})`,
|
|
503
|
-
);
|
|
504
|
-
}
|
|
533
|
+
// applied (review 2026-08-18 round 3): only a positive label proof
|
|
534
|
+
// authorizes deleting it, and an unobservable one is never recycled for.
|
|
535
|
+
const settled = await settleAppleOneOff(
|
|
536
|
+
{ cli, recycle: activation ? restart : askOwner },
|
|
537
|
+
oneOff,
|
|
538
|
+
);
|
|
539
|
+
// Unproven, or its client never reported its end (the container may yet
|
|
540
|
+
// be created): watched until it is settled.
|
|
541
|
+
const lateCreation = result.code === null;
|
|
542
|
+
if (appleOneOffNeedsWatch(settled, lateCreation)) void watchAppleOneOff(watched, oneOff, { lateCreation });
|
|
505
543
|
}
|
|
506
544
|
return result;
|
|
507
545
|
}
|
|
@@ -533,7 +571,7 @@ const VOLUME_AGENT_CLIS: {
|
|
|
533
571
|
* proof: another process can begin maintenance after a read-only check. The
|
|
534
572
|
* locked reconcile below always performs its own proof before releasing.
|
|
535
573
|
*/
|
|
536
|
-
async function probeVolumeAgentClis(): Promise<string[] | null> {
|
|
574
|
+
async function probeVolumeAgentClis(activation: ActivationIsCurrent | null): Promise<string[] | null> {
|
|
537
575
|
const checks = VOLUME_AGENT_CLIS.map((cli) => {
|
|
538
576
|
const missing = `echo "UAI_MISSING:${cli.bin}"`;
|
|
539
577
|
if (cli.expectedVersionOutput === undefined) {
|
|
@@ -565,7 +603,7 @@ async function probeVolumeAgentClis(): Promise<string[] | null> {
|
|
|
565
603
|
"/bin/bash",
|
|
566
604
|
"-c",
|
|
567
605
|
checks,
|
|
568
|
-
], SHORT_DOCKER_TIMEOUT_MS);
|
|
606
|
+
], SHORT_DOCKER_TIMEOUT_MS, activation);
|
|
569
607
|
if (check.code !== 0) {
|
|
570
608
|
console.warn(
|
|
571
609
|
`[host-agent] could not check agent CLIs in ${ASDF_DATA_VOLUME} (exit ` +
|
|
@@ -586,7 +624,7 @@ async function probeVolumeAgentClis(): Promise<string[] | null> {
|
|
|
586
624
|
* root-created file; opening it read-only permits flock, then metadata repair
|
|
587
625
|
* occurs while holding that same inode (never replace it under a waiter).
|
|
588
626
|
*/
|
|
589
|
-
async function prepareAsdfMaintenanceLock(): Promise<boolean> {
|
|
627
|
+
async function prepareAsdfMaintenanceLock(activation: ActivationIsCurrent | null): Promise<boolean> {
|
|
590
628
|
const command =
|
|
591
629
|
`set -eu; lock=${ASDF_MAINTENANCE_LOCK_PATH}; ` +
|
|
592
630
|
`if [ -L "$lock" ]; then echo "unsafe symlink lock: $lock" >&2; exit 1; fi; ` +
|
|
@@ -608,7 +646,7 @@ async function prepareAsdfMaintenanceLock(): Promise<boolean> {
|
|
|
608
646
|
"/bin/bash",
|
|
609
647
|
"-c",
|
|
610
648
|
command,
|
|
611
|
-
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
649
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
612
650
|
if (result.code === 0) return true;
|
|
613
651
|
lastMaintenanceStderr = result.stderr;
|
|
614
652
|
console.warn(
|
|
@@ -898,8 +936,8 @@ export function standardImageVolumeMaintenanceCommandForTest(
|
|
|
898
936
|
);
|
|
899
937
|
}
|
|
900
938
|
|
|
901
|
-
async function reconcileVolumeAgentClis(): Promise<boolean> {
|
|
902
|
-
const missingBefore = await probeVolumeAgentClis();
|
|
939
|
+
async function reconcileVolumeAgentClis(activation: ActivationIsCurrent | null): Promise<boolean> {
|
|
940
|
+
const missingBefore = await probeVolumeAgentClis(activation);
|
|
903
941
|
if (missingBefore?.length) {
|
|
904
942
|
console.log(
|
|
905
943
|
`[host-agent] repairing ${ASDF_DATA_VOLUME}: agent CLI(s) ` +
|
|
@@ -937,7 +975,7 @@ async function reconcileVolumeAgentClis(): Promise<boolean> {
|
|
|
937
975
|
"/bin/bash",
|
|
938
976
|
"-c",
|
|
939
977
|
command,
|
|
940
|
-
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
978
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
941
979
|
if (reconcile.code === 0) {
|
|
942
980
|
console.log(
|
|
943
981
|
`[host-agent] agent CLIs and Corepack current on ${ASDF_DATA_VOLUME}: ` +
|
|
@@ -1175,7 +1213,15 @@ function markAgentClisReady(): void {
|
|
|
1175
1213
|
* surface the reason). Resolves `agentClisReady` on every exit path.
|
|
1176
1214
|
*/
|
|
1177
1215
|
export function ensureStandardImage(
|
|
1178
|
-
opts: {
|
|
1216
|
+
opts: {
|
|
1217
|
+
maxAgeMs?: number;
|
|
1218
|
+
/** Only the runtime's activation passes this, answering whether it is
|
|
1219
|
+
* still the current activation: within it alone, and only while it is
|
|
1220
|
+
* current, may a wedged Apple runtime's services be restarted, since
|
|
1221
|
+
* its recovery follows its maintenance. Every other pass asks the
|
|
1222
|
+
* runtime's owner for that activation instead (#p2jdax). */
|
|
1223
|
+
withinActivation?: ActivationIsCurrent;
|
|
1224
|
+
} = {},
|
|
1179
1225
|
): Promise<StandardImageResult> {
|
|
1180
1226
|
// ADR-133 sidecar mode: a task machine has no container image to keep —
|
|
1181
1227
|
// its agent CLIs are baked into the machine image. Live 2026-09-10
|
|
@@ -1203,9 +1249,12 @@ export function ensureStandardImage(
|
|
|
1203
1249
|
// Startup, runtime recovery, and a newly-arrived task can all discover the
|
|
1204
1250
|
// same daemon transition. One shared volume repair/build must own that
|
|
1205
1251
|
// window; concurrent maintenance containers can otherwise replace the same
|
|
1206
|
-
// CLI shims underneath each other and defeat agentClisReady's fence.
|
|
1207
|
-
|
|
1208
|
-
|
|
1252
|
+
// CLI shims underneath each other and defeat agentClisReady's fence. The
|
|
1253
|
+
// activation's pass never joins one begun outside it (which may not
|
|
1254
|
+
// restart the runtime): it runs after it, as its own.
|
|
1255
|
+
const activation = opts.withinActivation ?? null;
|
|
1256
|
+
return ensureQueue.run(activation ? `activation:${key}` : key, () =>
|
|
1257
|
+
ensureStandardImageInner(engines, activation)
|
|
1209
1258
|
.then((result) => {
|
|
1210
1259
|
if (result.ok) lastOkByKey.set(key, { at: Date.now(), result });
|
|
1211
1260
|
else lastOkByKey.delete(key);
|
|
@@ -1222,6 +1271,10 @@ export function ensureStandardImage(
|
|
|
1222
1271
|
/** Last successful pass per engine configuration (in-process). */
|
|
1223
1272
|
const lastOkByKey = new Map<string, { at: number; result: StandardImageResult }>();
|
|
1224
1273
|
|
|
1274
|
+
/** The runtime's activation, asked whether it is still the current one: what
|
|
1275
|
+
* alone may restart a wedged Apple runtime's services (#p2jdax). */
|
|
1276
|
+
export type ActivationIsCurrent = () => boolean;
|
|
1277
|
+
|
|
1225
1278
|
/** Tests: forget every remembered pass. */
|
|
1226
1279
|
export function resetStandardImageMemo(): void {
|
|
1227
1280
|
lastOkByKey.clear();
|
|
@@ -1232,6 +1285,7 @@ export const MAINTENANCE_INTERVAL_MS = 60 * 60_000;
|
|
|
1232
1285
|
|
|
1233
1286
|
async function ensureStandardImageInner(
|
|
1234
1287
|
engines: OptionalEngineConfiguration,
|
|
1288
|
+
activation: ActivationIsCurrent | null,
|
|
1235
1289
|
): Promise<StandardImageResult> {
|
|
1236
1290
|
// 1. Shared asdf data volume — idempotent under Docker; the Apple CLI
|
|
1237
1291
|
// errors on an existing name, which is the same healthy outcome.
|
|
@@ -1459,7 +1513,7 @@ async function ensureStandardImageInner(
|
|
|
1459
1513
|
"/usr/bin/test",
|
|
1460
1514
|
"-d",
|
|
1461
1515
|
"/uai-volume-probe/plugins",
|
|
1462
|
-
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
1516
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
1463
1517
|
// Only `/usr/bin/test`'s own exit 1 means "not seeded". Everything else
|
|
1464
1518
|
// — a host-side deadline kill (null), an in-container SIGTERM/SIGKILL
|
|
1465
1519
|
// (143/137: live 2026-08-17, a wedged apiserver let the probe run ten
|
|
@@ -1508,7 +1562,7 @@ async function ensureStandardImageInner(
|
|
|
1508
1562
|
"/bin/sh",
|
|
1509
1563
|
"-c",
|
|
1510
1564
|
"cp -a /opt/asdf-data/. /uai-seed-target/ && chown -R node:node /uai-seed-target",
|
|
1511
|
-
], CLI_MAINTENANCE_TIMEOUT_MS);
|
|
1565
|
+
], CLI_MAINTENANCE_TIMEOUT_MS, activation);
|
|
1512
1566
|
if (seed.code !== 0) {
|
|
1513
1567
|
return {
|
|
1514
1568
|
ok: false,
|
|
@@ -1535,7 +1589,7 @@ async function ensureStandardImageInner(
|
|
|
1535
1589
|
// read-only CLI probe attaches fine next to tasks, and EXECUTING
|
|
1536
1590
|
// every agent CLI is the same positive evidence the full reconcile
|
|
1537
1591
|
// ends with — enough to defer on, and worth persisting as the proof.
|
|
1538
|
-
const missing = await probeVolumeAgentClis();
|
|
1592
|
+
const missing = await probeVolumeAgentClis(activation);
|
|
1539
1593
|
if (missing !== null && missing.length === 0) {
|
|
1540
1594
|
await writeVolumeReconcileProof();
|
|
1541
1595
|
console.log(
|
|
@@ -1566,7 +1620,7 @@ async function ensureStandardImageInner(
|
|
|
1566
1620
|
};
|
|
1567
1621
|
}
|
|
1568
1622
|
}
|
|
1569
|
-
const lockPrepared = await prepareAsdfMaintenanceLock();
|
|
1623
|
+
const lockPrepared = await prepareAsdfMaintenanceLock(activation);
|
|
1570
1624
|
if (!lockPrepared) {
|
|
1571
1625
|
if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
|
|
1572
1626
|
console.log(
|
|
@@ -1579,7 +1633,7 @@ async function ensureStandardImageInner(
|
|
|
1579
1633
|
error: `Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
|
|
1580
1634
|
};
|
|
1581
1635
|
}
|
|
1582
|
-
if (!(await reconcileVolumeAgentClis())) {
|
|
1636
|
+
if (!(await reconcileVolumeAgentClis(activation))) {
|
|
1583
1637
|
if (await volumeBusyDeferral(engine, lastMaintenanceStderr)) {
|
|
1584
1638
|
console.log(
|
|
1585
1639
|
`[host-agent] shared runtime volume ${ASDF_DATA_VOLUME} is held by running task containers; deferring maintenance (prior reconcile proof present)`,
|
|
@@ -17,22 +17,49 @@
|
|
|
17
17
|
*/
|
|
18
18
|
export type StoppedTaskReason = "manual";
|
|
19
19
|
|
|
20
|
+
interface StoppedTaskState {
|
|
21
|
+
status: string;
|
|
22
|
+
runEpoch: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
interface PendingStoppedTaskStatus {
|
|
26
|
+
reason: StoppedTaskReason | undefined;
|
|
27
|
+
runEpoch: number | undefined;
|
|
28
|
+
}
|
|
29
|
+
|
|
20
30
|
export class StoppedTaskStatusQueue {
|
|
21
|
-
private readonly pending = new Map<string,
|
|
31
|
+
private readonly pending = new Map<string, PendingStoppedTaskStatus>();
|
|
22
32
|
|
|
23
33
|
constructor(
|
|
24
|
-
private readonly
|
|
34
|
+
private readonly stateForTask: (
|
|
25
35
|
taskId: string,
|
|
26
|
-
) =>
|
|
36
|
+
) => StoppedTaskState | null | undefined,
|
|
27
37
|
) {}
|
|
28
38
|
|
|
29
|
-
enqueue(
|
|
39
|
+
enqueue(
|
|
40
|
+
taskId: string,
|
|
41
|
+
reason?: StoppedTaskReason,
|
|
42
|
+
runEpoch?: number,
|
|
43
|
+
): void {
|
|
44
|
+
const previous = this.pending.get(taskId);
|
|
45
|
+
const previousEpoch = previous?.runEpoch ?? 0;
|
|
46
|
+
const nextEpoch = runEpoch ?? 0;
|
|
47
|
+
// A command result can arrive after the next assignment has already
|
|
48
|
+
// queued its own status. Run epochs are monotonic, so an older result must
|
|
49
|
+
// not replace the newer pending truth (or donate its reason to it).
|
|
50
|
+
if (previous !== undefined && nextEpoch < previousEpoch) return;
|
|
30
51
|
// A reason, once known, STICKS across a re-enqueue. The same task can be
|
|
31
52
|
// queued twice before a flush — an operator Stop, then a quarantine sweep
|
|
32
53
|
// that finds its container already down — and the automatic pass arriving
|
|
33
54
|
// second does not make the human's decision retroactively automatic.
|
|
34
55
|
// Dropping it would silently hand the task back to every collaborator.
|
|
35
|
-
this.pending.set(taskId,
|
|
56
|
+
this.pending.set(taskId, {
|
|
57
|
+
reason:
|
|
58
|
+
previous !== undefined && previousEpoch === nextEpoch
|
|
59
|
+
? reason ?? previous.reason
|
|
60
|
+
: reason,
|
|
61
|
+
runEpoch,
|
|
62
|
+
});
|
|
36
63
|
}
|
|
37
64
|
|
|
38
65
|
/** A failed taskUp may have stopped a legacy runtime before recreation lost
|
|
@@ -46,16 +73,47 @@ export class StoppedTaskStatusQueue {
|
|
|
46
73
|
const task = (input as { task?: unknown }).task;
|
|
47
74
|
if (!task || typeof task !== "object" || Array.isArray(task)) return false;
|
|
48
75
|
const taskId = (task as { id?: unknown }).id;
|
|
49
|
-
|
|
76
|
+
const state = typeof taskId === "string" ? this.stateForTask(taskId) : null;
|
|
77
|
+
if (typeof taskId !== "string" || state?.status !== "stopped") {
|
|
50
78
|
return false;
|
|
51
79
|
}
|
|
52
|
-
|
|
80
|
+
const hasRunEpoch = Object.hasOwn(task, "runEpoch");
|
|
81
|
+
const inputRunEpoch = (task as { runEpoch?: unknown }).runEpoch;
|
|
82
|
+
if (hasRunEpoch) {
|
|
83
|
+
if (
|
|
84
|
+
!Number.isSafeInteger(inputRunEpoch) ||
|
|
85
|
+
(inputRunEpoch as number) < 1 ||
|
|
86
|
+
inputRunEpoch !== state.runEpoch
|
|
87
|
+
) {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
} else if (state.runEpoch !== 0) {
|
|
91
|
+
// Never relabel a delayed v0 taskUp result as the current v1 run.
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
const runEpoch = hasRunEpoch ? (inputRunEpoch as number) : undefined;
|
|
95
|
+
this.enqueue(taskId, undefined, runEpoch);
|
|
53
96
|
return true;
|
|
54
97
|
}
|
|
55
98
|
|
|
56
|
-
flush(
|
|
57
|
-
|
|
58
|
-
|
|
99
|
+
flush(
|
|
100
|
+
publish: (
|
|
101
|
+
taskId: string,
|
|
102
|
+
reason?: StoppedTaskReason,
|
|
103
|
+
runEpoch?: number,
|
|
104
|
+
) => void,
|
|
105
|
+
): void {
|
|
106
|
+
for (const [taskId, pending] of [...this.pending]) {
|
|
107
|
+
const state = this.stateForTask(taskId);
|
|
108
|
+
if (
|
|
109
|
+
state?.status === "stopped" &&
|
|
110
|
+
(pending.runEpoch === undefined
|
|
111
|
+
? state.runEpoch === 0
|
|
112
|
+
: state.runEpoch === pending.runEpoch)
|
|
113
|
+
) {
|
|
114
|
+
if (pending.runEpoch === undefined) publish(taskId, pending.reason);
|
|
115
|
+
else publish(taskId, pending.reason, pending.runEpoch);
|
|
116
|
+
}
|
|
59
117
|
this.pending.delete(taskId);
|
|
60
118
|
}
|
|
61
119
|
}
|