@runuai/host 0.9.168 → 0.9.170
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agent-cli.ts +82 -4
- package/lib/agent-errors.ts +22 -0
- package/lib/agents/codex.ts +222 -41
- package/lib/agents/machine-durable.ts +8 -2
- package/lib/agents/types.ts +16 -2
- package/lib/disk-pressure.ts +42 -0
- package/lib/orchestrator.ts +72 -33
- package/lib/standard-image-cleanup.ts +117 -0
- package/lib/standard-image.ts +31 -9
- package/lib/task-environment/types.ts +20 -15
- package/lib/task-project-ops.ts +17 -7
- package/package.json +1 -1
- package/src/main.ts +17 -0
- package/src/protocol.ts +19 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { statfs } from "node:fs/promises";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import type { HostCapabilities } from "../src/protocol";
|
|
5
|
+
|
|
6
|
+
type DiskSample = NonNullable<HostCapabilities["disk"]>;
|
|
7
|
+
const dockerInfo = (args: string[]): Promise<string | null> => new Promise(resolve => {
|
|
8
|
+
execFile("docker", args, { timeout: 5_000, maxBuffer: 4096 }, (error, stdout) => {
|
|
9
|
+
resolve(error ? null : stdout.trim());
|
|
10
|
+
});
|
|
11
|
+
});
|
|
12
|
+
async function dockerRoot(): Promise<string | null> {
|
|
13
|
+
// A remote daemon (including Docker Desktop's VM) has no local filesystem
|
|
14
|
+
// we can measure. Never report a coincidentally matching local path.
|
|
15
|
+
if (process.platform !== "linux") return null;
|
|
16
|
+
const endpoint = process.env.DOCKER_HOST && !process.env.DOCKER_CONTEXT
|
|
17
|
+
? process.env.DOCKER_HOST
|
|
18
|
+
: await dockerInfo(["context", "inspect", "--format", "{{.Endpoints.docker.Host}}"]);
|
|
19
|
+
if (!endpoint?.startsWith("unix://")) return null;
|
|
20
|
+
// Do not guess /var/lib/docker: data-root can be configured elsewhere.
|
|
21
|
+
const root = await dockerInfo(["info", "--format", "{{.DockerRootDir}}"]);
|
|
22
|
+
return root?.startsWith("/") ? root : null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function measure(path: string | null): Promise<DiskSample["home"]> {
|
|
26
|
+
if (!path) return null;
|
|
27
|
+
try {
|
|
28
|
+
const fs = await statfs(path);
|
|
29
|
+
const totalBytes = fs.blocks * fs.bsize;
|
|
30
|
+
// bavail (not bfree) accounts for root-reserved blocks: the host's
|
|
31
|
+
// unprivileged updater cannot use those even though root can.
|
|
32
|
+
const availableBytes = Math.max(0, fs.bavail * fs.bsize);
|
|
33
|
+
if (!Number.isSafeInteger(totalBytes) || totalBytes <= 0 || !Number.isSafeInteger(availableBytes) || availableBytes < 0 || availableBytes > totalBytes) return null;
|
|
34
|
+
return { totalBytes, availableBytes };
|
|
35
|
+
} catch { return null; }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function sampleDiskPressure(): Promise<DiskSample> {
|
|
39
|
+
const [home, docker] = await Promise.all([measure(homedir()), dockerRoot().then(measure)]);
|
|
40
|
+
// No local paths are sent to the cloud; an unavailable probe is unknown.
|
|
41
|
+
return { sampledAt: Date.now(), home, docker };
|
|
42
|
+
}
|
package/lib/orchestrator.ts
CHANGED
|
@@ -40,6 +40,7 @@ import { inArray } from "drizzle-orm";
|
|
|
40
40
|
|
|
41
41
|
import { getDb, schema } from "./db";
|
|
42
42
|
import { newId } from "./ulid";
|
|
43
|
+
import { agentErrorMessage, reportAgentError } from "./agent-errors";
|
|
43
44
|
import { mockAgentFactory } from "./agents/mock";
|
|
44
45
|
import { realAgentFactory } from "./agents/factory";
|
|
45
46
|
import { resolveAgentMode } from "./agents/mode";
|
|
@@ -50,6 +51,7 @@ import {
|
|
|
50
51
|
} from "./agents/session-state";
|
|
51
52
|
import {
|
|
52
53
|
type AgentEvent,
|
|
54
|
+
type AgentEventHandler,
|
|
53
55
|
type AgentSession,
|
|
54
56
|
type AgentSessionFactory,
|
|
55
57
|
type Roster,
|
|
@@ -709,7 +711,11 @@ const BROWSER_RETRY_COOLDOWN_MS = 60_000;
|
|
|
709
711
|
* poisons the runtime generation synchronously; normal detection/recovery owns
|
|
710
712
|
* the eventual ready edge and deferred-prompt replay.
|
|
711
713
|
*/
|
|
712
|
-
async function agentTerminationLostContainerRuntime(): Promise<boolean> {
|
|
714
|
+
async function agentTerminationLostContainerRuntime(channel: Channel): Promise<boolean> {
|
|
715
|
+
// Machine/self tasks have no Docker/Apple daemon. A failed Docker probe
|
|
716
|
+
// there would turn a real engine failure into silent runtime recovery.
|
|
717
|
+
if (isEnvironmentTransportProvider(channel.agentHandle?.descriptor.locator.provider) ||
|
|
718
|
+
isEnvironmentTransportProvider(getHostTask(channel.taskId)?.environmentProvider)) return false;
|
|
713
719
|
if (!backgroundContainerWorkAllowed()) return true;
|
|
714
720
|
let reachable = false;
|
|
715
721
|
try {
|
|
@@ -809,6 +815,7 @@ export interface OrchestratorTaskStopResult {
|
|
|
809
815
|
}
|
|
810
816
|
|
|
811
817
|
export class Orchestrator {
|
|
818
|
+
private readonly sessionEventHandlers = new WeakMap<AgentSession, AgentEventHandler>();
|
|
812
819
|
private readonly channels = new Map<string, Channel>();
|
|
813
820
|
private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
|
|
814
821
|
/** Teardown tombstones prevent a cloud ensure from reopening a task between
|
|
@@ -2915,7 +2922,7 @@ export class Orchestrator {
|
|
|
2915
2922
|
// transport emission order across that await; otherwise the later boundary
|
|
2916
2923
|
// can consume/reorder prompt state while the earlier error still owns it.
|
|
2917
2924
|
let eventTail: Promise<void> = Promise.resolve();
|
|
2918
|
-
|
|
2925
|
+
const onEvent: AgentEventHandler = (event) => {
|
|
2919
2926
|
this.markChannelActivity(channel.taskId);
|
|
2920
2927
|
// Recycle/teardown deletes the old session before calling close().
|
|
2921
2928
|
// Adapters deliberately emit a final exit event; without this identity
|
|
@@ -2957,7 +2964,9 @@ export class Orchestrator {
|
|
|
2957
2964
|
() => channel.eventOperations.delete(operation),
|
|
2958
2965
|
() => channel.eventOperations.delete(operation),
|
|
2959
2966
|
);
|
|
2960
|
-
}
|
|
2967
|
+
};
|
|
2968
|
+
this.sessionEventHandlers.set(session, onEvent);
|
|
2969
|
+
session.onEvent(onEvent);
|
|
2961
2970
|
}
|
|
2962
2971
|
|
|
2963
2972
|
/**
|
|
@@ -4616,11 +4625,11 @@ export class Orchestrator {
|
|
|
4616
4625
|
}
|
|
4617
4626
|
}
|
|
4618
4627
|
|
|
4619
|
-
/** Consume
|
|
4620
|
-
private decrementActiveTurns(channel: Channel, agentId: string): number {
|
|
4628
|
+
/** Consume deliveries settled by one provider boundary. */
|
|
4629
|
+
private decrementActiveTurns(channel: Channel, agentId: string, settles = 1): number {
|
|
4621
4630
|
const remaining = Math.max(
|
|
4622
4631
|
0,
|
|
4623
|
-
(channel.activeTurns.get(agentId) ?? 1) -
|
|
4632
|
+
(channel.activeTurns.get(agentId) ?? 1) - settles,
|
|
4624
4633
|
);
|
|
4625
4634
|
if (remaining === 0) channel.activeTurns.delete(agentId);
|
|
4626
4635
|
else channel.activeTurns.set(agentId, remaining);
|
|
@@ -4759,26 +4768,29 @@ export class Orchestrator {
|
|
|
4759
4768
|
this.incrementActiveTurns(channel, agentId);
|
|
4760
4769
|
void session.send(text).catch((err: unknown) => {
|
|
4761
4770
|
if (turn) journal?.failed(sessionId, turn.turnId);
|
|
4771
|
+
const diagnosticMessage = `Could not send the message to ${agentId}: ${err instanceof Error ? err.message : String(err)}`;
|
|
4772
|
+
reportAgentError(diagnosticMessage, { taskId: channel.taskId, agentId, engine: session.kind },
|
|
4773
|
+
{ capture: !(err instanceof ContainerRuntimeUnavailableError) });
|
|
4762
4774
|
if (channel.sessions.get(agentId) === session) {
|
|
4763
|
-
this.removeAcceptedPrompt(channel, agentId, accepted);
|
|
4764
|
-
const remaining = this.decrementActiveTurns(channel, agentId);
|
|
4765
|
-
if (
|
|
4766
|
-
remaining === 0 &&
|
|
4767
|
-
channel.lastPrompt.get(agentId) === accepted
|
|
4768
|
-
) {
|
|
4769
|
-
channel.lastPrompt.delete(agentId);
|
|
4770
|
-
}
|
|
4771
4775
|
if (err instanceof ContainerRuntimeUnavailableError) {
|
|
4776
|
+
this.removeAcceptedPrompt(channel, agentId, accepted);
|
|
4777
|
+
const remaining = this.decrementActiveTurns(channel, agentId);
|
|
4778
|
+
if (remaining === 0 && channel.lastPrompt.get(agentId) === accepted) {
|
|
4779
|
+
channel.lastPrompt.delete(agentId);
|
|
4780
|
+
}
|
|
4772
4781
|
this.deferPromptUntilRuntimeReady(channel, agentId, accepted);
|
|
4773
4782
|
} else {
|
|
4774
|
-
|
|
4783
|
+
// Share the ordered terminal-event path with adapter failures. It
|
|
4784
|
+
// retires the active count and publishes the error/aborted boundary;
|
|
4785
|
+
// a concurrent adapter error or exit can consume it only once.
|
|
4786
|
+
this.sessionEventHandlers.get(session)?.({
|
|
4787
|
+
type: "error",
|
|
4788
|
+
message: agentErrorMessage(diagnosticMessage),
|
|
4789
|
+
diagnosticMessage,
|
|
4790
|
+
aborted: true,
|
|
4791
|
+
});
|
|
4775
4792
|
}
|
|
4776
4793
|
}
|
|
4777
|
-
console.warn(
|
|
4778
|
-
`[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
|
|
4779
|
-
err instanceof Error ? err.message : String(err)
|
|
4780
|
-
}`,
|
|
4781
|
-
);
|
|
4782
4794
|
});
|
|
4783
4795
|
}
|
|
4784
4796
|
|
|
@@ -4794,13 +4806,14 @@ export class Orchestrator {
|
|
|
4794
4806
|
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
4795
4807
|
}
|
|
4796
4808
|
|
|
4797
|
-
private
|
|
4809
|
+
private completeAcceptedPrompts(
|
|
4798
4810
|
channel: Channel,
|
|
4799
4811
|
agentId: string,
|
|
4812
|
+
settles = 1,
|
|
4800
4813
|
): void {
|
|
4801
4814
|
const prompts = channel.acceptedPrompts.get(agentId);
|
|
4802
4815
|
if (!prompts) return;
|
|
4803
|
-
prompts.
|
|
4816
|
+
prompts.splice(0, settles);
|
|
4804
4817
|
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
4805
4818
|
}
|
|
4806
4819
|
|
|
@@ -5257,8 +5270,13 @@ export class Orchestrator {
|
|
|
5257
5270
|
});
|
|
5258
5271
|
break;
|
|
5259
5272
|
}
|
|
5273
|
+
case "notice": {
|
|
5274
|
+
this.emitSystemNote(channel.taskId, agentErrorMessage(`${agentId}: ${event.message}`));
|
|
5275
|
+
break;
|
|
5276
|
+
}
|
|
5260
5277
|
case "error": {
|
|
5261
|
-
|
|
5278
|
+
const diagnosticMessage = event.diagnosticMessage ?? event.message;
|
|
5279
|
+
if (await agentTerminationLostContainerRuntime(channel)) {
|
|
5262
5280
|
// `error` and `exit` can arrive back-to-back while the daemon probe
|
|
5263
5281
|
// is pending. Only the first terminal event still owns this session.
|
|
5264
5282
|
if (channel.sessions.get(agentId) !== session) break;
|
|
@@ -5272,7 +5290,7 @@ export class Orchestrator {
|
|
|
5272
5290
|
if (channel.sessions.get(agentId) !== session) break;
|
|
5273
5291
|
// The turn died with the session — drop its turn-state flags so a
|
|
5274
5292
|
// respawned session starts clean.
|
|
5275
|
-
|
|
5293
|
+
let abortBoundaryOnRosterChange =
|
|
5276
5294
|
this.needsAbortedTurnBoundary(channel, agentId);
|
|
5277
5295
|
this.retainRetiredTurnBoundary(
|
|
5278
5296
|
channel,
|
|
@@ -5280,6 +5298,14 @@ export class Orchestrator {
|
|
|
5280
5298
|
abortBoundaryOnRosterChange,
|
|
5281
5299
|
);
|
|
5282
5300
|
this.retireRunnerGeneration(channel, agentId);
|
|
5301
|
+
if (event.aborted) {
|
|
5302
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
5303
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
5304
|
+
if (change) change.abortPublished = true;
|
|
5305
|
+
// A recovery may replay the prompt, but the failed turn already
|
|
5306
|
+
// ended. Do not publish a second boundary for this generation.
|
|
5307
|
+
abortBoundaryOnRosterChange = false;
|
|
5308
|
+
}
|
|
5283
5309
|
// Claude under load (especially Docker Desktop macOS) occasionally
|
|
5284
5310
|
// unlinks ~/.claude.json mid-write during atomic config rewrites,
|
|
5285
5311
|
// and a concurrent claude spawn lands during the gap and exits 0
|
|
@@ -5325,7 +5351,7 @@ export class Orchestrator {
|
|
|
5325
5351
|
if (
|
|
5326
5352
|
agent &&
|
|
5327
5353
|
agent.kind !== "claude" &&
|
|
5328
|
-
isCredentialInvalidSignal(agent.kind,
|
|
5354
|
+
isCredentialInvalidSignal(agent.kind, diagnosticMessage)
|
|
5329
5355
|
) {
|
|
5330
5356
|
console.warn(
|
|
5331
5357
|
`[orchestrator] ${channel.taskId}/${agentId}: ${agent.kind} credential-invalid pattern matched: ${event.message.slice(0, 300)}`,
|
|
@@ -5407,7 +5433,7 @@ export class Orchestrator {
|
|
|
5407
5433
|
}
|
|
5408
5434
|
}
|
|
5409
5435
|
const isClaude = agent?.kind === "claude";
|
|
5410
|
-
if (isClaude && agent && isClaudeAuthRevoked(
|
|
5436
|
+
if (isClaude && agent && isClaudeAuthRevoked(diagnosticMessage)) {
|
|
5411
5437
|
// Token revoked/expired: the env-injected credential only changes on
|
|
5412
5438
|
// a fresh runner. Re-spawn on the re-resolved account (picks up a
|
|
5413
5439
|
// reconnected token); bounded so a still-bad token can't loop.
|
|
@@ -5424,7 +5450,7 @@ export class Orchestrator {
|
|
|
5424
5450
|
break;
|
|
5425
5451
|
}
|
|
5426
5452
|
const configMissing = CLAUDE_CONFIG_MISSING_PATTERNS.some((re) =>
|
|
5427
|
-
re.test(
|
|
5453
|
+
re.test(diagnosticMessage),
|
|
5428
5454
|
);
|
|
5429
5455
|
if (isClaude && configMissing) {
|
|
5430
5456
|
// This recovery path deliberately starts the next queued turn rather
|
|
@@ -5439,7 +5465,7 @@ export class Orchestrator {
|
|
|
5439
5465
|
break;
|
|
5440
5466
|
}
|
|
5441
5467
|
channel.lastPrompt.delete(agentId);
|
|
5442
|
-
|
|
5468
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
5443
5469
|
this.emitHost({
|
|
5444
5470
|
kind: "agent.exit",
|
|
5445
5471
|
taskId: channel.taskId,
|
|
@@ -5469,9 +5495,11 @@ export class Orchestrator {
|
|
|
5469
5495
|
// turn's messages and wakes the mentioned peers with the full turn.
|
|
5470
5496
|
// An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
|
|
5471
5497
|
// the cloud discards the buffer instead of handing it to peers.
|
|
5472
|
-
const aborted = channel.interrupted.delete(agentId);
|
|
5473
|
-
|
|
5474
|
-
|
|
5498
|
+
const aborted = channel.interrupted.delete(agentId) || event.aborted === true;
|
|
5499
|
+
const settles = event.settles !== undefined && Number.isSafeInteger(event.settles) && event.settles > 0
|
|
5500
|
+
? event.settles : 1;
|
|
5501
|
+
this.completeAcceptedPrompts(channel, agentId, settles);
|
|
5502
|
+
const remainingTurns = this.decrementActiveTurns(channel, agentId, settles);
|
|
5475
5503
|
channel.openTurns.delete(agentId);
|
|
5476
5504
|
// ADR-076: a turn that completed means the current account is healthy —
|
|
5477
5505
|
// reset its rotation budget so a LATER rate-limit gets a fresh failover.
|
|
@@ -5522,7 +5550,15 @@ export class Orchestrator {
|
|
|
5522
5550
|
break;
|
|
5523
5551
|
}
|
|
5524
5552
|
case "exit":
|
|
5525
|
-
if (
|
|
5553
|
+
if (event.code === 0 && this.needsAbortedTurnBoundary(channel, agentId)) {
|
|
5554
|
+
await this.handleAgentEvent(channel, agentId, session, {
|
|
5555
|
+
type: "error",
|
|
5556
|
+
message: `${session.kind} stopped before replying to the message. Please try again.`,
|
|
5557
|
+
aborted: true,
|
|
5558
|
+
});
|
|
5559
|
+
break;
|
|
5560
|
+
}
|
|
5561
|
+
if (await agentTerminationLostContainerRuntime(channel)) {
|
|
5526
5562
|
if (channel.sessions.get(agentId) !== session) break;
|
|
5527
5563
|
await this.retireSessionForRuntimeLoss(
|
|
5528
5564
|
channel,
|
|
@@ -7319,7 +7355,10 @@ export function buildSystemPreamble(
|
|
|
7319
7355
|
`${taskBranch}\` from inside the project, then open a PR with \`gh pr`,
|
|
7320
7356
|
"create` (the container has gh authenticated). For multi-project",
|
|
7321
7357
|
"tasks, each project's PR is independent — open one per project whose",
|
|
7322
|
-
"worktree you actually changed.",
|
|
7358
|
+
"worktree you actually changed. For a separate branch, create it in",
|
|
7359
|
+
"this task's namespace — `git switch -c \"$(uai branch name fix-foo)\"`",
|
|
7360
|
+
"makes `uai/<taskId>/fix-foo`. Only task-branch and namespace PRs link to",
|
|
7361
|
+
"this task (reviews, `--pr`, Usage); a PR from any other branch does not.",
|
|
7323
7362
|
"",
|
|
7324
7363
|
"The `.uai/` directory under each task is uai's own scaffolding",
|
|
7325
7364
|
"(rendered Dockerfile, compose file, container scripts) — it is NOT",
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { closeSync, constants, fstatSync, mkdirSync, openSync, readSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { CONTEXT_HASH_LABEL } from "./standard-image-pin";
|
|
5
|
+
|
|
6
|
+
type Result = { code: number | null; stdout: string; stderr: string };
|
|
7
|
+
export type DockerCommand = (command: string, args: string[], timeoutMs?: number) => Promise<Result>;
|
|
8
|
+
const IMAGE_ID = /^sha256:[a-f0-9]{64}$/;
|
|
9
|
+
const SWEEP_LIMIT = 16;
|
|
10
|
+
const HISTORY_LIMIT = 128;
|
|
11
|
+
const HISTORY_BYTES = 16 * 1024;
|
|
12
|
+
|
|
13
|
+
/** Context labels are inherited by task images. Authority to delete comes
|
|
14
|
+
* from observing this exact id under the standard tag, never from a scan. */
|
|
15
|
+
export class StandardImageCleanup {
|
|
16
|
+
private readonly observed = new Set<string>();
|
|
17
|
+
private loading: Promise<void> | undefined;
|
|
18
|
+
private statePath: string | undefined;
|
|
19
|
+
|
|
20
|
+
constructor(private readonly stateFile?: () => Promise<string>) {}
|
|
21
|
+
|
|
22
|
+
private async load(): Promise<void> {
|
|
23
|
+
if (!this.stateFile) return;
|
|
24
|
+
try {
|
|
25
|
+
this.statePath = await this.stateFile();
|
|
26
|
+
const fd = openSync(this.statePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
27
|
+
try {
|
|
28
|
+
const stat = fstatSync(fd);
|
|
29
|
+
if (!stat.isFile() || stat.size > HISTORY_BYTES) return;
|
|
30
|
+
const bytes = Buffer.alloc(HISTORY_BYTES + 1);
|
|
31
|
+
const length = readSync(fd, bytes, 0, bytes.length, 0);
|
|
32
|
+
if (length > HISTORY_BYTES) return;
|
|
33
|
+
const value = JSON.parse(bytes.subarray(0, length).toString("utf8")) as { version?: unknown; ids?: unknown };
|
|
34
|
+
if (value?.version !== 1 || !Array.isArray(value.ids) || value.ids.length > HISTORY_LIMIT || !value.ids.every(id => typeof id === "string" && IMAGE_ID.test(id))) return;
|
|
35
|
+
for (const id of value.ids) this.observed.add(id as string);
|
|
36
|
+
} finally { closeSync(fd); }
|
|
37
|
+
} catch { /* Missing, corrupt or unreadable history authorizes nothing. */ }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
private save(): void {
|
|
41
|
+
if (!this.statePath) return;
|
|
42
|
+
const temporary = `${this.statePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
43
|
+
try {
|
|
44
|
+
mkdirSync(dirname(this.statePath), { recursive: true, mode: 0o700 });
|
|
45
|
+
writeFileSync(temporary, JSON.stringify({ version: 1, ids: [...this.observed] }), { flag: "wx", mode: 0o600 });
|
|
46
|
+
renameSync(temporary, this.statePath);
|
|
47
|
+
} catch {
|
|
48
|
+
// ENOSPC must not block a build or safe cleanup using in-memory proof.
|
|
49
|
+
console.warn("[host-agent] could not persist standard-image history");
|
|
50
|
+
} finally {
|
|
51
|
+
try { unlinkSync(temporary); } catch { /* Renamed or never created. */ }
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async observe(run: DockerCommand, ref = "uai-standard:dev"): Promise<string | null> {
|
|
56
|
+
await (this.loading ??= this.load());
|
|
57
|
+
const result = await run("docker", ["image", "inspect", "--format", "{{json .}}", ref]);
|
|
58
|
+
if (result.code !== 0) return null;
|
|
59
|
+
try {
|
|
60
|
+
const image = JSON.parse(result.stdout) as { Id?: string; Config?: { Labels?: Record<string, string> } };
|
|
61
|
+
if (!IMAGE_ID.test(image.Id ?? "") || !/^[a-f0-9]{32}$/.test(image.Config?.Labels?.[CONTEXT_HASH_LABEL] ?? "")) return null;
|
|
62
|
+
if (ref === "uai-standard:dev") {
|
|
63
|
+
this.observed.delete(image.Id!);
|
|
64
|
+
this.observed.add(image.Id!);
|
|
65
|
+
if (this.observed.size > HISTORY_LIMIT) this.observed.delete(this.observed.values().next().value!);
|
|
66
|
+
this.save();
|
|
67
|
+
}
|
|
68
|
+
return image.Id!;
|
|
69
|
+
} catch { return null; }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async sweep(run: DockerCommand, previousId: string | null): Promise<void> {
|
|
73
|
+
const deadline = Date.now() + 30_000;
|
|
74
|
+
const boundedRun: DockerCommand = (command, args) => run(command, args, 5_000);
|
|
75
|
+
const currentId = await this.observe(boundedRun);
|
|
76
|
+
if (!currentId) return;
|
|
77
|
+
const history = await boundedRun("docker", ["image", "history", "--quiet", "--no-trunc", currentId]);
|
|
78
|
+
if (history.code !== 0) return;
|
|
79
|
+
const currentChain = new Set([currentId, ...history.stdout.trim().split(/\s+/)]);
|
|
80
|
+
const candidates = new Set([...(previousId ? [previousId] : []), ...this.observed]);
|
|
81
|
+
for (const id of [...candidates].filter(id => this.observed.has(id) && !currentChain.has(id)).slice(0, SWEEP_LIMIT)) {
|
|
82
|
+
if (Date.now() >= deadline) break;
|
|
83
|
+
// Round-robin retained/in-use ids so a busy prefix cannot starve later ids.
|
|
84
|
+
this.observed.delete(id);
|
|
85
|
+
this.observed.add(id);
|
|
86
|
+
const tags = await boundedRun("docker", ["image", "inspect", "--format", "{{json .RepoTags}}", id]);
|
|
87
|
+
if (tags.code !== 0) continue;
|
|
88
|
+
try { const parsed: unknown = JSON.parse(tags.stdout); if (parsed !== null && (!Array.isArray(parsed) || parsed.length !== 0)) continue; } catch { continue; }
|
|
89
|
+
const containers = await boundedRun("docker", ["ps", "--all", "--quiet", "--filter", `ancestor=${id}`]);
|
|
90
|
+
if (containers.code !== 0 || containers.stdout.trim()) continue;
|
|
91
|
+
// Normal Docker removal reclaims otherwise-unused untagged parents.
|
|
92
|
+
// Shared ancestors of the current image (or any other child/container)
|
|
93
|
+
// remain protected by the daemon. Never force or target a parent id.
|
|
94
|
+
const removed = await boundedRun("docker", ["image", "rm", id]);
|
|
95
|
+
if (removed.code === 0) {
|
|
96
|
+
this.observed.delete(id);
|
|
97
|
+
console.log("[host-agent] removed unused previous standard image", id);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
this.save();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Per-process exponential retry, keyed by the desired content/configuration.
|
|
105
|
+
* A new host payload/engine configuration retries immediately. A restart also
|
|
106
|
+
* retries once; repeated failures back off 2h, 4h, …, 24h instead of hourly. */
|
|
107
|
+
export class StandardImageBuildBackoff {
|
|
108
|
+
private state: { key: string; failures: number; retryAt: number; error: string } | null = null;
|
|
109
|
+
remaining(key: string, now = Date.now()): { delay: number; retryAt: number; error: string } | null {
|
|
110
|
+
return this.state?.key === key && this.state.retryAt > now ? { delay: this.state.retryAt - now, retryAt: this.state.retryAt, error: this.state.error } : null;
|
|
111
|
+
}
|
|
112
|
+
failed(key: string, error: string, now = Date.now()): void {
|
|
113
|
+
const failures = this.state?.key === key ? this.state.failures + 1 : 1;
|
|
114
|
+
this.state = { key, failures, retryAt: now + Math.min(24, 2 ** Math.min(failures, 5)) * 3_600_000, error };
|
|
115
|
+
}
|
|
116
|
+
clear(): void { this.state = null; }
|
|
117
|
+
}
|
package/lib/standard-image.ts
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Standard-image build pipeline (ADR-022 / docs/runtime.md).
|
|
3
3
|
*
|
|
4
4
|
* Every host builds exactly one standard image (`uai-standard:dev`) and
|
|
5
|
-
* caches it. `ensureStandardImage()`
|
|
5
|
+
* caches it. `ensureStandardImage()` runs at startup and on the owned-host
|
|
6
|
+
* hourly maintenance schedule (local build failures back off):
|
|
6
7
|
*
|
|
7
8
|
* 1. `docker volume create uai-asdf-data` — the host-wide asdf data volume
|
|
8
9
|
* that caches lazily-installed runtime versions across tasks (idempotent;
|
|
@@ -52,6 +53,7 @@ import {
|
|
|
52
53
|
standardImagePullDigest,
|
|
53
54
|
} from "./standard-image-pin";
|
|
54
55
|
import { sidecarMode } from "./sidecar";
|
|
56
|
+
import { StandardImageCleanup, StandardImageBuildBackoff } from "./standard-image-cleanup";
|
|
55
57
|
|
|
56
58
|
/** Pinned, host-wide constants (must match task-up.sh and the compose gen). */
|
|
57
59
|
export const STANDARD_IMAGE_TAG = "uai-standard:dev";
|
|
@@ -1296,8 +1298,15 @@ export type ActivationIsCurrent = () => boolean;
|
|
|
1296
1298
|
/** Tests: forget every remembered pass. */
|
|
1297
1299
|
export function resetStandardImageMemo(): void {
|
|
1298
1300
|
lastOkByKey.clear();
|
|
1301
|
+
buildBackoff.clear();
|
|
1299
1302
|
}
|
|
1300
1303
|
|
|
1304
|
+
const buildBackoff = new StandardImageBuildBackoff();
|
|
1305
|
+
const imageCleanup = new StandardImageCleanup(async () => {
|
|
1306
|
+
const { env } = await import("./env");
|
|
1307
|
+
return resolve(env.dataDir, "standard-image-seen.json");
|
|
1308
|
+
});
|
|
1309
|
+
|
|
1301
1310
|
/** The owned-host schedule (docs/agent-notes.md, "image maintenance"). */
|
|
1302
1311
|
export const MAINTENANCE_INTERVAL_MS = 60 * 60_000;
|
|
1303
1312
|
|
|
@@ -1418,6 +1427,7 @@ async function ensureStandardImageInner(
|
|
|
1418
1427
|
// hash, not this host's per-config hash, which would disagree by design
|
|
1419
1428
|
// and rebuild forever). Any pull-path failure falls through to the local
|
|
1420
1429
|
// build below, so offline and registry-down hosts behave exactly as today.
|
|
1430
|
+
const previousImageId = engine.apple ? null : await imageCleanup.observe(run);
|
|
1421
1431
|
const pin = await readStandardImagePin();
|
|
1422
1432
|
if (pin !== null) {
|
|
1423
1433
|
if (inspect.code === 0 && labeledHash === pin.contextHash) {
|
|
@@ -1473,13 +1483,18 @@ async function ensureStandardImageInner(
|
|
|
1473
1483
|
imageReady = true;
|
|
1474
1484
|
} else {
|
|
1475
1485
|
const context = standardImageDir();
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1486
|
+
const buildKey = `${contextHash ?? "unavailable"}:${JSON.stringify(engines)}:${pin?.contextHash ?? "local"}`;
|
|
1487
|
+
const deferred = engine.apple || labeledHash === null ? null : buildBackoff.remaining(buildKey);
|
|
1488
|
+
if (!deferred) {
|
|
1489
|
+
console.log(
|
|
1490
|
+
inspect.code === 0
|
|
1491
|
+
? `[host-agent] standard image ${STANDARD_IMAGE_TAG} stale (context changed) — rebuilding from ${context}`
|
|
1492
|
+
: `[host-agent] building standard image ${STANDARD_IMAGE_TAG} from ${context}`,
|
|
1493
|
+
);
|
|
1494
|
+
}
|
|
1495
|
+
const build = deferred ? { code: 1, stdout: "", stderr: `${deferred.error}\nStandard image build deferred until ${new Date(deferred.retryAt).toISOString()}.` } : await run(engine.command, [
|
|
1482
1496
|
"build",
|
|
1497
|
+
...(engine.apple ? [] : ["--force-rm"]),
|
|
1483
1498
|
"-t",
|
|
1484
1499
|
STANDARD_IMAGE_TAG,
|
|
1485
1500
|
...engineArgs,
|
|
@@ -1491,11 +1506,14 @@ async function ensureStandardImageInner(
|
|
|
1491
1506
|
if (build.code === 0) {
|
|
1492
1507
|
console.log(`[host-agent] built standard image ${STANDARD_IMAGE_TAG}`);
|
|
1493
1508
|
imageReady = true;
|
|
1509
|
+
buildBackoff.clear();
|
|
1494
1510
|
} else {
|
|
1495
1511
|
buildError =
|
|
1496
1512
|
build.stderr.trim() ||
|
|
1497
1513
|
`docker build exited ${build.code ?? "spawn error"}`;
|
|
1514
|
+
if (!deferred && !engine.apple) buildBackoff.failed(buildKey, buildError);
|
|
1498
1515
|
console.warn(
|
|
1516
|
+
deferred ? `[host-agent] ${buildError}` :
|
|
1499
1517
|
`[host-agent] standard image build failed (exit ${build.code ?? "spawn"}); ` +
|
|
1500
1518
|
"continuing. Tasks needing the image will surface this error.\n" +
|
|
1501
1519
|
buildError,
|
|
@@ -1505,6 +1523,10 @@ async function ensureStandardImageInner(
|
|
|
1505
1523
|
}
|
|
1506
1524
|
}
|
|
1507
1525
|
|
|
1526
|
+
// Revisit ids durably observed under the standard tag across restarts.
|
|
1527
|
+
// Inherited labels never authorize sweeping task/compose images.
|
|
1528
|
+
if (imageReady && !engine.apple) await imageCleanup.sweep(run, previousImageId);
|
|
1529
|
+
|
|
1508
1530
|
// 3. Reconcile every Uai-owned writer under the volume-global flock. Lock
|
|
1509
1531
|
// bootstrap/permission repair precedes the single upgrade→repair→proof
|
|
1510
1532
|
// transaction so node-run task materializers can acquire legacy volumes.
|
|
@@ -1648,7 +1670,7 @@ async function ensureStandardImageInner(
|
|
|
1648
1670
|
}
|
|
1649
1671
|
return {
|
|
1650
1672
|
ok: false,
|
|
1651
|
-
error: `Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
|
|
1673
|
+
error: `${buildError ? `${buildError} ` : ""}Could not prepare shared runtime maintenance lock ${ASDF_MAINTENANCE_LOCK_PATH}.`,
|
|
1652
1674
|
};
|
|
1653
1675
|
}
|
|
1654
1676
|
if (!(await reconcileVolumeAgentClis(activation))) {
|
|
@@ -1661,7 +1683,7 @@ async function ensureStandardImageInner(
|
|
|
1661
1683
|
return {
|
|
1662
1684
|
ok: false,
|
|
1663
1685
|
error:
|
|
1664
|
-
"Shared runtime maintenance failed; agent CLIs or Corepack did not pass execution proof
|
|
1686
|
+
`${buildError ? `${buildError} ` : ""}Shared runtime maintenance failed; agent CLIs or Corepack did not pass execution proof.`,
|
|
1665
1687
|
};
|
|
1666
1688
|
}
|
|
1667
1689
|
return { ok: true };
|
|
@@ -393,6 +393,11 @@ export function assertWorkspaceScopedPath(
|
|
|
393
393
|
}
|
|
394
394
|
}
|
|
395
395
|
|
|
396
|
+
/** A malformed process request, distinct from provider/runtime failures. */
|
|
397
|
+
export class TaskEnvironmentRequestValidationError extends Error {
|
|
398
|
+
override name = "TaskEnvironmentRequestValidationError";
|
|
399
|
+
}
|
|
400
|
+
|
|
396
401
|
const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
397
402
|
|
|
398
403
|
/** Shared admission guard for provider implementations and fake providers. */
|
|
@@ -402,7 +407,7 @@ const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
|
402
407
|
export function assertTaskEnvironmentExecRequest(request: TaskEnvironmentExecRequest): void {
|
|
403
408
|
const { privileged, ...unprivileged } = request;
|
|
404
409
|
if (privileged !== undefined && (privileged !== true || request.user !== "root")) {
|
|
405
|
-
throw new
|
|
410
|
+
throw new TaskEnvironmentRequestValidationError("a privileged task environment exec must run as root");
|
|
406
411
|
}
|
|
407
412
|
assertTaskEnvironmentProcessRequest(unprivileged);
|
|
408
413
|
}
|
|
@@ -412,7 +417,7 @@ export function assertTaskEnvironmentProcessRequest(
|
|
|
412
417
|
): void {
|
|
413
418
|
assertTaskEnvironmentSessionRequest(request);
|
|
414
419
|
if (!Number.isSafeInteger(request.timeoutMs) || request.timeoutMs <= 0) {
|
|
415
|
-
throw new
|
|
420
|
+
throw new TaskEnvironmentRequestValidationError("task environment timeout must be a positive integer");
|
|
416
421
|
}
|
|
417
422
|
}
|
|
418
423
|
|
|
@@ -420,7 +425,7 @@ export function assertTaskEnvironmentProcessRequest(
|
|
|
420
425
|
* step without the capability it asked for. */
|
|
421
426
|
export function refusePrivilegedExec(request: TaskEnvironmentExecRequest): void {
|
|
422
427
|
if (request.privileged === true) {
|
|
423
|
-
throw new
|
|
428
|
+
throw new TaskEnvironmentRequestValidationError("this task environment provider has no privileged exec");
|
|
424
429
|
}
|
|
425
430
|
}
|
|
426
431
|
|
|
@@ -434,13 +439,13 @@ export function assertTaskEnvironmentSessionRequest(
|
|
|
434
439
|
// Only a one-shot exec may be privileged (assertTaskEnvironmentExecRequest
|
|
435
440
|
// strips it after checking); anywhere else it is refused outright.
|
|
436
441
|
if ("privileged" in request) {
|
|
437
|
-
throw new
|
|
442
|
+
throw new TaskEnvironmentRequestValidationError("only a one-shot task environment exec may be privileged");
|
|
438
443
|
}
|
|
439
444
|
if (
|
|
440
445
|
request.projectEnvProfile !== "maintenance" &&
|
|
441
446
|
request.projectEnvProfile !== "task"
|
|
442
447
|
) {
|
|
443
|
-
throw new
|
|
448
|
+
throw new TaskEnvironmentRequestValidationError(
|
|
444
449
|
"task environment project env profile must be maintenance or task",
|
|
445
450
|
);
|
|
446
451
|
}
|
|
@@ -451,19 +456,19 @@ export function assertTaskEnvironmentSessionRequest(
|
|
|
451
456
|
(part) => typeof part !== "string" || part.length === 0 || part.includes("\0"),
|
|
452
457
|
)
|
|
453
458
|
) {
|
|
454
|
-
throw new
|
|
459
|
+
throw new TaskEnvironmentRequestValidationError("task environment argv must contain nonempty NUL-free strings");
|
|
455
460
|
}
|
|
456
461
|
if (
|
|
457
462
|
request.cwd !== undefined &&
|
|
458
|
-
(!request.cwd.startsWith("/") || request.cwd.includes("\0"))
|
|
463
|
+
(typeof request.cwd !== "string" || !request.cwd.startsWith("/") || request.cwd.includes("\0"))
|
|
459
464
|
) {
|
|
460
|
-
throw new
|
|
465
|
+
throw new TaskEnvironmentRequestValidationError("task environment cwd must be an absolute NUL-free path");
|
|
461
466
|
}
|
|
462
467
|
if (
|
|
463
468
|
request.user !== undefined &&
|
|
464
|
-
(request.user.length === 0 || request.user.includes("\0"))
|
|
469
|
+
(typeof request.user !== "string" || request.user.length === 0 || request.user.includes("\0"))
|
|
465
470
|
) {
|
|
466
|
-
throw new
|
|
471
|
+
throw new TaskEnvironmentRequestValidationError("task environment user must be a nonempty NUL-free value");
|
|
467
472
|
}
|
|
468
473
|
if (
|
|
469
474
|
request.anchor !== undefined &&
|
|
@@ -474,7 +479,7 @@ export function assertTaskEnvironmentSessionRequest(
|
|
|
474
479
|
(part) => typeof part !== "string" || part.length === 0 || part.includes("\0"),
|
|
475
480
|
))
|
|
476
481
|
) {
|
|
477
|
-
throw new
|
|
482
|
+
throw new TaskEnvironmentRequestValidationError("a task environment anchor must be a NUL-free argv for a task user's process");
|
|
478
483
|
}
|
|
479
484
|
if (
|
|
480
485
|
request.inheritEnv !== undefined &&
|
|
@@ -485,7 +490,7 @@ export function assertTaskEnvironmentSessionRequest(
|
|
|
485
490
|
(name) => typeof name !== "string" || !ENV_NAME.test(name),
|
|
486
491
|
))
|
|
487
492
|
) {
|
|
488
|
-
throw new
|
|
493
|
+
throw new TaskEnvironmentRequestValidationError("task environment inherited env names are invalid");
|
|
489
494
|
}
|
|
490
495
|
for (const [name, value] of Object.entries(request.env ?? {})) {
|
|
491
496
|
if (
|
|
@@ -493,21 +498,21 @@ export function assertTaskEnvironmentSessionRequest(
|
|
|
493
498
|
typeof value !== "string" ||
|
|
494
499
|
value.includes("\0")
|
|
495
500
|
) {
|
|
496
|
-
throw new
|
|
501
|
+
throw new TaskEnvironmentRequestValidationError("task environment env must contain valid NUL-free entries");
|
|
497
502
|
}
|
|
498
503
|
}
|
|
499
504
|
if (
|
|
500
505
|
!Number.isSafeInteger(request.maxOutputBytes) ||
|
|
501
506
|
request.maxOutputBytes <= 0
|
|
502
507
|
) {
|
|
503
|
-
throw new
|
|
508
|
+
throw new TaskEnvironmentRequestValidationError("task environment output limit must be a positive integer");
|
|
504
509
|
}
|
|
505
510
|
if (
|
|
506
511
|
"launchTimeoutMs" in request &&
|
|
507
512
|
(!Number.isSafeInteger(request.launchTimeoutMs) ||
|
|
508
513
|
request.launchTimeoutMs <= 0)
|
|
509
514
|
) {
|
|
510
|
-
throw new
|
|
515
|
+
throw new TaskEnvironmentRequestValidationError(
|
|
511
516
|
"task environment detached launch timeout must be a positive integer",
|
|
512
517
|
);
|
|
513
518
|
}
|