@runuai/host 0.9.14 → 0.9.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +80 -30
- package/images/standard/Dockerfile +36 -10
- package/images/standard/README.md +63 -18
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +308 -38
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/lib/agent-cli.ts +33 -2
- package/lib/agent.ts +46 -7
- package/lib/agents/claude.ts +13 -8
- package/lib/agents/codex.ts +11 -6
- package/lib/agents/cursor.ts +39 -29
- package/lib/agents/durable-proc.ts +20 -27
- package/lib/agents/factory.ts +9 -25
- package/lib/agents/grok.ts +43 -30
- package/lib/agents/kimi.ts +44 -29
- package/lib/agents/opencode.ts +43 -31
- package/lib/agents/proc.ts +149 -114
- package/lib/agents/transport.ts +62 -50
- package/lib/agents/types.ts +6 -4
- package/lib/apple-runtime-recycle.ts +236 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/browser-testing.ts +233 -93
- package/lib/codex-auth.ts +40 -6
- package/lib/command-db.ts +20 -0
- package/lib/container-runtime.ts +1338 -0
- package/lib/db.ts +1 -0
- package/lib/docker-exec.ts +87 -5
- package/lib/engine-accounts.ts +68 -5
- package/lib/engine-login.ts +1952 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +155 -0
- package/lib/env.ts +4 -0
- package/lib/git-diff.ts +98 -32
- package/lib/git-identity.ts +199 -87
- package/lib/github-tokens.ts +202 -91
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-logs.ts +962 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +49 -0
- package/lib/managed-runtime.ts +3644 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-gateway.ts +450 -23
- package/lib/orchestrator.ts +3051 -200
- package/lib/preview-sidecar.ts +68 -14
- package/lib/release-manifest.ts +708 -0
- package/lib/release-trust.ts +28 -0
- package/lib/runtime-activation-tail.ts +232 -0
- package/lib/runtime-archive.ts +1086 -0
- package/lib/runtime-authority.ts +79 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-state.ts +232 -12
- package/lib/skills.ts +24 -3
- package/lib/ssh.ts +18 -0
- package/lib/standard-image.ts +1104 -141
- package/lib/stopped-task-status-queue.ts +44 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-diff.ts +66 -46
- package/lib/task-environment/apple-container.ts +757 -0
- package/lib/task-environment/docker.ts +956 -0
- package/lib/task-environment/index.ts +364 -0
- package/lib/task-environment/legacy-adoption.ts +459 -0
- package/lib/task-environment/registry.ts +58 -0
- package/lib/task-environment/types.ts +408 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +585 -0
- package/lib/tunnel-registry.ts +135 -19
- package/lib/tunnel-runtime.ts +235 -0
- package/package.json +1 -1
- package/scripts/agent/_common.sh +123 -3
- package/scripts/agent/task-down.sh +146 -38
- package/scripts/agent/task-status.sh +19 -3
- package/scripts/agent/task-up.sh +1405 -107
- package/scripts/install/darwin.ts +848 -50
- package/scripts/install/linux.ts +838 -35
- package/scripts/install/types.ts +43 -0
- package/scripts/install/util.ts +215 -8
- package/scripts/install/win.ts +12 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/cli.ts +1464 -72
- package/src/event-outbox.ts +83 -4
- package/src/index.ts +766 -42
- package/src/main.ts +1398 -255
- package/src/paths.ts +17 -1
- package/src/protocol.ts +695 -1
- package/src/runtime-bootstrap.ts +165 -0
- package/src/ui/server.ts +46 -10
- package/src/ui/types.ts +37 -0
package/lib/orchestrator.ts
CHANGED
|
@@ -15,10 +15,18 @@
|
|
|
15
15
|
* instead of leaking a second orchestrator.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {
|
|
19
|
-
|
|
18
|
+
import {
|
|
19
|
+
existsSync,
|
|
20
|
+
lstatSync,
|
|
21
|
+
mkdtempSync,
|
|
22
|
+
readFileSync,
|
|
23
|
+
realpathSync,
|
|
24
|
+
renameSync,
|
|
25
|
+
rmSync,
|
|
26
|
+
writeFileSync,
|
|
27
|
+
} from "node:fs";
|
|
20
28
|
import { homedir } from "node:os";
|
|
21
|
-
import { join } from "node:path";
|
|
29
|
+
import { dirname, join } from "node:path";
|
|
22
30
|
|
|
23
31
|
import { inArray } from "drizzle-orm";
|
|
24
32
|
|
|
@@ -33,15 +41,52 @@ import {
|
|
|
33
41
|
type Roster,
|
|
34
42
|
type RosterAgent,
|
|
35
43
|
} from "./agents/types";
|
|
36
|
-
import { ACTIVE_STATUSES } from "./task-status";
|
|
44
|
+
import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
|
|
45
|
+
import { isSafeHostTaskId } from "./task-identity";
|
|
37
46
|
import { getHostTask, upsertHostTask } from "./runtime-state";
|
|
38
47
|
import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
|
|
39
48
|
import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
|
|
40
49
|
import { dockerCli } from "./docker-exec";
|
|
50
|
+
import {
|
|
51
|
+
taskContainerBackend,
|
|
52
|
+
taskContainerCli,
|
|
53
|
+
} from "./task-container-cli";
|
|
54
|
+
import {
|
|
55
|
+
RUNTIME_AUTHORITY_ENV,
|
|
56
|
+
runtimeAuthorityDockerExecArgs,
|
|
57
|
+
} from "./runtime-authority";
|
|
58
|
+
import {
|
|
59
|
+
backgroundContainerWorkAllowed,
|
|
60
|
+
ContainerRuntimeUnavailableError,
|
|
61
|
+
} from "./runtime-guard";
|
|
62
|
+
import {
|
|
63
|
+
appleContainerRuntimeBinding,
|
|
64
|
+
initializeContainerRuntime,
|
|
65
|
+
pinnedContainerRuntimeProvider,
|
|
66
|
+
suspendContainerRuntime,
|
|
67
|
+
} from "./container-runtime";
|
|
68
|
+
import {
|
|
69
|
+
reconstructPersistedTaskEnvironment,
|
|
70
|
+
setAppleTaskEnvironmentRecoveryDriver,
|
|
71
|
+
setDockerTaskEnvironmentRecoveryDriver,
|
|
72
|
+
} from "./task-environment";
|
|
73
|
+
import {
|
|
74
|
+
appleCliRunner,
|
|
75
|
+
appleTaskContainerName,
|
|
76
|
+
parseAppleTaskEnvironmentLocator,
|
|
77
|
+
} from "./task-environment/apple-container";
|
|
78
|
+
import { parseDockerTaskEnvironmentLocator } from "./task-environment/docker";
|
|
79
|
+
import type {
|
|
80
|
+
TaskEnvironmentDescriptor,
|
|
81
|
+
TaskEnvironmentHandle,
|
|
82
|
+
TaskEnvironmentRecoveryContext,
|
|
83
|
+
TaskEnvironmentRecoveryResult,
|
|
84
|
+
} from "./task-environment/types";
|
|
41
85
|
import {
|
|
42
86
|
containerSkillFile,
|
|
43
87
|
installedSkillPath,
|
|
44
88
|
installPackageSkills,
|
|
89
|
+
type SkillExec,
|
|
45
90
|
writeAgentSkills,
|
|
46
91
|
} from "./skills";
|
|
47
92
|
import {
|
|
@@ -66,7 +111,15 @@ import {
|
|
|
66
111
|
type ResolvedEngineAccount,
|
|
67
112
|
} from "./engine-accounts";
|
|
68
113
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
69
|
-
import {
|
|
114
|
+
import {
|
|
115
|
+
PREVIEW_SIDECAR_TASK_LABEL,
|
|
116
|
+
stopPreviewSidecars,
|
|
117
|
+
} from "./preview-sidecar";
|
|
118
|
+
import {
|
|
119
|
+
agentClisReady,
|
|
120
|
+
standardImageCorepackVersionPath,
|
|
121
|
+
standardImageRuntimeMaterializerPath,
|
|
122
|
+
} from "./standard-image";
|
|
70
123
|
import { env } from "./env";
|
|
71
124
|
import type {
|
|
72
125
|
ChannelEnsureInput,
|
|
@@ -102,6 +155,11 @@ interface Channel {
|
|
|
102
155
|
sessions: Map<string, AgentSession>;
|
|
103
156
|
/** Session-spawn inputs, kept so sessions can be started lazily. */
|
|
104
157
|
containerName: string;
|
|
158
|
+
/** Opaque provider-owned boundary used only for agent process creation. */
|
|
159
|
+
agentEnvironment: Promise<TaskEnvironmentHandle<unknown>> | null;
|
|
160
|
+
/** Settled handle for later config/setup passes; avoids a new await on
|
|
161
|
+
* legacy/bootstrap paths. */
|
|
162
|
+
agentHandle: TaskEnvironmentHandle<unknown> | null;
|
|
105
163
|
/** Per-agent system preamble (channel briefing + assembled persona). */
|
|
106
164
|
preambles: Map<string, string>;
|
|
107
165
|
/** Per-agent first-turn message — only agents whose `initialPrompt` is
|
|
@@ -167,11 +225,20 @@ interface Channel {
|
|
|
167
225
|
/** Agents with a delivered turn that has not reached a terminal event yet.
|
|
168
226
|
* Unlike openTurns, this covers thinking/tool/permission time before text. */
|
|
169
227
|
activeTurns: Map<string, number>;
|
|
228
|
+
/** Exact accepted prompts still owned by each runner generation, in send
|
|
229
|
+
* order. Open mode deliberately permits more than one; a process-level
|
|
230
|
+
* daemon-loss event must replay all of them rather than only `lastPrompt`. */
|
|
231
|
+
acceptedPrompts: Map<string, AcceptedPrompt[]>;
|
|
170
232
|
/** Crew replies waiting for the designated Secretary's current turn
|
|
171
233
|
* boundary. Adapters do not share one mid-turn input contract, so only this
|
|
172
234
|
* cross-lane role is serialized at the host boundary; open mode keeps its
|
|
173
235
|
* legacy immediate-concurrent delivery behavior. */
|
|
174
236
|
pendingPrompts: Map<string, string[]>;
|
|
237
|
+
/** Prompts accepted by channelDeliver whose final session boundary rejected
|
|
238
|
+
* specifically because the container runtime became unavailable. */
|
|
239
|
+
runtimeDeferredPrompts: Map<string, string[]>;
|
|
240
|
+
/** Missing/blocked sessions to reconcile exactly once after runtime ready. */
|
|
241
|
+
runtimeDeferredAgents: Set<string>;
|
|
175
242
|
/** Agents whose current turn was interrupted (ESC) — their next
|
|
176
243
|
* turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
|
|
177
244
|
* half-turn instead of delivering it to @-mentioned peers. */
|
|
@@ -230,6 +297,10 @@ interface Channel {
|
|
|
230
297
|
rotations: Map<string, number>;
|
|
231
298
|
}
|
|
232
299
|
|
|
300
|
+
interface AcceptedPrompt {
|
|
301
|
+
text: string;
|
|
302
|
+
}
|
|
303
|
+
|
|
233
304
|
/** Hard cap on automatic respawns per agent per channel lifetime. */
|
|
234
305
|
const MAX_RESPAWNS_PER_AGENT = 5;
|
|
235
306
|
/** ADR-076: hard cap on account rotations per agent between successful turns. */
|
|
@@ -241,6 +312,43 @@ const RESPAWN_COOLDOWN_MS = 10 * 60_000;
|
|
|
241
312
|
* retry would spin npx continuously for the life of a broken task. */
|
|
242
313
|
const BROWSER_RETRY_COOLDOWN_MS = 60_000;
|
|
243
314
|
|
|
315
|
+
/**
|
|
316
|
+
* A runner can be the first observable casualty of a daemon loss, before any
|
|
317
|
+
* command/tunnel path invalidates the cached ready verdict. Prove the daemon
|
|
318
|
+
* before classifying a terminal agent event as an agent crash. A failed proof
|
|
319
|
+
* poisons the runtime generation synchronously; normal detection/recovery owns
|
|
320
|
+
* the eventual ready edge and deferred-prompt replay.
|
|
321
|
+
*/
|
|
322
|
+
async function agentTerminationLostContainerRuntime(): Promise<boolean> {
|
|
323
|
+
if (!backgroundContainerWorkAllowed()) return true;
|
|
324
|
+
let reachable = false;
|
|
325
|
+
try {
|
|
326
|
+
// Backend-aware liveness: `docker` does not exist on an apple host, and
|
|
327
|
+
// an ENOENT here would poison the generation on EVERY abnormal agent
|
|
328
|
+
// termination (found in the 2026-08-18 spawn-docker-ENOENT sweep).
|
|
329
|
+
const proof = taskContainerBackend().apple
|
|
330
|
+
? await taskContainerCli(["system", "status"], { timeoutMs: 10_000 })
|
|
331
|
+
: await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
|
|
332
|
+
timeoutMs: 10_000,
|
|
333
|
+
});
|
|
334
|
+
reachable = proof.status === 0;
|
|
335
|
+
} catch {
|
|
336
|
+
reachable = false;
|
|
337
|
+
}
|
|
338
|
+
if (!backgroundContainerWorkAllowed()) return true;
|
|
339
|
+
if (reachable) return false;
|
|
340
|
+
|
|
341
|
+
suspendContainerRuntime(true);
|
|
342
|
+
void initializeContainerRuntime().catch((err) =>
|
|
343
|
+
console.warn(
|
|
344
|
+
`[orchestrator] container runtime recheck after agent termination failed: ${
|
|
345
|
+
err instanceof Error ? err.message : String(err)
|
|
346
|
+
}`,
|
|
347
|
+
),
|
|
348
|
+
);
|
|
349
|
+
return true;
|
|
350
|
+
}
|
|
351
|
+
|
|
244
352
|
/** Substrings in an agent's error output that mean "config was
|
|
245
353
|
* unlinked between runs" — repair-and-respawn covers the common
|
|
246
354
|
* Docker-Desktop-macOS race where Claude's atomic writes briefly
|
|
@@ -530,6 +638,8 @@ export class Orchestrator {
|
|
|
530
638
|
secretaryAgentId: spec.secretaryAgentId,
|
|
531
639
|
sessions: new Map(),
|
|
532
640
|
containerName: `task-${taskId.toLowerCase()}-app-1`,
|
|
641
|
+
agentEnvironment: null,
|
|
642
|
+
agentHandle: null,
|
|
533
643
|
preambles,
|
|
534
644
|
firstTurns,
|
|
535
645
|
sessionsReady: null,
|
|
@@ -547,7 +657,10 @@ export class Orchestrator {
|
|
|
547
657
|
closed: false,
|
|
548
658
|
openTurns: new Set(),
|
|
549
659
|
activeTurns: new Map(),
|
|
660
|
+
acceptedPrompts: new Map(),
|
|
550
661
|
pendingPrompts: new Map(),
|
|
662
|
+
runtimeDeferredPrompts: new Map(),
|
|
663
|
+
runtimeDeferredAgents: new Set(),
|
|
551
664
|
interrupted: new Set(),
|
|
552
665
|
respawns: new Map(),
|
|
553
666
|
respawnLastAt: new Map(),
|
|
@@ -587,6 +700,7 @@ export class Orchestrator {
|
|
|
587
700
|
// copy cannot land after browser/MCP materialization and erase config.
|
|
588
701
|
await taskRecoveryComplete(channel.taskId);
|
|
589
702
|
if (!this.isActiveChannel(channel)) return false;
|
|
703
|
+
if (!backgroundContainerWorkAllowed()) return false;
|
|
590
704
|
// Memoized: every caller awaits the SAME in-flight start, so a concurrent
|
|
591
705
|
// deliver() can't observe "ready" while the sessions map is still empty
|
|
592
706
|
// (startSessions awaits docker work before populating it — the old boolean
|
|
@@ -632,6 +746,7 @@ export class Orchestrator {
|
|
|
632
746
|
await this.ensureMcpConfig(channel);
|
|
633
747
|
if (!this.isActiveChannel(channel)) return false;
|
|
634
748
|
await this.reconcileSessions(channel);
|
|
749
|
+
this.drainRuntimeDeferredAgentWork(channel);
|
|
635
750
|
}
|
|
636
751
|
// The initial-start promise stays memoized after a healthy boot. A later
|
|
637
752
|
// fatal exit can therefore leave the designated Secretary missing even
|
|
@@ -735,6 +850,10 @@ export class Orchestrator {
|
|
|
735
850
|
changed: false,
|
|
736
851
|
};
|
|
737
852
|
if (!this.isActiveChannel(channel)) return unavailable;
|
|
853
|
+
if (!backgroundContainerWorkAllowed()) {
|
|
854
|
+
this.scheduleBrowserRetry(channel);
|
|
855
|
+
return unavailable;
|
|
856
|
+
}
|
|
738
857
|
const requestedBrowser = channel.browserTesting;
|
|
739
858
|
const requestedHasCodex =
|
|
740
859
|
hasCodex || channel.roster.some((agent) => agent.kind === "codex");
|
|
@@ -818,10 +937,17 @@ export class Orchestrator {
|
|
|
818
937
|
}
|
|
819
938
|
let copiedOrUnknown = false;
|
|
820
939
|
try {
|
|
821
|
-
const provisioned =
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
940
|
+
const provisioned = channel.agentHandle
|
|
941
|
+
? await provisionEngineAccounts(
|
|
942
|
+
channel.containerName,
|
|
943
|
+
requestedKinds,
|
|
944
|
+
{},
|
|
945
|
+
channel.agentHandle,
|
|
946
|
+
)
|
|
947
|
+
: await provisionEngineAccounts(
|
|
948
|
+
channel.containerName,
|
|
949
|
+
requestedKinds,
|
|
950
|
+
);
|
|
825
951
|
copiedOrUnknown =
|
|
826
952
|
provisioned.copied > 0 || provisioned.mayHaveMutated;
|
|
827
953
|
if (provisioned.failed > 0) {
|
|
@@ -873,12 +999,20 @@ export class Orchestrator {
|
|
|
873
999
|
return unavailable;
|
|
874
1000
|
}
|
|
875
1001
|
coverage.browserAttempted = true;
|
|
876
|
-
setup =
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
1002
|
+
setup = channel.agentHandle
|
|
1003
|
+
? await setupBrowserTesting(
|
|
1004
|
+
channel.taskId,
|
|
1005
|
+
channel.containerName,
|
|
1006
|
+
requestedHasCodex,
|
|
1007
|
+
requestedHomes,
|
|
1008
|
+
channel.agentHandle,
|
|
1009
|
+
)
|
|
1010
|
+
: await setupBrowserTesting(
|
|
1011
|
+
channel.taskId,
|
|
1012
|
+
channel.containerName,
|
|
1013
|
+
requestedHasCodex,
|
|
1014
|
+
requestedHomes,
|
|
1015
|
+
);
|
|
882
1016
|
} catch (err) {
|
|
883
1017
|
console.warn(
|
|
884
1018
|
`[browser] task ${channel.taskId}: setup failed: ${
|
|
@@ -977,6 +1111,10 @@ export class Orchestrator {
|
|
|
977
1111
|
channel.browserRetryTimer = null;
|
|
978
1112
|
}
|
|
979
1113
|
if (!this.isActiveChannel(channel)) return;
|
|
1114
|
+
if (!backgroundContainerWorkAllowed()) {
|
|
1115
|
+
this.scheduleBrowserRetry(channel);
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
980
1118
|
const forceProvision = Boolean(channel.provisionRetryFingerprint);
|
|
981
1119
|
void this.ensureBrowserSetup(
|
|
982
1120
|
channel,
|
|
@@ -1063,13 +1201,28 @@ export class Orchestrator {
|
|
|
1063
1201
|
engineKinds.filter((kind) => !previous?.engineKinds?.includes(kind)),
|
|
1064
1202
|
);
|
|
1065
1203
|
const hadLiveSessions = channel.sessions.size > 0;
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1204
|
+
// Once a provider-neutral handle has already been reconstructed for this
|
|
1205
|
+
// channel, keep subsequent config writes on that same environment. The
|
|
1206
|
+
// first legacy/bootstrap pass intentionally retains its Docker fallback;
|
|
1207
|
+
// it must not introduce an eager reconstruction dependency before the
|
|
1208
|
+
// channel has a durable provider locator.
|
|
1209
|
+
const providerEnvironment = channel.agentHandle ?? undefined;
|
|
1210
|
+
const configured = providerEnvironment
|
|
1211
|
+
? await setupMcpTaskConfig(
|
|
1212
|
+
channel.taskId,
|
|
1213
|
+
channel.containerName,
|
|
1214
|
+
connections,
|
|
1215
|
+
engineKinds,
|
|
1216
|
+
codexHomes,
|
|
1217
|
+
providerEnvironment,
|
|
1218
|
+
)
|
|
1219
|
+
: await setupMcpTaskConfig(
|
|
1220
|
+
channel.taskId,
|
|
1221
|
+
channel.containerName,
|
|
1222
|
+
connections,
|
|
1223
|
+
engineKinds,
|
|
1224
|
+
codexHomes,
|
|
1225
|
+
);
|
|
1073
1226
|
if (!configured || !this.isActiveChannel(channel)) return;
|
|
1074
1227
|
channel.configGeneration += 1;
|
|
1075
1228
|
channel.mcpConfigFingerprint = fingerprint;
|
|
@@ -1219,7 +1372,12 @@ export class Orchestrator {
|
|
|
1219
1372
|
// Same per-agent materialisation the initial start does. Browser setup
|
|
1220
1373
|
// is awaited before EVERY missing-session spawn: that reasserts configs
|
|
1221
1374
|
// clobbered by resume/auth injection and covers roster generation.
|
|
1222
|
-
|
|
1375
|
+
const skillExec = this.providerSkillExec(channel);
|
|
1376
|
+
if (skillExec) {
|
|
1377
|
+
await installPackageSkills(channel.taskId, missing, skillExec);
|
|
1378
|
+
} else {
|
|
1379
|
+
await installPackageSkills(channel.taskId, missing);
|
|
1380
|
+
}
|
|
1223
1381
|
if (!this.isActiveChannel(channel)) return;
|
|
1224
1382
|
|
|
1225
1383
|
// One materialization gate owns provisioning and (when enabled)
|
|
@@ -1270,7 +1428,6 @@ export class Orchestrator {
|
|
|
1270
1428
|
{
|
|
1271
1429
|
taskId: channel.taskId,
|
|
1272
1430
|
agent,
|
|
1273
|
-
containerName: channel.containerName,
|
|
1274
1431
|
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
1275
1432
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
1276
1433
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
@@ -1366,7 +1523,7 @@ export class Orchestrator {
|
|
|
1366
1523
|
}
|
|
1367
1524
|
channel.accountByAgent.set(agent.id, account.id);
|
|
1368
1525
|
noteEngineAccountUsed(account.id);
|
|
1369
|
-
return { ...
|
|
1526
|
+
return { ...account.execEnv, ...base };
|
|
1370
1527
|
}
|
|
1371
1528
|
|
|
1372
1529
|
/** Install a session with an event handler scoped to this exact generation. */
|
|
@@ -1376,6 +1533,11 @@ export class Orchestrator {
|
|
|
1376
1533
|
session: AgentSession,
|
|
1377
1534
|
): void {
|
|
1378
1535
|
channel.sessions.set(agentId, session);
|
|
1536
|
+
// Adapters may synchronously emit `error` followed by `turn_complete` (or
|
|
1537
|
+
// `exit`). Terminal classification now awaits a daemon proof, so preserve
|
|
1538
|
+
// transport emission order across that await; otherwise the later boundary
|
|
1539
|
+
// can consume/reorder prompt state while the earlier error still owns it.
|
|
1540
|
+
let eventTail: Promise<void> = Promise.resolve();
|
|
1379
1541
|
session.onEvent((event) => {
|
|
1380
1542
|
// Recycle/teardown deletes the old session before calling close().
|
|
1381
1543
|
// Adapters deliberately emit a final exit event; without this identity
|
|
@@ -1386,15 +1548,27 @@ export class Orchestrator {
|
|
|
1386
1548
|
) {
|
|
1387
1549
|
return;
|
|
1388
1550
|
}
|
|
1389
|
-
const operation =
|
|
1390
|
-
(
|
|
1551
|
+
const operation = eventTail
|
|
1552
|
+
.catch(() => {})
|
|
1553
|
+
.then(async () => {
|
|
1554
|
+
// The session may have been retired while an earlier event awaited.
|
|
1555
|
+
// Recheck identity at execution time, not only emission time.
|
|
1556
|
+
if (
|
|
1557
|
+
!this.isActiveChannel(channel) ||
|
|
1558
|
+
channel.sessions.get(agentId) !== session
|
|
1559
|
+
) {
|
|
1560
|
+
return;
|
|
1561
|
+
}
|
|
1562
|
+
await this.handleAgentEvent(channel, agentId, session, event);
|
|
1563
|
+
})
|
|
1564
|
+
.catch((err: unknown) => {
|
|
1391
1565
|
console.warn(
|
|
1392
1566
|
`[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
|
|
1393
1567
|
err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
1394
1568
|
}`,
|
|
1395
1569
|
);
|
|
1396
|
-
}
|
|
1397
|
-
|
|
1570
|
+
});
|
|
1571
|
+
eventTail = operation;
|
|
1398
1572
|
channel.eventOperations.add(operation);
|
|
1399
1573
|
operation.then(
|
|
1400
1574
|
() => channel.eventOperations.delete(operation),
|
|
@@ -1412,7 +1586,7 @@ export class Orchestrator {
|
|
|
1412
1586
|
private createAndBindStableSession(
|
|
1413
1587
|
channel: Channel,
|
|
1414
1588
|
agentId: string,
|
|
1415
|
-
args: Parameters<AgentSessionFactory["create"]>[0],
|
|
1589
|
+
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
1416
1590
|
): Promise<AgentSession | null> {
|
|
1417
1591
|
const operation = this.createAndBindStableSessionCore(
|
|
1418
1592
|
channel,
|
|
@@ -1430,11 +1604,12 @@ export class Orchestrator {
|
|
|
1430
1604
|
private async createAndBindStableSessionCore(
|
|
1431
1605
|
channel: Channel,
|
|
1432
1606
|
agentId: string,
|
|
1433
|
-
args: Parameters<AgentSessionFactory["create"]>[0],
|
|
1607
|
+
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
1434
1608
|
): Promise<AgentSession | null> {
|
|
1609
|
+
const environment = await this.agentEnvironment(channel);
|
|
1435
1610
|
while (this.isActiveChannel(channel)) {
|
|
1436
1611
|
const generation = channel.configGeneration;
|
|
1437
|
-
const session = await this.factory.create(args);
|
|
1612
|
+
const session = await this.factory.create({ ...args, environment });
|
|
1438
1613
|
if (!this.isActiveChannel(channel)) {
|
|
1439
1614
|
await session.close().catch(() => {});
|
|
1440
1615
|
return null;
|
|
@@ -1455,6 +1630,59 @@ export class Orchestrator {
|
|
|
1455
1630
|
return null;
|
|
1456
1631
|
}
|
|
1457
1632
|
|
|
1633
|
+
private agentEnvironment(
|
|
1634
|
+
channel: Channel,
|
|
1635
|
+
): Promise<TaskEnvironmentHandle<unknown>> {
|
|
1636
|
+
const existing = channel.agentEnvironment;
|
|
1637
|
+
if (existing) return existing;
|
|
1638
|
+
const created = (async () => {
|
|
1639
|
+
const task = getHostTask(channel.taskId);
|
|
1640
|
+
if (!task) throw new Error("task environment is not persisted");
|
|
1641
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
1642
|
+
if (!environment) throw new Error("task environment is not available");
|
|
1643
|
+
if (environment.descriptor.taskId !== channel.taskId) {
|
|
1644
|
+
throw new Error("task environment belongs to a different task");
|
|
1645
|
+
}
|
|
1646
|
+
return environment;
|
|
1647
|
+
})();
|
|
1648
|
+
const tracked = created.then((environment) => {
|
|
1649
|
+
if (this.isActiveChannel(channel)) channel.agentHandle = environment;
|
|
1650
|
+
return environment;
|
|
1651
|
+
});
|
|
1652
|
+
channel.agentEnvironment = tracked;
|
|
1653
|
+
tracked.catch(() => {
|
|
1654
|
+
if (this.isActiveChannel(channel)) {
|
|
1655
|
+
channel.agentEnvironment = null;
|
|
1656
|
+
channel.agentHandle = null;
|
|
1657
|
+
}
|
|
1658
|
+
});
|
|
1659
|
+
return tracked;
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
/** Provider-neutral skill installation for a settled task environment. The
|
|
1663
|
+
* legacy path remains the default until the first environment handle has
|
|
1664
|
+
* actually been reconstructed, avoiding a new await during cold admission.
|
|
1665
|
+
*/
|
|
1666
|
+
private providerSkillExec(channel: Channel): SkillExec | undefined {
|
|
1667
|
+
const environment = channel.agentHandle;
|
|
1668
|
+
if (!environment) return undefined;
|
|
1669
|
+
return async (_container, cmd, cwd) => {
|
|
1670
|
+
const argv = cmd as [string, ...string[]];
|
|
1671
|
+
const result = await environment.exec({
|
|
1672
|
+
argv,
|
|
1673
|
+
cwd,
|
|
1674
|
+
user: "node",
|
|
1675
|
+
env: RUNTIME_AUTHORITY_ENV,
|
|
1676
|
+
timeoutMs: 120_000,
|
|
1677
|
+
maxOutputBytes: 256 * 1024,
|
|
1678
|
+
});
|
|
1679
|
+
return {
|
|
1680
|
+
status: result.exitCode,
|
|
1681
|
+
stderr: Buffer.from(result.stderr).toString("utf8"),
|
|
1682
|
+
};
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1458
1686
|
private async startSessions(channel: Channel): Promise<boolean> {
|
|
1459
1687
|
const task = getHostTask(channel.taskId);
|
|
1460
1688
|
if (
|
|
@@ -1464,6 +1692,11 @@ export class Orchestrator {
|
|
|
1464
1692
|
) {
|
|
1465
1693
|
return false;
|
|
1466
1694
|
}
|
|
1695
|
+
// Every initial in-container materialization step shares the same boot
|
|
1696
|
+
// fence as the real agent factory. Gate before identity, auth, skills,
|
|
1697
|
+
// browser/MCP configuration, or runner creation can execute in Docker.
|
|
1698
|
+
await agentClisReady;
|
|
1699
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1467
1700
|
// Freeze the generation before any slow docker work. A roster add while
|
|
1468
1701
|
// setup awaits must reconcile under its own engine-aware browser pass,
|
|
1469
1702
|
// not slip into this factory loop under the old generation's config.
|
|
@@ -1474,7 +1707,13 @@ export class Orchestrator {
|
|
|
1474
1707
|
// task-up.sh using the creator's per-user key. Best-effort. Awaited but
|
|
1475
1708
|
// async (docker exec via dockerCli) so it doesn't block the event loop —
|
|
1476
1709
|
// this runs on every channel ensure, including each post-restart reconnect.
|
|
1477
|
-
await setupTaskGitIdentity(
|
|
1710
|
+
await setupTaskGitIdentity(
|
|
1711
|
+
channel.taskId,
|
|
1712
|
+
task.ownerName,
|
|
1713
|
+
task.ownerEmail,
|
|
1714
|
+
undefined,
|
|
1715
|
+
channel.agentHandle ?? undefined,
|
|
1716
|
+
);
|
|
1478
1717
|
if (!this.isActiveChannel(channel)) return false;
|
|
1479
1718
|
|
|
1480
1719
|
// GitHub auth for both `gh` and HTTPS Git (ADR-027): inject the token,
|
|
@@ -1482,7 +1721,11 @@ export class Orchestrator {
|
|
|
1482
1721
|
// serialized reconcile so agents cannot race their first fetch/push after
|
|
1483
1722
|
// task creation, reconnect, or host recovery.
|
|
1484
1723
|
if (task.ownerUserId) {
|
|
1485
|
-
await reconcileTaskGitAuth(channel.taskId, task.ownerUserId
|
|
1724
|
+
await reconcileTaskGitAuth(channel.taskId, task.ownerUserId, {
|
|
1725
|
+
setupDeps: {
|
|
1726
|
+
environment: () => channel.agentHandle,
|
|
1727
|
+
},
|
|
1728
|
+
});
|
|
1486
1729
|
}
|
|
1487
1730
|
|
|
1488
1731
|
// SSH signing identity, re-asserted independently of transport. Connected
|
|
@@ -1501,7 +1744,12 @@ export class Orchestrator {
|
|
|
1501
1744
|
// turn. Idempotent + best-effort (returns fast when there are none); a slow
|
|
1502
1745
|
// clone/install briefly delays start, which is acceptable for skill-bearing
|
|
1503
1746
|
// tasks. Never throws.
|
|
1504
|
-
|
|
1747
|
+
const skillExec = this.providerSkillExec(channel);
|
|
1748
|
+
if (skillExec) {
|
|
1749
|
+
await installPackageSkills(channel.taskId, initialRoster, skillExec);
|
|
1750
|
+
} else {
|
|
1751
|
+
await installPackageSkills(channel.taskId, initialRoster);
|
|
1752
|
+
}
|
|
1505
1753
|
if (!this.isActiveChannel(channel)) return false;
|
|
1506
1754
|
|
|
1507
1755
|
// ADR-053: wire the Playwright MCP browser before agents spawn — configs
|
|
@@ -1559,7 +1807,6 @@ export class Orchestrator {
|
|
|
1559
1807
|
{
|
|
1560
1808
|
taskId: channel.taskId,
|
|
1561
1809
|
agent,
|
|
1562
|
-
containerName: channel.containerName,
|
|
1563
1810
|
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
1564
1811
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
1565
1812
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
@@ -1608,6 +1855,61 @@ export class Orchestrator {
|
|
|
1608
1855
|
return channel ? this.ensureSessions(channel) : false;
|
|
1609
1856
|
}
|
|
1610
1857
|
|
|
1858
|
+
/** Recreate sessions lost with the daemon and replay only prompts whose
|
|
1859
|
+
* adapter boundary explicitly rejected for runtime unavailability. */
|
|
1860
|
+
async resumeRuntimeDeferredAgentWork(): Promise<void> {
|
|
1861
|
+
if (!backgroundContainerWorkAllowed()) return;
|
|
1862
|
+
await Promise.all(
|
|
1863
|
+
[...this.channels.values()].map(async (channel) => {
|
|
1864
|
+
if (
|
|
1865
|
+
!this.isActiveChannel(channel) ||
|
|
1866
|
+
channel.runtimeDeferredAgents.size === 0
|
|
1867
|
+
) {
|
|
1868
|
+
return;
|
|
1869
|
+
}
|
|
1870
|
+
try {
|
|
1871
|
+
await this.reconcileSessions(channel);
|
|
1872
|
+
this.drainRuntimeDeferredAgentWork(channel);
|
|
1873
|
+
} catch (err) {
|
|
1874
|
+
// A single ready edge fans out across channels. One broken task must
|
|
1875
|
+
// neither abort the others nor consume its own retryable prompts;
|
|
1876
|
+
// the next ordinary ensure/poll retries and drains this channel.
|
|
1877
|
+
console.warn(
|
|
1878
|
+
`[orchestrator] ${channel.taskId}: deferred runtime work remains queued: ${
|
|
1879
|
+
err instanceof Error ? err.message : String(err)
|
|
1880
|
+
}`,
|
|
1881
|
+
);
|
|
1882
|
+
}
|
|
1883
|
+
}),
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
private drainRuntimeDeferredAgentWork(channel: Channel): void {
|
|
1888
|
+
if (
|
|
1889
|
+
!this.isActiveChannel(channel) ||
|
|
1890
|
+
!backgroundContainerWorkAllowed()
|
|
1891
|
+
) {
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
for (const agentId of [...channel.runtimeDeferredAgents]) {
|
|
1895
|
+
const session = channel.sessions.get(agentId);
|
|
1896
|
+
if (!session) continue;
|
|
1897
|
+
|
|
1898
|
+
// Clear ownership before sending. If runtime drops at the same last hop,
|
|
1899
|
+
// startPrompt's typed catch atomically re-adds the exact work.
|
|
1900
|
+
channel.runtimeDeferredAgents.delete(agentId);
|
|
1901
|
+
if (this.isSecretary(channel, agentId)) {
|
|
1902
|
+
this.startNextPendingPrompt(channel, agentId);
|
|
1903
|
+
continue;
|
|
1904
|
+
}
|
|
1905
|
+
const prompts = channel.runtimeDeferredPrompts.get(agentId) ?? [];
|
|
1906
|
+
channel.runtimeDeferredPrompts.delete(agentId);
|
|
1907
|
+
for (const prompt of prompts) {
|
|
1908
|
+
this.startPrompt(channel, agentId, session, prompt);
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
|
|
1611
1913
|
private incrementActiveTurns(channel: Channel, agentId: string): void {
|
|
1612
1914
|
channel.activeTurns.set(
|
|
1613
1915
|
agentId,
|
|
@@ -1645,6 +1947,7 @@ export class Orchestrator {
|
|
|
1645
1947
|
private retireRunnerGeneration(channel: Channel, agentId: string): void {
|
|
1646
1948
|
const aborted = channel.interrupted.delete(agentId);
|
|
1647
1949
|
channel.activeTurns.delete(agentId);
|
|
1950
|
+
channel.acceptedPrompts.delete(agentId);
|
|
1648
1951
|
channel.openTurns.delete(agentId);
|
|
1649
1952
|
if (aborted) {
|
|
1650
1953
|
channel.lastPrompt.delete(agentId);
|
|
@@ -1666,9 +1969,14 @@ export class Orchestrator {
|
|
|
1666
1969
|
// ADR-076: this is the prompt actually in flight. A queued follow-up must
|
|
1667
1970
|
// not replace it or account rotation would replay the wrong turn.
|
|
1668
1971
|
channel.lastPrompt.set(agentId, prompt);
|
|
1972
|
+
const accepted = { text: prompt };
|
|
1973
|
+
const acceptedForAgent = channel.acceptedPrompts.get(agentId) ?? [];
|
|
1974
|
+
acceptedForAgent.push(accepted);
|
|
1975
|
+
channel.acceptedPrompts.set(agentId, acceptedForAgent);
|
|
1669
1976
|
this.incrementActiveTurns(channel, agentId);
|
|
1670
1977
|
void session.send(prompt).catch((err: unknown) => {
|
|
1671
1978
|
if (channel.sessions.get(agentId) === session) {
|
|
1979
|
+
this.removeAcceptedPrompt(channel, agentId, accepted);
|
|
1672
1980
|
const remaining = this.decrementActiveTurns(channel, agentId);
|
|
1673
1981
|
if (
|
|
1674
1982
|
remaining === 0 &&
|
|
@@ -1676,7 +1984,11 @@ export class Orchestrator {
|
|
|
1676
1984
|
) {
|
|
1677
1985
|
channel.lastPrompt.delete(agentId);
|
|
1678
1986
|
}
|
|
1679
|
-
|
|
1987
|
+
if (err instanceof ContainerRuntimeUnavailableError) {
|
|
1988
|
+
this.deferPromptUntilRuntimeReady(channel, agentId, prompt);
|
|
1989
|
+
} else {
|
|
1990
|
+
this.startNextPendingPrompt(channel, agentId);
|
|
1991
|
+
}
|
|
1680
1992
|
}
|
|
1681
1993
|
console.warn(
|
|
1682
1994
|
`[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
|
|
@@ -1686,6 +1998,49 @@ export class Orchestrator {
|
|
|
1686
1998
|
});
|
|
1687
1999
|
}
|
|
1688
2000
|
|
|
2001
|
+
private removeAcceptedPrompt(
|
|
2002
|
+
channel: Channel,
|
|
2003
|
+
agentId: string,
|
|
2004
|
+
accepted: AcceptedPrompt,
|
|
2005
|
+
): void {
|
|
2006
|
+
const prompts = channel.acceptedPrompts.get(agentId);
|
|
2007
|
+
if (!prompts) return;
|
|
2008
|
+
const index = prompts.indexOf(accepted);
|
|
2009
|
+
if (index >= 0) prompts.splice(index, 1);
|
|
2010
|
+
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
private completeNextAcceptedPrompt(
|
|
2014
|
+
channel: Channel,
|
|
2015
|
+
agentId: string,
|
|
2016
|
+
): void {
|
|
2017
|
+
const prompts = channel.acceptedPrompts.get(agentId);
|
|
2018
|
+
if (!prompts) return;
|
|
2019
|
+
prompts.shift();
|
|
2020
|
+
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
private deferPromptUntilRuntimeReady(
|
|
2024
|
+
channel: Channel,
|
|
2025
|
+
agentId: string,
|
|
2026
|
+
prompt: string,
|
|
2027
|
+
): void {
|
|
2028
|
+
channel.runtimeDeferredAgents.add(agentId);
|
|
2029
|
+
if (this.isSecretary(channel, agentId)) {
|
|
2030
|
+
// Secretary delivery is serialized. Put the failed in-flight prompt in
|
|
2031
|
+
// front of crew replies that were accepted behind it.
|
|
2032
|
+
const pending = channel.pendingPrompts.get(agentId) ?? [];
|
|
2033
|
+
pending.unshift(prompt);
|
|
2034
|
+
channel.pendingPrompts.set(agentId, pending);
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
2037
|
+
// Open-mode sends are intentionally concurrent. Promise rejection handlers
|
|
2038
|
+
// run in submission order, so append preserves that accepted order.
|
|
2039
|
+
const deferred = channel.runtimeDeferredPrompts.get(agentId) ?? [];
|
|
2040
|
+
deferred.push(prompt);
|
|
2041
|
+
channel.runtimeDeferredPrompts.set(agentId, deferred);
|
|
2042
|
+
}
|
|
2043
|
+
|
|
1689
2044
|
/** Start at most one queued prompt after the previous turn is terminal. */
|
|
1690
2045
|
private startNextPendingPrompt(channel: Channel, agentId: string): boolean {
|
|
1691
2046
|
if (
|
|
@@ -1804,7 +2159,11 @@ export class Orchestrator {
|
|
|
1804
2159
|
// Cancel any armed auto-retry chain so this manual attempt runs fresh
|
|
1805
2160
|
// (setup's duplicate-chain guard would otherwise no-op it).
|
|
1806
2161
|
clearRefresh(taskId);
|
|
1807
|
-
const ok = await reconcileTaskGitAuth(taskId, owner
|
|
2162
|
+
const ok = await reconcileTaskGitAuth(taskId, owner, {
|
|
2163
|
+
setupDeps: {
|
|
2164
|
+
environment: () => this.channels.get(taskId)?.agentHandle ?? null,
|
|
2165
|
+
},
|
|
2166
|
+
});
|
|
1808
2167
|
this.emitSystemNote(
|
|
1809
2168
|
taskId,
|
|
1810
2169
|
ok
|
|
@@ -1818,6 +2177,7 @@ export class Orchestrator {
|
|
|
1818
2177
|
private async handleAgentEvent(
|
|
1819
2178
|
channel: Channel,
|
|
1820
2179
|
agentId: string,
|
|
2180
|
+
session: AgentSession,
|
|
1821
2181
|
event: AgentEvent,
|
|
1822
2182
|
): Promise<void> {
|
|
1823
2183
|
if (!this.isActiveChannel(channel)) return;
|
|
@@ -1901,6 +2261,18 @@ export class Orchestrator {
|
|
|
1901
2261
|
break;
|
|
1902
2262
|
}
|
|
1903
2263
|
case "error": {
|
|
2264
|
+
if (await agentTerminationLostContainerRuntime()) {
|
|
2265
|
+
// `error` and `exit` can arrive back-to-back while the daemon probe
|
|
2266
|
+
// is pending. Only the first terminal event still owns this session.
|
|
2267
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
2268
|
+
await this.retireSessionForRuntimeLoss(
|
|
2269
|
+
channel,
|
|
2270
|
+
agentId,
|
|
2271
|
+
session,
|
|
2272
|
+
);
|
|
2273
|
+
break;
|
|
2274
|
+
}
|
|
2275
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
1904
2276
|
// The turn died with the session — drop its turn-state flags so a
|
|
1905
2277
|
// respawned session starts clean.
|
|
1906
2278
|
this.retireRunnerGeneration(channel, agentId);
|
|
@@ -1972,6 +2344,7 @@ export class Orchestrator {
|
|
|
1972
2344
|
// An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
|
|
1973
2345
|
// the cloud discards the buffer instead of handing it to peers.
|
|
1974
2346
|
const aborted = channel.interrupted.delete(agentId);
|
|
2347
|
+
this.completeNextAcceptedPrompt(channel, agentId);
|
|
1975
2348
|
const remainingTurns = this.decrementActiveTurns(channel, agentId);
|
|
1976
2349
|
channel.openTurns.delete(agentId);
|
|
1977
2350
|
// ADR-076: a turn that completed means the current account is healthy —
|
|
@@ -2005,6 +2378,16 @@ export class Orchestrator {
|
|
|
2005
2378
|
break;
|
|
2006
2379
|
}
|
|
2007
2380
|
case "exit":
|
|
2381
|
+
if (await agentTerminationLostContainerRuntime()) {
|
|
2382
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
2383
|
+
await this.retireSessionForRuntimeLoss(
|
|
2384
|
+
channel,
|
|
2385
|
+
agentId,
|
|
2386
|
+
session,
|
|
2387
|
+
);
|
|
2388
|
+
break;
|
|
2389
|
+
}
|
|
2390
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
2008
2391
|
// Same zombie hazard as the error path — a session whose process
|
|
2009
2392
|
// ended (even cleanly) can never carry another turn.
|
|
2010
2393
|
this.retireRunnerGeneration(channel, agentId);
|
|
@@ -2019,6 +2402,35 @@ export class Orchestrator {
|
|
|
2019
2402
|
}
|
|
2020
2403
|
}
|
|
2021
2404
|
|
|
2405
|
+
/** A daemon loss is infrastructure recovery, not an agent crash. Preserve
|
|
2406
|
+
* accepted work, drop the dead generation, and leave respawn/exit accounting
|
|
2407
|
+
* untouched; the runtime-ready hook recreates it deterministically. */
|
|
2408
|
+
private async retireSessionForRuntimeLoss(
|
|
2409
|
+
channel: Channel,
|
|
2410
|
+
agentId: string,
|
|
2411
|
+
session: AgentSession,
|
|
2412
|
+
): Promise<void> {
|
|
2413
|
+
const accepted = [
|
|
2414
|
+
...(channel.acceptedPrompts.get(agentId) ?? []),
|
|
2415
|
+
];
|
|
2416
|
+
const interrupted = channel.interrupted.has(agentId);
|
|
2417
|
+
this.retireRunnerGeneration(channel, agentId);
|
|
2418
|
+
channel.lastPrompt.delete(agentId);
|
|
2419
|
+
if (accepted.length > 0 && !interrupted) {
|
|
2420
|
+
for (const prompt of accepted) {
|
|
2421
|
+
this.deferPromptUntilRuntimeReady(channel, agentId, prompt.text);
|
|
2422
|
+
}
|
|
2423
|
+
} else {
|
|
2424
|
+
// Recreate even an idle session after readiness, without fabricating a
|
|
2425
|
+
// prompt or requiring another human delivery to discover the dead slot.
|
|
2426
|
+
channel.runtimeDeferredAgents.add(agentId);
|
|
2427
|
+
}
|
|
2428
|
+
channel.sessions.delete(agentId);
|
|
2429
|
+
channel.browserStaleSessions.delete(agentId);
|
|
2430
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
2431
|
+
await session.close().catch(() => {});
|
|
2432
|
+
}
|
|
2433
|
+
|
|
2022
2434
|
// -- session recovery -----------------------------------------------------
|
|
2023
2435
|
|
|
2024
2436
|
/**
|
|
@@ -2054,7 +2466,10 @@ export class Orchestrator {
|
|
|
2054
2466
|
|
|
2055
2467
|
channel.spawning.add(agentId);
|
|
2056
2468
|
try {
|
|
2057
|
-
const restored = repairClaudeConfigInContainer(
|
|
2469
|
+
const restored = await repairClaudeConfigInContainer(
|
|
2470
|
+
channel.containerName,
|
|
2471
|
+
channel.agentHandle ?? undefined,
|
|
2472
|
+
);
|
|
2058
2473
|
|
|
2059
2474
|
// Delete the old generation before awaiting close. Its adapter emits a
|
|
2060
2475
|
// terminal event during teardown; bindSession's identity guard must see
|
|
@@ -2097,7 +2512,6 @@ export class Orchestrator {
|
|
|
2097
2512
|
const session = await this.createAndBindStableSession(channel, agentId, {
|
|
2098
2513
|
taskId: channel.taskId,
|
|
2099
2514
|
agent,
|
|
2100
|
-
containerName: channel.containerName,
|
|
2101
2515
|
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
2102
2516
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2103
2517
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
@@ -2105,7 +2519,7 @@ export class Orchestrator {
|
|
|
2105
2519
|
agentId,
|
|
2106
2520
|
),
|
|
2107
2521
|
agentEnv: boundAccount
|
|
2108
|
-
? { ...
|
|
2522
|
+
? { ...boundAccount.execEnv, ...base }
|
|
2109
2523
|
: this.accountAgentEnv(channel, agent, base),
|
|
2110
2524
|
});
|
|
2111
2525
|
if (!session) return;
|
|
@@ -2226,14 +2640,13 @@ export class Orchestrator {
|
|
|
2226
2640
|
{
|
|
2227
2641
|
taskId: channel.taskId,
|
|
2228
2642
|
agent,
|
|
2229
|
-
containerName: channel.containerName,
|
|
2230
2643
|
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
2231
2644
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2232
2645
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2233
2646
|
channel,
|
|
2234
2647
|
agentId,
|
|
2235
2648
|
),
|
|
2236
|
-
agentEnv: { ...
|
|
2649
|
+
agentEnv: { ...next.execEnv, ...base },
|
|
2237
2650
|
},
|
|
2238
2651
|
);
|
|
2239
2652
|
if (!replacement) return true;
|
|
@@ -2341,14 +2754,13 @@ export class Orchestrator {
|
|
|
2341
2754
|
replacement = await this.createAndBindStableSession(channel, agentId, {
|
|
2342
2755
|
taskId: channel.taskId,
|
|
2343
2756
|
agent,
|
|
2344
|
-
containerName: channel.containerName,
|
|
2345
2757
|
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
2346
2758
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2347
2759
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2348
2760
|
channel,
|
|
2349
2761
|
agentId,
|
|
2350
2762
|
),
|
|
2351
|
-
agentEnv: { ...
|
|
2763
|
+
agentEnv: { ...account.execEnv, ...base },
|
|
2352
2764
|
});
|
|
2353
2765
|
if (!replacement) return true;
|
|
2354
2766
|
|
|
@@ -2487,7 +2899,10 @@ export class Orchestrator {
|
|
|
2487
2899
|
ch.browserStaleSessions.clear();
|
|
2488
2900
|
ch.browserPendingStaleSessions.clear();
|
|
2489
2901
|
ch.activeTurns.clear();
|
|
2902
|
+
ch.acceptedPrompts.clear();
|
|
2490
2903
|
ch.pendingPrompts.clear();
|
|
2904
|
+
ch.runtimeDeferredPrompts.clear();
|
|
2905
|
+
ch.runtimeDeferredAgents.clear();
|
|
2491
2906
|
ch.openTurns.clear();
|
|
2492
2907
|
ch.interrupted.clear();
|
|
2493
2908
|
ch.reconcileAgain = false;
|
|
@@ -2527,6 +2942,18 @@ export class Orchestrator {
|
|
|
2527
2942
|
return this.channelClosures.get(taskId) ?? Promise.resolve();
|
|
2528
2943
|
}
|
|
2529
2944
|
|
|
2945
|
+
/** Quarantine a live channel before recovery stops its app container.
|
|
2946
|
+
* Preserve the latest channel spec so a successful recovery can reopen the
|
|
2947
|
+
* task without waiting for another cloud message, while the ordinary close
|
|
2948
|
+
* tombstone prevents any agent process from respawning during maintenance. */
|
|
2949
|
+
async quarantineChannelForRecovery(taskId: string): Promise<void> {
|
|
2950
|
+
const currentSpec = this.channelSpecs.get(taskId);
|
|
2951
|
+
await this.closeChannel(taskId);
|
|
2952
|
+
if (currentSpec && !this.channelSpecs.has(taskId)) {
|
|
2953
|
+
this.channelSpecs.set(taskId, currentSpec);
|
|
2954
|
+
}
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2530
2957
|
/** A successful task-up is the only transition that reopens a tombstoned
|
|
2531
2958
|
* task id after stop/teardown. */
|
|
2532
2959
|
allowChannel(taskId: string): void {
|
|
@@ -2553,6 +2980,9 @@ export class Orchestrator {
|
|
|
2553
2980
|
private async stopTaskWithinLifecycle(
|
|
2554
2981
|
taskId: string,
|
|
2555
2982
|
): Promise<{ ok: boolean; error?: string }> {
|
|
2983
|
+
if (!backgroundContainerWorkAllowed()) {
|
|
2984
|
+
return { ok: false, error: "container runtime is not operational" };
|
|
2985
|
+
}
|
|
2556
2986
|
const task = getHostTask(taskId);
|
|
2557
2987
|
if (!task) return { ok: false, error: "unknown task" };
|
|
2558
2988
|
const project = task.composeProject;
|
|
@@ -2568,33 +2998,27 @@ export class Orchestrator {
|
|
|
2568
2998
|
}`,
|
|
2569
2999
|
);
|
|
2570
3000
|
}
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
3001
|
+
// Provider-neutral resumable stop. A row that reaches this point with
|
|
3002
|
+
// runtime resources always carries a proven locator (legacy rows cross
|
|
3003
|
+
// the adoption fence before the runtime becomes operational), so a
|
|
3004
|
+
// reconstruction failure is a stop failure, never a fallback hint. An
|
|
3005
|
+
// environment failure reopens the channel and leaves the runtime mirror
|
|
3006
|
+
// alone so cloud reconciliation cannot turn a live container into
|
|
3007
|
+
// "stopped".
|
|
3008
|
+
try {
|
|
3009
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
3010
|
+
if (!environment) {
|
|
3011
|
+
this.allowChannel(taskId);
|
|
3012
|
+
return { ok: false, error: "task has no persisted environment" };
|
|
3013
|
+
}
|
|
3014
|
+
await environment.stop();
|
|
3015
|
+
} catch (err) {
|
|
2581
3016
|
this.allowChannel(taskId);
|
|
2582
3017
|
return {
|
|
2583
3018
|
ok: false,
|
|
2584
|
-
error:
|
|
3019
|
+
error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
|
|
2585
3020
|
};
|
|
2586
3021
|
}
|
|
2587
|
-
const ids = ps.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
2588
|
-
if (ids.length > 0) {
|
|
2589
|
-
const stopped = await dockerCli(["stop", ...ids], { timeoutMs: 60_000 });
|
|
2590
|
-
if (stopped.status !== 0) {
|
|
2591
|
-
this.allowChannel(taskId);
|
|
2592
|
-
return {
|
|
2593
|
-
ok: false,
|
|
2594
|
-
error: stopped.stderr.trim().slice(0, 200) || "docker stop failed",
|
|
2595
|
-
};
|
|
2596
|
-
}
|
|
2597
|
-
}
|
|
2598
3022
|
upsertHostTask(taskId, { statusMirror: "stopped" });
|
|
2599
3023
|
clearTaskGatewayAcl(taskId);
|
|
2600
3024
|
return { ok: true };
|
|
@@ -2688,7 +3112,10 @@ export function resolveAddressing(
|
|
|
2688
3112
|
*
|
|
2689
3113
|
* Returns true when the file actually made it into the container.
|
|
2690
3114
|
*/
|
|
2691
|
-
function repairClaudeConfigInContainer(
|
|
3115
|
+
async function repairClaudeConfigInContainer(
|
|
3116
|
+
containerName: string,
|
|
3117
|
+
environment?: Pick<TaskEnvironmentHandle, "exec">,
|
|
3118
|
+
): Promise<boolean> {
|
|
2692
3119
|
// Operator can override the source path via UAI_OWNER_HOME so the
|
|
2693
3120
|
// server doesn't have to guess. Useful when Next.js's dev process
|
|
2694
3121
|
// env diverges from `os.homedir()` for any reason.
|
|
@@ -2733,25 +3160,44 @@ function repairClaudeConfigInContainer(containerName: string): boolean {
|
|
|
2733
3160
|
// bounces. `rm -f` punches through it (or no-ops on a genuine
|
|
2734
3161
|
// missing file). Then `cat >` creates the fresh inode. Both steps
|
|
2735
3162
|
// run in one `sh -c` so a partial failure leaves a clean state.
|
|
2736
|
-
const
|
|
2737
|
-
"
|
|
3163
|
+
const repairScript =
|
|
3164
|
+
"rm -f /home/node/.claude.json && cat > /home/node/.claude.json";
|
|
3165
|
+
if (environment) {
|
|
3166
|
+
const write = await environment.exec({
|
|
3167
|
+
argv: ["/bin/sh", "-c", repairScript],
|
|
3168
|
+
env: RUNTIME_AUTHORITY_ENV,
|
|
3169
|
+
stdin: content,
|
|
3170
|
+
timeoutMs: 30_000,
|
|
3171
|
+
maxOutputBytes: 64 * 1024,
|
|
3172
|
+
});
|
|
3173
|
+
if (write.exitCode !== 0) {
|
|
3174
|
+
console.error(
|
|
3175
|
+
"[orchestrator] repair: in-place write to .claude.json failed: " +
|
|
3176
|
+
Buffer.from(write.stderr).toString("utf8").trim(),
|
|
3177
|
+
);
|
|
3178
|
+
return false;
|
|
3179
|
+
}
|
|
3180
|
+
return true;
|
|
3181
|
+
}
|
|
3182
|
+
const write = await taskContainerCli(
|
|
2738
3183
|
[
|
|
2739
3184
|
"exec",
|
|
2740
3185
|
"-i",
|
|
2741
|
-
|
|
2742
|
-
|
|
3186
|
+
...runtimeAuthorityDockerExecArgs(),
|
|
3187
|
+
// The channel carries the Docker replica name; the apple container
|
|
3188
|
+
// drops the replica suffix (2026-08-18 spawn-docker-ENOENT sweep).
|
|
3189
|
+
taskContainerBackend().apple
|
|
3190
|
+
? containerName.replace(/-app-1$/, "-app")
|
|
3191
|
+
: containerName,
|
|
3192
|
+
"/bin/sh",
|
|
2743
3193
|
"-c",
|
|
2744
|
-
|
|
3194
|
+
repairScript,
|
|
2745
3195
|
],
|
|
2746
|
-
{ input: content,
|
|
3196
|
+
{ input: content.toString("utf8"), timeoutMs: 30_000 },
|
|
2747
3197
|
);
|
|
2748
3198
|
if (write.status !== 0) {
|
|
2749
|
-
const stderr =
|
|
2750
|
-
write.stderr instanceof Buffer
|
|
2751
|
-
? write.stderr.toString("utf8")
|
|
2752
|
-
: String(write.stderr ?? "");
|
|
2753
3199
|
console.error(
|
|
2754
|
-
`[orchestrator] repair: in-place write to .claude.json failed: ${stderr.trim()}`,
|
|
3200
|
+
`[orchestrator] repair: in-place write to .claude.json failed: ${write.stderr.trim()}`,
|
|
2755
3201
|
);
|
|
2756
3202
|
return false;
|
|
2757
3203
|
}
|
|
@@ -3407,19 +3853,60 @@ export function getOrchestrator(): Orchestrator {
|
|
|
3407
3853
|
const factory = mode === "real" ? realAgentFactory : mockAgentFactory;
|
|
3408
3854
|
globalForOrchestrator.__uaiOrchestrator = new Orchestrator(factory);
|
|
3409
3855
|
}
|
|
3410
|
-
if (!globalForOrchestrator.__uaiRecoverRan) {
|
|
3411
|
-
globalForOrchestrator.__uaiRecoverRan = true;
|
|
3412
|
-
// Fire-and-forget — the orchestrator is usable while recovery runs.
|
|
3413
|
-
// The promise is kept so boot steps that must NOT race recovery (the
|
|
3414
|
-
// Codex credential reinject) can sequence behind it via
|
|
3415
|
-
// recoveryComplete(). recoverRunningTasks never rejects.
|
|
3416
|
-
recoveryPromise = recoverRunningTasks();
|
|
3417
|
-
void recoveryPromise;
|
|
3418
|
-
}
|
|
3419
3856
|
return globalForOrchestrator.__uaiOrchestrator;
|
|
3420
3857
|
}
|
|
3421
3858
|
|
|
3422
|
-
|
|
3859
|
+
/** Runtime activation publication hook. Do not construct an orchestrator just
|
|
3860
|
+
* to run it: deferred work can only exist on the already-live singleton. */
|
|
3861
|
+
export function resumeRuntimeDeferredAgentWork(): void {
|
|
3862
|
+
const orchestrator = globalForOrchestrator.__uaiOrchestrator;
|
|
3863
|
+
if (!orchestrator) return;
|
|
3864
|
+
void orchestrator.resumeRuntimeDeferredAgentWork().catch((err) =>
|
|
3865
|
+
console.warn(
|
|
3866
|
+
`[orchestrator] deferred agent work resume failed: ${
|
|
3867
|
+
err instanceof Error ? err.message : String(err)
|
|
3868
|
+
}`,
|
|
3869
|
+
),
|
|
3870
|
+
);
|
|
3871
|
+
}
|
|
3872
|
+
|
|
3873
|
+
let recoveryPromise: Promise<boolean> = Promise.resolve(true);
|
|
3874
|
+
let recoveryInFlight: Promise<boolean> | null = null;
|
|
3875
|
+
|
|
3876
|
+
/** Start or join one recovery sweep. A runtime-ready recheck can call this
|
|
3877
|
+
* after the original boot window gave up, without racing an active sweep. */
|
|
3878
|
+
function beginRuntimeRecovery(
|
|
3879
|
+
lifecycle?: Orchestrator,
|
|
3880
|
+
isCurrent: () => boolean = () => true,
|
|
3881
|
+
): Promise<boolean> {
|
|
3882
|
+
// A caller that only wants the boot barrier joins it. A live-runtime request
|
|
3883
|
+
// is stronger: chain a fresh lifecycle-sequenced pass behind any active
|
|
3884
|
+
// scan. Promise-tail sequencing has no "last queue read vs finally" gap, so
|
|
3885
|
+
// a readiness request arriving in that microtask window cannot be lost.
|
|
3886
|
+
if (recoveryInFlight && !lifecycle) return recoveryInFlight;
|
|
3887
|
+
const predecessor = recoveryInFlight;
|
|
3888
|
+
const sweep = async (): Promise<boolean> => {
|
|
3889
|
+
try {
|
|
3890
|
+
return await recoverRunningTasks({ lifecycle, isCurrent });
|
|
3891
|
+
} catch (error) {
|
|
3892
|
+
console.error(
|
|
3893
|
+
"[orchestrator] recovery: unhandled failure",
|
|
3894
|
+
error instanceof Error ? error.message : error,
|
|
3895
|
+
);
|
|
3896
|
+
return false;
|
|
3897
|
+
}
|
|
3898
|
+
};
|
|
3899
|
+
let running!: Promise<boolean>;
|
|
3900
|
+
running = (predecessor
|
|
3901
|
+
? predecessor.catch(() => false).then(sweep)
|
|
3902
|
+
: sweep()
|
|
3903
|
+
).finally(() => {
|
|
3904
|
+
if (recoveryInFlight === running) recoveryInFlight = null;
|
|
3905
|
+
});
|
|
3906
|
+
recoveryInFlight = running;
|
|
3907
|
+
recoveryPromise = running;
|
|
3908
|
+
return running;
|
|
3909
|
+
}
|
|
3423
3910
|
|
|
3424
3911
|
interface TaskRecoveryBarrier {
|
|
3425
3912
|
promise: Promise<void>;
|
|
@@ -3456,13 +3943,73 @@ export function taskRecoveryComplete(taskId: string): Promise<void> {
|
|
|
3456
3943
|
|
|
3457
3944
|
/**
|
|
3458
3945
|
* Resolves once boot-time recovery has finished (starting the orchestrator —
|
|
3459
|
-
* and with it recovery — if that hasn't happened yet). Never rejects
|
|
3946
|
+
* and with it recovery — if that hasn't happened yet). Never rejects; false
|
|
3947
|
+
* means one or more active tasks remain unresolved.
|
|
3460
3948
|
*/
|
|
3461
|
-
export function recoveryComplete(
|
|
3949
|
+
export function recoveryComplete(
|
|
3950
|
+
isCurrent: () => boolean = () => true,
|
|
3951
|
+
): Promise<boolean> {
|
|
3462
3952
|
getOrchestrator();
|
|
3953
|
+
if (!globalForOrchestrator.__uaiRecoverRan) {
|
|
3954
|
+
globalForOrchestrator.__uaiRecoverRan = true;
|
|
3955
|
+
// Fire-and-forget — the orchestrator is usable while recovery runs. Main
|
|
3956
|
+
// calls this only after the selected container runtime is ready.
|
|
3957
|
+
void beginRuntimeRecovery(undefined, isCurrent);
|
|
3958
|
+
}
|
|
3463
3959
|
return recoveryPromise;
|
|
3464
3960
|
}
|
|
3465
3961
|
|
|
3962
|
+
/**
|
|
3963
|
+
* Re-run boot recovery after a container runtime transitions to ready.
|
|
3964
|
+
*
|
|
3965
|
+
* `recoveryComplete()` is a one-boot barrier; this is the explicit recovery
|
|
3966
|
+
* trigger for ADR-101 runtime rechecks. It joins an active boot sweep, or
|
|
3967
|
+
* starts a new sweep after the prior bounded retry window completed.
|
|
3968
|
+
*/
|
|
3969
|
+
export function requestRuntimeRecovery(
|
|
3970
|
+
isCurrent: () => boolean = () => true,
|
|
3971
|
+
): Promise<boolean> {
|
|
3972
|
+
const orchestrator = getOrchestrator();
|
|
3973
|
+
globalForOrchestrator.__uaiRecoverRan = true;
|
|
3974
|
+
return beginRuntimeRecovery(orchestrator, isCurrent);
|
|
3975
|
+
}
|
|
3976
|
+
|
|
3977
|
+
/** Join only work that is already mutating/reconciling runtime state. This is
|
|
3978
|
+
* used before image maintenance so an explicit recheck cannot overlap a
|
|
3979
|
+
* reconnect sweep on the shared daemon/CLI volume. */
|
|
3980
|
+
export function waitForRuntimeRecovery(): Promise<void> {
|
|
3981
|
+
return recoveryInFlight?.then(() => undefined) ?? Promise.resolve();
|
|
3982
|
+
}
|
|
3983
|
+
|
|
3984
|
+
export type RuntimeRecoveryVerdict = "clean" | "needed" | "unknown";
|
|
3985
|
+
|
|
3986
|
+
/** Cheap reconnect proof. Each active environment provider reports whether
|
|
3987
|
+
* its durable workload is still running; ordinary cloud/network flaps avoid
|
|
3988
|
+
* the heavier repair path, while an unseen runtime restart still forces full
|
|
3989
|
+
* lifecycle-sequenced recovery before commands reopen. */
|
|
3990
|
+
export async function runtimeRecoveryVerdict(): Promise<RuntimeRecoveryVerdict> {
|
|
3991
|
+
const activeRows = getDb()
|
|
3992
|
+
.select()
|
|
3993
|
+
.from(schema.hostTasks)
|
|
3994
|
+
.where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
|
|
3995
|
+
.all();
|
|
3996
|
+
for (const task of activeRows) {
|
|
3997
|
+
try {
|
|
3998
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
3999
|
+
// A pre-provision row has no provider-owned runtime to inventory.
|
|
4000
|
+
if (environment === null) continue;
|
|
4001
|
+
const status = await environment.status();
|
|
4002
|
+
if (status.state === "unknown") return "unknown";
|
|
4003
|
+
if (status.state !== "running") return "needed";
|
|
4004
|
+
} catch {
|
|
4005
|
+
// An unavailable provider, malformed locator, or backend mismatch is
|
|
4006
|
+
// unknown—not evidence that the selected runtime is clean or absent.
|
|
4007
|
+
return "unknown";
|
|
4008
|
+
}
|
|
4009
|
+
}
|
|
4010
|
+
return "clean";
|
|
4011
|
+
}
|
|
4012
|
+
|
|
3466
4013
|
// ---------------------------------------------------------------------------
|
|
3467
4014
|
// Boot-time recovery
|
|
3468
4015
|
//
|
|
@@ -3481,8 +4028,10 @@ export function recoveryComplete(): Promise<void> {
|
|
|
3481
4028
|
// id (task-up is idempotent).
|
|
3482
4029
|
// container gone, worktree gone → mark `error`. Data loss.
|
|
3483
4030
|
//
|
|
3484
|
-
// Idempotent per process —
|
|
3485
|
-
// so a Next.js HMR rebuild
|
|
4031
|
+
// Idempotent per process — recoveryComplete/requestRuntimeRecovery mark
|
|
4032
|
+
// `__uaiRecoverRan` on globalThis so a Next.js HMR rebuild cannot trigger the
|
|
4033
|
+
// boot sweep twice. Merely constructing the orchestrator is deliberately not
|
|
4034
|
+
// enough: a degraded host must not probe an unrelated ambient daemon.
|
|
3486
4035
|
// ---------------------------------------------------------------------------
|
|
3487
4036
|
|
|
3488
4037
|
/**
|
|
@@ -3529,32 +4078,1168 @@ async function dockerListContainersByLabel(
|
|
|
3529
4078
|
.filter((x): x is DockerPs => x !== null);
|
|
3530
4079
|
}
|
|
3531
4080
|
|
|
3532
|
-
|
|
4081
|
+
type DockerStartResult = "started" | "failed" | "unreachable";
|
|
4082
|
+
|
|
4083
|
+
async function dockerDaemonReachable(): Promise<boolean> {
|
|
4084
|
+
const probe = await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
|
|
4085
|
+
timeoutMs: 10_000,
|
|
4086
|
+
});
|
|
4087
|
+
return probe.status === 0 && probe.stdout.trim().length > 0;
|
|
4088
|
+
}
|
|
4089
|
+
|
|
4090
|
+
async function dockerStart(containerName: string): Promise<DockerStartResult> {
|
|
3533
4091
|
// Starting a big dev container can legitimately take a while — give it far
|
|
3534
4092
|
// more than dockerCli's 30s default before declaring the task stopped.
|
|
3535
4093
|
const res = await dockerCli(["start", containerName], { timeoutMs: 120_000 });
|
|
3536
|
-
if (res.status
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
4094
|
+
if (res.status === 0) return "started";
|
|
4095
|
+
// A timeout is ambiguous: Docker may have completed the start after our CLI
|
|
4096
|
+
// was killed. Inspect the exact container, then prove the daemon still
|
|
4097
|
+
// exists before deciding a non-inspectable container is a terminal failure.
|
|
4098
|
+
const inspected = await dockerCli([
|
|
4099
|
+
"inspect",
|
|
4100
|
+
"--format",
|
|
4101
|
+
"{{.State.Running}}",
|
|
4102
|
+
containerName,
|
|
4103
|
+
]);
|
|
4104
|
+
if (inspected.status === 0 && inspected.stdout.trim() === "true") {
|
|
4105
|
+
return "started";
|
|
3541
4106
|
}
|
|
3542
|
-
|
|
4107
|
+
if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
|
|
4108
|
+
return "unreachable";
|
|
4109
|
+
}
|
|
4110
|
+
console.error(
|
|
4111
|
+
`[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
|
|
4112
|
+
);
|
|
4113
|
+
return "failed";
|
|
4114
|
+
}
|
|
4115
|
+
|
|
4116
|
+
// The fourth column is the preview-sidecar marker: a sidecar can wear the
|
|
4117
|
+
// task's compose labels, and a row that is neither app-shaped nor a known
|
|
4118
|
+
// sidecar reads as AMBIGUOUS here — which fenced maintenance on a healthy
|
|
4119
|
+
// host (live 2026-08-18).
|
|
4120
|
+
const RUNTIME_WRITER_LIST_FORMAT =
|
|
4121
|
+
`{{json .Names}}\t{{.Label "com.docker.compose.project"}}\t{{.Label "com.docker.compose.service"}}\t{{.Label "${PREVIEW_SIDECAR_TASK_LABEL}"}}`;
|
|
4122
|
+
const RUNTIME_WRITER_INSPECT_FORMAT = [
|
|
4123
|
+
"{{json .Id}}",
|
|
4124
|
+
"{{json .Name}}",
|
|
4125
|
+
"{{json .State.Running}}",
|
|
4126
|
+
"{{json .State.Restarting}}",
|
|
4127
|
+
"{{json .State.Paused}}",
|
|
4128
|
+
"{{json .HostConfig.RestartPolicy.Name}}",
|
|
4129
|
+
'{{json (index .Config.Labels "com.docker.compose.project")}}',
|
|
4130
|
+
'{{json (index .Config.Labels "com.docker.compose.service")}}',
|
|
4131
|
+
"{{json .Mounts}}",
|
|
4132
|
+
"{{json .Config.Env}}",
|
|
4133
|
+
].join("\n");
|
|
4134
|
+
const RUNTIME_WRITER_CONTAINER_ID_RE = /^[0-9a-f]{64}$/;
|
|
4135
|
+
const RUNTIME_WRITER_SCAN_OUTPUT_BYTES = 2 * 1024 * 1024;
|
|
4136
|
+
const TASK_RUNTIME_PATH = RUNTIME_AUTHORITY_ENV.PATH;
|
|
4137
|
+
const TASK_OWNED_ENV = new Map<string, string>(
|
|
4138
|
+
Object.entries(RUNTIME_AUTHORITY_ENV),
|
|
4139
|
+
);
|
|
4140
|
+
const NPM_CONFIG_ENV_PREFIX = "npm_config_";
|
|
4141
|
+
function normalizeNpmConfigEnvKey(key: string): string | undefined {
|
|
4142
|
+
const folded = key.toLowerCase();
|
|
4143
|
+
if (!folded.startsWith(NPM_CONFIG_ENV_PREFIX)) return undefined;
|
|
4144
|
+
// npm converts underscores in the option suffix to hyphens. Docker can
|
|
4145
|
+
// preserve spellings a shell cannot assign (for example LOGS-DIR), so both
|
|
4146
|
+
// forms must be treated as the same owned option during immutable-env proof.
|
|
4147
|
+
return (
|
|
4148
|
+
NPM_CONFIG_ENV_PREFIX +
|
|
4149
|
+
folded.slice(NPM_CONFIG_ENV_PREFIX.length).replaceAll("_", "-")
|
|
4150
|
+
);
|
|
4151
|
+
}
|
|
4152
|
+
const TASK_OWNED_NORMALIZED_NPM_ENV_KEYS = new Set(
|
|
4153
|
+
[...TASK_OWNED_ENV.keys()]
|
|
4154
|
+
.map(normalizeNpmConfigEnvKey)
|
|
4155
|
+
.filter((key): key is string => key !== undefined),
|
|
4156
|
+
);
|
|
4157
|
+
|
|
4158
|
+
interface RuntimeWriterCandidate {
|
|
4159
|
+
taskId: string;
|
|
4160
|
+
composeProject: string;
|
|
4161
|
+
containerName: string;
|
|
4162
|
+
}
|
|
4163
|
+
|
|
4164
|
+
interface RuntimeWriterMetadata extends RuntimeWriterCandidate {
|
|
4165
|
+
containerId: string;
|
|
4166
|
+
running: boolean;
|
|
4167
|
+
restarting: boolean;
|
|
4168
|
+
paused: boolean;
|
|
4169
|
+
restartPolicy: string;
|
|
4170
|
+
sharedMount: "absent" | "read-only" | "writable" | "invalid";
|
|
4171
|
+
taskEnvironment: "current" | "invalid";
|
|
4172
|
+
}
|
|
4173
|
+
|
|
4174
|
+
type RuntimeWriterInspection =
|
|
4175
|
+
| { kind: "ready"; metadata: RuntimeWriterMetadata }
|
|
4176
|
+
| { kind: "failed"; detail: string }
|
|
4177
|
+
| { kind: "unreachable" };
|
|
4178
|
+
|
|
4179
|
+
export type TaskRuntimeContainerInspection =
|
|
4180
|
+
| {
|
|
4181
|
+
kind: "ready";
|
|
4182
|
+
containerId: string;
|
|
4183
|
+
running: boolean;
|
|
4184
|
+
restarting: boolean;
|
|
4185
|
+
paused: boolean;
|
|
4186
|
+
sharedMount: RuntimeWriterMetadata["sharedMount"];
|
|
4187
|
+
taskEnvironment: RuntimeWriterMetadata["taskEnvironment"];
|
|
4188
|
+
}
|
|
4189
|
+
| { kind: "failed"; detail: string }
|
|
4190
|
+
| { kind: "unreachable" };
|
|
4191
|
+
|
|
4192
|
+
export interface WritableRuntimeQuarantineResult {
|
|
4193
|
+
verdict: "ready" | "retry" | "stale";
|
|
4194
|
+
/** Known rows whose local `stopped` verdict must be published or replayed. */
|
|
4195
|
+
stoppedTaskIds: string[];
|
|
4196
|
+
}
|
|
4197
|
+
|
|
4198
|
+
function runtimeWriterCandidate(
|
|
4199
|
+
rawName: unknown,
|
|
4200
|
+
rawProject: string,
|
|
4201
|
+
rawService: string,
|
|
4202
|
+
): RuntimeWriterCandidate | "unrelated" | "invalid" {
|
|
4203
|
+
if (!rawProject.startsWith("task-")) return "unrelated";
|
|
4204
|
+
const taskId = rawProject.slice("task-".length);
|
|
4205
|
+
if (!isSafeHostTaskId(taskId)) return "invalid";
|
|
4206
|
+
if (rawService !== "app" || typeof rawName !== "string") return "invalid";
|
|
4207
|
+
const containerName = `${rawProject}-app-1`;
|
|
4208
|
+
if (rawName !== containerName) return "invalid";
|
|
4209
|
+
return { taskId, composeProject: rawProject, containerName };
|
|
3543
4210
|
}
|
|
3544
4211
|
|
|
3545
4212
|
/**
|
|
3546
|
-
*
|
|
3547
|
-
*
|
|
3548
|
-
*
|
|
3549
|
-
* managed User profile. `install -d` creates or repairs the exact directory
|
|
3550
|
-
* without recursively changing unrelated application state beneath it.
|
|
4213
|
+
* Enumerate the Docker authority, not only SQLite. A crashed task-down or an
|
|
4214
|
+
* older host can leave a Uai app container behind after its row changes, and
|
|
4215
|
+
* that orphan can still mutate the host-wide runtime volume.
|
|
3551
4216
|
*/
|
|
3552
|
-
async function
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
4217
|
+
async function listRuntimeWriterCandidates(): Promise<
|
|
4218
|
+
| { kind: "ready"; candidates: RuntimeWriterCandidate[]; ambiguous: boolean }
|
|
4219
|
+
| { kind: "unreachable" }
|
|
4220
|
+
> {
|
|
4221
|
+
const listed = await dockerCli(
|
|
4222
|
+
[
|
|
4223
|
+
"ps",
|
|
4224
|
+
"--all",
|
|
4225
|
+
"--filter",
|
|
4226
|
+
"label=com.docker.compose.project",
|
|
4227
|
+
"--filter",
|
|
4228
|
+
"label=com.docker.compose.service=app",
|
|
4229
|
+
"--format",
|
|
4230
|
+
RUNTIME_WRITER_LIST_FORMAT,
|
|
4231
|
+
],
|
|
4232
|
+
{ timeoutMs: 30_000, maxOutputBytes: RUNTIME_WRITER_SCAN_OUTPUT_BYTES },
|
|
4233
|
+
);
|
|
4234
|
+
if (listed.status !== 0 || listed.outputTruncated) {
|
|
4235
|
+
return { kind: "unreachable" };
|
|
4236
|
+
}
|
|
4237
|
+
|
|
4238
|
+
const candidates = new Map<string, RuntimeWriterCandidate>();
|
|
4239
|
+
let ambiguous = false;
|
|
4240
|
+
for (const rawLine of listed.stdout.split(/\r?\n/)) {
|
|
4241
|
+
if (!rawLine) continue;
|
|
4242
|
+
const fields = rawLine.split("\t");
|
|
4243
|
+
if (fields.length !== 4) {
|
|
4244
|
+
ambiguous = true;
|
|
4245
|
+
continue;
|
|
4246
|
+
}
|
|
4247
|
+
// A preview sidecar is a known non-writer, not an ambiguity.
|
|
4248
|
+
if (fields[3] !== "") continue;
|
|
4249
|
+
let name: unknown;
|
|
4250
|
+
try {
|
|
4251
|
+
name = JSON.parse(fields[0]!);
|
|
4252
|
+
} catch {
|
|
4253
|
+
ambiguous = true;
|
|
4254
|
+
continue;
|
|
4255
|
+
}
|
|
4256
|
+
const candidate = runtimeWriterCandidate(name, fields[1]!, fields[2]!);
|
|
4257
|
+
if (candidate === "unrelated") continue;
|
|
4258
|
+
if (candidate === "invalid") {
|
|
4259
|
+
ambiguous = true;
|
|
4260
|
+
continue;
|
|
4261
|
+
}
|
|
4262
|
+
if (candidates.has(candidate.containerName)) {
|
|
4263
|
+
ambiguous = true;
|
|
4264
|
+
continue;
|
|
4265
|
+
}
|
|
4266
|
+
candidates.set(candidate.containerName, candidate);
|
|
4267
|
+
}
|
|
4268
|
+
return { kind: "ready", candidates: [...candidates.values()], ambiguous };
|
|
4269
|
+
}
|
|
4270
|
+
|
|
4271
|
+
async function inspectRuntimeWriterContainer(
|
|
4272
|
+
candidate: RuntimeWriterCandidate,
|
|
4273
|
+
target: string = candidate.containerName,
|
|
4274
|
+
): Promise<RuntimeWriterInspection> {
|
|
4275
|
+
const inspected = await dockerCli(
|
|
4276
|
+
["inspect", "--format", RUNTIME_WRITER_INSPECT_FORMAT, target],
|
|
4277
|
+
{ timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
|
|
4278
|
+
);
|
|
4279
|
+
if (inspected.status !== 0 || inspected.outputTruncated) {
|
|
4280
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
4281
|
+
return {
|
|
4282
|
+
kind: "failed",
|
|
4283
|
+
detail:
|
|
4284
|
+
inspected.stderr.trim() ||
|
|
4285
|
+
`could not inspect Uai app container ${candidate.containerName}`,
|
|
4286
|
+
};
|
|
4287
|
+
}
|
|
4288
|
+
|
|
4289
|
+
const fields = inspected.stdout.trimEnd().split(/\r?\n/);
|
|
4290
|
+
if (fields.length !== 10) {
|
|
4291
|
+
return { kind: "failed", detail: "Docker returned malformed app metadata" };
|
|
4292
|
+
}
|
|
4293
|
+
let parsed: unknown[];
|
|
4294
|
+
try {
|
|
4295
|
+
parsed = fields.map((field) => JSON.parse(field));
|
|
4296
|
+
} catch {
|
|
4297
|
+
return { kind: "failed", detail: "Docker returned malformed app metadata" };
|
|
4298
|
+
}
|
|
4299
|
+
const [
|
|
4300
|
+
containerId,
|
|
4301
|
+
containerName,
|
|
4302
|
+
running,
|
|
4303
|
+
restarting,
|
|
4304
|
+
paused,
|
|
4305
|
+
restartPolicy,
|
|
4306
|
+
composeProject,
|
|
4307
|
+
composeService,
|
|
4308
|
+
mounts,
|
|
4309
|
+
environment,
|
|
4310
|
+
] = parsed;
|
|
4311
|
+
if (
|
|
4312
|
+
typeof containerId !== "string" ||
|
|
4313
|
+
!RUNTIME_WRITER_CONTAINER_ID_RE.test(containerId) ||
|
|
4314
|
+
containerName !== `/${candidate.containerName}` ||
|
|
4315
|
+
typeof running !== "boolean" ||
|
|
4316
|
+
typeof restarting !== "boolean" ||
|
|
4317
|
+
typeof paused !== "boolean" ||
|
|
4318
|
+
typeof restartPolicy !== "string" ||
|
|
4319
|
+
composeProject !== candidate.composeProject ||
|
|
4320
|
+
composeService !== "app" ||
|
|
4321
|
+
!Array.isArray(mounts)
|
|
4322
|
+
) {
|
|
4323
|
+
return {
|
|
4324
|
+
kind: "failed",
|
|
4325
|
+
detail: `Docker returned ambiguous identity/state metadata for ${candidate.containerName}`,
|
|
4326
|
+
};
|
|
4327
|
+
}
|
|
4328
|
+
|
|
4329
|
+
if (
|
|
4330
|
+
!mounts.every(
|
|
4331
|
+
(mount) => mount && typeof mount === "object" && !Array.isArray(mount),
|
|
4332
|
+
)
|
|
4333
|
+
) {
|
|
4334
|
+
return {
|
|
4335
|
+
kind: "failed",
|
|
4336
|
+
detail: `Docker returned malformed mount metadata for ${candidate.containerName}`,
|
|
4337
|
+
};
|
|
4338
|
+
}
|
|
4339
|
+
const mountRecords = mounts as Array<Record<string, unknown>>;
|
|
4340
|
+
const asdfMounts = mountRecords.filter(
|
|
4341
|
+
(mount) => mount.Destination === "/opt/asdf-data",
|
|
4342
|
+
);
|
|
4343
|
+
const namedSharedMounts = mountRecords.filter(
|
|
4344
|
+
(mount) => mount.Name === "uai-asdf-data",
|
|
4345
|
+
);
|
|
4346
|
+
let sharedMount: RuntimeWriterMetadata["sharedMount"];
|
|
4347
|
+
if (
|
|
4348
|
+
namedSharedMounts.some(
|
|
4349
|
+
(mount) => mount.Type === "volume" && mount.RW === true,
|
|
4350
|
+
)
|
|
4351
|
+
) {
|
|
4352
|
+
// The volume name, not its destination, is the shared writer authority.
|
|
4353
|
+
// An orphan can mount it at an arbitrary path and still race maintenance.
|
|
4354
|
+
sharedMount = "writable";
|
|
4355
|
+
} else if (
|
|
4356
|
+
namedSharedMounts.some(
|
|
4357
|
+
(mount) => mount.Type !== "volume" || typeof mount.RW !== "boolean",
|
|
4358
|
+
)
|
|
4359
|
+
) {
|
|
4360
|
+
sharedMount = "invalid";
|
|
4361
|
+
} else if (asdfMounts.length === 0) {
|
|
4362
|
+
sharedMount = "absent";
|
|
4363
|
+
} else if (
|
|
4364
|
+
asdfMounts.length === 1 &&
|
|
4365
|
+
namedSharedMounts.length === 1 &&
|
|
4366
|
+
asdfMounts[0]!.Type === "volume" &&
|
|
4367
|
+
asdfMounts[0]!.Name === "uai-asdf-data" &&
|
|
4368
|
+
asdfMounts[0]!.RW === false
|
|
4369
|
+
) {
|
|
4370
|
+
sharedMount = "read-only";
|
|
4371
|
+
} else {
|
|
4372
|
+
sharedMount = "invalid";
|
|
4373
|
+
}
|
|
4374
|
+
const environmentIsStringArray =
|
|
4375
|
+
Array.isArray(environment) &&
|
|
4376
|
+
environment.every((entry) => typeof entry === "string");
|
|
4377
|
+
const environmentEntries = environmentIsStringArray
|
|
4378
|
+
? (environment as string[])
|
|
4379
|
+
: [];
|
|
4380
|
+
const exactValues = new Map<string, string[]>();
|
|
4381
|
+
let taskEnvironment: RuntimeWriterMetadata["taskEnvironment"] =
|
|
4382
|
+
environmentIsStringArray ? "current" : "invalid";
|
|
4383
|
+
for (const entry of environmentEntries) {
|
|
4384
|
+
const divider = entry.indexOf("=");
|
|
4385
|
+
const key = divider < 0 ? entry : entry.slice(0, divider);
|
|
4386
|
+
const value = divider < 0 ? "" : entry.slice(divider + 1);
|
|
4387
|
+
if (divider < 1) taskEnvironment = "invalid";
|
|
4388
|
+
const normalizedNpmKey = normalizeNpmConfigEnvKey(key);
|
|
4389
|
+
if (
|
|
4390
|
+
normalizedNpmKey !== undefined &&
|
|
4391
|
+
TASK_OWNED_NORMALIZED_NPM_ENV_KEYS.has(normalizedNpmKey) &&
|
|
4392
|
+
!TASK_OWNED_ENV.has(key)
|
|
4393
|
+
) {
|
|
4394
|
+
taskEnvironment = "invalid";
|
|
4395
|
+
}
|
|
4396
|
+
const values = exactValues.get(key) ?? [];
|
|
4397
|
+
values.push(value);
|
|
4398
|
+
exactValues.set(key, values);
|
|
4399
|
+
}
|
|
4400
|
+
for (const [key, expected] of TASK_OWNED_ENV) {
|
|
4401
|
+
const values = exactValues.get(key);
|
|
4402
|
+
if (values?.length !== 1 || values[0] !== expected) {
|
|
4403
|
+
taskEnvironment = "invalid";
|
|
4404
|
+
}
|
|
4405
|
+
}
|
|
4406
|
+
// Keep an explicit assertion for the trust-precedence property even though
|
|
4407
|
+
// PATH is also part of TASK_OWNED_ENV above.
|
|
4408
|
+
if (exactValues.get("PATH")?.[0] !== TASK_RUNTIME_PATH) taskEnvironment = "invalid";
|
|
4409
|
+
return {
|
|
4410
|
+
kind: "ready",
|
|
4411
|
+
metadata: {
|
|
4412
|
+
...candidate,
|
|
4413
|
+
containerId,
|
|
4414
|
+
running,
|
|
4415
|
+
restarting,
|
|
4416
|
+
paused,
|
|
4417
|
+
restartPolicy,
|
|
4418
|
+
sharedMount,
|
|
4419
|
+
taskEnvironment,
|
|
4420
|
+
},
|
|
4421
|
+
};
|
|
4422
|
+
}
|
|
4423
|
+
|
|
4424
|
+
/** Exact single-task form used by duplicate taskUp admission. */
|
|
4425
|
+
export async function inspectTaskRuntimeContainer(
|
|
4426
|
+
taskId: string,
|
|
4427
|
+
composeProject: string,
|
|
4428
|
+
): Promise<TaskRuntimeContainerInspection> {
|
|
4429
|
+
if (!isSafeHostTaskId(taskId) || composeProject !== `task-${taskId}`) {
|
|
4430
|
+
return { kind: "failed", detail: "invalid task/Compose runtime identity" };
|
|
4431
|
+
}
|
|
4432
|
+
const inspected = await inspectRuntimeWriterContainer({
|
|
4433
|
+
taskId,
|
|
4434
|
+
composeProject,
|
|
4435
|
+
containerName: `${composeProject}-app-1`,
|
|
4436
|
+
});
|
|
4437
|
+
if (inspected.kind !== "ready") return inspected;
|
|
4438
|
+
return {
|
|
4439
|
+
kind: "ready",
|
|
4440
|
+
containerId: inspected.metadata.containerId,
|
|
4441
|
+
running: inspected.metadata.running,
|
|
4442
|
+
restarting: inspected.metadata.restarting,
|
|
4443
|
+
paused: inspected.metadata.paused,
|
|
4444
|
+
sharedMount: inspected.metadata.sharedMount,
|
|
4445
|
+
taskEnvironment: inspected.metadata.taskEnvironment,
|
|
4446
|
+
};
|
|
4447
|
+
}
|
|
4448
|
+
|
|
4449
|
+
function restartPolicyDisabled(name: string): boolean {
|
|
4450
|
+
return name === "" || name === "no";
|
|
4451
|
+
}
|
|
4452
|
+
|
|
4453
|
+
async function stopRuntimeWriter(
|
|
4454
|
+
metadata: RuntimeWriterMetadata,
|
|
4455
|
+
): Promise<"stopped" | "failed" | "unreachable"> {
|
|
4456
|
+
// A preserved legacy container must not spring back up under Docker's own
|
|
4457
|
+
// restart policy after the host has declared it stopped. Mount mode is
|
|
4458
|
+
// immutable, so only explicit taskUp may recreate it with the L5 contract.
|
|
4459
|
+
await dockerCli(["update", "--restart=no", metadata.containerId], {
|
|
4460
|
+
timeoutMs: 30_000,
|
|
4461
|
+
maxOutputBytes: 1024 * 1024,
|
|
4462
|
+
});
|
|
4463
|
+
let current = await inspectRuntimeWriterContainer(
|
|
4464
|
+
metadata,
|
|
4465
|
+
metadata.containerId,
|
|
4466
|
+
);
|
|
4467
|
+
if (current.kind !== "ready") return current.kind;
|
|
4468
|
+
if (
|
|
4469
|
+
current.metadata.containerId !== metadata.containerId ||
|
|
4470
|
+
!restartPolicyDisabled(current.metadata.restartPolicy)
|
|
4471
|
+
) {
|
|
4472
|
+
return "failed";
|
|
4473
|
+
}
|
|
4474
|
+
|
|
4475
|
+
if (current.metadata.paused) {
|
|
4476
|
+
// Docker refuses ordinary stop/kill operations for some paused states.
|
|
4477
|
+
// The restart policy is already disabled; unpause the exact immutable id,
|
|
4478
|
+
// re-prove identity/policy, then immediately enter the bounded stop path.
|
|
4479
|
+
await dockerCli(["unpause", metadata.containerId], {
|
|
4480
|
+
timeoutMs: 15_000,
|
|
4481
|
+
maxOutputBytes: 1024 * 1024,
|
|
4482
|
+
});
|
|
4483
|
+
current = await inspectRuntimeWriterContainer(
|
|
4484
|
+
metadata,
|
|
4485
|
+
metadata.containerId,
|
|
4486
|
+
);
|
|
4487
|
+
if (current.kind !== "ready") return current.kind;
|
|
4488
|
+
if (
|
|
4489
|
+
current.metadata.containerId !== metadata.containerId ||
|
|
4490
|
+
!restartPolicyDisabled(current.metadata.restartPolicy) ||
|
|
4491
|
+
current.metadata.paused
|
|
4492
|
+
) {
|
|
4493
|
+
return "failed";
|
|
4494
|
+
}
|
|
4495
|
+
}
|
|
4496
|
+
|
|
4497
|
+
if (!current.metadata.running && !current.metadata.restarting) {
|
|
4498
|
+
return "stopped";
|
|
4499
|
+
}
|
|
4500
|
+
await dockerCli(["stop", "--time", "10", metadata.containerId], {
|
|
4501
|
+
timeoutMs: 30_000,
|
|
4502
|
+
maxOutputBytes: 1024 * 1024,
|
|
4503
|
+
});
|
|
4504
|
+
current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
|
|
4505
|
+
if (current.kind !== "ready") return current.kind;
|
|
4506
|
+
if (
|
|
4507
|
+
current.metadata.containerId === metadata.containerId &&
|
|
4508
|
+
restartPolicyDisabled(current.metadata.restartPolicy) &&
|
|
4509
|
+
!current.metadata.running &&
|
|
4510
|
+
!current.metadata.restarting
|
|
4511
|
+
) {
|
|
4512
|
+
return "stopped";
|
|
4513
|
+
}
|
|
4514
|
+
|
|
4515
|
+
await dockerCli(["kill", metadata.containerId], {
|
|
4516
|
+
timeoutMs: 15_000,
|
|
4517
|
+
maxOutputBytes: 1024 * 1024,
|
|
4518
|
+
});
|
|
4519
|
+
current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
|
|
4520
|
+
if (current.kind !== "ready") return current.kind;
|
|
4521
|
+
return current.metadata.containerId === metadata.containerId &&
|
|
4522
|
+
restartPolicyDisabled(current.metadata.restartPolicy) &&
|
|
4523
|
+
!current.metadata.running &&
|
|
4524
|
+
!current.metadata.restarting
|
|
4525
|
+
? "stopped"
|
|
4526
|
+
: "failed";
|
|
4527
|
+
}
|
|
4528
|
+
|
|
4529
|
+
/** Stop the exact container previously inspected by taskUp. Re-inspection by
|
|
4530
|
+
* immutable id closes the name-reuse gap before any shared-volume writer. */
|
|
4531
|
+
export async function quarantineTaskRuntimeContainer(options: {
|
|
4532
|
+
taskId: string;
|
|
4533
|
+
composeProject: string;
|
|
4534
|
+
expectedContainerId: string;
|
|
4535
|
+
}): Promise<"stopped" | "failed" | "unreachable"> {
|
|
4536
|
+
if (
|
|
4537
|
+
!isSafeHostTaskId(options.taskId) ||
|
|
4538
|
+
options.composeProject !== `task-${options.taskId}` ||
|
|
4539
|
+
!RUNTIME_WRITER_CONTAINER_ID_RE.test(options.expectedContainerId)
|
|
4540
|
+
) {
|
|
4541
|
+
return "failed";
|
|
4542
|
+
}
|
|
4543
|
+
const candidate: RuntimeWriterCandidate = {
|
|
4544
|
+
taskId: options.taskId,
|
|
4545
|
+
composeProject: options.composeProject,
|
|
4546
|
+
containerName: `${options.composeProject}-app-1`,
|
|
4547
|
+
};
|
|
4548
|
+
const inspected = await inspectRuntimeWriterContainer(
|
|
4549
|
+
candidate,
|
|
4550
|
+
options.expectedContainerId,
|
|
4551
|
+
);
|
|
4552
|
+
if (inspected.kind !== "ready") return inspected.kind;
|
|
4553
|
+
if (inspected.metadata.containerId !== options.expectedContainerId) {
|
|
4554
|
+
return "failed";
|
|
4555
|
+
}
|
|
4556
|
+
return stopRuntimeWriter(inspected.metadata);
|
|
4557
|
+
}
|
|
4558
|
+
|
|
4559
|
+
/**
|
|
4560
|
+
* L5 rolling fence. It must complete before the first standard-image/runtime
|
|
4561
|
+
* volume writer of an activation generation. New task containers mount the
|
|
4562
|
+
* shared volume read-only; every preserved legacy writer is tombstoned and
|
|
4563
|
+
* stopped, while an exact RO container remains eligible for recovery.
|
|
4564
|
+
*/
|
|
4565
|
+
export async function quarantineWritableRuntimeContainers(options: {
|
|
4566
|
+
lifecycle: Orchestrator;
|
|
4567
|
+
isCurrent?: () => boolean;
|
|
4568
|
+
/** `contract` (default) sweeps only containers whose shared-volume
|
|
4569
|
+
* attachment violates the current contract. `volume-holders` sweeps EVERY
|
|
4570
|
+
* task container attached to the shared volume — the activation driver's
|
|
4571
|
+
* non-writer escape when the volume needs a repair the holders block
|
|
4572
|
+
* (ADR-108; apple only, a no-op on Docker). */
|
|
4573
|
+
scope?: "contract" | "volume-holders";
|
|
4574
|
+
}): Promise<WritableRuntimeQuarantineResult> {
|
|
4575
|
+
// Dispatch on the PIN: this sweep runs inside activation's
|
|
4576
|
+
// beforeMaintenance, where the public state is still `checking` and the
|
|
4577
|
+
// ready-gated identity is null — the docker fallback then retries forever
|
|
4578
|
+
// against a daemon that does not exist on a Docker-free Mac.
|
|
4579
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
4580
|
+
return quarantineWritableAppleRuntimeContainers(options);
|
|
4581
|
+
}
|
|
4582
|
+
const isCurrent = options.isCurrent ?? (() => true);
|
|
4583
|
+
const stoppedTaskIds = new Set<string>();
|
|
4584
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
|
|
4585
|
+
|
|
4586
|
+
const listed = await listRuntimeWriterCandidates();
|
|
4587
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
|
|
4588
|
+
if (listed.kind === "unreachable") {
|
|
4589
|
+
return { verdict: "retry", stoppedTaskIds: [] };
|
|
4590
|
+
}
|
|
4591
|
+
let resolved = !listed.ambiguous;
|
|
4592
|
+
|
|
4593
|
+
for (const candidate of listed.candidates) {
|
|
4594
|
+
if (!isCurrent()) {
|
|
4595
|
+
return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
4596
|
+
}
|
|
4597
|
+
await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
|
|
4598
|
+
if (!isCurrent()) return;
|
|
4599
|
+
const inspection = await inspectRuntimeWriterContainer(candidate);
|
|
4600
|
+
if (inspection.kind !== "ready") {
|
|
4601
|
+
resolved = false;
|
|
4602
|
+
console.error(
|
|
4603
|
+
`[orchestrator] runtime isolation: ${candidate.containerName} inspection did not prove a safe mount (${inspection.kind === "failed" ? inspection.detail : "Docker unreachable"})`,
|
|
4604
|
+
);
|
|
4605
|
+
return;
|
|
4606
|
+
}
|
|
4607
|
+
const task = getHostTask(candidate.taskId);
|
|
4608
|
+
const knownPreservedTask =
|
|
4609
|
+
task?.composeProject === candidate.composeProject &&
|
|
4610
|
+
typeof task.statusMirror === "string" &&
|
|
4611
|
+
isOnDashboard(task.statusMirror);
|
|
4612
|
+
if (
|
|
4613
|
+
inspection.metadata.sharedMount === "read-only" &&
|
|
4614
|
+
inspection.metadata.taskEnvironment === "current"
|
|
4615
|
+
) {
|
|
4616
|
+
return;
|
|
4617
|
+
}
|
|
4618
|
+
if (!knownPreservedTask) {
|
|
4619
|
+
// A task-shaped orphan with a proven RO shared mount cannot race
|
|
4620
|
+
// maintenance. Without a host row, its non-current task-local npm
|
|
4621
|
+
// environment is not enough ownership evidence to mutate it.
|
|
4622
|
+
if (inspection.metadata.sharedMount === "read-only") return;
|
|
4623
|
+
if (inspection.metadata.sharedMount === "absent") return;
|
|
4624
|
+
if (inspection.metadata.sharedMount === "invalid") {
|
|
4625
|
+
// `task-*` is Uai's namespace, but it is not by itself permission to
|
|
4626
|
+
// stop an unrelated/stale user stack. Only the exact named writable
|
|
4627
|
+
// Uai volume is positive ownership evidence without a host row.
|
|
4628
|
+
resolved = false;
|
|
4629
|
+
console.error(
|
|
4630
|
+
`[orchestrator] runtime isolation: orphan ${candidate.containerName} has ambiguous shared-runtime mount metadata; maintenance remains fenced`,
|
|
4631
|
+
);
|
|
4632
|
+
return;
|
|
4633
|
+
}
|
|
4634
|
+
}
|
|
4635
|
+
|
|
4636
|
+
// Drain before the first stop/update. A running agent continuation may
|
|
4637
|
+
// otherwise race the quarantine and respawn into the legacy container.
|
|
4638
|
+
await options.lifecycle.quarantineChannelForRecovery(candidate.taskId);
|
|
4639
|
+
const stopped = await stopRuntimeWriter(inspection.metadata);
|
|
4640
|
+
if (stopped !== "stopped") {
|
|
4641
|
+
resolved = false;
|
|
4642
|
+
console.error(
|
|
4643
|
+
`[orchestrator] runtime isolation: could not prove ${candidate.containerName} stopped (${stopped})`,
|
|
4644
|
+
);
|
|
4645
|
+
return;
|
|
4646
|
+
}
|
|
4647
|
+
|
|
4648
|
+
if (knownPreservedTask) {
|
|
4649
|
+
// Always re-publish the stopped verdict. A prior process may have
|
|
4650
|
+
// committed SQLite and crashed before its websocket frame left; the
|
|
4651
|
+
// cloud could otherwise remain ghost-running indefinitely.
|
|
4652
|
+
stoppedTaskIds.add(candidate.taskId);
|
|
4653
|
+
db_setStatus(candidate.taskId, "stopped", {
|
|
4654
|
+
codeServerPort: null,
|
|
4655
|
+
previewPorts: "[]",
|
|
4656
|
+
});
|
|
4657
|
+
}
|
|
4658
|
+
clearTaskGatewayAcl(candidate.taskId);
|
|
4659
|
+
// We deliberately reflect a proven physical stop even if the daemon
|
|
4660
|
+
// epoch changed while stopping: leaving the row active would invite a
|
|
4661
|
+
// later recovery to restart an immutable legacy writer. The next
|
|
4662
|
+
// activation generation independently re-enumerates Docker before
|
|
4663
|
+
// maintenance. Non-safety cleanup below may wait for that generation.
|
|
4664
|
+
if (!isCurrent()) return;
|
|
4665
|
+
try {
|
|
4666
|
+
await stopPreviewSidecars(candidate.taskId);
|
|
4667
|
+
} catch (error) {
|
|
4668
|
+
console.warn(
|
|
4669
|
+
`[orchestrator] runtime isolation: ${candidate.taskId} preview cleanup failed: ${
|
|
4670
|
+
error instanceof Error ? error.message : String(error)
|
|
4671
|
+
}`,
|
|
4672
|
+
);
|
|
4673
|
+
}
|
|
4674
|
+
console.warn(
|
|
4675
|
+
`[orchestrator] runtime isolation: ${candidate.containerName} used a writable/non-current shared runtime mount and remains stopped until explicit Resume recreates it`,
|
|
4676
|
+
);
|
|
4677
|
+
});
|
|
4678
|
+
}
|
|
4679
|
+
|
|
4680
|
+
if (!isCurrent()) {
|
|
4681
|
+
return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
4682
|
+
}
|
|
4683
|
+
return {
|
|
4684
|
+
verdict: resolved ? "ready" : "retry",
|
|
4685
|
+
stoppedTaskIds: [...stoppedTaskIds].sort(),
|
|
4686
|
+
};
|
|
4687
|
+
}
|
|
4688
|
+
|
|
4689
|
+
const RECOVERY_RUNTIME_INSPECT_FORMAT =
|
|
4690
|
+
"{{json .Image}}|{{json .State.Running}}|{{json .HostConfig.GroupAdd}}|{{json .Mounts}}";
|
|
4691
|
+
const RECOVERY_RUNTIME_IMAGE_RE = /^sha256:[0-9a-f]{64}$/;
|
|
4692
|
+
const RECOVERY_RUNTIME_SCRIPT_PATH =
|
|
4693
|
+
"/usr/local/bin/uai-materialize-runtimes";
|
|
4694
|
+
const RECOVERY_COREPACK_VERSION_PATH =
|
|
4695
|
+
"/usr/local/share/uai/corepack-version";
|
|
4696
|
+
const RECOVERY_INIT_PATH = "/uai-init-current";
|
|
4697
|
+
const RECOVERY_INIT_PROJECTS_PATH = "/uai-runtime-projects-current";
|
|
4698
|
+
const RECOVERY_RUNTIME_TIMEOUT_MS = 630_000;
|
|
4699
|
+
const RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES = 1024 * 1024;
|
|
4700
|
+
|
|
4701
|
+
interface RecoveryDockerMount {
|
|
4702
|
+
Type?: unknown;
|
|
4703
|
+
Name?: unknown;
|
|
4704
|
+
Source?: unknown;
|
|
4705
|
+
Destination?: unknown;
|
|
4706
|
+
RW?: unknown;
|
|
4707
|
+
}
|
|
4708
|
+
|
|
4709
|
+
interface RecoveryRuntimeSpec {
|
|
4710
|
+
imageId: string;
|
|
4711
|
+
workspaceSource: string;
|
|
4712
|
+
groupAdds: string[];
|
|
4713
|
+
projectCount: number;
|
|
4714
|
+
projectAllowlistSource: string;
|
|
4715
|
+
}
|
|
4716
|
+
|
|
4717
|
+
type RecoveryRuntimeInspection =
|
|
4718
|
+
| { kind: "ready"; spec: RecoveryRuntimeSpec }
|
|
4719
|
+
| { kind: "failed"; detail: string }
|
|
4720
|
+
| { kind: "unreachable" };
|
|
4721
|
+
|
|
4722
|
+
type RecoveryRuntimeResult =
|
|
4723
|
+
| { kind: "ready" }
|
|
4724
|
+
| { kind: "failed"; detail: string }
|
|
4725
|
+
| { kind: "unreachable" };
|
|
4726
|
+
|
|
4727
|
+
function recoveryRuntimeAssets():
|
|
4728
|
+
| { materializer: string; corepackVersion: string; uaiInit: string }
|
|
4729
|
+
| { error: string } {
|
|
4730
|
+
const materializer = standardImageRuntimeMaterializerPath();
|
|
4731
|
+
const corepackVersion = standardImageCorepackVersionPath();
|
|
4732
|
+
const uaiInit = join(dirname(materializer), "uai-init");
|
|
4733
|
+
try {
|
|
4734
|
+
const materializerStat = lstatSync(materializer);
|
|
4735
|
+
if (!materializerStat.isFile() || materializerStat.isSymbolicLink() ||
|
|
4736
|
+
(materializerStat.mode & 0o111) === 0) {
|
|
4737
|
+
return { error: `packaged materializer is not a regular executable: ${materializer}` };
|
|
4738
|
+
}
|
|
4739
|
+
const versionStat = lstatSync(corepackVersion);
|
|
4740
|
+
if (!versionStat.isFile() || versionStat.isSymbolicLink()) {
|
|
4741
|
+
return { error: `packaged Corepack pin is not a regular file: ${corepackVersion}` };
|
|
4742
|
+
}
|
|
4743
|
+
const version = readFileSync(corepackVersion, "utf8");
|
|
4744
|
+
if (!/^\d+\.\d+\.\d+\n?$/.test(version)) {
|
|
4745
|
+
return { error: `packaged Corepack pin is malformed: ${corepackVersion}` };
|
|
4746
|
+
}
|
|
4747
|
+
const initStat = lstatSync(uaiInit);
|
|
4748
|
+
if (
|
|
4749
|
+
!initStat.isFile() ||
|
|
4750
|
+
initStat.isSymbolicLink() ||
|
|
4751
|
+
(initStat.mode & 0o111) === 0
|
|
4752
|
+
) {
|
|
4753
|
+
return { error: `packaged task initializer is not a regular executable: ${uaiInit}` };
|
|
4754
|
+
}
|
|
4755
|
+
} catch (err) {
|
|
4756
|
+
return {
|
|
4757
|
+
error: `packaged runtime assets are unavailable: ${
|
|
4758
|
+
err instanceof Error ? err.message : String(err)
|
|
4759
|
+
}`,
|
|
4760
|
+
};
|
|
4761
|
+
}
|
|
4762
|
+
return { materializer, corepackVersion, uaiInit };
|
|
4763
|
+
}
|
|
4764
|
+
|
|
4765
|
+
function expectedRecoveryWorkspaceSources(
|
|
4766
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4767
|
+
): Set<string> {
|
|
4768
|
+
const expected = new Set<string>();
|
|
4769
|
+
if (!task.worktreePath) return expected;
|
|
4770
|
+
const lexical = join(task.worktreePath, "workspace");
|
|
4771
|
+
try {
|
|
4772
|
+
const stat = lstatSync(lexical);
|
|
4773
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) return expected;
|
|
4774
|
+
expected.add(lexical);
|
|
4775
|
+
expected.add(realpathSync(lexical));
|
|
4776
|
+
} catch {
|
|
4777
|
+
// Missing, unreadable, and unresolvable workspaces are never safe input
|
|
4778
|
+
// to `docker run`: a legacy `-v` bind would recreate a missing source as
|
|
4779
|
+
// an empty directory and falsely prove that the task had no projects.
|
|
4780
|
+
}
|
|
4781
|
+
return expected;
|
|
4782
|
+
}
|
|
4783
|
+
|
|
4784
|
+
function validRecoveryGroupAdds(value: unknown): string[] | null {
|
|
4785
|
+
if (value === null || value === undefined) return [];
|
|
4786
|
+
if (!Array.isArray(value)) return null;
|
|
4787
|
+
const groups: string[] = [];
|
|
4788
|
+
for (const candidate of value) {
|
|
4789
|
+
if (typeof candidate !== "string" || !/^[1-9]\d{0,9}$/.test(candidate)) {
|
|
4790
|
+
return null;
|
|
4791
|
+
}
|
|
4792
|
+
const numeric = Number(candidate);
|
|
4793
|
+
if (!Number.isSafeInteger(numeric) || numeric > 4_294_967_294) {
|
|
4794
|
+
return null;
|
|
4795
|
+
}
|
|
4796
|
+
if (!groups.includes(candidate)) groups.push(candidate);
|
|
4797
|
+
}
|
|
4798
|
+
return groups;
|
|
4799
|
+
}
|
|
4800
|
+
|
|
4801
|
+
function safeRecoveryProjectSlug(slug: unknown): slug is string {
|
|
4802
|
+
return typeof slug === "string" && /^[a-z0-9-]{1,64}$/.test(slug);
|
|
4803
|
+
}
|
|
4804
|
+
|
|
4805
|
+
function validateRecoveryProjects(
|
|
4806
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4807
|
+
workspaceSource: string,
|
|
4808
|
+
): { slugs: string[] } | { error: string } {
|
|
4809
|
+
let parsed: unknown;
|
|
4810
|
+
try {
|
|
4811
|
+
parsed = JSON.parse(task.projectSlugs);
|
|
4812
|
+
} catch {
|
|
4813
|
+
return { error: "the persisted project slug list is invalid JSON" };
|
|
4814
|
+
}
|
|
4815
|
+
if (!Array.isArray(parsed) || !parsed.every(safeRecoveryProjectSlug)) {
|
|
4816
|
+
return { error: "the persisted project slug list contains an unsafe entry" };
|
|
4817
|
+
}
|
|
4818
|
+
const slugs = parsed as string[];
|
|
4819
|
+
const expected = new Set(slugs);
|
|
4820
|
+
if (expected.size !== slugs.length) {
|
|
4821
|
+
return { error: "the persisted project slug list contains duplicates" };
|
|
4822
|
+
}
|
|
4823
|
+
|
|
4824
|
+
for (const slug of expected) {
|
|
4825
|
+
const project = join(workspaceSource, slug);
|
|
4826
|
+
let projectStat: ReturnType<typeof lstatSync>;
|
|
4827
|
+
let markerStat: ReturnType<typeof lstatSync>;
|
|
4828
|
+
try {
|
|
4829
|
+
projectStat = lstatSync(project);
|
|
4830
|
+
markerStat = lstatSync(join(project, ".git"));
|
|
4831
|
+
} catch {
|
|
4832
|
+
return { error: `expected project ${slug} or its .git marker is missing` };
|
|
4833
|
+
}
|
|
4834
|
+
if (
|
|
4835
|
+
projectStat.isSymbolicLink() ||
|
|
4836
|
+
!projectStat.isDirectory() ||
|
|
4837
|
+
markerStat.isSymbolicLink() ||
|
|
4838
|
+
(!markerStat.isFile() && !markerStat.isDirectory())
|
|
4839
|
+
) {
|
|
4840
|
+
return { error: `expected project ${slug} is not a safe Git worktree` };
|
|
4841
|
+
}
|
|
4842
|
+
}
|
|
4843
|
+
return { slugs };
|
|
4844
|
+
}
|
|
4845
|
+
|
|
4846
|
+
function validateRecoveryWorkspaceRuntimeRoot(
|
|
4847
|
+
workspaceSource: string,
|
|
4848
|
+
): string | null {
|
|
4849
|
+
const shadow = join(workspaceSource, ".tool-versions");
|
|
4850
|
+
try {
|
|
4851
|
+
// Any type is forbidden, including regular files and dangling symlinks.
|
|
4852
|
+
// lstat is metadata-only, so FIFO/device/socket entries cannot block the
|
|
4853
|
+
// host recovery loop.
|
|
4854
|
+
lstatSync(shadow);
|
|
4855
|
+
return "the workspace root contains a forbidden .tool-versions runtime override";
|
|
4856
|
+
} catch (err) {
|
|
4857
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
4858
|
+
return `the workspace-root runtime authority could not be verified: ${
|
|
4859
|
+
err instanceof Error ? err.message : String(err)
|
|
4860
|
+
}`;
|
|
4861
|
+
}
|
|
4862
|
+
}
|
|
4863
|
+
|
|
4864
|
+
function recoveryRuntimeProjectAllowlist(
|
|
4865
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4866
|
+
slugs: string[],
|
|
4867
|
+
): { path: string } | { error: string } {
|
|
4868
|
+
if (!task.worktreePath) {
|
|
4869
|
+
return { error: "the task has no persisted runtime directory" };
|
|
4870
|
+
}
|
|
4871
|
+
const runtimeDir = join(task.worktreePath, ".uai");
|
|
4872
|
+
const target = join(runtimeDir, "runtime-projects");
|
|
4873
|
+
const expected = slugs.length > 0 ? `${slugs.join("\n")}\n` : "";
|
|
4874
|
+
try {
|
|
4875
|
+
const runtimeDirStat = lstatSync(runtimeDir);
|
|
4876
|
+
if (!runtimeDirStat.isDirectory() || runtimeDirStat.isSymbolicLink()) {
|
|
4877
|
+
return { error: "the task runtime directory is not a safe directory" };
|
|
4878
|
+
}
|
|
4879
|
+
try {
|
|
4880
|
+
const targetStat = lstatSync(target);
|
|
4881
|
+
if (!targetStat.isFile() || targetStat.isSymbolicLink()) {
|
|
4882
|
+
return { error: "the task runtime-project allowlist is not a regular file" };
|
|
4883
|
+
}
|
|
4884
|
+
if (readFileSync(target, "utf8") !== expected) {
|
|
4885
|
+
return { error: "the task runtime-project allowlist does not match persisted projects" };
|
|
4886
|
+
}
|
|
4887
|
+
return { path: target };
|
|
4888
|
+
} catch (err) {
|
|
4889
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
|
|
4890
|
+
}
|
|
4891
|
+
|
|
4892
|
+
if (slugs.length === 0) {
|
|
4893
|
+
return {
|
|
4894
|
+
error: "zero-project recovery lacks an existing authoritative empty runtime-project allowlist",
|
|
4895
|
+
};
|
|
4896
|
+
}
|
|
4897
|
+
|
|
4898
|
+
const stagingDir = mkdtempSync(join(runtimeDir, ".runtime-projects-"));
|
|
4899
|
+
const staged = join(stagingDir, "runtime-projects");
|
|
4900
|
+
try {
|
|
4901
|
+
writeFileSync(staged, expected, { encoding: "utf8", flag: "wx", mode: 0o644 });
|
|
4902
|
+
renameSync(staged, target);
|
|
4903
|
+
} finally {
|
|
4904
|
+
rmSync(stagingDir, { recursive: true, force: true });
|
|
4905
|
+
}
|
|
4906
|
+
return { path: target };
|
|
4907
|
+
} catch (err) {
|
|
4908
|
+
return {
|
|
4909
|
+
error: `the task runtime-project allowlist is unavailable: ${
|
|
4910
|
+
err instanceof Error ? err.message : String(err)
|
|
4911
|
+
}`,
|
|
4912
|
+
};
|
|
4913
|
+
}
|
|
4914
|
+
}
|
|
4915
|
+
|
|
4916
|
+
async function inspectRecoveryRuntime(
|
|
4917
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4918
|
+
containerName: string,
|
|
4919
|
+
): Promise<RecoveryRuntimeInspection> {
|
|
4920
|
+
const inspected = await dockerCli(
|
|
4921
|
+
[
|
|
4922
|
+
"inspect",
|
|
4923
|
+
"--format",
|
|
4924
|
+
RECOVERY_RUNTIME_INSPECT_FORMAT,
|
|
4925
|
+
containerName,
|
|
4926
|
+
],
|
|
4927
|
+
{ timeoutMs: 30_000, maxOutputBytes: 128 * 1024 },
|
|
4928
|
+
);
|
|
4929
|
+
if (inspected.status !== 0) {
|
|
4930
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
4931
|
+
return {
|
|
4932
|
+
kind: "failed",
|
|
4933
|
+
detail:
|
|
4934
|
+
inspected.stderr.trim() ||
|
|
4935
|
+
`could not inspect stopped app container ${containerName}`,
|
|
4936
|
+
};
|
|
4937
|
+
}
|
|
4938
|
+
|
|
4939
|
+
const firstDivider = inspected.stdout.indexOf("|");
|
|
4940
|
+
const secondDivider = inspected.stdout.indexOf("|", firstDivider + 1);
|
|
4941
|
+
const thirdDivider = inspected.stdout.indexOf("|", secondDivider + 1);
|
|
4942
|
+
if (firstDivider < 0 || secondDivider < 0 || thirdDivider < 0) {
|
|
4943
|
+
return { kind: "failed", detail: "Docker returned malformed runtime metadata" };
|
|
4944
|
+
}
|
|
4945
|
+
|
|
4946
|
+
let imageId: unknown;
|
|
4947
|
+
let running: unknown;
|
|
4948
|
+
let rawGroupAdds: unknown;
|
|
4949
|
+
let mounts: unknown;
|
|
4950
|
+
try {
|
|
4951
|
+
imageId = JSON.parse(inspected.stdout.slice(0, firstDivider));
|
|
4952
|
+
running = JSON.parse(
|
|
4953
|
+
inspected.stdout.slice(firstDivider + 1, secondDivider),
|
|
4954
|
+
);
|
|
4955
|
+
rawGroupAdds = JSON.parse(
|
|
4956
|
+
inspected.stdout.slice(secondDivider + 1, thirdDivider),
|
|
4957
|
+
);
|
|
4958
|
+
mounts = JSON.parse(inspected.stdout.slice(thirdDivider + 1));
|
|
4959
|
+
} catch {
|
|
4960
|
+
return { kind: "failed", detail: "Docker returned invalid runtime metadata JSON" };
|
|
4961
|
+
}
|
|
4962
|
+
if (running !== false) {
|
|
4963
|
+
return {
|
|
4964
|
+
kind: "failed",
|
|
4965
|
+
detail: "the app container was not proven stopped before runtime maintenance",
|
|
4966
|
+
};
|
|
4967
|
+
}
|
|
4968
|
+
if (typeof imageId !== "string" || !RECOVERY_RUNTIME_IMAGE_RE.test(imageId)) {
|
|
4969
|
+
return { kind: "failed", detail: "the app container has an invalid immutable image ID" };
|
|
4970
|
+
}
|
|
4971
|
+
if (!Array.isArray(mounts)) {
|
|
4972
|
+
return { kind: "failed", detail: "the app container has invalid mount metadata" };
|
|
4973
|
+
}
|
|
4974
|
+
const groupAdds = validRecoveryGroupAdds(rawGroupAdds);
|
|
4975
|
+
if (groupAdds === null) {
|
|
4976
|
+
return {
|
|
4977
|
+
kind: "failed",
|
|
4978
|
+
detail: "the app container has malformed or unsafe supplemental groups",
|
|
4979
|
+
};
|
|
4980
|
+
}
|
|
4981
|
+
const typedMounts = mounts as RecoveryDockerMount[];
|
|
4982
|
+
const workspaceMounts = typedMounts.filter(
|
|
4983
|
+
(mount) => mount.Destination === "/workspace",
|
|
4984
|
+
);
|
|
4985
|
+
const expectedWorkspaceSources = expectedRecoveryWorkspaceSources(task);
|
|
4986
|
+
if (
|
|
4987
|
+
workspaceMounts.length !== 1 ||
|
|
4988
|
+
workspaceMounts[0]?.Type !== "bind" ||
|
|
4989
|
+
typeof workspaceMounts[0]?.Source !== "string" ||
|
|
4990
|
+
!expectedWorkspaceSources.has(workspaceMounts[0].Source)
|
|
4991
|
+
) {
|
|
4992
|
+
return {
|
|
4993
|
+
kind: "failed",
|
|
4994
|
+
detail: "the app container does not have the exact expected /workspace bind",
|
|
4995
|
+
};
|
|
4996
|
+
}
|
|
4997
|
+
try {
|
|
4998
|
+
const workspaceStat = lstatSync(workspaceMounts[0].Source);
|
|
4999
|
+
if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
|
|
5000
|
+
return {
|
|
5001
|
+
kind: "failed",
|
|
5002
|
+
detail: "the app container's /workspace source is not a real directory",
|
|
5003
|
+
};
|
|
5004
|
+
}
|
|
5005
|
+
} catch {
|
|
5006
|
+
return {
|
|
5007
|
+
kind: "failed",
|
|
5008
|
+
detail: "the app container's /workspace source no longer exists",
|
|
5009
|
+
};
|
|
5010
|
+
}
|
|
5011
|
+
const workspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
|
|
5012
|
+
workspaceMounts[0].Source,
|
|
5013
|
+
);
|
|
5014
|
+
if (workspaceRuntimeError) {
|
|
5015
|
+
return { kind: "failed", detail: workspaceRuntimeError };
|
|
5016
|
+
}
|
|
5017
|
+
const projects = validateRecoveryProjects(
|
|
5018
|
+
task,
|
|
5019
|
+
workspaceMounts[0].Source,
|
|
5020
|
+
);
|
|
5021
|
+
if ("error" in projects) {
|
|
5022
|
+
return { kind: "failed", detail: projects.error };
|
|
5023
|
+
}
|
|
5024
|
+
const projectAllowlist = recoveryRuntimeProjectAllowlist(
|
|
5025
|
+
task,
|
|
5026
|
+
projects.slugs,
|
|
5027
|
+
);
|
|
5028
|
+
if ("error" in projectAllowlist) {
|
|
5029
|
+
return { kind: "failed", detail: projectAllowlist.error };
|
|
5030
|
+
}
|
|
5031
|
+
const asdfMounts = typedMounts.filter(
|
|
5032
|
+
(mount) => mount.Destination === "/opt/asdf-data",
|
|
5033
|
+
);
|
|
5034
|
+
if (
|
|
5035
|
+
asdfMounts.length !== 1 ||
|
|
5036
|
+
asdfMounts[0]?.Type !== "volume" ||
|
|
5037
|
+
asdfMounts[0]?.Name !== "uai-asdf-data" ||
|
|
5038
|
+
asdfMounts[0]?.RW !== false
|
|
5039
|
+
) {
|
|
5040
|
+
return {
|
|
5041
|
+
kind: "failed",
|
|
5042
|
+
detail:
|
|
5043
|
+
"the app container does not use the exact read-only uai-asdf-data volume; explicit Resume must recreate legacy writable containers",
|
|
5044
|
+
};
|
|
5045
|
+
}
|
|
5046
|
+
return {
|
|
5047
|
+
kind: "ready",
|
|
5048
|
+
spec: {
|
|
5049
|
+
imageId,
|
|
5050
|
+
workspaceSource: workspaceMounts[0].Source,
|
|
5051
|
+
groupAdds,
|
|
5052
|
+
projectCount: projects.slugs.length,
|
|
5053
|
+
projectAllowlistSource: projectAllowlist.path,
|
|
5054
|
+
},
|
|
5055
|
+
};
|
|
5056
|
+
}
|
|
5057
|
+
|
|
5058
|
+
async function materializeRecoveryRuntimes(
|
|
5059
|
+
spec: RecoveryRuntimeSpec,
|
|
5060
|
+
): Promise<RecoveryRuntimeResult> {
|
|
5061
|
+
if (spec.projectCount === 0) return { kind: "ready" };
|
|
5062
|
+
const assets = recoveryRuntimeAssets();
|
|
5063
|
+
if ("error" in assets) return { kind: "failed", detail: assets.error };
|
|
5064
|
+
try {
|
|
5065
|
+
const workspaceStat = lstatSync(spec.workspaceSource);
|
|
5066
|
+
if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
|
|
5067
|
+
return {
|
|
5068
|
+
kind: "failed",
|
|
5069
|
+
detail: "the app container's /workspace source is no longer a real directory",
|
|
5070
|
+
};
|
|
5071
|
+
}
|
|
5072
|
+
} catch {
|
|
5073
|
+
return {
|
|
5074
|
+
kind: "failed",
|
|
5075
|
+
detail: "the app container's /workspace source disappeared before runtime proof",
|
|
5076
|
+
};
|
|
5077
|
+
}
|
|
5078
|
+
try {
|
|
5079
|
+
const allowlistStat = lstatSync(spec.projectAllowlistSource);
|
|
5080
|
+
if (!allowlistStat.isFile() || allowlistStat.isSymbolicLink()) {
|
|
5081
|
+
return {
|
|
5082
|
+
kind: "failed",
|
|
5083
|
+
detail: "the runtime-project allowlist is no longer a regular file",
|
|
5084
|
+
};
|
|
5085
|
+
}
|
|
5086
|
+
} catch {
|
|
5087
|
+
return {
|
|
5088
|
+
kind: "failed",
|
|
5089
|
+
detail: "the runtime-project allowlist disappeared before runtime proof",
|
|
5090
|
+
};
|
|
5091
|
+
}
|
|
5092
|
+
const supplementalGroups = spec.groupAdds.flatMap((group) => [
|
|
5093
|
+
"--group-add",
|
|
5094
|
+
group,
|
|
5095
|
+
]);
|
|
5096
|
+
const result = await dockerCli(
|
|
5097
|
+
[
|
|
5098
|
+
"run",
|
|
5099
|
+
"--rm",
|
|
5100
|
+
"--user",
|
|
5101
|
+
"node",
|
|
5102
|
+
...supplementalGroups,
|
|
5103
|
+
"--workdir",
|
|
5104
|
+
"/workspace",
|
|
5105
|
+
"--mount",
|
|
5106
|
+
`type=bind,src=${spec.workspaceSource},dst=/workspace,readonly`,
|
|
5107
|
+
"-v",
|
|
5108
|
+
"uai-asdf-data:/opt/asdf-data:rw",
|
|
5109
|
+
"-v",
|
|
5110
|
+
`${assets.materializer}:${RECOVERY_RUNTIME_SCRIPT_PATH}:ro`,
|
|
5111
|
+
"-v",
|
|
5112
|
+
`${assets.corepackVersion}:${RECOVERY_COREPACK_VERSION_PATH}:ro`,
|
|
5113
|
+
"-v",
|
|
5114
|
+
`${spec.projectAllowlistSource}:/run/uai/runtime-projects:ro`,
|
|
5115
|
+
"--env",
|
|
5116
|
+
"LD_PRELOAD=",
|
|
5117
|
+
"--env",
|
|
5118
|
+
"LD_AUDIT=",
|
|
5119
|
+
"--env",
|
|
5120
|
+
"LD_LIBRARY_PATH=",
|
|
5121
|
+
"--env",
|
|
5122
|
+
"BASH_ENV=",
|
|
5123
|
+
"--env",
|
|
5124
|
+
"ENV=",
|
|
5125
|
+
"--entrypoint",
|
|
5126
|
+
"/usr/bin/env",
|
|
5127
|
+
spec.imageId,
|
|
5128
|
+
"-i",
|
|
5129
|
+
"HOME=/home/node",
|
|
5130
|
+
"ASDF_DIR=/opt/asdf",
|
|
5131
|
+
"ASDF_DATA_DIR=/opt/asdf-data",
|
|
5132
|
+
"PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
|
5133
|
+
"/usr/bin/timeout",
|
|
5134
|
+
"--signal=TERM",
|
|
5135
|
+
"--kill-after=10s",
|
|
5136
|
+
"600s",
|
|
5137
|
+
RECOVERY_RUNTIME_SCRIPT_PATH,
|
|
5138
|
+
],
|
|
5139
|
+
{
|
|
5140
|
+
timeoutMs: RECOVERY_RUNTIME_TIMEOUT_MS,
|
|
5141
|
+
tailOutputBytes: RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES,
|
|
5142
|
+
},
|
|
5143
|
+
);
|
|
5144
|
+
if (result.status === 0) return { kind: "ready" };
|
|
5145
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
5146
|
+
return {
|
|
5147
|
+
kind: "failed",
|
|
5148
|
+
detail: `${result.outputTruncated ? "earlier runtime output was truncated; " : ""}${
|
|
5149
|
+
result.stderr.trim() ||
|
|
5150
|
+
`runtime materializer exited ${String(result.status)}`
|
|
5151
|
+
}`,
|
|
5152
|
+
};
|
|
5153
|
+
}
|
|
5154
|
+
|
|
5155
|
+
async function stageRecoveryInit(
|
|
5156
|
+
containerName: string,
|
|
5157
|
+
spec: RecoveryRuntimeSpec,
|
|
5158
|
+
): Promise<RecoveryRuntimeResult> {
|
|
5159
|
+
const assets = recoveryRuntimeAssets();
|
|
5160
|
+
if ("error" in assets) return { kind: "failed", detail: assets.error };
|
|
5161
|
+
const copies: Array<[string, string]> = [
|
|
5162
|
+
[assets.uaiInit, RECOVERY_INIT_PATH],
|
|
5163
|
+
[spec.projectAllowlistSource, RECOVERY_INIT_PROJECTS_PATH],
|
|
5164
|
+
];
|
|
5165
|
+
for (const [source, destination] of copies) {
|
|
5166
|
+
const copied = await dockerCli(
|
|
5167
|
+
["cp", source, `${containerName}:${destination}`],
|
|
5168
|
+
{ timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
|
|
5169
|
+
);
|
|
5170
|
+
if (copied.status === 0) continue;
|
|
5171
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
5172
|
+
return {
|
|
5173
|
+
kind: "failed",
|
|
5174
|
+
detail:
|
|
5175
|
+
copied.stderr.trim() ||
|
|
5176
|
+
`could not stage ${destination} in the stopped app container`,
|
|
5177
|
+
};
|
|
5178
|
+
}
|
|
5179
|
+
return { kind: "ready" };
|
|
5180
|
+
}
|
|
5181
|
+
|
|
5182
|
+
type DockerQuarantineResult = "stopped" | "failed" | "unreachable";
|
|
5183
|
+
|
|
5184
|
+
async function quarantineRunningContainer(
|
|
5185
|
+
containerName: string,
|
|
5186
|
+
): Promise<DockerQuarantineResult> {
|
|
5187
|
+
const stopped = await dockerCli(
|
|
5188
|
+
["stop", "--time", "10", containerName],
|
|
5189
|
+
{ timeoutMs: 30_000 },
|
|
5190
|
+
);
|
|
5191
|
+
const inspected = await dockerCli(
|
|
5192
|
+
["inspect", "--format", "{{.State.Running}}", containerName],
|
|
5193
|
+
{ timeoutMs: 10_000 },
|
|
5194
|
+
);
|
|
5195
|
+
if (inspected.status === 0 && inspected.stdout.trim() === "false") {
|
|
5196
|
+
return "stopped";
|
|
5197
|
+
}
|
|
5198
|
+
if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
|
|
5199
|
+
return "unreachable";
|
|
5200
|
+
}
|
|
5201
|
+
const killed = await dockerCli(["kill", containerName], {
|
|
5202
|
+
timeoutMs: 15_000,
|
|
5203
|
+
});
|
|
5204
|
+
const killInspection = await dockerCli(
|
|
5205
|
+
["inspect", "--format", "{{.State.Running}}", containerName],
|
|
5206
|
+
{ timeoutMs: 10_000 },
|
|
5207
|
+
);
|
|
5208
|
+
if (
|
|
5209
|
+
killInspection.status === 0 &&
|
|
5210
|
+
killInspection.stdout.trim() === "false"
|
|
5211
|
+
) {
|
|
5212
|
+
return "stopped";
|
|
5213
|
+
}
|
|
5214
|
+
if (killInspection.status !== 0 && !(await dockerDaemonReachable())) {
|
|
5215
|
+
return "unreachable";
|
|
5216
|
+
}
|
|
5217
|
+
console.error(
|
|
5218
|
+
`[orchestrator] recovery: could not quarantine ${containerName} before runtime proof: ${
|
|
5219
|
+
stopped.stderr.trim() ||
|
|
5220
|
+
inspected.stderr.trim() ||
|
|
5221
|
+
killed.stderr.trim() ||
|
|
5222
|
+
killInspection.stderr.trim() ||
|
|
5223
|
+
"container is still running after stop and kill"
|
|
5224
|
+
}`,
|
|
5225
|
+
);
|
|
5226
|
+
return "failed";
|
|
5227
|
+
}
|
|
5228
|
+
|
|
5229
|
+
/**
|
|
5230
|
+
* Repair the shared node data root before any recovered-container init.
|
|
5231
|
+
* Older task-up versions created the OpenCode leaf as root and accidentally
|
|
5232
|
+
* left this parent root-owned, which prevented code-server from creating its
|
|
5233
|
+
* managed User profile. `install -d` creates or repairs the exact directory
|
|
5234
|
+
* without recursively changing unrelated application state beneath it.
|
|
5235
|
+
*/
|
|
5236
|
+
async function repairNodeDataRoot(containerName: string): Promise<boolean> {
|
|
5237
|
+
const res = await dockerCli([
|
|
5238
|
+
"exec",
|
|
5239
|
+
"-u",
|
|
5240
|
+
"root",
|
|
5241
|
+
...runtimeAuthorityDockerExecArgs(),
|
|
5242
|
+
containerName,
|
|
3558
5243
|
"/usr/bin/install",
|
|
3559
5244
|
"-d",
|
|
3560
5245
|
"-o",
|
|
@@ -3574,18 +5259,32 @@ async function repairNodeDataRoot(containerName: string): Promise<boolean> {
|
|
|
3574
5259
|
return true;
|
|
3575
5260
|
}
|
|
3576
5261
|
|
|
5262
|
+
interface DockerPortResult {
|
|
5263
|
+
reachable: boolean;
|
|
5264
|
+
port: number | null;
|
|
5265
|
+
}
|
|
5266
|
+
|
|
3577
5267
|
async function dockerPort(
|
|
3578
5268
|
containerName: string,
|
|
3579
5269
|
containerPort: number,
|
|
3580
|
-
): Promise<
|
|
5270
|
+
): Promise<DockerPortResult> {
|
|
3581
5271
|
const res = await dockerCli(["port", containerName, String(containerPort)]);
|
|
3582
|
-
if (res.status
|
|
5272
|
+
if (res.status === null) return { reachable: false, port: null };
|
|
5273
|
+
if (res.status !== 0) {
|
|
5274
|
+
return {
|
|
5275
|
+
reachable: await dockerDaemonReachable(),
|
|
5276
|
+
port: null,
|
|
5277
|
+
};
|
|
5278
|
+
}
|
|
3583
5279
|
// Output: "127.0.0.1:32785\n"
|
|
3584
5280
|
const firstLine = res.stdout.split("\n")[0]?.trim() ?? "";
|
|
3585
5281
|
const colonIdx = firstLine.lastIndexOf(":");
|
|
3586
|
-
if (colonIdx === -1) return null;
|
|
5282
|
+
if (colonIdx === -1) return { reachable: true, port: null };
|
|
3587
5283
|
const port = Number(firstLine.slice(colonIdx + 1));
|
|
3588
|
-
return
|
|
5284
|
+
return {
|
|
5285
|
+
reachable: true,
|
|
5286
|
+
port: Number.isFinite(port) ? port : null,
|
|
5287
|
+
};
|
|
3589
5288
|
}
|
|
3590
5289
|
|
|
3591
5290
|
async function dockerExec(
|
|
@@ -3594,102 +5293,952 @@ async function dockerExec(
|
|
|
3594
5293
|
timeoutMs?: number,
|
|
3595
5294
|
): Promise<boolean> {
|
|
3596
5295
|
const res = await dockerCli(
|
|
3597
|
-
["exec", containerName, ...cmd],
|
|
5296
|
+
["exec", ...runtimeAuthorityDockerExecArgs(), containerName, ...cmd],
|
|
3598
5297
|
timeoutMs === undefined ? {} : { timeoutMs },
|
|
3599
5298
|
);
|
|
3600
5299
|
return res.status === 0;
|
|
3601
5300
|
}
|
|
3602
5301
|
|
|
3603
|
-
/** Exported for tests; production entry is the getOrchestrator() boot guard. */
|
|
3604
|
-
export async function recoverRunningTasks(opts?: {
|
|
3605
|
-
/** Delay between passes while docker is unreachable (tests shrink it). */
|
|
3606
|
-
retryMs?: number;
|
|
3607
|
-
/** Bound on passes; ~15 min at the default cadence covers slow boots. */
|
|
3608
|
-
maxPasses?: number;
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
5302
|
+
/** Exported for tests; production entry is the getOrchestrator() boot guard. */
|
|
5303
|
+
export async function recoverRunningTasks(opts?: {
|
|
5304
|
+
/** Delay between passes while docker is unreachable (tests shrink it). */
|
|
5305
|
+
retryMs?: number;
|
|
5306
|
+
/** Bound on passes; ~15 min at the default cadence covers slow boots. */
|
|
5307
|
+
maxPasses?: number;
|
|
5308
|
+
/** Post-boot recovery must queue behind live taskUp/taskDown operations.
|
|
5309
|
+
* Boot recovery omits this and uses its earlier per-task barriers instead. */
|
|
5310
|
+
lifecycle?: Orchestrator;
|
|
5311
|
+
/** Test seam; production waits for the shared image/CLI-volume reconcile. */
|
|
5312
|
+
maintenanceReady?: Promise<void>;
|
|
5313
|
+
/** A daemon-generation fence. False cancels every remaining side effect. */
|
|
5314
|
+
isCurrent?: () => boolean;
|
|
5315
|
+
}): Promise<boolean> {
|
|
5316
|
+
const retryMs = opts?.retryMs ?? 20_000;
|
|
5317
|
+
const maxPasses = opts?.maxPasses ?? 45;
|
|
5318
|
+
let pendingTaskIds: ReadonlySet<string> | undefined;
|
|
5319
|
+
// After a machine reboot the host service and its selected environment
|
|
5320
|
+
// provider start concurrently. Unknown provider state resolves nothing and
|
|
5321
|
+
// must retry once the runtime answers.
|
|
5322
|
+
for (let pass = 1; ; pass++) {
|
|
5323
|
+
if (opts?.isCurrent?.() === false) {
|
|
5324
|
+
if (!opts.lifecycle) finishAllTaskRecovery();
|
|
5325
|
+
return false;
|
|
5326
|
+
}
|
|
5327
|
+
const result = await recoveryPass(
|
|
5328
|
+
pendingTaskIds,
|
|
5329
|
+
opts?.lifecycle,
|
|
5330
|
+
opts?.maintenanceReady ?? agentClisReady,
|
|
5331
|
+
opts?.isCurrent ?? (() => true),
|
|
5332
|
+
);
|
|
5333
|
+
if (opts?.isCurrent?.() === false) {
|
|
5334
|
+
if (!opts.lifecycle) finishAllTaskRecovery();
|
|
5335
|
+
return false;
|
|
5336
|
+
}
|
|
5337
|
+
const { deferred } = result;
|
|
5338
|
+
if (!result.resolved) {
|
|
5339
|
+
if (!opts?.lifecycle) finishAllTaskRecovery();
|
|
5340
|
+
return false;
|
|
5341
|
+
}
|
|
5342
|
+
if (deferred.size === 0) {
|
|
5343
|
+
if (!opts?.lifecycle) finishAllTaskRecovery();
|
|
5344
|
+
return true;
|
|
5345
|
+
}
|
|
5346
|
+
pendingTaskIds = deferred;
|
|
5347
|
+
if (pass >= maxPasses) {
|
|
5348
|
+
console.warn(
|
|
5349
|
+
`[orchestrator] recovery: task environment runtime still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
|
|
5350
|
+
);
|
|
5351
|
+
if (!opts?.lifecycle) finishAllTaskRecovery();
|
|
5352
|
+
return false;
|
|
5353
|
+
}
|
|
5354
|
+
if (pass === 1) {
|
|
5355
|
+
console.log(
|
|
5356
|
+
`[orchestrator] recovery: task environment runtime not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
|
|
5357
|
+
);
|
|
5358
|
+
}
|
|
5359
|
+
await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
5360
|
+
}
|
|
5361
|
+
}
|
|
5362
|
+
|
|
5363
|
+
interface RecoveryPassResult {
|
|
5364
|
+
deferred: Set<string>;
|
|
5365
|
+
resolved: boolean;
|
|
5366
|
+
}
|
|
5367
|
+
|
|
5368
|
+
/** One recovery sweep. Returns only task ids deferred by an unreachable provider;
|
|
5369
|
+
* successful ids are never revisited by a later retry pass. `resolved` is
|
|
5370
|
+
* false for an unexpected failure whose runtime state could not be proved. */
|
|
5371
|
+
async function recoveryPass(
|
|
5372
|
+
onlyTaskIds?: ReadonlySet<string>,
|
|
5373
|
+
lifecycle?: Orchestrator,
|
|
5374
|
+
maintenanceReady: Promise<void> = agentClisReady,
|
|
5375
|
+
isCurrent: () => boolean = () => true,
|
|
5376
|
+
): Promise<RecoveryPassResult> {
|
|
5377
|
+
const deferred = new Set<string>();
|
|
5378
|
+
let resolved = true;
|
|
5379
|
+
try {
|
|
5380
|
+
if (!isCurrent()) return { deferred, resolved: false };
|
|
5381
|
+
const db = getDb();
|
|
5382
|
+
const activeRows = db
|
|
5383
|
+
.select()
|
|
5384
|
+
.from(schema.hostTasks)
|
|
5385
|
+
.where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
|
|
5386
|
+
.all();
|
|
5387
|
+
const rows = onlyTaskIds
|
|
5388
|
+
? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
|
|
5389
|
+
: activeRows;
|
|
5390
|
+
if (!lifecycle) {
|
|
5391
|
+
const activeTaskIds = new Set(rows.map((task) => task.taskId));
|
|
5392
|
+
const expectedTaskIds =
|
|
5393
|
+
onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
|
|
5394
|
+
for (const taskId of expectedTaskIds) {
|
|
5395
|
+
if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
|
|
5396
|
+
}
|
|
5397
|
+
for (const task of rows) openTaskRecoveryBarrier(task.taskId);
|
|
5398
|
+
}
|
|
5399
|
+
if (rows.length === 0) return { deferred, resolved };
|
|
5400
|
+
console.log(
|
|
5401
|
+
`[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
|
|
5402
|
+
);
|
|
5403
|
+
for (const task of rows) {
|
|
5404
|
+
if (!isCurrent()) break;
|
|
5405
|
+
try {
|
|
5406
|
+
const recovered = lifecycle
|
|
5407
|
+
? await lifecycle.runTaskLifecycle(task.taskId, async () => {
|
|
5408
|
+
// The row was selected before this lifecycle slot became ours.
|
|
5409
|
+
// A queued taskDown/taskUp may have completed meanwhile, so act
|
|
5410
|
+
// only on the current local authority—not the stale snapshot.
|
|
5411
|
+
const current = getHostTask(task.taskId);
|
|
5412
|
+
if (
|
|
5413
|
+
!current ||
|
|
5414
|
+
!isCurrent() ||
|
|
5415
|
+
!current.statusMirror ||
|
|
5416
|
+
!isActive(current.statusMirror)
|
|
5417
|
+
) {
|
|
5418
|
+
return true;
|
|
5419
|
+
}
|
|
5420
|
+
return recoverPersistedTaskEnvironment(
|
|
5421
|
+
current,
|
|
5422
|
+
maintenanceReady,
|
|
5423
|
+
isCurrent,
|
|
5424
|
+
lifecycle,
|
|
5425
|
+
);
|
|
5426
|
+
})
|
|
5427
|
+
: await recoverPersistedTaskEnvironment(
|
|
5428
|
+
task,
|
|
5429
|
+
maintenanceReady,
|
|
5430
|
+
isCurrent,
|
|
5431
|
+
);
|
|
5432
|
+
if (recovered) {
|
|
5433
|
+
if (!lifecycle) finishTaskRecovery(task.taskId);
|
|
5434
|
+
} else {
|
|
5435
|
+
deferred.add(task.taskId);
|
|
5436
|
+
}
|
|
5437
|
+
} catch (err) {
|
|
5438
|
+
resolved = false;
|
|
5439
|
+
if (!lifecycle) finishTaskRecovery(task.taskId);
|
|
5440
|
+
console.error(
|
|
5441
|
+
`[orchestrator] recovery: ${task.taskId} failed:`,
|
|
5442
|
+
err instanceof Error ? err.message : err,
|
|
5443
|
+
);
|
|
5444
|
+
}
|
|
5445
|
+
}
|
|
5446
|
+
} catch (err) {
|
|
5447
|
+
resolved = false;
|
|
5448
|
+
if (!lifecycle) finishAllTaskRecovery();
|
|
5449
|
+
console.error(
|
|
5450
|
+
"[orchestrator] recovery: top-level failure",
|
|
5451
|
+
err instanceof Error ? err.message : err,
|
|
5452
|
+
);
|
|
5453
|
+
}
|
|
5454
|
+
return { deferred, resolved };
|
|
5455
|
+
}
|
|
5456
|
+
|
|
5457
|
+
/** Apple-backend variant of the writable-runtime quarantine. Task containers
|
|
5458
|
+
* prove their shared-cache mount read-only at creation, so a writable mount
|
|
5459
|
+
* here is either a live materializer that outlived its bound or tampering —
|
|
5460
|
+
* both are stopped (task-labeled rows report `stopped`). A CLI that cannot
|
|
5461
|
+
* answer yields `retry`, exactly like an unreachable Docker daemon; it is
|
|
5462
|
+
* never treated as an empty inventory. */
|
|
5463
|
+
async function quarantineWritableAppleRuntimeContainers(options: {
|
|
5464
|
+
lifecycle: Orchestrator;
|
|
5465
|
+
isCurrent?: () => boolean;
|
|
5466
|
+
scope?: "contract" | "volume-holders";
|
|
5467
|
+
}): Promise<WritableRuntimeQuarantineResult> {
|
|
5468
|
+
const isCurrent = options.isCurrent ?? (() => true);
|
|
5469
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
|
|
5470
|
+
const binding = appleContainerRuntimeBinding();
|
|
5471
|
+
if (binding === null) return { verdict: "retry", stoppedTaskIds: [] };
|
|
5472
|
+
const cli = appleCliRunner(() => binding.containerCliPath);
|
|
5473
|
+
const listed = await cli(["list", "--all", "--format", "json"], {
|
|
5474
|
+
timeoutMs: 30_000,
|
|
5475
|
+
maxOutputBytes: 8 * 1024 * 1024,
|
|
5476
|
+
});
|
|
5477
|
+
if (listed.status !== 0 || listed.truncated === true) {
|
|
5478
|
+
return { verdict: "retry", stoppedTaskIds: [] };
|
|
5479
|
+
}
|
|
5480
|
+
let parsed: unknown;
|
|
5481
|
+
try {
|
|
5482
|
+
parsed = JSON.parse(listed.stdout);
|
|
5483
|
+
} catch {
|
|
5484
|
+
return { verdict: "retry", stoppedTaskIds: [] };
|
|
5485
|
+
}
|
|
5486
|
+
if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTaskIds: [] };
|
|
5487
|
+
const stoppedTaskIds = new Set<string>();
|
|
5488
|
+
let resolved = true;
|
|
5489
|
+
|
|
5490
|
+
const stopAndProve = async (id: string): Promise<boolean> => {
|
|
5491
|
+
const stopped = await cli(["stop", "--time", "30", id], {
|
|
5492
|
+
timeoutMs: 90_000,
|
|
5493
|
+
maxOutputBytes: 256 * 1024,
|
|
5494
|
+
});
|
|
5495
|
+
const after = await cli(["inspect", id], {
|
|
5496
|
+
timeoutMs: 15_000,
|
|
5497
|
+
maxOutputBytes: 1024 * 1024,
|
|
5498
|
+
});
|
|
5499
|
+
if (after.status !== 0) {
|
|
5500
|
+
return (
|
|
5501
|
+
after.stdout.trim() === "" &&
|
|
5502
|
+
after.stderr
|
|
5503
|
+
.split("\n")
|
|
5504
|
+
.some((line) => line.trim() === `Error: container not found: ${id}`)
|
|
5505
|
+
);
|
|
5506
|
+
}
|
|
5507
|
+
try {
|
|
5508
|
+
const parsedAfter = JSON.parse(after.stdout) as Array<{
|
|
5509
|
+
id?: unknown;
|
|
5510
|
+
status?: { state?: unknown };
|
|
5511
|
+
}>;
|
|
5512
|
+
// Only one exact-identity record in the runtime's terminal stopped
|
|
5513
|
+
// state settles the proof. A wrong-identity record, a transitional
|
|
5514
|
+
// state ("starting"), or an ambiguous shape must never release
|
|
5515
|
+
// shared-runtime maintenance while the writer may still be active.
|
|
5516
|
+
if (
|
|
5517
|
+
Array.isArray(parsedAfter) &&
|
|
5518
|
+
parsedAfter.length === 1 &&
|
|
5519
|
+
parsedAfter[0]?.id === id &&
|
|
5520
|
+
parsedAfter[0]?.status?.state === "stopped"
|
|
5521
|
+
) {
|
|
5522
|
+
return true;
|
|
5523
|
+
}
|
|
5524
|
+
} catch {
|
|
5525
|
+
// fall through to the failure log below
|
|
5526
|
+
}
|
|
5527
|
+
console.error(
|
|
5528
|
+
`[orchestrator] runtime isolation: apple container ${id} holds the shared runtime cache writable and could not be proven stopped (stop exit ${stopped.status})`,
|
|
5529
|
+
);
|
|
5530
|
+
return false;
|
|
5531
|
+
};
|
|
5532
|
+
|
|
5533
|
+
for (const item of parsed as Array<{
|
|
5534
|
+
configuration?: {
|
|
5535
|
+
id?: unknown;
|
|
5536
|
+
labels?: Record<string, unknown>;
|
|
5537
|
+
mounts?: Array<{
|
|
5538
|
+
destination?: unknown;
|
|
5539
|
+
options?: unknown;
|
|
5540
|
+
type?: { volume?: { name?: unknown } };
|
|
5541
|
+
}>;
|
|
5542
|
+
};
|
|
5543
|
+
}>) {
|
|
5544
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
5545
|
+
const id = item?.configuration?.id;
|
|
5546
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
5547
|
+
resolved = false;
|
|
5548
|
+
continue;
|
|
5549
|
+
}
|
|
5550
|
+
const mounts = item.configuration?.mounts;
|
|
5551
|
+
// Sweep every container whose shared-volume attachment violates the
|
|
5552
|
+
// CURRENT contract: writable (missing ro, or contradictory ro+rw), or at
|
|
5553
|
+
// any destination other than the ADR-108 staging path — which includes
|
|
5554
|
+
// every pre-ADR-108 container (read-only at /opt/asdf-data). Stopping
|
|
5555
|
+
// legacy containers HERE, before writer maintenance, is what lets a
|
|
5556
|
+
// legacy host migrate at all: a running legacy container would otherwise
|
|
5557
|
+
// hold the volume against the very reconcile whose completion recovery
|
|
5558
|
+
// needs (review 2026-08-18: maintenance retried forever and recovery —
|
|
5559
|
+
// the step that settles legacy containers — never ran).
|
|
5560
|
+
const writableShared = (Array.isArray(mounts) ? mounts : []).some(
|
|
5561
|
+
(mount) =>
|
|
5562
|
+
mount?.type?.volume?.name === "uai-asdf-data" &&
|
|
5563
|
+
(options.scope === "volume-holders" ||
|
|
5564
|
+
!(Array.isArray(mount.options) && mount.options.includes("ro")) ||
|
|
5565
|
+
(Array.isArray(mount.options) && mount.options.includes("rw")) ||
|
|
5566
|
+
mount?.destination !== "/run/uai/asdf-lower"),
|
|
5567
|
+
);
|
|
5568
|
+
if (!writableShared) continue;
|
|
5569
|
+
const label = item.configuration?.labels?.["com.uai.task"];
|
|
5570
|
+
// The label is container-influenced data. Only a syntactically safe task
|
|
5571
|
+
// id whose derived container name matches this exact container may drive
|
|
5572
|
+
// task-scoped mutations (lifecycle slot, DB row, ACL removal); anything
|
|
5573
|
+
// else is stopped as an anonymous writer, nothing task-scoped touched.
|
|
5574
|
+
const taskId =
|
|
5575
|
+
typeof label === "string" &&
|
|
5576
|
+
isSafeHostTaskId(label) &&
|
|
5577
|
+
id === appleTaskContainerName(label)
|
|
5578
|
+
? label
|
|
5579
|
+
: null;
|
|
5580
|
+
if (taskId === null) {
|
|
5581
|
+
if (!(await stopAndProve(id))) resolved = false;
|
|
5582
|
+
continue;
|
|
5583
|
+
}
|
|
5584
|
+
// Task-labeled writers stop inside the task's lifecycle slot so a
|
|
5585
|
+
// concurrent taskUp/taskDown cannot interleave, and a known row commits
|
|
5586
|
+
// its durable `stopped` verdict before the frame is queued — a crashed
|
|
5587
|
+
// publisher must never leave SQLite saying running.
|
|
5588
|
+
await options.lifecycle.runTaskLifecycle(taskId, async () => {
|
|
5589
|
+
if (!isCurrent()) return;
|
|
5590
|
+
await options.lifecycle.quarantineChannelForRecovery(taskId);
|
|
5591
|
+
if (!(await stopAndProve(id))) {
|
|
5592
|
+
resolved = false;
|
|
5593
|
+
return;
|
|
5594
|
+
}
|
|
5595
|
+
const task = getHostTask(taskId);
|
|
5596
|
+
if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
|
|
5597
|
+
db_setStatus(taskId, "stopped", {
|
|
5598
|
+
codeServerPort: null,
|
|
5599
|
+
previewPorts: "[]",
|
|
5600
|
+
});
|
|
5601
|
+
stoppedTaskIds.add(taskId);
|
|
5602
|
+
}
|
|
5603
|
+
// Revoke live access with the environment, like the Docker sweep.
|
|
5604
|
+
clearTaskGatewayAcl(taskId);
|
|
5605
|
+
});
|
|
5606
|
+
}
|
|
5607
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
5608
|
+
return {
|
|
5609
|
+
verdict: resolved ? "ready" : "retry",
|
|
5610
|
+
stoppedTaskIds: [...stoppedTaskIds],
|
|
5611
|
+
};
|
|
5612
|
+
}
|
|
5613
|
+
|
|
5614
|
+
/** Dispatch one durable row through its provider. Generic boot orchestration
|
|
5615
|
+
* owns DB/lifecycle sequencing but never derives a Compose/container identity. */
|
|
5616
|
+
async function recoverPersistedTaskEnvironment(
|
|
5617
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
5618
|
+
maintenanceReady: Promise<void>,
|
|
5619
|
+
isCurrent: () => boolean,
|
|
5620
|
+
lifecycle?: Orchestrator,
|
|
5621
|
+
): Promise<boolean> {
|
|
5622
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
5623
|
+
if (environment === null) return true;
|
|
5624
|
+
if (!isCurrent()) return true;
|
|
5625
|
+
const result = await environment.recover({
|
|
5626
|
+
maintenanceReady,
|
|
5627
|
+
isCurrent,
|
|
5628
|
+
...(lifecycle
|
|
5629
|
+
? {
|
|
5630
|
+
quarantineConsumers: () =>
|
|
5631
|
+
lifecycle.quarantineChannelForRecovery(task.taskId),
|
|
5632
|
+
allowConsumers: () => lifecycle.allowChannel(task.taskId),
|
|
5633
|
+
ensureConsumersStarted: async () => {
|
|
5634
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
5635
|
+
},
|
|
5636
|
+
}
|
|
5637
|
+
: {}),
|
|
5638
|
+
});
|
|
5639
|
+
return result.outcome === "recovered";
|
|
5640
|
+
}
|
|
5641
|
+
|
|
5642
|
+
interface RecoveryTaskConsumers {
|
|
5643
|
+
quarantineChannelForRecovery(taskId: string): Promise<void>;
|
|
5644
|
+
allowChannel(taskId: string): void;
|
|
5645
|
+
ensureStarted(taskId: string): Promise<boolean | void>;
|
|
5646
|
+
}
|
|
5647
|
+
|
|
5648
|
+
/** Apple provider driver. Recovery must end in a durable truth: a running
|
|
5649
|
+
* container is re-initialized (dependencies + code-server) under an
|
|
5650
|
+
* in-container deadline; a stopped one is started and initialized; a
|
|
5651
|
+
* container that is provenly absent — or cannot be started — marks the task
|
|
5652
|
+
* `stopped` so the host never reports operational with a phantom running
|
|
5653
|
+
* task. Unknown CLI answers stay deferred and retryable. */
|
|
5654
|
+
/** Exact TypeScript port of task-up.sh's container environment-authority
|
|
5655
|
+
* proof (the jq filter behind "verify task runtime isolation"): each required
|
|
5656
|
+
* canonical entry appears exactly once, every owned-npm-config spelling
|
|
5657
|
+
* (case-folded, underscores treated as hyphens) must be one of the canonical
|
|
5658
|
+
* eight, and PATH plus the asdf/corepack controls hold their exact values
|
|
5659
|
+
* exactly once. */
|
|
5660
|
+
export function appleTaskEnvironmentAuthorityHolds(value: unknown): boolean {
|
|
5661
|
+
if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) {
|
|
5662
|
+
return false;
|
|
5663
|
+
}
|
|
5664
|
+
const environment = value as string[];
|
|
5665
|
+
const npmPrefix = "/home/node/.local/share/uai-npm";
|
|
5666
|
+
const npmCache = "/home/node/.cache/uai-npm";
|
|
5667
|
+
const runtimePath =
|
|
5668
|
+
"/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:" +
|
|
5669
|
+
`/usr/sbin:/usr/bin:/sbin:/bin:${npmPrefix}/bin`;
|
|
5670
|
+
const required = [
|
|
5671
|
+
`NPM_CONFIG_PREFIX=${npmPrefix}`,
|
|
5672
|
+
`npm_config_prefix=${npmPrefix}`,
|
|
5673
|
+
`NPM_CONFIG_CACHE=${npmCache}`,
|
|
5674
|
+
`npm_config_cache=${npmCache}`,
|
|
5675
|
+
`NPM_CONFIG_LOGS_DIR=${npmCache}/_logs`,
|
|
5676
|
+
`npm_config_logs_dir=${npmCache}/_logs`,
|
|
5677
|
+
"NPM_CONFIG_UMASK=0002",
|
|
5678
|
+
"npm_config_umask=0002",
|
|
5679
|
+
];
|
|
5680
|
+
const owned = ["prefix", "cache", "logs-dir", "umask"];
|
|
5681
|
+
const countOf = (exact: string): number =>
|
|
5682
|
+
environment.filter((entry) => entry === exact).length;
|
|
5683
|
+
const isOwnedNpmConfig = (key: string): boolean => {
|
|
5684
|
+
const folded = key.toLowerCase();
|
|
5685
|
+
return (
|
|
5686
|
+
folded.startsWith("npm_config_") &&
|
|
5687
|
+
owned.includes(folded.slice("npm_config_".length).replace(/_/g, "-"))
|
|
5688
|
+
);
|
|
5689
|
+
};
|
|
5690
|
+
const ownedEntries = environment.filter((entry) =>
|
|
5691
|
+
isOwnedNpmConfig(entry.split("=")[0]!),
|
|
5692
|
+
);
|
|
5693
|
+
return (
|
|
5694
|
+
required.every((entry) => countOf(entry) === 1) &&
|
|
5695
|
+
ownedEntries.length === required.length &&
|
|
5696
|
+
countOf(`PATH=${runtimePath}`) === 1 &&
|
|
5697
|
+
countOf("ASDF_DATA_DIR=/opt/asdf-data") === 1 &&
|
|
5698
|
+
countOf("ASDF_SKIP_RESHIM=1") === 1 &&
|
|
5699
|
+
countOf("COREPACK_HOME=/opt/asdf-data/corepack") === 1 &&
|
|
5700
|
+
countOf("COREPACK_ENABLE_NETWORK=0") === 1
|
|
5701
|
+
);
|
|
5702
|
+
}
|
|
5703
|
+
|
|
5704
|
+
export type AppleTaskRuntimeContractVerdict =
|
|
5705
|
+
| { kind: "holds" }
|
|
5706
|
+
| { kind: "violated" }
|
|
5707
|
+
| { kind: "unreachable"; detail: string };
|
|
5708
|
+
|
|
5709
|
+
/**
|
|
5710
|
+
* The apple runtime-authority contract, in one inspection: this exact name
|
|
5711
|
+
* carries this task's label, is running, mounts the shared cache read-only
|
|
5712
|
+
* at exactly the ADR-108 staging path (and nowhere else, with no
|
|
5713
|
+
* contradictory options), binds /workspace as exactly one writable virtiofs
|
|
5714
|
+
* share of this task's real worktree leaf, and carries the same immutable
|
|
5715
|
+
* npm/runtime environment authority task-up proved at creation. Factored
|
|
5716
|
+
* from recovery so the taskUp lost-response fast path proves the SAME
|
|
5717
|
+
* invariants (review 2026-08-18 round 4). `unreachable` and `violated` are
|
|
5718
|
+
* distinct so the fast path can retry without stopping a container it could
|
|
5719
|
+
* not observe, while recovery keeps treating both as quarantine grounds.
|
|
5720
|
+
*/
|
|
5721
|
+
export async function proveAppleTaskRuntimeContract(options: {
|
|
5722
|
+
taskId: string;
|
|
5723
|
+
containerName: string;
|
|
5724
|
+
hostWorktreePath: string;
|
|
5725
|
+
cli?: ReturnType<typeof appleCliRunner>;
|
|
5726
|
+
}): Promise<AppleTaskRuntimeContractVerdict> {
|
|
5727
|
+
let cli = options.cli;
|
|
5728
|
+
if (!cli) {
|
|
5729
|
+
const binding = appleContainerRuntimeBinding();
|
|
5730
|
+
if (binding === null) {
|
|
5731
|
+
return { kind: "unreachable", detail: "apple runtime binding unavailable" };
|
|
5732
|
+
}
|
|
5733
|
+
cli = appleCliRunner(() => binding.containerCliPath);
|
|
5734
|
+
}
|
|
5735
|
+
const contract = await cli(["inspect", options.containerName], {
|
|
5736
|
+
timeoutMs: 15_000,
|
|
5737
|
+
maxOutputBytes: 4 * 1024 * 1024,
|
|
5738
|
+
});
|
|
5739
|
+
if (contract.status !== 0) {
|
|
5740
|
+
return {
|
|
5741
|
+
kind: "unreachable",
|
|
5742
|
+
detail:
|
|
5743
|
+
contract.stderr.trim().slice(0, 200) ||
|
|
5744
|
+
`inspect exited ${contract.status ?? "killed"}`,
|
|
5745
|
+
};
|
|
5746
|
+
}
|
|
5747
|
+
try {
|
|
5748
|
+
const records = JSON.parse(contract.stdout) as Array<{
|
|
5749
|
+
id?: unknown;
|
|
5750
|
+
status?: { state?: unknown };
|
|
5751
|
+
configuration?: {
|
|
5752
|
+
labels?: Record<string, unknown>;
|
|
5753
|
+
mounts?: Array<{
|
|
5754
|
+
destination?: unknown;
|
|
5755
|
+
options?: unknown;
|
|
5756
|
+
type?: { volume?: { name?: unknown } };
|
|
5757
|
+
}>;
|
|
5758
|
+
initProcess?: { environment?: unknown };
|
|
5759
|
+
};
|
|
5760
|
+
}>;
|
|
5761
|
+
// Same strictness as the provider's inspection: exactly one record and
|
|
5762
|
+
// it must identify itself as the requested container, or a malformed
|
|
5763
|
+
// response describing another container could authorize execution.
|
|
5764
|
+
if (!Array.isArray(records) || records.length !== 1) {
|
|
5765
|
+
throw new Error("contract inspection shape is invalid");
|
|
5766
|
+
}
|
|
5767
|
+
const record = records[0]!;
|
|
5768
|
+
if (record.id !== options.containerName) {
|
|
5769
|
+
throw new Error("contract inspection identity mismatch");
|
|
5770
|
+
}
|
|
5771
|
+
const mounts = record?.configuration?.mounts ?? [];
|
|
5772
|
+
// ADR-108: the shared volume attaches read-only at the overlay staging
|
|
5773
|
+
// path and /opt/asdf-data must NOT be a host mount at all — the
|
|
5774
|
+
// writable view there is the in-guest overlay.
|
|
5775
|
+
const runtimeMounts = mounts.filter(
|
|
5776
|
+
(mount) => mount?.destination === "/run/uai/asdf-lower",
|
|
5777
|
+
);
|
|
5778
|
+
const sharedMounts = mounts.filter(
|
|
5779
|
+
(mount) => mount?.type?.volume?.name === "uai-asdf-data",
|
|
5780
|
+
);
|
|
5781
|
+
const asdfHostMounts = mounts.filter(
|
|
5782
|
+
(mount) => mount?.destination === "/opt/asdf-data",
|
|
5783
|
+
);
|
|
5784
|
+
// The /workspace authority: exactly one bind (never a volume) whose
|
|
5785
|
+
// source is this task's exact worktree leaf, still a real non-symlink
|
|
5786
|
+
// directory on disk — the same proof Docker recovery performs.
|
|
5787
|
+
const workspaceMounts = mounts.filter(
|
|
5788
|
+
(mount) => mount?.destination === "/workspace",
|
|
5789
|
+
);
|
|
5790
|
+
const expectedWorkspaceSource = join(
|
|
5791
|
+
options.hostWorktreePath,
|
|
5792
|
+
"workspace",
|
|
5793
|
+
);
|
|
5794
|
+
// The verified Apple dialect for a host-directory bind is exactly
|
|
5795
|
+
// type:{virtiofs:{}} — a closed shape.
|
|
5796
|
+
const workspaceType = workspaceMounts[0]?.type as
|
|
5797
|
+
| Record<string, unknown>
|
|
5798
|
+
| undefined;
|
|
5799
|
+
const workspaceTypeIsVirtiofs =
|
|
5800
|
+
workspaceType !== undefined &&
|
|
5801
|
+
typeof workspaceType === "object" &&
|
|
5802
|
+
!Array.isArray(workspaceType) &&
|
|
5803
|
+
Object.keys(workspaceType).length === 1 &&
|
|
5804
|
+
typeof workspaceType.virtiofs === "object" &&
|
|
5805
|
+
workspaceType.virtiofs !== null &&
|
|
5806
|
+
!Array.isArray(workspaceType.virtiofs) &&
|
|
5807
|
+
Object.keys(workspaceType.virtiofs as object).length === 0;
|
|
5808
|
+
// Writable is part of the authority: a read-only /workspace would
|
|
5809
|
+
// expose a task whose agents cannot work.
|
|
5810
|
+
const workspaceOptions = workspaceMounts[0]?.options;
|
|
5811
|
+
let workspaceAuthorityHolds =
|
|
5812
|
+
workspaceMounts.length === 1 &&
|
|
5813
|
+
workspaceTypeIsVirtiofs &&
|
|
5814
|
+
Array.isArray(workspaceOptions) &&
|
|
5815
|
+
!(workspaceOptions as unknown[]).includes("ro") &&
|
|
5816
|
+
(workspaceMounts[0] as { source?: unknown } | undefined)?.source ===
|
|
5817
|
+
expectedWorkspaceSource;
|
|
5818
|
+
if (workspaceAuthorityHolds) {
|
|
5819
|
+
try {
|
|
5820
|
+
const workspaceStat = lstatSync(expectedWorkspaceSource);
|
|
5821
|
+
workspaceAuthorityHolds =
|
|
5822
|
+
workspaceStat.isDirectory() && !workspaceStat.isSymbolicLink();
|
|
5823
|
+
} catch {
|
|
5824
|
+
workspaceAuthorityHolds = false;
|
|
5825
|
+
}
|
|
5826
|
+
}
|
|
5827
|
+
const rawEnvironment = record?.configuration?.initProcess?.environment;
|
|
5828
|
+
const holds =
|
|
5829
|
+
record?.status?.state === "running" &&
|
|
5830
|
+
record?.configuration?.labels?.["com.uai.task"] === options.taskId &&
|
|
5831
|
+
runtimeMounts.length === 1 &&
|
|
5832
|
+
sharedMounts.length === 1 &&
|
|
5833
|
+
sharedMounts[0] === runtimeMounts[0] &&
|
|
5834
|
+
asdfHostMounts.length === 0 &&
|
|
5835
|
+
Array.isArray(runtimeMounts[0]?.options) &&
|
|
5836
|
+
(runtimeMounts[0]!.options as unknown[]).includes("ro") &&
|
|
5837
|
+
// Closed options contract: an array carrying BOTH ro and rw is
|
|
5838
|
+
// contradictory and proves nothing.
|
|
5839
|
+
!(runtimeMounts[0]!.options as unknown[]).includes("rw") &&
|
|
5840
|
+
workspaceAuthorityHolds &&
|
|
5841
|
+
appleTaskEnvironmentAuthorityHolds(rawEnvironment);
|
|
5842
|
+
return holds ? { kind: "holds" } : { kind: "violated" };
|
|
5843
|
+
} catch {
|
|
5844
|
+
return { kind: "violated" };
|
|
5845
|
+
}
|
|
5846
|
+
}
|
|
5847
|
+
|
|
5848
|
+
export async function recoverAppleTaskEnvironment(
|
|
5849
|
+
descriptor: TaskEnvironmentDescriptor,
|
|
5850
|
+
context: TaskEnvironmentRecoveryContext,
|
|
5851
|
+
): Promise<TaskEnvironmentRecoveryResult> {
|
|
5852
|
+
const locator = parseAppleTaskEnvironmentLocator(descriptor.locator);
|
|
5853
|
+
const task = getHostTask(descriptor.taskId);
|
|
5854
|
+
if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
|
|
5855
|
+
return { outcome: "recovered" };
|
|
5856
|
+
}
|
|
5857
|
+
// The boot maintenance latch (agentClisReady) is awaited ONLY before the
|
|
5858
|
+
// first in-container exec, exactly like Docker recovery. Awaiting it here
|
|
5859
|
+
// deadlocked the first fresh Apple host (live 2026-08-15): the on-connect
|
|
5860
|
+
// recovery sweep held this driver on the latch, activation joins in-flight
|
|
5861
|
+
// recovery before its maintenance, and the maintenance that settles the
|
|
5862
|
+
// latch could therefore never run — the host sat silently in `checking`
|
|
5863
|
+
// forever. Status verdicts, quarantines, and workspace settlement need no
|
|
5864
|
+
// maintenance and must never wait on it.
|
|
5865
|
+
if (!context.isCurrent()) {
|
|
5866
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5867
|
+
}
|
|
5868
|
+
const binding = appleContainerRuntimeBinding();
|
|
5869
|
+
if (binding === null) {
|
|
5870
|
+
return { outcome: "deferred", detail: "apple runtime binding unavailable" };
|
|
5871
|
+
}
|
|
5872
|
+
const cli = appleCliRunner(() => binding.containerCliPath);
|
|
5873
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
5874
|
+
if (environment === null) return { outcome: "recovered" };
|
|
5875
|
+
|
|
5876
|
+
const markStopped = (): TaskEnvironmentRecoveryResult => {
|
|
5877
|
+
db_setStatus(descriptor.taskId, "stopped", {
|
|
5878
|
+
codeServerPort: null,
|
|
5879
|
+
previewPorts: "[]",
|
|
5880
|
+
});
|
|
5881
|
+
// A stopped verdict must also revoke live access paths, exactly like the
|
|
5882
|
+
// Docker quarantine: the MCP gateway token dies with the environment.
|
|
5883
|
+
clearTaskGatewayAcl(descriptor.taskId);
|
|
5884
|
+
return { outcome: "recovered" };
|
|
5885
|
+
};
|
|
5886
|
+
// Without its workspace the task cannot resume — stopped would advertise a
|
|
5887
|
+
// restart that must fail. Docker recovery reconciles this as error.
|
|
5888
|
+
const markError = (): TaskEnvironmentRecoveryResult => {
|
|
5889
|
+
db_setStatus(descriptor.taskId, "error", {
|
|
5890
|
+
codeServerPort: null,
|
|
5891
|
+
previewPorts: "[]",
|
|
5892
|
+
composeProject: null,
|
|
5893
|
+
});
|
|
5894
|
+
clearTaskGatewayAcl(descriptor.taskId);
|
|
5895
|
+
return { outcome: "recovered" };
|
|
5896
|
+
};
|
|
5897
|
+
// Every settled verdict consults the freshest workspace observation: a
|
|
5898
|
+
// workspace that vanished mid-recovery must land as error exactly like one
|
|
5899
|
+
// that was already gone at the first look.
|
|
5900
|
+
const settleFromStatus = (settled: {
|
|
5901
|
+
workspacePresent: boolean;
|
|
5902
|
+
}): TaskEnvironmentRecoveryResult =>
|
|
5903
|
+
settled.workspacePresent ? markStopped() : markError();
|
|
5904
|
+
|
|
5905
|
+
let status = await environment.status();
|
|
5906
|
+
if (status.state === "unknown") {
|
|
5907
|
+
return { outcome: "deferred", detail: status.detail };
|
|
5908
|
+
}
|
|
5909
|
+
// Every awaited runtime observation can outlive this recovery's readiness
|
|
5910
|
+
// generation; a superseded pass must observe only, never mutate. Recheck
|
|
5911
|
+
// after each await and before each mutation from here on.
|
|
5912
|
+
if (!context.isCurrent()) {
|
|
5913
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5914
|
+
}
|
|
5915
|
+
// Drain consumers before ANY durable verdict or mutation — including the
|
|
5916
|
+
// absent path, where a live channel could otherwise keep serving a task
|
|
5917
|
+
// whose row is about to say stopped.
|
|
5918
|
+
await context.quarantineConsumers?.();
|
|
5919
|
+
if (!context.isCurrent()) {
|
|
5920
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5921
|
+
}
|
|
5922
|
+
if (status.state === "absent") {
|
|
5923
|
+
// Name-anchored absence came from the provider's inspect. The disk truth
|
|
5924
|
+
// decides the verdict: an intact workspace resumes via re-provision
|
|
5925
|
+
// (stopped); a vanished one cannot, and stopped would advertise a
|
|
5926
|
+
// restart that must fail (error).
|
|
5927
|
+
return status.workspacePresent ? markStopped() : markError();
|
|
5928
|
+
}
|
|
5929
|
+
if (status.state === "stopped" && !status.workspacePresent) {
|
|
5930
|
+
// Never start a container whose workspace is gone: dependency recovery
|
|
5931
|
+
// would mutate a recreated or missing tree.
|
|
5932
|
+
return markError();
|
|
5933
|
+
}
|
|
5934
|
+
if (status.state === "stopped") {
|
|
5935
|
+
const started = await cli(["start", locator.containerName], {
|
|
5936
|
+
timeoutMs: 120_000,
|
|
5937
|
+
maxOutputBytes: 256 * 1024,
|
|
5938
|
+
});
|
|
5939
|
+
if (!context.isCurrent()) {
|
|
5940
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
3622
5941
|
}
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
return;
|
|
5942
|
+
// A failed start command or an ambiguous follow-up is NOT proof of a
|
|
5943
|
+
// stopped container: only an exact stopped/absent answer may settle the
|
|
5944
|
+
// durable row, and unknown always stays deferred and retryable.
|
|
5945
|
+
status = await environment.status();
|
|
5946
|
+
if (status.state === "unknown") {
|
|
5947
|
+
return { outcome: "deferred", detail: status.detail };
|
|
3630
5948
|
}
|
|
3631
|
-
if (
|
|
3632
|
-
|
|
3633
|
-
|
|
5949
|
+
if (!context.isCurrent()) {
|
|
5950
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5951
|
+
}
|
|
5952
|
+
if (status.state === "stopped" || status.state === "absent") {
|
|
5953
|
+
return settleFromStatus(status);
|
|
5954
|
+
}
|
|
5955
|
+
if (started.status !== 0) {
|
|
5956
|
+
// The CLI reported failure yet the container runs — trust the state
|
|
5957
|
+
// probe over the command's exit and continue into revalidation.
|
|
5958
|
+
console.warn(
|
|
5959
|
+
`[orchestrator] recovery: ${descriptor.taskId} container start exited ${started.status} but the container is running; continuing`,
|
|
3634
5960
|
);
|
|
3635
5961
|
}
|
|
3636
|
-
await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
3637
5962
|
}
|
|
3638
|
-
}
|
|
3639
5963
|
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
const
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
const
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
5964
|
+
// One inspection proves the whole trust decision — factored into
|
|
5965
|
+
// proveAppleTaskRuntimeContract so the taskUp lost-response fast path can
|
|
5966
|
+
// prove the SAME invariants (review 2026-08-18 round 4). Recovery keeps
|
|
5967
|
+
// its historical mapping: anything short of `holds` — including an
|
|
5968
|
+
// unreachable inspect — quarantines below.
|
|
5969
|
+
const contractVerdict = await proveAppleTaskRuntimeContract({
|
|
5970
|
+
taskId: descriptor.taskId,
|
|
5971
|
+
containerName: locator.containerName,
|
|
5972
|
+
hostWorktreePath: locator.hostWorktreePath,
|
|
5973
|
+
cli,
|
|
5974
|
+
});
|
|
5975
|
+
const contractHolds = contractVerdict.kind === "holds";
|
|
5976
|
+
if (!context.isCurrent()) {
|
|
5977
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5978
|
+
}
|
|
5979
|
+
if (!contractHolds) {
|
|
5980
|
+
await environment.stop();
|
|
5981
|
+
const after = await environment.status();
|
|
5982
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
5983
|
+
return {
|
|
5984
|
+
outcome: "deferred",
|
|
5985
|
+
detail:
|
|
5986
|
+
"task container failed its runtime-authority contract and could not be proven stopped",
|
|
5987
|
+
};
|
|
5988
|
+
}
|
|
5989
|
+
if (!context.isCurrent()) {
|
|
5990
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5991
|
+
}
|
|
5992
|
+
console.error(
|
|
5993
|
+
`[orchestrator] recovery: ${descriptor.taskId} container failed its runtime-authority contract; task was stopped and remains quarantined`,
|
|
3666
5994
|
);
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
5995
|
+
return settleFromStatus(after);
|
|
5996
|
+
}
|
|
5997
|
+
// Even an already-running container's first exec resolves through shared
|
|
5998
|
+
// asdf shims; fence it behind the one-way boot maintenance latch, exactly
|
|
5999
|
+
// like Docker recovery fences its workbench probe.
|
|
6000
|
+
await context.maintenanceReady;
|
|
6001
|
+
if (!context.isCurrent()) {
|
|
6002
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6003
|
+
}
|
|
6004
|
+
// ADR-108: guest mounts do not survive a VM restart, so the runtime
|
|
6005
|
+
// overlay must be re-established (idempotently) before ANY exec resolves
|
|
6006
|
+
// through /opt/asdf-data shims — including uai-init below. The upper layer
|
|
6007
|
+
// persists in the container filesystem, so the task's own installs return
|
|
6008
|
+
// with the mount. Proven from the guest mount table, exactly like task-up.
|
|
6009
|
+
const overlay = await environment.exec({
|
|
6010
|
+
argv: [
|
|
6011
|
+
"/bin/sh",
|
|
6012
|
+
"-c",
|
|
6013
|
+
// Same establishment + token-exact proof as task-up §6a: exactly one
|
|
6014
|
+
// effective mount at the merged path and the lower, comma-delimited
|
|
6015
|
+
// option tokens (substring matching accepted a -evil suffix), a
|
|
6016
|
+
// read-only lower, and a real node-owned upper.
|
|
6017
|
+
"set -e; /bin/mkdir -p /var/lib/uai/asdf-upper /var/lib/uai/asdf-work /opt/asdf-data; " +
|
|
6018
|
+
"/bin/chown node:node /var/lib/uai/asdf-upper; " +
|
|
6019
|
+
'if ! /bin/grep -q "^overlay /opt/asdf-data overlay " /proc/mounts; then ' +
|
|
6020
|
+
"/bin/mount -t overlay overlay -o lowerdir=/run/uai/asdf-lower,upperdir=/var/lib/uai/asdf-upper,workdir=/var/lib/uai/asdf-work /opt/asdf-data; fi; " +
|
|
6021
|
+
'overlay_lines=$(/bin/grep -c "^overlay /opt/asdf-data overlay " /proc/mounts || true); ' +
|
|
6022
|
+
'[ "$overlay_lines" = "1" ] || { echo "expected exactly one /opt/asdf-data overlay, found $overlay_lines" >&2; exit 1; }; ' +
|
|
6023
|
+
'any_lines=$(/bin/grep -c " /opt/asdf-data " /proc/mounts || true); ' +
|
|
6024
|
+
'[ "$any_lines" = "1" ] || { echo "unexpected extra mounts at /opt/asdf-data" >&2; exit 1; }; ' +
|
|
6025
|
+
'opts=$(/bin/grep "^overlay /opt/asdf-data overlay " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
|
|
6026
|
+
'case ",$opts," in *",lowerdir=/run/uai/asdf-lower,"*) ;; *) echo "overlay lower is foreign" >&2; exit 1 ;; esac; ' +
|
|
6027
|
+
'case ",$opts," in *",upperdir=/var/lib/uai/asdf-upper,"*) ;; *) echo "overlay upper is foreign" >&2; exit 1 ;; esac; ' +
|
|
6028
|
+
'case ",$opts," in *",workdir=/var/lib/uai/asdf-work,"*) ;; *) echo "overlay workdir is foreign" >&2; exit 1 ;; esac; ' +
|
|
6029
|
+
'case ",$opts," in *",ro,"*) echo "overlay is unexpectedly read-only" >&2; exit 1 ;; esac; ' +
|
|
6030
|
+
'lower_lines=$(/bin/grep -c " /run/uai/asdf-lower " /proc/mounts || true); ' +
|
|
6031
|
+
'[ "$lower_lines" = "1" ] || { echo "expected exactly one lower attachment, found $lower_lines" >&2; exit 1; }; ' +
|
|
6032
|
+
'lopts=$(/bin/grep " /run/uai/asdf-lower " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
|
|
6033
|
+
'case ",$lopts," in *",ro,"*) ;; *) echo "lower attachment is not read-only" >&2; exit 1 ;; esac; ' +
|
|
6034
|
+
'[ ! -L /var/lib/uai/asdf-upper ] || { echo "upper is a symlink" >&2; exit 1; }; ' +
|
|
6035
|
+
'[ -d /var/lib/uai/asdf-upper ] || { echo "upper is not a directory" >&2; exit 1; }; ' +
|
|
6036
|
+
'[ "$(/usr/bin/stat -c %U /var/lib/uai/asdf-upper)" = "node" ] || { echo "upper is not node-owned" >&2; exit 1; }; ' +
|
|
6037
|
+
"/usr/bin/test -d /opt/asdf-data/plugins",
|
|
6038
|
+
],
|
|
6039
|
+
user: "root",
|
|
6040
|
+
env: {},
|
|
6041
|
+
timeoutMs: 30_000,
|
|
6042
|
+
maxOutputBytes: 64 * 1024,
|
|
6043
|
+
});
|
|
6044
|
+
if (!context.isCurrent()) {
|
|
6045
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6046
|
+
}
|
|
6047
|
+
if (overlay.exitCode !== 0 || overlay.signal !== null) {
|
|
6048
|
+
await environment.stop();
|
|
6049
|
+
const afterOverlay = await environment.status();
|
|
6050
|
+
if (afterOverlay.state === "running" || afterOverlay.state === "unknown") {
|
|
6051
|
+
return {
|
|
6052
|
+
outcome: "deferred",
|
|
6053
|
+
detail:
|
|
6054
|
+
"task runtime overlay could not be established and the container could not be proven stopped",
|
|
6055
|
+
};
|
|
6056
|
+
}
|
|
6057
|
+
if (!context.isCurrent()) {
|
|
6058
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
3678
6059
|
}
|
|
3679
|
-
} catch (err) {
|
|
3680
|
-
finishAllTaskRecovery();
|
|
3681
6060
|
console.error(
|
|
3682
|
-
|
|
3683
|
-
|
|
6061
|
+
`[orchestrator] recovery: ${descriptor.taskId} runtime overlay could not be established (exit ${overlay.exitCode}, signal ${overlay.signal}); task was stopped`,
|
|
6062
|
+
);
|
|
6063
|
+
return settleFromStatus(afterOverlay);
|
|
6064
|
+
}
|
|
6065
|
+
// Dependencies + code-server. The in-container deadline (TERM then KILL)
|
|
6066
|
+
// means a host-side expiry can never leave the initializer mutating the
|
|
6067
|
+
// workspace while agents use it; exit 78 is a runtime-authority violation
|
|
6068
|
+
// and quarantines exactly like at task-up.
|
|
6069
|
+
const init = await environment.exec({
|
|
6070
|
+
argv: [
|
|
6071
|
+
"/usr/bin/timeout",
|
|
6072
|
+
"--signal=TERM",
|
|
6073
|
+
"--kill-after=15s",
|
|
6074
|
+
"570s",
|
|
6075
|
+
"/usr/local/bin/uai-init",
|
|
6076
|
+
],
|
|
6077
|
+
env: {},
|
|
6078
|
+
timeoutMs: 10 * 60_000,
|
|
6079
|
+
maxOutputBytes: 1024 * 1024,
|
|
6080
|
+
});
|
|
6081
|
+
if (!context.isCurrent()) {
|
|
6082
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6083
|
+
}
|
|
6084
|
+
if (
|
|
6085
|
+
init.exitCode === null ||
|
|
6086
|
+
init.signal !== null ||
|
|
6087
|
+
init.stdoutTruncated ||
|
|
6088
|
+
init.stderrTruncated
|
|
6089
|
+
) {
|
|
6090
|
+
// The host-side client died (deadline, overflow, forced settlement)
|
|
6091
|
+
// without a real exit code, so the in-container initializer may STILL be
|
|
6092
|
+
// mutating dependencies for several minutes. Never expose the task on an
|
|
6093
|
+
// unknowable outcome: stop and prove, or stay deferred.
|
|
6094
|
+
await environment.stop();
|
|
6095
|
+
const after = await environment.status();
|
|
6096
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
6097
|
+
return {
|
|
6098
|
+
outcome: "deferred",
|
|
6099
|
+
detail:
|
|
6100
|
+
"task initializer outcome was unknowable and the container could not be proven stopped",
|
|
6101
|
+
};
|
|
6102
|
+
}
|
|
6103
|
+
if (!context.isCurrent()) {
|
|
6104
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6105
|
+
}
|
|
6106
|
+
console.error(
|
|
6107
|
+
`[orchestrator] recovery: ${descriptor.taskId} initializer outcome was unknowable (exit ${init.exitCode}, signal ${init.signal}); task was stopped pending a clean re-run`,
|
|
6108
|
+
);
|
|
6109
|
+
return settleFromStatus(after);
|
|
6110
|
+
}
|
|
6111
|
+
if (init.exitCode === 78) {
|
|
6112
|
+
await environment.stop();
|
|
6113
|
+
const after = await environment.status();
|
|
6114
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
6115
|
+
return {
|
|
6116
|
+
outcome: "deferred",
|
|
6117
|
+
detail:
|
|
6118
|
+
"runtime-authority violation detected and the container could not be proven stopped",
|
|
6119
|
+
};
|
|
6120
|
+
}
|
|
6121
|
+
if (!context.isCurrent()) {
|
|
6122
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6123
|
+
}
|
|
6124
|
+
console.error(
|
|
6125
|
+
`[orchestrator] recovery: ${descriptor.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
|
|
6126
|
+
);
|
|
6127
|
+
return settleFromStatus(after);
|
|
6128
|
+
}
|
|
6129
|
+
if (init.exitCode !== 0) {
|
|
6130
|
+
console.warn(
|
|
6131
|
+
`[orchestrator] recovery: ${descriptor.taskId} uai-init failed (exit ${init.exitCode}); container is up but Editor may be down`,
|
|
6132
|
+
);
|
|
6133
|
+
}
|
|
6134
|
+
if (!context.isCurrent()) {
|
|
6135
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6136
|
+
}
|
|
6137
|
+
db_setRuntime(descriptor.taskId, { codeServerPort: null });
|
|
6138
|
+
context.allowConsumers?.();
|
|
6139
|
+
await context.ensureConsumersStarted?.();
|
|
6140
|
+
if (!context.isCurrent()) {
|
|
6141
|
+
// Admission reopened against a superseded generation: re-quarantine so
|
|
6142
|
+
// the successor generation re-proves the environment before consumers
|
|
6143
|
+
// touch it — the same closing move Docker recovery makes.
|
|
6144
|
+
await context.quarantineConsumers?.();
|
|
6145
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6146
|
+
}
|
|
6147
|
+
return { outcome: "recovered" };
|
|
6148
|
+
}
|
|
6149
|
+
|
|
6150
|
+
/** Docker provider driver. The legacy repair implementation remains local to
|
|
6151
|
+
* this module for now, but it is reachable only through the provider handle;
|
|
6152
|
+
* the generic recovery sweep above never consumes its locator payload. */
|
|
6153
|
+
async function recoverDockerTaskEnvironment(
|
|
6154
|
+
descriptor: TaskEnvironmentDescriptor,
|
|
6155
|
+
context: TaskEnvironmentRecoveryContext,
|
|
6156
|
+
): Promise<TaskEnvironmentRecoveryResult> {
|
|
6157
|
+
const locator = parseDockerTaskEnvironmentLocator(descriptor.locator);
|
|
6158
|
+
const task = getHostTask(descriptor.taskId);
|
|
6159
|
+
if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
|
|
6160
|
+
return { outcome: "recovered" };
|
|
6161
|
+
}
|
|
6162
|
+
const isPreparedRow =
|
|
6163
|
+
(task.statusMirror === "starting" || task.statusMirror === "error") &&
|
|
6164
|
+
task.composeProject === null &&
|
|
6165
|
+
task.worktreePath === null;
|
|
6166
|
+
if (
|
|
6167
|
+
!isPreparedRow &&
|
|
6168
|
+
(task.composeProject !== locator.composeProject ||
|
|
6169
|
+
task.worktreePath !== locator.hostWorktreePath)
|
|
6170
|
+
) {
|
|
6171
|
+
throw new Error(
|
|
6172
|
+
`task ${descriptor.taskId} environment locator no longer matches its durable row`,
|
|
3684
6173
|
);
|
|
3685
6174
|
}
|
|
3686
|
-
|
|
6175
|
+
// Provider-owned provisional metadata is authoritative after a crash. Feed
|
|
6176
|
+
// the already-validated deterministic values to the Docker-private repair
|
|
6177
|
+
// helpers without teaching the generic sweep how to derive them.
|
|
6178
|
+
const recoveryTask = isPreparedRow
|
|
6179
|
+
? {
|
|
6180
|
+
...task,
|
|
6181
|
+
composeProject: locator.composeProject,
|
|
6182
|
+
worktreePath: locator.hostWorktreePath,
|
|
6183
|
+
}
|
|
6184
|
+
: task;
|
|
6185
|
+
|
|
6186
|
+
const lifecycleCallbacks = [
|
|
6187
|
+
context.quarantineConsumers,
|
|
6188
|
+
context.allowConsumers,
|
|
6189
|
+
context.ensureConsumersStarted,
|
|
6190
|
+
];
|
|
6191
|
+
const hasLifecycle = lifecycleCallbacks.some(Boolean);
|
|
6192
|
+
if (hasLifecycle && !lifecycleCallbacks.every(Boolean)) {
|
|
6193
|
+
throw new Error("task environment recovery lifecycle callbacks are incomplete");
|
|
6194
|
+
}
|
|
6195
|
+
const consumers: RecoveryTaskConsumers | undefined = hasLifecycle
|
|
6196
|
+
? {
|
|
6197
|
+
quarantineChannelForRecovery: async (taskId) => {
|
|
6198
|
+
if (taskId !== descriptor.taskId) {
|
|
6199
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
6200
|
+
}
|
|
6201
|
+
await context.quarantineConsumers!();
|
|
6202
|
+
},
|
|
6203
|
+
allowChannel: (taskId) => {
|
|
6204
|
+
if (taskId !== descriptor.taskId) {
|
|
6205
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
6206
|
+
}
|
|
6207
|
+
context.allowConsumers!();
|
|
6208
|
+
},
|
|
6209
|
+
ensureStarted: async (taskId) => {
|
|
6210
|
+
if (taskId !== descriptor.taskId) {
|
|
6211
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
6212
|
+
}
|
|
6213
|
+
await context.ensureConsumersStarted!();
|
|
6214
|
+
},
|
|
6215
|
+
}
|
|
6216
|
+
: undefined;
|
|
6217
|
+
const recovered = await recoverOneDockerTask(
|
|
6218
|
+
recoveryTask,
|
|
6219
|
+
context.maintenanceReady,
|
|
6220
|
+
context.isCurrent,
|
|
6221
|
+
consumers,
|
|
6222
|
+
);
|
|
6223
|
+
return recovered
|
|
6224
|
+
? { outcome: "recovered" }
|
|
6225
|
+
: {
|
|
6226
|
+
outcome: "deferred",
|
|
6227
|
+
detail: "container runtime did not answer during task recovery",
|
|
6228
|
+
};
|
|
3687
6229
|
}
|
|
3688
6230
|
|
|
6231
|
+
setDockerTaskEnvironmentRecoveryDriver(recoverDockerTaskEnvironment);
|
|
6232
|
+
setAppleTaskEnvironmentRecoveryDriver(recoverAppleTaskEnvironment);
|
|
6233
|
+
|
|
3689
6234
|
/** Returns false when docker was unreachable (caller retries the pass). */
|
|
3690
|
-
async function
|
|
6235
|
+
async function recoverOneDockerTask(
|
|
3691
6236
|
task: typeof schema.hostTasks.$inferSelect,
|
|
6237
|
+
maintenanceReady: Promise<void>,
|
|
6238
|
+
isCurrent: () => boolean = () => true,
|
|
6239
|
+
lifecycle?: RecoveryTaskConsumers,
|
|
3692
6240
|
): Promise<boolean> {
|
|
6241
|
+
if (!isCurrent()) return true;
|
|
3693
6242
|
const composeProject = task.composeProject;
|
|
3694
6243
|
if (!composeProject) {
|
|
3695
6244
|
// We never wrote a compose project name for this row — must be
|
|
@@ -3701,6 +6250,7 @@ async function recoverOneTask(
|
|
|
3701
6250
|
const containers = await dockerListContainersByLabel(
|
|
3702
6251
|
`com.docker.compose.project=${composeProject}`,
|
|
3703
6252
|
);
|
|
6253
|
+
if (!isCurrent()) return true;
|
|
3704
6254
|
if (containers === null) {
|
|
3705
6255
|
// Docker unreachable / timed out — actual state unknown. Leave the row
|
|
3706
6256
|
// alone; the retrying caller re-scans once docker answers.
|
|
@@ -3727,12 +6277,106 @@ async function recoverOneTask(
|
|
|
3727
6277
|
return true;
|
|
3728
6278
|
}
|
|
3729
6279
|
|
|
3730
|
-
|
|
6280
|
+
// Defense in depth for every recovery entry point: the maintained
|
|
6281
|
+
// activation normally quarantines legacy writers before its first shared
|
|
6282
|
+
// volume mutation, but recovery itself must never fast-path or `docker
|
|
6283
|
+
// start` a container whose immutable mount is not the exact L5 RO contract.
|
|
6284
|
+
const mountContract = await inspectRuntimeWriterContainer({
|
|
6285
|
+
taskId: task.taskId,
|
|
6286
|
+
composeProject,
|
|
6287
|
+
containerName,
|
|
6288
|
+
});
|
|
6289
|
+
if (!isCurrent()) return true;
|
|
6290
|
+
if (mountContract.kind === "unreachable") return false;
|
|
6291
|
+
if (mountContract.kind === "failed") {
|
|
6292
|
+
console.error(
|
|
6293
|
+
`[orchestrator] recovery: ${task.taskId} mount identity is ambiguous; deferring without starting it: ${mountContract.detail}`,
|
|
6294
|
+
);
|
|
6295
|
+
return false;
|
|
6296
|
+
}
|
|
6297
|
+
if (
|
|
6298
|
+
mountContract.metadata.sharedMount !== "read-only" ||
|
|
6299
|
+
mountContract.metadata.taskEnvironment !== "current"
|
|
6300
|
+
) {
|
|
6301
|
+
if (lifecycle) {
|
|
6302
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6303
|
+
if (!isCurrent()) return true;
|
|
6304
|
+
}
|
|
6305
|
+
const stopped = await stopRuntimeWriter(mountContract.metadata);
|
|
6306
|
+
if (!isCurrent()) return true;
|
|
6307
|
+
if (stopped === "unreachable") return false;
|
|
6308
|
+
if (stopped === "failed") {
|
|
6309
|
+
throw new Error(
|
|
6310
|
+
`${task.taskId} has a legacy/non-current runtime mount and could not be proven stopped`,
|
|
6311
|
+
);
|
|
6312
|
+
}
|
|
6313
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6314
|
+
codeServerPort: null,
|
|
6315
|
+
previewPorts: "[]",
|
|
6316
|
+
});
|
|
6317
|
+
clearTaskGatewayAcl(task.taskId);
|
|
6318
|
+
console.warn(
|
|
6319
|
+
`[orchestrator] recovery: ${task.taskId} uses a legacy/non-current runtime mount; it remains stopped until explicit Resume recreates it`,
|
|
6320
|
+
);
|
|
6321
|
+
return true;
|
|
6322
|
+
}
|
|
6323
|
+
|
|
6324
|
+
// Even a container already marked running needs an in-container workbench
|
|
6325
|
+
// probe. Fence that first exec, as well as the exited-container repair path,
|
|
6326
|
+
// behind the same one-way boot maintenance latch.
|
|
6327
|
+
await maintenanceReady;
|
|
6328
|
+
if (!isCurrent()) return true;
|
|
6329
|
+
|
|
6330
|
+
let running = containers.some(
|
|
6331
|
+
(container) =>
|
|
6332
|
+
container.Names === containerName && container.State === "running",
|
|
6333
|
+
);
|
|
3731
6334
|
if (running) {
|
|
6335
|
+
// A stale recovery generation may have successfully issued `docker start`
|
|
6336
|
+
// and then been cancelled before uai-init. Do not bless every running
|
|
6337
|
+
// container solely from Docker state: code-server is the cheap proof that
|
|
6338
|
+
// the workbench init completed. A fresh sweep repairs a merely-started
|
|
6339
|
+
// container through the shared init path below.
|
|
6340
|
+
const workbenchReady = await dockerExec(
|
|
6341
|
+
containerName,
|
|
6342
|
+
["/usr/bin/pgrep", "-f", "code-server.*--bind-addr"],
|
|
6343
|
+
10_000,
|
|
6344
|
+
);
|
|
6345
|
+
if (!isCurrent()) return true;
|
|
6346
|
+
if (!workbenchReady && !(await dockerDaemonReachable())) return false;
|
|
6347
|
+
if (!isCurrent()) return true;
|
|
6348
|
+
if (!workbenchReady) {
|
|
6349
|
+
console.log(
|
|
6350
|
+
`[orchestrator] recovery: ${task.taskId} running without its workbench; quarantining before runtime proof`,
|
|
6351
|
+
);
|
|
6352
|
+
// Post-boot recovery can race live agent sessions. Drain their external
|
|
6353
|
+
// work and tombstone the channel before stopping the shared-volume
|
|
6354
|
+
// writer; successful recovery reopens the preserved channel spec below.
|
|
6355
|
+
if (lifecycle) {
|
|
6356
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6357
|
+
if (!isCurrent()) return true;
|
|
6358
|
+
}
|
|
6359
|
+
const quarantined = await quarantineRunningContainer(containerName);
|
|
6360
|
+
if (!isCurrent()) return true;
|
|
6361
|
+
if (quarantined === "unreachable") return false;
|
|
6362
|
+
if (quarantined === "failed") {
|
|
6363
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6364
|
+
codeServerPort: null,
|
|
6365
|
+
previewPorts: "[]",
|
|
6366
|
+
});
|
|
6367
|
+
throw new Error(
|
|
6368
|
+
`${task.taskId} could not prove its running app stopped after stop and kill; task remains quarantined`,
|
|
6369
|
+
);
|
|
6370
|
+
}
|
|
6371
|
+
running = false;
|
|
6372
|
+
} else {
|
|
3732
6373
|
// Container is up. Re-discover the host port in case Docker
|
|
3733
6374
|
// remapped it across restarts (it usually does for ephemeral
|
|
3734
6375
|
// bindings).
|
|
3735
|
-
const
|
|
6376
|
+
const portResult = await dockerPort(containerName, 8080);
|
|
6377
|
+
if (!isCurrent()) return true;
|
|
6378
|
+
if (!portResult.reachable) return false;
|
|
6379
|
+
const port = portResult.port;
|
|
3736
6380
|
if (port && port !== task.codeServerPort) {
|
|
3737
6381
|
db_setRuntime(task.taskId, {
|
|
3738
6382
|
codeServerPort: port,
|
|
@@ -3759,30 +6403,113 @@ async function recoverOneTask(
|
|
|
3759
6403
|
}`,
|
|
3760
6404
|
),
|
|
3761
6405
|
);
|
|
6406
|
+
if (!isCurrent()) return true;
|
|
6407
|
+
if (lifecycle) {
|
|
6408
|
+
lifecycle.allowChannel(task.taskId);
|
|
6409
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
6410
|
+
if (!isCurrent()) {
|
|
6411
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6412
|
+
}
|
|
6413
|
+
}
|
|
3762
6414
|
return true;
|
|
6415
|
+
}
|
|
3763
6416
|
}
|
|
3764
6417
|
|
|
3765
|
-
//
|
|
3766
|
-
//
|
|
3767
|
-
|
|
3768
|
-
|
|
6418
|
+
// The exact app container is now stopped. Prove every project runtime from
|
|
6419
|
+
// its immutable image and mounts before bringing it back up. Rolling old
|
|
6420
|
+
// images receive the current packaged materializer/Corepack pin as read-only
|
|
6421
|
+
// binds, so recovery cannot bypass a newly-shipped proof contract.
|
|
6422
|
+
const runtimeInspection = await inspectRecoveryRuntime(task, containerName);
|
|
6423
|
+
if (!isCurrent()) return true;
|
|
6424
|
+
if (runtimeInspection.kind === "unreachable") return false;
|
|
6425
|
+
if (runtimeInspection.kind === "failed") {
|
|
6426
|
+
console.error(
|
|
6427
|
+
`[orchestrator] recovery: ${task.taskId} runtime inspection failed; keeping task stopped: ${runtimeInspection.detail}`,
|
|
6428
|
+
);
|
|
6429
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6430
|
+
codeServerPort: null,
|
|
6431
|
+
previewPorts: "[]",
|
|
6432
|
+
});
|
|
6433
|
+
return true;
|
|
6434
|
+
}
|
|
6435
|
+
const runtimeMaterialization = await materializeRecoveryRuntimes(
|
|
6436
|
+
runtimeInspection.spec,
|
|
6437
|
+
);
|
|
6438
|
+
if (!isCurrent()) return true;
|
|
6439
|
+
if (runtimeMaterialization.kind === "unreachable") return false;
|
|
6440
|
+
if (runtimeMaterialization.kind === "failed") {
|
|
6441
|
+
console.error(
|
|
6442
|
+
`[orchestrator] recovery: ${task.taskId} runtime materialization failed; keeping task stopped and requiring a complete retry: ${runtimeMaterialization.detail}`,
|
|
6443
|
+
);
|
|
6444
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6445
|
+
codeServerPort: null,
|
|
6446
|
+
previewPorts: "[]",
|
|
6447
|
+
});
|
|
6448
|
+
return true;
|
|
6449
|
+
}
|
|
6450
|
+
const stagedInit = await stageRecoveryInit(
|
|
6451
|
+
containerName,
|
|
6452
|
+
runtimeInspection.spec,
|
|
6453
|
+
);
|
|
6454
|
+
if (!isCurrent()) return true;
|
|
6455
|
+
if (stagedInit.kind === "unreachable") return false;
|
|
6456
|
+
if (stagedInit.kind === "failed") {
|
|
6457
|
+
console.error(
|
|
6458
|
+
`[orchestrator] recovery: ${task.taskId} could not stage the current task initializer; keeping task stopped: ${stagedInit.detail}`,
|
|
6459
|
+
);
|
|
6460
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6461
|
+
codeServerPort: null,
|
|
6462
|
+
previewPorts: "[]",
|
|
6463
|
+
});
|
|
6464
|
+
return true;
|
|
6465
|
+
}
|
|
6466
|
+
const preStartWorkspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
|
|
6467
|
+
runtimeInspection.spec.workspaceSource,
|
|
3769
6468
|
);
|
|
3770
|
-
if (
|
|
6469
|
+
if (preStartWorkspaceRuntimeError) {
|
|
6470
|
+
console.error(
|
|
6471
|
+
`[orchestrator] recovery: ${task.taskId} workspace runtime authority changed after proof; keeping task stopped: ${preStartWorkspaceRuntimeError}`,
|
|
6472
|
+
);
|
|
3771
6473
|
db_setStatus(task.taskId, "stopped", {
|
|
3772
6474
|
codeServerPort: null,
|
|
3773
6475
|
previewPorts: "[]",
|
|
3774
6476
|
});
|
|
3775
6477
|
return true;
|
|
3776
6478
|
}
|
|
6479
|
+
|
|
6480
|
+
// Runtime proof succeeded. Bring the preserved app back up + re-launch the
|
|
6481
|
+
// task-local dependency/editor init.
|
|
6482
|
+
// If Docker appeared after the host's initial no-runtime verdict, the
|
|
6483
|
+
// standard-image repair starts concurrently with this boot sweep. uai-init
|
|
6484
|
+
// is now task-local (dependencies + editor), but it still resolves through
|
|
6485
|
+
// shared asdf shims; never run it until the boot owner has finished
|
|
6486
|
+
// reconciling the image and agent CLI volume.
|
|
6487
|
+
if (!running) {
|
|
6488
|
+
console.log(
|
|
6489
|
+
`[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
|
|
6490
|
+
);
|
|
6491
|
+
const started = await dockerStart(containerName);
|
|
6492
|
+
if (!isCurrent()) return true;
|
|
6493
|
+
if (started === "unreachable") return false;
|
|
6494
|
+
if (started === "failed") {
|
|
6495
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6496
|
+
codeServerPort: null,
|
|
6497
|
+
previewPorts: "[]",
|
|
6498
|
+
});
|
|
6499
|
+
return true;
|
|
6500
|
+
}
|
|
6501
|
+
}
|
|
3777
6502
|
// uai-init seeds code-server's profile beneath this shared data root. Heal
|
|
3778
6503
|
// containers created by older hosts before running it as node.
|
|
3779
6504
|
await repairNodeDataRoot(containerName);
|
|
6505
|
+
if (!isCurrent()) return true;
|
|
3780
6506
|
// Correctly-owned Codex creds BEFORE uai-init / any agent respawn: the boot
|
|
3781
6507
|
// reinject sweep only targets containers already running, so a container
|
|
3782
6508
|
// recovered here would otherwise keep whatever it held when it exited —
|
|
3783
6509
|
// possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
|
|
3784
6510
|
// by the inject itself and must not abort the task's recovery.
|
|
3785
6511
|
await injectCodexIntoContainer(containerName);
|
|
6512
|
+
if (!isCurrent()) return true;
|
|
3786
6513
|
// Establish the task owner's GitHub transport BEFORE uai-init performs any
|
|
3787
6514
|
// dependency-network work. This matters for package manifests that refer to
|
|
3788
6515
|
// private GitHub repositories: gh's credential helper must already be live
|
|
@@ -3790,7 +6517,20 @@ async function recoverOneTask(
|
|
|
3790
6517
|
// like the prior post-init call, but it is now ordered rather than raced.
|
|
3791
6518
|
if (task.ownerUserId) {
|
|
3792
6519
|
try {
|
|
3793
|
-
await reconcileTaskGitAuth(task.taskId, task.ownerUserId
|
|
6520
|
+
await reconcileTaskGitAuth(task.taskId, task.ownerUserId, {
|
|
6521
|
+
// Recovery is sanctioned private maintenance while public admission is
|
|
6522
|
+
// deliberately `checking`. Fence it by this daemon epoch instead of
|
|
6523
|
+
// the public operational bit, preserving private-dependency setup
|
|
6524
|
+
// before uai-init without letting stale recovery keep mutating.
|
|
6525
|
+
admitContainerWork: () => {
|
|
6526
|
+
if (!isCurrent()) {
|
|
6527
|
+
throw new ContainerRuntimeUnavailableError(
|
|
6528
|
+
"container runtime recovery generation was superseded",
|
|
6529
|
+
);
|
|
6530
|
+
}
|
|
6531
|
+
},
|
|
6532
|
+
});
|
|
6533
|
+
if (!isCurrent()) return true;
|
|
3794
6534
|
} catch (err) {
|
|
3795
6535
|
console.warn(
|
|
3796
6536
|
`[orchestrator] recovery: ${task.taskId} GitHub auth setup failed before uai-init: ${
|
|
@@ -3798,6 +6538,7 @@ async function recoverOneTask(
|
|
|
3798
6538
|
}`,
|
|
3799
6539
|
);
|
|
3800
6540
|
}
|
|
6541
|
+
if (!isCurrent()) return true;
|
|
3801
6542
|
}
|
|
3802
6543
|
// uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
|
|
3803
6544
|
// repo. dockerCli's 30s default would SIGKILL it mid-install.
|
|
@@ -3806,17 +6547,119 @@ async function recoverOneTask(
|
|
|
3806
6547
|
"exec",
|
|
3807
6548
|
"-e",
|
|
3808
6549
|
"UAI_SKIP_GIT_TRANSPORT=1",
|
|
6550
|
+
"-e",
|
|
6551
|
+
"UAI_WORKSPACE=/workspace",
|
|
6552
|
+
"-e",
|
|
6553
|
+
`UAI_RUNTIME_PROJECTS=${RECOVERY_INIT_PROJECTS_PATH}`,
|
|
6554
|
+
"-e",
|
|
6555
|
+
"PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
|
6556
|
+
"-e",
|
|
6557
|
+
"HOME=/home/node",
|
|
6558
|
+
"-e",
|
|
6559
|
+
"XDG_CONFIG_HOME=/home/node/.config",
|
|
6560
|
+
"-e",
|
|
6561
|
+
"XDG_DATA_HOME=/home/node/.local/share",
|
|
6562
|
+
"-e",
|
|
6563
|
+
"ASDF_DIR=/opt/asdf",
|
|
6564
|
+
"-e",
|
|
6565
|
+
"ASDF_DATA_DIR=/opt/asdf-data",
|
|
6566
|
+
"-e",
|
|
6567
|
+
"ASDF_CONFIG_FILE=/dev/null",
|
|
6568
|
+
"-e",
|
|
6569
|
+
"ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions",
|
|
6570
|
+
"-e",
|
|
6571
|
+
"ASDF_NODEJS_VERSION=",
|
|
6572
|
+
"-e",
|
|
6573
|
+
"ASDF_PYTHON_VERSION=",
|
|
6574
|
+
"-e",
|
|
6575
|
+
"ASDF_GOLANG_VERSION=",
|
|
6576
|
+
"-e",
|
|
6577
|
+
"ASDF_RUBY_VERSION=",
|
|
6578
|
+
"-e",
|
|
6579
|
+
"ASDF_RUST_VERSION=",
|
|
6580
|
+
"-e",
|
|
6581
|
+
"COREPACK_HOME=/opt/asdf-data/corepack",
|
|
6582
|
+
"-e",
|
|
6583
|
+
"COREPACK_DEFAULT_TO_LATEST=0",
|
|
6584
|
+
"-e",
|
|
6585
|
+
"COREPACK_ENV_FILE=0",
|
|
6586
|
+
"-e",
|
|
6587
|
+
"COREPACK_NPM_REGISTRY=https://registry.npmjs.org",
|
|
6588
|
+
"-e",
|
|
6589
|
+
"COREPACK_ENABLE_NETWORK=0",
|
|
6590
|
+
"-e",
|
|
6591
|
+
"COREPACK_ENABLE_AUTO_PIN=0",
|
|
6592
|
+
"-e",
|
|
6593
|
+
"COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
|
|
6594
|
+
"-e",
|
|
6595
|
+
"COREPACK_ENABLE_PROJECT_SPEC=1",
|
|
6596
|
+
"-e",
|
|
6597
|
+
"COREPACK_ENABLE_STRICT=1",
|
|
6598
|
+
"-e",
|
|
6599
|
+
"COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0",
|
|
6600
|
+
"-e",
|
|
6601
|
+
"COREPACK_INTEGRITY_CHECK=1",
|
|
6602
|
+
"-e",
|
|
6603
|
+
"LD_PRELOAD=",
|
|
6604
|
+
"-e",
|
|
6605
|
+
"LD_AUDIT=",
|
|
6606
|
+
"-e",
|
|
6607
|
+
"LD_LIBRARY_PATH=",
|
|
6608
|
+
"-e",
|
|
6609
|
+
"NODE_OPTIONS=",
|
|
6610
|
+
"-e",
|
|
6611
|
+
"NODE_PATH=",
|
|
6612
|
+
// SHELLOPTS/BASHOPTS deliberately absent: present-in-env (even empty)
|
|
6613
|
+
// makes bash export its live options into children.
|
|
6614
|
+
"-e",
|
|
6615
|
+
"PS4=",
|
|
6616
|
+
"-e",
|
|
6617
|
+
"BASH_ENV=",
|
|
6618
|
+
"-e",
|
|
6619
|
+
"ENV=",
|
|
3809
6620
|
containerName,
|
|
3810
|
-
|
|
6621
|
+
// The host-side deadline below only kills the local docker client; the
|
|
6622
|
+
// server-side exec would keep mutating workspace dependencies while the
|
|
6623
|
+
// recovered task is already exposed to agents. This in-container
|
|
6624
|
+
// deadline kills the actual initializer first (TERM, then KILL), and
|
|
6625
|
+
// the host allowance is deliberately wider so a timed-out run settles
|
|
6626
|
+
// with exit 124 instead of racing the client kill.
|
|
6627
|
+
"/usr/bin/timeout",
|
|
6628
|
+
"--signal=TERM",
|
|
6629
|
+
"--kill-after=15s",
|
|
6630
|
+
"570s",
|
|
6631
|
+
RECOVERY_INIT_PATH,
|
|
3811
6632
|
],
|
|
3812
6633
|
{ timeoutMs: 10 * 60_000 },
|
|
3813
6634
|
);
|
|
6635
|
+
if (!isCurrent()) return true;
|
|
6636
|
+
if (initResult.status === 78) {
|
|
6637
|
+
const quarantined = await quarantineRunningContainer(containerName);
|
|
6638
|
+
if (!isCurrent()) return true;
|
|
6639
|
+
if (quarantined === "unreachable") return false;
|
|
6640
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6641
|
+
codeServerPort: null,
|
|
6642
|
+
previewPorts: "[]",
|
|
6643
|
+
});
|
|
6644
|
+
if (quarantined === "failed") {
|
|
6645
|
+
throw new Error(
|
|
6646
|
+
`${task.taskId} reported a runtime-authority violation and could not be proven stopped`,
|
|
6647
|
+
);
|
|
6648
|
+
}
|
|
6649
|
+
console.error(
|
|
6650
|
+
`[orchestrator] recovery: ${task.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
|
|
6651
|
+
);
|
|
6652
|
+
return true;
|
|
6653
|
+
}
|
|
3814
6654
|
if (initResult.status !== 0) {
|
|
3815
6655
|
console.warn(
|
|
3816
6656
|
`[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
|
|
3817
6657
|
);
|
|
3818
6658
|
}
|
|
3819
|
-
const
|
|
6659
|
+
const portResult = await dockerPort(containerName, 8080);
|
|
6660
|
+
if (!isCurrent()) return true;
|
|
6661
|
+
if (!portResult.reachable) return false;
|
|
6662
|
+
const port = portResult.port;
|
|
3820
6663
|
db_setRuntime(task.taskId, {
|
|
3821
6664
|
codeServerPort: port,
|
|
3822
6665
|
});
|
|
@@ -3830,6 +6673,14 @@ async function recoverOneTask(
|
|
|
3830
6673
|
console.log(
|
|
3831
6674
|
`[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
|
|
3832
6675
|
);
|
|
6676
|
+
if (!isCurrent()) return true;
|
|
6677
|
+
if (lifecycle) {
|
|
6678
|
+
lifecycle.allowChannel(task.taskId);
|
|
6679
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
6680
|
+
if (!isCurrent()) {
|
|
6681
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6682
|
+
}
|
|
6683
|
+
}
|
|
3833
6684
|
return true;
|
|
3834
6685
|
}
|
|
3835
6686
|
|