@runuai/host 0.9.14 → 0.9.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +80 -30
- package/images/standard/Dockerfile +36 -10
- package/images/standard/README.md +63 -18
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +308 -38
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/lib/agent-cli.ts +33 -2
- package/lib/agent.ts +46 -7
- package/lib/agents/claude.ts +13 -8
- package/lib/agents/codex.ts +11 -6
- package/lib/agents/cursor.ts +39 -29
- package/lib/agents/durable-proc.ts +20 -27
- package/lib/agents/factory.ts +9 -25
- package/lib/agents/grok.ts +43 -30
- package/lib/agents/kimi.ts +44 -29
- package/lib/agents/opencode.ts +43 -31
- package/lib/agents/proc.ts +149 -114
- package/lib/agents/transport.ts +62 -50
- package/lib/agents/types.ts +6 -4
- package/lib/apple-runtime-recycle.ts +236 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/browser-testing.ts +233 -93
- package/lib/codex-auth.ts +40 -6
- package/lib/command-db.ts +20 -0
- package/lib/container-runtime.ts +1338 -0
- package/lib/db.ts +1 -0
- package/lib/docker-exec.ts +87 -5
- package/lib/engine-accounts.ts +68 -5
- package/lib/engine-login.ts +1952 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +155 -0
- package/lib/env.ts +4 -0
- package/lib/git-diff.ts +98 -32
- package/lib/git-identity.ts +199 -87
- package/lib/github-tokens.ts +202 -91
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-logs.ts +962 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +49 -0
- package/lib/managed-runtime.ts +3644 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-gateway.ts +450 -23
- package/lib/orchestrator.ts +3060 -218
- package/lib/preview-sidecar.ts +57 -13
- package/lib/release-manifest.ts +708 -0
- package/lib/release-trust.ts +28 -0
- package/lib/runtime-activation-tail.ts +232 -0
- package/lib/runtime-archive.ts +1086 -0
- package/lib/runtime-authority.ts +79 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-state.ts +232 -12
- package/lib/skills.ts +24 -3
- package/lib/ssh.ts +18 -0
- package/lib/standard-image.ts +1104 -141
- package/lib/stopped-task-status-queue.ts +44 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-diff.ts +66 -46
- package/lib/task-environment/apple-container.ts +757 -0
- package/lib/task-environment/docker.ts +945 -0
- package/lib/task-environment/index.ts +364 -0
- package/lib/task-environment/legacy-adoption.ts +443 -0
- package/lib/task-environment/registry.ts +58 -0
- package/lib/task-environment/types.ts +408 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +585 -0
- package/lib/tunnel-registry.ts +135 -19
- package/lib/tunnel-runtime.ts +235 -0
- package/package.json +1 -1
- package/scripts/agent/_common.sh +123 -3
- package/scripts/agent/task-down.sh +146 -38
- package/scripts/agent/task-status.sh +19 -3
- package/scripts/agent/task-up.sh +1405 -107
- package/scripts/install/darwin.ts +848 -50
- package/scripts/install/linux.ts +838 -35
- package/scripts/install/types.ts +43 -0
- package/scripts/install/util.ts +215 -8
- package/scripts/install/win.ts +12 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/cli.ts +1464 -72
- package/src/event-outbox.ts +83 -4
- package/src/index.ts +766 -42
- package/src/main.ts +1398 -255
- package/src/paths.ts +17 -1
- package/src/protocol.ts +695 -1
- package/src/runtime-bootstrap.ts +165 -0
- package/src/ui/server.ts +46 -10
- package/src/ui/types.ts +37 -0
package/lib/orchestrator.ts
CHANGED
|
@@ -15,10 +15,18 @@
|
|
|
15
15
|
* instead of leaking a second orchestrator.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {
|
|
19
|
-
|
|
18
|
+
import {
|
|
19
|
+
existsSync,
|
|
20
|
+
lstatSync,
|
|
21
|
+
mkdtempSync,
|
|
22
|
+
readFileSync,
|
|
23
|
+
realpathSync,
|
|
24
|
+
renameSync,
|
|
25
|
+
rmSync,
|
|
26
|
+
writeFileSync,
|
|
27
|
+
} from "node:fs";
|
|
20
28
|
import { homedir } from "node:os";
|
|
21
|
-
import { join } from "node:path";
|
|
29
|
+
import { dirname, join } from "node:path";
|
|
22
30
|
|
|
23
31
|
import { inArray } from "drizzle-orm";
|
|
24
32
|
|
|
@@ -33,15 +41,52 @@ import {
|
|
|
33
41
|
type Roster,
|
|
34
42
|
type RosterAgent,
|
|
35
43
|
} from "./agents/types";
|
|
36
|
-
import { ACTIVE_STATUSES } from "./task-status";
|
|
44
|
+
import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
|
|
45
|
+
import { isSafeHostTaskId } from "./task-identity";
|
|
37
46
|
import { getHostTask, upsertHostTask } from "./runtime-state";
|
|
38
47
|
import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
|
|
39
48
|
import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
|
|
40
49
|
import { dockerCli } from "./docker-exec";
|
|
50
|
+
import {
|
|
51
|
+
taskContainerBackend,
|
|
52
|
+
taskContainerCli,
|
|
53
|
+
} from "./task-container-cli";
|
|
54
|
+
import {
|
|
55
|
+
RUNTIME_AUTHORITY_ENV,
|
|
56
|
+
runtimeAuthorityDockerExecArgs,
|
|
57
|
+
} from "./runtime-authority";
|
|
58
|
+
import {
|
|
59
|
+
backgroundContainerWorkAllowed,
|
|
60
|
+
ContainerRuntimeUnavailableError,
|
|
61
|
+
} from "./runtime-guard";
|
|
62
|
+
import {
|
|
63
|
+
appleContainerRuntimeBinding,
|
|
64
|
+
initializeContainerRuntime,
|
|
65
|
+
pinnedContainerRuntimeProvider,
|
|
66
|
+
suspendContainerRuntime,
|
|
67
|
+
} from "./container-runtime";
|
|
68
|
+
import {
|
|
69
|
+
reconstructPersistedTaskEnvironment,
|
|
70
|
+
setAppleTaskEnvironmentRecoveryDriver,
|
|
71
|
+
setDockerTaskEnvironmentRecoveryDriver,
|
|
72
|
+
} from "./task-environment";
|
|
73
|
+
import {
|
|
74
|
+
appleCliRunner,
|
|
75
|
+
appleTaskContainerName,
|
|
76
|
+
parseAppleTaskEnvironmentLocator,
|
|
77
|
+
} from "./task-environment/apple-container";
|
|
78
|
+
import { parseDockerTaskEnvironmentLocator } from "./task-environment/docker";
|
|
79
|
+
import type {
|
|
80
|
+
TaskEnvironmentDescriptor,
|
|
81
|
+
TaskEnvironmentHandle,
|
|
82
|
+
TaskEnvironmentRecoveryContext,
|
|
83
|
+
TaskEnvironmentRecoveryResult,
|
|
84
|
+
} from "./task-environment/types";
|
|
41
85
|
import {
|
|
42
86
|
containerSkillFile,
|
|
43
87
|
installedSkillPath,
|
|
44
88
|
installPackageSkills,
|
|
89
|
+
type SkillExec,
|
|
45
90
|
writeAgentSkills,
|
|
46
91
|
} from "./skills";
|
|
47
92
|
import {
|
|
@@ -67,6 +112,11 @@ import {
|
|
|
67
112
|
} from "./engine-accounts";
|
|
68
113
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
69
114
|
import { stopPreviewSidecars } from "./preview-sidecar";
|
|
115
|
+
import {
|
|
116
|
+
agentClisReady,
|
|
117
|
+
standardImageCorepackVersionPath,
|
|
118
|
+
standardImageRuntimeMaterializerPath,
|
|
119
|
+
} from "./standard-image";
|
|
70
120
|
import { env } from "./env";
|
|
71
121
|
import type {
|
|
72
122
|
ChannelEnsureInput,
|
|
@@ -102,6 +152,11 @@ interface Channel {
|
|
|
102
152
|
sessions: Map<string, AgentSession>;
|
|
103
153
|
/** Session-spawn inputs, kept so sessions can be started lazily. */
|
|
104
154
|
containerName: string;
|
|
155
|
+
/** Opaque provider-owned boundary used only for agent process creation. */
|
|
156
|
+
agentEnvironment: Promise<TaskEnvironmentHandle<unknown>> | null;
|
|
157
|
+
/** Settled handle for later config/setup passes; avoids a new await on
|
|
158
|
+
* legacy/bootstrap paths. */
|
|
159
|
+
agentHandle: TaskEnvironmentHandle<unknown> | null;
|
|
105
160
|
/** Per-agent system preamble (channel briefing + assembled persona). */
|
|
106
161
|
preambles: Map<string, string>;
|
|
107
162
|
/** Per-agent first-turn message — only agents whose `initialPrompt` is
|
|
@@ -167,11 +222,20 @@ interface Channel {
|
|
|
167
222
|
/** Agents with a delivered turn that has not reached a terminal event yet.
|
|
168
223
|
* Unlike openTurns, this covers thinking/tool/permission time before text. */
|
|
169
224
|
activeTurns: Map<string, number>;
|
|
225
|
+
/** Exact accepted prompts still owned by each runner generation, in send
|
|
226
|
+
* order. Open mode deliberately permits more than one; a process-level
|
|
227
|
+
* daemon-loss event must replay all of them rather than only `lastPrompt`. */
|
|
228
|
+
acceptedPrompts: Map<string, AcceptedPrompt[]>;
|
|
170
229
|
/** Crew replies waiting for the designated Secretary's current turn
|
|
171
230
|
* boundary. Adapters do not share one mid-turn input contract, so only this
|
|
172
231
|
* cross-lane role is serialized at the host boundary; open mode keeps its
|
|
173
232
|
* legacy immediate-concurrent delivery behavior. */
|
|
174
233
|
pendingPrompts: Map<string, string[]>;
|
|
234
|
+
/** Prompts accepted by channelDeliver whose final session boundary rejected
|
|
235
|
+
* specifically because the container runtime became unavailable. */
|
|
236
|
+
runtimeDeferredPrompts: Map<string, string[]>;
|
|
237
|
+
/** Missing/blocked sessions to reconcile exactly once after runtime ready. */
|
|
238
|
+
runtimeDeferredAgents: Set<string>;
|
|
175
239
|
/** Agents whose current turn was interrupted (ESC) — their next
|
|
176
240
|
* turn_complete is flagged `aborted` so the cloud DISCARDS the buffered
|
|
177
241
|
* half-turn instead of delivering it to @-mentioned peers. */
|
|
@@ -230,6 +294,10 @@ interface Channel {
|
|
|
230
294
|
rotations: Map<string, number>;
|
|
231
295
|
}
|
|
232
296
|
|
|
297
|
+
interface AcceptedPrompt {
|
|
298
|
+
text: string;
|
|
299
|
+
}
|
|
300
|
+
|
|
233
301
|
/** Hard cap on automatic respawns per agent per channel lifetime. */
|
|
234
302
|
const MAX_RESPAWNS_PER_AGENT = 5;
|
|
235
303
|
/** ADR-076: hard cap on account rotations per agent between successful turns. */
|
|
@@ -241,6 +309,43 @@ const RESPAWN_COOLDOWN_MS = 10 * 60_000;
|
|
|
241
309
|
* retry would spin npx continuously for the life of a broken task. */
|
|
242
310
|
const BROWSER_RETRY_COOLDOWN_MS = 60_000;
|
|
243
311
|
|
|
312
|
+
/**
|
|
313
|
+
* A runner can be the first observable casualty of a daemon loss, before any
|
|
314
|
+
* command/tunnel path invalidates the cached ready verdict. Prove the daemon
|
|
315
|
+
* before classifying a terminal agent event as an agent crash. A failed proof
|
|
316
|
+
* poisons the runtime generation synchronously; normal detection/recovery owns
|
|
317
|
+
* the eventual ready edge and deferred-prompt replay.
|
|
318
|
+
*/
|
|
319
|
+
async function agentTerminationLostContainerRuntime(): Promise<boolean> {
|
|
320
|
+
if (!backgroundContainerWorkAllowed()) return true;
|
|
321
|
+
let reachable = false;
|
|
322
|
+
try {
|
|
323
|
+
// Backend-aware liveness: `docker` does not exist on an apple host, and
|
|
324
|
+
// an ENOENT here would poison the generation on EVERY abnormal agent
|
|
325
|
+
// termination (found in the 2026-08-18 spawn-docker-ENOENT sweep).
|
|
326
|
+
const proof = taskContainerBackend().apple
|
|
327
|
+
? await taskContainerCli(["system", "status"], { timeoutMs: 10_000 })
|
|
328
|
+
: await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
|
|
329
|
+
timeoutMs: 10_000,
|
|
330
|
+
});
|
|
331
|
+
reachable = proof.status === 0;
|
|
332
|
+
} catch {
|
|
333
|
+
reachable = false;
|
|
334
|
+
}
|
|
335
|
+
if (!backgroundContainerWorkAllowed()) return true;
|
|
336
|
+
if (reachable) return false;
|
|
337
|
+
|
|
338
|
+
suspendContainerRuntime(true);
|
|
339
|
+
void initializeContainerRuntime().catch((err) =>
|
|
340
|
+
console.warn(
|
|
341
|
+
`[orchestrator] container runtime recheck after agent termination failed: ${
|
|
342
|
+
err instanceof Error ? err.message : String(err)
|
|
343
|
+
}`,
|
|
344
|
+
),
|
|
345
|
+
);
|
|
346
|
+
return true;
|
|
347
|
+
}
|
|
348
|
+
|
|
244
349
|
/** Substrings in an agent's error output that mean "config was
|
|
245
350
|
* unlinked between runs" — repair-and-respawn covers the common
|
|
246
351
|
* Docker-Desktop-macOS race where Claude's atomic writes briefly
|
|
@@ -530,6 +635,8 @@ export class Orchestrator {
|
|
|
530
635
|
secretaryAgentId: spec.secretaryAgentId,
|
|
531
636
|
sessions: new Map(),
|
|
532
637
|
containerName: `task-${taskId.toLowerCase()}-app-1`,
|
|
638
|
+
agentEnvironment: null,
|
|
639
|
+
agentHandle: null,
|
|
533
640
|
preambles,
|
|
534
641
|
firstTurns,
|
|
535
642
|
sessionsReady: null,
|
|
@@ -547,7 +654,10 @@ export class Orchestrator {
|
|
|
547
654
|
closed: false,
|
|
548
655
|
openTurns: new Set(),
|
|
549
656
|
activeTurns: new Map(),
|
|
657
|
+
acceptedPrompts: new Map(),
|
|
550
658
|
pendingPrompts: new Map(),
|
|
659
|
+
runtimeDeferredPrompts: new Map(),
|
|
660
|
+
runtimeDeferredAgents: new Set(),
|
|
551
661
|
interrupted: new Set(),
|
|
552
662
|
respawns: new Map(),
|
|
553
663
|
respawnLastAt: new Map(),
|
|
@@ -587,6 +697,7 @@ export class Orchestrator {
|
|
|
587
697
|
// copy cannot land after browser/MCP materialization and erase config.
|
|
588
698
|
await taskRecoveryComplete(channel.taskId);
|
|
589
699
|
if (!this.isActiveChannel(channel)) return false;
|
|
700
|
+
if (!backgroundContainerWorkAllowed()) return false;
|
|
590
701
|
// Memoized: every caller awaits the SAME in-flight start, so a concurrent
|
|
591
702
|
// deliver() can't observe "ready" while the sessions map is still empty
|
|
592
703
|
// (startSessions awaits docker work before populating it — the old boolean
|
|
@@ -632,6 +743,7 @@ export class Orchestrator {
|
|
|
632
743
|
await this.ensureMcpConfig(channel);
|
|
633
744
|
if (!this.isActiveChannel(channel)) return false;
|
|
634
745
|
await this.reconcileSessions(channel);
|
|
746
|
+
this.drainRuntimeDeferredAgentWork(channel);
|
|
635
747
|
}
|
|
636
748
|
// The initial-start promise stays memoized after a healthy boot. A later
|
|
637
749
|
// fatal exit can therefore leave the designated Secretary missing even
|
|
@@ -735,6 +847,10 @@ export class Orchestrator {
|
|
|
735
847
|
changed: false,
|
|
736
848
|
};
|
|
737
849
|
if (!this.isActiveChannel(channel)) return unavailable;
|
|
850
|
+
if (!backgroundContainerWorkAllowed()) {
|
|
851
|
+
this.scheduleBrowserRetry(channel);
|
|
852
|
+
return unavailable;
|
|
853
|
+
}
|
|
738
854
|
const requestedBrowser = channel.browserTesting;
|
|
739
855
|
const requestedHasCodex =
|
|
740
856
|
hasCodex || channel.roster.some((agent) => agent.kind === "codex");
|
|
@@ -818,10 +934,17 @@ export class Orchestrator {
|
|
|
818
934
|
}
|
|
819
935
|
let copiedOrUnknown = false;
|
|
820
936
|
try {
|
|
821
|
-
const provisioned =
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
937
|
+
const provisioned = channel.agentHandle
|
|
938
|
+
? await provisionEngineAccounts(
|
|
939
|
+
channel.containerName,
|
|
940
|
+
requestedKinds,
|
|
941
|
+
{},
|
|
942
|
+
channel.agentHandle,
|
|
943
|
+
)
|
|
944
|
+
: await provisionEngineAccounts(
|
|
945
|
+
channel.containerName,
|
|
946
|
+
requestedKinds,
|
|
947
|
+
);
|
|
825
948
|
copiedOrUnknown =
|
|
826
949
|
provisioned.copied > 0 || provisioned.mayHaveMutated;
|
|
827
950
|
if (provisioned.failed > 0) {
|
|
@@ -873,12 +996,20 @@ export class Orchestrator {
|
|
|
873
996
|
return unavailable;
|
|
874
997
|
}
|
|
875
998
|
coverage.browserAttempted = true;
|
|
876
|
-
setup =
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
999
|
+
setup = channel.agentHandle
|
|
1000
|
+
? await setupBrowserTesting(
|
|
1001
|
+
channel.taskId,
|
|
1002
|
+
channel.containerName,
|
|
1003
|
+
requestedHasCodex,
|
|
1004
|
+
requestedHomes,
|
|
1005
|
+
channel.agentHandle,
|
|
1006
|
+
)
|
|
1007
|
+
: await setupBrowserTesting(
|
|
1008
|
+
channel.taskId,
|
|
1009
|
+
channel.containerName,
|
|
1010
|
+
requestedHasCodex,
|
|
1011
|
+
requestedHomes,
|
|
1012
|
+
);
|
|
882
1013
|
} catch (err) {
|
|
883
1014
|
console.warn(
|
|
884
1015
|
`[browser] task ${channel.taskId}: setup failed: ${
|
|
@@ -977,6 +1108,10 @@ export class Orchestrator {
|
|
|
977
1108
|
channel.browserRetryTimer = null;
|
|
978
1109
|
}
|
|
979
1110
|
if (!this.isActiveChannel(channel)) return;
|
|
1111
|
+
if (!backgroundContainerWorkAllowed()) {
|
|
1112
|
+
this.scheduleBrowserRetry(channel);
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
980
1115
|
const forceProvision = Boolean(channel.provisionRetryFingerprint);
|
|
981
1116
|
void this.ensureBrowserSetup(
|
|
982
1117
|
channel,
|
|
@@ -1063,13 +1198,28 @@ export class Orchestrator {
|
|
|
1063
1198
|
engineKinds.filter((kind) => !previous?.engineKinds?.includes(kind)),
|
|
1064
1199
|
);
|
|
1065
1200
|
const hadLiveSessions = channel.sessions.size > 0;
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1201
|
+
// Once a provider-neutral handle has already been reconstructed for this
|
|
1202
|
+
// channel, keep subsequent config writes on that same environment. The
|
|
1203
|
+
// first legacy/bootstrap pass intentionally retains its Docker fallback;
|
|
1204
|
+
// it must not introduce an eager reconstruction dependency before the
|
|
1205
|
+
// channel has a durable provider locator.
|
|
1206
|
+
const providerEnvironment = channel.agentHandle ?? undefined;
|
|
1207
|
+
const configured = providerEnvironment
|
|
1208
|
+
? await setupMcpTaskConfig(
|
|
1209
|
+
channel.taskId,
|
|
1210
|
+
channel.containerName,
|
|
1211
|
+
connections,
|
|
1212
|
+
engineKinds,
|
|
1213
|
+
codexHomes,
|
|
1214
|
+
providerEnvironment,
|
|
1215
|
+
)
|
|
1216
|
+
: await setupMcpTaskConfig(
|
|
1217
|
+
channel.taskId,
|
|
1218
|
+
channel.containerName,
|
|
1219
|
+
connections,
|
|
1220
|
+
engineKinds,
|
|
1221
|
+
codexHomes,
|
|
1222
|
+
);
|
|
1073
1223
|
if (!configured || !this.isActiveChannel(channel)) return;
|
|
1074
1224
|
channel.configGeneration += 1;
|
|
1075
1225
|
channel.mcpConfigFingerprint = fingerprint;
|
|
@@ -1219,7 +1369,12 @@ export class Orchestrator {
|
|
|
1219
1369
|
// Same per-agent materialisation the initial start does. Browser setup
|
|
1220
1370
|
// is awaited before EVERY missing-session spawn: that reasserts configs
|
|
1221
1371
|
// clobbered by resume/auth injection and covers roster generation.
|
|
1222
|
-
|
|
1372
|
+
const skillExec = this.providerSkillExec(channel);
|
|
1373
|
+
if (skillExec) {
|
|
1374
|
+
await installPackageSkills(channel.taskId, missing, skillExec);
|
|
1375
|
+
} else {
|
|
1376
|
+
await installPackageSkills(channel.taskId, missing);
|
|
1377
|
+
}
|
|
1223
1378
|
if (!this.isActiveChannel(channel)) return;
|
|
1224
1379
|
|
|
1225
1380
|
// One materialization gate owns provisioning and (when enabled)
|
|
@@ -1270,7 +1425,6 @@ export class Orchestrator {
|
|
|
1270
1425
|
{
|
|
1271
1426
|
taskId: channel.taskId,
|
|
1272
1427
|
agent,
|
|
1273
|
-
containerName: channel.containerName,
|
|
1274
1428
|
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
1275
1429
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
1276
1430
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
@@ -1366,7 +1520,7 @@ export class Orchestrator {
|
|
|
1366
1520
|
}
|
|
1367
1521
|
channel.accountByAgent.set(agent.id, account.id);
|
|
1368
1522
|
noteEngineAccountUsed(account.id);
|
|
1369
|
-
return { ...
|
|
1523
|
+
return { ...account.execEnv, ...base };
|
|
1370
1524
|
}
|
|
1371
1525
|
|
|
1372
1526
|
/** Install a session with an event handler scoped to this exact generation. */
|
|
@@ -1376,6 +1530,11 @@ export class Orchestrator {
|
|
|
1376
1530
|
session: AgentSession,
|
|
1377
1531
|
): void {
|
|
1378
1532
|
channel.sessions.set(agentId, session);
|
|
1533
|
+
// Adapters may synchronously emit `error` followed by `turn_complete` (or
|
|
1534
|
+
// `exit`). Terminal classification now awaits a daemon proof, so preserve
|
|
1535
|
+
// transport emission order across that await; otherwise the later boundary
|
|
1536
|
+
// can consume/reorder prompt state while the earlier error still owns it.
|
|
1537
|
+
let eventTail: Promise<void> = Promise.resolve();
|
|
1379
1538
|
session.onEvent((event) => {
|
|
1380
1539
|
// Recycle/teardown deletes the old session before calling close().
|
|
1381
1540
|
// Adapters deliberately emit a final exit event; without this identity
|
|
@@ -1386,15 +1545,27 @@ export class Orchestrator {
|
|
|
1386
1545
|
) {
|
|
1387
1546
|
return;
|
|
1388
1547
|
}
|
|
1389
|
-
const operation =
|
|
1390
|
-
(
|
|
1548
|
+
const operation = eventTail
|
|
1549
|
+
.catch(() => {})
|
|
1550
|
+
.then(async () => {
|
|
1551
|
+
// The session may have been retired while an earlier event awaited.
|
|
1552
|
+
// Recheck identity at execution time, not only emission time.
|
|
1553
|
+
if (
|
|
1554
|
+
!this.isActiveChannel(channel) ||
|
|
1555
|
+
channel.sessions.get(agentId) !== session
|
|
1556
|
+
) {
|
|
1557
|
+
return;
|
|
1558
|
+
}
|
|
1559
|
+
await this.handleAgentEvent(channel, agentId, session, event);
|
|
1560
|
+
})
|
|
1561
|
+
.catch((err: unknown) => {
|
|
1391
1562
|
console.warn(
|
|
1392
1563
|
`[orchestrator] ${channel.taskId}/${agentId}: event handling failed: ${
|
|
1393
1564
|
err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
1394
1565
|
}`,
|
|
1395
1566
|
);
|
|
1396
|
-
}
|
|
1397
|
-
|
|
1567
|
+
});
|
|
1568
|
+
eventTail = operation;
|
|
1398
1569
|
channel.eventOperations.add(operation);
|
|
1399
1570
|
operation.then(
|
|
1400
1571
|
() => channel.eventOperations.delete(operation),
|
|
@@ -1412,7 +1583,7 @@ export class Orchestrator {
|
|
|
1412
1583
|
private createAndBindStableSession(
|
|
1413
1584
|
channel: Channel,
|
|
1414
1585
|
agentId: string,
|
|
1415
|
-
args: Parameters<AgentSessionFactory["create"]>[0],
|
|
1586
|
+
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
1416
1587
|
): Promise<AgentSession | null> {
|
|
1417
1588
|
const operation = this.createAndBindStableSessionCore(
|
|
1418
1589
|
channel,
|
|
@@ -1430,11 +1601,12 @@ export class Orchestrator {
|
|
|
1430
1601
|
private async createAndBindStableSessionCore(
|
|
1431
1602
|
channel: Channel,
|
|
1432
1603
|
agentId: string,
|
|
1433
|
-
args: Parameters<AgentSessionFactory["create"]>[0],
|
|
1604
|
+
args: Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">,
|
|
1434
1605
|
): Promise<AgentSession | null> {
|
|
1606
|
+
const environment = await this.agentEnvironment(channel);
|
|
1435
1607
|
while (this.isActiveChannel(channel)) {
|
|
1436
1608
|
const generation = channel.configGeneration;
|
|
1437
|
-
const session = await this.factory.create(args);
|
|
1609
|
+
const session = await this.factory.create({ ...args, environment });
|
|
1438
1610
|
if (!this.isActiveChannel(channel)) {
|
|
1439
1611
|
await session.close().catch(() => {});
|
|
1440
1612
|
return null;
|
|
@@ -1455,6 +1627,59 @@ export class Orchestrator {
|
|
|
1455
1627
|
return null;
|
|
1456
1628
|
}
|
|
1457
1629
|
|
|
1630
|
+
private agentEnvironment(
|
|
1631
|
+
channel: Channel,
|
|
1632
|
+
): Promise<TaskEnvironmentHandle<unknown>> {
|
|
1633
|
+
const existing = channel.agentEnvironment;
|
|
1634
|
+
if (existing) return existing;
|
|
1635
|
+
const created = (async () => {
|
|
1636
|
+
const task = getHostTask(channel.taskId);
|
|
1637
|
+
if (!task) throw new Error("task environment is not persisted");
|
|
1638
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
1639
|
+
if (!environment) throw new Error("task environment is not available");
|
|
1640
|
+
if (environment.descriptor.taskId !== channel.taskId) {
|
|
1641
|
+
throw new Error("task environment belongs to a different task");
|
|
1642
|
+
}
|
|
1643
|
+
return environment;
|
|
1644
|
+
})();
|
|
1645
|
+
const tracked = created.then((environment) => {
|
|
1646
|
+
if (this.isActiveChannel(channel)) channel.agentHandle = environment;
|
|
1647
|
+
return environment;
|
|
1648
|
+
});
|
|
1649
|
+
channel.agentEnvironment = tracked;
|
|
1650
|
+
tracked.catch(() => {
|
|
1651
|
+
if (this.isActiveChannel(channel)) {
|
|
1652
|
+
channel.agentEnvironment = null;
|
|
1653
|
+
channel.agentHandle = null;
|
|
1654
|
+
}
|
|
1655
|
+
});
|
|
1656
|
+
return tracked;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
/** Provider-neutral skill installation for a settled task environment. The
|
|
1660
|
+
* legacy path remains the default until the first environment handle has
|
|
1661
|
+
* actually been reconstructed, avoiding a new await during cold admission.
|
|
1662
|
+
*/
|
|
1663
|
+
private providerSkillExec(channel: Channel): SkillExec | undefined {
|
|
1664
|
+
const environment = channel.agentHandle;
|
|
1665
|
+
if (!environment) return undefined;
|
|
1666
|
+
return async (_container, cmd, cwd) => {
|
|
1667
|
+
const argv = cmd as [string, ...string[]];
|
|
1668
|
+
const result = await environment.exec({
|
|
1669
|
+
argv,
|
|
1670
|
+
cwd,
|
|
1671
|
+
user: "node",
|
|
1672
|
+
env: RUNTIME_AUTHORITY_ENV,
|
|
1673
|
+
timeoutMs: 120_000,
|
|
1674
|
+
maxOutputBytes: 256 * 1024,
|
|
1675
|
+
});
|
|
1676
|
+
return {
|
|
1677
|
+
status: result.exitCode,
|
|
1678
|
+
stderr: Buffer.from(result.stderr).toString("utf8"),
|
|
1679
|
+
};
|
|
1680
|
+
};
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1458
1683
|
private async startSessions(channel: Channel): Promise<boolean> {
|
|
1459
1684
|
const task = getHostTask(channel.taskId);
|
|
1460
1685
|
if (
|
|
@@ -1464,6 +1689,11 @@ export class Orchestrator {
|
|
|
1464
1689
|
) {
|
|
1465
1690
|
return false;
|
|
1466
1691
|
}
|
|
1692
|
+
// Every initial in-container materialization step shares the same boot
|
|
1693
|
+
// fence as the real agent factory. Gate before identity, auth, skills,
|
|
1694
|
+
// browser/MCP configuration, or runner creation can execute in Docker.
|
|
1695
|
+
await agentClisReady;
|
|
1696
|
+
if (!this.isActiveChannel(channel)) return false;
|
|
1467
1697
|
// Freeze the generation before any slow docker work. A roster add while
|
|
1468
1698
|
// setup awaits must reconcile under its own engine-aware browser pass,
|
|
1469
1699
|
// not slip into this factory loop under the old generation's config.
|
|
@@ -1474,7 +1704,13 @@ export class Orchestrator {
|
|
|
1474
1704
|
// task-up.sh using the creator's per-user key. Best-effort. Awaited but
|
|
1475
1705
|
// async (docker exec via dockerCli) so it doesn't block the event loop —
|
|
1476
1706
|
// this runs on every channel ensure, including each post-restart reconnect.
|
|
1477
|
-
await setupTaskGitIdentity(
|
|
1707
|
+
await setupTaskGitIdentity(
|
|
1708
|
+
channel.taskId,
|
|
1709
|
+
task.ownerName,
|
|
1710
|
+
task.ownerEmail,
|
|
1711
|
+
undefined,
|
|
1712
|
+
channel.agentHandle ?? undefined,
|
|
1713
|
+
);
|
|
1478
1714
|
if (!this.isActiveChannel(channel)) return false;
|
|
1479
1715
|
|
|
1480
1716
|
// GitHub auth for both `gh` and HTTPS Git (ADR-027): inject the token,
|
|
@@ -1482,7 +1718,11 @@ export class Orchestrator {
|
|
|
1482
1718
|
// serialized reconcile so agents cannot race their first fetch/push after
|
|
1483
1719
|
// task creation, reconnect, or host recovery.
|
|
1484
1720
|
if (task.ownerUserId) {
|
|
1485
|
-
await reconcileTaskGitAuth(channel.taskId, task.ownerUserId
|
|
1721
|
+
await reconcileTaskGitAuth(channel.taskId, task.ownerUserId, {
|
|
1722
|
+
setupDeps: {
|
|
1723
|
+
environment: () => channel.agentHandle,
|
|
1724
|
+
},
|
|
1725
|
+
});
|
|
1486
1726
|
}
|
|
1487
1727
|
|
|
1488
1728
|
// SSH signing identity, re-asserted independently of transport. Connected
|
|
@@ -1501,7 +1741,12 @@ export class Orchestrator {
|
|
|
1501
1741
|
// turn. Idempotent + best-effort (returns fast when there are none); a slow
|
|
1502
1742
|
// clone/install briefly delays start, which is acceptable for skill-bearing
|
|
1503
1743
|
// tasks. Never throws.
|
|
1504
|
-
|
|
1744
|
+
const skillExec = this.providerSkillExec(channel);
|
|
1745
|
+
if (skillExec) {
|
|
1746
|
+
await installPackageSkills(channel.taskId, initialRoster, skillExec);
|
|
1747
|
+
} else {
|
|
1748
|
+
await installPackageSkills(channel.taskId, initialRoster);
|
|
1749
|
+
}
|
|
1505
1750
|
if (!this.isActiveChannel(channel)) return false;
|
|
1506
1751
|
|
|
1507
1752
|
// ADR-053: wire the Playwright MCP browser before agents spawn — configs
|
|
@@ -1559,7 +1804,6 @@ export class Orchestrator {
|
|
|
1559
1804
|
{
|
|
1560
1805
|
taskId: channel.taskId,
|
|
1561
1806
|
agent,
|
|
1562
|
-
containerName: channel.containerName,
|
|
1563
1807
|
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
1564
1808
|
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
1565
1809
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
@@ -1608,6 +1852,61 @@ export class Orchestrator {
|
|
|
1608
1852
|
return channel ? this.ensureSessions(channel) : false;
|
|
1609
1853
|
}
|
|
1610
1854
|
|
|
1855
|
+
/** Recreate sessions lost with the daemon and replay only prompts whose
|
|
1856
|
+
* adapter boundary explicitly rejected for runtime unavailability. */
|
|
1857
|
+
async resumeRuntimeDeferredAgentWork(): Promise<void> {
|
|
1858
|
+
if (!backgroundContainerWorkAllowed()) return;
|
|
1859
|
+
await Promise.all(
|
|
1860
|
+
[...this.channels.values()].map(async (channel) => {
|
|
1861
|
+
if (
|
|
1862
|
+
!this.isActiveChannel(channel) ||
|
|
1863
|
+
channel.runtimeDeferredAgents.size === 0
|
|
1864
|
+
) {
|
|
1865
|
+
return;
|
|
1866
|
+
}
|
|
1867
|
+
try {
|
|
1868
|
+
await this.reconcileSessions(channel);
|
|
1869
|
+
this.drainRuntimeDeferredAgentWork(channel);
|
|
1870
|
+
} catch (err) {
|
|
1871
|
+
// A single ready edge fans out across channels. One broken task must
|
|
1872
|
+
// neither abort the others nor consume its own retryable prompts;
|
|
1873
|
+
// the next ordinary ensure/poll retries and drains this channel.
|
|
1874
|
+
console.warn(
|
|
1875
|
+
`[orchestrator] ${channel.taskId}: deferred runtime work remains queued: ${
|
|
1876
|
+
err instanceof Error ? err.message : String(err)
|
|
1877
|
+
}`,
|
|
1878
|
+
);
|
|
1879
|
+
}
|
|
1880
|
+
}),
|
|
1881
|
+
);
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
private drainRuntimeDeferredAgentWork(channel: Channel): void {
|
|
1885
|
+
if (
|
|
1886
|
+
!this.isActiveChannel(channel) ||
|
|
1887
|
+
!backgroundContainerWorkAllowed()
|
|
1888
|
+
) {
|
|
1889
|
+
return;
|
|
1890
|
+
}
|
|
1891
|
+
for (const agentId of [...channel.runtimeDeferredAgents]) {
|
|
1892
|
+
const session = channel.sessions.get(agentId);
|
|
1893
|
+
if (!session) continue;
|
|
1894
|
+
|
|
1895
|
+
// Clear ownership before sending. If runtime drops at the same last hop,
|
|
1896
|
+
// startPrompt's typed catch atomically re-adds the exact work.
|
|
1897
|
+
channel.runtimeDeferredAgents.delete(agentId);
|
|
1898
|
+
if (this.isSecretary(channel, agentId)) {
|
|
1899
|
+
this.startNextPendingPrompt(channel, agentId);
|
|
1900
|
+
continue;
|
|
1901
|
+
}
|
|
1902
|
+
const prompts = channel.runtimeDeferredPrompts.get(agentId) ?? [];
|
|
1903
|
+
channel.runtimeDeferredPrompts.delete(agentId);
|
|
1904
|
+
for (const prompt of prompts) {
|
|
1905
|
+
this.startPrompt(channel, agentId, session, prompt);
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1611
1910
|
private incrementActiveTurns(channel: Channel, agentId: string): void {
|
|
1612
1911
|
channel.activeTurns.set(
|
|
1613
1912
|
agentId,
|
|
@@ -1645,6 +1944,7 @@ export class Orchestrator {
|
|
|
1645
1944
|
private retireRunnerGeneration(channel: Channel, agentId: string): void {
|
|
1646
1945
|
const aborted = channel.interrupted.delete(agentId);
|
|
1647
1946
|
channel.activeTurns.delete(agentId);
|
|
1947
|
+
channel.acceptedPrompts.delete(agentId);
|
|
1648
1948
|
channel.openTurns.delete(agentId);
|
|
1649
1949
|
if (aborted) {
|
|
1650
1950
|
channel.lastPrompt.delete(agentId);
|
|
@@ -1666,9 +1966,14 @@ export class Orchestrator {
|
|
|
1666
1966
|
// ADR-076: this is the prompt actually in flight. A queued follow-up must
|
|
1667
1967
|
// not replace it or account rotation would replay the wrong turn.
|
|
1668
1968
|
channel.lastPrompt.set(agentId, prompt);
|
|
1969
|
+
const accepted = { text: prompt };
|
|
1970
|
+
const acceptedForAgent = channel.acceptedPrompts.get(agentId) ?? [];
|
|
1971
|
+
acceptedForAgent.push(accepted);
|
|
1972
|
+
channel.acceptedPrompts.set(agentId, acceptedForAgent);
|
|
1669
1973
|
this.incrementActiveTurns(channel, agentId);
|
|
1670
1974
|
void session.send(prompt).catch((err: unknown) => {
|
|
1671
1975
|
if (channel.sessions.get(agentId) === session) {
|
|
1976
|
+
this.removeAcceptedPrompt(channel, agentId, accepted);
|
|
1672
1977
|
const remaining = this.decrementActiveTurns(channel, agentId);
|
|
1673
1978
|
if (
|
|
1674
1979
|
remaining === 0 &&
|
|
@@ -1676,7 +1981,11 @@ export class Orchestrator {
|
|
|
1676
1981
|
) {
|
|
1677
1982
|
channel.lastPrompt.delete(agentId);
|
|
1678
1983
|
}
|
|
1679
|
-
|
|
1984
|
+
if (err instanceof ContainerRuntimeUnavailableError) {
|
|
1985
|
+
this.deferPromptUntilRuntimeReady(channel, agentId, prompt);
|
|
1986
|
+
} else {
|
|
1987
|
+
this.startNextPendingPrompt(channel, agentId);
|
|
1988
|
+
}
|
|
1680
1989
|
}
|
|
1681
1990
|
console.warn(
|
|
1682
1991
|
`[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
|
|
@@ -1686,6 +1995,49 @@ export class Orchestrator {
|
|
|
1686
1995
|
});
|
|
1687
1996
|
}
|
|
1688
1997
|
|
|
1998
|
+
private removeAcceptedPrompt(
|
|
1999
|
+
channel: Channel,
|
|
2000
|
+
agentId: string,
|
|
2001
|
+
accepted: AcceptedPrompt,
|
|
2002
|
+
): void {
|
|
2003
|
+
const prompts = channel.acceptedPrompts.get(agentId);
|
|
2004
|
+
if (!prompts) return;
|
|
2005
|
+
const index = prompts.indexOf(accepted);
|
|
2006
|
+
if (index >= 0) prompts.splice(index, 1);
|
|
2007
|
+
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
private completeNextAcceptedPrompt(
|
|
2011
|
+
channel: Channel,
|
|
2012
|
+
agentId: string,
|
|
2013
|
+
): void {
|
|
2014
|
+
const prompts = channel.acceptedPrompts.get(agentId);
|
|
2015
|
+
if (!prompts) return;
|
|
2016
|
+
prompts.shift();
|
|
2017
|
+
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
2018
|
+
}
|
|
2019
|
+
|
|
2020
|
+
private deferPromptUntilRuntimeReady(
|
|
2021
|
+
channel: Channel,
|
|
2022
|
+
agentId: string,
|
|
2023
|
+
prompt: string,
|
|
2024
|
+
): void {
|
|
2025
|
+
channel.runtimeDeferredAgents.add(agentId);
|
|
2026
|
+
if (this.isSecretary(channel, agentId)) {
|
|
2027
|
+
// Secretary delivery is serialized. Put the failed in-flight prompt in
|
|
2028
|
+
// front of crew replies that were accepted behind it.
|
|
2029
|
+
const pending = channel.pendingPrompts.get(agentId) ?? [];
|
|
2030
|
+
pending.unshift(prompt);
|
|
2031
|
+
channel.pendingPrompts.set(agentId, pending);
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
// Open-mode sends are intentionally concurrent. Promise rejection handlers
|
|
2035
|
+
// run in submission order, so append preserves that accepted order.
|
|
2036
|
+
const deferred = channel.runtimeDeferredPrompts.get(agentId) ?? [];
|
|
2037
|
+
deferred.push(prompt);
|
|
2038
|
+
channel.runtimeDeferredPrompts.set(agentId, deferred);
|
|
2039
|
+
}
|
|
2040
|
+
|
|
1689
2041
|
/** Start at most one queued prompt after the previous turn is terminal. */
|
|
1690
2042
|
private startNextPendingPrompt(channel: Channel, agentId: string): boolean {
|
|
1691
2043
|
if (
|
|
@@ -1804,7 +2156,11 @@ export class Orchestrator {
|
|
|
1804
2156
|
// Cancel any armed auto-retry chain so this manual attempt runs fresh
|
|
1805
2157
|
// (setup's duplicate-chain guard would otherwise no-op it).
|
|
1806
2158
|
clearRefresh(taskId);
|
|
1807
|
-
const ok = await reconcileTaskGitAuth(taskId, owner
|
|
2159
|
+
const ok = await reconcileTaskGitAuth(taskId, owner, {
|
|
2160
|
+
setupDeps: {
|
|
2161
|
+
environment: () => this.channels.get(taskId)?.agentHandle ?? null,
|
|
2162
|
+
},
|
|
2163
|
+
});
|
|
1808
2164
|
this.emitSystemNote(
|
|
1809
2165
|
taskId,
|
|
1810
2166
|
ok
|
|
@@ -1818,6 +2174,7 @@ export class Orchestrator {
|
|
|
1818
2174
|
private async handleAgentEvent(
|
|
1819
2175
|
channel: Channel,
|
|
1820
2176
|
agentId: string,
|
|
2177
|
+
session: AgentSession,
|
|
1821
2178
|
event: AgentEvent,
|
|
1822
2179
|
): Promise<void> {
|
|
1823
2180
|
if (!this.isActiveChannel(channel)) return;
|
|
@@ -1901,6 +2258,18 @@ export class Orchestrator {
|
|
|
1901
2258
|
break;
|
|
1902
2259
|
}
|
|
1903
2260
|
case "error": {
|
|
2261
|
+
if (await agentTerminationLostContainerRuntime()) {
|
|
2262
|
+
// `error` and `exit` can arrive back-to-back while the daemon probe
|
|
2263
|
+
// is pending. Only the first terminal event still owns this session.
|
|
2264
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
2265
|
+
await this.retireSessionForRuntimeLoss(
|
|
2266
|
+
channel,
|
|
2267
|
+
agentId,
|
|
2268
|
+
session,
|
|
2269
|
+
);
|
|
2270
|
+
break;
|
|
2271
|
+
}
|
|
2272
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
1904
2273
|
// The turn died with the session — drop its turn-state flags so a
|
|
1905
2274
|
// respawned session starts clean.
|
|
1906
2275
|
this.retireRunnerGeneration(channel, agentId);
|
|
@@ -1972,6 +2341,7 @@ export class Orchestrator {
|
|
|
1972
2341
|
// An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
|
|
1973
2342
|
// the cloud discards the buffer instead of handing it to peers.
|
|
1974
2343
|
const aborted = channel.interrupted.delete(agentId);
|
|
2344
|
+
this.completeNextAcceptedPrompt(channel, agentId);
|
|
1975
2345
|
const remainingTurns = this.decrementActiveTurns(channel, agentId);
|
|
1976
2346
|
channel.openTurns.delete(agentId);
|
|
1977
2347
|
// ADR-076: a turn that completed means the current account is healthy —
|
|
@@ -2005,6 +2375,16 @@ export class Orchestrator {
|
|
|
2005
2375
|
break;
|
|
2006
2376
|
}
|
|
2007
2377
|
case "exit":
|
|
2378
|
+
if (await agentTerminationLostContainerRuntime()) {
|
|
2379
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
2380
|
+
await this.retireSessionForRuntimeLoss(
|
|
2381
|
+
channel,
|
|
2382
|
+
agentId,
|
|
2383
|
+
session,
|
|
2384
|
+
);
|
|
2385
|
+
break;
|
|
2386
|
+
}
|
|
2387
|
+
if (channel.sessions.get(agentId) !== session) break;
|
|
2008
2388
|
// Same zombie hazard as the error path — a session whose process
|
|
2009
2389
|
// ended (even cleanly) can never carry another turn.
|
|
2010
2390
|
this.retireRunnerGeneration(channel, agentId);
|
|
@@ -2019,6 +2399,35 @@ export class Orchestrator {
|
|
|
2019
2399
|
}
|
|
2020
2400
|
}
|
|
2021
2401
|
|
|
2402
|
+
/** A daemon loss is infrastructure recovery, not an agent crash. Preserve
|
|
2403
|
+
* accepted work, drop the dead generation, and leave respawn/exit accounting
|
|
2404
|
+
* untouched; the runtime-ready hook recreates it deterministically. */
|
|
2405
|
+
private async retireSessionForRuntimeLoss(
|
|
2406
|
+
channel: Channel,
|
|
2407
|
+
agentId: string,
|
|
2408
|
+
session: AgentSession,
|
|
2409
|
+
): Promise<void> {
|
|
2410
|
+
const accepted = [
|
|
2411
|
+
...(channel.acceptedPrompts.get(agentId) ?? []),
|
|
2412
|
+
];
|
|
2413
|
+
const interrupted = channel.interrupted.has(agentId);
|
|
2414
|
+
this.retireRunnerGeneration(channel, agentId);
|
|
2415
|
+
channel.lastPrompt.delete(agentId);
|
|
2416
|
+
if (accepted.length > 0 && !interrupted) {
|
|
2417
|
+
for (const prompt of accepted) {
|
|
2418
|
+
this.deferPromptUntilRuntimeReady(channel, agentId, prompt.text);
|
|
2419
|
+
}
|
|
2420
|
+
} else {
|
|
2421
|
+
// Recreate even an idle session after readiness, without fabricating a
|
|
2422
|
+
// prompt or requiring another human delivery to discover the dead slot.
|
|
2423
|
+
channel.runtimeDeferredAgents.add(agentId);
|
|
2424
|
+
}
|
|
2425
|
+
channel.sessions.delete(agentId);
|
|
2426
|
+
channel.browserStaleSessions.delete(agentId);
|
|
2427
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
2428
|
+
await session.close().catch(() => {});
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2022
2431
|
// -- session recovery -----------------------------------------------------
|
|
2023
2432
|
|
|
2024
2433
|
/**
|
|
@@ -2054,7 +2463,10 @@ export class Orchestrator {
|
|
|
2054
2463
|
|
|
2055
2464
|
channel.spawning.add(agentId);
|
|
2056
2465
|
try {
|
|
2057
|
-
const restored = repairClaudeConfigInContainer(
|
|
2466
|
+
const restored = await repairClaudeConfigInContainer(
|
|
2467
|
+
channel.containerName,
|
|
2468
|
+
channel.agentHandle ?? undefined,
|
|
2469
|
+
);
|
|
2058
2470
|
|
|
2059
2471
|
// Delete the old generation before awaiting close. Its adapter emits a
|
|
2060
2472
|
// terminal event during teardown; bindSession's identity guard must see
|
|
@@ -2097,7 +2509,6 @@ export class Orchestrator {
|
|
|
2097
2509
|
const session = await this.createAndBindStableSession(channel, agentId, {
|
|
2098
2510
|
taskId: channel.taskId,
|
|
2099
2511
|
agent,
|
|
2100
|
-
containerName: channel.containerName,
|
|
2101
2512
|
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
2102
2513
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2103
2514
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
@@ -2105,7 +2516,7 @@ export class Orchestrator {
|
|
|
2105
2516
|
agentId,
|
|
2106
2517
|
),
|
|
2107
2518
|
agentEnv: boundAccount
|
|
2108
|
-
? { ...
|
|
2519
|
+
? { ...boundAccount.execEnv, ...base }
|
|
2109
2520
|
: this.accountAgentEnv(channel, agent, base),
|
|
2110
2521
|
});
|
|
2111
2522
|
if (!session) return;
|
|
@@ -2226,14 +2637,13 @@ export class Orchestrator {
|
|
|
2226
2637
|
{
|
|
2227
2638
|
taskId: channel.taskId,
|
|
2228
2639
|
agent,
|
|
2229
|
-
containerName: channel.containerName,
|
|
2230
2640
|
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
2231
2641
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2232
2642
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2233
2643
|
channel,
|
|
2234
2644
|
agentId,
|
|
2235
2645
|
),
|
|
2236
|
-
agentEnv: { ...
|
|
2646
|
+
agentEnv: { ...next.execEnv, ...base },
|
|
2237
2647
|
},
|
|
2238
2648
|
);
|
|
2239
2649
|
if (!replacement) return true;
|
|
@@ -2341,14 +2751,13 @@ export class Orchestrator {
|
|
|
2341
2751
|
replacement = await this.createAndBindStableSession(channel, agentId, {
|
|
2342
2752
|
taskId: channel.taskId,
|
|
2343
2753
|
agent,
|
|
2344
|
-
containerName: channel.containerName,
|
|
2345
2754
|
systemPreamble: channel.preambles.get(agentId) ?? "",
|
|
2346
2755
|
executionProfile: this.executionProfileFor(channel, agentId),
|
|
2347
2756
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2348
2757
|
channel,
|
|
2349
2758
|
agentId,
|
|
2350
2759
|
),
|
|
2351
|
-
agentEnv: { ...
|
|
2760
|
+
agentEnv: { ...account.execEnv, ...base },
|
|
2352
2761
|
});
|
|
2353
2762
|
if (!replacement) return true;
|
|
2354
2763
|
|
|
@@ -2487,7 +2896,10 @@ export class Orchestrator {
|
|
|
2487
2896
|
ch.browserStaleSessions.clear();
|
|
2488
2897
|
ch.browserPendingStaleSessions.clear();
|
|
2489
2898
|
ch.activeTurns.clear();
|
|
2899
|
+
ch.acceptedPrompts.clear();
|
|
2490
2900
|
ch.pendingPrompts.clear();
|
|
2901
|
+
ch.runtimeDeferredPrompts.clear();
|
|
2902
|
+
ch.runtimeDeferredAgents.clear();
|
|
2491
2903
|
ch.openTurns.clear();
|
|
2492
2904
|
ch.interrupted.clear();
|
|
2493
2905
|
ch.reconcileAgain = false;
|
|
@@ -2527,6 +2939,18 @@ export class Orchestrator {
|
|
|
2527
2939
|
return this.channelClosures.get(taskId) ?? Promise.resolve();
|
|
2528
2940
|
}
|
|
2529
2941
|
|
|
2942
|
+
/** Quarantine a live channel before recovery stops its app container.
|
|
2943
|
+
* Preserve the latest channel spec so a successful recovery can reopen the
|
|
2944
|
+
* task without waiting for another cloud message, while the ordinary close
|
|
2945
|
+
* tombstone prevents any agent process from respawning during maintenance. */
|
|
2946
|
+
async quarantineChannelForRecovery(taskId: string): Promise<void> {
|
|
2947
|
+
const currentSpec = this.channelSpecs.get(taskId);
|
|
2948
|
+
await this.closeChannel(taskId);
|
|
2949
|
+
if (currentSpec && !this.channelSpecs.has(taskId)) {
|
|
2950
|
+
this.channelSpecs.set(taskId, currentSpec);
|
|
2951
|
+
}
|
|
2952
|
+
}
|
|
2953
|
+
|
|
2530
2954
|
/** A successful task-up is the only transition that reopens a tombstoned
|
|
2531
2955
|
* task id after stop/teardown. */
|
|
2532
2956
|
allowChannel(taskId: string): void {
|
|
@@ -2553,6 +2977,9 @@ export class Orchestrator {
|
|
|
2553
2977
|
private async stopTaskWithinLifecycle(
|
|
2554
2978
|
taskId: string,
|
|
2555
2979
|
): Promise<{ ok: boolean; error?: string }> {
|
|
2980
|
+
if (!backgroundContainerWorkAllowed()) {
|
|
2981
|
+
return { ok: false, error: "container runtime is not operational" };
|
|
2982
|
+
}
|
|
2556
2983
|
const task = getHostTask(taskId);
|
|
2557
2984
|
if (!task) return { ok: false, error: "unknown task" };
|
|
2558
2985
|
const project = task.composeProject;
|
|
@@ -2568,33 +2995,27 @@ export class Orchestrator {
|
|
|
2568
2995
|
}`,
|
|
2569
2996
|
);
|
|
2570
2997
|
}
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2998
|
+
// Provider-neutral resumable stop. A row that reaches this point with
|
|
2999
|
+
// runtime resources always carries a proven locator (legacy rows cross
|
|
3000
|
+
// the adoption fence before the runtime becomes operational), so a
|
|
3001
|
+
// reconstruction failure is a stop failure, never a fallback hint. An
|
|
3002
|
+
// environment failure reopens the channel and leaves the runtime mirror
|
|
3003
|
+
// alone so cloud reconciliation cannot turn a live container into
|
|
3004
|
+
// "stopped".
|
|
3005
|
+
try {
|
|
3006
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
3007
|
+
if (!environment) {
|
|
3008
|
+
this.allowChannel(taskId);
|
|
3009
|
+
return { ok: false, error: "task has no persisted environment" };
|
|
3010
|
+
}
|
|
3011
|
+
await environment.stop();
|
|
3012
|
+
} catch (err) {
|
|
2581
3013
|
this.allowChannel(taskId);
|
|
2582
3014
|
return {
|
|
2583
3015
|
ok: false,
|
|
2584
|
-
error:
|
|
3016
|
+
error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
|
|
2585
3017
|
};
|
|
2586
3018
|
}
|
|
2587
|
-
const ids = ps.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
2588
|
-
if (ids.length > 0) {
|
|
2589
|
-
const stopped = await dockerCli(["stop", ...ids], { timeoutMs: 60_000 });
|
|
2590
|
-
if (stopped.status !== 0) {
|
|
2591
|
-
this.allowChannel(taskId);
|
|
2592
|
-
return {
|
|
2593
|
-
ok: false,
|
|
2594
|
-
error: stopped.stderr.trim().slice(0, 200) || "docker stop failed",
|
|
2595
|
-
};
|
|
2596
|
-
}
|
|
2597
|
-
}
|
|
2598
3019
|
upsertHostTask(taskId, { statusMirror: "stopped" });
|
|
2599
3020
|
clearTaskGatewayAcl(taskId);
|
|
2600
3021
|
return { ok: true };
|
|
@@ -2688,7 +3109,10 @@ export function resolveAddressing(
|
|
|
2688
3109
|
*
|
|
2689
3110
|
* Returns true when the file actually made it into the container.
|
|
2690
3111
|
*/
|
|
2691
|
-
function repairClaudeConfigInContainer(
|
|
3112
|
+
async function repairClaudeConfigInContainer(
|
|
3113
|
+
containerName: string,
|
|
3114
|
+
environment?: Pick<TaskEnvironmentHandle, "exec">,
|
|
3115
|
+
): Promise<boolean> {
|
|
2692
3116
|
// Operator can override the source path via UAI_OWNER_HOME so the
|
|
2693
3117
|
// server doesn't have to guess. Useful when Next.js's dev process
|
|
2694
3118
|
// env diverges from `os.homedir()` for any reason.
|
|
@@ -2733,25 +3157,44 @@ function repairClaudeConfigInContainer(containerName: string): boolean {
|
|
|
2733
3157
|
// bounces. `rm -f` punches through it (or no-ops on a genuine
|
|
2734
3158
|
// missing file). Then `cat >` creates the fresh inode. Both steps
|
|
2735
3159
|
// run in one `sh -c` so a partial failure leaves a clean state.
|
|
2736
|
-
const
|
|
2737
|
-
"
|
|
3160
|
+
const repairScript =
|
|
3161
|
+
"rm -f /home/node/.claude.json && cat > /home/node/.claude.json";
|
|
3162
|
+
if (environment) {
|
|
3163
|
+
const write = await environment.exec({
|
|
3164
|
+
argv: ["/bin/sh", "-c", repairScript],
|
|
3165
|
+
env: RUNTIME_AUTHORITY_ENV,
|
|
3166
|
+
stdin: content,
|
|
3167
|
+
timeoutMs: 30_000,
|
|
3168
|
+
maxOutputBytes: 64 * 1024,
|
|
3169
|
+
});
|
|
3170
|
+
if (write.exitCode !== 0) {
|
|
3171
|
+
console.error(
|
|
3172
|
+
"[orchestrator] repair: in-place write to .claude.json failed: " +
|
|
3173
|
+
Buffer.from(write.stderr).toString("utf8").trim(),
|
|
3174
|
+
);
|
|
3175
|
+
return false;
|
|
3176
|
+
}
|
|
3177
|
+
return true;
|
|
3178
|
+
}
|
|
3179
|
+
const write = await taskContainerCli(
|
|
2738
3180
|
[
|
|
2739
3181
|
"exec",
|
|
2740
3182
|
"-i",
|
|
2741
|
-
|
|
2742
|
-
|
|
3183
|
+
...runtimeAuthorityDockerExecArgs(),
|
|
3184
|
+
// The channel carries the Docker replica name; the apple container
|
|
3185
|
+
// drops the replica suffix (2026-08-18 spawn-docker-ENOENT sweep).
|
|
3186
|
+
taskContainerBackend().apple
|
|
3187
|
+
? containerName.replace(/-app-1$/, "-app")
|
|
3188
|
+
: containerName,
|
|
3189
|
+
"/bin/sh",
|
|
2743
3190
|
"-c",
|
|
2744
|
-
|
|
3191
|
+
repairScript,
|
|
2745
3192
|
],
|
|
2746
|
-
{ input: content,
|
|
3193
|
+
{ input: content.toString("utf8"), timeoutMs: 30_000 },
|
|
2747
3194
|
);
|
|
2748
3195
|
if (write.status !== 0) {
|
|
2749
|
-
const stderr =
|
|
2750
|
-
write.stderr instanceof Buffer
|
|
2751
|
-
? write.stderr.toString("utf8")
|
|
2752
|
-
: String(write.stderr ?? "");
|
|
2753
3196
|
console.error(
|
|
2754
|
-
`[orchestrator] repair: in-place write to .claude.json failed: ${stderr.trim()}`,
|
|
3197
|
+
`[orchestrator] repair: in-place write to .claude.json failed: ${write.stderr.trim()}`,
|
|
2755
3198
|
);
|
|
2756
3199
|
return false;
|
|
2757
3200
|
}
|
|
@@ -3407,19 +3850,60 @@ export function getOrchestrator(): Orchestrator {
|
|
|
3407
3850
|
const factory = mode === "real" ? realAgentFactory : mockAgentFactory;
|
|
3408
3851
|
globalForOrchestrator.__uaiOrchestrator = new Orchestrator(factory);
|
|
3409
3852
|
}
|
|
3410
|
-
if (!globalForOrchestrator.__uaiRecoverRan) {
|
|
3411
|
-
globalForOrchestrator.__uaiRecoverRan = true;
|
|
3412
|
-
// Fire-and-forget — the orchestrator is usable while recovery runs.
|
|
3413
|
-
// The promise is kept so boot steps that must NOT race recovery (the
|
|
3414
|
-
// Codex credential reinject) can sequence behind it via
|
|
3415
|
-
// recoveryComplete(). recoverRunningTasks never rejects.
|
|
3416
|
-
recoveryPromise = recoverRunningTasks();
|
|
3417
|
-
void recoveryPromise;
|
|
3418
|
-
}
|
|
3419
3853
|
return globalForOrchestrator.__uaiOrchestrator;
|
|
3420
3854
|
}
|
|
3421
3855
|
|
|
3422
|
-
|
|
3856
|
+
/** Runtime activation publication hook. Do not construct an orchestrator just
|
|
3857
|
+
* to run it: deferred work can only exist on the already-live singleton. */
|
|
3858
|
+
export function resumeRuntimeDeferredAgentWork(): void {
|
|
3859
|
+
const orchestrator = globalForOrchestrator.__uaiOrchestrator;
|
|
3860
|
+
if (!orchestrator) return;
|
|
3861
|
+
void orchestrator.resumeRuntimeDeferredAgentWork().catch((err) =>
|
|
3862
|
+
console.warn(
|
|
3863
|
+
`[orchestrator] deferred agent work resume failed: ${
|
|
3864
|
+
err instanceof Error ? err.message : String(err)
|
|
3865
|
+
}`,
|
|
3866
|
+
),
|
|
3867
|
+
);
|
|
3868
|
+
}
|
|
3869
|
+
|
|
3870
|
+
let recoveryPromise: Promise<boolean> = Promise.resolve(true);
|
|
3871
|
+
let recoveryInFlight: Promise<boolean> | null = null;
|
|
3872
|
+
|
|
3873
|
+
/** Start or join one recovery sweep. A runtime-ready recheck can call this
|
|
3874
|
+
* after the original boot window gave up, without racing an active sweep. */
|
|
3875
|
+
function beginRuntimeRecovery(
|
|
3876
|
+
lifecycle?: Orchestrator,
|
|
3877
|
+
isCurrent: () => boolean = () => true,
|
|
3878
|
+
): Promise<boolean> {
|
|
3879
|
+
// A caller that only wants the boot barrier joins it. A live-runtime request
|
|
3880
|
+
// is stronger: chain a fresh lifecycle-sequenced pass behind any active
|
|
3881
|
+
// scan. Promise-tail sequencing has no "last queue read vs finally" gap, so
|
|
3882
|
+
// a readiness request arriving in that microtask window cannot be lost.
|
|
3883
|
+
if (recoveryInFlight && !lifecycle) return recoveryInFlight;
|
|
3884
|
+
const predecessor = recoveryInFlight;
|
|
3885
|
+
const sweep = async (): Promise<boolean> => {
|
|
3886
|
+
try {
|
|
3887
|
+
return await recoverRunningTasks({ lifecycle, isCurrent });
|
|
3888
|
+
} catch (error) {
|
|
3889
|
+
console.error(
|
|
3890
|
+
"[orchestrator] recovery: unhandled failure",
|
|
3891
|
+
error instanceof Error ? error.message : error,
|
|
3892
|
+
);
|
|
3893
|
+
return false;
|
|
3894
|
+
}
|
|
3895
|
+
};
|
|
3896
|
+
let running!: Promise<boolean>;
|
|
3897
|
+
running = (predecessor
|
|
3898
|
+
? predecessor.catch(() => false).then(sweep)
|
|
3899
|
+
: sweep()
|
|
3900
|
+
).finally(() => {
|
|
3901
|
+
if (recoveryInFlight === running) recoveryInFlight = null;
|
|
3902
|
+
});
|
|
3903
|
+
recoveryInFlight = running;
|
|
3904
|
+
recoveryPromise = running;
|
|
3905
|
+
return running;
|
|
3906
|
+
}
|
|
3423
3907
|
|
|
3424
3908
|
interface TaskRecoveryBarrier {
|
|
3425
3909
|
promise: Promise<void>;
|
|
@@ -3456,13 +3940,73 @@ export function taskRecoveryComplete(taskId: string): Promise<void> {
|
|
|
3456
3940
|
|
|
3457
3941
|
/**
|
|
3458
3942
|
* Resolves once boot-time recovery has finished (starting the orchestrator —
|
|
3459
|
-
* and with it recovery — if that hasn't happened yet). Never rejects
|
|
3943
|
+
* and with it recovery — if that hasn't happened yet). Never rejects; false
|
|
3944
|
+
* means one or more active tasks remain unresolved.
|
|
3460
3945
|
*/
|
|
3461
|
-
export function recoveryComplete(
|
|
3946
|
+
export function recoveryComplete(
|
|
3947
|
+
isCurrent: () => boolean = () => true,
|
|
3948
|
+
): Promise<boolean> {
|
|
3462
3949
|
getOrchestrator();
|
|
3950
|
+
if (!globalForOrchestrator.__uaiRecoverRan) {
|
|
3951
|
+
globalForOrchestrator.__uaiRecoverRan = true;
|
|
3952
|
+
// Fire-and-forget — the orchestrator is usable while recovery runs. Main
|
|
3953
|
+
// calls this only after the selected container runtime is ready.
|
|
3954
|
+
void beginRuntimeRecovery(undefined, isCurrent);
|
|
3955
|
+
}
|
|
3463
3956
|
return recoveryPromise;
|
|
3464
3957
|
}
|
|
3465
3958
|
|
|
3959
|
+
/**
|
|
3960
|
+
* Re-run boot recovery after a container runtime transitions to ready.
|
|
3961
|
+
*
|
|
3962
|
+
* `recoveryComplete()` is a one-boot barrier; this is the explicit recovery
|
|
3963
|
+
* trigger for ADR-101 runtime rechecks. It joins an active boot sweep, or
|
|
3964
|
+
* starts a new sweep after the prior bounded retry window completed.
|
|
3965
|
+
*/
|
|
3966
|
+
export function requestRuntimeRecovery(
|
|
3967
|
+
isCurrent: () => boolean = () => true,
|
|
3968
|
+
): Promise<boolean> {
|
|
3969
|
+
const orchestrator = getOrchestrator();
|
|
3970
|
+
globalForOrchestrator.__uaiRecoverRan = true;
|
|
3971
|
+
return beginRuntimeRecovery(orchestrator, isCurrent);
|
|
3972
|
+
}
|
|
3973
|
+
|
|
3974
|
+
/** Join only work that is already mutating/reconciling runtime state. This is
|
|
3975
|
+
* used before image maintenance so an explicit recheck cannot overlap a
|
|
3976
|
+
* reconnect sweep on the shared daemon/CLI volume. */
|
|
3977
|
+
export function waitForRuntimeRecovery(): Promise<void> {
|
|
3978
|
+
return recoveryInFlight?.then(() => undefined) ?? Promise.resolve();
|
|
3979
|
+
}
|
|
3980
|
+
|
|
3981
|
+
export type RuntimeRecoveryVerdict = "clean" | "needed" | "unknown";
|
|
3982
|
+
|
|
3983
|
+
/** Cheap reconnect proof. Each active environment provider reports whether
|
|
3984
|
+
* its durable workload is still running; ordinary cloud/network flaps avoid
|
|
3985
|
+
* the heavier repair path, while an unseen runtime restart still forces full
|
|
3986
|
+
* lifecycle-sequenced recovery before commands reopen. */
|
|
3987
|
+
export async function runtimeRecoveryVerdict(): Promise<RuntimeRecoveryVerdict> {
|
|
3988
|
+
const activeRows = getDb()
|
|
3989
|
+
.select()
|
|
3990
|
+
.from(schema.hostTasks)
|
|
3991
|
+
.where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
|
|
3992
|
+
.all();
|
|
3993
|
+
for (const task of activeRows) {
|
|
3994
|
+
try {
|
|
3995
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
3996
|
+
// A pre-provision row has no provider-owned runtime to inventory.
|
|
3997
|
+
if (environment === null) continue;
|
|
3998
|
+
const status = await environment.status();
|
|
3999
|
+
if (status.state === "unknown") return "unknown";
|
|
4000
|
+
if (status.state !== "running") return "needed";
|
|
4001
|
+
} catch {
|
|
4002
|
+
// An unavailable provider, malformed locator, or backend mismatch is
|
|
4003
|
+
// unknown—not evidence that the selected runtime is clean or absent.
|
|
4004
|
+
return "unknown";
|
|
4005
|
+
}
|
|
4006
|
+
}
|
|
4007
|
+
return "clean";
|
|
4008
|
+
}
|
|
4009
|
+
|
|
3466
4010
|
// ---------------------------------------------------------------------------
|
|
3467
4011
|
// Boot-time recovery
|
|
3468
4012
|
//
|
|
@@ -3481,8 +4025,10 @@ export function recoveryComplete(): Promise<void> {
|
|
|
3481
4025
|
// id (task-up is idempotent).
|
|
3482
4026
|
// container gone, worktree gone → mark `error`. Data loss.
|
|
3483
4027
|
//
|
|
3484
|
-
// Idempotent per process —
|
|
3485
|
-
// so a Next.js HMR rebuild
|
|
4028
|
+
// Idempotent per process — recoveryComplete/requestRuntimeRecovery mark
|
|
4029
|
+
// `__uaiRecoverRan` on globalThis so a Next.js HMR rebuild cannot trigger the
|
|
4030
|
+
// boot sweep twice. Merely constructing the orchestrator is deliberately not
|
|
4031
|
+
// enough: a degraded host must not probe an unrelated ambient daemon.
|
|
3486
4032
|
// ---------------------------------------------------------------------------
|
|
3487
4033
|
|
|
3488
4034
|
/**
|
|
@@ -3529,63 +4075,1207 @@ async function dockerListContainersByLabel(
|
|
|
3529
4075
|
.filter((x): x is DockerPs => x !== null);
|
|
3530
4076
|
}
|
|
3531
4077
|
|
|
3532
|
-
|
|
4078
|
+
type DockerStartResult = "started" | "failed" | "unreachable";
|
|
4079
|
+
|
|
4080
|
+
async function dockerDaemonReachable(): Promise<boolean> {
|
|
4081
|
+
const probe = await dockerCli(["info", "--format", "{{.ServerVersion}}"], {
|
|
4082
|
+
timeoutMs: 10_000,
|
|
4083
|
+
});
|
|
4084
|
+
return probe.status === 0 && probe.stdout.trim().length > 0;
|
|
4085
|
+
}
|
|
4086
|
+
|
|
4087
|
+
async function dockerStart(containerName: string): Promise<DockerStartResult> {
|
|
3533
4088
|
// Starting a big dev container can legitimately take a while — give it far
|
|
3534
4089
|
// more than dockerCli's 30s default before declaring the task stopped.
|
|
3535
4090
|
const res = await dockerCli(["start", containerName], { timeoutMs: 120_000 });
|
|
3536
|
-
if (res.status
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
}
|
|
3544
|
-
|
|
3545
|
-
/**
|
|
3546
|
-
* Repair the shared node data root before any recovered-container init.
|
|
3547
|
-
* Older task-up versions created the OpenCode leaf as root and accidentally
|
|
3548
|
-
* left this parent root-owned, which prevented code-server from creating its
|
|
3549
|
-
* managed User profile. `install -d` creates or repairs the exact directory
|
|
3550
|
-
* without recursively changing unrelated application state beneath it.
|
|
3551
|
-
*/
|
|
3552
|
-
async function repairNodeDataRoot(containerName: string): Promise<boolean> {
|
|
3553
|
-
const res = await dockerCli([
|
|
3554
|
-
"exec",
|
|
3555
|
-
"-u",
|
|
3556
|
-
"root",
|
|
4091
|
+
if (res.status === 0) return "started";
|
|
4092
|
+
// A timeout is ambiguous: Docker may have completed the start after our CLI
|
|
4093
|
+
// was killed. Inspect the exact container, then prove the daemon still
|
|
4094
|
+
// exists before deciding a non-inspectable container is a terminal failure.
|
|
4095
|
+
const inspected = await dockerCli([
|
|
4096
|
+
"inspect",
|
|
4097
|
+
"--format",
|
|
4098
|
+
"{{.State.Running}}",
|
|
3557
4099
|
containerName,
|
|
3558
|
-
"/usr/bin/install",
|
|
3559
|
-
"-d",
|
|
3560
|
-
"-o",
|
|
3561
|
-
"node",
|
|
3562
|
-
"-g",
|
|
3563
|
-
"node",
|
|
3564
|
-
"-m",
|
|
3565
|
-
"0755",
|
|
3566
|
-
"/home/node/.local/share",
|
|
3567
4100
|
]);
|
|
3568
|
-
if (
|
|
3569
|
-
|
|
3570
|
-
`[orchestrator] recovery: ${containerName} could not repair /home/node/.local/share ownership: ${res.stderr.trim() || `docker exec exited ${String(res.status)}`}`,
|
|
3571
|
-
);
|
|
3572
|
-
return false;
|
|
4101
|
+
if (inspected.status === 0 && inspected.stdout.trim() === "true") {
|
|
4102
|
+
return "started";
|
|
3573
4103
|
}
|
|
3574
|
-
|
|
4104
|
+
if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
|
|
4105
|
+
return "unreachable";
|
|
4106
|
+
}
|
|
4107
|
+
console.error(
|
|
4108
|
+
`[orchestrator] docker start ${containerName} failed: ${res.stderr.trim()}`,
|
|
4109
|
+
);
|
|
4110
|
+
return "failed";
|
|
4111
|
+
}
|
|
4112
|
+
|
|
4113
|
+
const RUNTIME_WRITER_LIST_FORMAT =
|
|
4114
|
+
'{{json .Names}}\t{{.Label "com.docker.compose.project"}}\t{{.Label "com.docker.compose.service"}}';
|
|
4115
|
+
const RUNTIME_WRITER_INSPECT_FORMAT = [
|
|
4116
|
+
"{{json .Id}}",
|
|
4117
|
+
"{{json .Name}}",
|
|
4118
|
+
"{{json .State.Running}}",
|
|
4119
|
+
"{{json .State.Restarting}}",
|
|
4120
|
+
"{{json .State.Paused}}",
|
|
4121
|
+
"{{json .HostConfig.RestartPolicy.Name}}",
|
|
4122
|
+
'{{json (index .Config.Labels "com.docker.compose.project")}}',
|
|
4123
|
+
'{{json (index .Config.Labels "com.docker.compose.service")}}',
|
|
4124
|
+
"{{json .Mounts}}",
|
|
4125
|
+
"{{json .Config.Env}}",
|
|
4126
|
+
].join("\n");
|
|
4127
|
+
const RUNTIME_WRITER_CONTAINER_ID_RE = /^[0-9a-f]{64}$/;
|
|
4128
|
+
const RUNTIME_WRITER_SCAN_OUTPUT_BYTES = 2 * 1024 * 1024;
|
|
4129
|
+
const TASK_RUNTIME_PATH = RUNTIME_AUTHORITY_ENV.PATH;
|
|
4130
|
+
const TASK_OWNED_ENV = new Map<string, string>(
|
|
4131
|
+
Object.entries(RUNTIME_AUTHORITY_ENV),
|
|
4132
|
+
);
|
|
4133
|
+
const NPM_CONFIG_ENV_PREFIX = "npm_config_";
|
|
4134
|
+
function normalizeNpmConfigEnvKey(key: string): string | undefined {
|
|
4135
|
+
const folded = key.toLowerCase();
|
|
4136
|
+
if (!folded.startsWith(NPM_CONFIG_ENV_PREFIX)) return undefined;
|
|
4137
|
+
// npm converts underscores in the option suffix to hyphens. Docker can
|
|
4138
|
+
// preserve spellings a shell cannot assign (for example LOGS-DIR), so both
|
|
4139
|
+
// forms must be treated as the same owned option during immutable-env proof.
|
|
4140
|
+
return (
|
|
4141
|
+
NPM_CONFIG_ENV_PREFIX +
|
|
4142
|
+
folded.slice(NPM_CONFIG_ENV_PREFIX.length).replaceAll("_", "-")
|
|
4143
|
+
);
|
|
4144
|
+
}
|
|
4145
|
+
const TASK_OWNED_NORMALIZED_NPM_ENV_KEYS = new Set(
|
|
4146
|
+
[...TASK_OWNED_ENV.keys()]
|
|
4147
|
+
.map(normalizeNpmConfigEnvKey)
|
|
4148
|
+
.filter((key): key is string => key !== undefined),
|
|
4149
|
+
);
|
|
4150
|
+
|
|
4151
|
+
interface RuntimeWriterCandidate {
|
|
4152
|
+
taskId: string;
|
|
4153
|
+
composeProject: string;
|
|
4154
|
+
containerName: string;
|
|
4155
|
+
}
|
|
4156
|
+
|
|
4157
|
+
interface RuntimeWriterMetadata extends RuntimeWriterCandidate {
|
|
4158
|
+
containerId: string;
|
|
4159
|
+
running: boolean;
|
|
4160
|
+
restarting: boolean;
|
|
4161
|
+
paused: boolean;
|
|
4162
|
+
restartPolicy: string;
|
|
4163
|
+
sharedMount: "absent" | "read-only" | "writable" | "invalid";
|
|
4164
|
+
taskEnvironment: "current" | "invalid";
|
|
4165
|
+
}
|
|
4166
|
+
|
|
4167
|
+
type RuntimeWriterInspection =
|
|
4168
|
+
| { kind: "ready"; metadata: RuntimeWriterMetadata }
|
|
4169
|
+
| { kind: "failed"; detail: string }
|
|
4170
|
+
| { kind: "unreachable" };
|
|
4171
|
+
|
|
4172
|
+
export type TaskRuntimeContainerInspection =
|
|
4173
|
+
| {
|
|
4174
|
+
kind: "ready";
|
|
4175
|
+
containerId: string;
|
|
4176
|
+
running: boolean;
|
|
4177
|
+
restarting: boolean;
|
|
4178
|
+
paused: boolean;
|
|
4179
|
+
sharedMount: RuntimeWriterMetadata["sharedMount"];
|
|
4180
|
+
taskEnvironment: RuntimeWriterMetadata["taskEnvironment"];
|
|
4181
|
+
}
|
|
4182
|
+
| { kind: "failed"; detail: string }
|
|
4183
|
+
| { kind: "unreachable" };
|
|
4184
|
+
|
|
4185
|
+
export interface WritableRuntimeQuarantineResult {
|
|
4186
|
+
verdict: "ready" | "retry" | "stale";
|
|
4187
|
+
/** Known rows whose local `stopped` verdict must be published or replayed. */
|
|
4188
|
+
stoppedTaskIds: string[];
|
|
4189
|
+
}
|
|
4190
|
+
|
|
4191
|
+
function runtimeWriterCandidate(
|
|
4192
|
+
rawName: unknown,
|
|
4193
|
+
rawProject: string,
|
|
4194
|
+
rawService: string,
|
|
4195
|
+
): RuntimeWriterCandidate | "unrelated" | "invalid" {
|
|
4196
|
+
if (!rawProject.startsWith("task-")) return "unrelated";
|
|
4197
|
+
const taskId = rawProject.slice("task-".length);
|
|
4198
|
+
if (!isSafeHostTaskId(taskId)) return "invalid";
|
|
4199
|
+
if (rawService !== "app" || typeof rawName !== "string") return "invalid";
|
|
4200
|
+
const containerName = `${rawProject}-app-1`;
|
|
4201
|
+
if (rawName !== containerName) return "invalid";
|
|
4202
|
+
return { taskId, composeProject: rawProject, containerName };
|
|
4203
|
+
}
|
|
4204
|
+
|
|
4205
|
+
/**
|
|
4206
|
+
* Enumerate the Docker authority, not only SQLite. A crashed task-down or an
|
|
4207
|
+
* older host can leave a Uai app container behind after its row changes, and
|
|
4208
|
+
* that orphan can still mutate the host-wide runtime volume.
|
|
4209
|
+
*/
|
|
4210
|
+
async function listRuntimeWriterCandidates(): Promise<
|
|
4211
|
+
| { kind: "ready"; candidates: RuntimeWriterCandidate[]; ambiguous: boolean }
|
|
4212
|
+
| { kind: "unreachable" }
|
|
4213
|
+
> {
|
|
4214
|
+
const listed = await dockerCli(
|
|
4215
|
+
[
|
|
4216
|
+
"ps",
|
|
4217
|
+
"--all",
|
|
4218
|
+
"--filter",
|
|
4219
|
+
"label=com.docker.compose.project",
|
|
4220
|
+
"--filter",
|
|
4221
|
+
"label=com.docker.compose.service=app",
|
|
4222
|
+
"--format",
|
|
4223
|
+
RUNTIME_WRITER_LIST_FORMAT,
|
|
4224
|
+
],
|
|
4225
|
+
{ timeoutMs: 30_000, maxOutputBytes: RUNTIME_WRITER_SCAN_OUTPUT_BYTES },
|
|
4226
|
+
);
|
|
4227
|
+
if (listed.status !== 0 || listed.outputTruncated) {
|
|
4228
|
+
return { kind: "unreachable" };
|
|
4229
|
+
}
|
|
4230
|
+
|
|
4231
|
+
const candidates = new Map<string, RuntimeWriterCandidate>();
|
|
4232
|
+
let ambiguous = false;
|
|
4233
|
+
for (const rawLine of listed.stdout.split(/\r?\n/)) {
|
|
4234
|
+
if (!rawLine) continue;
|
|
4235
|
+
const fields = rawLine.split("\t");
|
|
4236
|
+
if (fields.length !== 3) {
|
|
4237
|
+
ambiguous = true;
|
|
4238
|
+
continue;
|
|
4239
|
+
}
|
|
4240
|
+
let name: unknown;
|
|
4241
|
+
try {
|
|
4242
|
+
name = JSON.parse(fields[0]!);
|
|
4243
|
+
} catch {
|
|
4244
|
+
ambiguous = true;
|
|
4245
|
+
continue;
|
|
4246
|
+
}
|
|
4247
|
+
const candidate = runtimeWriterCandidate(name, fields[1]!, fields[2]!);
|
|
4248
|
+
if (candidate === "unrelated") continue;
|
|
4249
|
+
if (candidate === "invalid") {
|
|
4250
|
+
ambiguous = true;
|
|
4251
|
+
continue;
|
|
4252
|
+
}
|
|
4253
|
+
if (candidates.has(candidate.containerName)) {
|
|
4254
|
+
ambiguous = true;
|
|
4255
|
+
continue;
|
|
4256
|
+
}
|
|
4257
|
+
candidates.set(candidate.containerName, candidate);
|
|
4258
|
+
}
|
|
4259
|
+
return { kind: "ready", candidates: [...candidates.values()], ambiguous };
|
|
4260
|
+
}
|
|
4261
|
+
|
|
4262
|
+
async function inspectRuntimeWriterContainer(
|
|
4263
|
+
candidate: RuntimeWriterCandidate,
|
|
4264
|
+
target: string = candidate.containerName,
|
|
4265
|
+
): Promise<RuntimeWriterInspection> {
|
|
4266
|
+
const inspected = await dockerCli(
|
|
4267
|
+
["inspect", "--format", RUNTIME_WRITER_INSPECT_FORMAT, target],
|
|
4268
|
+
{ timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
|
|
4269
|
+
);
|
|
4270
|
+
if (inspected.status !== 0 || inspected.outputTruncated) {
|
|
4271
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
4272
|
+
return {
|
|
4273
|
+
kind: "failed",
|
|
4274
|
+
detail:
|
|
4275
|
+
inspected.stderr.trim() ||
|
|
4276
|
+
`could not inspect Uai app container ${candidate.containerName}`,
|
|
4277
|
+
};
|
|
4278
|
+
}
|
|
4279
|
+
|
|
4280
|
+
const fields = inspected.stdout.trimEnd().split(/\r?\n/);
|
|
4281
|
+
if (fields.length !== 10) {
|
|
4282
|
+
return { kind: "failed", detail: "Docker returned malformed app metadata" };
|
|
4283
|
+
}
|
|
4284
|
+
let parsed: unknown[];
|
|
4285
|
+
try {
|
|
4286
|
+
parsed = fields.map((field) => JSON.parse(field));
|
|
4287
|
+
} catch {
|
|
4288
|
+
return { kind: "failed", detail: "Docker returned malformed app metadata" };
|
|
4289
|
+
}
|
|
4290
|
+
const [
|
|
4291
|
+
containerId,
|
|
4292
|
+
containerName,
|
|
4293
|
+
running,
|
|
4294
|
+
restarting,
|
|
4295
|
+
paused,
|
|
4296
|
+
restartPolicy,
|
|
4297
|
+
composeProject,
|
|
4298
|
+
composeService,
|
|
4299
|
+
mounts,
|
|
4300
|
+
environment,
|
|
4301
|
+
] = parsed;
|
|
4302
|
+
if (
|
|
4303
|
+
typeof containerId !== "string" ||
|
|
4304
|
+
!RUNTIME_WRITER_CONTAINER_ID_RE.test(containerId) ||
|
|
4305
|
+
containerName !== `/${candidate.containerName}` ||
|
|
4306
|
+
typeof running !== "boolean" ||
|
|
4307
|
+
typeof restarting !== "boolean" ||
|
|
4308
|
+
typeof paused !== "boolean" ||
|
|
4309
|
+
typeof restartPolicy !== "string" ||
|
|
4310
|
+
composeProject !== candidate.composeProject ||
|
|
4311
|
+
composeService !== "app" ||
|
|
4312
|
+
!Array.isArray(mounts)
|
|
4313
|
+
) {
|
|
4314
|
+
return {
|
|
4315
|
+
kind: "failed",
|
|
4316
|
+
detail: `Docker returned ambiguous identity/state metadata for ${candidate.containerName}`,
|
|
4317
|
+
};
|
|
4318
|
+
}
|
|
4319
|
+
|
|
4320
|
+
if (
|
|
4321
|
+
!mounts.every(
|
|
4322
|
+
(mount) => mount && typeof mount === "object" && !Array.isArray(mount),
|
|
4323
|
+
)
|
|
4324
|
+
) {
|
|
4325
|
+
return {
|
|
4326
|
+
kind: "failed",
|
|
4327
|
+
detail: `Docker returned malformed mount metadata for ${candidate.containerName}`,
|
|
4328
|
+
};
|
|
4329
|
+
}
|
|
4330
|
+
const mountRecords = mounts as Array<Record<string, unknown>>;
|
|
4331
|
+
const asdfMounts = mountRecords.filter(
|
|
4332
|
+
(mount) => mount.Destination === "/opt/asdf-data",
|
|
4333
|
+
);
|
|
4334
|
+
const namedSharedMounts = mountRecords.filter(
|
|
4335
|
+
(mount) => mount.Name === "uai-asdf-data",
|
|
4336
|
+
);
|
|
4337
|
+
let sharedMount: RuntimeWriterMetadata["sharedMount"];
|
|
4338
|
+
if (
|
|
4339
|
+
namedSharedMounts.some(
|
|
4340
|
+
(mount) => mount.Type === "volume" && mount.RW === true,
|
|
4341
|
+
)
|
|
4342
|
+
) {
|
|
4343
|
+
// The volume name, not its destination, is the shared writer authority.
|
|
4344
|
+
// An orphan can mount it at an arbitrary path and still race maintenance.
|
|
4345
|
+
sharedMount = "writable";
|
|
4346
|
+
} else if (
|
|
4347
|
+
namedSharedMounts.some(
|
|
4348
|
+
(mount) => mount.Type !== "volume" || typeof mount.RW !== "boolean",
|
|
4349
|
+
)
|
|
4350
|
+
) {
|
|
4351
|
+
sharedMount = "invalid";
|
|
4352
|
+
} else if (asdfMounts.length === 0) {
|
|
4353
|
+
sharedMount = "absent";
|
|
4354
|
+
} else if (
|
|
4355
|
+
asdfMounts.length === 1 &&
|
|
4356
|
+
namedSharedMounts.length === 1 &&
|
|
4357
|
+
asdfMounts[0]!.Type === "volume" &&
|
|
4358
|
+
asdfMounts[0]!.Name === "uai-asdf-data" &&
|
|
4359
|
+
asdfMounts[0]!.RW === false
|
|
4360
|
+
) {
|
|
4361
|
+
sharedMount = "read-only";
|
|
4362
|
+
} else {
|
|
4363
|
+
sharedMount = "invalid";
|
|
4364
|
+
}
|
|
4365
|
+
const environmentIsStringArray =
|
|
4366
|
+
Array.isArray(environment) &&
|
|
4367
|
+
environment.every((entry) => typeof entry === "string");
|
|
4368
|
+
const environmentEntries = environmentIsStringArray
|
|
4369
|
+
? (environment as string[])
|
|
4370
|
+
: [];
|
|
4371
|
+
const exactValues = new Map<string, string[]>();
|
|
4372
|
+
let taskEnvironment: RuntimeWriterMetadata["taskEnvironment"] =
|
|
4373
|
+
environmentIsStringArray ? "current" : "invalid";
|
|
4374
|
+
for (const entry of environmentEntries) {
|
|
4375
|
+
const divider = entry.indexOf("=");
|
|
4376
|
+
const key = divider < 0 ? entry : entry.slice(0, divider);
|
|
4377
|
+
const value = divider < 0 ? "" : entry.slice(divider + 1);
|
|
4378
|
+
if (divider < 1) taskEnvironment = "invalid";
|
|
4379
|
+
const normalizedNpmKey = normalizeNpmConfigEnvKey(key);
|
|
4380
|
+
if (
|
|
4381
|
+
normalizedNpmKey !== undefined &&
|
|
4382
|
+
TASK_OWNED_NORMALIZED_NPM_ENV_KEYS.has(normalizedNpmKey) &&
|
|
4383
|
+
!TASK_OWNED_ENV.has(key)
|
|
4384
|
+
) {
|
|
4385
|
+
taskEnvironment = "invalid";
|
|
4386
|
+
}
|
|
4387
|
+
const values = exactValues.get(key) ?? [];
|
|
4388
|
+
values.push(value);
|
|
4389
|
+
exactValues.set(key, values);
|
|
4390
|
+
}
|
|
4391
|
+
for (const [key, expected] of TASK_OWNED_ENV) {
|
|
4392
|
+
const values = exactValues.get(key);
|
|
4393
|
+
if (values?.length !== 1 || values[0] !== expected) {
|
|
4394
|
+
taskEnvironment = "invalid";
|
|
4395
|
+
}
|
|
4396
|
+
}
|
|
4397
|
+
// Keep an explicit assertion for the trust-precedence property even though
|
|
4398
|
+
// PATH is also part of TASK_OWNED_ENV above.
|
|
4399
|
+
if (exactValues.get("PATH")?.[0] !== TASK_RUNTIME_PATH) taskEnvironment = "invalid";
|
|
4400
|
+
return {
|
|
4401
|
+
kind: "ready",
|
|
4402
|
+
metadata: {
|
|
4403
|
+
...candidate,
|
|
4404
|
+
containerId,
|
|
4405
|
+
running,
|
|
4406
|
+
restarting,
|
|
4407
|
+
paused,
|
|
4408
|
+
restartPolicy,
|
|
4409
|
+
sharedMount,
|
|
4410
|
+
taskEnvironment,
|
|
4411
|
+
},
|
|
4412
|
+
};
|
|
4413
|
+
}
|
|
4414
|
+
|
|
4415
|
+
/** Exact single-task form used by duplicate taskUp admission. */
|
|
4416
|
+
export async function inspectTaskRuntimeContainer(
|
|
4417
|
+
taskId: string,
|
|
4418
|
+
composeProject: string,
|
|
4419
|
+
): Promise<TaskRuntimeContainerInspection> {
|
|
4420
|
+
if (!isSafeHostTaskId(taskId) || composeProject !== `task-${taskId}`) {
|
|
4421
|
+
return { kind: "failed", detail: "invalid task/Compose runtime identity" };
|
|
4422
|
+
}
|
|
4423
|
+
const inspected = await inspectRuntimeWriterContainer({
|
|
4424
|
+
taskId,
|
|
4425
|
+
composeProject,
|
|
4426
|
+
containerName: `${composeProject}-app-1`,
|
|
4427
|
+
});
|
|
4428
|
+
if (inspected.kind !== "ready") return inspected;
|
|
4429
|
+
return {
|
|
4430
|
+
kind: "ready",
|
|
4431
|
+
containerId: inspected.metadata.containerId,
|
|
4432
|
+
running: inspected.metadata.running,
|
|
4433
|
+
restarting: inspected.metadata.restarting,
|
|
4434
|
+
paused: inspected.metadata.paused,
|
|
4435
|
+
sharedMount: inspected.metadata.sharedMount,
|
|
4436
|
+
taskEnvironment: inspected.metadata.taskEnvironment,
|
|
4437
|
+
};
|
|
4438
|
+
}
|
|
4439
|
+
|
|
4440
|
+
function restartPolicyDisabled(name: string): boolean {
|
|
4441
|
+
return name === "" || name === "no";
|
|
4442
|
+
}
|
|
4443
|
+
|
|
4444
|
+
async function stopRuntimeWriter(
|
|
4445
|
+
metadata: RuntimeWriterMetadata,
|
|
4446
|
+
): Promise<"stopped" | "failed" | "unreachable"> {
|
|
4447
|
+
// A preserved legacy container must not spring back up under Docker's own
|
|
4448
|
+
// restart policy after the host has declared it stopped. Mount mode is
|
|
4449
|
+
// immutable, so only explicit taskUp may recreate it with the L5 contract.
|
|
4450
|
+
await dockerCli(["update", "--restart=no", metadata.containerId], {
|
|
4451
|
+
timeoutMs: 30_000,
|
|
4452
|
+
maxOutputBytes: 1024 * 1024,
|
|
4453
|
+
});
|
|
4454
|
+
let current = await inspectRuntimeWriterContainer(
|
|
4455
|
+
metadata,
|
|
4456
|
+
metadata.containerId,
|
|
4457
|
+
);
|
|
4458
|
+
if (current.kind !== "ready") return current.kind;
|
|
4459
|
+
if (
|
|
4460
|
+
current.metadata.containerId !== metadata.containerId ||
|
|
4461
|
+
!restartPolicyDisabled(current.metadata.restartPolicy)
|
|
4462
|
+
) {
|
|
4463
|
+
return "failed";
|
|
4464
|
+
}
|
|
4465
|
+
|
|
4466
|
+
if (current.metadata.paused) {
|
|
4467
|
+
// Docker refuses ordinary stop/kill operations for some paused states.
|
|
4468
|
+
// The restart policy is already disabled; unpause the exact immutable id,
|
|
4469
|
+
// re-prove identity/policy, then immediately enter the bounded stop path.
|
|
4470
|
+
await dockerCli(["unpause", metadata.containerId], {
|
|
4471
|
+
timeoutMs: 15_000,
|
|
4472
|
+
maxOutputBytes: 1024 * 1024,
|
|
4473
|
+
});
|
|
4474
|
+
current = await inspectRuntimeWriterContainer(
|
|
4475
|
+
metadata,
|
|
4476
|
+
metadata.containerId,
|
|
4477
|
+
);
|
|
4478
|
+
if (current.kind !== "ready") return current.kind;
|
|
4479
|
+
if (
|
|
4480
|
+
current.metadata.containerId !== metadata.containerId ||
|
|
4481
|
+
!restartPolicyDisabled(current.metadata.restartPolicy) ||
|
|
4482
|
+
current.metadata.paused
|
|
4483
|
+
) {
|
|
4484
|
+
return "failed";
|
|
4485
|
+
}
|
|
4486
|
+
}
|
|
4487
|
+
|
|
4488
|
+
if (!current.metadata.running && !current.metadata.restarting) {
|
|
4489
|
+
return "stopped";
|
|
4490
|
+
}
|
|
4491
|
+
await dockerCli(["stop", "--time", "10", metadata.containerId], {
|
|
4492
|
+
timeoutMs: 30_000,
|
|
4493
|
+
maxOutputBytes: 1024 * 1024,
|
|
4494
|
+
});
|
|
4495
|
+
current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
|
|
4496
|
+
if (current.kind !== "ready") return current.kind;
|
|
4497
|
+
if (
|
|
4498
|
+
current.metadata.containerId === metadata.containerId &&
|
|
4499
|
+
restartPolicyDisabled(current.metadata.restartPolicy) &&
|
|
4500
|
+
!current.metadata.running &&
|
|
4501
|
+
!current.metadata.restarting
|
|
4502
|
+
) {
|
|
4503
|
+
return "stopped";
|
|
4504
|
+
}
|
|
4505
|
+
|
|
4506
|
+
await dockerCli(["kill", metadata.containerId], {
|
|
4507
|
+
timeoutMs: 15_000,
|
|
4508
|
+
maxOutputBytes: 1024 * 1024,
|
|
4509
|
+
});
|
|
4510
|
+
current = await inspectRuntimeWriterContainer(metadata, metadata.containerId);
|
|
4511
|
+
if (current.kind !== "ready") return current.kind;
|
|
4512
|
+
return current.metadata.containerId === metadata.containerId &&
|
|
4513
|
+
restartPolicyDisabled(current.metadata.restartPolicy) &&
|
|
4514
|
+
!current.metadata.running &&
|
|
4515
|
+
!current.metadata.restarting
|
|
4516
|
+
? "stopped"
|
|
4517
|
+
: "failed";
|
|
4518
|
+
}
|
|
4519
|
+
|
|
4520
|
+
/** Stop the exact container previously inspected by taskUp. Re-inspection by
|
|
4521
|
+
* immutable id closes the name-reuse gap before any shared-volume writer. */
|
|
4522
|
+
export async function quarantineTaskRuntimeContainer(options: {
|
|
4523
|
+
taskId: string;
|
|
4524
|
+
composeProject: string;
|
|
4525
|
+
expectedContainerId: string;
|
|
4526
|
+
}): Promise<"stopped" | "failed" | "unreachable"> {
|
|
4527
|
+
if (
|
|
4528
|
+
!isSafeHostTaskId(options.taskId) ||
|
|
4529
|
+
options.composeProject !== `task-${options.taskId}` ||
|
|
4530
|
+
!RUNTIME_WRITER_CONTAINER_ID_RE.test(options.expectedContainerId)
|
|
4531
|
+
) {
|
|
4532
|
+
return "failed";
|
|
4533
|
+
}
|
|
4534
|
+
const candidate: RuntimeWriterCandidate = {
|
|
4535
|
+
taskId: options.taskId,
|
|
4536
|
+
composeProject: options.composeProject,
|
|
4537
|
+
containerName: `${options.composeProject}-app-1`,
|
|
4538
|
+
};
|
|
4539
|
+
const inspected = await inspectRuntimeWriterContainer(
|
|
4540
|
+
candidate,
|
|
4541
|
+
options.expectedContainerId,
|
|
4542
|
+
);
|
|
4543
|
+
if (inspected.kind !== "ready") return inspected.kind;
|
|
4544
|
+
if (inspected.metadata.containerId !== options.expectedContainerId) {
|
|
4545
|
+
return "failed";
|
|
4546
|
+
}
|
|
4547
|
+
return stopRuntimeWriter(inspected.metadata);
|
|
4548
|
+
}
|
|
4549
|
+
|
|
4550
|
+
/**
|
|
4551
|
+
* L5 rolling fence. It must complete before the first standard-image/runtime
|
|
4552
|
+
* volume writer of an activation generation. New task containers mount the
|
|
4553
|
+
* shared volume read-only; every preserved legacy writer is tombstoned and
|
|
4554
|
+
* stopped, while an exact RO container remains eligible for recovery.
|
|
4555
|
+
*/
|
|
4556
|
+
export async function quarantineWritableRuntimeContainers(options: {
|
|
4557
|
+
lifecycle: Orchestrator;
|
|
4558
|
+
isCurrent?: () => boolean;
|
|
4559
|
+
/** `contract` (default) sweeps only containers whose shared-volume
|
|
4560
|
+
* attachment violates the current contract. `volume-holders` sweeps EVERY
|
|
4561
|
+
* task container attached to the shared volume — the activation driver's
|
|
4562
|
+
* non-writer escape when the volume needs a repair the holders block
|
|
4563
|
+
* (ADR-108; apple only, a no-op on Docker). */
|
|
4564
|
+
scope?: "contract" | "volume-holders";
|
|
4565
|
+
}): Promise<WritableRuntimeQuarantineResult> {
|
|
4566
|
+
// Dispatch on the PIN: this sweep runs inside activation's
|
|
4567
|
+
// beforeMaintenance, where the public state is still `checking` and the
|
|
4568
|
+
// ready-gated identity is null — the docker fallback then retries forever
|
|
4569
|
+
// against a daemon that does not exist on a Docker-free Mac.
|
|
4570
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
4571
|
+
return quarantineWritableAppleRuntimeContainers(options);
|
|
4572
|
+
}
|
|
4573
|
+
const isCurrent = options.isCurrent ?? (() => true);
|
|
4574
|
+
const stoppedTaskIds = new Set<string>();
|
|
4575
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
|
|
4576
|
+
|
|
4577
|
+
const listed = await listRuntimeWriterCandidates();
|
|
4578
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
|
|
4579
|
+
if (listed.kind === "unreachable") {
|
|
4580
|
+
return { verdict: "retry", stoppedTaskIds: [] };
|
|
4581
|
+
}
|
|
4582
|
+
let resolved = !listed.ambiguous;
|
|
4583
|
+
|
|
4584
|
+
for (const candidate of listed.candidates) {
|
|
4585
|
+
if (!isCurrent()) {
|
|
4586
|
+
return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
4587
|
+
}
|
|
4588
|
+
await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
|
|
4589
|
+
if (!isCurrent()) return;
|
|
4590
|
+
const inspection = await inspectRuntimeWriterContainer(candidate);
|
|
4591
|
+
if (inspection.kind !== "ready") {
|
|
4592
|
+
resolved = false;
|
|
4593
|
+
console.error(
|
|
4594
|
+
`[orchestrator] runtime isolation: ${candidate.containerName} inspection did not prove a safe mount (${inspection.kind === "failed" ? inspection.detail : "Docker unreachable"})`,
|
|
4595
|
+
);
|
|
4596
|
+
return;
|
|
4597
|
+
}
|
|
4598
|
+
const task = getHostTask(candidate.taskId);
|
|
4599
|
+
const knownPreservedTask =
|
|
4600
|
+
task?.composeProject === candidate.composeProject &&
|
|
4601
|
+
typeof task.statusMirror === "string" &&
|
|
4602
|
+
isOnDashboard(task.statusMirror);
|
|
4603
|
+
if (
|
|
4604
|
+
inspection.metadata.sharedMount === "read-only" &&
|
|
4605
|
+
inspection.metadata.taskEnvironment === "current"
|
|
4606
|
+
) {
|
|
4607
|
+
return;
|
|
4608
|
+
}
|
|
4609
|
+
if (!knownPreservedTask) {
|
|
4610
|
+
// A task-shaped orphan with a proven RO shared mount cannot race
|
|
4611
|
+
// maintenance. Without a host row, its non-current task-local npm
|
|
4612
|
+
// environment is not enough ownership evidence to mutate it.
|
|
4613
|
+
if (inspection.metadata.sharedMount === "read-only") return;
|
|
4614
|
+
if (inspection.metadata.sharedMount === "absent") return;
|
|
4615
|
+
if (inspection.metadata.sharedMount === "invalid") {
|
|
4616
|
+
// `task-*` is Uai's namespace, but it is not by itself permission to
|
|
4617
|
+
// stop an unrelated/stale user stack. Only the exact named writable
|
|
4618
|
+
// Uai volume is positive ownership evidence without a host row.
|
|
4619
|
+
resolved = false;
|
|
4620
|
+
console.error(
|
|
4621
|
+
`[orchestrator] runtime isolation: orphan ${candidate.containerName} has ambiguous shared-runtime mount metadata; maintenance remains fenced`,
|
|
4622
|
+
);
|
|
4623
|
+
return;
|
|
4624
|
+
}
|
|
4625
|
+
}
|
|
4626
|
+
|
|
4627
|
+
// Drain before the first stop/update. A running agent continuation may
|
|
4628
|
+
// otherwise race the quarantine and respawn into the legacy container.
|
|
4629
|
+
await options.lifecycle.quarantineChannelForRecovery(candidate.taskId);
|
|
4630
|
+
const stopped = await stopRuntimeWriter(inspection.metadata);
|
|
4631
|
+
if (stopped !== "stopped") {
|
|
4632
|
+
resolved = false;
|
|
4633
|
+
console.error(
|
|
4634
|
+
`[orchestrator] runtime isolation: could not prove ${candidate.containerName} stopped (${stopped})`,
|
|
4635
|
+
);
|
|
4636
|
+
return;
|
|
4637
|
+
}
|
|
4638
|
+
|
|
4639
|
+
if (knownPreservedTask) {
|
|
4640
|
+
// Always re-publish the stopped verdict. A prior process may have
|
|
4641
|
+
// committed SQLite and crashed before its websocket frame left; the
|
|
4642
|
+
// cloud could otherwise remain ghost-running indefinitely.
|
|
4643
|
+
stoppedTaskIds.add(candidate.taskId);
|
|
4644
|
+
db_setStatus(candidate.taskId, "stopped", {
|
|
4645
|
+
codeServerPort: null,
|
|
4646
|
+
previewPorts: "[]",
|
|
4647
|
+
});
|
|
4648
|
+
}
|
|
4649
|
+
clearTaskGatewayAcl(candidate.taskId);
|
|
4650
|
+
// We deliberately reflect a proven physical stop even if the daemon
|
|
4651
|
+
// epoch changed while stopping: leaving the row active would invite a
|
|
4652
|
+
// later recovery to restart an immutable legacy writer. The next
|
|
4653
|
+
// activation generation independently re-enumerates Docker before
|
|
4654
|
+
// maintenance. Non-safety cleanup below may wait for that generation.
|
|
4655
|
+
if (!isCurrent()) return;
|
|
4656
|
+
try {
|
|
4657
|
+
await stopPreviewSidecars(candidate.taskId);
|
|
4658
|
+
} catch (error) {
|
|
4659
|
+
console.warn(
|
|
4660
|
+
`[orchestrator] runtime isolation: ${candidate.taskId} preview cleanup failed: ${
|
|
4661
|
+
error instanceof Error ? error.message : String(error)
|
|
4662
|
+
}`,
|
|
4663
|
+
);
|
|
4664
|
+
}
|
|
4665
|
+
console.warn(
|
|
4666
|
+
`[orchestrator] runtime isolation: ${candidate.containerName} used a writable/non-current shared runtime mount and remains stopped until explicit Resume recreates it`,
|
|
4667
|
+
);
|
|
4668
|
+
});
|
|
4669
|
+
}
|
|
4670
|
+
|
|
4671
|
+
if (!isCurrent()) {
|
|
4672
|
+
return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
4673
|
+
}
|
|
4674
|
+
return {
|
|
4675
|
+
verdict: resolved ? "ready" : "retry",
|
|
4676
|
+
stoppedTaskIds: [...stoppedTaskIds].sort(),
|
|
4677
|
+
};
|
|
4678
|
+
}
|
|
4679
|
+
|
|
4680
|
+
const RECOVERY_RUNTIME_INSPECT_FORMAT =
|
|
4681
|
+
"{{json .Image}}|{{json .State.Running}}|{{json .HostConfig.GroupAdd}}|{{json .Mounts}}";
|
|
4682
|
+
const RECOVERY_RUNTIME_IMAGE_RE = /^sha256:[0-9a-f]{64}$/;
|
|
4683
|
+
const RECOVERY_RUNTIME_SCRIPT_PATH =
|
|
4684
|
+
"/usr/local/bin/uai-materialize-runtimes";
|
|
4685
|
+
const RECOVERY_COREPACK_VERSION_PATH =
|
|
4686
|
+
"/usr/local/share/uai/corepack-version";
|
|
4687
|
+
const RECOVERY_INIT_PATH = "/uai-init-current";
|
|
4688
|
+
const RECOVERY_INIT_PROJECTS_PATH = "/uai-runtime-projects-current";
|
|
4689
|
+
const RECOVERY_RUNTIME_TIMEOUT_MS = 630_000;
|
|
4690
|
+
const RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES = 1024 * 1024;
|
|
4691
|
+
|
|
4692
|
+
interface RecoveryDockerMount {
|
|
4693
|
+
Type?: unknown;
|
|
4694
|
+
Name?: unknown;
|
|
4695
|
+
Source?: unknown;
|
|
4696
|
+
Destination?: unknown;
|
|
4697
|
+
RW?: unknown;
|
|
4698
|
+
}
|
|
4699
|
+
|
|
4700
|
+
interface RecoveryRuntimeSpec {
|
|
4701
|
+
imageId: string;
|
|
4702
|
+
workspaceSource: string;
|
|
4703
|
+
groupAdds: string[];
|
|
4704
|
+
projectCount: number;
|
|
4705
|
+
projectAllowlistSource: string;
|
|
4706
|
+
}
|
|
4707
|
+
|
|
4708
|
+
type RecoveryRuntimeInspection =
|
|
4709
|
+
| { kind: "ready"; spec: RecoveryRuntimeSpec }
|
|
4710
|
+
| { kind: "failed"; detail: string }
|
|
4711
|
+
| { kind: "unreachable" };
|
|
4712
|
+
|
|
4713
|
+
type RecoveryRuntimeResult =
|
|
4714
|
+
| { kind: "ready" }
|
|
4715
|
+
| { kind: "failed"; detail: string }
|
|
4716
|
+
| { kind: "unreachable" };
|
|
4717
|
+
|
|
4718
|
+
function recoveryRuntimeAssets():
|
|
4719
|
+
| { materializer: string; corepackVersion: string; uaiInit: string }
|
|
4720
|
+
| { error: string } {
|
|
4721
|
+
const materializer = standardImageRuntimeMaterializerPath();
|
|
4722
|
+
const corepackVersion = standardImageCorepackVersionPath();
|
|
4723
|
+
const uaiInit = join(dirname(materializer), "uai-init");
|
|
4724
|
+
try {
|
|
4725
|
+
const materializerStat = lstatSync(materializer);
|
|
4726
|
+
if (!materializerStat.isFile() || materializerStat.isSymbolicLink() ||
|
|
4727
|
+
(materializerStat.mode & 0o111) === 0) {
|
|
4728
|
+
return { error: `packaged materializer is not a regular executable: ${materializer}` };
|
|
4729
|
+
}
|
|
4730
|
+
const versionStat = lstatSync(corepackVersion);
|
|
4731
|
+
if (!versionStat.isFile() || versionStat.isSymbolicLink()) {
|
|
4732
|
+
return { error: `packaged Corepack pin is not a regular file: ${corepackVersion}` };
|
|
4733
|
+
}
|
|
4734
|
+
const version = readFileSync(corepackVersion, "utf8");
|
|
4735
|
+
if (!/^\d+\.\d+\.\d+\n?$/.test(version)) {
|
|
4736
|
+
return { error: `packaged Corepack pin is malformed: ${corepackVersion}` };
|
|
4737
|
+
}
|
|
4738
|
+
const initStat = lstatSync(uaiInit);
|
|
4739
|
+
if (
|
|
4740
|
+
!initStat.isFile() ||
|
|
4741
|
+
initStat.isSymbolicLink() ||
|
|
4742
|
+
(initStat.mode & 0o111) === 0
|
|
4743
|
+
) {
|
|
4744
|
+
return { error: `packaged task initializer is not a regular executable: ${uaiInit}` };
|
|
4745
|
+
}
|
|
4746
|
+
} catch (err) {
|
|
4747
|
+
return {
|
|
4748
|
+
error: `packaged runtime assets are unavailable: ${
|
|
4749
|
+
err instanceof Error ? err.message : String(err)
|
|
4750
|
+
}`,
|
|
4751
|
+
};
|
|
4752
|
+
}
|
|
4753
|
+
return { materializer, corepackVersion, uaiInit };
|
|
4754
|
+
}
|
|
4755
|
+
|
|
4756
|
+
function expectedRecoveryWorkspaceSources(
|
|
4757
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4758
|
+
): Set<string> {
|
|
4759
|
+
const expected = new Set<string>();
|
|
4760
|
+
if (!task.worktreePath) return expected;
|
|
4761
|
+
const lexical = join(task.worktreePath, "workspace");
|
|
4762
|
+
try {
|
|
4763
|
+
const stat = lstatSync(lexical);
|
|
4764
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) return expected;
|
|
4765
|
+
expected.add(lexical);
|
|
4766
|
+
expected.add(realpathSync(lexical));
|
|
4767
|
+
} catch {
|
|
4768
|
+
// Missing, unreadable, and unresolvable workspaces are never safe input
|
|
4769
|
+
// to `docker run`: a legacy `-v` bind would recreate a missing source as
|
|
4770
|
+
// an empty directory and falsely prove that the task had no projects.
|
|
4771
|
+
}
|
|
4772
|
+
return expected;
|
|
4773
|
+
}
|
|
4774
|
+
|
|
4775
|
+
function validRecoveryGroupAdds(value: unknown): string[] | null {
|
|
4776
|
+
if (value === null || value === undefined) return [];
|
|
4777
|
+
if (!Array.isArray(value)) return null;
|
|
4778
|
+
const groups: string[] = [];
|
|
4779
|
+
for (const candidate of value) {
|
|
4780
|
+
if (typeof candidate !== "string" || !/^[1-9]\d{0,9}$/.test(candidate)) {
|
|
4781
|
+
return null;
|
|
4782
|
+
}
|
|
4783
|
+
const numeric = Number(candidate);
|
|
4784
|
+
if (!Number.isSafeInteger(numeric) || numeric > 4_294_967_294) {
|
|
4785
|
+
return null;
|
|
4786
|
+
}
|
|
4787
|
+
if (!groups.includes(candidate)) groups.push(candidate);
|
|
4788
|
+
}
|
|
4789
|
+
return groups;
|
|
4790
|
+
}
|
|
4791
|
+
|
|
4792
|
+
function safeRecoveryProjectSlug(slug: unknown): slug is string {
|
|
4793
|
+
return typeof slug === "string" && /^[a-z0-9-]{1,64}$/.test(slug);
|
|
4794
|
+
}
|
|
4795
|
+
|
|
4796
|
+
function validateRecoveryProjects(
|
|
4797
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4798
|
+
workspaceSource: string,
|
|
4799
|
+
): { slugs: string[] } | { error: string } {
|
|
4800
|
+
let parsed: unknown;
|
|
4801
|
+
try {
|
|
4802
|
+
parsed = JSON.parse(task.projectSlugs);
|
|
4803
|
+
} catch {
|
|
4804
|
+
return { error: "the persisted project slug list is invalid JSON" };
|
|
4805
|
+
}
|
|
4806
|
+
if (!Array.isArray(parsed) || !parsed.every(safeRecoveryProjectSlug)) {
|
|
4807
|
+
return { error: "the persisted project slug list contains an unsafe entry" };
|
|
4808
|
+
}
|
|
4809
|
+
const slugs = parsed as string[];
|
|
4810
|
+
const expected = new Set(slugs);
|
|
4811
|
+
if (expected.size !== slugs.length) {
|
|
4812
|
+
return { error: "the persisted project slug list contains duplicates" };
|
|
4813
|
+
}
|
|
4814
|
+
|
|
4815
|
+
for (const slug of expected) {
|
|
4816
|
+
const project = join(workspaceSource, slug);
|
|
4817
|
+
let projectStat: ReturnType<typeof lstatSync>;
|
|
4818
|
+
let markerStat: ReturnType<typeof lstatSync>;
|
|
4819
|
+
try {
|
|
4820
|
+
projectStat = lstatSync(project);
|
|
4821
|
+
markerStat = lstatSync(join(project, ".git"));
|
|
4822
|
+
} catch {
|
|
4823
|
+
return { error: `expected project ${slug} or its .git marker is missing` };
|
|
4824
|
+
}
|
|
4825
|
+
if (
|
|
4826
|
+
projectStat.isSymbolicLink() ||
|
|
4827
|
+
!projectStat.isDirectory() ||
|
|
4828
|
+
markerStat.isSymbolicLink() ||
|
|
4829
|
+
(!markerStat.isFile() && !markerStat.isDirectory())
|
|
4830
|
+
) {
|
|
4831
|
+
return { error: `expected project ${slug} is not a safe Git worktree` };
|
|
4832
|
+
}
|
|
4833
|
+
}
|
|
4834
|
+
return { slugs };
|
|
4835
|
+
}
|
|
4836
|
+
|
|
4837
|
+
function validateRecoveryWorkspaceRuntimeRoot(
|
|
4838
|
+
workspaceSource: string,
|
|
4839
|
+
): string | null {
|
|
4840
|
+
const shadow = join(workspaceSource, ".tool-versions");
|
|
4841
|
+
try {
|
|
4842
|
+
// Any type is forbidden, including regular files and dangling symlinks.
|
|
4843
|
+
// lstat is metadata-only, so FIFO/device/socket entries cannot block the
|
|
4844
|
+
// host recovery loop.
|
|
4845
|
+
lstatSync(shadow);
|
|
4846
|
+
return "the workspace root contains a forbidden .tool-versions runtime override";
|
|
4847
|
+
} catch (err) {
|
|
4848
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
4849
|
+
return `the workspace-root runtime authority could not be verified: ${
|
|
4850
|
+
err instanceof Error ? err.message : String(err)
|
|
4851
|
+
}`;
|
|
4852
|
+
}
|
|
4853
|
+
}
|
|
4854
|
+
|
|
4855
|
+
function recoveryRuntimeProjectAllowlist(
|
|
4856
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4857
|
+
slugs: string[],
|
|
4858
|
+
): { path: string } | { error: string } {
|
|
4859
|
+
if (!task.worktreePath) {
|
|
4860
|
+
return { error: "the task has no persisted runtime directory" };
|
|
4861
|
+
}
|
|
4862
|
+
const runtimeDir = join(task.worktreePath, ".uai");
|
|
4863
|
+
const target = join(runtimeDir, "runtime-projects");
|
|
4864
|
+
const expected = slugs.length > 0 ? `${slugs.join("\n")}\n` : "";
|
|
4865
|
+
try {
|
|
4866
|
+
const runtimeDirStat = lstatSync(runtimeDir);
|
|
4867
|
+
if (!runtimeDirStat.isDirectory() || runtimeDirStat.isSymbolicLink()) {
|
|
4868
|
+
return { error: "the task runtime directory is not a safe directory" };
|
|
4869
|
+
}
|
|
4870
|
+
try {
|
|
4871
|
+
const targetStat = lstatSync(target);
|
|
4872
|
+
if (!targetStat.isFile() || targetStat.isSymbolicLink()) {
|
|
4873
|
+
return { error: "the task runtime-project allowlist is not a regular file" };
|
|
4874
|
+
}
|
|
4875
|
+
if (readFileSync(target, "utf8") !== expected) {
|
|
4876
|
+
return { error: "the task runtime-project allowlist does not match persisted projects" };
|
|
4877
|
+
}
|
|
4878
|
+
return { path: target };
|
|
4879
|
+
} catch (err) {
|
|
4880
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
|
|
4881
|
+
}
|
|
4882
|
+
|
|
4883
|
+
if (slugs.length === 0) {
|
|
4884
|
+
return {
|
|
4885
|
+
error: "zero-project recovery lacks an existing authoritative empty runtime-project allowlist",
|
|
4886
|
+
};
|
|
4887
|
+
}
|
|
4888
|
+
|
|
4889
|
+
const stagingDir = mkdtempSync(join(runtimeDir, ".runtime-projects-"));
|
|
4890
|
+
const staged = join(stagingDir, "runtime-projects");
|
|
4891
|
+
try {
|
|
4892
|
+
writeFileSync(staged, expected, { encoding: "utf8", flag: "wx", mode: 0o644 });
|
|
4893
|
+
renameSync(staged, target);
|
|
4894
|
+
} finally {
|
|
4895
|
+
rmSync(stagingDir, { recursive: true, force: true });
|
|
4896
|
+
}
|
|
4897
|
+
return { path: target };
|
|
4898
|
+
} catch (err) {
|
|
4899
|
+
return {
|
|
4900
|
+
error: `the task runtime-project allowlist is unavailable: ${
|
|
4901
|
+
err instanceof Error ? err.message : String(err)
|
|
4902
|
+
}`,
|
|
4903
|
+
};
|
|
4904
|
+
}
|
|
4905
|
+
}
|
|
4906
|
+
|
|
4907
|
+
async function inspectRecoveryRuntime(
|
|
4908
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
4909
|
+
containerName: string,
|
|
4910
|
+
): Promise<RecoveryRuntimeInspection> {
|
|
4911
|
+
const inspected = await dockerCli(
|
|
4912
|
+
[
|
|
4913
|
+
"inspect",
|
|
4914
|
+
"--format",
|
|
4915
|
+
RECOVERY_RUNTIME_INSPECT_FORMAT,
|
|
4916
|
+
containerName,
|
|
4917
|
+
],
|
|
4918
|
+
{ timeoutMs: 30_000, maxOutputBytes: 128 * 1024 },
|
|
4919
|
+
);
|
|
4920
|
+
if (inspected.status !== 0) {
|
|
4921
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
4922
|
+
return {
|
|
4923
|
+
kind: "failed",
|
|
4924
|
+
detail:
|
|
4925
|
+
inspected.stderr.trim() ||
|
|
4926
|
+
`could not inspect stopped app container ${containerName}`,
|
|
4927
|
+
};
|
|
4928
|
+
}
|
|
4929
|
+
|
|
4930
|
+
const firstDivider = inspected.stdout.indexOf("|");
|
|
4931
|
+
const secondDivider = inspected.stdout.indexOf("|", firstDivider + 1);
|
|
4932
|
+
const thirdDivider = inspected.stdout.indexOf("|", secondDivider + 1);
|
|
4933
|
+
if (firstDivider < 0 || secondDivider < 0 || thirdDivider < 0) {
|
|
4934
|
+
return { kind: "failed", detail: "Docker returned malformed runtime metadata" };
|
|
4935
|
+
}
|
|
4936
|
+
|
|
4937
|
+
let imageId: unknown;
|
|
4938
|
+
let running: unknown;
|
|
4939
|
+
let rawGroupAdds: unknown;
|
|
4940
|
+
let mounts: unknown;
|
|
4941
|
+
try {
|
|
4942
|
+
imageId = JSON.parse(inspected.stdout.slice(0, firstDivider));
|
|
4943
|
+
running = JSON.parse(
|
|
4944
|
+
inspected.stdout.slice(firstDivider + 1, secondDivider),
|
|
4945
|
+
);
|
|
4946
|
+
rawGroupAdds = JSON.parse(
|
|
4947
|
+
inspected.stdout.slice(secondDivider + 1, thirdDivider),
|
|
4948
|
+
);
|
|
4949
|
+
mounts = JSON.parse(inspected.stdout.slice(thirdDivider + 1));
|
|
4950
|
+
} catch {
|
|
4951
|
+
return { kind: "failed", detail: "Docker returned invalid runtime metadata JSON" };
|
|
4952
|
+
}
|
|
4953
|
+
if (running !== false) {
|
|
4954
|
+
return {
|
|
4955
|
+
kind: "failed",
|
|
4956
|
+
detail: "the app container was not proven stopped before runtime maintenance",
|
|
4957
|
+
};
|
|
4958
|
+
}
|
|
4959
|
+
if (typeof imageId !== "string" || !RECOVERY_RUNTIME_IMAGE_RE.test(imageId)) {
|
|
4960
|
+
return { kind: "failed", detail: "the app container has an invalid immutable image ID" };
|
|
4961
|
+
}
|
|
4962
|
+
if (!Array.isArray(mounts)) {
|
|
4963
|
+
return { kind: "failed", detail: "the app container has invalid mount metadata" };
|
|
4964
|
+
}
|
|
4965
|
+
const groupAdds = validRecoveryGroupAdds(rawGroupAdds);
|
|
4966
|
+
if (groupAdds === null) {
|
|
4967
|
+
return {
|
|
4968
|
+
kind: "failed",
|
|
4969
|
+
detail: "the app container has malformed or unsafe supplemental groups",
|
|
4970
|
+
};
|
|
4971
|
+
}
|
|
4972
|
+
const typedMounts = mounts as RecoveryDockerMount[];
|
|
4973
|
+
const workspaceMounts = typedMounts.filter(
|
|
4974
|
+
(mount) => mount.Destination === "/workspace",
|
|
4975
|
+
);
|
|
4976
|
+
const expectedWorkspaceSources = expectedRecoveryWorkspaceSources(task);
|
|
4977
|
+
if (
|
|
4978
|
+
workspaceMounts.length !== 1 ||
|
|
4979
|
+
workspaceMounts[0]?.Type !== "bind" ||
|
|
4980
|
+
typeof workspaceMounts[0]?.Source !== "string" ||
|
|
4981
|
+
!expectedWorkspaceSources.has(workspaceMounts[0].Source)
|
|
4982
|
+
) {
|
|
4983
|
+
return {
|
|
4984
|
+
kind: "failed",
|
|
4985
|
+
detail: "the app container does not have the exact expected /workspace bind",
|
|
4986
|
+
};
|
|
4987
|
+
}
|
|
4988
|
+
try {
|
|
4989
|
+
const workspaceStat = lstatSync(workspaceMounts[0].Source);
|
|
4990
|
+
if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
|
|
4991
|
+
return {
|
|
4992
|
+
kind: "failed",
|
|
4993
|
+
detail: "the app container's /workspace source is not a real directory",
|
|
4994
|
+
};
|
|
4995
|
+
}
|
|
4996
|
+
} catch {
|
|
4997
|
+
return {
|
|
4998
|
+
kind: "failed",
|
|
4999
|
+
detail: "the app container's /workspace source no longer exists",
|
|
5000
|
+
};
|
|
5001
|
+
}
|
|
5002
|
+
const workspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
|
|
5003
|
+
workspaceMounts[0].Source,
|
|
5004
|
+
);
|
|
5005
|
+
if (workspaceRuntimeError) {
|
|
5006
|
+
return { kind: "failed", detail: workspaceRuntimeError };
|
|
5007
|
+
}
|
|
5008
|
+
const projects = validateRecoveryProjects(
|
|
5009
|
+
task,
|
|
5010
|
+
workspaceMounts[0].Source,
|
|
5011
|
+
);
|
|
5012
|
+
if ("error" in projects) {
|
|
5013
|
+
return { kind: "failed", detail: projects.error };
|
|
5014
|
+
}
|
|
5015
|
+
const projectAllowlist = recoveryRuntimeProjectAllowlist(
|
|
5016
|
+
task,
|
|
5017
|
+
projects.slugs,
|
|
5018
|
+
);
|
|
5019
|
+
if ("error" in projectAllowlist) {
|
|
5020
|
+
return { kind: "failed", detail: projectAllowlist.error };
|
|
5021
|
+
}
|
|
5022
|
+
const asdfMounts = typedMounts.filter(
|
|
5023
|
+
(mount) => mount.Destination === "/opt/asdf-data",
|
|
5024
|
+
);
|
|
5025
|
+
if (
|
|
5026
|
+
asdfMounts.length !== 1 ||
|
|
5027
|
+
asdfMounts[0]?.Type !== "volume" ||
|
|
5028
|
+
asdfMounts[0]?.Name !== "uai-asdf-data" ||
|
|
5029
|
+
asdfMounts[0]?.RW !== false
|
|
5030
|
+
) {
|
|
5031
|
+
return {
|
|
5032
|
+
kind: "failed",
|
|
5033
|
+
detail:
|
|
5034
|
+
"the app container does not use the exact read-only uai-asdf-data volume; explicit Resume must recreate legacy writable containers",
|
|
5035
|
+
};
|
|
5036
|
+
}
|
|
5037
|
+
return {
|
|
5038
|
+
kind: "ready",
|
|
5039
|
+
spec: {
|
|
5040
|
+
imageId,
|
|
5041
|
+
workspaceSource: workspaceMounts[0].Source,
|
|
5042
|
+
groupAdds,
|
|
5043
|
+
projectCount: projects.slugs.length,
|
|
5044
|
+
projectAllowlistSource: projectAllowlist.path,
|
|
5045
|
+
},
|
|
5046
|
+
};
|
|
5047
|
+
}
|
|
5048
|
+
|
|
5049
|
+
async function materializeRecoveryRuntimes(
|
|
5050
|
+
spec: RecoveryRuntimeSpec,
|
|
5051
|
+
): Promise<RecoveryRuntimeResult> {
|
|
5052
|
+
if (spec.projectCount === 0) return { kind: "ready" };
|
|
5053
|
+
const assets = recoveryRuntimeAssets();
|
|
5054
|
+
if ("error" in assets) return { kind: "failed", detail: assets.error };
|
|
5055
|
+
try {
|
|
5056
|
+
const workspaceStat = lstatSync(spec.workspaceSource);
|
|
5057
|
+
if (!workspaceStat.isDirectory() || workspaceStat.isSymbolicLink()) {
|
|
5058
|
+
return {
|
|
5059
|
+
kind: "failed",
|
|
5060
|
+
detail: "the app container's /workspace source is no longer a real directory",
|
|
5061
|
+
};
|
|
5062
|
+
}
|
|
5063
|
+
} catch {
|
|
5064
|
+
return {
|
|
5065
|
+
kind: "failed",
|
|
5066
|
+
detail: "the app container's /workspace source disappeared before runtime proof",
|
|
5067
|
+
};
|
|
5068
|
+
}
|
|
5069
|
+
try {
|
|
5070
|
+
const allowlistStat = lstatSync(spec.projectAllowlistSource);
|
|
5071
|
+
if (!allowlistStat.isFile() || allowlistStat.isSymbolicLink()) {
|
|
5072
|
+
return {
|
|
5073
|
+
kind: "failed",
|
|
5074
|
+
detail: "the runtime-project allowlist is no longer a regular file",
|
|
5075
|
+
};
|
|
5076
|
+
}
|
|
5077
|
+
} catch {
|
|
5078
|
+
return {
|
|
5079
|
+
kind: "failed",
|
|
5080
|
+
detail: "the runtime-project allowlist disappeared before runtime proof",
|
|
5081
|
+
};
|
|
5082
|
+
}
|
|
5083
|
+
const supplementalGroups = spec.groupAdds.flatMap((group) => [
|
|
5084
|
+
"--group-add",
|
|
5085
|
+
group,
|
|
5086
|
+
]);
|
|
5087
|
+
const result = await dockerCli(
|
|
5088
|
+
[
|
|
5089
|
+
"run",
|
|
5090
|
+
"--rm",
|
|
5091
|
+
"--user",
|
|
5092
|
+
"node",
|
|
5093
|
+
...supplementalGroups,
|
|
5094
|
+
"--workdir",
|
|
5095
|
+
"/workspace",
|
|
5096
|
+
"--mount",
|
|
5097
|
+
`type=bind,src=${spec.workspaceSource},dst=/workspace,readonly`,
|
|
5098
|
+
"-v",
|
|
5099
|
+
"uai-asdf-data:/opt/asdf-data:rw",
|
|
5100
|
+
"-v",
|
|
5101
|
+
`${assets.materializer}:${RECOVERY_RUNTIME_SCRIPT_PATH}:ro`,
|
|
5102
|
+
"-v",
|
|
5103
|
+
`${assets.corepackVersion}:${RECOVERY_COREPACK_VERSION_PATH}:ro`,
|
|
5104
|
+
"-v",
|
|
5105
|
+
`${spec.projectAllowlistSource}:/run/uai/runtime-projects:ro`,
|
|
5106
|
+
"--env",
|
|
5107
|
+
"LD_PRELOAD=",
|
|
5108
|
+
"--env",
|
|
5109
|
+
"LD_AUDIT=",
|
|
5110
|
+
"--env",
|
|
5111
|
+
"LD_LIBRARY_PATH=",
|
|
5112
|
+
"--env",
|
|
5113
|
+
"BASH_ENV=",
|
|
5114
|
+
"--env",
|
|
5115
|
+
"ENV=",
|
|
5116
|
+
"--entrypoint",
|
|
5117
|
+
"/usr/bin/env",
|
|
5118
|
+
spec.imageId,
|
|
5119
|
+
"-i",
|
|
5120
|
+
"HOME=/home/node",
|
|
5121
|
+
"ASDF_DIR=/opt/asdf",
|
|
5122
|
+
"ASDF_DATA_DIR=/opt/asdf-data",
|
|
5123
|
+
"PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
|
5124
|
+
"/usr/bin/timeout",
|
|
5125
|
+
"--signal=TERM",
|
|
5126
|
+
"--kill-after=10s",
|
|
5127
|
+
"600s",
|
|
5128
|
+
RECOVERY_RUNTIME_SCRIPT_PATH,
|
|
5129
|
+
],
|
|
5130
|
+
{
|
|
5131
|
+
timeoutMs: RECOVERY_RUNTIME_TIMEOUT_MS,
|
|
5132
|
+
tailOutputBytes: RECOVERY_RUNTIME_OUTPUT_TAIL_BYTES,
|
|
5133
|
+
},
|
|
5134
|
+
);
|
|
5135
|
+
if (result.status === 0) return { kind: "ready" };
|
|
5136
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
5137
|
+
return {
|
|
5138
|
+
kind: "failed",
|
|
5139
|
+
detail: `${result.outputTruncated ? "earlier runtime output was truncated; " : ""}${
|
|
5140
|
+
result.stderr.trim() ||
|
|
5141
|
+
`runtime materializer exited ${String(result.status)}`
|
|
5142
|
+
}`,
|
|
5143
|
+
};
|
|
5144
|
+
}
|
|
5145
|
+
|
|
5146
|
+
async function stageRecoveryInit(
|
|
5147
|
+
containerName: string,
|
|
5148
|
+
spec: RecoveryRuntimeSpec,
|
|
5149
|
+
): Promise<RecoveryRuntimeResult> {
|
|
5150
|
+
const assets = recoveryRuntimeAssets();
|
|
5151
|
+
if ("error" in assets) return { kind: "failed", detail: assets.error };
|
|
5152
|
+
const copies: Array<[string, string]> = [
|
|
5153
|
+
[assets.uaiInit, RECOVERY_INIT_PATH],
|
|
5154
|
+
[spec.projectAllowlistSource, RECOVERY_INIT_PROJECTS_PATH],
|
|
5155
|
+
];
|
|
5156
|
+
for (const [source, destination] of copies) {
|
|
5157
|
+
const copied = await dockerCli(
|
|
5158
|
+
["cp", source, `${containerName}:${destination}`],
|
|
5159
|
+
{ timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
|
|
5160
|
+
);
|
|
5161
|
+
if (copied.status === 0) continue;
|
|
5162
|
+
if (!(await dockerDaemonReachable())) return { kind: "unreachable" };
|
|
5163
|
+
return {
|
|
5164
|
+
kind: "failed",
|
|
5165
|
+
detail:
|
|
5166
|
+
copied.stderr.trim() ||
|
|
5167
|
+
`could not stage ${destination} in the stopped app container`,
|
|
5168
|
+
};
|
|
5169
|
+
}
|
|
5170
|
+
return { kind: "ready" };
|
|
5171
|
+
}
|
|
5172
|
+
|
|
5173
|
+
type DockerQuarantineResult = "stopped" | "failed" | "unreachable";
|
|
5174
|
+
|
|
5175
|
+
async function quarantineRunningContainer(
|
|
5176
|
+
containerName: string,
|
|
5177
|
+
): Promise<DockerQuarantineResult> {
|
|
5178
|
+
const stopped = await dockerCli(
|
|
5179
|
+
["stop", "--time", "10", containerName],
|
|
5180
|
+
{ timeoutMs: 30_000 },
|
|
5181
|
+
);
|
|
5182
|
+
const inspected = await dockerCli(
|
|
5183
|
+
["inspect", "--format", "{{.State.Running}}", containerName],
|
|
5184
|
+
{ timeoutMs: 10_000 },
|
|
5185
|
+
);
|
|
5186
|
+
if (inspected.status === 0 && inspected.stdout.trim() === "false") {
|
|
5187
|
+
return "stopped";
|
|
5188
|
+
}
|
|
5189
|
+
if (inspected.status !== 0 && !(await dockerDaemonReachable())) {
|
|
5190
|
+
return "unreachable";
|
|
5191
|
+
}
|
|
5192
|
+
const killed = await dockerCli(["kill", containerName], {
|
|
5193
|
+
timeoutMs: 15_000,
|
|
5194
|
+
});
|
|
5195
|
+
const killInspection = await dockerCli(
|
|
5196
|
+
["inspect", "--format", "{{.State.Running}}", containerName],
|
|
5197
|
+
{ timeoutMs: 10_000 },
|
|
5198
|
+
);
|
|
5199
|
+
if (
|
|
5200
|
+
killInspection.status === 0 &&
|
|
5201
|
+
killInspection.stdout.trim() === "false"
|
|
5202
|
+
) {
|
|
5203
|
+
return "stopped";
|
|
5204
|
+
}
|
|
5205
|
+
if (killInspection.status !== 0 && !(await dockerDaemonReachable())) {
|
|
5206
|
+
return "unreachable";
|
|
5207
|
+
}
|
|
5208
|
+
console.error(
|
|
5209
|
+
`[orchestrator] recovery: could not quarantine ${containerName} before runtime proof: ${
|
|
5210
|
+
stopped.stderr.trim() ||
|
|
5211
|
+
inspected.stderr.trim() ||
|
|
5212
|
+
killed.stderr.trim() ||
|
|
5213
|
+
killInspection.stderr.trim() ||
|
|
5214
|
+
"container is still running after stop and kill"
|
|
5215
|
+
}`,
|
|
5216
|
+
);
|
|
5217
|
+
return "failed";
|
|
5218
|
+
}
|
|
5219
|
+
|
|
5220
|
+
/**
|
|
5221
|
+
* Repair the shared node data root before any recovered-container init.
|
|
5222
|
+
* Older task-up versions created the OpenCode leaf as root and accidentally
|
|
5223
|
+
* left this parent root-owned, which prevented code-server from creating its
|
|
5224
|
+
* managed User profile. `install -d` creates or repairs the exact directory
|
|
5225
|
+
* without recursively changing unrelated application state beneath it.
|
|
5226
|
+
*/
|
|
5227
|
+
async function repairNodeDataRoot(containerName: string): Promise<boolean> {
|
|
5228
|
+
const res = await dockerCli([
|
|
5229
|
+
"exec",
|
|
5230
|
+
"-u",
|
|
5231
|
+
"root",
|
|
5232
|
+
...runtimeAuthorityDockerExecArgs(),
|
|
5233
|
+
containerName,
|
|
5234
|
+
"/usr/bin/install",
|
|
5235
|
+
"-d",
|
|
5236
|
+
"-o",
|
|
5237
|
+
"node",
|
|
5238
|
+
"-g",
|
|
5239
|
+
"node",
|
|
5240
|
+
"-m",
|
|
5241
|
+
"0755",
|
|
5242
|
+
"/home/node/.local/share",
|
|
5243
|
+
]);
|
|
5244
|
+
if (res.status !== 0) {
|
|
5245
|
+
console.error(
|
|
5246
|
+
`[orchestrator] recovery: ${containerName} could not repair /home/node/.local/share ownership: ${res.stderr.trim() || `docker exec exited ${String(res.status)}`}`,
|
|
5247
|
+
);
|
|
5248
|
+
return false;
|
|
5249
|
+
}
|
|
5250
|
+
return true;
|
|
5251
|
+
}
|
|
5252
|
+
|
|
5253
|
+
interface DockerPortResult {
|
|
5254
|
+
reachable: boolean;
|
|
5255
|
+
port: number | null;
|
|
3575
5256
|
}
|
|
3576
5257
|
|
|
3577
5258
|
async function dockerPort(
|
|
3578
5259
|
containerName: string,
|
|
3579
5260
|
containerPort: number,
|
|
3580
|
-
): Promise<
|
|
5261
|
+
): Promise<DockerPortResult> {
|
|
3581
5262
|
const res = await dockerCli(["port", containerName, String(containerPort)]);
|
|
3582
|
-
if (res.status
|
|
5263
|
+
if (res.status === null) return { reachable: false, port: null };
|
|
5264
|
+
if (res.status !== 0) {
|
|
5265
|
+
return {
|
|
5266
|
+
reachable: await dockerDaemonReachable(),
|
|
5267
|
+
port: null,
|
|
5268
|
+
};
|
|
5269
|
+
}
|
|
3583
5270
|
// Output: "127.0.0.1:32785\n"
|
|
3584
5271
|
const firstLine = res.stdout.split("\n")[0]?.trim() ?? "";
|
|
3585
5272
|
const colonIdx = firstLine.lastIndexOf(":");
|
|
3586
|
-
if (colonIdx === -1) return null;
|
|
5273
|
+
if (colonIdx === -1) return { reachable: true, port: null };
|
|
3587
5274
|
const port = Number(firstLine.slice(colonIdx + 1));
|
|
3588
|
-
return
|
|
5275
|
+
return {
|
|
5276
|
+
reachable: true,
|
|
5277
|
+
port: Number.isFinite(port) ? port : null,
|
|
5278
|
+
};
|
|
3589
5279
|
}
|
|
3590
5280
|
|
|
3591
5281
|
async function dockerExec(
|
|
@@ -3594,102 +5284,952 @@ async function dockerExec(
|
|
|
3594
5284
|
timeoutMs?: number,
|
|
3595
5285
|
): Promise<boolean> {
|
|
3596
5286
|
const res = await dockerCli(
|
|
3597
|
-
["exec", containerName, ...cmd],
|
|
5287
|
+
["exec", ...runtimeAuthorityDockerExecArgs(), containerName, ...cmd],
|
|
3598
5288
|
timeoutMs === undefined ? {} : { timeoutMs },
|
|
3599
5289
|
);
|
|
3600
5290
|
return res.status === 0;
|
|
3601
5291
|
}
|
|
3602
5292
|
|
|
3603
|
-
/** Exported for tests; production entry is the getOrchestrator() boot guard. */
|
|
3604
|
-
export async function recoverRunningTasks(opts?: {
|
|
3605
|
-
/** Delay between passes while docker is unreachable (tests shrink it). */
|
|
3606
|
-
retryMs?: number;
|
|
3607
|
-
/** Bound on passes; ~15 min at the default cadence covers slow boots. */
|
|
3608
|
-
maxPasses?: number;
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
5293
|
+
/** Exported for tests; production entry is the getOrchestrator() boot guard. */
|
|
5294
|
+
export async function recoverRunningTasks(opts?: {
|
|
5295
|
+
/** Delay between passes while docker is unreachable (tests shrink it). */
|
|
5296
|
+
retryMs?: number;
|
|
5297
|
+
/** Bound on passes; ~15 min at the default cadence covers slow boots. */
|
|
5298
|
+
maxPasses?: number;
|
|
5299
|
+
/** Post-boot recovery must queue behind live taskUp/taskDown operations.
|
|
5300
|
+
* Boot recovery omits this and uses its earlier per-task barriers instead. */
|
|
5301
|
+
lifecycle?: Orchestrator;
|
|
5302
|
+
/** Test seam; production waits for the shared image/CLI-volume reconcile. */
|
|
5303
|
+
maintenanceReady?: Promise<void>;
|
|
5304
|
+
/** A daemon-generation fence. False cancels every remaining side effect. */
|
|
5305
|
+
isCurrent?: () => boolean;
|
|
5306
|
+
}): Promise<boolean> {
|
|
5307
|
+
const retryMs = opts?.retryMs ?? 20_000;
|
|
5308
|
+
const maxPasses = opts?.maxPasses ?? 45;
|
|
5309
|
+
let pendingTaskIds: ReadonlySet<string> | undefined;
|
|
5310
|
+
// After a machine reboot the host service and its selected environment
|
|
5311
|
+
// provider start concurrently. Unknown provider state resolves nothing and
|
|
5312
|
+
// must retry once the runtime answers.
|
|
5313
|
+
for (let pass = 1; ; pass++) {
|
|
5314
|
+
if (opts?.isCurrent?.() === false) {
|
|
5315
|
+
if (!opts.lifecycle) finishAllTaskRecovery();
|
|
5316
|
+
return false;
|
|
5317
|
+
}
|
|
5318
|
+
const result = await recoveryPass(
|
|
5319
|
+
pendingTaskIds,
|
|
5320
|
+
opts?.lifecycle,
|
|
5321
|
+
opts?.maintenanceReady ?? agentClisReady,
|
|
5322
|
+
opts?.isCurrent ?? (() => true),
|
|
5323
|
+
);
|
|
5324
|
+
if (opts?.isCurrent?.() === false) {
|
|
5325
|
+
if (!opts.lifecycle) finishAllTaskRecovery();
|
|
5326
|
+
return false;
|
|
5327
|
+
}
|
|
5328
|
+
const { deferred } = result;
|
|
5329
|
+
if (!result.resolved) {
|
|
5330
|
+
if (!opts?.lifecycle) finishAllTaskRecovery();
|
|
5331
|
+
return false;
|
|
5332
|
+
}
|
|
5333
|
+
if (deferred.size === 0) {
|
|
5334
|
+
if (!opts?.lifecycle) finishAllTaskRecovery();
|
|
5335
|
+
return true;
|
|
5336
|
+
}
|
|
5337
|
+
pendingTaskIds = deferred;
|
|
5338
|
+
if (pass >= maxPasses) {
|
|
5339
|
+
console.warn(
|
|
5340
|
+
`[orchestrator] recovery: task environment runtime still unreachable after ${pass} passes — giving up (${deferred.size} task(s) unresolved)`,
|
|
5341
|
+
);
|
|
5342
|
+
if (!opts?.lifecycle) finishAllTaskRecovery();
|
|
5343
|
+
return false;
|
|
5344
|
+
}
|
|
5345
|
+
if (pass === 1) {
|
|
5346
|
+
console.log(
|
|
5347
|
+
`[orchestrator] recovery: task environment runtime not ready — retrying every ${Math.round(retryMs / 1000)}s for ${deferred.size} task(s)`,
|
|
5348
|
+
);
|
|
5349
|
+
}
|
|
5350
|
+
await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
5351
|
+
}
|
|
5352
|
+
}
|
|
5353
|
+
|
|
5354
|
+
interface RecoveryPassResult {
|
|
5355
|
+
deferred: Set<string>;
|
|
5356
|
+
resolved: boolean;
|
|
5357
|
+
}
|
|
5358
|
+
|
|
5359
|
+
/** One recovery sweep. Returns only task ids deferred by an unreachable provider;
|
|
5360
|
+
* successful ids are never revisited by a later retry pass. `resolved` is
|
|
5361
|
+
* false for an unexpected failure whose runtime state could not be proved. */
|
|
5362
|
+
async function recoveryPass(
|
|
5363
|
+
onlyTaskIds?: ReadonlySet<string>,
|
|
5364
|
+
lifecycle?: Orchestrator,
|
|
5365
|
+
maintenanceReady: Promise<void> = agentClisReady,
|
|
5366
|
+
isCurrent: () => boolean = () => true,
|
|
5367
|
+
): Promise<RecoveryPassResult> {
|
|
5368
|
+
const deferred = new Set<string>();
|
|
5369
|
+
let resolved = true;
|
|
5370
|
+
try {
|
|
5371
|
+
if (!isCurrent()) return { deferred, resolved: false };
|
|
5372
|
+
const db = getDb();
|
|
5373
|
+
const activeRows = db
|
|
5374
|
+
.select()
|
|
5375
|
+
.from(schema.hostTasks)
|
|
5376
|
+
.where(inArray(schema.hostTasks.statusMirror, [...ACTIVE_STATUSES]))
|
|
5377
|
+
.all();
|
|
5378
|
+
const rows = onlyTaskIds
|
|
5379
|
+
? activeRows.filter((task) => onlyTaskIds.has(task.taskId))
|
|
5380
|
+
: activeRows;
|
|
5381
|
+
if (!lifecycle) {
|
|
5382
|
+
const activeTaskIds = new Set(rows.map((task) => task.taskId));
|
|
5383
|
+
const expectedTaskIds =
|
|
5384
|
+
onlyTaskIds ?? new Set(taskRecoveryBarriers.keys());
|
|
5385
|
+
for (const taskId of expectedTaskIds) {
|
|
5386
|
+
if (!activeTaskIds.has(taskId)) finishTaskRecovery(taskId);
|
|
5387
|
+
}
|
|
5388
|
+
for (const task of rows) openTaskRecoveryBarrier(task.taskId);
|
|
5389
|
+
}
|
|
5390
|
+
if (rows.length === 0) return { deferred, resolved };
|
|
5391
|
+
console.log(
|
|
5392
|
+
`[orchestrator] recovery: scanning ${rows.length} active task row(s)`,
|
|
5393
|
+
);
|
|
5394
|
+
for (const task of rows) {
|
|
5395
|
+
if (!isCurrent()) break;
|
|
5396
|
+
try {
|
|
5397
|
+
const recovered = lifecycle
|
|
5398
|
+
? await lifecycle.runTaskLifecycle(task.taskId, async () => {
|
|
5399
|
+
// The row was selected before this lifecycle slot became ours.
|
|
5400
|
+
// A queued taskDown/taskUp may have completed meanwhile, so act
|
|
5401
|
+
// only on the current local authority—not the stale snapshot.
|
|
5402
|
+
const current = getHostTask(task.taskId);
|
|
5403
|
+
if (
|
|
5404
|
+
!current ||
|
|
5405
|
+
!isCurrent() ||
|
|
5406
|
+
!current.statusMirror ||
|
|
5407
|
+
!isActive(current.statusMirror)
|
|
5408
|
+
) {
|
|
5409
|
+
return true;
|
|
5410
|
+
}
|
|
5411
|
+
return recoverPersistedTaskEnvironment(
|
|
5412
|
+
current,
|
|
5413
|
+
maintenanceReady,
|
|
5414
|
+
isCurrent,
|
|
5415
|
+
lifecycle,
|
|
5416
|
+
);
|
|
5417
|
+
})
|
|
5418
|
+
: await recoverPersistedTaskEnvironment(
|
|
5419
|
+
task,
|
|
5420
|
+
maintenanceReady,
|
|
5421
|
+
isCurrent,
|
|
5422
|
+
);
|
|
5423
|
+
if (recovered) {
|
|
5424
|
+
if (!lifecycle) finishTaskRecovery(task.taskId);
|
|
5425
|
+
} else {
|
|
5426
|
+
deferred.add(task.taskId);
|
|
5427
|
+
}
|
|
5428
|
+
} catch (err) {
|
|
5429
|
+
resolved = false;
|
|
5430
|
+
if (!lifecycle) finishTaskRecovery(task.taskId);
|
|
5431
|
+
console.error(
|
|
5432
|
+
`[orchestrator] recovery: ${task.taskId} failed:`,
|
|
5433
|
+
err instanceof Error ? err.message : err,
|
|
5434
|
+
);
|
|
5435
|
+
}
|
|
5436
|
+
}
|
|
5437
|
+
} catch (err) {
|
|
5438
|
+
resolved = false;
|
|
5439
|
+
if (!lifecycle) finishAllTaskRecovery();
|
|
5440
|
+
console.error(
|
|
5441
|
+
"[orchestrator] recovery: top-level failure",
|
|
5442
|
+
err instanceof Error ? err.message : err,
|
|
5443
|
+
);
|
|
5444
|
+
}
|
|
5445
|
+
return { deferred, resolved };
|
|
5446
|
+
}
|
|
5447
|
+
|
|
5448
|
+
/** Apple-backend variant of the writable-runtime quarantine. Task containers
|
|
5449
|
+
* prove their shared-cache mount read-only at creation, so a writable mount
|
|
5450
|
+
* here is either a live materializer that outlived its bound or tampering —
|
|
5451
|
+
* both are stopped (task-labeled rows report `stopped`). A CLI that cannot
|
|
5452
|
+
* answer yields `retry`, exactly like an unreachable Docker daemon; it is
|
|
5453
|
+
* never treated as an empty inventory. */
|
|
5454
|
+
async function quarantineWritableAppleRuntimeContainers(options: {
|
|
5455
|
+
lifecycle: Orchestrator;
|
|
5456
|
+
isCurrent?: () => boolean;
|
|
5457
|
+
scope?: "contract" | "volume-holders";
|
|
5458
|
+
}): Promise<WritableRuntimeQuarantineResult> {
|
|
5459
|
+
const isCurrent = options.isCurrent ?? (() => true);
|
|
5460
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [] };
|
|
5461
|
+
const binding = appleContainerRuntimeBinding();
|
|
5462
|
+
if (binding === null) return { verdict: "retry", stoppedTaskIds: [] };
|
|
5463
|
+
const cli = appleCliRunner(() => binding.containerCliPath);
|
|
5464
|
+
const listed = await cli(["list", "--all", "--format", "json"], {
|
|
5465
|
+
timeoutMs: 30_000,
|
|
5466
|
+
maxOutputBytes: 8 * 1024 * 1024,
|
|
5467
|
+
});
|
|
5468
|
+
if (listed.status !== 0 || listed.truncated === true) {
|
|
5469
|
+
return { verdict: "retry", stoppedTaskIds: [] };
|
|
5470
|
+
}
|
|
5471
|
+
let parsed: unknown;
|
|
5472
|
+
try {
|
|
5473
|
+
parsed = JSON.parse(listed.stdout);
|
|
5474
|
+
} catch {
|
|
5475
|
+
return { verdict: "retry", stoppedTaskIds: [] };
|
|
5476
|
+
}
|
|
5477
|
+
if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTaskIds: [] };
|
|
5478
|
+
const stoppedTaskIds = new Set<string>();
|
|
5479
|
+
let resolved = true;
|
|
5480
|
+
|
|
5481
|
+
const stopAndProve = async (id: string): Promise<boolean> => {
|
|
5482
|
+
const stopped = await cli(["stop", "--time", "30", id], {
|
|
5483
|
+
timeoutMs: 90_000,
|
|
5484
|
+
maxOutputBytes: 256 * 1024,
|
|
5485
|
+
});
|
|
5486
|
+
const after = await cli(["inspect", id], {
|
|
5487
|
+
timeoutMs: 15_000,
|
|
5488
|
+
maxOutputBytes: 1024 * 1024,
|
|
5489
|
+
});
|
|
5490
|
+
if (after.status !== 0) {
|
|
5491
|
+
return (
|
|
5492
|
+
after.stdout.trim() === "" &&
|
|
5493
|
+
after.stderr
|
|
5494
|
+
.split("\n")
|
|
5495
|
+
.some((line) => line.trim() === `Error: container not found: ${id}`)
|
|
5496
|
+
);
|
|
5497
|
+
}
|
|
5498
|
+
try {
|
|
5499
|
+
const parsedAfter = JSON.parse(after.stdout) as Array<{
|
|
5500
|
+
id?: unknown;
|
|
5501
|
+
status?: { state?: unknown };
|
|
5502
|
+
}>;
|
|
5503
|
+
// Only one exact-identity record in the runtime's terminal stopped
|
|
5504
|
+
// state settles the proof. A wrong-identity record, a transitional
|
|
5505
|
+
// state ("starting"), or an ambiguous shape must never release
|
|
5506
|
+
// shared-runtime maintenance while the writer may still be active.
|
|
5507
|
+
if (
|
|
5508
|
+
Array.isArray(parsedAfter) &&
|
|
5509
|
+
parsedAfter.length === 1 &&
|
|
5510
|
+
parsedAfter[0]?.id === id &&
|
|
5511
|
+
parsedAfter[0]?.status?.state === "stopped"
|
|
5512
|
+
) {
|
|
5513
|
+
return true;
|
|
5514
|
+
}
|
|
5515
|
+
} catch {
|
|
5516
|
+
// fall through to the failure log below
|
|
5517
|
+
}
|
|
5518
|
+
console.error(
|
|
5519
|
+
`[orchestrator] runtime isolation: apple container ${id} holds the shared runtime cache writable and could not be proven stopped (stop exit ${stopped.status})`,
|
|
5520
|
+
);
|
|
5521
|
+
return false;
|
|
5522
|
+
};
|
|
5523
|
+
|
|
5524
|
+
for (const item of parsed as Array<{
|
|
5525
|
+
configuration?: {
|
|
5526
|
+
id?: unknown;
|
|
5527
|
+
labels?: Record<string, unknown>;
|
|
5528
|
+
mounts?: Array<{
|
|
5529
|
+
destination?: unknown;
|
|
5530
|
+
options?: unknown;
|
|
5531
|
+
type?: { volume?: { name?: unknown } };
|
|
5532
|
+
}>;
|
|
5533
|
+
};
|
|
5534
|
+
}>) {
|
|
5535
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
5536
|
+
const id = item?.configuration?.id;
|
|
5537
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
5538
|
+
resolved = false;
|
|
5539
|
+
continue;
|
|
5540
|
+
}
|
|
5541
|
+
const mounts = item.configuration?.mounts;
|
|
5542
|
+
// Sweep every container whose shared-volume attachment violates the
|
|
5543
|
+
// CURRENT contract: writable (missing ro, or contradictory ro+rw), or at
|
|
5544
|
+
// any destination other than the ADR-108 staging path — which includes
|
|
5545
|
+
// every pre-ADR-108 container (read-only at /opt/asdf-data). Stopping
|
|
5546
|
+
// legacy containers HERE, before writer maintenance, is what lets a
|
|
5547
|
+
// legacy host migrate at all: a running legacy container would otherwise
|
|
5548
|
+
// hold the volume against the very reconcile whose completion recovery
|
|
5549
|
+
// needs (review 2026-08-18: maintenance retried forever and recovery —
|
|
5550
|
+
// the step that settles legacy containers — never ran).
|
|
5551
|
+
const writableShared = (Array.isArray(mounts) ? mounts : []).some(
|
|
5552
|
+
(mount) =>
|
|
5553
|
+
mount?.type?.volume?.name === "uai-asdf-data" &&
|
|
5554
|
+
(options.scope === "volume-holders" ||
|
|
5555
|
+
!(Array.isArray(mount.options) && mount.options.includes("ro")) ||
|
|
5556
|
+
(Array.isArray(mount.options) && mount.options.includes("rw")) ||
|
|
5557
|
+
mount?.destination !== "/run/uai/asdf-lower"),
|
|
5558
|
+
);
|
|
5559
|
+
if (!writableShared) continue;
|
|
5560
|
+
const label = item.configuration?.labels?.["com.uai.task"];
|
|
5561
|
+
// The label is container-influenced data. Only a syntactically safe task
|
|
5562
|
+
// id whose derived container name matches this exact container may drive
|
|
5563
|
+
// task-scoped mutations (lifecycle slot, DB row, ACL removal); anything
|
|
5564
|
+
// else is stopped as an anonymous writer, nothing task-scoped touched.
|
|
5565
|
+
const taskId =
|
|
5566
|
+
typeof label === "string" &&
|
|
5567
|
+
isSafeHostTaskId(label) &&
|
|
5568
|
+
id === appleTaskContainerName(label)
|
|
5569
|
+
? label
|
|
5570
|
+
: null;
|
|
5571
|
+
if (taskId === null) {
|
|
5572
|
+
if (!(await stopAndProve(id))) resolved = false;
|
|
5573
|
+
continue;
|
|
5574
|
+
}
|
|
5575
|
+
// Task-labeled writers stop inside the task's lifecycle slot so a
|
|
5576
|
+
// concurrent taskUp/taskDown cannot interleave, and a known row commits
|
|
5577
|
+
// its durable `stopped` verdict before the frame is queued — a crashed
|
|
5578
|
+
// publisher must never leave SQLite saying running.
|
|
5579
|
+
await options.lifecycle.runTaskLifecycle(taskId, async () => {
|
|
5580
|
+
if (!isCurrent()) return;
|
|
5581
|
+
await options.lifecycle.quarantineChannelForRecovery(taskId);
|
|
5582
|
+
if (!(await stopAndProve(id))) {
|
|
5583
|
+
resolved = false;
|
|
5584
|
+
return;
|
|
5585
|
+
}
|
|
5586
|
+
const task = getHostTask(taskId);
|
|
5587
|
+
if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
|
|
5588
|
+
db_setStatus(taskId, "stopped", {
|
|
5589
|
+
codeServerPort: null,
|
|
5590
|
+
previewPorts: "[]",
|
|
5591
|
+
});
|
|
5592
|
+
stoppedTaskIds.add(taskId);
|
|
5593
|
+
}
|
|
5594
|
+
// Revoke live access with the environment, like the Docker sweep.
|
|
5595
|
+
clearTaskGatewayAcl(taskId);
|
|
5596
|
+
});
|
|
5597
|
+
}
|
|
5598
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTaskIds: [...stoppedTaskIds] };
|
|
5599
|
+
return {
|
|
5600
|
+
verdict: resolved ? "ready" : "retry",
|
|
5601
|
+
stoppedTaskIds: [...stoppedTaskIds],
|
|
5602
|
+
};
|
|
5603
|
+
}
|
|
5604
|
+
|
|
5605
|
+
/** Dispatch one durable row through its provider. Generic boot orchestration
|
|
5606
|
+
* owns DB/lifecycle sequencing but never derives a Compose/container identity. */
|
|
5607
|
+
async function recoverPersistedTaskEnvironment(
|
|
5608
|
+
task: typeof schema.hostTasks.$inferSelect,
|
|
5609
|
+
maintenanceReady: Promise<void>,
|
|
5610
|
+
isCurrent: () => boolean,
|
|
5611
|
+
lifecycle?: Orchestrator,
|
|
5612
|
+
): Promise<boolean> {
|
|
5613
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
5614
|
+
if (environment === null) return true;
|
|
5615
|
+
if (!isCurrent()) return true;
|
|
5616
|
+
const result = await environment.recover({
|
|
5617
|
+
maintenanceReady,
|
|
5618
|
+
isCurrent,
|
|
5619
|
+
...(lifecycle
|
|
5620
|
+
? {
|
|
5621
|
+
quarantineConsumers: () =>
|
|
5622
|
+
lifecycle.quarantineChannelForRecovery(task.taskId),
|
|
5623
|
+
allowConsumers: () => lifecycle.allowChannel(task.taskId),
|
|
5624
|
+
ensureConsumersStarted: async () => {
|
|
5625
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
5626
|
+
},
|
|
5627
|
+
}
|
|
5628
|
+
: {}),
|
|
5629
|
+
});
|
|
5630
|
+
return result.outcome === "recovered";
|
|
5631
|
+
}
|
|
5632
|
+
|
|
5633
|
+
interface RecoveryTaskConsumers {
|
|
5634
|
+
quarantineChannelForRecovery(taskId: string): Promise<void>;
|
|
5635
|
+
allowChannel(taskId: string): void;
|
|
5636
|
+
ensureStarted(taskId: string): Promise<boolean | void>;
|
|
5637
|
+
}
|
|
5638
|
+
|
|
5639
|
+
/** Apple provider driver. Recovery must end in a durable truth: a running
|
|
5640
|
+
* container is re-initialized (dependencies + code-server) under an
|
|
5641
|
+
* in-container deadline; a stopped one is started and initialized; a
|
|
5642
|
+
* container that is provenly absent — or cannot be started — marks the task
|
|
5643
|
+
* `stopped` so the host never reports operational with a phantom running
|
|
5644
|
+
* task. Unknown CLI answers stay deferred and retryable. */
|
|
5645
|
+
/** Exact TypeScript port of task-up.sh's container environment-authority
|
|
5646
|
+
* proof (the jq filter behind "verify task runtime isolation"): each required
|
|
5647
|
+
* canonical entry appears exactly once, every owned-npm-config spelling
|
|
5648
|
+
* (case-folded, underscores treated as hyphens) must be one of the canonical
|
|
5649
|
+
* eight, and PATH plus the asdf/corepack controls hold their exact values
|
|
5650
|
+
* exactly once. */
|
|
5651
|
+
export function appleTaskEnvironmentAuthorityHolds(value: unknown): boolean {
|
|
5652
|
+
if (!Array.isArray(value) || !value.every((entry) => typeof entry === "string")) {
|
|
5653
|
+
return false;
|
|
5654
|
+
}
|
|
5655
|
+
const environment = value as string[];
|
|
5656
|
+
const npmPrefix = "/home/node/.local/share/uai-npm";
|
|
5657
|
+
const npmCache = "/home/node/.cache/uai-npm";
|
|
5658
|
+
const runtimePath =
|
|
5659
|
+
"/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:" +
|
|
5660
|
+
`/usr/sbin:/usr/bin:/sbin:/bin:${npmPrefix}/bin`;
|
|
5661
|
+
const required = [
|
|
5662
|
+
`NPM_CONFIG_PREFIX=${npmPrefix}`,
|
|
5663
|
+
`npm_config_prefix=${npmPrefix}`,
|
|
5664
|
+
`NPM_CONFIG_CACHE=${npmCache}`,
|
|
5665
|
+
`npm_config_cache=${npmCache}`,
|
|
5666
|
+
`NPM_CONFIG_LOGS_DIR=${npmCache}/_logs`,
|
|
5667
|
+
`npm_config_logs_dir=${npmCache}/_logs`,
|
|
5668
|
+
"NPM_CONFIG_UMASK=0002",
|
|
5669
|
+
"npm_config_umask=0002",
|
|
5670
|
+
];
|
|
5671
|
+
const owned = ["prefix", "cache", "logs-dir", "umask"];
|
|
5672
|
+
const countOf = (exact: string): number =>
|
|
5673
|
+
environment.filter((entry) => entry === exact).length;
|
|
5674
|
+
const isOwnedNpmConfig = (key: string): boolean => {
|
|
5675
|
+
const folded = key.toLowerCase();
|
|
5676
|
+
return (
|
|
5677
|
+
folded.startsWith("npm_config_") &&
|
|
5678
|
+
owned.includes(folded.slice("npm_config_".length).replace(/_/g, "-"))
|
|
5679
|
+
);
|
|
5680
|
+
};
|
|
5681
|
+
const ownedEntries = environment.filter((entry) =>
|
|
5682
|
+
isOwnedNpmConfig(entry.split("=")[0]!),
|
|
5683
|
+
);
|
|
5684
|
+
return (
|
|
5685
|
+
required.every((entry) => countOf(entry) === 1) &&
|
|
5686
|
+
ownedEntries.length === required.length &&
|
|
5687
|
+
countOf(`PATH=${runtimePath}`) === 1 &&
|
|
5688
|
+
countOf("ASDF_DATA_DIR=/opt/asdf-data") === 1 &&
|
|
5689
|
+
countOf("ASDF_SKIP_RESHIM=1") === 1 &&
|
|
5690
|
+
countOf("COREPACK_HOME=/opt/asdf-data/corepack") === 1 &&
|
|
5691
|
+
countOf("COREPACK_ENABLE_NETWORK=0") === 1
|
|
5692
|
+
);
|
|
5693
|
+
}
|
|
5694
|
+
|
|
5695
|
+
export type AppleTaskRuntimeContractVerdict =
|
|
5696
|
+
| { kind: "holds" }
|
|
5697
|
+
| { kind: "violated" }
|
|
5698
|
+
| { kind: "unreachable"; detail: string };
|
|
5699
|
+
|
|
5700
|
+
/**
|
|
5701
|
+
* The apple runtime-authority contract, in one inspection: this exact name
|
|
5702
|
+
* carries this task's label, is running, mounts the shared cache read-only
|
|
5703
|
+
* at exactly the ADR-108 staging path (and nowhere else, with no
|
|
5704
|
+
* contradictory options), binds /workspace as exactly one writable virtiofs
|
|
5705
|
+
* share of this task's real worktree leaf, and carries the same immutable
|
|
5706
|
+
* npm/runtime environment authority task-up proved at creation. Factored
|
|
5707
|
+
* from recovery so the taskUp lost-response fast path proves the SAME
|
|
5708
|
+
* invariants (review 2026-08-18 round 4). `unreachable` and `violated` are
|
|
5709
|
+
* distinct so the fast path can retry without stopping a container it could
|
|
5710
|
+
* not observe, while recovery keeps treating both as quarantine grounds.
|
|
5711
|
+
*/
|
|
5712
|
+
export async function proveAppleTaskRuntimeContract(options: {
|
|
5713
|
+
taskId: string;
|
|
5714
|
+
containerName: string;
|
|
5715
|
+
hostWorktreePath: string;
|
|
5716
|
+
cli?: ReturnType<typeof appleCliRunner>;
|
|
5717
|
+
}): Promise<AppleTaskRuntimeContractVerdict> {
|
|
5718
|
+
let cli = options.cli;
|
|
5719
|
+
if (!cli) {
|
|
5720
|
+
const binding = appleContainerRuntimeBinding();
|
|
5721
|
+
if (binding === null) {
|
|
5722
|
+
return { kind: "unreachable", detail: "apple runtime binding unavailable" };
|
|
5723
|
+
}
|
|
5724
|
+
cli = appleCliRunner(() => binding.containerCliPath);
|
|
5725
|
+
}
|
|
5726
|
+
const contract = await cli(["inspect", options.containerName], {
|
|
5727
|
+
timeoutMs: 15_000,
|
|
5728
|
+
maxOutputBytes: 4 * 1024 * 1024,
|
|
5729
|
+
});
|
|
5730
|
+
if (contract.status !== 0) {
|
|
5731
|
+
return {
|
|
5732
|
+
kind: "unreachable",
|
|
5733
|
+
detail:
|
|
5734
|
+
contract.stderr.trim().slice(0, 200) ||
|
|
5735
|
+
`inspect exited ${contract.status ?? "killed"}`,
|
|
5736
|
+
};
|
|
5737
|
+
}
|
|
5738
|
+
try {
|
|
5739
|
+
const records = JSON.parse(contract.stdout) as Array<{
|
|
5740
|
+
id?: unknown;
|
|
5741
|
+
status?: { state?: unknown };
|
|
5742
|
+
configuration?: {
|
|
5743
|
+
labels?: Record<string, unknown>;
|
|
5744
|
+
mounts?: Array<{
|
|
5745
|
+
destination?: unknown;
|
|
5746
|
+
options?: unknown;
|
|
5747
|
+
type?: { volume?: { name?: unknown } };
|
|
5748
|
+
}>;
|
|
5749
|
+
initProcess?: { environment?: unknown };
|
|
5750
|
+
};
|
|
5751
|
+
}>;
|
|
5752
|
+
// Same strictness as the provider's inspection: exactly one record and
|
|
5753
|
+
// it must identify itself as the requested container, or a malformed
|
|
5754
|
+
// response describing another container could authorize execution.
|
|
5755
|
+
if (!Array.isArray(records) || records.length !== 1) {
|
|
5756
|
+
throw new Error("contract inspection shape is invalid");
|
|
5757
|
+
}
|
|
5758
|
+
const record = records[0]!;
|
|
5759
|
+
if (record.id !== options.containerName) {
|
|
5760
|
+
throw new Error("contract inspection identity mismatch");
|
|
5761
|
+
}
|
|
5762
|
+
const mounts = record?.configuration?.mounts ?? [];
|
|
5763
|
+
// ADR-108: the shared volume attaches read-only at the overlay staging
|
|
5764
|
+
// path and /opt/asdf-data must NOT be a host mount at all — the
|
|
5765
|
+
// writable view there is the in-guest overlay.
|
|
5766
|
+
const runtimeMounts = mounts.filter(
|
|
5767
|
+
(mount) => mount?.destination === "/run/uai/asdf-lower",
|
|
5768
|
+
);
|
|
5769
|
+
const sharedMounts = mounts.filter(
|
|
5770
|
+
(mount) => mount?.type?.volume?.name === "uai-asdf-data",
|
|
5771
|
+
);
|
|
5772
|
+
const asdfHostMounts = mounts.filter(
|
|
5773
|
+
(mount) => mount?.destination === "/opt/asdf-data",
|
|
5774
|
+
);
|
|
5775
|
+
// The /workspace authority: exactly one bind (never a volume) whose
|
|
5776
|
+
// source is this task's exact worktree leaf, still a real non-symlink
|
|
5777
|
+
// directory on disk — the same proof Docker recovery performs.
|
|
5778
|
+
const workspaceMounts = mounts.filter(
|
|
5779
|
+
(mount) => mount?.destination === "/workspace",
|
|
5780
|
+
);
|
|
5781
|
+
const expectedWorkspaceSource = join(
|
|
5782
|
+
options.hostWorktreePath,
|
|
5783
|
+
"workspace",
|
|
5784
|
+
);
|
|
5785
|
+
// The verified Apple dialect for a host-directory bind is exactly
|
|
5786
|
+
// type:{virtiofs:{}} — a closed shape.
|
|
5787
|
+
const workspaceType = workspaceMounts[0]?.type as
|
|
5788
|
+
| Record<string, unknown>
|
|
5789
|
+
| undefined;
|
|
5790
|
+
const workspaceTypeIsVirtiofs =
|
|
5791
|
+
workspaceType !== undefined &&
|
|
5792
|
+
typeof workspaceType === "object" &&
|
|
5793
|
+
!Array.isArray(workspaceType) &&
|
|
5794
|
+
Object.keys(workspaceType).length === 1 &&
|
|
5795
|
+
typeof workspaceType.virtiofs === "object" &&
|
|
5796
|
+
workspaceType.virtiofs !== null &&
|
|
5797
|
+
!Array.isArray(workspaceType.virtiofs) &&
|
|
5798
|
+
Object.keys(workspaceType.virtiofs as object).length === 0;
|
|
5799
|
+
// Writable is part of the authority: a read-only /workspace would
|
|
5800
|
+
// expose a task whose agents cannot work.
|
|
5801
|
+
const workspaceOptions = workspaceMounts[0]?.options;
|
|
5802
|
+
let workspaceAuthorityHolds =
|
|
5803
|
+
workspaceMounts.length === 1 &&
|
|
5804
|
+
workspaceTypeIsVirtiofs &&
|
|
5805
|
+
Array.isArray(workspaceOptions) &&
|
|
5806
|
+
!(workspaceOptions as unknown[]).includes("ro") &&
|
|
5807
|
+
(workspaceMounts[0] as { source?: unknown } | undefined)?.source ===
|
|
5808
|
+
expectedWorkspaceSource;
|
|
5809
|
+
if (workspaceAuthorityHolds) {
|
|
5810
|
+
try {
|
|
5811
|
+
const workspaceStat = lstatSync(expectedWorkspaceSource);
|
|
5812
|
+
workspaceAuthorityHolds =
|
|
5813
|
+
workspaceStat.isDirectory() && !workspaceStat.isSymbolicLink();
|
|
5814
|
+
} catch {
|
|
5815
|
+
workspaceAuthorityHolds = false;
|
|
5816
|
+
}
|
|
5817
|
+
}
|
|
5818
|
+
const rawEnvironment = record?.configuration?.initProcess?.environment;
|
|
5819
|
+
const holds =
|
|
5820
|
+
record?.status?.state === "running" &&
|
|
5821
|
+
record?.configuration?.labels?.["com.uai.task"] === options.taskId &&
|
|
5822
|
+
runtimeMounts.length === 1 &&
|
|
5823
|
+
sharedMounts.length === 1 &&
|
|
5824
|
+
sharedMounts[0] === runtimeMounts[0] &&
|
|
5825
|
+
asdfHostMounts.length === 0 &&
|
|
5826
|
+
Array.isArray(runtimeMounts[0]?.options) &&
|
|
5827
|
+
(runtimeMounts[0]!.options as unknown[]).includes("ro") &&
|
|
5828
|
+
// Closed options contract: an array carrying BOTH ro and rw is
|
|
5829
|
+
// contradictory and proves nothing.
|
|
5830
|
+
!(runtimeMounts[0]!.options as unknown[]).includes("rw") &&
|
|
5831
|
+
workspaceAuthorityHolds &&
|
|
5832
|
+
appleTaskEnvironmentAuthorityHolds(rawEnvironment);
|
|
5833
|
+
return holds ? { kind: "holds" } : { kind: "violated" };
|
|
5834
|
+
} catch {
|
|
5835
|
+
return { kind: "violated" };
|
|
5836
|
+
}
|
|
5837
|
+
}
|
|
5838
|
+
|
|
5839
|
+
export async function recoverAppleTaskEnvironment(
|
|
5840
|
+
descriptor: TaskEnvironmentDescriptor,
|
|
5841
|
+
context: TaskEnvironmentRecoveryContext,
|
|
5842
|
+
): Promise<TaskEnvironmentRecoveryResult> {
|
|
5843
|
+
const locator = parseAppleTaskEnvironmentLocator(descriptor.locator);
|
|
5844
|
+
const task = getHostTask(descriptor.taskId);
|
|
5845
|
+
if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
|
|
5846
|
+
return { outcome: "recovered" };
|
|
5847
|
+
}
|
|
5848
|
+
// The boot maintenance latch (agentClisReady) is awaited ONLY before the
|
|
5849
|
+
// first in-container exec, exactly like Docker recovery. Awaiting it here
|
|
5850
|
+
// deadlocked the first fresh Apple host (live 2026-08-15): the on-connect
|
|
5851
|
+
// recovery sweep held this driver on the latch, activation joins in-flight
|
|
5852
|
+
// recovery before its maintenance, and the maintenance that settles the
|
|
5853
|
+
// latch could therefore never run — the host sat silently in `checking`
|
|
5854
|
+
// forever. Status verdicts, quarantines, and workspace settlement need no
|
|
5855
|
+
// maintenance and must never wait on it.
|
|
5856
|
+
if (!context.isCurrent()) {
|
|
5857
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5858
|
+
}
|
|
5859
|
+
const binding = appleContainerRuntimeBinding();
|
|
5860
|
+
if (binding === null) {
|
|
5861
|
+
return { outcome: "deferred", detail: "apple runtime binding unavailable" };
|
|
5862
|
+
}
|
|
5863
|
+
const cli = appleCliRunner(() => binding.containerCliPath);
|
|
5864
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
5865
|
+
if (environment === null) return { outcome: "recovered" };
|
|
5866
|
+
|
|
5867
|
+
const markStopped = (): TaskEnvironmentRecoveryResult => {
|
|
5868
|
+
db_setStatus(descriptor.taskId, "stopped", {
|
|
5869
|
+
codeServerPort: null,
|
|
5870
|
+
previewPorts: "[]",
|
|
5871
|
+
});
|
|
5872
|
+
// A stopped verdict must also revoke live access paths, exactly like the
|
|
5873
|
+
// Docker quarantine: the MCP gateway token dies with the environment.
|
|
5874
|
+
clearTaskGatewayAcl(descriptor.taskId);
|
|
5875
|
+
return { outcome: "recovered" };
|
|
5876
|
+
};
|
|
5877
|
+
// Without its workspace the task cannot resume — stopped would advertise a
|
|
5878
|
+
// restart that must fail. Docker recovery reconciles this as error.
|
|
5879
|
+
const markError = (): TaskEnvironmentRecoveryResult => {
|
|
5880
|
+
db_setStatus(descriptor.taskId, "error", {
|
|
5881
|
+
codeServerPort: null,
|
|
5882
|
+
previewPorts: "[]",
|
|
5883
|
+
composeProject: null,
|
|
5884
|
+
});
|
|
5885
|
+
clearTaskGatewayAcl(descriptor.taskId);
|
|
5886
|
+
return { outcome: "recovered" };
|
|
5887
|
+
};
|
|
5888
|
+
// Every settled verdict consults the freshest workspace observation: a
|
|
5889
|
+
// workspace that vanished mid-recovery must land as error exactly like one
|
|
5890
|
+
// that was already gone at the first look.
|
|
5891
|
+
const settleFromStatus = (settled: {
|
|
5892
|
+
workspacePresent: boolean;
|
|
5893
|
+
}): TaskEnvironmentRecoveryResult =>
|
|
5894
|
+
settled.workspacePresent ? markStopped() : markError();
|
|
5895
|
+
|
|
5896
|
+
let status = await environment.status();
|
|
5897
|
+
if (status.state === "unknown") {
|
|
5898
|
+
return { outcome: "deferred", detail: status.detail };
|
|
5899
|
+
}
|
|
5900
|
+
// Every awaited runtime observation can outlive this recovery's readiness
|
|
5901
|
+
// generation; a superseded pass must observe only, never mutate. Recheck
|
|
5902
|
+
// after each await and before each mutation from here on.
|
|
5903
|
+
if (!context.isCurrent()) {
|
|
5904
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5905
|
+
}
|
|
5906
|
+
// Drain consumers before ANY durable verdict or mutation — including the
|
|
5907
|
+
// absent path, where a live channel could otherwise keep serving a task
|
|
5908
|
+
// whose row is about to say stopped.
|
|
5909
|
+
await context.quarantineConsumers?.();
|
|
5910
|
+
if (!context.isCurrent()) {
|
|
5911
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5912
|
+
}
|
|
5913
|
+
if (status.state === "absent") {
|
|
5914
|
+
// Name-anchored absence came from the provider's inspect. The disk truth
|
|
5915
|
+
// decides the verdict: an intact workspace resumes via re-provision
|
|
5916
|
+
// (stopped); a vanished one cannot, and stopped would advertise a
|
|
5917
|
+
// restart that must fail (error).
|
|
5918
|
+
return status.workspacePresent ? markStopped() : markError();
|
|
5919
|
+
}
|
|
5920
|
+
if (status.state === "stopped" && !status.workspacePresent) {
|
|
5921
|
+
// Never start a container whose workspace is gone: dependency recovery
|
|
5922
|
+
// would mutate a recreated or missing tree.
|
|
5923
|
+
return markError();
|
|
5924
|
+
}
|
|
5925
|
+
if (status.state === "stopped") {
|
|
5926
|
+
const started = await cli(["start", locator.containerName], {
|
|
5927
|
+
timeoutMs: 120_000,
|
|
5928
|
+
maxOutputBytes: 256 * 1024,
|
|
5929
|
+
});
|
|
5930
|
+
if (!context.isCurrent()) {
|
|
5931
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
3622
5932
|
}
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
return;
|
|
5933
|
+
// A failed start command or an ambiguous follow-up is NOT proof of a
|
|
5934
|
+
// stopped container: only an exact stopped/absent answer may settle the
|
|
5935
|
+
// durable row, and unknown always stays deferred and retryable.
|
|
5936
|
+
status = await environment.status();
|
|
5937
|
+
if (status.state === "unknown") {
|
|
5938
|
+
return { outcome: "deferred", detail: status.detail };
|
|
3630
5939
|
}
|
|
3631
|
-
if (
|
|
3632
|
-
|
|
3633
|
-
|
|
5940
|
+
if (!context.isCurrent()) {
|
|
5941
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5942
|
+
}
|
|
5943
|
+
if (status.state === "stopped" || status.state === "absent") {
|
|
5944
|
+
return settleFromStatus(status);
|
|
5945
|
+
}
|
|
5946
|
+
if (started.status !== 0) {
|
|
5947
|
+
// The CLI reported failure yet the container runs — trust the state
|
|
5948
|
+
// probe over the command's exit and continue into revalidation.
|
|
5949
|
+
console.warn(
|
|
5950
|
+
`[orchestrator] recovery: ${descriptor.taskId} container start exited ${started.status} but the container is running; continuing`,
|
|
3634
5951
|
);
|
|
3635
5952
|
}
|
|
3636
|
-
await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
3637
5953
|
}
|
|
3638
|
-
}
|
|
3639
5954
|
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
const
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
const
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
5955
|
+
// One inspection proves the whole trust decision — factored into
|
|
5956
|
+
// proveAppleTaskRuntimeContract so the taskUp lost-response fast path can
|
|
5957
|
+
// prove the SAME invariants (review 2026-08-18 round 4). Recovery keeps
|
|
5958
|
+
// its historical mapping: anything short of `holds` — including an
|
|
5959
|
+
// unreachable inspect — quarantines below.
|
|
5960
|
+
const contractVerdict = await proveAppleTaskRuntimeContract({
|
|
5961
|
+
taskId: descriptor.taskId,
|
|
5962
|
+
containerName: locator.containerName,
|
|
5963
|
+
hostWorktreePath: locator.hostWorktreePath,
|
|
5964
|
+
cli,
|
|
5965
|
+
});
|
|
5966
|
+
const contractHolds = contractVerdict.kind === "holds";
|
|
5967
|
+
if (!context.isCurrent()) {
|
|
5968
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5969
|
+
}
|
|
5970
|
+
if (!contractHolds) {
|
|
5971
|
+
await environment.stop();
|
|
5972
|
+
const after = await environment.status();
|
|
5973
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
5974
|
+
return {
|
|
5975
|
+
outcome: "deferred",
|
|
5976
|
+
detail:
|
|
5977
|
+
"task container failed its runtime-authority contract and could not be proven stopped",
|
|
5978
|
+
};
|
|
5979
|
+
}
|
|
5980
|
+
if (!context.isCurrent()) {
|
|
5981
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5982
|
+
}
|
|
5983
|
+
console.error(
|
|
5984
|
+
`[orchestrator] recovery: ${descriptor.taskId} container failed its runtime-authority contract; task was stopped and remains quarantined`,
|
|
3666
5985
|
);
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
5986
|
+
return settleFromStatus(after);
|
|
5987
|
+
}
|
|
5988
|
+
// Even an already-running container's first exec resolves through shared
|
|
5989
|
+
// asdf shims; fence it behind the one-way boot maintenance latch, exactly
|
|
5990
|
+
// like Docker recovery fences its workbench probe.
|
|
5991
|
+
await context.maintenanceReady;
|
|
5992
|
+
if (!context.isCurrent()) {
|
|
5993
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
5994
|
+
}
|
|
5995
|
+
// ADR-108: guest mounts do not survive a VM restart, so the runtime
|
|
5996
|
+
// overlay must be re-established (idempotently) before ANY exec resolves
|
|
5997
|
+
// through /opt/asdf-data shims — including uai-init below. The upper layer
|
|
5998
|
+
// persists in the container filesystem, so the task's own installs return
|
|
5999
|
+
// with the mount. Proven from the guest mount table, exactly like task-up.
|
|
6000
|
+
const overlay = await environment.exec({
|
|
6001
|
+
argv: [
|
|
6002
|
+
"/bin/sh",
|
|
6003
|
+
"-c",
|
|
6004
|
+
// Same establishment + token-exact proof as task-up §6a: exactly one
|
|
6005
|
+
// effective mount at the merged path and the lower, comma-delimited
|
|
6006
|
+
// option tokens (substring matching accepted a -evil suffix), a
|
|
6007
|
+
// read-only lower, and a real node-owned upper.
|
|
6008
|
+
"set -e; /bin/mkdir -p /var/lib/uai/asdf-upper /var/lib/uai/asdf-work /opt/asdf-data; " +
|
|
6009
|
+
"/bin/chown node:node /var/lib/uai/asdf-upper; " +
|
|
6010
|
+
'if ! /bin/grep -q "^overlay /opt/asdf-data overlay " /proc/mounts; then ' +
|
|
6011
|
+
"/bin/mount -t overlay overlay -o lowerdir=/run/uai/asdf-lower,upperdir=/var/lib/uai/asdf-upper,workdir=/var/lib/uai/asdf-work /opt/asdf-data; fi; " +
|
|
6012
|
+
'overlay_lines=$(/bin/grep -c "^overlay /opt/asdf-data overlay " /proc/mounts || true); ' +
|
|
6013
|
+
'[ "$overlay_lines" = "1" ] || { echo "expected exactly one /opt/asdf-data overlay, found $overlay_lines" >&2; exit 1; }; ' +
|
|
6014
|
+
'any_lines=$(/bin/grep -c " /opt/asdf-data " /proc/mounts || true); ' +
|
|
6015
|
+
'[ "$any_lines" = "1" ] || { echo "unexpected extra mounts at /opt/asdf-data" >&2; exit 1; }; ' +
|
|
6016
|
+
'opts=$(/bin/grep "^overlay /opt/asdf-data overlay " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
|
|
6017
|
+
'case ",$opts," in *",lowerdir=/run/uai/asdf-lower,"*) ;; *) echo "overlay lower is foreign" >&2; exit 1 ;; esac; ' +
|
|
6018
|
+
'case ",$opts," in *",upperdir=/var/lib/uai/asdf-upper,"*) ;; *) echo "overlay upper is foreign" >&2; exit 1 ;; esac; ' +
|
|
6019
|
+
'case ",$opts," in *",workdir=/var/lib/uai/asdf-work,"*) ;; *) echo "overlay workdir is foreign" >&2; exit 1 ;; esac; ' +
|
|
6020
|
+
'case ",$opts," in *",ro,"*) echo "overlay is unexpectedly read-only" >&2; exit 1 ;; esac; ' +
|
|
6021
|
+
'lower_lines=$(/bin/grep -c " /run/uai/asdf-lower " /proc/mounts || true); ' +
|
|
6022
|
+
'[ "$lower_lines" = "1" ] || { echo "expected exactly one lower attachment, found $lower_lines" >&2; exit 1; }; ' +
|
|
6023
|
+
'lopts=$(/bin/grep " /run/uai/asdf-lower " /proc/mounts | /usr/bin/cut -d" " -f4); ' +
|
|
6024
|
+
'case ",$lopts," in *",ro,"*) ;; *) echo "lower attachment is not read-only" >&2; exit 1 ;; esac; ' +
|
|
6025
|
+
'[ ! -L /var/lib/uai/asdf-upper ] || { echo "upper is a symlink" >&2; exit 1; }; ' +
|
|
6026
|
+
'[ -d /var/lib/uai/asdf-upper ] || { echo "upper is not a directory" >&2; exit 1; }; ' +
|
|
6027
|
+
'[ "$(/usr/bin/stat -c %U /var/lib/uai/asdf-upper)" = "node" ] || { echo "upper is not node-owned" >&2; exit 1; }; ' +
|
|
6028
|
+
"/usr/bin/test -d /opt/asdf-data/plugins",
|
|
6029
|
+
],
|
|
6030
|
+
user: "root",
|
|
6031
|
+
env: {},
|
|
6032
|
+
timeoutMs: 30_000,
|
|
6033
|
+
maxOutputBytes: 64 * 1024,
|
|
6034
|
+
});
|
|
6035
|
+
if (!context.isCurrent()) {
|
|
6036
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6037
|
+
}
|
|
6038
|
+
if (overlay.exitCode !== 0 || overlay.signal !== null) {
|
|
6039
|
+
await environment.stop();
|
|
6040
|
+
const afterOverlay = await environment.status();
|
|
6041
|
+
if (afterOverlay.state === "running" || afterOverlay.state === "unknown") {
|
|
6042
|
+
return {
|
|
6043
|
+
outcome: "deferred",
|
|
6044
|
+
detail:
|
|
6045
|
+
"task runtime overlay could not be established and the container could not be proven stopped",
|
|
6046
|
+
};
|
|
6047
|
+
}
|
|
6048
|
+
if (!context.isCurrent()) {
|
|
6049
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
3678
6050
|
}
|
|
3679
|
-
} catch (err) {
|
|
3680
|
-
finishAllTaskRecovery();
|
|
3681
6051
|
console.error(
|
|
3682
|
-
|
|
3683
|
-
|
|
6052
|
+
`[orchestrator] recovery: ${descriptor.taskId} runtime overlay could not be established (exit ${overlay.exitCode}, signal ${overlay.signal}); task was stopped`,
|
|
6053
|
+
);
|
|
6054
|
+
return settleFromStatus(afterOverlay);
|
|
6055
|
+
}
|
|
6056
|
+
// Dependencies + code-server. The in-container deadline (TERM then KILL)
|
|
6057
|
+
// means a host-side expiry can never leave the initializer mutating the
|
|
6058
|
+
// workspace while agents use it; exit 78 is a runtime-authority violation
|
|
6059
|
+
// and quarantines exactly like at task-up.
|
|
6060
|
+
const init = await environment.exec({
|
|
6061
|
+
argv: [
|
|
6062
|
+
"/usr/bin/timeout",
|
|
6063
|
+
"--signal=TERM",
|
|
6064
|
+
"--kill-after=15s",
|
|
6065
|
+
"570s",
|
|
6066
|
+
"/usr/local/bin/uai-init",
|
|
6067
|
+
],
|
|
6068
|
+
env: {},
|
|
6069
|
+
timeoutMs: 10 * 60_000,
|
|
6070
|
+
maxOutputBytes: 1024 * 1024,
|
|
6071
|
+
});
|
|
6072
|
+
if (!context.isCurrent()) {
|
|
6073
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6074
|
+
}
|
|
6075
|
+
if (
|
|
6076
|
+
init.exitCode === null ||
|
|
6077
|
+
init.signal !== null ||
|
|
6078
|
+
init.stdoutTruncated ||
|
|
6079
|
+
init.stderrTruncated
|
|
6080
|
+
) {
|
|
6081
|
+
// The host-side client died (deadline, overflow, forced settlement)
|
|
6082
|
+
// without a real exit code, so the in-container initializer may STILL be
|
|
6083
|
+
// mutating dependencies for several minutes. Never expose the task on an
|
|
6084
|
+
// unknowable outcome: stop and prove, or stay deferred.
|
|
6085
|
+
await environment.stop();
|
|
6086
|
+
const after = await environment.status();
|
|
6087
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
6088
|
+
return {
|
|
6089
|
+
outcome: "deferred",
|
|
6090
|
+
detail:
|
|
6091
|
+
"task initializer outcome was unknowable and the container could not be proven stopped",
|
|
6092
|
+
};
|
|
6093
|
+
}
|
|
6094
|
+
if (!context.isCurrent()) {
|
|
6095
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6096
|
+
}
|
|
6097
|
+
console.error(
|
|
6098
|
+
`[orchestrator] recovery: ${descriptor.taskId} initializer outcome was unknowable (exit ${init.exitCode}, signal ${init.signal}); task was stopped pending a clean re-run`,
|
|
6099
|
+
);
|
|
6100
|
+
return settleFromStatus(after);
|
|
6101
|
+
}
|
|
6102
|
+
if (init.exitCode === 78) {
|
|
6103
|
+
await environment.stop();
|
|
6104
|
+
const after = await environment.status();
|
|
6105
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
6106
|
+
return {
|
|
6107
|
+
outcome: "deferred",
|
|
6108
|
+
detail:
|
|
6109
|
+
"runtime-authority violation detected and the container could not be proven stopped",
|
|
6110
|
+
};
|
|
6111
|
+
}
|
|
6112
|
+
if (!context.isCurrent()) {
|
|
6113
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6114
|
+
}
|
|
6115
|
+
console.error(
|
|
6116
|
+
`[orchestrator] recovery: ${descriptor.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
|
|
6117
|
+
);
|
|
6118
|
+
return settleFromStatus(after);
|
|
6119
|
+
}
|
|
6120
|
+
if (init.exitCode !== 0) {
|
|
6121
|
+
console.warn(
|
|
6122
|
+
`[orchestrator] recovery: ${descriptor.taskId} uai-init failed (exit ${init.exitCode}); container is up but Editor may be down`,
|
|
6123
|
+
);
|
|
6124
|
+
}
|
|
6125
|
+
if (!context.isCurrent()) {
|
|
6126
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6127
|
+
}
|
|
6128
|
+
db_setRuntime(descriptor.taskId, { codeServerPort: null });
|
|
6129
|
+
context.allowConsumers?.();
|
|
6130
|
+
await context.ensureConsumersStarted?.();
|
|
6131
|
+
if (!context.isCurrent()) {
|
|
6132
|
+
// Admission reopened against a superseded generation: re-quarantine so
|
|
6133
|
+
// the successor generation re-proves the environment before consumers
|
|
6134
|
+
// touch it — the same closing move Docker recovery makes.
|
|
6135
|
+
await context.quarantineConsumers?.();
|
|
6136
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
6137
|
+
}
|
|
6138
|
+
return { outcome: "recovered" };
|
|
6139
|
+
}
|
|
6140
|
+
|
|
6141
|
+
/** Docker provider driver. The legacy repair implementation remains local to
|
|
6142
|
+
* this module for now, but it is reachable only through the provider handle;
|
|
6143
|
+
* the generic recovery sweep above never consumes its locator payload. */
|
|
6144
|
+
async function recoverDockerTaskEnvironment(
|
|
6145
|
+
descriptor: TaskEnvironmentDescriptor,
|
|
6146
|
+
context: TaskEnvironmentRecoveryContext,
|
|
6147
|
+
): Promise<TaskEnvironmentRecoveryResult> {
|
|
6148
|
+
const locator = parseDockerTaskEnvironmentLocator(descriptor.locator);
|
|
6149
|
+
const task = getHostTask(descriptor.taskId);
|
|
6150
|
+
if (!task || !task.statusMirror || !isActive(task.statusMirror)) {
|
|
6151
|
+
return { outcome: "recovered" };
|
|
6152
|
+
}
|
|
6153
|
+
const isPreparedRow =
|
|
6154
|
+
(task.statusMirror === "starting" || task.statusMirror === "error") &&
|
|
6155
|
+
task.composeProject === null &&
|
|
6156
|
+
task.worktreePath === null;
|
|
6157
|
+
if (
|
|
6158
|
+
!isPreparedRow &&
|
|
6159
|
+
(task.composeProject !== locator.composeProject ||
|
|
6160
|
+
task.worktreePath !== locator.hostWorktreePath)
|
|
6161
|
+
) {
|
|
6162
|
+
throw new Error(
|
|
6163
|
+
`task ${descriptor.taskId} environment locator no longer matches its durable row`,
|
|
3684
6164
|
);
|
|
3685
6165
|
}
|
|
3686
|
-
|
|
6166
|
+
// Provider-owned provisional metadata is authoritative after a crash. Feed
|
|
6167
|
+
// the already-validated deterministic values to the Docker-private repair
|
|
6168
|
+
// helpers without teaching the generic sweep how to derive them.
|
|
6169
|
+
const recoveryTask = isPreparedRow
|
|
6170
|
+
? {
|
|
6171
|
+
...task,
|
|
6172
|
+
composeProject: locator.composeProject,
|
|
6173
|
+
worktreePath: locator.hostWorktreePath,
|
|
6174
|
+
}
|
|
6175
|
+
: task;
|
|
6176
|
+
|
|
6177
|
+
const lifecycleCallbacks = [
|
|
6178
|
+
context.quarantineConsumers,
|
|
6179
|
+
context.allowConsumers,
|
|
6180
|
+
context.ensureConsumersStarted,
|
|
6181
|
+
];
|
|
6182
|
+
const hasLifecycle = lifecycleCallbacks.some(Boolean);
|
|
6183
|
+
if (hasLifecycle && !lifecycleCallbacks.every(Boolean)) {
|
|
6184
|
+
throw new Error("task environment recovery lifecycle callbacks are incomplete");
|
|
6185
|
+
}
|
|
6186
|
+
const consumers: RecoveryTaskConsumers | undefined = hasLifecycle
|
|
6187
|
+
? {
|
|
6188
|
+
quarantineChannelForRecovery: async (taskId) => {
|
|
6189
|
+
if (taskId !== descriptor.taskId) {
|
|
6190
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
6191
|
+
}
|
|
6192
|
+
await context.quarantineConsumers!();
|
|
6193
|
+
},
|
|
6194
|
+
allowChannel: (taskId) => {
|
|
6195
|
+
if (taskId !== descriptor.taskId) {
|
|
6196
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
6197
|
+
}
|
|
6198
|
+
context.allowConsumers!();
|
|
6199
|
+
},
|
|
6200
|
+
ensureStarted: async (taskId) => {
|
|
6201
|
+
if (taskId !== descriptor.taskId) {
|
|
6202
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
6203
|
+
}
|
|
6204
|
+
await context.ensureConsumersStarted!();
|
|
6205
|
+
},
|
|
6206
|
+
}
|
|
6207
|
+
: undefined;
|
|
6208
|
+
const recovered = await recoverOneDockerTask(
|
|
6209
|
+
recoveryTask,
|
|
6210
|
+
context.maintenanceReady,
|
|
6211
|
+
context.isCurrent,
|
|
6212
|
+
consumers,
|
|
6213
|
+
);
|
|
6214
|
+
return recovered
|
|
6215
|
+
? { outcome: "recovered" }
|
|
6216
|
+
: {
|
|
6217
|
+
outcome: "deferred",
|
|
6218
|
+
detail: "container runtime did not answer during task recovery",
|
|
6219
|
+
};
|
|
3687
6220
|
}
|
|
3688
6221
|
|
|
6222
|
+
setDockerTaskEnvironmentRecoveryDriver(recoverDockerTaskEnvironment);
|
|
6223
|
+
setAppleTaskEnvironmentRecoveryDriver(recoverAppleTaskEnvironment);
|
|
6224
|
+
|
|
3689
6225
|
/** Returns false when docker was unreachable (caller retries the pass). */
|
|
3690
|
-
async function
|
|
6226
|
+
async function recoverOneDockerTask(
|
|
3691
6227
|
task: typeof schema.hostTasks.$inferSelect,
|
|
6228
|
+
maintenanceReady: Promise<void>,
|
|
6229
|
+
isCurrent: () => boolean = () => true,
|
|
6230
|
+
lifecycle?: RecoveryTaskConsumers,
|
|
3692
6231
|
): Promise<boolean> {
|
|
6232
|
+
if (!isCurrent()) return true;
|
|
3693
6233
|
const composeProject = task.composeProject;
|
|
3694
6234
|
if (!composeProject) {
|
|
3695
6235
|
// We never wrote a compose project name for this row — must be
|
|
@@ -3701,6 +6241,7 @@ async function recoverOneTask(
|
|
|
3701
6241
|
const containers = await dockerListContainersByLabel(
|
|
3702
6242
|
`com.docker.compose.project=${composeProject}`,
|
|
3703
6243
|
);
|
|
6244
|
+
if (!isCurrent()) return true;
|
|
3704
6245
|
if (containers === null) {
|
|
3705
6246
|
// Docker unreachable / timed out — actual state unknown. Leave the row
|
|
3706
6247
|
// alone; the retrying caller re-scans once docker answers.
|
|
@@ -3727,12 +6268,106 @@ async function recoverOneTask(
|
|
|
3727
6268
|
return true;
|
|
3728
6269
|
}
|
|
3729
6270
|
|
|
3730
|
-
|
|
6271
|
+
// Defense in depth for every recovery entry point: the maintained
|
|
6272
|
+
// activation normally quarantines legacy writers before its first shared
|
|
6273
|
+
// volume mutation, but recovery itself must never fast-path or `docker
|
|
6274
|
+
// start` a container whose immutable mount is not the exact L5 RO contract.
|
|
6275
|
+
const mountContract = await inspectRuntimeWriterContainer({
|
|
6276
|
+
taskId: task.taskId,
|
|
6277
|
+
composeProject,
|
|
6278
|
+
containerName,
|
|
6279
|
+
});
|
|
6280
|
+
if (!isCurrent()) return true;
|
|
6281
|
+
if (mountContract.kind === "unreachable") return false;
|
|
6282
|
+
if (mountContract.kind === "failed") {
|
|
6283
|
+
console.error(
|
|
6284
|
+
`[orchestrator] recovery: ${task.taskId} mount identity is ambiguous; deferring without starting it: ${mountContract.detail}`,
|
|
6285
|
+
);
|
|
6286
|
+
return false;
|
|
6287
|
+
}
|
|
6288
|
+
if (
|
|
6289
|
+
mountContract.metadata.sharedMount !== "read-only" ||
|
|
6290
|
+
mountContract.metadata.taskEnvironment !== "current"
|
|
6291
|
+
) {
|
|
6292
|
+
if (lifecycle) {
|
|
6293
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6294
|
+
if (!isCurrent()) return true;
|
|
6295
|
+
}
|
|
6296
|
+
const stopped = await stopRuntimeWriter(mountContract.metadata);
|
|
6297
|
+
if (!isCurrent()) return true;
|
|
6298
|
+
if (stopped === "unreachable") return false;
|
|
6299
|
+
if (stopped === "failed") {
|
|
6300
|
+
throw new Error(
|
|
6301
|
+
`${task.taskId} has a legacy/non-current runtime mount and could not be proven stopped`,
|
|
6302
|
+
);
|
|
6303
|
+
}
|
|
6304
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6305
|
+
codeServerPort: null,
|
|
6306
|
+
previewPorts: "[]",
|
|
6307
|
+
});
|
|
6308
|
+
clearTaskGatewayAcl(task.taskId);
|
|
6309
|
+
console.warn(
|
|
6310
|
+
`[orchestrator] recovery: ${task.taskId} uses a legacy/non-current runtime mount; it remains stopped until explicit Resume recreates it`,
|
|
6311
|
+
);
|
|
6312
|
+
return true;
|
|
6313
|
+
}
|
|
6314
|
+
|
|
6315
|
+
// Even a container already marked running needs an in-container workbench
|
|
6316
|
+
// probe. Fence that first exec, as well as the exited-container repair path,
|
|
6317
|
+
// behind the same one-way boot maintenance latch.
|
|
6318
|
+
await maintenanceReady;
|
|
6319
|
+
if (!isCurrent()) return true;
|
|
6320
|
+
|
|
6321
|
+
let running = containers.some(
|
|
6322
|
+
(container) =>
|
|
6323
|
+
container.Names === containerName && container.State === "running",
|
|
6324
|
+
);
|
|
3731
6325
|
if (running) {
|
|
6326
|
+
// A stale recovery generation may have successfully issued `docker start`
|
|
6327
|
+
// and then been cancelled before uai-init. Do not bless every running
|
|
6328
|
+
// container solely from Docker state: code-server is the cheap proof that
|
|
6329
|
+
// the workbench init completed. A fresh sweep repairs a merely-started
|
|
6330
|
+
// container through the shared init path below.
|
|
6331
|
+
const workbenchReady = await dockerExec(
|
|
6332
|
+
containerName,
|
|
6333
|
+
["/usr/bin/pgrep", "-f", "code-server.*--bind-addr"],
|
|
6334
|
+
10_000,
|
|
6335
|
+
);
|
|
6336
|
+
if (!isCurrent()) return true;
|
|
6337
|
+
if (!workbenchReady && !(await dockerDaemonReachable())) return false;
|
|
6338
|
+
if (!isCurrent()) return true;
|
|
6339
|
+
if (!workbenchReady) {
|
|
6340
|
+
console.log(
|
|
6341
|
+
`[orchestrator] recovery: ${task.taskId} running without its workbench; quarantining before runtime proof`,
|
|
6342
|
+
);
|
|
6343
|
+
// Post-boot recovery can race live agent sessions. Drain their external
|
|
6344
|
+
// work and tombstone the channel before stopping the shared-volume
|
|
6345
|
+
// writer; successful recovery reopens the preserved channel spec below.
|
|
6346
|
+
if (lifecycle) {
|
|
6347
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6348
|
+
if (!isCurrent()) return true;
|
|
6349
|
+
}
|
|
6350
|
+
const quarantined = await quarantineRunningContainer(containerName);
|
|
6351
|
+
if (!isCurrent()) return true;
|
|
6352
|
+
if (quarantined === "unreachable") return false;
|
|
6353
|
+
if (quarantined === "failed") {
|
|
6354
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6355
|
+
codeServerPort: null,
|
|
6356
|
+
previewPorts: "[]",
|
|
6357
|
+
});
|
|
6358
|
+
throw new Error(
|
|
6359
|
+
`${task.taskId} could not prove its running app stopped after stop and kill; task remains quarantined`,
|
|
6360
|
+
);
|
|
6361
|
+
}
|
|
6362
|
+
running = false;
|
|
6363
|
+
} else {
|
|
3732
6364
|
// Container is up. Re-discover the host port in case Docker
|
|
3733
6365
|
// remapped it across restarts (it usually does for ephemeral
|
|
3734
6366
|
// bindings).
|
|
3735
|
-
const
|
|
6367
|
+
const portResult = await dockerPort(containerName, 8080);
|
|
6368
|
+
if (!isCurrent()) return true;
|
|
6369
|
+
if (!portResult.reachable) return false;
|
|
6370
|
+
const port = portResult.port;
|
|
3736
6371
|
if (port && port !== task.codeServerPort) {
|
|
3737
6372
|
db_setRuntime(task.taskId, {
|
|
3738
6373
|
codeServerPort: port,
|
|
@@ -3759,30 +6394,113 @@ async function recoverOneTask(
|
|
|
3759
6394
|
}`,
|
|
3760
6395
|
),
|
|
3761
6396
|
);
|
|
6397
|
+
if (!isCurrent()) return true;
|
|
6398
|
+
if (lifecycle) {
|
|
6399
|
+
lifecycle.allowChannel(task.taskId);
|
|
6400
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
6401
|
+
if (!isCurrent()) {
|
|
6402
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6403
|
+
}
|
|
6404
|
+
}
|
|
3762
6405
|
return true;
|
|
6406
|
+
}
|
|
3763
6407
|
}
|
|
3764
6408
|
|
|
3765
|
-
//
|
|
3766
|
-
//
|
|
3767
|
-
|
|
3768
|
-
|
|
6409
|
+
// The exact app container is now stopped. Prove every project runtime from
|
|
6410
|
+
// its immutable image and mounts before bringing it back up. Rolling old
|
|
6411
|
+
// images receive the current packaged materializer/Corepack pin as read-only
|
|
6412
|
+
// binds, so recovery cannot bypass a newly-shipped proof contract.
|
|
6413
|
+
const runtimeInspection = await inspectRecoveryRuntime(task, containerName);
|
|
6414
|
+
if (!isCurrent()) return true;
|
|
6415
|
+
if (runtimeInspection.kind === "unreachable") return false;
|
|
6416
|
+
if (runtimeInspection.kind === "failed") {
|
|
6417
|
+
console.error(
|
|
6418
|
+
`[orchestrator] recovery: ${task.taskId} runtime inspection failed; keeping task stopped: ${runtimeInspection.detail}`,
|
|
6419
|
+
);
|
|
6420
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6421
|
+
codeServerPort: null,
|
|
6422
|
+
previewPorts: "[]",
|
|
6423
|
+
});
|
|
6424
|
+
return true;
|
|
6425
|
+
}
|
|
6426
|
+
const runtimeMaterialization = await materializeRecoveryRuntimes(
|
|
6427
|
+
runtimeInspection.spec,
|
|
6428
|
+
);
|
|
6429
|
+
if (!isCurrent()) return true;
|
|
6430
|
+
if (runtimeMaterialization.kind === "unreachable") return false;
|
|
6431
|
+
if (runtimeMaterialization.kind === "failed") {
|
|
6432
|
+
console.error(
|
|
6433
|
+
`[orchestrator] recovery: ${task.taskId} runtime materialization failed; keeping task stopped and requiring a complete retry: ${runtimeMaterialization.detail}`,
|
|
6434
|
+
);
|
|
6435
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6436
|
+
codeServerPort: null,
|
|
6437
|
+
previewPorts: "[]",
|
|
6438
|
+
});
|
|
6439
|
+
return true;
|
|
6440
|
+
}
|
|
6441
|
+
const stagedInit = await stageRecoveryInit(
|
|
6442
|
+
containerName,
|
|
6443
|
+
runtimeInspection.spec,
|
|
6444
|
+
);
|
|
6445
|
+
if (!isCurrent()) return true;
|
|
6446
|
+
if (stagedInit.kind === "unreachable") return false;
|
|
6447
|
+
if (stagedInit.kind === "failed") {
|
|
6448
|
+
console.error(
|
|
6449
|
+
`[orchestrator] recovery: ${task.taskId} could not stage the current task initializer; keeping task stopped: ${stagedInit.detail}`,
|
|
6450
|
+
);
|
|
6451
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6452
|
+
codeServerPort: null,
|
|
6453
|
+
previewPorts: "[]",
|
|
6454
|
+
});
|
|
6455
|
+
return true;
|
|
6456
|
+
}
|
|
6457
|
+
const preStartWorkspaceRuntimeError = validateRecoveryWorkspaceRuntimeRoot(
|
|
6458
|
+
runtimeInspection.spec.workspaceSource,
|
|
3769
6459
|
);
|
|
3770
|
-
if (
|
|
6460
|
+
if (preStartWorkspaceRuntimeError) {
|
|
6461
|
+
console.error(
|
|
6462
|
+
`[orchestrator] recovery: ${task.taskId} workspace runtime authority changed after proof; keeping task stopped: ${preStartWorkspaceRuntimeError}`,
|
|
6463
|
+
);
|
|
3771
6464
|
db_setStatus(task.taskId, "stopped", {
|
|
3772
6465
|
codeServerPort: null,
|
|
3773
6466
|
previewPorts: "[]",
|
|
3774
6467
|
});
|
|
3775
6468
|
return true;
|
|
3776
6469
|
}
|
|
6470
|
+
|
|
6471
|
+
// Runtime proof succeeded. Bring the preserved app back up + re-launch the
|
|
6472
|
+
// task-local dependency/editor init.
|
|
6473
|
+
// If Docker appeared after the host's initial no-runtime verdict, the
|
|
6474
|
+
// standard-image repair starts concurrently with this boot sweep. uai-init
|
|
6475
|
+
// is now task-local (dependencies + editor), but it still resolves through
|
|
6476
|
+
// shared asdf shims; never run it until the boot owner has finished
|
|
6477
|
+
// reconciling the image and agent CLI volume.
|
|
6478
|
+
if (!running) {
|
|
6479
|
+
console.log(
|
|
6480
|
+
`[orchestrator] recovery: ${task.taskId} starting exited container ${containerName}`,
|
|
6481
|
+
);
|
|
6482
|
+
const started = await dockerStart(containerName);
|
|
6483
|
+
if (!isCurrent()) return true;
|
|
6484
|
+
if (started === "unreachable") return false;
|
|
6485
|
+
if (started === "failed") {
|
|
6486
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6487
|
+
codeServerPort: null,
|
|
6488
|
+
previewPorts: "[]",
|
|
6489
|
+
});
|
|
6490
|
+
return true;
|
|
6491
|
+
}
|
|
6492
|
+
}
|
|
3777
6493
|
// uai-init seeds code-server's profile beneath this shared data root. Heal
|
|
3778
6494
|
// containers created by older hosts before running it as node.
|
|
3779
6495
|
await repairNodeDataRoot(containerName);
|
|
6496
|
+
if (!isCurrent()) return true;
|
|
3780
6497
|
// Correctly-owned Codex creds BEFORE uai-init / any agent respawn: the boot
|
|
3781
6498
|
// reinject sweep only targets containers already running, so a container
|
|
3782
6499
|
// recovered here would otherwise keep whatever it held when it exited —
|
|
3783
6500
|
// possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
|
|
3784
6501
|
// by the inject itself and must not abort the task's recovery.
|
|
3785
6502
|
await injectCodexIntoContainer(containerName);
|
|
6503
|
+
if (!isCurrent()) return true;
|
|
3786
6504
|
// Establish the task owner's GitHub transport BEFORE uai-init performs any
|
|
3787
6505
|
// dependency-network work. This matters for package manifests that refer to
|
|
3788
6506
|
// private GitHub repositories: gh's credential helper must already be live
|
|
@@ -3790,7 +6508,20 @@ async function recoverOneTask(
|
|
|
3790
6508
|
// like the prior post-init call, but it is now ordered rather than raced.
|
|
3791
6509
|
if (task.ownerUserId) {
|
|
3792
6510
|
try {
|
|
3793
|
-
await reconcileTaskGitAuth(task.taskId, task.ownerUserId
|
|
6511
|
+
await reconcileTaskGitAuth(task.taskId, task.ownerUserId, {
|
|
6512
|
+
// Recovery is sanctioned private maintenance while public admission is
|
|
6513
|
+
// deliberately `checking`. Fence it by this daemon epoch instead of
|
|
6514
|
+
// the public operational bit, preserving private-dependency setup
|
|
6515
|
+
// before uai-init without letting stale recovery keep mutating.
|
|
6516
|
+
admitContainerWork: () => {
|
|
6517
|
+
if (!isCurrent()) {
|
|
6518
|
+
throw new ContainerRuntimeUnavailableError(
|
|
6519
|
+
"container runtime recovery generation was superseded",
|
|
6520
|
+
);
|
|
6521
|
+
}
|
|
6522
|
+
},
|
|
6523
|
+
});
|
|
6524
|
+
if (!isCurrent()) return true;
|
|
3794
6525
|
} catch (err) {
|
|
3795
6526
|
console.warn(
|
|
3796
6527
|
`[orchestrator] recovery: ${task.taskId} GitHub auth setup failed before uai-init: ${
|
|
@@ -3798,6 +6529,7 @@ async function recoverOneTask(
|
|
|
3798
6529
|
}`,
|
|
3799
6530
|
);
|
|
3800
6531
|
}
|
|
6532
|
+
if (!isCurrent()) return true;
|
|
3801
6533
|
}
|
|
3802
6534
|
// uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
|
|
3803
6535
|
// repo. dockerCli's 30s default would SIGKILL it mid-install.
|
|
@@ -3806,17 +6538,119 @@ async function recoverOneTask(
|
|
|
3806
6538
|
"exec",
|
|
3807
6539
|
"-e",
|
|
3808
6540
|
"UAI_SKIP_GIT_TRANSPORT=1",
|
|
6541
|
+
"-e",
|
|
6542
|
+
"UAI_WORKSPACE=/workspace",
|
|
6543
|
+
"-e",
|
|
6544
|
+
`UAI_RUNTIME_PROJECTS=${RECOVERY_INIT_PROJECTS_PATH}`,
|
|
6545
|
+
"-e",
|
|
6546
|
+
"PATH=/opt/asdf-data/shims:/opt/asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
|
6547
|
+
"-e",
|
|
6548
|
+
"HOME=/home/node",
|
|
6549
|
+
"-e",
|
|
6550
|
+
"XDG_CONFIG_HOME=/home/node/.config",
|
|
6551
|
+
"-e",
|
|
6552
|
+
"XDG_DATA_HOME=/home/node/.local/share",
|
|
6553
|
+
"-e",
|
|
6554
|
+
"ASDF_DIR=/opt/asdf",
|
|
6555
|
+
"-e",
|
|
6556
|
+
"ASDF_DATA_DIR=/opt/asdf-data",
|
|
6557
|
+
"-e",
|
|
6558
|
+
"ASDF_CONFIG_FILE=/dev/null",
|
|
6559
|
+
"-e",
|
|
6560
|
+
"ASDF_DEFAULT_TOOL_VERSIONS_FILENAME=.tool-versions",
|
|
6561
|
+
"-e",
|
|
6562
|
+
"ASDF_NODEJS_VERSION=",
|
|
6563
|
+
"-e",
|
|
6564
|
+
"ASDF_PYTHON_VERSION=",
|
|
6565
|
+
"-e",
|
|
6566
|
+
"ASDF_GOLANG_VERSION=",
|
|
6567
|
+
"-e",
|
|
6568
|
+
"ASDF_RUBY_VERSION=",
|
|
6569
|
+
"-e",
|
|
6570
|
+
"ASDF_RUST_VERSION=",
|
|
6571
|
+
"-e",
|
|
6572
|
+
"COREPACK_HOME=/opt/asdf-data/corepack",
|
|
6573
|
+
"-e",
|
|
6574
|
+
"COREPACK_DEFAULT_TO_LATEST=0",
|
|
6575
|
+
"-e",
|
|
6576
|
+
"COREPACK_ENV_FILE=0",
|
|
6577
|
+
"-e",
|
|
6578
|
+
"COREPACK_NPM_REGISTRY=https://registry.npmjs.org",
|
|
6579
|
+
"-e",
|
|
6580
|
+
"COREPACK_ENABLE_NETWORK=0",
|
|
6581
|
+
"-e",
|
|
6582
|
+
"COREPACK_ENABLE_AUTO_PIN=0",
|
|
6583
|
+
"-e",
|
|
6584
|
+
"COREPACK_ENABLE_DOWNLOAD_PROMPT=0",
|
|
6585
|
+
"-e",
|
|
6586
|
+
"COREPACK_ENABLE_PROJECT_SPEC=1",
|
|
6587
|
+
"-e",
|
|
6588
|
+
"COREPACK_ENABLE_STRICT=1",
|
|
6589
|
+
"-e",
|
|
6590
|
+
"COREPACK_ENABLE_UNSAFE_CUSTOM_URLS=0",
|
|
6591
|
+
"-e",
|
|
6592
|
+
"COREPACK_INTEGRITY_CHECK=1",
|
|
6593
|
+
"-e",
|
|
6594
|
+
"LD_PRELOAD=",
|
|
6595
|
+
"-e",
|
|
6596
|
+
"LD_AUDIT=",
|
|
6597
|
+
"-e",
|
|
6598
|
+
"LD_LIBRARY_PATH=",
|
|
6599
|
+
"-e",
|
|
6600
|
+
"NODE_OPTIONS=",
|
|
6601
|
+
"-e",
|
|
6602
|
+
"NODE_PATH=",
|
|
6603
|
+
// SHELLOPTS/BASHOPTS deliberately absent: present-in-env (even empty)
|
|
6604
|
+
// makes bash export its live options into children.
|
|
6605
|
+
"-e",
|
|
6606
|
+
"PS4=",
|
|
6607
|
+
"-e",
|
|
6608
|
+
"BASH_ENV=",
|
|
6609
|
+
"-e",
|
|
6610
|
+
"ENV=",
|
|
3809
6611
|
containerName,
|
|
3810
|
-
|
|
6612
|
+
// The host-side deadline below only kills the local docker client; the
|
|
6613
|
+
// server-side exec would keep mutating workspace dependencies while the
|
|
6614
|
+
// recovered task is already exposed to agents. This in-container
|
|
6615
|
+
// deadline kills the actual initializer first (TERM, then KILL), and
|
|
6616
|
+
// the host allowance is deliberately wider so a timed-out run settles
|
|
6617
|
+
// with exit 124 instead of racing the client kill.
|
|
6618
|
+
"/usr/bin/timeout",
|
|
6619
|
+
"--signal=TERM",
|
|
6620
|
+
"--kill-after=15s",
|
|
6621
|
+
"570s",
|
|
6622
|
+
RECOVERY_INIT_PATH,
|
|
3811
6623
|
],
|
|
3812
6624
|
{ timeoutMs: 10 * 60_000 },
|
|
3813
6625
|
);
|
|
6626
|
+
if (!isCurrent()) return true;
|
|
6627
|
+
if (initResult.status === 78) {
|
|
6628
|
+
const quarantined = await quarantineRunningContainer(containerName);
|
|
6629
|
+
if (!isCurrent()) return true;
|
|
6630
|
+
if (quarantined === "unreachable") return false;
|
|
6631
|
+
db_setStatus(task.taskId, "stopped", {
|
|
6632
|
+
codeServerPort: null,
|
|
6633
|
+
previewPorts: "[]",
|
|
6634
|
+
});
|
|
6635
|
+
if (quarantined === "failed") {
|
|
6636
|
+
throw new Error(
|
|
6637
|
+
`${task.taskId} reported a runtime-authority violation and could not be proven stopped`,
|
|
6638
|
+
);
|
|
6639
|
+
}
|
|
6640
|
+
console.error(
|
|
6641
|
+
`[orchestrator] recovery: ${task.taskId} initializer detected a workspace runtime-authority violation; task was stopped and remains quarantined`,
|
|
6642
|
+
);
|
|
6643
|
+
return true;
|
|
6644
|
+
}
|
|
3814
6645
|
if (initResult.status !== 0) {
|
|
3815
6646
|
console.warn(
|
|
3816
6647
|
`[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
|
|
3817
6648
|
);
|
|
3818
6649
|
}
|
|
3819
|
-
const
|
|
6650
|
+
const portResult = await dockerPort(containerName, 8080);
|
|
6651
|
+
if (!isCurrent()) return true;
|
|
6652
|
+
if (!portResult.reachable) return false;
|
|
6653
|
+
const port = portResult.port;
|
|
3820
6654
|
db_setRuntime(task.taskId, {
|
|
3821
6655
|
codeServerPort: port,
|
|
3822
6656
|
});
|
|
@@ -3830,6 +6664,14 @@ async function recoverOneTask(
|
|
|
3830
6664
|
console.log(
|
|
3831
6665
|
`[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
|
|
3832
6666
|
);
|
|
6667
|
+
if (!isCurrent()) return true;
|
|
6668
|
+
if (lifecycle) {
|
|
6669
|
+
lifecycle.allowChannel(task.taskId);
|
|
6670
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
6671
|
+
if (!isCurrent()) {
|
|
6672
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
6673
|
+
}
|
|
6674
|
+
}
|
|
3833
6675
|
return true;
|
|
3834
6676
|
}
|
|
3835
6677
|
|