@runuai/host 0.9.14 → 0.9.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +80 -30
- package/images/standard/Dockerfile +36 -10
- package/images/standard/README.md +63 -18
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +308 -38
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/lib/agent-cli.ts +33 -2
- package/lib/agent.ts +46 -7
- package/lib/agents/claude.ts +13 -8
- package/lib/agents/codex.ts +11 -6
- package/lib/agents/cursor.ts +39 -29
- package/lib/agents/durable-proc.ts +20 -27
- package/lib/agents/factory.ts +9 -25
- package/lib/agents/grok.ts +43 -30
- package/lib/agents/kimi.ts +44 -29
- package/lib/agents/opencode.ts +43 -31
- package/lib/agents/proc.ts +149 -114
- package/lib/agents/transport.ts +62 -50
- package/lib/agents/types.ts +6 -4
- package/lib/apple-runtime-recycle.ts +236 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/browser-testing.ts +233 -93
- package/lib/codex-auth.ts +40 -6
- package/lib/command-db.ts +20 -0
- package/lib/container-runtime.ts +1338 -0
- package/lib/db.ts +1 -0
- package/lib/docker-exec.ts +87 -5
- package/lib/engine-accounts.ts +68 -5
- package/lib/engine-login.ts +1952 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +155 -0
- package/lib/env.ts +4 -0
- package/lib/git-diff.ts +98 -32
- package/lib/git-identity.ts +199 -87
- package/lib/github-tokens.ts +202 -91
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-logs.ts +962 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +49 -0
- package/lib/managed-runtime.ts +3644 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-gateway.ts +450 -23
- package/lib/orchestrator.ts +3060 -218
- package/lib/preview-sidecar.ts +57 -13
- package/lib/release-manifest.ts +708 -0
- package/lib/release-trust.ts +28 -0
- package/lib/runtime-activation-tail.ts +232 -0
- package/lib/runtime-archive.ts +1086 -0
- package/lib/runtime-authority.ts +79 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-state.ts +232 -12
- package/lib/skills.ts +24 -3
- package/lib/ssh.ts +18 -0
- package/lib/standard-image.ts +1104 -141
- package/lib/stopped-task-status-queue.ts +44 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-diff.ts +66 -46
- package/lib/task-environment/apple-container.ts +757 -0
- package/lib/task-environment/docker.ts +945 -0
- package/lib/task-environment/index.ts +364 -0
- package/lib/task-environment/legacy-adoption.ts +443 -0
- package/lib/task-environment/registry.ts +58 -0
- package/lib/task-environment/types.ts +408 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +585 -0
- package/lib/tunnel-registry.ts +135 -19
- package/lib/tunnel-runtime.ts +235 -0
- package/package.json +1 -1
- package/scripts/agent/_common.sh +123 -3
- package/scripts/agent/task-down.sh +146 -38
- package/scripts/agent/task-status.sh +19 -3
- package/scripts/agent/task-up.sh +1405 -107
- package/scripts/install/darwin.ts +848 -50
- package/scripts/install/linux.ts +838 -35
- package/scripts/install/types.ts +43 -0
- package/scripts/install/util.ts +215 -8
- package/scripts/install/win.ts +12 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/cli.ts +1464 -72
- package/src/event-outbox.ts +83 -4
- package/src/index.ts +766 -42
- package/src/main.ts +1398 -255
- package/src/paths.ts +17 -1
- package/src/protocol.ts +695 -1
- package/src/runtime-bootstrap.ts +165 -0
- package/src/ui/server.ts +46 -10
- package/src/ui/types.ts +37 -0
package/src/index.ts
CHANGED
|
@@ -1,9 +1,22 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
agent,
|
|
3
|
+
AgentError,
|
|
4
|
+
toolStderrDetail,
|
|
5
|
+
type TaskDownResult,
|
|
6
|
+
} from "../lib/agent";
|
|
7
|
+
import { agentClisReady, ensureStandardImage } from "../lib/standard-image";
|
|
3
8
|
import { cloneRepo } from "../lib/repo-clone";
|
|
4
9
|
import { handleFilesOp } from "../lib/shared-files";
|
|
5
|
-
import {
|
|
6
|
-
|
|
10
|
+
import {
|
|
11
|
+
getOrchestrator,
|
|
12
|
+
inspectTaskRuntimeContainer,
|
|
13
|
+
proveAppleTaskRuntimeContract,
|
|
14
|
+
quarantineTaskRuntimeContainer,
|
|
15
|
+
} from "../lib/orchestrator";
|
|
16
|
+
import {
|
|
17
|
+
removeTaskCliSecretStrict,
|
|
18
|
+
storeTaskCliSecret,
|
|
19
|
+
} from "../lib/agent-cli";
|
|
7
20
|
import {
|
|
8
21
|
clearRefresh,
|
|
9
22
|
reconcileTaskGitAuth,
|
|
@@ -16,21 +29,56 @@ import { readAttachment, writeAttachment } from "../lib/attachments";
|
|
|
16
29
|
import { appendTranscript as writeTranscript } from "../lib/transcript";
|
|
17
30
|
import { buildTaskDiff } from "../lib/task-diff";
|
|
18
31
|
import {
|
|
32
|
+
deleteHostTask,
|
|
19
33
|
getHostTask,
|
|
20
34
|
recordHostEvent,
|
|
21
35
|
recordTaskDown,
|
|
36
|
+
recordTaskEnvironmentPrepared,
|
|
22
37
|
recordTaskError,
|
|
23
38
|
recordTaskOwner,
|
|
24
39
|
recordTaskStarting,
|
|
25
40
|
recordTaskUpResult,
|
|
41
|
+
purgeHostTaskState,
|
|
42
|
+
upsertHostTask,
|
|
26
43
|
} from "../lib/runtime-state";
|
|
27
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
clearTaskGatewayAcl,
|
|
46
|
+
clearTaskGatewayAclStrict,
|
|
47
|
+
} from "../lib/mcp-gateway";
|
|
28
48
|
import { parsePreviewPortRuntimes } from "../lib/preview-ports";
|
|
29
49
|
import { dockerCli } from "../lib/docker-exec";
|
|
50
|
+
import {
|
|
51
|
+
claimContainerRuntimeForTask,
|
|
52
|
+
containerRuntimeGenerationOperational,
|
|
53
|
+
containerRuntimeProblem,
|
|
54
|
+
containerRuntimeReadinessEpoch,
|
|
55
|
+
containerRuntimeTeardownProblem,
|
|
56
|
+
ensureContainerRuntimeForTask,
|
|
57
|
+
initializeContainerRuntime,
|
|
58
|
+
reprobeContainerRuntimeMachineIdentity,
|
|
59
|
+
suspendContainerRuntime,
|
|
60
|
+
waitForContainerRuntimeOperational,
|
|
61
|
+
} from "../lib/container-runtime";
|
|
62
|
+
import {
|
|
63
|
+
provisionTaskEnvironment,
|
|
64
|
+
reconstructPersistedTaskEnvironment,
|
|
65
|
+
} from "../lib/task-environment";
|
|
66
|
+
import { appleTaskContainerName } from "../lib/task-environment/apple-container";
|
|
67
|
+
import type {
|
|
68
|
+
TaskEnvironmentHandle,
|
|
69
|
+
TaskEnvironmentLocator,
|
|
70
|
+
TaskEnvironmentStatus,
|
|
71
|
+
} from "../lib/task-environment/types";
|
|
30
72
|
import {
|
|
31
73
|
ensurePreviewSidecar,
|
|
32
74
|
stopPreviewSidecars,
|
|
33
75
|
} from "../lib/preview-sidecar";
|
|
76
|
+
import { inspectTunnelContainer } from "../lib/tunnel-runtime";
|
|
77
|
+
import { ContainerRuntimeUnavailableError } from "../lib/runtime-guard";
|
|
78
|
+
import { managedRuntimeAdmissionPending } from "../lib/managed-runtime";
|
|
79
|
+
import { retireTaskSshIdentityEnsures } from "../lib/git-identity";
|
|
80
|
+
import { removeTaskIdentityStrict } from "../lib/ssh";
|
|
81
|
+
import { proveOrphanTaskResourcesAbsent } from "../lib/task-inventory";
|
|
34
82
|
import {
|
|
35
83
|
HostErrorCode,
|
|
36
84
|
type CommandContext,
|
|
@@ -118,18 +166,212 @@ export const hostEvents: HostEventStream = {
|
|
|
118
166
|
export const hostCommands: HostCommands = {
|
|
119
167
|
async taskUp(ctx, input) {
|
|
120
168
|
logCommand(ctx, "taskUp", input.task.id);
|
|
169
|
+
const admissionFailure = managedMaintenanceUnavailable();
|
|
170
|
+
if (admissionFailure) return admissionFailure;
|
|
171
|
+
const runtimePreparationFailure = await prepareContainerRuntimeForTask();
|
|
172
|
+
if (runtimePreparationFailure) return runtimePreparationFailure;
|
|
173
|
+
const runtimeFailure = runtimeUnavailable();
|
|
174
|
+
if (runtimeFailure) return runtimeFailure;
|
|
175
|
+
// The boot reconcile briefly replaces CLI shims in the shared volume.
|
|
176
|
+
// Do not create local lifecycle/channel state until its one-way latch has
|
|
177
|
+
// settled (success or best-effort failure), then revalidate both mutable
|
|
178
|
+
// admission authorities after the await.
|
|
179
|
+
await agentClisReady;
|
|
180
|
+
const readyAdmissionFailure = managedMaintenanceUnavailable();
|
|
181
|
+
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
182
|
+
const readyRuntimeFailure = runtimeUnavailable();
|
|
183
|
+
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
121
184
|
const orchestrator = getOrchestrator();
|
|
122
185
|
return orchestrator.runTaskLifecycle(input.task.id, async () => {
|
|
123
186
|
// A direct duplicate teardown may still be draining operations that
|
|
124
187
|
// target this task's stable compose/container names. Never recreate
|
|
125
188
|
// those names until every close caller has crossed the shared fence.
|
|
126
189
|
await orchestrator.waitForChannelClose(input.task.id);
|
|
190
|
+
const inSlotAdmissionFailure = managedMaintenanceUnavailable();
|
|
191
|
+
if (inSlotAdmissionFailure) return inSlotAdmissionFailure;
|
|
192
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
193
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
127
194
|
// A command retry that arrives after the original start completed must
|
|
128
195
|
// not rerender/compose-up/uai-init underneath live agent sessions. Treat
|
|
129
196
|
// that lost-response retry as the successful operation it already was:
|
|
130
197
|
// every cloud caller records a non-ok taskUp as an errored task.
|
|
131
198
|
const existingTask = getHostTask(input.task.id);
|
|
132
|
-
|
|
199
|
+
let runtimeQuarantinedForRecreate = false;
|
|
200
|
+
let channelClosedForRecreate = false;
|
|
201
|
+
if (
|
|
202
|
+
existingTask?.statusMirror === "running" &&
|
|
203
|
+
existingTask.environmentProvider === "apple-container"
|
|
204
|
+
) {
|
|
205
|
+
// ADR-106/round-4: the lost-response fast path must speak the
|
|
206
|
+
// provider's language — the Docker branch below ENOENTs on a
|
|
207
|
+
// Docker-free Mac and suspended the generation instead of returning
|
|
208
|
+
// the persisted success.
|
|
209
|
+
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
210
|
+
return {
|
|
211
|
+
ok: false,
|
|
212
|
+
code: HostErrorCode.Internal,
|
|
213
|
+
message:
|
|
214
|
+
`task ${input.task.id} is running but its persisted runtime ` +
|
|
215
|
+
"metadata is incomplete",
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
// Round 5: every await below can outlive the runtime generation
|
|
219
|
+
// (bridge revalidation, forced recheck). A superseded pass must
|
|
220
|
+
// observe only — never reopen consumers or mutate the container —
|
|
221
|
+
// the same discipline apple recovery applies.
|
|
222
|
+
// Round 6: the fence requires the OPERATIONAL generation, not merely
|
|
223
|
+
// the epoch — plain suspension (connection revalidation) keeps the
|
|
224
|
+
// epoch and ready status while the daemon is unverified.
|
|
225
|
+
const fastEpoch = containerRuntimeReadinessEpoch();
|
|
226
|
+
const fastStale = (): boolean =>
|
|
227
|
+
!containerRuntimeGenerationOperational(fastEpoch);
|
|
228
|
+
const staleRetry = {
|
|
229
|
+
ok: false as const,
|
|
230
|
+
code: HostErrorCode.HostUnavailable,
|
|
231
|
+
message:
|
|
232
|
+
`the container runtime generation changed while verifying task ${input.task.id}; retry`,
|
|
233
|
+
retryable: true,
|
|
234
|
+
};
|
|
235
|
+
const environment = await reconstructPersistedTaskEnvironment(
|
|
236
|
+
existingTask,
|
|
237
|
+
);
|
|
238
|
+
if (fastStale()) return staleRetry;
|
|
239
|
+
if (environment === null) {
|
|
240
|
+
// The persisted locator no longer reconstructs (e.g. an engine
|
|
241
|
+
// identity change) — recreate through the normal path.
|
|
242
|
+
await orchestrator.closeChannel(input.task.id);
|
|
243
|
+
// Session drain can outlive the generation too: a superseded pass
|
|
244
|
+
// must not reach recordTaskStarting/provisioning below.
|
|
245
|
+
if (fastStale()) return staleRetry;
|
|
246
|
+
channelClosedForRecreate = true;
|
|
247
|
+
} else {
|
|
248
|
+
const appleStatus = await environment.status();
|
|
249
|
+
if (fastStale()) return staleRetry;
|
|
250
|
+
if (appleStatus.state === "unknown") {
|
|
251
|
+
return {
|
|
252
|
+
ok: false,
|
|
253
|
+
code: HostErrorCode.HostUnavailable,
|
|
254
|
+
message:
|
|
255
|
+
`could not verify runtime state for task ${input.task.id}: ` +
|
|
256
|
+
appleStatus.detail,
|
|
257
|
+
retryable: true,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
if (appleStatus.state === "running") {
|
|
261
|
+
const verdict = await proveAppleTaskRuntimeContract({
|
|
262
|
+
taskId: input.task.id,
|
|
263
|
+
containerName: appleTaskContainerName(input.task.id),
|
|
264
|
+
// The row's worktreePath is task-up's $task_dir — the same
|
|
265
|
+
// directory the apple locator records as hostWorktreePath.
|
|
266
|
+
hostWorktreePath: existingTask.worktreePath,
|
|
267
|
+
});
|
|
268
|
+
if (fastStale()) return staleRetry;
|
|
269
|
+
if (verdict.kind === "unreachable") {
|
|
270
|
+
return {
|
|
271
|
+
ok: false,
|
|
272
|
+
code: HostErrorCode.HostUnavailable,
|
|
273
|
+
message:
|
|
274
|
+
`could not prove the exact runtime contract for task ${input.task.id}: ` +
|
|
275
|
+
verdict.detail,
|
|
276
|
+
retryable: true,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
let projectsHealthy = true;
|
|
280
|
+
for (const project of verdict.kind === "holds"
|
|
281
|
+
? input.projects
|
|
282
|
+
: []) {
|
|
283
|
+
const gitHealth = await environment.exec({
|
|
284
|
+
argv: [
|
|
285
|
+
"/usr/bin/env",
|
|
286
|
+
"-i",
|
|
287
|
+
"HOME=/home/node",
|
|
288
|
+
"PATH=/usr/bin:/bin",
|
|
289
|
+
"/usr/bin/git",
|
|
290
|
+
"rev-parse",
|
|
291
|
+
"--is-inside-work-tree",
|
|
292
|
+
],
|
|
293
|
+
user: "node",
|
|
294
|
+
cwd: `/workspace/${project.slug}`,
|
|
295
|
+
env: {},
|
|
296
|
+
timeoutMs: 10_000,
|
|
297
|
+
maxOutputBytes: 64 * 1024,
|
|
298
|
+
});
|
|
299
|
+
if (fastStale()) return staleRetry;
|
|
300
|
+
if (gitHealth.exitCode === null || gitHealth.signal !== null) {
|
|
301
|
+
return {
|
|
302
|
+
ok: false,
|
|
303
|
+
code: HostErrorCode.HostUnavailable,
|
|
304
|
+
message:
|
|
305
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
306
|
+
`project ${project.id}: the runtime did not answer`,
|
|
307
|
+
retryable: true,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
if (
|
|
311
|
+
gitHealth.exitCode !== 0 ||
|
|
312
|
+
Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
|
|
313
|
+
"true"
|
|
314
|
+
) {
|
|
315
|
+
projectsHealthy = false;
|
|
316
|
+
console.warn(
|
|
317
|
+
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
|
|
318
|
+
);
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (verdict.kind === "holds" && projectsHealthy) {
|
|
323
|
+
if (fastStale()) return staleRetry;
|
|
324
|
+
orchestrator.allowChannel(input.task.id);
|
|
325
|
+
return {
|
|
326
|
+
ok: true,
|
|
327
|
+
value: {
|
|
328
|
+
composeProject: existingTask.composeProject,
|
|
329
|
+
worktreePath: existingTask.worktreePath,
|
|
330
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
331
|
+
previewPorts: parsePreviewPortRuntimes(
|
|
332
|
+
existingTask.previewPorts,
|
|
333
|
+
),
|
|
334
|
+
},
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
// Violated contract or broken project repo: drain, stop, and
|
|
338
|
+
// prove before task-up recreates the same stable name.
|
|
339
|
+
if (fastStale()) return staleRetry;
|
|
340
|
+
await orchestrator.closeChannel(input.task.id);
|
|
341
|
+
channelClosedForRecreate = true;
|
|
342
|
+
if (fastStale()) return staleRetry;
|
|
343
|
+
await environment.stop();
|
|
344
|
+
const after = await environment.status();
|
|
345
|
+
if (fastStale()) return staleRetry;
|
|
346
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
347
|
+
return {
|
|
348
|
+
ok: false,
|
|
349
|
+
code: HostErrorCode.HostUnavailable,
|
|
350
|
+
message:
|
|
351
|
+
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
352
|
+
"could not be proven stopped; retry after the runtime is healthy",
|
|
353
|
+
retryable: true,
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
// Round 5: the container is PHYSICALLY stopped now, whatever the
|
|
357
|
+
// reason (violated contract or a broken project repo). A failed
|
|
358
|
+
// recreation must roll the mirror back to stopped — restoring
|
|
359
|
+
// "running" would advertise a container proven not to run.
|
|
360
|
+
runtimeQuarantinedForRecreate = true;
|
|
361
|
+
} else {
|
|
362
|
+
// Provider proved the container absent/stopped: normal self-heal
|
|
363
|
+
// recreate — and the rollback state must say stopped for the
|
|
364
|
+
// same reason as above.
|
|
365
|
+
if (fastStale()) return staleRetry;
|
|
366
|
+
await orchestrator.closeChannel(input.task.id);
|
|
367
|
+
// Fence AFTER the drain as well: provisioning must never start
|
|
368
|
+
// under a generation this pass no longer owns.
|
|
369
|
+
if (fastStale()) return staleRetry;
|
|
370
|
+
channelClosedForRecreate = true;
|
|
371
|
+
runtimeQuarantinedForRecreate = true;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
} else if (existingTask?.statusMirror === "running") {
|
|
133
375
|
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
134
376
|
return {
|
|
135
377
|
ok: false,
|
|
@@ -156,21 +398,72 @@ export const hostCommands: HostCommands = {
|
|
|
156
398
|
{ timeoutMs: 10_000 },
|
|
157
399
|
);
|
|
158
400
|
if (appRuntime.status !== 0) {
|
|
159
|
-
return
|
|
160
|
-
|
|
161
|
-
code: HostErrorCode.HostUnavailable,
|
|
162
|
-
message:
|
|
163
|
-
`could not verify runtime state for task ${input.task.id}: ` +
|
|
401
|
+
return dockerUnavailableResult(
|
|
402
|
+
`could not verify runtime state for task ${input.task.id}: ` +
|
|
164
403
|
(appRuntime.stderr.trim().slice(0, 200) ||
|
|
165
404
|
"docker did not answer"),
|
|
166
|
-
|
|
167
|
-
};
|
|
405
|
+
);
|
|
168
406
|
}
|
|
169
407
|
if (
|
|
170
408
|
appRuntime.stdout
|
|
171
409
|
.split(/\r?\n/)
|
|
172
410
|
.some((state) => state.trim() === "running")
|
|
173
411
|
) {
|
|
412
|
+
const appContainer = `${existingTask.composeProject}-app-1`;
|
|
413
|
+
const runtimeContract = await inspectTaskRuntimeContainer(
|
|
414
|
+
input.task.id,
|
|
415
|
+
existingTask.composeProject,
|
|
416
|
+
);
|
|
417
|
+
if (runtimeContract.kind === "unreachable") {
|
|
418
|
+
return dockerUnavailableResult(
|
|
419
|
+
`could not inspect the exact runtime mount for task ${input.task.id}`,
|
|
420
|
+
);
|
|
421
|
+
}
|
|
422
|
+
if (runtimeContract.kind === "failed") {
|
|
423
|
+
return {
|
|
424
|
+
ok: false,
|
|
425
|
+
code: HostErrorCode.HostUnavailable,
|
|
426
|
+
message:
|
|
427
|
+
`could not prove the exact runtime contract for task ${input.task.id}: ` +
|
|
428
|
+
runtimeContract.detail,
|
|
429
|
+
retryable: true,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
const reusable =
|
|
433
|
+
runtimeContract.running &&
|
|
434
|
+
!runtimeContract.restarting &&
|
|
435
|
+
!runtimeContract.paused &&
|
|
436
|
+
runtimeContract.sharedMount === "read-only" &&
|
|
437
|
+
runtimeContract.taskEnvironment === "current";
|
|
438
|
+
if (!reusable) {
|
|
439
|
+
// Drain/tombstone before touching the exact container. task-up's
|
|
440
|
+
// pre-materialization cleanup then removes this stopped immutable
|
|
441
|
+
// generation, so Compose must create the current RO/private-prefix
|
|
442
|
+
// contract rather than `docker start`ing it again.
|
|
443
|
+
await orchestrator.closeChannel(input.task.id);
|
|
444
|
+
channelClosedForRecreate = true;
|
|
445
|
+
const quarantined = await quarantineTaskRuntimeContainer({
|
|
446
|
+
taskId: input.task.id,
|
|
447
|
+
composeProject: existingTask.composeProject,
|
|
448
|
+
expectedContainerId: runtimeContract.containerId,
|
|
449
|
+
});
|
|
450
|
+
if (quarantined === "unreachable") {
|
|
451
|
+
return dockerUnavailableResult(
|
|
452
|
+
`Docker became unavailable while quarantining task ${input.task.id}`,
|
|
453
|
+
);
|
|
454
|
+
}
|
|
455
|
+
if (quarantined === "failed") {
|
|
456
|
+
return {
|
|
457
|
+
ok: false,
|
|
458
|
+
code: HostErrorCode.HostUnavailable,
|
|
459
|
+
message:
|
|
460
|
+
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
461
|
+
"could not be proven stopped; retry after Docker is healthy",
|
|
462
|
+
retryable: true,
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
runtimeQuarantinedForRecreate = true;
|
|
466
|
+
}
|
|
174
467
|
// Docker being up is not enough to prove a lost-response retry is
|
|
175
468
|
// healthy. The worktree's `.git` file contains an absolute pointer
|
|
176
469
|
// into the task-private repository; after a host workspace-root
|
|
@@ -178,9 +471,8 @@ export const hostCommands: HostCommands = {
|
|
|
178
471
|
// sees `fatal: not a git repository`. Probe from INSIDE the live
|
|
179
472
|
// container, which is the filesystem view the agents actually use.
|
|
180
473
|
// Scratchpads have no projects, so this remains an O(1) fast path.
|
|
181
|
-
const appContainer = `${existingTask.composeProject}-app-1`;
|
|
182
474
|
let projectsHealthy = true;
|
|
183
|
-
for (const project of input.projects) {
|
|
475
|
+
for (const project of reusable ? input.projects : []) {
|
|
184
476
|
const gitHealth = await dockerCli(
|
|
185
477
|
[
|
|
186
478
|
"exec",
|
|
@@ -243,21 +535,26 @@ export const hostCommands: HostCommands = {
|
|
|
243
535
|
// broken. Preserve the running task and let the caller retry once
|
|
244
536
|
// Docker can answer, just as the runtime-state probe above does.
|
|
245
537
|
if (gitHealth.status === null) {
|
|
246
|
-
return
|
|
247
|
-
|
|
248
|
-
code: HostErrorCode.HostUnavailable,
|
|
249
|
-
message:
|
|
250
|
-
`could not verify Git state for task ${input.task.id} ` +
|
|
538
|
+
return dockerUnavailableResult(
|
|
539
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
251
540
|
`project ${project.id}: ` +
|
|
252
541
|
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
253
542
|
"docker did not answer"),
|
|
254
|
-
|
|
255
|
-
};
|
|
543
|
+
);
|
|
256
544
|
}
|
|
257
545
|
if (
|
|
258
546
|
gitHealth.status !== 0 ||
|
|
259
547
|
gitHealth.stdout.trim() !== "true"
|
|
260
548
|
) {
|
|
549
|
+
if (gitHealth.status !== 0) {
|
|
550
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(
|
|
551
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
552
|
+
`project ${project.id}: ` +
|
|
553
|
+
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
554
|
+
`git exited ${String(gitHealth.status)}`),
|
|
555
|
+
);
|
|
556
|
+
if (unavailable) return unavailable;
|
|
557
|
+
}
|
|
261
558
|
projectsHealthy = false;
|
|
262
559
|
console.warn(
|
|
263
560
|
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
|
|
@@ -265,7 +562,7 @@ export const hostCommands: HostCommands = {
|
|
|
265
562
|
break;
|
|
266
563
|
}
|
|
267
564
|
}
|
|
268
|
-
if (projectsHealthy) {
|
|
565
|
+
if (reusable && projectsHealthy) {
|
|
269
566
|
// waitForChannelClose crossed the teardown fence above. A completed
|
|
270
567
|
// close-only teardown may have left this running task tombstoned,
|
|
271
568
|
// so the idempotent-success transition must reopen delivery just
|
|
@@ -288,9 +585,51 @@ export const hostCommands: HostCommands = {
|
|
|
288
585
|
// live container proved one of its project repositories is broken.
|
|
289
586
|
// Drain and invalidate any stale in-memory channel before taskUp
|
|
290
587
|
// recreates the same stable compose/container names and repairs Git.
|
|
291
|
-
|
|
588
|
+
if (!channelClosedForRecreate) {
|
|
589
|
+
await orchestrator.closeChannel(input.task.id);
|
|
590
|
+
}
|
|
292
591
|
}
|
|
592
|
+
const priorRuntimeState = existingTask
|
|
593
|
+
? runtimeQuarantinedForRecreate
|
|
594
|
+
? {
|
|
595
|
+
statusMirror: "stopped",
|
|
596
|
+
codeServerPort: null,
|
|
597
|
+
previewPorts: "[]",
|
|
598
|
+
lockedAt: null,
|
|
599
|
+
startedAt: existingTask.startedAt,
|
|
600
|
+
endedAt: existingTask.endedAt,
|
|
601
|
+
environmentProvider: existingTask.environmentProvider,
|
|
602
|
+
environmentLocator: existingTask.environmentLocator,
|
|
603
|
+
}
|
|
604
|
+
: {
|
|
605
|
+
statusMirror: existingTask.statusMirror,
|
|
606
|
+
lockedAt: existingTask.lockedAt,
|
|
607
|
+
startedAt: existingTask.startedAt,
|
|
608
|
+
endedAt: existingTask.endedAt,
|
|
609
|
+
environmentProvider: existingTask.environmentProvider,
|
|
610
|
+
environmentLocator: existingTask.environmentLocator,
|
|
611
|
+
}
|
|
612
|
+
: {
|
|
613
|
+
statusMirror: input.task.status,
|
|
614
|
+
lockedAt: null,
|
|
615
|
+
startedAt: null,
|
|
616
|
+
endedAt: null,
|
|
617
|
+
};
|
|
293
618
|
recordTaskStarting(input.task.id);
|
|
619
|
+
// Cross-process managed maintenance and task admission use a two-sided
|
|
620
|
+
// handshake: update/uninstall writes its durable marker before checking
|
|
621
|
+
// this DB row; task-up writes `starting` before checking admission again.
|
|
622
|
+
// Whichever arrives second observes the first, so a runtime cannot be
|
|
623
|
+
// swapped or removed while an unrecorded task enters Docker side effects.
|
|
624
|
+
const admittedMaintenanceFailure = managedMaintenanceUnavailable();
|
|
625
|
+
if (admittedMaintenanceFailure) {
|
|
626
|
+
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
627
|
+
else deleteHostTask(input.task.id);
|
|
628
|
+
if (!runtimeQuarantinedForRecreate) {
|
|
629
|
+
orchestrator.allowChannel(input.task.id);
|
|
630
|
+
}
|
|
631
|
+
return admittedMaintenanceFailure;
|
|
632
|
+
}
|
|
294
633
|
recordTaskOwner(
|
|
295
634
|
input.task.id,
|
|
296
635
|
input.task.ownerUserId,
|
|
@@ -304,6 +643,7 @@ export const hostCommands: HostCommands = {
|
|
|
304
643
|
storeTaskCliSecret(input.task.id, input.task.cliSecret);
|
|
305
644
|
}
|
|
306
645
|
recordHostEvent(input.task.id, "task.created");
|
|
646
|
+
let environmentLocator: TaskEnvironmentLocator | undefined;
|
|
307
647
|
const result = await wrapAgent(ctx, "taskUp", async () => {
|
|
308
648
|
// Self-heal the standard base image before task-up needs it. The
|
|
309
649
|
// boot-time build is best-effort + one-shot: a host that started before
|
|
@@ -337,13 +677,21 @@ export const hostCommands: HostCommands = {
|
|
|
337
677
|
);
|
|
338
678
|
}
|
|
339
679
|
try {
|
|
340
|
-
|
|
680
|
+
const provisioned = gitCredential
|
|
341
681
|
? await gitCredential.run(() =>
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
682
|
+
provisionTaskEnvironment(
|
|
683
|
+
input,
|
|
684
|
+
{ githubCredentialSocket: gitCredential.socketPath },
|
|
685
|
+
async (locator) => {
|
|
686
|
+
recordTaskEnvironmentPrepared(input.task.id, locator);
|
|
687
|
+
},
|
|
688
|
+
),
|
|
345
689
|
)
|
|
346
|
-
: await
|
|
690
|
+
: await provisionTaskEnvironment(input, {}, async (locator) => {
|
|
691
|
+
recordTaskEnvironmentPrepared(input.task.id, locator);
|
|
692
|
+
});
|
|
693
|
+
environmentLocator = provisioned.handle.descriptor.locator;
|
|
694
|
+
return provisioned.result;
|
|
347
695
|
} finally {
|
|
348
696
|
await gitCredential?.close().catch((error: unknown) => {
|
|
349
697
|
// Cleanup must not replace the actual task-up result/error. The
|
|
@@ -357,7 +705,10 @@ export const hostCommands: HostCommands = {
|
|
|
357
705
|
});
|
|
358
706
|
if (result.ok) {
|
|
359
707
|
orchestrator.allowChannel(input.task.id);
|
|
360
|
-
|
|
708
|
+
if (environmentLocator === undefined) {
|
|
709
|
+
throw new Error("task environment provision returned no locator");
|
|
710
|
+
}
|
|
711
|
+
recordTaskUpResult(input.task.id, result.value, environmentLocator);
|
|
361
712
|
recordHostEvent(input.task.id, "task.started");
|
|
362
713
|
// Degraded start (uai-init failed twice): the task is up, but say so
|
|
363
714
|
// in the feed — a silent half-start reads as a broken product.
|
|
@@ -384,6 +735,16 @@ export const hostCommands: HostCommands = {
|
|
|
384
735
|
}`,
|
|
385
736
|
),
|
|
386
737
|
);
|
|
738
|
+
} else if (result.code === HostErrorCode.HostUnavailable) {
|
|
739
|
+
// A daemon outage is retryable infrastructure state, not proof that
|
|
740
|
+
// the task itself failed. Undo the optimistic local `starting` mirror
|
|
741
|
+
// and normally reopen delivery; a runtime already proven stopped for
|
|
742
|
+
// L5 recreation stays stopped/tombstoned for the cloud's retry.
|
|
743
|
+
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
744
|
+
else deleteHostTask(input.task.id);
|
|
745
|
+
if (!runtimeQuarantinedForRecreate) {
|
|
746
|
+
orchestrator.allowChannel(input.task.id);
|
|
747
|
+
}
|
|
387
748
|
} else {
|
|
388
749
|
recordTaskError(input.task.id);
|
|
389
750
|
}
|
|
@@ -393,11 +754,55 @@ export const hostCommands: HostCommands = {
|
|
|
393
754
|
|
|
394
755
|
async taskDown(ctx, input) {
|
|
395
756
|
logCommand(ctx, "taskDown", input.taskId);
|
|
757
|
+
const orphanGc = input.cleanup?.kind === "orphan-gc";
|
|
758
|
+
// A GC command is already authoritative evidence that this cloud task is
|
|
759
|
+
// gone. Enter the lifecycle slot and tombstone its channel even when the
|
|
760
|
+
// runtime is temporarily unavailable; the durable cloud intent will retry
|
|
761
|
+
// destruction later. Ordinary teardown retains its fast preflight.
|
|
762
|
+
if (!orphanGc) {
|
|
763
|
+
// Teardown-specific admission: `checking` with a pinned backend ADMITS
|
|
764
|
+
// teardown. Gating kills on `operational` created the 2026-08-16 ring —
|
|
765
|
+
// a task holding the shared volume blocked the very maintenance whose
|
|
766
|
+
// completion the kill was waiting on. Removing load is how a degraded
|
|
767
|
+
// host heals.
|
|
768
|
+
const runtimeFailure = teardownRuntimeUnavailable();
|
|
769
|
+
if (runtimeFailure) return runtimeFailure;
|
|
770
|
+
}
|
|
396
771
|
const orchestrator = getOrchestrator();
|
|
397
772
|
return orchestrator.runTaskLifecycle(input.taskId, async () => {
|
|
773
|
+
// Preserve ordinary teardown's runtime-race behavior: if the daemon was
|
|
774
|
+
// lost while waiting for the lifecycle slot, leave its channel open.
|
|
775
|
+
if (!orphanGc) {
|
|
776
|
+
const inSlotRuntimeFailure = teardownRuntimeUnavailable();
|
|
777
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
778
|
+
}
|
|
779
|
+
let environment: TaskEnvironmentHandle<TaskDownResult> | null;
|
|
780
|
+
if (!orphanGc) {
|
|
781
|
+
try {
|
|
782
|
+
environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
783
|
+
} catch (error) {
|
|
784
|
+
return failFromUnknown(error);
|
|
785
|
+
}
|
|
786
|
+
} else {
|
|
787
|
+
environment = null;
|
|
788
|
+
}
|
|
398
789
|
// Tombstone + close before container destruction. A concurrent cloud
|
|
399
790
|
// ensure cannot reopen the channel while teardown is awaiting Docker.
|
|
400
791
|
await orchestrator.closeChannel(input.taskId);
|
|
792
|
+
if (orphanGc) {
|
|
793
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
794
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
795
|
+
try {
|
|
796
|
+
environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
797
|
+
} catch (error) {
|
|
798
|
+
return failFromUnknown(error);
|
|
799
|
+
}
|
|
800
|
+
// Channel startup and runtime recovery launch SSH reconciliation in
|
|
801
|
+
// the background. Fence every future launch and join the exact older
|
|
802
|
+
// ensure before container destruction; otherwise a paused health probe
|
|
803
|
+
// could materialize the private key after localStatePruned was sent.
|
|
804
|
+
await retireTaskSshIdentityEnsures(input.taskId);
|
|
805
|
+
}
|
|
401
806
|
// Ad-hoc preview proxies share this task's compose network and stable
|
|
402
807
|
// names. Remove them inside the same lifecycle slot before compose down.
|
|
403
808
|
try {
|
|
@@ -409,10 +814,34 @@ export const hostCommands: HostCommands = {
|
|
|
409
814
|
}`,
|
|
410
815
|
);
|
|
411
816
|
}
|
|
412
|
-
const result = await wrapAgent(ctx, "taskDown", () =>
|
|
413
|
-
agent.taskDown(input)
|
|
414
|
-
|
|
817
|
+
const result = await wrapAgent(ctx, "taskDown", async () => {
|
|
818
|
+
if (environment === null) return agent.taskDown(input);
|
|
819
|
+
return taskDownResultForInput(input, await environment.teardown());
|
|
820
|
+
});
|
|
415
821
|
if (result.ok) {
|
|
822
|
+
if (orphanGc) {
|
|
823
|
+
try {
|
|
824
|
+
// task-down.sh is destructive but its exit alone is not the proof:
|
|
825
|
+
// independently re-query every exact Docker label and the one
|
|
826
|
+
// derived task directory before deleting the remaining authority.
|
|
827
|
+
await proveOrphanTaskResourcesAbsent(input.taskId);
|
|
828
|
+
clearTaskGatewayAclStrict(input.taskId);
|
|
829
|
+
removeTaskCliSecretStrict(input.taskId);
|
|
830
|
+
removeTaskIdentityStrict(input.taskId);
|
|
831
|
+
clearRefresh(input.taskId);
|
|
832
|
+
// Last, atomically. Do not record taskDown/task.ended first: both
|
|
833
|
+
// would recreate the very inventory evidence being pruned.
|
|
834
|
+
purgeHostTaskState(input.taskId);
|
|
835
|
+
return ok({ ...result.value, localStatePruned: true as const });
|
|
836
|
+
} catch (error) {
|
|
837
|
+
console.warn(
|
|
838
|
+
`[host-agent] task ${input.taskId}: orphan cleanup proof failed: ${
|
|
839
|
+
error instanceof Error ? error.message : String(error)
|
|
840
|
+
}`,
|
|
841
|
+
);
|
|
842
|
+
return failFromUnknown(error);
|
|
843
|
+
}
|
|
844
|
+
}
|
|
416
845
|
// Revoke only after destruction succeeds. On failure the task is
|
|
417
846
|
// reopened below and its on-disk MCP definitions must retain the same
|
|
418
847
|
// gateway token; deleting it early would make those exact definitions
|
|
@@ -424,7 +853,7 @@ export const hostCommands: HostCommands = {
|
|
|
424
853
|
} else {
|
|
425
854
|
// Destruction failed; let a subsequent ensure restore the still-running
|
|
426
855
|
// task instead of leaving it permanently tombstoned.
|
|
427
|
-
orchestrator.allowChannel(input.taskId);
|
|
856
|
+
if (!orphanGc) orchestrator.allowChannel(input.taskId);
|
|
428
857
|
}
|
|
429
858
|
return result;
|
|
430
859
|
});
|
|
@@ -432,9 +861,19 @@ export const hostCommands: HostCommands = {
|
|
|
432
861
|
|
|
433
862
|
async taskStatus(ctx, taskId) {
|
|
434
863
|
logCommand(ctx, "taskStatus", taskId);
|
|
435
|
-
const
|
|
436
|
-
|
|
437
|
-
)
|
|
864
|
+
const runtimeFailure = runtimeUnavailable();
|
|
865
|
+
if (runtimeFailure) return runtimeFailure;
|
|
866
|
+
const result = await wrapAgent(ctx, "taskStatus", async () => {
|
|
867
|
+
const environment = await reconstructHostTaskEnvironment(taskId);
|
|
868
|
+
if (environment === null) {
|
|
869
|
+
return {
|
|
870
|
+
composeRunning: false,
|
|
871
|
+
containers: [],
|
|
872
|
+
worktreePresent: false,
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
return taskEnvironmentStatusResult(await environment.status());
|
|
876
|
+
});
|
|
438
877
|
// ADR-051: attach the published preview ports from the host DB so the
|
|
439
878
|
// cloud can backfill its mirror on reconcile (tasks launched before the
|
|
440
879
|
// cloud started recording them at task-up).
|
|
@@ -451,6 +890,15 @@ export const hostCommands: HostCommands = {
|
|
|
451
890
|
|
|
452
891
|
async channelEnsure(ctx, input) {
|
|
453
892
|
logCommand(ctx, "channelEnsure", input.taskId);
|
|
893
|
+
const admissionFailure = managedMaintenanceUnavailable();
|
|
894
|
+
if (admissionFailure) return admissionFailure;
|
|
895
|
+
const runtimeFailure = runtimeUnavailable();
|
|
896
|
+
if (runtimeFailure) return runtimeFailure;
|
|
897
|
+
await agentClisReady;
|
|
898
|
+
const readyAdmissionFailure = managedMaintenanceUnavailable();
|
|
899
|
+
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
900
|
+
const readyRuntimeFailure = runtimeUnavailable();
|
|
901
|
+
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
454
902
|
try {
|
|
455
903
|
getOrchestrator().registerChannelSpec(normalizeChannelSpec(input));
|
|
456
904
|
const ready = await getOrchestrator().ensureStarted(input.taskId);
|
|
@@ -491,6 +939,8 @@ export const hostCommands: HostCommands = {
|
|
|
491
939
|
|
|
492
940
|
async channelDeliver(ctx, taskId, agentId, text) {
|
|
493
941
|
logCommand(ctx, "channelDeliver", taskId, agentId);
|
|
942
|
+
const runtimeFailure = runtimeUnavailable();
|
|
943
|
+
if (runtimeFailure) return runtimeFailure;
|
|
494
944
|
try {
|
|
495
945
|
const result = await getOrchestrator().deliver(taskId, agentId, text);
|
|
496
946
|
if (!result.ok) {
|
|
@@ -508,9 +958,13 @@ export const hostCommands: HostCommands = {
|
|
|
508
958
|
|
|
509
959
|
async previewEnsure(ctx, taskId, name, containerPort) {
|
|
510
960
|
logCommand(ctx, "previewEnsure", taskId, name);
|
|
961
|
+
const runtimeFailure = runtimeUnavailable();
|
|
962
|
+
if (runtimeFailure) return runtimeFailure;
|
|
511
963
|
const orchestrator = getOrchestrator();
|
|
512
964
|
return orchestrator.runTaskLifecycle(taskId, async () => {
|
|
513
965
|
try {
|
|
966
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
967
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
514
968
|
const task = getHostTask(taskId);
|
|
515
969
|
if (!task) {
|
|
516
970
|
return {
|
|
@@ -522,20 +976,49 @@ export const hostCommands: HostCommands = {
|
|
|
522
976
|
if (task.statusMirror !== "running") {
|
|
523
977
|
return ok({ hostPort: null });
|
|
524
978
|
}
|
|
525
|
-
|
|
979
|
+
if (!task.composeProject) return ok({ hostPort: null });
|
|
526
980
|
const declared = parsePreviewPortRuntimes(task.previewPorts).find(
|
|
527
981
|
(port) => port.name === name,
|
|
528
982
|
);
|
|
529
|
-
|
|
983
|
+
// A stored host port is never routing truth after an unnoticed daemon
|
|
984
|
+
// restart. Prove the exact app container and current mapping before
|
|
985
|
+
// either returning a declared port or creating an ad-hoc sidecar.
|
|
986
|
+
const appProof = await inspectTunnelContainer(
|
|
987
|
+
`${task.composeProject}-app-1`,
|
|
988
|
+
containerPort,
|
|
989
|
+
);
|
|
990
|
+
if (appProof.kind === "daemon-unavailable") {
|
|
991
|
+
return dockerUnavailableResult(
|
|
992
|
+
`could not verify preview runtime for task ${taskId}: ${appProof.detail}`,
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
// The proof awaited Docker. A concurrent runtime recheck can invalidate
|
|
996
|
+
// its generation before this continuation resumes, so fence the result
|
|
997
|
+
// again before returning any route to the cloud.
|
|
998
|
+
const postInspectRuntimeFailure = runtimeUnavailable();
|
|
999
|
+
if (postInspectRuntimeFailure) return postInspectRuntimeFailure;
|
|
1000
|
+
if (appProof.kind !== "running") return ok({ hostPort: null });
|
|
1001
|
+
// Task-up published port (preview enabled at launch) wins, but use the
|
|
1002
|
+
// mapping Docker just proved rather than the persisted hostPort.
|
|
1003
|
+
if (declared) {
|
|
1004
|
+
return ok({ hostPort: appProof.published?.port ?? null });
|
|
1005
|
+
}
|
|
530
1006
|
// Else start (or find) the node-proxy sidecar NOW — the same one the
|
|
531
1007
|
// tunnel lazy-starts on first access — and hand back its host port.
|
|
532
|
-
if (!task.composeProject) return ok({ hostPort: null });
|
|
533
1008
|
const hostPort = await ensurePreviewSidecar({
|
|
534
1009
|
taskId,
|
|
535
1010
|
composeProject: task.composeProject,
|
|
536
1011
|
name,
|
|
537
1012
|
containerPort,
|
|
538
1013
|
});
|
|
1014
|
+
const postSidecarRuntimeFailure = runtimeUnavailable();
|
|
1015
|
+
if (postSidecarRuntimeFailure) return postSidecarRuntimeFailure;
|
|
1016
|
+
if (hostPort === null) {
|
|
1017
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(
|
|
1018
|
+
`could not prepare preview runtime for task ${taskId}`,
|
|
1019
|
+
);
|
|
1020
|
+
if (unavailable) return unavailable;
|
|
1021
|
+
}
|
|
539
1022
|
return ok({ hostPort });
|
|
540
1023
|
} catch (err) {
|
|
541
1024
|
return failFromUnknown(err);
|
|
@@ -576,8 +1059,24 @@ export const hostCommands: HostCommands = {
|
|
|
576
1059
|
|
|
577
1060
|
async taskDiff(ctx, input) {
|
|
578
1061
|
logCommand(ctx, "taskDiff", input.taskId);
|
|
1062
|
+
const runtimeFailure = runtimeUnavailable();
|
|
1063
|
+
if (runtimeFailure) return runtimeFailure;
|
|
579
1064
|
try {
|
|
580
|
-
|
|
1065
|
+
const task = getHostTask(input.taskId);
|
|
1066
|
+
// A stopped/terminal task retains its host worktree, so preserve the
|
|
1067
|
+
// historical host-side fallback there. A running task must reconstruct
|
|
1068
|
+
// its exact persisted provider identity: malformed/foreign metadata is
|
|
1069
|
+
// not permission to run Git against a different filesystem view.
|
|
1070
|
+
const environment =
|
|
1071
|
+
task?.statusMirror === "running"
|
|
1072
|
+
? await reconstructPersistedTaskEnvironment(task)
|
|
1073
|
+
: null;
|
|
1074
|
+
if (task?.statusMirror === "running" && environment === null) {
|
|
1075
|
+
throw new Error(
|
|
1076
|
+
`running task ${input.taskId} has no persisted task environment`,
|
|
1077
|
+
);
|
|
1078
|
+
}
|
|
1079
|
+
return ok(await buildTaskDiff(input, environment));
|
|
581
1080
|
} catch (err) {
|
|
582
1081
|
return failFromUnknown(err);
|
|
583
1082
|
}
|
|
@@ -655,6 +1154,35 @@ function normalizeChannelSpec(input: ChannelEnsureInput): ChannelEnsureInput {
|
|
|
655
1154
|
return { ...input, workspacePath: "/workspace" };
|
|
656
1155
|
}
|
|
657
1156
|
|
|
1157
|
+
async function reconstructHostTaskEnvironment(
|
|
1158
|
+
taskId: string,
|
|
1159
|
+
): Promise<TaskEnvironmentHandle<TaskDownResult> | null> {
|
|
1160
|
+
const task = getHostTask(taskId);
|
|
1161
|
+
return task === null ? null : reconstructPersistedTaskEnvironment(task);
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
function taskEnvironmentStatusResult(status: TaskEnvironmentStatus) {
|
|
1165
|
+
if (status.state === "unknown") {
|
|
1166
|
+
throw new Error(`task environment status is unknown: ${status.detail}`);
|
|
1167
|
+
}
|
|
1168
|
+
return {
|
|
1169
|
+
composeRunning: status.state === "running",
|
|
1170
|
+
containers: [...status.instances],
|
|
1171
|
+
worktreePresent: status.workspacePresent,
|
|
1172
|
+
};
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
function taskDownResultForInput(
|
|
1176
|
+
input: TaskDownInput,
|
|
1177
|
+
providerResult: TaskDownResult,
|
|
1178
|
+
): TaskDownResult {
|
|
1179
|
+
const status = input.task.status;
|
|
1180
|
+
return {
|
|
1181
|
+
...providerResult,
|
|
1182
|
+
status: status === "shipped" || status === "error" ? status : "killed",
|
|
1183
|
+
};
|
|
1184
|
+
}
|
|
1185
|
+
|
|
658
1186
|
function logCommand(
|
|
659
1187
|
ctx: CommandContext,
|
|
660
1188
|
command: keyof HostCommands,
|
|
@@ -671,6 +1199,13 @@ async function wrapAgent<T>(
|
|
|
671
1199
|
try {
|
|
672
1200
|
return ok(await fn());
|
|
673
1201
|
} catch (err) {
|
|
1202
|
+
if (AgentError.is(err) && isRuntimeSensitiveAgentError(err.code)) {
|
|
1203
|
+
if (err.code === "DOCKER_PS_FAILED") {
|
|
1204
|
+
return dockerUnavailableResult(err.message);
|
|
1205
|
+
}
|
|
1206
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(err.message);
|
|
1207
|
+
if (unavailable) return unavailable;
|
|
1208
|
+
}
|
|
674
1209
|
return failFromUnknown(err);
|
|
675
1210
|
}
|
|
676
1211
|
}
|
|
@@ -679,7 +1214,193 @@ function ok<T>(value: T): HostCommandResult<T> {
|
|
|
679
1214
|
return { ok: true, value };
|
|
680
1215
|
}
|
|
681
1216
|
|
|
1217
|
+
/** Docker-dependent commands must not reinterpret an unavailable or
|
|
1218
|
+
* half-recovered backend as an absent task/container. */
|
|
1219
|
+
function runtimeUnavailable(): HostCommandResult<never> | null {
|
|
1220
|
+
const message = containerRuntimeProblem();
|
|
1221
|
+
if (!message) return null;
|
|
1222
|
+
return {
|
|
1223
|
+
ok: false,
|
|
1224
|
+
code: HostErrorCode.HostUnavailable,
|
|
1225
|
+
message,
|
|
1226
|
+
retryable: true,
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
function teardownRuntimeUnavailable(): HostCommandResult<never> | null {
|
|
1231
|
+
const message = containerRuntimeTeardownProblem();
|
|
1232
|
+
if (!message) return null;
|
|
1233
|
+
return {
|
|
1234
|
+
ok: false,
|
|
1235
|
+
code: HostErrorCode.HostUnavailable,
|
|
1236
|
+
message,
|
|
1237
|
+
retryable: true,
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
const RUNTIME_TASK_ADMISSION_WAIT_MS = 3_000;
|
|
1242
|
+
|
|
1243
|
+
/**
|
|
1244
|
+
* Runtime activation is intentionally detached from one cloud command's
|
|
1245
|
+
* 15-minute deadline. A first attempt joins for a few seconds, then returns a
|
|
1246
|
+
* narrowly-tagged retry while the single detection/activation promise keeps
|
|
1247
|
+
* running. Every retry rechecks the exact generation and durable provider
|
|
1248
|
+
* claim before task-owned side effects begin.
|
|
1249
|
+
*/
|
|
1250
|
+
async function prepareContainerRuntimeForTask(): Promise<
|
|
1251
|
+
HostCommandResult<never> | null
|
|
1252
|
+
> {
|
|
1253
|
+
let prepared:
|
|
1254
|
+
| Awaited<ReturnType<typeof ensureContainerRuntimeForTask>>
|
|
1255
|
+
| null;
|
|
1256
|
+
try {
|
|
1257
|
+
prepared = await settleWithin(
|
|
1258
|
+
ensureContainerRuntimeForTask(),
|
|
1259
|
+
RUNTIME_TASK_ADMISSION_WAIT_MS,
|
|
1260
|
+
);
|
|
1261
|
+
} catch (error) {
|
|
1262
|
+
return runtimeSetupRetry(
|
|
1263
|
+
`container runtime setup failed: ${boundedFailureMessage(error)}`,
|
|
1264
|
+
);
|
|
1265
|
+
}
|
|
1266
|
+
if (prepared === null) {
|
|
1267
|
+
return runtimeSetupRetry(
|
|
1268
|
+
"The bundled container runtime is still being provisioned on this host.",
|
|
1269
|
+
);
|
|
1270
|
+
}
|
|
1271
|
+
if (prepared.status !== "ready") {
|
|
1272
|
+
return {
|
|
1273
|
+
ok: false,
|
|
1274
|
+
code: HostErrorCode.HostUnavailable,
|
|
1275
|
+
message:
|
|
1276
|
+
prepared.status === "no-runtime"
|
|
1277
|
+
? prepared.message
|
|
1278
|
+
: "The container runtime is still being detected on this host.",
|
|
1279
|
+
retryable: true,
|
|
1280
|
+
};
|
|
1281
|
+
}
|
|
1282
|
+
const epoch = containerRuntimeReadinessEpoch();
|
|
1283
|
+
const operational = await waitForContainerRuntimeOperational(
|
|
1284
|
+
epoch,
|
|
1285
|
+
RUNTIME_TASK_ADMISSION_WAIT_MS,
|
|
1286
|
+
);
|
|
1287
|
+
if (!operational) {
|
|
1288
|
+
const failure = runtimeUnavailable();
|
|
1289
|
+
if (failure && !failure.ok && !failure.message.includes("still preparing")) {
|
|
1290
|
+
return failure;
|
|
1291
|
+
}
|
|
1292
|
+
return runtimeSetupRetry(
|
|
1293
|
+
"The container runtime is completing image maintenance and task recovery.",
|
|
1294
|
+
);
|
|
1295
|
+
}
|
|
1296
|
+
try {
|
|
1297
|
+
if (await claimContainerRuntimeForTask(epoch)) return null;
|
|
1298
|
+
} catch (error) {
|
|
1299
|
+
return runtimeSetupRetry(
|
|
1300
|
+
`The container runtime could not persist its task-state claim: ${boundedFailureMessage(error)}`,
|
|
1301
|
+
);
|
|
1302
|
+
}
|
|
1303
|
+
return runtimeSetupRetry(
|
|
1304
|
+
"The container runtime changed while task admission was being prepared.",
|
|
1305
|
+
);
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
function runtimeSetupRetry(message: string): HostCommandResult<never> {
|
|
1309
|
+
return {
|
|
1310
|
+
ok: false,
|
|
1311
|
+
code: HostErrorCode.HostUnavailable,
|
|
1312
|
+
message,
|
|
1313
|
+
retryable: true,
|
|
1314
|
+
retryReason: "runtime_setup",
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
function settleWithin<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
|
|
1319
|
+
return new Promise<T | null>((resolve, reject) => {
|
|
1320
|
+
let settled = false;
|
|
1321
|
+
const timer = setTimeout(() => {
|
|
1322
|
+
if (settled) return;
|
|
1323
|
+
settled = true;
|
|
1324
|
+
resolve(null);
|
|
1325
|
+
}, timeoutMs);
|
|
1326
|
+
timer.unref?.();
|
|
1327
|
+
void promise.then(
|
|
1328
|
+
(value) => {
|
|
1329
|
+
if (settled) return;
|
|
1330
|
+
settled = true;
|
|
1331
|
+
clearTimeout(timer);
|
|
1332
|
+
resolve(value);
|
|
1333
|
+
},
|
|
1334
|
+
(error: unknown) => {
|
|
1335
|
+
if (settled) return;
|
|
1336
|
+
settled = true;
|
|
1337
|
+
clearTimeout(timer);
|
|
1338
|
+
reject(error);
|
|
1339
|
+
},
|
|
1340
|
+
);
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
function boundedFailureMessage(error: unknown): string {
|
|
1345
|
+
return (error instanceof Error ? error.message : String(error)).slice(0, 512);
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
/** Managed installation, activation, and uninstall drain instead of admitting work. */
|
|
1349
|
+
function managedMaintenanceUnavailable(): HostCommandResult<never> | null {
|
|
1350
|
+
if (!managedRuntimeAdmissionPending()) return null;
|
|
1351
|
+
return {
|
|
1352
|
+
ok: false,
|
|
1353
|
+
code: HostErrorCode.HostUnavailable,
|
|
1354
|
+
message: "managed host maintenance is waiting for this runtime to restart",
|
|
1355
|
+
retryable: true,
|
|
1356
|
+
retryReason: "managed_restart",
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
function dockerUnavailableResult<T = never>(
|
|
1361
|
+
message: string,
|
|
1362
|
+
): HostCommandResult<T> {
|
|
1363
|
+
// Invalidate synchronously so a second command in this event-loop turn
|
|
1364
|
+
// cannot touch the daemon under a stale ready verdict. Detection/recovery is
|
|
1365
|
+
// asynchronous and deliberately does not delay the retryable response.
|
|
1366
|
+
suspendContainerRuntime(true);
|
|
1367
|
+
void initializeContainerRuntime();
|
|
1368
|
+
return {
|
|
1369
|
+
ok: false,
|
|
1370
|
+
code: HostErrorCode.HostUnavailable,
|
|
1371
|
+
message,
|
|
1372
|
+
retryable: true,
|
|
1373
|
+
};
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
async function runtimeFailureIfDaemonUnavailable<T = never>(
|
|
1377
|
+
message: string,
|
|
1378
|
+
): Promise<HostCommandResult<T> | null> {
|
|
1379
|
+
return (await reprobeContainerRuntimeMachineIdentity()) !== null
|
|
1380
|
+
? null
|
|
1381
|
+
: dockerUnavailableResult(message);
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
function isRuntimeSensitiveAgentError(code: string): boolean {
|
|
1385
|
+
return (
|
|
1386
|
+
code === "STANDARD_IMAGE_UNAVAILABLE" ||
|
|
1387
|
+
code === "STANDARD_IMAGE_MISSING" ||
|
|
1388
|
+
code === "COMPOSE_UP_FAILED" ||
|
|
1389
|
+
code === "COMPOSE_DOWN_FAILED" ||
|
|
1390
|
+
code === "CONTAINER_INIT_FAILED" ||
|
|
1391
|
+
code === "DOCKER_PS_FAILED"
|
|
1392
|
+
);
|
|
1393
|
+
}
|
|
1394
|
+
|
|
682
1395
|
function failFromUnknown<T = never>(err: unknown): HostCommandResult<T> {
|
|
1396
|
+
if (err instanceof ContainerRuntimeUnavailableError) {
|
|
1397
|
+
return {
|
|
1398
|
+
ok: false,
|
|
1399
|
+
code: HostErrorCode.HostUnavailable,
|
|
1400
|
+
message: err.message,
|
|
1401
|
+
retryable: true,
|
|
1402
|
+
};
|
|
1403
|
+
}
|
|
683
1404
|
if (AgentError.is(err)) {
|
|
684
1405
|
return {
|
|
685
1406
|
ok: false,
|
|
@@ -727,6 +1448,8 @@ function mapAgentError(code: string): HostErrorCode {
|
|
|
727
1448
|
return HostErrorCode.ContainerInitFailed;
|
|
728
1449
|
case "DB_FAILED":
|
|
729
1450
|
return HostErrorCode.DbFailed;
|
|
1451
|
+
case "DOCKER_PS_FAILED":
|
|
1452
|
+
return HostErrorCode.HostUnavailable;
|
|
730
1453
|
default:
|
|
731
1454
|
return HostErrorCode.Internal;
|
|
732
1455
|
}
|
|
@@ -758,7 +1481,8 @@ function isRetryableAgentError(code: string): boolean {
|
|
|
758
1481
|
return (
|
|
759
1482
|
code === "FETCH_FAILED" ||
|
|
760
1483
|
code === "CONTAINER_INIT_FAILED" ||
|
|
761
|
-
code === "GITHUB_AUTH_UNAVAILABLE"
|
|
1484
|
+
code === "GITHUB_AUTH_UNAVAILABLE" ||
|
|
1485
|
+
code === "DOCKER_PS_FAILED"
|
|
762
1486
|
);
|
|
763
1487
|
}
|
|
764
1488
|
|