@runuai/host 0.9.14 → 0.9.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +22 -5
  2. package/db/migrations/0014_host_inventory_event_index.sql +1 -0
  3. package/db/migrations/0015_host_settings.sql +9 -0
  4. package/db/migrations/0016_task_environment.sql +2 -0
  5. package/db/migrations/meta/_journal.json +21 -0
  6. package/db/schema.ts +80 -30
  7. package/images/standard/Dockerfile +36 -10
  8. package/images/standard/README.md +63 -18
  9. package/images/standard/container/corepack-version +1 -0
  10. package/images/standard/container/uai-init +308 -38
  11. package/images/standard/container/uai-materialize-runtimes +1527 -0
  12. package/lib/agent-cli.ts +33 -2
  13. package/lib/agent.ts +46 -7
  14. package/lib/agents/claude.ts +13 -8
  15. package/lib/agents/codex.ts +11 -6
  16. package/lib/agents/cursor.ts +39 -29
  17. package/lib/agents/durable-proc.ts +20 -27
  18. package/lib/agents/factory.ts +9 -25
  19. package/lib/agents/grok.ts +43 -30
  20. package/lib/agents/kimi.ts +44 -29
  21. package/lib/agents/opencode.ts +43 -31
  22. package/lib/agents/proc.ts +149 -114
  23. package/lib/agents/transport.ts +62 -50
  24. package/lib/agents/types.ts +6 -4
  25. package/lib/apple-runtime-recycle.ts +236 -0
  26. package/lib/apple-uninstall-teardown.ts +224 -0
  27. package/lib/browser-testing.ts +233 -93
  28. package/lib/codex-auth.ts +40 -6
  29. package/lib/command-db.ts +20 -0
  30. package/lib/container-runtime.ts +1338 -0
  31. package/lib/db.ts +1 -0
  32. package/lib/docker-exec.ts +87 -5
  33. package/lib/engine-accounts.ts +68 -5
  34. package/lib/engine-login.ts +1952 -0
  35. package/lib/enrollment-state.ts +251 -0
  36. package/lib/env-file.ts +155 -0
  37. package/lib/env.ts +4 -0
  38. package/lib/git-diff.ts +98 -32
  39. package/lib/git-identity.ts +199 -87
  40. package/lib/github-tokens.ts +202 -91
  41. package/lib/host-cloud-url.ts +62 -0
  42. package/lib/host-config.ts +279 -0
  43. package/lib/host-logs.ts +962 -0
  44. package/lib/keyed-promise-tail.ts +23 -0
  45. package/lib/legacy-runtime-v1.fixture.ts +627 -0
  46. package/lib/managed-activation-watcher.ts +72 -0
  47. package/lib/managed-install-owner-watcher.ts +55 -0
  48. package/lib/managed-operation-drain.ts +49 -0
  49. package/lib/managed-runtime.ts +3644 -0
  50. package/lib/managed-update-scheduler.ts +125 -0
  51. package/lib/mcp-gateway.ts +450 -23
  52. package/lib/orchestrator.ts +3051 -200
  53. package/lib/preview-sidecar.ts +68 -14
  54. package/lib/release-manifest.ts +708 -0
  55. package/lib/release-trust.ts +28 -0
  56. package/lib/runtime-activation-tail.ts +232 -0
  57. package/lib/runtime-archive.ts +1086 -0
  58. package/lib/runtime-authority.ts +79 -0
  59. package/lib/runtime-guard.ts +36 -0
  60. package/lib/runtime-provider-state.ts +169 -0
  61. package/lib/runtime-state.ts +232 -12
  62. package/lib/skills.ts +24 -3
  63. package/lib/ssh.ts +18 -0
  64. package/lib/standard-image.ts +1104 -141
  65. package/lib/stopped-task-status-queue.ts +44 -0
  66. package/lib/task-container-cli.ts +269 -0
  67. package/lib/task-diff.ts +66 -46
  68. package/lib/task-environment/apple-container.ts +757 -0
  69. package/lib/task-environment/docker.ts +956 -0
  70. package/lib/task-environment/index.ts +364 -0
  71. package/lib/task-environment/legacy-adoption.ts +459 -0
  72. package/lib/task-environment/registry.ts +58 -0
  73. package/lib/task-environment/types.ts +408 -0
  74. package/lib/task-identity.ts +19 -0
  75. package/lib/task-inventory.ts +585 -0
  76. package/lib/tunnel-registry.ts +135 -19
  77. package/lib/tunnel-runtime.ts +235 -0
  78. package/package.json +1 -1
  79. package/scripts/agent/_common.sh +123 -3
  80. package/scripts/agent/task-down.sh +146 -38
  81. package/scripts/agent/task-status.sh +19 -3
  82. package/scripts/agent/task-up.sh +1405 -107
  83. package/scripts/install/darwin.ts +848 -50
  84. package/scripts/install/linux.ts +838 -35
  85. package/scripts/install/types.ts +43 -0
  86. package/scripts/install/util.ts +215 -8
  87. package/scripts/install/win.ts +12 -0
  88. package/src/apple-tunnel-route.ts +104 -0
  89. package/src/cli.ts +1464 -72
  90. package/src/event-outbox.ts +83 -4
  91. package/src/index.ts +766 -42
  92. package/src/main.ts +1398 -255
  93. package/src/paths.ts +17 -1
  94. package/src/protocol.ts +695 -1
  95. package/src/runtime-bootstrap.ts +165 -0
  96. package/src/ui/server.ts +46 -10
  97. package/src/ui/types.ts +37 -0
package/src/index.ts CHANGED
@@ -1,9 +1,22 @@
1
- import { agent, AgentError, toolStderrDetail } from "../lib/agent";
2
- import { ensureStandardImage } from "../lib/standard-image";
1
+ import {
2
+ agent,
3
+ AgentError,
4
+ toolStderrDetail,
5
+ type TaskDownResult,
6
+ } from "../lib/agent";
7
+ import { agentClisReady, ensureStandardImage } from "../lib/standard-image";
3
8
  import { cloneRepo } from "../lib/repo-clone";
4
9
  import { handleFilesOp } from "../lib/shared-files";
5
- import { getOrchestrator } from "../lib/orchestrator";
6
- import { storeTaskCliSecret } from "../lib/agent-cli";
10
+ import {
11
+ getOrchestrator,
12
+ inspectTaskRuntimeContainer,
13
+ proveAppleTaskRuntimeContract,
14
+ quarantineTaskRuntimeContainer,
15
+ } from "../lib/orchestrator";
16
+ import {
17
+ removeTaskCliSecretStrict,
18
+ storeTaskCliSecret,
19
+ } from "../lib/agent-cli";
7
20
  import {
8
21
  clearRefresh,
9
22
  reconcileTaskGitAuth,
@@ -16,21 +29,56 @@ import { readAttachment, writeAttachment } from "../lib/attachments";
16
29
  import { appendTranscript as writeTranscript } from "../lib/transcript";
17
30
  import { buildTaskDiff } from "../lib/task-diff";
18
31
  import {
32
+ deleteHostTask,
19
33
  getHostTask,
20
34
  recordHostEvent,
21
35
  recordTaskDown,
36
+ recordTaskEnvironmentPrepared,
22
37
  recordTaskError,
23
38
  recordTaskOwner,
24
39
  recordTaskStarting,
25
40
  recordTaskUpResult,
41
+ purgeHostTaskState,
42
+ upsertHostTask,
26
43
  } from "../lib/runtime-state";
27
- import { clearTaskGatewayAcl } from "../lib/mcp-gateway";
44
+ import {
45
+ clearTaskGatewayAcl,
46
+ clearTaskGatewayAclStrict,
47
+ } from "../lib/mcp-gateway";
28
48
  import { parsePreviewPortRuntimes } from "../lib/preview-ports";
29
49
  import { dockerCli } from "../lib/docker-exec";
50
+ import {
51
+ claimContainerRuntimeForTask,
52
+ containerRuntimeGenerationOperational,
53
+ containerRuntimeProblem,
54
+ containerRuntimeReadinessEpoch,
55
+ containerRuntimeTeardownProblem,
56
+ ensureContainerRuntimeForTask,
57
+ initializeContainerRuntime,
58
+ reprobeContainerRuntimeMachineIdentity,
59
+ suspendContainerRuntime,
60
+ waitForContainerRuntimeOperational,
61
+ } from "../lib/container-runtime";
62
+ import {
63
+ provisionTaskEnvironment,
64
+ reconstructPersistedTaskEnvironment,
65
+ } from "../lib/task-environment";
66
+ import { appleTaskContainerName } from "../lib/task-environment/apple-container";
67
+ import type {
68
+ TaskEnvironmentHandle,
69
+ TaskEnvironmentLocator,
70
+ TaskEnvironmentStatus,
71
+ } from "../lib/task-environment/types";
30
72
  import {
31
73
  ensurePreviewSidecar,
32
74
  stopPreviewSidecars,
33
75
  } from "../lib/preview-sidecar";
76
+ import { inspectTunnelContainer } from "../lib/tunnel-runtime";
77
+ import { ContainerRuntimeUnavailableError } from "../lib/runtime-guard";
78
+ import { managedRuntimeAdmissionPending } from "../lib/managed-runtime";
79
+ import { retireTaskSshIdentityEnsures } from "../lib/git-identity";
80
+ import { removeTaskIdentityStrict } from "../lib/ssh";
81
+ import { proveOrphanTaskResourcesAbsent } from "../lib/task-inventory";
34
82
  import {
35
83
  HostErrorCode,
36
84
  type CommandContext,
@@ -118,18 +166,212 @@ export const hostEvents: HostEventStream = {
118
166
  export const hostCommands: HostCommands = {
119
167
  async taskUp(ctx, input) {
120
168
  logCommand(ctx, "taskUp", input.task.id);
169
+ const admissionFailure = managedMaintenanceUnavailable();
170
+ if (admissionFailure) return admissionFailure;
171
+ const runtimePreparationFailure = await prepareContainerRuntimeForTask();
172
+ if (runtimePreparationFailure) return runtimePreparationFailure;
173
+ const runtimeFailure = runtimeUnavailable();
174
+ if (runtimeFailure) return runtimeFailure;
175
+ // The boot reconcile briefly replaces CLI shims in the shared volume.
176
+ // Do not create local lifecycle/channel state until its one-way latch has
177
+ // settled (success or best-effort failure), then revalidate both mutable
178
+ // admission authorities after the await.
179
+ await agentClisReady;
180
+ const readyAdmissionFailure = managedMaintenanceUnavailable();
181
+ if (readyAdmissionFailure) return readyAdmissionFailure;
182
+ const readyRuntimeFailure = runtimeUnavailable();
183
+ if (readyRuntimeFailure) return readyRuntimeFailure;
121
184
  const orchestrator = getOrchestrator();
122
185
  return orchestrator.runTaskLifecycle(input.task.id, async () => {
123
186
  // A direct duplicate teardown may still be draining operations that
124
187
  // target this task's stable compose/container names. Never recreate
125
188
  // those names until every close caller has crossed the shared fence.
126
189
  await orchestrator.waitForChannelClose(input.task.id);
190
+ const inSlotAdmissionFailure = managedMaintenanceUnavailable();
191
+ if (inSlotAdmissionFailure) return inSlotAdmissionFailure;
192
+ const inSlotRuntimeFailure = runtimeUnavailable();
193
+ if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
127
194
  // A command retry that arrives after the original start completed must
128
195
  // not rerender/compose-up/uai-init underneath live agent sessions. Treat
129
196
  // that lost-response retry as the successful operation it already was:
130
197
  // every cloud caller records a non-ok taskUp as an errored task.
131
198
  const existingTask = getHostTask(input.task.id);
132
- if (existingTask?.statusMirror === "running") {
199
+ let runtimeQuarantinedForRecreate = false;
200
+ let channelClosedForRecreate = false;
201
+ if (
202
+ existingTask?.statusMirror === "running" &&
203
+ existingTask.environmentProvider === "apple-container"
204
+ ) {
205
+ // ADR-106/round-4: the lost-response fast path must speak the
206
+ // provider's language — the Docker branch below ENOENTs on a
207
+ // Docker-free Mac and suspended the generation instead of returning
208
+ // the persisted success.
209
+ if (!existingTask.composeProject || !existingTask.worktreePath) {
210
+ return {
211
+ ok: false,
212
+ code: HostErrorCode.Internal,
213
+ message:
214
+ `task ${input.task.id} is running but its persisted runtime ` +
215
+ "metadata is incomplete",
216
+ };
217
+ }
218
+ // Round 5: every await below can outlive the runtime generation
219
+ // (bridge revalidation, forced recheck). A superseded pass must
220
+ // observe only — never reopen consumers or mutate the container —
221
+ // the same discipline apple recovery applies.
222
+ // Round 6: the fence requires the OPERATIONAL generation, not merely
223
+ // the epoch — plain suspension (connection revalidation) keeps the
224
+ // epoch and ready status while the daemon is unverified.
225
+ const fastEpoch = containerRuntimeReadinessEpoch();
226
+ const fastStale = (): boolean =>
227
+ !containerRuntimeGenerationOperational(fastEpoch);
228
+ const staleRetry = {
229
+ ok: false as const,
230
+ code: HostErrorCode.HostUnavailable,
231
+ message:
232
+ `the container runtime generation changed while verifying task ${input.task.id}; retry`,
233
+ retryable: true,
234
+ };
235
+ const environment = await reconstructPersistedTaskEnvironment(
236
+ existingTask,
237
+ );
238
+ if (fastStale()) return staleRetry;
239
+ if (environment === null) {
240
+ // The persisted locator no longer reconstructs (e.g. an engine
241
+ // identity change) — recreate through the normal path.
242
+ await orchestrator.closeChannel(input.task.id);
243
+ // Session drain can outlive the generation too: a superseded pass
244
+ // must not reach recordTaskStarting/provisioning below.
245
+ if (fastStale()) return staleRetry;
246
+ channelClosedForRecreate = true;
247
+ } else {
248
+ const appleStatus = await environment.status();
249
+ if (fastStale()) return staleRetry;
250
+ if (appleStatus.state === "unknown") {
251
+ return {
252
+ ok: false,
253
+ code: HostErrorCode.HostUnavailable,
254
+ message:
255
+ `could not verify runtime state for task ${input.task.id}: ` +
256
+ appleStatus.detail,
257
+ retryable: true,
258
+ };
259
+ }
260
+ if (appleStatus.state === "running") {
261
+ const verdict = await proveAppleTaskRuntimeContract({
262
+ taskId: input.task.id,
263
+ containerName: appleTaskContainerName(input.task.id),
264
+ // The row's worktreePath is task-up's $task_dir — the same
265
+ // directory the apple locator records as hostWorktreePath.
266
+ hostWorktreePath: existingTask.worktreePath,
267
+ });
268
+ if (fastStale()) return staleRetry;
269
+ if (verdict.kind === "unreachable") {
270
+ return {
271
+ ok: false,
272
+ code: HostErrorCode.HostUnavailable,
273
+ message:
274
+ `could not prove the exact runtime contract for task ${input.task.id}: ` +
275
+ verdict.detail,
276
+ retryable: true,
277
+ };
278
+ }
279
+ let projectsHealthy = true;
280
+ for (const project of verdict.kind === "holds"
281
+ ? input.projects
282
+ : []) {
283
+ const gitHealth = await environment.exec({
284
+ argv: [
285
+ "/usr/bin/env",
286
+ "-i",
287
+ "HOME=/home/node",
288
+ "PATH=/usr/bin:/bin",
289
+ "/usr/bin/git",
290
+ "rev-parse",
291
+ "--is-inside-work-tree",
292
+ ],
293
+ user: "node",
294
+ cwd: `/workspace/${project.slug}`,
295
+ env: {},
296
+ timeoutMs: 10_000,
297
+ maxOutputBytes: 64 * 1024,
298
+ });
299
+ if (fastStale()) return staleRetry;
300
+ if (gitHealth.exitCode === null || gitHealth.signal !== null) {
301
+ return {
302
+ ok: false,
303
+ code: HostErrorCode.HostUnavailable,
304
+ message:
305
+ `could not verify Git state for task ${input.task.id} ` +
306
+ `project ${project.id}: the runtime did not answer`,
307
+ retryable: true,
308
+ };
309
+ }
310
+ if (
311
+ gitHealth.exitCode !== 0 ||
312
+ Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
313
+ "true"
314
+ ) {
315
+ projectsHealthy = false;
316
+ console.warn(
317
+ `[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
318
+ );
319
+ break;
320
+ }
321
+ }
322
+ if (verdict.kind === "holds" && projectsHealthy) {
323
+ if (fastStale()) return staleRetry;
324
+ orchestrator.allowChannel(input.task.id);
325
+ return {
326
+ ok: true,
327
+ value: {
328
+ composeProject: existingTask.composeProject,
329
+ worktreePath: existingTask.worktreePath,
330
+ codeServerPort: existingTask.codeServerPort ?? undefined,
331
+ previewPorts: parsePreviewPortRuntimes(
332
+ existingTask.previewPorts,
333
+ ),
334
+ },
335
+ };
336
+ }
337
+ // Violated contract or broken project repo: drain, stop, and
338
+ // prove before task-up recreates the same stable name.
339
+ if (fastStale()) return staleRetry;
340
+ await orchestrator.closeChannel(input.task.id);
341
+ channelClosedForRecreate = true;
342
+ if (fastStale()) return staleRetry;
343
+ await environment.stop();
344
+ const after = await environment.status();
345
+ if (fastStale()) return staleRetry;
346
+ if (after.state === "running" || after.state === "unknown") {
347
+ return {
348
+ ok: false,
349
+ code: HostErrorCode.HostUnavailable,
350
+ message:
351
+ `task ${input.task.id} has a non-current runtime contract and ` +
352
+ "could not be proven stopped; retry after the runtime is healthy",
353
+ retryable: true,
354
+ };
355
+ }
356
+ // Round 5: the container is PHYSICALLY stopped now, whatever the
357
+ // reason (violated contract or a broken project repo). A failed
358
+ // recreation must roll the mirror back to stopped — restoring
359
+ // "running" would advertise a container proven not to run.
360
+ runtimeQuarantinedForRecreate = true;
361
+ } else {
362
+ // Provider proved the container absent/stopped: normal self-heal
363
+ // recreate — and the rollback state must say stopped for the
364
+ // same reason as above.
365
+ if (fastStale()) return staleRetry;
366
+ await orchestrator.closeChannel(input.task.id);
367
+ // Fence AFTER the drain as well: provisioning must never start
368
+ // under a generation this pass no longer owns.
369
+ if (fastStale()) return staleRetry;
370
+ channelClosedForRecreate = true;
371
+ runtimeQuarantinedForRecreate = true;
372
+ }
373
+ }
374
+ } else if (existingTask?.statusMirror === "running") {
133
375
  if (!existingTask.composeProject || !existingTask.worktreePath) {
134
376
  return {
135
377
  ok: false,
@@ -156,21 +398,72 @@ export const hostCommands: HostCommands = {
156
398
  { timeoutMs: 10_000 },
157
399
  );
158
400
  if (appRuntime.status !== 0) {
159
- return {
160
- ok: false,
161
- code: HostErrorCode.HostUnavailable,
162
- message:
163
- `could not verify runtime state for task ${input.task.id}: ` +
401
+ return dockerUnavailableResult(
402
+ `could not verify runtime state for task ${input.task.id}: ` +
164
403
  (appRuntime.stderr.trim().slice(0, 200) ||
165
404
  "docker did not answer"),
166
- retryable: true,
167
- };
405
+ );
168
406
  }
169
407
  if (
170
408
  appRuntime.stdout
171
409
  .split(/\r?\n/)
172
410
  .some((state) => state.trim() === "running")
173
411
  ) {
412
+ const appContainer = `${existingTask.composeProject}-app-1`;
413
+ const runtimeContract = await inspectTaskRuntimeContainer(
414
+ input.task.id,
415
+ existingTask.composeProject,
416
+ );
417
+ if (runtimeContract.kind === "unreachable") {
418
+ return dockerUnavailableResult(
419
+ `could not inspect the exact runtime mount for task ${input.task.id}`,
420
+ );
421
+ }
422
+ if (runtimeContract.kind === "failed") {
423
+ return {
424
+ ok: false,
425
+ code: HostErrorCode.HostUnavailable,
426
+ message:
427
+ `could not prove the exact runtime contract for task ${input.task.id}: ` +
428
+ runtimeContract.detail,
429
+ retryable: true,
430
+ };
431
+ }
432
+ const reusable =
433
+ runtimeContract.running &&
434
+ !runtimeContract.restarting &&
435
+ !runtimeContract.paused &&
436
+ runtimeContract.sharedMount === "read-only" &&
437
+ runtimeContract.taskEnvironment === "current";
438
+ if (!reusable) {
439
+ // Drain/tombstone before touching the exact container. task-up's
440
+ // pre-materialization cleanup then removes this stopped immutable
441
+ // generation, so Compose must create the current RO/private-prefix
442
+ // contract rather than `docker start`ing it again.
443
+ await orchestrator.closeChannel(input.task.id);
444
+ channelClosedForRecreate = true;
445
+ const quarantined = await quarantineTaskRuntimeContainer({
446
+ taskId: input.task.id,
447
+ composeProject: existingTask.composeProject,
448
+ expectedContainerId: runtimeContract.containerId,
449
+ });
450
+ if (quarantined === "unreachable") {
451
+ return dockerUnavailableResult(
452
+ `Docker became unavailable while quarantining task ${input.task.id}`,
453
+ );
454
+ }
455
+ if (quarantined === "failed") {
456
+ return {
457
+ ok: false,
458
+ code: HostErrorCode.HostUnavailable,
459
+ message:
460
+ `task ${input.task.id} has a non-current runtime contract and ` +
461
+ "could not be proven stopped; retry after Docker is healthy",
462
+ retryable: true,
463
+ };
464
+ }
465
+ runtimeQuarantinedForRecreate = true;
466
+ }
174
467
  // Docker being up is not enough to prove a lost-response retry is
175
468
  // healthy. The worktree's `.git` file contains an absolute pointer
176
469
  // into the task-private repository; after a host workspace-root
@@ -178,9 +471,8 @@ export const hostCommands: HostCommands = {
178
471
  // sees `fatal: not a git repository`. Probe from INSIDE the live
179
472
  // container, which is the filesystem view the agents actually use.
180
473
  // Scratchpads have no projects, so this remains an O(1) fast path.
181
- const appContainer = `${existingTask.composeProject}-app-1`;
182
474
  let projectsHealthy = true;
183
- for (const project of input.projects) {
475
+ for (const project of reusable ? input.projects : []) {
184
476
  const gitHealth = await dockerCli(
185
477
  [
186
478
  "exec",
@@ -243,21 +535,26 @@ export const hostCommands: HostCommands = {
243
535
  // broken. Preserve the running task and let the caller retry once
244
536
  // Docker can answer, just as the runtime-state probe above does.
245
537
  if (gitHealth.status === null) {
246
- return {
247
- ok: false,
248
- code: HostErrorCode.HostUnavailable,
249
- message:
250
- `could not verify Git state for task ${input.task.id} ` +
538
+ return dockerUnavailableResult(
539
+ `could not verify Git state for task ${input.task.id} ` +
251
540
  `project ${project.id}: ` +
252
541
  (gitHealth.stderr.trim().slice(0, 200) ||
253
542
  "docker did not answer"),
254
- retryable: true,
255
- };
543
+ );
256
544
  }
257
545
  if (
258
546
  gitHealth.status !== 0 ||
259
547
  gitHealth.stdout.trim() !== "true"
260
548
  ) {
549
+ if (gitHealth.status !== 0) {
550
+ const unavailable = await runtimeFailureIfDaemonUnavailable(
551
+ `could not verify Git state for task ${input.task.id} ` +
552
+ `project ${project.id}: ` +
553
+ (gitHealth.stderr.trim().slice(0, 200) ||
554
+ `git exited ${String(gitHealth.status)}`),
555
+ );
556
+ if (unavailable) return unavailable;
557
+ }
261
558
  projectsHealthy = false;
262
559
  console.warn(
263
560
  `[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
@@ -265,7 +562,7 @@ export const hostCommands: HostCommands = {
265
562
  break;
266
563
  }
267
564
  }
268
- if (projectsHealthy) {
565
+ if (reusable && projectsHealthy) {
269
566
  // waitForChannelClose crossed the teardown fence above. A completed
270
567
  // close-only teardown may have left this running task tombstoned,
271
568
  // so the idempotent-success transition must reopen delivery just
@@ -288,9 +585,51 @@ export const hostCommands: HostCommands = {
288
585
  // live container proved one of its project repositories is broken.
289
586
  // Drain and invalidate any stale in-memory channel before taskUp
290
587
  // recreates the same stable compose/container names and repairs Git.
291
- await orchestrator.closeChannel(input.task.id);
588
+ if (!channelClosedForRecreate) {
589
+ await orchestrator.closeChannel(input.task.id);
590
+ }
292
591
  }
592
+ const priorRuntimeState = existingTask
593
+ ? runtimeQuarantinedForRecreate
594
+ ? {
595
+ statusMirror: "stopped",
596
+ codeServerPort: null,
597
+ previewPorts: "[]",
598
+ lockedAt: null,
599
+ startedAt: existingTask.startedAt,
600
+ endedAt: existingTask.endedAt,
601
+ environmentProvider: existingTask.environmentProvider,
602
+ environmentLocator: existingTask.environmentLocator,
603
+ }
604
+ : {
605
+ statusMirror: existingTask.statusMirror,
606
+ lockedAt: existingTask.lockedAt,
607
+ startedAt: existingTask.startedAt,
608
+ endedAt: existingTask.endedAt,
609
+ environmentProvider: existingTask.environmentProvider,
610
+ environmentLocator: existingTask.environmentLocator,
611
+ }
612
+ : {
613
+ statusMirror: input.task.status,
614
+ lockedAt: null,
615
+ startedAt: null,
616
+ endedAt: null,
617
+ };
293
618
  recordTaskStarting(input.task.id);
619
+ // Cross-process managed maintenance and task admission use a two-sided
620
+ // handshake: update/uninstall writes its durable marker before checking
621
+ // this DB row; task-up writes `starting` before checking admission again.
622
+ // Whichever arrives second observes the first, so a runtime cannot be
623
+ // swapped or removed while an unrecorded task enters Docker side effects.
624
+ const admittedMaintenanceFailure = managedMaintenanceUnavailable();
625
+ if (admittedMaintenanceFailure) {
626
+ if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
627
+ else deleteHostTask(input.task.id);
628
+ if (!runtimeQuarantinedForRecreate) {
629
+ orchestrator.allowChannel(input.task.id);
630
+ }
631
+ return admittedMaintenanceFailure;
632
+ }
294
633
  recordTaskOwner(
295
634
  input.task.id,
296
635
  input.task.ownerUserId,
@@ -304,6 +643,7 @@ export const hostCommands: HostCommands = {
304
643
  storeTaskCliSecret(input.task.id, input.task.cliSecret);
305
644
  }
306
645
  recordHostEvent(input.task.id, "task.created");
646
+ let environmentLocator: TaskEnvironmentLocator | undefined;
307
647
  const result = await wrapAgent(ctx, "taskUp", async () => {
308
648
  // Self-heal the standard base image before task-up needs it. The
309
649
  // boot-time build is best-effort + one-shot: a host that started before
@@ -337,13 +677,21 @@ export const hostCommands: HostCommands = {
337
677
  );
338
678
  }
339
679
  try {
340
- return gitCredential
680
+ const provisioned = gitCredential
341
681
  ? await gitCredential.run(() =>
342
- agent.taskUp(input, {
343
- githubCredentialSocket: gitCredential.socketPath,
344
- }),
682
+ provisionTaskEnvironment(
683
+ input,
684
+ { githubCredentialSocket: gitCredential.socketPath },
685
+ async (locator) => {
686
+ recordTaskEnvironmentPrepared(input.task.id, locator);
687
+ },
688
+ ),
345
689
  )
346
- : await agent.taskUp(input);
690
+ : await provisionTaskEnvironment(input, {}, async (locator) => {
691
+ recordTaskEnvironmentPrepared(input.task.id, locator);
692
+ });
693
+ environmentLocator = provisioned.handle.descriptor.locator;
694
+ return provisioned.result;
347
695
  } finally {
348
696
  await gitCredential?.close().catch((error: unknown) => {
349
697
  // Cleanup must not replace the actual task-up result/error. The
@@ -357,7 +705,10 @@ export const hostCommands: HostCommands = {
357
705
  });
358
706
  if (result.ok) {
359
707
  orchestrator.allowChannel(input.task.id);
360
- recordTaskUpResult(input.task.id, result.value);
708
+ if (environmentLocator === undefined) {
709
+ throw new Error("task environment provision returned no locator");
710
+ }
711
+ recordTaskUpResult(input.task.id, result.value, environmentLocator);
361
712
  recordHostEvent(input.task.id, "task.started");
362
713
  // Degraded start (uai-init failed twice): the task is up, but say so
363
714
  // in the feed — a silent half-start reads as a broken product.
@@ -384,6 +735,16 @@ export const hostCommands: HostCommands = {
384
735
  }`,
385
736
  ),
386
737
  );
738
+ } else if (result.code === HostErrorCode.HostUnavailable) {
739
+ // A daemon outage is retryable infrastructure state, not proof that
740
+ // the task itself failed. Undo the optimistic local `starting` mirror
741
+ // and normally reopen delivery; a runtime already proven stopped for
742
+ // L5 recreation stays stopped/tombstoned for the cloud's retry.
743
+ if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
744
+ else deleteHostTask(input.task.id);
745
+ if (!runtimeQuarantinedForRecreate) {
746
+ orchestrator.allowChannel(input.task.id);
747
+ }
387
748
  } else {
388
749
  recordTaskError(input.task.id);
389
750
  }
@@ -393,11 +754,55 @@ export const hostCommands: HostCommands = {
393
754
 
394
755
  async taskDown(ctx, input) {
395
756
  logCommand(ctx, "taskDown", input.taskId);
757
+ const orphanGc = input.cleanup?.kind === "orphan-gc";
758
+ // A GC command is already authoritative evidence that this cloud task is
759
+ // gone. Enter the lifecycle slot and tombstone its channel even when the
760
+ // runtime is temporarily unavailable; the durable cloud intent will retry
761
+ // destruction later. Ordinary teardown retains its fast preflight.
762
+ if (!orphanGc) {
763
+ // Teardown-specific admission: `checking` with a pinned backend ADMITS
764
+ // teardown. Gating kills on `operational` created the 2026-08-16 ring —
765
+ // a task holding the shared volume blocked the very maintenance whose
766
+ // completion the kill was waiting on. Removing load is how a degraded
767
+ // host heals.
768
+ const runtimeFailure = teardownRuntimeUnavailable();
769
+ if (runtimeFailure) return runtimeFailure;
770
+ }
396
771
  const orchestrator = getOrchestrator();
397
772
  return orchestrator.runTaskLifecycle(input.taskId, async () => {
773
+ // Preserve ordinary teardown's runtime-race behavior: if the daemon was
774
+ // lost while waiting for the lifecycle slot, leave its channel open.
775
+ if (!orphanGc) {
776
+ const inSlotRuntimeFailure = teardownRuntimeUnavailable();
777
+ if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
778
+ }
779
+ let environment: TaskEnvironmentHandle<TaskDownResult> | null;
780
+ if (!orphanGc) {
781
+ try {
782
+ environment = await reconstructHostTaskEnvironment(input.taskId);
783
+ } catch (error) {
784
+ return failFromUnknown(error);
785
+ }
786
+ } else {
787
+ environment = null;
788
+ }
398
789
  // Tombstone + close before container destruction. A concurrent cloud
399
790
  // ensure cannot reopen the channel while teardown is awaiting Docker.
400
791
  await orchestrator.closeChannel(input.taskId);
792
+ if (orphanGc) {
793
+ const inSlotRuntimeFailure = runtimeUnavailable();
794
+ if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
795
+ try {
796
+ environment = await reconstructHostTaskEnvironment(input.taskId);
797
+ } catch (error) {
798
+ return failFromUnknown(error);
799
+ }
800
+ // Channel startup and runtime recovery launch SSH reconciliation in
801
+ // the background. Fence every future launch and join the exact older
802
+ // ensure before container destruction; otherwise a paused health probe
803
+ // could materialize the private key after localStatePruned was sent.
804
+ await retireTaskSshIdentityEnsures(input.taskId);
805
+ }
401
806
  // Ad-hoc preview proxies share this task's compose network and stable
402
807
  // names. Remove them inside the same lifecycle slot before compose down.
403
808
  try {
@@ -409,10 +814,34 @@ export const hostCommands: HostCommands = {
409
814
  }`,
410
815
  );
411
816
  }
412
- const result = await wrapAgent(ctx, "taskDown", () =>
413
- agent.taskDown(input),
414
- );
817
+ const result = await wrapAgent(ctx, "taskDown", async () => {
818
+ if (environment === null) return agent.taskDown(input);
819
+ return taskDownResultForInput(input, await environment.teardown());
820
+ });
415
821
  if (result.ok) {
822
+ if (orphanGc) {
823
+ try {
824
+ // task-down.sh is destructive but its exit alone is not the proof:
825
+ // independently re-query every exact Docker label and the one
826
+ // derived task directory before deleting the remaining authority.
827
+ await proveOrphanTaskResourcesAbsent(input.taskId);
828
+ clearTaskGatewayAclStrict(input.taskId);
829
+ removeTaskCliSecretStrict(input.taskId);
830
+ removeTaskIdentityStrict(input.taskId);
831
+ clearRefresh(input.taskId);
832
+ // Last, atomically. Do not record taskDown/task.ended first: both
833
+ // would recreate the very inventory evidence being pruned.
834
+ purgeHostTaskState(input.taskId);
835
+ return ok({ ...result.value, localStatePruned: true as const });
836
+ } catch (error) {
837
+ console.warn(
838
+ `[host-agent] task ${input.taskId}: orphan cleanup proof failed: ${
839
+ error instanceof Error ? error.message : String(error)
840
+ }`,
841
+ );
842
+ return failFromUnknown(error);
843
+ }
844
+ }
416
845
  // Revoke only after destruction succeeds. On failure the task is
417
846
  // reopened below and its on-disk MCP definitions must retain the same
418
847
  // gateway token; deleting it early would make those exact definitions
@@ -424,7 +853,7 @@ export const hostCommands: HostCommands = {
424
853
  } else {
425
854
  // Destruction failed; let a subsequent ensure restore the still-running
426
855
  // task instead of leaving it permanently tombstoned.
427
- orchestrator.allowChannel(input.taskId);
856
+ if (!orphanGc) orchestrator.allowChannel(input.taskId);
428
857
  }
429
858
  return result;
430
859
  });
@@ -432,9 +861,19 @@ export const hostCommands: HostCommands = {
432
861
 
433
862
  async taskStatus(ctx, taskId) {
434
863
  logCommand(ctx, "taskStatus", taskId);
435
- const result = await wrapAgent(ctx, "taskStatus", () =>
436
- agent.taskStatus(taskId),
437
- );
864
+ const runtimeFailure = runtimeUnavailable();
865
+ if (runtimeFailure) return runtimeFailure;
866
+ const result = await wrapAgent(ctx, "taskStatus", async () => {
867
+ const environment = await reconstructHostTaskEnvironment(taskId);
868
+ if (environment === null) {
869
+ return {
870
+ composeRunning: false,
871
+ containers: [],
872
+ worktreePresent: false,
873
+ };
874
+ }
875
+ return taskEnvironmentStatusResult(await environment.status());
876
+ });
438
877
  // ADR-051: attach the published preview ports from the host DB so the
439
878
  // cloud can backfill its mirror on reconcile (tasks launched before the
440
879
  // cloud started recording them at task-up).
@@ -451,6 +890,15 @@ export const hostCommands: HostCommands = {
451
890
 
452
891
  async channelEnsure(ctx, input) {
453
892
  logCommand(ctx, "channelEnsure", input.taskId);
893
+ const admissionFailure = managedMaintenanceUnavailable();
894
+ if (admissionFailure) return admissionFailure;
895
+ const runtimeFailure = runtimeUnavailable();
896
+ if (runtimeFailure) return runtimeFailure;
897
+ await agentClisReady;
898
+ const readyAdmissionFailure = managedMaintenanceUnavailable();
899
+ if (readyAdmissionFailure) return readyAdmissionFailure;
900
+ const readyRuntimeFailure = runtimeUnavailable();
901
+ if (readyRuntimeFailure) return readyRuntimeFailure;
454
902
  try {
455
903
  getOrchestrator().registerChannelSpec(normalizeChannelSpec(input));
456
904
  const ready = await getOrchestrator().ensureStarted(input.taskId);
@@ -491,6 +939,8 @@ export const hostCommands: HostCommands = {
491
939
 
492
940
  async channelDeliver(ctx, taskId, agentId, text) {
493
941
  logCommand(ctx, "channelDeliver", taskId, agentId);
942
+ const runtimeFailure = runtimeUnavailable();
943
+ if (runtimeFailure) return runtimeFailure;
494
944
  try {
495
945
  const result = await getOrchestrator().deliver(taskId, agentId, text);
496
946
  if (!result.ok) {
@@ -508,9 +958,13 @@ export const hostCommands: HostCommands = {
508
958
 
509
959
  async previewEnsure(ctx, taskId, name, containerPort) {
510
960
  logCommand(ctx, "previewEnsure", taskId, name);
961
+ const runtimeFailure = runtimeUnavailable();
962
+ if (runtimeFailure) return runtimeFailure;
511
963
  const orchestrator = getOrchestrator();
512
964
  return orchestrator.runTaskLifecycle(taskId, async () => {
513
965
  try {
966
+ const inSlotRuntimeFailure = runtimeUnavailable();
967
+ if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
514
968
  const task = getHostTask(taskId);
515
969
  if (!task) {
516
970
  return {
@@ -522,20 +976,49 @@ export const hostCommands: HostCommands = {
522
976
  if (task.statusMirror !== "running") {
523
977
  return ok({ hostPort: null });
524
978
  }
525
- // Task-up published port (preview enabled at launch) wins.
979
+ if (!task.composeProject) return ok({ hostPort: null });
526
980
  const declared = parsePreviewPortRuntimes(task.previewPorts).find(
527
981
  (port) => port.name === name,
528
982
  );
529
- if (declared) return ok({ hostPort: declared.hostPort });
983
+ // A stored host port is never routing truth after an unnoticed daemon
984
+ // restart. Prove the exact app container and current mapping before
985
+ // either returning a declared port or creating an ad-hoc sidecar.
986
+ const appProof = await inspectTunnelContainer(
987
+ `${task.composeProject}-app-1`,
988
+ containerPort,
989
+ );
990
+ if (appProof.kind === "daemon-unavailable") {
991
+ return dockerUnavailableResult(
992
+ `could not verify preview runtime for task ${taskId}: ${appProof.detail}`,
993
+ );
994
+ }
995
+ // The proof awaited Docker. A concurrent runtime recheck can invalidate
996
+ // its generation before this continuation resumes, so fence the result
997
+ // again before returning any route to the cloud.
998
+ const postInspectRuntimeFailure = runtimeUnavailable();
999
+ if (postInspectRuntimeFailure) return postInspectRuntimeFailure;
1000
+ if (appProof.kind !== "running") return ok({ hostPort: null });
1001
+ // Task-up published port (preview enabled at launch) wins, but use the
1002
+ // mapping Docker just proved rather than the persisted hostPort.
1003
+ if (declared) {
1004
+ return ok({ hostPort: appProof.published?.port ?? null });
1005
+ }
530
1006
  // Else start (or find) the node-proxy sidecar NOW — the same one the
531
1007
  // tunnel lazy-starts on first access — and hand back its host port.
532
- if (!task.composeProject) return ok({ hostPort: null });
533
1008
  const hostPort = await ensurePreviewSidecar({
534
1009
  taskId,
535
1010
  composeProject: task.composeProject,
536
1011
  name,
537
1012
  containerPort,
538
1013
  });
1014
+ const postSidecarRuntimeFailure = runtimeUnavailable();
1015
+ if (postSidecarRuntimeFailure) return postSidecarRuntimeFailure;
1016
+ if (hostPort === null) {
1017
+ const unavailable = await runtimeFailureIfDaemonUnavailable(
1018
+ `could not prepare preview runtime for task ${taskId}`,
1019
+ );
1020
+ if (unavailable) return unavailable;
1021
+ }
539
1022
  return ok({ hostPort });
540
1023
  } catch (err) {
541
1024
  return failFromUnknown(err);
@@ -576,8 +1059,24 @@ export const hostCommands: HostCommands = {
576
1059
 
577
1060
  async taskDiff(ctx, input) {
578
1061
  logCommand(ctx, "taskDiff", input.taskId);
1062
+ const runtimeFailure = runtimeUnavailable();
1063
+ if (runtimeFailure) return runtimeFailure;
579
1064
  try {
580
- return ok(await buildTaskDiff(input));
1065
+ const task = getHostTask(input.taskId);
1066
+ // A stopped/terminal task retains its host worktree, so preserve the
1067
+ // historical host-side fallback there. A running task must reconstruct
1068
+ // its exact persisted provider identity: malformed/foreign metadata is
1069
+ // not permission to run Git against a different filesystem view.
1070
+ const environment =
1071
+ task?.statusMirror === "running"
1072
+ ? await reconstructPersistedTaskEnvironment(task)
1073
+ : null;
1074
+ if (task?.statusMirror === "running" && environment === null) {
1075
+ throw new Error(
1076
+ `running task ${input.taskId} has no persisted task environment`,
1077
+ );
1078
+ }
1079
+ return ok(await buildTaskDiff(input, environment));
581
1080
  } catch (err) {
582
1081
  return failFromUnknown(err);
583
1082
  }
@@ -655,6 +1154,35 @@ function normalizeChannelSpec(input: ChannelEnsureInput): ChannelEnsureInput {
655
1154
  return { ...input, workspacePath: "/workspace" };
656
1155
  }
657
1156
 
1157
+ async function reconstructHostTaskEnvironment(
1158
+ taskId: string,
1159
+ ): Promise<TaskEnvironmentHandle<TaskDownResult> | null> {
1160
+ const task = getHostTask(taskId);
1161
+ return task === null ? null : reconstructPersistedTaskEnvironment(task);
1162
+ }
1163
+
1164
+ function taskEnvironmentStatusResult(status: TaskEnvironmentStatus) {
1165
+ if (status.state === "unknown") {
1166
+ throw new Error(`task environment status is unknown: ${status.detail}`);
1167
+ }
1168
+ return {
1169
+ composeRunning: status.state === "running",
1170
+ containers: [...status.instances],
1171
+ worktreePresent: status.workspacePresent,
1172
+ };
1173
+ }
1174
+
1175
+ function taskDownResultForInput(
1176
+ input: TaskDownInput,
1177
+ providerResult: TaskDownResult,
1178
+ ): TaskDownResult {
1179
+ const status = input.task.status;
1180
+ return {
1181
+ ...providerResult,
1182
+ status: status === "shipped" || status === "error" ? status : "killed",
1183
+ };
1184
+ }
1185
+
658
1186
  function logCommand(
659
1187
  ctx: CommandContext,
660
1188
  command: keyof HostCommands,
@@ -671,6 +1199,13 @@ async function wrapAgent<T>(
671
1199
  try {
672
1200
  return ok(await fn());
673
1201
  } catch (err) {
1202
+ if (AgentError.is(err) && isRuntimeSensitiveAgentError(err.code)) {
1203
+ if (err.code === "DOCKER_PS_FAILED") {
1204
+ return dockerUnavailableResult(err.message);
1205
+ }
1206
+ const unavailable = await runtimeFailureIfDaemonUnavailable(err.message);
1207
+ if (unavailable) return unavailable;
1208
+ }
674
1209
  return failFromUnknown(err);
675
1210
  }
676
1211
  }
@@ -679,7 +1214,193 @@ function ok<T>(value: T): HostCommandResult<T> {
679
1214
  return { ok: true, value };
680
1215
  }
681
1216
 
1217
+ /** Docker-dependent commands must not reinterpret an unavailable or
1218
+ * half-recovered backend as an absent task/container. */
1219
+ function runtimeUnavailable(): HostCommandResult<never> | null {
1220
+ const message = containerRuntimeProblem();
1221
+ if (!message) return null;
1222
+ return {
1223
+ ok: false,
1224
+ code: HostErrorCode.HostUnavailable,
1225
+ message,
1226
+ retryable: true,
1227
+ };
1228
+ }
1229
+
1230
+ function teardownRuntimeUnavailable(): HostCommandResult<never> | null {
1231
+ const message = containerRuntimeTeardownProblem();
1232
+ if (!message) return null;
1233
+ return {
1234
+ ok: false,
1235
+ code: HostErrorCode.HostUnavailable,
1236
+ message,
1237
+ retryable: true,
1238
+ };
1239
+ }
1240
+
1241
+ const RUNTIME_TASK_ADMISSION_WAIT_MS = 3_000;
1242
+
1243
+ /**
1244
+ * Runtime activation is intentionally detached from one cloud command's
1245
+ * 15-minute deadline. A first attempt joins for a few seconds, then returns a
1246
+ * narrowly-tagged retry while the single detection/activation promise keeps
1247
+ * running. Every retry rechecks the exact generation and durable provider
1248
+ * claim before task-owned side effects begin.
1249
+ */
1250
+ async function prepareContainerRuntimeForTask(): Promise<
1251
+ HostCommandResult<never> | null
1252
+ > {
1253
+ let prepared:
1254
+ | Awaited<ReturnType<typeof ensureContainerRuntimeForTask>>
1255
+ | null;
1256
+ try {
1257
+ prepared = await settleWithin(
1258
+ ensureContainerRuntimeForTask(),
1259
+ RUNTIME_TASK_ADMISSION_WAIT_MS,
1260
+ );
1261
+ } catch (error) {
1262
+ return runtimeSetupRetry(
1263
+ `container runtime setup failed: ${boundedFailureMessage(error)}`,
1264
+ );
1265
+ }
1266
+ if (prepared === null) {
1267
+ return runtimeSetupRetry(
1268
+ "The bundled container runtime is still being provisioned on this host.",
1269
+ );
1270
+ }
1271
+ if (prepared.status !== "ready") {
1272
+ return {
1273
+ ok: false,
1274
+ code: HostErrorCode.HostUnavailable,
1275
+ message:
1276
+ prepared.status === "no-runtime"
1277
+ ? prepared.message
1278
+ : "The container runtime is still being detected on this host.",
1279
+ retryable: true,
1280
+ };
1281
+ }
1282
+ const epoch = containerRuntimeReadinessEpoch();
1283
+ const operational = await waitForContainerRuntimeOperational(
1284
+ epoch,
1285
+ RUNTIME_TASK_ADMISSION_WAIT_MS,
1286
+ );
1287
+ if (!operational) {
1288
+ const failure = runtimeUnavailable();
1289
+ if (failure && !failure.ok && !failure.message.includes("still preparing")) {
1290
+ return failure;
1291
+ }
1292
+ return runtimeSetupRetry(
1293
+ "The container runtime is completing image maintenance and task recovery.",
1294
+ );
1295
+ }
1296
+ try {
1297
+ if (await claimContainerRuntimeForTask(epoch)) return null;
1298
+ } catch (error) {
1299
+ return runtimeSetupRetry(
1300
+ `The container runtime could not persist its task-state claim: ${boundedFailureMessage(error)}`,
1301
+ );
1302
+ }
1303
+ return runtimeSetupRetry(
1304
+ "The container runtime changed while task admission was being prepared.",
1305
+ );
1306
+ }
1307
+
1308
+ function runtimeSetupRetry(message: string): HostCommandResult<never> {
1309
+ return {
1310
+ ok: false,
1311
+ code: HostErrorCode.HostUnavailable,
1312
+ message,
1313
+ retryable: true,
1314
+ retryReason: "runtime_setup",
1315
+ };
1316
+ }
1317
+
1318
+ function settleWithin<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
1319
+ return new Promise<T | null>((resolve, reject) => {
1320
+ let settled = false;
1321
+ const timer = setTimeout(() => {
1322
+ if (settled) return;
1323
+ settled = true;
1324
+ resolve(null);
1325
+ }, timeoutMs);
1326
+ timer.unref?.();
1327
+ void promise.then(
1328
+ (value) => {
1329
+ if (settled) return;
1330
+ settled = true;
1331
+ clearTimeout(timer);
1332
+ resolve(value);
1333
+ },
1334
+ (error: unknown) => {
1335
+ if (settled) return;
1336
+ settled = true;
1337
+ clearTimeout(timer);
1338
+ reject(error);
1339
+ },
1340
+ );
1341
+ });
1342
+ }
1343
+
1344
+ function boundedFailureMessage(error: unknown): string {
1345
+ return (error instanceof Error ? error.message : String(error)).slice(0, 512);
1346
+ }
1347
+
1348
+ /** Managed installation, activation, and uninstall drain instead of admitting work. */
1349
+ function managedMaintenanceUnavailable(): HostCommandResult<never> | null {
1350
+ if (!managedRuntimeAdmissionPending()) return null;
1351
+ return {
1352
+ ok: false,
1353
+ code: HostErrorCode.HostUnavailable,
1354
+ message: "managed host maintenance is waiting for this runtime to restart",
1355
+ retryable: true,
1356
+ retryReason: "managed_restart",
1357
+ };
1358
+ }
1359
+
1360
+ function dockerUnavailableResult<T = never>(
1361
+ message: string,
1362
+ ): HostCommandResult<T> {
1363
+ // Invalidate synchronously so a second command in this event-loop turn
1364
+ // cannot touch the daemon under a stale ready verdict. Detection/recovery is
1365
+ // asynchronous and deliberately does not delay the retryable response.
1366
+ suspendContainerRuntime(true);
1367
+ void initializeContainerRuntime();
1368
+ return {
1369
+ ok: false,
1370
+ code: HostErrorCode.HostUnavailable,
1371
+ message,
1372
+ retryable: true,
1373
+ };
1374
+ }
1375
+
1376
+ async function runtimeFailureIfDaemonUnavailable<T = never>(
1377
+ message: string,
1378
+ ): Promise<HostCommandResult<T> | null> {
1379
+ return (await reprobeContainerRuntimeMachineIdentity()) !== null
1380
+ ? null
1381
+ : dockerUnavailableResult(message);
1382
+ }
1383
+
1384
+ function isRuntimeSensitiveAgentError(code: string): boolean {
1385
+ return (
1386
+ code === "STANDARD_IMAGE_UNAVAILABLE" ||
1387
+ code === "STANDARD_IMAGE_MISSING" ||
1388
+ code === "COMPOSE_UP_FAILED" ||
1389
+ code === "COMPOSE_DOWN_FAILED" ||
1390
+ code === "CONTAINER_INIT_FAILED" ||
1391
+ code === "DOCKER_PS_FAILED"
1392
+ );
1393
+ }
1394
+
682
1395
  function failFromUnknown<T = never>(err: unknown): HostCommandResult<T> {
1396
+ if (err instanceof ContainerRuntimeUnavailableError) {
1397
+ return {
1398
+ ok: false,
1399
+ code: HostErrorCode.HostUnavailable,
1400
+ message: err.message,
1401
+ retryable: true,
1402
+ };
1403
+ }
683
1404
  if (AgentError.is(err)) {
684
1405
  return {
685
1406
  ok: false,
@@ -727,6 +1448,8 @@ function mapAgentError(code: string): HostErrorCode {
727
1448
  return HostErrorCode.ContainerInitFailed;
728
1449
  case "DB_FAILED":
729
1450
  return HostErrorCode.DbFailed;
1451
+ case "DOCKER_PS_FAILED":
1452
+ return HostErrorCode.HostUnavailable;
730
1453
  default:
731
1454
  return HostErrorCode.Internal;
732
1455
  }
@@ -758,7 +1481,8 @@ function isRetryableAgentError(code: string): boolean {
758
1481
  return (
759
1482
  code === "FETCH_FAILED" ||
760
1483
  code === "CONTAINER_INIT_FAILED" ||
761
- code === "GITHUB_AUTH_UNAVAILABLE"
1484
+ code === "GITHUB_AUTH_UNAVAILABLE" ||
1485
+ code === "DOCKER_PS_FAILED"
762
1486
  );
763
1487
  }
764
1488