@runuai/host 0.9.149 → 0.9.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
  2. package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
  3. package/db/migrations/meta/_journal.json +14 -0
  4. package/db/schema.ts +188 -0
  5. package/lib/agent-cli.ts +11 -13
  6. package/lib/agent.ts +177 -7
  7. package/lib/agents/durable-proc.ts +56 -44
  8. package/lib/agents/registry.ts +19 -3
  9. package/lib/agents/session-state.ts +23 -10
  10. package/lib/agents/transport.ts +101 -69
  11. package/lib/apple-one-off.ts +277 -0
  12. package/lib/apple-runtime-recycle.ts +4 -3
  13. package/lib/attachments.ts +19 -17
  14. package/lib/capture-flow.ts +319 -0
  15. package/lib/cloud-credential-cleanup-adapter.ts +123 -0
  16. package/lib/cloud-credential-host.ts +512 -0
  17. package/lib/cloud-credential-leases.ts +2544 -0
  18. package/lib/cloud-credential-runtime.ts +1035 -0
  19. package/lib/codex-auth.ts +21 -1
  20. package/lib/container-runtime.ts +76 -0
  21. package/lib/credential-cleanup-inventory.ts +405 -0
  22. package/lib/credential-consumers/engine-anchor.ts +1876 -0
  23. package/lib/credential-consumers/engines.ts +773 -0
  24. package/lib/credential-consumers/github.ts +594 -0
  25. package/lib/credential-consumers/mcp.ts +384 -0
  26. package/lib/credential-consumers/task-files.ts +127 -0
  27. package/lib/credential-store-pickup.ts +15 -2
  28. package/lib/credential-store.ts +361 -33
  29. package/lib/credentials-activation-authority.ts +546 -0
  30. package/lib/credentials-purge.ts +1006 -0
  31. package/lib/db.ts +11 -0
  32. package/lib/docker-exec.ts +3 -1
  33. package/lib/engine-accounts.ts +42 -32
  34. package/lib/engine-connect.ts +47 -2
  35. package/lib/engine-credential-env.ts +50 -0
  36. package/lib/engine-login.ts +83 -1
  37. package/lib/env-file.ts +62 -8
  38. package/lib/env.ts +5 -0
  39. package/lib/fs-aliases.ts +49 -0
  40. package/lib/git-clone-credentials.ts +334 -0
  41. package/lib/git-diff.ts +219 -177
  42. package/lib/git-identity.ts +59 -14
  43. package/lib/github-origin.ts +28 -0
  44. package/lib/github-tokens.ts +56 -21
  45. package/lib/host-env.ts +17 -4
  46. package/lib/launch-mode.ts +25 -0
  47. package/lib/legacy-runtime-retirement.ts +388 -0
  48. package/lib/machine-exec.ts +14 -3
  49. package/lib/machine-provider-aws.ts +50 -8
  50. package/lib/machine-provider-local.ts +4 -0
  51. package/lib/machine-provider.ts +4 -0
  52. package/lib/managed-operation-drain.ts +8 -0
  53. package/lib/managed-runtime.ts +371 -7
  54. package/lib/mcp-capture.ts +860 -0
  55. package/lib/mcp-connections.ts +8 -1
  56. package/lib/mcp-gateway.ts +168 -10
  57. package/lib/obs.ts +2 -6
  58. package/lib/orchestrator.ts +1847 -236
  59. package/lib/person-capture.ts +327 -0
  60. package/lib/project-env-legacy-tags.ts +130 -0
  61. package/lib/project-env-task-replica.ts +1752 -0
  62. package/lib/runner-boot.ts +10 -0
  63. package/lib/runtime-activation-tail.ts +50 -3
  64. package/lib/runtime-archive.ts +68 -1
  65. package/lib/runtime-reconnect.ts +86 -0
  66. package/lib/runtime-state.ts +429 -13
  67. package/lib/safe-http.ts +364 -0
  68. package/lib/secrets-proxy/task-trust.ts +34 -3
  69. package/lib/sidecar-resume.ts +18 -5
  70. package/lib/skills.ts +47 -8
  71. package/lib/standard-image.ts +157 -103
  72. package/lib/stopped-task-status-queue.ts +68 -10
  73. package/lib/task-credential-cleanup.ts +525 -0
  74. package/lib/task-diff-sandbox.ts +464 -0
  75. package/lib/task-diff.ts +66 -163
  76. package/lib/task-env-source.ts +68 -14
  77. package/lib/task-environment/anchor.ts +31 -0
  78. package/lib/task-environment/apple-container.ts +69 -28
  79. package/lib/task-environment/docker.ts +101 -20
  80. package/lib/task-environment/machine-environment.ts +61 -0
  81. package/lib/task-environment/machine-task-up.ts +459 -96
  82. package/lib/task-environment/machine.ts +101 -11
  83. package/lib/task-environment/self.ts +108 -50
  84. package/lib/task-environment/types.ts +89 -5
  85. package/lib/task-environment/workspace-files.ts +623 -25
  86. package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
  87. package/lib/task-gateway-recovery.ts +4 -1
  88. package/lib/task-gateway-state.ts +15 -0
  89. package/lib/task-machine-swap-safety.ts +729 -0
  90. package/lib/token-broker-adopt.ts +344 -0
  91. package/lib/token-broker-lambda.ts +56 -2
  92. package/lib/token-broker-local.ts +28 -4
  93. package/lib/token-broker-revision.ts +310 -0
  94. package/lib/token-broker-ssm.ts +368 -28
  95. package/lib/token-lanes/openai-oauth.ts +7 -3
  96. package/lib/token-lanes/xai-oauth.ts +7 -3
  97. package/lib/transcript.ts +9 -5
  98. package/package.json +6 -2
  99. package/scripts/agent/task-up.sh +242 -100
  100. package/src/cloud-credentials.ts +2399 -0
  101. package/src/event-outbox.ts +8 -6
  102. package/src/index.ts +1386 -875
  103. package/src/main.ts +1062 -140
  104. package/src/outbound-frame.ts +31 -0
  105. package/src/project-env-policy.ts +125 -10
  106. package/src/protocol.ts +699 -23
  107. package/src/ui/server.ts +48 -10
package/src/index.ts CHANGED
@@ -1,9 +1,28 @@
1
1
  import {
2
2
  agent,
3
3
  AgentError,
4
+ CREDENTIALS_WITHDRAWN_MESSAGE,
4
5
  toolStderrDetail,
6
+ type LeasedTaskUp,
5
7
  type TaskDownResult,
6
8
  } from "../lib/agent";
9
+ import {
10
+ activeCloudCredentials,
11
+ admitLegacyStart,
12
+ cloudCredentialRuntime,
13
+ credentialsMode,
14
+ onOwnCredentials,
15
+ whileOnOwnCredentials,
16
+ type LegacyStartPermit,
17
+ } from "../lib/cloud-credential-host";
18
+ import { CredentialLeaseUnavailableError, type TaskAssignment } from "../lib/cloud-credential-leases";
19
+ import {
20
+ StartInterruptedError,
21
+ type CloudCredentialRuntime,
22
+ type TaskCredentialStart,
23
+ type TaskUpHold,
24
+ } from "../lib/cloud-credential-runtime";
25
+ import { writeTaskGithub, writeTaskSshKey } from "../lib/credential-consumers/github";
7
26
  import { agentClisReady, ensureStandardImage } from "../lib/standard-image";
8
27
  import { cloneRepo } from "../lib/repo-clone";
9
28
  import { handleFilesOp } from "../lib/shared-files";
@@ -43,16 +62,23 @@ import {
43
62
  appendTranscriptViaEnvironment,
44
63
  } from "../lib/transcript";
45
64
  import { buildTaskDiff } from "../lib/task-diff";
65
+ import { stoppedTaskDiff } from "../lib/task-diff-sandbox";
46
66
  import {
67
+ admitTaskRunEpoch,
68
+ canReuseTaskRuntimeForRun,
69
+ closeTaskRunAssignment,
70
+ closeTaskRunAssignmentForStop,
47
71
  deleteHostTask,
48
72
  getHostTask,
49
73
  hostTaskEnvMode,
74
+ readTaskRunEpochState,
50
75
  recordHostEvent,
51
76
  recordTaskDown,
52
77
  recordTaskEnvironmentPrepared,
53
78
  recordTaskError,
54
79
  recordTaskOwner,
55
80
  recordTaskProjectEnvSpec,
81
+ recordTaskRuntimeEstablished,
56
82
  recordTaskStarting,
57
83
  recordTaskUpResult,
58
84
  taskProjectEnvSpec,
@@ -78,7 +104,10 @@ import { activeSecretsProxy } from "../lib/secrets-proxy/host";
78
104
  import { secretsProxyEnabled } from "../lib/secrets-proxy/runtime";
79
105
  import { forgetTaskProxyReadiness } from "../lib/secrets-proxy/readiness";
80
106
  import { admitFreshTaskView } from "../lib/secrets-proxy/sidecar-view";
81
- import { invalidateTaskWrites, reconcileTaskProxyState } from "../lib/secrets-proxy/task-trust";
107
+ import {
108
+ invalidateTaskWrites,
109
+ reconcileTaskProxyState,
110
+ } from "../lib/secrets-proxy/task-trust";
82
111
  import { parsePreviewPortRuntimes } from "../lib/preview-ports";
83
112
  import { dockerCli } from "../lib/docker-exec";
84
113
  import {
@@ -187,7 +216,7 @@ export {
187
216
  export { cloneRepo } from "../lib/repo-clone";
188
217
  export type { CloneRepoInput, CloneRepoResult } from "../lib/repo-clone";
189
218
 
190
- export { parseGitDiff, runGitDiff } from "../lib/git-diff";
219
+ export { parseGitDiff } from "../lib/git-diff";
191
220
  export type { DiffFile, DiffHunk, DiffLine, DiffStatus } from "../lib/git-diff";
192
221
 
193
222
  export {
@@ -242,88 +271,292 @@ function sidecarRunnerIsPinned(): HostCommandResult<never> | null {
242
271
  };
243
272
  }
244
273
 
245
- export const hostCommands: HostCommands = {
246
- // The hostId argument is cloud-side routing (host-scoped commands carry
247
- // their target in args, like filesOp); this process IS the target.
248
- async hostUpdate(ctx, _hostId) {
249
- logCommand(ctx, "hostUpdate");
250
- const pinned = sidecarRunnerIsPinned();
251
- if (pinned) return pinned;
252
- try {
253
- const { updateManagedRuntime } = await import("../lib/managed-runtime");
254
- const outcome = await updateManagedRuntime();
255
- if (outcome.status === "not-managed") {
256
- return {
257
- ok: false,
258
- code: HostErrorCode.Internal,
259
- message:
260
- "this host is not a managed install — update it where it runs",
261
- retryable: false,
262
- };
263
- }
264
- if (outcome.status === "deferred") {
265
- // A rollback hold is an operator verdict that the newer release is
266
- // unsafe on this host; a remote click must not override it.
267
- return {
268
- ok: true,
269
- value: { version: outcome.version, staged: false, deferred: true },
270
- };
271
- }
272
- if (outcome.status === "current") {
273
- return { ok: true, value: { version: outcome.version, staged: false } };
274
+ type TaskUpCommand = HostCommands["taskUp"];
275
+ type TaskUpOutcome = Awaited<ReturnType<TaskUpCommand>>;
276
+
277
+ /**
278
+ * ADR-154: a start on cloud credentials, as task-up carries it. It begins in
279
+ * the short admission slot (the run's startup; every older start of the task
280
+ * interrupted), is awaited outside every slot, holds its installed grant
281
+ * through the long slot (the clone, provisioning and the runtime-ready
282
+ * writes), publishes its success only while it is still the task's current
283
+ * start with its credentials open, and is settled after: ended, or abandoned
284
+ * with its hold let go first.
285
+ */
286
+ interface CloudCredentialTaskStart {
287
+ readonly runtime: CloudCredentialRuntime;
288
+ readonly assignment: TaskAssignment;
289
+ start: TaskCredentialStart | null;
290
+ hold: TaskUpHold | null;
291
+ }
292
+
293
+ /** The start's lease for task-up: its whole grant's hold, and what the
294
+ * runner writes into the runtime once it exists (the owner's GitHub sign-in
295
+ * and SSH key, recorded for cleanup) before anything in it runs. */
296
+ function leasedTaskUp(start: TaskCredentialStart, hold: TaskUpHold): LeasedTaskUp {
297
+ return {
298
+ // The whole grant's signal: its gate closing, a cleanup reaching it or
299
+ // the start being interrupted all end task-up.
300
+ clone: { ...hold.git, signal: hold.signal },
301
+ runtimeReady: async (environment, signal) => {
302
+ for (const write of [writeTaskGithub, writeTaskSshKey]) {
303
+ if (signal.aborted) throw new CredentialLeaseUnavailableError("the task's credentials were withdrawn");
304
+ await write(environment, start.assignment);
274
305
  }
275
- managedHostRestartHook?.();
276
- return { ok: true, value: { version: outcome.version, staged: true } };
277
- } catch (error) {
278
- return {
279
- ok: false,
280
- code: HostErrorCode.Internal,
281
- message: `update failed: ${
282
- error instanceof Error ? error.message : "unknown error"
283
- }`,
284
- retryable: true,
285
- };
306
+ },
307
+ };
308
+ }
309
+
310
+ /** `wait`, given up (rejecting) as soon as the runner begins leaving its own
311
+ * credentials, when what it waits for may never come. */
312
+ function untilLeavingLegacy<T>(legacy: LegacyStartPermit | null, wait: Promise<T>): Promise<T> {
313
+ if (legacy === null) return wait;
314
+ const signal = legacy.signal;
315
+ return new Promise<T>((resolve, reject) => {
316
+ const onAbort = (): void => reject(signal.reason);
317
+ if (signal.aborted) onAbort();
318
+ else signal.addEventListener("abort", onAbort, { once: true });
319
+ wait.then(
320
+ (value) => {
321
+ signal.removeEventListener("abort", onAbort);
322
+ resolve(value);
323
+ },
324
+ (error: unknown) => {
325
+ signal.removeEventListener("abort", onAbort);
326
+ reject(error);
327
+ },
328
+ );
329
+ });
330
+ }
331
+
332
+ /** A start the runner does not take while it switches its tasks to cloud
333
+ * credentials: the cloud starts it again shortly. */
334
+ function switchingToCloudCredentials(): TaskUpOutcome {
335
+ return {
336
+ ok: false,
337
+ code: HostErrorCode.HostUnavailable,
338
+ message: "This host is switching its tasks to cloud credentials; the task starts once it is done.",
339
+ retryable: true,
340
+ retryReason: "managed_restart",
341
+ };
342
+ }
343
+
344
+ /** What a start reports when Stop, teardown, a newer run or shutdown
345
+ * interrupted it, in place of its own outcome; null when none did. The
346
+ * interruption that won decides: shutdown asks the cloud to start the
347
+ * task again once the runner is back, anything else leaves a run that is
348
+ * no longer current. */
349
+ function interruptedStartResult(cloud: CloudCredentialTaskStart | null): TaskUpOutcome | null {
350
+ if (!cloud?.start?.signal.aborted) return null;
351
+ const reason: unknown = cloud.start.signal.reason;
352
+ if (reason instanceof StartInterruptedError && reason.by === "shutdown") {
353
+ return {
354
+ ok: false,
355
+ code: HostErrorCode.HostUnavailable,
356
+ message: "This host is restarting; the task starts once it is back.",
357
+ retryable: true,
358
+ retryReason: "managed_restart",
359
+ };
360
+ }
361
+ return {
362
+ ok: false,
363
+ code: HostErrorCode.TaskAlreadyRunning,
364
+ message: `task ${cloud.assignment.taskId} run ${String(cloud.assignment.runEpoch)} is no longer current: Stop, teardown or a newer run took over its start`,
365
+ };
366
+ }
367
+
368
+ /** The start's credentials closed for its run without an interruption:
369
+ * removed, refused or replaced in the cloud. */
370
+ function withdrawnStart(): TaskUpOutcome {
371
+ return { ok: false, code: HostErrorCode.Internal, message: CREDENTIALS_WITHDRAWN_MESSAGE };
372
+ }
373
+
374
+ /** The last word before a start publishes its success: null while it may.
375
+ * Otherwise what it reports instead (`interrupted` when Stop, teardown, a
376
+ * newer run or shutdown took over): its interruption's result, or a
377
+ * withdrawn start's failure once its credentials are no longer open for
378
+ * its run. */
379
+ function cloudStartCannotPublish(
380
+ cloud: CloudCredentialTaskStart | null,
381
+ ): { result: TaskUpOutcome; interrupted: boolean } | null {
382
+ if (!cloud?.start) return null;
383
+ const interrupted = interruptedStartResult(cloud);
384
+ if (interrupted) return { result: interrupted, interrupted: true };
385
+ try {
386
+ cloud.start.assertCurrent();
387
+ return null;
388
+ } catch {
389
+ return { result: withdrawnStart(), interrupted: false };
390
+ }
391
+ }
392
+
393
+ /** The long slot's operation, the start's hold let go as it ends: a hold
394
+ * never outlives the slot it was taken in, so a Stop or teardown queued
395
+ * behind it never waits on it. */
396
+ function releasingHold(
397
+ cloud: CloudCredentialTaskStart | null,
398
+ operation: () => Promise<TaskUpOutcome>,
399
+ ): () => Promise<TaskUpOutcome> {
400
+ return async () => {
401
+ try {
402
+ return await operation();
403
+ } finally {
404
+ cloud?.hold?.release();
286
405
  }
287
- },
288
- async hostRestart(ctx, _hostId) {
289
- logCommand(ctx, "hostRestart");
290
- const pinned = sidecarRunnerIsPinned();
291
- if (pinned) return pinned;
292
- if (!managedHostRestartHook) {
406
+ };
407
+ }
408
+
409
+ async function taskUpOn(
410
+ ctx: Parameters<TaskUpCommand>[0],
411
+ input: Parameters<TaskUpCommand>[1],
412
+ cloud: CloudCredentialTaskStart | null,
413
+ legacy: LegacyStartPermit | null,
414
+ ): ReturnType<TaskUpCommand> {
415
+ const orchestrator = getOrchestrator();
416
+ let runEpochAdmission: ReturnType<typeof admitTaskRunEpoch>;
417
+ try {
418
+ if (input.task.runEpoch === undefined) {
419
+ // ADR-154: a start on the runner's own credentials only while the
420
+ // runner is still on them.
421
+ if (legacy?.valid() === false) return switchingToCloudCredentials();
422
+ // Preserve the v0 path exactly during the rolling window. It has no
423
+ // assignment floor to serialize and must keep failing cheap host
424
+ // preflights before entering the runtime lifecycle queue.
425
+ runEpochAdmission = admitTaskRunEpoch(input.task.id, undefined);
426
+ } else {
427
+ const admitted = await orchestrator.runTaskLifecycle(
428
+ input.task.id,
429
+ async () => {
430
+ // ADR-154: a start on the runner's own credentials is admitted
431
+ // only while the runner is still on them.
432
+ if (legacy?.valid() === false) {
433
+ return { ok: false as const, result: switchingToCloudCredentials() };
434
+ }
435
+ const admission = admitTaskRunEpoch(
436
+ input.task.id,
437
+ input.task.runEpoch,
438
+ );
439
+ if (admission.kind === "stale") {
440
+ return {
441
+ ok: false as const,
442
+ result: {
443
+ ok: false as const,
444
+ code: HostErrorCode.TaskAlreadyRunning,
445
+ message:
446
+ `task ${input.task.id} run ${String(input.task.runEpoch)} is stale; ` +
447
+ `this runner has already admitted run ${String(admission.currentRunEpoch)}`,
448
+ },
449
+ };
450
+ }
451
+ // ADR-154: the run's credential startup begins in this short slot;
452
+ // its predecessor's cleanup and its grant are awaited outside it.
453
+ if (cloud !== null) {
454
+ try {
455
+ cloud.start = cloud.runtime.beginStart(cloud.assignment);
456
+ } catch (error) {
457
+ return {
458
+ ok: false as const,
459
+ result: {
460
+ ok: false as const,
461
+ code: HostErrorCode.HostUnavailable,
462
+ message: `task ${input.task.id} could not begin on its cloud credentials: ${
463
+ error instanceof Error ? error.message : String(error)
464
+ }`,
465
+ retryable: true,
466
+ ...(error instanceof StartInterruptedError ? { retryReason: "managed_restart" as const } : {}),
467
+ },
468
+ };
469
+ }
470
+ }
471
+ return { ok: true as const, admission };
472
+ },
473
+ );
474
+ if (!admitted.ok) return admitted.result;
475
+ runEpochAdmission = admitted.admission;
476
+ }
477
+ } catch (error) {
478
+ return {
479
+ ok: false,
480
+ code: HostErrorCode.Internal,
481
+ message:
482
+ error instanceof Error
483
+ ? error.message
484
+ : "the task run epoch is invalid",
485
+ };
486
+ }
487
+ // Admission alone never proves a runtime belongs to this assignment. A
488
+ // failed prior start can leave an open floor beside the older runtime.
489
+ const mayReuseRuntime = canReuseTaskRuntimeForRun(
490
+ input.task.id,
491
+ input.task.runEpoch,
492
+ );
493
+ const admissionFailure = managedMaintenanceUnavailable();
494
+ if (admissionFailure) return admissionFailure;
495
+ const runtimePreparationFailure = await prepareContainerRuntimeForTask();
496
+ if (runtimePreparationFailure) return runtimePreparationFailure;
497
+ const runtimeFailure = runtimeUnavailable();
498
+ if (runtimeFailure) return runtimeFailure;
499
+ // The boot reconcile briefly replaces CLI shims in the shared volume.
500
+ // Do not create local lifecycle/channel state until its one-way latch has
501
+ // settled (success or best-effort failure), then revalidate both mutable
502
+ // admission authorities after the await.
503
+ await agentClisReady;
504
+ const readyAdmissionFailure = managedMaintenanceUnavailable();
505
+ if (readyAdmissionFailure) return readyAdmissionFailure;
506
+ const readyRuntimeFailure = runtimeUnavailable();
507
+ if (readyRuntimeFailure) return readyRuntimeFailure;
508
+ try {
509
+ await waitForMcpGatewayReady();
510
+ } catch (error) {
511
+ console.warn(
512
+ `[task-gateway] refusing task ${input.task.id} before provisioning: ${
513
+ error instanceof Error ? error.message : String(error)
514
+ }`,
515
+ );
516
+ return {
517
+ ok: false,
518
+ code: HostErrorCode.HostUnavailable,
519
+ message:
520
+ "The task environment gateway is not ready on this host. Retry the task; if it persists, check the host service log.",
521
+ retryable: true,
522
+ };
523
+ }
524
+ // A task machine's own guards first: its runner's data directory root's
525
+ // alone, and the metadata service closed to the task's user. Without
526
+ // them the task does not start: it could read the sidecar's host token.
527
+ if (sidecarMode()) {
528
+ const guard = guardSidecarMachine({ dataDir: env.dataDir });
529
+ if (!guard.ok) {
530
+ console.warn(`[sidecar] refusing task ${input.task.id}: ${guard.reason}`);
293
531
  return {
294
532
  ok: false,
295
533
  code: HostErrorCode.Internal,
296
- message: "restart is unavailable in this process",
534
+ message: `This task machine is not safe to run the task on: ${guard.reason}. Close the task and start it again on a new machine.`,
297
535
  retryable: false,
298
536
  };
299
537
  }
300
- // The result frame flushes before exit: shutdown drains in-flight
301
- // commands (this one included) before the process terminates.
302
- managedHostRestartHook();
303
- return { ok: true, value: undefined };
304
- },
305
- async taskUp(ctx, input) {
306
- logCommand(ctx, "taskUp", input.task.id);
307
- const admissionFailure = managedMaintenanceUnavailable();
308
- if (admissionFailure) return admissionFailure;
309
- const runtimePreparationFailure = await prepareContainerRuntimeForTask();
310
- if (runtimePreparationFailure) return runtimePreparationFailure;
311
- const runtimeFailure = runtimeUnavailable();
312
- if (runtimeFailure) return runtimeFailure;
313
- // The boot reconcile briefly replaces CLI shims in the shared volume.
314
- // Do not create local lifecycle/channel state until its one-way latch has
315
- // settled (success or best-effort failure), then revalidate both mutable
316
- // admission authorities after the await.
317
- await agentClisReady;
318
- const readyAdmissionFailure = managedMaintenanceUnavailable();
319
- if (readyAdmissionFailure) return readyAdmissionFailure;
320
- const readyRuntimeFailure = runtimeUnavailable();
321
- if (readyRuntimeFailure) return readyRuntimeFailure;
538
+ }
539
+ // A bindings-capable task sidecar must activate and commit its exact
540
+ // replica before any task state, /env generation or process can exist.
541
+ // Otherwise the source's intentional pre-activation legacy fallback
542
+ // could expose stored plaintext during the first reconciliation.
543
+ if (sidecarMode()) {
322
544
  try {
323
- await waitForMcpGatewayReady();
545
+ // ADR-154: on cloud credentials the start's own grant is what readies
546
+ // it (awaited below); the v0 lease that readies the machine's stores
547
+ // never runs there.
548
+ if (cloud === null) await untilLeavingLegacy(legacy, awaitSidecarReady());
549
+ // ADR-154: and its env is the composer's, never the machine's replica.
550
+ if (cloud === null && projectEnvBindingCapability() !== null) {
551
+ await untilLeavingLegacy(legacy, awaitProjectEnvReplicaReady());
552
+ }
324
553
  } catch (error) {
554
+ // ADR-154: the runner began leaving its own credentials, and with them
555
+ // the v0 lease this readiness comes from: the cloud starts the task
556
+ // again on cloud credentials.
557
+ if (legacy?.signal.aborted) return switchingToCloudCredentials();
325
558
  console.warn(
326
- `[task-gateway] refusing task ${input.task.id} before provisioning: ${
559
+ `[task-env] refusing task ${input.task.id} before provisioning: ${
327
560
  error instanceof Error ? error.message : String(error)
328
561
  }`,
329
562
  );
@@ -331,881 +564,1072 @@ export const hostCommands: HostCommands = {
331
564
  ok: false,
332
565
  code: HostErrorCode.HostUnavailable,
333
566
  message:
334
- "The task environment gateway is not ready on this host. Retry the task; if it persists, check the host service log.",
567
+ "This task machine is still synchronizing its environment. Retry shortly.",
335
568
  retryable: true,
336
569
  };
337
570
  }
338
- // A task machine's own guards first: its runner's data directory root's
339
- // alone, and the metadata service closed to the task's user. Without
340
- // them the task does not start: it could read the sidecar's host token.
341
- if (sidecarMode()) {
342
- const guard = guardSidecarMachine({ dataDir: env.dataDir });
343
- if (!guard.ok) {
344
- console.warn(`[sidecar] refusing task ${input.task.id}: ${guard.reason}`);
345
- return {
571
+ }
572
+ // ADR-118 follow-up: each apple-container task is its own VM, and each
573
+ // VM pins host vnodes for its life. Refuse to start ANOTHER one when the
574
+ // table is nearly full — a clear, retryable refusal now beats ENFILE
575
+ // storms (and the suspected panic path) for the whole machine later.
576
+ if (pinnedContainerRuntimeProvider() === "apple-container") {
577
+ const vnodeProblem = await vnodeAdmissionProblem();
578
+ if (vnodeProblem) {
579
+ return {
580
+ ok: false,
581
+ code: HostErrorCode.HostUnavailable,
582
+ message: vnodeProblem,
583
+ retryable: true,
584
+ };
585
+ }
586
+ }
587
+ // ADR-154: the grant installed and its gate open, its env snapshot exact:
588
+ // awaited outside every lifecycle slot, and bounded.
589
+ if (cloud?.start) {
590
+ try {
591
+ await cloud.runtime.waitForLaunchReady(cloud.start);
592
+ } catch (error) {
593
+ return (
594
+ interruptedStartResult(cloud) ?? {
346
595
  ok: false,
347
- code: HostErrorCode.Internal,
348
- message: `This task machine is not safe to run the task on: ${guard.reason}. Close the task and start it again on a new machine.`,
349
- retryable: false,
350
- };
351
- }
596
+ code: HostErrorCode.HostUnavailable,
597
+ message: `The task's cloud credentials did not become usable (${
598
+ error instanceof Error ? error.message : String(error)
599
+ }). Retry the task.`,
600
+ retryable: true,
601
+ }
602
+ );
352
603
  }
353
- // A bindings-capable task sidecar must activate and commit its exact
354
- // replica before any task state, /env generation or process can exist.
355
- // Otherwise the source's intentional pre-activation legacy fallback
356
- // could expose stored plaintext during the first reconciliation.
357
- if (sidecarMode()) {
604
+ }
605
+ return orchestrator.runTaskLifecycle(input.task.id, releasingHold(cloud, async () => {
606
+ // A direct duplicate teardown may still be draining operations that
607
+ // target this task's stable compose/container names. Never recreate
608
+ // those names until every close caller has crossed the shared fence.
609
+ await orchestrator.waitForChannelClose(input.task.id);
610
+ // Admission is a short lifecycle transaction before the long host
611
+ // preflights above. A newer dispatch may have advanced the durable
612
+ // floor while this command was waiting; once this command owns the
613
+ // lifecycle slot, reject it before it can observe or mutate a newer
614
+ // assignment's runtime.
615
+ const assignmentState = readTaskRunEpochState(input.task.id);
616
+ const assignmentCurrent =
617
+ input.task.runEpoch === undefined
618
+ ? assignmentState === null
619
+ : assignmentState !== null &&
620
+ assignmentState.assignmentOpen &&
621
+ assignmentState.highestRunEpoch === input.task.runEpoch;
622
+ if (!assignmentCurrent) {
623
+ const currentRunEpoch = assignmentState?.highestRunEpoch ?? 0;
624
+ return {
625
+ ok: false,
626
+ code: HostErrorCode.TaskAlreadyRunning,
627
+ message:
628
+ `task ${input.task.id} run ${String(input.task.runEpoch)} is no longer current; ` +
629
+ `this runner has admitted run ${String(currentRunEpoch)}`,
630
+ };
631
+ }
632
+ const inSlotAdmissionFailure = managedMaintenanceUnavailable();
633
+ if (inSlotAdmissionFailure) return inSlotAdmissionFailure;
634
+ const inSlotRuntimeFailure = runtimeUnavailable();
635
+ if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
636
+ // ADR-154: nothing of a start on the runner's own credentials happens
637
+ // once the runner has begun leaving them.
638
+ if (legacy?.valid() === false) return switchingToCloudCredentials();
639
+ // ADR-154: the start holds its installed grant from here through the
640
+ // clone, provisioning and the runtime-ready writes; without it nothing
641
+ // starts.
642
+ if (cloud?.start) {
358
643
  try {
359
- await awaitSidecarReady();
360
- if (projectEnvBindingCapability() !== null) {
361
- await awaitProjectEnvReplicaReady();
362
- }
644
+ cloud.hold = cloud.runtime.tryAcquireTaskUpHold(cloud.start);
363
645
  } catch (error) {
364
- console.warn(
365
- `[task-env] refusing task ${input.task.id} before provisioning: ${
366
- error instanceof Error ? error.message : String(error)
367
- }`,
368
- );
369
646
  return {
370
647
  ok: false,
371
648
  code: HostErrorCode.HostUnavailable,
372
- message:
373
- "This task machine is still synchronizing its environment. Retry shortly.",
649
+ message: `The task's lease names credentials that could not be read (${
650
+ error instanceof Error ? error.message : String(error)
651
+ }). Retry the task.`,
374
652
  retryable: true,
375
653
  };
376
654
  }
655
+ if (cloud.hold === null) return interruptedStartResult(cloud) ?? withdrawnStart();
656
+ }
657
+ // A command retry that arrives after the original start completed must
658
+ // not rerender/compose-up/uai-init underneath live agent sessions. Treat
659
+ // that lost-response retry as the successful operation it already was:
660
+ // every cloud caller records a non-ok taskUp as an errored task.
661
+ const existingTask = getHostTask(input.task.id);
662
+ const incomingEnvSpec = taskProjectEnvSpec(input);
663
+ let existingGatewayReady = true;
664
+ if (existingTask && hostTaskEnvMode(existingTask) === "loader") {
665
+ // A lost-response retry is also the repair path after a host process
666
+ // restart: reattach the live resolver before any task process can be
667
+ // treated as reusable. Missing/corrupt durable identity forces normal
668
+ // recreation, which mints a generation and restarts its forwarder.
669
+ recordTaskProjectEnvSpec(input.task.id, input);
670
+ existingGatewayReady =
671
+ recoverTaskGatewayTask(input.task.id, {
672
+ // The generation serves the run its runtime was established for.
673
+ resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec, existingTask.runEpoch > 0 ? existingTask.runEpoch : null),
674
+ runEpoch: existingTask.runEpoch > 0 ? existingTask.runEpoch : null,
675
+ }) !== null;
377
676
  }
378
- // ADR-118 follow-up: each apple-container task is its own VM, and each
379
- // VM pins host vnodes for its life. Refuse to start ANOTHER one when the
380
- // table is nearly full — a clear, retryable refusal now beats ENFILE
381
- // storms (and the suspected panic path) for the whole machine later.
382
- if (pinnedContainerRuntimeProvider() === "apple-container") {
383
- const vnodeProblem = await vnodeAdmissionProblem();
384
- if (vnodeProblem) {
677
+ let runtimeQuarantinedForRecreate = false;
678
+ let channelClosedForRecreate = false;
679
+ if (
680
+ existingTask?.statusMirror === "running" &&
681
+ existingTask.environmentProvider === "apple-container"
682
+ ) {
683
+ // ADR-106/round-4: the lost-response fast path must speak the
684
+ // provider's language — the Docker branch below ENOENTs on a
685
+ // Docker-free Mac and suspended the generation instead of returning
686
+ // the persisted success.
687
+ if (!existingTask.composeProject || !existingTask.worktreePath) {
385
688
  return {
386
689
  ok: false,
387
- code: HostErrorCode.HostUnavailable,
388
- message: vnodeProblem,
389
- retryable: true,
690
+ code: HostErrorCode.Internal,
691
+ message:
692
+ `task ${input.task.id} is running but its persisted runtime ` +
693
+ "metadata is incomplete",
390
694
  };
391
695
  }
392
- }
393
- const orchestrator = getOrchestrator();
394
- return orchestrator.runTaskLifecycle(input.task.id, async () => {
395
- // A direct duplicate teardown may still be draining operations that
396
- // target this task's stable compose/container names. Never recreate
397
- // those names until every close caller has crossed the shared fence.
398
- await orchestrator.waitForChannelClose(input.task.id);
399
- const inSlotAdmissionFailure = managedMaintenanceUnavailable();
400
- if (inSlotAdmissionFailure) return inSlotAdmissionFailure;
401
- const inSlotRuntimeFailure = runtimeUnavailable();
402
- if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
403
- // A command retry that arrives after the original start completed must
404
- // not rerender/compose-up/uai-init underneath live agent sessions. Treat
405
- // that lost-response retry as the successful operation it already was:
406
- // every cloud caller records a non-ok taskUp as an errored task.
407
- const existingTask = getHostTask(input.task.id);
408
- const incomingEnvSpec = taskProjectEnvSpec(input);
409
- let existingGatewayReady = true;
410
- if (existingTask && hostTaskEnvMode(existingTask) === "loader") {
411
- // A lost-response retry is also the repair path after a host process
412
- // restart: reattach the live resolver before any task process can be
413
- // treated as reusable. Missing/corrupt durable identity forces normal
414
- // recreation, which mints a generation and restarts its forwarder.
415
- recordTaskProjectEnvSpec(input.task.id, input);
416
- existingGatewayReady =
417
- recoverTaskGatewayTask(input.task.id, {
418
- resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec),
419
- }) !== null;
420
- }
421
- let runtimeQuarantinedForRecreate = false;
422
- let channelClosedForRecreate = false;
423
- if (
424
- existingTask?.statusMirror === "running" &&
425
- existingTask.environmentProvider === "apple-container"
426
- ) {
427
- // ADR-106/round-4: the lost-response fast path must speak the
428
- // provider's language — the Docker branch below ENOENTs on a
429
- // Docker-free Mac and suspended the generation instead of returning
430
- // the persisted success.
431
- if (!existingTask.composeProject || !existingTask.worktreePath) {
696
+ // Round 5: every await below can outlive the runtime generation
697
+ // (bridge revalidation, forced recheck). A superseded pass must
698
+ // observe only — never reopen consumers or mutate the container —
699
+ // the same discipline apple recovery applies.
700
+ // Round 6: the fence requires the OPERATIONAL generation, not merely
701
+ // the epoch — plain suspension (connection revalidation) keeps the
702
+ // epoch and ready status while the daemon is unverified.
703
+ const fastEpoch = containerRuntimeReadinessEpoch();
704
+ const fastStale = (): boolean =>
705
+ !containerRuntimeGenerationOperational(fastEpoch);
706
+ const staleRetry = {
707
+ ok: false as const,
708
+ code: HostErrorCode.HostUnavailable,
709
+ message:
710
+ `the container runtime generation changed while verifying task ${input.task.id}; retry`,
711
+ retryable: true,
712
+ };
713
+ const environment = await reconstructPersistedTaskEnvironment(
714
+ existingTask,
715
+ );
716
+ if (fastStale()) return staleRetry;
717
+ if (environment === null) {
718
+ // The persisted locator no longer reconstructs (e.g. an engine
719
+ // identity change) — recreate through the normal path.
720
+ suspendTaskGatewayTask(input.task.id);
721
+ await orchestrator.closeChannel(input.task.id);
722
+ // Session drain can outlive the generation too: a superseded pass
723
+ // must not reach recordTaskStarting/provisioning below.
724
+ if (fastStale()) return staleRetry;
725
+ channelClosedForRecreate = true;
726
+ } else {
727
+ const appleStatus = await environment.status();
728
+ if (fastStale()) return staleRetry;
729
+ if (appleStatus.state === "unknown") {
432
730
  return {
433
731
  ok: false,
434
- code: HostErrorCode.Internal,
732
+ code: HostErrorCode.HostUnavailable,
435
733
  message:
436
- `task ${input.task.id} is running but its persisted runtime ` +
437
- "metadata is incomplete",
734
+ `could not verify runtime state for task ${input.task.id}: ` +
735
+ appleStatus.detail,
736
+ retryable: true,
438
737
  };
439
738
  }
440
- // Round 5: every await below can outlive the runtime generation
441
- // (bridge revalidation, forced recheck). A superseded pass must
442
- // observe only — never reopen consumers or mutate the container —
443
- // the same discipline apple recovery applies.
444
- // Round 6: the fence requires the OPERATIONAL generation, not merely
445
- // the epoch — plain suspension (connection revalidation) keeps the
446
- // epoch and ready status while the daemon is unverified.
447
- const fastEpoch = containerRuntimeReadinessEpoch();
448
- const fastStale = (): boolean =>
449
- !containerRuntimeGenerationOperational(fastEpoch);
450
- const staleRetry = {
451
- ok: false as const,
452
- code: HostErrorCode.HostUnavailable,
453
- message:
454
- `the container runtime generation changed while verifying task ${input.task.id}; retry`,
455
- retryable: true,
456
- };
457
- const environment = await reconstructPersistedTaskEnvironment(
458
- existingTask,
459
- );
460
- if (fastStale()) return staleRetry;
461
- if (environment === null) {
462
- // The persisted locator no longer reconstructs (e.g. an engine
463
- // identity change) — recreate through the normal path.
464
- suspendTaskGatewayTask(input.task.id);
465
- await orchestrator.closeChannel(input.task.id);
466
- // Session drain can outlive the generation too: a superseded pass
467
- // must not reach recordTaskStarting/provisioning below.
468
- if (fastStale()) return staleRetry;
469
- channelClosedForRecreate = true;
470
- } else {
471
- const appleStatus = await environment.status();
739
+ if (appleStatus.state === "running") {
740
+ const verdict = await proveAppleTaskRuntimeContract({
741
+ taskId: input.task.id,
742
+ containerName: appleTaskContainerName(input.task.id),
743
+ // The row's worktreePath is task-up's $task_dir — the same
744
+ // directory the apple locator records as hostWorktreePath.
745
+ hostWorktreePath: existingTask.worktreePath,
746
+ });
472
747
  if (fastStale()) return staleRetry;
473
- if (appleStatus.state === "unknown") {
748
+ if (verdict.kind === "unreachable") {
474
749
  return {
475
750
  ok: false,
476
751
  code: HostErrorCode.HostUnavailable,
477
752
  message:
478
- `could not verify runtime state for task ${input.task.id}: ` +
479
- appleStatus.detail,
753
+ `could not prove the exact runtime contract for task ${input.task.id}: ` +
754
+ verdict.detail,
480
755
  retryable: true,
481
756
  };
482
757
  }
483
- if (appleStatus.state === "running") {
484
- const verdict = await proveAppleTaskRuntimeContract({
485
- taskId: input.task.id,
486
- containerName: appleTaskContainerName(input.task.id),
487
- // The row's worktreePath is task-up's $task_dir — the same
488
- // directory the apple locator records as hostWorktreePath.
489
- hostWorktreePath: existingTask.worktreePath,
758
+ let projectsHealthy = true;
759
+ for (const project of verdict.kind === "holds"
760
+ ? input.projects
761
+ : []) {
762
+ const gitHealth = await environment.exec({
763
+ projectEnvProfile: "maintenance",
764
+ argv: [
765
+ "/usr/bin/env",
766
+ "-i",
767
+ "HOME=/home/node",
768
+ "PATH=/usr/bin:/bin",
769
+ "/usr/bin/git",
770
+ "rev-parse",
771
+ "--is-inside-work-tree",
772
+ ],
773
+ user: "node",
774
+ cwd: `/workspace/${project.slug}`,
775
+ env: {},
776
+ timeoutMs: 10_000,
777
+ maxOutputBytes: 64 * 1024,
490
778
  });
491
779
  if (fastStale()) return staleRetry;
492
- if (verdict.kind === "unreachable") {
780
+ if (gitHealth.exitCode === null || gitHealth.signal !== null) {
493
781
  return {
494
782
  ok: false,
495
783
  code: HostErrorCode.HostUnavailable,
496
784
  message:
497
- `could not prove the exact runtime contract for task ${input.task.id}: ` +
498
- verdict.detail,
785
+ `could not verify Git state for task ${input.task.id} ` +
786
+ `project ${project.id}: the runtime did not answer`,
499
787
  retryable: true,
500
788
  };
501
789
  }
502
- let projectsHealthy = true;
503
- for (const project of verdict.kind === "holds"
504
- ? input.projects
505
- : []) {
506
- const gitHealth = await environment.exec({
507
- projectEnvProfile: "maintenance",
508
- argv: [
509
- "/usr/bin/env",
510
- "-i",
511
- "HOME=/home/node",
512
- "PATH=/usr/bin:/bin",
513
- "/usr/bin/git",
514
- "rev-parse",
515
- "--is-inside-work-tree",
516
- ],
517
- user: "node",
518
- cwd: `/workspace/${project.slug}`,
519
- env: {},
520
- timeoutMs: 10_000,
521
- maxOutputBytes: 64 * 1024,
522
- });
523
- if (fastStale()) return staleRetry;
524
- if (gitHealth.exitCode === null || gitHealth.signal !== null) {
525
- return {
526
- ok: false,
527
- code: HostErrorCode.HostUnavailable,
528
- message:
529
- `could not verify Git state for task ${input.task.id} ` +
530
- `project ${project.id}: the runtime did not answer`,
531
- retryable: true,
532
- };
533
- }
534
- if (
535
- gitHealth.exitCode !== 0 ||
536
- Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
537
- "true"
538
- ) {
539
- projectsHealthy = false;
540
- console.warn(
541
- `[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
542
- );
543
- break;
544
- }
545
- }
546
790
  if (
547
- verdict.kind === "holds" &&
548
- projectsHealthy &&
549
- existingGatewayReady
791
+ gitHealth.exitCode !== 0 ||
792
+ Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
793
+ "true"
550
794
  ) {
551
- if (fastStale()) return staleRetry;
552
- orchestrator.allowChannel(input.task.id);
553
- return {
554
- ok: true,
555
- value: {
556
- composeProject: existingTask.composeProject,
557
- worktreePath: existingTask.worktreePath,
558
- codeServerPort: existingTask.codeServerPort ?? undefined,
559
- previewPorts: parsePreviewPortRuntimes(
560
- existingTask.previewPorts,
561
- ),
562
- envMode: hostTaskEnvMode(existingTask),
563
- },
564
- };
565
- }
566
- // Violated contract or broken project repo: drain, stop, and
567
- // prove before task-up recreates the same stable name.
568
- if (fastStale()) return staleRetry;
569
- suspendTaskGatewayTask(input.task.id);
570
- await orchestrator.closeChannel(input.task.id);
571
- channelClosedForRecreate = true;
572
- if (fastStale()) return staleRetry;
573
- await environment.stop();
574
- const after = await environment.status();
575
- if (fastStale()) return staleRetry;
576
- if (after.state === "running" || after.state === "unknown") {
577
- return {
578
- ok: false,
579
- code: HostErrorCode.HostUnavailable,
580
- message:
581
- `task ${input.task.id} has a non-current runtime contract and ` +
582
- "could not be proven stopped; retry after the runtime is healthy",
583
- retryable: true,
584
- };
795
+ projectsHealthy = false;
796
+ console.warn(
797
+ `[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
798
+ );
799
+ break;
585
800
  }
586
- // Round 5: the container is PHYSICALLY stopped now, whatever the
587
- // reason (violated contract or a broken project repo). A failed
588
- // recreation must roll the mirror back to stopped — restoring
589
- // "running" would advertise a container proven not to run.
590
- runtimeQuarantinedForRecreate = true;
591
- } else {
592
- // Provider proved the container absent/stopped: normal self-heal
593
- // recreate — and the rollback state must say stopped for the
594
- // same reason as above.
595
- if (fastStale()) return staleRetry;
596
- suspendTaskGatewayTask(input.task.id);
597
- await orchestrator.closeChannel(input.task.id);
598
- // Fence AFTER the drain as well: provisioning must never start
599
- // under a generation this pass no longer owns.
801
+ }
802
+ if (
803
+ verdict.kind === "holds" &&
804
+ projectsHealthy &&
805
+ existingGatewayReady &&
806
+ mayReuseRuntime
807
+ ) {
600
808
  if (fastStale()) return staleRetry;
601
- channelClosedForRecreate = true;
602
- runtimeQuarantinedForRecreate = true;
809
+ const cannotPublish = cloudStartCannotPublish(cloud);
810
+ if (cannotPublish) return cannotPublish.result;
811
+ orchestrator.allowChannel(input.task.id);
812
+ return {
813
+ ok: true,
814
+ value: {
815
+ composeProject: existingTask.composeProject,
816
+ worktreePath: existingTask.worktreePath,
817
+ codeServerPort: existingTask.codeServerPort ?? undefined,
818
+ previewPorts: parsePreviewPortRuntimes(
819
+ existingTask.previewPorts,
820
+ ),
821
+ envMode: hostTaskEnvMode(existingTask),
822
+ },
823
+ };
603
824
  }
825
+ // Violated contract or broken project repo: drain, stop, and
826
+ // prove before task-up recreates the same stable name.
827
+ if (fastStale()) return staleRetry;
828
+ suspendTaskGatewayTask(input.task.id);
829
+ await orchestrator.closeChannel(input.task.id);
830
+ channelClosedForRecreate = true;
831
+ if (fastStale()) return staleRetry;
832
+ await environment.stop();
833
+ const after = await environment.status();
834
+ if (fastStale()) return staleRetry;
835
+ if (after.state === "running" || after.state === "unknown") {
836
+ return {
837
+ ok: false,
838
+ code: HostErrorCode.HostUnavailable,
839
+ message:
840
+ `task ${input.task.id} has a non-current runtime contract and ` +
841
+ "could not be proven stopped; retry after the runtime is healthy",
842
+ retryable: true,
843
+ };
844
+ }
845
+ // Round 5: the container is PHYSICALLY stopped now, whatever the
846
+ // reason (violated contract or a broken project repo). A failed
847
+ // recreation must roll the mirror back to stopped — restoring
848
+ // "running" would advertise a container proven not to run.
849
+ runtimeQuarantinedForRecreate = true;
850
+ } else {
851
+ // Provider proved the container absent/stopped: normal self-heal
852
+ // recreate — and the rollback state must say stopped for the
853
+ // same reason as above.
854
+ if (fastStale()) return staleRetry;
855
+ suspendTaskGatewayTask(input.task.id);
856
+ await orchestrator.closeChannel(input.task.id);
857
+ // Fence AFTER the drain as well: provisioning must never start
858
+ // under a generation this pass no longer owns.
859
+ if (fastStale()) return staleRetry;
860
+ channelClosedForRecreate = true;
861
+ runtimeQuarantinedForRecreate = true;
604
862
  }
605
- } else if (
606
- existingTask?.statusMirror === "running" &&
607
- isEnvironmentTransportProvider(existingTask.environmentProvider)
608
- ) {
609
- // ADR-121: a machine-backed task revalidates through its own
610
- // provider — the compose fast path below speaks docker and would
611
- // misread a machine row. Running machine → persisted success;
612
- // anything else falls through to provisionTaskEnvironment, whose
613
- // machine provider owns recovery semantics.
614
- try {
615
- const environment = await reconstructPersistedTaskEnvironment(
616
- existingTask,
617
- );
618
- if (environment) {
619
- const machineStatus = await environment.status();
620
- if (machineStatus.state === "running" && existingGatewayReady) {
621
- orchestrator.allowChannel(input.task.id);
622
- return {
623
- ok: true,
624
- value: {
625
- composeProject: existingTask.composeProject ?? "",
626
- worktreePath: existingTask.worktreePath ?? "",
627
- codeServerPort: existingTask.codeServerPort ?? undefined,
628
- envMode: hostTaskEnvMode(existingTask),
629
- },
630
- };
631
- }
863
+ }
864
+ } else if (
865
+ existingTask?.statusMirror === "running" &&
866
+ isEnvironmentTransportProvider(existingTask.environmentProvider)
867
+ ) {
868
+ // ADR-121: a machine-backed task revalidates through its own
869
+ // provider — the compose fast path below speaks docker and would
870
+ // misread a machine row. Running machine → persisted success;
871
+ // anything else falls through to provisionTaskEnvironment, whose
872
+ // machine provider owns recovery semantics.
873
+ try {
874
+ const environment = await reconstructPersistedTaskEnvironment(
875
+ existingTask,
876
+ );
877
+ if (environment) {
878
+ const machineStatus = await environment.status();
879
+ if (
880
+ machineStatus.state === "running" &&
881
+ existingGatewayReady &&
882
+ mayReuseRuntime
883
+ ) {
884
+ const cannotPublish = cloudStartCannotPublish(cloud);
885
+ if (cannotPublish) return cannotPublish.result;
886
+ orchestrator.allowChannel(input.task.id);
887
+ return {
888
+ ok: true,
889
+ value: {
890
+ composeProject: existingTask.composeProject ?? "",
891
+ worktreePath: existingTask.worktreePath ?? "",
892
+ codeServerPort: existingTask.codeServerPort ?? undefined,
893
+ envMode: hostTaskEnvMode(existingTask),
894
+ },
895
+ };
632
896
  }
633
- } catch (error) {
634
- console.warn(
635
- `[machine] task ${input.task.id}: running-row revalidation failed, continuing to provision: ${error instanceof Error ? error.message : String(error)}`,
897
+ }
898
+ } catch (error) {
899
+ console.warn(
900
+ `[machine] task ${input.task.id}: running-row revalidation failed, continuing to provision: ${error instanceof Error ? error.message : String(error)}`,
901
+ );
902
+ }
903
+ } else if (existingTask?.statusMirror === "running") {
904
+ if (!existingTask.composeProject || !existingTask.worktreePath) {
905
+ return {
906
+ ok: false,
907
+ code: HostErrorCode.Internal,
908
+ message:
909
+ `task ${input.task.id} is running but its persisted runtime ` +
910
+ "metadata is incomplete",
911
+ };
912
+ }
913
+ // The mirror may be stale after a manual `docker compose down`.
914
+ // Confirm the app itself is live before treating this as a lost-response
915
+ // retry; otherwise fall through to the normal taskUp self-heal.
916
+ const appRuntime = await dockerCli(
917
+ [
918
+ "ps",
919
+ "-a",
920
+ "--filter",
921
+ `label=com.docker.compose.project=${existingTask.composeProject}`,
922
+ "--filter",
923
+ "label=com.docker.compose.service=app",
924
+ "--format",
925
+ "{{.State}}",
926
+ ],
927
+ { timeoutMs: 10_000 },
928
+ );
929
+ if (appRuntime.status !== 0) {
930
+ return dockerUnavailableResult(
931
+ `could not verify runtime state for task ${input.task.id}: ` +
932
+ (appRuntime.stderr.trim().slice(0, 200) ||
933
+ "docker did not answer"),
934
+ );
935
+ }
936
+ if (
937
+ appRuntime.stdout
938
+ .split(/\r?\n/)
939
+ .some((state) => state.trim() === "running")
940
+ ) {
941
+ const appContainer = `${existingTask.composeProject}-app-1`;
942
+ const runtimeContract = await inspectTaskRuntimeContainer(
943
+ input.task.id,
944
+ existingTask.composeProject,
945
+ );
946
+ if (runtimeContract.kind === "unreachable") {
947
+ return dockerUnavailableResult(
948
+ `could not inspect the exact runtime mount for task ${input.task.id}`,
636
949
  );
637
950
  }
638
- } else if (existingTask?.statusMirror === "running") {
639
- if (!existingTask.composeProject || !existingTask.worktreePath) {
951
+ if (runtimeContract.kind === "failed") {
640
952
  return {
641
953
  ok: false,
642
- code: HostErrorCode.Internal,
954
+ code: HostErrorCode.HostUnavailable,
643
955
  message:
644
- `task ${input.task.id} is running but its persisted runtime ` +
645
- "metadata is incomplete",
956
+ `could not prove the exact runtime contract for task ${input.task.id}: ` +
957
+ runtimeContract.detail,
958
+ retryable: true,
646
959
  };
647
960
  }
648
- // The mirror may be stale after a manual `docker compose down`.
649
- // Confirm the app itself is live before treating this as a lost-response
650
- // retry; otherwise fall through to the normal taskUp self-heal.
651
- const appRuntime = await dockerCli(
652
- [
653
- "ps",
654
- "-a",
655
- "--filter",
656
- `label=com.docker.compose.project=${existingTask.composeProject}`,
657
- "--filter",
658
- "label=com.docker.compose.service=app",
659
- "--format",
660
- "{{.State}}",
661
- ],
662
- { timeoutMs: 10_000 },
663
- );
664
- if (appRuntime.status !== 0) {
665
- return dockerUnavailableResult(
666
- `could not verify runtime state for task ${input.task.id}: ` +
667
- (appRuntime.stderr.trim().slice(0, 200) ||
668
- "docker did not answer"),
669
- );
670
- }
671
- if (
672
- appRuntime.stdout
673
- .split(/\r?\n/)
674
- .some((state) => state.trim() === "running")
675
- ) {
676
- const appContainer = `${existingTask.composeProject}-app-1`;
677
- const runtimeContract = await inspectTaskRuntimeContainer(
678
- input.task.id,
679
- existingTask.composeProject,
680
- );
681
- if (runtimeContract.kind === "unreachable") {
961
+ const reusable =
962
+ runtimeContract.running &&
963
+ !runtimeContract.restarting &&
964
+ !runtimeContract.paused &&
965
+ runtimeContract.sharedMount === "read-only" &&
966
+ runtimeContract.taskEnvironment === "current" &&
967
+ existingGatewayReady &&
968
+ mayReuseRuntime;
969
+ if (!reusable) {
970
+ // Drain/tombstone before touching the exact container. task-up's
971
+ // pre-materialization cleanup then removes this stopped immutable
972
+ // generation, so Compose must create the current RO/private-prefix
973
+ // contract rather than `docker start`ing it again.
974
+ suspendTaskGatewayTask(input.task.id);
975
+ await orchestrator.closeChannel(input.task.id);
976
+ channelClosedForRecreate = true;
977
+ const quarantined = await quarantineTaskRuntimeContainer({
978
+ taskId: input.task.id,
979
+ composeProject: existingTask.composeProject,
980
+ expectedContainerId: runtimeContract.containerId,
981
+ });
982
+ if (quarantined === "unreachable") {
682
983
  return dockerUnavailableResult(
683
- `could not inspect the exact runtime mount for task ${input.task.id}`,
984
+ `Docker became unavailable while quarantining task ${input.task.id}`,
684
985
  );
685
986
  }
686
- if (runtimeContract.kind === "failed") {
987
+ if (quarantined === "failed") {
687
988
  return {
688
989
  ok: false,
689
990
  code: HostErrorCode.HostUnavailable,
690
991
  message:
691
- `could not prove the exact runtime contract for task ${input.task.id}: ` +
692
- runtimeContract.detail,
992
+ `task ${input.task.id} has a non-current runtime contract and ` +
993
+ "could not be proven stopped; retry after Docker is healthy",
693
994
  retryable: true,
694
995
  };
695
996
  }
696
- const reusable =
697
- runtimeContract.running &&
698
- !runtimeContract.restarting &&
699
- !runtimeContract.paused &&
700
- runtimeContract.sharedMount === "read-only" &&
701
- runtimeContract.taskEnvironment === "current" &&
702
- existingGatewayReady;
703
- if (!reusable) {
704
- // Drain/tombstone before touching the exact container. task-up's
705
- // pre-materialization cleanup then removes this stopped immutable
706
- // generation, so Compose must create the current RO/private-prefix
707
- // contract rather than `docker start`ing it again.
708
- suspendTaskGatewayTask(input.task.id);
709
- await orchestrator.closeChannel(input.task.id);
710
- channelClosedForRecreate = true;
711
- const quarantined = await quarantineTaskRuntimeContainer({
712
- taskId: input.task.id,
713
- composeProject: existingTask.composeProject,
714
- expectedContainerId: runtimeContract.containerId,
715
- });
716
- if (quarantined === "unreachable") {
717
- return dockerUnavailableResult(
718
- `Docker became unavailable while quarantining task ${input.task.id}`,
719
- );
720
- }
721
- if (quarantined === "failed") {
722
- return {
723
- ok: false,
724
- code: HostErrorCode.HostUnavailable,
725
- message:
726
- `task ${input.task.id} has a non-current runtime contract and ` +
727
- "could not be proven stopped; retry after Docker is healthy",
728
- retryable: true,
729
- };
730
- }
731
- runtimeQuarantinedForRecreate = true;
732
- }
733
- // Docker being up is not enough to prove a lost-response retry is
734
- // healthy. The worktree's `.git` file contains an absolute pointer
735
- // into the task-private repository; after a host workspace-root
736
- // migration the old container can keep running while every agent
737
- // sees `fatal: not a git repository`. Probe from INSIDE the live
738
- // container, which is the filesystem view the agents actually use.
739
- // Scratchpads have no projects, so this remains an O(1) fast path.
740
- let projectsHealthy = true;
741
- for (const project of reusable ? input.projects : []) {
742
- const gitHealth = await dockerCli(
743
- [
744
- "exec",
745
- "-u",
746
- "node",
747
- // docker exec inherits the Compose/project environment. Clear
748
- // loader and Git controls before the first executable, then
749
- // give Git a minimal environment of its own. Keep this aligned
750
- // with task-up.sh's managed container operations.
751
- "-e",
752
- "HOME=/home/node",
753
- "-e",
754
- "PATH=/usr/bin:/bin",
755
- "-e",
756
- "LD_PRELOAD=",
757
- "-e",
758
- "LD_LIBRARY_PATH=",
759
- "-e",
760
- "DYLD_INSERT_LIBRARIES=",
761
- "-e",
762
- "DYLD_LIBRARY_PATH=",
763
- "-e",
764
- "BASH_ENV=",
765
- "-e",
766
- "ENV=",
767
- "-e",
768
- "GIT_CONFIG=",
769
- "-e",
770
- "GIT_CONFIG_GLOBAL=",
771
- "-e",
772
- "GIT_CONFIG_SYSTEM=",
773
- "-e",
774
- "GIT_CONFIG_NOSYSTEM=",
775
- "-e",
776
- "GIT_CONFIG_COUNT=0",
777
- "-e",
778
- "GIT_EXEC_PATH=",
779
- "-e",
780
- "GIT_SSH=",
781
- "-e",
782
- "GIT_SSH_COMMAND=",
783
- "-e",
784
- "GIT_ASKPASS=",
785
- "-e",
786
- "SSH_ASKPASS=",
787
- "-w",
788
- `/workspace/${project.slug}`,
789
- appContainer,
790
- "/usr/bin/env",
791
- "-i",
792
- "HOME=/home/node",
793
- "PATH=/usr/bin:/bin",
794
- "/usr/bin/git",
795
- "rev-parse",
796
- "--is-inside-work-tree",
797
- ],
798
- { timeoutMs: 10_000 },
997
+ runtimeQuarantinedForRecreate = true;
998
+ }
999
+ // Docker being up is not enough to prove a lost-response retry is
1000
+ // healthy. The worktree's `.git` file contains an absolute pointer
1001
+ // into the task-private repository; after a host workspace-root
1002
+ // migration the old container can keep running while every agent
1003
+ // sees `fatal: not a git repository`. Probe from INSIDE the live
1004
+ // container, which is the filesystem view the agents actually use.
1005
+ // Scratchpads have no projects, so this remains an O(1) fast path.
1006
+ let projectsHealthy = true;
1007
+ for (const project of reusable ? input.projects : []) {
1008
+ const gitHealth = await dockerCli(
1009
+ [
1010
+ "exec",
1011
+ "-u",
1012
+ "node",
1013
+ // docker exec inherits the Compose/project environment. Clear
1014
+ // loader and Git controls before the first executable, then
1015
+ // give Git a minimal environment of its own. Keep this aligned
1016
+ // with task-up.sh's managed container operations.
1017
+ "-e",
1018
+ "HOME=/home/node",
1019
+ "-e",
1020
+ "PATH=/usr/bin:/bin",
1021
+ "-e",
1022
+ "LD_PRELOAD=",
1023
+ "-e",
1024
+ "LD_LIBRARY_PATH=",
1025
+ "-e",
1026
+ "DYLD_INSERT_LIBRARIES=",
1027
+ "-e",
1028
+ "DYLD_LIBRARY_PATH=",
1029
+ "-e",
1030
+ "BASH_ENV=",
1031
+ "-e",
1032
+ "ENV=",
1033
+ "-e",
1034
+ "GIT_CONFIG=",
1035
+ "-e",
1036
+ "GIT_CONFIG_GLOBAL=",
1037
+ "-e",
1038
+ "GIT_CONFIG_SYSTEM=",
1039
+ "-e",
1040
+ "GIT_CONFIG_NOSYSTEM=",
1041
+ "-e",
1042
+ "GIT_CONFIG_COUNT=0",
1043
+ "-e",
1044
+ "GIT_EXEC_PATH=",
1045
+ "-e",
1046
+ "GIT_SSH=",
1047
+ "-e",
1048
+ "GIT_SSH_COMMAND=",
1049
+ "-e",
1050
+ "GIT_ASKPASS=",
1051
+ "-e",
1052
+ "SSH_ASKPASS=",
1053
+ "-w",
1054
+ `/workspace/${project.slug}`,
1055
+ appContainer,
1056
+ "/usr/bin/env",
1057
+ "-i",
1058
+ "HOME=/home/node",
1059
+ "PATH=/usr/bin:/bin",
1060
+ "/usr/bin/git",
1061
+ "rev-parse",
1062
+ "--is-inside-work-tree",
1063
+ ],
1064
+ { timeoutMs: 10_000 },
1065
+ );
1066
+ // A timeout/spawn error is not proof that the repository is
1067
+ // broken. Preserve the running task and let the caller retry once
1068
+ // Docker can answer, just as the runtime-state probe above does.
1069
+ if (gitHealth.status === null) {
1070
+ return dockerUnavailableResult(
1071
+ `could not verify Git state for task ${input.task.id} ` +
1072
+ `project ${project.id}: ` +
1073
+ (gitHealth.stderr.trim().slice(0, 200) ||
1074
+ "docker did not answer"),
799
1075
  );
800
- // A timeout/spawn error is not proof that the repository is
801
- // broken. Preserve the running task and let the caller retry once
802
- // Docker can answer, just as the runtime-state probe above does.
803
- if (gitHealth.status === null) {
804
- return dockerUnavailableResult(
1076
+ }
1077
+ if (
1078
+ gitHealth.status !== 0 ||
1079
+ gitHealth.stdout.trim() !== "true"
1080
+ ) {
1081
+ if (gitHealth.status !== 0) {
1082
+ const unavailable = await runtimeFailureIfDaemonUnavailable(
805
1083
  `could not verify Git state for task ${input.task.id} ` +
806
1084
  `project ${project.id}: ` +
807
1085
  (gitHealth.stderr.trim().slice(0, 200) ||
808
- "docker did not answer"),
1086
+ `git exited ${String(gitHealth.status)}`),
809
1087
  );
1088
+ if (unavailable) return unavailable;
810
1089
  }
811
- if (
812
- gitHealth.status !== 0 ||
813
- gitHealth.stdout.trim() !== "true"
814
- ) {
815
- if (gitHealth.status !== 0) {
816
- const unavailable = await runtimeFailureIfDaemonUnavailable(
817
- `could not verify Git state for task ${input.task.id} ` +
818
- `project ${project.id}: ` +
819
- (gitHealth.stderr.trim().slice(0, 200) ||
820
- `git exited ${String(gitHealth.status)}`),
821
- );
822
- if (unavailable) return unavailable;
823
- }
824
- projectsHealthy = false;
825
- console.warn(
826
- `[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
827
- );
828
- break;
829
- }
830
- }
831
- if (reusable && projectsHealthy) {
832
- // waitForChannelClose crossed the teardown fence above. A completed
833
- // close-only teardown may have left this running task tombstoned,
834
- // so the idempotent-success transition must reopen delivery just
835
- // like a fresh successful taskUp does.
836
- orchestrator.allowChannel(input.task.id);
837
- return {
838
- ok: true,
839
- value: {
840
- composeProject: existingTask.composeProject,
841
- worktreePath: existingTask.worktreePath,
842
- codeServerPort: existingTask.codeServerPort ?? undefined,
843
- previewPorts: parsePreviewPortRuntimes(
844
- existingTask.previewPorts,
845
- ),
846
- envMode: hostTaskEnvMode(existingTask),
847
- },
848
- };
1090
+ projectsHealthy = false;
1091
+ console.warn(
1092
+ `[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
1093
+ );
1094
+ break;
849
1095
  }
850
1096
  }
851
- // Docker positively proved the mirrored app is absent/stopped, or the
852
- // live container proved one of its project repositories is broken.
853
- // Drain and invalidate any stale in-memory channel before taskUp
854
- // recreates the same stable compose/container names and repairs Git.
855
- if (!channelClosedForRecreate) {
856
- suspendTaskGatewayTask(input.task.id);
857
- await orchestrator.closeChannel(input.task.id);
1097
+ if (reusable && projectsHealthy) {
1098
+ // waitForChannelClose crossed the teardown fence above. A completed
1099
+ // close-only teardown may have left this running task tombstoned,
1100
+ // so the idempotent-success transition must reopen delivery just
1101
+ // like a fresh successful taskUp does.
1102
+ const cannotPublish = cloudStartCannotPublish(cloud);
1103
+ if (cannotPublish) return cannotPublish.result;
1104
+ orchestrator.allowChannel(input.task.id);
1105
+ return {
1106
+ ok: true,
1107
+ value: {
1108
+ composeProject: existingTask.composeProject,
1109
+ worktreePath: existingTask.worktreePath,
1110
+ codeServerPort: existingTask.codeServerPort ?? undefined,
1111
+ previewPorts: parsePreviewPortRuntimes(
1112
+ existingTask.previewPorts,
1113
+ ),
1114
+ envMode: hostTaskEnvMode(existingTask),
1115
+ },
1116
+ };
858
1117
  }
859
1118
  }
860
- const priorRuntimeState = existingTask
861
- ? runtimeQuarantinedForRecreate
862
- ? {
863
- statusMirror: "stopped",
864
- codeServerPort: null,
865
- previewPorts: "[]",
866
- lockedAt: null,
867
- startedAt: existingTask.startedAt,
868
- endedAt: existingTask.endedAt,
869
- environmentProvider: existingTask.environmentProvider,
870
- environmentLocator: existingTask.environmentLocator,
871
- envMode: existingTask.envMode,
872
- projectEnvSpec: existingTask.projectEnvSpec,
873
- }
874
- : {
875
- statusMirror: existingTask.statusMirror,
876
- lockedAt: existingTask.lockedAt,
877
- startedAt: existingTask.startedAt,
878
- endedAt: existingTask.endedAt,
879
- environmentProvider: existingTask.environmentProvider,
880
- environmentLocator: existingTask.environmentLocator,
881
- envMode: existingTask.envMode,
882
- projectEnvSpec: existingTask.projectEnvSpec,
883
- }
884
- : {
885
- statusMirror: input.task.status,
1119
+ // Docker positively proved the mirrored app is absent/stopped, or the
1120
+ // live container proved one of its project repositories is broken.
1121
+ // Drain and invalidate any stale in-memory channel before taskUp
1122
+ // recreates the same stable compose/container names and repairs Git.
1123
+ if (!channelClosedForRecreate) {
1124
+ suspendTaskGatewayTask(input.task.id);
1125
+ await orchestrator.closeChannel(input.task.id);
1126
+ }
1127
+ }
1128
+ const priorRuntimeState = existingTask
1129
+ ? runtimeQuarantinedForRecreate
1130
+ ? {
1131
+ statusMirror: "stopped",
1132
+ codeServerPort: null,
1133
+ previewPorts: "[]",
886
1134
  lockedAt: null,
887
- startedAt: null,
888
- endedAt: null,
889
- };
890
- // ADR-153 machines: a task with no row has never run on this machine,
891
- // so everything it runs comes through its hosts view. Recorded before
892
- // the row, and so before the task's first process; a task with a row
893
- // (a start driven again, a start after a stop) stands as recorded.
894
- if (sidecarMode() && !existingTask) admitFreshTaskView(input.task.id);
1135
+ startedAt: existingTask.startedAt,
1136
+ endedAt: existingTask.endedAt,
1137
+ environmentProvider: existingTask.environmentProvider,
1138
+ environmentLocator: existingTask.environmentLocator,
1139
+ envMode: existingTask.envMode,
1140
+ projectEnvSpec: existingTask.projectEnvSpec,
1141
+ runEpoch: existingTask.runEpoch,
1142
+ }
1143
+ : {
1144
+ statusMirror: existingTask.statusMirror,
1145
+ lockedAt: existingTask.lockedAt,
1146
+ startedAt: existingTask.startedAt,
1147
+ endedAt: existingTask.endedAt,
1148
+ environmentProvider: existingTask.environmentProvider,
1149
+ environmentLocator: existingTask.environmentLocator,
1150
+ envMode: existingTask.envMode,
1151
+ projectEnvSpec: existingTask.projectEnvSpec,
1152
+ runEpoch: existingTask.runEpoch,
1153
+ }
1154
+ : {
1155
+ statusMirror: input.task.status,
1156
+ lockedAt: null,
1157
+ startedAt: null,
1158
+ endedAt: null,
1159
+ };
1160
+ // ADR-153 machines: a task with no row has never run on this machine,
1161
+ // so everything it runs comes through its hosts view. Recorded before
1162
+ // the row, and so before the task's first process; a task with a row
1163
+ // (a start driven again, a start after a stop) stands as recorded.
1164
+ const taskWasKnownBeforeAssignment =
1165
+ runEpochAdmission.kind === "accepted"
1166
+ ? runEpochAdmission.hadTask
1167
+ : existingTask !== null;
1168
+ if (sidecarMode() && !taskWasKnownBeforeAssignment) {
1169
+ admitFreshTaskView(input.task.id);
1170
+ }
1171
+ if (input.task.runEpoch === undefined) {
895
1172
  recordTaskStarting(input.task.id);
896
- // Cross-process managed maintenance and task admission use a two-sided
897
- // handshake: update/uninstall writes its durable marker before checking
898
- // this DB row; task-up writes `starting` before checking admission again.
899
- // Whichever arrives second observes the first, so a runtime cannot be
900
- // swapped or removed while an unrecorded task enters Docker side effects.
901
- const admittedMaintenanceFailure = managedMaintenanceUnavailable();
902
- if (admittedMaintenanceFailure) {
903
- if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
904
- else deleteHostTask(input.task.id);
905
- if (!runtimeQuarantinedForRecreate) {
906
- orchestrator.allowChannel(input.task.id);
907
- }
908
- return admittedMaintenanceFailure;
1173
+ } else {
1174
+ recordTaskStarting(input.task.id, input.task.runEpoch);
1175
+ }
1176
+ // Cross-process managed maintenance and task admission use a two-sided
1177
+ // handshake: update/uninstall writes its durable marker before checking
1178
+ // this DB row; task-up writes `starting` before checking admission again.
1179
+ // Whichever arrives second observes the first, so a runtime cannot be
1180
+ // swapped or removed while an unrecorded task enters Docker side effects.
1181
+ const admittedMaintenanceFailure = managedMaintenanceUnavailable();
1182
+ if (admittedMaintenanceFailure) {
1183
+ if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
1184
+ else deleteHostTask(input.task.id);
1185
+ if (!runtimeQuarantinedForRecreate) {
1186
+ orchestrator.allowChannel(input.task.id);
909
1187
  }
910
- recordTaskOwner(
911
- input.task.id,
912
- input.task.ownerUserId,
913
- input.ownerEmail ?? null,
914
- input.ownerName ?? null,
915
- input.projects.map((p) => p.slug),
916
- // ADR-131 D3: persisted so recovery re-provisions the same scope.
917
- input.task.engineAccountScope ?? null,
918
- // Lead project first (position 0 wins env collisions, as in task-up).
919
- [...input.projects]
920
- .sort((a, b) => a.position - b.position)
921
- .map((p) => p.id),
922
- );
923
- recordTaskProjectEnvSpec(input.task.id, input);
924
- // ADR-048: persist the cloud-issued per-task cli secret (host-private) so the
925
- // `uai` CLI's per-agent tokens can be minted now + after a host restart.
926
- if (input.task.cliSecret) {
927
- storeTaskCliSecret(input.task.id, input.task.cliSecret);
1188
+ return admittedMaintenanceFailure;
1189
+ }
1190
+ recordTaskOwner(
1191
+ input.task.id,
1192
+ input.task.ownerUserId,
1193
+ input.ownerEmail ?? null,
1194
+ input.ownerName ?? null,
1195
+ input.projects.map((p) => p.slug),
1196
+ // ADR-131 D3: persisted so recovery re-provisions the same scope.
1197
+ input.task.engineAccountScope ?? null,
1198
+ // Lead project first (position 0 wins env collisions, as in task-up).
1199
+ [...input.projects]
1200
+ .sort((a, b) => a.position - b.position)
1201
+ .map((p) => p.id),
1202
+ );
1203
+ recordTaskProjectEnvSpec(input.task.id, input);
1204
+ // ADR-048: persist the cloud-issued per-task cli secret (host-private) so the
1205
+ // `uai` CLI's per-agent tokens can be minted now + after a host restart.
1206
+ if (input.task.cliSecret) {
1207
+ storeTaskCliSecret(input.task.id, input.task.cliSecret);
1208
+ }
1209
+ recordHostEvent(input.task.id, "task.created");
1210
+ let environmentLocator: TaskEnvironmentLocator | undefined;
1211
+ // The locator of everything this start may create, recorded before any
1212
+ // of it exists: once set, the row must keep it.
1213
+ let runtimePrepared = false;
1214
+ const recordPrepared = async (locator: TaskEnvironmentLocator): Promise<void> => {
1215
+ recordTaskEnvironmentPrepared(input.task.id, locator, "loader");
1216
+ runtimePrepared = true;
1217
+ };
1218
+ let registeredTaskGatewayGeneration: string | null = null;
1219
+ let result: TaskUpOutcome = await wrapAgent(ctx, "taskUp", async () => {
1220
+ // Self-heal the standard base image before task-up needs it. The
1221
+ // boot-time build is best-effort + one-shot: a host that started before
1222
+ // Docker was ready (common on Docker Desktop) never built
1223
+ // uai-standard:dev, so every task-up died on a doomed registry pull
1224
+ // (live 2026-07-22). This is idempotent — a fast inspect when present,
1225
+ // a real build only when missing/stale (Docker is definitely up now, a
1226
+ // task just arrived). On a build FAILURE, surface WHY to the cloud (not
1227
+ // just the host log — the operator can't read a remote user's machine).
1228
+ // A pass that succeeded within the hour is reused (2026-09-09: the
1229
+ // re-check cost 16 s of every task start); the hourly schedule in
1230
+ // main.ts and the boot pass keep it fresh.
1231
+ const img = await ensureStandardImage({ maxAgeMs: 60 * 60_000 });
1232
+ if (!img.ok) {
1233
+ throw new AgentError(
1234
+ "STANDARD_IMAGE_UNAVAILABLE",
1235
+ img.error
1236
+ ? `could not prepare the host base image uai-standard:dev — the build failed. ${toolStderrDetail(img.error)}`
1237
+ : "could not prepare the host base image uai-standard:dev — see the host log.",
1238
+ {},
1239
+ );
928
1240
  }
929
- recordHostEvent(input.task.id, "task.created");
930
- let environmentLocator: TaskEnvironmentLocator | undefined;
931
- let registeredTaskGatewayGeneration: string | null = null;
932
- const result = await wrapAgent(ctx, "taskUp", async () => {
933
- // Self-heal the standard base image before task-up needs it. The
934
- // boot-time build is best-effort + one-shot: a host that started before
935
- // Docker was ready (common on Docker Desktop) never built
936
- // uai-standard:dev, so every task-up died on a doomed registry pull
937
- // (live 2026-07-22). This is idempotent — a fast inspect when present,
938
- // a real build only when missing/stale (Docker is definitely up now, a
939
- // task just arrived). On a build FAILURE, surface WHY to the cloud (not
940
- // just the host log — the operator can't read a remote user's machine).
941
- // A pass that succeeded within the hour is reused (2026-09-09: the
942
- // re-check cost 16 s of every task start); the hourly schedule in
943
- // main.ts and the boot pass keep it fresh.
944
- const img = await ensureStandardImage({ maxAgeMs: 60 * 60_000 });
945
- if (!img.ok) {
946
- throw new AgentError(
947
- "STANDARD_IMAGE_UNAVAILABLE",
948
- img.error
949
- ? `could not prepare the host base image uai-standard:dev — the build failed. ${toolStderrDetail(img.error)}`
950
- : "could not prepare the host base image uai-standard:dev — see the host log.",
951
- {},
952
- );
953
- }
954
- let gitCredential: TaskGithubGitCredential | null;
955
- try {
956
- gitCredential = input.projects.some((project) =>
957
- isGithubRepo(project.repoUrl),
958
- )
959
- ? await prepareTaskGithubGitCredential(input.task.ownerUserId)
960
- : null;
961
- } catch {
962
- throw new AgentError(
963
- "GITHUB_AUTH_UNAVAILABLE",
964
- "GitHub is connected on this host, but Uai could not prepare its credential for Git. Retry the task; if it persists, reconnect GitHub on this host.",
1241
+ let gitCredential: TaskGithubGitCredential | null;
1242
+ try {
1243
+ // ADR-154: a start on cloud credentials clones with its lease's,
1244
+ // never the runner's own store.
1245
+ gitCredential = cloud === null && input.projects.some((project) =>
1246
+ isGithubRepo(project.repoUrl),
1247
+ )
1248
+ ? await prepareTaskGithubGitCredential(input.task.ownerUserId)
1249
+ : null;
1250
+ } catch {
1251
+ throw new AgentError(
1252
+ "GITHUB_AUTH_UNAVAILABLE",
1253
+ "GitHub is connected on this host, but Uai could not prepare its credential for Git. Retry the task; if it persists, reconnect GitHub on this host.",
1254
+ );
1255
+ }
1256
+ if (sidecarMode() && cloud === null) {
1257
+ // ADR-133: project env lives under /uai/projects/<id>, readable by
1258
+ // the lease only for parameters tagged with this org, and only the
1259
+ // task's own projects are worth listing — their ids are known here,
1260
+ // not at the first grant (docs/credential-store.md).
1261
+ const { reconcileProjectEnvFromStore } = await import("../lib/user-credentials-store");
1262
+ const projectIds = input.projects.map((project) => project.id);
1263
+ const env = await reconcileProjectEnvFromStore({ projectIds }).catch((error: unknown) => {
1264
+ console.warn(
1265
+ `[project-env] task ${input.task.id}: credential store read failed: ${error instanceof Error ? error.message : String(error)}`,
965
1266
  );
966
- }
967
- if (sidecarMode()) {
968
- // ADR-133: project env lives under /uai/projects/<id>, readable by
969
- // the lease only for parameters tagged with this org, and only the
970
- // task's own projects are worth listing — their ids are known here,
971
- // not at the first grant (docs/credential-store.md).
972
- const { reconcileProjectEnvFromStore } = await import("../lib/user-credentials-store");
973
- const projectIds = input.projects.map((project) => project.id);
974
- const env = await reconcileProjectEnvFromStore({ projectIds }).catch((error: unknown) => {
975
- console.warn(
976
- `[project-env] task ${input.task.id}: credential store read failed: ${error instanceof Error ? error.message : String(error)}`,
977
- );
978
- return null;
979
- });
980
- if (env && !env.skipped) {
981
- console.log(
982
- `[project-env] task ${input.task.id}: credential store reconciled for ${projectIds.length} project(s): ${env.imported} imported, ${env.removed} removed, ${env.errors} errors`,
983
- );
984
- }
985
- }
986
- // Mint the authenticated task generation before its environment
987
- // starts. The declarations are pinned; values remain live DB reads.
988
- const gatewayRegistration = registerTaskGatewayTask(input.task.id, {
989
- resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec),
1267
+ return null;
990
1268
  });
991
- registeredTaskGatewayGeneration = gatewayRegistration.generation;
992
- const gatewaySecret = taskGatewayForwarderSecretInput(
993
- input.task.id,
994
- gatewayRegistration.generation,
995
- );
996
- if (gatewaySecret === null) {
997
- removeTaskGatewayTask(
998
- input.task.id,
999
- gatewayRegistration.generation,
1269
+ if (env && !env.skipped) {
1270
+ console.log(
1271
+ `[project-env] task ${input.task.id}: credential store reconciled for ${projectIds.length} project(s): ${env.imported} imported, ${env.removed} removed, ${env.errors} errors`,
1000
1272
  );
1001
- registeredTaskGatewayGeneration = null;
1002
- throw new Error("could not export the task gateway generation secret");
1003
1273
  }
1004
- // Launch screen: the first stage boundary. A machine-backed task
1005
- // spends the next minute launching an EC2 instance; a container
1006
- // task goes straight to preparing its workspace.
1007
- orchestrator.emitTaskStage(
1274
+ }
1275
+ // Mint the authenticated task generation before its environment
1276
+ // starts. The declarations are pinned; values remain live DB reads.
1277
+ const gatewayRegistration = registerTaskGatewayTask(input.task.id, {
1278
+ resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec, input.task.runEpoch ?? null),
1279
+ runEpoch: input.task.runEpoch ?? null,
1280
+ });
1281
+ registeredTaskGatewayGeneration = gatewayRegistration.generation;
1282
+ const gatewaySecret = taskGatewayForwarderSecretInput(
1283
+ input.task.id,
1284
+ gatewayRegistration.generation,
1285
+ );
1286
+ if (gatewaySecret === null) {
1287
+ removeTaskGatewayTask(
1008
1288
  input.task.id,
1009
- machineTasksEnabled() ? "machine" : "workspace",
1289
+ gatewayRegistration.generation,
1010
1290
  );
1011
- try {
1012
- const taskGateway = {
1013
- generation: gatewayRegistration.generation,
1014
- secret: gatewaySecret,
1015
- };
1016
- const provisioned = gitCredential
1017
- ? await gitCredential.run(() =>
1018
- provisionTaskEnvironment(
1019
- input,
1020
- {
1021
- githubCredentialSocket: gitCredential.socketPath,
1022
- taskGateway,
1023
- },
1024
- async (locator) => {
1025
- recordTaskEnvironmentPrepared(
1026
- input.task.id,
1027
- locator,
1028
- "loader",
1029
- );
1030
- },
1031
- ),
1032
- )
1033
- : await provisionTaskEnvironment(
1291
+ registeredTaskGatewayGeneration = null;
1292
+ throw new Error("could not export the task gateway generation secret");
1293
+ }
1294
+ // Launch screen: the first stage boundary. A machine-backed task
1295
+ // spends the next minute launching an EC2 instance; a container
1296
+ // task goes straight to preparing its workspace.
1297
+ orchestrator.emitTaskStage(
1298
+ input.task.id,
1299
+ machineTasksEnabled() ? "machine" : "workspace",
1300
+ );
1301
+ try {
1302
+ const taskGateway = {
1303
+ generation: gatewayRegistration.generation,
1304
+ secret: gatewaySecret,
1305
+ };
1306
+ const provisioned = gitCredential
1307
+ ? await gitCredential.run(() =>
1308
+ provisionTaskEnvironment(
1034
1309
  input,
1035
- { taskGateway },
1036
- async (locator) => {
1037
- recordTaskEnvironmentPrepared(
1038
- input.task.id,
1039
- locator,
1040
- "loader",
1041
- );
1310
+ {
1311
+ githubCredentialSocket: gitCredential.socketPath,
1312
+ taskGateway,
1042
1313
  },
1043
- );
1044
- environmentLocator = provisioned.handle.descriptor.locator;
1045
- // ADR-153 §4: the task's own schema observations, read from the
1046
- // bases task-up recorded before its container started (a machine
1047
- // captures its own). Stored once; a resume reuses them.
1048
- const envSchemas = isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)
1049
- ? undefined
1050
- : captureTaskEnvSchemas(
1051
- input.task.id,
1052
- incomingEnvSpec.projects.map((project) => project.projectId),
1053
- );
1054
- // ADR-153: before any consumer is let in, bring the task to now:
1055
- // a CA successor minted while it started, and the domains its
1056
- // policy has now rather than those task-up was handed. A failure
1057
- // leaves those keys withheld, and the trust watcher retries.
1058
- const secretsProxy = activeSecretsProxy();
1059
- if (secretsProxy !== null) {
1060
- await reconcileTaskProxyState(provisioned.handle, secretsProxy.runtime).catch((error: unknown) => {
1061
- console.warn(
1062
- `[secrets-proxy] task ${input.task.id}: could not bring the task's trust and pins up to date: ${
1063
- error instanceof Error ? error.message : String(error)
1064
- }`,
1065
- );
1066
- secretsProxy.runtime.requestTrustDelivery();
1067
- });
1068
- }
1069
- return envSchemas === undefined || envSchemas.length === 0
1070
- ? { ...provisioned.result, envMode: "loader" as const }
1071
- : { ...provisioned.result, envMode: "loader" as const, envSchemas };
1072
- } finally {
1073
- gatewaySecret.fill(0);
1074
- await gitCredential?.close().catch((error: unknown) => {
1075
- // Cleanup must not replace the actual task-up result/error. The
1076
- // credential is short-lived and close() also removes its private
1077
- // directory in a finally block; retain diagnostics for operators.
1314
+ recordPrepared,
1315
+ ),
1316
+ )
1317
+ : await provisionTaskEnvironment(
1318
+ input,
1319
+ {
1320
+ taskGateway,
1321
+ ...(cloud?.start && cloud.hold ? { leased: leasedTaskUp(cloud.start, cloud.hold) } : {}),
1322
+ },
1323
+ recordPrepared,
1324
+ );
1325
+ environmentLocator = provisioned.handle.descriptor.locator;
1326
+ // ADR-153 §4: the task's own schema observations, read from the
1327
+ // bases task-up recorded before its container started (a machine
1328
+ // captures its own). Stored once; a resume reuses them.
1329
+ const envSchemas = isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)
1330
+ ? undefined
1331
+ : captureTaskEnvSchemas(
1332
+ input.task.id,
1333
+ incomingEnvSpec.projects.map((project) => project.projectId),
1334
+ );
1335
+ // ADR-153: before any consumer is let in, bring the task to now:
1336
+ // a CA successor minted while it started, and the domains its
1337
+ // policy has now rather than those task-up was handed. A failure
1338
+ // leaves those keys withheld, and the trust watcher retries.
1339
+ const secretsProxy = activeSecretsProxy();
1340
+ if (secretsProxy !== null) {
1341
+ await reconcileTaskProxyState(provisioned.handle, secretsProxy.runtime).catch((error: unknown) => {
1078
1342
  console.warn(
1079
- `[github] task ${input.task.id}: credential cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
1343
+ `[secrets-proxy] task ${input.task.id}: could not bring the task's trust and pins up to date: ${
1344
+ error instanceof Error ? error.message : String(error)
1345
+ }`,
1080
1346
  );
1347
+ secretsProxy.runtime.requestTrustDelivery();
1081
1348
  });
1082
1349
  }
1083
- });
1084
- if (!result.ok && registeredTaskGatewayGeneration !== null) {
1085
- forgetTaskEnvSnapshot(input.task.id);
1086
- try {
1087
- removeTaskGatewayTask(
1088
- input.task.id,
1089
- registeredTaskGatewayGeneration,
1090
- );
1091
- } catch (error) {
1092
- // The in-process authority is revoked before durable deletion is
1093
- // attempted. Preserve the task-up error while retaining a cleanup
1094
- // diagnostic; a leftover file loads inactive after restart.
1350
+ return envSchemas === undefined || envSchemas.length === 0
1351
+ ? { ...provisioned.result, envMode: "loader" as const }
1352
+ : { ...provisioned.result, envMode: "loader" as const, envSchemas };
1353
+ } finally {
1354
+ gatewaySecret.fill(0);
1355
+ await gitCredential?.close().catch((error: unknown) => {
1356
+ // Cleanup must not replace the actual task-up result/error. The
1357
+ // credential is short-lived and close() also removes its private
1358
+ // directory in a finally block; retain diagnostics for operators.
1095
1359
  console.warn(
1096
- `[task-gateway] task ${input.task.id}: failed to remove the rejected generation: ${
1097
- error instanceof Error ? error.message : String(error)
1098
- }`,
1360
+ `[github] task ${input.task.id}: credential cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
1099
1361
  );
1100
- }
1362
+ });
1101
1363
  }
1102
- if (result.ok) {
1103
- orchestrator.allowChannel(input.task.id);
1104
- if (environmentLocator === undefined) {
1105
- throw new Error("task environment provision returned no locator");
1106
- }
1107
- recordTaskUpResult(
1364
+ });
1365
+ // ADR-154: the last word before the start publishes its outcome. A
1366
+ // success needs the start still the task's current one, its credentials
1367
+ // open for its run; an interrupted start reports its interruption, never
1368
+ // what it failed of. Nothing it made is destroyed here: a teardown would
1369
+ // delete a resumed task's workspace, and whoever took over acts on the
1370
+ // runtime itself.
1371
+ const interrupted = interruptedStartResult(cloud);
1372
+ const replaced = result.ok
1373
+ ? cloudStartCannotPublish(cloud)
1374
+ : interrupted
1375
+ ? { result: interrupted, interrupted: true }
1376
+ : null;
1377
+ if (replaced) result = replaced.result;
1378
+ if (!result.ok && registeredTaskGatewayGeneration !== null) {
1379
+ forgetTaskEnvSnapshot(input.task.id);
1380
+ try {
1381
+ removeTaskGatewayTask(
1108
1382
  input.task.id,
1109
- result.value,
1110
- environmentLocator,
1111
- "loader",
1383
+ registeredTaskGatewayGeneration,
1112
1384
  );
1113
- recordHostEvent(input.task.id, "task.started");
1114
- // ADR-153: with the proxy off, every start says so, since the task's
1115
- // proxied keys are withheld rather than delivered.
1116
- if (activeSecretsProxy() === null) {
1117
- orchestrator.emitSystemNote(
1118
- input.task.id,
1119
- secretsProxyEnabled()
1120
- ? "The secrets proxy could not start on this host, so this task's proxied env vars are withheld. See the host log."
1121
- : "The secrets proxy is off on this host (UAI_SECRETS_PROXY=off), so this task's proxied env vars are withheld.",
1122
- );
1123
- }
1124
- // ADR-153 §4: one note naming the keys this task does not get, and
1125
- // why, so nobody hunts for a value Uai withholds on purpose.
1126
- try {
1127
- const withheld = taskEnvWithheldNoteNow(input.task.id, incomingEnvSpec);
1128
- if (withheld !== null) orchestrator.emitSystemNote(input.task.id, withheld);
1129
- } catch (error) {
1385
+ } catch (error) {
1386
+ // The in-process authority is revoked before durable deletion is
1387
+ // attempted. Preserve the task-up error while retaining a cleanup
1388
+ // diagnostic; a leftover file loads inactive after restart.
1389
+ console.warn(
1390
+ `[task-gateway] task ${input.task.id}: failed to remove the rejected generation: ${
1391
+ error instanceof Error ? error.message : String(error)
1392
+ }`,
1393
+ );
1394
+ }
1395
+ }
1396
+ if (result.ok) {
1397
+ orchestrator.allowChannel(input.task.id);
1398
+ if (environmentLocator === undefined) {
1399
+ throw new Error("task environment provision returned no locator");
1400
+ }
1401
+ recordTaskUpResult(
1402
+ input.task.id,
1403
+ result.value,
1404
+ environmentLocator,
1405
+ "loader",
1406
+ );
1407
+ if (input.task.runEpoch !== undefined) {
1408
+ recordTaskRuntimeEstablished(
1409
+ input.task.id,
1410
+ input.task.runEpoch,
1411
+ );
1412
+ }
1413
+ recordHostEvent(input.task.id, "task.started");
1414
+ // ADR-153: with the proxy off, every start says so, since the task's
1415
+ // proxied keys are withheld rather than delivered.
1416
+ if (activeSecretsProxy() === null) {
1417
+ orchestrator.emitSystemNote(
1418
+ input.task.id,
1419
+ secretsProxyEnabled()
1420
+ ? "The secrets proxy could not start on this host, so this task's proxied env vars are withheld. See the host log."
1421
+ : "The secrets proxy is off on this host (UAI_SECRETS_PROXY=off), so this task's proxied env vars are withheld.",
1422
+ );
1423
+ }
1424
+ // ADR-153 §4: one note naming the keys this task does not get, and
1425
+ // why, so nobody hunts for a value Uai withholds on purpose.
1426
+ try {
1427
+ const withheld = taskEnvWithheldNoteNow(input.task.id, incomingEnvSpec, input.task.runEpoch ?? null);
1428
+ if (withheld !== null) orchestrator.emitSystemNote(input.task.id, withheld);
1429
+ } catch (error) {
1430
+ console.warn(
1431
+ `[project-env] task ${input.task.id}: withheld-keys note failed: ${
1432
+ error instanceof Error ? error.message : String(error)
1433
+ }`,
1434
+ );
1435
+ }
1436
+ // Degraded start (uai-init failed twice): the task is up, but say so
1437
+ // in the feed — a silent half-start reads as a broken product.
1438
+ if (result.value.initWarning) {
1439
+ orchestrator.emitSystemNote(
1440
+ input.task.id,
1441
+ result.value.initWarning,
1442
+ );
1443
+ }
1444
+ // ADR-122 phase 1: a project that declares @required env keys with
1445
+ // no value on this host gets ONE visible note now — not a blank 500
1446
+ // hours later (live 2026-08-25). Best-effort by design: a schema
1447
+ // read hiccup must never fail a completed task-up.
1448
+ void noteMissingRequiredEnvKeys(input).catch((err) =>
1449
+ console.warn(
1450
+ `[env-schema] task ${input.task.id}: validation note failed: ${
1451
+ err instanceof Error ? err.message : String(err)
1452
+ }`,
1453
+ ),
1454
+ );
1455
+ // GitHub auth for the container is best-effort (ADR-027) and runs in the
1456
+ // background — it must never block or fail task-up. Awaiting it here would
1457
+ // couple the command result to a network token-exchange: a slow/hung
1458
+ // exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
1459
+ // and mark a running task as errored. The reconciler injects + schedules
1460
+ // (or emits a system note on failure) on its own; the agents come up
1461
+ // regardless and the token lands well before the first `gh` call.
1462
+ // ADR-121: machine-backed tasks take a direct environment injection —
1463
+ // the compose reconciler would probe for an app container, find none,
1464
+ // and silently skip. Reconnect re-injection for machines is a
1465
+ // follow-up alongside the machine recovery pass.
1466
+ if (cloud !== null) {
1467
+ // ADR-154: the runtime-ready writes put the lease's GitHub sign-in
1468
+ // and SSH key in; the runner's own store has nothing to add.
1469
+ } else if (isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)) {
1470
+ void machineTaskGithubAuth(
1471
+ input.task.id,
1472
+ input.task.ownerUserId,
1473
+ ).catch((err) =>
1130
1474
  console.warn(
1131
- `[project-env] task ${input.task.id}: withheld-keys note failed: ${
1132
- error instanceof Error ? error.message : String(error)
1475
+ `[github] machine task ${input.task.id}: auth injection failed: ${
1476
+ err instanceof Error ? err.message : String(err)
1133
1477
  }`,
1134
- );
1135
- }
1136
- // Degraded start (uai-init failed twice): the task is up, but say so
1137
- // in the feed — a silent half-start reads as a broken product.
1138
- if (result.value.initWarning) {
1139
- orchestrator.emitSystemNote(
1140
- input.task.id,
1141
- result.value.initWarning,
1142
- );
1143
- }
1144
- // ADR-122 phase 1: a project that declares @required env keys with
1145
- // no value on this host gets ONE visible note now — not a blank 500
1146
- // hours later (live 2026-08-25). Best-effort by design: a schema
1147
- // read hiccup must never fail a completed task-up.
1148
- void noteMissingRequiredEnvKeys(input).catch((err) =>
1478
+ ),
1479
+ );
1480
+ } else {
1481
+ void reconcileTaskGitAuth(
1482
+ input.task.id,
1483
+ input.task.ownerUserId,
1484
+ ).catch((err) =>
1149
1485
  console.warn(
1150
- `[env-schema] task ${input.task.id}: validation note failed: ${
1486
+ `[github] task ${input.task.id}: post-start reconciliation failed: ${
1151
1487
  err instanceof Error ? err.message : String(err)
1152
1488
  }`,
1153
1489
  ),
1154
1490
  );
1155
- // GitHub auth for the container is best-effort (ADR-027) and runs in the
1156
- // background — it must never block or fail task-up. Awaiting it here would
1157
- // couple the command result to a network token-exchange: a slow/hung
1158
- // exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
1159
- // and mark a running task as errored. The reconciler injects + schedules
1160
- // (or emits a system note on failure) on its own; the agents come up
1161
- // regardless and the token lands well before the first `gh` call.
1162
- // ADR-121: machine-backed tasks take a direct environment injection —
1163
- // the compose reconciler would probe for an app container, find none,
1164
- // and silently skip. Reconnect re-injection for machines is a
1165
- // follow-up alongside the machine recovery pass.
1166
- if (isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)) {
1167
- void machineTaskGithubAuth(
1168
- input.task.id,
1169
- input.task.ownerUserId,
1170
- ).catch((err) =>
1171
- console.warn(
1172
- `[github] machine task ${input.task.id}: auth injection failed: ${
1173
- err instanceof Error ? err.message : String(err)
1174
- }`,
1175
- ),
1176
- );
1177
- } else {
1178
- void reconcileTaskGitAuth(
1179
- input.task.id,
1180
- input.task.ownerUserId,
1181
- ).catch((err) =>
1182
- console.warn(
1183
- `[github] task ${input.task.id}: post-start reconciliation failed: ${
1184
- err instanceof Error ? err.message : String(err)
1185
- }`,
1186
- ),
1187
- );
1188
- }
1189
- } else if (result.code === HostErrorCode.HostUnavailable) {
1190
- // A daemon outage is retryable infrastructure state, not proof that
1191
- // the task itself failed. Undo the optimistic local `starting` mirror
1192
- // and normally reopen delivery; a runtime already proven stopped for
1193
- // L5 recreation stays stopped/tombstoned for the cloud's retry.
1194
- if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
1195
- else deleteHostTask(input.task.id);
1196
- if (!runtimeQuarantinedForRecreate) {
1197
- orchestrator.allowChannel(input.task.id);
1198
- }
1199
- } else {
1200
- recordTaskError(input.task.id);
1201
1491
  }
1202
- return result;
1203
- });
1492
+ } else if (replaced?.interrupted) {
1493
+ // ADR-154: Stop, teardown, a newer run or shutdown took over, and the
1494
+ // channel stays closed for it. A start that recorded its runtime's
1495
+ // locator (before any of that runtime existed) keeps it exactly, so
1496
+ // whoever took over stops, destroys or starts again that runtime and
1497
+ // nothing that may exist is forgotten; its credentials are cleaned up
1498
+ // while it still runs. A start that recorded none made nothing, and
1499
+ // leaves no trace.
1500
+ if (runtimePrepared) recordTaskError(input.task.id);
1501
+ else if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
1502
+ else deleteHostTask(input.task.id);
1503
+ } else if (result.code === HostErrorCode.HostUnavailable) {
1504
+ // A daemon outage is retryable infrastructure state, not proof that
1505
+ // the task itself failed. Undo the optimistic local `starting` mirror
1506
+ // and normally reopen delivery; a runtime already proven stopped for
1507
+ // L5 recreation stays stopped/tombstoned for the cloud's retry.
1508
+ if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
1509
+ else deleteHostTask(input.task.id);
1510
+ if (!runtimeQuarantinedForRecreate) {
1511
+ orchestrator.allowChannel(input.task.id);
1512
+ }
1513
+ } else {
1514
+ recordTaskError(input.task.id);
1515
+ }
1516
+ return result;
1517
+ }));
1518
+ }
1519
+
1520
+ export const hostCommands: HostCommands = {
1521
+ // The hostId argument is cloud-side routing (host-scoped commands carry
1522
+ // their target in args, like filesOp); this process IS the target.
1523
+ async hostUpdate(ctx, _hostId) {
1524
+ logCommand(ctx, "hostUpdate");
1525
+ const pinned = sidecarRunnerIsPinned();
1526
+ if (pinned) return pinned;
1527
+ try {
1528
+ const { updateManagedRuntime } = await import("../lib/managed-runtime");
1529
+ const outcome = await updateManagedRuntime();
1530
+ if (outcome.status === "not-managed") {
1531
+ return {
1532
+ ok: false,
1533
+ code: HostErrorCode.Internal,
1534
+ message:
1535
+ "this host is not a managed install — update it where it runs",
1536
+ retryable: false,
1537
+ };
1538
+ }
1539
+ if (outcome.status === "deferred") {
1540
+ // A rollback hold is an operator verdict that the newer release is
1541
+ // unsafe on this host; a remote click must not override it.
1542
+ return {
1543
+ ok: true,
1544
+ value: { version: outcome.version, staged: false, deferred: true },
1545
+ };
1546
+ }
1547
+ if (outcome.status === "current") {
1548
+ return { ok: true, value: { version: outcome.version, staged: false } };
1549
+ }
1550
+ managedHostRestartHook?.();
1551
+ return { ok: true, value: { version: outcome.version, staged: true } };
1552
+ } catch (error) {
1553
+ return {
1554
+ ok: false,
1555
+ code: HostErrorCode.Internal,
1556
+ message: `update failed: ${
1557
+ error instanceof Error ? error.message : "unknown error"
1558
+ }`,
1559
+ retryable: true,
1560
+ };
1561
+ }
1562
+ },
1563
+ async hostRestart(ctx, _hostId) {
1564
+ logCommand(ctx, "hostRestart");
1565
+ const pinned = sidecarRunnerIsPinned();
1566
+ if (pinned) return pinned;
1567
+ if (!managedHostRestartHook) {
1568
+ return {
1569
+ ok: false,
1570
+ code: HostErrorCode.Internal,
1571
+ message: "restart is unavailable in this process",
1572
+ retryable: false,
1573
+ };
1574
+ }
1575
+ // The result frame flushes before exit: shutdown drains in-flight
1576
+ // commands (this one included) before the process terminates.
1577
+ managedHostRestartHook();
1578
+ return { ok: true, value: undefined };
1579
+ },
1580
+ async taskUp(ctx, input) {
1581
+ logCommand(ctx, "taskUp", input.task.id);
1582
+ // ADR-154: on cloud credentials every start goes through the composer,
1583
+ // and none starts while the runner's activation is pending.
1584
+ const mode = credentialsMode();
1585
+ if (mode === "pending") return switchingToCloudCredentials();
1586
+ // On the runner's own credentials, a counted permit from here to the
1587
+ // outcome: leaving them waits for it.
1588
+ let legacy: LegacyStartPermit | null = null;
1589
+ if (mode === "legacy") {
1590
+ legacy = admitLegacyStart();
1591
+ if (legacy === null) return switchingToCloudCredentials();
1592
+ }
1593
+ let cloud: CloudCredentialTaskStart | null = null;
1594
+ if (mode === "active") {
1595
+ const runtime = cloudCredentialRuntime();
1596
+ if (runtime === null || input.task.runEpoch === undefined) {
1597
+ return {
1598
+ ok: false,
1599
+ code: HostErrorCode.Internal,
1600
+ message: "This host runs tasks only on cloud credentials, and could not start this one on them.",
1601
+ retryable: false,
1602
+ };
1603
+ }
1604
+ cloud = { runtime, assignment: { taskId: input.task.id, runEpoch: input.task.runEpoch }, start: null, hold: null };
1605
+ // A newer run's start ends an older one's before it queues behind it.
1606
+ runtime.interruptStarts(input.task.id, { below: input.task.runEpoch });
1607
+ }
1608
+ let outcome: TaskUpOutcome | undefined;
1609
+ try {
1610
+ outcome = await taskUpOn(ctx, input, cloud, legacy);
1611
+ return outcome;
1612
+ } finally {
1613
+ legacy?.release();
1614
+ // Its hold already let go with the long slot: the cleanup an abandon
1615
+ // runs never waits on it.
1616
+ if (cloud?.start) {
1617
+ if (outcome?.ok === true) cloud.runtime.endStart(cloud.start);
1618
+ else await cloud.runtime.abandonStart(cloud.start, cloud.hold);
1619
+ }
1620
+ }
1204
1621
  },
1205
1622
 
1206
1623
  async taskDown(ctx, input) {
1207
1624
  logCommand(ctx, "taskDown", input.taskId);
1208
1625
  const orphanGc = input.cleanup?.kind === "orphan-gc";
1626
+ // ADR-154: teardown ends its run's start (orphan GC: every start of the
1627
+ // task) before it queues behind it.
1628
+ const interrupting = activeCloudCredentials();
1629
+ if (interrupting) {
1630
+ if (orphanGc) interrupting.interruptStarts(input.taskId);
1631
+ else if (input.task.runEpoch !== undefined) interrupting.interruptStarts(input.taskId, { exact: input.task.runEpoch });
1632
+ }
1209
1633
  // A GC command is already authoritative evidence that this cloud task is
1210
1634
  // gone. Enter the lifecycle slot and tombstone its channel even when the
1211
1635
  // runtime is temporarily unavailable; the durable cloud intent will retry
@@ -1237,6 +1661,40 @@ export const hostCommands: HostCommands = {
1237
1661
  } else {
1238
1662
  environment = null;
1239
1663
  }
1664
+ // Close the exact v1 assignment before any terminal side effect. A
1665
+ // delayed teardown for an older run must never tombstone or destroy the
1666
+ // newer run. Orphan GC is authoritative for the task id and therefore
1667
+ // closes whichever assignment still survives locally.
1668
+ let v1AssignmentClosed = false;
1669
+ let closedRunEpoch: number | null = null;
1670
+ const assignmentState = readTaskRunEpochState(input.taskId);
1671
+ if (assignmentState !== null || input.task.runEpoch !== undefined) {
1672
+ const requestedRunEpoch = orphanGc
1673
+ ? assignmentState?.highestRunEpoch
1674
+ : input.task.runEpoch;
1675
+ if (assignmentState === null || requestedRunEpoch === undefined) {
1676
+ return {
1677
+ ok: false,
1678
+ code: HostErrorCode.TaskAlreadyRunning,
1679
+ message:
1680
+ `task ${input.taskId} teardown does not name its current run`,
1681
+ };
1682
+ }
1683
+ const closed = closeTaskRunAssignment(
1684
+ input.taskId,
1685
+ requestedRunEpoch,
1686
+ );
1687
+ if (closed.kind === "missing" || closed.kind === "mismatch") {
1688
+ return {
1689
+ ok: false,
1690
+ code: HostErrorCode.TaskAlreadyRunning,
1691
+ message:
1692
+ `task ${input.taskId} run ${String(requestedRunEpoch)} is no longer current`,
1693
+ };
1694
+ }
1695
+ v1AssignmentClosed = true;
1696
+ closedRunEpoch = requestedRunEpoch;
1697
+ }
1240
1698
  // Tombstone + close before container destruction. A concurrent cloud
1241
1699
  // ensure cannot reopen the channel while teardown is awaiting Docker.
1242
1700
  const suspendedTaskGateway = suspendTaskGatewayTask(input.taskId);
@@ -1269,6 +1727,17 @@ export const hostCommands: HostCommands = {
1269
1727
  }`,
1270
1728
  );
1271
1729
  }
1730
+ // ADR-154 §6: the closed run's credentials cleaned up, every path
1731
+ // proven gone (or the runtime destroyed), before the runtime goes; by
1732
+ // the composer as it is now, in the slot.
1733
+ const credentials = activeCloudCredentials();
1734
+ if (credentials && closedRunEpoch !== null) {
1735
+ try {
1736
+ await credentials.closeAssignment({ taskId: input.taskId, runEpoch: closedRunEpoch }, "teardown");
1737
+ } catch (error) {
1738
+ return failFromUnknown(error);
1739
+ }
1740
+ }
1272
1741
  const result = await wrapAgent(ctx, "taskDown", async () => {
1273
1742
  if (environment === null) return agent.taskDown(input);
1274
1743
  // taskDown is TERMINAL — its only cloud callers are task close and
@@ -1341,7 +1810,7 @@ export const hostCommands: HostCommands = {
1341
1810
  } else {
1342
1811
  // Destruction failed; let a subsequent ensure restore the still-running
1343
1812
  // task instead of leaving it permanently tombstoned.
1344
- if (!orphanGc) {
1813
+ if (!orphanGc && !v1AssignmentClosed) {
1345
1814
  if (suspendedTaskGateway) {
1346
1815
  reactivateTaskGatewayTask(suspendedTaskGateway);
1347
1816
  }
@@ -1354,22 +1823,55 @@ export const hostCommands: HostCommands = {
1354
1823
 
1355
1824
  async taskStop(ctx, taskId, input) {
1356
1825
  logCommand(ctx, "taskStop", taskId, input.reason);
1357
- // ADR-125: idle stops are the cloud's judgment from what it can see;
1358
- // only the host can see a turn that is mid-flight RIGHT NOW. Manual
1359
- // pause carries no veto — a human's explicit gesture wins.
1360
- if (input.reason === "idle") {
1361
- const activeAt = getOrchestrator().lastChannelActivity(taskId);
1362
- if (activeAt !== null && Date.now() - activeAt < IDLE_STOP_VETO_MS) {
1363
- return { ok: true, value: { vetoed: "active" as const } };
1364
- }
1826
+ const orchestrator = getOrchestrator();
1827
+ // ADR-154: a manual Stop ends its run's start before it queues behind
1828
+ // it; an idle Stop never does, since a start vetoes it in the slot.
1829
+ const interrupting = activeCloudCredentials();
1830
+ if (interrupting && input.reason !== "idle" && input.runEpoch !== undefined) {
1831
+ interrupting.interruptStarts(taskId, { exact: input.runEpoch });
1832
+ }
1833
+ const result = await orchestrator.stopTask(taskId, {
1834
+ // ADR-154: the run Stop closes, cleaned up by the composer as it is
1835
+ // when Stop gets there, in the slot.
1836
+ closeCredentials: (runEpoch: number) =>
1837
+ activeCloudCredentials()?.closeAssignment({ taskId, runEpoch }, "stop") ?? Promise.resolve(),
1838
+ // Resolve legacy versus v1 only after Stop owns the task lifecycle slot.
1839
+ // Otherwise task-up can admit v1 between an outside read and the stop.
1840
+ admitAssignment: () => {
1841
+ if (input.runEpoch === undefined) {
1842
+ return readTaskRunEpochState(taskId) === null
1843
+ ? { kind: "legacy" }
1844
+ : { kind: "stale" };
1845
+ }
1846
+ const closed = closeTaskRunAssignmentForStop(taskId, input.runEpoch);
1847
+ return closed.kind === "closed" || closed.kind === "already_closed"
1848
+ ? { kind: "closed", runEpoch: input.runEpoch }
1849
+ : { kind: "stale" };
1850
+ },
1851
+ // ADR-125: idle stops are the cloud's judgment from what it can see;
1852
+ // only the host can see a turn that is mid-flight RIGHT NOW. Evaluate
1853
+ // immediately before assignment closure. Manual pause has no veto.
1854
+ shouldVeto:
1855
+ input.reason === "idle"
1856
+ ? () => {
1857
+ const activeAt = orchestrator.lastChannelActivity(taskId);
1858
+ return (
1859
+ activeAt !== null && Date.now() - activeAt < IDLE_STOP_VETO_MS
1860
+ );
1861
+ }
1862
+ : undefined,
1863
+ });
1864
+ if (result.ok && result.vetoed) {
1865
+ return { ok: true, value: { vetoed: "active" as const } };
1365
1866
  }
1366
- const result = await getOrchestrator().stopTask(taskId);
1367
1867
  if (!result.ok) {
1368
1868
  return {
1369
1869
  ok: false,
1370
- code: HostErrorCode.Internal,
1870
+ code: result.assignmentStale
1871
+ ? HostErrorCode.TaskAlreadyRunning
1872
+ : HostErrorCode.Internal,
1371
1873
  message: result.error ?? "stop failed",
1372
- retryable: true,
1874
+ ...(result.assignmentStale ? {} : { retryable: true }),
1373
1875
  };
1374
1876
  }
1375
1877
  return { ok: true, value: {} };
@@ -1635,15 +2137,19 @@ export const hostCommands: HostCommands = {
1635
2137
  if (runtimeFailure) return runtimeFailure;
1636
2138
  try {
1637
2139
  const task = getHostTask(input.taskId);
1638
- // A stopped/terminal task retains its host worktree, so preserve the
1639
- // historical host-side fallback there. A running task must reconstruct
1640
- // its exact persisted provider identity: malformed/foreign metadata is
1641
- // not permission to run Git against a different filesystem view.
1642
- const environment =
1643
- task?.statusMirror === "running"
1644
- ? await reconstructPersistedTaskEnvironment(task)
1645
- : null;
1646
- if (task?.statusMirror === "running" && environment === null) {
2140
+ // #cb3j0z: Git never runs on the host against a task's repositories;
2141
+ // what a task configured there would run as the runner's user. A
2142
+ // running task must reconstruct its exact persisted provider identity
2143
+ // (malformed/foreign metadata is not permission to run Git against a
2144
+ // different filesystem view) and diffs there; a stopped one diffs in
2145
+ // a throwaway sandbox of its retained worktree.
2146
+ if (task?.statusMirror !== "running") {
2147
+ return ok(
2148
+ await stoppedTaskDiff(input, task ? { environmentProvider: task.environmentProvider ?? null } : null),
2149
+ );
2150
+ }
2151
+ const environment = await reconstructPersistedTaskEnvironment(task);
2152
+ if (environment === null) {
1647
2153
  throw new Error(
1648
2154
  `running task ${input.taskId} has no persisted task environment`,
1649
2155
  );
@@ -1671,7 +2177,7 @@ export const hostCommands: HostCommands = {
1671
2177
  );
1672
2178
  return ok(undefined);
1673
2179
  }
1674
- writeAttachment(
2180
+ await writeAttachment(
1675
2181
  input.taskId,
1676
2182
  input.filename,
1677
2183
  Buffer.from(input.dataBase64, "base64"),
@@ -1690,7 +2196,7 @@ export const hostCommands: HostCommands = {
1690
2196
  ? await environment.readWorkspaceFile(
1691
2197
  attachmentEnvironmentPath(environment, input.filename),
1692
2198
  )
1693
- : readAttachment(input.taskId, input.filename);
2199
+ : await readAttachment(input.taskId, input.filename);
1694
2200
  if (!bytes) {
1695
2201
  return {
1696
2202
  ok: false,
@@ -1780,22 +2286,27 @@ function normalizeChannelSpec(input: ChannelEnsureInput): ChannelEnsureInput {
1780
2286
  * `setup-git` gesture as containers, no compose probing). Best-effort like
1781
2287
  * the compose reconciler: absence of a GitHub connection is a quiet no-op.
1782
2288
  */
2289
+ /** ADR-154: only while the runner is on its own credentials. Leaving them
2290
+ * waits for one under way, and the write is fenced on them: none starts
2291
+ * once leaving has begun. */
1783
2292
  async function machineTaskGithubAuth(
1784
2293
  taskId: string,
1785
2294
  userId: string,
1786
2295
  ): Promise<void> {
1787
- const environment = await reconstructHostTaskEnvironment(taskId);
1788
- if (!environment) return;
1789
- const token = await requestAccessToken(userId);
1790
- if (!token) return;
1791
- // Machine work is not container work: no docker/apple runtime admission.
1792
- await injectIntoContainer(
1793
- taskId,
1794
- token.accessToken,
1795
- undefined,
1796
- () => {},
1797
- environment,
1798
- );
2296
+ await whileOnOwnCredentials(async () => {
2297
+ const environment = await reconstructHostTaskEnvironment(taskId);
2298
+ if (!environment) return;
2299
+ const token = await requestAccessToken(userId);
2300
+ if (!token || !onOwnCredentials()) return;
2301
+ // Machine work is not container work: no docker/apple runtime admission.
2302
+ await injectIntoContainer(
2303
+ taskId,
2304
+ token.accessToken,
2305
+ undefined,
2306
+ () => {},
2307
+ environment,
2308
+ );
2309
+ });
1799
2310
  }
1800
2311
 
1801
2312
  async function reconstructHostTaskEnvironment(