@runuai/host 0.9.152 → 0.9.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ -- ADR-154 Files v1: persist the cloud-issued mode for crash recovery. The
2
+ -- task-owned `.uai/files-mounted` marker is an observation, never authority.
3
+ ALTER TABLE `uai_host_tasks` ADD `shared_files_mode` text
4
+ CONSTRAINT `uai_host_tasks_shared_files_mode`
5
+ CHECK (`shared_files_mode` IS NULL OR `shared_files_mode` IN ('off', 'ro', 'rw'));
@@ -204,6 +204,13 @@
204
204
  "when": 1791331200001,
205
205
  "tag": "0028_leased_engine_runtimes",
206
206
  "breakpoints": true
207
+ },
208
+ {
209
+ "idx": 29,
210
+ "version": "6",
211
+ "when": 1791590400000,
212
+ "tag": "0029_task_shared_files_mode",
213
+ "breakpoints": true
207
214
  }
208
215
  ]
209
216
  }
package/db/schema.ts CHANGED
@@ -39,6 +39,12 @@ export const hostTasks = sqliteTable(
39
39
  // Value-free resolver metadata for rebuilding the project-env gateway
40
40
  // after a runner restart, before the cloud re-sends task-up.
41
41
  projectEnvSpec: text("project_env_spec"),
42
+ // ADR-154 Files v1: the normalized mode from the cloud's task launch.
43
+ // Recovery must never trust the task-writable `.uai/files-mounted`
44
+ // marker as authority for what may be mounted.
45
+ sharedFilesMode: text("shared_files_mode", {
46
+ enum: ["off", "ro", "rw"],
47
+ }),
42
48
  statusMirror: text("status_mirror"),
43
49
  // Owner + project slugs — received in TaskLaunchInput at task-up and
44
50
  // persisted here so the local UI (ADR-028) can show "who/what" without a
@@ -70,6 +76,10 @@ export const hostTasks = sqliteTable(
70
76
  "uai_host_tasks_environment_pair",
71
77
  sql`(${table.environmentProvider} IS NULL) = (${table.environmentLocator} IS NULL)`,
72
78
  ),
79
+ sharedFilesModeCheck: check(
80
+ "uai_host_tasks_shared_files_mode",
81
+ sql`${table.sharedFilesMode} IS NULL OR ${table.sharedFilesMode} IN ('off', 'ro', 'rw')`,
82
+ ),
73
83
  }),
74
84
  );
75
85
 
package/lib/agent.ts CHANGED
@@ -68,6 +68,10 @@ export interface LeasedTaskUp {
68
68
  * the start.
69
69
  */
70
70
  runtimeReady: (environment: TaskWriteEnvironment, signal: AbortSignal) => Promise<void>;
71
+ /** This exact start carries a task-machine Files session. Machine task-up
72
+ * skips its legacy copy/mount path only in this case; owned leased tasks do
73
+ * not receive a Files session and keep their normal Files behavior. */
74
+ filesV1?: true;
71
75
  }
72
76
 
73
77
  /** Refuses a start on cloud credentials that is not whole, or not this
@@ -1390,11 +1390,16 @@ export async function setupBrowserTesting(
1390
1390
  }
1391
1391
  }
1392
1392
 
1393
- // Grok registers its servers through its own CLI (project scope, see
1394
- // GROK_BROWSER_ADD_CMD) rather than a file we own; a failure here leaves
1395
- // the task configured-false so the next ensure retries.
1393
+ // Grok registers through its own read-modify-write CLI (user scope,
1394
+ // see grokBrowserAddCommand), under the same lock as gateway reconciliation
1395
+ // so two successful CLI processes cannot silently lose each other's entry.
1396
+ // A failure leaves the task configured-false so the next ensure retries.
1396
1397
  if (hasGrok) {
1397
1398
  const grokArgv = [
1399
+ "flock",
1400
+ "-w",
1401
+ "10",
1402
+ MCP_CONFIG_LOCK_PATH,
1398
1403
  "sh",
1399
1404
  "-lc",
1400
1405
  selectedGrokHomes.map(grokBrowserAddCommand).join(" && "),
@@ -257,6 +257,12 @@ export interface TaskCredentialLeaseRegistryOptions {
257
257
  assignment: Readonly<TaskAssignment>,
258
258
  view: TaskCredentialView,
259
259
  ): void;
260
+ /** A material update committed while the existing gate stayed open.
261
+ * Followers may reconcile runtime files from this authoritative view. */
262
+ credentialMaterialChanged?(
263
+ assignment: Readonly<TaskAssignment>,
264
+ view: TaskCredentialView,
265
+ ): void;
260
266
  cleanupExternal(request: CredentialCleanupRequest): Promise<void>;
261
267
  /**
262
268
  * Installs files/proxy/engine consumers transactionally. The env-replica
@@ -1126,7 +1132,10 @@ export class TaskCredentialLeaseRegistry {
1126
1132
  await this.#applyCredentialVersion(state, frame);
1127
1133
  } catch (error) {
1128
1134
  this.#report(error, state.assignment);
1129
- if (this.#isActive(state)) this.#request(state);
1135
+ // A failed consumer install may already have completed full cleanup
1136
+ // and armed the bounded retry which owns what comes next. Do not cancel
1137
+ // that backoff by immediately requesting the same refused manifest.
1138
+ if (this.#isActive(state) && !state.retryTimer) this.#request(state);
1130
1139
  }
1131
1140
  }
1132
1141
 
@@ -1226,7 +1235,7 @@ export class TaskCredentialLeaseRegistry {
1226
1235
  });
1227
1236
  } catch (error) {
1228
1237
  this.#report(error, state.assignment);
1229
- this.#request(state);
1238
+ if (this.#isActive(state) && !state.retryTimer) this.#request(state);
1230
1239
  }
1231
1240
  } else if (frame.outcome === "needs_sign_in") {
1232
1241
  // The authoritative stale frame supplies any floor and removals.
@@ -2276,6 +2285,9 @@ export class TaskCredentialLeaseRegistry {
2276
2285
  state.bundle = next;
2277
2286
  state.bundleGeneration++;
2278
2287
  this.#armBundleTimers(state);
2288
+ if (!refused) {
2289
+ this.#opts.credentialMaterialChanged?.(cloneAssignment(state.assignment), next.view);
2290
+ }
2279
2291
  // A refusal opens through its release, inside this boundary: an open
2280
2292
  // that fails, its compensating close included, escalates below.
2281
2293
  release();
@@ -2452,8 +2464,16 @@ export class TaskCredentialLeaseRegistry {
2452
2464
  // A permanent revocation answers every request with another denial.
2453
2465
  // Keep asking, but do not turn that into a tight
2454
2466
  // deny/cleanup/request loop against the cloud.
2455
- if (reason === "denied") this.#armRetry(state);
2456
- else this.#request(state);
2467
+ // A denial is stable until cloud state changes, and a consumer which
2468
+ // refused a grant can keep refusing the same manifest (for example
2469
+ // while a task-owned workspace entry is being repaired). Neither is
2470
+ // allowed to become a cleanup/request loop which repeatedly mints STS
2471
+ // sessions and reads Parameter Store without delay.
2472
+ if (reason === "denied" || reason === "manifest-narrowed") {
2473
+ this.#armRetry(state);
2474
+ } else {
2475
+ this.#request(state);
2476
+ }
2457
2477
  }
2458
2478
  }
2459
2479
  }
@@ -63,9 +63,11 @@ import {
63
63
  cleanupPersonCredentials,
64
64
  closePersonCredentialGate,
65
65
  drainPersonCredentialWrites,
66
+ holdPersonCredentialGate,
66
67
  holdTaskGitCredentials,
67
68
  installPersonCredentials,
68
69
  openPersonCredentialGate,
70
+ type PersonCredentialLane,
69
71
  type TaskGitCredentialHold,
70
72
  } from "./credential-consumers/github";
71
73
  import {
@@ -79,6 +81,8 @@ import {
79
81
  setMcpCredentialRefresher,
80
82
  } from "./credential-consumers/mcp";
81
83
  import type { ProjectEnvTaskReplicaManager, ProjectEnvTaskReplicaSnapshot } from "./project-env-task-replica";
84
+ import type { SharedFilesMode } from "./shared-files-s3";
85
+ import type { TaskEnvironmentHandle } from "./task-environment/types";
82
86
  import {
83
87
  TaskCredentialCleanupCoordinator,
84
88
  type TaskCredentialCleanupDeps,
@@ -95,6 +99,17 @@ const ENV_READY_POLL_MS = 50;
95
99
  * way; the next process's recovery sweep finishes what is left. */
96
100
  export const SHUTDOWN_DRAIN_MS = 30_000;
97
101
 
102
+ /** The only MCP state an agent config exposes. Material, provider URL,
103
+ * revision and availability stay behind the task gateway, so changing any of
104
+ * them must not churn a long-lived agent process. */
105
+ function mcpAgentMembershipFingerprint(view: TaskCredentialView): string {
106
+ return JSON.stringify(
107
+ view.mcp
108
+ .map((connection) => connection.manifest.slug)
109
+ .sort((left, right) => left.localeCompare(right)),
110
+ );
111
+ }
112
+
98
113
  /** The task's Files (ADR-154 §12): installed, opened and closed with the
99
114
  * rest. `close` is synchronous: no new Files admission after it. */
100
115
  export interface TaskFilesConsumer {
@@ -102,6 +117,32 @@ export interface TaskFilesConsumer {
102
117
  open(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void;
103
118
  close(assignment: Readonly<TaskAssignment>): void;
104
119
  closeConsumers(request: TaskCredentialCleanupRequest): Promise<void>;
120
+ /** A trusted task-machine runtime could not establish its Files boundary.
121
+ * This run cannot be reopened on the same machine. */
122
+ retire(assignment: Readonly<TaskAssignment>, error: unknown): void;
123
+ /** Optional only so owned runners can use the same composer without a
124
+ * task-machine Files implementation. */
125
+ runtimeReady?(
126
+ assignment: Readonly<TaskAssignment>,
127
+ environment: TaskEnvironmentHandle<unknown>,
128
+ mode: SharedFilesMode,
129
+ signal: AbortSignal,
130
+ ): Promise<void>;
131
+ /** Drop secrets at process shutdown without starting cleanup. */
132
+ shutdown?(): void;
133
+ }
134
+
135
+ /** A Files implementation proved that the runtime-ready operation which
136
+ * failed still owns the exact open run and requested terminal retirement of
137
+ * its one-task machine; a stale operation throws its ordinary error. */
138
+ export class AuthoritativeTaskFilesFailure extends Error {
139
+ override readonly cause: unknown;
140
+
141
+ constructor(cause: unknown) {
142
+ super(cause instanceof Error ? cause.message : "the task's Files boundary failed");
143
+ this.name = "AuthoritativeTaskFilesFailure";
144
+ this.cause = cause;
145
+ }
105
146
  }
106
147
 
107
148
  export interface CloudCredentialRuntimeDeps {
@@ -121,18 +162,90 @@ export interface CloudCredentialRuntimeDeps {
121
162
  * on a grant's snapshot, or closed): whoever follows the task's env
122
163
  * (the proxy's trust delivery) looks again. */
123
164
  envServedChanged?(taskId: string): void;
165
+ /** Reconcile the owner's GitHub/SSH files in an already-running task.
166
+ * The signal closes with the exact person-credential gate, and the caller
167
+ * retries a transient rejection while that gate and material stay current. */
168
+ personCredentialsReady?(
169
+ assignment: Readonly<TaskAssignment>,
170
+ lanes: readonly PersonCredentialLane[],
171
+ signal: AbortSignal,
172
+ ): Promise<void>;
124
173
  files?: TaskFilesConsumer | null;
125
174
  launchDeadlineMs?: number;
126
175
  envReadyTimeoutMs?: number;
127
176
  shutdownDrainMs?: number;
128
177
  /** The first retry's delay for a recovered start whose startup failed. */
129
178
  recoveryRetryBaseMs?: number;
179
+ /** The first retry's delay for task-runtime person-credential writes. */
180
+ personReconcileRetryBaseMs?: number;
130
181
  registry?: Partial<
131
182
  Pick<TaskCredentialLeaseRegistryOptions, "createSessionReader" | "now" | "setTimer" | "clearTimer" | "randomId" | "onError">
132
183
  >;
133
184
  cleanup?: Pick<TaskCredentialCleanupDeps, "db" | "removePath" | "destroyRuntime">;
134
185
  }
135
186
 
187
+ interface PersonCredentialStamp {
188
+ /** Stable without retaining another copy of any credential material. */
189
+ identity: string;
190
+ available: boolean;
191
+ }
192
+
193
+ interface PersonCredentialSnapshot {
194
+ runEpoch: number;
195
+ github: PersonCredentialStamp;
196
+ ssh: PersonCredentialStamp;
197
+ }
198
+
199
+ function personCredentialStamp(
200
+ credential: TaskCredentialView["github"] | TaskCredentialView["ssh"],
201
+ ): PersonCredentialStamp {
202
+ if (credential === null) return { identity: "absent", available: false };
203
+ return {
204
+ identity: [
205
+ credential.manifest.credentialId,
206
+ credential.manifest.revision,
207
+ credential.availability,
208
+ credential.observedVersion ?? "unreadable",
209
+ ].join("\u0000"),
210
+ available: credential.availability === "available",
211
+ };
212
+ }
213
+
214
+ function personCredentialSnapshot(view: TaskCredentialView): PersonCredentialSnapshot {
215
+ return {
216
+ runEpoch: view.assignment.runEpoch,
217
+ github: personCredentialStamp(view.github),
218
+ ssh: personCredentialStamp(view.ssh),
219
+ };
220
+ }
221
+
222
+ function samePersonCredentialSnapshot(
223
+ left: PersonCredentialSnapshot | undefined,
224
+ right: PersonCredentialSnapshot,
225
+ ): boolean {
226
+ return Boolean(
227
+ left &&
228
+ left.runEpoch === right.runEpoch &&
229
+ left.github.identity === right.github.identity &&
230
+ left.ssh.identity === right.ssh.identity,
231
+ );
232
+ }
233
+
234
+ /** What must be rewritten for `next`. GitHub and SSH transport are coupled:
235
+ * when GitHub leaves, the surviving SSH key must select SSH again. */
236
+ function changedPersonCredentialLanes(
237
+ previous: PersonCredentialSnapshot | undefined,
238
+ next: PersonCredentialSnapshot,
239
+ ): PersonCredentialLane[] {
240
+ if (samePersonCredentialSnapshot(previous, next)) return [];
241
+ // The pair shares Git transport configuration. Reconcile both available
242
+ // lanes, including a write interrupted by the other lane changing.
243
+ const lanes: PersonCredentialLane[] = [];
244
+ if (next.github.available) lanes.push("github");
245
+ if (next.ssh.available) lanes.push("ssh");
246
+ return lanes;
247
+ }
248
+
136
249
  /**
137
250
  * Why a start was interrupted, as its signal's reason: Stop, teardown or a
138
251
  * newer run took it over, or the runner is shutting down. The first
@@ -266,6 +379,16 @@ export class CloudCredentialRuntime {
266
379
  /** Work accepted and not yet over (startups, frames, installs, holds not
267
380
  * yet released, closes): what shutdown joins. */
268
381
  readonly #work = new Set<Promise<unknown>>();
382
+ /** Last agent-facing MCP membership published for each task. Kept across a
383
+ * gate close/renewal because the config remains in the task runtime. */
384
+ readonly #publishedMcpMembership = new Map<string, string>();
385
+ /** Person material last opened or installed for each task. Only identities,
386
+ * availability and observed versions: never another copy of a secret. */
387
+ readonly #personCredentialSnapshots = new Map<string, PersonCredentialSnapshot>();
388
+ readonly #personReconciles = new Map<string, { runEpoch: number; controller: AbortController }>();
389
+ /** Stop fences work before it can acquire the lifecycle slot or close the
390
+ * lease. A late grant cannot restart writes in that interval. */
391
+ readonly #personInterrupted = new Map<string, number>();
269
392
  readonly #setTimer: typeof setTimeout;
270
393
  readonly #clearTimer: typeof clearTimeout;
271
394
  #shutDown = false;
@@ -295,6 +418,7 @@ export class CloudCredentialRuntime {
295
418
  send: (frame) => deps.send(frame),
296
419
  closeTaskGate: (assignment) => this.#close(assignment),
297
420
  openTaskGate: (assignment, view) => this.#openGate(assignment, view),
421
+ credentialMaterialChanged: (assignment, view) => this.#personCredentialsChanged(assignment, view),
298
422
  installExternal: (request) => this.#track(this.#install(request)),
299
423
  // From a grant's acceptance until its own reader is active, the env
300
424
  // replica refuses the run's frames for the cloud to retry: none is
@@ -337,6 +461,7 @@ export class CloudCredentialRuntime {
337
461
  throw new CredentialLeaseUnavailableError("a newer run of this task has begun");
338
462
  }
339
463
  this.#floors.set(assignment.taskId, assignment.runEpoch);
464
+ if (!options.recovery) this.#personInterrupted.delete(assignment.taskId);
340
465
  this.#interruptTokens(assignment.taskId, {});
341
466
  // A newer run's start ends an older run's recovery; an ordinary start of
342
467
  // the run being recovered takes its startup over.
@@ -376,6 +501,11 @@ export class CloudCredentialRuntime {
376
501
  */
377
502
  interruptStarts(taskId: string, which: { exact?: number; below?: number } = {}): void {
378
503
  this.#interruptTokens(taskId, which);
504
+ const assignment = this.#assignments.get(taskId);
505
+ if (assignment && (which.exact === undefined || assignment.runEpoch === which.exact) &&
506
+ (which.below === undefined || assignment.runEpoch < which.below)) {
507
+ this.#interruptPersonCredentials(assignment);
508
+ }
379
509
  // A recovery of the run ends with it, ended start token or not.
380
510
  const recovered = this.#recovered.get(taskId);
381
511
  if (
@@ -399,6 +529,8 @@ export class CloudCredentialRuntime {
399
529
  /** A local Stop: the task's recovery, whatever its run, ends now. One
400
530
  * only exists for the run this runner owned at its start. */
401
531
  interruptRecovery(taskId: string): void {
532
+ const assignment = this.#assignments.get(taskId);
533
+ if (assignment) this.#interruptPersonCredentials(assignment);
402
534
  const recovered = this.#recovered.get(taskId);
403
535
  if (recovered) this.#endRecovery(taskId, recovered.runEpoch);
404
536
  }
@@ -500,6 +632,69 @@ export class CloudCredentialRuntime {
500
632
  };
501
633
  }
502
634
 
635
+ /** Whether this process has the task-machine Files-v1 consumer wired. */
636
+ taskFilesV1(): boolean {
637
+ return typeof this.#deps.files?.runtimeReady === "function";
638
+ }
639
+
640
+ /** Runtime-ready half of Files v1. It is tracked with the composer's
641
+ * accepted work so shutdown joins a write/mount already admitted. */
642
+ prepareTaskFiles(
643
+ assignment: Readonly<TaskAssignment>,
644
+ environment: TaskEnvironmentHandle<unknown>,
645
+ mode: SharedFilesMode,
646
+ signal: AbortSignal,
647
+ ): Promise<void> {
648
+ const ready = this.#deps.files?.runtimeReady;
649
+ if (!ready) return Promise.reject(new Error("task-machine Files v1 is unavailable"));
650
+ return this.#track(
651
+ ready.call(this.#deps.files, assignment, environment, mode, signal).catch((error: unknown) => {
652
+ if (error instanceof AuthoritativeTaskFilesFailure) {
653
+ // Never reopen this run on the same machine. The consumer already
654
+ // proved this is authoritative; centrally latch/request exact-run
655
+ // terminal retirement too, so an implementation cannot turn this
656
+ // into an ordinary cleanup/regrant by omission.
657
+ let retirementError: unknown = null;
658
+ try {
659
+ this.#deps.files!.retire(assignment, error.cause);
660
+ } catch (failure) {
661
+ retirementError = failure;
662
+ }
663
+ void this.closeAssignment({ ...assignment }, "stop").catch(() => {});
664
+ if (retirementError !== null) throw retirementError;
665
+ throw error.cause;
666
+ }
667
+ throw error;
668
+ }),
669
+ );
670
+ }
671
+
672
+ /** A recovered task whose trusted Files metadata is absent or invalid has
673
+ * failed the same boundary before the consumer can be called. Request its
674
+ * terminal machine retirement, close its gate synchronously, and run the
675
+ * run's full cleanup. */
676
+ failTaskFilesSetup(
677
+ assignment: Readonly<TaskAssignment>,
678
+ error: unknown = new Error("the recovered task has no trusted Files-v1 metadata"),
679
+ ): void {
680
+ try {
681
+ this.#deps.files?.retire(assignment, error);
682
+ } catch (retirementError) {
683
+ console.warn(
684
+ `[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: Files machine retirement could not be requested: ${
685
+ retirementError instanceof Error ? retirementError.message : String(retirementError)
686
+ }`,
687
+ );
688
+ }
689
+ void this.closeAssignment({ ...assignment }, "stop").catch((error: unknown) => {
690
+ console.warn(
691
+ `[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: Files recovery cleanup failed: ${
692
+ error instanceof Error ? error.message : String(error)
693
+ }`,
694
+ );
695
+ });
696
+ }
697
+
503
698
  /** The start is over, successful or not: its token has no authority left,
504
699
  * and it is no longer the task's newest. */
505
700
  endStart(start: TaskCredentialStart): void {
@@ -671,34 +866,50 @@ export class CloudCredentialRuntime {
671
866
  if (recovered.running === null) this.#recovered.delete(taskId);
672
867
  }
673
868
 
674
- /** A recovered run's gate opened: its recovery is done, and its setup
675
- * runs, once, in the background, its failure its own. */
869
+ /** A recovered runtime is ready only after setup succeeds. Transient
870
+ * reconstruction/exec failures retry; Stop, cleanup and takeover fence
871
+ * every attempt and its retry delay. */
676
872
  #runRecoveredSetup(assignment: Readonly<TaskAssignment>): void {
677
873
  const recovered = this.#recovered.get(assignment.taskId);
678
874
  if (!recovered || recovered.runEpoch !== assignment.runEpoch || recovered.running !== null) return;
679
- if (recovered.controller.signal.aborted) return;
875
+ if (recovered.controller.signal.aborted || this.#open.get(assignment.taskId) !== assignment.runEpoch) return;
680
876
  if (recovered.retry !== null) this.#clearTimer(recovered.retry);
681
877
  recovered.retry = null;
682
878
  const setup = recovered.setup;
683
- recovered.setup = null;
684
879
  if (setup === null) {
685
880
  this.#recovered.delete(assignment.taskId);
686
881
  return;
687
882
  }
688
- const signal = recovered.controller.signal;
883
+ const hold = holdPersonCredentialGate(assignment);
884
+ const signal = AbortSignal.any([recovered.controller.signal, hold.signal]);
885
+ let complete = false;
689
886
  recovered.running = this.#track(
690
887
  Promise.resolve()
691
- .then(() => setup(signal))
692
- .catch((error: unknown) =>
693
- console.warn(
694
- `[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: recovered runtime setup failed: ${
695
- error instanceof Error ? error.message : String(error)
696
- }`,
697
- ),
698
- )
888
+ .then(async () => {
889
+ signal.throwIfAborted();
890
+ await setup(signal);
891
+ signal.throwIfAborted();
892
+ complete = true;
893
+ })
894
+ .catch(() => {
895
+ if (!signal.aborted) {
896
+ console.warn(`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: recovered runtime setup failed; retrying`);
897
+ }
898
+ })
699
899
  .finally(() => {
900
+ hold.release();
700
901
  recovered.running = null;
701
- if (this.#recovered.get(assignment.taskId) === recovered) this.#recovered.delete(assignment.taskId);
902
+ if (this.#recovered.get(assignment.taskId) !== recovered) return;
903
+ if (complete || recovered.controller.signal.aborted || this.#shutDown) {
904
+ this.#recovered.delete(assignment.taskId);
905
+ return;
906
+ }
907
+ const base = this.#deps.recoveryRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
908
+ const delay = Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** Math.min(recovered.attempts++, 20));
909
+ recovered.retry = this.#setTimer(() => {
910
+ recovered.retry = null;
911
+ this.#runRecoveredSetup(assignment);
912
+ }, delay);
702
913
  }),
703
914
  );
704
915
  }
@@ -830,6 +1041,7 @@ export class CloudCredentialRuntime {
830
1041
  setEngineCredentialRefresher(null);
831
1042
  setEngineLaunchFenceEscalator(null);
832
1043
  setMcpCredentialRefresher(null);
1044
+ this.#deps.files?.shutdown?.();
833
1045
  }
834
1046
 
835
1047
  // -- the registry's gate and install --------------------------------------------
@@ -860,16 +1072,82 @@ export class CloudCredentialRuntime {
860
1072
  openEngineGate(assignment, view);
861
1073
  openMcpGate(assignment, view);
862
1074
  openPersonCredentialGate(assignment, view);
863
- this.#deps.files?.open(assignment, view);
864
1075
  this.#open.set(assignment.taskId, assignment.runEpoch);
865
1076
  const generation = this.#generations.get(view);
866
1077
  if (generation === undefined) this.#served.delete(assignment.taskId);
867
1078
  else this.#served.set(assignment.taskId, generation);
868
- this.#deps.mcpServedChanged?.(assignment.taskId);
869
1079
  this.#envServedChanged(assignment.taskId);
1080
+ // Last throwing boundary: a recovered runtime's mounts stay staged and
1081
+ // root-inaccessible until every other consumer accepted this exact view.
1082
+ // Nothing which follows publishes work synchronously. If Files itself
1083
+ // fails, the registry compensates by closing the earlier gates.
1084
+ this.#deps.files?.open(assignment, view);
1085
+ this.#personCredentialsChanged(assignment, view);
1086
+ this.#publishMcpMembership(assignment.taskId, view);
870
1087
  this.#runRecoveredSetup(assignment);
871
1088
  }
872
1089
 
1090
+ #personCredentialsChanged(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void {
1091
+ const reconcile = this.#deps.personCredentialsReady;
1092
+ if (!reconcile || this.#shutDown || this.#open.get(assignment.taskId) !== assignment.runEpoch) return;
1093
+ if ((this.#personInterrupted.get(assignment.taskId) ?? -1) >= assignment.runEpoch) return;
1094
+ const previous = this.#personCredentialSnapshots.get(assignment.taskId);
1095
+ const snapshot = personCredentialSnapshot(view);
1096
+ if (samePersonCredentialSnapshot(previous, snapshot)) return;
1097
+ this.#personCredentialSnapshots.set(assignment.taskId, snapshot);
1098
+ this.#personReconciles.get(assignment.taskId)?.controller.abort();
1099
+ const lanes = changedPersonCredentialLanes(previous, snapshot);
1100
+ if (lanes.length === 0) return;
1101
+ const entry = { runEpoch: assignment.runEpoch, controller: new AbortController() };
1102
+ const hold = holdPersonCredentialGate(assignment);
1103
+ const signal = AbortSignal.any([entry.controller.signal, hold.signal]);
1104
+ this.#personReconciles.set(assignment.taskId, entry);
1105
+ this.#background(
1106
+ Promise.resolve().then(async () => {
1107
+ let attempts = 0;
1108
+ while (!signal.aborted) {
1109
+ try {
1110
+ await reconcile(assignment, lanes, signal);
1111
+ signal.throwIfAborted();
1112
+ return;
1113
+ } catch {
1114
+ if (signal.aborted) return;
1115
+ if (attempts === 0) {
1116
+ console.warn(`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: person credential files not ready; retrying`);
1117
+ }
1118
+ const base = this.#deps.personReconcileRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
1119
+ await this.#retryDelay(Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** Math.min(attempts++, 20)), signal);
1120
+ }
1121
+ }
1122
+ }).finally(() => {
1123
+ hold.release();
1124
+ if (this.#personReconciles.get(assignment.taskId) === entry) this.#personReconciles.delete(assignment.taskId);
1125
+ }),
1126
+ );
1127
+ }
1128
+
1129
+ #interruptPersonCredentials(assignment: Readonly<TaskAssignment>): void {
1130
+ this.#personInterrupted.set(assignment.taskId, Math.max(assignment.runEpoch, this.#personInterrupted.get(assignment.taskId) ?? -1));
1131
+ const snapshot = this.#personCredentialSnapshots.get(assignment.taskId);
1132
+ if (snapshot && snapshot.runEpoch <= assignment.runEpoch) this.#personCredentialSnapshots.delete(assignment.taskId);
1133
+ const reconcile = this.#personReconciles.get(assignment.taskId);
1134
+ if (reconcile && reconcile.runEpoch <= assignment.runEpoch) reconcile.controller.abort();
1135
+ }
1136
+
1137
+ /** A retry never holds cleanup for its backoff interval. */
1138
+ #retryDelay(delay: number, signal: AbortSignal): Promise<void> {
1139
+ return new Promise((resolve) => {
1140
+ const finish = (): void => {
1141
+ this.#clearTimer(timer);
1142
+ signal.removeEventListener("abort", finish);
1143
+ resolve();
1144
+ };
1145
+ const timer = this.#setTimer(finish, delay);
1146
+ signal.addEventListener("abort", finish, { once: true });
1147
+ if (signal.aborted) finish();
1148
+ });
1149
+ }
1150
+
873
1151
  #envServedChanged(taskId: string): void {
874
1152
  try {
875
1153
  this.#deps.envServedChanged?.(taskId);
@@ -878,6 +1156,21 @@ export class CloudCredentialRuntime {
878
1156
  }
879
1157
  }
880
1158
 
1159
+ /** Publish only a normalized change visible to agent configuration. A
1160
+ * renewed lease, a refreshed OAuth token, and available/unavailable
1161
+ * transitions all keep the same named gateway route and therefore do not
1162
+ * rewrite config or restart agents; an unavailable route answers 503. */
1163
+ #publishMcpMembership(taskId: string, view: TaskCredentialView): void {
1164
+ const fingerprint = mcpAgentMembershipFingerprint(view);
1165
+ if (this.#publishedMcpMembership.get(taskId) === fingerprint) return;
1166
+ this.#publishedMcpMembership.set(taskId, fingerprint);
1167
+ try {
1168
+ this.#deps.mcpServedChanged?.(taskId);
1169
+ } catch {
1170
+ // The gateway gate is authoritative even when its config follower fails.
1171
+ }
1172
+ }
1173
+
881
1174
  /**
882
1175
  * Synchronous and idempotent: every consumer of this run (and older)
883
1176
  * stops being served, env and Files included, and every start holding its
@@ -898,8 +1191,11 @@ export class CloudCredentialRuntime {
898
1191
  if (open !== undefined && open <= assignment.runEpoch) {
899
1192
  this.#open.delete(assignment.taskId);
900
1193
  this.#served.delete(assignment.taskId);
1194
+ this.#personCredentialSnapshots.delete(assignment.taskId);
901
1195
  this.#envServedChanged(assignment.taskId);
902
1196
  }
1197
+ const person = this.#personReconciles.get(assignment.taskId);
1198
+ if (person && person.runEpoch <= assignment.runEpoch) person.controller.abort();
903
1199
  for (const entry of this.#holds.get(assignment.taskId) ?? []) {
904
1200
  if (entry.runEpoch <= assignment.runEpoch) {
905
1201
  attempt(() => entry.controller.abort(new CredentialLeaseUnavailableError("the task's credentials were withdrawn")));
@@ -947,6 +1243,12 @@ export class CloudCredentialRuntime {
947
1243
  installPersonCredentials(request);
948
1244
  await this.#deps.files?.install(request);
949
1245
  this.#assertInstallCurrent(request);
1246
+ // Credential changes normally affect material or availability only. Keep
1247
+ // this normalized comparison at the install boundary so even a future
1248
+ // same-run membership change publishes exactly once.
1249
+ if (request.reason === "credential-change") {
1250
+ this.#publishMcpMembership(request.assignment.taskId, request.next);
1251
+ }
950
1252
  }
951
1253
 
952
1254
  #assertInstallCurrent(request: CredentialInstallRequest): void {
@@ -996,7 +1298,9 @@ export class CloudCredentialRuntime {
996
1298
  const recovered = this.#recovered.get(request.taskId);
997
1299
  const setups: Promise<void>[] = [];
998
1300
  if (request.credentialIds === undefined && recovered && recovered.runEpoch <= request.runEpoch && recovered.running) {
999
- recovered.controller.abort(new StartInterruptedError("takeover"));
1301
+ // The gate hold already cancelled this attempt. A transient full
1302
+ // cleanup may reacquire the same run's lease, so retain its setup for
1303
+ // that next open. Stop/takeover cancel recovery at their own boundary.
1000
1304
  setups.push(recovered.running);
1001
1305
  }
1002
1306
  const consumer = consumerCleanupRequest(request);
@@ -1019,6 +1323,7 @@ export class CloudCredentialRuntime {
1019
1323
  if (request.credentialIds !== undefined) return;
1020
1324
  this.#deps.envReplica?.dropAssignment(request.taskId, request.runEpoch);
1021
1325
  if (request.reason !== "teardown") return;
1326
+ this.#publishedMcpMembership.delete(request.taskId);
1022
1327
  forgetLeasedEngines({ taskId: request.taskId, runEpoch: request.runEpoch });
1023
1328
  const mcp = leasedMcpFor(request.taskId);
1024
1329
  if (mcp && mcp.assignment.runEpoch <= request.runEpoch) forgetLeasedMcp(request.taskId);
@@ -165,7 +165,9 @@ const SSH_PATHS = new Set(
165
165
  );
166
166
  const FILES_SESSION_PATHS = new Set([
167
167
  LEASE_CREDENTIALS_FILE,
168
+ `${LEASE_CREDENTIALS_FILE}.uai-tmp`,
168
169
  LEASE_AWS_CONFIG_FILE,
170
+ `${LEASE_AWS_CONFIG_FILE}.uai-tmp`,
169
171
  ]);
170
172
 
171
173
  /** Destructive cleanup accepts only lane-owned files. The executor repeats