@runuai/host 0.9.152 → 0.9.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0029_task_shared_files_mode.sql +5 -0
- package/db/migrations/meta/_journal.json +7 -0
- package/db/schema.ts +10 -0
- package/lib/agent.ts +4 -0
- package/lib/browser-testing.ts +8 -3
- package/lib/cloud-credential-leases.ts +24 -4
- package/lib/cloud-credential-runtime.ts +322 -17
- package/lib/credential-cleanup-inventory.ts +2 -0
- package/lib/credential-consumers/files.ts +934 -0
- package/lib/credential-consumers/github.ts +140 -32
- package/lib/credential-consumers/mcp.ts +6 -1
- package/lib/mcp-gateway.ts +576 -122
- package/lib/orchestrator.ts +91 -9
- package/lib/runtime-state.ts +88 -0
- package/lib/shared-files-s3.ts +743 -3
- package/lib/sidecar-resume.ts +9 -0
- package/lib/task-credential-cleanup.ts +36 -5
- package/lib/task-environment/machine-task-up.ts +35 -19
- package/lib/task-files-machine-retirement.ts +88 -0
- package/lib/task-files-readiness.ts +127 -0
- package/lib/task-person-credentials.ts +76 -0
- package/package.json +1 -1
- package/src/cloud-credentials.ts +10 -0
- package/src/index.ts +40 -7
- package/src/main.ts +49 -2
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
-- ADR-154 Files v1: persist the cloud-issued mode for crash recovery. The
|
|
2
|
+
-- task-owned `.uai/files-mounted` marker is an observation, never authority.
|
|
3
|
+
ALTER TABLE `uai_host_tasks` ADD `shared_files_mode` text
|
|
4
|
+
CONSTRAINT `uai_host_tasks_shared_files_mode`
|
|
5
|
+
CHECK (`shared_files_mode` IS NULL OR `shared_files_mode` IN ('off', 'ro', 'rw'));
|
|
@@ -204,6 +204,13 @@
|
|
|
204
204
|
"when": 1791331200001,
|
|
205
205
|
"tag": "0028_leased_engine_runtimes",
|
|
206
206
|
"breakpoints": true
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
"idx": 29,
|
|
210
|
+
"version": "6",
|
|
211
|
+
"when": 1791590400000,
|
|
212
|
+
"tag": "0029_task_shared_files_mode",
|
|
213
|
+
"breakpoints": true
|
|
207
214
|
}
|
|
208
215
|
]
|
|
209
216
|
}
|
package/db/schema.ts
CHANGED
|
@@ -39,6 +39,12 @@ export const hostTasks = sqliteTable(
|
|
|
39
39
|
// Value-free resolver metadata for rebuilding the project-env gateway
|
|
40
40
|
// after a runner restart, before the cloud re-sends task-up.
|
|
41
41
|
projectEnvSpec: text("project_env_spec"),
|
|
42
|
+
// ADR-154 Files v1: the normalized mode from the cloud's task launch.
|
|
43
|
+
// Recovery must never trust the task-writable `.uai/files-mounted`
|
|
44
|
+
// marker as authority for what may be mounted.
|
|
45
|
+
sharedFilesMode: text("shared_files_mode", {
|
|
46
|
+
enum: ["off", "ro", "rw"],
|
|
47
|
+
}),
|
|
42
48
|
statusMirror: text("status_mirror"),
|
|
43
49
|
// Owner + project slugs — received in TaskLaunchInput at task-up and
|
|
44
50
|
// persisted here so the local UI (ADR-028) can show "who/what" without a
|
|
@@ -70,6 +76,10 @@ export const hostTasks = sqliteTable(
|
|
|
70
76
|
"uai_host_tasks_environment_pair",
|
|
71
77
|
sql`(${table.environmentProvider} IS NULL) = (${table.environmentLocator} IS NULL)`,
|
|
72
78
|
),
|
|
79
|
+
sharedFilesModeCheck: check(
|
|
80
|
+
"uai_host_tasks_shared_files_mode",
|
|
81
|
+
sql`${table.sharedFilesMode} IS NULL OR ${table.sharedFilesMode} IN ('off', 'ro', 'rw')`,
|
|
82
|
+
),
|
|
73
83
|
}),
|
|
74
84
|
);
|
|
75
85
|
|
package/lib/agent.ts
CHANGED
|
@@ -68,6 +68,10 @@ export interface LeasedTaskUp {
|
|
|
68
68
|
* the start.
|
|
69
69
|
*/
|
|
70
70
|
runtimeReady: (environment: TaskWriteEnvironment, signal: AbortSignal) => Promise<void>;
|
|
71
|
+
/** This exact start carries a task-machine Files session. Machine task-up
|
|
72
|
+
* skips its legacy copy/mount path only in this case; owned leased tasks do
|
|
73
|
+
* not receive a Files session and keep their normal Files behavior. */
|
|
74
|
+
filesV1?: true;
|
|
71
75
|
}
|
|
72
76
|
|
|
73
77
|
/** Refuses a start on cloud credentials that is not whole, or not this
|
package/lib/browser-testing.ts
CHANGED
|
@@ -1390,11 +1390,16 @@ export async function setupBrowserTesting(
|
|
|
1390
1390
|
}
|
|
1391
1391
|
}
|
|
1392
1392
|
|
|
1393
|
-
// Grok registers
|
|
1394
|
-
//
|
|
1395
|
-
//
|
|
1393
|
+
// Grok registers through its own read-modify-write CLI (user scope,
|
|
1394
|
+
// see grokBrowserAddCommand), under the same lock as gateway reconciliation
|
|
1395
|
+
// so two successful CLI processes cannot silently lose each other's entry.
|
|
1396
|
+
// A failure leaves the task configured-false so the next ensure retries.
|
|
1396
1397
|
if (hasGrok) {
|
|
1397
1398
|
const grokArgv = [
|
|
1399
|
+
"flock",
|
|
1400
|
+
"-w",
|
|
1401
|
+
"10",
|
|
1402
|
+
MCP_CONFIG_LOCK_PATH,
|
|
1398
1403
|
"sh",
|
|
1399
1404
|
"-lc",
|
|
1400
1405
|
selectedGrokHomes.map(grokBrowserAddCommand).join(" && "),
|
|
@@ -257,6 +257,12 @@ export interface TaskCredentialLeaseRegistryOptions {
|
|
|
257
257
|
assignment: Readonly<TaskAssignment>,
|
|
258
258
|
view: TaskCredentialView,
|
|
259
259
|
): void;
|
|
260
|
+
/** A material update committed while the existing gate stayed open.
|
|
261
|
+
* Followers may reconcile runtime files from this authoritative view. */
|
|
262
|
+
credentialMaterialChanged?(
|
|
263
|
+
assignment: Readonly<TaskAssignment>,
|
|
264
|
+
view: TaskCredentialView,
|
|
265
|
+
): void;
|
|
260
266
|
cleanupExternal(request: CredentialCleanupRequest): Promise<void>;
|
|
261
267
|
/**
|
|
262
268
|
* Installs files/proxy/engine consumers transactionally. The env-replica
|
|
@@ -1126,7 +1132,10 @@ export class TaskCredentialLeaseRegistry {
|
|
|
1126
1132
|
await this.#applyCredentialVersion(state, frame);
|
|
1127
1133
|
} catch (error) {
|
|
1128
1134
|
this.#report(error, state.assignment);
|
|
1129
|
-
|
|
1135
|
+
// A failed consumer install may already have completed full cleanup
|
|
1136
|
+
// and armed the bounded retry which owns what comes next. Do not cancel
|
|
1137
|
+
// that backoff by immediately requesting the same refused manifest.
|
|
1138
|
+
if (this.#isActive(state) && !state.retryTimer) this.#request(state);
|
|
1130
1139
|
}
|
|
1131
1140
|
}
|
|
1132
1141
|
|
|
@@ -1226,7 +1235,7 @@ export class TaskCredentialLeaseRegistry {
|
|
|
1226
1235
|
});
|
|
1227
1236
|
} catch (error) {
|
|
1228
1237
|
this.#report(error, state.assignment);
|
|
1229
|
-
this.#request(state);
|
|
1238
|
+
if (this.#isActive(state) && !state.retryTimer) this.#request(state);
|
|
1230
1239
|
}
|
|
1231
1240
|
} else if (frame.outcome === "needs_sign_in") {
|
|
1232
1241
|
// The authoritative stale frame supplies any floor and removals.
|
|
@@ -2276,6 +2285,9 @@ export class TaskCredentialLeaseRegistry {
|
|
|
2276
2285
|
state.bundle = next;
|
|
2277
2286
|
state.bundleGeneration++;
|
|
2278
2287
|
this.#armBundleTimers(state);
|
|
2288
|
+
if (!refused) {
|
|
2289
|
+
this.#opts.credentialMaterialChanged?.(cloneAssignment(state.assignment), next.view);
|
|
2290
|
+
}
|
|
2279
2291
|
// A refusal opens through its release, inside this boundary: an open
|
|
2280
2292
|
// that fails, its compensating close included, escalates below.
|
|
2281
2293
|
release();
|
|
@@ -2452,8 +2464,16 @@ export class TaskCredentialLeaseRegistry {
|
|
|
2452
2464
|
// A permanent revocation answers every request with another denial.
|
|
2453
2465
|
// Keep asking, but do not turn that into a tight
|
|
2454
2466
|
// deny/cleanup/request loop against the cloud.
|
|
2455
|
-
|
|
2456
|
-
|
|
2467
|
+
// A denial is stable until cloud state changes, and a consumer which
|
|
2468
|
+
// refused a grant can keep refusing the same manifest (for example
|
|
2469
|
+
// while a task-owned workspace entry is being repaired). Neither is
|
|
2470
|
+
// allowed to become a cleanup/request loop which repeatedly mints STS
|
|
2471
|
+
// sessions and reads Parameter Store without delay.
|
|
2472
|
+
if (reason === "denied" || reason === "manifest-narrowed") {
|
|
2473
|
+
this.#armRetry(state);
|
|
2474
|
+
} else {
|
|
2475
|
+
this.#request(state);
|
|
2476
|
+
}
|
|
2457
2477
|
}
|
|
2458
2478
|
}
|
|
2459
2479
|
}
|
|
@@ -63,9 +63,11 @@ import {
|
|
|
63
63
|
cleanupPersonCredentials,
|
|
64
64
|
closePersonCredentialGate,
|
|
65
65
|
drainPersonCredentialWrites,
|
|
66
|
+
holdPersonCredentialGate,
|
|
66
67
|
holdTaskGitCredentials,
|
|
67
68
|
installPersonCredentials,
|
|
68
69
|
openPersonCredentialGate,
|
|
70
|
+
type PersonCredentialLane,
|
|
69
71
|
type TaskGitCredentialHold,
|
|
70
72
|
} from "./credential-consumers/github";
|
|
71
73
|
import {
|
|
@@ -79,6 +81,8 @@ import {
|
|
|
79
81
|
setMcpCredentialRefresher,
|
|
80
82
|
} from "./credential-consumers/mcp";
|
|
81
83
|
import type { ProjectEnvTaskReplicaManager, ProjectEnvTaskReplicaSnapshot } from "./project-env-task-replica";
|
|
84
|
+
import type { SharedFilesMode } from "./shared-files-s3";
|
|
85
|
+
import type { TaskEnvironmentHandle } from "./task-environment/types";
|
|
82
86
|
import {
|
|
83
87
|
TaskCredentialCleanupCoordinator,
|
|
84
88
|
type TaskCredentialCleanupDeps,
|
|
@@ -95,6 +99,17 @@ const ENV_READY_POLL_MS = 50;
|
|
|
95
99
|
* way; the next process's recovery sweep finishes what is left. */
|
|
96
100
|
export const SHUTDOWN_DRAIN_MS = 30_000;
|
|
97
101
|
|
|
102
|
+
/** The only MCP state an agent config exposes. Material, provider URL,
|
|
103
|
+
* revision and availability stay behind the task gateway, so changing any of
|
|
104
|
+
* them must not churn a long-lived agent process. */
|
|
105
|
+
function mcpAgentMembershipFingerprint(view: TaskCredentialView): string {
|
|
106
|
+
return JSON.stringify(
|
|
107
|
+
view.mcp
|
|
108
|
+
.map((connection) => connection.manifest.slug)
|
|
109
|
+
.sort((left, right) => left.localeCompare(right)),
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
98
113
|
/** The task's Files (ADR-154 §12): installed, opened and closed with the
|
|
99
114
|
* rest. `close` is synchronous: no new Files admission after it. */
|
|
100
115
|
export interface TaskFilesConsumer {
|
|
@@ -102,6 +117,32 @@ export interface TaskFilesConsumer {
|
|
|
102
117
|
open(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void;
|
|
103
118
|
close(assignment: Readonly<TaskAssignment>): void;
|
|
104
119
|
closeConsumers(request: TaskCredentialCleanupRequest): Promise<void>;
|
|
120
|
+
/** A trusted task-machine runtime could not establish its Files boundary.
|
|
121
|
+
* This run cannot be reopened on the same machine. */
|
|
122
|
+
retire(assignment: Readonly<TaskAssignment>, error: unknown): void;
|
|
123
|
+
/** Optional only so owned runners can use the same composer without a
|
|
124
|
+
* task-machine Files implementation. */
|
|
125
|
+
runtimeReady?(
|
|
126
|
+
assignment: Readonly<TaskAssignment>,
|
|
127
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
128
|
+
mode: SharedFilesMode,
|
|
129
|
+
signal: AbortSignal,
|
|
130
|
+
): Promise<void>;
|
|
131
|
+
/** Drop secrets at process shutdown without starting cleanup. */
|
|
132
|
+
shutdown?(): void;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** A Files implementation proved that the runtime-ready operation which
|
|
136
|
+
* failed still owns the exact open run and requested terminal retirement of
|
|
137
|
+
* its one-task machine; a stale operation throws its ordinary error. */
|
|
138
|
+
export class AuthoritativeTaskFilesFailure extends Error {
|
|
139
|
+
override readonly cause: unknown;
|
|
140
|
+
|
|
141
|
+
constructor(cause: unknown) {
|
|
142
|
+
super(cause instanceof Error ? cause.message : "the task's Files boundary failed");
|
|
143
|
+
this.name = "AuthoritativeTaskFilesFailure";
|
|
144
|
+
this.cause = cause;
|
|
145
|
+
}
|
|
105
146
|
}
|
|
106
147
|
|
|
107
148
|
export interface CloudCredentialRuntimeDeps {
|
|
@@ -121,18 +162,90 @@ export interface CloudCredentialRuntimeDeps {
|
|
|
121
162
|
* on a grant's snapshot, or closed): whoever follows the task's env
|
|
122
163
|
* (the proxy's trust delivery) looks again. */
|
|
123
164
|
envServedChanged?(taskId: string): void;
|
|
165
|
+
/** Reconcile the owner's GitHub/SSH files in an already-running task.
|
|
166
|
+
* The signal closes with the exact person-credential gate, and the caller
|
|
167
|
+
* retries a transient rejection while that gate and material stay current. */
|
|
168
|
+
personCredentialsReady?(
|
|
169
|
+
assignment: Readonly<TaskAssignment>,
|
|
170
|
+
lanes: readonly PersonCredentialLane[],
|
|
171
|
+
signal: AbortSignal,
|
|
172
|
+
): Promise<void>;
|
|
124
173
|
files?: TaskFilesConsumer | null;
|
|
125
174
|
launchDeadlineMs?: number;
|
|
126
175
|
envReadyTimeoutMs?: number;
|
|
127
176
|
shutdownDrainMs?: number;
|
|
128
177
|
/** The first retry's delay for a recovered start whose startup failed. */
|
|
129
178
|
recoveryRetryBaseMs?: number;
|
|
179
|
+
/** The first retry's delay for task-runtime person-credential writes. */
|
|
180
|
+
personReconcileRetryBaseMs?: number;
|
|
130
181
|
registry?: Partial<
|
|
131
182
|
Pick<TaskCredentialLeaseRegistryOptions, "createSessionReader" | "now" | "setTimer" | "clearTimer" | "randomId" | "onError">
|
|
132
183
|
>;
|
|
133
184
|
cleanup?: Pick<TaskCredentialCleanupDeps, "db" | "removePath" | "destroyRuntime">;
|
|
134
185
|
}
|
|
135
186
|
|
|
187
|
+
interface PersonCredentialStamp {
|
|
188
|
+
/** Stable without retaining another copy of any credential material. */
|
|
189
|
+
identity: string;
|
|
190
|
+
available: boolean;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
interface PersonCredentialSnapshot {
|
|
194
|
+
runEpoch: number;
|
|
195
|
+
github: PersonCredentialStamp;
|
|
196
|
+
ssh: PersonCredentialStamp;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function personCredentialStamp(
|
|
200
|
+
credential: TaskCredentialView["github"] | TaskCredentialView["ssh"],
|
|
201
|
+
): PersonCredentialStamp {
|
|
202
|
+
if (credential === null) return { identity: "absent", available: false };
|
|
203
|
+
return {
|
|
204
|
+
identity: [
|
|
205
|
+
credential.manifest.credentialId,
|
|
206
|
+
credential.manifest.revision,
|
|
207
|
+
credential.availability,
|
|
208
|
+
credential.observedVersion ?? "unreadable",
|
|
209
|
+
].join("\u0000"),
|
|
210
|
+
available: credential.availability === "available",
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function personCredentialSnapshot(view: TaskCredentialView): PersonCredentialSnapshot {
|
|
215
|
+
return {
|
|
216
|
+
runEpoch: view.assignment.runEpoch,
|
|
217
|
+
github: personCredentialStamp(view.github),
|
|
218
|
+
ssh: personCredentialStamp(view.ssh),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function samePersonCredentialSnapshot(
|
|
223
|
+
left: PersonCredentialSnapshot | undefined,
|
|
224
|
+
right: PersonCredentialSnapshot,
|
|
225
|
+
): boolean {
|
|
226
|
+
return Boolean(
|
|
227
|
+
left &&
|
|
228
|
+
left.runEpoch === right.runEpoch &&
|
|
229
|
+
left.github.identity === right.github.identity &&
|
|
230
|
+
left.ssh.identity === right.ssh.identity,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** What must be rewritten for `next`. GitHub and SSH transport are coupled:
|
|
235
|
+
* when GitHub leaves, the surviving SSH key must select SSH again. */
|
|
236
|
+
function changedPersonCredentialLanes(
|
|
237
|
+
previous: PersonCredentialSnapshot | undefined,
|
|
238
|
+
next: PersonCredentialSnapshot,
|
|
239
|
+
): PersonCredentialLane[] {
|
|
240
|
+
if (samePersonCredentialSnapshot(previous, next)) return [];
|
|
241
|
+
// The pair shares Git transport configuration. Reconcile both available
|
|
242
|
+
// lanes, including a write interrupted by the other lane changing.
|
|
243
|
+
const lanes: PersonCredentialLane[] = [];
|
|
244
|
+
if (next.github.available) lanes.push("github");
|
|
245
|
+
if (next.ssh.available) lanes.push("ssh");
|
|
246
|
+
return lanes;
|
|
247
|
+
}
|
|
248
|
+
|
|
136
249
|
/**
|
|
137
250
|
* Why a start was interrupted, as its signal's reason: Stop, teardown or a
|
|
138
251
|
* newer run took it over, or the runner is shutting down. The first
|
|
@@ -266,6 +379,16 @@ export class CloudCredentialRuntime {
|
|
|
266
379
|
/** Work accepted and not yet over (startups, frames, installs, holds not
|
|
267
380
|
* yet released, closes): what shutdown joins. */
|
|
268
381
|
readonly #work = new Set<Promise<unknown>>();
|
|
382
|
+
/** Last agent-facing MCP membership published for each task. Kept across a
|
|
383
|
+
* gate close/renewal because the config remains in the task runtime. */
|
|
384
|
+
readonly #publishedMcpMembership = new Map<string, string>();
|
|
385
|
+
/** Person material last opened or installed for each task. Only identities,
|
|
386
|
+
* availability and observed versions: never another copy of a secret. */
|
|
387
|
+
readonly #personCredentialSnapshots = new Map<string, PersonCredentialSnapshot>();
|
|
388
|
+
readonly #personReconciles = new Map<string, { runEpoch: number; controller: AbortController }>();
|
|
389
|
+
/** Stop fences work before it can acquire the lifecycle slot or close the
|
|
390
|
+
* lease. A late grant cannot restart writes in that interval. */
|
|
391
|
+
readonly #personInterrupted = new Map<string, number>();
|
|
269
392
|
readonly #setTimer: typeof setTimeout;
|
|
270
393
|
readonly #clearTimer: typeof clearTimeout;
|
|
271
394
|
#shutDown = false;
|
|
@@ -295,6 +418,7 @@ export class CloudCredentialRuntime {
|
|
|
295
418
|
send: (frame) => deps.send(frame),
|
|
296
419
|
closeTaskGate: (assignment) => this.#close(assignment),
|
|
297
420
|
openTaskGate: (assignment, view) => this.#openGate(assignment, view),
|
|
421
|
+
credentialMaterialChanged: (assignment, view) => this.#personCredentialsChanged(assignment, view),
|
|
298
422
|
installExternal: (request) => this.#track(this.#install(request)),
|
|
299
423
|
// From a grant's acceptance until its own reader is active, the env
|
|
300
424
|
// replica refuses the run's frames for the cloud to retry: none is
|
|
@@ -337,6 +461,7 @@ export class CloudCredentialRuntime {
|
|
|
337
461
|
throw new CredentialLeaseUnavailableError("a newer run of this task has begun");
|
|
338
462
|
}
|
|
339
463
|
this.#floors.set(assignment.taskId, assignment.runEpoch);
|
|
464
|
+
if (!options.recovery) this.#personInterrupted.delete(assignment.taskId);
|
|
340
465
|
this.#interruptTokens(assignment.taskId, {});
|
|
341
466
|
// A newer run's start ends an older run's recovery; an ordinary start of
|
|
342
467
|
// the run being recovered takes its startup over.
|
|
@@ -376,6 +501,11 @@ export class CloudCredentialRuntime {
|
|
|
376
501
|
*/
|
|
377
502
|
interruptStarts(taskId: string, which: { exact?: number; below?: number } = {}): void {
|
|
378
503
|
this.#interruptTokens(taskId, which);
|
|
504
|
+
const assignment = this.#assignments.get(taskId);
|
|
505
|
+
if (assignment && (which.exact === undefined || assignment.runEpoch === which.exact) &&
|
|
506
|
+
(which.below === undefined || assignment.runEpoch < which.below)) {
|
|
507
|
+
this.#interruptPersonCredentials(assignment);
|
|
508
|
+
}
|
|
379
509
|
// A recovery of the run ends with it, ended start token or not.
|
|
380
510
|
const recovered = this.#recovered.get(taskId);
|
|
381
511
|
if (
|
|
@@ -399,6 +529,8 @@ export class CloudCredentialRuntime {
|
|
|
399
529
|
/** A local Stop: the task's recovery, whatever its run, ends now. One
|
|
400
530
|
* only exists for the run this runner owned at its start. */
|
|
401
531
|
interruptRecovery(taskId: string): void {
|
|
532
|
+
const assignment = this.#assignments.get(taskId);
|
|
533
|
+
if (assignment) this.#interruptPersonCredentials(assignment);
|
|
402
534
|
const recovered = this.#recovered.get(taskId);
|
|
403
535
|
if (recovered) this.#endRecovery(taskId, recovered.runEpoch);
|
|
404
536
|
}
|
|
@@ -500,6 +632,69 @@ export class CloudCredentialRuntime {
|
|
|
500
632
|
};
|
|
501
633
|
}
|
|
502
634
|
|
|
635
|
+
/** Whether this process has the task-machine Files-v1 consumer wired. */
|
|
636
|
+
taskFilesV1(): boolean {
|
|
637
|
+
return typeof this.#deps.files?.runtimeReady === "function";
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** Runtime-ready half of Files v1. It is tracked with the composer's
|
|
641
|
+
* accepted work so shutdown joins a write/mount already admitted. */
|
|
642
|
+
prepareTaskFiles(
|
|
643
|
+
assignment: Readonly<TaskAssignment>,
|
|
644
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
645
|
+
mode: SharedFilesMode,
|
|
646
|
+
signal: AbortSignal,
|
|
647
|
+
): Promise<void> {
|
|
648
|
+
const ready = this.#deps.files?.runtimeReady;
|
|
649
|
+
if (!ready) return Promise.reject(new Error("task-machine Files v1 is unavailable"));
|
|
650
|
+
return this.#track(
|
|
651
|
+
ready.call(this.#deps.files, assignment, environment, mode, signal).catch((error: unknown) => {
|
|
652
|
+
if (error instanceof AuthoritativeTaskFilesFailure) {
|
|
653
|
+
// Never reopen this run on the same machine. The consumer already
|
|
654
|
+
// proved this is authoritative; centrally latch/request exact-run
|
|
655
|
+
// terminal retirement too, so an implementation cannot turn this
|
|
656
|
+
// into an ordinary cleanup/regrant by omission.
|
|
657
|
+
let retirementError: unknown = null;
|
|
658
|
+
try {
|
|
659
|
+
this.#deps.files!.retire(assignment, error.cause);
|
|
660
|
+
} catch (failure) {
|
|
661
|
+
retirementError = failure;
|
|
662
|
+
}
|
|
663
|
+
void this.closeAssignment({ ...assignment }, "stop").catch(() => {});
|
|
664
|
+
if (retirementError !== null) throw retirementError;
|
|
665
|
+
throw error.cause;
|
|
666
|
+
}
|
|
667
|
+
throw error;
|
|
668
|
+
}),
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/** A recovered task whose trusted Files metadata is absent or invalid has
|
|
673
|
+
* failed the same boundary before the consumer can be called. Request its
|
|
674
|
+
* terminal machine retirement, close its gate synchronously, and run the
|
|
675
|
+
* run's full cleanup. */
|
|
676
|
+
failTaskFilesSetup(
|
|
677
|
+
assignment: Readonly<TaskAssignment>,
|
|
678
|
+
error: unknown = new Error("the recovered task has no trusted Files-v1 metadata"),
|
|
679
|
+
): void {
|
|
680
|
+
try {
|
|
681
|
+
this.#deps.files?.retire(assignment, error);
|
|
682
|
+
} catch (retirementError) {
|
|
683
|
+
console.warn(
|
|
684
|
+
`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: Files machine retirement could not be requested: ${
|
|
685
|
+
retirementError instanceof Error ? retirementError.message : String(retirementError)
|
|
686
|
+
}`,
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
void this.closeAssignment({ ...assignment }, "stop").catch((error: unknown) => {
|
|
690
|
+
console.warn(
|
|
691
|
+
`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: Files recovery cleanup failed: ${
|
|
692
|
+
error instanceof Error ? error.message : String(error)
|
|
693
|
+
}`,
|
|
694
|
+
);
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
|
|
503
698
|
/** The start is over, successful or not: its token has no authority left,
|
|
504
699
|
* and it is no longer the task's newest. */
|
|
505
700
|
endStart(start: TaskCredentialStart): void {
|
|
@@ -671,34 +866,50 @@ export class CloudCredentialRuntime {
|
|
|
671
866
|
if (recovered.running === null) this.#recovered.delete(taskId);
|
|
672
867
|
}
|
|
673
868
|
|
|
674
|
-
/** A recovered
|
|
675
|
-
*
|
|
869
|
+
/** A recovered runtime is ready only after setup succeeds. Transient
|
|
870
|
+
* reconstruction/exec failures retry; Stop, cleanup and takeover fence
|
|
871
|
+
* every attempt and its retry delay. */
|
|
676
872
|
#runRecoveredSetup(assignment: Readonly<TaskAssignment>): void {
|
|
677
873
|
const recovered = this.#recovered.get(assignment.taskId);
|
|
678
874
|
if (!recovered || recovered.runEpoch !== assignment.runEpoch || recovered.running !== null) return;
|
|
679
|
-
if (recovered.controller.signal.aborted) return;
|
|
875
|
+
if (recovered.controller.signal.aborted || this.#open.get(assignment.taskId) !== assignment.runEpoch) return;
|
|
680
876
|
if (recovered.retry !== null) this.#clearTimer(recovered.retry);
|
|
681
877
|
recovered.retry = null;
|
|
682
878
|
const setup = recovered.setup;
|
|
683
|
-
recovered.setup = null;
|
|
684
879
|
if (setup === null) {
|
|
685
880
|
this.#recovered.delete(assignment.taskId);
|
|
686
881
|
return;
|
|
687
882
|
}
|
|
688
|
-
const
|
|
883
|
+
const hold = holdPersonCredentialGate(assignment);
|
|
884
|
+
const signal = AbortSignal.any([recovered.controller.signal, hold.signal]);
|
|
885
|
+
let complete = false;
|
|
689
886
|
recovered.running = this.#track(
|
|
690
887
|
Promise.resolve()
|
|
691
|
-
.then(() =>
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
888
|
+
.then(async () => {
|
|
889
|
+
signal.throwIfAborted();
|
|
890
|
+
await setup(signal);
|
|
891
|
+
signal.throwIfAborted();
|
|
892
|
+
complete = true;
|
|
893
|
+
})
|
|
894
|
+
.catch(() => {
|
|
895
|
+
if (!signal.aborted) {
|
|
896
|
+
console.warn(`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: recovered runtime setup failed; retrying`);
|
|
897
|
+
}
|
|
898
|
+
})
|
|
699
899
|
.finally(() => {
|
|
900
|
+
hold.release();
|
|
700
901
|
recovered.running = null;
|
|
701
|
-
if (this.#recovered.get(assignment.taskId)
|
|
902
|
+
if (this.#recovered.get(assignment.taskId) !== recovered) return;
|
|
903
|
+
if (complete || recovered.controller.signal.aborted || this.#shutDown) {
|
|
904
|
+
this.#recovered.delete(assignment.taskId);
|
|
905
|
+
return;
|
|
906
|
+
}
|
|
907
|
+
const base = this.#deps.recoveryRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
|
|
908
|
+
const delay = Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** Math.min(recovered.attempts++, 20));
|
|
909
|
+
recovered.retry = this.#setTimer(() => {
|
|
910
|
+
recovered.retry = null;
|
|
911
|
+
this.#runRecoveredSetup(assignment);
|
|
912
|
+
}, delay);
|
|
702
913
|
}),
|
|
703
914
|
);
|
|
704
915
|
}
|
|
@@ -830,6 +1041,7 @@ export class CloudCredentialRuntime {
|
|
|
830
1041
|
setEngineCredentialRefresher(null);
|
|
831
1042
|
setEngineLaunchFenceEscalator(null);
|
|
832
1043
|
setMcpCredentialRefresher(null);
|
|
1044
|
+
this.#deps.files?.shutdown?.();
|
|
833
1045
|
}
|
|
834
1046
|
|
|
835
1047
|
// -- the registry's gate and install --------------------------------------------
|
|
@@ -860,16 +1072,82 @@ export class CloudCredentialRuntime {
|
|
|
860
1072
|
openEngineGate(assignment, view);
|
|
861
1073
|
openMcpGate(assignment, view);
|
|
862
1074
|
openPersonCredentialGate(assignment, view);
|
|
863
|
-
this.#deps.files?.open(assignment, view);
|
|
864
1075
|
this.#open.set(assignment.taskId, assignment.runEpoch);
|
|
865
1076
|
const generation = this.#generations.get(view);
|
|
866
1077
|
if (generation === undefined) this.#served.delete(assignment.taskId);
|
|
867
1078
|
else this.#served.set(assignment.taskId, generation);
|
|
868
|
-
this.#deps.mcpServedChanged?.(assignment.taskId);
|
|
869
1079
|
this.#envServedChanged(assignment.taskId);
|
|
1080
|
+
// Last throwing boundary: a recovered runtime's mounts stay staged and
|
|
1081
|
+
// root-inaccessible until every other consumer accepted this exact view.
|
|
1082
|
+
// Nothing which follows publishes work synchronously. If Files itself
|
|
1083
|
+
// fails, the registry compensates by closing the earlier gates.
|
|
1084
|
+
this.#deps.files?.open(assignment, view);
|
|
1085
|
+
this.#personCredentialsChanged(assignment, view);
|
|
1086
|
+
this.#publishMcpMembership(assignment.taskId, view);
|
|
870
1087
|
this.#runRecoveredSetup(assignment);
|
|
871
1088
|
}
|
|
872
1089
|
|
|
1090
|
+
#personCredentialsChanged(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void {
|
|
1091
|
+
const reconcile = this.#deps.personCredentialsReady;
|
|
1092
|
+
if (!reconcile || this.#shutDown || this.#open.get(assignment.taskId) !== assignment.runEpoch) return;
|
|
1093
|
+
if ((this.#personInterrupted.get(assignment.taskId) ?? -1) >= assignment.runEpoch) return;
|
|
1094
|
+
const previous = this.#personCredentialSnapshots.get(assignment.taskId);
|
|
1095
|
+
const snapshot = personCredentialSnapshot(view);
|
|
1096
|
+
if (samePersonCredentialSnapshot(previous, snapshot)) return;
|
|
1097
|
+
this.#personCredentialSnapshots.set(assignment.taskId, snapshot);
|
|
1098
|
+
this.#personReconciles.get(assignment.taskId)?.controller.abort();
|
|
1099
|
+
const lanes = changedPersonCredentialLanes(previous, snapshot);
|
|
1100
|
+
if (lanes.length === 0) return;
|
|
1101
|
+
const entry = { runEpoch: assignment.runEpoch, controller: new AbortController() };
|
|
1102
|
+
const hold = holdPersonCredentialGate(assignment);
|
|
1103
|
+
const signal = AbortSignal.any([entry.controller.signal, hold.signal]);
|
|
1104
|
+
this.#personReconciles.set(assignment.taskId, entry);
|
|
1105
|
+
this.#background(
|
|
1106
|
+
Promise.resolve().then(async () => {
|
|
1107
|
+
let attempts = 0;
|
|
1108
|
+
while (!signal.aborted) {
|
|
1109
|
+
try {
|
|
1110
|
+
await reconcile(assignment, lanes, signal);
|
|
1111
|
+
signal.throwIfAborted();
|
|
1112
|
+
return;
|
|
1113
|
+
} catch {
|
|
1114
|
+
if (signal.aborted) return;
|
|
1115
|
+
if (attempts === 0) {
|
|
1116
|
+
console.warn(`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: person credential files not ready; retrying`);
|
|
1117
|
+
}
|
|
1118
|
+
const base = this.#deps.personReconcileRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
|
|
1119
|
+
await this.#retryDelay(Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** Math.min(attempts++, 20)), signal);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
}).finally(() => {
|
|
1123
|
+
hold.release();
|
|
1124
|
+
if (this.#personReconciles.get(assignment.taskId) === entry) this.#personReconciles.delete(assignment.taskId);
|
|
1125
|
+
}),
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
#interruptPersonCredentials(assignment: Readonly<TaskAssignment>): void {
|
|
1130
|
+
this.#personInterrupted.set(assignment.taskId, Math.max(assignment.runEpoch, this.#personInterrupted.get(assignment.taskId) ?? -1));
|
|
1131
|
+
const snapshot = this.#personCredentialSnapshots.get(assignment.taskId);
|
|
1132
|
+
if (snapshot && snapshot.runEpoch <= assignment.runEpoch) this.#personCredentialSnapshots.delete(assignment.taskId);
|
|
1133
|
+
const reconcile = this.#personReconciles.get(assignment.taskId);
|
|
1134
|
+
if (reconcile && reconcile.runEpoch <= assignment.runEpoch) reconcile.controller.abort();
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
/** A retry never holds cleanup for its backoff interval. */
|
|
1138
|
+
#retryDelay(delay: number, signal: AbortSignal): Promise<void> {
|
|
1139
|
+
return new Promise((resolve) => {
|
|
1140
|
+
const finish = (): void => {
|
|
1141
|
+
this.#clearTimer(timer);
|
|
1142
|
+
signal.removeEventListener("abort", finish);
|
|
1143
|
+
resolve();
|
|
1144
|
+
};
|
|
1145
|
+
const timer = this.#setTimer(finish, delay);
|
|
1146
|
+
signal.addEventListener("abort", finish, { once: true });
|
|
1147
|
+
if (signal.aborted) finish();
|
|
1148
|
+
});
|
|
1149
|
+
}
|
|
1150
|
+
|
|
873
1151
|
#envServedChanged(taskId: string): void {
|
|
874
1152
|
try {
|
|
875
1153
|
this.#deps.envServedChanged?.(taskId);
|
|
@@ -878,6 +1156,21 @@ export class CloudCredentialRuntime {
|
|
|
878
1156
|
}
|
|
879
1157
|
}
|
|
880
1158
|
|
|
1159
|
+
/** Publish only a normalized change visible to agent configuration. A
|
|
1160
|
+
* renewed lease, a refreshed OAuth token, and available/unavailable
|
|
1161
|
+
* transitions all keep the same named gateway route and therefore do not
|
|
1162
|
+
* rewrite config or restart agents; an unavailable route answers 503. */
|
|
1163
|
+
#publishMcpMembership(taskId: string, view: TaskCredentialView): void {
|
|
1164
|
+
const fingerprint = mcpAgentMembershipFingerprint(view);
|
|
1165
|
+
if (this.#publishedMcpMembership.get(taskId) === fingerprint) return;
|
|
1166
|
+
this.#publishedMcpMembership.set(taskId, fingerprint);
|
|
1167
|
+
try {
|
|
1168
|
+
this.#deps.mcpServedChanged?.(taskId);
|
|
1169
|
+
} catch {
|
|
1170
|
+
// The gateway gate is authoritative even when its config follower fails.
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
881
1174
|
/**
|
|
882
1175
|
* Synchronous and idempotent: every consumer of this run (and older)
|
|
883
1176
|
* stops being served, env and Files included, and every start holding its
|
|
@@ -898,8 +1191,11 @@ export class CloudCredentialRuntime {
|
|
|
898
1191
|
if (open !== undefined && open <= assignment.runEpoch) {
|
|
899
1192
|
this.#open.delete(assignment.taskId);
|
|
900
1193
|
this.#served.delete(assignment.taskId);
|
|
1194
|
+
this.#personCredentialSnapshots.delete(assignment.taskId);
|
|
901
1195
|
this.#envServedChanged(assignment.taskId);
|
|
902
1196
|
}
|
|
1197
|
+
const person = this.#personReconciles.get(assignment.taskId);
|
|
1198
|
+
if (person && person.runEpoch <= assignment.runEpoch) person.controller.abort();
|
|
903
1199
|
for (const entry of this.#holds.get(assignment.taskId) ?? []) {
|
|
904
1200
|
if (entry.runEpoch <= assignment.runEpoch) {
|
|
905
1201
|
attempt(() => entry.controller.abort(new CredentialLeaseUnavailableError("the task's credentials were withdrawn")));
|
|
@@ -947,6 +1243,12 @@ export class CloudCredentialRuntime {
|
|
|
947
1243
|
installPersonCredentials(request);
|
|
948
1244
|
await this.#deps.files?.install(request);
|
|
949
1245
|
this.#assertInstallCurrent(request);
|
|
1246
|
+
// Credential changes normally affect material or availability only. Keep
|
|
1247
|
+
// this normalized comparison at the install boundary so even a future
|
|
1248
|
+
// same-run membership change publishes exactly once.
|
|
1249
|
+
if (request.reason === "credential-change") {
|
|
1250
|
+
this.#publishMcpMembership(request.assignment.taskId, request.next);
|
|
1251
|
+
}
|
|
950
1252
|
}
|
|
951
1253
|
|
|
952
1254
|
#assertInstallCurrent(request: CredentialInstallRequest): void {
|
|
@@ -996,7 +1298,9 @@ export class CloudCredentialRuntime {
|
|
|
996
1298
|
const recovered = this.#recovered.get(request.taskId);
|
|
997
1299
|
const setups: Promise<void>[] = [];
|
|
998
1300
|
if (request.credentialIds === undefined && recovered && recovered.runEpoch <= request.runEpoch && recovered.running) {
|
|
999
|
-
|
|
1301
|
+
// The gate hold already cancelled this attempt. A transient full
|
|
1302
|
+
// cleanup may reacquire the same run's lease, so retain its setup for
|
|
1303
|
+
// that next open. Stop/takeover cancel recovery at their own boundary.
|
|
1000
1304
|
setups.push(recovered.running);
|
|
1001
1305
|
}
|
|
1002
1306
|
const consumer = consumerCleanupRequest(request);
|
|
@@ -1019,6 +1323,7 @@ export class CloudCredentialRuntime {
|
|
|
1019
1323
|
if (request.credentialIds !== undefined) return;
|
|
1020
1324
|
this.#deps.envReplica?.dropAssignment(request.taskId, request.runEpoch);
|
|
1021
1325
|
if (request.reason !== "teardown") return;
|
|
1326
|
+
this.#publishedMcpMembership.delete(request.taskId);
|
|
1022
1327
|
forgetLeasedEngines({ taskId: request.taskId, runEpoch: request.runEpoch });
|
|
1023
1328
|
const mcp = leasedMcpFor(request.taskId);
|
|
1024
1329
|
if (mcp && mcp.assignment.runEpoch <= request.runEpoch) forgetLeasedMcp(request.taskId);
|
|
@@ -165,7 +165,9 @@ const SSH_PATHS = new Set(
|
|
|
165
165
|
);
|
|
166
166
|
const FILES_SESSION_PATHS = new Set([
|
|
167
167
|
LEASE_CREDENTIALS_FILE,
|
|
168
|
+
`${LEASE_CREDENTIALS_FILE}.uai-tmp`,
|
|
168
169
|
LEASE_AWS_CONFIG_FILE,
|
|
170
|
+
`${LEASE_AWS_CONFIG_FILE}.uai-tmp`,
|
|
169
171
|
]);
|
|
170
172
|
|
|
171
173
|
/** Destructive cleanup accepts only lane-owned files. The executor repeats
|