@runuai/host 0.9.153 → 0.9.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cloud-credential-leases.ts +9 -0
- package/lib/cloud-credential-runtime.ts +187 -21
- package/lib/credential-consumers/github.ts +140 -32
- package/lib/task-person-credentials.ts +76 -0
- package/package.json +1 -1
- package/src/index.ts +3 -6
- package/src/main.ts +2 -0
|
@@ -257,6 +257,12 @@ export interface TaskCredentialLeaseRegistryOptions {
|
|
|
257
257
|
assignment: Readonly<TaskAssignment>,
|
|
258
258
|
view: TaskCredentialView,
|
|
259
259
|
): void;
|
|
260
|
+
/** A material update committed while the existing gate stayed open.
|
|
261
|
+
* Followers may reconcile runtime files from this authoritative view. */
|
|
262
|
+
credentialMaterialChanged?(
|
|
263
|
+
assignment: Readonly<TaskAssignment>,
|
|
264
|
+
view: TaskCredentialView,
|
|
265
|
+
): void;
|
|
260
266
|
cleanupExternal(request: CredentialCleanupRequest): Promise<void>;
|
|
261
267
|
/**
|
|
262
268
|
* Installs files/proxy/engine consumers transactionally. The env-replica
|
|
@@ -2279,6 +2285,9 @@ export class TaskCredentialLeaseRegistry {
|
|
|
2279
2285
|
state.bundle = next;
|
|
2280
2286
|
state.bundleGeneration++;
|
|
2281
2287
|
this.#armBundleTimers(state);
|
|
2288
|
+
if (!refused) {
|
|
2289
|
+
this.#opts.credentialMaterialChanged?.(cloneAssignment(state.assignment), next.view);
|
|
2290
|
+
}
|
|
2282
2291
|
// A refusal opens through its release, inside this boundary: an open
|
|
2283
2292
|
// that fails, its compensating close included, escalates below.
|
|
2284
2293
|
release();
|
|
@@ -63,9 +63,11 @@ import {
|
|
|
63
63
|
cleanupPersonCredentials,
|
|
64
64
|
closePersonCredentialGate,
|
|
65
65
|
drainPersonCredentialWrites,
|
|
66
|
+
holdPersonCredentialGate,
|
|
66
67
|
holdTaskGitCredentials,
|
|
67
68
|
installPersonCredentials,
|
|
68
69
|
openPersonCredentialGate,
|
|
70
|
+
type PersonCredentialLane,
|
|
69
71
|
type TaskGitCredentialHold,
|
|
70
72
|
} from "./credential-consumers/github";
|
|
71
73
|
import {
|
|
@@ -160,18 +162,90 @@ export interface CloudCredentialRuntimeDeps {
|
|
|
160
162
|
* on a grant's snapshot, or closed): whoever follows the task's env
|
|
161
163
|
* (the proxy's trust delivery) looks again. */
|
|
162
164
|
envServedChanged?(taskId: string): void;
|
|
165
|
+
/** Reconcile the owner's GitHub/SSH files in an already-running task.
|
|
166
|
+
* The signal closes with the exact person-credential gate, and the caller
|
|
167
|
+
* retries a transient rejection while that gate and material stay current. */
|
|
168
|
+
personCredentialsReady?(
|
|
169
|
+
assignment: Readonly<TaskAssignment>,
|
|
170
|
+
lanes: readonly PersonCredentialLane[],
|
|
171
|
+
signal: AbortSignal,
|
|
172
|
+
): Promise<void>;
|
|
163
173
|
files?: TaskFilesConsumer | null;
|
|
164
174
|
launchDeadlineMs?: number;
|
|
165
175
|
envReadyTimeoutMs?: number;
|
|
166
176
|
shutdownDrainMs?: number;
|
|
167
177
|
/** The first retry's delay for a recovered start whose startup failed. */
|
|
168
178
|
recoveryRetryBaseMs?: number;
|
|
179
|
+
/** The first retry's delay for task-runtime person-credential writes. */
|
|
180
|
+
personReconcileRetryBaseMs?: number;
|
|
169
181
|
registry?: Partial<
|
|
170
182
|
Pick<TaskCredentialLeaseRegistryOptions, "createSessionReader" | "now" | "setTimer" | "clearTimer" | "randomId" | "onError">
|
|
171
183
|
>;
|
|
172
184
|
cleanup?: Pick<TaskCredentialCleanupDeps, "db" | "removePath" | "destroyRuntime">;
|
|
173
185
|
}
|
|
174
186
|
|
|
187
|
+
interface PersonCredentialStamp {
|
|
188
|
+
/** Stable without retaining another copy of any credential material. */
|
|
189
|
+
identity: string;
|
|
190
|
+
available: boolean;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
interface PersonCredentialSnapshot {
|
|
194
|
+
runEpoch: number;
|
|
195
|
+
github: PersonCredentialStamp;
|
|
196
|
+
ssh: PersonCredentialStamp;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function personCredentialStamp(
|
|
200
|
+
credential: TaskCredentialView["github"] | TaskCredentialView["ssh"],
|
|
201
|
+
): PersonCredentialStamp {
|
|
202
|
+
if (credential === null) return { identity: "absent", available: false };
|
|
203
|
+
return {
|
|
204
|
+
identity: [
|
|
205
|
+
credential.manifest.credentialId,
|
|
206
|
+
credential.manifest.revision,
|
|
207
|
+
credential.availability,
|
|
208
|
+
credential.observedVersion ?? "unreadable",
|
|
209
|
+
].join("\u0000"),
|
|
210
|
+
available: credential.availability === "available",
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function personCredentialSnapshot(view: TaskCredentialView): PersonCredentialSnapshot {
|
|
215
|
+
return {
|
|
216
|
+
runEpoch: view.assignment.runEpoch,
|
|
217
|
+
github: personCredentialStamp(view.github),
|
|
218
|
+
ssh: personCredentialStamp(view.ssh),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function samePersonCredentialSnapshot(
|
|
223
|
+
left: PersonCredentialSnapshot | undefined,
|
|
224
|
+
right: PersonCredentialSnapshot,
|
|
225
|
+
): boolean {
|
|
226
|
+
return Boolean(
|
|
227
|
+
left &&
|
|
228
|
+
left.runEpoch === right.runEpoch &&
|
|
229
|
+
left.github.identity === right.github.identity &&
|
|
230
|
+
left.ssh.identity === right.ssh.identity,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** What must be rewritten for `next`. GitHub and SSH transport are coupled:
|
|
235
|
+
* when GitHub leaves, the surviving SSH key must select SSH again. */
|
|
236
|
+
function changedPersonCredentialLanes(
|
|
237
|
+
previous: PersonCredentialSnapshot | undefined,
|
|
238
|
+
next: PersonCredentialSnapshot,
|
|
239
|
+
): PersonCredentialLane[] {
|
|
240
|
+
if (samePersonCredentialSnapshot(previous, next)) return [];
|
|
241
|
+
// The pair shares Git transport configuration. Reconcile both available
|
|
242
|
+
// lanes, including a write interrupted by the other lane changing.
|
|
243
|
+
const lanes: PersonCredentialLane[] = [];
|
|
244
|
+
if (next.github.available) lanes.push("github");
|
|
245
|
+
if (next.ssh.available) lanes.push("ssh");
|
|
246
|
+
return lanes;
|
|
247
|
+
}
|
|
248
|
+
|
|
175
249
|
/**
|
|
176
250
|
* Why a start was interrupted, as its signal's reason: Stop, teardown or a
|
|
177
251
|
* newer run took it over, or the runner is shutting down. The first
|
|
@@ -308,6 +382,13 @@ export class CloudCredentialRuntime {
|
|
|
308
382
|
/** Last agent-facing MCP membership published for each task. Kept across a
|
|
309
383
|
* gate close/renewal because the config remains in the task runtime. */
|
|
310
384
|
readonly #publishedMcpMembership = new Map<string, string>();
|
|
385
|
+
/** Person material last opened or installed for each task. Only identities,
|
|
386
|
+
* availability and observed versions: never another copy of a secret. */
|
|
387
|
+
readonly #personCredentialSnapshots = new Map<string, PersonCredentialSnapshot>();
|
|
388
|
+
readonly #personReconciles = new Map<string, { runEpoch: number; controller: AbortController }>();
|
|
389
|
+
/** Stop fences work before it can acquire the lifecycle slot or close the
|
|
390
|
+
* lease. A late grant cannot restart writes in that interval. */
|
|
391
|
+
readonly #personInterrupted = new Map<string, number>();
|
|
311
392
|
readonly #setTimer: typeof setTimeout;
|
|
312
393
|
readonly #clearTimer: typeof clearTimeout;
|
|
313
394
|
#shutDown = false;
|
|
@@ -337,6 +418,7 @@ export class CloudCredentialRuntime {
|
|
|
337
418
|
send: (frame) => deps.send(frame),
|
|
338
419
|
closeTaskGate: (assignment) => this.#close(assignment),
|
|
339
420
|
openTaskGate: (assignment, view) => this.#openGate(assignment, view),
|
|
421
|
+
credentialMaterialChanged: (assignment, view) => this.#personCredentialsChanged(assignment, view),
|
|
340
422
|
installExternal: (request) => this.#track(this.#install(request)),
|
|
341
423
|
// From a grant's acceptance until its own reader is active, the env
|
|
342
424
|
// replica refuses the run's frames for the cloud to retry: none is
|
|
@@ -379,6 +461,7 @@ export class CloudCredentialRuntime {
|
|
|
379
461
|
throw new CredentialLeaseUnavailableError("a newer run of this task has begun");
|
|
380
462
|
}
|
|
381
463
|
this.#floors.set(assignment.taskId, assignment.runEpoch);
|
|
464
|
+
if (!options.recovery) this.#personInterrupted.delete(assignment.taskId);
|
|
382
465
|
this.#interruptTokens(assignment.taskId, {});
|
|
383
466
|
// A newer run's start ends an older run's recovery; an ordinary start of
|
|
384
467
|
// the run being recovered takes its startup over.
|
|
@@ -418,6 +501,11 @@ export class CloudCredentialRuntime {
|
|
|
418
501
|
*/
|
|
419
502
|
interruptStarts(taskId: string, which: { exact?: number; below?: number } = {}): void {
|
|
420
503
|
this.#interruptTokens(taskId, which);
|
|
504
|
+
const assignment = this.#assignments.get(taskId);
|
|
505
|
+
if (assignment && (which.exact === undefined || assignment.runEpoch === which.exact) &&
|
|
506
|
+
(which.below === undefined || assignment.runEpoch < which.below)) {
|
|
507
|
+
this.#interruptPersonCredentials(assignment);
|
|
508
|
+
}
|
|
421
509
|
// A recovery of the run ends with it, ended start token or not.
|
|
422
510
|
const recovered = this.#recovered.get(taskId);
|
|
423
511
|
if (
|
|
@@ -441,6 +529,8 @@ export class CloudCredentialRuntime {
|
|
|
441
529
|
/** A local Stop: the task's recovery, whatever its run, ends now. One
|
|
442
530
|
* only exists for the run this runner owned at its start. */
|
|
443
531
|
interruptRecovery(taskId: string): void {
|
|
532
|
+
const assignment = this.#assignments.get(taskId);
|
|
533
|
+
if (assignment) this.#interruptPersonCredentials(assignment);
|
|
444
534
|
const recovered = this.#recovered.get(taskId);
|
|
445
535
|
if (recovered) this.#endRecovery(taskId, recovered.runEpoch);
|
|
446
536
|
}
|
|
@@ -776,41 +866,50 @@ export class CloudCredentialRuntime {
|
|
|
776
866
|
if (recovered.running === null) this.#recovered.delete(taskId);
|
|
777
867
|
}
|
|
778
868
|
|
|
779
|
-
/** A recovered
|
|
780
|
-
*
|
|
869
|
+
/** A recovered runtime is ready only after setup succeeds. Transient
|
|
870
|
+
* reconstruction/exec failures retry; Stop, cleanup and takeover fence
|
|
871
|
+
* every attempt and its retry delay. */
|
|
781
872
|
#runRecoveredSetup(assignment: Readonly<TaskAssignment>): void {
|
|
782
873
|
const recovered = this.#recovered.get(assignment.taskId);
|
|
783
874
|
if (!recovered || recovered.runEpoch !== assignment.runEpoch || recovered.running !== null) return;
|
|
784
|
-
if (recovered.controller.signal.aborted) return;
|
|
875
|
+
if (recovered.controller.signal.aborted || this.#open.get(assignment.taskId) !== assignment.runEpoch) return;
|
|
785
876
|
if (recovered.retry !== null) this.#clearTimer(recovered.retry);
|
|
786
877
|
recovered.retry = null;
|
|
787
878
|
const setup = recovered.setup;
|
|
788
|
-
recovered.setup = null;
|
|
789
879
|
if (setup === null) {
|
|
790
880
|
this.#recovered.delete(assignment.taskId);
|
|
791
881
|
return;
|
|
792
882
|
}
|
|
793
|
-
const
|
|
883
|
+
const hold = holdPersonCredentialGate(assignment);
|
|
884
|
+
const signal = AbortSignal.any([recovered.controller.signal, hold.signal]);
|
|
885
|
+
let complete = false;
|
|
794
886
|
recovered.running = this.#track(
|
|
795
887
|
Promise.resolve()
|
|
796
|
-
.then(() => {
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
888
|
+
.then(async () => {
|
|
889
|
+
signal.throwIfAborted();
|
|
890
|
+
await setup(signal);
|
|
891
|
+
signal.throwIfAborted();
|
|
892
|
+
complete = true;
|
|
893
|
+
})
|
|
894
|
+
.catch(() => {
|
|
895
|
+
if (!signal.aborted) {
|
|
896
|
+
console.warn(`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: recovered runtime setup failed; retrying`);
|
|
897
|
+
}
|
|
803
898
|
})
|
|
804
|
-
.catch((error: unknown) =>
|
|
805
|
-
console.warn(
|
|
806
|
-
`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: recovered runtime setup failed: ${
|
|
807
|
-
error instanceof Error ? error.message : String(error)
|
|
808
|
-
}`,
|
|
809
|
-
),
|
|
810
|
-
)
|
|
811
899
|
.finally(() => {
|
|
900
|
+
hold.release();
|
|
812
901
|
recovered.running = null;
|
|
813
|
-
if (this.#recovered.get(assignment.taskId)
|
|
902
|
+
if (this.#recovered.get(assignment.taskId) !== recovered) return;
|
|
903
|
+
if (complete || recovered.controller.signal.aborted || this.#shutDown) {
|
|
904
|
+
this.#recovered.delete(assignment.taskId);
|
|
905
|
+
return;
|
|
906
|
+
}
|
|
907
|
+
const base = this.#deps.recoveryRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
|
|
908
|
+
const delay = Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** Math.min(recovered.attempts++, 20));
|
|
909
|
+
recovered.retry = this.#setTimer(() => {
|
|
910
|
+
recovered.retry = null;
|
|
911
|
+
this.#runRecoveredSetup(assignment);
|
|
912
|
+
}, delay);
|
|
814
913
|
}),
|
|
815
914
|
);
|
|
816
915
|
}
|
|
@@ -983,10 +1082,72 @@ export class CloudCredentialRuntime {
|
|
|
983
1082
|
// Nothing which follows publishes work synchronously. If Files itself
|
|
984
1083
|
// fails, the registry compensates by closing the earlier gates.
|
|
985
1084
|
this.#deps.files?.open(assignment, view);
|
|
1085
|
+
this.#personCredentialsChanged(assignment, view);
|
|
986
1086
|
this.#publishMcpMembership(assignment.taskId, view);
|
|
987
1087
|
this.#runRecoveredSetup(assignment);
|
|
988
1088
|
}
|
|
989
1089
|
|
|
1090
|
+
#personCredentialsChanged(assignment: Readonly<TaskAssignment>, view: TaskCredentialView): void {
|
|
1091
|
+
const reconcile = this.#deps.personCredentialsReady;
|
|
1092
|
+
if (!reconcile || this.#shutDown || this.#open.get(assignment.taskId) !== assignment.runEpoch) return;
|
|
1093
|
+
if ((this.#personInterrupted.get(assignment.taskId) ?? -1) >= assignment.runEpoch) return;
|
|
1094
|
+
const previous = this.#personCredentialSnapshots.get(assignment.taskId);
|
|
1095
|
+
const snapshot = personCredentialSnapshot(view);
|
|
1096
|
+
if (samePersonCredentialSnapshot(previous, snapshot)) return;
|
|
1097
|
+
this.#personCredentialSnapshots.set(assignment.taskId, snapshot);
|
|
1098
|
+
this.#personReconciles.get(assignment.taskId)?.controller.abort();
|
|
1099
|
+
const lanes = changedPersonCredentialLanes(previous, snapshot);
|
|
1100
|
+
if (lanes.length === 0) return;
|
|
1101
|
+
const entry = { runEpoch: assignment.runEpoch, controller: new AbortController() };
|
|
1102
|
+
const hold = holdPersonCredentialGate(assignment);
|
|
1103
|
+
const signal = AbortSignal.any([entry.controller.signal, hold.signal]);
|
|
1104
|
+
this.#personReconciles.set(assignment.taskId, entry);
|
|
1105
|
+
this.#background(
|
|
1106
|
+
Promise.resolve().then(async () => {
|
|
1107
|
+
let attempts = 0;
|
|
1108
|
+
while (!signal.aborted) {
|
|
1109
|
+
try {
|
|
1110
|
+
await reconcile(assignment, lanes, signal);
|
|
1111
|
+
signal.throwIfAborted();
|
|
1112
|
+
return;
|
|
1113
|
+
} catch {
|
|
1114
|
+
if (signal.aborted) return;
|
|
1115
|
+
if (attempts === 0) {
|
|
1116
|
+
console.warn(`[credentials] task ${assignment.taskId} run ${String(assignment.runEpoch)}: person credential files not ready; retrying`);
|
|
1117
|
+
}
|
|
1118
|
+
const base = this.#deps.personReconcileRetryBaseMs ?? RECOVERY_RETRY_BASE_MS;
|
|
1119
|
+
await this.#retryDelay(Math.min(RECOVERY_RETRY_MAX_MS, base * 2 ** Math.min(attempts++, 20)), signal);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
}).finally(() => {
|
|
1123
|
+
hold.release();
|
|
1124
|
+
if (this.#personReconciles.get(assignment.taskId) === entry) this.#personReconciles.delete(assignment.taskId);
|
|
1125
|
+
}),
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
#interruptPersonCredentials(assignment: Readonly<TaskAssignment>): void {
|
|
1130
|
+
this.#personInterrupted.set(assignment.taskId, Math.max(assignment.runEpoch, this.#personInterrupted.get(assignment.taskId) ?? -1));
|
|
1131
|
+
const snapshot = this.#personCredentialSnapshots.get(assignment.taskId);
|
|
1132
|
+
if (snapshot && snapshot.runEpoch <= assignment.runEpoch) this.#personCredentialSnapshots.delete(assignment.taskId);
|
|
1133
|
+
const reconcile = this.#personReconciles.get(assignment.taskId);
|
|
1134
|
+
if (reconcile && reconcile.runEpoch <= assignment.runEpoch) reconcile.controller.abort();
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
/** A retry never holds cleanup for its backoff interval. */
|
|
1138
|
+
#retryDelay(delay: number, signal: AbortSignal): Promise<void> {
|
|
1139
|
+
return new Promise((resolve) => {
|
|
1140
|
+
const finish = (): void => {
|
|
1141
|
+
this.#clearTimer(timer);
|
|
1142
|
+
signal.removeEventListener("abort", finish);
|
|
1143
|
+
resolve();
|
|
1144
|
+
};
|
|
1145
|
+
const timer = this.#setTimer(finish, delay);
|
|
1146
|
+
signal.addEventListener("abort", finish, { once: true });
|
|
1147
|
+
if (signal.aborted) finish();
|
|
1148
|
+
});
|
|
1149
|
+
}
|
|
1150
|
+
|
|
990
1151
|
#envServedChanged(taskId: string): void {
|
|
991
1152
|
try {
|
|
992
1153
|
this.#deps.envServedChanged?.(taskId);
|
|
@@ -1030,8 +1191,11 @@ export class CloudCredentialRuntime {
|
|
|
1030
1191
|
if (open !== undefined && open <= assignment.runEpoch) {
|
|
1031
1192
|
this.#open.delete(assignment.taskId);
|
|
1032
1193
|
this.#served.delete(assignment.taskId);
|
|
1194
|
+
this.#personCredentialSnapshots.delete(assignment.taskId);
|
|
1033
1195
|
this.#envServedChanged(assignment.taskId);
|
|
1034
1196
|
}
|
|
1197
|
+
const person = this.#personReconciles.get(assignment.taskId);
|
|
1198
|
+
if (person && person.runEpoch <= assignment.runEpoch) person.controller.abort();
|
|
1035
1199
|
for (const entry of this.#holds.get(assignment.taskId) ?? []) {
|
|
1036
1200
|
if (entry.runEpoch <= assignment.runEpoch) {
|
|
1037
1201
|
attempt(() => entry.controller.abort(new CredentialLeaseUnavailableError("the task's credentials were withdrawn")));
|
|
@@ -1134,7 +1298,9 @@ export class CloudCredentialRuntime {
|
|
|
1134
1298
|
const recovered = this.#recovered.get(request.taskId);
|
|
1135
1299
|
const setups: Promise<void>[] = [];
|
|
1136
1300
|
if (request.credentialIds === undefined && recovered && recovered.runEpoch <= request.runEpoch && recovered.running) {
|
|
1137
|
-
|
|
1301
|
+
// The gate hold already cancelled this attempt. A transient full
|
|
1302
|
+
// cleanup may reacquire the same run's lease, so retain its setup for
|
|
1303
|
+
// that next open. Stop/takeover cancel recovery at their own boundary.
|
|
1138
1304
|
setups.push(recovered.running);
|
|
1139
1305
|
}
|
|
1140
1306
|
const consumer = consumerCleanupRequest(request);
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
* is recorded in the cleanup inventory before anything is written to it.
|
|
24
24
|
* - Writes for one task run one at a time, and each exec is admitted at the
|
|
25
25
|
* provider's last moment against the task's live state: the same
|
|
26
|
-
* assignment,
|
|
26
|
+
* assignment, credential identity, revision and material version, not
|
|
27
|
+
* sealed. A cleanup seals
|
|
27
28
|
* (or drops the credentials it names), then waits for every write under
|
|
28
29
|
* way to end, so nothing it is about to delete can be written again after.
|
|
29
30
|
*/
|
|
@@ -64,6 +65,16 @@ export interface LeasedSshKey {
|
|
|
64
65
|
publicKey: string;
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
export type PersonCredentialLane = "github" | "ssh";
|
|
69
|
+
|
|
70
|
+
/** Writer-only authority: clone callers do not need to know the version. */
|
|
71
|
+
type VersionedCredential<T> = T & { version: number };
|
|
72
|
+
|
|
73
|
+
interface PersonCredentialSlots {
|
|
74
|
+
github: Slot<VersionedCredential<LeasedGithub>>;
|
|
75
|
+
ssh: Slot<VersionedCredential<LeasedSshKey>>;
|
|
76
|
+
}
|
|
77
|
+
|
|
67
78
|
/** A manifest lane: not named, named but unreadable, or in hand. */
|
|
68
79
|
type Slot<T extends { credentialId: string; revision: string }> =
|
|
69
80
|
| { kind: "absent" }
|
|
@@ -85,14 +96,12 @@ interface GitHold {
|
|
|
85
96
|
release(): void;
|
|
86
97
|
}
|
|
87
98
|
|
|
88
|
-
|
|
99
|
+
/** The inherited slots are what the registry's gate opened on: the exact
|
|
100
|
+
* committed view, absent while closed. Only these serve holds and writers. */
|
|
101
|
+
interface TaskPersonCredentials extends PersonCredentialSlots {
|
|
89
102
|
assignment: TaskAssignment;
|
|
90
|
-
/** What the registry's gate opened on (the exact committed view): the
|
|
91
|
-
* only thing holds and writers read. Absent while closed. */
|
|
92
|
-
github: Slot<LeasedGithub>;
|
|
93
|
-
ssh: Slot<LeasedSshKey>;
|
|
94
103
|
/** What the last install proposed: never served until a gate opens on it. */
|
|
95
|
-
candidate:
|
|
104
|
+
candidate: PersonCredentialSlots;
|
|
96
105
|
/** The registry's gate for this run: nothing is held or written while
|
|
97
106
|
* closed. */
|
|
98
107
|
open: boolean;
|
|
@@ -109,33 +118,59 @@ interface TaskPersonCredentials {
|
|
|
109
118
|
|
|
110
119
|
const byTask = new Map<string, TaskPersonCredentials>();
|
|
111
120
|
|
|
112
|
-
function slotOf<
|
|
121
|
+
function slotOf<
|
|
122
|
+
T extends GithubLeasedValue | SshLeasedValue,
|
|
123
|
+
V extends { credentialId: string; revision: string; version: number },
|
|
124
|
+
>(
|
|
113
125
|
credential: TaskStaticCredential<T> | null,
|
|
114
|
-
value: (material: Readonly<T>) => Omit<V, "credentialId" | "revision">,
|
|
126
|
+
value: (material: Readonly<T>) => Omit<V, "credentialId" | "revision" | "version">,
|
|
115
127
|
): Slot<V> {
|
|
116
128
|
if (!credential) return { kind: "absent" };
|
|
117
129
|
const identity = { credentialId: credential.manifest.credentialId, revision: credential.manifest.revision };
|
|
118
|
-
if (
|
|
119
|
-
|
|
130
|
+
if (
|
|
131
|
+
credential.availability !== "available" ||
|
|
132
|
+
!credential.material ||
|
|
133
|
+
credential.observedVersion === null
|
|
134
|
+
) {
|
|
135
|
+
return { kind: "unavailable", ...identity };
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
kind: "present",
|
|
139
|
+
value: {
|
|
140
|
+
...identity,
|
|
141
|
+
version: credential.observedVersion,
|
|
142
|
+
...value(credential.material),
|
|
143
|
+
} as V,
|
|
144
|
+
};
|
|
120
145
|
}
|
|
121
146
|
|
|
122
|
-
function slotsOf(view: TaskCredentialView):
|
|
147
|
+
function slotsOf(view: TaskCredentialView): PersonCredentialSlots {
|
|
123
148
|
return {
|
|
124
|
-
github: slotOf<GithubLeasedValue, LeasedGithub
|
|
125
|
-
ssh: slotOf<SshLeasedValue, LeasedSshKey
|
|
149
|
+
github: slotOf<GithubLeasedValue, VersionedCredential<LeasedGithub>>(view.github, (material) => ({ token: material.token })),
|
|
150
|
+
ssh: slotOf<SshLeasedValue, VersionedCredential<LeasedSshKey>>(view.ssh, (material) => ({
|
|
126
151
|
privateKey: material.privateKey,
|
|
127
152
|
publicKey: material.publicKey,
|
|
128
153
|
})),
|
|
129
154
|
};
|
|
130
155
|
}
|
|
131
156
|
|
|
132
|
-
|
|
157
|
+
function identityOf(slot: Slot<CredentialIdentity>): CredentialIdentity | null {
|
|
158
|
+
return slot.kind === "present"
|
|
159
|
+
? { credentialId: slot.value.credentialId, revision: slot.value.revision }
|
|
160
|
+
: slot.kind === "unavailable"
|
|
161
|
+
? { credentialId: slot.credentialId, revision: slot.revision }
|
|
162
|
+
: null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** The same named credential at the same revision, readable or not. */
|
|
133
166
|
function sameIdentity<T extends CredentialIdentity>(a: Slot<T>, b: Slot<T>): boolean {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
167
|
+
const left = identityOf(a);
|
|
168
|
+
const right = identityOf(b);
|
|
169
|
+
return Boolean(
|
|
170
|
+
left &&
|
|
171
|
+
right &&
|
|
172
|
+
left.credentialId === right.credentialId &&
|
|
173
|
+
left.revision === right.revision,
|
|
139
174
|
);
|
|
140
175
|
}
|
|
141
176
|
|
|
@@ -194,12 +229,6 @@ export async function cleanupPersonCredentials(request: CredentialCleanupRequest
|
|
|
194
229
|
removed.credentialId === identity.credentialId &&
|
|
195
230
|
(removed.revision === null || removed.revision === identity.revision),
|
|
196
231
|
));
|
|
197
|
-
const identityOf = (slot: Slot<CredentialIdentity>): CredentialIdentity | null =>
|
|
198
|
-
slot.kind === "present"
|
|
199
|
-
? { credentialId: slot.value.credentialId, revision: slot.value.revision }
|
|
200
|
-
: slot.kind === "unavailable"
|
|
201
|
-
? { credentialId: slot.credentialId, revision: slot.revision }
|
|
202
|
-
: null;
|
|
203
232
|
if (request.mode === "full") {
|
|
204
233
|
current.sealed = true;
|
|
205
234
|
current.open = false;
|
|
@@ -286,6 +315,44 @@ export interface TaskGitCredentialHold {
|
|
|
286
315
|
release(): void;
|
|
287
316
|
}
|
|
288
317
|
|
|
318
|
+
/**
|
|
319
|
+
* A task-runtime reconciliation's hold on the exact open person-credential
|
|
320
|
+
* gate. Unlike a clone's hold, it does not require either credential to be
|
|
321
|
+
* readable and exposes no material. Cleanup aborts it and waits for release,
|
|
322
|
+
* so a retry delay or runtime reconstruction cannot outlive the run whose
|
|
323
|
+
* files it is about to write.
|
|
324
|
+
*/
|
|
325
|
+
export interface TaskPersonCredentialGateHold {
|
|
326
|
+
signal: AbortSignal;
|
|
327
|
+
release(): void;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export function holdPersonCredentialGate(
|
|
331
|
+
assignment: TaskAssignment,
|
|
332
|
+
): TaskPersonCredentialGateHold {
|
|
333
|
+
const state = liveState(assignment);
|
|
334
|
+
const controller = new AbortController();
|
|
335
|
+
let settle!: () => void;
|
|
336
|
+
const released = new Promise<void>((resolve) => {
|
|
337
|
+
settle = resolve;
|
|
338
|
+
});
|
|
339
|
+
let releasedOnce = false;
|
|
340
|
+
const hold: GitHold = {
|
|
341
|
+
controller,
|
|
342
|
+
github: identityOf(state.github),
|
|
343
|
+
ssh: identityOf(state.ssh),
|
|
344
|
+
released,
|
|
345
|
+
release: () => {
|
|
346
|
+
if (releasedOnce) return;
|
|
347
|
+
releasedOnce = true;
|
|
348
|
+
state.holds.delete(hold);
|
|
349
|
+
settle();
|
|
350
|
+
},
|
|
351
|
+
};
|
|
352
|
+
state.holds.add(hold);
|
|
353
|
+
return { signal: controller.signal, release: hold.release };
|
|
354
|
+
}
|
|
355
|
+
|
|
289
356
|
/**
|
|
290
357
|
* Hold the owner's GitHub sign-in and SSH key for one host-side git
|
|
291
358
|
* operation of the exact assignment, while the registry's gate is open on
|
|
@@ -361,7 +428,7 @@ export const TASK_SSH_ALLOWED_SIGNERS_PATH = "/home/node/.ssh/allowed_signers";
|
|
|
361
428
|
export const TASK_SSH_SIGNING_GITCONFIG_PATH = "/home/node/.ssh/uai-signing.gitconfig";
|
|
362
429
|
/** SSH as git's transport to GitHub, while the task has no GitHub sign-in. */
|
|
363
430
|
export const TASK_SSH_TRANSPORT_GITCONFIG_PATH = "/home/node/.ssh/uai-transport.gitconfig";
|
|
364
|
-
/** The writer's step was refused: the run,
|
|
431
|
+
/** The writer's step was refused: the run, material or gate moved. */
|
|
365
432
|
export class PersonCredentialWriteRefused extends Error {
|
|
366
433
|
constructor() {
|
|
367
434
|
super("the task's credentials changed or were cleaned up; the write was not made");
|
|
@@ -505,11 +572,18 @@ export function writeTaskGithub(environment: Environment, assignment: TaskAssign
|
|
|
505
572
|
const slot = current().github;
|
|
506
573
|
if (slot.kind === "absent") return false;
|
|
507
574
|
if (slot.kind === "unavailable") throw new CredentialLeaseUnavailableError("the GitHub sign-in could not be read");
|
|
508
|
-
const { credentialId, revision, token } = slot.value;
|
|
509
|
-
// Admitted only while this exact
|
|
575
|
+
const { credentialId, revision, version, token } = slot.value;
|
|
576
|
+
// Admitted only while this exact material version is still the task's.
|
|
510
577
|
const admit = (): void => {
|
|
511
578
|
const live = current().github;
|
|
512
|
-
if (
|
|
579
|
+
if (
|
|
580
|
+
live.kind !== "present" ||
|
|
581
|
+
live.value.credentialId !== credentialId ||
|
|
582
|
+
live.value.revision !== revision ||
|
|
583
|
+
live.value.version !== version
|
|
584
|
+
) {
|
|
585
|
+
throw new PersonCredentialWriteRefused();
|
|
586
|
+
}
|
|
513
587
|
};
|
|
514
588
|
inventory(environment, assignment, "github", credentialId, withStaged(TASK_GH_HOSTS_PATH, TASK_GH_GITCONFIG_PATH));
|
|
515
589
|
const hosts = await run(environment, ["/bin/sh", "-c", GH_LOGIN_SCRIPT], `${token}\n`, "gh auth login", admit, "root");
|
|
@@ -548,10 +622,17 @@ export function writeTaskSshKey(environment: Environment, assignment: TaskAssign
|
|
|
548
622
|
const slot = current().ssh;
|
|
549
623
|
if (slot.kind === "absent") return false;
|
|
550
624
|
if (slot.kind === "unavailable") throw new CredentialLeaseUnavailableError("the SSH key could not be read");
|
|
551
|
-
const { credentialId, revision, privateKey, publicKey } = slot.value;
|
|
625
|
+
const { credentialId, revision, version, privateKey, publicKey } = slot.value;
|
|
552
626
|
const admit = (): void => {
|
|
553
627
|
const live = current().ssh;
|
|
554
|
-
if (
|
|
628
|
+
if (
|
|
629
|
+
live.kind !== "present" ||
|
|
630
|
+
live.value.credentialId !== credentialId ||
|
|
631
|
+
live.value.revision !== revision ||
|
|
632
|
+
live.value.version !== version
|
|
633
|
+
) {
|
|
634
|
+
throw new PersonCredentialWriteRefused();
|
|
635
|
+
}
|
|
555
636
|
};
|
|
556
637
|
inventory(
|
|
557
638
|
environment,
|
|
@@ -589,6 +670,33 @@ export function writeTaskSshKey(environment: Environment, assignment: TaskAssign
|
|
|
589
670
|
});
|
|
590
671
|
}
|
|
591
672
|
|
|
673
|
+
/**
|
|
674
|
+
* Reconcile the task-visible person credentials in their coupled order.
|
|
675
|
+
* GitHub chooses HTTPS and removes SSH transport; SSH then observes whether
|
|
676
|
+
* GitHub is present and either keeps HTTPS or restores SSH transport. Every
|
|
677
|
+
* individual exec remains fenced by the exact run and material version.
|
|
678
|
+
* Absent lanes rely on the coordinator having removed their inventoried
|
|
679
|
+
* files. When GitHub is removed, callers must select the surviving SSH lane
|
|
680
|
+
* even if its credential has not changed, so transport is restored too.
|
|
681
|
+
*/
|
|
682
|
+
export async function writeTaskPersonCredentials(
|
|
683
|
+
environment: Environment,
|
|
684
|
+
assignment: TaskAssignment,
|
|
685
|
+
lanes: readonly PersonCredentialLane[] = ["github", "ssh"],
|
|
686
|
+
signal?: AbortSignal,
|
|
687
|
+
): Promise<void> {
|
|
688
|
+
const selected = new Set(lanes);
|
|
689
|
+
for (const [lane, write] of [
|
|
690
|
+
["github", writeTaskGithub],
|
|
691
|
+
["ssh", writeTaskSshKey],
|
|
692
|
+
] as const) {
|
|
693
|
+
if (!selected.has(lane)) continue;
|
|
694
|
+
signal?.throwIfAborted();
|
|
695
|
+
await write(environment, assignment);
|
|
696
|
+
}
|
|
697
|
+
signal?.throwIfAborted();
|
|
698
|
+
}
|
|
699
|
+
|
|
592
700
|
function sameAssignment(a: TaskAssignment, b: TaskAssignment): boolean {
|
|
593
701
|
return a.taskId === b.taskId && a.runEpoch === b.runEpoch;
|
|
594
702
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Restore GitHub/SSH files when a running task's cloud credentials become
|
|
3
|
+
* usable, after recovery or a live change. The persisted provider locator
|
|
4
|
+
* is the only runtime authority; a missing runtime is a retryable failure.
|
|
5
|
+
* Fresh task-up uses the same writer on its newly provisioned handle.
|
|
6
|
+
*/
|
|
7
|
+
import type { HostTask } from "../db/schema";
|
|
8
|
+
import { CredentialLeaseUnavailableError, type TaskAssignment } from "./cloud-credential-leases";
|
|
9
|
+
import { writeTaskPersonCredentials, type PersonCredentialLane } from "./credential-consumers/github";
|
|
10
|
+
import type { TaskWriteEnvironment } from "./credential-consumers/task-files";
|
|
11
|
+
import { getHostTask, readTaskRunEpochState } from "./runtime-state";
|
|
12
|
+
import { reconstructPersistedTaskEnvironment } from "./task-environment";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The composer retries a rejection while the same material is current and
|
|
16
|
+
* aborts this signal before cleanup or takeover. In particular, a fresh
|
|
17
|
+
* task that is still starting must retry: a rotation after its runtime-ready
|
|
18
|
+
* writes but before it is marked running must not be silently missed.
|
|
19
|
+
*/
|
|
20
|
+
export async function reconcileRunningTaskPersonCredentials(
|
|
21
|
+
assignment: Readonly<TaskAssignment>,
|
|
22
|
+
lanes: readonly PersonCredentialLane[],
|
|
23
|
+
signal: AbortSignal,
|
|
24
|
+
): Promise<void> {
|
|
25
|
+
const current = (): HostTask => {
|
|
26
|
+
signal.throwIfAborted();
|
|
27
|
+
const task = getHostTask(assignment.taskId);
|
|
28
|
+
const run = readTaskRunEpochState(assignment.taskId);
|
|
29
|
+
if (
|
|
30
|
+
task === null ||
|
|
31
|
+
task.statusMirror !== "running" ||
|
|
32
|
+
task.runEpoch !== assignment.runEpoch ||
|
|
33
|
+
run === null ||
|
|
34
|
+
!run.assignmentOpen ||
|
|
35
|
+
run.highestRunEpoch !== assignment.runEpoch ||
|
|
36
|
+
run.runtimeEstablishedEpoch !== assignment.runEpoch
|
|
37
|
+
) {
|
|
38
|
+
throw new CredentialLeaseUnavailableError("the task's running credential assignment is not current");
|
|
39
|
+
}
|
|
40
|
+
return task;
|
|
41
|
+
};
|
|
42
|
+
const task = current();
|
|
43
|
+
const assertCurrent = (): void => {
|
|
44
|
+
const live = current();
|
|
45
|
+
if (
|
|
46
|
+
live.environmentProvider !== task.environmentProvider ||
|
|
47
|
+
live.environmentLocator !== task.environmentLocator ||
|
|
48
|
+
live.envMode !== task.envMode
|
|
49
|
+
) {
|
|
50
|
+
throw new CredentialLeaseUnavailableError("the task's credential runtime has changed");
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
54
|
+
assertCurrent();
|
|
55
|
+
if (environment === null) {
|
|
56
|
+
throw new CredentialLeaseUnavailableError("the task's credential runtime is unavailable");
|
|
57
|
+
}
|
|
58
|
+
// Keep the exact running assignment and cancellation at each provider's
|
|
59
|
+
// last exec boundary too: reconstruction or address lookup can suspend
|
|
60
|
+
// while Stop or a new run replaces the runtime.
|
|
61
|
+
const writable: TaskWriteEnvironment = {
|
|
62
|
+
descriptor: environment.descriptor,
|
|
63
|
+
exec: (request) => {
|
|
64
|
+
assertCurrent();
|
|
65
|
+
return environment.exec({
|
|
66
|
+
...request,
|
|
67
|
+
admit: () => {
|
|
68
|
+
assertCurrent();
|
|
69
|
+
request.admit?.();
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
await writeTaskPersonCredentials(writable, assignment, lanes, signal);
|
|
75
|
+
assertCurrent();
|
|
76
|
+
}
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -15,14 +15,14 @@ import {
|
|
|
15
15
|
whileOnOwnCredentials,
|
|
16
16
|
type LegacyStartPermit,
|
|
17
17
|
} from "../lib/cloud-credential-host";
|
|
18
|
-
import {
|
|
18
|
+
import type { TaskAssignment } from "../lib/cloud-credential-leases";
|
|
19
19
|
import {
|
|
20
20
|
StartInterruptedError,
|
|
21
21
|
type CloudCredentialRuntime,
|
|
22
22
|
type TaskCredentialStart,
|
|
23
23
|
type TaskUpHold,
|
|
24
24
|
} from "../lib/cloud-credential-runtime";
|
|
25
|
-
import {
|
|
25
|
+
import { writeTaskPersonCredentials } from "../lib/credential-consumers/github";
|
|
26
26
|
import { agentClisReady, ensureStandardImage } from "../lib/standard-image";
|
|
27
27
|
import { cloneRepo } from "../lib/repo-clone";
|
|
28
28
|
import { handleFilesOp } from "../lib/shared-files";
|
|
@@ -315,10 +315,7 @@ function leasedTaskUp(
|
|
|
315
315
|
signal,
|
|
316
316
|
);
|
|
317
317
|
}
|
|
318
|
-
|
|
319
|
-
if (signal.aborted) throw new CredentialLeaseUnavailableError("the task's credentials were withdrawn");
|
|
320
|
-
await write(environment, start.assignment);
|
|
321
|
-
}
|
|
318
|
+
await writeTaskPersonCredentials(environment, start.assignment, undefined, signal);
|
|
322
319
|
},
|
|
323
320
|
...(filesV1 ? { filesV1: true as const } : {}),
|
|
324
321
|
};
|
package/src/main.ts
CHANGED
|
@@ -178,6 +178,7 @@ import {
|
|
|
178
178
|
reprobeContainerRuntimeMachineIdentity,
|
|
179
179
|
} from "../lib/container-runtime";
|
|
180
180
|
import { persistedTaskEnvironmentsMatchMachine, reconstructPersistedTaskEnvironment } from "../lib/task-environment";
|
|
181
|
+
import { reconcileRunningTaskPersonCredentials } from "../lib/task-person-credentials";
|
|
181
182
|
import { logVnodePressure } from "../lib/vnode-pressure";
|
|
182
183
|
import { resolveAppleTunnelRoute } from "./apple-tunnel-route";
|
|
183
184
|
import {
|
|
@@ -939,6 +940,7 @@ const cloudCredentials = new CloudCredentialRuntime({
|
|
|
939
940
|
// delivery look again; /env reads it afresh on every command.
|
|
940
941
|
envReplica: new ProjectEnvTaskReplicaManager({ onChange: () => noteProjectEnvViewChange() }),
|
|
941
942
|
envServedChanged: () => noteProjectEnvViewChange(),
|
|
943
|
+
personCredentialsReady: reconcileRunningTaskPersonCredentials,
|
|
942
944
|
files: taskFiles,
|
|
943
945
|
});
|
|
944
946
|
// Leaving its own credentials ends every task's env from the runner's
|