@runuai/host 0.9.150 → 0.9.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1876 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1830 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +623 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
|
@@ -0,0 +1,773 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR-154 §6, §8, §10 on the runner: a task's AI accounts, from its lease.
|
|
3
|
+
*
|
|
4
|
+
* - The registry installs each grant and credential change here, in memory
|
|
5
|
+
* for the task's run only. A grant is a candidate: spawns use exactly the
|
|
6
|
+
* accounts of the view the registry opened its gate on (`openEngineGate`),
|
|
7
|
+
* in the manifest's order, which per engine kind is the owner's personal
|
|
8
|
+
* accounts, then the workspace's (§10). A credential change within the run
|
|
9
|
+
* updates an account already served that is otherwise the same
|
|
10
|
+
* (credential, revision, kind): a refreshed token, or its material
|
|
11
|
+
* becoming readable again or no longer readable.
|
|
12
|
+
* - A task on cloud credentials never falls back to the runner's own store.
|
|
13
|
+
* It has a record here from the registry's first callback for its
|
|
14
|
+
* assignment (the gate closing at its start), closed until a grant opens
|
|
15
|
+
* it: `leasedEnginesFor` is undefined only for a task that is not on cloud
|
|
16
|
+
* credentials. An account the manifest names but the lease could not read
|
|
17
|
+
* is served as unavailable, never as absent.
|
|
18
|
+
* - Env-token engines (Claude, Cursor, pasted API keys) take their account
|
|
19
|
+
* at spawn as exec env values (`engineSpawnEnv`).
|
|
20
|
+
* - Config-directory engines (Codex, Grok sign-ins, OpenCode) read a
|
|
21
|
+
* temporary task-local file: only the lane's files the lease-readable
|
|
22
|
+
* parameter carries (`auth.json`, its refresh token the broker's
|
|
23
|
+
* placeholder), in the account's home (`/home/node/.<kind>-acct-<id>`),
|
|
24
|
+
* owned by `node`, 0600. Each is written only by streaming over an exec's
|
|
25
|
+
* stdin through the no-follow writer, every path and intermediate
|
|
26
|
+
* inventoried first. A home derived from the account's (Codex's per-agent
|
|
27
|
+
* copy) is written the same way, never copied from it.
|
|
28
|
+
* - Before an account is served at a newer version than the run has
|
|
29
|
+
* served, its older versions are held back in every runtime of the run
|
|
30
|
+
* (§6): a launch admitted on one of them that has not started never will.
|
|
31
|
+
* - A changed credential rewrites only its lane's files, atomically, in
|
|
32
|
+
* every home this run wrote it to (`syncEngineHomes`). A removed or
|
|
33
|
+
* refused one is deleted from all of them by the cleanup coordinator, from
|
|
34
|
+
* the inventory; nothing here deletes.
|
|
35
|
+
* - Writes for one task run one at a time, and each exec is admitted at the
|
|
36
|
+
* provider's last moment against the task's live state: the same run, the
|
|
37
|
+
* gate open, the account served readable at the same revision and version.
|
|
38
|
+
* A cleanup closes the gate (or drops the accounts it names), then waits
|
|
39
|
+
* for every write under way to end, so nothing it is about to delete can
|
|
40
|
+
* be written again after.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
import {
|
|
44
|
+
CredentialLeaseUnavailableError,
|
|
45
|
+
type CredentialCleanupRequest,
|
|
46
|
+
type CredentialInstallRequest,
|
|
47
|
+
type TaskAssignment,
|
|
48
|
+
type TaskCredentialLeaseRegistry,
|
|
49
|
+
type TaskCredentialView,
|
|
50
|
+
type TaskEngineCredential,
|
|
51
|
+
} from "../cloud-credential-leases";
|
|
52
|
+
import { rememberCredentialCleanupPath } from "../credential-cleanup-inventory";
|
|
53
|
+
import { idPathSegment, type CredentialScope } from "../../src/cloud-credentials";
|
|
54
|
+
import type { EngineAdmissionFloor } from "./engine-anchor";
|
|
55
|
+
import {
|
|
56
|
+
assertTaskRuntime,
|
|
57
|
+
runInTask,
|
|
58
|
+
TASK_HOME,
|
|
59
|
+
withStaged,
|
|
60
|
+
writeTaskFileArgv,
|
|
61
|
+
type TaskWriteEnvironment,
|
|
62
|
+
} from "./task-files";
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Where a config-directory engine's account lives in the task, the variable
|
|
66
|
+
* that points the engine at it, and whether the engine reads it again at
|
|
67
|
+
* every turn (a one-shot CLI) or only when it starts (Codex's app-server,
|
|
68
|
+
* which must respawn to see a rewritten file).
|
|
69
|
+
*/
|
|
70
|
+
export const ENGINE_ACCOUNT_HOMES: Readonly<
|
|
71
|
+
Record<string, { selectVar: string; prefix: string; readsEachTurn: boolean }>
|
|
72
|
+
> = Object.freeze({
|
|
73
|
+
codex: { selectVar: "CODEX_HOME", prefix: `${TASK_HOME}/.codex-acct-`, readsEachTurn: false },
|
|
74
|
+
grok: { selectVar: "GROK_HOME", prefix: `${TASK_HOME}/.grok-acct-`, readsEachTurn: true },
|
|
75
|
+
opencode: {
|
|
76
|
+
selectVar: "OPENCODE_DATA_DIR",
|
|
77
|
+
prefix: `${TASK_HOME}/.local/share/opencode-acct-`,
|
|
78
|
+
readsEachTurn: true,
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
/** A derived home's suffix: `<home>-agent-<agent>`. */
|
|
83
|
+
const DERIVED_HOME_SUFFIX = /^-agent-[A-Za-z0-9_-]{1,128}$/;
|
|
84
|
+
|
|
85
|
+
/** An account the task may use, readable, as the lease read it. */
|
|
86
|
+
export interface LeasedEngineAccount {
|
|
87
|
+
credentialId: string;
|
|
88
|
+
revision: string;
|
|
89
|
+
engineKind: string;
|
|
90
|
+
scope: CredentialScope;
|
|
91
|
+
label: string;
|
|
92
|
+
authKind: "env" | "config-dir";
|
|
93
|
+
/** The lease parameter's version this material was read at. */
|
|
94
|
+
version: number;
|
|
95
|
+
/** Env-token accounts: the variables the engine's spawn takes. */
|
|
96
|
+
env: Readonly<Record<string, string>> | null;
|
|
97
|
+
/** Config-directory accounts: the lane's files, by name. */
|
|
98
|
+
files: Readonly<Record<string, string>> | null;
|
|
99
|
+
/** Config-directory accounts: the account's home in the task. */
|
|
100
|
+
home: string | null;
|
|
101
|
+
expiresAt: number | null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** An account the manifest names but the lease could not read, under the
|
|
105
|
+
* identity it was named with: never used, never "no such account". */
|
|
106
|
+
export interface UnavailableEngineAccount {
|
|
107
|
+
credentialId: string;
|
|
108
|
+
revision: string;
|
|
109
|
+
engineKind: string;
|
|
110
|
+
scope: CredentialScope;
|
|
111
|
+
label: string;
|
|
112
|
+
authKind: "env" | "config-dir";
|
|
113
|
+
home: string | null;
|
|
114
|
+
unavailable: true;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export type ServedEngineAccount = LeasedEngineAccount | UnavailableEngineAccount;
|
|
118
|
+
|
|
119
|
+
export interface TaskEngineLease {
|
|
120
|
+
assignment: TaskAssignment;
|
|
121
|
+
/** False while the registry's gate is closed: spawn nothing. */
|
|
122
|
+
open: boolean;
|
|
123
|
+
/** In try order; empty while the gate is closed. */
|
|
124
|
+
accounts: readonly ServedEngineAccount[];
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The registry's answer to a reactive refresh (the frozen contract). */
|
|
128
|
+
type RefreshResult = Awaited<ReturnType<TaskCredentialLeaseRegistry["requestCredentialRefresh"]>>;
|
|
129
|
+
|
|
130
|
+
export type EngineCredentialRefresher = (
|
|
131
|
+
assignment: TaskAssignment,
|
|
132
|
+
credentialId: string,
|
|
133
|
+
revision: string,
|
|
134
|
+
) => Promise<Pick<RefreshResult, "outcome">>;
|
|
135
|
+
|
|
136
|
+
interface WrittenHome {
|
|
137
|
+
credentialId: string;
|
|
138
|
+
revision: string;
|
|
139
|
+
/** Null while a write is under way or after one failed: stale until a
|
|
140
|
+
* write completes. */
|
|
141
|
+
version: number | null;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
interface TaskEngineState {
|
|
145
|
+
assignment: TaskAssignment;
|
|
146
|
+
/** The accounts of the view the gate opened on; null while it is closed. */
|
|
147
|
+
served: ServedEngineAccount[] | null;
|
|
148
|
+
/** The writes for this task, one after another. */
|
|
149
|
+
queue: Promise<void>;
|
|
150
|
+
/** Writes queued or under way and not yet ended. */
|
|
151
|
+
writing: number;
|
|
152
|
+
drained: Array<() => void>;
|
|
153
|
+
/** Every home this run wrote an account's files to, and what it wrote. */
|
|
154
|
+
written: Map<string, WrittenHome>;
|
|
155
|
+
/** The newest version this run served of each account's revision: a
|
|
156
|
+
* newer one needs the runtimes' floors raised first. */
|
|
157
|
+
exposed: Map<string, number>;
|
|
158
|
+
/** The version below which this run admits nothing of each account's
|
|
159
|
+
* revision. It rises before the runtimes' floors do and stays risen if
|
|
160
|
+
* they could not: the next install of that version raises them again. */
|
|
161
|
+
heldBack: Map<string, number>;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const tasks = new Map<string, TaskEngineState>();
|
|
165
|
+
/** Every record with writes queued or under way, the replaced ones
|
|
166
|
+
* included: a cleanup waits for those of its run and every older one. */
|
|
167
|
+
const writers = new Set<TaskEngineState>();
|
|
168
|
+
const listeners = new Set<(taskId: string) => void>();
|
|
169
|
+
let refresher: EngineCredentialRefresher | null = null;
|
|
170
|
+
|
|
171
|
+
/** Holds back older versions of accounts in every runtime of the task's
|
|
172
|
+
* run, before a newer one is served (engine-anchor.ts). */
|
|
173
|
+
export type EngineAdmissionFloorRaiser = (
|
|
174
|
+
assignment: TaskAssignment,
|
|
175
|
+
floors: readonly EngineAdmissionFloor[],
|
|
176
|
+
) => Promise<void>;
|
|
177
|
+
let floorRaiser: EngineAdmissionFloorRaiser | null = null;
|
|
178
|
+
|
|
179
|
+
/** Escalates a launch slot of this exact run that could not be retired in
|
|
180
|
+
* its runtime (ADR-154 §6): the run's full credential cleanup, which
|
|
181
|
+
* closes its gate first and proves no launch of it runs (the lease
|
|
182
|
+
* registry's). */
|
|
183
|
+
export type EngineLaunchFenceEscalator = (assignment: TaskAssignment) => void;
|
|
184
|
+
let fenceEscalator: EngineLaunchFenceEscalator | null = null;
|
|
185
|
+
|
|
186
|
+
/** The account's home in the task: null for a kind that reads no file. */
|
|
187
|
+
export function engineAccountHome(engineKind: string, credentialId: string): string | null {
|
|
188
|
+
const home = ENGINE_ACCOUNT_HOMES[engineKind];
|
|
189
|
+
return home ? `${home.prefix}${idPathSegment(credentialId)}` : null;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function accountOf(credential: TaskEngineCredential): ServedEngineAccount {
|
|
193
|
+
const { manifest, material } = credential;
|
|
194
|
+
const home = manifest.authKind === "config-dir" ? engineAccountHome(manifest.engineKind, manifest.credentialId) : null;
|
|
195
|
+
const identity = {
|
|
196
|
+
credentialId: manifest.credentialId,
|
|
197
|
+
revision: manifest.revision,
|
|
198
|
+
engineKind: manifest.engineKind,
|
|
199
|
+
scope: manifest.scope,
|
|
200
|
+
label: manifest.label,
|
|
201
|
+
authKind: manifest.authKind,
|
|
202
|
+
home,
|
|
203
|
+
};
|
|
204
|
+
if (
|
|
205
|
+
credential.availability !== "available" ||
|
|
206
|
+
!material ||
|
|
207
|
+
credential.observedVersion === null ||
|
|
208
|
+
material.authKind !== manifest.authKind ||
|
|
209
|
+
(manifest.authKind === "config-dir" && home === null)
|
|
210
|
+
) {
|
|
211
|
+
return { ...identity, unavailable: true };
|
|
212
|
+
}
|
|
213
|
+
return material.authKind === "env"
|
|
214
|
+
? { ...identity, version: credential.observedVersion, env: { ...material.env }, files: null, expiresAt: null }
|
|
215
|
+
: {
|
|
216
|
+
...identity,
|
|
217
|
+
version: credential.observedVersion,
|
|
218
|
+
env: null,
|
|
219
|
+
files: { ...material.files },
|
|
220
|
+
expiresAt: material.expiresAt,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function accountsOf(view: TaskCredentialView): ServedEngineAccount[] {
|
|
225
|
+
return view.engines.map(accountOf);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export function isLeasedEngineAccount(account: ServedEngineAccount | undefined): account is LeasedEngineAccount {
|
|
229
|
+
return account !== undefined && !("unavailable" in account);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** The same account, all but its material (or whether it could be read). */
|
|
233
|
+
function sameAccount(a: ServedEngineAccount, b: ServedEngineAccount): boolean {
|
|
234
|
+
return (
|
|
235
|
+
a.credentialId === b.credentialId &&
|
|
236
|
+
a.revision === b.revision &&
|
|
237
|
+
a.engineKind === b.engineKind &&
|
|
238
|
+
a.authKind === b.authKind
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function sameAssignment(a: TaskAssignment, b: TaskAssignment): boolean {
|
|
243
|
+
return a.taskId === b.taskId && a.runEpoch === b.runEpoch;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function notify(taskId: string): void {
|
|
247
|
+
for (const listener of [...listeners]) {
|
|
248
|
+
try {
|
|
249
|
+
listener(taskId);
|
|
250
|
+
} catch {
|
|
251
|
+
// A listener's failure is its own; the others still hear.
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** The record for this run: the current one, or a fresh, closed one when
|
|
257
|
+
* this is a newer run (whatever the last run had is closed with it). */
|
|
258
|
+
function stateFor(assignment: TaskAssignment): TaskEngineState {
|
|
259
|
+
const current = tasks.get(assignment.taskId);
|
|
260
|
+
if (current && sameAssignment(current.assignment, assignment)) return current;
|
|
261
|
+
if (current) current.served = null;
|
|
262
|
+
const fresh: TaskEngineState = {
|
|
263
|
+
assignment: { taskId: assignment.taskId, runEpoch: assignment.runEpoch },
|
|
264
|
+
served: null,
|
|
265
|
+
queue: Promise.resolve(),
|
|
266
|
+
writing: 0,
|
|
267
|
+
drained: [],
|
|
268
|
+
written: new Map(),
|
|
269
|
+
exposed: new Map(),
|
|
270
|
+
heldBack: new Map(),
|
|
271
|
+
};
|
|
272
|
+
tasks.set(assignment.taskId, fresh);
|
|
273
|
+
return fresh;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** True when `assignment` is older than the run this runner now holds. */
|
|
277
|
+
function olderRun(assignment: TaskAssignment): boolean {
|
|
278
|
+
const current = tasks.get(assignment.taskId);
|
|
279
|
+
return current !== undefined && current.assignment.runEpoch > assignment.runEpoch;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function exposureKey(account: { credentialId: string; revision: string }): string {
|
|
283
|
+
return `${account.credentialId}\u0000${account.revision}`;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Note the versions `accounts` serve as served this run. */
|
|
287
|
+
function expose(state: TaskEngineState, accounts: readonly ServedEngineAccount[]): void {
|
|
288
|
+
for (const account of accounts) {
|
|
289
|
+
if (!isLeasedEngineAccount(account)) continue;
|
|
290
|
+
const key = exposureKey(account);
|
|
291
|
+
state.exposed.set(key, Math.max(state.exposed.get(key) ?? 0, account.version));
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* A grant's install is a candidate: nothing of it is served until the
|
|
297
|
+
* registry opens its gate on it. A credential change of this run takes
|
|
298
|
+
* effect now, on an account already served that is otherwise the same.
|
|
299
|
+
* Before either can serve an account at a newer version than this run has
|
|
300
|
+
* served, the account's older versions are held back in every runtime of
|
|
301
|
+
* the run (§6, `setEngineAdmissionFloorRaiser`): a launch admitted on an
|
|
302
|
+
* older version that has not started never will. One that cannot be held
|
|
303
|
+
* back fails the install, and none of it is served.
|
|
304
|
+
*/
|
|
305
|
+
export async function installEngineCredentials(request: CredentialInstallRequest): Promise<void> {
|
|
306
|
+
if (olderRun(request.assignment)) return;
|
|
307
|
+
const current = tasks.get(request.assignment.taskId);
|
|
308
|
+
if (request.reason === "credential-change" && current && !sameAssignment(current.assignment, request.assignment)) {
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
// On cloud credentials from its first install, closed until opened: never
|
|
312
|
+
// the runner's own store.
|
|
313
|
+
const state = stateFor(request.assignment);
|
|
314
|
+
const next = accountsOf(request.next);
|
|
315
|
+
const floors = next.filter(isLeasedEngineAccount).flatMap((account) => {
|
|
316
|
+
const served = state.exposed.get(exposureKey(account));
|
|
317
|
+
return served !== undefined && account.version > served
|
|
318
|
+
? [{ credentialId: account.credentialId, revision: account.revision, version: account.version }]
|
|
319
|
+
: [];
|
|
320
|
+
});
|
|
321
|
+
// From here nothing older is admitted on this runner (`engineVersionHeldBack`):
|
|
322
|
+
// a launch checked after this is refused, and the floors revoke in the
|
|
323
|
+
// runtimes what was admitted before. A failed raise leaves it so, and does
|
|
324
|
+
// not count as raised: until the newer version is served, every install of
|
|
325
|
+
// it raises the floors again.
|
|
326
|
+
for (const floor of floors) {
|
|
327
|
+
const key = exposureKey(floor);
|
|
328
|
+
state.heldBack.set(key, Math.max(state.heldBack.get(key) ?? 0, floor.version));
|
|
329
|
+
}
|
|
330
|
+
if (floors.length > 0) {
|
|
331
|
+
// Nobody to hold them back in the runtimes is a raise that failed.
|
|
332
|
+
if (!floorRaiser) throw new Error("an AI account's older version could not be held back: no runtime is reachable");
|
|
333
|
+
await floorRaiser({ ...request.assignment }, floors);
|
|
334
|
+
// The run moved on meanwhile: this install is no longer its.
|
|
335
|
+
if (tasks.get(request.assignment.taskId) !== state) return;
|
|
336
|
+
}
|
|
337
|
+
if (request.reason !== "credential-change" || !state.served) return;
|
|
338
|
+
state.served = state.served.flatMap((served) => {
|
|
339
|
+
const proposed = next.find((account) => account.credentialId === served.credentialId);
|
|
340
|
+
// Gone from the view: never served past it, gate or not.
|
|
341
|
+
if (!proposed) return [];
|
|
342
|
+
return [sameAccount(served, proposed) ? proposed : served];
|
|
343
|
+
});
|
|
344
|
+
expose(state, state.served);
|
|
345
|
+
notify(request.assignment.taskId);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/** The registry opened its gate on this committed view: serve exactly its
|
|
349
|
+
* accounts, in its order. An older run's opening changes nothing. */
|
|
350
|
+
export function openEngineGate(assignment: TaskAssignment, view: TaskCredentialView): void {
|
|
351
|
+
if (olderRun(assignment)) return;
|
|
352
|
+
const state = stateFor(assignment);
|
|
353
|
+
state.served = accountsOf(view);
|
|
354
|
+
expose(state, state.served);
|
|
355
|
+
notify(assignment.taskId);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* The registry closed its gate: spawn and write nothing for the task until
|
|
360
|
+
* it opens again. The first close (at the start of the task's first run
|
|
361
|
+
* here) is what puts the task on cloud credentials. A newer run's close
|
|
362
|
+
* closes the record it finds and does not replace it: the previous run's
|
|
363
|
+
* cleanup, which comes after, still finds its record and waits for its
|
|
364
|
+
* writes. The newer run's record comes with its first install.
|
|
365
|
+
*/
|
|
366
|
+
export function closeEngineGate(assignment: TaskAssignment): void {
|
|
367
|
+
if (olderRun(assignment)) return;
|
|
368
|
+
const current = tasks.get(assignment.taskId);
|
|
369
|
+
if (current) current.served = null;
|
|
370
|
+
else stateFor(assignment);
|
|
371
|
+
notify(assignment.taskId);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* A cleanup: a full one closes the gate and forgets every home this run
|
|
376
|
+
* wrote, a selective one drops the accounts it names and their homes; then
|
|
377
|
+
* it waits for every write under way for the task's run, and any older
|
|
378
|
+
* run's, to end. The coordinator deletes the files, from the inventory.
|
|
379
|
+
*/
|
|
380
|
+
export async function cleanupEngineCredentials(request: CredentialCleanupRequest): Promise<void> {
|
|
381
|
+
const { taskId, runEpoch } = request.assignment;
|
|
382
|
+
const current = tasks.get(taskId);
|
|
383
|
+
if (current && sameAssignment(current.assignment, request.assignment)) {
|
|
384
|
+
if (request.mode === "full") {
|
|
385
|
+
current.served = null;
|
|
386
|
+
current.written.clear();
|
|
387
|
+
} else {
|
|
388
|
+
const removed = new Set(
|
|
389
|
+
request.removed.filter((identity) => identity.class === "engine").map((identity) => identity.credentialId),
|
|
390
|
+
);
|
|
391
|
+
if (current.served) current.served = current.served.filter((account) => !removed.has(account.credentialId));
|
|
392
|
+
for (const [home, written] of current.written) {
|
|
393
|
+
if (removed.has(written.credentialId)) current.written.delete(home);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
notify(taskId);
|
|
397
|
+
} else if (request.mode === "full" && (!current || current.assignment.runEpoch < runEpoch)) {
|
|
398
|
+
// A full cleanup for a run never opened here still puts the task on
|
|
399
|
+
// cloud credentials, closed.
|
|
400
|
+
closeEngineGate(request.assignment);
|
|
401
|
+
}
|
|
402
|
+
await Promise.all(
|
|
403
|
+
[...writers]
|
|
404
|
+
.filter((state) => state.assignment.taskId === taskId && state.assignment.runEpoch <= runEpoch)
|
|
405
|
+
.map(drain),
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function drain(state: TaskEngineState): Promise<void> {
|
|
410
|
+
if (state.writing === 0) return Promise.resolve();
|
|
411
|
+
return new Promise((resolve) => state.drained.push(resolve));
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/** Process shutdown: every engine-home write under way, for any task, over.
|
|
415
|
+
* With every gate closed nothing new is admitted, so this ends. */
|
|
416
|
+
export function drainLeasedEngineWrites(): Promise<void> {
|
|
417
|
+
return Promise.all([...writers].map(drain)).then(() => undefined);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** The task is gone (teardown): forget its record, this run's or an older
|
|
421
|
+
* one's. A newer run's record stays: a delayed teardown never takes it. */
|
|
422
|
+
export function forgetLeasedEngines(assignment: TaskAssignment): void {
|
|
423
|
+
const state = tasks.get(assignment.taskId);
|
|
424
|
+
if (!state || state.assignment.runEpoch > assignment.runEpoch) return;
|
|
425
|
+
state.served = null;
|
|
426
|
+
tasks.delete(assignment.taskId);
|
|
427
|
+
notify(assignment.taskId);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** Whether this run has moved `account` past that version (or is moving it):
|
|
431
|
+
* nothing is admitted on it any more, served or not. */
|
|
432
|
+
export function engineVersionHeldBack(
|
|
433
|
+
assignment: TaskAssignment,
|
|
434
|
+
account: { credentialId: string; revision: string; version: number },
|
|
435
|
+
): boolean {
|
|
436
|
+
const state = tasks.get(assignment.taskId);
|
|
437
|
+
if (!state || !sameAssignment(state.assignment, assignment)) return true;
|
|
438
|
+
const key = exposureKey(account);
|
|
439
|
+
return account.version < Math.max(state.exposed.get(key) ?? 0, state.heldBack.get(key) ?? 0);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** The task's AI accounts, or undefined for a task that is not on cloud
|
|
443
|
+
* credentials. */
|
|
444
|
+
export function leasedEnginesFor(taskId: string): TaskEngineLease | undefined {
|
|
445
|
+
const state = tasks.get(taskId);
|
|
446
|
+
if (!state) return undefined;
|
|
447
|
+
return { assignment: { ...state.assignment }, open: state.served !== null, accounts: [...(state.served ?? [])] };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** The live, open record for this exact run, or a failure: never "none". */
|
|
451
|
+
function openState(assignment: TaskAssignment): TaskEngineState {
|
|
452
|
+
const state = tasks.get(assignment.taskId);
|
|
453
|
+
if (!state || !state.served || !sameAssignment(state.assignment, assignment)) {
|
|
454
|
+
throw new CredentialLeaseUnavailableError("the task's AI accounts are not available for this run");
|
|
455
|
+
}
|
|
456
|
+
return state;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** The accounts of `engineKind` the task may use now, in try order, the
|
|
460
|
+
* unavailable ones included under their identity. Empty when the lease
|
|
461
|
+
* names none of that kind; throws while the gate is closed. */
|
|
462
|
+
export function leasedEngineAccounts(assignment: TaskAssignment, engineKind: string): ServedEngineAccount[] {
|
|
463
|
+
return openState(assignment).served!.filter((account) => account.engineKind === engineKind);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Wait, bounded, for this exact run's gate to be open, and resolve with its
|
|
468
|
+
* accounts. A record for an older run means the run's own record has not
|
|
469
|
+
* come yet: keep waiting. Rejects with `CredentialLeaseUnavailableError` once
|
|
470
|
+
* `timeoutMs` passes, the signal aborts, the task holds a newer run, or the
|
|
471
|
+
* task is not (or no longer) on cloud credentials: a wait never resolves with
|
|
472
|
+
* another run's accounts.
|
|
473
|
+
*/
|
|
474
|
+
export function waitForEngineGate(
|
|
475
|
+
assignment: TaskAssignment,
|
|
476
|
+
timeoutMs: number,
|
|
477
|
+
signal?: AbortSignal,
|
|
478
|
+
): Promise<TaskEngineLease> {
|
|
479
|
+
const check = (): TaskEngineLease | "closed" | Error => {
|
|
480
|
+
const state = tasks.get(assignment.taskId);
|
|
481
|
+
if (!state) return new CredentialLeaseUnavailableError("the task is not on cloud credentials");
|
|
482
|
+
if (state.assignment.runEpoch > assignment.runEpoch) {
|
|
483
|
+
return new CredentialLeaseUnavailableError("the task moved on to another run");
|
|
484
|
+
}
|
|
485
|
+
if (!sameAssignment(state.assignment, assignment) || !state.served) return "closed";
|
|
486
|
+
return { assignment: { ...state.assignment }, open: true, accounts: [...state.served] };
|
|
487
|
+
};
|
|
488
|
+
const now = check();
|
|
489
|
+
if (now instanceof Error) return Promise.reject(now);
|
|
490
|
+
if (now !== "closed") return Promise.resolve(now);
|
|
491
|
+
return new Promise((resolve, reject) => {
|
|
492
|
+
let unsubscribe = (): void => {};
|
|
493
|
+
const finish = (outcome: TaskEngineLease | Error): void => {
|
|
494
|
+
clearTimeout(timer);
|
|
495
|
+
unsubscribe();
|
|
496
|
+
signal?.removeEventListener("abort", onAbort);
|
|
497
|
+
if (outcome instanceof Error) reject(outcome);
|
|
498
|
+
else resolve(outcome);
|
|
499
|
+
};
|
|
500
|
+
const onAbort = (): void => finish(new CredentialLeaseUnavailableError("credentials unavailable"));
|
|
501
|
+
const timer = setTimeout(() => finish(new CredentialLeaseUnavailableError("credentials unavailable")), timeoutMs);
|
|
502
|
+
timer.unref?.();
|
|
503
|
+
unsubscribe = onLeasedEnginesChanged((changed) => {
|
|
504
|
+
if (changed !== assignment.taskId) return;
|
|
505
|
+
const state = check();
|
|
506
|
+
if (state !== "closed") finish(state);
|
|
507
|
+
});
|
|
508
|
+
if (signal?.aborted) onAbort();
|
|
509
|
+
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** Hear every change to what a task's accounts are or whether its gate is
|
|
514
|
+
* open; returns the unsubscribe. */
|
|
515
|
+
export function onLeasedEnginesChanged(listener: (taskId: string) => void): () => void {
|
|
516
|
+
listeners.add(listener);
|
|
517
|
+
return () => {
|
|
518
|
+
listeners.delete(listener);
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/** `account`'s home or a home derived from it; anything else is refused. */
|
|
523
|
+
function homeOf(account: LeasedEngineAccount, home: string | undefined): string {
|
|
524
|
+
if (account.home === null) throw new Error("the account reads no file");
|
|
525
|
+
if (home === undefined || home === account.home) return account.home;
|
|
526
|
+
if (home.startsWith(account.home) && DERIVED_HOME_SUFFIX.test(home.slice(account.home.length))) return home;
|
|
527
|
+
throw new Error("an engine home outside the account's");
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* What a spawn on `account` takes: an env-token account's variables, or the
|
|
532
|
+
* variable that points a config-directory engine at the account's home (or
|
|
533
|
+
* a home derived from it). Never the files' content.
|
|
534
|
+
*/
|
|
535
|
+
export function engineSpawnEnv(account: LeasedEngineAccount, home?: string): Record<string, string> {
|
|
536
|
+
if (account.authKind === "env") return { ...(account.env ?? {}) };
|
|
537
|
+
const selector = ENGINE_ACCOUNT_HOMES[account.engineKind];
|
|
538
|
+
if (!selector) throw new Error("the account's engine reads no file");
|
|
539
|
+
return { [selector.selectVar]: homeOf(account, home) };
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** The writer's step was refused: the run, the gate, the revision or the
|
|
543
|
+
* version moved. */
|
|
544
|
+
export class EngineCredentialWriteRefused extends Error {
|
|
545
|
+
constructor() {
|
|
546
|
+
super("the task's AI accounts changed or were cleaned up; the write was not made");
|
|
547
|
+
this.name = "EngineCredentialWriteRefused";
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* Queue one write for the task: it starts once the writes before it ended,
|
|
553
|
+
* counts as under way until it ends (a cleanup waits for it), and reads the
|
|
554
|
+
* task's live state through `current`, which refuses once the run moved on.
|
|
555
|
+
*/
|
|
556
|
+
function enqueueWrite<T>(assignment: TaskAssignment, write: (current: () => TaskEngineState) => Promise<T>): Promise<T> {
|
|
557
|
+
const state = openState(assignment);
|
|
558
|
+
state.writing += 1;
|
|
559
|
+
writers.add(state);
|
|
560
|
+
const settle = (): void => {
|
|
561
|
+
state.writing -= 1;
|
|
562
|
+
if (state.writing > 0) return;
|
|
563
|
+
writers.delete(state);
|
|
564
|
+
for (const resolve of state.drained.splice(0)) resolve();
|
|
565
|
+
};
|
|
566
|
+
const current = (): TaskEngineState => {
|
|
567
|
+
if (tasks.get(assignment.taskId) !== state || !state.served || !sameAssignment(state.assignment, assignment)) {
|
|
568
|
+
throw new EngineCredentialWriteRefused();
|
|
569
|
+
}
|
|
570
|
+
return state;
|
|
571
|
+
};
|
|
572
|
+
const result = state.queue.then(() => write(current));
|
|
573
|
+
state.queue = result.then(
|
|
574
|
+
() => undefined,
|
|
575
|
+
() => undefined,
|
|
576
|
+
);
|
|
577
|
+
return result.finally(settle);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
/** The account as served now, readable, or a refusal. */
|
|
581
|
+
function servedNow(state: TaskEngineState, credentialId: string): LeasedEngineAccount {
|
|
582
|
+
const account = state.served?.find((candidate) => candidate.credentialId === credentialId);
|
|
583
|
+
if (!account) throw new EngineCredentialWriteRefused();
|
|
584
|
+
if (!isLeasedEngineAccount(account)) throw new CredentialLeaseUnavailableError("the AI account could not be read");
|
|
585
|
+
return account;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/**
|
|
589
|
+
* Write `credentialId`'s lane files into its home in the task (or a home
|
|
590
|
+
* derived from it): every path and intermediate inventoried first, each file
|
|
591
|
+
* through the no-follow writer, owner-only, admitted only while the account
|
|
592
|
+
* is served readable at the revision and version it began with. Resolves to
|
|
593
|
+
* the version written. Throws `CredentialLeaseUnavailableError` when the
|
|
594
|
+
* account is named but unreadable, and `EngineCredentialWriteRefused` when it
|
|
595
|
+
* is not served, or the run, gate, revision or version moved under it.
|
|
596
|
+
*/
|
|
597
|
+
export function writeEngineHome(
|
|
598
|
+
environment: TaskWriteEnvironment,
|
|
599
|
+
assignment: TaskAssignment,
|
|
600
|
+
credentialId: string,
|
|
601
|
+
home?: string,
|
|
602
|
+
): Promise<number> {
|
|
603
|
+
assertTaskRuntime(environment, assignment.taskId);
|
|
604
|
+
return enqueueWrite(assignment, async (current) => {
|
|
605
|
+
const account = servedNow(current(), credentialId);
|
|
606
|
+
if (account.authKind !== "config-dir" || !account.files) throw new Error("the account reads no file");
|
|
607
|
+
const target = homeOf(account, home);
|
|
608
|
+
const { revision, version } = account;
|
|
609
|
+
const admit = (): void => {
|
|
610
|
+
const live = servedNow(current(), credentialId);
|
|
611
|
+
if (live.revision !== revision || live.version !== version) throw new EngineCredentialWriteRefused();
|
|
612
|
+
};
|
|
613
|
+
const names = Object.keys(account.files).sort();
|
|
614
|
+
for (const path of withStaged(...names.map((name) => `${target}/${name}`))) {
|
|
615
|
+
rememberCredentialCleanupPath({
|
|
616
|
+
taskId: assignment.taskId,
|
|
617
|
+
runEpoch: assignment.runEpoch,
|
|
618
|
+
runtime: environment.descriptor.locator,
|
|
619
|
+
lane: "engine-auth",
|
|
620
|
+
path,
|
|
621
|
+
credentialId,
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
// Stale until this write completes: a refused or failed one is written
|
|
625
|
+
// again by the next sync.
|
|
626
|
+
current().written.set(target, { credentialId, revision, version: null });
|
|
627
|
+
for (const name of names) {
|
|
628
|
+
await runInTask(
|
|
629
|
+
environment,
|
|
630
|
+
writeTaskFileArgv(`${target}/${name}`, "600"),
|
|
631
|
+
account.files[name]!,
|
|
632
|
+
`writing the ${account.engineKind} account`,
|
|
633
|
+
admit,
|
|
634
|
+
);
|
|
635
|
+
}
|
|
636
|
+
// Each exec was admitted at its last moment, but the account can still
|
|
637
|
+
// have moved while the last one ran: the home holds what was written,
|
|
638
|
+
// and is current only if that is still what the task is served.
|
|
639
|
+
const live = current();
|
|
640
|
+
if (live.written.get(target)?.credentialId === credentialId) {
|
|
641
|
+
live.written.set(target, { credentialId, revision, version });
|
|
642
|
+
}
|
|
643
|
+
const now = servedNow(live, credentialId);
|
|
644
|
+
if (now.revision !== revision || now.version !== version) throw new EngineCredentialWriteRefused();
|
|
645
|
+
return version;
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* What a spawn on a config-directory account needs first: `credentialId`'s
|
|
651
|
+
* files in `home` (its home, or one derived from it) at the version served
|
|
652
|
+
* now. Writes them unless this run already did; resolves to that version.
|
|
653
|
+
*/
|
|
654
|
+
export async function ensureEngineHome(
|
|
655
|
+
environment: TaskWriteEnvironment,
|
|
656
|
+
assignment: TaskAssignment,
|
|
657
|
+
credentialId: string,
|
|
658
|
+
home?: string,
|
|
659
|
+
): Promise<number> {
|
|
660
|
+
const state = openState(assignment);
|
|
661
|
+
const account = state.served!.find((candidate) => candidate.credentialId === credentialId);
|
|
662
|
+
if (isLeasedEngineAccount(account) && account.home !== null) {
|
|
663
|
+
const written = state.written.get(homeOf(account, home));
|
|
664
|
+
if (
|
|
665
|
+
written?.credentialId === credentialId &&
|
|
666
|
+
written.revision === account.revision &&
|
|
667
|
+
written.version === account.version
|
|
668
|
+
) {
|
|
669
|
+
return account.version;
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
return writeEngineHome(environment, assignment, credentialId, home);
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Write again, in every home this run wrote it to, each account whose
|
|
677
|
+
* material moved since (a refreshed token, a write that did not complete).
|
|
678
|
+
* Resolves to the accounts written again; an account served unreadable is
|
|
679
|
+
* left as it is. Throws on the first write that fails.
|
|
680
|
+
*/
|
|
681
|
+
export async function syncEngineHomes(environment: TaskWriteEnvironment, assignment: TaskAssignment): Promise<string[]> {
|
|
682
|
+
const state = openState(assignment);
|
|
683
|
+
const stale: Array<[string, string]> = [];
|
|
684
|
+
for (const [home, written] of state.written) {
|
|
685
|
+
const account = state.served!.find((candidate) => candidate.credentialId === written.credentialId);
|
|
686
|
+
if (!isLeasedEngineAccount(account)) continue;
|
|
687
|
+
if (account.revision !== written.revision || account.version !== written.version) {
|
|
688
|
+
stale.push([home, written.credentialId]);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
const rewritten = new Set<string>();
|
|
692
|
+
for (const [home, credentialId] of stale) {
|
|
693
|
+
await writeEngineHome(environment, assignment, credentialId, home);
|
|
694
|
+
rewritten.add(credentialId);
|
|
695
|
+
}
|
|
696
|
+
return [...rewritten];
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/** The homes this run wrote, with the account each holds; none for another
|
|
700
|
+
* run. */
|
|
701
|
+
export function writtenEngineHomes(assignment: TaskAssignment): Array<{ home: string; credentialId: string }> {
|
|
702
|
+
const state = tasks.get(assignment.taskId);
|
|
703
|
+
if (!state || !sameAssignment(state.assignment, assignment)) return [];
|
|
704
|
+
return [...state.written].map(([home, written]) => ({ home, credentialId: written.credentialId }));
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
export function setEngineCredentialRefresher(next: EngineCredentialRefresher | null): void {
|
|
708
|
+
refresher = next;
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/** Who holds back an account's older versions in the task's runtimes
|
|
712
|
+
* before a newer one is served (the orchestrator, which knows them). */
|
|
713
|
+
export function setEngineAdmissionFloorRaiser(next: EngineAdmissionFloorRaiser | null): void {
|
|
714
|
+
floorRaiser = next;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/** Who escalates a launch slot that could not be retired in its runtime
|
|
718
|
+
* (the runner's credentials runtime, through its lease registry). */
|
|
719
|
+
export function setEngineLaunchFenceEscalator(next: EngineLaunchFenceEscalator | null): void {
|
|
720
|
+
fenceEscalator = next;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* ADR-154 §6: a launch slot of `assignment`'s run could not be retired in
|
|
725
|
+
* its runtime, so a launch handed over before its retirement may still
|
|
726
|
+
* start: the run's full credential cleanup proves none does, its gate closed
|
|
727
|
+
* first (no runner on cloud credentials: nothing to escalate to). Returns
|
|
728
|
+
* whether it was asked for.
|
|
729
|
+
*/
|
|
730
|
+
export function escalateEngineLaunchFence(assignment: TaskAssignment): boolean {
|
|
731
|
+
if (!fenceEscalator) return false;
|
|
732
|
+
fenceEscalator({ taskId: assignment.taskId, runEpoch: assignment.runEpoch });
|
|
733
|
+
return true;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/**
|
|
737
|
+
* The reactive path (§7): an engine reported its credential invalid. Ask
|
|
738
|
+
* the cloud for a refresh of the account it ran on, for this exact run, and
|
|
739
|
+
* report whether the run now holds a newer version of it, readable
|
|
740
|
+
* ("refreshed"); "unchanged" when the answer brought nothing newer (busy,
|
|
741
|
+
* unavailable, a new sign-in needed, or the account moved on), "unavailable"
|
|
742
|
+
* when the run cannot ask: its gate closed, another run, no such account.
|
|
743
|
+
*/
|
|
744
|
+
export async function refreshLeasedEngineAccount(
|
|
745
|
+
assignment: TaskAssignment,
|
|
746
|
+
credentialId: string,
|
|
747
|
+
): Promise<"refreshed" | "unchanged" | "unavailable"> {
|
|
748
|
+
const state = tasks.get(assignment.taskId);
|
|
749
|
+
if (!state || !sameAssignment(state.assignment, assignment) || !refresher) return "unavailable";
|
|
750
|
+
const used = state.served?.find((account) => account.credentialId === credentialId);
|
|
751
|
+
if (!isLeasedEngineAccount(used)) return "unavailable";
|
|
752
|
+
try {
|
|
753
|
+
const { outcome } = await refresher({ ...assignment }, credentialId, used.revision);
|
|
754
|
+
if (outcome !== "refreshed" && outcome !== "fresh") return "unchanged";
|
|
755
|
+
} catch {
|
|
756
|
+
return "unavailable";
|
|
757
|
+
}
|
|
758
|
+
if (tasks.get(assignment.taskId) !== state || !sameAssignment(state.assignment, assignment)) return "unavailable";
|
|
759
|
+
const now = state.served?.find((account) => account.credentialId === credentialId);
|
|
760
|
+
return isLeasedEngineAccount(now) && now.revision === used.revision && now.version > used.version
|
|
761
|
+
? "refreshed"
|
|
762
|
+
: "unchanged";
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
/** Tests only. */
|
|
766
|
+
export function resetLeasedEnginesForTests(): void {
|
|
767
|
+
tasks.clear();
|
|
768
|
+
writers.clear();
|
|
769
|
+
listeners.clear();
|
|
770
|
+
refresher = null;
|
|
771
|
+
floorRaiser = null;
|
|
772
|
+
fenceEscalator = null;
|
|
773
|
+
}
|