@runuai/host 0.9.150 → 0.9.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1876 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1830 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +623 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
|
@@ -21,11 +21,13 @@ import { dirname, resolve } from "node:path";
|
|
|
21
21
|
import { fileURLToPath } from "node:url";
|
|
22
22
|
import { promisify } from "node:util";
|
|
23
23
|
|
|
24
|
-
import
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
import {
|
|
25
|
+
AgentError,
|
|
26
|
+
assertLeasedTaskUp,
|
|
27
|
+
type TaskDownResult,
|
|
28
|
+
type TaskLaunchInput,
|
|
29
|
+
type TaskUpCredentials,
|
|
30
|
+
type TaskUpResult,
|
|
29
31
|
} from "../agent";
|
|
30
32
|
import { injectCodexIntoMachine } from "../codex-auth";
|
|
31
33
|
import {
|
|
@@ -59,13 +61,16 @@ import {
|
|
|
59
61
|
type MachineTaskInput,
|
|
60
62
|
} from "./machine";
|
|
61
63
|
import type {
|
|
64
|
+
TaskEnvironmentCallOptions,
|
|
62
65
|
ProjectEnvMode,
|
|
63
66
|
TaskEnvironmentDescriptor,
|
|
64
67
|
TaskEnvironmentHandle,
|
|
68
|
+
TaskEnvironmentProcess,
|
|
65
69
|
TaskEnvironmentProvider,
|
|
66
70
|
TaskEnvironmentProvisionRequest,
|
|
67
71
|
TaskEnvironmentProvisioned,
|
|
68
72
|
} from "./types";
|
|
73
|
+
import { TASK_SSH_KEY_PATH } from "../credential-consumers/github";
|
|
69
74
|
|
|
70
75
|
/** In-machine clone budget per project. */
|
|
71
76
|
const CLONE_TIMEOUT_MS = 10 * 60_000;
|
|
@@ -102,6 +107,9 @@ export interface MachineHostTaskUpDeps {
|
|
|
102
107
|
) => Promise<string | null>;
|
|
103
108
|
/** Test seam for the host-side persistent reverse-tunnel supervisor. */
|
|
104
109
|
ensureGatewayTunnel?: typeof ensureMachineGatewayTunnelReady;
|
|
110
|
+
/** How long a start that is not one waits for the stages it left running
|
|
111
|
+
* (tests). */
|
|
112
|
+
abandonJoinMs?: number;
|
|
105
113
|
}
|
|
106
114
|
|
|
107
115
|
async function startMachineTaskForwarder(
|
|
@@ -196,15 +204,31 @@ async function execWithStdin(
|
|
|
196
204
|
argv: readonly [string, ...string[]],
|
|
197
205
|
stdin: string,
|
|
198
206
|
timeoutMs: number,
|
|
207
|
+
/** Aborted: the process is ended at once (ADR-154 §12, a withdrawn start). */
|
|
208
|
+
signal?: AbortSignal,
|
|
209
|
+
user?: string,
|
|
199
210
|
): Promise<StepResult> {
|
|
211
|
+
if (signal?.aborted) throw new Error(WITHDRAWN);
|
|
200
212
|
const proc = await handle.spawn({
|
|
201
213
|
argv,
|
|
214
|
+
...(user === undefined ? {} : { user }),
|
|
202
215
|
projectEnvProfile: "maintenance",
|
|
203
216
|
inheritEnv: [],
|
|
204
217
|
env: { GIT_TERMINAL_PROMPT: "0" },
|
|
205
218
|
timeoutMs,
|
|
206
219
|
maxOutputBytes: STEP_OUTPUT_BYTES,
|
|
207
220
|
});
|
|
221
|
+
const onAbort = (): void => void proc.terminate("SIGKILL").catch(() => {});
|
|
222
|
+
if (signal?.aborted) onAbort();
|
|
223
|
+
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
224
|
+
try {
|
|
225
|
+
return await drainStep(proc, stdin);
|
|
226
|
+
} finally {
|
|
227
|
+
signal?.removeEventListener("abort", onAbort);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
async function drainStep(proc: TaskEnvironmentProcess, stdin: string): Promise<StepResult> {
|
|
208
232
|
let stderr = "";
|
|
209
233
|
const drainOut = (async () => {
|
|
210
234
|
for await (const chunk of proc.stdout) {
|
|
@@ -247,6 +271,97 @@ async function execStep(
|
|
|
247
271
|
const CREDENTIALED_CLONE_SCRIPT =
|
|
248
272
|
'IFS= read -r UAI_GIT_TOKEN && export UAI_GIT_TOKEN GIT_ASKPASS=/usr/local/bin/uai-git-askpass GIT_TERMINAL_PROMPT=0 && exec git clone -- "$1" "$2"';
|
|
249
273
|
|
|
274
|
+
const WITHDRAWN = "the task's credentials were withdrawn while it was starting; start the task again";
|
|
275
|
+
|
|
276
|
+
/** The task's user on a machine, and in sidecar mode. */
|
|
277
|
+
const MACHINE_TASK_USER = "node";
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* ADR-154 §12: a task on cloud credentials clones by exactly the route its
|
|
281
|
+
* lease names, with nothing of the machine's own and nothing the task could
|
|
282
|
+
* have put in its own login:
|
|
283
|
+
* - the clone starts as root, whose login is not the task's: a profile the
|
|
284
|
+
* task wrote could otherwise read the token off stdin before anything
|
|
285
|
+
* else, or put its own `env`, `git`, `GIT_EXEC_PATH` or `LD_PRELOAD` in
|
|
286
|
+
* the clone's way;
|
|
287
|
+
* - in an environment rebuilt from nothing (`env -i`): a fixed PATH and no
|
|
288
|
+
* home (so no .netrc, ~/.gitconfig or ~/.ssh/config), no global or system
|
|
289
|
+
* git configuration;
|
|
290
|
+
* - then as the task's user (`setpriv`, when root), in a shell that reads no
|
|
291
|
+
* profile. It alone reads the token from stdin, and runs git and ssh by
|
|
292
|
+
* absolute path, with no credential helper and no agent.
|
|
293
|
+
*/
|
|
294
|
+
const LEASED_CLONE_ENV = [
|
|
295
|
+
"PATH=/usr/bin:/bin",
|
|
296
|
+
"HOME=/nonexistent",
|
|
297
|
+
"XDG_CONFIG_HOME=/nonexistent",
|
|
298
|
+
"LC_ALL=C",
|
|
299
|
+
"GIT_CONFIG_NOSYSTEM=1",
|
|
300
|
+
"GIT_CONFIG_GLOBAL=/dev/null",
|
|
301
|
+
"GIT_TERMINAL_PROMPT=0",
|
|
302
|
+
] as const;
|
|
303
|
+
/** Root drops to the task's user; anyone else (a non-root dev sidecar, whose
|
|
304
|
+
* user is the task's) stays. The route's script arrives as $1, never on
|
|
305
|
+
* stdin, which only it reads. */
|
|
306
|
+
const LEASED_CLONE_DROP =
|
|
307
|
+
'if [ "$(/usr/bin/id -u)" = 0 ]; then ' +
|
|
308
|
+
`exec /usr/bin/setpriv --reuid=${MACHINE_TASK_USER} --regid=${MACHINE_TASK_USER} --init-groups -- /bin/sh -c "$1" clone "$2" "$3"; ` +
|
|
309
|
+
'fi; exec /bin/sh -c "$1" clone "$2" "$3"';
|
|
310
|
+
const LEASED_GIT = "exec /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null";
|
|
311
|
+
const LEASED_TOKEN_CLONE_SCRIPT =
|
|
312
|
+
'IFS= read -r UAI_GIT_TOKEN && export UAI_GIT_TOKEN GIT_ASKPASS=/usr/local/bin/uai-git-askpass && ' +
|
|
313
|
+
`${LEASED_GIT} clone -- "$1" "$2"`;
|
|
314
|
+
const LEASED_SSH_CLONE_SCRIPT =
|
|
315
|
+
`export GIT_SSH_COMMAND="/usr/bin/ssh -F /dev/null -i ${TASK_SSH_KEY_PATH} -o IdentitiesOnly=yes -o IdentityAgent=none -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/home/node/.ssh/known_hosts" && ` +
|
|
316
|
+
`${LEASED_GIT} -c url.git@github.com:.insteadOf=https://github.com/ clone -- "$1" "$2"`;
|
|
317
|
+
const LEASED_ANONYMOUS_CLONE_SCRIPT = `${LEASED_GIT} clone -- "$1" "$2"`;
|
|
318
|
+
|
|
319
|
+
/** The leased clone's command, run as root (see above). */
|
|
320
|
+
export function leasedCloneArgv(script: string, url: string, destination: string): [string, ...string[]] {
|
|
321
|
+
return ["/usr/bin/env", "-i", ...LEASED_CLONE_ENV, "/bin/sh", "-c", LEASED_CLONE_DROP, "drop", script, url, destination];
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** How a task on cloud credentials clones: its lease's token, its SSH key,
|
|
325
|
+
* or neither (it names neither). */
|
|
326
|
+
interface LeasedCloneRoute {
|
|
327
|
+
ssh: boolean;
|
|
328
|
+
signal: AbortSignal;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const DENIED =
|
|
332
|
+
/repository not found|permission denied|access rights|could not read username|authentication failed|invalid username or password|returned error: 40[134]|HTTP 40[134]/i;
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* A fresh clone by a leased route that failed: the start fails and the
|
|
336
|
+
* machine goes (unlike the legacy route's warning, ADR-121), saying which of
|
|
337
|
+
* the owner's credentials GitHub refused and what to do about it.
|
|
338
|
+
*/
|
|
339
|
+
function leasedCloneFailure(repoUrl: string, route: "token" | "ssh" | "none", stderr: string): AgentError {
|
|
340
|
+
const step = "clone";
|
|
341
|
+
if (isGithubRepoUrl(repoUrl) && DENIED.test(stderr)) {
|
|
342
|
+
if (route === "token") {
|
|
343
|
+
return new AgentError(
|
|
344
|
+
"GITHUB_TOKEN_ACCESS_DENIED",
|
|
345
|
+
`GitHub denied the task owner's GitHub sign-in access to ${repoUrl}. Confirm that the Uai GitHub App is installed on the owning account or organization, this repository is included in its grant, and the owner can read it. Then start the task again.`,
|
|
346
|
+
{ step },
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
if (route === "ssh") {
|
|
350
|
+
return new AgentError(
|
|
351
|
+
"GITHUB_SSH_ACCESS_DENIED",
|
|
352
|
+
`GitHub denied the task owner's SSH key access to ${repoUrl}. Register that key with an account that can read the repository (and authorize it for SAML SSO when required), or connect GitHub in Settings. Then start the task again.`,
|
|
353
|
+
{ step },
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
return new AgentError(
|
|
357
|
+
"GITHUB_CONNECTION_REQUIRED",
|
|
358
|
+
`The task owner has neither a GitHub sign-in nor an SSH key in Uai that can read ${repoUrl}. Connect GitHub in Settings, then start the task again.`,
|
|
359
|
+
{ step },
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
return new AgentError("CLONE_FAILED", `could not clone ${repoUrl}: ${stderr || "clone failed"}`, { step });
|
|
363
|
+
}
|
|
364
|
+
|
|
250
365
|
type ProjectEnvSchemaObservation = NonNullable<TaskUpResult["envSchemas"]>[number];
|
|
251
366
|
|
|
252
367
|
/** What a project's clone step did: cloned it now, found the task's own
|
|
@@ -263,6 +378,8 @@ async function cloneProject(
|
|
|
263
378
|
taskBranch: string,
|
|
264
379
|
owner: { name?: string; email?: string },
|
|
265
380
|
token: string | null,
|
|
381
|
+
/** A task on cloud credentials: its exact route, and its withdrawal. */
|
|
382
|
+
leased?: LeasedCloneRoute,
|
|
266
383
|
): Promise<CloneOutcome> {
|
|
267
384
|
const url = machineCloneUrl(project.repoUrl);
|
|
268
385
|
const useToken = token !== null && isGithubRepoUrl(project.repoUrl);
|
|
@@ -275,19 +392,37 @@ async function cloneProject(
|
|
|
275
392
|
GIT_STEP_TIMEOUT_MS,
|
|
276
393
|
);
|
|
277
394
|
if (already.exitCode === 0) return { clone: "existing", warning: null };
|
|
395
|
+
// Withdrawn while the probe ran: the clone never starts.
|
|
396
|
+
if (leased?.signal.aborted) throw new Error(WITHDRAWN);
|
|
278
397
|
|
|
279
|
-
const
|
|
398
|
+
const route = useToken ? "token" : leased?.ssh && isGithubRepoUrl(project.repoUrl) ? "ssh" : "none";
|
|
399
|
+
const clone = leased
|
|
280
400
|
? await execWithStdin(
|
|
281
401
|
handle,
|
|
282
|
-
|
|
283
|
-
|
|
402
|
+
leasedCloneArgv(
|
|
403
|
+
route === "token" ? LEASED_TOKEN_CLONE_SCRIPT : route === "ssh" ? LEASED_SSH_CLONE_SCRIPT : LEASED_ANONYMOUS_CLONE_SCRIPT,
|
|
404
|
+
url,
|
|
405
|
+
destination,
|
|
406
|
+
),
|
|
407
|
+
useToken ? `${token}\n` : "",
|
|
284
408
|
CLONE_TIMEOUT_MS,
|
|
409
|
+
leased.signal,
|
|
410
|
+
"root",
|
|
285
411
|
)
|
|
286
|
-
:
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
412
|
+
: useToken
|
|
413
|
+
? await execWithStdin(
|
|
414
|
+
handle,
|
|
415
|
+
["/bin/sh", "-c", CREDENTIALED_CLONE_SCRIPT, "clone", url, destination],
|
|
416
|
+
`${token}\n`,
|
|
417
|
+
CLONE_TIMEOUT_MS,
|
|
418
|
+
)
|
|
419
|
+
: await execStep(
|
|
420
|
+
handle,
|
|
421
|
+
["git", "clone", "--", url, destination],
|
|
422
|
+
CLONE_TIMEOUT_MS,
|
|
423
|
+
);
|
|
424
|
+
if (leased?.signal.aborted) throw new Error(WITHDRAWN);
|
|
425
|
+
if (clone.exitCode !== 0 && leased) throw leasedCloneFailure(project.repoUrl, route, clone.stderr);
|
|
291
426
|
if (clone.exitCode !== 0) {
|
|
292
427
|
// The commonest failure by far is an ANONYMOUS clone of a private GitHub
|
|
293
428
|
// repo because this host has no GitHub credential for the task owner —
|
|
@@ -629,6 +764,122 @@ export async function ensureCodeServer(
|
|
|
629
764
|
}
|
|
630
765
|
}
|
|
631
766
|
|
|
767
|
+
/** How long a start that is not one waits for the stages it left running. */
|
|
768
|
+
const ABANDON_JOIN_MS = 15_000;
|
|
769
|
+
|
|
770
|
+
/** Until every one of `work` is settled, or `ms` has passed. */
|
|
771
|
+
async function joinBounded(work: readonly Promise<unknown>[], ms: number): Promise<void> {
|
|
772
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
773
|
+
await Promise.race([
|
|
774
|
+
Promise.allSettled(work),
|
|
775
|
+
new Promise<void>((resolve) => {
|
|
776
|
+
timer = setTimeout(resolve, ms);
|
|
777
|
+
timer.unref?.();
|
|
778
|
+
}),
|
|
779
|
+
]);
|
|
780
|
+
if (timer) clearTimeout(timer);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* A start's own view of its machine: every call through it, and every write
|
|
785
|
+
* to a process started through it, is refused once `over()` holds. Work the
|
|
786
|
+
* start left behind therefore never reaches the machine again, nor a
|
|
787
|
+
* replacement that reuses its name.
|
|
788
|
+
*/
|
|
789
|
+
function fencedHandle(handle: TaskEnvironmentHandle<unknown>, over: () => boolean): TaskEnvironmentHandle<unknown> {
|
|
790
|
+
const admit = (): void => {
|
|
791
|
+
if (over()) throw new Error(WITHDRAWN);
|
|
792
|
+
};
|
|
793
|
+
// Checked again by the provider at its last moment, after anything it
|
|
794
|
+
// awaits first (a machine looks up its address): a call admitted before
|
|
795
|
+
// the start ended never starts after it.
|
|
796
|
+
const lastly = (inner?: () => void) => (): void => {
|
|
797
|
+
admit();
|
|
798
|
+
inner?.();
|
|
799
|
+
};
|
|
800
|
+
const fenced = (proc: TaskEnvironmentProcess): TaskEnvironmentProcess => ({
|
|
801
|
+
stdout: proc.stdout,
|
|
802
|
+
stderr: proc.stderr,
|
|
803
|
+
completion: proc.completion,
|
|
804
|
+
write: async (chunk) => {
|
|
805
|
+
admit();
|
|
806
|
+
await proc.write(chunk);
|
|
807
|
+
},
|
|
808
|
+
closeInput: () => proc.closeInput(),
|
|
809
|
+
terminate: (signal) => proc.terminate(signal),
|
|
810
|
+
});
|
|
811
|
+
return {
|
|
812
|
+
descriptor: handle.descriptor,
|
|
813
|
+
durableIdentity: handle.durableIdentity,
|
|
814
|
+
...(handle.anchors === undefined ? {} : { anchors: handle.anchors }),
|
|
815
|
+
...(handle.credentialDelivery === undefined ? {} : { credentialDelivery: handle.credentialDelivery }),
|
|
816
|
+
readWorkspaceFile: async (path, options) => {
|
|
817
|
+
admit();
|
|
818
|
+
return handle.readWorkspaceFile(path, { ...options, admit: lastly(options?.admit) });
|
|
819
|
+
},
|
|
820
|
+
writeWorkspaceFile: async (path, bytes, options) => {
|
|
821
|
+
admit();
|
|
822
|
+
await handle.writeWorkspaceFile(path, bytes, { ...options, admit: lastly(options?.admit) });
|
|
823
|
+
},
|
|
824
|
+
exec: async (request) => {
|
|
825
|
+
admit();
|
|
826
|
+
return handle.exec({ ...request, admit: lastly(request.admit) });
|
|
827
|
+
},
|
|
828
|
+
spawn: async (request) => {
|
|
829
|
+
admit();
|
|
830
|
+
return fenced(await handle.spawn({ ...request, admit: lastly(request.admit) }));
|
|
831
|
+
},
|
|
832
|
+
spawnSession: async (request) => {
|
|
833
|
+
admit();
|
|
834
|
+
return fenced(await handle.spawnSession({ ...request, admit: lastly(request.admit) }));
|
|
835
|
+
},
|
|
836
|
+
launchDetachedSession: async (request) => {
|
|
837
|
+
admit();
|
|
838
|
+
return handle.launchDetachedSession({ ...request, admit: lastly(request.admit) });
|
|
839
|
+
},
|
|
840
|
+
copy: async (request) => {
|
|
841
|
+
admit();
|
|
842
|
+
await handle.copy({ ...request, admit: lastly(request.admit) });
|
|
843
|
+
},
|
|
844
|
+
ports: async () => {
|
|
845
|
+
admit();
|
|
846
|
+
return handle.ports();
|
|
847
|
+
},
|
|
848
|
+
status: async () => {
|
|
849
|
+
admit();
|
|
850
|
+
return handle.status();
|
|
851
|
+
},
|
|
852
|
+
recover: async (context) => {
|
|
853
|
+
admit();
|
|
854
|
+
return handle.recover(context);
|
|
855
|
+
},
|
|
856
|
+
stop: async () => {
|
|
857
|
+
admit();
|
|
858
|
+
await handle.stop();
|
|
859
|
+
},
|
|
860
|
+
teardown: async () => {
|
|
861
|
+
admit();
|
|
862
|
+
await handle.teardown();
|
|
863
|
+
},
|
|
864
|
+
...(handle.inspectRoute
|
|
865
|
+
? {
|
|
866
|
+
inspectRoute: async () => {
|
|
867
|
+
admit();
|
|
868
|
+
return handle.inspectRoute!();
|
|
869
|
+
},
|
|
870
|
+
}
|
|
871
|
+
: {}),
|
|
872
|
+
...(handle.gatewayTunnelSshArgs
|
|
873
|
+
? {
|
|
874
|
+
gatewayTunnelSshArgs: async (machinePort: number, hostPort: number, options?: TaskEnvironmentCallOptions) => {
|
|
875
|
+
admit();
|
|
876
|
+
return handle.gatewayTunnelSshArgs!(machinePort, hostPort, { ...options, admit: lastly(options?.admit) });
|
|
877
|
+
},
|
|
878
|
+
}
|
|
879
|
+
: {}),
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
|
|
632
883
|
/** The identity the host records for the task's runtime: the machine id for
|
|
633
884
|
* a machine over ssh (what recovery and orphan sweeps key on), the handle's
|
|
634
885
|
* durable identity for anything else (sidecar mode: `self:<task>`). */
|
|
@@ -649,8 +900,10 @@ function withTaskDownResult(
|
|
|
649
900
|
return {
|
|
650
901
|
descriptor: handle.descriptor,
|
|
651
902
|
durableIdentity: handle.durableIdentity,
|
|
652
|
-
|
|
653
|
-
|
|
903
|
+
...(handle.anchors === undefined ? {} : { anchors: handle.anchors }),
|
|
904
|
+
...(handle.credentialDelivery === undefined ? {} : { credentialDelivery: handle.credentialDelivery }),
|
|
905
|
+
readWorkspaceFile: (path, options) => handle.readWorkspaceFile(path, options),
|
|
906
|
+
writeWorkspaceFile: (path, bytes, options) => handle.writeWorkspaceFile(path, bytes, options),
|
|
654
907
|
exec: (request) => handle.exec(request),
|
|
655
908
|
spawn: (request) => handle.spawn(request),
|
|
656
909
|
spawnSession: (request) => handle.spawnSession(request),
|
|
@@ -669,8 +922,8 @@ function withTaskDownResult(
|
|
|
669
922
|
: {}),
|
|
670
923
|
...(handle.gatewayTunnelSshArgs
|
|
671
924
|
? {
|
|
672
|
-
gatewayTunnelSshArgs: (machinePort: number, hostPort: number) =>
|
|
673
|
-
handle.gatewayTunnelSshArgs!(machinePort, hostPort),
|
|
925
|
+
gatewayTunnelSshArgs: (machinePort: number, hostPort: number, options?: TaskEnvironmentCallOptions) =>
|
|
926
|
+
handle.gatewayTunnelSshArgs!(machinePort, hostPort, options),
|
|
674
927
|
}
|
|
675
928
|
: {}),
|
|
676
929
|
};
|
|
@@ -694,7 +947,36 @@ export function createMachineHostTaskEnvironmentProvider(
|
|
|
694
947
|
): Promise<TaskEnvironmentProvisioned<TaskUpResult, TaskDownResult>> {
|
|
695
948
|
const { input } = request;
|
|
696
949
|
const envMode = request.envMode ?? "loader";
|
|
697
|
-
|
|
950
|
+
// ADR-154 §12: a task on cloud credentials holds its grant for the
|
|
951
|
+
// whole start; withdrawn (its gate closed, a cleanup reached it), the
|
|
952
|
+
// start stops wherever it is, its clone included, and the machine goes.
|
|
953
|
+
const leased = request.credentials.leased;
|
|
954
|
+
assertLeasedTaskUp(leased, input.task);
|
|
955
|
+
const clone = leased?.clone;
|
|
956
|
+
const withdrawal = (): Error => new Error(WITHDRAWN);
|
|
957
|
+
if (clone?.signal.aborted) throw withdrawal();
|
|
958
|
+
const untilWithdrawn = <T>(work: Promise<T>): Promise<T> => {
|
|
959
|
+
if (!clone) return work;
|
|
960
|
+
return new Promise<T>((resolve, reject) => {
|
|
961
|
+
const onAbort = (): void => reject(withdrawal());
|
|
962
|
+
if (clone.signal.aborted) {
|
|
963
|
+
onAbort();
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
clone.signal.addEventListener("abort", onAbort, { once: true });
|
|
967
|
+
work.then(
|
|
968
|
+
(value) => {
|
|
969
|
+
clone.signal.removeEventListener("abort", onAbort);
|
|
970
|
+
resolve(value);
|
|
971
|
+
},
|
|
972
|
+
(error: unknown) => {
|
|
973
|
+
clone.signal.removeEventListener("abort", onAbort);
|
|
974
|
+
reject(error);
|
|
975
|
+
},
|
|
976
|
+
);
|
|
977
|
+
});
|
|
978
|
+
};
|
|
979
|
+
const provisioning = deps.environment.provision({
|
|
698
980
|
taskId: request.taskId,
|
|
699
981
|
input: {
|
|
700
982
|
image: deps.machineImage(),
|
|
@@ -706,21 +988,75 @@ export function createMachineHostTaskEnvironmentProvider(
|
|
|
706
988
|
envMode,
|
|
707
989
|
onPrepared: request.onPrepared,
|
|
708
990
|
});
|
|
991
|
+
let provisioned: Awaited<typeof provisioning>;
|
|
992
|
+
try {
|
|
993
|
+
provisioned = await untilWithdrawn(provisioning);
|
|
994
|
+
} catch (error) {
|
|
995
|
+
if (clone?.signal.aborted) {
|
|
996
|
+
// Withdrawn while the machine came up: the start returns only once
|
|
997
|
+
// that is over and the machine is gone again (the provider's own
|
|
998
|
+
// timeouts bound the wait). A teardown left for later could take a
|
|
999
|
+
// replacement, which reuses the task's machine name.
|
|
1000
|
+
const late = await provisioning.catch(() => null);
|
|
1001
|
+
if (late) await late.handle.teardown().catch(() => {});
|
|
1002
|
+
}
|
|
1003
|
+
throw error;
|
|
1004
|
+
}
|
|
709
1005
|
const handle = provisioned.handle;
|
|
710
1006
|
const workspacePath = handle.descriptor.workspacePath;
|
|
711
1007
|
const warnings: string[] = [];
|
|
712
1008
|
const clones: Array<{ projectId: string; destination: string; clone: CloneOutcome["clone"] }> = [];
|
|
1009
|
+
// The start's own view of the machine: once the start is withdrawn or
|
|
1010
|
+
// failed, every call through it is refused, so work it began and left
|
|
1011
|
+
// behind never reaches the machine again, nor the replacement that
|
|
1012
|
+
// reuses its name: no exec, write, secret or teardown.
|
|
1013
|
+
let over = false;
|
|
1014
|
+
const own = fencedHandle(handle, () => over || clone?.signal.aborted === true);
|
|
1015
|
+
// Every later stage races the withdrawal, through the start's last
|
|
1016
|
+
// step: withdrawn, the start stops wherever it is. What it began is
|
|
1017
|
+
// fenced off, the machine goes, and the stages still running get a
|
|
1018
|
+
// bounded chance to end before the start returns.
|
|
1019
|
+
const begun: Array<Promise<unknown>> = [];
|
|
1020
|
+
const stage = <T>(work: Promise<T>): Promise<T> => {
|
|
1021
|
+
begun.push(work);
|
|
1022
|
+
work.catch(() => {});
|
|
1023
|
+
return untilWithdrawn(work);
|
|
1024
|
+
};
|
|
1025
|
+
let abandoning: Promise<void> | null = null;
|
|
1026
|
+
const abandon = (): Promise<void> => {
|
|
1027
|
+
over = true;
|
|
1028
|
+
abandoning ??= (async () => {
|
|
1029
|
+
await handle.teardown().catch(() => {});
|
|
1030
|
+
await joinBounded(begun, deps.abandonJoinMs ?? ABANDON_JOIN_MS);
|
|
1031
|
+
})();
|
|
1032
|
+
return abandoning;
|
|
1033
|
+
};
|
|
713
1034
|
|
|
714
1035
|
try {
|
|
1036
|
+
if (clone?.signal.aborted) throw withdrawal();
|
|
1037
|
+
// ADR-154 §12: the runtime exists and nothing in it has run yet: the
|
|
1038
|
+
// runner's writes (the owner's GitHub configuration and SSH key,
|
|
1039
|
+
// recorded for cleanup) come first, so the clones below authenticate
|
|
1040
|
+
// with them, over SSH too.
|
|
1041
|
+
if (leased) {
|
|
1042
|
+
const writes = leased.runtimeReady(own, leased.clone.signal);
|
|
1043
|
+
try {
|
|
1044
|
+
await untilWithdrawn(writes);
|
|
1045
|
+
} catch (error) {
|
|
1046
|
+
// Joined before anything else happens: nothing still writes into
|
|
1047
|
+
// the machine once the start has stopped.
|
|
1048
|
+
over = true;
|
|
1049
|
+
await writes.catch(() => {});
|
|
1050
|
+
throw error;
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
715
1053
|
if (envMode === "loader") {
|
|
716
|
-
await installMachineProjectEnvLoader(
|
|
1054
|
+
await stage(installMachineProjectEnvLoader(own));
|
|
717
1055
|
// ADR-153, on a machine's own task (the sidecar's): the proxy's CA
|
|
718
1056
|
// and pins before anything in the task loads its env. Never fails
|
|
719
1057
|
// the start; its proxied keys stay withheld until they are in.
|
|
720
|
-
const proxyWarning = await
|
|
721
|
-
|
|
722
|
-
activeSecretsProxy()?.runtime ?? null,
|
|
723
|
-
{ start: true },
|
|
1058
|
+
const proxyWarning = await stage(
|
|
1059
|
+
reconcileSidecarTaskProxy(own, activeSecretsProxy()?.runtime ?? null, { start: true }),
|
|
724
1060
|
);
|
|
725
1061
|
if (proxyWarning !== null) console.warn(`[machine] task ${request.taskId}: ${proxyWarning}`);
|
|
726
1062
|
}
|
|
@@ -730,95 +1066,122 @@ export function createMachineHostTaskEnvironmentProvider(
|
|
|
730
1066
|
const wantsGithub = projects.some((project) =>
|
|
731
1067
|
isGithubRepoUrl(project.repoUrl),
|
|
732
1068
|
);
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
1069
|
+
// ADR-154 §12: a task on cloud credentials clones with the token its
|
|
1070
|
+
// lease holds for this start, never the runner's own store.
|
|
1071
|
+
const token = !wantsGithub
|
|
1072
|
+
? null
|
|
1073
|
+
: clone
|
|
1074
|
+
? clone.github && { accessToken: clone.github.token }
|
|
1075
|
+
: await deps.githubToken(input.task.ownerUserId);
|
|
736
1076
|
for (const project of projects) {
|
|
1077
|
+
if (clone?.signal.aborted) throw withdrawal();
|
|
737
1078
|
const destination = `${workspacePath}/${projectDirectoryName(project.slug)}`;
|
|
738
|
-
const outcome = await
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
1079
|
+
const outcome = await untilWithdrawn(
|
|
1080
|
+
cloneProject(
|
|
1081
|
+
own,
|
|
1082
|
+
destination,
|
|
1083
|
+
project,
|
|
1084
|
+
input.task.branch,
|
|
1085
|
+
{ name: input.ownerName, email: input.ownerEmail },
|
|
1086
|
+
token?.accessToken ?? null,
|
|
1087
|
+
clone ? { ssh: clone.ssh !== null, signal: clone.signal } : undefined,
|
|
1088
|
+
),
|
|
745
1089
|
);
|
|
746
1090
|
clones.push({ projectId: project.id, destination, clone: outcome.clone });
|
|
747
1091
|
if (outcome.warning) warnings.push(outcome.warning);
|
|
748
1092
|
}
|
|
1093
|
+
if (clone?.signal.aborted) throw withdrawal();
|
|
749
1094
|
} catch (error) {
|
|
750
1095
|
// Provisioning-infrastructure failures (ssh died, unsafe slug) tear
|
|
751
|
-
// the machine down rather than stranding an orphan
|
|
752
|
-
//
|
|
753
|
-
|
|
1096
|
+
// the machine down rather than stranding an orphan, and so does a
|
|
1097
|
+
// leased clone that failed; a legacy route's per-repo problems were
|
|
1098
|
+
// already converted to warnings above and do not land here.
|
|
1099
|
+
await abandon();
|
|
754
1100
|
throw error;
|
|
755
1101
|
}
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
1102
|
+
let envSchemas: ProjectEnvSchemaObservation[];
|
|
1103
|
+
try {
|
|
1104
|
+
// ADR-153 §4: right after the clones, before shared files, the
|
|
1105
|
+
// forwarder, the editor or any agent.
|
|
1106
|
+
envSchemas = await stage(captureMachineEnvSchemas(request.taskId, own, clones));
|
|
1107
|
+
|
|
1108
|
+
// ADR-062 shared files: point-in-time copy (see copySharedFiles).
|
|
1109
|
+
const sharedWarning = await stage(
|
|
1110
|
+
(deps.sharedFiles ?? copySharedFiles)(own, workspacePath, {
|
|
1111
|
+
...(input.task.ownerOrgId ? { ownerOrgId: input.task.ownerOrgId } : {}),
|
|
1112
|
+
ownerUserId: input.task.ownerUserId,
|
|
1113
|
+
...(input.task.sharedFiles ? { sharedFiles: input.task.sharedFiles } : {}),
|
|
1114
|
+
}).catch((error: unknown) => {
|
|
1115
|
+
console.warn(
|
|
1116
|
+
`[machine] task ${request.taskId}: shared files copy failed: ${
|
|
1117
|
+
error instanceof Error ? error.message : String(error)
|
|
1118
|
+
}`,
|
|
1119
|
+
);
|
|
1120
|
+
return null;
|
|
1121
|
+
}),
|
|
770
1122
|
);
|
|
771
|
-
|
|
772
|
-
});
|
|
773
|
-
if (sharedWarning) warnings.push(sharedWarning);
|
|
774
|
-
|
|
775
|
-
// Codex default-slot credentials: the legacy delivery paths are all
|
|
776
|
-
// docker grammar; machines get theirs over the transport. Best-effort
|
|
777
|
-
// — a failed inject degrades codex agents, never the task.
|
|
778
|
-
if (input.task.agents.some((agent) => agent.kind === "codex")) {
|
|
779
|
-
await injectCodexIntoMachine(handle).catch((error: unknown) => {
|
|
780
|
-
console.warn(
|
|
781
|
-
`[machine] task ${request.taskId}: codex inject failed: ${
|
|
782
|
-
error instanceof Error ? error.message : String(error)
|
|
783
|
-
}`,
|
|
784
|
-
);
|
|
785
|
-
});
|
|
786
|
-
}
|
|
1123
|
+
if (sharedWarning) warnings.push(sharedWarning);
|
|
787
1124
|
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
await handle.teardown().catch(() => {});
|
|
802
|
-
throw error;
|
|
1125
|
+
// Codex default-slot credentials: the legacy delivery paths are all
|
|
1126
|
+
// docker grammar; machines get theirs over the transport. Best-effort
|
|
1127
|
+
// — a failed inject degrades codex agents, never the task. A task on
|
|
1128
|
+
// cloud credentials gets its accounts from its lease, never the
|
|
1129
|
+
// runner's own (ADR-154 §6).
|
|
1130
|
+
if (!clone && input.task.agents.some((agent) => agent.kind === "codex")) {
|
|
1131
|
+
await injectCodexIntoMachine(own).catch((error: unknown) => {
|
|
1132
|
+
console.warn(
|
|
1133
|
+
`[machine] task ${request.taskId}: codex inject failed: ${
|
|
1134
|
+
error instanceof Error ? error.message : String(error)
|
|
1135
|
+
}`,
|
|
1136
|
+
);
|
|
1137
|
+
});
|
|
803
1138
|
}
|
|
804
|
-
}
|
|
805
1139
|
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
1140
|
+
if (envMode === "loader") {
|
|
1141
|
+
try {
|
|
1142
|
+
await stage(
|
|
1143
|
+
startMachineTaskForwarder(
|
|
1144
|
+
request.taskId,
|
|
1145
|
+
own,
|
|
1146
|
+
request.credentials,
|
|
1147
|
+
deps.ensureGatewayTunnel ?? ensureMachineGatewayTunnelReady,
|
|
1148
|
+
),
|
|
1149
|
+
);
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
// Unlike editor/auth conveniences, the loader is the only project
|
|
1152
|
+
// env delivery path. A task without it must not start and silently
|
|
1153
|
+
// hand consumers an incomplete or legacy environment.
|
|
1154
|
+
stopMachineGatewayTunnel(request.taskId);
|
|
1155
|
+
await abandon();
|
|
1156
|
+
throw error;
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
// Editor pane: code-server on the machine (same flags as uai-init's
|
|
1161
|
+
// container launch), idempotent across provision re-runs and resume.
|
|
1162
|
+
// Best-effort — a missing binary degrades the Editor tab, never the task.
|
|
1163
|
+
await stage(
|
|
1164
|
+
ensureCodeServer(own, workspacePath, input.task.previewEnv?.UAI_EDITOR_THEME, envMode).catch(
|
|
1165
|
+
(error: unknown) => {
|
|
1166
|
+
console.warn(
|
|
1167
|
+
`[machine] task ${request.taskId}: code-server launch failed: ${
|
|
1168
|
+
error instanceof Error ? error.message : String(error)
|
|
1169
|
+
}`,
|
|
1170
|
+
);
|
|
1171
|
+
},
|
|
1172
|
+
),
|
|
819
1173
|
);
|
|
820
|
-
});
|
|
821
1174
|
|
|
1175
|
+
// Withdrawn while the rest came up: the start is not one.
|
|
1176
|
+
if (clone?.signal.aborted) throw withdrawal();
|
|
1177
|
+
} catch (error) {
|
|
1178
|
+
if (clone?.signal.aborted) {
|
|
1179
|
+
stopMachineGatewayTunnel(request.taskId);
|
|
1180
|
+
await abandon();
|
|
1181
|
+
throw withdrawal();
|
|
1182
|
+
}
|
|
1183
|
+
throw error;
|
|
1184
|
+
}
|
|
822
1185
|
return {
|
|
823
1186
|
handle: withTaskDownResult(handle),
|
|
824
1187
|
result: {
|