@runuai/host 0.9.150 → 0.9.152

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
  2. package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
  3. package/db/migrations/meta/_journal.json +14 -0
  4. package/db/schema.ts +188 -0
  5. package/lib/agent-cli.ts +11 -13
  6. package/lib/agent.ts +177 -7
  7. package/lib/agents/durable-proc.ts +56 -44
  8. package/lib/agents/registry.ts +19 -3
  9. package/lib/agents/session-state.ts +23 -10
  10. package/lib/agents/transport.ts +101 -69
  11. package/lib/apple-one-off.ts +277 -0
  12. package/lib/apple-runtime-recycle.ts +4 -3
  13. package/lib/attachments.ts +19 -17
  14. package/lib/capture-flow.ts +319 -0
  15. package/lib/cloud-credential-cleanup-adapter.ts +123 -0
  16. package/lib/cloud-credential-host.ts +512 -0
  17. package/lib/cloud-credential-leases.ts +2544 -0
  18. package/lib/cloud-credential-runtime.ts +1035 -0
  19. package/lib/codex-auth.ts +21 -1
  20. package/lib/container-runtime.ts +76 -0
  21. package/lib/credential-cleanup-inventory.ts +405 -0
  22. package/lib/credential-consumers/engine-anchor.ts +1880 -0
  23. package/lib/credential-consumers/engines.ts +773 -0
  24. package/lib/credential-consumers/github.ts +594 -0
  25. package/lib/credential-consumers/mcp.ts +384 -0
  26. package/lib/credential-consumers/task-files.ts +127 -0
  27. package/lib/credential-store-pickup.ts +15 -2
  28. package/lib/credential-store.ts +361 -33
  29. package/lib/credentials-activation-authority.ts +546 -0
  30. package/lib/credentials-purge.ts +1006 -0
  31. package/lib/db.ts +11 -0
  32. package/lib/docker-exec.ts +3 -1
  33. package/lib/engine-accounts.ts +42 -32
  34. package/lib/engine-connect.ts +47 -2
  35. package/lib/engine-credential-env.ts +50 -0
  36. package/lib/engine-login.ts +83 -1
  37. package/lib/env-file.ts +62 -8
  38. package/lib/env.ts +5 -0
  39. package/lib/fs-aliases.ts +49 -0
  40. package/lib/git-clone-credentials.ts +334 -0
  41. package/lib/git-diff.ts +219 -177
  42. package/lib/git-identity.ts +59 -14
  43. package/lib/github-origin.ts +28 -0
  44. package/lib/github-tokens.ts +56 -21
  45. package/lib/host-env.ts +17 -4
  46. package/lib/launch-mode.ts +25 -0
  47. package/lib/legacy-runtime-retirement.ts +388 -0
  48. package/lib/machine-exec.ts +14 -3
  49. package/lib/machine-provider-aws.ts +50 -8
  50. package/lib/machine-provider-local.ts +4 -0
  51. package/lib/machine-provider.ts +4 -0
  52. package/lib/managed-operation-drain.ts +8 -0
  53. package/lib/managed-runtime.ts +371 -7
  54. package/lib/mcp-capture.ts +860 -0
  55. package/lib/mcp-connections.ts +8 -1
  56. package/lib/mcp-gateway.ts +168 -10
  57. package/lib/obs.ts +2 -6
  58. package/lib/orchestrator.ts +1837 -224
  59. package/lib/person-capture.ts +327 -0
  60. package/lib/project-env-legacy-tags.ts +130 -0
  61. package/lib/project-env-task-replica.ts +1752 -0
  62. package/lib/runner-boot.ts +10 -0
  63. package/lib/runtime-activation-tail.ts +50 -3
  64. package/lib/runtime-archive.ts +68 -1
  65. package/lib/runtime-reconnect.ts +86 -0
  66. package/lib/runtime-state.ts +429 -13
  67. package/lib/safe-http.ts +364 -0
  68. package/lib/secrets-proxy/task-trust.ts +34 -3
  69. package/lib/sidecar-resume.ts +18 -5
  70. package/lib/skills.ts +47 -8
  71. package/lib/standard-image.ts +157 -103
  72. package/lib/stopped-task-status-queue.ts +68 -10
  73. package/lib/task-credential-cleanup.ts +525 -0
  74. package/lib/task-diff-sandbox.ts +464 -0
  75. package/lib/task-diff.ts +66 -163
  76. package/lib/task-env-source.ts +68 -14
  77. package/lib/task-environment/anchor.ts +31 -0
  78. package/lib/task-environment/apple-container.ts +69 -28
  79. package/lib/task-environment/docker.ts +101 -20
  80. package/lib/task-environment/machine-environment.ts +61 -0
  81. package/lib/task-environment/machine-task-up.ts +459 -96
  82. package/lib/task-environment/machine.ts +101 -11
  83. package/lib/task-environment/self.ts +108 -50
  84. package/lib/task-environment/types.ts +89 -5
  85. package/lib/task-environment/workspace-files.ts +650 -25
  86. package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
  87. package/lib/task-gateway-recovery.ts +4 -1
  88. package/lib/task-gateway-state.ts +15 -0
  89. package/lib/task-machine-swap-safety.ts +729 -0
  90. package/lib/token-broker-adopt.ts +344 -0
  91. package/lib/token-broker-lambda.ts +56 -2
  92. package/lib/token-broker-local.ts +28 -4
  93. package/lib/token-broker-revision.ts +310 -0
  94. package/lib/token-broker-ssm.ts +368 -28
  95. package/lib/token-lanes/openai-oauth.ts +7 -3
  96. package/lib/token-lanes/xai-oauth.ts +7 -3
  97. package/lib/transcript.ts +9 -5
  98. package/package.json +6 -2
  99. package/scripts/agent/task-up.sh +242 -100
  100. package/src/cloud-credentials.ts +2399 -0
  101. package/src/event-outbox.ts +8 -6
  102. package/src/index.ts +1386 -875
  103. package/src/main.ts +1061 -139
  104. package/src/outbound-frame.ts +31 -0
  105. package/src/project-env-policy.ts +125 -10
  106. package/src/protocol.ts +699 -23
  107. package/src/ui/server.ts +48 -10
@@ -0,0 +1,1880 @@
1
+ /**
2
+ * ADR-154 §6: every process a leased engine starts from is admitted by the
3
+ * runner where it really starts, and everything it leads to stays owned
4
+ * until a cleanup has killed it.
5
+ *
6
+ * - The gate. Before the runner hands a launch to a provider, it writes a
7
+ * single-use admission into the runtime, as root, under a root-only
8
+ * directory (`/run/uai/engine-gates/<task>/<run>/<account>/admit-<token>`,
9
+ * the launch's revision and version in it, and the runtime instance it
10
+ * was written in: a stopped container's next start, or a machine's next
11
+ * boot, admits nothing written before). The provider starts the
12
+ * launch as root through the supervisor, which admits it by consuming
13
+ * that token. A cleanup closes the runner's admission first (no new
14
+ * token is written for the accounts it takes), waits for the writes
15
+ * under way, then revokes every token left for its runs and accounts and
16
+ * reads them back gone. So a launch handed over before a cleanup that
17
+ * starts after it (a machine still looking up its address, a durable
18
+ * runner still being deployed) finds no token and never starts. A token
19
+ * is never reused, so a write that lands late admits nothing.
20
+ * - The floor. An account's admissions are also held to the newest version
21
+ * the runner has moved it to (`floor-<revision hash>`, monotonic): before
22
+ * a newer version of an account is served, the runner raises the floor
23
+ * in every running runtime of the task (`raiseEngineAdmissionFloors`; a
24
+ * stopped one's admissions end with its instance). A launch admitted on
25
+ * the older version that has not started by then never does.
26
+ * - The slot. An agent's launches are admitted for its spawn's slot (the
27
+ * slot's id in each admission). When the runner retires a slot (the agent
28
+ * removed, changed or started again, its session ended), it marks the
29
+ * slot retired in the runtime (`retired-<slot>`, under the task's lock,
30
+ * `retireEngineLaunchSlot`): every admission of that slot not yet used is
31
+ * revoked, with what it carried, and none of its own is written or used
32
+ * after. So a launch handed to its provider just before the retirement,
33
+ * and not yet started there, never starts with the agent's old task token,
34
+ * however late it gets there.
35
+ * - The subjects. An admission is for an AI account's exact material, or
36
+ * for an agent's task token alone (the Managed Engine, ADR-128, which
37
+ * runs on Uai's gateway with that token as its bearer). Each has its own
38
+ * namespace in the runtime: an account's admissions are under its id's
39
+ * segment, which never starts with "_" (`idPathSegment`); every task-token
40
+ * admission is under `TASK_TOKEN_SEGMENT`. A cleanup of named accounts
41
+ * never reaches a task-token admission; a whole run's reaches both.
42
+ * - The anchor. The supervisor the provider starts (the launcher, W) forks
43
+ * the anchor (R), a child subreaper named `uai-anchor`, before anything is
44
+ * admitted. R takes the task's gate lock, consumes the token, writes its
45
+ * receipt (`live-<token>`: its pid, its start time and the runtime
46
+ * instance it runs in), forks the engine (C), which drops to the task
47
+ * user (groups, gid, uid; the capabilities go with root), and releases
48
+ * the lock: a cleanup, which revokes under the same lock, either finds
49
+ * the token gone and the receipt written, or revokes it before R looks.
50
+ * Everything C starts, double-forked, setsid or with its environment
51
+ * cleared, as the task user or as root through sudo, reparents to R,
52
+ * never to PID 1. R tells W how C ended and W ends the same way, so turns
53
+ * and sessions end as before; R reaps until it has no child left, and
54
+ * only then, under the lock, removes its receipt and exits: what an
55
+ * engine leaves running keeps running, owned. A removed receipt is the
56
+ * kernel's word (no child left) that nothing R owned still runs.
57
+ * - The cleanup (`stopAnchoredEngines`), as root: revoke, then, for every
58
+ * receipt of its runs and accounts, kill everything under R, over and
59
+ * over, until R ends on its own. The cleanup never kills R: when R has
60
+ * ended, its receipt says how. Gone, R proved its subtree empty. Still
61
+ * there, R died with things it owned possibly reparented away: the anchor
62
+ * is lost, absence is never proof, and the runtime must be stopped
63
+ * (`EngineAnchorLost`). The receipt stays as the witness until that
64
+ * runtime instance is gone (a stopped container's next start, a machine's
65
+ * next boot): every cleanup until then fails the same way. An R still
66
+ * there at the bound (stopped, wedged), or anything else the cleanup
67
+ * killed that will not end, is lost the same way: stopping the runtime is
68
+ * the one proof left, where retrying would wait on it forever.
69
+ * - Whose they are. Gates, floors, receipts and anchors are root's: the
70
+ * task user can neither forge an admission nor kill an anchor. Inside its
71
+ * runtime the task user can become root (the task image's sudo), so none
72
+ * of this holds against task code that sets out to defeat it; such code
73
+ * already holds the credential it was given (ADR-154 §Limits). It
74
+ * contains the runner's own launches and everything engines start in
75
+ * ordinary use.
76
+ * - A runtime that cannot run it (no Python 3.9, no pidfds, no subreaper)
77
+ * refuses the launch: a leased engine stays down rather than run unowned.
78
+ */
79
+
80
+ import { randomBytes } from "node:crypto";
81
+
82
+ import { idPathSegment } from "../../src/cloud-credentials";
83
+ import { LAUNCH_SECRET_ENV_NAMES, isLaunchSecretEnvName } from "../engine-credential-env";
84
+ import type {
85
+ TaskEnvironmentAnchor,
86
+ TaskEnvironmentExecResult,
87
+ TaskEnvironmentHandle,
88
+ TaskEnvironmentProcess,
89
+ } from "../task-environment/types";
90
+
91
+ /** The anchor's process name (comm), how a cleanup knows it. */
92
+ export const ENGINE_ANCHOR_NAME = "uai-anchor";
93
+ /** The launcher's process name. */
94
+ export const ENGINE_LAUNCH_NAME = "uai-launch";
95
+ /** Where a runtime's engine gates live, root's alone. */
96
+ export const ENGINE_GATE_ROOT = "/run/uai/engine-gates";
97
+ /** The run marker every process of a leased engine inherits from its spawn. */
98
+ export const ENGINE_RUN_ENV = "UAI_ENGINE_RUN";
99
+ /** The account marker every process of a leased engine inherits: the
100
+ * credential it runs on. */
101
+ export const ENGINE_ACCOUNT_ENV = "UAI_ENGINE_ACCOUNT";
102
+
103
+ /** Where a launch's carried engine credentials wait when the gate area is
104
+ * not held in memory (ADR-154 §6: a tmpfs, or nowhere). */
105
+ export const ENGINE_CARRIER_FALLBACK_ROOT = "/dev/shm/uai-engine-gates";
106
+
107
+ let gateRoot = ENGINE_GATE_ROOT;
108
+ let carrierFallbackRoot = ENGINE_CARRIER_FALLBACK_ROOT;
109
+
110
+ /** The areas a carried credential may wait in, in the order tried. */
111
+ function carrierRoots(): string {
112
+ return [gateRoot, carrierFallbackRoot].join("\n");
113
+ }
114
+
115
+ /** Where every task-token admission is, in a namespace of its own: no
116
+ * account's segment starts with "_" (`idPathSegment`). */
117
+ export const TASK_TOKEN_SEGMENT = "_task-token";
118
+ /** A task-token admission's revision: it has no account material, and no
119
+ * floor is ever raised for it. */
120
+ const TASK_TOKEN_REVISION = "task-token";
121
+
122
+ /** What a launch is admitted for (ADR-154 §6): an AI account's exact
123
+ * material, or an agent's task token alone (the Managed Engine). */
124
+ export type EngineLaunchSubject =
125
+ | { subject: "account"; credentialId: string; revision: string; version: number }
126
+ | { subject: "task-token" };
127
+
128
+ /** What a launch is admitted as: the task's run, its subject, and the
129
+ * agent's launch slot it belongs to (`newEngineLaunchSlotId`; null for a
130
+ * one-shot that belongs to no agent's spawn). */
131
+ export type EngineLaunchIdentity = { taskId: string; runEpoch: number; slotId: string | null } & EngineLaunchSubject;
132
+
133
+ /** A new launch slot's id: one per spawn, never reused. */
134
+ export function newEngineLaunchSlotId(): string {
135
+ return randomBytes(16).toString("hex");
136
+ }
137
+
138
+ /** The segment, revision and version an admission is written under, and
139
+ * its slot ("-": none). */
140
+ function gateSegment(subject: EngineLaunchSubject): string {
141
+ return subject.subject === "account" ? idPathSegment(subject.credentialId) : TASK_TOKEN_SEGMENT;
142
+ }
143
+
144
+ function gateRevision(subject: EngineLaunchSubject): string {
145
+ return subject.subject === "account" ? subject.revision : TASK_TOKEN_REVISION;
146
+ }
147
+
148
+ function gateVersion(subject: EngineLaunchSubject): string {
149
+ return subject.subject === "account" ? String(subject.version) : "0";
150
+ }
151
+
152
+ function gateSlot(identity: EngineLaunchIdentity): string {
153
+ return identity.slotId ?? "-";
154
+ }
155
+
156
+ /** The newest version of an account's revision a launch may start on. */
157
+ export interface EngineAdmissionFloor {
158
+ credentialId: string;
159
+ revision: string;
160
+ version: number;
161
+ }
162
+
163
+ /** The launch was not admitted: it starts nothing. */
164
+ export class EngineLaunchRefused extends Error {
165
+ constructor(message = "this AI account may not start an engine now") {
166
+ super(message);
167
+ this.name = "EngineLaunchRefused";
168
+ }
169
+ }
170
+
171
+ /** An anchor ended without proving its subtree empty, or the runtime's gate
172
+ * area is not the runner's: only stopping the runtime proves its engines
173
+ * gone. */
174
+ export class EngineAnchorLost extends Error {
175
+ constructor() {
176
+ super("an engine supervisor in the task runtime was lost; only stopping the runtime proves its engines gone");
177
+ this.name = "EngineAnchorLost";
178
+ }
179
+ }
180
+
181
+ /**
182
+ * Shared by the helpers: the gate area, every directory to the task's
183
+ * opened without following a link, those above the root root's or this
184
+ * user's and letting nobody else rename what is in them, the root and below
185
+ * this user's alone; the task's lock; an account's floor; the runtime
186
+ * instance (the kernel's boot and its PID 1's start: a stopped container's
187
+ * next start, or a machine's next boot, is another instance, with none of
188
+ * the last one's processes).
189
+ */
190
+ const GATE_PRELUDE_BASE = String.raw`
191
+ import errno
192
+ import fcntl
193
+ import hashlib
194
+ import os
195
+ import re
196
+ import stat
197
+ import sys
198
+ import time
199
+
200
+ SEGMENT = re.compile(r"[A-Za-z0-9_-]{1,128}\Z")
201
+ TOKEN = re.compile(r"[0-9a-f]{32}\Z")
202
+ NUMBER = re.compile(r"(?:0|[1-9][0-9]{0,15})\Z")
203
+ DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
204
+ ME = os.geteuid()
205
+
206
+
207
+ class Refused(Exception):
208
+ pass
209
+
210
+
211
+ def revision_ok(revision):
212
+ return 0 < len(revision) <= 512 and "\n" not in revision and "\t" not in revision
213
+
214
+
215
+ def checked(fd, kind):
216
+ st = os.fstat(fd)
217
+ if not kind(st.st_mode) or st.st_uid != ME or st.st_mode & 0o077:
218
+ raise Refused("not this user's alone")
219
+ return st
220
+
221
+
222
+ def private(parent, name, create):
223
+ if create:
224
+ try:
225
+ os.mkdir(name, 0o700, dir_fd=parent)
226
+ except FileExistsError:
227
+ pass
228
+ fd = os.open(name, DIRECTORY, dir_fd=parent)
229
+ try:
230
+ checked(fd, stat.S_ISDIR)
231
+ except BaseException:
232
+ os.close(fd)
233
+ raise
234
+ return fd
235
+
236
+
237
+ def guarded(fd):
238
+ st = os.fstat(fd)
239
+ if st.st_uid not in (0, ME) or (st.st_mode & 0o022 and not st.st_mode & stat.S_ISVTX):
240
+ raise Refused("an unsafe directory above the gates")
241
+
242
+
243
+ def task_directory(root, task, create):
244
+ parts = root.split("/")[1:]
245
+ if not root.startswith("/") or not SEGMENT.match(task) or any(part in ("", ".", "..") for part in parts):
246
+ raise Refused("a malformed gate root")
247
+ fd = os.open("/", DIRECTORY)
248
+ try:
249
+ for part in parts[:-1]:
250
+ guarded(fd)
251
+ if create:
252
+ try:
253
+ os.mkdir(part, 0o755, dir_fd=fd)
254
+ except FileExistsError:
255
+ pass
256
+ child = os.open(part, DIRECTORY, dir_fd=fd)
257
+ os.close(fd)
258
+ fd = child
259
+ guarded(fd)
260
+ gates = private(fd, parts[-1], create)
261
+ finally:
262
+ os.close(fd)
263
+ try:
264
+ return private(gates, task, create)
265
+ finally:
266
+ os.close(gates)
267
+
268
+
269
+ def locked(directory):
270
+ fd = os.open(".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=directory)
271
+ try:
272
+ if checked(fd, stat.S_ISREG).st_nlink != 1:
273
+ raise Refused("a linked lock")
274
+ fcntl.flock(fd, fcntl.LOCK_EX)
275
+ except BaseException:
276
+ os.close(fd)
277
+ raise
278
+ return fd
279
+
280
+
281
+ def descend(directory, names, create):
282
+ try:
283
+ for name in names:
284
+ child = private(directory, name, create)
285
+ os.close(directory)
286
+ directory = child
287
+ except BaseException:
288
+ os.close(directory)
289
+ raise
290
+ return directory
291
+
292
+
293
+ def read_private(directory, name):
294
+ fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=directory)
295
+ try:
296
+ if checked(fd, stat.S_ISREG).st_nlink != 1:
297
+ raise Refused("a linked gate file")
298
+ return os.read(fd, 4096)
299
+ finally:
300
+ os.close(fd)
301
+
302
+
303
+ def write_private(directory, name, content):
304
+ fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=directory)
305
+ try:
306
+ if os.write(fd, content) != len(content):
307
+ raise Refused("a short write")
308
+ except BaseException:
309
+ os.close(fd)
310
+ os.unlink(name, dir_fd=directory)
311
+ raise
312
+ os.close(fd)
313
+
314
+
315
+ def floor_name(revision):
316
+ return "floor-" + hashlib.sha256(revision.encode()).hexdigest()[:40]
317
+
318
+
319
+ def floor_of(directory, revision):
320
+ try:
321
+ held = read_private(directory, floor_name(revision))
322
+ except FileNotFoundError:
323
+ return 0
324
+ if not NUMBER.match(held.decode().strip()):
325
+ raise Refused("a malformed floor")
326
+ return int(held)
327
+
328
+
329
+ RETIRED = "retired-"
330
+
331
+
332
+ def slot_ok(slot):
333
+ return slot == "-" or TOKEN.match(slot) is not None
334
+
335
+
336
+ def retired(directory, slot):
337
+ if slot == "-":
338
+ return False
339
+ try:
340
+ os.stat(RETIRED + slot, dir_fd=directory, follow_symlinks=False)
341
+ except FileNotFoundError:
342
+ return False
343
+ return True
344
+
345
+
346
+ BOOT_ID = "/proc/sys/kernel/random/boot_id"
347
+ INIT_STAT = "/proc/1/stat"
348
+
349
+
350
+ def instance():
351
+ with open(BOOT_ID, "rb") as source:
352
+ boot = source.read().strip()
353
+ with open(INIT_STAT, "rb") as source:
354
+ data = source.read()
355
+ return boot + b":" + data[data.rindex(b")") + 2:].split()[19]
356
+
357
+
358
+ TMPFS_MAGIC = 0x01021994
359
+ SECRET = "secret-"
360
+ SECRET_SECONDS = 120
361
+ SECRET_BYTES = 65536
362
+ VALUE_BYTES = 8192
363
+ CARRIER_FD = 9
364
+
365
+
366
+ def tmpfs(fd):
367
+ import ctypes
368
+
369
+ found = ctypes.create_string_buffer(512)
370
+ if ctypes.CDLL(None, use_errno=True).fstatfs(fd, found) != 0:
371
+ raise OSError(ctypes.get_errno(), "fstatfs")
372
+ return ctypes.c_long.from_buffer(found).value == TMPFS_MAGIC
373
+
374
+
375
+ def carried(data):
376
+ import json
377
+
378
+ if len(data) > SECRET_BYTES:
379
+ raise Refused("an oversized carried credential")
380
+ values = json.loads(data.decode("utf-8"))
381
+ if not isinstance(values, dict) or not values:
382
+ raise Refused("a malformed carried credential")
383
+ for name, value in values.items():
384
+ if (
385
+ name not in CARRIED
386
+ or not isinstance(value, str)
387
+ or not value
388
+ or "\0" in value
389
+ or len(value.encode("utf-8")) > VALUE_BYTES
390
+ ):
391
+ raise Refused("a malformed carried credential")
392
+ return values
393
+
394
+
395
+ def account_area(root, task, run, account, create):
396
+ return descend(task_directory(root, task, create), (run, account), create)
397
+
398
+
399
+ def drop_secret(roots, task, run, account, token):
400
+ for root in roots:
401
+ try:
402
+ area = account_area(root, task, run, account, False)
403
+ except FileNotFoundError:
404
+ continue
405
+ try:
406
+ os.unlink(SECRET + token, dir_fd=area)
407
+ except FileNotFoundError:
408
+ pass
409
+ finally:
410
+ os.close(area)
411
+
412
+
413
+ def inject(fd):
414
+ data = b""
415
+ while len(data) <= SECRET_BYTES:
416
+ chunk = os.read(fd, 65536)
417
+ if not chunk:
418
+ break
419
+ data += chunk
420
+ os.close(fd)
421
+ for name, value in carried(data).items():
422
+ os.environ[name] = value
423
+ `;
424
+
425
+ /**
426
+ * Shared by the helpers, with the prelude above: what a launch's carried
427
+ * engine credentials may be (ADR-154 §6). They wait in memory only (a
428
+ * tmpfs), each in a file of its own beside its admission, root's alone,
429
+ * for two minutes at most; a cleanup, a revocation or a floor removes one
430
+ * without ever reading it.
431
+ */
432
+ const GATE_PRELUDE =
433
+ GATE_PRELUDE_BASE +
434
+ `
435
+ CARRIED = frozenset(${JSON.stringify([...LAUNCH_SECRET_ENV_NAMES].sort())})
436
+ `;
437
+
438
+ /**
439
+ * Writes one admission, as root. argv: gate root, task, run, account
440
+ * segment, token, revision, version, slot ("-": none), the areas a carried
441
+ * credential may wait in (one per line, in order), and "carry" when the
442
+ * launch carries engine credentials (as JSON on stdin) or "-". A carried
443
+ * credential is written first, into the first of those areas held in memory
444
+ * (a tmpfs), and the admission names where it is. Prints that area, or "-".
445
+ * Every carried credential of the task past its bound goes first, unread.
446
+ * Exits 2 on malformed arguments or a malformed carried credential, 4 when
447
+ * the gate area is not the runner's, the version is below the account's
448
+ * floor, the slot is retired, no area is held in memory, or a write fails.
449
+ */
450
+ export const ENGINE_GATE_WRITE_HELPER =
451
+ GATE_PRELUDE +
452
+ String.raw`
453
+ root, task, run, account, token, revision, version, slot, carriers, carry = sys.argv[1:11]
454
+ roots = [candidate for candidate in carriers.split("\n") if candidate]
455
+ if not (
456
+ NUMBER.match(run)
457
+ and SEGMENT.match(account)
458
+ and TOKEN.match(token)
459
+ and NUMBER.match(version)
460
+ and revision_ok(revision)
461
+ and slot_ok(slot)
462
+ ):
463
+ sys.exit(2)
464
+ if carry not in ("carry", "-") or not roots:
465
+ sys.exit(2)
466
+ payload = None
467
+ if carry == "carry":
468
+ try:
469
+ payload = sys.stdin.buffer.read(SECRET_BYTES + 1)
470
+ carried(payload)
471
+ except (Refused, ValueError, UnicodeDecodeError):
472
+ sys.exit(2)
473
+
474
+
475
+ def stale(directory, name):
476
+ try:
477
+ return time.time() - os.stat(name, dir_fd=directory, follow_symlinks=False).st_mtime > SECRET_SECONDS
478
+ except FileNotFoundError:
479
+ return False
480
+
481
+
482
+ def expire():
483
+ # Past its bound, a carried credential goes, and so does the admission
484
+ # that named one: neither is ever read for its value.
485
+ for area_root in [root] + [candidate for candidate in roots if candidate != root]:
486
+ try:
487
+ directory = task_directory(area_root, task, False)
488
+ except FileNotFoundError:
489
+ continue
490
+ try:
491
+ for run_name in os.listdir(directory):
492
+ if not NUMBER.match(run_name):
493
+ continue
494
+ run_directory = private(directory, run_name, False)
495
+ try:
496
+ for account_name in os.listdir(run_directory):
497
+ if not SEGMENT.match(account_name):
498
+ continue
499
+ area = private(run_directory, account_name, False)
500
+ try:
501
+ for name in os.listdir(area):
502
+ if name.startswith(SECRET) and stale(area, name):
503
+ os.unlink(name, dir_fd=area)
504
+ elif name.startswith("admit-") and area_root == root and stale(area, name):
505
+ held = read_private(area, name).split(b"\n")
506
+ if len(held) > 3 and held[3] != b"-":
507
+ os.unlink(name, dir_fd=area)
508
+ drop_secret(roots, task, run_name, account_name, name[len("admit-"):])
509
+ finally:
510
+ os.close(area)
511
+ finally:
512
+ os.close(run_directory)
513
+ finally:
514
+ os.close(directory)
515
+
516
+
517
+ def hold(payload):
518
+ for candidate in roots:
519
+ area = account_area(candidate, task, run, account, True)
520
+ try:
521
+ if not tmpfs(area):
522
+ continue
523
+ write_private(area, SECRET + token, payload)
524
+ return candidate
525
+ finally:
526
+ os.close(area)
527
+ raise Refused("no area held in memory for an engine's credentials")
528
+
529
+
530
+ held_at = "-"
531
+ try:
532
+ directory = task_directory(root, task, True)
533
+ lock = locked(directory)
534
+ expire()
535
+ directory = descend(directory, (run, account), True)
536
+ if int(version) < floor_of(directory, revision):
537
+ raise Refused("an older version")
538
+ if retired(directory, slot):
539
+ raise Refused("a retired launch slot")
540
+ if payload is not None:
541
+ held_at = hold(payload)
542
+ try:
543
+ write_private(
544
+ directory,
545
+ "admit-" + token,
546
+ (revision + "\n" + version + "\n").encode()
547
+ + instance()
548
+ + b"\n"
549
+ + held_at.encode()
550
+ + b"\n"
551
+ + slot.encode()
552
+ + b"\n",
553
+ )
554
+ except BaseException:
555
+ if held_at != "-":
556
+ drop_secret([held_at], task, run, account, token)
557
+ raise
558
+ except (OSError, Refused, ValueError):
559
+ sys.exit(4)
560
+ os.write(1, held_at.encode() + b"\n")
561
+ `;
562
+
563
+ /**
564
+ * Raises accounts' floors, as root, and revokes their admissions below
565
+ * them, with what they carried, unread. argv: gate root, task, run, the
566
+ * floors (one per line: account segment, revision, version,
567
+ * tab-separated), the areas a carried credential may wait in (one per
568
+ * line). A floor only ever rises. Exits 2 on malformed arguments, 4 when
569
+ * the gate area is not the runner's or a write fails.
570
+ */
571
+ export const ENGINE_GATE_FLOOR_HELPER =
572
+ GATE_PRELUDE +
573
+ String.raw`
574
+ root, task, run, wanted, carriers = sys.argv[1:6]
575
+ roots = [candidate for candidate in carriers.split("\n") if candidate]
576
+ floors = []
577
+ for line in wanted.split("\n"):
578
+ if not line:
579
+ continue
580
+ parts = line.split("\t")
581
+ if len(parts) != 3 or not SEGMENT.match(parts[0]) or not revision_ok(parts[1]) or not NUMBER.match(parts[2]):
582
+ sys.exit(2)
583
+ floors.append(parts)
584
+ if not NUMBER.match(run) or not floors:
585
+ sys.exit(2)
586
+ try:
587
+ directory = task_directory(root, task, True)
588
+ lock = locked(directory)
589
+ for account, revision, version in floors:
590
+ account_directory = descend(os.dup(directory), (run, account), True)
591
+ try:
592
+ name = floor_name(revision)
593
+ if int(version) > floor_of(account_directory, revision):
594
+ staged = name + ".next"
595
+ try:
596
+ os.unlink(staged, dir_fd=account_directory)
597
+ except FileNotFoundError:
598
+ pass
599
+ write_private(account_directory, staged, version.encode() + b"\n")
600
+ os.rename(staged, name, src_dir_fd=account_directory, dst_dir_fd=account_directory)
601
+ floor = floor_of(account_directory, revision)
602
+ for entry in os.listdir(account_directory):
603
+ if not entry.startswith("admit-"):
604
+ continue
605
+ held = read_private(account_directory, entry).decode().split("\n")
606
+ if len(held) < 2 or (held[0] == revision and NUMBER.match(held[1]) and int(held[1]) < floor):
607
+ os.unlink(entry, dir_fd=account_directory)
608
+ drop_secret(roots, task, run, account, entry[len("admit-"):])
609
+ finally:
610
+ os.close(account_directory)
611
+ except (OSError, Refused, ValueError, UnicodeDecodeError):
612
+ sys.exit(4)
613
+ `;
614
+
615
+ /**
616
+ * The supervisor. argv: gate root, task, run, account segment, token,
617
+ * revision, version, slot ("-": none), the area its carried credentials
618
+ * wait in ("-": none), the user the engine drops to ("-": the provider drops
619
+ * it itself), "--", the engine's argv. The anchor opens what the launch
620
+ * carries and removes it from its area, but never reads it, and closes it
621
+ * right after it forks the engine; the engine reads it once it has dropped
622
+ * to the task user (or, when the provider drops it, hands it on as fd 9 to
623
+ * the injector the provider runs right after its drop).
624
+ */
625
+ export const ENGINE_ANCHOR_SHIM =
626
+ GATE_PRELUDE +
627
+ String.raw`
628
+ import ctypes
629
+ import signal
630
+
631
+
632
+ def refuse(message):
633
+ try:
634
+ os.write(2, b"uai: " + message.encode() + b"\n")
635
+ finally:
636
+ os._exit(126)
637
+
638
+
639
+ arguments = sys.argv[1:]
640
+ if len(arguments) < 12 or arguments[10] != "--":
641
+ refuse("a malformed engine launch")
642
+ root, task, run, account, token, revision, version, slot, held_at, user = arguments[:10]
643
+ argv = arguments[11:]
644
+ if not (
645
+ NUMBER.match(run)
646
+ and SEGMENT.match(account)
647
+ and TOKEN.match(token)
648
+ and NUMBER.match(version)
649
+ and revision_ok(revision)
650
+ and slot_ok(slot)
651
+ and (held_at == "-" or held_at.startswith("/"))
652
+ ):
653
+ refuse("a malformed engine launch")
654
+
655
+
656
+ def claim(area_root):
657
+ # The carried credential's file, open, and gone from its area. Never
658
+ # read here.
659
+ area = account_area(area_root, task, run, account, False)
660
+ try:
661
+ fd = os.open(SECRET + token, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=area)
662
+ try:
663
+ st = checked(fd, stat.S_ISREG)
664
+ single = st.st_nlink == 1
665
+ os.unlink(SECRET + token, dir_fd=area)
666
+ if not single or time.time() - st.st_mtime > SECRET_SECONDS or not tmpfs(fd):
667
+ raise Refused("a carried credential not held as it was written")
668
+ except BaseException:
669
+ os.close(fd)
670
+ raise
671
+ return fd
672
+ finally:
673
+ os.close(area)
674
+ if not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal"):
675
+ refuse("this runtime cannot supervise an engine")
676
+ libc = ctypes.CDLL(None, use_errno=True)
677
+ STOPS = (signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT)
678
+ pending = []
679
+ engine = None
680
+
681
+
682
+ def forward(signum, frame):
683
+ if engine is None:
684
+ pending.append(signum)
685
+ return
686
+ if engine is False:
687
+ return
688
+ try:
689
+ signal.pidfd_send_signal(engine, signum)
690
+ except ProcessLookupError:
691
+ pass
692
+
693
+
694
+ for signum in STOPS:
695
+ signal.signal(signum, forward)
696
+ libc.prctl(15, b"uai-launch", 0, 0, 0)
697
+ reader, writer = os.pipe()
698
+ anchor = os.fork()
699
+ if anchor == 0:
700
+ # R: the anchor, owning everything the engine starts before it admits.
701
+ os.close(reader)
702
+ for signum in STOPS + (signal.SIGPIPE,):
703
+ signal.signal(signum, signal.SIG_IGN)
704
+ if libc.prctl(36, 1, 0, 0, 0) != 0 or libc.prctl(15, b"uai-anchor", 0, 0, 0) != 0:
705
+ refuse("this runtime cannot supervise an engine")
706
+ try:
707
+ with open("/proc/self/oom_score_adj", "rb") as source:
708
+ score = source.read().strip()
709
+ except OSError:
710
+ score = None
711
+ try:
712
+ with open("/proc/self/oom_score_adj", "wb") as sink:
713
+ sink.write(b"-1000")
714
+ except OSError:
715
+ pass
716
+ receipt = "live-" + token
717
+ secret = None
718
+ try:
719
+ directory = task_directory(root, task, False)
720
+ lock = locked(directory)
721
+ directory = descend(directory, (run, account), False)
722
+ gate = "admit-" + token
723
+ held = read_private(directory, gate)
724
+ if held != (
725
+ (revision + "\n" + version + "\n").encode()
726
+ + instance()
727
+ + b"\n"
728
+ + held_at.encode()
729
+ + b"\n"
730
+ + slot.encode()
731
+ + b"\n"
732
+ ):
733
+ raise Refused("another launch's gate, or another runtime instance's")
734
+ if int(version) < floor_of(directory, revision):
735
+ raise Refused("an older version")
736
+ if retired(directory, slot):
737
+ raise Refused("a retired launch slot")
738
+ os.unlink(gate, dir_fd=directory)
739
+ if held_at != "-":
740
+ secret = claim(held_at)
741
+ with open("/proc/self/stat", "rb") as source:
742
+ line = source.read()
743
+ started = line[line.rindex(b")") + 2:].split()[19]
744
+ write_private(directory, receipt, b"%d %s %s\n" % (os.getpid(), started, instance()))
745
+ except (OSError, Refused, ValueError):
746
+ if held_at != "-":
747
+ try:
748
+ drop_secret([held_at], task, run, account, token)
749
+ except (OSError, Refused):
750
+ pass
751
+ refuse("this AI account may not start an engine now")
752
+ engine = os.fork()
753
+ if engine == 0:
754
+ # C: the engine, as the task user.
755
+ try:
756
+ os.close(writer)
757
+ os.close(lock)
758
+ os.close(directory)
759
+ for signum in STOPS + (signal.SIGPIPE,):
760
+ signal.signal(signum, signal.SIG_DFL)
761
+ if score is not None:
762
+ try:
763
+ with open("/proc/self/oom_score_adj", "wb") as sink:
764
+ sink.write(score)
765
+ except OSError:
766
+ pass
767
+ if user != "-":
768
+ import pwd
769
+
770
+ entry = pwd.getpwnam(user)
771
+ superuser = pwd.getpwuid(0)
772
+ os.initgroups(entry.pw_name, entry.pw_gid)
773
+ os.setresgid(entry.pw_gid, entry.pw_gid, entry.pw_gid)
774
+ os.setresuid(entry.pw_uid, entry.pw_uid, entry.pw_uid)
775
+ if entry.pw_uid != 0:
776
+ try:
777
+ os.setuid(0)
778
+ except PermissionError:
779
+ pass
780
+ else:
781
+ os._exit(126)
782
+ if os.environ.get("HOME", superuser.pw_dir) == superuser.pw_dir:
783
+ os.environ["HOME"] = entry.pw_dir
784
+ for name, theirs, ours in (
785
+ ("USER", superuser.pw_name, entry.pw_name),
786
+ ("LOGNAME", superuser.pw_name, entry.pw_name),
787
+ ("SHELL", superuser.pw_shell, entry.pw_shell),
788
+ ("MAIL", "/var/mail/" + superuser.pw_name, "/var/mail/" + entry.pw_name),
789
+ ):
790
+ if os.environ.get(name) == theirs:
791
+ os.environ[name] = ours
792
+ if os.environ.get("XDG_RUNTIME_DIR") == "/run/user/0":
793
+ mine = "/run/user/%d" % entry.pw_uid
794
+ if os.path.isdir(mine):
795
+ os.environ["XDG_RUNTIME_DIR"] = mine
796
+ else:
797
+ del os.environ["XDG_RUNTIME_DIR"]
798
+ if secret is not None:
799
+ if user != "-":
800
+ # Dropped: the engine's own credentials, read now.
801
+ inject(secret)
802
+ elif secret == CARRIER_FD:
803
+ os.set_inheritable(secret, True)
804
+ else:
805
+ os.dup2(secret, CARRIER_FD)
806
+ os.close(secret)
807
+ except BaseException:
808
+ os._exit(126)
809
+ try:
810
+ os.execvp(argv[0], argv)
811
+ except BaseException:
812
+ os._exit(127)
813
+ if secret is not None:
814
+ os.close(secret)
815
+ fcntl.flock(lock, fcntl.LOCK_UN)
816
+ try:
817
+ os.write(writer, b"%d\n" % engine)
818
+ except OSError:
819
+ pass
820
+ null = os.open(os.devnull, os.O_RDWR)
821
+ for stream in (0, 1, 2):
822
+ os.dup2(null, stream)
823
+ os.close(null)
824
+ reported = False
825
+ while True:
826
+ try:
827
+ pid, status = os.wait()
828
+ except ChildProcessError:
829
+ break
830
+ if pid == engine and not reported:
831
+ reported = True
832
+ try:
833
+ os.write(writer, b"%d\n" % status)
834
+ os.close(writer)
835
+ except OSError:
836
+ pass
837
+ # No child left: nothing it owned still runs. The receipt goes, under
838
+ # the lock, as the word of it.
839
+ fcntl.flock(lock, fcntl.LOCK_EX)
840
+ try:
841
+ os.unlink(receipt, dir_fd=directory)
842
+ except OSError:
843
+ pass
844
+ os._exit(0)
845
+
846
+ # W: the launcher, standing for the engine to whoever started it.
847
+ os.close(writer)
848
+ received = b""
849
+
850
+
851
+ def line():
852
+ global received
853
+ while b"\n" not in received:
854
+ chunk = os.read(reader, 64)
855
+ if not chunk:
856
+ return None
857
+ received += chunk
858
+ value, _, received = received.partition(b"\n")
859
+ return int(value)
860
+
861
+
862
+ child = line()
863
+ if child is None:
864
+ os._exit(126)
865
+ try:
866
+ handle = os.pidfd_open(child)
867
+ with open("/proc/%d/stat" % child, "rb") as source:
868
+ data = source.read()
869
+ if int(data[data.rindex(b")") + 2:].split()[1]) != anchor:
870
+ os.close(handle)
871
+ handle = False
872
+ except OSError:
873
+ handle = False
874
+ engine = handle
875
+ for signum in pending:
876
+ forward(signum, None)
877
+ status = line()
878
+ if status is None:
879
+ os._exit(126)
880
+ if os.WIFSIGNALED(status):
881
+ signum = os.WTERMSIG(status)
882
+ try:
883
+ signal.signal(signum, signal.SIG_DFL)
884
+ except (OSError, ValueError):
885
+ pass
886
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum])
887
+ os.kill(os.getpid(), signum)
888
+ os._exit(128 + signum)
889
+ os._exit(os.WEXITSTATUS(status))
890
+ `;
891
+
892
+ /**
893
+ * The cleanup, as root. argv: gate root, task, run (this and older),
894
+ * "named" or "all", the account segments (named; one per line), the bound
895
+ * in seconds, the areas a carried credential may wait in (one per line):
896
+ * every carried credential of what it takes goes with its admission,
897
+ * unread. "all" takes every subject's, a task token's included, and once
898
+ * everything is proven gone, the runs' floors and retired slots with them.
899
+ * Exits 0 when every selected anchor ended having proved its
900
+ * subtree empty and nothing it was admitted for remains; 5 when something
901
+ * must be killed on a kernel or Python without pidfds; 6 when an anchor of
902
+ * this runtime instance ended without proving its subtree empty (its
903
+ * receipt is kept: every cleanup fails so until the instance is gone), or
904
+ * when anything selected (an anchor that cannot reap, stopped or wedged, a
905
+ * process that will not die) is still there at the bound: only stopping the
906
+ * runtime proves it gone; 7 when the gate area is not the runner's.
907
+ */
908
+ export const ENGINE_ANCHOR_STOP_HELPER =
909
+ GATE_PRELUDE +
910
+ String.raw`
911
+ import select
912
+ import signal
913
+ import time
914
+
915
+ root, task, through, mode, wanted, seconds, carriers = sys.argv[1:8]
916
+ through = int(through)
917
+ roots = [candidate for candidate in carriers.split("\n") if candidate]
918
+ accounts = set(name for name in wanted.split("\n") if name)
919
+ deadline = time.monotonic() + float(seconds)
920
+ ANCHOR = b"uai-anchor"
921
+ LAUNCH = b"uai-launch"
922
+
923
+
924
+ def selected(run, account):
925
+ return (
926
+ NUMBER.match(run) is not None
927
+ and int(run) <= through
928
+ and SEGMENT.match(account) is not None
929
+ and (mode == "all" or account in accounts)
930
+ )
931
+
932
+
933
+ def described(pid):
934
+ try:
935
+ with open("/proc/%d/stat" % pid, "rb") as source:
936
+ data = source.read()
937
+ owner = os.stat("/proc/%d" % pid).st_uid
938
+ except (FileNotFoundError, ProcessLookupError):
939
+ return None
940
+ fields = data[data.rindex(b")") + 2:].split()
941
+ return data[data.index(b"(") + 1:data.rindex(b")")], fields[0], int(fields[1]), int(fields[19]), owner
942
+
943
+
944
+ def table():
945
+ processes = {}
946
+ for entry in os.listdir("/proc"):
947
+ if entry.isdigit():
948
+ process = described(int(entry))
949
+ if process is not None:
950
+ processes[int(entry)] = process
951
+ return processes
952
+
953
+
954
+ def under(processes, ancestor):
955
+ children = {}
956
+ for pid, process in processes.items():
957
+ children.setdefault(process[2], []).append(pid)
958
+ found = []
959
+ stack = list(children.get(ancestor, ()))
960
+ while stack:
961
+ pid = stack.pop()
962
+ stack.extend(children.get(pid, ()))
963
+ if processes[pid][1] not in (b"Z", b"X"):
964
+ found.append(pid)
965
+ return found
966
+
967
+
968
+ def launch_of(pid):
969
+ try:
970
+ with open("/proc/%d/cmdline" % pid, "rb") as source:
971
+ parts = source.read().split(b"\0")
972
+ at = parts.index(b"-c")
973
+ named = [part.decode() for part in parts[at + 2:at + 6]]
974
+ except (FileNotFoundError, ProcessLookupError, ValueError, UnicodeDecodeError):
975
+ return None
976
+ if len(named) != 4 or named[0] != root or named[1] != task:
977
+ return None
978
+ return named[2], named[3]
979
+
980
+
981
+ def pin(pid, started):
982
+ if not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal"):
983
+ sys.exit(5)
984
+ try:
985
+ handle = os.pidfd_open(pid)
986
+ except (FileNotFoundError, ProcessLookupError):
987
+ return None
988
+ except OSError as error:
989
+ if error.errno == errno.ENOSYS:
990
+ sys.exit(5)
991
+ raise
992
+ now = described(pid)
993
+ if now is None or now[3] != started:
994
+ os.close(handle)
995
+ return None
996
+ return handle
997
+
998
+
999
+ def kill(handle):
1000
+ try:
1001
+ signal.pidfd_send_signal(handle, signal.SIGKILL)
1002
+ except ProcessLookupError:
1003
+ pass
1004
+
1005
+
1006
+ def kill_all(processes, pids):
1007
+ for pid in pids:
1008
+ handle = pin(pid, processes[pid][3])
1009
+ if handle is not None:
1010
+ kill(handle)
1011
+ os.close(handle)
1012
+
1013
+
1014
+ def drain(handle, anchor):
1015
+ """Kill everything under the anchor until the anchor ends: it ends on its
1016
+ own once it has no child left. False when it has not by the bound."""
1017
+ poller = select.poll()
1018
+ poller.register(handle, select.POLLIN)
1019
+ while not poller.poll(0):
1020
+ processes = table()
1021
+ kill_all(processes, under(processes, anchor))
1022
+ if time.monotonic() > deadline:
1023
+ return False
1024
+ poller.poll(20)
1025
+ return True
1026
+
1027
+
1028
+ def receipt_left(run, account, name):
1029
+ fcntl.flock(lock, fcntl.LOCK_EX)
1030
+ try:
1031
+ try:
1032
+ account_directory = descend(os.dup(directory), (run, account), False)
1033
+ except FileNotFoundError:
1034
+ return False
1035
+ try:
1036
+ os.stat(name, dir_fd=account_directory, follow_symlinks=False)
1037
+ return True
1038
+ except FileNotFoundError:
1039
+ return False
1040
+ finally:
1041
+ os.close(account_directory)
1042
+ finally:
1043
+ fcntl.flock(lock, fcntl.LOCK_UN)
1044
+
1045
+
1046
+ try:
1047
+ current = instance()
1048
+ except OSError:
1049
+ sys.exit(7)
1050
+ try:
1051
+ directory = task_directory(root, task, False)
1052
+ except FileNotFoundError:
1053
+ directory = None
1054
+ except (OSError, Refused):
1055
+ sys.exit(7)
1056
+
1057
+ receipts = []
1058
+ if directory is not None:
1059
+ try:
1060
+ lock = locked(directory)
1061
+ for run in os.listdir(directory):
1062
+ if run == ".lock" or not NUMBER.match(run) or int(run) > through:
1063
+ continue
1064
+ run_directory = private(directory, run, False)
1065
+ try:
1066
+ for account in os.listdir(run_directory):
1067
+ if not selected(run, account):
1068
+ continue
1069
+ account_directory = private(run_directory, account, False)
1070
+ try:
1071
+ for name in os.listdir(account_directory):
1072
+ if name.startswith("admit-") or name.startswith(SECRET):
1073
+ os.unlink(name, dir_fd=account_directory)
1074
+ elif name.startswith("live-"):
1075
+ pid, started, owner = read_private(account_directory, name).split()
1076
+ if owner != current:
1077
+ # That runtime instance is gone, and
1078
+ # every process of it.
1079
+ os.unlink(name, dir_fd=account_directory)
1080
+ else:
1081
+ receipts.append((run, account, name, int(pid), int(started)))
1082
+ elif not (name.startswith("floor-") or name.startswith(RETIRED)):
1083
+ raise Refused("an unknown gate entry")
1084
+ if any(name.startswith("admit-") for name in os.listdir(account_directory)):
1085
+ raise Refused("a revoked gate came back")
1086
+ finally:
1087
+ os.close(account_directory)
1088
+ finally:
1089
+ os.close(run_directory)
1090
+ # What waits elsewhere in memory for what it takes goes too, unread.
1091
+ for area_root in roots:
1092
+ if area_root == root:
1093
+ continue
1094
+ try:
1095
+ area_directory = task_directory(area_root, task, False)
1096
+ except FileNotFoundError:
1097
+ continue
1098
+ try:
1099
+ for run in os.listdir(area_directory):
1100
+ if not NUMBER.match(run) or int(run) > through:
1101
+ continue
1102
+ run_directory = private(area_directory, run, False)
1103
+ try:
1104
+ for account in os.listdir(run_directory):
1105
+ if not selected(run, account):
1106
+ continue
1107
+ account_directory = private(run_directory, account, False)
1108
+ try:
1109
+ for name in os.listdir(account_directory):
1110
+ if name.startswith(SECRET):
1111
+ os.unlink(name, dir_fd=account_directory)
1112
+ finally:
1113
+ os.close(account_directory)
1114
+ try:
1115
+ os.rmdir(account, dir_fd=run_directory)
1116
+ except OSError:
1117
+ pass
1118
+ finally:
1119
+ os.close(run_directory)
1120
+ try:
1121
+ os.rmdir(run, dir_fd=area_directory)
1122
+ except OSError:
1123
+ pass
1124
+ finally:
1125
+ os.close(area_directory)
1126
+ fcntl.flock(lock, fcntl.LOCK_UN)
1127
+ except (OSError, Refused, ValueError):
1128
+ sys.exit(7)
1129
+
1130
+ lost = False
1131
+ for run, account, name, pid, started in receipts:
1132
+ handle = pin(pid, started)
1133
+ if handle is not None:
1134
+ try:
1135
+ process = described(pid)
1136
+ if process is not None and process[0] == ANCHOR and process[4] == ME and not drain(handle, pid):
1137
+ # Alive at the bound (stopped, wedged): it cannot prove its
1138
+ # subtree empty, and nothing here can.
1139
+ lost = True
1140
+ continue
1141
+ finally:
1142
+ os.close(handle)
1143
+ # The anchor has ended (or was never this process): its receipt says
1144
+ # how. Gone, it proved its subtree empty; left, it died owning what may
1145
+ # still run elsewhere, and the receipt stays as the witness.
1146
+ if receipt_left(run, account, name):
1147
+ lost = True
1148
+
1149
+ # Launchers, and anchors never admitted: they hold the launch's
1150
+ # environment, and start nothing now. An anchor that owns anything without a
1151
+ # receipt of this instance was not admitted here: lost. An admitted one is
1152
+ # never killed here either: its receipt decided above.
1153
+ admitted = set((pid, started) for _, _, _, pid, started in receipts)
1154
+ processes = table()
1155
+ for pid, process in processes.items():
1156
+ if process[0] not in (ANCHOR, LAUNCH) or process[4] != ME or process[1] in (b"Z", b"X"):
1157
+ continue
1158
+ if (pid, process[3]) in admitted:
1159
+ continue
1160
+ launch = launch_of(pid)
1161
+ if launch is None or not selected(*launch):
1162
+ continue
1163
+ if process[0] == ANCHOR and under(processes, pid):
1164
+ lost = True
1165
+ continue
1166
+ handle = pin(pid, process[3])
1167
+ if handle is None:
1168
+ continue
1169
+ try:
1170
+ kill(handle)
1171
+ poller = select.poll()
1172
+ poller.register(handle, select.POLLIN)
1173
+ if not poller.poll(max(deadline - time.monotonic(), 0.5) * 1000):
1174
+ # Killed and still there at the bound.
1175
+ lost = True
1176
+ finally:
1177
+ os.close(handle)
1178
+
1179
+ if lost:
1180
+ sys.exit(6)
1181
+ if mode == "all" and directory is not None:
1182
+ fcntl.flock(lock, fcntl.LOCK_EX)
1183
+ for run in os.listdir(directory):
1184
+ if run == ".lock" or not NUMBER.match(run) or int(run) > through:
1185
+ continue
1186
+ run_directory = private(directory, run, False)
1187
+ for account in os.listdir(run_directory):
1188
+ account_directory = private(run_directory, account, False)
1189
+ for name in os.listdir(account_directory):
1190
+ if name.startswith("floor-") or name.startswith(RETIRED):
1191
+ os.unlink(name, dir_fd=account_directory)
1192
+ os.close(account_directory)
1193
+ try:
1194
+ os.rmdir(account, dir_fd=run_directory)
1195
+ except OSError:
1196
+ pass
1197
+ os.close(run_directory)
1198
+ try:
1199
+ os.rmdir(run, dir_fd=directory)
1200
+ except OSError:
1201
+ pass
1202
+ fcntl.flock(lock, fcntl.LOCK_UN)
1203
+ `;
1204
+
1205
+ /**
1206
+ * Revokes one admission, as root, with what it carried, unread: a launch
1207
+ * refused after it was admitted, or that its provider could not start.
1208
+ * argv: gate root, task, run, account segment, token, the areas a carried
1209
+ * credential may wait in (one per line). Exits 0 once neither is left
1210
+ * (whether or not the launch had already used them), 2 on malformed
1211
+ * arguments, 7 when the gate area is not the runner's.
1212
+ */
1213
+ export const ENGINE_GATE_REVOKE_HELPER =
1214
+ GATE_PRELUDE +
1215
+ String.raw`
1216
+ root, task, run, account, token, carriers = sys.argv[1:7]
1217
+ roots = [candidate for candidate in carriers.split("\n") if candidate]
1218
+ if not (NUMBER.match(run) and SEGMENT.match(account) and TOKEN.match(token)):
1219
+ sys.exit(2)
1220
+ try:
1221
+ try:
1222
+ directory = task_directory(root, task, False)
1223
+ except FileNotFoundError:
1224
+ sys.exit(0)
1225
+ lock = locked(directory)
1226
+ try:
1227
+ area = descend(os.dup(directory), (run, account), False)
1228
+ except FileNotFoundError:
1229
+ area = None
1230
+ if area is not None:
1231
+ try:
1232
+ os.unlink("admit-" + token, dir_fd=area)
1233
+ except FileNotFoundError:
1234
+ pass
1235
+ finally:
1236
+ os.close(area)
1237
+ drop_secret(roots, task, run, account, token)
1238
+ except (OSError, Refused):
1239
+ sys.exit(7)
1240
+ `;
1241
+
1242
+ /**
1243
+ * Retires one launch slot, as root: marks it retired (`retired-<slot>`),
1244
+ * then revokes every admission of it not yet used, with what it carried,
1245
+ * unread. From then on, under the same lock, no admission of the slot is
1246
+ * written or used. Where its run and subject have no gate area, nothing of
1247
+ * it is there to retire, and nothing is made: a launch handed over has its
1248
+ * admission written first, so its area is there. argv: gate root, task,
1249
+ * run, account segment, slot, the areas a carried credential may wait in
1250
+ * (one per line). Exits 0 once the slot is marked and none of its
1251
+ * admissions is left (or nothing of it is there), 2 on malformed
1252
+ * arguments, 7 when the gate area is not the runner's or a write fails.
1253
+ */
1254
+ export const ENGINE_GATE_RETIRE_HELPER =
1255
+ GATE_PRELUDE +
1256
+ String.raw`
1257
+ root, task, run, account, slot, carriers = sys.argv[1:7]
1258
+ roots = [candidate for candidate in carriers.split("\n") if candidate]
1259
+ if not (NUMBER.match(run) and SEGMENT.match(account) and TOKEN.match(slot)):
1260
+ sys.exit(2)
1261
+ try:
1262
+ try:
1263
+ directory = task_directory(root, task, False)
1264
+ except FileNotFoundError:
1265
+ sys.exit(0)
1266
+ lock = locked(directory)
1267
+ try:
1268
+ directory = descend(directory, (run, account), False)
1269
+ except FileNotFoundError:
1270
+ sys.exit(0)
1271
+ if not retired(directory, slot):
1272
+ write_private(directory, RETIRED + slot, b"")
1273
+ for entry in os.listdir(directory):
1274
+ if not entry.startswith("admit-"):
1275
+ continue
1276
+ held = read_private(directory, entry).split(b"\n")
1277
+ # The slot's own, and any written before admissions named their
1278
+ # slot (one line short): revoked.
1279
+ if len(held) < 6 or held[4] == slot.encode():
1280
+ os.unlink(entry, dir_fd=directory)
1281
+ drop_secret(roots, task, run, account, entry[len("admit-"):])
1282
+ except (OSError, Refused, ValueError):
1283
+ sys.exit(7)
1284
+ `;
1285
+
1286
+ /**
1287
+ * The injector a provider that drops to the task user itself runs right
1288
+ * after that drop (ADR-154 §6): it reads the launch's carried credentials
1289
+ * from fd 9, the supervisor's hand-off, into its own environment, closes
1290
+ * it, and runs the rest of its argv. argv: "9", "--", the process.
1291
+ */
1292
+ export const ENGINE_CREDENTIAL_INJECTOR =
1293
+ GATE_PRELUDE +
1294
+ String.raw`
1295
+ arguments = sys.argv[1:]
1296
+ if len(arguments) < 3 or arguments[0] != str(CARRIER_FD) or arguments[1] != "--":
1297
+ os._exit(126)
1298
+ try:
1299
+ inject(CARRIER_FD)
1300
+ except BaseException:
1301
+ os._exit(126)
1302
+ try:
1303
+ os.execvp(arguments[2], arguments[2:])
1304
+ except BaseException:
1305
+ os._exit(127)
1306
+ `;
1307
+
1308
+ const ENGINE_ANCHOR_STOP_SECONDS = 15;
1309
+ let anchorStopSeconds = ENGINE_ANCHOR_STOP_SECONDS;
1310
+
1311
+ /** The helpers' Python ignores the task's module paths (-I) and runs with
1312
+ * no preloads from the environment it is exec'd with. */
1313
+ const CLEAN_PYTHON_ENV: Readonly<Record<string, string>> = Object.freeze({
1314
+ LD_AUDIT: "",
1315
+ LD_LIBRARY_PATH: "",
1316
+ LD_PRELOAD: "",
1317
+ PYTHONHOME: "",
1318
+ PYTHONPATH: "",
1319
+ });
1320
+
1321
+ /** The supervisor's argv for one admitted launch, up to the user it drops
1322
+ * to (the provider appends that, "--" and the launch's own argv);
1323
+ * `heldAt`, the area its carried credentials wait in ("-": none). */
1324
+ export function engineAnchorPrefix(
1325
+ identity: EngineLaunchIdentity,
1326
+ token: string,
1327
+ heldAt = "-",
1328
+ ): [string, ...string[]] {
1329
+ return [
1330
+ "/usr/bin/python3",
1331
+ "-I",
1332
+ "-c",
1333
+ ENGINE_ANCHOR_SHIM,
1334
+ gateRoot,
1335
+ identity.taskId,
1336
+ String(identity.runEpoch),
1337
+ gateSegment(identity),
1338
+ token,
1339
+ gateRevision(identity),
1340
+ gateVersion(identity),
1341
+ gateSlot(identity),
1342
+ heldAt,
1343
+ ];
1344
+ }
1345
+
1346
+ /** The injector a provider that drops to the task user itself runs right
1347
+ * after its drop, before the launch, when the launch carries credentials. */
1348
+ const INJECTOR_PREFIX: readonly [string, ...string[]] = Object.freeze([
1349
+ "/usr/bin/python3",
1350
+ "-I",
1351
+ "-c",
1352
+ ENGINE_CREDENTIAL_INJECTOR,
1353
+ "9",
1354
+ "--",
1355
+ ]) as unknown as readonly [string, ...string[]];
1356
+
1357
+ /** One written admission: the anchor its launch starts under, and its
1358
+ * revocation. */
1359
+ export interface EngineAdmission {
1360
+ readonly anchor: TaskEnvironmentAnchor;
1361
+ /** Revoke the admission and what it carried, unread: the launch never
1362
+ * started, or must not. Resolves once neither is left, whether or not the
1363
+ * launch had already used them. Rejects when that could not be done
1364
+ * now; the revocation is then retained and retried until done, or until
1365
+ * a cleanup's stop of its run supersedes it (`retainedEngineRevocations`). */
1366
+ revoke(): Promise<void>;
1367
+ }
1368
+
1369
+ /**
1370
+ * Admit one launch of `identity` in the task's runtime: write its token,
1371
+ * as root, the exec admitted by `admit` at the provider's last moment.
1372
+ * `carried`, the engine credentials the launch carries (ADR-154 §6, for a
1373
+ * provider whose process environment would otherwise put them on a command
1374
+ * line), travel on the helper's stdin and wait in memory beside the token
1375
+ * until the supervisor hands them to the engine. Resolves to the admission.
1376
+ * Rejects with `EngineLaunchRefused` when it could not be written (a
1377
+ * version below the account's floor, a retired slot, or no memory to hold
1378
+ * what it carries, included): the launch must not start.
1379
+ */
1380
+ export async function admitEngineLaunch(
1381
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1382
+ identity: EngineLaunchIdentity,
1383
+ admit: () => void,
1384
+ carried: Readonly<Record<string, string>> | null = null,
1385
+ ): Promise<EngineAdmission> {
1386
+ const token = randomBytes(16).toString("hex");
1387
+ const result = await environment.exec({
1388
+ argv: [
1389
+ "/usr/bin/python3",
1390
+ "-I",
1391
+ "-c",
1392
+ ENGINE_GATE_WRITE_HELPER,
1393
+ gateRoot,
1394
+ identity.taskId,
1395
+ String(identity.runEpoch),
1396
+ gateSegment(identity),
1397
+ token,
1398
+ gateRevision(identity),
1399
+ gateVersion(identity),
1400
+ gateSlot(identity),
1401
+ carrierRoots(),
1402
+ carried === null ? "-" : "carry",
1403
+ ],
1404
+ user: "root",
1405
+ env: CLEAN_PYTHON_ENV,
1406
+ projectEnvProfile: "maintenance",
1407
+ ...(carried === null ? {} : { stdin: Buffer.from(JSON.stringify(carried), "utf8") }),
1408
+ timeoutMs: 30_000,
1409
+ maxOutputBytes: 4 * 1024,
1410
+ admit,
1411
+ });
1412
+ const heldAt = Buffer.from(result.stdout).toString("utf8").trim();
1413
+ if (result.exitCode !== 0 || (carried === null ? heldAt !== "-" : !heldAt.startsWith("/"))) {
1414
+ // The helper ran, and may have written the token and what it carries
1415
+ // though its report was lost: both are revoked, unread, before the
1416
+ // launch is refused (retained when that cannot be done now).
1417
+ await revokeOrRetain(environment, identity, token).catch(() => {});
1418
+ throw new EngineLaunchRefused("the task runtime could not admit the engine launch");
1419
+ }
1420
+ const anchor: TaskEnvironmentAnchor =
1421
+ carried === null
1422
+ ? { prefix: engineAnchorPrefix(identity, token) }
1423
+ : { prefix: engineAnchorPrefix(identity, token, heldAt), injector: INJECTOR_PREFIX };
1424
+ return { anchor, revoke: () => revokeOrRetain(environment, identity, token) };
1425
+ }
1426
+
1427
+ /** A revocation, or a slot's retirement, not done yet: retried until it is,
1428
+ * or a cleanup's stop of its run supersedes it. */
1429
+ interface RetainedRevocation {
1430
+ readonly environment: Pick<TaskEnvironmentHandle, "exec">;
1431
+ readonly identity: EngineLaunchIdentity;
1432
+ /** The admission revoked; null: the identity's slot, retired. */
1433
+ readonly token: string | null;
1434
+ attempts: number;
1435
+ }
1436
+
1437
+ const retained = new Map<string, RetainedRevocation>();
1438
+ /** Revocations and slot retirements under way. A cleanup that proves them
1439
+ * done (`dischargeEngineRevocations`) marks them, and one so marked that
1440
+ * fails after is not retained: nothing is left to do, nor retried. */
1441
+ const underWay = new Set<{
1442
+ readonly environment: Pick<TaskEnvironmentHandle, "exec">;
1443
+ readonly identity: EngineLaunchIdentity;
1444
+ discharged: boolean;
1445
+ }>();
1446
+ const REVOCATION_RETRY_FIRST_MS = 1_000;
1447
+ const REVOCATION_RETRY_MOST_MS = 60_000;
1448
+ let revocationRetryFirstMs = REVOCATION_RETRY_FIRST_MS;
1449
+
1450
+ function retainedKey(identity: EngineLaunchIdentity, token: string | null): string {
1451
+ return [identity.taskId, identity.runEpoch, gateSegment(identity), token ?? `slot:${gateSlot(identity)}`].join("\0");
1452
+ }
1453
+
1454
+ /** Revoke one admission (`token`), or retire the identity's slot (null). */
1455
+ function revocation(
1456
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1457
+ identity: EngineLaunchIdentity,
1458
+ token: string | null,
1459
+ ): Promise<void> {
1460
+ return token === null ? retireSlot(environment, identity) : revokeEngineAdmission(environment, identity, token);
1461
+ }
1462
+
1463
+ /**
1464
+ * Revoke one admission and what it carried, or retire a slot. When that
1465
+ * cannot be done now (its runtime unreachable for a moment), it is retained
1466
+ * and retried, backing off, until it is done or a cleanup's stop of its run
1467
+ * in that runtime supersedes it: an admission and its plaintext are never
1468
+ * left only because one attempt failed. Rejects all the same.
1469
+ */
1470
+ async function revokeOrRetain(
1471
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1472
+ identity: EngineLaunchIdentity,
1473
+ token: string | null,
1474
+ ): Promise<void> {
1475
+ const key = retainedKey(identity, token);
1476
+ const attempt = { environment, identity, discharged: false };
1477
+ underWay.add(attempt);
1478
+ try {
1479
+ await revocation(environment, identity, token);
1480
+ } catch (error) {
1481
+ if (!attempt.discharged && !retained.has(key)) {
1482
+ const entry: RetainedRevocation = { environment, identity, token, attempts: 0 };
1483
+ retained.set(key, entry);
1484
+ console.warn(
1485
+ `[engine-anchor] ${identity.taskId}: ${
1486
+ token === null ? "a retired launch slot could not be closed" : "an engine admission could not be revoked"
1487
+ } yet; retrying (${error instanceof Error ? error.message : String(error)})`,
1488
+ );
1489
+ retryRevocation(key, entry);
1490
+ }
1491
+ throw error;
1492
+ } finally {
1493
+ underWay.delete(attempt);
1494
+ }
1495
+ // Done: whatever was retained for it is too.
1496
+ retained.delete(key);
1497
+ }
1498
+
1499
+ function retryRevocation(key: string, entry: RetainedRevocation): void {
1500
+ const delay = Math.min(REVOCATION_RETRY_MOST_MS, revocationRetryFirstMs * 2 ** entry.attempts);
1501
+ const timer = setTimeout(() => {
1502
+ if (retained.get(key) !== entry) return;
1503
+ entry.attempts += 1;
1504
+ revocation(entry.environment, entry.identity, entry.token).then(
1505
+ () => {
1506
+ if (retained.get(key) === entry) retained.delete(key);
1507
+ },
1508
+ () => {
1509
+ if (retained.get(key) === entry) retryRevocation(key, entry);
1510
+ },
1511
+ );
1512
+ }, delay);
1513
+ timer.unref?.();
1514
+ }
1515
+
1516
+ /** The revocations and slot retirements retained for `taskId` (every
1517
+ * task's when absent): what a cleanup of their runs has yet to supersede.
1518
+ * `credentialId` null: an agent's task token's. */
1519
+ export function retainedEngineRevocations(
1520
+ taskId?: string,
1521
+ ): Array<{ taskId: string; runEpoch: number; credentialId: string | null; slot: boolean }> {
1522
+ return [...retained.values()]
1523
+ .filter((entry) => taskId === undefined || entry.identity.taskId === taskId)
1524
+ .map(({ identity, token }) => ({
1525
+ taskId: identity.taskId,
1526
+ runEpoch: identity.runEpoch,
1527
+ credentialId: identity.subject === "account" ? identity.credentialId : null,
1528
+ slot: token === null,
1529
+ }));
1530
+ }
1531
+
1532
+ /**
1533
+ * A cleanup proved every admission of these runs and accounts (of every
1534
+ * subject, a task token's included, when none is named) revoked in
1535
+ * `environment` (its stop revokes every one not yet used), or, when no
1536
+ * environment is named, in every runtime they were written in: the
1537
+ * revocations and slot retirements retained for them are done.
1538
+ */
1539
+ export function dischargeEngineRevocations(
1540
+ scope: { taskId: string; runEpoch: number; credentialIds?: readonly string[] },
1541
+ environment?: Pick<TaskEnvironmentHandle, "exec">,
1542
+ ): void {
1543
+ const named = scope.credentialIds === undefined ? null : new Set(scope.credentialIds);
1544
+ const covered = (entry: { identity: EngineLaunchIdentity; environment: object }): boolean =>
1545
+ entry.identity.taskId === scope.taskId &&
1546
+ entry.identity.runEpoch <= scope.runEpoch &&
1547
+ (!named || (entry.identity.subject === "account" && named.has(entry.identity.credentialId))) &&
1548
+ (!environment || sameRuntime(environment, entry.environment));
1549
+ for (const [key, entry] of retained) {
1550
+ if (covered(entry)) retained.delete(key);
1551
+ }
1552
+ // One still under way is done too: should it fail now, it is not kept.
1553
+ for (const attempt of underWay) {
1554
+ if (covered(attempt)) attempt.discharged = true;
1555
+ }
1556
+ }
1557
+
1558
+ /** Whether two task environments are the same runtime. */
1559
+ export function sameRuntime(left: object, right: object): boolean {
1560
+ if (left === right) return true;
1561
+ const identity = (environment: object): unknown => (environment as { durableIdentity?: unknown }).durableIdentity;
1562
+ return typeof identity(left) === "string" && identity(left) === identity(right);
1563
+ }
1564
+
1565
+ /** Revoke one admission and what it carried, as root, unread. */
1566
+ async function revokeEngineAdmission(
1567
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1568
+ identity: EngineLaunchIdentity,
1569
+ token: string,
1570
+ ): Promise<void> {
1571
+ const result = await environment.exec({
1572
+ argv: [
1573
+ "/usr/bin/python3",
1574
+ "-I",
1575
+ "-c",
1576
+ ENGINE_GATE_REVOKE_HELPER,
1577
+ gateRoot,
1578
+ identity.taskId,
1579
+ String(identity.runEpoch),
1580
+ gateSegment(identity),
1581
+ token,
1582
+ carrierRoots(),
1583
+ ],
1584
+ user: "root",
1585
+ env: CLEAN_PYTHON_ENV,
1586
+ projectEnvProfile: "maintenance",
1587
+ timeoutMs: 30_000,
1588
+ maxOutputBytes: 4 * 1024,
1589
+ });
1590
+ if (result.exitCode !== 0) throw new Error("the task runtime could not revoke an engine admission");
1591
+ }
1592
+
1593
+ /**
1594
+ * Retire `identity`'s launch slot in one task runtime (ADR-154 §6): from
1595
+ * then on no admission of it is written or used there, and every one it
1596
+ * wrote that no launch has used is revoked, with what it carried, unread.
1597
+ * Resolves once that is so. Rejects when it could not be done now; it is
1598
+ * then retained and retried, backing off, until it is done or a cleanup's
1599
+ * stop of its run supersedes it.
1600
+ */
1601
+ export function retireEngineLaunchSlot(
1602
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1603
+ identity: EngineLaunchIdentity & { slotId: string },
1604
+ ): Promise<void> {
1605
+ return revokeOrRetain(environment, identity, null);
1606
+ }
1607
+
1608
+ /** Retire one slot, as root. */
1609
+ async function retireSlot(environment: Pick<TaskEnvironmentHandle, "exec">, identity: EngineLaunchIdentity): Promise<void> {
1610
+ if (identity.slotId === null) throw new Error("a launch with no slot cannot be retired");
1611
+ const result = await environment.exec({
1612
+ argv: [
1613
+ "/usr/bin/python3",
1614
+ "-I",
1615
+ "-c",
1616
+ ENGINE_GATE_RETIRE_HELPER,
1617
+ gateRoot,
1618
+ identity.taskId,
1619
+ String(identity.runEpoch),
1620
+ gateSegment(identity),
1621
+ identity.slotId,
1622
+ carrierRoots(),
1623
+ ],
1624
+ user: "root",
1625
+ env: CLEAN_PYTHON_ENV,
1626
+ projectEnvProfile: "maintenance",
1627
+ timeoutMs: 30_000,
1628
+ maxOutputBytes: 4 * 1024,
1629
+ });
1630
+ if (result.exitCode !== 0) throw new Error("the task runtime could not retire an engine launch slot");
1631
+ }
1632
+
1633
+ /**
1634
+ * Raise, in one task runtime, each account's floor for `runEpoch` to the
1635
+ * version given (a floor never falls), and revoke its admissions below it:
1636
+ * from then on, nothing admitted on an older version of that revision
1637
+ * starts there. Rejects when it could not.
1638
+ */
1639
+ export async function raiseEngineAdmissionFloors(
1640
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1641
+ request: { taskId: string; runEpoch: number; floors: readonly EngineAdmissionFloor[] },
1642
+ ): Promise<void> {
1643
+ if (request.floors.length === 0) return;
1644
+ const result = await environment.exec({
1645
+ argv: [
1646
+ "/usr/bin/python3",
1647
+ "-I",
1648
+ "-",
1649
+ gateRoot,
1650
+ request.taskId,
1651
+ String(request.runEpoch),
1652
+ request.floors
1653
+ .map((floor) => `${idPathSegment(floor.credentialId)}\t${floor.revision}\t${floor.version}`)
1654
+ .join("\n"),
1655
+ carrierRoots(),
1656
+ ],
1657
+ user: "root",
1658
+ env: CLEAN_PYTHON_ENV,
1659
+ projectEnvProfile: "maintenance",
1660
+ stdin: Buffer.from(ENGINE_GATE_FLOOR_HELPER, "utf8"),
1661
+ timeoutMs: 30_000,
1662
+ maxOutputBytes: 4 * 1024,
1663
+ });
1664
+ if (result.exitCode !== 0) throw new Error("the task runtime could not hold back an AI account's older version");
1665
+ }
1666
+
1667
+ /**
1668
+ * Stop, in one task runtime, every anchored launch of `credentialIds` (of
1669
+ * every account when absent) for `runEpoch` and older runs, and everything
1670
+ * each started, and prove none is left; revoke every admission not yet
1671
+ * used. Rejects with `EngineAnchorLost` when only stopping the runtime can
1672
+ * prove it, otherwise with an error when it cannot be proven at all.
1673
+ */
1674
+ export async function stopAnchoredEngines(
1675
+ environment: Pick<TaskEnvironmentHandle, "exec">,
1676
+ request: { taskId: string; runEpoch: number; credentialIds?: readonly string[] },
1677
+ ): Promise<void> {
1678
+ const named = request.credentialIds;
1679
+ const result = await environment.exec({
1680
+ argv: [
1681
+ "/usr/bin/python3",
1682
+ "-I",
1683
+ "-",
1684
+ gateRoot,
1685
+ request.taskId,
1686
+ String(request.runEpoch),
1687
+ named === undefined ? "all" : "named",
1688
+ // Every provider refuses an empty argv entry, and "all" names no
1689
+ // account: a lone newline stands for the empty list (the helper drops
1690
+ // blank names). Live 2026-10-11: the empty string failed every
1691
+ // stop-all, so no cleanup, recovery sweep or relaunch could finish.
1692
+ (named ?? []).map(idPathSegment).join("\n") || "\n",
1693
+ String(anchorStopSeconds),
1694
+ carrierRoots(),
1695
+ ],
1696
+ user: "root",
1697
+ env: CLEAN_PYTHON_ENV,
1698
+ projectEnvProfile: "maintenance",
1699
+ stdin: Buffer.from(ENGINE_ANCHOR_STOP_HELPER, "utf8"),
1700
+ timeoutMs: (anchorStopSeconds + 10) * 1000,
1701
+ maxOutputBytes: 4 * 1024,
1702
+ });
1703
+ if (result.exitCode === 6 || result.exitCode === 7) throw new EngineAnchorLost();
1704
+ if (result.exitCode !== 0) {
1705
+ throw new Error("an engine on a removed AI account could not be proven stopped");
1706
+ }
1707
+ // Every admission of these runs and accounts not yet used is revoked here.
1708
+ dischargeEngineRevocations(request, environment);
1709
+ }
1710
+
1711
+ /** Whether a request starts something that carries, or may carry, leased
1712
+ * engine material: what a task's engines run as, or anything with an
1713
+ * engine credential, an agent's task token or a leased engine's marker in
1714
+ * its environment. */
1715
+ export function launchesEngine(request: {
1716
+ projectEnvProfile: string;
1717
+ env?: Readonly<Record<string, string>>;
1718
+ inheritEnv?: readonly string[];
1719
+ }): boolean {
1720
+ return (
1721
+ request.projectEnvProfile === "task" ||
1722
+ Object.keys(request.env ?? {}).some(
1723
+ (name) => isLaunchSecretEnvName(name) || name === ENGINE_RUN_ENV || name === ENGINE_ACCOUNT_ENV,
1724
+ ) ||
1725
+ (request.inheritEnv ?? []).some(isLaunchSecretEnvName)
1726
+ );
1727
+ }
1728
+
1729
+ /** The process-start methods an adapter may call on its environment. */
1730
+ const LAUNCHES: ReadonlySet<PropertyKey> = new Set(["exec", "spawn", "spawnSession", "launchDetachedSession"]);
1731
+
1732
+ /** How the launches through an anchored environment are admitted. */
1733
+ export interface EngineLaunchGate {
1734
+ /** Admit one launch, carrying `carried` (see `admitEngineLaunch`):
1735
+ * resolves to its admission, or rejects and it starts nothing. */
1736
+ admit(carried: Readonly<Record<string, string>> | null): Promise<EngineAdmission>;
1737
+ /** Whether a launch may still start; asked once more just before the
1738
+ * launch is handed to the provider, and again at the provider's last
1739
+ * moment (the request's `admit`), after anything it awaits first. */
1740
+ held(): boolean;
1741
+ }
1742
+
1743
+ /**
1744
+ * ADR-154 §6, for a provider that carries a launch's engine credentials in
1745
+ * its admission (`credentialDelivery: "admission"`): the request without
1746
+ * them, and them, with the agent's task token under any of its names. A
1747
+ * credential with a value is carried, never left in the request's
1748
+ * environment, nor its name inherited from the runner's own; an explicit
1749
+ * blank stays, so no other value of that name reaches the engine.
1750
+ */
1751
+ export function carriedCredentials<
1752
+ R extends { env?: Readonly<Record<string, string>>; inheritEnv?: readonly string[] },
1753
+ >(request: R): { delivered: R; carried: Record<string, string> | null } {
1754
+ const env: Record<string, string> = {};
1755
+ const carried: Record<string, string> = {};
1756
+ for (const [name, value] of Object.entries(request.env ?? {})) {
1757
+ if (isLaunchSecretEnvName(name) && value !== "") carried[name] = value;
1758
+ else env[name] = value;
1759
+ }
1760
+ const delivered: R = {
1761
+ ...request,
1762
+ ...(request.env === undefined ? {} : { env }),
1763
+ ...(request.inheritEnv === undefined
1764
+ ? {}
1765
+ : { inheritEnv: request.inheritEnv.filter((name) => !isLaunchSecretEnvName(name)) }),
1766
+ };
1767
+ return { delivered, carried: Object.keys(carried).length > 0 ? carried : null };
1768
+ }
1769
+
1770
+ function isProcess(value: unknown): value is TaskEnvironmentProcess {
1771
+ return typeof value === "object" && value !== null && "completion" in value;
1772
+ }
1773
+
1774
+ /** A bounded exec's or detached launch's result that reports no exit, or a
1775
+ * failed one. */
1776
+ function failedResult(value: unknown): boolean {
1777
+ return (
1778
+ typeof value === "object" &&
1779
+ value !== null &&
1780
+ "exitCode" in value &&
1781
+ (value as Pick<TaskEnvironmentExecResult, "exitCode">).exitCode !== 0
1782
+ );
1783
+ }
1784
+
1785
+ /**
1786
+ * `environment`, with every engine launch an adapter makes through it (a
1787
+ * session, a one-shot turn, a durable runner, a bounded exec) admitted by
1788
+ * `gate` and started under the anchor it resolves to; a launch it refuses
1789
+ * rejects unstarted. The gate is asked again at the provider's last moment,
1790
+ * after the request's own last-moment check: a launch whose gate closed
1791
+ * while its provider waited (a machine looking up its address) starts
1792
+ * nothing. On a provider that carries engine credentials in the
1793
+ * admission, the launch's credentials go there, never into its command. An
1794
+ * admission refused after it was written, or whose launch the provider
1795
+ * could not start (it threw, or a bounded exec or detached launch reports
1796
+ * no exit or a failed one), is revoked before the launch returns; one that
1797
+ * carried credentials is revoked once its bounded exec or process has ended
1798
+ * too, so nothing it carried outlives it unused (a detached launch's own
1799
+ * supervisor uses it, or it expires). A revocation that cannot be done now
1800
+ * is retained and retried (`EngineAdmission.revoke`), never dropped.
1801
+ * Anything else passes through, bound to the original.
1802
+ */
1803
+ export function anchoredEnvironment<T extends object>(environment: T, gate: EngineLaunchGate): T {
1804
+ const carries = (environment as { credentialDelivery?: unknown }).credentialDelivery === "admission";
1805
+ return new Proxy(environment, {
1806
+ get(target, property) {
1807
+ const value: unknown = Reflect.get(target, property, target);
1808
+ if (typeof value !== "function") return value;
1809
+ const method = value as (...args: unknown[]) => unknown;
1810
+ if (!LAUNCHES.has(property)) return method.bind(target);
1811
+ return async (
1812
+ request: {
1813
+ projectEnvProfile: string;
1814
+ env?: Readonly<Record<string, string>>;
1815
+ inheritEnv?: readonly string[];
1816
+ },
1817
+ ...rest: unknown[]
1818
+ ) => {
1819
+ if (!launchesEngine(request)) return method.call(target, request, ...rest);
1820
+ const { delivered, carried } = carries ? carriedCredentials(request) : { delivered: request, carried: null };
1821
+ const admission = await gate.admit(carried);
1822
+ // A failure is retained by the admission and retried there.
1823
+ const revoke = (): Promise<void> => admission.revoke().catch(() => undefined);
1824
+ if (!gate.held()) {
1825
+ await revoke();
1826
+ throw new EngineLaunchRefused();
1827
+ }
1828
+ const own = (delivered as { admit?: () => void }).admit;
1829
+ const admit = (): void => {
1830
+ own?.();
1831
+ if (!gate.held()) throw new EngineLaunchRefused();
1832
+ };
1833
+ let started: unknown;
1834
+ try {
1835
+ started = await method.call(target, { ...delivered, anchor: admission.anchor, admit }, ...rest);
1836
+ } catch (error) {
1837
+ await revoke();
1838
+ throw error;
1839
+ }
1840
+ if ((property === "exec" || property === "launchDetachedSession") && failedResult(started)) {
1841
+ // The provider reports no start, or a failed one: what it was
1842
+ // admitted with may be left unused, so it is revoked first.
1843
+ await revoke();
1844
+ return started;
1845
+ }
1846
+ if (carried !== null) {
1847
+ if (property === "exec") void revoke();
1848
+ else if (isProcess(started)) void started.completion.then(revoke, revoke);
1849
+ }
1850
+ return started;
1851
+ };
1852
+ },
1853
+ });
1854
+ }
1855
+
1856
+ /** Tests only: where carried credentials wait when the gate area is not
1857
+ * held in memory. */
1858
+ export function setEngineCarrierRootForTests(root: string | null): void {
1859
+ carrierFallbackRoot = root ?? ENGINE_CARRIER_FALLBACK_ROOT;
1860
+ }
1861
+
1862
+ /** Tests only: where the gates live (a directory the test user owns). */
1863
+ export function setEngineGateRootForTests(root: string | null): void {
1864
+ gateRoot = root ?? ENGINE_GATE_ROOT;
1865
+ }
1866
+
1867
+ /** Tests only: the cleanup's bound, in seconds. */
1868
+ export function setEngineAnchorStopBoundForTests(seconds: number | null): void {
1869
+ anchorStopSeconds = seconds ?? ENGINE_ANCHOR_STOP_SECONDS;
1870
+ }
1871
+
1872
+ /** Tests only: the first retry of a retained revocation, in milliseconds,
1873
+ * and every retained revocation forgotten. */
1874
+ export function resetEngineRevocationsForTests(firstRetryMs: number | null = null): void {
1875
+ revocationRetryFirstMs = firstRetryMs ?? REVOCATION_RETRY_FIRST_MS;
1876
+ retained.clear();
1877
+ // One still under way keeps nothing for the next test.
1878
+ for (const attempt of underWay) attempt.discharged = true;
1879
+ underWay.clear();
1880
+ }